Line data Source code
1 : /* If-conversion support.
2 : Copyright (C) 2000-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 : or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
14 : License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "cfghooks.h"
28 : #include "df.h"
29 : #include "memmodel.h"
30 : #include "tm_p.h"
31 : #include "expmed.h"
32 : #include "optabs.h"
33 : #include "regs.h"
34 : #include "emit-rtl.h"
35 : #include "recog.h"
36 :
37 : #include "cfgrtl.h"
38 : #include "cfganal.h"
39 : #include "cfgcleanup.h"
40 : #include "explow.h"
41 : #include "expr.h"
42 : #include "output.h"
43 : #include "cfgloop.h"
44 : #include "tree-pass.h"
45 : #include "dbgcnt.h"
46 : #include "shrink-wrap.h"
47 : #include "rtl-iter.h"
48 : #include "ifcvt.h"
49 :
50 : /* The number of instructions this pass may make run unconditionally in place
51 : of a branch. The conditional-execution path uses it as the number of insns
52 : it may predicate. The branchless paths use it as the number of insns they
53 : may speculate. The default charges the cost of an unpredictable branch,
54 : plus one. Some targets override it. */
55 :
56 : #ifndef MAX_CONDITIONAL_EXECUTE
57 : #define MAX_CONDITIONAL_EXECUTE \
58 : (BRANCH_COST (optimize_function_for_speed_p (cfun), false) \
59 : + 1)
60 : #endif
61 :
62 : #define NULL_BLOCK ((basic_block) NULL)
63 :
64 : /* Which of the three RTL if-conversion passes is running. ce1 runs before
65 : combine, ce2 after combine and ce3 after reload. The cost model treats the
66 : pre-combine run specially, so the phase is recorded here. */
67 : enum ifcvt_phase
68 : {
69 : IFCVT_BEFORE_COMBINE, /* ce1. */
70 : IFCVT_AFTER_COMBINE, /* ce2. */
71 : IFCVT_AFTER_RELOAD /* ce3. */
72 : };
73 :
74 : /* The if-conversion pass currently running. */
75 : static ifcvt_phase ifcvt_pass_phase;
76 :
77 : /* True if the target has the cbranchcc4 optab. */
78 : static bool have_cbranchcc4;
79 :
80 : /* # of IF-THEN or IF-THEN-ELSE blocks we looked at */
81 : static int num_possible_if_blocks;
82 :
83 : /* # of IF-THEN or IF-THEN-ELSE blocks were converted to conditional
84 : execution. */
85 : static int num_updated_if_blocks;
86 :
87 : /* # of changes made. */
88 : static int num_true_changes;
89 :
90 : /* Whether this pass over the function converted anything, so that another
91 : pass may find further opportunities in what it left behind. Every
92 : successful conversion sets it, not only the conditional-execution one. */
93 : static bool ifcvt_changed_p;
94 :
95 : /* Forward references. */
96 : static bool noce_operand_ok (const_rtx);
97 : static void merge_if_block (ce_if_block *);
98 : static bool find_cond_trap (basic_block, edge, edge);
99 : static bool cond_exec_find_if_block (ce_if_block *);
100 : static bool find_if_case_1 (basic_block, edge, edge);
101 : static bool find_if_case_2 (basic_block, edge, edge);
102 : static bool dead_or_predicable (basic_block, basic_block, basic_block,
103 : edge, bool);
104 : static rtx_insn *block_has_only_trap (basic_block);
105 : static void init_noce_multiple_sets_info (basic_block,
106 : auto_delete_vec<noce_multiple_sets_info> &);
107 : static bool noce_convert_multiple_sets_1 (noce_if_info *, rtx,
108 : auto_delete_vec<noce_multiple_sets_info> &,
109 : auto_delete_vec<noce_multiple_sets_info> &, unsigned,
110 : const vec<unsigned> &, int *, bool *);
111 :
112 : /* Count the number of non-jump active insns in BB. */
113 :
114 : static int
115 0 : count_bb_insns (const_basic_block bb)
116 : {
117 0 : int count = 0;
118 0 : rtx_insn *insn = BB_HEAD (bb);
119 :
120 0 : while (1)
121 : {
122 0 : if (active_insn_p (insn) && !JUMP_P (insn))
123 0 : count++;
124 :
125 0 : if (insn == BB_END (bb))
126 : break;
127 0 : insn = NEXT_INSN (insn);
128 : }
129 :
130 0 : return count;
131 : }
132 :
133 : /* Determine whether the total insn_cost on non-jump insns in
134 : basic block BB is less than MAX_COST. This function returns
135 : false if the cost of any instruction could not be estimated.
136 :
137 : The cost of the non-jump insns in BB is scaled by REG_BR_PROB_BASE
138 : as those insns are being speculated. MAX_COST is scaled with SCALE
139 : plus a small fudge factor. */
140 :
141 : static bool
142 3059923 : cheap_bb_rtx_cost_p (const_basic_block bb,
143 : profile_probability prob, int max_cost)
144 : {
145 3059923 : int count = 0;
146 3059923 : rtx_insn *insn = BB_HEAD (bb);
147 3059923 : bool speed = optimize_bb_for_speed_p (bb);
148 3059923 : int scale = prob.initialized_p () ? prob.to_reg_br_prob_base ()
149 : : REG_BR_PROB_BASE;
150 :
151 : /* Set scale to REG_BR_PROB_BASE to void the identical scaling
152 : applied to insn_cost when optimizing for size. Only do
153 : this after combine because if-conversion might interfere with
154 : passes before combine.
155 :
156 : Use optimize_function_for_speed_p instead of the pre-defined
157 : variable speed to make sure it is set to same value for all
158 : basic blocks in one if-conversion transformation. */
159 3059923 : if (!optimize_function_for_speed_p (cfun)
160 3059923 : && ifcvt_pass_phase != IFCVT_BEFORE_COMBINE)
161 : scale = REG_BR_PROB_BASE;
162 : /* Our branch probability/scaling factors are just estimates and don't
163 : account for cases where we can get speculation for free and other
164 : secondary benefits. So we fudge the scale factor to make speculating
165 : appear a little more profitable when optimizing for performance. */
166 : else
167 3008926 : scale += REG_BR_PROB_BASE / 8;
168 :
169 :
170 3059923 : max_cost *= scale;
171 :
172 12920075 : while (1)
173 : {
174 15979998 : if (NONJUMP_INSN_P (insn))
175 : {
176 : /* Inline-asm's cost is not very estimatable.
177 : It could be a costly instruction but the
178 : estimate would be the same as a non costly
179 : instruction. */
180 4580298 : if (asm_noperands (PATTERN (insn)) >= 0)
181 : return false;
182 :
183 4577597 : int cost = insn_cost (insn, speed) * REG_BR_PROB_BASE;
184 4577597 : if (cost == 0)
185 : return false;
186 :
187 : /* If this instruction is the load or set of a "stack" register,
188 : such as a floating point register on x87, then the cost of
189 : speculatively executing this insn may need to include
190 : the additional cost of popping its result off of the
191 : register stack. Unfortunately, correctly recognizing and
192 : accounting for this additional overhead is tricky, so for
193 : now we simply prohibit such speculative execution. */
194 : #ifdef STACK_REGS
195 4527060 : {
196 4527060 : rtx set = single_set (insn);
197 4527060 : if (set && STACK_REG_P (SET_DEST (set)))
198 : return false;
199 : }
200 : #endif
201 :
202 4523154 : count += cost;
203 4523154 : if (count >= max_cost)
204 : return false;
205 : }
206 11399700 : else if (CALL_P (insn))
207 : return false;
208 :
209 13508430 : if (insn == BB_END (bb))
210 : break;
211 12920075 : insn = NEXT_INSN (insn);
212 12920075 : }
213 :
214 : return true;
215 : }
216 :
217 : /* Return the first non-jump active insn in the basic block. */
218 :
219 : static rtx_insn *
220 1533450 : first_active_insn (basic_block bb)
221 : {
222 1533450 : rtx_insn *insn = BB_HEAD (bb);
223 :
224 1533450 : if (LABEL_P (insn))
225 : {
226 280884 : if (insn == BB_END (bb))
227 : return NULL;
228 280884 : insn = NEXT_INSN (insn);
229 : }
230 :
231 5157291 : while (NOTE_P (insn) || DEBUG_INSN_P (insn))
232 : {
233 3623841 : if (insn == BB_END (bb))
234 : return NULL;
235 3623841 : insn = NEXT_INSN (insn);
236 : }
237 :
238 1533450 : if (JUMP_P (insn))
239 0 : return NULL;
240 :
241 : return insn;
242 : }
243 :
244 : /* Return the last non-jump active insn in the basic block. */
245 :
246 : static rtx_insn *
247 2570283 : last_active_insn (basic_block bb, bool skip_use_p)
248 : {
249 2570283 : rtx_insn *insn = BB_END (bb);
250 2570283 : rtx_insn *head = BB_HEAD (bb);
251 :
252 2570283 : while (NOTE_P (insn)
253 : || JUMP_P (insn)
254 : || DEBUG_INSN_P (insn)
255 6248977 : || (skip_use_p
256 0 : && NONJUMP_INSN_P (insn)
257 0 : && GET_CODE (PATTERN (insn)) == USE))
258 : {
259 3678930 : if (insn == head)
260 : return NULL;
261 3678694 : insn = PREV_INSN (insn);
262 : }
263 :
264 2570047 : if (LABEL_P (insn))
265 76 : return NULL;
266 :
267 : return insn;
268 : }
269 :
270 : /* Return the active insn before INSN inside basic block CURR_BB. */
271 :
272 : static rtx_insn *
273 0 : find_active_insn_before (basic_block curr_bb, rtx_insn *insn)
274 : {
275 0 : if (!insn || insn == BB_HEAD (curr_bb))
276 : return NULL;
277 :
278 0 : while ((insn = PREV_INSN (insn)) != NULL_RTX)
279 : {
280 0 : if (NONJUMP_INSN_P (insn) || JUMP_P (insn) || CALL_P (insn))
281 : break;
282 :
283 : /* No other active insn all the way to the start of the basic block. */
284 0 : if (insn == BB_HEAD (curr_bb))
285 : return NULL;
286 : }
287 :
288 : return insn;
289 : }
290 :
291 : /* Return the active insn after INSN inside basic block CURR_BB. */
292 :
293 : static rtx_insn *
294 0 : find_active_insn_after (basic_block curr_bb, rtx_insn *insn)
295 : {
296 0 : if (!insn || insn == BB_END (curr_bb))
297 : return NULL;
298 :
299 0 : while ((insn = NEXT_INSN (insn)) != NULL_RTX)
300 : {
301 0 : if (NONJUMP_INSN_P (insn) || JUMP_P (insn) || CALL_P (insn))
302 : break;
303 :
304 : /* No other active insn all the way to the end of the basic block. */
305 0 : if (insn == BB_END (curr_bb))
306 : return NULL;
307 : }
308 :
309 : return insn;
310 : }
311 :
312 : /* Return the basic block reached by falling through the basic block BB. */
313 :
314 : static basic_block
315 0 : block_fallthru (basic_block bb)
316 : {
317 0 : edge e = find_fallthru_edge (bb->succs);
318 :
319 0 : return (e) ? e->dest : NULL_BLOCK;
320 : }
321 :
322 : /* Return true if RTXs A and B can be safely interchanged. */
323 :
324 : static bool
325 495834 : rtx_interchangeable_p (const_rtx a, const_rtx b)
326 : {
327 495834 : if (!rtx_equal_p (a, b))
328 : return false;
329 :
330 183925 : if (GET_CODE (a) != MEM)
331 : return true;
332 :
333 : /* A dead type-unsafe memory reference is legal, but a live type-unsafe memory
334 : reference is not. Interchanging a dead type-unsafe memory reference with
335 : a live type-safe one creates a live type-unsafe memory reference, in other
336 : words, it makes the program illegal.
337 : We check here conservatively whether the two memory references have equal
338 : memory attributes. */
339 :
340 2373 : return mem_attrs_eq_p (get_mem_attrs (a), get_mem_attrs (b));
341 : }
342 :
343 :
344 : /* Go through the insns from START to END, converting them to conditional
345 : execution format if possible. CE_INFO describes the if-block being
346 : converted and is only read by the target IFCVT_MODIFY_INSN macro. TEST is
347 : the conditional-execution test to predicate them on and PROB_VAL the
348 : probability that it holds. MOD_OK allows one insn that modifies TEST, in
349 : which case it must be the last one. Return TRUE if all of the non-note
350 : insns were processed. */
351 :
352 : static bool
353 0 : cond_exec_process_insns (ce_if_block *ce_info ATTRIBUTE_UNUSED,
354 : rtx_insn *start, rtx end, rtx test,
355 : profile_probability prob_val, bool mod_ok)
356 : {
357 0 : bool must_be_last = false;
358 0 : rtx_insn *insn;
359 0 : rtx xtest;
360 0 : rtx pattern;
361 :
362 0 : if (!start || !end)
363 : return false;
364 :
365 0 : for (insn = start; ; insn = NEXT_INSN (insn))
366 : {
367 : /* dwarf2out can't cope with conditional prologues. */
368 0 : if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_PROLOGUE_END)
369 : return false;
370 :
371 0 : if (NOTE_P (insn) || DEBUG_INSN_P (insn))
372 0 : goto insn_done;
373 :
374 0 : gcc_assert (NONJUMP_INSN_P (insn) || CALL_P (insn));
375 :
376 : /* dwarf2out can't cope with conditional unwind info. */
377 0 : if (RTX_FRAME_RELATED_P (insn))
378 : return false;
379 :
380 : /* Remove USE insns that get in the way. */
381 0 : if (reload_completed && GET_CODE (PATTERN (insn)) == USE)
382 : {
383 : /* ??? Ug. Actually unlinking the thing is problematic,
384 : given what we'd have to coordinate with our callers. */
385 0 : SET_INSN_DELETED (insn);
386 0 : goto insn_done;
387 : }
388 :
389 : /* Last insn wasn't last? */
390 0 : if (must_be_last)
391 : return false;
392 :
393 0 : if (modified_in_p (test, insn))
394 : {
395 0 : if (!mod_ok)
396 : return false;
397 : must_be_last = true;
398 : }
399 :
400 : /* Now build the conditional form of the instruction. */
401 0 : pattern = PATTERN (insn);
402 0 : xtest = copy_rtx (test);
403 :
404 : /* If this is already a COND_EXEC, rewrite the test to be an AND of the
405 : two conditions. */
406 0 : if (GET_CODE (pattern) == COND_EXEC)
407 : {
408 0 : if (GET_MODE (xtest) != GET_MODE (COND_EXEC_TEST (pattern)))
409 : return false;
410 :
411 0 : xtest = gen_rtx_AND (GET_MODE (xtest), xtest,
412 : COND_EXEC_TEST (pattern));
413 0 : pattern = COND_EXEC_CODE (pattern);
414 : }
415 :
416 0 : pattern = gen_rtx_COND_EXEC (VOIDmode, xtest, pattern);
417 :
418 : /* If the machine needs to modify the insn being conditionally executed,
419 : say for example to force a constant integer operand into a temp
420 : register, do so here. */
421 : #ifdef IFCVT_MODIFY_INSN
422 : IFCVT_MODIFY_INSN (ce_info, pattern, insn);
423 : if (! pattern)
424 : return false;
425 : #endif
426 :
427 0 : validate_change (insn, &PATTERN (insn), pattern, 1);
428 :
429 0 : if (CALL_P (insn) && prob_val.initialized_p ())
430 0 : validate_change (insn, ®_NOTES (insn),
431 : gen_rtx_INT_LIST ((machine_mode) REG_BR_PROB,
432 : prob_val.to_reg_br_prob_note (),
433 : REG_NOTES (insn)), 1);
434 :
435 0 : insn_done:
436 0 : if (insn == end)
437 : break;
438 0 : }
439 :
440 : return true;
441 : }
442 :
443 : /* Return the condition for a jump. Do not do any special processing. */
444 :
445 : static rtx
446 196748 : cond_exec_get_condition (rtx_insn *jump, bool get_reversed = false)
447 : {
448 196748 : rtx test_if, cond;
449 :
450 196748 : if (any_condjump_p (jump))
451 196748 : test_if = SET_SRC (pc_set (jump));
452 : else
453 : return NULL_RTX;
454 196748 : cond = XEXP (test_if, 0);
455 :
456 : /* If this branches to JUMP_LABEL when the condition is false,
457 : reverse the condition. */
458 196748 : if (get_reversed
459 196748 : || (GET_CODE (XEXP (test_if, 2)) == LABEL_REF
460 0 : && label_ref_label (XEXP (test_if, 2))
461 0 : == JUMP_LABEL (jump)))
462 : {
463 98374 : enum rtx_code rev = reversed_comparison_code (cond, jump);
464 98374 : if (rev == UNKNOWN)
465 : return NULL_RTX;
466 :
467 98374 : cond = gen_rtx_fmt_ee (rev, GET_MODE (cond), XEXP (cond, 0),
468 : XEXP (cond, 1));
469 : }
470 :
471 : return cond;
472 : }
473 :
474 : /* Given a simple IF-THEN or IF-THEN-ELSE block described by CE_INFO, attempt
475 : to convert it to conditional execution. DO_MULTIPLE_P allows CE_INFO's
476 : recorded chain of && or || test blocks to be converted as well. Return
477 : TRUE if we were successful at converting the block. */
478 :
479 : static bool
480 0 : cond_exec_process_if_block (ce_if_block *ce_info, bool do_multiple_p)
481 : {
482 0 : basic_block test_bb = ce_info->test_bb; /* last test block */
483 0 : basic_block then_bb = ce_info->then_bb; /* THEN */
484 0 : basic_block else_bb = ce_info->else_bb; /* ELSE or NULL */
485 0 : rtx test_expr; /* expression in IF_THEN_ELSE that is tested */
486 0 : rtx_insn *then_start; /* first insn in THEN block */
487 0 : rtx_insn *then_end; /* last insn + 1 in THEN block */
488 0 : rtx_insn *else_start = NULL; /* first insn in ELSE block or NULL */
489 0 : rtx_insn *else_end = NULL; /* last insn + 1 in ELSE block */
490 0 : int max; /* max # of insns to convert. */
491 0 : bool then_mod_ok; /* whether conditional mods are ok in THEN */
492 0 : rtx true_expr; /* test for else block insns */
493 0 : rtx false_expr; /* test for then block insns */
494 0 : profile_probability true_prob_val;/* probability of else block */
495 0 : profile_probability false_prob_val;/* probability of then block */
496 0 : rtx_insn *then_last_head = NULL; /* Last match at the head of THEN */
497 0 : rtx_insn *else_last_head = NULL; /* Last match at the head of ELSE */
498 0 : rtx_insn *then_first_tail = NULL; /* First match at the tail of THEN */
499 0 : rtx_insn *else_first_tail = NULL; /* First match at the tail of ELSE */
500 0 : int then_n_insns, else_n_insns, n_insns;
501 0 : enum rtx_code false_code;
502 0 : rtx note;
503 :
504 : /* If test is comprised of && or || elements, and we've failed at handling
505 : all of them together, just use the last test if it is the special case of
506 : && elements without an ELSE block. */
507 0 : if (!do_multiple_p && ce_info->num_multiple_test_blocks)
508 : {
509 0 : if (else_bb || ! ce_info->and_and_p)
510 : return false;
511 :
512 0 : ce_info->test_bb = test_bb = ce_info->last_test_bb;
513 0 : ce_info->num_multiple_test_blocks = 0;
514 0 : ce_info->num_and_and_blocks = 0;
515 0 : ce_info->num_or_or_blocks = 0;
516 : }
517 :
518 : /* Find the conditional jump to the ELSE or JOIN part, and isolate
519 : the test. */
520 0 : test_expr = cond_exec_get_condition (BB_END (test_bb));
521 0 : if (! test_expr)
522 : return false;
523 :
524 : /* If the conditional jump is more than just a conditional jump,
525 : then we cannot do conditional execution conversion on this block. */
526 0 : if (! onlyjump_p (BB_END (test_bb)))
527 : return false;
528 :
529 : /* Collect the bounds of where we're to search, skipping any labels, jumps
530 : and notes at the beginning and end of the block. Then count the total
531 : number of insns and see if it is small enough to convert. */
532 0 : then_start = first_active_insn (then_bb);
533 0 : then_end = last_active_insn (then_bb, true);
534 0 : then_n_insns = ce_info->num_then_insns = count_bb_insns (then_bb);
535 0 : n_insns = then_n_insns;
536 0 : max = MAX_CONDITIONAL_EXECUTE;
537 :
538 0 : if (else_bb)
539 : {
540 0 : int n_matching;
541 :
542 0 : max *= 2;
543 0 : else_start = first_active_insn (else_bb);
544 0 : else_end = last_active_insn (else_bb, true);
545 0 : else_n_insns = ce_info->num_else_insns = count_bb_insns (else_bb);
546 0 : n_insns += else_n_insns;
547 :
548 : /* Look for matching sequences at the head and tail of the two blocks,
549 : and limit the range of insns to be converted if possible. */
550 0 : n_matching = flow_find_cross_jump (then_bb, else_bb,
551 : &then_first_tail, &else_first_tail,
552 : NULL);
553 0 : if (then_first_tail == BB_HEAD (then_bb))
554 0 : then_start = then_end = NULL;
555 0 : if (else_first_tail == BB_HEAD (else_bb))
556 0 : else_start = else_end = NULL;
557 :
558 0 : if (n_matching > 0)
559 : {
560 0 : if (then_end)
561 0 : then_end = find_active_insn_before (then_bb, then_first_tail);
562 0 : if (else_end)
563 0 : else_end = find_active_insn_before (else_bb, else_first_tail);
564 0 : n_insns -= 2 * n_matching;
565 : }
566 :
567 0 : if (then_start
568 0 : && else_start
569 : && then_n_insns > n_matching
570 0 : && else_n_insns > n_matching)
571 : {
572 0 : int longest_match = MIN (then_n_insns - n_matching,
573 : else_n_insns - n_matching);
574 0 : n_matching
575 0 : = flow_find_head_matching_sequence (then_bb, else_bb,
576 : &then_last_head,
577 : &else_last_head,
578 : longest_match);
579 :
580 0 : if (n_matching > 0)
581 : {
582 0 : rtx_insn *insn;
583 :
584 : /* We won't pass the insns in the head sequence to
585 : cond_exec_process_insns, so we need to test them here
586 : to make sure that they don't clobber the condition. */
587 0 : for (insn = BB_HEAD (then_bb);
588 0 : insn != NEXT_INSN (then_last_head);
589 0 : insn = NEXT_INSN (insn))
590 0 : if (!LABEL_P (insn) && !NOTE_P (insn)
591 0 : && !DEBUG_INSN_P (insn)
592 0 : && modified_in_p (test_expr, insn))
593 : return false;
594 : }
595 :
596 0 : if (then_last_head == then_end)
597 0 : then_start = then_end = NULL;
598 0 : if (else_last_head == else_end)
599 0 : else_start = else_end = NULL;
600 :
601 0 : if (n_matching > 0)
602 : {
603 0 : if (then_start)
604 0 : then_start = find_active_insn_after (then_bb, then_last_head);
605 0 : if (else_start)
606 0 : else_start = find_active_insn_after (else_bb, else_last_head);
607 0 : n_insns -= 2 * n_matching;
608 : }
609 : }
610 : }
611 :
612 0 : if (n_insns > max)
613 : return false;
614 :
615 : /* Map test_expr/test_jump into the appropriate MD tests to use on
616 : the conditionally executed code. */
617 :
618 0 : true_expr = test_expr;
619 :
620 0 : false_code = reversed_comparison_code (true_expr, BB_END (test_bb));
621 0 : if (false_code != UNKNOWN)
622 0 : false_expr = gen_rtx_fmt_ee (false_code, GET_MODE (true_expr),
623 : XEXP (true_expr, 0), XEXP (true_expr, 1));
624 : else
625 : false_expr = NULL_RTX;
626 :
627 : #ifdef IFCVT_MODIFY_TESTS
628 : /* If the machine description needs to modify the tests, such as setting a
629 : conditional execution register from a comparison, it can do so here. */
630 : IFCVT_MODIFY_TESTS (ce_info, true_expr, false_expr);
631 :
632 : /* See if the conversion failed. */
633 : if (!true_expr || !false_expr)
634 : goto fail;
635 : #endif
636 :
637 0 : note = find_reg_note (BB_END (test_bb), REG_BR_PROB, NULL_RTX);
638 0 : if (note)
639 : {
640 0 : true_prob_val = profile_probability::from_reg_br_prob_note (XINT (note, 0));
641 0 : false_prob_val = true_prob_val.invert ();
642 : }
643 : else
644 : {
645 : true_prob_val = profile_probability::uninitialized ();
646 : false_prob_val = profile_probability::uninitialized ();
647 : }
648 :
649 : /* If we have && or || tests, do them here. These tests are in the adjacent
650 : blocks after the first block containing the test. */
651 0 : if (ce_info->num_multiple_test_blocks > 0)
652 : {
653 0 : basic_block bb = test_bb;
654 0 : basic_block last_test_bb = ce_info->last_test_bb;
655 :
656 0 : if (! false_expr)
657 0 : goto fail;
658 :
659 0 : do
660 : {
661 0 : rtx_insn *start, *end;
662 0 : rtx t, f;
663 0 : enum rtx_code f_code;
664 :
665 0 : bb = block_fallthru (bb);
666 0 : start = first_active_insn (bb);
667 0 : end = last_active_insn (bb, true);
668 0 : if (start
669 0 : && ! cond_exec_process_insns (ce_info, start, end, false_expr,
670 : false_prob_val, false))
671 0 : goto fail;
672 :
673 : /* If the conditional jump is more than just a conditional jump, then
674 : we cannot do conditional execution conversion on this block. */
675 0 : if (! onlyjump_p (BB_END (bb)))
676 0 : goto fail;
677 :
678 : /* Find the conditional jump and isolate the test. */
679 0 : t = cond_exec_get_condition (BB_END (bb));
680 0 : if (! t)
681 0 : goto fail;
682 :
683 0 : f_code = reversed_comparison_code (t, BB_END (bb));
684 0 : if (f_code == UNKNOWN)
685 0 : goto fail;
686 :
687 0 : f = gen_rtx_fmt_ee (f_code, GET_MODE (t), XEXP (t, 0), XEXP (t, 1));
688 0 : if (ce_info->and_and_p)
689 : {
690 0 : t = gen_rtx_AND (GET_MODE (t), true_expr, t);
691 0 : f = gen_rtx_IOR (GET_MODE (t), false_expr, f);
692 : }
693 : else
694 : {
695 0 : t = gen_rtx_IOR (GET_MODE (t), true_expr, t);
696 0 : f = gen_rtx_AND (GET_MODE (t), false_expr, f);
697 : }
698 :
699 : /* If the machine description needs to modify the tests, such as
700 : setting a conditional execution register from a comparison, it can
701 : do so here. */
702 : #ifdef IFCVT_MODIFY_MULTIPLE_TESTS
703 : IFCVT_MODIFY_MULTIPLE_TESTS (ce_info, bb, t, f);
704 :
705 : /* See if the conversion failed. */
706 : if (!t || !f)
707 : goto fail;
708 : #endif
709 :
710 0 : true_expr = t;
711 0 : false_expr = f;
712 : }
713 0 : while (bb != last_test_bb);
714 : }
715 :
716 : /* For IF-THEN-ELSE blocks, we don't allow modifications of the test
717 : on then THEN block. */
718 0 : then_mod_ok = (else_bb == NULL_BLOCK);
719 :
720 : /* Go through the THEN and ELSE blocks converting the insns if possible
721 : to conditional execution. */
722 :
723 0 : if (then_end
724 0 : && (! false_expr
725 0 : || ! cond_exec_process_insns (ce_info, then_start, then_end,
726 : false_expr, false_prob_val,
727 : then_mod_ok)))
728 0 : goto fail;
729 :
730 0 : if (else_bb && else_end
731 0 : && ! cond_exec_process_insns (ce_info, else_start, else_end,
732 : true_expr, true_prob_val, true))
733 0 : goto fail;
734 :
735 : /* If we cannot apply the changes, fail. Do not go through the normal fail
736 : processing, since apply_change_group will call cancel_changes. */
737 0 : if (! apply_change_group ())
738 : {
739 : #ifdef IFCVT_MODIFY_CANCEL
740 : /* Cancel any machine dependent changes. */
741 : IFCVT_MODIFY_CANCEL (ce_info);
742 : #endif
743 : return false;
744 : }
745 :
746 : #ifdef IFCVT_MODIFY_FINAL
747 : /* Do any machine dependent final modifications. */
748 : IFCVT_MODIFY_FINAL (ce_info);
749 : #endif
750 :
751 : /* Conversion succeeded. */
752 0 : if (dump_file)
753 0 : fprintf (dump_file, "%d insn%s converted to conditional execution.\n",
754 : n_insns, (n_insns == 1) ? " was" : "s were");
755 :
756 : /* Merge the blocks! If we had matching sequences, make sure to delete one
757 : copy at the appropriate location first: delete the copy in the THEN branch
758 : for a tail sequence so that the remaining one is executed last for both
759 : branches, and delete the copy in the ELSE branch for a head sequence so
760 : that the remaining one is executed first for both branches. */
761 0 : if (then_first_tail)
762 : {
763 0 : rtx_insn *from = then_first_tail;
764 0 : if (!INSN_P (from))
765 0 : from = find_active_insn_after (then_bb, from);
766 0 : delete_insn_chain (from, get_last_bb_insn (then_bb), false);
767 : }
768 0 : if (else_last_head)
769 0 : delete_insn_chain (first_active_insn (else_bb), else_last_head, false);
770 :
771 0 : merge_if_block (ce_info);
772 0 : ifcvt_changed_p = true;
773 0 : return true;
774 :
775 0 : fail:
776 : #ifdef IFCVT_MODIFY_CANCEL
777 : /* Cancel any machine dependent changes. */
778 : IFCVT_MODIFY_CANCEL (ce_info);
779 : #endif
780 :
781 0 : cancel_changes (0);
782 0 : return false;
783 : }
784 :
785 : /* Return the condition to use when the arms of IF_INFO's if-region are
786 : swapped, and set *CODE to its comparison code. When the reversed condition
787 : has no rtx of its own the original condition is returned alongside the
788 : reversed code, which is the form the store-flag and conditional-move
789 : expanders take. *CODE is UNKNOWN if the condition cannot be reversed. */
790 :
791 : static rtx
792 179624 : noce_reversed_cond (noce_if_info *if_info, enum rtx_code *code)
793 : {
794 179624 : if (if_info->rev_cond)
795 : {
796 179624 : *code = GET_CODE (if_info->rev_cond);
797 179624 : return if_info->rev_cond;
798 : }
799 :
800 0 : *code = reversed_comparison_code (if_info->cond, if_info->jump);
801 0 : return if_info->cond;
802 : }
803 :
804 : /* Return the comparison code for reversed condition for IF_INFO,
805 : or UNKNOWN if reversing the condition is not possible. */
806 :
807 : static inline enum rtx_code
808 104395 : noce_reversed_cond_code (noce_if_info *if_info)
809 : {
810 104395 : enum rtx_code code;
811 11230 : noce_reversed_cond (if_info, &code);
812 104395 : return code;
813 : }
814 :
815 : /* A register definition and its dependency level. */
816 :
817 : struct noce_parallel_cost_node
818 : {
819 : rtx dest;
820 : unsigned int level;
821 : };
822 :
823 : /* Return true if INSN is a unit-cost register SET supported by the parallel
824 : cost model. Store the SET in *SET. */
825 :
826 : static bool
827 473240 : noce_parallel_costed_insn_p (rtx_insn *insn, bool speed_p, rtx *set)
828 : {
829 473240 : *set = single_set (insn);
830 473240 : if (!*set)
831 : return false;
832 :
833 473019 : rtx dest = SET_DEST (*set);
834 473019 : rtx src = SET_SRC (*set);
835 :
836 473019 : return (REG_P (dest)
837 471438 : && !contains_mem_rtx_p (src)
838 452123 : && !side_effects_p (src)
839 452123 : && !may_trap_p (src)
840 922292 : && set_rtx_cost (*set, speed_p) == COSTS_N_INSNS (1));
841 : }
842 :
843 : /* Return the dependency level for an instruction reading SRC. A use of a
844 : register defined at level N requires level N + 1. */
845 :
846 : static unsigned int
847 429790 : noce_dependency_level (const vec<noce_parallel_cost_node> &defs, rtx src)
848 : {
849 429790 : unsigned int level = 0;
850 :
851 1300333 : for (unsigned int i = 0; i < defs.length (); ++i)
852 870543 : if (reg_overlap_mentioned_p (defs[i].dest, src))
853 870543 : level = MAX (level, defs[i].level + 1);
854 :
855 429790 : return level;
856 : }
857 :
858 : /* Estimate the cost of SEQ using the target issue rate. Unit-cost register
859 : operations are grouped by RAW dependency level. The cost of each level is
860 : its instruction count divided by the issue rate, rounded up. Fall back to
861 : serial cost if any instruction cannot be modeled.
862 :
863 : In an instrumented build of all SPEC CPU2017 Integer rate benchmarks
864 : for RISC-V, the analysis handled 16,947 of 30,120 candidates (56.26%).
865 : It reduced the cost for 15,024 of the handled candidates (88.65%).
866 : The parallel-to-serial cost ratios for the handled candidates were:
867 :
868 : 0.8 < ratio <= 1.0 11.54 percent
869 : 0.6 < ratio <= 0.8 47.31 percent
870 : 0.4 < ratio <= 0.6 40.24 percent
871 : 0.2 < ratio <= 0.4 0.91 percent
872 : 0.0 < ratio <= 0.2 0.00 percent. */
873 :
874 : static unsigned int
875 186371 : noce_parallel_seq_cost (rtx_insn *seq, bool speed_p)
876 : {
877 186371 : unsigned int serial_cost = seq_cost (seq, speed_p);
878 :
879 186371 : if (!speed_p || !targetm.sched.issue_rate)
880 : return serial_cost;
881 :
882 126856 : unsigned int issue_rate = MAX (targetm.sched.issue_rate (), 1);
883 :
884 : /* Track prior definitions and instruction counts per dependency level. */
885 126856 : auto_vec<noce_parallel_cost_node> defs;
886 126856 : auto_vec<unsigned int> insns_per_level;
887 :
888 556646 : for (rtx_insn *insn = seq; insn; insn = NEXT_INSN (insn))
889 : {
890 473240 : rtx set;
891 :
892 473240 : if (!NONDEBUG_INSN_P (insn))
893 0 : continue;
894 :
895 473240 : if (!noce_parallel_costed_insn_p (insn, speed_p, &set))
896 43450 : return serial_cost;
897 :
898 429790 : unsigned int level = noce_dependency_level (defs, SET_SRC (set));
899 :
900 429790 : if (insns_per_level.length () <= level)
901 249341 : insns_per_level.safe_grow_cleared (level + 1, true);
902 429790 : ++insns_per_level[level];
903 :
904 429790 : noce_parallel_cost_node node = { SET_DEST (set), level };
905 429790 : defs.safe_push (node);
906 : }
907 :
908 : unsigned int parallel_cost = 0;
909 292956 : for (unsigned int i = 0; i < insns_per_level.length (); ++i)
910 209550 : parallel_cost += COSTS_N_INSNS (CEIL (insns_per_level[i], issue_rate));
911 :
912 : /* set_rtx_cost and insn_cost can disagree, so cap the estimate at the
913 : serial cost. */
914 83406 : return MIN (serial_cost, parallel_cost);
915 126856 : }
916 :
917 : /* Return true if SEQ is a good candidate as a replacement for the
918 : if-convertible sequence described in IF_INFO.
919 : This is the default implementation that targets can override
920 : through a target hook. */
921 :
922 : bool
923 186371 : default_noce_conversion_profitable_p (rtx_insn *seq,
924 : noce_if_info *if_info)
925 : {
926 186371 : bool speed_p = if_info->speed_p;
927 :
928 : /* Cost up the new sequence. */
929 186371 : unsigned int cost = noce_parallel_seq_cost (seq, speed_p);
930 :
931 186371 : if (cost <= if_info->original_cost)
932 : return true;
933 :
934 : /* When compiling for size, we can make a reasonably accurately guess
935 : at the size growth. When compiling for speed, use the maximum. */
936 83146 : return speed_p && cost <= if_info->max_seq_cost;
937 : }
938 :
939 : /* Helper function for noce_try_store_flag*. Return NULL_RTX on failure,
940 : including when REVERSEP is requested but the condition cannot be
941 : reversed; callers may rely on this and need not pre-check. */
942 :
943 : static rtx
944 58521 : noce_emit_store_flag (noce_if_info *if_info, rtx x, bool reversep,
945 : int normalize)
946 : {
947 58521 : rtx cond = if_info->cond;
948 58521 : bool cond_complex;
949 58521 : enum rtx_code code;
950 :
951 58521 : cond_complex = (! general_operand (XEXP (cond, 0), VOIDmode)
952 58521 : || ! general_operand (XEXP (cond, 1), VOIDmode));
953 :
954 : /* If earliest == jump, or when the condition is complex, try to
955 : build the store_flag insn directly. */
956 :
957 34606 : if (cond_complex)
958 : {
959 23915 : rtx set = pc_set (if_info->jump);
960 23915 : cond = XEXP (SET_SRC (set), 0);
961 23915 : if (GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
962 23915 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (if_info->jump))
963 0 : reversep = !reversep;
964 23915 : if (if_info->then_else_reversed)
965 21162 : reversep = !reversep;
966 : }
967 34606 : else if (reversep
968 17781 : && if_info->rev_cond
969 17781 : && general_operand (XEXP (if_info->rev_cond, 0), VOIDmode)
970 52387 : && general_operand (XEXP (if_info->rev_cond, 1), VOIDmode))
971 : {
972 17781 : cond = if_info->rev_cond;
973 17781 : reversep = false;
974 : }
975 :
976 58521 : if (reversep)
977 4410 : code = reversed_comparison_code (cond, if_info->jump);
978 : else
979 54111 : code = GET_CODE (cond);
980 :
981 : /* reversed_comparison_code returns UNKNOWN for an unordered code, or a
982 : CC-mode compare it cannot trace; neither path below can use that. */
983 58521 : if (code == UNKNOWN)
984 : return NULL_RTX;
985 :
986 58521 : if ((if_info->cond_earliest == if_info->jump || cond_complex)
987 23915 : && (normalize == 0 || STORE_FLAG_VALUE == normalize))
988 : {
989 18195 : rtx src = gen_rtx_fmt_ee (code, GET_MODE (x), XEXP (cond, 0),
990 : XEXP (cond, 1));
991 18195 : rtx set = gen_rtx_SET (x, src);
992 :
993 18195 : start_sequence ();
994 18195 : rtx_insn *insn = emit_insn (set);
995 :
996 18195 : if (recog_memoized (insn) >= 0)
997 : {
998 7910 : rtx_insn *seq = end_sequence ();
999 7910 : emit_insn (seq);
1000 :
1001 7910 : if_info->cond_earliest = if_info->jump;
1002 :
1003 7910 : return x;
1004 : }
1005 :
1006 10285 : end_sequence ();
1007 : }
1008 :
1009 : /* Don't even try if the comparison operands or the mode of X are weird. */
1010 50611 : if (cond_complex || !SCALAR_INT_MODE_P (GET_MODE (x)))
1011 : return NULL_RTX;
1012 :
1013 : /* Don't try if mode of X is more than the max fixed mode size. */
1014 103788 : if (known_le (MAX_FIXED_MODE_SIZE, GET_MODE_BITSIZE (GET_MODE (x))))
1015 : return NULL_RTX;
1016 :
1017 33261 : return emit_store_flag (x, code, XEXP (cond, 0),
1018 : XEXP (cond, 1), VOIDmode,
1019 33261 : (code == LTU || code == LEU
1020 66522 : || code == GEU || code == GTU), normalize);
1021 : }
1022 :
1023 : /* Return true if X can be safely forced into a register by copy_to_mode_reg
1024 : / force_operand. */
1025 :
1026 : static bool
1027 2096855 : noce_can_force_operand (rtx x)
1028 : {
1029 2097300 : if (general_operand (x, VOIDmode))
1030 : return true;
1031 557361 : if (SUBREG_P (x))
1032 : {
1033 445 : if (!noce_can_force_operand (SUBREG_REG (x)))
1034 : return false;
1035 : return true;
1036 : }
1037 556916 : if (ARITHMETIC_P (x))
1038 : {
1039 463646 : if (!noce_can_force_operand (XEXP (x, 0))
1040 463646 : || !noce_can_force_operand (XEXP (x, 1)))
1041 : return false;
1042 457872 : switch (GET_CODE (x))
1043 : {
1044 : case MULT:
1045 : case MOD:
1046 : case UDIV:
1047 : case UMOD:
1048 : return true;
1049 1851 : case DIV:
1050 1851 : if (INTEGRAL_MODE_P (GET_MODE (x)))
1051 : return true;
1052 : /* FALLTHRU */
1053 443963 : default:
1054 443963 : auto optab = code_to_optab (GET_CODE (x));
1055 443963 : if (!optab)
1056 : return false;
1057 433991 : return optab_handler (optab, GET_MODE (x)) != CODE_FOR_nothing;
1058 : }
1059 : }
1060 93270 : if (UNARY_P (x))
1061 : {
1062 84196 : if (!noce_can_force_operand (XEXP (x, 0)))
1063 : return false;
1064 84175 : switch (GET_CODE (x))
1065 : {
1066 : case ZERO_EXTEND:
1067 : case SIGN_EXTEND:
1068 : case TRUNCATE:
1069 : case FLOAT_EXTEND:
1070 : case FLOAT_TRUNCATE:
1071 : case FIX:
1072 : case UNSIGNED_FIX:
1073 : case FLOAT:
1074 : case UNSIGNED_FLOAT:
1075 : return true;
1076 13265 : default:
1077 13265 : auto optab = code_to_optab (GET_CODE (x));
1078 13265 : if (!optab)
1079 : return false;
1080 13265 : return optab_handler (optab, GET_MODE (x)) != CODE_FOR_nothing;
1081 : }
1082 : }
1083 : return false;
1084 : }
1085 :
1086 : /* Emit instruction to move an rtx, possibly into STRICT_LOW_PART.
1087 : X is the destination/target and Y is the value to copy. */
1088 :
1089 : static void
1090 148219 : noce_emit_move_insn (rtx x, rtx y)
1091 : {
1092 148219 : machine_mode outmode;
1093 148219 : rtx outer, inner;
1094 148219 : poly_int64 bitpos;
1095 :
1096 148219 : if (GET_CODE (x) != STRICT_LOW_PART)
1097 : {
1098 148219 : rtx_insn *seq, *insn;
1099 148219 : rtx target;
1100 148219 : optab ot;
1101 :
1102 148219 : start_sequence ();
1103 : /* Check that the SET_SRC is reasonable before calling emit_move_insn,
1104 : otherwise construct a suitable SET pattern ourselves. */
1105 148299 : insn = (OBJECT_P (y) || CONSTANT_P (y) || GET_CODE (y) == SUBREG)
1106 148299 : ? emit_move_insn (x, y)
1107 109324 : : emit_insn (gen_rtx_SET (x, y));
1108 148219 : seq = end_sequence ();
1109 :
1110 148219 : if (recog_memoized (insn) <= 0)
1111 : {
1112 68706 : if (GET_CODE (x) == ZERO_EXTRACT)
1113 : {
1114 0 : rtx op = XEXP (x, 0);
1115 0 : unsigned HOST_WIDE_INT size = INTVAL (XEXP (x, 1));
1116 0 : unsigned HOST_WIDE_INT start = INTVAL (XEXP (x, 2));
1117 :
1118 : /* store_bit_field expects START to be relative to
1119 : BYTES_BIG_ENDIAN and adjusts this value for machines with
1120 : BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN. In order to be able to
1121 : invoke store_bit_field again it is necessary to have the START
1122 : value from the first call. */
1123 0 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
1124 : {
1125 : if (MEM_P (op))
1126 : start = BITS_PER_UNIT - start - size;
1127 : else
1128 : {
1129 : gcc_assert (REG_P (op));
1130 : start = BITS_PER_WORD - start - size;
1131 : }
1132 : }
1133 :
1134 0 : gcc_assert (start < (MEM_P (op) ? BITS_PER_UNIT : BITS_PER_WORD));
1135 0 : store_bit_field (op, size, start, 0, 0, GET_MODE (x), y, false,
1136 : false);
1137 0 : return;
1138 : }
1139 :
1140 68706 : switch (GET_RTX_CLASS (GET_CODE (y)))
1141 : {
1142 1058 : case RTX_UNARY:
1143 1058 : ot = code_to_optab (GET_CODE (y));
1144 1058 : if (ot && noce_can_force_operand (XEXP (y, 0)))
1145 : {
1146 820 : start_sequence ();
1147 820 : target = expand_unop (GET_MODE (y), ot, XEXP (y, 0), x, 0);
1148 820 : if (target != NULL_RTX)
1149 : {
1150 820 : if (target != x)
1151 0 : emit_move_insn (x, target);
1152 820 : seq = get_insns ();
1153 : }
1154 820 : end_sequence ();
1155 : }
1156 : break;
1157 :
1158 36815 : case RTX_BIN_ARITH:
1159 36815 : case RTX_COMM_ARITH:
1160 36815 : ot = code_to_optab (GET_CODE (y));
1161 36815 : if (ot
1162 36815 : && noce_can_force_operand (XEXP (y, 0))
1163 73630 : && noce_can_force_operand (XEXP (y, 1)))
1164 : {
1165 36815 : start_sequence ();
1166 36815 : target = expand_binop (GET_MODE (y), ot,
1167 : XEXP (y, 0), XEXP (y, 1),
1168 : x, 0, OPTAB_DIRECT);
1169 36815 : if (target != NULL_RTX)
1170 : {
1171 36815 : if (target != x)
1172 0 : emit_move_insn (x, target);
1173 36815 : seq = get_insns ();
1174 : }
1175 36815 : end_sequence ();
1176 : }
1177 : break;
1178 :
1179 : default:
1180 : break;
1181 : }
1182 : }
1183 :
1184 148219 : emit_insn (seq);
1185 148219 : return;
1186 : }
1187 :
1188 0 : outer = XEXP (x, 0);
1189 0 : inner = XEXP (outer, 0);
1190 0 : outmode = GET_MODE (outer);
1191 0 : bitpos = SUBREG_BYTE (outer) * BITS_PER_UNIT;
1192 0 : store_bit_field (inner, GET_MODE_BITSIZE (outmode), bitpos,
1193 : 0, 0, outmode, y, false, false);
1194 : }
1195 :
1196 : /* Return the CC reg if it is used in COND. */
1197 :
1198 : static rtx
1199 4151628 : cc_in_cond (rtx cond)
1200 : {
1201 4151628 : if (have_cbranchcc4 && cond
1202 4151628 : && GET_MODE_CLASS (GET_MODE (XEXP (cond, 0))) == MODE_CC)
1203 142740 : return XEXP (cond, 0);
1204 :
1205 : return NULL_RTX;
1206 : }
1207 :
1208 : struct noce_cc_reg_set
1209 : {
1210 : HARD_REG_SET set;
1211 : unsigned int fixed_regno1;
1212 : unsigned int fixed_regno2;
1213 : };
1214 :
1215 : /* If DEST is a hard condition-code register, add it to the set in DATA.
1216 : PAT is the SET or CLOBBER that writes DEST. */
1217 :
1218 : static void
1219 1013247 : noce_record_cc_reg_set (rtx dest, const_rtx pat ATTRIBUTE_UNUSED, void *data)
1220 : {
1221 1013247 : noce_cc_reg_set *cc_regs = (noce_cc_reg_set *) data;
1222 :
1223 1011645 : if (REG_P (dest) && HARD_REGISTER_P (dest)
1224 1370516 : && (GET_MODE_CLASS (GET_MODE (dest)) == MODE_CC
1225 0 : || REGNO (dest) == targetm.flags_regnum
1226 0 : || REGNO (dest) == cc_regs->fixed_regno1
1227 0 : || REGNO (dest) == cc_regs->fixed_regno2))
1228 357269 : add_to_hard_reg_set (&cc_regs->set, GET_MODE (dest), REGNO (dest));
1229 1013247 : }
1230 :
1231 : /* Return true if SEQ writes a condition-code register that still holds a
1232 : value which is live on exit from TEST_BB.
1233 :
1234 : An if-converted sequence is emitted immediately before the jump that ends
1235 : TEST_BB, and the conversion then removes that jump, so a condition code
1236 : that is live out of TEST_BB is one which a later block still reads.
1237 : Expanding a conditional move can emit a fresh comparison, and that would
1238 : destroy it. */
1239 :
1240 : static bool
1241 227241 : noce_clobbers_live_cc_p (basic_block test_bb, rtx_insn *seq)
1242 : {
1243 227241 : noce_cc_reg_set cc_regs;
1244 227241 : CLEAR_HARD_REG_SET (cc_regs.set);
1245 227241 : cc_regs.fixed_regno1 = INVALID_REGNUM;
1246 227241 : cc_regs.fixed_regno2 = INVALID_REGNUM;
1247 227241 : targetm.fixed_condition_code_regs (&cc_regs.fixed_regno1,
1248 : &cc_regs.fixed_regno2);
1249 1318380 : for (rtx_insn *insn = seq; insn; insn = NEXT_INSN (insn))
1250 863898 : note_stores (insn, noce_record_cc_reg_set, &cc_regs);
1251 :
1252 227241 : bitmap live_out = df_get_live_out (test_bb);
1253 227241 : hard_reg_set_iterator hrsi;
1254 227241 : unsigned int regno;
1255 415080 : EXECUTE_IF_SET_IN_HARD_REG_SET (cc_regs.set, 0, regno, hrsi)
1256 188251 : if (bitmap_bit_p (live_out, regno))
1257 : return true;
1258 :
1259 : return false;
1260 : }
1261 :
1262 : /* Return sequence of instructions generated by if conversion. This
1263 : function calls end_sequence() to end the current stream, ensures
1264 : that the instructions are unshared, recognizable non-jump insns.
1265 : On failure, this function returns a NULL_RTX. */
1266 :
1267 : static rtx_insn *
1268 226073 : end_ifcvt_sequence (noce_if_info *if_info)
1269 : {
1270 226073 : rtx_insn *insn;
1271 226073 : rtx_insn *seq = get_insns ();
1272 226073 : rtx cc = cc_in_cond (if_info->cond);
1273 :
1274 226073 : set_used_flags (if_info->x);
1275 226073 : set_used_flags (if_info->cond);
1276 226073 : set_used_flags (if_info->a);
1277 226073 : set_used_flags (if_info->b);
1278 :
1279 1166065 : for (insn = seq; insn; insn = NEXT_INSN (insn))
1280 713919 : set_used_flags (insn);
1281 :
1282 226073 : unshare_all_rtl_in_chain (seq);
1283 226073 : end_sequence ();
1284 :
1285 : /* Make sure that all of the instructions emitted are recognizable,
1286 : and that we haven't introduced a new jump instruction.
1287 : As an exercise for the reader, build a general mechanism that
1288 : allows proper placement of required clobbers. */
1289 1133617 : for (insn = seq; insn; insn = NEXT_INSN (insn))
1290 713417 : if (JUMP_P (insn)
1291 713417 : || recog_memoized (insn) == -1
1292 : /* Make sure new generated code does not clobber CC. */
1293 1394963 : || (cc && set_of (cc, insn)))
1294 : return NULL;
1295 :
1296 : /* CC above only covers the condition code read by the converted branch. */
1297 194127 : if (noce_clobbers_live_cc_p (if_info->test_bb, seq))
1298 412 : return NULL;
1299 :
1300 : return seq;
1301 : }
1302 :
1303 : /* Return true iff the then and else basic block (if it exists)
1304 : consist of a single simple set instruction. */
1305 :
1306 : static bool
1307 3171219 : noce_simple_bbs (noce_if_info *if_info)
1308 : {
1309 0 : if (!if_info->then_simple)
1310 : return false;
1311 :
1312 2937608 : if (if_info->else_bb)
1313 0 : return if_info->else_simple;
1314 :
1315 : return true;
1316 : }
1317 :
1318 : /* Commit the wound-up candidate SEQ produced by a noce transform for IF_INFO.
1319 : Reject it (returning false, emitting nothing) if it failed to build, or if
1320 : CHECK_PROFITABLE and the target deems it not worthwhile. Otherwise emit it
1321 : before the branch, record NAME as the winning transform and return true.
1322 : This is the shared tail of the noce_try_* matchers. */
1323 :
1324 : static bool
1325 211217 : noce_commit_sequence (noce_if_info *if_info, rtx_insn *seq,
1326 : bool check_profitable, const char *name)
1327 : {
1328 211217 : if (!seq
1329 136585 : || (check_profitable
1330 135073 : && !targetm.noce_conversion_profitable_p (seq, if_info)))
1331 : return false;
1332 :
1333 141164 : emit_insn_before_setloc (seq, if_info->jump,
1334 141164 : INSN_LOCATION (if_info->insn_a));
1335 141164 : if_info->transform_name = name;
1336 141164 : return true;
1337 : }
1338 :
1339 : /* Convert "if (a != b) x = a; else x = b" into "x = a" and
1340 : "if (a == b) x = a; else x = b" into "x = b". */
1341 :
1342 : static bool
1343 252720 : noce_try_move (noce_if_info *if_info)
1344 : {
1345 252720 : rtx cond = if_info->cond;
1346 252720 : enum rtx_code code = GET_CODE (cond);
1347 252720 : rtx y;
1348 252720 : rtx_insn *seq;
1349 :
1350 252720 : if (code != NE && code != EQ)
1351 : return false;
1352 :
1353 256651 : if (!noce_simple_bbs (if_info))
1354 : return false;
1355 :
1356 : /* This optimization isn't valid if either A or B could be a NaN
1357 : or a signed zero. */
1358 176527 : if (HONOR_NANS (if_info->x)
1359 176527 : || HONOR_SIGNED_ZEROS (if_info->x))
1360 : return false;
1361 :
1362 : /* Check whether the operands of the comparison are A and B, in
1363 : either order. */
1364 151421 : if ((rtx_equal_p (if_info->a, XEXP (cond, 0))
1365 2311 : && rtx_equal_p (if_info->b, XEXP (cond, 1)))
1366 153720 : || (rtx_equal_p (if_info->a, XEXP (cond, 1))
1367 15645 : && rtx_equal_p (if_info->b, XEXP (cond, 0))))
1368 : {
1369 21 : if (!rtx_interchangeable_p (if_info->a, if_info->b))
1370 : return false;
1371 :
1372 0 : y = (code == EQ) ? if_info->a : if_info->b;
1373 :
1374 : /* Avoid generating the move if the source is the destination. */
1375 0 : if (! rtx_equal_p (if_info->x, y))
1376 : {
1377 0 : start_sequence ();
1378 0 : noce_emit_move_insn (if_info->x, y);
1379 0 : seq = end_ifcvt_sequence (if_info);
1380 0 : if (!seq)
1381 : return false;
1382 :
1383 0 : emit_insn_before_setloc (seq, if_info->jump,
1384 0 : INSN_LOCATION (if_info->insn_a));
1385 : }
1386 0 : if_info->transform_name = "noce_try_move";
1387 0 : return true;
1388 : }
1389 : return false;
1390 : }
1391 :
1392 : /* TEMP holds an all-ones or all-zeros splat of a tested sign bit. Return an
1393 : rtx that narrows it to VAL or zero, expanded into TARGET where possible, or
1394 : NULL_RTX if the expansion failed.
1395 :
1396 : The obvious form is a mask, but since the value is known to be -1 or 0 a
1397 : shift is sometimes cheaper: a logical right shift constructs 2^n-1 and a
1398 : left shift constructs ~(2^n-1). Some targets do not have efficient shifts,
1399 : so build the RTL for each applicable form and expand the cheapest. */
1400 :
1401 : static rtx
1402 82 : noce_splat_to_const (machine_mode mode, rtx temp, HOST_WIDE_INT val,
1403 : rtx target)
1404 : {
1405 82 : rtx and_form = gen_rtx_AND (mode, temp, GEN_INT (val));
1406 164 : rtx shift_left = gen_rtx_ASHIFT (mode, temp, GEN_INT (ctz_hwi (val)));
1407 82 : HOST_WIDE_INT rshift_count
1408 164 : = clz_hwi (val) & (GET_MODE_PRECISION (mode).to_constant () - 1);
1409 82 : rtx shift_right = gen_rtx_LSHIFTRT (mode, temp, GEN_INT (rshift_count));
1410 82 : bool speed_p = optimize_insn_for_speed_p ();
1411 :
1412 82 : if (exact_log2 (val + 1) >= 0
1413 50 : && (rtx_cost (shift_right, mode, SET, 1, speed_p)
1414 25 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1415 3 : return expand_simple_binop (mode, LSHIFTRT, temp, GEN_INT (rshift_count),
1416 3 : target, false, OPTAB_WIDEN);
1417 :
1418 79 : if (exact_log2 (~val + 1) >= 0
1419 0 : && (rtx_cost (shift_left, mode, SET, 1, speed_p)
1420 0 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1421 0 : return expand_simple_binop (mode, ASHIFT, temp, GEN_INT (ctz_hwi (val)),
1422 0 : target, false, OPTAB_WIDEN);
1423 :
1424 79 : return expand_simple_binop (mode, AND, temp, GEN_INT (val), target, false,
1425 79 : OPTAB_WIDEN);
1426 : }
1427 :
1428 : /* If a sign bit test is selecting across constants, we may be able
1429 : to generate efficient code utilizing the -1/0 result of a sign
1430 : bit splat idiom.
1431 :
1432 : Do this before trying the generalized conditional move as these
1433 : (when applicable) are hopefully faster than a conditional move. */
1434 :
1435 : static bool
1436 200275 : noce_try_sign_bit_splat (noce_if_info *if_info)
1437 : {
1438 200275 : rtx cond = if_info->cond;
1439 200275 : enum rtx_code code = GET_CODE (cond);
1440 :
1441 : /* We're looking for sign bit tests, so only a few cases are
1442 : interesting. LT/GE 0 LE/GT -1. */
1443 200275 : if (((code == LT || code == GE)
1444 11194 : && XEXP (cond, 1) == CONST0_RTX (GET_MODE (cond)))
1445 197766 : || ((code == LE || code == GT)
1446 11352 : && XEXP (cond, 1) == CONSTM1_RTX (GET_MODE (cond))))
1447 : ;
1448 : else
1449 : return false;
1450 :
1451 : /* It would be good if this could be extended since constant synthesis
1452 : on some platforms will result in blocks which fail this test. */
1453 10876 : if (!noce_simple_bbs (if_info))
1454 : return false;
1455 :
1456 : /* Only try this for constants in the true/false arms and a REG
1457 : destination. We could select between 0 and a REG pretty
1458 : easily with a logical AND. */
1459 6059 : if (!CONST_INT_P (if_info->a)
1460 1990 : || !CONST_INT_P (if_info->b)
1461 681 : || !REG_P (if_info->x))
1462 : return false;
1463 :
1464 681 : machine_mode mode = GET_MODE (if_info->x);
1465 :
1466 : /* If the mode of the destination does not match the mode of
1467 : the value we're testing, then this optimization is not valid. */
1468 681 : if (mode != GET_MODE (XEXP (cond, 0)))
1469 : return false;
1470 :
1471 212 : HOST_WIDE_INT val_a = INTVAL (if_info->a);
1472 212 : HOST_WIDE_INT val_b = INTVAL (if_info->b);
1473 :
1474 212 : rtx_insn *seq;
1475 212 : start_sequence ();
1476 :
1477 : /* We're testing the sign bit of this operand. */
1478 212 : rtx condop = XEXP (cond, 0);
1479 :
1480 : /* To splat the sign bit we arithmetically shift the
1481 : input value right by the size of the object - 1 bits.
1482 :
1483 : Note some targets do not have strong shifters, but do have
1484 : alternative ways to generate the sign bit splat. The
1485 : profitability test when we end the sequence should reject
1486 : cases when the branchy sequence is better. */
1487 212 : int splat_count = GET_MODE_BITSIZE (GET_MODE (condop)).to_constant () - 1;
1488 212 : rtx splat = GEN_INT (splat_count);
1489 :
1490 212 : rtx temp;
1491 212 : temp = expand_simple_binop (GET_MODE (XEXP (cond, 0)), ASHIFTRT,
1492 : XEXP (cond, 0), splat, NULL_RTX,
1493 : false, OPTAB_WIDEN);
1494 212 : if (!temp)
1495 0 : goto fail;
1496 :
1497 : /* IOR of anything with -1 still results in -1. So we can
1498 : IOR the other operand to generate a select between -1 and
1499 : an arbitrary constant. */
1500 212 : if (val_a == -1)
1501 : {
1502 50 : if (code == LT || code == LE)
1503 : {
1504 2 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1505 2 : if (!temp)
1506 0 : goto fail;
1507 : }
1508 :
1509 50 : temp = expand_simple_binop (mode, IOR, temp, GEN_INT (val_b),
1510 : if_info->x, false, OPTAB_WIDEN);
1511 : }
1512 : /* AND of anything with 0 is still zero. So we can AND
1513 : with the -1 operand with the a constant to select
1514 : between the constant and zero. */
1515 162 : else if (val_b == 0)
1516 : {
1517 0 : if (code == LT || code == LE)
1518 : {
1519 0 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1520 0 : if (!temp)
1521 0 : goto fail;
1522 : }
1523 :
1524 0 : temp = noce_splat_to_const (mode, temp, val_a, if_info->x);
1525 : }
1526 : /* Same cases, but with the test or arms swapped. These
1527 : can be realized as well, though it typically costs
1528 : an extra instruction. */
1529 162 : else if (val_b == -1)
1530 : {
1531 46 : if (code != LT && code != LE)
1532 : {
1533 2 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1534 2 : if (!temp)
1535 0 : goto fail;
1536 : }
1537 :
1538 46 : temp = expand_simple_binop (mode, IOR, temp, GEN_INT (val_a),
1539 : if_info->x, false, OPTAB_WIDEN);
1540 : }
1541 116 : else if (val_a == 0)
1542 : {
1543 82 : if (code != LT && code != LE)
1544 : {
1545 9 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1546 9 : if (!temp)
1547 0 : goto fail;
1548 : }
1549 :
1550 82 : temp = noce_splat_to_const (mode, temp, val_b, if_info->x);
1551 : }
1552 : /* Nothing worked. */
1553 : else
1554 : temp = NULL_RTX;
1555 :
1556 178 : if (!temp)
1557 34 : goto fail;
1558 :
1559 : /* Move into the final destination if the value wasn't
1560 : constructed there. */
1561 178 : if (if_info->x != temp)
1562 0 : emit_move_insn (if_info->x, temp);
1563 :
1564 : /* This ends the sequence and tests the cost model. */
1565 178 : seq = end_ifcvt_sequence (if_info);
1566 178 : return noce_commit_sequence (if_info, seq, true, "splat_sign_bit_trivial");
1567 :
1568 34 : fail:
1569 34 : end_ifcvt_sequence (if_info);
1570 34 : return false;
1571 : }
1572 :
1573 :
1574 : /* Try forming an IF_THEN_ELSE (cond, b, a) and collapsing that
1575 : through simplify_rtx. Sometimes that can eliminate the IF_THEN_ELSE.
1576 : If that is the case, emit the result into x. */
1577 :
1578 : static bool
1579 252720 : noce_try_ifelse_collapse (noce_if_info * if_info)
1580 : {
1581 338605 : if (!noce_simple_bbs (if_info))
1582 : return false;
1583 :
1584 234042 : machine_mode mode = GET_MODE (if_info->x);
1585 234042 : rtx if_then_else = simplify_gen_ternary (IF_THEN_ELSE, mode, mode,
1586 : if_info->cond, if_info->b,
1587 : if_info->a);
1588 :
1589 234042 : if (GET_CODE (if_then_else) == IF_THEN_ELSE)
1590 : return false;
1591 :
1592 48631 : rtx_insn *seq;
1593 48631 : start_sequence ();
1594 48631 : noce_emit_move_insn (if_info->x, if_then_else);
1595 48631 : seq = end_ifcvt_sequence (if_info);
1596 48631 : return noce_commit_sequence (if_info, seq, false, "noce_try_ifelse_collapse");
1597 : }
1598 :
1599 :
1600 : /* Convert "if (test) x = 1; else x = 0".
1601 :
1602 : Only try 0 and STORE_FLAG_VALUE here. Other combinations will be
1603 : tried in noce_try_store_flag_constants after noce_try_cmove has had
1604 : a go at the conversion. */
1605 :
1606 : static bool
1607 234922 : noce_try_store_flag (noce_if_info *if_info)
1608 : {
1609 234922 : bool reversep;
1610 234922 : rtx target;
1611 234922 : rtx_insn *seq;
1612 :
1613 307523 : if (!noce_simple_bbs (if_info))
1614 : return false;
1615 :
1616 216244 : if (CONST_INT_P (if_info->b)
1617 73873 : && INTVAL (if_info->b) == STORE_FLAG_VALUE
1618 13355 : && if_info->a == const0_rtx)
1619 : reversep = false;
1620 205759 : else if (if_info->b == const0_rtx
1621 30457 : && CONST_INT_P (if_info->a)
1622 20804 : && INTVAL (if_info->a) == STORE_FLAG_VALUE)
1623 : reversep = true;
1624 : else
1625 : return false;
1626 :
1627 30834 : start_sequence ();
1628 :
1629 30834 : target = noce_emit_store_flag (if_info, if_info->x, reversep, 0);
1630 30834 : if (target)
1631 : {
1632 26001 : if (target != if_info->x)
1633 2982 : noce_emit_move_insn (if_info->x, target);
1634 :
1635 26001 : seq = end_ifcvt_sequence (if_info);
1636 26001 : return noce_commit_sequence (if_info, seq, false, "noce_try_store_flag");
1637 : }
1638 : else
1639 : {
1640 4833 : end_sequence ();
1641 4833 : return false;
1642 : }
1643 : }
1644 :
1645 :
1646 : /* Convert "if (test) x = -A; else x = A" into
1647 : x = A; if (test) x = -x if the machine can do the
1648 : conditional negate form of this cheaply.
1649 : Try this before noce_try_cmove that will just load the
1650 : immediates into two registers and do a conditional select
1651 : between them. If the target has a conditional negate or
1652 : conditional invert operation we can save a potentially
1653 : expensive constant synthesis. */
1654 :
1655 : static bool
1656 208837 : noce_try_inverse_constants (noce_if_info *if_info)
1657 : {
1658 271799 : if (!noce_simple_bbs (if_info))
1659 : return false;
1660 :
1661 190159 : if (!CONST_INT_P (if_info->a)
1662 51593 : || !CONST_INT_P (if_info->b)
1663 21282 : || !REG_P (if_info->x))
1664 : return false;
1665 :
1666 21282 : machine_mode mode = GET_MODE (if_info->x);
1667 :
1668 21282 : HOST_WIDE_INT val_a = INTVAL (if_info->a);
1669 21282 : HOST_WIDE_INT val_b = INTVAL (if_info->b);
1670 :
1671 21282 : rtx cond = if_info->cond;
1672 :
1673 21282 : rtx x = if_info->x;
1674 21282 : rtx target;
1675 :
1676 21282 : start_sequence ();
1677 :
1678 21282 : rtx_code code;
1679 21282 : if (val_b != HOST_WIDE_INT_MIN && val_a == -val_b)
1680 : code = NEG;
1681 19991 : else if (val_a == ~val_b)
1682 : code = NOT;
1683 : else
1684 : {
1685 19923 : end_sequence ();
1686 19923 : return false;
1687 : }
1688 :
1689 1359 : rtx tmp = gen_reg_rtx (mode);
1690 1359 : noce_emit_move_insn (tmp, if_info->a);
1691 :
1692 1359 : target = emit_conditional_neg_or_complement (x, code, mode, cond, tmp, tmp);
1693 :
1694 1359 : if (target)
1695 : {
1696 0 : rtx_insn *seq = get_insns ();
1697 :
1698 0 : if (!seq)
1699 : {
1700 0 : end_sequence ();
1701 0 : return false;
1702 : }
1703 :
1704 0 : if (target != if_info->x)
1705 0 : noce_emit_move_insn (if_info->x, target);
1706 :
1707 0 : seq = end_ifcvt_sequence (if_info);
1708 0 : return noce_commit_sequence (if_info, seq, false,
1709 : "noce_try_inverse_constants");
1710 : }
1711 :
1712 1359 : end_sequence ();
1713 1359 : return false;
1714 : }
1715 :
1716 : /* Check if OP is supported by conditional zero based if conversion,
1717 : returning TRUE if satisfied otherwise FALSE.
1718 :
1719 : OP is the operation to check. */
1720 :
1721 : static bool
1722 586348 : noce_cond_zero_binary_op_supported (rtx op)
1723 : {
1724 586348 : enum rtx_code opcode = GET_CODE (op);
1725 :
1726 0 : if (opcode == PLUS || opcode == MINUS || opcode == IOR || opcode == XOR
1727 : || opcode == ASHIFT || opcode == ASHIFTRT || opcode == LSHIFTRT
1728 : || opcode == ROTATE || opcode == ROTATERT || opcode == AND)
1729 0 : return true;
1730 :
1731 : return false;
1732 : }
1733 :
1734 : /* Convert "if (test) x = a; else x = a OP c " where c is 2^n.
1735 :
1736 : We can shift the output of a set flag insn left to generate 2^n
1737 : cheaply, then apply OP unconditionally. This works even if the
1738 : target does not have a conditional zero idiom. On targets such
1739 : as RISC-V this sequence may also encode better.
1740 :
1741 : This is based on noce_try_store_flag_constants. */
1742 :
1743 : static bool
1744 205676 : noce_try_shifted_store_flag (noce_if_info *if_info)
1745 : {
1746 205676 : rtx target;
1747 205676 : rtx_insn *seq;
1748 205676 : machine_mode mode = GET_MODE (if_info->x);
1749 205676 : rtx common = NULL_RTX;
1750 205676 : rtx_code code = UNKNOWN;
1751 :
1752 205676 : if (STORE_FLAG_VALUE != 1)
1753 : return false;
1754 :
1755 265551 : if (!noce_simple_bbs (if_info))
1756 : return false;
1757 :
1758 186998 : rtx a = if_info->a;
1759 186998 : rtx b = if_info->b;
1760 :
1761 : /* Most binary operators are allowed, essentially the same set
1762 : as the condzero arithmetic paths, with the exception of AND
1763 : which could be handled if we were inclined. */
1764 186998 : int shiftval;
1765 186998 : bool reversed = false;
1766 186998 : if (noce_cond_zero_binary_op_supported (a)
1767 24216 : && GET_CODE (a) != AND
1768 24010 : && GET_CODE (b) == REG
1769 20385 : && rtx_equal_p (XEXP (a, 0), b)
1770 18403 : && CONST_INT_P (XEXP (a, 1))
1771 8979 : && pow2p_hwi (INTVAL (XEXP (a, 1))))
1772 : {
1773 6634 : code = GET_CODE (a);
1774 6634 : common = XEXP (a, 0);
1775 6634 : shiftval = exact_log2 (INTVAL (XEXP (a, 1)));
1776 :
1777 : /* When the condition is true, we branch around A and thus we want
1778 : the condition reversed from what you might expect. */
1779 : reversed = true;
1780 : }
1781 180364 : else if (noce_cond_zero_binary_op_supported (b)
1782 2141 : && GET_CODE (b) != AND
1783 2110 : && GET_CODE (a) == REG
1784 230 : && rtx_equal_p (XEXP (b, 0), a)
1785 89 : && CONST_INT_P (XEXP (b, 1))
1786 62 : && pow2p_hwi (INTVAL (XEXP (b, 1))))
1787 : {
1788 31 : code = GET_CODE (b);
1789 31 : common = XEXP (b, 0);
1790 31 : shiftval = exact_log2 (INTVAL (XEXP (b, 1)));
1791 : }
1792 :
1793 :
1794 : /* If CODE hasn't been reset, then the basic expression form was wrong and
1795 : we can not optimize. */
1796 6665 : if (code == UNKNOWN)
1797 : return false;
1798 :
1799 6665 : start_sequence ();
1800 :
1801 : /* If X and COMMON are the same, then we're going to need a temporary. */
1802 6665 : if (rtx_equal_p (common, if_info->x))
1803 : {
1804 6068 : common = gen_reg_rtx (mode);
1805 6068 : noce_emit_move_insn (common, if_info->x);
1806 : }
1807 :
1808 6665 : target = noce_emit_store_flag (if_info, if_info->x, reversed, 1);
1809 6665 : if (!target)
1810 : {
1811 745 : end_sequence ();
1812 745 : return false;
1813 : }
1814 :
1815 : /* Right now TARGET has the value 1/0. A left shift will produce the
1816 : target value. Unnecessary if the shift value is zero. */
1817 5920 : if (shiftval != 0)
1818 1218 : target = expand_simple_binop (mode, ASHIFT, target,
1819 : GEN_INT (shiftval),
1820 : NULL_RTX, 0, OPTAB_WIDEN);
1821 :
1822 5920 : if (!target)
1823 : {
1824 0 : end_sequence ();
1825 0 : return false;
1826 : }
1827 :
1828 : /* Now we just need to apply the code to COMMON and TARGET. */
1829 5920 : target = expand_simple_binop (mode, code,
1830 : common, target,
1831 : NULL_RTX, 0, OPTAB_WIDEN);
1832 :
1833 5920 : if (!target)
1834 : {
1835 0 : end_sequence ();
1836 0 : return false;
1837 : }
1838 :
1839 : /* If necessary, store the result into the proper destination. */
1840 5920 : if (target != if_info->x)
1841 5920 : noce_emit_move_insn (if_info->x, target);
1842 :
1843 5920 : seq = end_ifcvt_sequence (if_info);
1844 5920 : return noce_commit_sequence (if_info, seq, true,
1845 5920 : "noce_try_shifted_store_flag");
1846 : }
1847 :
1848 :
1849 : /* Set *DIFF to ITRUE - IFALSE truncated to MODE and return true. Return
1850 : false, leaving *DIFF alone, when that difference is not representable and
1851 : so cannot be used to build a store-flag sequence. */
1852 :
1853 : static bool
1854 21774 : noce_representable_diff_p (HOST_WIDE_INT ifalse, HOST_WIDE_INT itrue,
1855 : machine_mode mode, HOST_WIDE_INT *diff)
1856 : {
1857 21774 : HOST_WIDE_INT d = (unsigned HOST_WIDE_INT) itrue - ifalse;
1858 :
1859 21774 : if ((d > 0) != ((ifalse < 0) != (itrue < 0) ? ifalse < 0 : ifalse < itrue))
1860 : return false;
1861 :
1862 21639 : *diff = trunc_int_for_mode (d, mode);
1863 21639 : return true;
1864 : }
1865 :
1866 : /* Convert "if (test) x = a; else x = b", for A and B constant.
1867 : Also allow A = y + c1, B = y + c2, with a common y between A
1868 : and B. */
1869 :
1870 : static bool
1871 208837 : noce_try_store_flag_constants (noce_if_info *if_info)
1872 : {
1873 208837 : rtx target;
1874 208837 : rtx_insn *seq;
1875 208837 : bool reversep;
1876 208837 : HOST_WIDE_INT itrue, ifalse, diff, tmp;
1877 208837 : int normalize;
1878 208837 : bool can_reverse;
1879 208837 : machine_mode mode = GET_MODE (if_info->x);
1880 208837 : rtx common = NULL_RTX;
1881 :
1882 208837 : rtx a = if_info->a;
1883 208837 : rtx b = if_info->b;
1884 :
1885 : /* Handle cases like x := test ? y + 3 : y + 4. */
1886 208837 : if (GET_CODE (a) == PLUS
1887 22334 : && GET_CODE (b) == PLUS
1888 1447 : && CONST_INT_P (XEXP (a, 1))
1889 570 : && CONST_INT_P (XEXP (b, 1))
1890 494 : && rtx_equal_p (XEXP (a, 0), XEXP (b, 0))
1891 : /* Allow expressions that are not using the result or plain
1892 : registers where we handle overlap below. */
1893 209281 : && (REG_P (XEXP (a, 0))
1894 6 : || (noce_operand_ok (XEXP (a, 0))
1895 6 : && ! reg_overlap_mentioned_p (if_info->x, XEXP (a, 0)))))
1896 : {
1897 444 : common = XEXP (a, 0);
1898 444 : a = XEXP (a, 1);
1899 444 : b = XEXP (b, 1);
1900 : }
1901 :
1902 271799 : if (!noce_simple_bbs (if_info))
1903 : return false;
1904 :
1905 190159 : if (CONST_INT_P (a)
1906 52025 : && CONST_INT_P (b))
1907 : {
1908 21714 : ifalse = INTVAL (a);
1909 21714 : itrue = INTVAL (b);
1910 21714 : bool subtract_flag_p = false;
1911 :
1912 21714 : if (!noce_representable_diff_p (ifalse, itrue, mode, &diff))
1913 : return false;
1914 :
1915 21600 : can_reverse = noce_reversed_cond_code (if_info) != UNKNOWN;
1916 21600 : reversep = false;
1917 21600 : if (diff == STORE_FLAG_VALUE || diff == -STORE_FLAG_VALUE)
1918 : {
1919 12009 : normalize = 0;
1920 : /* We could collapse these cases but it is easier to follow the
1921 : diff/STORE_FLAG_VALUE combinations when they are listed
1922 : explicitly. */
1923 :
1924 : /* test ? 3 : 4
1925 : => 4 + (test != 0). */
1926 12009 : if (diff < 0 && STORE_FLAG_VALUE < 0)
1927 : reversep = false;
1928 : /* test ? 4 : 3
1929 : => can_reverse | 4 + (test == 0)
1930 : !can_reverse | 3 - (test != 0). */
1931 12009 : else if (diff > 0 && STORE_FLAG_VALUE < 0)
1932 : {
1933 : reversep = can_reverse;
1934 : subtract_flag_p = !can_reverse;
1935 : /* If we need to subtract the flag and we have PLUS-immediate
1936 : A and B then it is unlikely to be beneficial to play tricks
1937 : here. */
1938 : if (subtract_flag_p && common)
1939 : return false;
1940 : }
1941 : /* test ? 3 : 4
1942 : => can_reverse | 3 + (test == 0)
1943 : !can_reverse | 4 - (test != 0). */
1944 12009 : else if (diff < 0 && STORE_FLAG_VALUE > 0)
1945 : {
1946 6105 : reversep = can_reverse;
1947 6105 : subtract_flag_p = !can_reverse;
1948 : /* If we need to subtract the flag and we have PLUS-immediate
1949 : A and B then it is unlikely to be beneficial to play tricks
1950 : here. */
1951 6105 : if (subtract_flag_p && common)
1952 : return false;
1953 : }
1954 : /* test ? 4 : 3
1955 : => 4 + (test != 0). */
1956 : else if (diff > 0 && STORE_FLAG_VALUE > 0)
1957 : reversep = false;
1958 : else
1959 : gcc_unreachable ();
1960 : }
1961 : /* Is this (cond) ? 2^n : 0? */
1962 532 : else if (ifalse == 0 && pow2p_hwi (itrue)
1963 9591 : && STORE_FLAG_VALUE == 1)
1964 : normalize = 1;
1965 : /* Is this (cond) ? 0 : 2^n? */
1966 388 : else if (itrue == 0 && pow2p_hwi (ifalse)
1967 9584 : && STORE_FLAG_VALUE == 1)
1968 : {
1969 : normalize = 1;
1970 : reversep = true;
1971 : }
1972 : /* Is this (cond) ? -1 : x? */
1973 : else if (itrue == -1
1974 : && STORE_FLAG_VALUE == -1)
1975 : normalize = -1;
1976 : /* Is this (cond) ? x : -1? */
1977 : else if (ifalse == -1
1978 : && STORE_FLAG_VALUE == -1)
1979 : {
1980 : normalize = -1;
1981 : reversep = true;
1982 : }
1983 : else
1984 : return false;
1985 :
1986 6105 : if (reversep)
1987 : {
1988 6106 : std::swap (itrue, ifalse);
1989 6106 : diff = trunc_int_for_mode (-(unsigned HOST_WIDE_INT) diff, mode);
1990 : }
1991 :
1992 12017 : start_sequence ();
1993 :
1994 : /* If we have x := test ? x + 3 : x + 4 then move the original
1995 : x out of the way while we store flags. */
1996 12017 : if (common && rtx_equal_p (common, if_info->x))
1997 : {
1998 5 : common = gen_reg_rtx (mode);
1999 5 : noce_emit_move_insn (common, if_info->x);
2000 : }
2001 :
2002 12017 : target = noce_emit_store_flag (if_info, if_info->x, reversep, normalize);
2003 12017 : if (! target)
2004 : {
2005 4869 : end_sequence ();
2006 4869 : return false;
2007 : }
2008 :
2009 : /* if (test) x = 3; else x = 4;
2010 : => x = 3 + (test == 0); */
2011 7148 : if (diff == STORE_FLAG_VALUE || diff == -STORE_FLAG_VALUE)
2012 : {
2013 : /* Add the common part now. This may allow combine to merge this
2014 : with the store flag operation earlier into some sort of conditional
2015 : increment/decrement if the target allows it. */
2016 7140 : if (common)
2017 28 : target = expand_simple_binop (mode, PLUS,
2018 : target, common,
2019 : target, 0, OPTAB_WIDEN);
2020 :
2021 : /* Always use ifalse here. It should have been swapped with itrue
2022 : when appropriate when reversep is true. */
2023 14280 : target = expand_simple_binop (mode, subtract_flag_p ? MINUS : PLUS,
2024 7140 : gen_int_mode (ifalse, mode), target,
2025 : if_info->x, 0, OPTAB_WIDEN);
2026 : }
2027 : /* Other cases are not beneficial when the original A and B are PLUS
2028 : expressions. */
2029 8 : else if (common)
2030 : {
2031 0 : end_sequence ();
2032 0 : return false;
2033 : }
2034 : /* if (test) x = 8; else x = 0;
2035 : => x = (test != 0) << 3; */
2036 8 : else if (ifalse == 0 && (tmp = exact_log2 (itrue)) >= 0)
2037 : {
2038 8 : target = expand_simple_binop (mode, ASHIFT,
2039 : target, GEN_INT (tmp), if_info->x, 0,
2040 : OPTAB_WIDEN);
2041 : }
2042 :
2043 : /* if (test) x = -1; else x = b;
2044 : => x = -(test != 0) | b; */
2045 0 : else if (itrue == -1)
2046 : {
2047 0 : target = expand_simple_binop (mode, IOR,
2048 0 : target, gen_int_mode (ifalse, mode),
2049 : if_info->x, 0, OPTAB_WIDEN);
2050 : }
2051 : else
2052 : {
2053 0 : end_sequence ();
2054 0 : return false;
2055 : }
2056 :
2057 7148 : if (! target)
2058 : {
2059 0 : end_sequence ();
2060 0 : return false;
2061 : }
2062 :
2063 7148 : if (target != if_info->x)
2064 0 : noce_emit_move_insn (if_info->x, target);
2065 :
2066 7148 : seq = end_ifcvt_sequence (if_info);
2067 7148 : return noce_commit_sequence (if_info, seq, true,
2068 7148 : "noce_try_store_flag_constants");
2069 : }
2070 :
2071 : return false;
2072 : }
2073 :
2074 : /* This is trying to capture cases like dest = cond ? a : -1.
2075 :
2076 : The basic idea is to use a store-flag insn to generate a -1/0
2077 : value (directly or indirectly), then IOR that with the other
2078 : input. */
2079 : static bool
2080 200097 : noce_try_store_flag_logical (noce_if_info *if_info)
2081 : {
2082 200097 : rtx a = if_info->a;
2083 200097 : rtx b = if_info->b;
2084 200097 : rtx dest = if_info->x;
2085 200097 : machine_mode mode = GET_MODE (dest);
2086 :
2087 200097 : if (STORE_FLAG_VALUE != -1 && STORE_FLAG_VALUE != 1)
2088 : return false;
2089 :
2090 259669 : if (!noce_simple_bbs (if_info))
2091 : return false;
2092 :
2093 181419 : bool swapped = STORE_FLAG_VALUE == -1;
2094 181419 : if (a == CONSTM1_RTX (GET_MODE (dest))
2095 1644 : && if_info->rev_cond)
2096 : {
2097 : std::swap (a, b);
2098 181419 : swapped = !swapped;
2099 : }
2100 :
2101 181419 : if (b != CONSTM1_RTX (GET_MODE (dest)))
2102 : return false;
2103 :
2104 : /* If the other arm is not a REG/SUBREG, then punt. This is primarily to
2105 : let the target handle the constant case, which it can likely do better.
2106 : It also means we don't have to worry about non terminal expressions. */
2107 2518 : if (!REG_P (a) && !SUBREG_P (a))
2108 : return false;
2109 :
2110 : /* At this point we've got dest = cond ? a : -1. Emit the store flag and
2111 : adjust its value (if necessary) to -1/0. */
2112 999 : start_sequence ();
2113 999 : rtx temp = gen_reg_rtx (mode);
2114 999 : rtx target = noce_emit_store_flag (if_info, temp, !swapped, 0);
2115 999 : if (!target)
2116 : {
2117 88 : end_sequence ();
2118 88 : return false;
2119 : }
2120 :
2121 911 : if (STORE_FLAG_VALUE == 1)
2122 : {
2123 911 : rtx x = gen_rtx_PLUS (mode, target, CONSTM1_RTX (mode));
2124 911 : emit_move_insn (target, x);
2125 : }
2126 :
2127 : /* Now we've got -1/0 in TARGET. We can just IOR with A. */
2128 911 : rtx x = gen_rtx_IOR (mode, target, a);
2129 911 : emit_move_insn (dest, x);
2130 :
2131 : /* We've generated all the RTL, make sure it recognizes and is
2132 : profitable. */
2133 911 : rtx_insn *seq = end_ifcvt_sequence (if_info);
2134 911 : return noce_commit_sequence (if_info, seq, true,
2135 911 : "noce_try_store_flag_logical");
2136 : }
2137 :
2138 : /* Convert "if (test) foo++" into "foo += (test != 0)", and
2139 : similarly for "foo--". */
2140 :
2141 : static bool
2142 157509 : noce_try_addcc (noce_if_info *if_info)
2143 : {
2144 157509 : rtx target;
2145 157509 : rtx_insn *seq;
2146 157509 : bool subtract;
2147 157509 : int normalize;
2148 :
2149 200578 : if (!noce_simple_bbs (if_info))
2150 : return false;
2151 :
2152 138831 : if (GET_CODE (if_info->a) == PLUS
2153 14775 : && rtx_equal_p (XEXP (if_info->a, 0), if_info->b)
2154 150061 : && noce_reversed_cond_code (if_info) != UNKNOWN)
2155 : {
2156 11230 : enum rtx_code code;
2157 11230 : rtx cond = noce_reversed_cond (if_info, &code);
2158 :
2159 : /* First try to use addcc pattern. */
2160 11230 : if (general_operand (XEXP (cond, 0), VOIDmode)
2161 11230 : && general_operand (XEXP (cond, 1), VOIDmode))
2162 : {
2163 10924 : start_sequence ();
2164 32772 : target = emit_conditional_add (if_info->x, code,
2165 : XEXP (cond, 0),
2166 : XEXP (cond, 1),
2167 : VOIDmode,
2168 : if_info->b,
2169 10924 : XEXP (if_info->a, 1),
2170 10924 : GET_MODE (if_info->x),
2171 10533 : (code == LTU || code == GEU
2172 21404 : || code == LEU || code == GTU));
2173 10924 : if (target)
2174 : {
2175 332 : if (target != if_info->x)
2176 0 : noce_emit_move_insn (if_info->x, target);
2177 :
2178 332 : seq = end_ifcvt_sequence (if_info);
2179 332 : return noce_commit_sequence (if_info, seq, true,
2180 1162 : "noce_try_addcc");
2181 : }
2182 10592 : end_sequence ();
2183 : }
2184 :
2185 : /* If that fails, construct conditional increment or decrement using
2186 : setcc. We're changing a branch and an increment to a comparison and
2187 : an ADD/SUB. */
2188 10898 : if (XEXP (if_info->a, 1) == const1_rtx
2189 9934 : || XEXP (if_info->a, 1) == constm1_rtx)
2190 : {
2191 1519 : start_sequence ();
2192 1519 : if (STORE_FLAG_VALUE == INTVAL (XEXP (if_info->a, 1)))
2193 : subtract = false, normalize = 0;
2194 555 : else if (-STORE_FLAG_VALUE == INTVAL (XEXP (if_info->a, 1)))
2195 : subtract = true, normalize = 0;
2196 : else
2197 0 : subtract = false, normalize = INTVAL (XEXP (if_info->a, 1));
2198 :
2199 :
2200 1519 : target = noce_emit_store_flag (if_info,
2201 1519 : gen_reg_rtx (GET_MODE (if_info->x)),
2202 : true, normalize);
2203 :
2204 1519 : if (target)
2205 1128 : target = expand_simple_binop (GET_MODE (if_info->x),
2206 : subtract ? MINUS : PLUS,
2207 : if_info->b, target, if_info->x,
2208 : 0, OPTAB_WIDEN);
2209 830 : if (target)
2210 : {
2211 830 : if (target != if_info->x)
2212 0 : noce_emit_move_insn (if_info->x, target);
2213 :
2214 830 : seq = end_ifcvt_sequence (if_info);
2215 830 : return noce_commit_sequence (if_info, seq, true,
2216 830 : "noce_try_addcc");
2217 : }
2218 689 : end_sequence ();
2219 : }
2220 : }
2221 :
2222 : return false;
2223 : }
2224 :
2225 : /* Convert "if (test) x = 0;" to "x &= -(test == 0);" */
2226 :
2227 : static bool
2228 156606 : noce_try_store_flag_mask (noce_if_info *if_info)
2229 : {
2230 156606 : rtx target;
2231 156606 : rtx_insn *seq;
2232 :
2233 199631 : if (!noce_simple_bbs (if_info))
2234 : return false;
2235 :
2236 : /* Match "if (test) x = 0;" and, with the arms the other way round,
2237 : "if (!test) x = 0;". */
2238 137928 : bool zero_a = (if_info->a == const0_rtx
2239 137928 : && (REG_P (if_info->b)
2240 1681 : || rtx_equal_p (if_info->b, if_info->x)));
2241 137928 : bool zero_b = (if_info->b == const0_rtx
2242 137928 : && (REG_P (if_info->a)
2243 13145 : || rtx_equal_p (if_info->a, if_info->x)));
2244 :
2245 137926 : if (zero_a || zero_b)
2246 : {
2247 1151 : bool reversep = !zero_a;
2248 1151 : start_sequence ();
2249 1151 : target = noce_emit_store_flag (if_info,
2250 1151 : gen_reg_rtx (GET_MODE (if_info->x)),
2251 : reversep, -1);
2252 1151 : if (target)
2253 301 : target = expand_simple_binop (GET_MODE (if_info->x), AND,
2254 : reversep ? if_info->a : if_info->b,
2255 : target, if_info->x, 0,
2256 : OPTAB_WIDEN);
2257 :
2258 301 : if (target)
2259 : {
2260 301 : if (target != if_info->x)
2261 0 : noce_emit_move_insn (if_info->x, target);
2262 :
2263 301 : seq = end_ifcvt_sequence (if_info);
2264 301 : return noce_commit_sequence (if_info, seq, true,
2265 301 : "noce_try_store_flag_mask");
2266 : }
2267 :
2268 850 : end_sequence ();
2269 : }
2270 :
2271 : return false;
2272 : }
2273 :
2274 : /* Emit a conditional move selecting VTRUE or VFALSE into X, and return the
2275 : destination it landed in, or NULL_RTX on failure. CODE, CMP_A and CMP_B
2276 : give the canonicalized comparison. CC_CMP and REV_CC_CMP, when given, are
2277 : the non-canonicalized condition and its reverse, which let a target emit
2278 : the move without materializing its own compare. */
2279 :
2280 : static rtx
2281 518668 : noce_emit_cmove (noce_if_info *if_info, rtx x, enum rtx_code code,
2282 : rtx cmp_a, rtx cmp_b, rtx vfalse, rtx vtrue,
2283 : rtx cc_cmp = NULL, rtx rev_cc_cmp = NULL)
2284 : {
2285 518668 : rtx target;
2286 518668 : bool unsignedp;
2287 :
2288 : /* If earliest == jump, try to build the cmove insn directly.
2289 : This is helpful when combine has created some complex condition
2290 : (like for alpha's cmovlbs) that we can't hope to regenerate
2291 : through the normal interface. */
2292 :
2293 518668 : if (if_info->cond_earliest == if_info->jump)
2294 : {
2295 4758 : rtx cond = gen_rtx_fmt_ee (code, GET_MODE (if_info->cond), cmp_a, cmp_b);
2296 4758 : rtx if_then_else = gen_rtx_IF_THEN_ELSE (GET_MODE (x),
2297 : cond, vtrue, vfalse);
2298 4758 : rtx set = gen_rtx_SET (x, if_then_else);
2299 :
2300 4758 : start_sequence ();
2301 4758 : rtx_insn *insn = emit_insn (set);
2302 :
2303 4758 : if (recog_memoized (insn) >= 0)
2304 : {
2305 1451 : rtx_insn *seq = end_sequence ();
2306 1451 : emit_insn (seq);
2307 :
2308 1451 : return x;
2309 : }
2310 :
2311 3307 : end_sequence ();
2312 : }
2313 :
2314 1034434 : unsignedp = (code == LTU || code == GEU
2315 517217 : || code == LEU || code == GTU);
2316 :
2317 517217 : if (cc_cmp != NULL_RTX && rev_cc_cmp != NULL_RTX)
2318 106350 : target = emit_conditional_move (x, cc_cmp, rev_cc_cmp,
2319 106350 : vtrue, vfalse, GET_MODE (x));
2320 : else
2321 : {
2322 : /* Don't even try if the comparison operands are weird
2323 : except that the target supports cbranchcc4. */
2324 410867 : if (! general_operand (cmp_a, GET_MODE (cmp_a))
2325 410867 : || ! general_operand (cmp_b, GET_MODE (cmp_b)))
2326 : {
2327 25272 : if (!have_cbranchcc4
2328 25272 : || GET_MODE_CLASS (GET_MODE (cmp_a)) != MODE_CC
2329 3048 : || cmp_b != const0_rtx)
2330 : return NULL_RTX;
2331 : }
2332 :
2333 388643 : target = emit_conditional_move (x, { code, cmp_a, cmp_b, VOIDmode },
2334 388643 : vtrue, vfalse, GET_MODE (x),
2335 : unsignedp);
2336 : }
2337 :
2338 494993 : if (target)
2339 : return target;
2340 :
2341 : /* We might be faced with a situation like:
2342 :
2343 : x = (reg:M TARGET)
2344 : vtrue = (subreg:M (reg:N VTRUE) BYTE)
2345 : vfalse = (subreg:M (reg:N VFALSE) BYTE)
2346 :
2347 : We can't do a conditional move in mode M, but it's possible that we
2348 : could do a conditional move in mode N instead and take a subreg of
2349 : the result.
2350 :
2351 : If we can't create new pseudos, though, don't bother. */
2352 43188 : if (reload_completed)
2353 : return NULL_RTX;
2354 :
2355 43188 : if (GET_CODE (vtrue) == SUBREG && GET_CODE (vfalse) == SUBREG)
2356 : {
2357 23 : rtx reg_vtrue = SUBREG_REG (vtrue);
2358 23 : rtx reg_vfalse = SUBREG_REG (vfalse);
2359 23 : poly_uint64 byte_vtrue = SUBREG_BYTE (vtrue);
2360 23 : poly_uint64 byte_vfalse = SUBREG_BYTE (vfalse);
2361 23 : rtx promoted_target;
2362 :
2363 23 : if (GET_MODE (reg_vtrue) != GET_MODE (reg_vfalse)
2364 23 : || maybe_ne (byte_vtrue, byte_vfalse)
2365 23 : || (SUBREG_PROMOTED_VAR_P (vtrue)
2366 23 : != SUBREG_PROMOTED_VAR_P (vfalse))
2367 23 : || (SUBREG_PROMOTED_GET (vtrue)
2368 23 : != SUBREG_PROMOTED_GET (vfalse)))
2369 : return NULL_RTX;
2370 :
2371 23 : promoted_target = gen_reg_rtx (GET_MODE (reg_vtrue));
2372 :
2373 46 : target = emit_conditional_move (promoted_target,
2374 : { code, cmp_a, cmp_b, VOIDmode },
2375 : reg_vtrue, reg_vfalse,
2376 23 : GET_MODE (reg_vtrue), unsignedp);
2377 : /* Nope, couldn't do it in that mode either. */
2378 23 : if (!target)
2379 : return NULL_RTX;
2380 :
2381 6 : target = gen_rtx_SUBREG (GET_MODE (vtrue), promoted_target, byte_vtrue);
2382 6 : SUBREG_PROMOTED_VAR_P (target) = SUBREG_PROMOTED_VAR_P (vtrue);
2383 6 : SUBREG_PROMOTED_SET (target, SUBREG_PROMOTED_GET (vtrue));
2384 6 : emit_move_insn (x, target);
2385 6 : return x;
2386 : }
2387 : else
2388 : return NULL_RTX;
2389 : }
2390 :
2391 : /* Return true if X is a constant or register operand suitable for simple
2392 : conditional-move handling. */
2393 :
2394 : static bool
2395 2577490 : noce_simple_cmove_operand_p (rtx x)
2396 : {
2397 2577490 : return CONSTANT_P (x) || register_operand (x, VOIDmode);
2398 : }
2399 :
2400 : /* Try only simple constants and registers here. More complex cases are
2401 : handled in noce_try_cmove_arith, which the dispatch table reaches after
2402 : the store-flag matchers have had a go. */
2403 :
2404 : static bool
2405 200097 : noce_try_cmove (noce_if_info *if_info)
2406 : {
2407 200097 : enum rtx_code code;
2408 200097 : rtx target;
2409 200097 : rtx_insn *seq;
2410 :
2411 259669 : if (!noce_simple_bbs (if_info))
2412 : return false;
2413 :
2414 181419 : if (noce_simple_cmove_operand_p (if_info->a)
2415 181419 : && noce_simple_cmove_operand_p (if_info->b))
2416 : {
2417 82692 : start_sequence ();
2418 :
2419 82692 : code = GET_CODE (if_info->cond);
2420 82692 : target = noce_emit_cmove (if_info, if_info->x, code,
2421 : XEXP (if_info->cond, 0),
2422 : XEXP (if_info->cond, 1),
2423 : if_info->a, if_info->b);
2424 :
2425 82692 : if (target)
2426 : {
2427 56583 : if (target != if_info->x)
2428 0 : noce_emit_move_insn (if_info->x, target);
2429 :
2430 56583 : seq = end_ifcvt_sequence (if_info);
2431 56583 : return noce_commit_sequence (if_info, seq, true, "noce_try_cmove");
2432 : }
2433 : /* If both a and b are constants try a last-ditch transformation:
2434 : if (test) x = a; else x = b;
2435 : => x = (-(test != 0) & (b - a)) + a;
2436 : Try this only if the target-specific expansion above has failed.
2437 : The target-specific expander may want to generate sequences that
2438 : we don't know about, so give them a chance before trying this
2439 : approach. */
2440 26109 : else if (!targetm.have_conditional_execution ()
2441 26109 : && CONST_INT_P (if_info->a) && CONST_INT_P (if_info->b))
2442 : {
2443 5336 : machine_mode mode = GET_MODE (if_info->x);
2444 5336 : HOST_WIDE_INT ifalse = INTVAL (if_info->a);
2445 5336 : HOST_WIDE_INT itrue = INTVAL (if_info->b);
2446 5336 : rtx target = noce_emit_store_flag (if_info, if_info->x, false, -1);
2447 5336 : if (!target)
2448 : {
2449 5276 : end_sequence ();
2450 5276 : return false;
2451 : }
2452 :
2453 60 : HOST_WIDE_INT diff;
2454 60 : if (!noce_representable_diff_p (ifalse, itrue, mode, &diff))
2455 : {
2456 21 : end_sequence ();
2457 21 : return false;
2458 : }
2459 :
2460 39 : target = expand_simple_binop (mode, AND,
2461 39 : target, gen_int_mode (diff, mode),
2462 : if_info->x, 0, OPTAB_WIDEN);
2463 39 : if (target)
2464 39 : target = expand_simple_binop (mode, PLUS,
2465 39 : target, gen_int_mode (ifalse, mode),
2466 : if_info->x, 0, OPTAB_WIDEN);
2467 39 : if (target)
2468 : {
2469 39 : if (target != if_info->x)
2470 0 : noce_emit_move_insn (if_info->x, target);
2471 :
2472 39 : seq = end_ifcvt_sequence (if_info);
2473 39 : return noce_commit_sequence (if_info, seq, true,
2474 39 : "noce_try_cmove");
2475 : }
2476 : else
2477 : {
2478 0 : end_sequence ();
2479 0 : return false;
2480 : }
2481 : }
2482 : else
2483 20773 : end_sequence ();
2484 : }
2485 :
2486 : return false;
2487 : }
2488 :
2489 : /* Return true if X contains a conditional code mode rtx. */
2490 :
2491 : static bool
2492 2879813 : contains_ccmode_rtx_p (rtx x)
2493 : {
2494 2879813 : subrtx_iterator::array_type array;
2495 7364177 : FOR_EACH_SUBRTX (iter, array, x, ALL)
2496 4531650 : if (GET_MODE_CLASS (GET_MODE (*iter)) == MODE_CC)
2497 47286 : return true;
2498 :
2499 2832527 : return false;
2500 2879813 : }
2501 :
2502 : /* Helper for bb_valid_for_noce_process_p. Validate that
2503 : the rtx insn INSN is a single set that does not set
2504 : the conditional register CC and is in general valid for
2505 : if-conversion. */
2506 :
2507 : static bool
2508 3630250 : insn_valid_noce_process_p (rtx_insn *insn, rtx cc)
2509 : {
2510 3630250 : if (!insn
2511 3629938 : || !NONJUMP_INSN_P (insn)
2512 6541768 : || (cc && set_of (cc, insn)))
2513 : return false;
2514 :
2515 2899938 : rtx sset = single_set (insn);
2516 :
2517 : /* Currently support only simple single sets in test_bb. */
2518 2899938 : if (!sset
2519 2893501 : || !noce_operand_ok (SET_DEST (sset))
2520 2879813 : || contains_ccmode_rtx_p (SET_DEST (sset))
2521 5732465 : || !noce_operand_ok (SET_SRC (sset)))
2522 130699 : return false;
2523 :
2524 : return true;
2525 : }
2526 :
2527 :
2528 : /* Return true iff the registers that the insns in BB_A set do not get
2529 : used in BB_B. If TO_RENAME is non-NULL then it is a location that will be
2530 : renamed later by the caller and so conflicts on it should be ignored
2531 : in this function. */
2532 :
2533 : static bool
2534 53908 : bbs_ok_for_cmove_arith (basic_block bb_a, basic_block bb_b, rtx to_rename)
2535 : {
2536 53908 : rtx_insn *a_insn;
2537 53908 : auto_bitmap bba_sets (®_obstack);
2538 :
2539 53908 : df_ref def;
2540 53908 : df_ref use;
2541 :
2542 285234 : FOR_BB_INSNS (bb_a, a_insn)
2543 : {
2544 231326 : if (!active_insn_p (a_insn))
2545 155022 : continue;
2546 :
2547 76304 : rtx sset_a = single_set (a_insn);
2548 :
2549 76304 : if (!sset_a)
2550 : return false;
2551 : /* Record all registers that BB_A sets. */
2552 185721 : FOR_EACH_INSN_DEF (def, a_insn)
2553 109417 : if (!(to_rename && DF_REF_REG (def) == to_rename))
2554 56752 : bitmap_set_bit (bba_sets, DF_REF_REGNO (def));
2555 : }
2556 :
2557 53908 : rtx_insn *b_insn;
2558 :
2559 285298 : FOR_BB_INSNS (bb_b, b_insn)
2560 : {
2561 231490 : if (!active_insn_p (b_insn))
2562 155193 : continue;
2563 :
2564 76297 : rtx sset_b = single_set (b_insn);
2565 :
2566 76297 : if (!sset_b)
2567 : return false;
2568 :
2569 : /* Make sure this is a REG and not some instance
2570 : of ZERO_EXTRACT or non-paradoxical SUBREG or other dangerous stuff.
2571 : If we have a memory destination then we have a pair of simple
2572 : basic blocks performing an operation of the form [addr] = c ? a : b.
2573 : bb_valid_for_noce_process_p will have ensured that these are
2574 : the only stores present. In that case [addr] should be the location
2575 : to be renamed. Assert that the callers set this up properly. */
2576 76297 : if (MEM_P (SET_DEST (sset_b)))
2577 1234 : gcc_assert (rtx_equal_p (SET_DEST (sset_b), to_rename));
2578 75063 : else if (!REG_P (SET_DEST (sset_b))
2579 128928 : && !paradoxical_subreg_p (SET_DEST (sset_b)))
2580 : return false;
2581 :
2582 : /* If the insn uses a reg set in BB_A return false. */
2583 148436 : FOR_EACH_INSN_USE (use, b_insn)
2584 72239 : if (bitmap_bit_p (bba_sets, DF_REF_REGNO (use)))
2585 : return false;
2586 :
2587 : }
2588 :
2589 : return true;
2590 53908 : }
2591 :
2592 : /* Emit copies of all the active instructions in BB except the last.
2593 : This is a helper for noce_try_cmove_arith. */
2594 :
2595 : static void
2596 14132 : noce_emit_all_but_last (basic_block bb)
2597 : {
2598 14132 : rtx_insn *last = last_active_insn (bb, false);
2599 14132 : rtx_insn *insn;
2600 144654 : FOR_BB_INSNS (bb, insn)
2601 : {
2602 116390 : if (insn != last && active_insn_p (insn))
2603 : {
2604 35989 : rtx_insn *to_emit = as_a <rtx_insn *> (copy_rtx (insn));
2605 :
2606 35989 : emit_insn (PATTERN (to_emit));
2607 : }
2608 : }
2609 14132 : }
2610 :
2611 : /* Helper for noce_try_cmove_arith. Emit the pattern TO_EMIT and return
2612 : the resulting insn or NULL if it's not a valid insn. */
2613 :
2614 : static rtx_insn *
2615 140548 : noce_emit_insn (rtx to_emit)
2616 : {
2617 140548 : gcc_assert (to_emit);
2618 140548 : rtx_insn *insn = emit_insn (to_emit);
2619 :
2620 140548 : if (recog_memoized (insn) < 0)
2621 193 : return NULL;
2622 :
2623 : return insn;
2624 : }
2625 :
2626 : /* Helper for noce_try_cmove_arith. Emit a copy of the insns up to
2627 : and including the penultimate one in BB if it is not simple
2628 : (as indicated by SIMPLE). Then emit LAST_INSN as the last
2629 : insn in the block. The reason for that is that LAST_INSN may
2630 : have been modified by the preparation in noce_try_cmove_arith. */
2631 :
2632 : static bool
2633 142930 : noce_emit_bb (rtx last_insn, basic_block bb, bool simple)
2634 : {
2635 142930 : if (bb && !simple)
2636 14132 : noce_emit_all_but_last (bb);
2637 :
2638 142930 : if (last_insn && !noce_emit_insn (last_insn))
2639 193 : return false;
2640 :
2641 : return true;
2642 : }
2643 :
2644 : /* Try more complex cases involving conditional_move. */
2645 :
2646 : static bool
2647 146415 : noce_try_cmove_arith (noce_if_info *if_info)
2648 : {
2649 146415 : rtx a = if_info->a;
2650 146415 : rtx b = if_info->b;
2651 146415 : rtx x = if_info->x;
2652 146415 : rtx orig_a, orig_b;
2653 146415 : rtx_insn *insn_a, *insn_b;
2654 146415 : bool a_simple = if_info->then_simple;
2655 146415 : bool b_simple = if_info->else_simple;
2656 146415 : basic_block then_bb = if_info->then_bb;
2657 146415 : basic_block else_bb = if_info->else_bb;
2658 146415 : rtx target;
2659 146415 : bool is_mem = false;
2660 146415 : enum rtx_code code;
2661 146415 : rtx cond = if_info->cond;
2662 146415 : rtx_insn *ifcvt_seq;
2663 :
2664 : /* A conditional move from two memory sources is equivalent to a
2665 : conditional on their addresses followed by a load. Don't do this
2666 : early because it'll screw alias analysis. Note that we've
2667 : already checked for no side effects. */
2668 146415 : if (cse_not_expected
2669 48015 : && MEM_P (a) && MEM_P (b)
2670 149089 : && MEM_ADDR_SPACE (a) == MEM_ADDR_SPACE (b))
2671 : {
2672 2673 : machine_mode address_mode = get_address_mode (a);
2673 :
2674 2673 : a = XEXP (a, 0);
2675 2673 : b = XEXP (b, 0);
2676 2673 : x = gen_reg_rtx (address_mode);
2677 2673 : is_mem = true;
2678 : }
2679 :
2680 : /* ??? We could handle this if we knew that a load from A or B could
2681 : not trap or fault. This is also true if we've already loaded
2682 : from the address along the path from ENTRY. */
2683 143742 : else if (may_trap_or_fault_p (a) || may_trap_or_fault_p (b))
2684 : return false;
2685 :
2686 : /* if (test) x = a + b; else x = c - d;
2687 : => y = a + b;
2688 : x = c - d;
2689 : if (test)
2690 : x = y;
2691 : */
2692 :
2693 91809 : code = GET_CODE (cond);
2694 91809 : insn_a = if_info->insn_a;
2695 91809 : insn_b = if_info->insn_b;
2696 :
2697 91809 : machine_mode x_mode = GET_MODE (x);
2698 :
2699 91809 : if (!can_conditionally_move_p (x_mode))
2700 : return false;
2701 :
2702 : /* Possibly rearrange operands to make things come out more natural. */
2703 71565 : if (noce_reversed_cond_code (if_info) != UNKNOWN)
2704 : {
2705 71565 : bool reversep = false;
2706 71565 : if (rtx_equal_p (b, x))
2707 : reversep = true;
2708 45548 : else if (general_operand (b, GET_MODE (b)))
2709 : reversep = true;
2710 :
2711 : if (reversep)
2712 : {
2713 63795 : cond = noce_reversed_cond (if_info, &code);
2714 63795 : std::swap (a, b);
2715 63795 : std::swap (insn_a, insn_b);
2716 63795 : std::swap (a_simple, b_simple);
2717 63795 : std::swap (then_bb, else_bb);
2718 : }
2719 : }
2720 :
2721 27768 : if (then_bb && else_bb
2722 98569 : && (!bbs_ok_for_cmove_arith (then_bb, else_bb, if_info->orig_x)
2723 26904 : || !bbs_ok_for_cmove_arith (else_bb, then_bb, if_info->orig_x)))
2724 : return false;
2725 :
2726 71465 : start_sequence ();
2727 :
2728 : /* If one of the blocks is empty then the corresponding B or A value
2729 : came from the test block. The non-empty complex block that we will
2730 : emit might clobber the register used by B or A, so move it to a pseudo
2731 : first. */
2732 :
2733 71465 : rtx tmp_a = NULL_RTX;
2734 71465 : rtx tmp_b = NULL_RTX;
2735 :
2736 71465 : if (b_simple || !else_bb)
2737 59353 : tmp_b = gen_reg_rtx (x_mode);
2738 :
2739 71465 : if (a_simple || !then_bb)
2740 69445 : tmp_a = gen_reg_rtx (x_mode);
2741 :
2742 71465 : orig_a = a;
2743 71465 : orig_b = b;
2744 :
2745 71465 : rtx emit_a = NULL_RTX;
2746 71465 : rtx emit_b = NULL_RTX;
2747 71465 : rtx_insn *tmp_insn = NULL;
2748 71465 : bool modified_in_a = false;
2749 71465 : bool modified_in_b = false;
2750 : /* If either operand is complex, load it into a register first.
2751 : The best way to do this is to copy the original insn. In this
2752 : way we preserve any clobbers etc that the insn may have had.
2753 : This is of course not possible in the IS_MEM case. */
2754 :
2755 71465 : if (! general_operand (a, GET_MODE (a)) || tmp_a)
2756 : {
2757 :
2758 70749 : if (is_mem)
2759 : {
2760 2672 : rtx reg = gen_reg_rtx (GET_MODE (a));
2761 2672 : emit_a = gen_rtx_SET (reg, a);
2762 : }
2763 : else
2764 : {
2765 68077 : if (insn_a)
2766 : {
2767 42005 : a = tmp_a ? tmp_a : gen_reg_rtx (GET_MODE (a));
2768 :
2769 42005 : rtx_insn *copy_of_a = as_a <rtx_insn *> (copy_rtx (insn_a));
2770 42005 : rtx set = single_set (copy_of_a);
2771 42005 : SET_DEST (set) = a;
2772 :
2773 42005 : emit_a = PATTERN (copy_of_a);
2774 : }
2775 : else
2776 : {
2777 26072 : rtx tmp_reg = tmp_a ? tmp_a : gen_reg_rtx (GET_MODE (a));
2778 26072 : emit_a = gen_rtx_SET (tmp_reg, a);
2779 26072 : a = tmp_reg;
2780 : }
2781 : }
2782 : }
2783 :
2784 71465 : if (! general_operand (b, GET_MODE (b)) || tmp_b)
2785 : {
2786 69799 : if (is_mem)
2787 : {
2788 2672 : rtx reg = gen_reg_rtx (GET_MODE (b));
2789 2672 : emit_b = gen_rtx_SET (reg, b);
2790 : }
2791 : else
2792 : {
2793 67127 : if (insn_b)
2794 : {
2795 66937 : b = tmp_b ? tmp_b : gen_reg_rtx (GET_MODE (b));
2796 66937 : rtx_insn *copy_of_b = as_a <rtx_insn *> (copy_rtx (insn_b));
2797 66937 : rtx set = single_set (copy_of_b);
2798 :
2799 66937 : SET_DEST (set) = b;
2800 66937 : emit_b = PATTERN (copy_of_b);
2801 : }
2802 : else
2803 : {
2804 190 : rtx tmp_reg = tmp_b ? tmp_b : gen_reg_rtx (GET_MODE (b));
2805 190 : emit_b = gen_rtx_SET (tmp_reg, b);
2806 190 : b = tmp_reg;
2807 : }
2808 : }
2809 : }
2810 :
2811 71465 : modified_in_a = emit_a != NULL_RTX && modified_in_p (orig_b, emit_a);
2812 71465 : if (tmp_b && then_bb)
2813 : {
2814 76620 : FOR_BB_INSNS (then_bb, tmp_insn)
2815 : /* Don't check inside insn_a. We will have changed it to emit_a
2816 : with a destination that doesn't conflict. */
2817 57889 : if (!(insn_a && tmp_insn == insn_a)
2818 97047 : && modified_in_p (orig_b, tmp_insn))
2819 : {
2820 : modified_in_a = true;
2821 : break;
2822 : }
2823 :
2824 : }
2825 :
2826 71465 : modified_in_b = emit_b != NULL_RTX && modified_in_p (orig_a, emit_b);
2827 71465 : if (tmp_a && else_bb)
2828 : {
2829 330167 : FOR_BB_INSNS (else_bb, tmp_insn)
2830 : /* Don't check inside insn_b. We will have changed it to emit_b
2831 : with a destination that doesn't conflict. */
2832 261465 : if (!(insn_b && tmp_insn == insn_b)
2833 454228 : && modified_in_p (orig_a, tmp_insn))
2834 : {
2835 : modified_in_b = true;
2836 : break;
2837 : }
2838 : }
2839 :
2840 : /* If insn to set up A clobbers any registers B depends on, try to
2841 : swap insn that sets up A with the one that sets up B. If even
2842 : that doesn't help, punt. */
2843 71465 : if (modified_in_a && !modified_in_b)
2844 : {
2845 0 : if (!noce_emit_bb (emit_b, else_bb, b_simple))
2846 0 : goto end_seq_and_fail;
2847 :
2848 0 : if (!noce_emit_bb (emit_a, then_bb, a_simple))
2849 0 : goto end_seq_and_fail;
2850 : }
2851 71465 : else if (!modified_in_a)
2852 : {
2853 71465 : if (!noce_emit_bb (emit_a, then_bb, a_simple))
2854 0 : goto end_seq_and_fail;
2855 :
2856 71465 : if (!noce_emit_bb (emit_b, else_bb, b_simple))
2857 193 : goto end_seq_and_fail;
2858 : }
2859 : else
2860 0 : goto end_seq_and_fail;
2861 :
2862 71272 : target = noce_emit_cmove (if_info, x, code, XEXP (cond, 0), XEXP (cond, 1),
2863 : a, b);
2864 :
2865 71272 : if (! target)
2866 17466 : goto end_seq_and_fail;
2867 :
2868 : /* If we're handling a memory for above, emit the load now. */
2869 53806 : if (is_mem)
2870 : {
2871 2281 : rtx mem = gen_rtx_MEM (GET_MODE (if_info->x), target);
2872 :
2873 : /* Copy over flags as appropriate. */
2874 2281 : if (MEM_VOLATILE_P (if_info->a) || MEM_VOLATILE_P (if_info->b))
2875 0 : MEM_VOLATILE_P (mem) = 1;
2876 2281 : if (MEM_ALIAS_SET (if_info->a) == MEM_ALIAS_SET (if_info->b))
2877 2084 : set_mem_alias_set (mem, MEM_ALIAS_SET (if_info->a));
2878 4562 : set_mem_align (mem,
2879 2281 : MIN (MEM_ALIGN (if_info->a), MEM_ALIGN (if_info->b)));
2880 :
2881 2281 : gcc_assert (MEM_ADDR_SPACE (if_info->a) == MEM_ADDR_SPACE (if_info->b));
2882 2281 : set_mem_addr_space (mem, MEM_ADDR_SPACE (if_info->a));
2883 :
2884 2281 : noce_emit_move_insn (if_info->x, mem);
2885 : }
2886 51525 : else if (target != x)
2887 0 : noce_emit_move_insn (x, target);
2888 :
2889 53806 : ifcvt_seq = end_ifcvt_sequence (if_info);
2890 53806 : return noce_commit_sequence (if_info, ifcvt_seq, true,
2891 53806 : "noce_try_cmove_arith");
2892 :
2893 17659 : end_seq_and_fail:
2894 17659 : end_sequence ();
2895 17659 : return false;
2896 : }
2897 :
2898 : /* For most cases, the simplified condition we found is the best
2899 : choice, but this is not the case for the min/max/abs transforms.
2900 : For these we wish to know that it is A or B in the condition. */
2901 :
2902 : static rtx
2903 161328 : noce_get_alt_condition (noce_if_info *if_info, rtx target,
2904 : rtx_insn **earliest)
2905 : {
2906 161328 : rtx cond, set;
2907 161328 : rtx_insn *insn;
2908 161328 : bool reverse;
2909 :
2910 : /* If target is already mentioned in the known condition, return it. */
2911 161328 : if (reg_mentioned_p (target, if_info->cond))
2912 : {
2913 8252 : *earliest = if_info->cond_earliest;
2914 8252 : return if_info->cond;
2915 : }
2916 :
2917 153076 : set = pc_set (if_info->jump);
2918 153076 : cond = XEXP (SET_SRC (set), 0);
2919 153076 : reverse
2920 306152 : = GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
2921 153076 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (if_info->jump);
2922 153076 : if (if_info->then_else_reversed)
2923 24490 : reverse = !reverse;
2924 :
2925 : /* If we're looking for a constant, try to make the conditional
2926 : have that constant in it. There are two reasons why it may
2927 : not have the constant we want:
2928 :
2929 : 1. GCC may have needed to put the constant in a register, because
2930 : the target can't compare directly against that constant. For
2931 : this case, we look for a SET immediately before the comparison
2932 : that puts a constant in that register.
2933 :
2934 : 2. GCC may have canonicalized the conditional, for example
2935 : replacing "if x < 4" with "if x <= 3". We can undo that (or
2936 : make equivalent types of changes) to get the constants we need
2937 : if they're off by one in the right direction. */
2938 :
2939 153076 : if (CONST_INT_P (target))
2940 : {
2941 46664 : enum rtx_code code = GET_CODE (if_info->cond);
2942 46664 : rtx op_a = XEXP (if_info->cond, 0);
2943 46664 : rtx op_b = XEXP (if_info->cond, 1);
2944 46664 : rtx_insn *prev_insn;
2945 :
2946 : /* First, look to see if we put a constant in a register. */
2947 46664 : prev_insn = prev_nonnote_nondebug_insn (if_info->cond_earliest);
2948 46664 : if (prev_insn
2949 46553 : && BLOCK_FOR_INSN (prev_insn)
2950 46553 : == BLOCK_FOR_INSN (if_info->cond_earliest)
2951 37100 : && INSN_P (prev_insn)
2952 79642 : && GET_CODE (PATTERN (prev_insn)) == SET)
2953 : {
2954 25524 : rtx src = find_reg_equal_equiv_note (prev_insn);
2955 25524 : if (!src)
2956 25358 : src = SET_SRC (PATTERN (prev_insn));
2957 25524 : if (CONST_INT_P (src))
2958 : {
2959 15012 : if (rtx_equal_p (op_a, SET_DEST (PATTERN (prev_insn))))
2960 : op_a = src;
2961 14651 : else if (rtx_equal_p (op_b, SET_DEST (PATTERN (prev_insn))))
2962 1073 : op_b = src;
2963 :
2964 15012 : if (CONST_INT_P (op_a))
2965 : {
2966 361 : std::swap (op_a, op_b);
2967 361 : code = swap_condition (code);
2968 : }
2969 : }
2970 : }
2971 :
2972 : /* Now, look to see if we can get the right constant by
2973 : adjusting the conditional. */
2974 46664 : if (CONST_INT_P (op_b))
2975 : {
2976 24180 : HOST_WIDE_INT desired_val = INTVAL (target);
2977 24180 : HOST_WIDE_INT actual_val = INTVAL (op_b);
2978 :
2979 24180 : switch (code)
2980 : {
2981 1162 : case LT:
2982 1162 : if (desired_val != HOST_WIDE_INT_MAX
2983 1147 : && actual_val == desired_val + 1)
2984 : {
2985 174 : code = LE;
2986 174 : op_b = GEN_INT (desired_val);
2987 : }
2988 : break;
2989 3 : case LE:
2990 3 : if (desired_val != HOST_WIDE_INT_MIN
2991 3 : && actual_val == desired_val - 1)
2992 : {
2993 0 : code = LT;
2994 0 : op_b = GEN_INT (desired_val);
2995 : }
2996 : break;
2997 1700 : case GT:
2998 1700 : if (desired_val != HOST_WIDE_INT_MIN
2999 1700 : && actual_val == desired_val - 1)
3000 : {
3001 235 : code = GE;
3002 235 : op_b = GEN_INT (desired_val);
3003 : }
3004 : break;
3005 40 : case GE:
3006 40 : if (desired_val != HOST_WIDE_INT_MAX
3007 25 : && actual_val == desired_val + 1)
3008 : {
3009 3 : code = GT;
3010 3 : op_b = GEN_INT (desired_val);
3011 : }
3012 : break;
3013 : default:
3014 : break;
3015 : }
3016 : }
3017 :
3018 : /* If we made any changes, generate a new conditional that is
3019 : equivalent to what we started with, but has the right
3020 : constants in it. */
3021 46664 : if (code != GET_CODE (if_info->cond)
3022 45892 : || op_a != XEXP (if_info->cond, 0)
3023 45891 : || op_b != XEXP (if_info->cond, 1))
3024 : {
3025 1846 : cond = gen_rtx_fmt_ee (code, GET_MODE (cond), op_a, op_b);
3026 1846 : *earliest = if_info->cond_earliest;
3027 1846 : return cond;
3028 : }
3029 : }
3030 :
3031 151230 : cond = canonicalize_condition (if_info->jump, cond, reverse,
3032 : earliest, target, have_cbranchcc4, true);
3033 151230 : if (! cond || ! reg_mentioned_p (target, cond))
3034 : return NULL;
3035 :
3036 : /* We almost certainly searched back to a different place.
3037 : Need to re-verify correct lifetimes. */
3038 :
3039 : /* X may not be mentioned in the range (cond_earliest, jump]. */
3040 0 : for (insn = if_info->jump; insn != *earliest; insn = PREV_INSN (insn))
3041 0 : if (INSN_P (insn) && reg_overlap_mentioned_p (if_info->x, PATTERN (insn)))
3042 : return NULL;
3043 :
3044 : /* A and B may not be modified in the range [cond_earliest, jump). */
3045 0 : for (insn = *earliest; insn != if_info->jump; insn = NEXT_INSN (insn))
3046 0 : if (INSN_P (insn)
3047 0 : && (modified_in_p (if_info->a, insn)
3048 0 : || modified_in_p (if_info->b, insn)))
3049 : return NULL;
3050 :
3051 : return cond;
3052 : }
3053 :
3054 : /* Convert "if (a < b) x = a; else x = b;" to "x = min(a, b);", etc. */
3055 :
3056 : static bool
3057 208921 : noce_try_minmax (noce_if_info *if_info)
3058 : {
3059 208921 : rtx cond, target;
3060 208921 : rtx_insn *earliest, *seq;
3061 208921 : enum rtx_code code, op;
3062 208921 : bool unsignedp;
3063 :
3064 271910 : if (!noce_simple_bbs (if_info))
3065 : return false;
3066 :
3067 : /* ??? Reject modes with NaNs or signed zeros since we don't know how
3068 : they will be resolved with an SMIN/SMAX. It wouldn't be too hard
3069 : to get the target to tell us... */
3070 190243 : if (HONOR_SIGNED_ZEROS (if_info->x)
3071 190243 : || HONOR_NANS (if_info->x))
3072 : return false;
3073 :
3074 160351 : cond = noce_get_alt_condition (if_info, if_info->a, &earliest);
3075 160351 : if (!cond)
3076 : return false;
3077 :
3078 : /* Verify the condition is of the form we expect, and canonicalize
3079 : the comparison code. */
3080 10073 : code = GET_CODE (cond);
3081 10073 : if (rtx_equal_p (XEXP (cond, 0), if_info->a))
3082 : {
3083 2562 : if (! rtx_equal_p (XEXP (cond, 1), if_info->b))
3084 : return false;
3085 : }
3086 7511 : else if (rtx_equal_p (XEXP (cond, 1), if_info->a))
3087 : {
3088 4698 : if (! rtx_equal_p (XEXP (cond, 0), if_info->b))
3089 : return false;
3090 45 : code = swap_condition (code);
3091 : }
3092 : else
3093 : return false;
3094 :
3095 : /* Determine what sort of operation this is. Note that the code is for
3096 : a taken branch, so the code->operation mapping appears backwards. */
3097 60 : switch (code)
3098 : {
3099 : case LT:
3100 : case LE:
3101 : case UNLT:
3102 : case UNLE:
3103 : op = SMAX;
3104 : unsignedp = false;
3105 : break;
3106 17 : case GT:
3107 17 : case GE:
3108 17 : case UNGT:
3109 17 : case UNGE:
3110 17 : op = SMIN;
3111 17 : unsignedp = false;
3112 17 : break;
3113 3 : case LTU:
3114 3 : case LEU:
3115 3 : op = UMAX;
3116 3 : unsignedp = true;
3117 3 : break;
3118 20 : case GTU:
3119 20 : case GEU:
3120 20 : op = UMIN;
3121 20 : unsignedp = true;
3122 20 : break;
3123 : default:
3124 : return false;
3125 : }
3126 :
3127 60 : start_sequence ();
3128 :
3129 60 : target = expand_simple_binop (GET_MODE (if_info->x), op,
3130 : if_info->a, if_info->b,
3131 : if_info->x, unsignedp, OPTAB_WIDEN);
3132 60 : if (! target)
3133 : {
3134 0 : end_sequence ();
3135 0 : return false;
3136 : }
3137 60 : if (target != if_info->x)
3138 0 : noce_emit_move_insn (if_info->x, target);
3139 :
3140 60 : seq = end_ifcvt_sequence (if_info);
3141 60 : if (!seq)
3142 : return false;
3143 :
3144 60 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3145 60 : if_info->cond = cond;
3146 60 : if_info->cond_earliest = earliest;
3147 60 : if_info->rev_cond = NULL_RTX;
3148 60 : if_info->transform_name = "noce_try_minmax";
3149 :
3150 60 : return true;
3151 : }
3152 :
3153 : /* Convert "if (a < 0) x = -a; else x = a;" to "x = abs(a);",
3154 : "if (a < 0) x = ~a; else x = a;" to "x = one_cmpl_abs(a);",
3155 : etc. */
3156 :
3157 : static bool
3158 208861 : noce_try_abs (noce_if_info *if_info)
3159 : {
3160 208861 : rtx cond, target, a, b, c;
3161 208861 : rtx_insn *earliest, *seq;
3162 208861 : bool negate;
3163 208861 : bool one_cmpl = false;
3164 :
3165 271841 : if (!noce_simple_bbs (if_info))
3166 : return false;
3167 :
3168 : /* Reject modes with signed zeros. */
3169 190183 : if (HONOR_SIGNED_ZEROS (if_info->x))
3170 : return false;
3171 :
3172 : /* Recognize A and B as constituting an ABS or NABS. The canonical
3173 : form is a branch around the negation, taken when the object is the
3174 : first operand of a comparison against 0 that evaluates to true. */
3175 160299 : a = if_info->a;
3176 160299 : b = if_info->b;
3177 160299 : if (GET_CODE (a) == NEG && rtx_equal_p (XEXP (a, 0), b))
3178 : negate = false;
3179 159352 : else if (GET_CODE (b) == NEG && rtx_equal_p (XEXP (b, 0), a))
3180 : {
3181 : std::swap (a, b);
3182 : negate = true;
3183 : }
3184 159322 : else if (GET_CODE (a) == NOT && rtx_equal_p (XEXP (a, 0), b))
3185 : {
3186 : negate = false;
3187 : one_cmpl = true;
3188 : }
3189 159322 : else if (GET_CODE (b) == NOT && rtx_equal_p (XEXP (b, 0), a))
3190 : {
3191 : std::swap (a, b);
3192 : negate = true;
3193 : one_cmpl = true;
3194 : }
3195 : else
3196 : return false;
3197 :
3198 977 : cond = noce_get_alt_condition (if_info, b, &earliest);
3199 977 : if (!cond)
3200 : return false;
3201 :
3202 : /* Verify the condition is of the form we expect. */
3203 25 : if (rtx_equal_p (XEXP (cond, 0), b))
3204 25 : c = XEXP (cond, 1);
3205 0 : else if (rtx_equal_p (XEXP (cond, 1), b))
3206 : {
3207 0 : c = XEXP (cond, 0);
3208 0 : negate = !negate;
3209 : }
3210 : else
3211 : return false;
3212 :
3213 : /* Verify that C is zero. Search one step backward for a
3214 : REG_EQUAL note or a simple source if necessary. */
3215 25 : if (REG_P (c))
3216 : {
3217 0 : rtx set;
3218 0 : rtx_insn *insn = prev_nonnote_nondebug_insn (earliest);
3219 0 : if (insn
3220 0 : && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (earliest)
3221 0 : && (set = single_set (insn))
3222 0 : && rtx_equal_p (SET_DEST (set), c))
3223 : {
3224 0 : rtx note = find_reg_equal_equiv_note (insn);
3225 0 : if (note)
3226 0 : c = XEXP (note, 0);
3227 : else
3228 0 : c = SET_SRC (set);
3229 : }
3230 : else
3231 : return false;
3232 : }
3233 25 : if (MEM_P (c)
3234 0 : && GET_CODE (XEXP (c, 0)) == SYMBOL_REF
3235 25 : && CONSTANT_POOL_ADDRESS_P (XEXP (c, 0)))
3236 0 : c = get_pool_constant (XEXP (c, 0));
3237 :
3238 : /* Work around funny ideas get_condition has wrt canonicalization.
3239 : Note that these rtx constants are known to be CONST_INT, and
3240 : therefore imply integer comparisons.
3241 : The one_cmpl case is more complicated, as we want to handle
3242 : only x < 0 ? ~x : x or x >= 0 ? x : ~x to one_cmpl_abs (x)
3243 : and x < 0 ? x : ~x or x >= 0 ? ~x : x to ~one_cmpl_abs (x),
3244 : but not other cases (x > -1 is equivalent of x >= 0). */
3245 25 : if (c == constm1_rtx && GET_CODE (cond) == GT)
3246 : ;
3247 1 : else if (c == const1_rtx && GET_CODE (cond) == LT)
3248 : {
3249 0 : if (one_cmpl)
3250 : return false;
3251 : }
3252 1 : else if (c == CONST0_RTX (GET_MODE (b)))
3253 : {
3254 0 : if (one_cmpl
3255 0 : && GET_CODE (cond) != GE
3256 0 : && GET_CODE (cond) != LT)
3257 : return false;
3258 : }
3259 : else
3260 : return false;
3261 :
3262 : /* Determine what sort of operation this is. */
3263 24 : switch (GET_CODE (cond))
3264 : {
3265 0 : case LT:
3266 0 : case LE:
3267 0 : case UNLT:
3268 0 : case UNLE:
3269 0 : negate = !negate;
3270 0 : break;
3271 : case GT:
3272 : case GE:
3273 : case UNGT:
3274 : case UNGE:
3275 : break;
3276 : default:
3277 : return false;
3278 : }
3279 :
3280 24 : start_sequence ();
3281 24 : if (one_cmpl)
3282 0 : target = expand_one_cmpl_abs_nojump (GET_MODE (if_info->x), b,
3283 : if_info->x);
3284 : else
3285 24 : target = expand_abs_nojump (GET_MODE (if_info->x), b, if_info->x, 1);
3286 :
3287 : /* ??? It's a quandary whether cmove would be better here, especially
3288 : for integers. Perhaps combine will clean things up. */
3289 24 : if (target && negate)
3290 : {
3291 0 : if (one_cmpl)
3292 0 : target = expand_simple_unop (GET_MODE (target), NOT, target,
3293 : if_info->x, 0);
3294 : else
3295 0 : target = expand_simple_unop (GET_MODE (target), NEG, target,
3296 : if_info->x, 0);
3297 : }
3298 :
3299 24 : if (! target)
3300 : {
3301 0 : end_sequence ();
3302 0 : return false;
3303 : }
3304 :
3305 24 : if (target != if_info->x)
3306 0 : noce_emit_move_insn (if_info->x, target);
3307 :
3308 24 : seq = end_ifcvt_sequence (if_info);
3309 24 : if (!seq)
3310 : return false;
3311 :
3312 24 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3313 24 : if_info->cond = cond;
3314 24 : if_info->cond_earliest = earliest;
3315 24 : if_info->rev_cond = NULL_RTX;
3316 24 : if_info->transform_name = "noce_try_abs";
3317 :
3318 24 : return true;
3319 : }
3320 :
3321 : /* Convert "if (m < 0) x = b; else x = 0;" to "x = (m >> C) & b;". */
3322 :
3323 : static bool
3324 111472 : noce_try_sign_mask (noce_if_info *if_info)
3325 : {
3326 111472 : rtx cond, t, m, c;
3327 111472 : rtx_insn *seq;
3328 111472 : machine_mode mode;
3329 111472 : enum rtx_code code;
3330 111472 : bool t_unconditional;
3331 :
3332 137053 : if (!noce_simple_bbs (if_info))
3333 : return false;
3334 :
3335 99962 : cond = if_info->cond;
3336 99962 : code = GET_CODE (cond);
3337 99962 : m = XEXP (cond, 0);
3338 99962 : c = XEXP (cond, 1);
3339 :
3340 99962 : t = NULL_RTX;
3341 99962 : if (if_info->a == const0_rtx)
3342 : {
3343 2650 : if ((code == LT && c == const0_rtx)
3344 2621 : || (code == LE && c == constm1_rtx))
3345 29 : t = if_info->b;
3346 : }
3347 97312 : else if (if_info->b == const0_rtx)
3348 : {
3349 10905 : if ((code == GE && c == const0_rtx)
3350 10905 : || (code == GT && c == constm1_rtx))
3351 : t = if_info->a;
3352 : }
3353 :
3354 95 : if (! t || side_effects_p (t))
3355 : return false;
3356 :
3357 : /* We currently don't handle different modes. */
3358 95 : mode = GET_MODE (t);
3359 95 : if (GET_MODE (m) != mode)
3360 : return false;
3361 :
3362 : /* This is only profitable if T is unconditionally executed/evaluated in the
3363 : original insn sequence or T is cheap and can't trap or fault. The former
3364 : happens if B is the non-zero (T) value and if INSN_B was taken from
3365 : TEST_BB, or there was no INSN_B which can happen for e.g. conditional
3366 : stores to memory. For the cost computation use the block TEST_BB where
3367 : the evaluation will end up after the transformation. */
3368 43 : t_unconditional
3369 86 : = (t == if_info->b
3370 43 : && (if_info->insn_b == NULL_RTX
3371 0 : || BLOCK_FOR_INSN (if_info->insn_b) == if_info->test_bb));
3372 86 : if (!(t_unconditional
3373 43 : || ((set_src_cost (t, mode, if_info->speed_p)
3374 : < COSTS_N_INSNS (2))
3375 37 : && !may_trap_or_fault_p (t))))
3376 : return false;
3377 :
3378 0 : if (!noce_can_force_operand (t))
3379 : return false;
3380 :
3381 0 : start_sequence ();
3382 : /* Use emit_store_flag to generate "m < 0 ? -1 : 0" instead of expanding
3383 : "(signed) m >> 31" directly. This benefits targets with specialized
3384 : insns to obtain the signmask, but still uses ashr_optab otherwise. */
3385 0 : m = emit_store_flag (gen_reg_rtx (mode), LT, m, const0_rtx, mode, 0, -1);
3386 0 : t = m ? expand_binop (mode, and_optab, m, t, NULL_RTX, 0, OPTAB_DIRECT)
3387 : : NULL_RTX;
3388 :
3389 0 : if (!t)
3390 : {
3391 0 : end_sequence ();
3392 0 : return false;
3393 : }
3394 :
3395 0 : noce_emit_move_insn (if_info->x, t);
3396 :
3397 0 : seq = end_ifcvt_sequence (if_info);
3398 0 : return noce_commit_sequence (if_info, seq, false, "noce_try_sign_mask");
3399 : }
3400 :
3401 : /* Return the arithmetic operation in X, looking through an extension. */
3402 :
3403 : static rtx
3404 218986 : noce_cond_arith_operation (rtx x)
3405 : {
3406 218986 : enum rtx_code code = GET_CODE (x);
3407 218986 : if (code == SIGN_EXTEND || code == ZERO_EXTEND)
3408 527 : x = XEXP (x, 0);
3409 :
3410 218986 : return noce_cond_zero_binary_op_supported (x) ? x : NULL_RTX;
3411 : }
3412 :
3413 : /* Return true if X contains an EXT_CODE extension from its mode to
3414 : OUTER_MODE. */
3415 :
3416 : static bool
3417 0 : noce_operand_known_extended_p (rtx x, machine_mode outer_mode,
3418 : enum rtx_code ext_code)
3419 : {
3420 0 : machine_mode inner_mode = GET_MODE (x);
3421 0 : unsigned int precision = GET_MODE_PRECISION (outer_mode).to_constant ();
3422 0 : unsigned int inner_precision
3423 0 : = GET_MODE_PRECISION (inner_mode).to_constant ();
3424 0 : if (ext_code == SIGN_EXTEND)
3425 0 : return (num_sign_bit_copies (x, outer_mode)
3426 0 : > precision - inner_precision);
3427 :
3428 0 : if (ext_code == ZERO_EXTEND)
3429 0 : return (HWI_COMPUTABLE_MODE_P (outer_mode)
3430 0 : && (nonzero_bits (x, outer_mode)
3431 0 : & ~GET_MODE_MASK (inner_mode)) == 0);
3432 :
3433 : return false;
3434 : }
3435 :
3436 : /* Return the register or constant that EXP selects on: EXP itself for a
3437 : register or a constant, the inner register of a subreg, and NULL_RTX for
3438 : anything else. */
3439 :
3440 : static rtx
3441 28176 : get_base_reg_or_constant (rtx exp)
3442 : {
3443 0 : if (REG_P (exp))
3444 : return exp;
3445 5124 : else if (SUBREG_P (exp))
3446 309 : return SUBREG_REG (exp);
3447 4815 : else if (CONST_INT_P (exp))
3448 0 : return exp;
3449 : return NULL_RTX;
3450 : }
3451 :
3452 : /* Try to convert if-then-else with conditional zero, returning TRUE on
3453 : success or FALSE on failure. IF_INFO describes the if-conversion scenario
3454 : under consideration.
3455 :
3456 : It verifies the branch structure on the left and transforms it into the
3457 : branchless sequence on the right, with a backend-provided conditional zero
3458 : or orig for operand z. If true, tmp is z, 0 otherwise (y op 0 is the same
3459 : as y for most op).
3460 :
3461 : if (cond) | tmp = cond ? z : 0
3462 : x = y op z | x = y op tmp
3463 : else |
3464 : x = y |
3465 :
3466 : AND is special as it needs to be handled differently.
3467 :
3468 : tmp = !cond ? y : 0
3469 : x = (y & z) | tmp
3470 :
3471 : Also for AND try:
3472 :
3473 : tmp = cond ? z : -1
3474 : x = y op tmp
3475 :
3476 : to see if it is cheaper to produce `!cond ? y : 0` or `cond ? z : -1`. */
3477 :
3478 : static bool
3479 156597 : noce_try_cond_arith (noce_if_info *if_info)
3480 : {
3481 156597 : rtx target, a, b, a_op0, a_op1, outer_a;
3482 156597 : rtx cond = if_info->cond;
3483 156597 : rtx_code code = GET_CODE (cond);
3484 156597 : rtx_insn *seq;
3485 156597 : rtx_code op;
3486 156597 : machine_mode mode = GET_MODE (if_info->x);
3487 :
3488 : /* Scalar integral modes are only supported here.
3489 : Could support scalar floating point but that
3490 : would be only with -ffast-math and might
3491 : be worse than a branch. */
3492 156597 : if (!SCALAR_INT_MODE_P (mode))
3493 : return false;
3494 :
3495 165691 : if (!noce_simple_bbs (if_info))
3496 : return false;
3497 :
3498 109493 : a = copy_rtx (if_info->a);
3499 109493 : b = copy_rtx (if_info->b);
3500 109493 : rtx a_arith = noce_cond_arith_operation (a);
3501 109493 : rtx b_arith = noce_cond_arith_operation (b);
3502 :
3503 : /* Canonicalize x = y : (y op z) to x = (y op z) : y. */
3504 109493 : if (REG_P (a) && b_arith)
3505 : {
3506 204 : cond = noce_reversed_cond (if_info, &code);
3507 204 : std::swap (a, b);
3508 204 : a_arith = b_arith;
3509 : }
3510 :
3511 : /* Check if x = (y op z) : y is supported by czero based ifcvt. */
3512 109289 : else if (!(a_arith && REG_P (b)))
3513 95077 : goto fail;
3514 :
3515 14416 : if (code == UNKNOWN)
3516 0 : goto fail;
3517 :
3518 14416 : outer_a = a == a_arith ? NULL_RTX : a;
3519 14416 : a = a_arith;
3520 14416 : op = GET_CODE (a);
3521 :
3522 : /* EXTEND (y op z) : y is valid only when Y already contains the same
3523 : extension of its lowpart. AND uses a different transformation. */
3524 14416 : if (outer_a
3525 0 : && (op == AND
3526 0 : || GET_MODE (outer_a) != mode
3527 0 : || !SCALAR_INT_MODE_P (GET_MODE (a))))
3528 0 : goto fail;
3529 :
3530 : /* Canonicalize x = (z op y) : y to x = (y op z) : y */
3531 14416 : a_op1 = get_base_reg_or_constant (XEXP (a, 1));
3532 13755 : if (a_op1 && rtx_equal_p (a_op1, b) && COMMUTATIVE_ARITH_P (a))
3533 : {
3534 18 : std::swap (XEXP (a, 0), XEXP (a, 1));
3535 18 : a_op1 = get_base_reg_or_constant (XEXP (a, 1));
3536 : }
3537 :
3538 14398 : if (a_op1 == NULL_RTX)
3539 674 : goto fail;
3540 :
3541 : /* Ensure the cond is of form: x = (y op z) : y */
3542 13742 : a_op0 = get_base_reg_or_constant (XEXP (a, 0));
3543 13737 : if (!(a_op0 && rtx_equal_p (a_op0, b)))
3544 1984 : goto fail;
3545 :
3546 11758 : if (outer_a
3547 11758 : && (!SUBREG_P (XEXP (a, 0))
3548 0 : || !subreg_lowpart_p (XEXP (a, 0))
3549 0 : || GET_MODE (SUBREG_REG (XEXP (a, 0))) != mode
3550 0 : || !noce_operand_known_extended_p (XEXP (a, 0), mode,
3551 0 : GET_CODE (outer_a))))
3552 0 : goto fail;
3553 :
3554 11758 : start_sequence ();
3555 :
3556 11758 : if (CONST_INT_P (XEXP (a, 1)))
3557 2880 : target = gen_reg_rtx (GET_MODE (XEXP (a, 0)));
3558 : else
3559 8878 : target = gen_reg_rtx (GET_MODE (XEXP (a, op != AND)));
3560 :
3561 : /* AND requires !cond, instead we swap ops around. */
3562 11758 : target = noce_emit_cmove (if_info, target, code,
3563 : XEXP (cond, 0), XEXP (cond, 1),
3564 : op != AND ? XEXP (a, 1) : const0_rtx,
3565 : op != AND ? const0_rtx : XEXP (a, 0));
3566 11758 : if (!target)
3567 : {
3568 1221 : end_sequence ();
3569 1221 : rtx tmp = XEXP (a, op != AND);
3570 : /* If the cmove fails and this was a lowpart subreg,
3571 : then try the reg part and then putting back the lowpart
3572 : afterwards. */
3573 1221 : if (GET_CODE (tmp) != SUBREG || !subreg_lowpart_p (tmp))
3574 : return false;
3575 0 : tmp = SUBREG_REG (tmp);
3576 : /* Only handle integer scalar modes for the inner mode of
3577 : the subreg. */
3578 0 : if (!SCALAR_INT_MODE_P (GET_MODE (tmp)))
3579 : return false;
3580 :
3581 0 : start_sequence ();
3582 0 : target = gen_reg_rtx (GET_MODE (tmp));
3583 0 : target = noce_emit_cmove (if_info, target, code,
3584 : XEXP (cond, 0), XEXP (cond, 1),
3585 : op != AND ? tmp : const0_rtx,
3586 : op != AND ? const0_rtx : tmp);
3587 0 : if (!target)
3588 0 : goto end_seq_n_fail;
3589 0 : target = rtl_hooks.gen_lowpart_no_emit (GET_MODE (XEXP (a, op != AND)), target);
3590 0 : gcc_assert (target);
3591 : }
3592 :
3593 : /* Every path through the fallback above either returns or produces a
3594 : conditional move, so TARGET is non-null below. */
3595 : gcc_checking_assert (target);
3596 :
3597 10537 : if (outer_a)
3598 : {
3599 0 : XEXP (a, 1) = target;
3600 0 : noce_emit_move_insn (if_info->x, outer_a);
3601 0 : target = if_info->x;
3602 0 : goto success;
3603 : }
3604 :
3605 : /* For AND, try `cond ? z : -1` to see if that is cheaper or the same cost.
3606 : In some cases it will be cheaper to produce the -1 rather than the 0 case. */
3607 10537 : if (op == AND)
3608 : {
3609 84 : rtx_insn *seq0 = end_sequence ();
3610 84 : unsigned cost0 = seq_cost (seq0, if_info->speed_p);
3611 :
3612 : /* Produce `cond ? z : -1`. */
3613 84 : rtx targetm1;
3614 84 : start_sequence ();
3615 84 : targetm1 = gen_reg_rtx (GET_MODE (XEXP (a, 1)));
3616 84 : targetm1 = noce_emit_cmove (if_info, targetm1, code,
3617 : XEXP (cond, 0), XEXP (cond, 1),
3618 : XEXP (a, 1), constm1_rtx);
3619 84 : rtx_insn *seqm1 = end_sequence ();
3620 84 : unsigned costm1 = seq_cost (seqm1, if_info->speed_p);
3621 84 : if (!targetm1)
3622 78 : costm1 = -1u;
3623 :
3624 : /* If -1 is cheaper or the same cost to producing 0, then use that. */
3625 84 : if (costm1 <= cost0)
3626 : {
3627 6 : push_to_sequence (seqm1);
3628 6 : targetm1 = expand_simple_binop (mode, op, a_op0, targetm1,
3629 : if_info->x, 0, OPTAB_WIDEN);
3630 6 : if (targetm1)
3631 : {
3632 6 : target = targetm1;
3633 6 : goto success;
3634 : }
3635 0 : end_sequence ();
3636 : }
3637 :
3638 : /* For 0 the produce sequence is:
3639 : tmp = !cond ? y : 0
3640 : x = (y & z) | tmp */
3641 78 : push_to_sequence (seq0);
3642 78 : rtx a_bin = gen_reg_rtx (mode);
3643 78 : noce_emit_move_insn (a_bin, a);
3644 :
3645 78 : target = expand_simple_binop (mode, IOR, a_bin, target, if_info->x, 0,
3646 : OPTAB_WIDEN);
3647 78 : if (!target)
3648 0 : goto end_seq_n_fail;
3649 78 : goto success;
3650 : }
3651 :
3652 10453 : target = expand_simple_binop (mode, op, a_op0, target, if_info->x, 0,
3653 : OPTAB_WIDEN);
3654 :
3655 10453 : if (!target)
3656 0 : goto end_seq_n_fail;
3657 :
3658 10453 : success:
3659 10537 : if (target != if_info->x)
3660 0 : noce_emit_move_insn (if_info->x, target);
3661 :
3662 10537 : seq = end_ifcvt_sequence (if_info);
3663 10537 : return noce_commit_sequence (if_info, seq, true, "noce_try_cond_arith");
3664 :
3665 0 : end_seq_n_fail:
3666 0 : end_sequence ();
3667 :
3668 156597 : fail:
3669 :
3670 : return false;
3671 : }
3672 :
3673 : /* Optimize away "if (x & C) x |= C" and similar bit manipulation
3674 : transformations. */
3675 :
3676 : static bool
3677 208921 : noce_try_bitop (noce_if_info *if_info)
3678 : {
3679 208921 : rtx cond, x, a, result;
3680 208921 : rtx_insn *seq;
3681 208921 : scalar_int_mode mode;
3682 208921 : enum rtx_code code;
3683 208921 : int bitnum;
3684 :
3685 208921 : x = if_info->x;
3686 208921 : cond = if_info->cond;
3687 208921 : code = GET_CODE (cond);
3688 :
3689 : /* Check for an integer operation. */
3690 208921 : if (!is_a <scalar_int_mode> (GET_MODE (x), &mode))
3691 : return false;
3692 :
3693 234876 : if (!noce_simple_bbs (if_info))
3694 : return false;
3695 :
3696 : /* Check for no else condition. */
3697 158979 : if (! rtx_equal_p (x, if_info->b))
3698 : return false;
3699 :
3700 : /* Check for a suitable condition. */
3701 84053 : if (code != NE && code != EQ)
3702 : return false;
3703 59828 : if (XEXP (cond, 1) != const0_rtx)
3704 : return false;
3705 41439 : cond = XEXP (cond, 0);
3706 :
3707 : /* ??? We could also handle AND here. */
3708 41439 : if (GET_CODE (cond) == ZERO_EXTRACT)
3709 : {
3710 228 : if (XEXP (cond, 1) != const1_rtx
3711 113 : || !CONST_INT_P (XEXP (cond, 2))
3712 341 : || ! rtx_equal_p (x, XEXP (cond, 0)))
3713 : return false;
3714 4 : bitnum = INTVAL (XEXP (cond, 2));
3715 4 : if (BITS_BIG_ENDIAN)
3716 : bitnum = GET_MODE_BITSIZE (mode) - 1 - bitnum;
3717 4 : if (bitnum < 0 || bitnum >= HOST_BITS_PER_WIDE_INT)
3718 : return false;
3719 : }
3720 : else
3721 : return false;
3722 :
3723 4 : a = if_info->a;
3724 4 : if (GET_CODE (a) == IOR || GET_CODE (a) == XOR)
3725 : {
3726 : /* Check for "if (X & C) x = x op C". */
3727 0 : if (! rtx_equal_p (x, XEXP (a, 0))
3728 0 : || !CONST_INT_P (XEXP (a, 1))
3729 0 : || (INTVAL (XEXP (a, 1)) & GET_MODE_MASK (mode))
3730 0 : != HOST_WIDE_INT_1U << bitnum)
3731 : return false;
3732 :
3733 : /* if ((x & C) == 0) x |= C; is transformed to x |= C. */
3734 : /* if ((x & C) != 0) x |= C; is transformed to nothing. */
3735 0 : if (GET_CODE (a) == IOR)
3736 0 : result = (code == NE) ? a : NULL_RTX;
3737 0 : else if (code == NE)
3738 : {
3739 : /* if ((x & C) == 0) x ^= C; is transformed to x |= C. */
3740 0 : result = gen_int_mode (HOST_WIDE_INT_1 << bitnum, mode);
3741 0 : result = simplify_gen_binary (IOR, mode, x, result);
3742 : }
3743 : else
3744 : {
3745 : /* if ((x & C) != 0) x ^= C; is transformed to x &= ~C. */
3746 0 : result = gen_int_mode (~(HOST_WIDE_INT_1 << bitnum), mode);
3747 0 : result = simplify_gen_binary (AND, mode, x, result);
3748 : }
3749 : }
3750 4 : else if (GET_CODE (a) == AND)
3751 : {
3752 : /* Check for "if (X & C) x &= ~C". */
3753 0 : if (! rtx_equal_p (x, XEXP (a, 0))
3754 0 : || !CONST_INT_P (XEXP (a, 1))
3755 0 : || (INTVAL (XEXP (a, 1)) & GET_MODE_MASK (mode))
3756 0 : != (~(HOST_WIDE_INT_1 << bitnum) & GET_MODE_MASK (mode)))
3757 : return false;
3758 :
3759 : /* if ((x & C) == 0) x &= ~C; is transformed to nothing. */
3760 : /* if ((x & C) != 0) x &= ~C; is transformed to x &= ~C. */
3761 0 : result = (code == EQ) ? a : NULL_RTX;
3762 : }
3763 : else
3764 : return false;
3765 :
3766 0 : if (result)
3767 : {
3768 0 : start_sequence ();
3769 0 : noce_emit_move_insn (x, result);
3770 0 : seq = end_ifcvt_sequence (if_info);
3771 0 : if (!seq)
3772 : return false;
3773 :
3774 0 : emit_insn_before_setloc (seq, if_info->jump,
3775 0 : INSN_LOCATION (if_info->insn_a));
3776 : }
3777 0 : if_info->transform_name = "noce_try_bitop";
3778 0 : return true;
3779 : }
3780 :
3781 :
3782 : /* Similar to get_condition, only the resulting condition must be
3783 : valid at JUMP, instead of at EARLIEST.
3784 :
3785 : If THEN_ELSE_REVERSED is true, the fallthrough does not go to the
3786 : THEN block of the caller, and we have to reverse the condition. */
3787 :
3788 : static rtx
3789 4316765 : noce_get_condition (rtx_insn *jump, rtx_insn **earliest,
3790 : bool then_else_reversed)
3791 : {
3792 4316765 : rtx cond, set, tmp;
3793 4316765 : bool reverse;
3794 :
3795 4316765 : if (! any_condjump_p (jump))
3796 : return NULL_RTX;
3797 :
3798 4316765 : set = pc_set (jump);
3799 :
3800 : /* If this branches to JUMP_LABEL when the condition is false,
3801 : reverse the condition. */
3802 8633530 : reverse = (GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
3803 4316765 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (jump));
3804 :
3805 : /* We may have to reverse because the caller's if block is not canonical,
3806 : i.e. the THEN block isn't the fallthrough block for the TEST block
3807 : (see find_if_header). */
3808 4316765 : if (then_else_reversed)
3809 1914842 : reverse = !reverse;
3810 :
3811 : /* If the condition variable is a register and is MODE_INT, accept it. */
3812 :
3813 4316765 : cond = XEXP (SET_SRC (set), 0);
3814 4316765 : tmp = XEXP (cond, 0);
3815 4316091 : if (REG_P (tmp) && GET_MODE_CLASS (GET_MODE (tmp)) == MODE_INT
3816 4316783 : && (GET_MODE (tmp) != BImode
3817 0 : || !targetm.small_register_classes_for_mode_p (BImode)))
3818 : {
3819 18 : *earliest = jump;
3820 :
3821 18 : if (reverse)
3822 9 : cond = gen_rtx_fmt_ee (reverse_condition (GET_CODE (cond)),
3823 : GET_MODE (cond), tmp, XEXP (cond, 1));
3824 : return cond;
3825 : }
3826 :
3827 : /* Otherwise, fall back on canonicalize_condition to do the dirty
3828 : work of manipulating MODE_CC values and COMPARE rtx codes. */
3829 4316747 : tmp = canonicalize_condition (jump, cond, reverse, earliest,
3830 : NULL_RTX, have_cbranchcc4, true);
3831 :
3832 : /* We don't handle side-effects in the condition, like handling
3833 : REG_INC notes and making sure no duplicate conditions are emitted. */
3834 4316747 : if (tmp != NULL_RTX && side_effects_p (tmp))
3835 8494 : return NULL_RTX;
3836 :
3837 : return tmp;
3838 : }
3839 :
3840 : /* Return true if OP is ok for if-then-else processing. */
3841 :
3842 : static bool
3843 8935745 : noce_operand_ok (const_rtx op)
3844 : {
3845 8935745 : if (side_effects_p (op))
3846 : return false;
3847 :
3848 : /* We special-case memories, so handle any of them with
3849 : no address side effects. */
3850 8914308 : if (MEM_P (op))
3851 1595467 : return ! side_effects_p (XEXP (op, 0));
3852 :
3853 7318841 : return ! may_trap_p (op);
3854 : }
3855 :
3856 : /* Return true iff basic block TEST_BB is valid for noce if-conversion.
3857 : The condition used in this if-conversion is in COND.
3858 : In practice, check that TEST_BB ends with a single set
3859 : x := a and all previous computations
3860 : in TEST_BB don't produce any values that are live after TEST_BB.
3861 : In other words, all the insns in TEST_BB are there only
3862 : to compute a value for x. Add the rtx cost of the insns
3863 : in TEST_BB to COST. Record whether TEST_BB is a single simple
3864 : set instruction in SIMPLE_P. */
3865 :
3866 : static bool
3867 2159531 : bb_valid_for_noce_process_p (basic_block test_bb, rtx cond,
3868 : unsigned int *cost, bool *simple_p)
3869 : {
3870 2159531 : if (!test_bb)
3871 : return false;
3872 :
3873 2159531 : rtx_insn *last_insn = last_active_insn (test_bb, false);
3874 2159531 : rtx last_set = NULL_RTX;
3875 :
3876 2159531 : rtx cc = cc_in_cond (cond);
3877 :
3878 2159531 : if (!insn_valid_noce_process_p (last_insn, cc))
3879 : return false;
3880 :
3881 : /* Punt on blocks ending with asm goto or jumps with other side-effects,
3882 : last_active_insn ignores JUMP_INSNs. */
3883 1530083 : if (JUMP_P (BB_END (test_bb)) && !onlyjump_p (BB_END (test_bb)))
3884 : return false;
3885 :
3886 1530081 : last_set = single_set (last_insn);
3887 :
3888 1530081 : rtx x = SET_DEST (last_set);
3889 1530081 : rtx_insn *first_insn = first_active_insn (test_bb);
3890 1530081 : rtx first_set = single_set (first_insn);
3891 :
3892 1530081 : if (!first_set)
3893 : return false;
3894 :
3895 : /* We have a single simple set, that's okay. */
3896 1516076 : bool speed_p = optimize_bb_for_speed_p (test_bb);
3897 :
3898 1516076 : if (first_insn == last_insn)
3899 : {
3900 549103 : *simple_p = noce_operand_ok (SET_DEST (first_set));
3901 549103 : *cost += pattern_cost (first_set, speed_p);
3902 549103 : return *simple_p;
3903 : }
3904 :
3905 966973 : rtx_insn *prev_last_insn = PREV_INSN (last_insn);
3906 966973 : gcc_assert (prev_last_insn);
3907 :
3908 : /* For now, disallow setting x multiple times in test_bb. */
3909 966973 : if (REG_P (x) && reg_set_between_p (x, first_insn, prev_last_insn))
3910 : return false;
3911 :
3912 819846 : auto_bitmap test_bb_temps (®_obstack);
3913 :
3914 : /* The regs that are live out of test_bb. */
3915 819846 : bitmap test_bb_live_out = df_get_live_out (test_bb);
3916 :
3917 819846 : int potential_cost = pattern_cost (last_set, speed_p);
3918 819846 : rtx_insn *insn;
3919 4490479 : FOR_BB_INSNS (test_bb, insn)
3920 : {
3921 4411774 : if (insn != last_insn)
3922 : {
3923 4333069 : if (!active_insn_p (insn))
3924 2862350 : continue;
3925 :
3926 1470719 : if (!insn_valid_noce_process_p (insn, cc))
3927 : return false;
3928 :
3929 1239156 : rtx sset = single_set (insn);
3930 1239156 : gcc_assert (sset);
3931 1239156 : rtx dest = SET_DEST (sset);
3932 1239156 : if (SUBREG_P (dest))
3933 18583 : dest = SUBREG_REG (dest);
3934 :
3935 1239156 : if (contains_mem_rtx_p (SET_SRC (sset))
3936 866879 : || !REG_P (dest)
3937 1977116 : || reg_overlap_mentioned_p (dest, cond))
3938 : return false;
3939 :
3940 729578 : potential_cost += pattern_cost (sset, speed_p);
3941 729578 : bitmap_set_bit (test_bb_temps, REGNO (dest));
3942 : }
3943 : }
3944 :
3945 : /* If any of the intermediate results in test_bb are live after test_bb
3946 : then fail. */
3947 78705 : if (bitmap_intersect_p (test_bb_live_out, test_bb_temps))
3948 : return false;
3949 :
3950 30516 : *cost += potential_cost;
3951 30516 : *simple_p = false;
3952 30516 : return true;
3953 819846 : }
3954 :
3955 : /* Helper function to emit a cmov sequence encapsulated in
3956 : start_sequence () and end_sequence (). If NEED_CMOV is true
3957 : we call noce_emit_cmove to create a cmove sequence. Otherwise emit
3958 : a simple move. If successful, store the first instruction of the
3959 : sequence in TEMP_DEST and the sequence costs in SEQ_COST. */
3960 :
3961 : static rtx_insn*
3962 393689 : try_emit_cmove_seq (noce_if_info *if_info, rtx temp,
3963 : rtx cond, rtx new_val, rtx old_val, bool need_cmov,
3964 : unsigned *cost, rtx *temp_dest,
3965 : rtx cc_cmp = NULL, rtx rev_cc_cmp = NULL)
3966 : {
3967 393689 : rtx_insn *seq = NULL;
3968 393689 : *cost = 0;
3969 :
3970 393689 : rtx x = XEXP (cond, 0);
3971 393689 : rtx y = XEXP (cond, 1);
3972 393689 : rtx_code cond_code = GET_CODE (cond);
3973 :
3974 393689 : start_sequence ();
3975 :
3976 393689 : if (need_cmov)
3977 317573 : *temp_dest = noce_emit_cmove (if_info, temp, cond_code,
3978 : x, y, new_val, old_val, cc_cmp, rev_cc_cmp);
3979 : else
3980 : {
3981 76116 : *temp_dest = temp;
3982 76116 : if (if_info->then_else_reversed)
3983 2044 : noce_emit_move_insn (temp, old_val);
3984 : else
3985 74072 : noce_emit_move_insn (temp, new_val);
3986 : }
3987 :
3988 393689 : if (*temp_dest != NULL_RTX)
3989 : {
3990 387525 : seq = get_insns ();
3991 387525 : *cost = seq_cost (seq, if_info->speed_p);
3992 : }
3993 :
3994 393689 : end_sequence ();
3995 :
3996 393689 : return seq;
3997 : }
3998 :
3999 : /* Finish a successful noce if-conversion of IF_INFO whose replacement insns
4000 : have already been emitted before the branch. Delete the now-dead THEN
4001 : block, and ELSE too if this was a diamond. An IF-THEN-JOIN also has an edge
4002 : from TEST_BB straight to JOIN_BB that bypassed THEN, and that goes as well.
4003 : An IF-THEN-ELSE-JOIN has no such edge, and deleting ELSE_BB removes its two
4004 : edges instead. Redirect TEST_BB to JOIN_BB and merge the two when the tail
4005 : no longer needs its own block. Count the converted if-block and the CFG
4006 : changes it took. */
4007 :
4008 : static void
4009 178112 : noce_finish_if_conversion (noce_if_info *if_info)
4010 : {
4011 178112 : basic_block test_bb = if_info->test_bb;
4012 178112 : basic_block then_bb = if_info->then_bb;
4013 178112 : basic_block else_bb = if_info->else_bb;
4014 178112 : basic_block join_bb = if_info->join_bb;
4015 :
4016 178112 : if (else_bb)
4017 : {
4018 76209 : delete_basic_block (else_bb);
4019 76209 : num_true_changes++;
4020 : }
4021 : else
4022 101903 : remove_edge (find_edge (test_bb, join_bb));
4023 :
4024 178112 : remove_edge (find_edge (then_bb, join_bb));
4025 178112 : redirect_edge_and_branch_force (single_succ_edge (test_bb), join_bb);
4026 178112 : delete_basic_block (then_bb);
4027 178112 : num_true_changes++;
4028 :
4029 178112 : if (can_merge_blocks_p (test_bb, join_bb))
4030 : {
4031 134343 : merge_blocks (test_bb, join_bb);
4032 134343 : num_true_changes++;
4033 : }
4034 :
4035 178112 : num_updated_if_blocks++;
4036 178112 : }
4037 :
4038 : /* We have something like:
4039 :
4040 : if (x > y)
4041 : { i = EXPR_A; j = EXPR_B; k = EXPR_C; }
4042 :
4043 : Make it:
4044 :
4045 : tmp_i = (x > y) ? EXPR_A : i;
4046 : tmp_j = (x > y) ? EXPR_B : j;
4047 : tmp_k = (x > y) ? EXPR_C : k;
4048 : i = tmp_i;
4049 : j = tmp_j;
4050 : k = tmp_k;
4051 :
4052 : Subsequent passes are expected to clean up the extra moves.
4053 :
4054 : Look for special cases such as writes to one register which are
4055 : read back in another SET, as might occur in a swap idiom or
4056 : similar.
4057 :
4058 : These look like:
4059 :
4060 : if (x > y)
4061 : i = a;
4062 : j = i;
4063 :
4064 : Which we want to rewrite to:
4065 :
4066 : tmp_i = (x > y) ? a : i;
4067 : tmp_j = (x > y) ? tmp_i : j;
4068 : i = tmp_i;
4069 : j = tmp_j;
4070 :
4071 : We can catch these when looking at (SET x y) by keeping a list of the
4072 : registers we would have targeted before if-conversion and looking back
4073 : through it for an overlap with Y. If we find one, we rewire the
4074 : conditional set to use the temporary we introduced earlier.
4075 :
4076 : IF_INFO contains the useful information about the block structure and
4077 : jump instructions. For an IF-THEN-ELSE-JOIN, first evaluate the else
4078 : arm's sets into temporaries and retain their final values for the
4079 : conditional moves. Return true if the replacement is valid and profitable
4080 : and its CFG changes have been committed, otherwise return false. */
4081 :
4082 : static bool
4083 50594 : noce_convert_multiple_sets (noce_if_info *if_info)
4084 : {
4085 50594 : basic_block test_bb = if_info->test_bb;
4086 50594 : basic_block then_bb = if_info->then_bb;
4087 50594 : basic_block else_bb = if_info->else_bb;
4088 50594 : rtx_insn *jump = if_info->jump;
4089 50594 : rtx_insn *cond_earliest;
4090 50594 : rtx_insn *insn;
4091 :
4092 50594 : start_sequence ();
4093 :
4094 : /* Decompose the condition attached to the jump. */
4095 50594 : rtx cond = noce_get_condition (jump, &cond_earliest, false);
4096 :
4097 50594 : auto_delete_vec<noce_multiple_sets_info> insn_info;
4098 50594 : init_noce_multiple_sets_info (then_bb, insn_info);
4099 50594 : gcc_assert (!insn_info.is_empty ());
4100 50594 : unsigned then_count = insn_info.length ();
4101 50594 : location_t sequence_location
4102 50594 : = INSN_LOCATION (insn_info.last ()->unmodified_insn);
4103 :
4104 50594 : auto_delete_vec<noce_multiple_sets_info> else_insn_info;
4105 50594 : auto_vec<unsigned> else_only_indices;
4106 50594 : if (else_bb)
4107 : {
4108 22944 : init_noce_multiple_sets_info (else_bb, else_insn_info);
4109 :
4110 : /* Add one output entry for each live-out destination that is set only
4111 : by the else arm. Record the last definition, which holds the
4112 : value that reaches the join. */
4113 22944 : auto_bitmap then_dests, seen;
4114 86739 : for (unsigned i = 0; i < then_count; ++i)
4115 63795 : bitmap_set_bit (then_dests, REGNO (insn_info[i]->target));
4116 :
4117 86228 : for (int i = else_insn_info.length () - 1; i >= 0; --i)
4118 40340 : if (else_insn_info[i]->need_cmov
4119 38782 : && !bitmap_bit_p (then_dests,
4120 38782 : REGNO (else_insn_info[i]->target))
4121 47269 : && bitmap_set_bit (seen, REGNO (else_insn_info[i]->target)))
4122 6929 : else_only_indices.safe_push (i);
4123 22944 : else_only_indices.reverse ();
4124 :
4125 22944 : unsigned i, else_index;
4126 59426 : FOR_EACH_VEC_ELT (else_only_indices, i, else_index)
4127 : {
4128 6929 : noce_multiple_sets_info *info = new noce_multiple_sets_info;
4129 6929 : noce_multiple_sets_info *else_info = else_insn_info[else_index];
4130 6929 : info->target = else_info->target;
4131 6929 : info->temporary = NULL_RTX;
4132 6929 : info->unmodified_insn = else_info->unmodified_insn;
4133 6929 : info->need_cmov = true;
4134 6929 : insn_info.safe_push (info);
4135 : }
4136 22944 : }
4137 :
4138 : /* Include else-only outputs when enforcing the conversion limit. */
4139 50594 : if (insn_info.length () > (unsigned) param_max_rtl_if_conversion_insns)
4140 : {
4141 0 : end_sequence ();
4142 0 : return false;
4143 : }
4144 :
4145 50594 : int last_needs_comparison = -1;
4146 :
4147 50594 : bool use_cond_earliest = false;
4148 :
4149 50594 : bool ok = noce_convert_multiple_sets_1
4150 50594 : (if_info, cond, insn_info, else_insn_info, then_count, else_only_indices,
4151 : &last_needs_comparison, &use_cond_earliest);
4152 50594 : if (!ok)
4153 : return false;
4154 :
4155 : /* Always perform a second attempt that uses information gathered in the
4156 : first. At least we can omit creating temporaries until we definitely
4157 : need them. The sequence created in the second attempt is never worse
4158 : than the first. */
4159 :
4160 47780 : end_sequence ();
4161 47780 : start_sequence ();
4162 47780 : ok = noce_convert_multiple_sets_1
4163 47780 : (if_info, cond, insn_info, else_insn_info, then_count, else_only_indices,
4164 : &last_needs_comparison, &use_cond_earliest);
4165 :
4166 : /* Actually we should not fail anymore if we reached here,
4167 : but better still check. */
4168 47780 : if (!ok)
4169 : return false;
4170 :
4171 : /* Now fixup the assignments.
4172 : PR116405: Iterate in reverse order and keep track of the targets so that
4173 : a move does not overwrite a subsequent value when multiple instructions
4174 : have the same target. */
4175 47780 : unsigned i;
4176 47780 : noce_multiple_sets_info *info;
4177 47780 : auto_bitmap set_targets (®_obstack);
4178 226242 : FOR_EACH_VEC_ELT_REVERSE (insn_info, i, info)
4179 : {
4180 130682 : gcc_checking_assert (REG_P (info->target));
4181 :
4182 130682 : if (info->target != info->temporary
4183 130682 : && !bitmap_bit_p (set_targets, REGNO (info->target)))
4184 4104 : noce_emit_move_insn (info->target, info->temporary);
4185 :
4186 130682 : bitmap_set_bit (set_targets, REGNO (info->target));
4187 : }
4188 :
4189 : /* Actually emit the sequence if it isn't too expensive. */
4190 47780 : rtx_insn *seq = get_insns ();
4191 :
4192 : /* If the created sequence does not use cond_earliest (but the jump
4193 : does) add its cost to the original_cost before comparing costs. */
4194 47780 : unsigned int original_cost = if_info->original_cost;
4195 47780 : if (if_info->jump != if_info->cond_earliest && !use_cond_earliest)
4196 36273 : if_info->original_cost += insn_cost (if_info->cond_earliest,
4197 : if_info->speed_p);
4198 :
4199 47780 : if (!targetm.noce_conversion_profitable_p (seq, if_info))
4200 : {
4201 14619 : end_sequence ();
4202 14619 : return false;
4203 : }
4204 :
4205 : /* Restore the original cost in case we do not succeed below. */
4206 33161 : if_info->original_cost = original_cost;
4207 :
4208 219423 : for (insn = seq; insn; insn = NEXT_INSN (insn))
4209 186262 : set_used_flags (insn);
4210 :
4211 : /* Mark all our temporaries and targets as used. */
4212 181731 : for (const noce_multiple_sets_info *msi : insn_info)
4213 : {
4214 82248 : set_used_flags (msi->temporary);
4215 82248 : set_used_flags (msi->target);
4216 : }
4217 :
4218 33161 : set_used_flags (cond);
4219 :
4220 33161 : unshare_all_rtl_in_chain (seq);
4221 33161 : end_sequence ();
4222 :
4223 33161 : if (!seq)
4224 : return false;
4225 :
4226 219221 : for (insn = seq; insn; insn = NEXT_INSN (insn))
4227 186107 : if (JUMP_P (insn) || CALL_P (insn)
4228 186107 : || recog_memoized (insn) == -1)
4229 : return false;
4230 :
4231 : /* This path does not go through end_ifcvt_sequence, so apply the same rule
4232 : about condition codes that outlive the if-region. */
4233 33114 : if (noce_clobbers_live_cc_p (test_bb, seq))
4234 : return false;
4235 :
4236 33114 : emit_insn_before_setloc (seq, if_info->jump, sequence_location);
4237 :
4238 33114 : noce_finish_if_conversion (if_info);
4239 :
4240 33114 : if_info->transform_name = "noce_convert_multiple_sets";
4241 33114 : return true;
4242 98374 : }
4243 :
4244 : /* Return true if any insn in SEQ modifies CC_CMP, or REV_CC_CMP when there is
4245 : one. Such a sequence cannot be used to select on that condition. */
4246 :
4247 : static bool
4248 207461 : noce_seq_clobbers_cc_cmp_p (rtx_insn *seq, rtx cc_cmp, rtx rev_cc_cmp)
4249 : {
4250 314583 : for (rtx_insn *insn = seq; insn; insn = NEXT_INSN (insn))
4251 223450 : if (modified_in_p (cc_cmp, insn)
4252 223450 : || (rev_cc_cmp && modified_in_p (rev_cc_cmp, insn)))
4253 : return true;
4254 :
4255 : return false;
4256 : }
4257 :
4258 : /* Try to emit the multiple-set conversion described by IF_INFO, selecting on
4259 : the already decoded jump condition COND. INSN_INFO holds THEN_COUNT
4260 : then-arm entries followed by entries for the else-only definitions recorded
4261 : in ELSE_ONLY_INDICES. For a diamond, evaluate ELSE_INSN_INFO first and
4262 : retain its final values for the conditional moves.
4263 :
4264 : LAST_NEEDS_COMPARISON is -1 on the first attempt. Record in it the last set
4265 : that needs a temporary to preserve the comparison, then use that boundary
4266 : on the second attempt. Set USE_COND_EARLIEST if the emitted sequence uses
4267 : IF_INFO->cond_earliest. Return true if the complete sequence was
4268 : emitted. */
4269 :
4270 : static bool
4271 98374 : noce_convert_multiple_sets_1 (noce_if_info *if_info, rtx cond,
4272 : auto_delete_vec<noce_multiple_sets_info> &insn_info,
4273 : auto_delete_vec<noce_multiple_sets_info>
4274 : &else_insn_info,
4275 : unsigned then_count,
4276 : const vec<unsigned> &else_only_indices,
4277 : int *last_needs_comparison,
4278 : bool *use_cond_earliest)
4279 : {
4280 98374 : rtx_insn *jump = if_info->jump;
4281 :
4282 98374 : rtx cc_cmp = cond_exec_get_condition (jump);
4283 98374 : if (cc_cmp)
4284 98374 : cc_cmp = copy_rtx (cc_cmp);
4285 98374 : rtx rev_cc_cmp = cond_exec_get_condition (jump, /* get_reversed */ true);
4286 98374 : if (rev_cc_cmp)
4287 98374 : rev_cc_cmp = copy_rtx (rev_cc_cmp);
4288 :
4289 98374 : int count = 0;
4290 98374 : bool second_try = *last_needs_comparison != -1;
4291 98374 : *use_cond_earliest = false;
4292 :
4293 : /* For an IF-THEN-ELSE-JOIN, emit the else block's computations first into
4294 : fresh temporaries. This leaves the incoming register values available to
4295 : the then arm. The conditional moves below select the then or else
4296 : values. */
4297 98374 : if (if_info->else_bb)
4298 : {
4299 44038 : int else_count = 0;
4300 44038 : rtx_insn *before_else = get_last_insn ();
4301 44038 : location_t saved_location = curr_insn_location ();
4302 44038 : rtx_insn *else_insn;
4303 255885 : FOR_BB_INSNS (if_info->else_bb, else_insn)
4304 : {
4305 211847 : if (!active_insn_p (else_insn))
4306 138374 : continue;
4307 :
4308 73473 : noce_multiple_sets_info *info = else_insn_info[else_count];
4309 73473 : rtx set = single_set (else_insn);
4310 73473 : gcc_checking_assert (set && REG_P (SET_DEST (set))
4311 : && !HARD_REGISTER_P (SET_DEST (set)));
4312 :
4313 73473 : rtx target = SET_DEST (set);
4314 73473 : rtx value = copy_rtx (SET_SRC (set));
4315 73473 : int i, ii;
4316 153667 : FOR_EACH_VEC_ELT (info->rewired_src, i, ii)
4317 6721 : value = simplify_replace_rtx (value,
4318 6721 : else_insn_info[ii]->target,
4319 6721 : else_insn_info[ii]->temporary);
4320 :
4321 73473 : set_curr_insn_location (INSN_LOCATION (else_insn));
4322 73473 : rtx temporary = copy_to_mode_reg (GET_MODE (target), value);
4323 :
4324 73473 : info->temporary = temporary;
4325 73473 : else_count++;
4326 : }
4327 :
4328 44038 : set_curr_insn_location (saved_location);
4329 :
4330 88076 : gcc_checking_assert (else_count == (int) else_insn_info.length ());
4331 :
4332 : /* These insns run ahead of the conditional moves. If they change a
4333 : register the comparison reads we cannot reuse it, so bail. If they
4334 : only clobber the condition code, drop the shared compare so that every
4335 : move re-materializes its own. */
4336 :
4337 44038 : rtx_insn *first_else
4338 44038 : = before_else ? NEXT_INSN (before_else) : get_insns ();
4339 111361 : for (rtx_insn *ei = first_else; ei; ei = NEXT_INSN (ei))
4340 : {
4341 69080 : if (modified_in_p (cond, ei))
4342 : {
4343 1757 : end_sequence ();
4344 1757 : return false;
4345 : }
4346 67323 : if (cc_cmp
4347 67323 : && (modified_in_p (cc_cmp, ei)
4348 35214 : || (rev_cc_cmp && modified_in_p (rev_cc_cmp, ei))))
4349 : {
4350 : cc_cmp = NULL_RTX;
4351 : rev_cc_cmp = NULL_RTX;
4352 : }
4353 : }
4354 : }
4355 :
4356 719008 : for (count = 0; count < (int) insn_info.length (); ++count)
4357 : {
4358 263944 : noce_multiple_sets_info *info = insn_info[count];
4359 263944 : rtx_insn *insn = info->unmodified_insn;
4360 263944 : rtx set = single_set (insn);
4361 263944 : gcc_checking_assert (set);
4362 :
4363 263944 : rtx target = info->target;
4364 263944 : rtx temp;
4365 263944 : rtx new_val;
4366 263944 : rtx old_val;
4367 :
4368 263944 : if ((unsigned) count < then_count)
4369 : {
4370 250167 : new_val = SET_SRC (set);
4371 :
4372 250167 : int i, ii;
4373 309059 : FOR_EACH_VEC_ELT (info->rewired_src, i, ii)
4374 58892 : new_val = simplify_replace_rtx (new_val, insn_info[ii]->target,
4375 58892 : insn_info[ii]->temporary);
4376 :
4377 250167 : old_val = target;
4378 :
4379 : /* Use the final value assigned to TARGET on the else arm.
4380 : Scanning in reverse is important when the arm assigns the same
4381 : register more than once. */
4382 250167 : if (if_info->else_bb)
4383 332393 : for (int j = else_insn_info.length () - 1; j >= 0; --j)
4384 155915 : if (rtx_equal_p (target, else_insn_info[j]->target))
4385 : {
4386 49498 : old_val = else_insn_info[j]->temporary;
4387 49498 : break;
4388 : }
4389 : }
4390 : else
4391 : {
4392 13777 : unsigned else_index
4393 13777 : = else_only_indices[(unsigned) count - then_count];
4394 13777 : noce_multiple_sets_info *else_info = else_insn_info[else_index];
4395 13777 : gcc_checking_assert (rtx_equal_p (target, else_info->target));
4396 :
4397 13777 : new_val = target;
4398 13777 : old_val = else_info->temporary;
4399 : }
4400 :
4401 : /* As we are transforming
4402 : if (x > y)
4403 : {
4404 : a = b;
4405 : c = d;
4406 : }
4407 : into
4408 : a = (x > y) ...
4409 : c = (x > y) ...
4410 :
4411 : we potentially check x > y before every set.
4412 : Even though the check might be removed by subsequent passes, this means
4413 : that we cannot transform
4414 : if (x > y)
4415 : {
4416 : x = y;
4417 : ...
4418 : }
4419 : into
4420 : x = (x > y) ...
4421 : ...
4422 : since this would invalidate x and the following to-be-removed checks.
4423 : Therefore we introduce a temporary every time we are about to
4424 : overwrite a variable used in the check. Costing of a sequence with
4425 : these is going to be inaccurate so only use temporaries when
4426 : needed.
4427 :
4428 : If performing a second try, we know how many insns require a
4429 : temporary. For the last of these, we can omit creating one. */
4430 263944 : if (reg_overlap_mentioned_p (target, cond)
4431 263944 : && (!second_try || count < *last_needs_comparison))
4432 26365 : temp = gen_reg_rtx (GET_MODE (target));
4433 : else
4434 : temp = target;
4435 :
4436 : /* We have identified swap-style idioms before. A normal
4437 : set will need to be a cmov while the first instruction of a swap-style
4438 : idiom can be a regular move. This helps with costing. */
4439 263944 : bool need_cmov = info->need_cmov;
4440 :
4441 : /* If we had a non-canonical conditional jump (i.e. one where
4442 : the fallthrough is to the "else" case) we need to reverse
4443 : the conditional select. */
4444 263944 : if (if_info->then_else_reversed)
4445 51149 : std::swap (old_val, new_val);
4446 :
4447 : /* Try emitting a conditional move passing the backend the
4448 : canonicalized comparison. The backend is then able to
4449 : recognize expressions like
4450 :
4451 : if (x > y)
4452 : y = x;
4453 :
4454 : as min/max and emit an insn, accordingly. */
4455 263944 : unsigned cost1 = 0, cost2 = 0;
4456 263944 : rtx_insn *seq, *seq1, *seq2 = NULL;
4457 263944 : rtx temp_dest = NULL_RTX, temp_dest1 = NULL_RTX, temp_dest2 = NULL_RTX;
4458 263944 : bool read_comparison = false;
4459 :
4460 263944 : seq1 = try_emit_cmove_seq (if_info, temp, cond,
4461 : new_val, old_val, need_cmov,
4462 : &cost1, &temp_dest1);
4463 :
4464 : /* Here, we try to pass the backend a non-canonicalized cc comparison
4465 : as well. This allows the backend to emit a cmov directly without
4466 : creating an additional compare for each. If successful, costing
4467 : is easier and this sequence is usually preferred. */
4468 263944 : if (cc_cmp)
4469 : {
4470 129745 : seq2 = try_emit_cmove_seq (if_info, temp, cond,
4471 : new_val, old_val, need_cmov,
4472 : &cost2, &temp_dest2, cc_cmp, rev_cc_cmp);
4473 :
4474 : /* The if_then_else in SEQ2 may be affected when cc_cmp/rev_cc_cmp is
4475 : clobbered. We can't safely use the sequence in this case. */
4476 129745 : if (noce_seq_clobbers_cc_cmp_p (seq2, cc_cmp, rev_cc_cmp))
4477 192507 : seq2 = NULL;
4478 : }
4479 :
4480 : /* The backend might have created a sequence that uses the
4481 : condition as a value. Check this. */
4482 :
4483 : /* We cannot handle anything more complex than a reg or constant. */
4484 263944 : if (!REG_P (XEXP (cond, 0)) && !CONSTANT_P (XEXP (cond, 0)))
4485 263944 : read_comparison = true;
4486 :
4487 263944 : if (!REG_P (XEXP (cond, 1)) && !CONSTANT_P (XEXP (cond, 1)))
4488 263944 : read_comparison = true;
4489 :
4490 263944 : rtx_insn *walk = seq2;
4491 263944 : int if_then_else_count = 0;
4492 343754 : while (walk && !read_comparison)
4493 : {
4494 79810 : rtx exprs_to_check[2];
4495 79810 : unsigned int exprs_count = 0;
4496 :
4497 79810 : rtx set = single_set (walk);
4498 79810 : if (set && XEXP (set, 1)
4499 79810 : && GET_CODE (XEXP (set, 1)) == IF_THEN_ELSE)
4500 : {
4501 : /* We assume that this is the cmove created by the backend that
4502 : naturally uses the condition. */
4503 49884 : exprs_to_check[exprs_count++] = XEXP (XEXP (set, 1), 1);
4504 49884 : exprs_to_check[exprs_count++] = XEXP (XEXP (set, 1), 2);
4505 49884 : if_then_else_count++;
4506 : }
4507 29926 : else if (NONDEBUG_INSN_P (walk))
4508 29926 : exprs_to_check[exprs_count++] = PATTERN (walk);
4509 :
4510 : /* Bail if we get more than one if_then_else because the assumption
4511 : above may be incorrect. */
4512 79810 : if (if_then_else_count > 1)
4513 : {
4514 0 : read_comparison = true;
4515 0 : break;
4516 : }
4517 :
4518 209504 : for (unsigned int i = 0; i < exprs_count; i++)
4519 : {
4520 129694 : subrtx_iterator::array_type array;
4521 289723 : FOR_EACH_SUBRTX (iter, array, exprs_to_check[i], NONCONST)
4522 190991 : if (*iter != NULL_RTX
4523 190991 : && (reg_overlap_mentioned_p (XEXP (cond, 0), *iter)
4524 172789 : || reg_overlap_mentioned_p (XEXP (cond, 1), *iter)))
4525 : {
4526 : read_comparison = true;
4527 : break;
4528 : }
4529 129694 : }
4530 :
4531 79810 : walk = NEXT_INSN (walk);
4532 : }
4533 :
4534 : /* Check which version is less expensive. */
4535 263944 : if (seq1 != NULL_RTX && (cost1 <= cost2 || seq2 == NULL_RTX))
4536 : {
4537 211445 : seq = seq1;
4538 211445 : temp_dest = temp_dest1;
4539 211445 : if (!second_try)
4540 106545 : *last_needs_comparison = count;
4541 : }
4542 1124 : else if (seq2 != NULL_RTX)
4543 : {
4544 51595 : seq = seq2;
4545 51595 : temp_dest = temp_dest2;
4546 51595 : if (!second_try && read_comparison)
4547 8750 : *last_needs_comparison = count;
4548 51595 : *use_cond_earliest = true;
4549 : }
4550 : else
4551 : {
4552 : /* Nothing worked, bail out. */
4553 904 : end_sequence ();
4554 1057 : return false;
4555 : }
4556 :
4557 : /* Although we use temporaries if there is register overlap of COND and
4558 : TARGET, it is possible that SEQ modifies COND anyway. For example,
4559 : COND may use the flags register and if INSN clobbers flags then
4560 : we may be unable to emit a valid sequence (e.g. in x86 that would
4561 : require saving and restoring the flags register). */
4562 158140 : if (!second_try)
4563 415962 : for (rtx_insn *iter = seq; iter; iter = NEXT_INSN (iter))
4564 283757 : if (modified_in_p (cond, iter))
4565 : {
4566 153 : end_sequence ();
4567 153 : return false;
4568 : }
4569 :
4570 : /* If SEQ clobbers registers mentioned in cc_cmp/rev_cc_cmp we have to
4571 : fall back on SEQ1 from that point on. Only check when we use SEQ1,
4572 : since SEQ2 has been tested already. */
4573 262887 : if (cc_cmp && seq == seq1
4574 262887 : && noce_seq_clobbers_cc_cmp_p (seq, cc_cmp, rev_cc_cmp))
4575 : {
4576 : cc_cmp = NULL_RTX;
4577 : rev_cc_cmp = NULL_RTX;
4578 : }
4579 :
4580 : /* End the sub sequence and emit to the main sequence. */
4581 262887 : emit_insn (seq);
4582 :
4583 : /* Bookkeeping. */
4584 262887 : info->temporary = temp_dest;
4585 : }
4586 :
4587 : /* Even if we did not actually need the comparison, we want to make sure
4588 : to try a second time in order to get rid of the temporaries. */
4589 95560 : if (*last_needs_comparison == -1)
4590 965 : *last_needs_comparison = 0;
4591 :
4592 : return true;
4593 : }
4594 :
4595 : /* Fill INSN_INFO with one entry per active insn in BB. Find local swap-style
4596 : idioms and mark the first insn (1) that is only a temporary as not needing
4597 : a conditional move, as it is going to be dead afterwards anyway.
4598 :
4599 : (1) int tmp = a;
4600 : a = b;
4601 : b = tmp;
4602 :
4603 : ifcvt
4604 : -->
4605 :
4606 : tmp = a;
4607 : a = cond ? b : a_old;
4608 : b = cond ? tmp : b_old;
4609 :
4610 : Additionally, store the index of insns like (2) when a subsequent
4611 : SET reads from their destination.
4612 :
4613 : (2) int c = a;
4614 : int d = c;
4615 :
4616 : ifcvt
4617 : -->
4618 :
4619 : c = cond ? a : c_old;
4620 : d = cond ? d : c; // Need to use c rather than c_old here. */
4621 :
4622 : static void
4623 73538 : init_noce_multiple_sets_info (basic_block bb,
4624 : auto_delete_vec<noce_multiple_sets_info> &insn_info)
4625 : {
4626 73538 : rtx_insn *insn;
4627 73538 : int count = 0;
4628 73538 : auto_vec<rtx> dests;
4629 73538 : bitmap bb_live_out = df_get_live_out (bb);
4630 :
4631 : /* Iterate over all SETs, storing the destinations in DEST.
4632 : - If we encounter a previously changed register,
4633 : rewire the read to the original source.
4634 : - If we encounter a SET that writes to a destination
4635 : that is not live after this block then the register
4636 : does not need to be moved conditionally. */
4637 494767 : FOR_BB_INSNS (bb, insn)
4638 : {
4639 421229 : if (!active_insn_p (insn))
4640 246965 : continue;
4641 :
4642 174264 : noce_multiple_sets_info *info = new noce_multiple_sets_info;
4643 174264 : info->temporary = NULL_RTX;
4644 174264 : insn_info.safe_push (info);
4645 :
4646 174264 : rtx set = single_set (insn);
4647 174264 : gcc_checking_assert (set);
4648 :
4649 174264 : rtx src = SET_SRC (set);
4650 174264 : rtx dest = SET_DEST (set);
4651 :
4652 174264 : gcc_checking_assert (REG_P (dest) && !HARD_REGISTER_P (dest));
4653 174264 : info->target = dest;
4654 174264 : info->unmodified_insn = insn;
4655 174264 : info->need_cmov = bitmap_bit_p (bb_live_out, REGNO (dest));
4656 :
4657 : /* Check if the current SET's source mentions any previously seen
4658 : destination. Keep only the newest definition of each pseudo register.
4659 : This is quadratic but the number of insns in BB
4660 : is bounded by PARAM_MAX_RTL_IF_CONVERSION_INSNS. */
4661 174264 : auto_bitmap rewired_regs;
4662 343306 : for (int i = count - 1; i >= 0; --i)
4663 169042 : if (reg_mentioned_p (dests[i], src)
4664 169042 : && bitmap_set_bit (rewired_regs, REGNO (dests[i])))
4665 33709 : insn_info[count]->rewired_src.safe_push (i);
4666 :
4667 174264 : dests.safe_push (dest);
4668 174264 : count++;
4669 174264 : }
4670 73538 : }
4671 :
4672 : /* Return true iff basic block TEST_BB is suitable for conversion to a
4673 : series of conditional moves. Unless REQUIRE_MULTIPLE is false, also check
4674 : that we have more than one set (other routines can handle a single set
4675 : better than we would). A diamond arm may have a single set.
4676 : Require no more than PARAM_MAX_RTL_IF_CONVERSION_INSNS sets. While going
4677 : through the insns, store the sum of their potential costs in COST. On
4678 : success, if LIVE_OUT_DESTS is nonnull, record the distinct pseudo
4679 : destinations that are live out of
4680 : TEST_BB. If HAS_NON_SIMPLE_SRC is nonnull, set it when an instruction
4681 : source is neither a constant nor a register operand. On success, if
4682 : INSN_COUNT is nonnull, store the number of active sets in it. */
4683 :
4684 : static bool
4685 2047172 : bb_ok_for_noce_convert_multiple_sets (basic_block test_bb, unsigned *cost,
4686 : bool require_multiple = true,
4687 : bitmap live_out_dests = NULL,
4688 : bool *has_non_simple_src = NULL,
4689 : unsigned *insn_count = NULL)
4690 : {
4691 2047172 : rtx_insn *insn;
4692 2047172 : unsigned count = 0;
4693 2047172 : unsigned param = param_max_rtl_if_conversion_insns;
4694 2047172 : bool speed_p = optimize_bb_for_speed_p (test_bb);
4695 2047172 : unsigned potential_cost = 0;
4696 2047172 : if (live_out_dests)
4697 959734 : bitmap_clear (live_out_dests);
4698 2047172 : if (has_non_simple_src)
4699 959734 : *has_non_simple_src = false;
4700 2047172 : bitmap bb_live_out = NULL;
4701 2047172 : if (live_out_dests)
4702 959734 : bb_live_out = df_get_live_out (test_bb);
4703 :
4704 12474158 : FOR_BB_INSNS (test_bb, insn)
4705 : {
4706 : /* Skip over notes etc. */
4707 12104911 : if (!active_insn_p (insn))
4708 9485955 : continue;
4709 :
4710 : /* We only handle SET insns. */
4711 2618956 : rtx set = single_set (insn);
4712 2618956 : if (set == NULL_RTX)
4713 : return false;
4714 :
4715 2543207 : rtx dest = SET_DEST (set);
4716 2543207 : rtx src = SET_SRC (set);
4717 :
4718 2543207 : if (has_non_simple_src && !noce_simple_cmove_operand_p (src))
4719 594844 : *has_non_simple_src = true;
4720 :
4721 : /* Dependency rewiring is keyed by register number, so restrict
4722 : destinations to pseudos. Hard-register definitions can overlap
4723 : without having the same mode. Do not handle anything involving
4724 : memory loads/stores since it might violate data-race-freedom
4725 : guarantees. Make sure we can force SRC to a register as that may
4726 : be needed in try_emit_cmove_seq. */
4727 2118906 : if (!REG_P (dest) || HARD_REGISTER_P (dest)
4728 1682499 : || contains_mem_rtx_p (src)
4729 3555695 : || !noce_can_force_operand (src))
4730 : return false;
4731 :
4732 : /* Destination and source must be appropriate. */
4733 993315 : if (!noce_operand_ok (dest) || !noce_operand_ok (src))
4734 : return false;
4735 :
4736 : /* We must be able to conditionally move in this mode. */
4737 970543 : if (!can_conditionally_move_p (GET_MODE (dest)))
4738 : return false;
4739 :
4740 941031 : if (bb_live_out && bitmap_bit_p (bb_live_out, REGNO (dest)))
4741 403669 : bitmap_set_bit (live_out_dests, REGNO (dest));
4742 :
4743 941031 : potential_cost += insn_cost (insn, speed_p);
4744 :
4745 941031 : count++;
4746 : }
4747 :
4748 369247 : *cost += potential_cost;
4749 :
4750 : /* If we would only put out one conditional move, the other strategies
4751 : this pass tries are better optimized and will be more appropriate, so
4752 : require more than one set unless REQUIRE_MULTIPLE is false. A diamond
4753 : arm may have one set.
4754 : Some targets want to strictly limit the number of conditional moves
4755 : that are emitted, they set this through PARAM, we need to respect
4756 : that. */
4757 623647 : bool ok = count >= (require_multiple ? 2u : 1u) && count <= param;
4758 369247 : if (ok && insn_count)
4759 252706 : *insn_count = count;
4760 : return ok;
4761 : }
4762 :
4763 : /* Compute average of two given costs weighted by relative probabilities
4764 : of respective basic blocks in an IF-THEN-ELSE. E is the IF-THEN edge.
4765 : With P as the probability to take the IF-THEN branch, return
4766 : P * THEN_COST + (1 - P) * ELSE_COST. Evaluate this as
4767 : ELSE_COST + P * (THEN_COST - ELSE_COST) when THEN_COST >= ELSE_COST, and
4768 : THEN_COST + (1 - P) * (ELSE_COST - THEN_COST) otherwise. Both forms pass
4769 : a nonnegative value to profile_probability::apply and make its rounding
4770 : independent of the CFG arm order. */
4771 : static unsigned
4772 277189 : average_cost (unsigned then_cost, unsigned else_cost, edge e)
4773 : {
4774 277189 : if (then_cost < else_cost)
4775 9997 : return then_cost
4776 9997 : + e->probability.invert ().apply ((gcov_type) else_cost - then_cost);
4777 :
4778 267192 : return else_cost
4779 267192 : + e->probability.apply ((gcov_type) then_cost - else_cost);
4780 : }
4781 :
4782 : /* Return the estimated cost of the original, un-converted if-region described
4783 : by IF_INFO whose THEN and ELSE arms cost THEN_COST and ELSE_COST. BASE_COST
4784 : already accounts for the branch, and any compare, that the conversion
4785 : removes. When optimizing for speed (SPEED_P) only one arm runs, so charge
4786 : the branch-probability-weighted average of the two. When optimizing for
4787 : size both arms are emitted, so sum them. */
4788 :
4789 : static unsigned
4790 320916 : noce_original_region_cost (const noce_if_info *if_info, bool speed_p,
4791 : unsigned base_cost, unsigned then_cost,
4792 : unsigned else_cost)
4793 : {
4794 320916 : if (speed_p)
4795 277189 : return base_cost + average_cost (then_cost, else_cost,
4796 277189 : find_edge (if_info->test_bb,
4797 554378 : if_info->then_bb));
4798 43727 : return base_cost + then_cost + else_cost;
4799 : }
4800 :
4801 : /* Given a simple IF-THEN-JOIN or IF-THEN-ELSE-JOIN block, attempt to convert
4802 : it without using conditional execution. Return TRUE if we were successful
4803 : at converting the block. */
4804 :
4805 : static bool
4806 1910998 : noce_process_if_block (noce_if_info *if_info)
4807 : {
4808 1910998 : basic_block test_bb = if_info->test_bb; /* test block */
4809 1910998 : basic_block then_bb = if_info->then_bb; /* THEN */
4810 1910998 : basic_block else_bb = if_info->else_bb; /* ELSE or NULL */
4811 1910998 : rtx_insn *jump = if_info->jump;
4812 1910998 : rtx cond = if_info->cond;
4813 1910998 : rtx_insn *insn_a, *insn_b;
4814 1910998 : rtx set_a, set_b;
4815 1910998 : rtx orig_x, x, a, b;
4816 :
4817 : /* We're looking for patterns of the form
4818 :
4819 : (1) if (...) x = a; else x = b;
4820 : (2) x = b; if (...) x = a;
4821 : (3) if (...) x = a; // as if with an initial x = x.
4822 : (4) if (...) { x = a; y = b; z = c; } // Like 3, for multiple SETS.
4823 : (5) A multi-set IF-THEN-ELSE-JOIN.
4824 : The later patterns require jumps to be more expensive.
4825 : For the diamond case, defer diamonds with only register and constant
4826 : assignments in both arms to the existing simple conditional-move
4827 : handling. Require at least two distinct live-out pseudos across the
4828 : arms.
4829 : ??? For future expansion, further expand the "multiple X" rules. */
4830 :
4831 : /* The base cost recorded so far covers the branch, and any compare, that
4832 : if-conversion removes. Each candidate path below derives its estimate
4833 : from this base plus the cost of the arms it folds in. If a target
4834 : re-uses the existing CC comparison, noce_convert_multiple_sets accounts
4835 : for that against the base before it calls
4836 : noce_conversion_profitable_p. */
4837 1910998 : unsigned base_cost = if_info->original_cost;
4838 1910998 : bool speed_p = optimize_bb_for_speed_p (test_bb);
4839 :
4840 1910998 : unsigned ms_then_cost = 0, ms_else_cost = 0;
4841 1910998 : unsigned ms_then_insn_count = 0, ms_else_insn_count = 0;
4842 1910998 : bool ms_then_has_non_simple_src, ms_else_has_non_simple_src;
4843 1910998 : auto_bitmap ms_then_live_out_dests, ms_else_live_out_dests;
4844 1910998 : noce_if_info ms_if_info = *if_info;
4845 1910998 : bool multiple_sets_p = false;
4846 :
4847 1910998 : if (HAVE_conditional_move)
4848 : {
4849 1910998 : if (!else_bb)
4850 1087438 : multiple_sets_p
4851 1087438 : = bb_ok_for_noce_convert_multiple_sets (then_bb, &ms_then_cost);
4852 823560 : else if (!if_info->then_else_reversed
4853 823560 : && bb_ok_for_noce_convert_multiple_sets
4854 823560 : (then_bb, &ms_then_cost, false,
4855 : ms_then_live_out_dests, &ms_then_has_non_simple_src,
4856 : &ms_then_insn_count)
4857 959734 : && bb_ok_for_noce_convert_multiple_sets
4858 136174 : (else_bb, &ms_else_cost, false,
4859 : ms_else_live_out_dests, &ms_else_has_non_simple_src,
4860 : &ms_else_insn_count))
4861 : {
4862 116532 : auto_bitmap ms_live_out_dests;
4863 116532 : bitmap_ior (ms_live_out_dests, ms_then_live_out_dests,
4864 116532 : ms_else_live_out_dests);
4865 116532 : unsigned output_count = bitmap_count_bits (ms_live_out_dests);
4866 :
4867 116532 : if (output_count >= 2
4868 50151 : && output_count <= (unsigned) param_max_rtl_if_conversion_insns
4869 50149 : && (ms_then_has_non_simple_src || ms_else_has_non_simple_src))
4870 116532 : multiple_sets_p = true;
4871 116532 : }
4872 : }
4873 :
4874 : /* The diamond conversion evaluates the else arm unconditionally.
4875 : Honor the target limit on the number of instructions made
4876 : unconditional. */
4877 1203970 : if (multiple_sets_p
4878 53625 : && ms_if_info.else_bb
4879 1229945 : && ms_else_insn_count > (unsigned) MAX_CONDITIONAL_EXECUTE)
4880 : multiple_sets_p = false;
4881 :
4882 1907967 : if (multiple_sets_p)
4883 : {
4884 : /* The original code runs the comparison and one arm. Estimate that cost
4885 : and let noce_convert_multiple_sets convert only if the conditional
4886 : moves come out cheaper. */
4887 50594 : ms_if_info.original_cost
4888 101188 : = ms_if_info.else_bb
4889 50594 : ? noce_original_region_cost (&ms_if_info, speed_p, base_cost,
4890 : ms_then_cost, ms_else_cost)
4891 27650 : : base_cost + ms_then_cost;
4892 50594 : if (noce_convert_multiple_sets (&ms_if_info))
4893 : {
4894 33114 : if (dump_file && ms_if_info.transform_name)
4895 2 : fprintf (dump_file, "if-conversion succeeded through %s\n",
4896 : ms_if_info.transform_name);
4897 : return true;
4898 : }
4899 : }
4900 :
4901 1877884 : unsigned int then_cost = 0, else_cost = 0;
4902 1877884 : if (!bb_valid_for_noce_process_p (then_bb, cond, &then_cost,
4903 : &if_info->then_simple))
4904 : return false;
4905 :
4906 480971 : if (else_bb
4907 480971 : && !bb_valid_for_noce_process_p (else_bb, cond, &else_cost,
4908 : &if_info->else_simple))
4909 : return false;
4910 :
4911 297972 : if_info->original_cost
4912 297972 : = noce_original_region_cost (if_info, speed_p, base_cost, then_cost,
4913 : else_cost);
4914 :
4915 297972 : insn_a = last_active_insn (then_bb, false);
4916 297972 : set_a = single_set (insn_a);
4917 297972 : gcc_assert (set_a);
4918 :
4919 297972 : x = SET_DEST (set_a);
4920 297972 : a = SET_SRC (set_a);
4921 :
4922 : /* Look for the other potential set. Make sure we've got equivalent
4923 : destinations. */
4924 : /* ??? This is overconservative. Storing to two different mems is
4925 : as easy as conditionally computing the address. Storing to a
4926 : single mem merely requires a scratch memory to use as one of the
4927 : destination addresses; often the memory immediately below the
4928 : stack pointer is available for this. */
4929 297972 : set_b = NULL_RTX;
4930 297972 : if (else_bb)
4931 : {
4932 98648 : insn_b = last_active_insn (else_bb, false);
4933 98648 : set_b = single_set (insn_b);
4934 98648 : gcc_assert (set_b);
4935 :
4936 98648 : if (!rtx_interchangeable_p (x, SET_DEST (set_b)))
4937 : return false;
4938 : }
4939 : else
4940 : {
4941 199324 : insn_b = if_info->cond_earliest;
4942 539415 : do
4943 539415 : insn_b = prev_nonnote_nondebug_insn (insn_b);
4944 : while (insn_b
4945 536707 : && (BLOCK_FOR_INSN (insn_b)
4946 536707 : == BLOCK_FOR_INSN (if_info->cond_earliest))
4947 1189991 : && !modified_in_p (x, insn_b));
4948 :
4949 : /* We're going to be moving the evaluation of B down from above
4950 : COND_EARLIEST to JUMP. Make sure the relevant data is still
4951 : intact. */
4952 199324 : if (! insn_b
4953 196616 : || BLOCK_FOR_INSN (insn_b) != BLOCK_FOR_INSN (if_info->cond_earliest)
4954 111161 : || !NONJUMP_INSN_P (insn_b)
4955 105437 : || (set_b = single_set (insn_b)) == NULL_RTX
4956 103941 : || ! rtx_interchangeable_p (x, SET_DEST (set_b))
4957 89392 : || ! noce_operand_ok (SET_SRC (set_b))
4958 87421 : || reg_overlap_mentioned_p (x, SET_SRC (set_b))
4959 85964 : || modified_between_p (SET_SRC (set_b), insn_b, jump)
4960 : /* Avoid extending the lifetime of hard registers on small
4961 : register class machines. */
4962 83743 : || (REG_P (SET_SRC (set_b))
4963 20673 : && HARD_REGISTER_P (SET_SRC (set_b))
4964 3671 : && targetm.small_register_classes_for_mode_p
4965 3671 : (GET_MODE (SET_SRC (set_b))))
4966 : /* Likewise with X. In particular this can happen when
4967 : noce_get_condition looks farther back in the instruction
4968 : stream than one might expect. */
4969 80072 : || reg_overlap_mentioned_p (x, cond)
4970 66438 : || reg_overlap_mentioned_p (x, a)
4971 259798 : || modified_between_p (x, insn_b, jump))
4972 : {
4973 : insn_b = NULL;
4974 : set_b = NULL_RTX;
4975 : }
4976 : }
4977 :
4978 : /* If x has side effects then only the if-then-else form is safe to
4979 : convert. But even in that case we would need to restore any notes
4980 : (such as REG_INC) at then end. That can be tricky if
4981 : noce_emit_move_insn expands to more than one insn, so disable the
4982 : optimization entirely for now if there are side effects. */
4983 292535 : if (side_effects_p (x))
4984 : return false;
4985 :
4986 292535 : b = (set_b ? SET_SRC (set_b) : x);
4987 :
4988 : /* Only operate on register destinations, and even then avoid extending
4989 : the lifetime of hard registers on small register class machines. */
4990 292535 : orig_x = x;
4991 292535 : if_info->orig_x = orig_x;
4992 292535 : if (!REG_P (x)
4993 292535 : || (HARD_REGISTER_P (x)
4994 2 : && targetm.small_register_classes_for_mode_p (GET_MODE (x))))
4995 : {
4996 68124 : if (GET_MODE (x) == BLKmode)
4997 : return false;
4998 :
4999 67882 : if (GET_CODE (x) == ZERO_EXTRACT
5000 1 : && (!CONST_INT_P (XEXP (x, 1))
5001 1 : || !CONST_INT_P (XEXP (x, 2))))
5002 : return false;
5003 :
5004 67882 : x = gen_reg_rtx (GET_MODE (GET_CODE (x) == STRICT_LOW_PART
5005 : ? XEXP (x, 0) : x));
5006 : }
5007 :
5008 : /* Don't operate on sources that may trap or are volatile. */
5009 292293 : if (! noce_operand_ok (a) || ! noce_operand_ok (b))
5010 : return false;
5011 :
5012 319302 : retry:
5013 : /* Set up the info block for our subroutines. */
5014 319302 : if_info->insn_a = insn_a;
5015 319302 : if_info->insn_b = insn_b;
5016 319302 : if_info->x = x;
5017 319302 : if_info->a = a;
5018 319302 : if_info->b = b;
5019 :
5020 : /* Try optimizations in some approximation of a useful order. */
5021 : /* ??? Should first look to see if X is live incoming at all. If it
5022 : isn't, we don't need anything but an unconditional set. */
5023 :
5024 : /* Look and see if A and B are really the same. Avoid creating silly
5025 : cmove constructs that no one will fix up later. */
5026 319302 : if (noce_simple_bbs (if_info)
5027 296991 : && rtx_interchangeable_p (a, b))
5028 : {
5029 : /* If we have an INSN_B, we don't have to create any new rtl. Just
5030 : move the instruction that we already have. If we don't have an
5031 : INSN_B, that means that A == X, and we've got a noop move. In
5032 : that case don't do anything and let the code below delete INSN_A. */
5033 381 : if (insn_b && else_bb)
5034 : {
5035 372 : rtx note;
5036 :
5037 372 : if (insn_b == BB_END (else_bb))
5038 261 : BB_END (else_bb) = PREV_INSN (insn_b);
5039 372 : reorder_insns (insn_b, insn_b, PREV_INSN (jump));
5040 :
5041 : /* If there was a REG_EQUAL note, delete it since it may have been
5042 : true due to this insn being after a jump. */
5043 372 : if ((note = find_reg_note (insn_b, REG_EQUAL, NULL_RTX)) != 0)
5044 0 : remove_note (insn_b, note);
5045 :
5046 381 : insn_b = NULL;
5047 : }
5048 : /* If we have "x = b; if (...) x = a;", and x has side-effects, then
5049 : x must be executed twice. */
5050 9 : else if (insn_b && side_effects_p (orig_x))
5051 : return false;
5052 :
5053 381 : x = orig_x;
5054 381 : goto success;
5055 : }
5056 :
5057 318921 : if (!set_b && MEM_P (orig_x))
5058 : /* We want to avoid store speculation to avoid cases like
5059 : if (pthread_mutex_trylock(mutex))
5060 : ++global_variable;
5061 : Rather than go to much effort here, we rely on the SSA optimizers,
5062 : which do a good enough job these days. */
5063 : return false;
5064 :
5065 252720 : if (noce_try_move (if_info))
5066 0 : goto success;
5067 252720 : if (noce_try_ifelse_collapse (if_info))
5068 17798 : goto success;
5069 234922 : if (noce_try_store_flag (if_info))
5070 26001 : goto success;
5071 208921 : if (noce_try_bitop (if_info))
5072 0 : goto success;
5073 208921 : if (noce_try_minmax (if_info))
5074 60 : goto success;
5075 208861 : if (noce_try_abs (if_info))
5076 24 : goto success;
5077 208837 : if (noce_try_inverse_constants (if_info))
5078 0 : goto success;
5079 208837 : if (!targetm.have_conditional_execution ()
5080 208837 : && noce_try_store_flag_constants (if_info))
5081 3161 : goto success;
5082 205676 : if (!targetm.have_conditional_execution ()
5083 205676 : && noce_try_shifted_store_flag (if_info))
5084 5401 : goto success;
5085 200275 : if (noce_try_sign_bit_splat (if_info))
5086 178 : goto success;
5087 200097 : if (!targetm.have_conditional_execution ()
5088 200097 : && noce_try_store_flag_logical (if_info))
5089 0 : goto success;
5090 200097 : if (HAVE_conditional_move
5091 200097 : && noce_try_cmove (if_info))
5092 42588 : goto success;
5093 157509 : if (! targetm.have_conditional_execution ())
5094 : {
5095 157509 : if (noce_try_addcc (if_info))
5096 903 : goto success;
5097 156606 : if (noce_try_store_flag_mask (if_info))
5098 9 : goto success;
5099 156597 : if (HAVE_conditional_move
5100 156597 : && noce_try_cond_arith (if_info))
5101 10182 : goto success;
5102 146415 : if (HAVE_conditional_move
5103 146415 : && noce_try_cmove_arith (if_info))
5104 34943 : goto success;
5105 111472 : if (noce_try_sign_mask (if_info))
5106 0 : goto success;
5107 : }
5108 :
5109 111472 : if (!else_bb && set_b)
5110 : {
5111 27009 : insn_b = NULL;
5112 27009 : set_b = NULL_RTX;
5113 27009 : b = orig_x;
5114 27009 : goto retry;
5115 : }
5116 :
5117 : return false;
5118 :
5119 141629 : success:
5120 141629 : if (dump_file && if_info->transform_name)
5121 7 : fprintf (dump_file, "if-conversion succeeded through %s\n",
5122 : if_info->transform_name);
5123 :
5124 : /* If we used a temporary, fix it up now. */
5125 141629 : if (orig_x != x)
5126 : {
5127 675 : rtx_insn *seq;
5128 :
5129 675 : start_sequence ();
5130 675 : noce_emit_move_insn (orig_x, x);
5131 675 : seq = get_insns ();
5132 675 : set_used_flags (orig_x);
5133 675 : unshare_all_rtl_in_chain (seq);
5134 675 : end_sequence ();
5135 :
5136 675 : emit_insn_before_setloc (seq, BB_END (test_bb), INSN_LOCATION (insn_a));
5137 : }
5138 :
5139 : /* The original THEN and ELSE blocks may now be removed and the test block
5140 : redirected to the join block. */
5141 141629 : noce_finish_if_conversion (if_info);
5142 :
5143 141629 : return true;
5144 1910998 : }
5145 :
5146 : /* Check whether a block is suitable for conditional move conversion.
5147 : Every insn must be a simple set of a register to a constant or a
5148 : register. For each assignment, store the value in the pointer map
5149 : VALS, keyed by register pointer, then store the register pointer in
5150 : REGS. COND is the condition we will test. */
5151 :
5152 : static bool
5153 1766024 : check_cond_move_block (basic_block bb,
5154 : hash_map<rtx, rtx> *vals,
5155 : vec<rtx> *regs,
5156 : rtx cond)
5157 : {
5158 1766024 : rtx_insn *insn;
5159 1766024 : rtx cc = cc_in_cond (cond);
5160 :
5161 : /* We can only handle simple jumps at the end of the basic block.
5162 : It is almost impossible to update the CFG otherwise. */
5163 1766024 : insn = BB_END (bb);
5164 1766024 : if (JUMP_P (insn) && !onlyjump_p (insn))
5165 : return false;
5166 :
5167 10511451 : FOR_BB_INSNS (bb, insn)
5168 : {
5169 10449542 : rtx set, dest, src;
5170 :
5171 10449542 : if (!NONDEBUG_INSN_P (insn) || JUMP_P (insn))
5172 8611265 : continue;
5173 1838277 : set = single_set (insn);
5174 1838277 : if (!set)
5175 1704062 : return false;
5176 :
5177 1764595 : dest = SET_DEST (set);
5178 1764595 : src = SET_SRC (set);
5179 1764595 : if (!REG_P (dest)
5180 1764595 : || (HARD_REGISTER_P (dest)
5181 313353 : && targetm.small_register_classes_for_mode_p (GET_MODE (dest))))
5182 : return false;
5183 :
5184 1062375 : if (!noce_simple_cmove_operand_p (src))
5185 : return false;
5186 :
5187 156635 : if (side_effects_p (src) || side_effects_p (dest))
5188 : return false;
5189 :
5190 156635 : if (may_trap_p (src) || may_trap_p (dest))
5191 : return false;
5192 :
5193 : /* Don't try to handle this if the source register was
5194 : modified earlier in the block. */
5195 156635 : if ((REG_P (src)
5196 51402 : && vals->get (src))
5197 207776 : || (GET_CODE (src) == SUBREG && REG_P (SUBREG_REG (src))
5198 5259 : && vals->get (SUBREG_REG (src))))
5199 : return false;
5200 :
5201 : /* Don't try to handle this if the destination register was
5202 : modified earlier in the block. */
5203 156320 : if (vals->get (dest))
5204 : return false;
5205 :
5206 : /* Don't try to handle this if the condition uses the
5207 : destination register. */
5208 156320 : if (reg_overlap_mentioned_p (dest, cond))
5209 : return false;
5210 :
5211 : /* Don't try to handle this if the source register is modified
5212 : later in the block. */
5213 137439 : if (!CONSTANT_P (src)
5214 137439 : && modified_between_p (src, insn, NEXT_INSN (BB_END (bb))))
5215 : return false;
5216 :
5217 : /* Skip it if the instruction to be moved might clobber CC. */
5218 134215 : if (cc && set_of (cc, insn))
5219 : return false;
5220 :
5221 134215 : vals->put (dest, src);
5222 :
5223 134215 : regs->safe_push (dest);
5224 : }
5225 :
5226 : return true;
5227 : }
5228 :
5229 : /* Given a basic block BB suitable for conditional move conversion,
5230 : a condition COND, and pointer maps THEN_VALS and ELSE_VALS containing
5231 : the register values depending on COND, emit the insns in the block as
5232 : conditional moves. If ELSE_BLOCK is true, THEN_BB was already
5233 : processed. The caller has started a sequence for the conversion.
5234 : Return true if successful, false if something goes wrong. */
5235 :
5236 : static bool
5237 36785 : cond_move_convert_if_block (noce_if_info *if_infop,
5238 : basic_block bb, rtx cond,
5239 : hash_map<rtx, rtx> *then_vals,
5240 : hash_map<rtx, rtx> *else_vals,
5241 : bool else_block_p)
5242 : {
5243 36785 : enum rtx_code code;
5244 36785 : rtx_insn *insn;
5245 36785 : rtx cond_arg0, cond_arg1;
5246 :
5247 36785 : code = GET_CODE (cond);
5248 36785 : cond_arg0 = XEXP (cond, 0);
5249 36785 : cond_arg1 = XEXP (cond, 1);
5250 :
5251 126752 : FOR_BB_INSNS (bb, insn)
5252 : {
5253 104335 : rtx set, target, dest, t, e;
5254 :
5255 : /* ??? Maybe emit conditional debug insn? */
5256 104335 : if (!NONDEBUG_INSN_P (insn) || JUMP_P (insn))
5257 69046 : continue;
5258 46417 : set = single_set (insn);
5259 46417 : gcc_assert (set && REG_P (SET_DEST (set)));
5260 :
5261 46417 : dest = SET_DEST (set);
5262 :
5263 46417 : rtx *then_slot = then_vals->get (dest);
5264 46417 : rtx *else_slot = else_vals->get (dest);
5265 46417 : t = then_slot ? *then_slot : NULL_RTX;
5266 46417 : e = else_slot ? *else_slot : NULL_RTX;
5267 :
5268 46417 : if (else_block_p)
5269 : {
5270 : /* If this register was set in the then block, we already
5271 : handled this case there. */
5272 12400 : if (t)
5273 11128 : continue;
5274 1272 : t = dest;
5275 1272 : gcc_assert (e);
5276 : }
5277 : else
5278 : {
5279 34017 : gcc_assert (t);
5280 34017 : if (!e)
5281 22191 : e = dest;
5282 : }
5283 :
5284 35289 : if (if_infop->cond_inverted)
5285 0 : std::swap (t, e);
5286 :
5287 35289 : target = noce_emit_cmove (if_infop, dest, code, cond_arg0, cond_arg1,
5288 : t, e);
5289 35289 : if (!target)
5290 14368 : return false;
5291 :
5292 20921 : if (target != dest)
5293 0 : noce_emit_move_insn (dest, target);
5294 : }
5295 :
5296 : return true;
5297 : }
5298 :
5299 : /* Given a simple IF-THEN-JOIN or IF-THEN-ELSE-JOIN block, attempt to convert
5300 : it using only conditional moves. Return TRUE if we were successful at
5301 : converting the block. */
5302 :
5303 : static bool
5304 1736255 : cond_move_process_if_block (noce_if_info *if_info)
5305 : {
5306 1736255 : basic_block then_bb = if_info->then_bb;
5307 1736255 : basic_block else_bb = if_info->else_bb;
5308 1736255 : rtx_insn *jump = if_info->jump;
5309 1736255 : rtx cond = if_info->cond;
5310 1736255 : rtx_insn *seq, *loc_insn;
5311 1736255 : int c;
5312 1736255 : auto_vec<rtx> then_regs;
5313 1736255 : auto_vec<rtx> else_regs;
5314 1736255 : int limit = param_max_rtl_if_conversion_insns;
5315 :
5316 : /* Build a mapping for each block to the value used for each
5317 : register. */
5318 1736255 : hash_map<rtx, rtx> then_vals;
5319 1736255 : hash_map<rtx, rtx> else_vals;
5320 :
5321 : /* Make sure the blocks are suitable. */
5322 1736255 : if (!check_cond_move_block (then_bb, &then_vals, &then_regs, cond)
5323 1736255 : || (else_bb
5324 29769 : && !check_cond_move_block (else_bb, &else_vals, &else_regs, cond)))
5325 : return false;
5326 :
5327 : /* Make sure the blocks can be used together. If the same register
5328 : is set in both blocks, and is not set to a constant in both
5329 : cases, then both blocks must set it to the same register. We
5330 : have already verified that if it is set to a register, that the
5331 : source register does not change after the assignment. Also count
5332 : the number of registers set in only one of the blocks. */
5333 32140 : c = 0;
5334 131179 : for (rtx reg : then_regs)
5335 : {
5336 38175 : rtx *then_slot = then_vals.get (reg);
5337 38175 : rtx *else_slot = else_vals.get (reg);
5338 :
5339 38175 : gcc_checking_assert (then_slot);
5340 38175 : if (!else_slot)
5341 22712 : ++c;
5342 : else
5343 : {
5344 15463 : rtx then_val = *then_slot;
5345 15463 : rtx else_val = *else_slot;
5346 4561 : if (!CONSTANT_P (then_val) && !CONSTANT_P (else_val)
5347 19340 : && !rtx_equal_p (then_val, else_val))
5348 3020 : return false;
5349 : }
5350 : }
5351 :
5352 : /* Finish off c for MAX_CONDITIONAL_EXECUTE. */
5353 59529 : for (rtx reg : else_regs)
5354 : {
5355 13673 : gcc_checking_assert (else_vals.get (reg));
5356 13673 : if (!then_vals.get (reg))
5357 1381 : ++c;
5358 : }
5359 :
5360 : /* Make sure it is reasonable to convert this block. What matters
5361 : is the number of assignments currently made in only one of the
5362 : branches, since if we convert we are going to always execute
5363 : them. */
5364 29120 : if (c > MAX_CONDITIONAL_EXECUTE
5365 29120 : || c > limit)
5366 : return false;
5367 :
5368 : /* Try to emit the conditional moves. First do the then block,
5369 : then do anything left in the else blocks. */
5370 29106 : start_sequence ();
5371 29106 : if (!cond_move_convert_if_block (if_info, then_bb, cond,
5372 : &then_vals, &else_vals, false)
5373 29106 : || (else_bb
5374 7679 : && !cond_move_convert_if_block (if_info, else_bb, cond,
5375 : &then_vals, &else_vals, true)))
5376 : {
5377 14368 : end_sequence ();
5378 14368 : return false;
5379 : }
5380 14738 : seq = end_ifcvt_sequence (if_info);
5381 14738 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
5382 : return false;
5383 :
5384 3369 : loc_insn = first_active_insn (then_bb);
5385 3369 : if (!loc_insn)
5386 : {
5387 0 : loc_insn = first_active_insn (else_bb);
5388 0 : gcc_assert (loc_insn);
5389 : }
5390 3369 : emit_insn_before_setloc (seq, jump, INSN_LOCATION (loc_insn));
5391 :
5392 3369 : noce_finish_if_conversion (if_info);
5393 :
5394 3369 : return true;
5395 1736255 : }
5396 :
5397 :
5398 : /* Determine if a given basic block heads a simple IF-THEN-JOIN or an
5399 : IF-THEN-ELSE-JOIN block.
5400 :
5401 : If so, we'll try to convert the insns to not require the branch,
5402 : using only transformations that do not require conditional execution.
5403 :
5404 : Return TRUE if we were successful at converting the block. */
5405 :
5406 : static bool
5407 9935554 : noce_find_if_block (basic_block test_bb, edge then_edge, edge else_edge,
5408 : int pass)
5409 : {
5410 9935554 : basic_block then_bb, else_bb, join_bb;
5411 9935554 : bool then_else_reversed = false;
5412 9935554 : rtx_insn *jump;
5413 9935554 : rtx_insn *cond_earliest;
5414 9935554 : noce_if_info if_info = {};
5415 9935554 : bool speed_p = optimize_bb_for_speed_p (test_bb);
5416 :
5417 : /* We only ever should get here before reload. */
5418 9935554 : gcc_assert (!reload_completed);
5419 :
5420 : /* Recognize an IF-THEN-ELSE-JOIN block. */
5421 9935554 : if (single_pred_p (then_edge->dest)
5422 12605501 : && single_succ_p (then_edge->dest)
5423 3559397 : && single_pred_p (else_edge->dest)
5424 1854831 : && single_succ_p (else_edge->dest)
5425 11100750 : && single_succ (then_edge->dest) == single_succ (else_edge->dest))
5426 : {
5427 : then_bb = then_edge->dest;
5428 : else_bb = else_edge->dest;
5429 : join_bb = single_succ (then_bb);
5430 : }
5431 : /* Recognize an IF-THEN-JOIN block. */
5432 17134437 : else if (single_pred_p (then_edge->dest)
5433 7273442 : && single_succ_p (then_edge->dest)
5434 11716051 : && single_succ (then_edge->dest) == else_edge->dest)
5435 : {
5436 : then_bb = then_edge->dest;
5437 : else_bb = NULL_BLOCK;
5438 : join_bb = else_edge->dest;
5439 : }
5440 : /* Recognize an IF-ELSE-JOIN block. We can have those because the order
5441 : of basic blocks in cfglayout mode does not matter, so the fallthrough
5442 : edge can go to any basic block (and not just to bb->next_bb, like in
5443 : cfgrtl mode). */
5444 8088333 : else if (single_pred_p (else_edge->dest)
5445 3858092 : && single_succ_p (else_edge->dest)
5446 9497751 : && single_succ (else_edge->dest) == then_edge->dest)
5447 : {
5448 : /* The noce transformations do not apply to IF-ELSE-JOIN blocks.
5449 : To make this work, we have to invert the THEN and ELSE blocks
5450 : and reverse the jump condition. */
5451 : then_bb = else_edge->dest;
5452 : else_bb = NULL_BLOCK;
5453 : join_bb = single_succ (then_bb);
5454 : then_else_reversed = true;
5455 : }
5456 : else
5457 : /* Not a form we can handle. */
5458 : return false;
5459 :
5460 : /* The edges of the THEN and ELSE blocks cannot have complex edges. */
5461 1980603 : if (single_succ_edge (then_bb)->flags & EDGE_COMPLEX)
5462 : return false;
5463 1925111 : if (else_bb
5464 1925111 : && single_succ_edge (else_bb)->flags & EDGE_COMPLEX)
5465 : return false;
5466 :
5467 1915103 : num_possible_if_blocks++;
5468 :
5469 1915103 : if (dump_file)
5470 : {
5471 93 : fprintf (dump_file,
5472 : "\nIF-THEN%s-JOIN block found, pass %d, test %d, then %d",
5473 : (else_bb) ? "-ELSE" : "",
5474 : pass, test_bb->index, then_bb->index);
5475 :
5476 56 : if (else_bb)
5477 19 : fprintf (dump_file, ", else %d", else_bb->index);
5478 :
5479 56 : fprintf (dump_file, ", join %d\n", join_bb->index);
5480 : }
5481 :
5482 : /* If the conditional jump is more than just a conditional
5483 : jump, then we cannot do if-conversion on this block. */
5484 1915103 : jump = BB_END (test_bb);
5485 1915103 : if (! onlyjump_p (jump))
5486 : return false;
5487 :
5488 : /* Initialize an IF_INFO struct to pass around. */
5489 1914842 : if_info.test_bb = test_bb;
5490 1914842 : if_info.then_bb = then_bb;
5491 1914842 : if_info.else_bb = else_bb;
5492 1914842 : if_info.join_bb = join_bb;
5493 1914842 : if_info.cond = noce_get_condition (jump, &cond_earliest,
5494 : then_else_reversed);
5495 1914842 : rtx_insn *rev_cond_earliest;
5496 1914842 : if_info.rev_cond = noce_get_condition (jump, &rev_cond_earliest,
5497 : !then_else_reversed);
5498 1914842 : if (!if_info.cond && !if_info.rev_cond)
5499 : return false;
5500 1910998 : if (!if_info.cond)
5501 : {
5502 0 : std::swap (if_info.cond, if_info.rev_cond);
5503 0 : std::swap (cond_earliest, rev_cond_earliest);
5504 0 : if_info.cond_inverted = true;
5505 : }
5506 : /* We must be comparing objects whose modes imply the size. */
5507 1910998 : if (GET_MODE (XEXP (if_info.cond, 0)) == BLKmode)
5508 : return false;
5509 1910998 : gcc_assert (if_info.rev_cond == NULL_RTX
5510 : || rev_cond_earliest == cond_earliest);
5511 1910998 : if_info.cond_earliest = cond_earliest;
5512 1910998 : if_info.jump = jump;
5513 1910998 : if_info.then_else_reversed = then_else_reversed;
5514 1910998 : if_info.speed_p = speed_p;
5515 1910998 : if_info.max_seq_cost
5516 1910998 : = targetm.max_noce_ifcvt_seq_cost (then_edge);
5517 : /* We'll add in the cost of THEN_BB and ELSE_BB later, when we check
5518 : that they are valid to transform. We can't easily get back to the insn
5519 : for COND (and it may not exist if we had to canonicalize to get COND).
5520 : It is assumed that the costs of a jump insn are dependent on the
5521 : branch costs. */
5522 1910998 : if_info.original_cost += insn_cost (if_info.jump, if_info.speed_p);
5523 :
5524 : /* Do the real work. */
5525 :
5526 : /* ??? noce_process_if_block has not yet been updated to handle
5527 : inverted conditions. */
5528 1910998 : if (!if_info.cond_inverted && noce_process_if_block (&if_info))
5529 : return true;
5530 :
5531 1736255 : if (HAVE_conditional_move
5532 1736255 : && cond_move_process_if_block (&if_info))
5533 : return true;
5534 :
5535 : return false;
5536 : }
5537 :
5538 :
5539 : /* Merge BB into COMBO_BB. BB has no successor edges left, so if COMBO_BB
5540 : still has another successor the BARRIER that follows BB is no longer needed
5541 : and it is in fact incorrect to leave it in the insn stream. */
5542 :
5543 : static void
5544 0 : merge_block_into_combo (basic_block combo_bb, basic_block bb)
5545 : {
5546 0 : if (EDGE_COUNT (bb->succs) == 0 && EDGE_COUNT (combo_bb->succs) > 1)
5547 : {
5548 0 : rtx_insn *end = NEXT_INSN (BB_END (bb));
5549 0 : while (end && NOTE_P (end) && !NOTE_INSN_BASIC_BLOCK_P (end))
5550 0 : end = NEXT_INSN (end);
5551 :
5552 0 : if (end && BARRIER_P (end))
5553 0 : delete_insn (end);
5554 : }
5555 :
5556 0 : merge_blocks (combo_bb, bb);
5557 0 : num_true_changes++;
5558 0 : }
5559 :
5560 : /* Merge the blocks and mark for local life update. */
5561 :
5562 : static void
5563 0 : merge_if_block (ce_if_block *ce_info)
5564 : {
5565 0 : basic_block test_bb = ce_info->test_bb; /* last test block */
5566 0 : basic_block then_bb = ce_info->then_bb; /* THEN */
5567 0 : basic_block else_bb = ce_info->else_bb; /* ELSE or NULL */
5568 0 : basic_block join_bb = ce_info->join_bb; /* join block */
5569 0 : basic_block combo_bb;
5570 :
5571 : /* All block merging is done into the lower block numbers. */
5572 :
5573 0 : combo_bb = test_bb;
5574 0 : df_set_bb_dirty (test_bb);
5575 :
5576 : /* Merge any basic blocks to handle && and || subtests. Each of
5577 : the blocks are on the fallthru path from the predecessor block. */
5578 0 : if (ce_info->num_multiple_test_blocks > 0)
5579 : {
5580 0 : basic_block bb = test_bb;
5581 0 : basic_block last_test_bb = ce_info->last_test_bb;
5582 0 : basic_block fallthru = block_fallthru (bb);
5583 :
5584 0 : do
5585 : {
5586 0 : bb = fallthru;
5587 0 : fallthru = block_fallthru (bb);
5588 0 : merge_blocks (combo_bb, bb);
5589 0 : num_true_changes++;
5590 : }
5591 0 : while (bb != last_test_bb);
5592 : }
5593 :
5594 : /* Merge TEST block into THEN block. Normally the THEN block won't have a
5595 : label, but it might if there were || tests. That label's count should be
5596 : zero, and it normally should be removed. */
5597 :
5598 0 : if (then_bb)
5599 0 : merge_block_into_combo (combo_bb, then_bb);
5600 :
5601 : /* The ELSE block, if it existed, had a label. That label count
5602 : will almost always be zero, but odd things can happen when labels
5603 : get their addresses taken. */
5604 0 : if (else_bb)
5605 0 : merge_block_into_combo (combo_bb, else_bb);
5606 :
5607 : /* If there was no join block reported, that means it was not adjacent
5608 : to the others, and so we cannot merge them. */
5609 :
5610 0 : if (! join_bb)
5611 : {
5612 0 : rtx_insn *last = BB_END (combo_bb);
5613 :
5614 : /* The outgoing edge for the current COMBO block should already
5615 : be correct. Verify this. */
5616 0 : if (EDGE_COUNT (combo_bb->succs) == 0)
5617 0 : gcc_assert (find_reg_note (last, REG_NORETURN, NULL)
5618 : || (NONJUMP_INSN_P (last)
5619 : && GET_CODE (PATTERN (last)) == TRAP_IF
5620 : && (TRAP_CONDITION (PATTERN (last))
5621 : == const_true_rtx)));
5622 :
5623 : else
5624 : /* There should still be something at the end of the THEN or ELSE
5625 : blocks taking us to our final destination. */
5626 0 : gcc_assert (JUMP_P (last)
5627 : || (EDGE_SUCC (combo_bb, 0)->dest
5628 : == EXIT_BLOCK_PTR_FOR_FN (cfun)
5629 : && CALL_P (last)
5630 : && SIBLING_CALL_P (last))
5631 : || ((EDGE_SUCC (combo_bb, 0)->flags & EDGE_EH)
5632 : && can_throw_internal (last)));
5633 : }
5634 :
5635 : /* The JOIN block may have had quite a number of other predecessors too.
5636 : Since we've already merged the TEST, THEN and ELSE blocks, we should
5637 : have only one remaining edge from our if-then-else diamond. If there
5638 : is more than one remaining edge, it must come from elsewhere. There
5639 : may be zero incoming edges if the THEN block didn't actually join
5640 : back up (as with a call to a non-return function). */
5641 0 : else if (EDGE_COUNT (join_bb->preds) < 2
5642 0 : && join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5643 : {
5644 : /* We can merge the JOIN cleanly and update the dataflow try
5645 : again on this pass.*/
5646 0 : merge_blocks (combo_bb, join_bb);
5647 0 : num_true_changes++;
5648 : }
5649 : else
5650 : {
5651 : /* We cannot merge the JOIN. */
5652 :
5653 : /* The outgoing edge for the current COMBO block should already
5654 : be correct. Verify this. */
5655 0 : gcc_assert (single_succ_p (combo_bb)
5656 : && single_succ (combo_bb) == join_bb);
5657 :
5658 : /* Remove the jump and cruft from the end of the COMBO block. */
5659 0 : if (join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5660 0 : tidy_fallthru_edge (single_succ_edge (combo_bb));
5661 : }
5662 :
5663 0 : num_updated_if_blocks++;
5664 0 : }
5665 :
5666 : /* Find a block ending in a simple IF condition and try to transform it
5667 : in some way. When converting a multi-block condition, put the new code
5668 : in the first such block and delete the rest. Return a pointer to this
5669 : first block if some transformation was done. Return NULL otherwise. */
5670 :
5671 : static basic_block
5672 39927116 : find_if_header (basic_block test_bb, int pass)
5673 : {
5674 39927116 : ce_if_block ce_info;
5675 39927116 : edge then_edge;
5676 39927116 : edge else_edge;
5677 :
5678 : /* The kind of block we're looking for has exactly two successors. */
5679 39927116 : if (EDGE_COUNT (test_bb->succs) != 2)
5680 : return NULL;
5681 :
5682 20561052 : then_edge = EDGE_SUCC (test_bb, 0);
5683 20561052 : else_edge = EDGE_SUCC (test_bb, 1);
5684 :
5685 20561052 : if (df_get_bb_dirty (then_edge->dest))
5686 : return NULL;
5687 20539274 : if (df_get_bb_dirty (else_edge->dest))
5688 : return NULL;
5689 :
5690 : /* Neither edge should be abnormal. */
5691 20503845 : if ((then_edge->flags & EDGE_COMPLEX)
5692 18744210 : || (else_edge->flags & EDGE_COMPLEX))
5693 : return NULL;
5694 :
5695 : /* Nor exit the loop. */
5696 18350465 : if ((then_edge->flags & EDGE_LOOP_EXIT)
5697 16871128 : || (else_edge->flags & EDGE_LOOP_EXIT))
5698 : return NULL;
5699 :
5700 : /* The THEN edge is canonically the one that falls through. */
5701 14587007 : if (then_edge->flags & EDGE_FALLTHRU)
5702 : ;
5703 7074430 : else if (else_edge->flags & EDGE_FALLTHRU)
5704 : std::swap (then_edge, else_edge);
5705 : else
5706 : /* Otherwise this must be a multiway branch of some sort. */
5707 : return NULL;
5708 :
5709 14586667 : memset (&ce_info, 0, sizeof (ce_info));
5710 14586667 : ce_info.test_bb = test_bb;
5711 14586667 : ce_info.then_bb = then_edge->dest;
5712 14586667 : ce_info.else_bb = else_edge->dest;
5713 14586667 : ce_info.pass = pass;
5714 :
5715 : #ifdef IFCVT_MACHDEP_INIT
5716 : IFCVT_MACHDEP_INIT (&ce_info);
5717 : #endif
5718 :
5719 14586667 : if (!reload_completed
5720 14586667 : && noce_find_if_block (test_bb, then_edge, else_edge, pass))
5721 178112 : goto success;
5722 :
5723 14408555 : if (reload_completed
5724 4651113 : && targetm.have_conditional_execution ()
5725 14408555 : && cond_exec_find_if_block (&ce_info))
5726 0 : goto success;
5727 :
5728 9757442 : if (!reload_completed && targetm.have_trap ()
5729 9757442 : && optab_handler (ctrap_optab, word_mode) != CODE_FOR_nothing
5730 14408555 : && find_cond_trap (test_bb, then_edge, else_edge))
5731 0 : goto success;
5732 :
5733 14408555 : if (dom_info_state (CDI_POST_DOMINATORS) >= DOM_NO_FAST_QUERY
5734 14408555 : && (reload_completed || !targetm.have_conditional_execution ()))
5735 : {
5736 14408555 : if (find_if_case_1 (test_bb, then_edge, else_edge))
5737 20708 : goto success;
5738 14387847 : if (find_if_case_2 (test_bb, then_edge, else_edge))
5739 125685 : goto success;
5740 : }
5741 :
5742 : return NULL;
5743 :
5744 324505 : success:
5745 324505 : if (dump_file)
5746 22 : fprintf (dump_file, "Conversion succeeded on pass %d.\n", pass);
5747 : /* Set this so we continue looking. */
5748 324505 : ifcvt_changed_p = true;
5749 324505 : return ce_info.test_bb;
5750 : }
5751 :
5752 : /* CUR_BB has two edges, one falling through to the next block and one
5753 : jumping to TARGET_BB, so it can be part of an && test or an || test.
5754 : Return the number of non-note, non-jump, non-USE/CLOBBER insns in it, or
5755 : -1 if CUR_BB is not of that form. */
5756 :
5757 : static int
5758 0 : block_jumps_and_fallthru (basic_block cur_bb, basic_block target_bb)
5759 : {
5760 0 : edge cur_edge;
5761 0 : bool fallthru_p = false;
5762 0 : bool jump_p = false;
5763 0 : rtx_insn *insn;
5764 0 : rtx_insn *end;
5765 0 : int n_insns = 0;
5766 0 : edge_iterator ei;
5767 :
5768 0 : if (!cur_bb || !target_bb)
5769 : return -1;
5770 :
5771 : /* If no edges, obviously it doesn't jump or fallthru. */
5772 0 : if (EDGE_COUNT (cur_bb->succs) == 0)
5773 : return 0;
5774 :
5775 0 : FOR_EACH_EDGE (cur_edge, ei, cur_bb->succs)
5776 : {
5777 0 : if (cur_edge->flags & EDGE_COMPLEX)
5778 : /* Anything complex isn't what we want. */
5779 : return -1;
5780 :
5781 0 : else if (cur_edge->flags & EDGE_FALLTHRU)
5782 : fallthru_p = true;
5783 :
5784 0 : else if (cur_edge->dest == target_bb)
5785 : jump_p = true;
5786 :
5787 : else
5788 : return -1;
5789 : }
5790 :
5791 0 : if (!jump_p || !fallthru_p)
5792 : return -1;
5793 :
5794 : /* Don't allow calls in the block, since this is used to group && and ||
5795 : together for conditional execution support. ??? we should support
5796 : conditional execution support across calls for IA-64 some day, but
5797 : for now it makes the code simpler. */
5798 0 : end = BB_END (cur_bb);
5799 0 : insn = BB_HEAD (cur_bb);
5800 :
5801 0 : while (insn)
5802 : {
5803 0 : if (CALL_P (insn))
5804 : return -1;
5805 :
5806 0 : if (INSN_P (insn)
5807 0 : && !JUMP_P (insn)
5808 0 : && !DEBUG_INSN_P (insn)
5809 0 : && GET_CODE (PATTERN (insn)) != USE
5810 0 : && GET_CODE (PATTERN (insn)) != CLOBBER)
5811 0 : n_insns++;
5812 :
5813 0 : if (insn == end)
5814 : break;
5815 :
5816 0 : insn = NEXT_INSN (insn);
5817 : }
5818 :
5819 : return n_insns;
5820 : }
5821 :
5822 : /* Determine if a given basic block heads a simple IF-THEN or IF-THEN-ELSE
5823 : block. If so, we'll try to convert the insns to not require the branch.
5824 : Return TRUE if we were successful at converting the block. */
5825 :
5826 : static bool
5827 0 : cond_exec_find_if_block (ce_if_block *ce_info)
5828 : {
5829 0 : basic_block test_bb = ce_info->test_bb;
5830 0 : basic_block then_bb = ce_info->then_bb;
5831 0 : basic_block else_bb = ce_info->else_bb;
5832 0 : basic_block join_bb = NULL_BLOCK;
5833 0 : edge cur_edge;
5834 0 : basic_block next;
5835 0 : edge_iterator ei;
5836 :
5837 0 : ce_info->last_test_bb = test_bb;
5838 :
5839 : /* We only ever should get here after reload,
5840 : and if we have conditional execution. */
5841 0 : gcc_assert (reload_completed && targetm.have_conditional_execution ());
5842 :
5843 : /* Discover if any fall through predecessors of the current test basic block
5844 : were && tests (which jump to the else block) or || tests (which jump to
5845 : the then block). */
5846 0 : if (single_pred_p (test_bb)
5847 0 : && single_pred_edge (test_bb)->flags == EDGE_FALLTHRU)
5848 : {
5849 0 : basic_block bb = single_pred (test_bb);
5850 0 : basic_block target_bb;
5851 0 : int max_insns = MAX_CONDITIONAL_EXECUTE;
5852 0 : int n_insns;
5853 :
5854 : /* Determine if the preceding block is an && or || block. */
5855 0 : if ((n_insns = block_jumps_and_fallthru (bb, else_bb)) >= 0)
5856 : {
5857 0 : ce_info->and_and_p = true;
5858 0 : target_bb = else_bb;
5859 : }
5860 0 : else if ((n_insns = block_jumps_and_fallthru (bb, then_bb)) >= 0)
5861 : {
5862 0 : ce_info->and_and_p = false;
5863 0 : target_bb = then_bb;
5864 : }
5865 : else
5866 : target_bb = NULL_BLOCK;
5867 :
5868 0 : if (target_bb && n_insns <= max_insns)
5869 : {
5870 : int total_insns = 0;
5871 : int blocks = 0;
5872 :
5873 : /* Found at least one && or || block, look for more. */
5874 0 : do
5875 : {
5876 0 : ce_info->test_bb = test_bb = bb;
5877 0 : total_insns += n_insns;
5878 0 : blocks++;
5879 :
5880 0 : if (!single_pred_p (bb))
5881 : break;
5882 :
5883 0 : bb = single_pred (bb);
5884 0 : n_insns = block_jumps_and_fallthru (bb, target_bb);
5885 : }
5886 0 : while (n_insns >= 0 && (total_insns + n_insns) <= max_insns);
5887 :
5888 0 : ce_info->num_multiple_test_blocks = blocks;
5889 0 : ce_info->num_multiple_test_insns = total_insns;
5890 :
5891 0 : if (ce_info->and_and_p)
5892 0 : ce_info->num_and_and_blocks = blocks;
5893 : else
5894 0 : ce_info->num_or_or_blocks = blocks;
5895 : }
5896 : }
5897 :
5898 : /* The THEN block of an IF-THEN combo must have exactly one predecessor,
5899 : other than any || blocks which jump to the THEN block. */
5900 0 : if ((EDGE_COUNT (then_bb->preds) - ce_info->num_or_or_blocks) != 1)
5901 : return false;
5902 :
5903 : /* The edges of the THEN and ELSE blocks cannot have complex edges. */
5904 0 : FOR_EACH_EDGE (cur_edge, ei, then_bb->preds)
5905 : {
5906 0 : if (cur_edge->flags & EDGE_COMPLEX)
5907 : return false;
5908 : }
5909 :
5910 0 : FOR_EACH_EDGE (cur_edge, ei, else_bb->preds)
5911 : {
5912 0 : if (cur_edge->flags & EDGE_COMPLEX)
5913 : return false;
5914 : }
5915 :
5916 : /* The THEN block of an IF-THEN combo must have zero or one successors. */
5917 0 : if (EDGE_COUNT (then_bb->succs) > 0
5918 0 : && (!single_succ_p (then_bb)
5919 0 : || (single_succ_edge (then_bb)->flags & EDGE_COMPLEX)
5920 0 : || (epilogue_completed
5921 0 : && tablejump_p (BB_END (then_bb), NULL, NULL))))
5922 : return false;
5923 :
5924 : /* If the THEN block has no successors, conditional execution can still
5925 : make a conditional call. Don't do this unless the ELSE block has
5926 : only one incoming edge -- the CFG manipulation is too ugly otherwise.
5927 : Check for the last insn of the THEN block being an indirect jump, which
5928 : is listed as not having any successors, but confuses the rest of the CE
5929 : code processing. ??? we should fix this in the future. */
5930 0 : if (EDGE_COUNT (then_bb->succs) == 0)
5931 : {
5932 0 : if (single_pred_p (else_bb) && else_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5933 : {
5934 0 : rtx_insn *last_insn = BB_END (then_bb);
5935 :
5936 0 : while (last_insn
5937 0 : && NOTE_P (last_insn)
5938 0 : && last_insn != BB_HEAD (then_bb))
5939 0 : last_insn = PREV_INSN (last_insn);
5940 :
5941 0 : if (last_insn
5942 0 : && JUMP_P (last_insn)
5943 0 : && ! simplejump_p (last_insn))
5944 : return false;
5945 :
5946 : join_bb = else_bb;
5947 : else_bb = NULL_BLOCK;
5948 : }
5949 : else
5950 : return false;
5951 : }
5952 :
5953 : /* If the THEN block's successor is the other edge out of the TEST block,
5954 : then we have an IF-THEN combo without an ELSE. */
5955 0 : else if (single_succ (then_bb) == else_bb)
5956 : {
5957 : join_bb = else_bb;
5958 : else_bb = NULL_BLOCK;
5959 : }
5960 :
5961 : /* If the THEN and ELSE block meet in a subsequent block, and the ELSE
5962 : has exactly one predecessor and one successor, and the outgoing edge
5963 : is not complex, then we have an IF-THEN-ELSE combo. */
5964 0 : else if (single_succ_p (else_bb)
5965 0 : && single_succ (then_bb) == single_succ (else_bb)
5966 0 : && single_pred_p (else_bb)
5967 0 : && !(single_succ_edge (else_bb)->flags & EDGE_COMPLEX)
5968 0 : && !(epilogue_completed
5969 0 : && tablejump_p (BB_END (else_bb), NULL, NULL)))
5970 0 : join_bb = single_succ (else_bb);
5971 :
5972 : /* Otherwise it is not an IF-THEN or IF-THEN-ELSE combination. */
5973 : else
5974 : return false;
5975 :
5976 0 : num_possible_if_blocks++;
5977 :
5978 0 : if (dump_file)
5979 : {
5980 0 : fprintf (dump_file,
5981 : "\nIF-THEN%s block found, pass %d, start block %d "
5982 : "[insn %d], then %d [%d]",
5983 : (else_bb) ? "-ELSE" : "",
5984 : ce_info->pass,
5985 : test_bb->index,
5986 0 : BB_HEAD (test_bb) ? (int)INSN_UID (BB_HEAD (test_bb)) : -1,
5987 : then_bb->index,
5988 0 : BB_HEAD (then_bb) ? (int)INSN_UID (BB_HEAD (then_bb)) : -1);
5989 :
5990 0 : if (else_bb)
5991 0 : fprintf (dump_file, ", else %d [%d]",
5992 : else_bb->index,
5993 0 : BB_HEAD (else_bb) ? (int)INSN_UID (BB_HEAD (else_bb)) : -1);
5994 :
5995 0 : fprintf (dump_file, ", join %d [%d]",
5996 : join_bb->index,
5997 0 : BB_HEAD (join_bb) ? (int)INSN_UID (BB_HEAD (join_bb)) : -1);
5998 :
5999 0 : if (ce_info->num_multiple_test_blocks > 0)
6000 0 : fprintf (dump_file, ", %d %s block%s last test %d [%d]",
6001 : ce_info->num_multiple_test_blocks,
6002 0 : (ce_info->and_and_p) ? "&&" : "||",
6003 : (ce_info->num_multiple_test_blocks == 1) ? "" : "s",
6004 : ce_info->last_test_bb->index,
6005 0 : ((BB_HEAD (ce_info->last_test_bb))
6006 0 : ? (int)INSN_UID (BB_HEAD (ce_info->last_test_bb))
6007 : : -1));
6008 :
6009 0 : fputc ('\n', dump_file);
6010 : }
6011 :
6012 : /* Make sure IF, THEN, and ELSE, blocks are adjacent. Actually, we get the
6013 : first condition for free, since we've already asserted that there's a
6014 : fallthru edge from IF to THEN. Likewise for the && and || blocks, since
6015 : we checked the FALLTHRU flag, those are already adjacent to the last IF
6016 : block. */
6017 : /* ??? As an enhancement, move the ELSE block. Have to deal with
6018 : BLOCK notes, if by no other means than backing out the merge if they
6019 : exist. Sticky enough I don't want to think about it now. */
6020 0 : next = then_bb;
6021 0 : if (else_bb && (next = next->next_bb) != else_bb)
6022 : return false;
6023 0 : if ((next = next->next_bb) != join_bb
6024 0 : && join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
6025 : {
6026 0 : if (else_bb)
6027 : join_bb = NULL;
6028 : else
6029 : return false;
6030 : }
6031 :
6032 : /* Do the real work. */
6033 :
6034 0 : ce_info->else_bb = else_bb;
6035 0 : ce_info->join_bb = join_bb;
6036 :
6037 : /* If we have && and || tests, try to first handle combining the && and ||
6038 : tests into the conditional code, and if that fails, go back and handle
6039 : it without the && and ||, which at present handles the && case if there
6040 : was no ELSE block. */
6041 0 : if (cond_exec_process_if_block (ce_info, true))
6042 : return true;
6043 :
6044 0 : if (ce_info->num_multiple_test_blocks)
6045 : {
6046 0 : cancel_changes (0);
6047 :
6048 0 : if (cond_exec_process_if_block (ce_info, false))
6049 : return true;
6050 : }
6051 :
6052 : return false;
6053 : }
6054 :
6055 : /* Convert a branch over a trap, or a branch
6056 : to a trap, into a conditional trap. */
6057 :
6058 : static bool
6059 0 : find_cond_trap (basic_block test_bb, edge then_edge, edge else_edge)
6060 : {
6061 0 : basic_block then_bb = then_edge->dest;
6062 0 : basic_block else_bb = else_edge->dest;
6063 0 : basic_block other_bb, trap_bb;
6064 0 : rtx_insn *trap, *jump;
6065 0 : rtx cond;
6066 0 : rtx_insn *cond_earliest;
6067 :
6068 : /* Locate the block with the trap instruction. */
6069 : /* ??? While we look for no successors, we really ought to allow
6070 : EH successors. Need to fix merge_if_block for that to work. */
6071 0 : if ((trap = block_has_only_trap (then_bb)) != NULL)
6072 : trap_bb = then_bb, other_bb = else_bb;
6073 0 : else if ((trap = block_has_only_trap (else_bb)) != NULL)
6074 : trap_bb = else_bb, other_bb = then_bb;
6075 : else
6076 : return false;
6077 :
6078 0 : if (dump_file)
6079 : {
6080 0 : fprintf (dump_file, "\nTRAP-IF block found, start %d, trap %d\n",
6081 : test_bb->index, trap_bb->index);
6082 : }
6083 :
6084 : /* If this is not a standard conditional jump, we can't parse it. */
6085 0 : jump = BB_END (test_bb);
6086 0 : cond = noce_get_condition (jump, &cond_earliest, then_bb == trap_bb);
6087 0 : if (! cond)
6088 : return false;
6089 :
6090 : /* If the conditional jump is more than just a conditional jump, then
6091 : we cannot do if-conversion on this block. Give up for returnjump_p,
6092 : changing a conditional return followed by unconditional trap for
6093 : conditional trap followed by unconditional return is likely not
6094 : beneficial and harder to handle. */
6095 0 : if (! onlyjump_p (jump) || returnjump_p (jump))
6096 : return false;
6097 :
6098 : /* We must be comparing objects whose modes imply the size. */
6099 0 : if (GET_MODE (XEXP (cond, 0)) == BLKmode)
6100 : return false;
6101 :
6102 : /* Attempt to generate the conditional trap. */
6103 0 : rtx_insn *seq = gen_cond_trap (GET_CODE (cond), copy_rtx (XEXP (cond, 0)),
6104 : copy_rtx (XEXP (cond, 1)),
6105 0 : TRAP_CODE (PATTERN (trap)));
6106 0 : if (seq == NULL)
6107 : return false;
6108 :
6109 : /* If that results in an invalid insn, back out. */
6110 0 : for (rtx_insn *x = seq; x; x = NEXT_INSN (x))
6111 0 : if (reload_completed
6112 0 : ? !valid_insn_p (x)
6113 0 : : recog_memoized (x) < 0)
6114 : return false;
6115 :
6116 : /* Emit the new insns before cond_earliest. */
6117 0 : emit_insn_before_setloc (seq, cond_earliest, INSN_LOCATION (trap));
6118 :
6119 : /* Delete the trap block if possible. */
6120 0 : remove_edge (trap_bb == then_bb ? then_edge : else_edge);
6121 0 : df_set_bb_dirty (test_bb);
6122 0 : df_set_bb_dirty (then_bb);
6123 0 : df_set_bb_dirty (else_bb);
6124 :
6125 0 : if (EDGE_COUNT (trap_bb->preds) == 0)
6126 : {
6127 0 : delete_basic_block (trap_bb);
6128 0 : num_true_changes++;
6129 : }
6130 :
6131 : /* Wire together the blocks again. */
6132 0 : if (current_ir_type () == IR_RTL_CFGLAYOUT)
6133 0 : single_succ_edge (test_bb)->flags |= EDGE_FALLTHRU;
6134 0 : else if (trap_bb == then_bb)
6135 : {
6136 0 : rtx lab = JUMP_LABEL (jump);
6137 0 : rtx_insn *seq = targetm.gen_jump (lab);
6138 0 : rtx_jump_insn *newjump = emit_jump_insn_after (seq, jump);
6139 0 : LABEL_NUSES (lab) += 1;
6140 0 : JUMP_LABEL (newjump) = lab;
6141 0 : emit_barrier_after (newjump);
6142 : }
6143 0 : delete_insn (jump);
6144 :
6145 0 : if (can_merge_blocks_p (test_bb, other_bb))
6146 : {
6147 0 : merge_blocks (test_bb, other_bb);
6148 0 : num_true_changes++;
6149 : }
6150 :
6151 0 : num_updated_if_blocks++;
6152 0 : return true;
6153 : }
6154 :
6155 : /* Subroutine of find_cond_trap: if BB contains only a trap insn,
6156 : return it. */
6157 :
6158 : static rtx_insn *
6159 0 : block_has_only_trap (basic_block bb)
6160 : {
6161 0 : rtx_insn *trap;
6162 :
6163 : /* We're not the exit block. */
6164 0 : if (bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
6165 : return NULL;
6166 :
6167 : /* The block must have no successors. */
6168 0 : if (EDGE_COUNT (bb->succs) > 0)
6169 : return NULL;
6170 :
6171 : /* The only instruction in the THEN block must be the trap. */
6172 0 : trap = first_active_insn (bb);
6173 0 : if (! (trap == BB_END (bb)
6174 0 : && GET_CODE (PATTERN (trap)) == TRAP_IF
6175 0 : && TRAP_CONDITION (PATTERN (trap)) == const_true_rtx))
6176 0 : return NULL;
6177 :
6178 : return trap;
6179 : }
6180 :
6181 : /* Look for IF-THEN-ELSE cases in which one of THEN or ELSE is
6182 : transformable, but not necessarily the other. There need be no
6183 : JOIN block.
6184 :
6185 : Return TRUE if we were successful at converting the block.
6186 :
6187 : Cases we'd like to look at:
6188 :
6189 : (1)
6190 : if (test) goto over; // x not live
6191 : x = a;
6192 : goto label;
6193 : over:
6194 :
6195 : becomes
6196 :
6197 : x = a;
6198 : if (! test) goto label;
6199 :
6200 : (2)
6201 : if (test) goto E; // x not live
6202 : x = big();
6203 : goto L;
6204 : E:
6205 : x = b;
6206 : goto M;
6207 :
6208 : becomes
6209 :
6210 : x = b;
6211 : if (test) goto M;
6212 : x = big();
6213 : goto L;
6214 :
6215 : (3) // This one's really only interesting for targets that can do
6216 : // multiway branching, e.g. IA-64 BBB bundles. For other targets
6217 : // it results in multiple branches on a cache line, which often
6218 : // does not sit well with predictors.
6219 :
6220 : if (test1) goto E; // predicted not taken
6221 : x = a;
6222 : if (test2) goto F;
6223 : ...
6224 : E:
6225 : x = b;
6226 : J:
6227 :
6228 : becomes
6229 :
6230 : x = a;
6231 : if (test1) goto E;
6232 : if (test2) goto F;
6233 :
6234 : Notes:
6235 :
6236 : (A) Don't do (2) if the branch is predicted against the block we're
6237 : eliminating. Do it anyway if we can eliminate a branch; this requires
6238 : that the sole successor of the eliminated block postdominate the other
6239 : side of the if.
6240 :
6241 : (B) With CE, on (3) we can steal from both sides of the if, creating
6242 :
6243 : if (test1) x = a;
6244 : if (!test1) x = b;
6245 : if (test1) goto J;
6246 : if (test2) goto F;
6247 : ...
6248 : J:
6249 :
6250 : Again, this is most useful if J postdominates.
6251 :
6252 : (C) CE substitutes for helpful life information.
6253 :
6254 : (D) These heuristics need a lot of work. */
6255 :
6256 : /* Return true if the if-case formed by TEST_BB, THEN_BB and ELSE_BB is one we
6257 : may convert at all.
6258 :
6259 : If we are partitioning hot/cold basic blocks, we don't want to mess up
6260 : unconditional or indirect jumps that cross between hot and cold sections.
6261 : Basic block partitioning may result in some jumps that appear to be
6262 : optimizable (or blocks that appear to be mergeable), but which really must
6263 : be left untouched (they are required to make it safely across partition
6264 : boundaries). See the comments at the top of
6265 : bb-reorder.cc:partition_hot_cold_basic_blocks for complete details.
6266 :
6267 : TEST_BB must also end in a conditional jump with no other side-effects. */
6268 :
6269 : static bool
6270 28643784 : if_case_blocks_ok_p (basic_block test_bb, basic_block then_bb,
6271 : basic_block else_bb)
6272 : {
6273 28643784 : if ((BB_END (then_bb)
6274 28643784 : && JUMP_P (BB_END (then_bb))
6275 14957922 : && CROSSING_JUMP_P (BB_END (then_bb)))
6276 28167214 : || (JUMP_P (BB_END (test_bb))
6277 28167214 : && CROSSING_JUMP_P (BB_END (test_bb)))
6278 56596126 : || (BB_END (else_bb)
6279 27952342 : && JUMP_P (BB_END (else_bb))
6280 14082273 : && CROSSING_JUMP_P (BB_END (else_bb))))
6281 : return false;
6282 :
6283 27905254 : return onlyjump_p (BB_END (test_bb));
6284 : }
6285 :
6286 : /* Tests for case 1 above. */
6287 :
6288 : static bool
6289 14408555 : find_if_case_1 (basic_block test_bb, edge then_edge, edge else_edge)
6290 : {
6291 14408555 : basic_block then_bb = then_edge->dest;
6292 14408555 : basic_block else_bb = else_edge->dest;
6293 14408555 : basic_block new_bb;
6294 14408555 : int then_bb_index;
6295 14408555 : profile_probability then_prob;
6296 14408555 : rtx else_target = NULL_RTX;
6297 :
6298 14408555 : if (!if_case_blocks_ok_p (test_bb, then_bb, else_bb))
6299 : return false;
6300 :
6301 : /* THEN has one successor. */
6302 14038618 : if (!single_succ_p (then_bb))
6303 : return false;
6304 :
6305 : /* THEN does not fall through, but is not strange either. */
6306 6350270 : if (single_succ_edge (then_bb)->flags & (EDGE_COMPLEX | EDGE_FALLTHRU))
6307 : return false;
6308 :
6309 : /* THEN has one predecessor. */
6310 1247171 : if (!single_pred_p (then_bb))
6311 : return false;
6312 :
6313 : /* THEN must do something. */
6314 1123709 : if (forwarder_block_p (then_bb))
6315 : return false;
6316 :
6317 711456 : num_possible_if_blocks++;
6318 711456 : if (dump_file)
6319 8 : fprintf (dump_file,
6320 : "\nIF-CASE-1 found, start %d, then %d\n",
6321 : test_bb->index, then_bb->index);
6322 :
6323 711456 : then_prob = then_edge->probability.invert ();
6324 :
6325 : /* We're speculating from the THEN path, we want to make sure the cost
6326 : of speculation is within reason. */
6327 1345145 : if (! cheap_bb_rtx_cost_p (then_bb, then_prob,
6328 1345145 : COSTS_N_INSNS (BRANCH_COST (optimize_bb_for_speed_p (then_edge->src),
6329 : predictable_edge_p (then_edge)))))
6330 : return false;
6331 :
6332 139082 : if (else_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
6333 : {
6334 0 : rtx_insn *jump = BB_END (else_edge->src);
6335 0 : gcc_assert (JUMP_P (jump));
6336 0 : else_target = JUMP_LABEL (jump);
6337 : }
6338 :
6339 : /* Registers set are dead, or are predicable. */
6340 139082 : if (! dead_or_predicable (test_bb, then_bb, else_bb,
6341 : single_succ_edge (then_bb), true))
6342 : return false;
6343 :
6344 : /* Conversion went ok, including moving the insns and fixing up the
6345 : jump. Adjust the CFG to match. */
6346 :
6347 : /* We can avoid creating a new basic block if then_bb is immediately
6348 : followed by else_bb, i.e. deleting then_bb allows test_bb to fall
6349 : through to else_bb. */
6350 :
6351 20708 : if (then_bb->next_bb == else_bb
6352 9361 : && then_bb->prev_bb == test_bb
6353 9361 : && else_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
6354 : {
6355 9361 : redirect_edge_succ (FALLTHRU_EDGE (test_bb), else_bb);
6356 9361 : new_bb = 0;
6357 : }
6358 11347 : else if (else_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
6359 0 : new_bb = force_nonfallthru_and_redirect (FALLTHRU_EDGE (test_bb),
6360 : else_bb, else_target);
6361 : else
6362 11347 : new_bb = redirect_edge_and_branch_force (FALLTHRU_EDGE (test_bb),
6363 : else_bb);
6364 :
6365 20708 : df_set_bb_dirty (test_bb);
6366 20708 : df_set_bb_dirty (else_bb);
6367 :
6368 20708 : then_bb_index = then_bb->index;
6369 20708 : delete_basic_block (then_bb);
6370 :
6371 : /* Make rest of code believe that the newly created block is the THEN_BB
6372 : block we removed. */
6373 20708 : if (new_bb)
6374 : {
6375 11347 : df_bb_replace (then_bb_index, new_bb);
6376 : /* This should have been done above via force_nonfallthru_and_redirect
6377 : (possibly called from redirect_edge_and_branch_force). */
6378 11347 : gcc_checking_assert (BB_PARTITION (new_bb) == BB_PARTITION (test_bb));
6379 : }
6380 :
6381 20708 : num_true_changes++;
6382 20708 : num_updated_if_blocks++;
6383 20708 : return true;
6384 : }
6385 :
6386 : /* Test for case 2 above. */
6387 :
6388 : static bool
6389 14387847 : find_if_case_2 (basic_block test_bb, edge then_edge, edge else_edge)
6390 : {
6391 14387847 : basic_block then_bb = then_edge->dest;
6392 14387847 : basic_block else_bb = else_edge->dest;
6393 14387847 : edge else_succ;
6394 14387847 : profile_probability then_prob, else_prob;
6395 :
6396 : /* We do not want to speculate (empty) loop latches. */
6397 14387847 : if (current_loops
6398 5623943 : && else_bb->loop_father->latch == else_bb)
6399 : return false;
6400 :
6401 14235229 : if (!if_case_blocks_ok_p (test_bb, then_bb, else_bb))
6402 : return false;
6403 :
6404 : /* ELSE has one successor. */
6405 13865293 : if (!single_succ_p (else_bb))
6406 : return false;
6407 : else
6408 5848102 : else_succ = single_succ_edge (else_bb);
6409 :
6410 : /* ELSE outgoing edge is not complex. */
6411 5848102 : if (else_succ->flags & EDGE_COMPLEX)
6412 : return false;
6413 :
6414 : /* ELSE has one predecessor. */
6415 5689433 : if (!single_pred_p (else_bb))
6416 : return false;
6417 :
6418 : /* THEN is not EXIT. */
6419 2972081 : if (then_bb->index < NUM_FIXED_BLOCKS)
6420 : return false;
6421 :
6422 2972081 : else_prob = else_edge->probability;
6423 2972081 : then_prob = else_prob.invert ();
6424 :
6425 : /* ELSE is predicted or SUCC(ELSE) postdominates THEN. */
6426 2972081 : if (else_prob > then_prob)
6427 : ;
6428 2000459 : else if (else_succ->dest->index < NUM_FIXED_BLOCKS
6429 2000459 : || dominated_by_p (CDI_POST_DOMINATORS, then_bb,
6430 : else_succ->dest))
6431 : ;
6432 : else
6433 : return false;
6434 :
6435 2348467 : num_possible_if_blocks++;
6436 2348467 : if (dump_file)
6437 44 : fprintf (dump_file,
6438 : "\nIF-CASE-2 found, start %d, else %d\n",
6439 : test_bb->index, else_bb->index);
6440 :
6441 : /* We're speculating from the ELSE path, we want to make sure the cost
6442 : of speculation is within reason. */
6443 4525339 : if (! cheap_bb_rtx_cost_p (else_bb, else_prob,
6444 4525339 : COSTS_N_INSNS (BRANCH_COST (optimize_bb_for_speed_p (else_edge->src),
6445 : predictable_edge_p (else_edge)))))
6446 : return false;
6447 :
6448 : /* Registers set are dead, or are predicable. */
6449 449273 : if (! dead_or_predicable (test_bb, else_bb, then_bb, else_succ, false))
6450 : return false;
6451 :
6452 : /* Conversion went ok, including moving the insns and fixing up the
6453 : jump. Adjust the CFG to match. */
6454 :
6455 125685 : df_set_bb_dirty (test_bb);
6456 125685 : df_set_bb_dirty (then_bb);
6457 125685 : delete_basic_block (else_bb);
6458 :
6459 125685 : num_true_changes++;
6460 125685 : num_updated_if_blocks++;
6461 :
6462 : /* ??? We may now fallthru from one of THEN's successors into a join
6463 : block. Rerun cleanup_cfg? Examine things manually? Wait? */
6464 :
6465 125685 : return true;
6466 : }
6467 :
6468 : /* Used by the code above to perform the actual rtl transformations.
6469 : Return TRUE if successful.
6470 :
6471 : TEST_BB is the block containing the conditional branch. MERGE_BB is the
6472 : block containing the code to manipulate. OTHER_BB is the other successor
6473 : of TEST_BB, the one the code is being moved past. DEST_EDGE is an edge
6474 : representing a jump to the join block; after the conversion, TEST_BB should
6475 : be branching to its destination. REVERSEP is true if the sense of the
6476 : branch should be reversed. */
6477 :
6478 : static bool
6479 588355 : dead_or_predicable (basic_block test_bb, basic_block merge_bb,
6480 : basic_block other_bb, edge dest_edge, bool reversep)
6481 : {
6482 588355 : basic_block new_dest = dest_edge->dest;
6483 588355 : rtx_insn *head, *end, *jump;
6484 588355 : rtx_insn *earliest = NULL;
6485 588355 : rtx old_dest;
6486 588355 : auto_bitmap merge_set (®_obstack);
6487 : /* Number of pending changes. */
6488 588355 : int n_validated_changes = 0;
6489 588355 : rtx new_dest_label = NULL_RTX;
6490 :
6491 588355 : jump = BB_END (test_bb);
6492 :
6493 : /* Find the extent of the real code in the merge block. */
6494 588355 : head = BB_HEAD (merge_bb);
6495 588355 : end = BB_END (merge_bb);
6496 :
6497 769326 : while (DEBUG_INSN_P (end) && end != head)
6498 180971 : end = PREV_INSN (end);
6499 :
6500 : /* If merge_bb ends with a tablejump, predicating/moving insn's
6501 : into test_bb and then deleting merge_bb will result in the jumptable
6502 : that follows merge_bb being removed along with merge_bb and then we
6503 : get an unresolved reference to the jumptable. */
6504 588355 : if (tablejump_p (end, NULL, NULL))
6505 : return false;
6506 :
6507 588355 : if (LABEL_P (head))
6508 449343 : head = NEXT_INSN (head);
6509 588355 : while (DEBUG_INSN_P (head) && head != end)
6510 0 : head = NEXT_INSN (head);
6511 588355 : if (NOTE_P (head))
6512 : {
6513 588355 : if (head == end)
6514 : {
6515 103978 : head = end = NULL;
6516 103978 : goto no_body;
6517 : }
6518 484377 : head = NEXT_INSN (head);
6519 1401866 : while (DEBUG_INSN_P (head) && head != end)
6520 433112 : head = NEXT_INSN (head);
6521 : }
6522 :
6523 484377 : if (JUMP_P (end))
6524 : {
6525 201242 : if (!onlyjump_p (end))
6526 : return false;
6527 159977 : if (head == end)
6528 : {
6529 6 : head = end = NULL;
6530 6 : goto no_body;
6531 : }
6532 159971 : end = PREV_INSN (end);
6533 429037 : while (DEBUG_INSN_P (end) && end != head)
6534 109095 : end = PREV_INSN (end);
6535 : }
6536 :
6537 : /* Don't move frame-related insn across the conditional branch. This
6538 : can lead to one of the paths of the branch having wrong unwind info. */
6539 443106 : if (epilogue_completed)
6540 : {
6541 : rtx_insn *insn = head;
6542 82397 : while (1)
6543 : {
6544 282531 : if (INSN_P (insn) && RTX_FRAME_RELATED_P (insn))
6545 : return false;
6546 275912 : if (insn == end)
6547 : break;
6548 82397 : insn = NEXT_INSN (insn);
6549 82397 : }
6550 : }
6551 :
6552 : /* Disable handling dead code by conditional execution if the machine needs
6553 : to do anything funny with the tests, etc. */
6554 : #ifndef IFCVT_MODIFY_TESTS
6555 436487 : if (targetm.have_conditional_execution ())
6556 : {
6557 : /* In the conditional execution case, we have things easy. We know
6558 : the condition is reversible. We don't have to check life info
6559 : because we're going to conditionally execute the code anyway.
6560 : All that's left is making sure the insns involved can actually
6561 : be predicated. */
6562 :
6563 0 : rtx cond;
6564 :
6565 : /* If the conditional jump is more than just a conditional jump,
6566 : then we cannot do conditional execution conversion on this block. */
6567 0 : if (!onlyjump_p (jump))
6568 0 : goto nce;
6569 :
6570 0 : cond = cond_exec_get_condition (jump);
6571 0 : if (! cond)
6572 0 : goto nce;
6573 :
6574 0 : rtx note = find_reg_note (jump, REG_BR_PROB, NULL_RTX);
6575 0 : profile_probability prob_val
6576 0 : = (note ? profile_probability::from_reg_br_prob_note (XINT (note, 0))
6577 : : profile_probability::uninitialized ());
6578 :
6579 0 : if (reversep)
6580 : {
6581 0 : enum rtx_code rev = reversed_comparison_code (cond, jump);
6582 0 : if (rev == UNKNOWN)
6583 0 : return false;
6584 0 : cond = gen_rtx_fmt_ee (rev, GET_MODE (cond), XEXP (cond, 0),
6585 : XEXP (cond, 1));
6586 0 : prob_val = prob_val.invert ();
6587 : }
6588 :
6589 0 : if (cond_exec_process_insns (NULL, head, end, cond, prob_val, false)
6590 0 : && verify_changes (0))
6591 0 : n_validated_changes = num_validated_changes ();
6592 : else
6593 0 : cancel_changes (0);
6594 :
6595 0 : earliest = jump;
6596 : }
6597 436487 : nce:
6598 : #endif
6599 :
6600 : /* If we allocated new pseudos (e.g. in the conditional move
6601 : expander called from noce_emit_cmove), we must resize the
6602 : array first. */
6603 436487 : if (max_regno < max_reg_num ())
6604 84491 : max_regno = max_reg_num ();
6605 :
6606 : /* Try the NCE path if the CE path did not result in any changes. */
6607 436487 : if (n_validated_changes == 0)
6608 : {
6609 436487 : rtx cond;
6610 436487 : rtx_insn *insn;
6611 436487 : bool success;
6612 :
6613 : /* In the non-conditional execution case, we have to verify that there
6614 : are no trapping operations, no calls, no references to memory, and
6615 : that any registers modified are dead at the branch site. */
6616 :
6617 436487 : if (!any_condjump_p (jump))
6618 293179 : return false;
6619 :
6620 : /* Find the extent of the conditional. */
6621 436487 : cond = noce_get_condition (jump, &earliest, false);
6622 436487 : if (!cond)
6623 : return false;
6624 :
6625 435681 : auto_bitmap live (®_obstack);
6626 435681 : simulate_backwards_to_point (merge_bb, live, end);
6627 435681 : success = can_move_insns_across (head, end, earliest, jump,
6628 : merge_bb, live,
6629 : df_get_live_in (other_bb), NULL);
6630 435681 : if (!success)
6631 : return false;
6632 :
6633 : /* Collect the set of registers set in MERGE_BB. */
6634 720229 : FOR_BB_INSNS (merge_bb, insn)
6635 576384 : if (NONDEBUG_INSN_P (insn))
6636 187027 : df_simulate_find_defs (insn, merge_set);
6637 :
6638 : /* If shrink-wrapping, disable this optimization when test_bb is
6639 : the first basic block and merge_bb exits. The idea is to not
6640 : move code setting up a return register as that may clobber a
6641 : register used to pass function parameters, which then must be
6642 : saved in caller-saved regs. A caller-saved reg requires the
6643 : prologue, killing a shrink-wrap opportunity. */
6644 143844 : if ((SHRINK_WRAPPING_ENABLED && !epilogue_completed)
6645 114004 : && ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb == test_bb
6646 157301 : && single_succ_p (new_dest)
6647 13993 : && single_succ (new_dest) == EXIT_BLOCK_PTR_FOR_FN (cfun)
6648 156828 : && bitmap_intersect_p (df_get_live_in (new_dest), merge_set))
6649 : {
6650 12983 : unsigned int i;
6651 :
6652 12983 : auto_bitmap return_regs (®_obstack);
6653 :
6654 : /* Start off with the intersection of regs used to pass
6655 : params and regs used to return values. */
6656 1233385 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
6657 1220402 : if (FUNCTION_ARG_REGNO_P (i)
6658 1220402 : && targetm.calls.function_value_regno_p (i))
6659 50754 : bitmap_set_bit (return_regs, INCOMING_REGNO (i));
6660 :
6661 12983 : bitmap_and_into (return_regs,
6662 12983 : df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
6663 12983 : bitmap_and_into (return_regs,
6664 12983 : df_get_live_in (EXIT_BLOCK_PTR_FOR_FN (cfun)));
6665 12983 : if (!bitmap_empty_p (return_regs))
6666 : {
6667 4888 : FOR_BB_INSNS_REVERSE (new_dest, insn)
6668 4333 : if (NONDEBUG_INSN_P (insn))
6669 : {
6670 2207 : df_ref def;
6671 :
6672 : /* If this insn sets any reg in return_regs, add all
6673 : reg uses to the set of regs we're interested in. */
6674 2796 : FOR_EACH_INSN_DEF (def, insn)
6675 1706 : if (bitmap_bit_p (return_regs, DF_REF_REGNO (def)))
6676 : {
6677 1117 : df_simulate_uses (insn, return_regs);
6678 1117 : break;
6679 : }
6680 : }
6681 555 : if (bitmap_intersect_p (merge_set, return_regs))
6682 537 : return false;
6683 : }
6684 12983 : }
6685 435681 : }
6686 :
6687 0 : no_body:
6688 : /* We don't want to use normal invert_jump or redirect_jump because
6689 : we don't want to delete_insn called. Also, we want to do our own
6690 : change group management. */
6691 :
6692 247292 : old_dest = JUMP_LABEL (jump);
6693 247292 : if (other_bb != new_dest)
6694 : {
6695 247277 : if (!any_condjump_p (jump))
6696 0 : goto cancel;
6697 :
6698 247277 : if (JUMP_P (BB_END (dest_edge->src)))
6699 21778 : new_dest_label = JUMP_LABEL (BB_END (dest_edge->src));
6700 225499 : else if (new_dest == EXIT_BLOCK_PTR_FOR_FN (cfun))
6701 100899 : new_dest_label = ret_rtx;
6702 : else
6703 124600 : new_dest_label = block_label (new_dest);
6704 :
6705 247277 : rtx_jump_insn *jump_insn = as_a <rtx_jump_insn *> (jump);
6706 247277 : if (reversep
6707 247277 : ? ! invert_jump_1 (jump_insn, new_dest_label)
6708 226569 : : ! redirect_jump_1 (jump_insn, new_dest_label))
6709 0 : goto cancel;
6710 : }
6711 :
6712 247292 : if (verify_changes (n_validated_changes))
6713 146393 : confirm_change_group ();
6714 : else
6715 100899 : goto cancel;
6716 :
6717 146393 : if (other_bb != new_dest)
6718 : {
6719 146378 : redirect_jump_2 (as_a <rtx_jump_insn *> (jump), old_dest, new_dest_label,
6720 : 0, reversep);
6721 :
6722 146378 : redirect_edge_succ (BRANCH_EDGE (test_bb), new_dest);
6723 146378 : if (reversep)
6724 : {
6725 20708 : std::swap (BRANCH_EDGE (test_bb)->probability,
6726 20708 : FALLTHRU_EDGE (test_bb)->probability);
6727 20708 : update_br_prob_note (test_bb);
6728 : }
6729 : }
6730 :
6731 : /* Move the insns out of MERGE_BB to before the branch. */
6732 146393 : if (head != NULL)
6733 : {
6734 143308 : rtx_insn *insn;
6735 :
6736 143308 : if (end == BB_END (merge_bb))
6737 110364 : BB_END (merge_bb) = PREV_INSN (head);
6738 :
6739 : /* PR 21767: when moving insns above a conditional branch, the REG_EQUAL
6740 : notes being moved might become invalid. */
6741 143308 : insn = head;
6742 171625 : do
6743 : {
6744 171625 : rtx note;
6745 :
6746 171625 : if (! INSN_P (insn))
6747 3099 : continue;
6748 168526 : note = find_reg_note (insn, REG_EQUAL, NULL_RTX);
6749 168526 : if (! note)
6750 157510 : continue;
6751 11016 : remove_note (insn, note);
6752 199942 : } while (insn != end && (insn = NEXT_INSN (insn)));
6753 :
6754 : /* PR46315: when moving insns above a conditional branch, the REG_EQUAL
6755 : notes referring to the registers being set might become invalid.
6756 : MERGE_SET is empty on the conditional-execution path, which does not
6757 : move anything. */
6758 143308 : unsigned i;
6759 143308 : bitmap_iterator bi;
6760 322776 : EXECUTE_IF_SET_IN_BITMAP (merge_set, 0, i, bi)
6761 179468 : remove_reg_equal_equiv_notes_for_regno (i);
6762 :
6763 143308 : reorder_insns (head, end, PREV_INSN (earliest));
6764 : }
6765 :
6766 : /* Remove the jump and edge if we can. */
6767 146393 : if (other_bb == new_dest)
6768 : {
6769 15 : delete_insn (jump);
6770 15 : remove_edge (BRANCH_EDGE (test_bb));
6771 : /* ??? Can't merge blocks here, as then_bb is still in use.
6772 : At minimum, the merge will get done just before bb-reorder. */
6773 : }
6774 :
6775 : return true;
6776 :
6777 100899 : cancel:
6778 100899 : cancel_changes (0);
6779 :
6780 100899 : return false;
6781 588355 : }
6782 :
6783 : /* Main entry point for all if-conversion. PHASE selects the ce1, ce2 or ce3
6784 : run: before combine, after combine, or after reload. */
6785 :
6786 : static void
6787 3185827 : if_convert (ifcvt_phase phase)
6788 : {
6789 3185827 : basic_block bb;
6790 3185827 : int pass;
6791 :
6792 3185827 : if (optimize == 1)
6793 : {
6794 236611 : df_live_add_problem ();
6795 236611 : df_live_set_all_dirty ();
6796 : }
6797 :
6798 : /* Record which pass is running for the cost model. */
6799 3185827 : ifcvt_pass_phase = phase;
6800 3185827 : have_cbranchcc4 = (direct_optab_handler (cbranch_optab, CCmode)
6801 3185827 : != CODE_FOR_nothing);
6802 3185827 : num_possible_if_blocks = 0;
6803 3185827 : num_updated_if_blocks = 0;
6804 3185827 : num_true_changes = 0;
6805 :
6806 3185827 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
6807 3185827 : mark_loop_exit_edges ();
6808 3185827 : loop_optimizer_finalize ();
6809 3185827 : free_dominance_info (CDI_DOMINATORS);
6810 :
6811 : /* Compute postdominators. */
6812 3185827 : calculate_dominance_info (CDI_POST_DOMINATORS);
6813 :
6814 3185827 : df_set_flags (DF_LR_RUN_DCE);
6815 :
6816 : /* Go through each of the basic blocks looking for things to convert. If we
6817 : have conditional execution, we make multiple passes to allow us to handle
6818 : IF-THEN{-ELSE} blocks within other IF-THEN{-ELSE} blocks. */
6819 3185827 : pass = 0;
6820 3352497 : do
6821 : {
6822 3352497 : df_analyze ();
6823 : /* Only need to do dce on the first pass. */
6824 3352497 : df_clear_flags (DF_LR_RUN_DCE);
6825 3352497 : ifcvt_changed_p = false;
6826 3352497 : pass++;
6827 :
6828 3352497 : if (dump_file && pass > 1)
6829 22 : fprintf (dump_file, "\n\n========== Pass %d ==========\n", pass);
6830 :
6831 43443063 : FOR_EACH_BB_FN (bb, cfun)
6832 : {
6833 : basic_block new_bb;
6834 40415071 : while (!df_get_bb_dirty (bb)
6835 80017682 : && (new_bb = find_if_header (bb, pass)) != NULL)
6836 : bb = new_bb;
6837 : }
6838 :
6839 3352497 : if (dump_file && ifcvt_changed_p)
6840 22 : print_rtl_with_bb (dump_file, get_insns (), dump_flags);
6841 : }
6842 : while (ifcvt_changed_p);
6843 :
6844 3185827 : if (dump_file)
6845 88 : fprintf (dump_file, "\n\n========== no more changes\n");
6846 :
6847 3185827 : free_dominance_info (CDI_POST_DOMINATORS);
6848 :
6849 3185827 : if (dump_file)
6850 88 : fflush (dump_file);
6851 :
6852 3185827 : clear_aux_for_blocks ();
6853 :
6854 : /* If we allocated new pseudos, we must resize the array for sched1. */
6855 3185827 : if (max_regno < max_reg_num ())
6856 649165 : max_regno = max_reg_num ();
6857 :
6858 : /* Write the final stats. */
6859 3185827 : if (dump_file && num_possible_if_blocks > 0)
6860 : {
6861 62 : fprintf (dump_file,
6862 : "\n%d possible IF blocks searched.\n",
6863 : num_possible_if_blocks);
6864 62 : fprintf (dump_file,
6865 : "%d IF blocks converted.\n",
6866 : num_updated_if_blocks);
6867 62 : fprintf (dump_file,
6868 : "%d true changes made.\n\n\n",
6869 : num_true_changes);
6870 : }
6871 :
6872 3185827 : if (optimize == 1)
6873 236611 : df_remove_problem (df_live);
6874 :
6875 : /* Some non-cold blocks may now be only reachable from cold blocks.
6876 : Fix that up. */
6877 3185827 : fixup_partitions ();
6878 :
6879 3185827 : checking_verify_flow_info ();
6880 3185827 : }
6881 :
6882 : /* If-conversion and CFG cleanup. */
6883 : static void
6884 1064387 : rest_of_handle_if_conversion (void)
6885 : {
6886 1064387 : int flags = 0;
6887 :
6888 1064387 : if (flag_if_conversion)
6889 : {
6890 1061938 : if (dump_file)
6891 : {
6892 36 : dump_reg_info (dump_file);
6893 36 : dump_flow_info (dump_file, dump_flags);
6894 : }
6895 1061938 : cleanup_cfg (CLEANUP_EXPENSIVE);
6896 1061938 : if_convert (IFCVT_BEFORE_COMBINE);
6897 1061938 : if (num_updated_if_blocks)
6898 : /* Get rid of any dead CC-related instructions. */
6899 1064387 : flags |= CLEANUP_FORCE_FAST_DCE;
6900 : }
6901 :
6902 1064387 : cleanup_cfg (flags);
6903 1064387 : }
6904 :
6905 : namespace {
6906 :
6907 : const pass_data pass_data_rtl_ifcvt =
6908 : {
6909 : RTL_PASS, /* type */
6910 : "ce1", /* name */
6911 : OPTGROUP_NONE, /* optinfo_flags */
6912 : TV_IFCVT, /* tv_id */
6913 : 0, /* properties_required */
6914 : 0, /* properties_provided */
6915 : 0, /* properties_destroyed */
6916 : 0, /* todo_flags_start */
6917 : TODO_df_finish, /* todo_flags_finish */
6918 : };
6919 :
6920 : /* ce2 reruns if-conversion after combine has simplified things enough to
6921 : meet the sequence-length restrictions. */
6922 : const pass_data pass_data_if_after_combine =
6923 : {
6924 : RTL_PASS, /* type */
6925 : "ce2", /* name */
6926 : OPTGROUP_NONE, /* optinfo_flags */
6927 : TV_IFCVT, /* tv_id */
6928 : 0, /* properties_required */
6929 : 0, /* properties_provided */
6930 : 0, /* properties_destroyed */
6931 : 0, /* todo_flags_start */
6932 : TODO_df_finish, /* todo_flags_finish */
6933 : };
6934 :
6935 : /* ce3 reruns if-conversion after reload. */
6936 : const pass_data pass_data_if_after_reload =
6937 : {
6938 : RTL_PASS, /* type */
6939 : "ce3", /* name */
6940 : OPTGROUP_NONE, /* optinfo_flags */
6941 : TV_IFCVT2, /* tv_id */
6942 : 0, /* properties_required */
6943 : 0, /* properties_provided */
6944 : 0, /* properties_destroyed */
6945 : 0, /* todo_flags_start */
6946 : TODO_df_finish, /* todo_flags_finish */
6947 : };
6948 :
6949 : /* The three if-conversion passes (ce1/ce2/ce3) share this class. M_PHASE
6950 : selects the gate condition and the work: ce1 (before combine) also cleans
6951 : up the CFG and is gated only on optimize, while ce2 and ce3 rerun once
6952 : combine and reload have simplified the RTL. */
6953 :
6954 : class pass_rtl_ifcvt : public rtl_opt_pass
6955 : {
6956 : public:
6957 883761 : pass_rtl_ifcvt (gcc::context *ctxt, const pass_data &data, ifcvt_phase phase)
6958 1767522 : : rtl_opt_pass (data, ctxt), m_phase (phase)
6959 : {}
6960 :
6961 : /* opt_pass methods: */
6962 4534176 : bool gate (function *) final override
6963 : {
6964 4534176 : if (optimize == 0)
6965 : return false;
6966 3193164 : switch (m_phase)
6967 : {
6968 1064388 : case IFCVT_BEFORE_COMBINE:
6969 1064388 : return dbg_cnt (if_conversion);
6970 1064388 : case IFCVT_AFTER_COMBINE:
6971 1064388 : return flag_if_conversion && dbg_cnt (if_after_combine);
6972 1064388 : case IFCVT_AFTER_RELOAD:
6973 1064388 : return flag_if_conversion2 && dbg_cnt (if_after_reload);
6974 : }
6975 0 : gcc_unreachable ();
6976 : }
6977 :
6978 3188276 : unsigned int execute (function *) final override
6979 : {
6980 3188276 : if (m_phase == IFCVT_BEFORE_COMBINE)
6981 1064387 : rest_of_handle_if_conversion ();
6982 : else
6983 2123889 : if_convert (m_phase);
6984 3188276 : return 0;
6985 : }
6986 :
6987 : private:
6988 : ifcvt_phase m_phase;
6989 :
6990 : }; // class pass_rtl_ifcvt
6991 :
6992 : } // anon namespace
6993 :
6994 : rtl_opt_pass *
6995 294587 : make_pass_rtl_ifcvt (gcc::context *ctxt)
6996 : {
6997 294587 : return new pass_rtl_ifcvt (ctxt, pass_data_rtl_ifcvt, IFCVT_BEFORE_COMBINE);
6998 : }
6999 :
7000 : rtl_opt_pass *
7001 294587 : make_pass_if_after_combine (gcc::context *ctxt)
7002 : {
7003 294587 : return new pass_rtl_ifcvt (ctxt, pass_data_if_after_combine,
7004 294587 : IFCVT_AFTER_COMBINE);
7005 : }
7006 :
7007 : rtl_opt_pass *
7008 294587 : make_pass_if_after_reload (gcc::context *ctxt)
7009 : {
7010 294587 : return new pass_rtl_ifcvt (ctxt, pass_data_if_after_reload,
7011 294587 : IFCVT_AFTER_RELOAD);
7012 : }
|
