Line data Source code
1 : /* If-conversion support.
2 : Copyright (C) 2000-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 : or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
14 : License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "cfghooks.h"
28 : #include "df.h"
29 : #include "memmodel.h"
30 : #include "tm_p.h"
31 : #include "expmed.h"
32 : #include "optabs.h"
33 : #include "regs.h"
34 : #include "emit-rtl.h"
35 : #include "recog.h"
36 :
37 : #include "cfgrtl.h"
38 : #include "cfganal.h"
39 : #include "cfgcleanup.h"
40 : #include "expr.h"
41 : #include "output.h"
42 : #include "cfgloop.h"
43 : #include "tree-pass.h"
44 : #include "dbgcnt.h"
45 : #include "shrink-wrap.h"
46 : #include "rtl-iter.h"
47 : #include "ifcvt.h"
48 :
49 : #ifndef MAX_CONDITIONAL_EXECUTE
50 : #define MAX_CONDITIONAL_EXECUTE \
51 : (BRANCH_COST (optimize_function_for_speed_p (cfun), false) \
52 : + 1)
53 : #endif
54 :
55 : #define IFCVT_MULTIPLE_DUMPS 1
56 :
57 : #define NULL_BLOCK ((basic_block) NULL)
58 :
59 : /* True if after combine pass. */
60 : static bool ifcvt_after_combine;
61 :
62 : /* True if the target has the cbranchcc4 optab. */
63 : static bool have_cbranchcc4;
64 :
65 : /* # of IF-THEN or IF-THEN-ELSE blocks we looked at */
66 : static int num_possible_if_blocks;
67 :
68 : /* # of IF-THEN or IF-THEN-ELSE blocks were converted to conditional
69 : execution. */
70 : static int num_updated_if_blocks;
71 :
72 : /* # of changes made. */
73 : static int num_true_changes;
74 :
75 : /* Whether conditional execution changes were made. */
76 : static bool cond_exec_changed_p;
77 :
78 : /* Forward references. */
79 : static int count_bb_insns (const_basic_block);
80 : static bool cheap_bb_rtx_cost_p (const_basic_block, profile_probability, int);
81 : static rtx_insn *first_active_insn (basic_block);
82 : static rtx_insn *last_active_insn (basic_block, bool);
83 : static rtx_insn *find_active_insn_before (basic_block, rtx_insn *);
84 : static rtx_insn *find_active_insn_after (basic_block, rtx_insn *);
85 : static basic_block block_fallthru (basic_block);
86 : static rtx cond_exec_get_condition (rtx_insn *, bool);
87 : static rtx noce_get_condition (rtx_insn *, rtx_insn **, bool);
88 : static bool noce_operand_ok (const_rtx);
89 : static void merge_if_block (ce_if_block *);
90 : static bool find_cond_trap (basic_block, edge, edge);
91 : static basic_block find_if_header (basic_block, int);
92 : static int block_jumps_and_fallthru (basic_block, basic_block);
93 : static bool noce_find_if_block (basic_block, edge, edge, int);
94 : static bool cond_exec_find_if_block (ce_if_block *);
95 : static bool find_if_case_1 (basic_block, edge, edge);
96 : static bool find_if_case_2 (basic_block, edge, edge);
97 : static bool dead_or_predicable (basic_block, basic_block, basic_block,
98 : edge, bool);
99 : static void noce_emit_move_insn (rtx, rtx);
100 : static rtx_insn *block_has_only_trap (basic_block);
101 : static void init_noce_multiple_sets_info (basic_block,
102 : auto_delete_vec<noce_multiple_sets_info> &);
103 : static bool noce_convert_multiple_sets_1 (struct noce_if_info *,
104 : auto_delete_vec<noce_multiple_sets_info> &, int *, bool *);
105 :
106 : /* Count the number of non-jump active insns in BB. */
107 :
108 : static int
109 0 : count_bb_insns (const_basic_block bb)
110 : {
111 0 : int count = 0;
112 0 : rtx_insn *insn = BB_HEAD (bb);
113 :
114 0 : while (1)
115 : {
116 0 : if (active_insn_p (insn) && !JUMP_P (insn))
117 0 : count++;
118 :
119 0 : if (insn == BB_END (bb))
120 : break;
121 0 : insn = NEXT_INSN (insn);
122 : }
123 :
124 0 : return count;
125 : }
126 :
127 : /* Determine whether the total insn_cost on non-jump insns in
128 : basic block BB is less than MAX_COST. This function returns
129 : false if the cost of any instruction could not be estimated.
130 :
131 : The cost of the non-jump insns in BB is scaled by REG_BR_PROB_BASE
132 : as those insns are being speculated. MAX_COST is scaled with SCALE
133 : plus a small fudge factor. */
134 :
135 : static bool
136 3095551 : cheap_bb_rtx_cost_p (const_basic_block bb,
137 : profile_probability prob, int max_cost)
138 : {
139 3095551 : int count = 0;
140 3095551 : rtx_insn *insn = BB_HEAD (bb);
141 3095551 : bool speed = optimize_bb_for_speed_p (bb);
142 3095551 : int scale = prob.initialized_p () ? prob.to_reg_br_prob_base ()
143 : : REG_BR_PROB_BASE;
144 :
145 : /* Set scale to REG_BR_PROB_BASE to void the identical scaling
146 : applied to insn_cost when optimizing for size. Only do
147 : this after combine because if-conversion might interfere with
148 : passes before combine.
149 :
150 : Use optimize_function_for_speed_p instead of the pre-defined
151 : variable speed to make sure it is set to same value for all
152 : basic blocks in one if-conversion transformation. */
153 3095551 : if (!optimize_function_for_speed_p (cfun) && ifcvt_after_combine)
154 : scale = REG_BR_PROB_BASE;
155 : /* Our branch probability/scaling factors are just estimates and don't
156 : account for cases where we can get speculation for free and other
157 : secondary benefits. So we fudge the scale factor to make speculating
158 : appear a little more profitable when optimizing for performance. */
159 : else
160 3044103 : scale += REG_BR_PROB_BASE / 8;
161 :
162 :
163 3095551 : max_cost *= scale;
164 :
165 13075491 : while (1)
166 : {
167 16171042 : if (NONJUMP_INSN_P (insn))
168 : {
169 : /* Inline-asm's cost is not very estimatable.
170 : It could be a costly instruction but the
171 : estimate would be the same as a non costly
172 : instruction. */
173 4626971 : if (asm_noperands (PATTERN (insn)) >= 0)
174 : return false;
175 :
176 4624270 : int cost = insn_cost (insn, speed) * REG_BR_PROB_BASE;
177 4624270 : if (cost == 0)
178 : return false;
179 :
180 : /* If this instruction is the load or set of a "stack" register,
181 : such as a floating point register on x87, then the cost of
182 : speculatively executing this insn may need to include
183 : the additional cost of popping its result off of the
184 : register stack. Unfortunately, correctly recognizing and
185 : accounting for this additional overhead is tricky, so for
186 : now we simply prohibit such speculative execution. */
187 : #ifdef STACK_REGS
188 4573961 : {
189 4573961 : rtx set = single_set (insn);
190 4573961 : if (set && STACK_REG_P (SET_DEST (set)))
191 : return false;
192 : }
193 : #endif
194 :
195 4569989 : count += cost;
196 4569989 : if (count >= max_cost)
197 : return false;
198 : }
199 11544071 : else if (CALL_P (insn))
200 : return false;
201 :
202 13681885 : if (insn == BB_END (bb))
203 : break;
204 13075491 : insn = NEXT_INSN (insn);
205 13075491 : }
206 :
207 : return true;
208 : }
209 :
210 : /* Return the first non-jump active insn in the basic block. */
211 :
212 : static rtx_insn *
213 1559273 : first_active_insn (basic_block bb)
214 : {
215 1559273 : rtx_insn *insn = BB_HEAD (bb);
216 :
217 1559273 : if (LABEL_P (insn))
218 : {
219 289169 : if (insn == BB_END (bb))
220 : return NULL;
221 289169 : insn = NEXT_INSN (insn);
222 : }
223 :
224 5199753 : while (NOTE_P (insn) || DEBUG_INSN_P (insn))
225 : {
226 3640480 : if (insn == BB_END (bb))
227 : return NULL;
228 3640480 : insn = NEXT_INSN (insn);
229 : }
230 :
231 1559273 : if (JUMP_P (insn))
232 0 : return NULL;
233 :
234 : return insn;
235 : }
236 :
237 : /* Return the last non-jump active (non-jump) insn in the basic block. */
238 :
239 : static rtx_insn *
240 2608810 : last_active_insn (basic_block bb, bool skip_use_p)
241 : {
242 2608810 : rtx_insn *insn = BB_END (bb);
243 2608810 : rtx_insn *head = BB_HEAD (bb);
244 :
245 2608810 : while (NOTE_P (insn)
246 : || JUMP_P (insn)
247 : || DEBUG_INSN_P (insn)
248 6304529 : || (skip_use_p
249 0 : && NONJUMP_INSN_P (insn)
250 0 : && GET_CODE (PATTERN (insn)) == USE))
251 : {
252 3695953 : if (insn == head)
253 : return NULL;
254 3695719 : insn = PREV_INSN (insn);
255 : }
256 :
257 2608576 : if (LABEL_P (insn))
258 77 : return NULL;
259 :
260 : return insn;
261 : }
262 :
263 : /* Return the active insn before INSN inside basic block CURR_BB. */
264 :
265 : static rtx_insn *
266 0 : find_active_insn_before (basic_block curr_bb, rtx_insn *insn)
267 : {
268 0 : if (!insn || insn == BB_HEAD (curr_bb))
269 : return NULL;
270 :
271 0 : while ((insn = PREV_INSN (insn)) != NULL_RTX)
272 : {
273 0 : if (NONJUMP_INSN_P (insn) || JUMP_P (insn) || CALL_P (insn))
274 : break;
275 :
276 : /* No other active insn all the way to the start of the basic block. */
277 0 : if (insn == BB_HEAD (curr_bb))
278 : return NULL;
279 : }
280 :
281 : return insn;
282 : }
283 :
284 : /* Return the active insn after INSN inside basic block CURR_BB. */
285 :
286 : static rtx_insn *
287 0 : find_active_insn_after (basic_block curr_bb, rtx_insn *insn)
288 : {
289 0 : if (!insn || insn == BB_END (curr_bb))
290 : return NULL;
291 :
292 0 : while ((insn = NEXT_INSN (insn)) != NULL_RTX)
293 : {
294 0 : if (NONJUMP_INSN_P (insn) || JUMP_P (insn) || CALL_P (insn))
295 : break;
296 :
297 : /* No other active insn all the way to the end of the basic block. */
298 0 : if (insn == BB_END (curr_bb))
299 : return NULL;
300 : }
301 :
302 : return insn;
303 : }
304 :
305 : /* Return the basic block reached by falling though the basic block BB. */
306 :
307 : static basic_block
308 0 : block_fallthru (basic_block bb)
309 : {
310 0 : edge e = find_fallthru_edge (bb->succs);
311 :
312 0 : return (e) ? e->dest : NULL_BLOCK;
313 : }
314 :
315 : /* Return true if RTXs A and B can be safely interchanged. */
316 :
317 : static bool
318 500324 : rtx_interchangeable_p (const_rtx a, const_rtx b)
319 : {
320 500324 : if (!rtx_equal_p (a, b))
321 : return false;
322 :
323 183855 : if (GET_CODE (a) != MEM)
324 : return true;
325 :
326 : /* A dead type-unsafe memory reference is legal, but a live type-unsafe memory
327 : reference is not. Interchanging a dead type-unsafe memory reference with
328 : a live type-safe one creates a live type-unsafe memory reference, in other
329 : words, it makes the program illegal.
330 : We check here conservatively whether the two memory references have equal
331 : memory attributes. */
332 :
333 2326 : return mem_attrs_eq_p (get_mem_attrs (a), get_mem_attrs (b));
334 : }
335 :
336 :
337 : /* Go through a bunch of insns, converting them to conditional
338 : execution format if possible. Return TRUE if all of the non-note
339 : insns were processed. */
340 :
341 : static bool
342 0 : cond_exec_process_insns (ce_if_block *ce_info ATTRIBUTE_UNUSED,
343 : /* if block information */rtx_insn *start,
344 : /* first insn to look at */rtx end,
345 : /* last insn to look at */rtx test,
346 : /* conditional execution test */profile_probability
347 : prob_val,
348 : /* probability of branch taken. */bool mod_ok)
349 : {
350 0 : bool must_be_last = false;
351 0 : rtx_insn *insn;
352 0 : rtx xtest;
353 0 : rtx pattern;
354 :
355 0 : if (!start || !end)
356 : return false;
357 :
358 0 : for (insn = start; ; insn = NEXT_INSN (insn))
359 : {
360 : /* dwarf2out can't cope with conditional prologues. */
361 0 : if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_PROLOGUE_END)
362 : return false;
363 :
364 0 : if (NOTE_P (insn) || DEBUG_INSN_P (insn))
365 0 : goto insn_done;
366 :
367 0 : gcc_assert (NONJUMP_INSN_P (insn) || CALL_P (insn));
368 :
369 : /* dwarf2out can't cope with conditional unwind info. */
370 0 : if (RTX_FRAME_RELATED_P (insn))
371 : return false;
372 :
373 : /* Remove USE insns that get in the way. */
374 0 : if (reload_completed && GET_CODE (PATTERN (insn)) == USE)
375 : {
376 : /* ??? Ug. Actually unlinking the thing is problematic,
377 : given what we'd have to coordinate with our callers. */
378 0 : SET_INSN_DELETED (insn);
379 0 : goto insn_done;
380 : }
381 :
382 : /* Last insn wasn't last? */
383 0 : if (must_be_last)
384 : return false;
385 :
386 0 : if (modified_in_p (test, insn))
387 : {
388 0 : if (!mod_ok)
389 : return false;
390 : must_be_last = true;
391 : }
392 :
393 : /* Now build the conditional form of the instruction. */
394 0 : pattern = PATTERN (insn);
395 0 : xtest = copy_rtx (test);
396 :
397 : /* If this is already a COND_EXEC, rewrite the test to be an AND of the
398 : two conditions. */
399 0 : if (GET_CODE (pattern) == COND_EXEC)
400 : {
401 0 : if (GET_MODE (xtest) != GET_MODE (COND_EXEC_TEST (pattern)))
402 : return false;
403 :
404 0 : xtest = gen_rtx_AND (GET_MODE (xtest), xtest,
405 : COND_EXEC_TEST (pattern));
406 0 : pattern = COND_EXEC_CODE (pattern);
407 : }
408 :
409 0 : pattern = gen_rtx_COND_EXEC (VOIDmode, xtest, pattern);
410 :
411 : /* If the machine needs to modify the insn being conditionally executed,
412 : say for example to force a constant integer operand into a temp
413 : register, do so here. */
414 : #ifdef IFCVT_MODIFY_INSN
415 : IFCVT_MODIFY_INSN (ce_info, pattern, insn);
416 : if (! pattern)
417 : return false;
418 : #endif
419 :
420 0 : validate_change (insn, &PATTERN (insn), pattern, 1);
421 :
422 0 : if (CALL_P (insn) && prob_val.initialized_p ())
423 0 : validate_change (insn, ®_NOTES (insn),
424 : gen_rtx_INT_LIST ((machine_mode) REG_BR_PROB,
425 : prob_val.to_reg_br_prob_note (),
426 : REG_NOTES (insn)), 1);
427 :
428 0 : insn_done:
429 0 : if (insn == end)
430 : break;
431 0 : }
432 :
433 : return true;
434 : }
435 :
436 : /* Return the condition for a jump. Do not do any special processing. */
437 :
438 : static rtx
439 118124 : cond_exec_get_condition (rtx_insn *jump, bool get_reversed = false)
440 : {
441 118124 : rtx test_if, cond;
442 :
443 118124 : if (any_condjump_p (jump))
444 118124 : test_if = SET_SRC (pc_set (jump));
445 : else
446 : return NULL_RTX;
447 118124 : cond = XEXP (test_if, 0);
448 :
449 : /* If this branches to JUMP_LABEL when the condition is false,
450 : reverse the condition. */
451 118124 : if (get_reversed
452 118124 : || (GET_CODE (XEXP (test_if, 2)) == LABEL_REF
453 0 : && label_ref_label (XEXP (test_if, 2))
454 0 : == JUMP_LABEL (jump)))
455 : {
456 59062 : enum rtx_code rev = reversed_comparison_code (cond, jump);
457 59062 : if (rev == UNKNOWN)
458 : return NULL_RTX;
459 :
460 59062 : cond = gen_rtx_fmt_ee (rev, GET_MODE (cond), XEXP (cond, 0),
461 : XEXP (cond, 1));
462 : }
463 :
464 : return cond;
465 : }
466 :
467 : /* Given a simple IF-THEN or IF-THEN-ELSE block, attempt to convert it
468 : to conditional execution. Return TRUE if we were successful at
469 : converting the block. */
470 :
471 : static bool
472 0 : cond_exec_process_if_block (ce_if_block * ce_info,
473 : /* if block information */bool do_multiple_p)
474 : {
475 0 : basic_block test_bb = ce_info->test_bb; /* last test block */
476 0 : basic_block then_bb = ce_info->then_bb; /* THEN */
477 0 : basic_block else_bb = ce_info->else_bb; /* ELSE or NULL */
478 0 : rtx test_expr; /* expression in IF_THEN_ELSE that is tested */
479 0 : rtx_insn *then_start; /* first insn in THEN block */
480 0 : rtx_insn *then_end; /* last insn + 1 in THEN block */
481 0 : rtx_insn *else_start = NULL; /* first insn in ELSE block or NULL */
482 0 : rtx_insn *else_end = NULL; /* last insn + 1 in ELSE block */
483 0 : int max; /* max # of insns to convert. */
484 0 : bool then_mod_ok; /* whether conditional mods are ok in THEN */
485 0 : rtx true_expr; /* test for else block insns */
486 0 : rtx false_expr; /* test for then block insns */
487 0 : profile_probability true_prob_val;/* probability of else block */
488 0 : profile_probability false_prob_val;/* probability of then block */
489 0 : rtx_insn *then_last_head = NULL; /* Last match at the head of THEN */
490 0 : rtx_insn *else_last_head = NULL; /* Last match at the head of ELSE */
491 0 : rtx_insn *then_first_tail = NULL; /* First match at the tail of THEN */
492 0 : rtx_insn *else_first_tail = NULL; /* First match at the tail of ELSE */
493 0 : int then_n_insns, else_n_insns, n_insns;
494 0 : enum rtx_code false_code;
495 0 : rtx note;
496 :
497 : /* If test is comprised of && or || elements, and we've failed at handling
498 : all of them together, just use the last test if it is the special case of
499 : && elements without an ELSE block. */
500 0 : if (!do_multiple_p && ce_info->num_multiple_test_blocks)
501 : {
502 0 : if (else_bb || ! ce_info->and_and_p)
503 : return false;
504 :
505 0 : ce_info->test_bb = test_bb = ce_info->last_test_bb;
506 0 : ce_info->num_multiple_test_blocks = 0;
507 0 : ce_info->num_and_and_blocks = 0;
508 0 : ce_info->num_or_or_blocks = 0;
509 : }
510 :
511 : /* Find the conditional jump to the ELSE or JOIN part, and isolate
512 : the test. */
513 0 : test_expr = cond_exec_get_condition (BB_END (test_bb));
514 0 : if (! test_expr)
515 : return false;
516 :
517 : /* If the conditional jump is more than just a conditional jump,
518 : then we cannot do conditional execution conversion on this block. */
519 0 : if (! onlyjump_p (BB_END (test_bb)))
520 : return false;
521 :
522 : /* Collect the bounds of where we're to search, skipping any labels, jumps
523 : and notes at the beginning and end of the block. Then count the total
524 : number of insns and see if it is small enough to convert. */
525 0 : then_start = first_active_insn (then_bb);
526 0 : then_end = last_active_insn (then_bb, true);
527 0 : then_n_insns = ce_info->num_then_insns = count_bb_insns (then_bb);
528 0 : n_insns = then_n_insns;
529 0 : max = MAX_CONDITIONAL_EXECUTE;
530 :
531 0 : if (else_bb)
532 : {
533 0 : int n_matching;
534 :
535 0 : max *= 2;
536 0 : else_start = first_active_insn (else_bb);
537 0 : else_end = last_active_insn (else_bb, true);
538 0 : else_n_insns = ce_info->num_else_insns = count_bb_insns (else_bb);
539 0 : n_insns += else_n_insns;
540 :
541 : /* Look for matching sequences at the head and tail of the two blocks,
542 : and limit the range of insns to be converted if possible. */
543 0 : n_matching = flow_find_cross_jump (then_bb, else_bb,
544 : &then_first_tail, &else_first_tail,
545 : NULL);
546 0 : if (then_first_tail == BB_HEAD (then_bb))
547 0 : then_start = then_end = NULL;
548 0 : if (else_first_tail == BB_HEAD (else_bb))
549 0 : else_start = else_end = NULL;
550 :
551 0 : if (n_matching > 0)
552 : {
553 0 : if (then_end)
554 0 : then_end = find_active_insn_before (then_bb, then_first_tail);
555 0 : if (else_end)
556 0 : else_end = find_active_insn_before (else_bb, else_first_tail);
557 0 : n_insns -= 2 * n_matching;
558 : }
559 :
560 0 : if (then_start
561 0 : && else_start
562 : && then_n_insns > n_matching
563 0 : && else_n_insns > n_matching)
564 : {
565 0 : int longest_match = MIN (then_n_insns - n_matching,
566 : else_n_insns - n_matching);
567 0 : n_matching
568 0 : = flow_find_head_matching_sequence (then_bb, else_bb,
569 : &then_last_head,
570 : &else_last_head,
571 : longest_match);
572 :
573 0 : if (n_matching > 0)
574 : {
575 0 : rtx_insn *insn;
576 :
577 : /* We won't pass the insns in the head sequence to
578 : cond_exec_process_insns, so we need to test them here
579 : to make sure that they don't clobber the condition. */
580 0 : for (insn = BB_HEAD (then_bb);
581 0 : insn != NEXT_INSN (then_last_head);
582 0 : insn = NEXT_INSN (insn))
583 0 : if (!LABEL_P (insn) && !NOTE_P (insn)
584 0 : && !DEBUG_INSN_P (insn)
585 0 : && modified_in_p (test_expr, insn))
586 : return false;
587 : }
588 :
589 0 : if (then_last_head == then_end)
590 0 : then_start = then_end = NULL;
591 0 : if (else_last_head == else_end)
592 0 : else_start = else_end = NULL;
593 :
594 0 : if (n_matching > 0)
595 : {
596 0 : if (then_start)
597 0 : then_start = find_active_insn_after (then_bb, then_last_head);
598 0 : if (else_start)
599 0 : else_start = find_active_insn_after (else_bb, else_last_head);
600 0 : n_insns -= 2 * n_matching;
601 : }
602 : }
603 : }
604 :
605 0 : if (n_insns > max)
606 : return false;
607 :
608 : /* Map test_expr/test_jump into the appropriate MD tests to use on
609 : the conditionally executed code. */
610 :
611 0 : true_expr = test_expr;
612 :
613 0 : false_code = reversed_comparison_code (true_expr, BB_END (test_bb));
614 0 : if (false_code != UNKNOWN)
615 0 : false_expr = gen_rtx_fmt_ee (false_code, GET_MODE (true_expr),
616 : XEXP (true_expr, 0), XEXP (true_expr, 1));
617 : else
618 : false_expr = NULL_RTX;
619 :
620 : #ifdef IFCVT_MODIFY_TESTS
621 : /* If the machine description needs to modify the tests, such as setting a
622 : conditional execution register from a comparison, it can do so here. */
623 : IFCVT_MODIFY_TESTS (ce_info, true_expr, false_expr);
624 :
625 : /* See if the conversion failed. */
626 : if (!true_expr || !false_expr)
627 : goto fail;
628 : #endif
629 :
630 0 : note = find_reg_note (BB_END (test_bb), REG_BR_PROB, NULL_RTX);
631 0 : if (note)
632 : {
633 0 : true_prob_val = profile_probability::from_reg_br_prob_note (XINT (note, 0));
634 0 : false_prob_val = true_prob_val.invert ();
635 : }
636 : else
637 : {
638 : true_prob_val = profile_probability::uninitialized ();
639 : false_prob_val = profile_probability::uninitialized ();
640 : }
641 :
642 : /* If we have && or || tests, do them here. These tests are in the adjacent
643 : blocks after the first block containing the test. */
644 0 : if (ce_info->num_multiple_test_blocks > 0)
645 : {
646 0 : basic_block bb = test_bb;
647 0 : basic_block last_test_bb = ce_info->last_test_bb;
648 :
649 0 : if (! false_expr)
650 0 : goto fail;
651 :
652 0 : do
653 : {
654 0 : rtx_insn *start, *end;
655 0 : rtx t, f;
656 0 : enum rtx_code f_code;
657 :
658 0 : bb = block_fallthru (bb);
659 0 : start = first_active_insn (bb);
660 0 : end = last_active_insn (bb, true);
661 0 : if (start
662 0 : && ! cond_exec_process_insns (ce_info, start, end, false_expr,
663 : false_prob_val, false))
664 0 : goto fail;
665 :
666 : /* If the conditional jump is more than just a conditional jump, then
667 : we cannot do conditional execution conversion on this block. */
668 0 : if (! onlyjump_p (BB_END (bb)))
669 0 : goto fail;
670 :
671 : /* Find the conditional jump and isolate the test. */
672 0 : t = cond_exec_get_condition (BB_END (bb));
673 0 : if (! t)
674 0 : goto fail;
675 :
676 0 : f_code = reversed_comparison_code (t, BB_END (bb));
677 0 : if (f_code == UNKNOWN)
678 0 : goto fail;
679 :
680 0 : f = gen_rtx_fmt_ee (f_code, GET_MODE (t), XEXP (t, 0), XEXP (t, 1));
681 0 : if (ce_info->and_and_p)
682 : {
683 0 : t = gen_rtx_AND (GET_MODE (t), true_expr, t);
684 0 : f = gen_rtx_IOR (GET_MODE (t), false_expr, f);
685 : }
686 : else
687 : {
688 0 : t = gen_rtx_IOR (GET_MODE (t), true_expr, t);
689 0 : f = gen_rtx_AND (GET_MODE (t), false_expr, f);
690 : }
691 :
692 : /* If the machine description needs to modify the tests, such as
693 : setting a conditional execution register from a comparison, it can
694 : do so here. */
695 : #ifdef IFCVT_MODIFY_MULTIPLE_TESTS
696 : IFCVT_MODIFY_MULTIPLE_TESTS (ce_info, bb, t, f);
697 :
698 : /* See if the conversion failed. */
699 : if (!t || !f)
700 : goto fail;
701 : #endif
702 :
703 0 : true_expr = t;
704 0 : false_expr = f;
705 : }
706 0 : while (bb != last_test_bb);
707 : }
708 :
709 : /* For IF-THEN-ELSE blocks, we don't allow modifications of the test
710 : on then THEN block. */
711 0 : then_mod_ok = (else_bb == NULL_BLOCK);
712 :
713 : /* Go through the THEN and ELSE blocks converting the insns if possible
714 : to conditional execution. */
715 :
716 0 : if (then_end
717 0 : && (! false_expr
718 0 : || ! cond_exec_process_insns (ce_info, then_start, then_end,
719 : false_expr, false_prob_val,
720 : then_mod_ok)))
721 0 : goto fail;
722 :
723 0 : if (else_bb && else_end
724 0 : && ! cond_exec_process_insns (ce_info, else_start, else_end,
725 : true_expr, true_prob_val, true))
726 0 : goto fail;
727 :
728 : /* If we cannot apply the changes, fail. Do not go through the normal fail
729 : processing, since apply_change_group will call cancel_changes. */
730 0 : if (! apply_change_group ())
731 : {
732 : #ifdef IFCVT_MODIFY_CANCEL
733 : /* Cancel any machine dependent changes. */
734 : IFCVT_MODIFY_CANCEL (ce_info);
735 : #endif
736 : return false;
737 : }
738 :
739 : #ifdef IFCVT_MODIFY_FINAL
740 : /* Do any machine dependent final modifications. */
741 : IFCVT_MODIFY_FINAL (ce_info);
742 : #endif
743 :
744 : /* Conversion succeeded. */
745 0 : if (dump_file)
746 0 : fprintf (dump_file, "%d insn%s converted to conditional execution.\n",
747 : n_insns, (n_insns == 1) ? " was" : "s were");
748 :
749 : /* Merge the blocks! If we had matching sequences, make sure to delete one
750 : copy at the appropriate location first: delete the copy in the THEN branch
751 : for a tail sequence so that the remaining one is executed last for both
752 : branches, and delete the copy in the ELSE branch for a head sequence so
753 : that the remaining one is executed first for both branches. */
754 0 : if (then_first_tail)
755 : {
756 0 : rtx_insn *from = then_first_tail;
757 0 : if (!INSN_P (from))
758 0 : from = find_active_insn_after (then_bb, from);
759 0 : delete_insn_chain (from, get_last_bb_insn (then_bb), false);
760 : }
761 0 : if (else_last_head)
762 0 : delete_insn_chain (first_active_insn (else_bb), else_last_head, false);
763 :
764 0 : merge_if_block (ce_info);
765 0 : cond_exec_changed_p = true;
766 0 : return true;
767 :
768 0 : fail:
769 : #ifdef IFCVT_MODIFY_CANCEL
770 : /* Cancel any machine dependent changes. */
771 : IFCVT_MODIFY_CANCEL (ce_info);
772 : #endif
773 :
774 0 : cancel_changes (0);
775 0 : return false;
776 : }
777 :
778 : static rtx noce_emit_store_flag (struct noce_if_info *, rtx, bool, int);
779 : static bool noce_try_move (struct noce_if_info *);
780 : static bool noce_try_ifelse_collapse (struct noce_if_info *);
781 : static bool noce_try_store_flag (struct noce_if_info *);
782 : static bool noce_try_addcc (struct noce_if_info *);
783 : static bool noce_try_store_flag_constants (struct noce_if_info *);
784 : static bool noce_try_shifted_store_flag (struct noce_if_info *);
785 : static bool noce_try_store_flag_mask (struct noce_if_info *);
786 : static rtx noce_emit_cmove (struct noce_if_info *, rtx, enum rtx_code, rtx,
787 : rtx, rtx, rtx, rtx = NULL, rtx = NULL);
788 : static bool noce_try_cmove (struct noce_if_info *);
789 : static bool noce_try_cmove_arith (struct noce_if_info *);
790 : static rtx noce_get_alt_condition (struct noce_if_info *, rtx, rtx_insn **);
791 : static bool noce_try_minmax (struct noce_if_info *);
792 : static bool noce_try_abs (struct noce_if_info *);
793 : static bool noce_try_sign_mask (struct noce_if_info *);
794 :
795 : /* Return the comparison code for reversed condition for IF_INFO,
796 : or UNKNOWN if reversing the condition is not possible. */
797 :
798 : static inline enum rtx_code
799 105341 : noce_reversed_cond_code (struct noce_if_info *if_info)
800 : {
801 105341 : if (if_info->rev_cond)
802 105341 : return GET_CODE (if_info->rev_cond);
803 0 : return reversed_comparison_code (if_info->cond, if_info->jump);
804 : }
805 :
806 : /* Return true if SEQ is a good candidate as a replacement for the
807 : if-convertible sequence described in IF_INFO.
808 : This is the default implementation that targets can override
809 : through a target hook. */
810 :
811 : bool
812 169367 : default_noce_conversion_profitable_p (rtx_insn *seq,
813 : struct noce_if_info *if_info)
814 : {
815 169367 : bool speed_p = if_info->speed_p;
816 :
817 : /* Cost up the new sequence. */
818 169367 : unsigned int cost = seq_cost (seq, speed_p);
819 :
820 169367 : if (cost <= if_info->original_cost)
821 : return true;
822 :
823 : /* When compiling for size, we can make a reasonably accurately guess
824 : at the size growth. When compiling for speed, use the maximum. */
825 88344 : return speed_p && cost <= if_info->max_seq_cost;
826 : }
827 :
828 : /* Helper function for noce_try_store_flag*. Return NULL_RTX on failure,
829 : including when REVERSEP is requested but the condition cannot be
830 : reversed; callers may rely on this and need not pre-check. */
831 :
832 : static rtx
833 59171 : noce_emit_store_flag (struct noce_if_info *if_info, rtx x, bool reversep,
834 : int normalize)
835 : {
836 59171 : rtx cond = if_info->cond;
837 59171 : bool cond_complex;
838 59171 : enum rtx_code code;
839 :
840 59171 : cond_complex = (! general_operand (XEXP (cond, 0), VOIDmode)
841 59171 : || ! general_operand (XEXP (cond, 1), VOIDmode));
842 :
843 : /* If earliest == jump, or when the condition is complex, try to
844 : build the store_flag insn directly. */
845 :
846 34603 : if (cond_complex)
847 : {
848 24568 : rtx set = pc_set (if_info->jump);
849 24568 : cond = XEXP (SET_SRC (set), 0);
850 24568 : if (GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
851 24568 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (if_info->jump))
852 0 : reversep = !reversep;
853 24568 : if (if_info->then_else_reversed)
854 21800 : reversep = !reversep;
855 : }
856 34603 : else if (reversep
857 17784 : && if_info->rev_cond
858 17784 : && general_operand (XEXP (if_info->rev_cond, 0), VOIDmode)
859 52387 : && general_operand (XEXP (if_info->rev_cond, 1), VOIDmode))
860 : {
861 17784 : cond = if_info->rev_cond;
862 17784 : reversep = false;
863 : }
864 :
865 59171 : if (reversep)
866 4455 : code = reversed_comparison_code (cond, if_info->jump);
867 : else
868 54716 : code = GET_CODE (cond);
869 :
870 : /* reversed_comparison_code returns UNKNOWN for an unordered code, or a
871 : CC-mode compare it cannot trace; neither path below can use that. */
872 59171 : if (code == UNKNOWN)
873 : return NULL_RTX;
874 :
875 59171 : if ((if_info->cond_earliest == if_info->jump || cond_complex)
876 24568 : && (normalize == 0 || STORE_FLAG_VALUE == normalize))
877 : {
878 18814 : rtx src = gen_rtx_fmt_ee (code, GET_MODE (x), XEXP (cond, 0),
879 : XEXP (cond, 1));
880 18814 : rtx set = gen_rtx_SET (x, src);
881 :
882 18814 : start_sequence ();
883 18814 : rtx_insn *insn = emit_insn (set);
884 :
885 18814 : if (recog_memoized (insn) >= 0)
886 : {
887 8453 : rtx_insn *seq = end_sequence ();
888 8453 : emit_insn (seq);
889 :
890 8453 : if_info->cond_earliest = if_info->jump;
891 :
892 8453 : return x;
893 : }
894 :
895 10361 : end_sequence ();
896 : }
897 :
898 : /* Don't even try if the comparison operands or the mode of X are weird. */
899 50718 : if (cond_complex || !SCALAR_INT_MODE_P (GET_MODE (x)))
900 : return NULL_RTX;
901 :
902 : /* Don't try if mode of X is more than the max fixed mode size. */
903 103779 : if (known_le (MAX_FIXED_MODE_SIZE, GET_MODE_BITSIZE (GET_MODE (x))))
904 : return NULL_RTX;
905 :
906 33261 : return emit_store_flag (x, code, XEXP (cond, 0),
907 : XEXP (cond, 1), VOIDmode,
908 33261 : (code == LTU || code == LEU
909 66522 : || code == GEU || code == GTU), normalize);
910 : }
911 :
912 : /* Return true if X can be safely forced into a register by copy_to_mode_reg
913 : / force_operand. */
914 :
915 : static bool
916 994426 : noce_can_force_operand (rtx x)
917 : {
918 994806 : if (general_operand (x, VOIDmode))
919 : return true;
920 292934 : if (SUBREG_P (x))
921 : {
922 380 : if (!noce_can_force_operand (SUBREG_REG (x)))
923 : return false;
924 : return true;
925 : }
926 292554 : if (ARITHMETIC_P (x))
927 : {
928 247276 : if (!noce_can_force_operand (XEXP (x, 0))
929 247276 : || !noce_can_force_operand (XEXP (x, 1)))
930 : return false;
931 244850 : switch (GET_CODE (x))
932 : {
933 : case MULT:
934 : case MOD:
935 : case UDIV:
936 : case UMOD:
937 : return true;
938 483 : case DIV:
939 483 : if (INTEGRAL_MODE_P (GET_MODE (x)))
940 : return true;
941 : /* FALLTHRU */
942 237865 : default:
943 237865 : auto optab = code_to_optab (GET_CODE (x));
944 237865 : if (!optab)
945 : return false;
946 235016 : return optab_handler (optab, GET_MODE (x));
947 : }
948 : }
949 45278 : if (UNARY_P (x))
950 : {
951 40047 : if (!noce_can_force_operand (XEXP (x, 0)))
952 : return false;
953 40034 : switch (GET_CODE (x))
954 : {
955 : case ZERO_EXTEND:
956 : case SIGN_EXTEND:
957 : case TRUNCATE:
958 : case FLOAT_EXTEND:
959 : case FLOAT_TRUNCATE:
960 : case FIX:
961 : case UNSIGNED_FIX:
962 : case FLOAT:
963 : case UNSIGNED_FLOAT:
964 : return true;
965 5023 : default:
966 5023 : auto optab = code_to_optab (GET_CODE (x));
967 5023 : if (!optab)
968 : return false;
969 5023 : return optab_handler (optab, GET_MODE (x));
970 : }
971 : }
972 : return false;
973 : }
974 :
975 : /* Emit instruction to move an rtx, possibly into STRICT_LOW_PART.
976 : X is the destination/target and Y is the value to copy. */
977 :
978 : static void
979 114242 : noce_emit_move_insn (rtx x, rtx y)
980 : {
981 114242 : machine_mode outmode;
982 114242 : rtx outer, inner;
983 114242 : poly_int64 bitpos;
984 :
985 114242 : if (GET_CODE (x) != STRICT_LOW_PART)
986 : {
987 114242 : rtx_insn *seq, *insn;
988 114242 : rtx target;
989 114242 : optab ot;
990 :
991 114242 : start_sequence ();
992 : /* Check that the SET_SRC is reasonable before calling emit_move_insn,
993 : otherwise construct a suitable SET pattern ourselves. */
994 114322 : insn = (OBJECT_P (y) || CONSTANT_P (y) || GET_CODE (y) == SUBREG)
995 114322 : ? emit_move_insn (x, y)
996 81066 : : emit_insn (gen_rtx_SET (x, y));
997 114242 : seq = end_sequence ();
998 :
999 114242 : if (recog_memoized (insn) <= 0)
1000 : {
1001 46824 : if (GET_CODE (x) == ZERO_EXTRACT)
1002 : {
1003 0 : rtx op = XEXP (x, 0);
1004 0 : unsigned HOST_WIDE_INT size = INTVAL (XEXP (x, 1));
1005 0 : unsigned HOST_WIDE_INT start = INTVAL (XEXP (x, 2));
1006 :
1007 : /* store_bit_field expects START to be relative to
1008 : BYTES_BIG_ENDIAN and adjusts this value for machines with
1009 : BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN. In order to be able to
1010 : invoke store_bit_field again it is necessary to have the START
1011 : value from the first call. */
1012 0 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
1013 : {
1014 : if (MEM_P (op))
1015 : start = BITS_PER_UNIT - start - size;
1016 : else
1017 : {
1018 : gcc_assert (REG_P (op));
1019 : start = BITS_PER_WORD - start - size;
1020 : }
1021 : }
1022 :
1023 0 : gcc_assert (start < (MEM_P (op) ? BITS_PER_UNIT : BITS_PER_WORD));
1024 0 : store_bit_field (op, size, start, 0, 0, GET_MODE (x), y, false,
1025 : false);
1026 0 : return;
1027 : }
1028 :
1029 46824 : switch (GET_RTX_CLASS (GET_CODE (y)))
1030 : {
1031 926 : case RTX_UNARY:
1032 926 : ot = code_to_optab (GET_CODE (y));
1033 926 : if (ot && noce_can_force_operand (XEXP (y, 0)))
1034 : {
1035 732 : start_sequence ();
1036 732 : target = expand_unop (GET_MODE (y), ot, XEXP (y, 0), x, 0);
1037 732 : if (target != NULL_RTX)
1038 : {
1039 732 : if (target != x)
1040 0 : emit_move_insn (x, target);
1041 732 : seq = get_insns ();
1042 : }
1043 732 : end_sequence ();
1044 : }
1045 : break;
1046 :
1047 15107 : case RTX_BIN_ARITH:
1048 15107 : case RTX_COMM_ARITH:
1049 15107 : ot = code_to_optab (GET_CODE (y));
1050 15107 : if (ot
1051 15107 : && noce_can_force_operand (XEXP (y, 0))
1052 30214 : && noce_can_force_operand (XEXP (y, 1)))
1053 : {
1054 15107 : start_sequence ();
1055 15107 : target = expand_binop (GET_MODE (y), ot,
1056 : XEXP (y, 0), XEXP (y, 1),
1057 : x, 0, OPTAB_DIRECT);
1058 15107 : if (target != NULL_RTX)
1059 : {
1060 15107 : if (target != x)
1061 0 : emit_move_insn (x, target);
1062 15107 : seq = get_insns ();
1063 : }
1064 15107 : end_sequence ();
1065 : }
1066 : break;
1067 :
1068 : default:
1069 : break;
1070 : }
1071 : }
1072 :
1073 114242 : emit_insn (seq);
1074 114242 : return;
1075 : }
1076 :
1077 0 : outer = XEXP (x, 0);
1078 0 : inner = XEXP (outer, 0);
1079 0 : outmode = GET_MODE (outer);
1080 0 : bitpos = SUBREG_BYTE (outer) * BITS_PER_UNIT;
1081 0 : store_bit_field (inner, GET_MODE_BITSIZE (outmode), bitpos,
1082 : 0, 0, outmode, y, false, false);
1083 : }
1084 :
1085 : /* Return the CC reg if it is used in COND. */
1086 :
1087 : static rtx
1088 4200409 : cc_in_cond (rtx cond)
1089 : {
1090 4200409 : if (have_cbranchcc4 && cond
1091 4200409 : && GET_MODE_CLASS (GET_MODE (XEXP (cond, 0))) == MODE_CC)
1092 141146 : return XEXP (cond, 0);
1093 :
1094 : return NULL_RTX;
1095 : }
1096 :
1097 : /* Return sequence of instructions generated by if conversion. This
1098 : function calls end_sequence() to end the current stream, ensures
1099 : that the instructions are unshared, recognizable non-jump insns.
1100 : On failure, this function returns a NULL_RTX. */
1101 :
1102 : static rtx_insn *
1103 227239 : end_ifcvt_sequence (struct noce_if_info *if_info)
1104 : {
1105 227239 : rtx_insn *insn;
1106 227239 : rtx_insn *seq = get_insns ();
1107 227239 : rtx cc = cc_in_cond (if_info->cond);
1108 :
1109 227239 : set_used_flags (if_info->x);
1110 227239 : set_used_flags (if_info->cond);
1111 227239 : set_used_flags (if_info->a);
1112 227239 : set_used_flags (if_info->b);
1113 :
1114 1175795 : for (insn = seq; insn; insn = NEXT_INSN (insn))
1115 721317 : set_used_flags (insn);
1116 :
1117 227239 : unshare_all_rtl_in_chain (seq);
1118 227239 : end_sequence ();
1119 :
1120 : /* Make sure that all of the instructions emitted are recognizable,
1121 : and that we haven't introduced a new jump instruction.
1122 : As an exercise for the reader, build a general mechanism that
1123 : allows proper placement of required clobbers. */
1124 1143385 : for (insn = seq; insn; insn = NEXT_INSN (insn))
1125 720812 : if (JUMP_P (insn)
1126 720812 : || recog_memoized (insn) == -1
1127 : /* Make sure new generated code does not clobber CC. */
1128 1409794 : || (cc && set_of (cc, insn)))
1129 : return NULL;
1130 :
1131 : return seq;
1132 : }
1133 :
1134 : /* Return true iff the then and else basic block (if it exists)
1135 : consist of a single simple set instruction. */
1136 :
1137 : static bool
1138 3173589 : noce_simple_bbs (struct noce_if_info *if_info)
1139 : {
1140 0 : if (!if_info->then_simple)
1141 : return false;
1142 :
1143 2941459 : if (if_info->else_bb)
1144 0 : return if_info->else_simple;
1145 :
1146 : return true;
1147 : }
1148 :
1149 : /* Convert "if (a != b) x = a; else x = b" into "x = a" and
1150 : "if (a == b) x = a; else x = b" into "x = b". */
1151 :
1152 : static bool
1153 252809 : noce_try_move (struct noce_if_info *if_info)
1154 : {
1155 252809 : rtx cond = if_info->cond;
1156 252809 : enum rtx_code code = GET_CODE (cond);
1157 252809 : rtx y;
1158 252809 : rtx_insn *seq;
1159 :
1160 252809 : if (code != NE && code != EQ)
1161 : return false;
1162 :
1163 256857 : if (!noce_simple_bbs (if_info))
1164 : return false;
1165 :
1166 : /* This optimization isn't valid if either A or B could be a NaN
1167 : or a signed zero. */
1168 176559 : if (HONOR_NANS (if_info->x)
1169 176559 : || HONOR_SIGNED_ZEROS (if_info->x))
1170 : return false;
1171 :
1172 : /* Check whether the operands of the comparison are A and in
1173 : either order. */
1174 151323 : if ((rtx_equal_p (if_info->a, XEXP (cond, 0))
1175 2320 : && rtx_equal_p (if_info->b, XEXP (cond, 1)))
1176 153631 : || (rtx_equal_p (if_info->a, XEXP (cond, 1))
1177 15618 : && rtx_equal_p (if_info->b, XEXP (cond, 0))))
1178 : {
1179 21 : if (!rtx_interchangeable_p (if_info->a, if_info->b))
1180 : return false;
1181 :
1182 0 : y = (code == EQ) ? if_info->a : if_info->b;
1183 :
1184 : /* Avoid generating the move if the source is the destination. */
1185 0 : if (! rtx_equal_p (if_info->x, y))
1186 : {
1187 0 : start_sequence ();
1188 0 : noce_emit_move_insn (if_info->x, y);
1189 0 : seq = end_ifcvt_sequence (if_info);
1190 0 : if (!seq)
1191 : return false;
1192 :
1193 0 : emit_insn_before_setloc (seq, if_info->jump,
1194 0 : INSN_LOCATION (if_info->insn_a));
1195 : }
1196 0 : if_info->transform_name = "noce_try_move";
1197 0 : return true;
1198 : }
1199 : return false;
1200 : }
1201 :
1202 : /* If a sign bit test is selecting across constants, we may be able
1203 : to generate efficient code utilizing the -1/0 result of a sign
1204 : bit splat idiom.
1205 :
1206 : Do this before trying the generalized conditional move as these
1207 : (when applicable) are hopefully faster than a conditional move. */
1208 :
1209 : static bool
1210 201070 : noce_try_sign_bit_splat (struct noce_if_info *if_info)
1211 : {
1212 201070 : rtx cond = if_info->cond;
1213 201070 : enum rtx_code code = GET_CODE (cond);
1214 :
1215 : /* We're looking for sign bit tests, so only a few cases are
1216 : interesting. LT/GE 0 LE/GT -1. */
1217 201070 : if (((code == LT || code == GE)
1218 10993 : && XEXP (cond, 1) == CONST0_RTX (GET_MODE (cond)))
1219 198741 : || ((code == LE || code == GT)
1220 11350 : && XEXP (cond, 1) == CONSTM1_RTX (GET_MODE (cond))))
1221 : ;
1222 : else
1223 : return false;
1224 :
1225 : /* It would be good if this could be extended since constant synthesis
1226 : on some platforms will result in blocks which fail this test. */
1227 10753 : if (!noce_simple_bbs (if_info))
1228 : return false;
1229 :
1230 : /* Only try this for constants in the true/false arms and a REG
1231 : destination. We could select between 0 and a REG pretty
1232 : easily with a logical AND. */
1233 6050 : if (!CONST_INT_P (if_info->a)
1234 1962 : || !CONST_INT_P (if_info->b)
1235 660 : || !REG_P (if_info->x))
1236 : return false;
1237 :
1238 660 : machine_mode mode = GET_MODE (if_info->x);
1239 :
1240 : /* If the mode of the destination does not match the mode of
1241 : the value we're testing, then this optimization is not valid. */
1242 660 : if (mode != GET_MODE (XEXP (cond, 0)))
1243 : return false;
1244 :
1245 206 : HOST_WIDE_INT val_a = INTVAL (if_info->a);
1246 206 : HOST_WIDE_INT val_b = INTVAL (if_info->b);
1247 :
1248 206 : rtx_insn *seq;
1249 206 : start_sequence ();
1250 :
1251 : /* We're testing the sign bit of this operand. */
1252 206 : rtx condop = XEXP (cond, 0);
1253 :
1254 : /* To splat the sign bit we arithmetically shift the
1255 : input value right by the size of the object - 1 bits.
1256 :
1257 : Note some targets do not have strong shifters, but do have
1258 : alternative ways to generate the sign bit splat. The
1259 : profitability test when we end the sequence should reject
1260 : cases when the branchy sequence is better. */
1261 206 : int splat_count = GET_MODE_BITSIZE (GET_MODE (condop)).to_constant () - 1;
1262 206 : rtx splat = GEN_INT (splat_count);
1263 :
1264 206 : rtx temp;
1265 206 : temp = expand_simple_binop (GET_MODE (XEXP (cond, 0)), ASHIFTRT,
1266 : XEXP (cond, 0), splat, NULL_RTX,
1267 : false, OPTAB_WIDEN);
1268 206 : if (!temp)
1269 0 : goto fail;
1270 :
1271 : /* IOR of anything with -1 still results in -1. So we can
1272 : IOR the other operand to generate a select between -1 and
1273 : an arbitrary constant. */
1274 206 : if (val_a == -1)
1275 : {
1276 47 : if (code == LT || code == LE)
1277 : {
1278 2 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1279 2 : if (!temp)
1280 0 : goto fail;
1281 : }
1282 :
1283 47 : temp = expand_simple_binop (mode, IOR, temp, GEN_INT (val_b),
1284 : if_info->x, false, OPTAB_WIDEN);
1285 : }
1286 : /* AND of anything with 0 is still zero. So we can AND
1287 : with the -1 operand with the a constant to select
1288 : between the constant and zero. */
1289 159 : else if (val_b == 0)
1290 : {
1291 0 : if (code == LT || code == LE)
1292 : {
1293 0 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1294 0 : if (!temp)
1295 0 : goto fail;
1296 : }
1297 :
1298 : /* Since we know the value is currently -1 or 0, some constants may
1299 : be more easily handled by shifting the value again. A right
1300 : logical shift constructs 2^n-1 constants a left shift constructs
1301 : ~(2^n-1) constants. Given some targets don't have efficient
1302 : shifts, generate the obvious RTL for both forms and select the
1303 : one with smaller cost. */
1304 0 : rtx and_form = gen_rtx_AND (mode, temp, GEN_INT (val_a));
1305 0 : rtx shift_left = gen_rtx_ASHIFT (mode, temp, GEN_INT (ctz_hwi (val_a)));
1306 0 : HOST_WIDE_INT rshift_count
1307 0 : = (clz_hwi (val_a) & (GET_MODE_PRECISION (mode).to_constant() - 1));
1308 0 : rtx shift_right = gen_rtx_LSHIFTRT (mode, temp, GEN_INT (rshift_count));
1309 0 : bool speed_p = optimize_insn_for_speed_p ();
1310 0 : if (exact_log2 (val_a + 1) >= 0
1311 0 : && (rtx_cost (shift_right, mode, SET, 1, speed_p)
1312 0 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1313 0 : temp = expand_simple_binop (mode, LSHIFTRT, temp,
1314 : GEN_INT (rshift_count),
1315 : if_info->x, false, OPTAB_WIDEN);
1316 0 : else if (exact_log2 (~val_a + 1) >= 0
1317 0 : && (rtx_cost (shift_left, mode, SET, 1, speed_p)
1318 0 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1319 0 : temp = expand_simple_binop (mode, ASHIFT, temp,
1320 0 : GEN_INT (ctz_hwi (val_a)),
1321 : if_info->x, false, OPTAB_WIDEN);
1322 : else
1323 0 : temp = expand_simple_binop (mode, AND, temp, GEN_INT (val_a),
1324 : if_info->x, false, OPTAB_WIDEN);
1325 : }
1326 : /* Same cases, but with the test or arms swapped. These
1327 : can be realized as well, though it typically costs
1328 : an extra instruction. */
1329 159 : else if (val_b == -1)
1330 : {
1331 46 : if (code != LT && code != LE)
1332 : {
1333 2 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1334 2 : if (!temp)
1335 0 : goto fail;
1336 : }
1337 :
1338 46 : temp = expand_simple_binop (mode, IOR, temp, GEN_INT (val_a),
1339 : if_info->x, false, OPTAB_WIDEN);
1340 : }
1341 113 : else if (val_a == 0)
1342 : {
1343 82 : if (code != LT && code != LE)
1344 : {
1345 9 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1346 9 : if (!temp)
1347 0 : goto fail;
1348 : }
1349 :
1350 : /* Since we know the value is currently -1 or 0, some constants may
1351 : be more easily handled by shifting the value again. A right
1352 : logical shift constructs 2^n-1 constants a left shift constructs
1353 : ~(2^n-1) constants. Given some targets don't have efficient
1354 : shifts, generate the obvious RTL for both forms and select the
1355 : one with smaller cost. */
1356 82 : rtx and_form = gen_rtx_AND (mode, temp, GEN_INT (val_b));
1357 82 : rtx shift_left = gen_rtx_ASHIFT (mode, temp, GEN_INT (ctz_hwi (val_b)));
1358 82 : HOST_WIDE_INT rshift_count
1359 82 : = (clz_hwi (val_b) & (GET_MODE_PRECISION (mode).to_constant() - 1));
1360 82 : rtx shift_right = gen_rtx_LSHIFTRT (mode, temp, GEN_INT (rshift_count));
1361 82 : bool speed_p = optimize_insn_for_speed_p ();
1362 82 : if (exact_log2 (val_b + 1) >= 0
1363 50 : && (rtx_cost (shift_right, mode, SET, 1, speed_p)
1364 25 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1365 3 : temp = expand_simple_binop (mode, LSHIFTRT, temp,
1366 : GEN_INT (rshift_count),
1367 : if_info->x, false, OPTAB_WIDEN);
1368 79 : else if (exact_log2 (~val_b + 1) >= 0
1369 0 : && (rtx_cost (shift_left, mode, SET, 1, speed_p)
1370 0 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1371 0 : temp = expand_simple_binop (mode, ASHIFT, temp,
1372 0 : GEN_INT (ctz_hwi (val_b)),
1373 : if_info->x, false, OPTAB_WIDEN);
1374 : else
1375 79 : temp = expand_simple_binop (mode, AND, temp, GEN_INT (val_b),
1376 : if_info->x, false, OPTAB_WIDEN);
1377 : }
1378 : /* Nothing worked. */
1379 : else
1380 : temp = NULL_RTX;
1381 :
1382 175 : if (!temp)
1383 31 : goto fail;
1384 :
1385 : /* Move into the final destination if the value wasn't
1386 : constructed there. */
1387 175 : if (if_info->x != temp)
1388 0 : emit_move_insn (if_info->x, temp);
1389 :
1390 : /* This ends the sequence and tests the cost model. */
1391 175 : seq = end_ifcvt_sequence (if_info);
1392 175 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
1393 : return false;
1394 :
1395 : /* Everything looks good. Install the if-converted sequence. */
1396 175 : emit_insn_before_setloc (seq, if_info->jump,
1397 175 : INSN_LOCATION (if_info->insn_a));
1398 175 : if_info->transform_name = "splat_sign_bit_trivial";
1399 175 : return true;
1400 :
1401 31 : fail:
1402 31 : end_ifcvt_sequence (if_info);
1403 31 : return false;
1404 : }
1405 :
1406 :
1407 : /* Try forming an IF_THEN_ELSE (cond, b, a) and collapsing that
1408 : through simplify_rtx. Sometimes that can eliminate the IF_THEN_ELSE.
1409 : If that is the case, emit the result into x. */
1410 :
1411 : static bool
1412 252809 : noce_try_ifelse_collapse (struct noce_if_info * if_info)
1413 : {
1414 338830 : if (!noce_simple_bbs (if_info))
1415 : return false;
1416 :
1417 234203 : machine_mode mode = GET_MODE (if_info->x);
1418 234203 : rtx if_then_else = simplify_gen_ternary (IF_THEN_ELSE, mode, mode,
1419 : if_info->cond, if_info->b,
1420 : if_info->a);
1421 :
1422 234203 : if (GET_CODE (if_then_else) == IF_THEN_ELSE)
1423 : return false;
1424 :
1425 48582 : rtx_insn *seq;
1426 48582 : start_sequence ();
1427 48582 : noce_emit_move_insn (if_info->x, if_then_else);
1428 48582 : seq = end_ifcvt_sequence (if_info);
1429 48582 : if (!seq)
1430 : return false;
1431 :
1432 17791 : emit_insn_before_setloc (seq, if_info->jump,
1433 17791 : INSN_LOCATION (if_info->insn_a));
1434 :
1435 17791 : if_info->transform_name = "noce_try_ifelse_collapse";
1436 17791 : return true;
1437 : }
1438 :
1439 :
1440 : /* Convert "if (test) x = 1; else x = 0".
1441 :
1442 : Only try 0 and STORE_FLAG_VALUE here. Other combinations will be
1443 : tried in noce_try_store_flag_constants after noce_try_cmove has had
1444 : a go at the conversion. */
1445 :
1446 : static bool
1447 235018 : noce_try_store_flag (struct noce_if_info *if_info)
1448 : {
1449 235018 : bool reversep;
1450 235018 : rtx target;
1451 235018 : rtx_insn *seq;
1452 :
1453 307755 : if (!noce_simple_bbs (if_info))
1454 : return false;
1455 :
1456 216412 : if (CONST_INT_P (if_info->b)
1457 73779 : && INTVAL (if_info->b) == STORE_FLAG_VALUE
1458 13410 : && if_info->a == const0_rtx)
1459 : reversep = false;
1460 205933 : else if (if_info->b == const0_rtx
1461 30391 : && CONST_INT_P (if_info->a)
1462 20752 : && INTVAL (if_info->a) == STORE_FLAG_VALUE)
1463 : reversep = true;
1464 : else
1465 : return false;
1466 :
1467 30792 : start_sequence ();
1468 :
1469 30792 : target = noce_emit_store_flag (if_info, if_info->x, reversep, 0);
1470 30792 : if (target)
1471 : {
1472 25925 : if (target != if_info->x)
1473 2982 : noce_emit_move_insn (if_info->x, target);
1474 :
1475 25925 : seq = end_ifcvt_sequence (if_info);
1476 25925 : if (! seq)
1477 : return false;
1478 :
1479 25925 : emit_insn_before_setloc (seq, if_info->jump,
1480 25925 : INSN_LOCATION (if_info->insn_a));
1481 25925 : if_info->transform_name = "noce_try_store_flag";
1482 25925 : return true;
1483 : }
1484 : else
1485 : {
1486 4867 : end_sequence ();
1487 4867 : return false;
1488 : }
1489 : }
1490 :
1491 :
1492 : /* Convert "if (test) x = -A; else x = A" into
1493 : x = A; if (test) x = -x if the machine can do the
1494 : conditional negate form of this cheaply.
1495 : Try this before noce_try_cmove that will just load the
1496 : immediates into two registers and do a conditional select
1497 : between them. If the target has a conditional negate or
1498 : conditional invert operation we can save a potentially
1499 : expensive constant synthesis. */
1500 :
1501 : static bool
1502 208996 : noce_try_inverse_constants (struct noce_if_info *if_info)
1503 : {
1504 272102 : if (!noce_simple_bbs (if_info))
1505 : return false;
1506 :
1507 190390 : if (!CONST_INT_P (if_info->a)
1508 51739 : || !CONST_INT_P (if_info->b)
1509 21211 : || !REG_P (if_info->x))
1510 : return false;
1511 :
1512 21211 : machine_mode mode = GET_MODE (if_info->x);
1513 :
1514 21211 : HOST_WIDE_INT val_a = INTVAL (if_info->a);
1515 21211 : HOST_WIDE_INT val_b = INTVAL (if_info->b);
1516 :
1517 21211 : rtx cond = if_info->cond;
1518 :
1519 21211 : rtx x = if_info->x;
1520 21211 : rtx target;
1521 :
1522 21211 : start_sequence ();
1523 :
1524 21211 : rtx_code code;
1525 21211 : if (val_b != HOST_WIDE_INT_MIN && val_a == -val_b)
1526 : code = NEG;
1527 19937 : else if (val_a == ~val_b)
1528 : code = NOT;
1529 : else
1530 : {
1531 19869 : end_sequence ();
1532 19869 : return false;
1533 : }
1534 :
1535 1342 : rtx tmp = gen_reg_rtx (mode);
1536 1342 : noce_emit_move_insn (tmp, if_info->a);
1537 :
1538 1342 : target = emit_conditional_neg_or_complement (x, code, mode, cond, tmp, tmp);
1539 :
1540 1342 : if (target)
1541 : {
1542 0 : rtx_insn *seq = get_insns ();
1543 :
1544 0 : if (!seq)
1545 : {
1546 0 : end_sequence ();
1547 0 : return false;
1548 : }
1549 :
1550 0 : if (target != if_info->x)
1551 0 : noce_emit_move_insn (if_info->x, target);
1552 :
1553 0 : seq = end_ifcvt_sequence (if_info);
1554 :
1555 0 : if (!seq)
1556 : return false;
1557 :
1558 0 : emit_insn_before_setloc (seq, if_info->jump,
1559 0 : INSN_LOCATION (if_info->insn_a));
1560 0 : if_info->transform_name = "noce_try_inverse_constants";
1561 0 : return true;
1562 : }
1563 :
1564 1342 : end_sequence ();
1565 1342 : return false;
1566 : }
1567 :
1568 : /* Check if OP is supported by conditional zero based if conversion,
1569 : returning TRUE if satisfied otherwise FALSE.
1570 :
1571 : OP is the operation to check. */
1572 :
1573 : static bool
1574 478670 : noce_cond_zero_binary_op_supported (rtx op)
1575 : {
1576 478670 : enum rtx_code opcode = GET_CODE (op);
1577 :
1578 0 : if (opcode == PLUS || opcode == MINUS || opcode == IOR || opcode == XOR
1579 : || opcode == ASHIFT || opcode == ASHIFTRT || opcode == LSHIFTRT
1580 : || opcode == ROTATE || opcode == ROTATERT || opcode == AND)
1581 0 : return true;
1582 :
1583 : return false;
1584 : }
1585 :
1586 : /* Convert "if (test) x = a; else x = a OP c " where c is 2^n.
1587 :
1588 : We can shift the output of a set flag insn left to generate 2^n
1589 : cheaply, then apply OP unconditionally. This works even if the
1590 : target does not have a conditional zero idiom. On targets such
1591 : as RISC-V this sequence may also encode better.
1592 :
1593 : This is based on noce_try_store_flag_constants. */
1594 :
1595 : static bool
1596 205839 : noce_try_shifted_store_flag (struct noce_if_info *if_info)
1597 : {
1598 205839 : rtx target;
1599 205839 : rtx_insn *seq;
1600 205839 : machine_mode mode = GET_MODE (if_info->x);
1601 205839 : rtx common = NULL_RTX;
1602 205839 : rtx_code code = UNKNOWN;
1603 :
1604 205839 : if (STORE_FLAG_VALUE != 1)
1605 : return false;
1606 :
1607 265862 : if (!noce_simple_bbs (if_info))
1608 : return false;
1609 :
1610 187233 : rtx a = if_info->a;
1611 187233 : rtx b = if_info->b;
1612 :
1613 : /* Most binary operators are allowed, essentially the same set
1614 : as the condzero arithmetic paths, with the exception of AND
1615 : which could be handled if we were inclined. */
1616 187233 : int shiftval;
1617 187233 : bool reversed = false;
1618 187233 : if (noce_cond_zero_binary_op_supported (a)
1619 24493 : && GET_CODE (a) != AND
1620 24284 : && GET_CODE (b) == REG
1621 20552 : && rtx_equal_p (XEXP (a, 0), b)
1622 18575 : && CONST_INT_P (XEXP (a, 1))
1623 9105 : && pow2p_hwi (INTVAL (XEXP (a, 1))))
1624 : {
1625 6633 : code = GET_CODE (a);
1626 6633 : common = XEXP (a, 0);
1627 6633 : shiftval = exact_log2 (INTVAL (XEXP (a, 1)));
1628 :
1629 : /* When the condition is true, we branch around A and thus we want
1630 : the condition reversed from what you might expect. */
1631 : reversed = true;
1632 : }
1633 180600 : else if (noce_cond_zero_binary_op_supported (b)
1634 2348 : && GET_CODE (b) != AND
1635 2317 : && GET_CODE (a) == REG
1636 253 : && rtx_equal_p (XEXP (b, 0), a)
1637 112 : && CONST_INT_P (XEXP (b, 1))
1638 70 : && pow2p_hwi (INTVAL (XEXP (b, 1))))
1639 : {
1640 31 : code = GET_CODE (b);
1641 31 : common = XEXP (b, 0);
1642 31 : shiftval = exact_log2 (INTVAL (XEXP (b, 1)));
1643 : }
1644 :
1645 :
1646 : /* If CODE hasn't been reset, then the basic expression form was wrong and
1647 : we can not optimize. */
1648 6664 : if (code == UNKNOWN)
1649 : return false;
1650 :
1651 6664 : start_sequence ();
1652 :
1653 : /* If X and COMMON are the same, then we're going to need a temporary. */
1654 6664 : if (common && rtx_equal_p (common, if_info->x))
1655 : {
1656 6089 : common = gen_reg_rtx (mode);
1657 6089 : noce_emit_move_insn (common, if_info->x);
1658 : }
1659 :
1660 6664 : target = noce_emit_store_flag (if_info, if_info->x, reversed, 1);
1661 6664 : if (!target)
1662 : {
1663 749 : end_sequence ();
1664 749 : return false;
1665 : }
1666 :
1667 : /* Right now TARGET has the value 1/0. A left shift will produce the
1668 : target value. Unnecessary if the shift value is zero. */
1669 5915 : if (shiftval != 0)
1670 1207 : target = expand_simple_binop (mode, ASHIFT, target,
1671 : GEN_INT (shiftval),
1672 : NULL_RTX, 0, OPTAB_WIDEN);
1673 :
1674 5915 : if (!target)
1675 : {
1676 0 : end_sequence ();
1677 0 : return false;
1678 : }
1679 :
1680 : /* Now we just need to apply the code to COMMON and TARGET. */
1681 5915 : target = expand_simple_binop (mode, code,
1682 : common, target,
1683 : NULL_RTX, 0, OPTAB_WIDEN);
1684 :
1685 5915 : if (!target)
1686 : {
1687 0 : end_sequence ();
1688 0 : return false;
1689 : }
1690 :
1691 : /* If necessary, store the result into the proper destination. */
1692 5915 : if (target != if_info->x)
1693 5915 : noce_emit_move_insn (if_info->x, target);
1694 :
1695 5915 : seq = end_ifcvt_sequence (if_info);
1696 5915 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
1697 : return false;
1698 :
1699 4769 : emit_insn_before_setloc (seq, if_info->jump,
1700 4769 : INSN_LOCATION (if_info->insn_a));
1701 4769 : if_info->transform_name = "noce_try_shifted_store_flag";
1702 :
1703 4769 : return true;
1704 : }
1705 :
1706 :
1707 : /* Convert "if (test) x = a; else x = b", for A and B constant.
1708 : Also allow A = y + c1, B = y + c2, with a common y between A
1709 : and B. */
1710 :
1711 : static bool
1712 208996 : noce_try_store_flag_constants (struct noce_if_info *if_info)
1713 : {
1714 208996 : rtx target;
1715 208996 : rtx_insn *seq;
1716 208996 : bool reversep;
1717 208996 : HOST_WIDE_INT itrue, ifalse, diff, tmp;
1718 208996 : int normalize;
1719 208996 : bool can_reverse;
1720 208996 : machine_mode mode = GET_MODE (if_info->x);
1721 208996 : rtx common = NULL_RTX;
1722 :
1723 208996 : rtx a = if_info->a;
1724 208996 : rtx b = if_info->b;
1725 :
1726 : /* Handle cases like x := test ? y + 3 : y + 4. */
1727 208996 : if (GET_CODE (a) == PLUS
1728 22535 : && GET_CODE (b) == PLUS
1729 1495 : && CONST_INT_P (XEXP (a, 1))
1730 574 : && CONST_INT_P (XEXP (b, 1))
1731 494 : && rtx_equal_p (XEXP (a, 0), XEXP (b, 0))
1732 : /* Allow expressions that are not using the result or plain
1733 : registers where we handle overlap below. */
1734 209432 : && (REG_P (XEXP (a, 0))
1735 6 : || (noce_operand_ok (XEXP (a, 0))
1736 6 : && ! reg_overlap_mentioned_p (if_info->x, XEXP (a, 0)))))
1737 : {
1738 436 : common = XEXP (a, 0);
1739 436 : a = XEXP (a, 1);
1740 436 : b = XEXP (b, 1);
1741 : }
1742 :
1743 272102 : if (!noce_simple_bbs (if_info))
1744 : return false;
1745 :
1746 190390 : if (CONST_INT_P (a)
1747 52163 : && CONST_INT_P (b))
1748 : {
1749 21635 : ifalse = INTVAL (a);
1750 21635 : itrue = INTVAL (b);
1751 21635 : bool subtract_flag_p = false;
1752 :
1753 21635 : diff = (unsigned HOST_WIDE_INT) itrue - ifalse;
1754 : /* Make sure we can represent the difference between the two values. */
1755 21635 : if ((diff > 0)
1756 21635 : != ((ifalse < 0) != (itrue < 0) ? ifalse < 0 : ifalse < itrue))
1757 : return false;
1758 :
1759 21521 : diff = trunc_int_for_mode (diff, mode);
1760 :
1761 21521 : can_reverse = noce_reversed_cond_code (if_info) != UNKNOWN;
1762 21521 : reversep = false;
1763 21521 : if (diff == STORE_FLAG_VALUE || diff == -STORE_FLAG_VALUE)
1764 : {
1765 12040 : normalize = 0;
1766 : /* We could collapse these cases but it is easier to follow the
1767 : diff/STORE_FLAG_VALUE combinations when they are listed
1768 : explicitly. */
1769 :
1770 : /* test ? 3 : 4
1771 : => 4 + (test != 0). */
1772 12040 : if (diff < 0 && STORE_FLAG_VALUE < 0)
1773 : reversep = false;
1774 : /* test ? 4 : 3
1775 : => can_reverse | 4 + (test == 0)
1776 : !can_reverse | 3 - (test != 0). */
1777 12040 : else if (diff > 0 && STORE_FLAG_VALUE < 0)
1778 : {
1779 : reversep = can_reverse;
1780 : subtract_flag_p = !can_reverse;
1781 : /* If we need to subtract the flag and we have PLUS-immediate
1782 : A and B then it is unlikely to be beneficial to play tricks
1783 : here. */
1784 : if (subtract_flag_p && common)
1785 : return false;
1786 : }
1787 : /* test ? 3 : 4
1788 : => can_reverse | 3 + (test == 0)
1789 : !can_reverse | 4 - (test != 0). */
1790 12040 : else if (diff < 0 && STORE_FLAG_VALUE > 0)
1791 : {
1792 6136 : reversep = can_reverse;
1793 6136 : subtract_flag_p = !can_reverse;
1794 : /* If we need to subtract the flag and we have PLUS-immediate
1795 : A and B then it is unlikely to be beneficial to play tricks
1796 : here. */
1797 6136 : if (subtract_flag_p && common)
1798 : return false;
1799 : }
1800 : /* test ? 4 : 3
1801 : => 4 + (test != 0). */
1802 : else if (diff > 0 && STORE_FLAG_VALUE > 0)
1803 : reversep = false;
1804 : else
1805 : gcc_unreachable ();
1806 : }
1807 : /* Is this (cond) ? 2^n : 0? */
1808 527 : else if (ifalse == 0 && pow2p_hwi (itrue)
1809 9481 : && STORE_FLAG_VALUE == 1)
1810 : normalize = 1;
1811 : /* Is this (cond) ? 0 : 2^n? */
1812 372 : else if (itrue == 0 && pow2p_hwi (ifalse)
1813 9474 : && STORE_FLAG_VALUE == 1)
1814 : {
1815 : normalize = 1;
1816 : reversep = true;
1817 : }
1818 : /* Is this (cond) ? -1 : x? */
1819 : else if (itrue == -1
1820 : && STORE_FLAG_VALUE == -1)
1821 : normalize = -1;
1822 : /* Is this (cond) ? x : -1? */
1823 : else if (ifalse == -1
1824 : && STORE_FLAG_VALUE == -1)
1825 : {
1826 : normalize = -1;
1827 : reversep = true;
1828 : }
1829 : else
1830 : return false;
1831 :
1832 6136 : if (reversep)
1833 : {
1834 6137 : std::swap (itrue, ifalse);
1835 6137 : diff = trunc_int_for_mode (-(unsigned HOST_WIDE_INT) diff, mode);
1836 : }
1837 :
1838 12048 : start_sequence ();
1839 :
1840 : /* If we have x := test ? x + 3 : x + 4 then move the original
1841 : x out of the way while we store flags. */
1842 12048 : if (common && rtx_equal_p (common, if_info->x))
1843 : {
1844 5 : common = gen_reg_rtx (mode);
1845 5 : noce_emit_move_insn (common, if_info->x);
1846 : }
1847 :
1848 12048 : target = noce_emit_store_flag (if_info, if_info->x, reversep, normalize);
1849 12048 : if (! target)
1850 : {
1851 4903 : end_sequence ();
1852 4903 : return false;
1853 : }
1854 :
1855 : /* if (test) x = 3; else x = 4;
1856 : => x = 3 + (test == 0); */
1857 7145 : if (diff == STORE_FLAG_VALUE || diff == -STORE_FLAG_VALUE)
1858 : {
1859 : /* Add the common part now. This may allow combine to merge this
1860 : with the store flag operation earlier into some sort of conditional
1861 : increment/decrement if the target allows it. */
1862 7137 : if (common)
1863 28 : target = expand_simple_binop (mode, PLUS,
1864 : target, common,
1865 : target, 0, OPTAB_WIDEN);
1866 :
1867 : /* Always use ifalse here. It should have been swapped with itrue
1868 : when appropriate when reversep is true. */
1869 14274 : target = expand_simple_binop (mode, subtract_flag_p ? MINUS : PLUS,
1870 7137 : gen_int_mode (ifalse, mode), target,
1871 : if_info->x, 0, OPTAB_WIDEN);
1872 : }
1873 : /* Other cases are not beneficial when the original A and B are PLUS
1874 : expressions. */
1875 8 : else if (common)
1876 : {
1877 0 : end_sequence ();
1878 0 : return false;
1879 : }
1880 : /* if (test) x = 8; else x = 0;
1881 : => x = (test != 0) << 3; */
1882 8 : else if (ifalse == 0 && (tmp = exact_log2 (itrue)) >= 0)
1883 : {
1884 8 : target = expand_simple_binop (mode, ASHIFT,
1885 : target, GEN_INT (tmp), if_info->x, 0,
1886 : OPTAB_WIDEN);
1887 : }
1888 :
1889 : /* if (test) x = -1; else x = b;
1890 : => x = -(test != 0) | b; */
1891 0 : else if (itrue == -1)
1892 : {
1893 0 : target = expand_simple_binop (mode, IOR,
1894 0 : target, gen_int_mode (ifalse, mode),
1895 : if_info->x, 0, OPTAB_WIDEN);
1896 : }
1897 : else
1898 : {
1899 0 : end_sequence ();
1900 0 : return false;
1901 : }
1902 :
1903 7145 : if (! target)
1904 : {
1905 0 : end_sequence ();
1906 0 : return false;
1907 : }
1908 :
1909 7145 : if (target != if_info->x)
1910 0 : noce_emit_move_insn (if_info->x, target);
1911 :
1912 7145 : seq = end_ifcvt_sequence (if_info);
1913 7145 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
1914 : return false;
1915 :
1916 3157 : emit_insn_before_setloc (seq, if_info->jump,
1917 3157 : INSN_LOCATION (if_info->insn_a));
1918 3157 : if_info->transform_name = "noce_try_store_flag_constants";
1919 :
1920 3157 : return true;
1921 : }
1922 :
1923 : return false;
1924 : }
1925 :
1926 : /* This is trying to capture cases like dest = cond ? a : -1.
1927 :
1928 : The basic idea is to use a store-flag insn to generate a -1/0
1929 : value (directly or indirectly), then IOR that with the other
1930 : input. */
1931 : static bool
1932 200895 : noce_try_store_flag_logical (struct noce_if_info *if_info)
1933 : {
1934 200895 : rtx a = if_info->a;
1935 200895 : rtx b = if_info->b;
1936 200895 : rtx dest = if_info->x;
1937 200895 : machine_mode mode = GET_MODE (dest);
1938 :
1939 200895 : if (STORE_FLAG_VALUE != -1 && STORE_FLAG_VALUE != 1)
1940 : return false;
1941 :
1942 260636 : if (!noce_simple_bbs (if_info))
1943 : return false;
1944 :
1945 182289 : bool swapped = STORE_FLAG_VALUE == -1;
1946 182289 : if (a == CONSTM1_RTX (GET_MODE (dest))
1947 1642 : && if_info->rev_cond)
1948 : {
1949 : std::swap (a, b);
1950 182289 : swapped = !swapped;
1951 : }
1952 :
1953 182289 : if (b != CONSTM1_RTX (GET_MODE (dest)))
1954 : return false;
1955 :
1956 : /* If the other arm is not a REG/SUBREG, then punt. This is primarily to
1957 : let the target handle the constant case, which it can likely do better.
1958 : It also means we don't have to worry about non terminal expressions. */
1959 2497 : if (!REG_P (a) && !SUBREG_P (a))
1960 : return false;
1961 :
1962 : /* At this point we've got dest = cond ? a : -1. Emit the store flag and
1963 : adjust its value (if necessary) to -1/0. */
1964 1000 : start_sequence ();
1965 1000 : rtx temp = gen_reg_rtx (mode);
1966 1000 : rtx target = noce_emit_store_flag (if_info, temp, !swapped, false);
1967 1000 : if (!target)
1968 : {
1969 88 : end_sequence ();
1970 88 : return false;
1971 : }
1972 :
1973 912 : if (STORE_FLAG_VALUE == 1)
1974 : {
1975 912 : rtx x = gen_rtx_PLUS (mode, target, CONSTM1_RTX (mode));
1976 912 : emit_move_insn (target, x);
1977 : }
1978 :
1979 : /* Now we've got -1/0 in TARGET. We can just IOR with A. */
1980 912 : rtx x = gen_rtx_IOR (mode, target, a);
1981 912 : emit_move_insn (dest, x);
1982 :
1983 : /* We've generated all the RTL, make sure it recognizes and is
1984 : profitable. */
1985 912 : rtx_insn *seq = end_ifcvt_sequence (if_info);
1986 912 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
1987 : return false;
1988 :
1989 0 : emit_insn_before_setloc (seq, if_info->jump,
1990 0 : INSN_LOCATION (if_info->insn_a));
1991 0 : if_info->transform_name = "noce_try_store_flag_logical";
1992 0 : return true;
1993 : }
1994 :
1995 : /* Convert "if (test) foo++" into "foo += (test != 0)", and
1996 : similarly for "foo--". */
1997 :
1998 : static bool
1999 158292 : noce_try_addcc (struct noce_if_info *if_info)
2000 : {
2001 158292 : rtx target;
2002 158292 : rtx_insn *seq;
2003 158292 : bool subtract;
2004 158292 : int normalize;
2005 :
2006 201547 : if (!noce_simple_bbs (if_info))
2007 : return false;
2008 :
2009 139686 : if (GET_CODE (if_info->a) == PLUS
2010 15585 : && rtx_equal_p (XEXP (if_info->a, 0), if_info->b)
2011 151743 : && noce_reversed_cond_code (if_info) != UNKNOWN)
2012 : {
2013 12057 : rtx cond = if_info->rev_cond;
2014 12057 : enum rtx_code code;
2015 :
2016 12057 : if (cond == NULL_RTX)
2017 : {
2018 0 : cond = if_info->cond;
2019 0 : code = reversed_comparison_code (cond, if_info->jump);
2020 : }
2021 : else
2022 12057 : code = GET_CODE (cond);
2023 :
2024 : /* First try to use addcc pattern. */
2025 12057 : if (general_operand (XEXP (cond, 0), VOIDmode)
2026 12057 : && general_operand (XEXP (cond, 1), VOIDmode))
2027 : {
2028 11139 : start_sequence ();
2029 33417 : target = emit_conditional_add (if_info->x, code,
2030 : XEXP (cond, 0),
2031 : XEXP (cond, 1),
2032 : VOIDmode,
2033 : if_info->b,
2034 11139 : XEXP (if_info->a, 1),
2035 11139 : GET_MODE (if_info->x),
2036 11139 : (code == LTU || code == GEU
2037 11139 : || code == LEU || code == GTU));
2038 11139 : if (target)
2039 : {
2040 328 : if (target != if_info->x)
2041 0 : noce_emit_move_insn (if_info->x, target);
2042 :
2043 328 : seq = end_ifcvt_sequence (if_info);
2044 328 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
2045 : return false;
2046 :
2047 328 : emit_insn_before_setloc (seq, if_info->jump,
2048 328 : INSN_LOCATION (if_info->insn_a));
2049 328 : if_info->transform_name = "noce_try_addcc";
2050 :
2051 328 : return true;
2052 : }
2053 10811 : end_sequence ();
2054 : }
2055 :
2056 : /* If that fails, construct conditional increment or decrement using
2057 : setcc. We're changing a branch and an increment to a comparison and
2058 : an ADD/SUB. */
2059 11729 : if (XEXP (if_info->a, 1) == const1_rtx
2060 10139 : || XEXP (if_info->a, 1) == constm1_rtx)
2061 : {
2062 2145 : start_sequence ();
2063 2145 : if (STORE_FLAG_VALUE == INTVAL (XEXP (if_info->a, 1)))
2064 : subtract = false, normalize = 0;
2065 555 : else if (-STORE_FLAG_VALUE == INTVAL (XEXP (if_info->a, 1)))
2066 : subtract = true, normalize = 0;
2067 : else
2068 0 : subtract = false, normalize = INTVAL (XEXP (if_info->a, 1));
2069 :
2070 :
2071 2145 : target = noce_emit_store_flag (if_info,
2072 2145 : gen_reg_rtx (GET_MODE (if_info->x)),
2073 : true, normalize);
2074 :
2075 2145 : if (target)
2076 2374 : target = expand_simple_binop (GET_MODE (if_info->x),
2077 : subtract ? MINUS : PLUS,
2078 : if_info->b, target, if_info->x,
2079 : 0, OPTAB_WIDEN);
2080 1454 : if (target)
2081 : {
2082 1454 : if (target != if_info->x)
2083 0 : noce_emit_move_insn (if_info->x, target);
2084 :
2085 1454 : seq = end_ifcvt_sequence (if_info);
2086 1454 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
2087 : return false;
2088 :
2089 1186 : emit_insn_before_setloc (seq, if_info->jump,
2090 1186 : INSN_LOCATION (if_info->insn_a));
2091 1186 : if_info->transform_name = "noce_try_addcc";
2092 1186 : return true;
2093 : }
2094 691 : end_sequence ();
2095 : }
2096 : }
2097 :
2098 : return false;
2099 : }
2100 :
2101 : /* Convert "if (test) x = 0;" to "x &= -(test == 0);" */
2102 :
2103 : static bool
2104 156778 : noce_try_store_flag_mask (struct noce_if_info *if_info)
2105 : {
2106 156778 : rtx target;
2107 156778 : rtx_insn *seq;
2108 156778 : bool reversep;
2109 :
2110 199993 : if (!noce_simple_bbs (if_info))
2111 : return false;
2112 :
2113 138172 : reversep = false;
2114 :
2115 138172 : if ((if_info->a == const0_rtx
2116 2824 : && (REG_P (if_info->b) || rtx_equal_p (if_info->b, if_info->x)))
2117 138172 : || ((reversep = true)
2118 137021 : && if_info->b == const0_rtx
2119 13176 : && (REG_P (if_info->a) || rtx_equal_p (if_info->a, if_info->x))))
2120 : {
2121 1153 : start_sequence ();
2122 1153 : target = noce_emit_store_flag (if_info,
2123 1153 : gen_reg_rtx (GET_MODE (if_info->x)),
2124 : reversep, -1);
2125 1153 : if (target)
2126 303 : target = expand_simple_binop (GET_MODE (if_info->x), AND,
2127 : reversep ? if_info->a : if_info->b,
2128 : target, if_info->x, 0,
2129 : OPTAB_WIDEN);
2130 :
2131 303 : if (target)
2132 : {
2133 303 : if (target != if_info->x)
2134 0 : noce_emit_move_insn (if_info->x, target);
2135 :
2136 303 : seq = end_ifcvt_sequence (if_info);
2137 303 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
2138 : return false;
2139 :
2140 11 : emit_insn_before_setloc (seq, if_info->jump,
2141 11 : INSN_LOCATION (if_info->insn_a));
2142 11 : if_info->transform_name = "noce_try_store_flag_mask";
2143 :
2144 11 : return true;
2145 : }
2146 :
2147 850 : end_sequence ();
2148 : }
2149 :
2150 : return false;
2151 : }
2152 :
2153 : /* Helper function for noce_try_cmove and noce_try_cmove_arith. */
2154 :
2155 : static rtx
2156 419852 : noce_emit_cmove (struct noce_if_info *if_info, rtx x, enum rtx_code code,
2157 : rtx cmp_a, rtx cmp_b, rtx vfalse, rtx vtrue, rtx cc_cmp,
2158 : rtx rev_cc_cmp)
2159 : {
2160 419852 : rtx target ATTRIBUTE_UNUSED;
2161 419852 : bool unsignedp ATTRIBUTE_UNUSED;
2162 :
2163 : /* If earliest == jump, try to build the cmove insn directly.
2164 : This is helpful when combine has created some complex condition
2165 : (like for alpha's cmovlbs) that we can't hope to regenerate
2166 : through the normal interface. */
2167 :
2168 419852 : if (if_info->cond_earliest == if_info->jump)
2169 : {
2170 4614 : rtx cond = gen_rtx_fmt_ee (code, GET_MODE (if_info->cond), cmp_a, cmp_b);
2171 4614 : rtx if_then_else = gen_rtx_IF_THEN_ELSE (GET_MODE (x),
2172 : cond, vtrue, vfalse);
2173 4614 : rtx set = gen_rtx_SET (x, if_then_else);
2174 :
2175 4614 : start_sequence ();
2176 4614 : rtx_insn *insn = emit_insn (set);
2177 :
2178 4614 : if (recog_memoized (insn) >= 0)
2179 : {
2180 1347 : rtx_insn *seq = end_sequence ();
2181 1347 : emit_insn (seq);
2182 :
2183 1347 : return x;
2184 : }
2185 :
2186 3267 : end_sequence ();
2187 : }
2188 :
2189 837010 : unsignedp = (code == LTU || code == GEU
2190 418505 : || code == LEU || code == GTU);
2191 :
2192 418505 : if (cc_cmp != NULL_RTX && rev_cc_cmp != NULL_RTX)
2193 86964 : target = emit_conditional_move (x, cc_cmp, rev_cc_cmp,
2194 86964 : vtrue, vfalse, GET_MODE (x));
2195 : else
2196 : {
2197 : /* Don't even try if the comparison operands are weird
2198 : except that the target supports cbranchcc4. */
2199 331541 : if (! general_operand (cmp_a, GET_MODE (cmp_a))
2200 331541 : || ! general_operand (cmp_b, GET_MODE (cmp_b)))
2201 : {
2202 25236 : if (!have_cbranchcc4
2203 25236 : || GET_MODE_CLASS (GET_MODE (cmp_a)) != MODE_CC
2204 3015 : || cmp_b != const0_rtx)
2205 : return NULL_RTX;
2206 : }
2207 :
2208 309320 : target = emit_conditional_move (x, { code, cmp_a, cmp_b, VOIDmode },
2209 309320 : vtrue, vfalse, GET_MODE (x),
2210 : unsignedp);
2211 : }
2212 :
2213 396284 : if (target)
2214 : return target;
2215 :
2216 : /* We might be faced with a situation like:
2217 :
2218 : x = (reg:M TARGET)
2219 : vtrue = (subreg:M (reg:N VTRUE) BYTE)
2220 : vfalse = (subreg:M (reg:N VFALSE) BYTE)
2221 :
2222 : We can't do a conditional move in mode M, but it's possible that we
2223 : could do a conditional move in mode N instead and take a subreg of
2224 : the result.
2225 :
2226 : If we can't create new pseudos, though, don't bother. */
2227 43129 : if (reload_completed)
2228 : return NULL_RTX;
2229 :
2230 43129 : if (GET_CODE (vtrue) == SUBREG && GET_CODE (vfalse) == SUBREG)
2231 : {
2232 23 : rtx reg_vtrue = SUBREG_REG (vtrue);
2233 23 : rtx reg_vfalse = SUBREG_REG (vfalse);
2234 23 : poly_uint64 byte_vtrue = SUBREG_BYTE (vtrue);
2235 23 : poly_uint64 byte_vfalse = SUBREG_BYTE (vfalse);
2236 23 : rtx promoted_target;
2237 :
2238 23 : if (GET_MODE (reg_vtrue) != GET_MODE (reg_vfalse)
2239 23 : || maybe_ne (byte_vtrue, byte_vfalse)
2240 23 : || (SUBREG_PROMOTED_VAR_P (vtrue)
2241 23 : != SUBREG_PROMOTED_VAR_P (vfalse))
2242 23 : || (SUBREG_PROMOTED_GET (vtrue)
2243 23 : != SUBREG_PROMOTED_GET (vfalse)))
2244 : return NULL_RTX;
2245 :
2246 23 : promoted_target = gen_reg_rtx (GET_MODE (reg_vtrue));
2247 :
2248 46 : target = emit_conditional_move (promoted_target,
2249 : { code, cmp_a, cmp_b, VOIDmode },
2250 : reg_vtrue, reg_vfalse,
2251 23 : GET_MODE (reg_vtrue), unsignedp);
2252 : /* Nope, couldn't do it in that mode either. */
2253 23 : if (!target)
2254 : return NULL_RTX;
2255 :
2256 6 : target = gen_rtx_SUBREG (GET_MODE (vtrue), promoted_target, byte_vtrue);
2257 6 : SUBREG_PROMOTED_VAR_P (target) = SUBREG_PROMOTED_VAR_P (vtrue);
2258 6 : SUBREG_PROMOTED_SET (target, SUBREG_PROMOTED_GET (vtrue));
2259 6 : emit_move_insn (x, target);
2260 6 : return x;
2261 : }
2262 : else
2263 : return NULL_RTX;
2264 : }
2265 :
2266 : /* Try only simple constants and registers here. More complex cases
2267 : are handled in noce_try_cmove_arith after noce_try_store_flag_arith
2268 : has had a go at it. */
2269 :
2270 : static bool
2271 200895 : noce_try_cmove (struct noce_if_info *if_info)
2272 : {
2273 200895 : enum rtx_code code;
2274 200895 : rtx target;
2275 200895 : rtx_insn *seq;
2276 :
2277 260636 : if (!noce_simple_bbs (if_info))
2278 : return false;
2279 :
2280 124136 : if ((CONSTANT_P (if_info->a) || register_operand (if_info->a, VOIDmode))
2281 213511 : && (CONSTANT_P (if_info->b) || register_operand (if_info->b, VOIDmode)))
2282 : {
2283 82754 : start_sequence ();
2284 :
2285 82754 : code = GET_CODE (if_info->cond);
2286 82754 : target = noce_emit_cmove (if_info, if_info->x, code,
2287 : XEXP (if_info->cond, 0),
2288 : XEXP (if_info->cond, 1),
2289 : if_info->a, if_info->b);
2290 :
2291 82754 : if (target)
2292 : {
2293 56592 : if (target != if_info->x)
2294 0 : noce_emit_move_insn (if_info->x, target);
2295 :
2296 56592 : seq = end_ifcvt_sequence (if_info);
2297 56592 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
2298 : return false;
2299 :
2300 42567 : emit_insn_before_setloc (seq, if_info->jump,
2301 42567 : INSN_LOCATION (if_info->insn_a));
2302 42567 : if_info->transform_name = "noce_try_cmove";
2303 :
2304 42567 : return true;
2305 : }
2306 : /* If both a and b are constants try a last-ditch transformation:
2307 : if (test) x = a; else x = b;
2308 : => x = (-(test != 0) & (b - a)) + a;
2309 : Try this only if the target-specific expansion above has failed.
2310 : The target-specific expander may want to generate sequences that
2311 : we don't know about, so give them a chance before trying this
2312 : approach. */
2313 26162 : else if (!targetm.have_conditional_execution ()
2314 26162 : && CONST_INT_P (if_info->a) && CONST_INT_P (if_info->b))
2315 : {
2316 5369 : machine_mode mode = GET_MODE (if_info->x);
2317 5369 : HOST_WIDE_INT ifalse = INTVAL (if_info->a);
2318 5369 : HOST_WIDE_INT itrue = INTVAL (if_info->b);
2319 5369 : rtx target = noce_emit_store_flag (if_info, if_info->x, false, -1);
2320 5369 : if (!target)
2321 : {
2322 5309 : end_sequence ();
2323 5309 : return false;
2324 : }
2325 :
2326 60 : HOST_WIDE_INT diff = (unsigned HOST_WIDE_INT) itrue - ifalse;
2327 : /* Make sure we can represent the difference
2328 : between the two values. */
2329 60 : if ((diff > 0)
2330 60 : != ((ifalse < 0) != (itrue < 0) ? ifalse < 0 : ifalse < itrue))
2331 : {
2332 21 : end_sequence ();
2333 21 : return false;
2334 : }
2335 :
2336 39 : diff = trunc_int_for_mode (diff, mode);
2337 39 : target = expand_simple_binop (mode, AND,
2338 39 : target, gen_int_mode (diff, mode),
2339 : if_info->x, 0, OPTAB_WIDEN);
2340 39 : if (target)
2341 39 : target = expand_simple_binop (mode, PLUS,
2342 39 : target, gen_int_mode (ifalse, mode),
2343 : if_info->x, 0, OPTAB_WIDEN);
2344 39 : if (target)
2345 : {
2346 39 : if (target != if_info->x)
2347 0 : noce_emit_move_insn (if_info->x, target);
2348 :
2349 39 : seq = end_ifcvt_sequence (if_info);
2350 39 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
2351 : return false;
2352 :
2353 36 : emit_insn_before_setloc (seq, if_info->jump,
2354 36 : INSN_LOCATION (if_info->insn_a));
2355 36 : if_info->transform_name = "noce_try_cmove";
2356 36 : return true;
2357 : }
2358 : else
2359 : {
2360 0 : end_sequence ();
2361 0 : return false;
2362 : }
2363 : }
2364 : else
2365 20793 : end_sequence ();
2366 : }
2367 :
2368 : return false;
2369 : }
2370 :
2371 : /* Return true if X contains a conditional code mode rtx. */
2372 :
2373 : static bool
2374 2924856 : contains_ccmode_rtx_p (rtx x)
2375 : {
2376 2924856 : subrtx_iterator::array_type array;
2377 7450007 : FOR_EACH_SUBRTX (iter, array, x, ALL)
2378 4570505 : if (GET_MODE_CLASS (GET_MODE (*iter)) == MODE_CC)
2379 45354 : return true;
2380 :
2381 2879502 : return false;
2382 2924856 : }
2383 :
2384 : /* Helper for bb_valid_for_noce_process_p. Validate that
2385 : the rtx insn INSN is a single set that does not set
2386 : the conditional register CC and is in general valid for
2387 : if-conversion. */
2388 :
2389 : static bool
2390 3674980 : insn_valid_noce_process_p (rtx_insn *insn, rtx cc)
2391 : {
2392 3674980 : if (!insn
2393 3674669 : || !NONJUMP_INSN_P (insn)
2394 6630908 : || (cc && set_of (cc, insn)))
2395 : return false;
2396 :
2397 2944307 : rtx sset = single_set (insn);
2398 :
2399 : /* Currently support only simple single sets in test_bb. */
2400 2944307 : if (!sset
2401 2937939 : || !noce_operand_ok (SET_DEST (sset))
2402 2924856 : || contains_ccmode_rtx_p (SET_DEST (sset))
2403 5823809 : || !noce_operand_ok (SET_SRC (sset)))
2404 130455 : return false;
2405 :
2406 : return true;
2407 : }
2408 :
2409 :
2410 : /* Return true iff the registers that the insns in BB_A set do not get
2411 : used in BB_B. If TO_RENAME is non-NULL then it is a location that will be
2412 : renamed later by the caller and so conflicts on it should be ignored
2413 : in this function. */
2414 :
2415 : static bool
2416 54248 : bbs_ok_for_cmove_arith (basic_block bb_a, basic_block bb_b, rtx to_rename)
2417 : {
2418 54248 : rtx_insn *a_insn;
2419 54248 : bitmap bba_sets = BITMAP_ALLOC (®_obstack);
2420 :
2421 54248 : df_ref def;
2422 54248 : df_ref use;
2423 :
2424 284792 : FOR_BB_INSNS (bb_a, a_insn)
2425 : {
2426 230544 : if (!active_insn_p (a_insn))
2427 153564 : continue;
2428 :
2429 76980 : rtx sset_a = single_set (a_insn);
2430 :
2431 76980 : if (!sset_a)
2432 : {
2433 0 : BITMAP_FREE (bba_sets);
2434 0 : return false;
2435 : }
2436 : /* Record all registers that BB_A sets. */
2437 187984 : FOR_EACH_INSN_DEF (def, a_insn)
2438 111004 : if (!(to_rename && DF_REF_REG (def) == to_rename))
2439 57885 : bitmap_set_bit (bba_sets, DF_REF_REGNO (def));
2440 : }
2441 :
2442 54248 : rtx_insn *b_insn;
2443 :
2444 284856 : FOR_BB_INSNS (bb_b, b_insn)
2445 : {
2446 230708 : if (!active_insn_p (b_insn))
2447 153735 : continue;
2448 :
2449 76973 : rtx sset_b = single_set (b_insn);
2450 :
2451 76973 : if (!sset_b)
2452 : {
2453 0 : BITMAP_FREE (bba_sets);
2454 0 : return false;
2455 : }
2456 :
2457 : /* Make sure this is a REG and not some instance
2458 : of ZERO_EXTRACT or non-paradoxical SUBREG or other dangerous stuff.
2459 : If we have a memory destination then we have a pair of simple
2460 : basic blocks performing an operation of the form [addr] = c ? a : b.
2461 : bb_valid_for_noce_process_p will have ensured that these are
2462 : the only stores present. In that case [addr] should be the location
2463 : to be renamed. Assert that the callers set this up properly. */
2464 76973 : if (MEM_P (SET_DEST (sset_b)))
2465 1116 : gcc_assert (rtx_equal_p (SET_DEST (sset_b), to_rename));
2466 75857 : else if (!REG_P (SET_DEST (sset_b))
2467 75857 : && !paradoxical_subreg_p (SET_DEST (sset_b)))
2468 : {
2469 43 : BITMAP_FREE (bba_sets);
2470 43 : return false;
2471 : }
2472 :
2473 : /* If the insn uses a reg set in BB_A return false. */
2474 150254 : FOR_EACH_INSN_USE (use, b_insn)
2475 : {
2476 73381 : if (bitmap_bit_p (bba_sets, DF_REF_REGNO (use)))
2477 : {
2478 57 : BITMAP_FREE (bba_sets);
2479 57 : return false;
2480 : }
2481 : }
2482 :
2483 : }
2484 :
2485 54148 : BITMAP_FREE (bba_sets);
2486 54148 : return true;
2487 : }
2488 :
2489 : /* Emit copies of all the active instructions in BB except the last.
2490 : This is a helper for noce_try_cmove_arith. */
2491 :
2492 : static void
2493 14095 : noce_emit_all_but_last (basic_block bb)
2494 : {
2495 14095 : rtx_insn *last = last_active_insn (bb, false);
2496 14095 : rtx_insn *insn;
2497 145074 : FOR_BB_INSNS (bb, insn)
2498 : {
2499 116884 : if (insn != last && active_insn_p (insn))
2500 : {
2501 37711 : rtx_insn *to_emit = as_a <rtx_insn *> (copy_rtx (insn));
2502 :
2503 37711 : emit_insn (PATTERN (to_emit));
2504 : }
2505 : }
2506 14095 : }
2507 :
2508 : /* Helper for noce_try_cmove_arith. Emit the pattern TO_EMIT and return
2509 : the resulting insn or NULL if it's not a valid insn. */
2510 :
2511 : static rtx_insn *
2512 141074 : noce_emit_insn (rtx to_emit)
2513 : {
2514 141074 : gcc_assert (to_emit);
2515 141074 : rtx_insn *insn = emit_insn (to_emit);
2516 :
2517 141074 : if (recog_memoized (insn) < 0)
2518 193 : return NULL;
2519 :
2520 : return insn;
2521 : }
2522 :
2523 : /* Helper for noce_try_cmove_arith. Emit a copy of the insns up to
2524 : and including the penultimate one in BB if it is not simple
2525 : (as indicated by SIMPLE). Then emit LAST_INSN as the last
2526 : insn in the block. The reason for that is that LAST_INSN may
2527 : have been modified by the preparation in noce_try_cmove_arith. */
2528 :
2529 : static bool
2530 143326 : noce_emit_bb (rtx last_insn, basic_block bb, bool simple)
2531 : {
2532 143326 : if (bb && !simple)
2533 14095 : noce_emit_all_but_last (bb);
2534 :
2535 143326 : if (last_insn && !noce_emit_insn (last_insn))
2536 193 : return false;
2537 :
2538 : return true;
2539 : }
2540 :
2541 : /* Try more complex cases involving conditional_move. */
2542 :
2543 : static bool
2544 146385 : noce_try_cmove_arith (struct noce_if_info *if_info)
2545 : {
2546 146385 : rtx a = if_info->a;
2547 146385 : rtx b = if_info->b;
2548 146385 : rtx x = if_info->x;
2549 146385 : rtx orig_a, orig_b;
2550 146385 : rtx_insn *insn_a, *insn_b;
2551 146385 : bool a_simple = if_info->then_simple;
2552 146385 : bool b_simple = if_info->else_simple;
2553 146385 : basic_block then_bb = if_info->then_bb;
2554 146385 : basic_block else_bb = if_info->else_bb;
2555 146385 : rtx target;
2556 146385 : bool is_mem = false;
2557 146385 : enum rtx_code code;
2558 146385 : rtx cond = if_info->cond;
2559 146385 : rtx_insn *ifcvt_seq;
2560 :
2561 : /* A conditional move from two memory sources is equivalent to a
2562 : conditional on their addresses followed by a load. Don't do this
2563 : early because it'll screw alias analysis. Note that we've
2564 : already checked for no side effects. */
2565 146385 : if (cse_not_expected
2566 47820 : && MEM_P (a) && MEM_P (b)
2567 149044 : && MEM_ADDR_SPACE (a) == MEM_ADDR_SPACE (b))
2568 : {
2569 2658 : machine_mode address_mode = get_address_mode (a);
2570 :
2571 2658 : a = XEXP (a, 0);
2572 2658 : b = XEXP (b, 0);
2573 2658 : x = gen_reg_rtx (address_mode);
2574 2658 : is_mem = true;
2575 : }
2576 :
2577 : /* ??? We could handle this if we knew that a load from A or B could
2578 : not trap or fault. This is also true if we've already loaded
2579 : from the address along the path from ENTRY. */
2580 143727 : else if (may_trap_or_fault_p (a) || may_trap_or_fault_p (b))
2581 : return false;
2582 :
2583 : /* if (test) x = a + b; else x = c - d;
2584 : => y = a + b;
2585 : x = c - d;
2586 : if (test)
2587 : x = y;
2588 : */
2589 :
2590 92165 : code = GET_CODE (cond);
2591 92165 : insn_a = if_info->insn_a;
2592 92165 : insn_b = if_info->insn_b;
2593 :
2594 92165 : machine_mode x_mode = GET_MODE (x);
2595 :
2596 92165 : if (!can_conditionally_move_p (x_mode))
2597 : return false;
2598 :
2599 : /* Possibly rearrange operands to make things come out more natural. */
2600 71763 : if (noce_reversed_cond_code (if_info) != UNKNOWN)
2601 : {
2602 71763 : bool reversep = false;
2603 71763 : if (rtx_equal_p (b, x))
2604 : reversep = true;
2605 45766 : else if (general_operand (b, GET_MODE (b)))
2606 : reversep = true;
2607 :
2608 : if (reversep)
2609 : {
2610 63668 : if (if_info->rev_cond)
2611 : {
2612 63668 : cond = if_info->rev_cond;
2613 63668 : code = GET_CODE (cond);
2614 : }
2615 : else
2616 0 : code = reversed_comparison_code (cond, if_info->jump);
2617 : std::swap (a, b);
2618 : std::swap (insn_a, insn_b);
2619 : std::swap (a_simple, b_simple);
2620 : std::swap (then_bb, else_bb);
2621 : }
2622 : }
2623 :
2624 27943 : if (then_bb && else_bb
2625 98937 : && (!bbs_ok_for_cmove_arith (then_bb, else_bb, if_info->orig_x)
2626 27074 : || !bbs_ok_for_cmove_arith (else_bb, then_bb, if_info->orig_x)))
2627 : return false;
2628 :
2629 71663 : start_sequence ();
2630 :
2631 : /* If one of the blocks is empty then the corresponding B or A value
2632 : came from the test block. The non-empty complex block that we will
2633 : emit might clobber the register used by B or A, so move it to a pseudo
2634 : first. */
2635 :
2636 71663 : rtx tmp_a = NULL_RTX;
2637 71663 : rtx tmp_b = NULL_RTX;
2638 :
2639 71663 : if (b_simple || !else_bb)
2640 59652 : tmp_b = gen_reg_rtx (x_mode);
2641 :
2642 71663 : if (a_simple || !then_bb)
2643 69579 : tmp_a = gen_reg_rtx (x_mode);
2644 :
2645 71663 : orig_a = a;
2646 71663 : orig_b = b;
2647 :
2648 71663 : rtx emit_a = NULL_RTX;
2649 71663 : rtx emit_b = NULL_RTX;
2650 71663 : rtx_insn *tmp_insn = NULL;
2651 71663 : bool modified_in_a = false;
2652 71663 : bool modified_in_b = false;
2653 : /* If either operand is complex, load it into a register first.
2654 : The best way to do this is to copy the original insn. In this
2655 : way we preserve any clobbers etc that the insn may have had.
2656 : This is of course not possible in the IS_MEM case. */
2657 :
2658 71663 : if (! general_operand (a, GET_MODE (a)) || tmp_a)
2659 : {
2660 :
2661 71013 : if (is_mem)
2662 : {
2663 2657 : rtx reg = gen_reg_rtx (GET_MODE (a));
2664 2657 : emit_a = gen_rtx_SET (reg, a);
2665 : }
2666 : else
2667 : {
2668 68356 : if (insn_a)
2669 : {
2670 42300 : a = tmp_a ? tmp_a : gen_reg_rtx (GET_MODE (a));
2671 :
2672 42300 : rtx_insn *copy_of_a = as_a <rtx_insn *> (copy_rtx (insn_a));
2673 42300 : rtx set = single_set (copy_of_a);
2674 42300 : SET_DEST (set) = a;
2675 :
2676 42300 : emit_a = PATTERN (copy_of_a);
2677 : }
2678 : else
2679 : {
2680 26056 : rtx tmp_reg = tmp_a ? tmp_a : gen_reg_rtx (GET_MODE (a));
2681 26056 : emit_a = gen_rtx_SET (tmp_reg, a);
2682 26056 : a = tmp_reg;
2683 : }
2684 : }
2685 : }
2686 :
2687 71663 : if (! general_operand (b, GET_MODE (b)) || tmp_b)
2688 : {
2689 70061 : if (is_mem)
2690 : {
2691 2657 : rtx reg = gen_reg_rtx (GET_MODE (b));
2692 2657 : emit_b = gen_rtx_SET (reg, b);
2693 : }
2694 : else
2695 : {
2696 67404 : if (insn_b)
2697 : {
2698 67214 : b = tmp_b ? tmp_b : gen_reg_rtx (GET_MODE (b));
2699 67214 : rtx_insn *copy_of_b = as_a <rtx_insn *> (copy_rtx (insn_b));
2700 67214 : rtx set = single_set (copy_of_b);
2701 :
2702 67214 : SET_DEST (set) = b;
2703 67214 : emit_b = PATTERN (copy_of_b);
2704 : }
2705 : else
2706 : {
2707 190 : rtx tmp_reg = tmp_b ? tmp_b : gen_reg_rtx (GET_MODE (b));
2708 190 : emit_b = gen_rtx_SET (tmp_reg, b);
2709 190 : b = tmp_reg;
2710 : }
2711 : }
2712 : }
2713 :
2714 71663 : modified_in_a = emit_a != NULL_RTX && modified_in_p (orig_b, emit_a);
2715 71663 : if (tmp_b && then_bb)
2716 : {
2717 77083 : FOR_BB_INSNS (then_bb, tmp_insn)
2718 : /* Don't check inside insn_a. We will have changed it to emit_a
2719 : with a destination that doesn't conflict. */
2720 58155 : if (!(insn_a && tmp_insn == insn_a)
2721 97382 : && modified_in_p (orig_b, tmp_insn))
2722 : {
2723 : modified_in_a = true;
2724 : break;
2725 : }
2726 :
2727 : }
2728 :
2729 71663 : modified_in_b = emit_b != NULL_RTX && modified_in_p (orig_a, emit_b);
2730 71663 : if (tmp_a && else_bb)
2731 : {
2732 332060 : FOR_BB_INSNS (else_bb, tmp_insn)
2733 : /* Don't check inside insn_b. We will have changed it to emit_b
2734 : with a destination that doesn't conflict. */
2735 263226 : if (!(insn_b && tmp_insn == insn_b)
2736 457618 : && modified_in_p (orig_a, tmp_insn))
2737 : {
2738 : modified_in_b = true;
2739 : break;
2740 : }
2741 : }
2742 :
2743 : /* If insn to set up A clobbers any registers B depends on, try to
2744 : swap insn that sets up A with the one that sets up B. If even
2745 : that doesn't help, punt. */
2746 71663 : if (modified_in_a && !modified_in_b)
2747 : {
2748 0 : if (!noce_emit_bb (emit_b, else_bb, b_simple))
2749 0 : goto end_seq_and_fail;
2750 :
2751 0 : if (!noce_emit_bb (emit_a, then_bb, a_simple))
2752 0 : goto end_seq_and_fail;
2753 : }
2754 71663 : else if (!modified_in_a)
2755 : {
2756 71663 : if (!noce_emit_bb (emit_a, then_bb, a_simple))
2757 0 : goto end_seq_and_fail;
2758 :
2759 71663 : if (!noce_emit_bb (emit_b, else_bb, b_simple))
2760 193 : goto end_seq_and_fail;
2761 : }
2762 : else
2763 0 : goto end_seq_and_fail;
2764 :
2765 71470 : target = noce_emit_cmove (if_info, x, code, XEXP (cond, 0), XEXP (cond, 1),
2766 : a, b);
2767 :
2768 71470 : if (! target)
2769 17499 : goto end_seq_and_fail;
2770 :
2771 : /* If we're handling a memory for above, emit the load now. */
2772 53971 : if (is_mem)
2773 : {
2774 2266 : rtx mem = gen_rtx_MEM (GET_MODE (if_info->x), target);
2775 :
2776 : /* Copy over flags as appropriate. */
2777 2266 : if (MEM_VOLATILE_P (if_info->a) || MEM_VOLATILE_P (if_info->b))
2778 0 : MEM_VOLATILE_P (mem) = 1;
2779 2266 : if (MEM_ALIAS_SET (if_info->a) == MEM_ALIAS_SET (if_info->b))
2780 2070 : set_mem_alias_set (mem, MEM_ALIAS_SET (if_info->a));
2781 4532 : set_mem_align (mem,
2782 2266 : MIN (MEM_ALIGN (if_info->a), MEM_ALIGN (if_info->b)));
2783 :
2784 2266 : gcc_assert (MEM_ADDR_SPACE (if_info->a) == MEM_ADDR_SPACE (if_info->b));
2785 2266 : set_mem_addr_space (mem, MEM_ADDR_SPACE (if_info->a));
2786 :
2787 2266 : noce_emit_move_insn (if_info->x, mem);
2788 : }
2789 51705 : else if (target != x)
2790 0 : noce_emit_move_insn (x, target);
2791 :
2792 53971 : ifcvt_seq = end_ifcvt_sequence (if_info);
2793 53971 : if (!ifcvt_seq || !targetm.noce_conversion_profitable_p (ifcvt_seq, if_info))
2794 : return false;
2795 :
2796 35085 : emit_insn_before_setloc (ifcvt_seq, if_info->jump,
2797 35085 : INSN_LOCATION (if_info->insn_a));
2798 35085 : if_info->transform_name = "noce_try_cmove_arith";
2799 35085 : return true;
2800 :
2801 17692 : end_seq_and_fail:
2802 17692 : end_sequence ();
2803 17692 : return false;
2804 : }
2805 :
2806 : /* For most cases, the simplified condition we found is the best
2807 : choice, but this is not the case for the min/max/abs transforms.
2808 : For these we wish to know that it is A or B in the condition. */
2809 :
2810 : static rtx
2811 161558 : noce_get_alt_condition (struct noce_if_info *if_info, rtx target,
2812 : rtx_insn **earliest)
2813 : {
2814 161558 : rtx cond, set;
2815 161558 : rtx_insn *insn;
2816 161558 : bool reverse;
2817 :
2818 : /* If target is already mentioned in the known condition, return it. */
2819 161558 : if (reg_mentioned_p (target, if_info->cond))
2820 : {
2821 8258 : *earliest = if_info->cond_earliest;
2822 8258 : return if_info->cond;
2823 : }
2824 :
2825 153300 : set = pc_set (if_info->jump);
2826 153300 : cond = XEXP (SET_SRC (set), 0);
2827 153300 : reverse
2828 306600 : = GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
2829 153300 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (if_info->jump);
2830 153300 : if (if_info->then_else_reversed)
2831 23819 : reverse = !reverse;
2832 :
2833 : /* If we're looking for a constant, try to make the conditional
2834 : have that constant in it. There are two reasons why it may
2835 : not have the constant we want:
2836 :
2837 : 1. GCC may have needed to put the constant in a register, because
2838 : the target can't compare directly against that constant. For
2839 : this case, we look for a SET immediately before the comparison
2840 : that puts a constant in that register.
2841 :
2842 : 2. GCC may have canonicalized the conditional, for example
2843 : replacing "if x < 4" with "if x <= 3". We can undo that (or
2844 : make equivalent types of changes) to get the constants we need
2845 : if they're off by one in the right direction. */
2846 :
2847 153300 : if (CONST_INT_P (target))
2848 : {
2849 46839 : enum rtx_code code = GET_CODE (if_info->cond);
2850 46839 : rtx op_a = XEXP (if_info->cond, 0);
2851 46839 : rtx op_b = XEXP (if_info->cond, 1);
2852 46839 : rtx_insn *prev_insn;
2853 :
2854 : /* First, look to see if we put a constant in a register. */
2855 46839 : prev_insn = prev_nonnote_nondebug_insn (if_info->cond_earliest);
2856 46839 : if (prev_insn
2857 46728 : && BLOCK_FOR_INSN (prev_insn)
2858 46728 : == BLOCK_FOR_INSN (if_info->cond_earliest)
2859 37324 : && INSN_P (prev_insn)
2860 80016 : && GET_CODE (PATTERN (prev_insn)) == SET)
2861 : {
2862 25764 : rtx src = find_reg_equal_equiv_note (prev_insn);
2863 25764 : if (!src)
2864 25598 : src = SET_SRC (PATTERN (prev_insn));
2865 25764 : if (CONST_INT_P (src))
2866 : {
2867 15258 : if (rtx_equal_p (op_a, SET_DEST (PATTERN (prev_insn))))
2868 : op_a = src;
2869 14716 : else if (rtx_equal_p (op_b, SET_DEST (PATTERN (prev_insn))))
2870 1073 : op_b = src;
2871 :
2872 15258 : if (CONST_INT_P (op_a))
2873 : {
2874 542 : std::swap (op_a, op_b);
2875 542 : code = swap_condition (code);
2876 : }
2877 : }
2878 : }
2879 :
2880 : /* Now, look to see if we can get the right constant by
2881 : adjusting the conditional. */
2882 46839 : if (CONST_INT_P (op_b))
2883 : {
2884 24323 : HOST_WIDE_INT desired_val = INTVAL (target);
2885 24323 : HOST_WIDE_INT actual_val = INTVAL (op_b);
2886 :
2887 24323 : switch (code)
2888 : {
2889 1177 : case LT:
2890 1177 : if (desired_val != HOST_WIDE_INT_MAX
2891 1162 : && actual_val == desired_val + 1)
2892 : {
2893 173 : code = LE;
2894 173 : op_b = GEN_INT (desired_val);
2895 : }
2896 : break;
2897 3 : case LE:
2898 3 : if (desired_val != HOST_WIDE_INT_MIN
2899 3 : && actual_val == desired_val - 1)
2900 : {
2901 0 : code = LT;
2902 0 : op_b = GEN_INT (desired_val);
2903 : }
2904 : break;
2905 1683 : case GT:
2906 1683 : if (desired_val != HOST_WIDE_INT_MIN
2907 1683 : && actual_val == desired_val - 1)
2908 : {
2909 235 : code = GE;
2910 235 : op_b = GEN_INT (desired_val);
2911 : }
2912 : break;
2913 40 : case GE:
2914 40 : if (desired_val != HOST_WIDE_INT_MAX
2915 25 : && actual_val == desired_val + 1)
2916 : {
2917 3 : code = GT;
2918 3 : op_b = GEN_INT (desired_val);
2919 : }
2920 : break;
2921 : default:
2922 : break;
2923 : }
2924 : }
2925 :
2926 : /* If we made any changes, generate a new conditional that is
2927 : equivalent to what we started with, but has the right
2928 : constants in it. */
2929 46839 : if (code != GET_CODE (if_info->cond)
2930 45887 : || op_a != XEXP (if_info->cond, 0)
2931 45886 : || op_b != XEXP (if_info->cond, 1))
2932 : {
2933 2026 : cond = gen_rtx_fmt_ee (code, GET_MODE (cond), op_a, op_b);
2934 2026 : *earliest = if_info->cond_earliest;
2935 2026 : return cond;
2936 : }
2937 : }
2938 :
2939 151274 : cond = canonicalize_condition (if_info->jump, cond, reverse,
2940 : earliest, target, have_cbranchcc4, true);
2941 151274 : if (! cond || ! reg_mentioned_p (target, cond))
2942 : return NULL;
2943 :
2944 : /* We almost certainly searched back to a different place.
2945 : Need to re-verify correct lifetimes. */
2946 :
2947 : /* X may not be mentioned in the range (cond_earliest, jump]. */
2948 0 : for (insn = if_info->jump; insn != *earliest; insn = PREV_INSN (insn))
2949 0 : if (INSN_P (insn) && reg_overlap_mentioned_p (if_info->x, PATTERN (insn)))
2950 : return NULL;
2951 :
2952 : /* A and B may not be modified in the range [cond_earliest, jump). */
2953 0 : for (insn = *earliest; insn != if_info->jump; insn = NEXT_INSN (insn))
2954 0 : if (INSN_P (insn)
2955 0 : && (modified_in_p (if_info->a, insn)
2956 0 : || modified_in_p (if_info->b, insn)))
2957 : return NULL;
2958 :
2959 : return cond;
2960 : }
2961 :
2962 : /* Convert "if (a < b) x = a; else x = b;" to "x = min(a, b);", etc. */
2963 :
2964 : static bool
2965 209093 : noce_try_minmax (struct noce_if_info *if_info)
2966 : {
2967 209093 : rtx cond, target;
2968 209093 : rtx_insn *earliest, *seq;
2969 209093 : enum rtx_code code, op;
2970 209093 : bool unsignedp;
2971 :
2972 272226 : if (!noce_simple_bbs (if_info))
2973 : return false;
2974 :
2975 : /* ??? Reject modes with NaNs or signed zeros since we don't know how
2976 : they will be resolved with an SMIN/SMAX. It wouldn't be too hard
2977 : to get the target to tell us... */
2978 190487 : if (HONOR_SIGNED_ZEROS (if_info->x)
2979 190487 : || HONOR_NANS (if_info->x))
2980 : return false;
2981 :
2982 160568 : cond = noce_get_alt_condition (if_info, if_info->a, &earliest);
2983 160568 : if (!cond)
2984 : return false;
2985 :
2986 : /* Verify the condition is of the form we expect, and canonicalize
2987 : the comparison code. */
2988 10259 : code = GET_CODE (cond);
2989 10259 : if (rtx_equal_p (XEXP (cond, 0), if_info->a))
2990 : {
2991 2575 : if (! rtx_equal_p (XEXP (cond, 1), if_info->b))
2992 : return false;
2993 : }
2994 7684 : else if (rtx_equal_p (XEXP (cond, 1), if_info->a))
2995 : {
2996 4681 : if (! rtx_equal_p (XEXP (cond, 0), if_info->b))
2997 : return false;
2998 54 : code = swap_condition (code);
2999 : }
3000 : else
3001 : return false;
3002 :
3003 : /* Determine what sort of operation this is. Note that the code is for
3004 : a taken branch, so the code->operation mapping appears backwards. */
3005 73 : switch (code)
3006 : {
3007 : case LT:
3008 : case LE:
3009 : case UNLT:
3010 : case UNLE:
3011 : op = SMAX;
3012 : unsignedp = false;
3013 : break;
3014 21 : case GT:
3015 21 : case GE:
3016 21 : case UNGT:
3017 21 : case UNGE:
3018 21 : op = SMIN;
3019 21 : unsignedp = false;
3020 21 : break;
3021 3 : case LTU:
3022 3 : case LEU:
3023 3 : op = UMAX;
3024 3 : unsignedp = true;
3025 3 : break;
3026 21 : case GTU:
3027 21 : case GEU:
3028 21 : op = UMIN;
3029 21 : unsignedp = true;
3030 21 : break;
3031 : default:
3032 : return false;
3033 : }
3034 :
3035 73 : start_sequence ();
3036 :
3037 73 : target = expand_simple_binop (GET_MODE (if_info->x), op,
3038 : if_info->a, if_info->b,
3039 : if_info->x, unsignedp, OPTAB_WIDEN);
3040 73 : if (! target)
3041 : {
3042 0 : end_sequence ();
3043 0 : return false;
3044 : }
3045 73 : if (target != if_info->x)
3046 0 : noce_emit_move_insn (if_info->x, target);
3047 :
3048 73 : seq = end_ifcvt_sequence (if_info);
3049 73 : if (!seq)
3050 : return false;
3051 :
3052 73 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3053 73 : if_info->cond = cond;
3054 73 : if_info->cond_earliest = earliest;
3055 73 : if_info->rev_cond = NULL_RTX;
3056 73 : if_info->transform_name = "noce_try_minmax";
3057 :
3058 73 : return true;
3059 : }
3060 :
3061 : /* Convert "if (a < 0) x = -a; else x = a;" to "x = abs(a);",
3062 : "if (a < 0) x = ~a; else x = a;" to "x = one_cmpl_abs(a);",
3063 : etc. */
3064 :
3065 : static bool
3066 209020 : noce_try_abs (struct noce_if_info *if_info)
3067 : {
3068 209020 : rtx cond, target, a, b, c;
3069 209020 : rtx_insn *earliest, *seq;
3070 209020 : bool negate;
3071 209020 : bool one_cmpl = false;
3072 :
3073 272144 : if (!noce_simple_bbs (if_info))
3074 : return false;
3075 :
3076 : /* Reject modes with signed zeros. */
3077 190414 : if (HONOR_SIGNED_ZEROS (if_info->x))
3078 : return false;
3079 :
3080 : /* Recognize A and B as constituting an ABS or NABS. The canonical
3081 : form is a branch around the negation, taken when the object is the
3082 : first operand of a comparison against 0 that evaluates to true. */
3083 160503 : a = if_info->a;
3084 160503 : b = if_info->b;
3085 160503 : if (GET_CODE (a) == NEG && rtx_equal_p (XEXP (a, 0), b))
3086 : negate = false;
3087 159543 : else if (GET_CODE (b) == NEG && rtx_equal_p (XEXP (b, 0), a))
3088 : {
3089 : std::swap (a, b);
3090 : negate = true;
3091 : }
3092 159513 : else if (GET_CODE (a) == NOT && rtx_equal_p (XEXP (a, 0), b))
3093 : {
3094 : negate = false;
3095 : one_cmpl = true;
3096 : }
3097 159513 : else if (GET_CODE (b) == NOT && rtx_equal_p (XEXP (b, 0), a))
3098 : {
3099 : std::swap (a, b);
3100 : negate = true;
3101 : one_cmpl = true;
3102 : }
3103 : else
3104 : return false;
3105 :
3106 990 : cond = noce_get_alt_condition (if_info, b, &earliest);
3107 990 : if (!cond)
3108 : return false;
3109 :
3110 : /* Verify the condition is of the form we expect. */
3111 25 : if (rtx_equal_p (XEXP (cond, 0), b))
3112 25 : c = XEXP (cond, 1);
3113 0 : else if (rtx_equal_p (XEXP (cond, 1), b))
3114 : {
3115 0 : c = XEXP (cond, 0);
3116 0 : negate = !negate;
3117 : }
3118 : else
3119 : return false;
3120 :
3121 : /* Verify that C is zero. Search one step backward for a
3122 : REG_EQUAL note or a simple source if necessary. */
3123 25 : if (REG_P (c))
3124 : {
3125 0 : rtx set;
3126 0 : rtx_insn *insn = prev_nonnote_nondebug_insn (earliest);
3127 0 : if (insn
3128 0 : && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (earliest)
3129 0 : && (set = single_set (insn))
3130 0 : && rtx_equal_p (SET_DEST (set), c))
3131 : {
3132 0 : rtx note = find_reg_equal_equiv_note (insn);
3133 0 : if (note)
3134 0 : c = XEXP (note, 0);
3135 : else
3136 0 : c = SET_SRC (set);
3137 : }
3138 : else
3139 : return false;
3140 : }
3141 25 : if (MEM_P (c)
3142 0 : && GET_CODE (XEXP (c, 0)) == SYMBOL_REF
3143 25 : && CONSTANT_POOL_ADDRESS_P (XEXP (c, 0)))
3144 0 : c = get_pool_constant (XEXP (c, 0));
3145 :
3146 : /* Work around funny ideas get_condition has wrt canonicalization.
3147 : Note that these rtx constants are known to be CONST_INT, and
3148 : therefore imply integer comparisons.
3149 : The one_cmpl case is more complicated, as we want to handle
3150 : only x < 0 ? ~x : x or x >= 0 ? x : ~x to one_cmpl_abs (x)
3151 : and x < 0 ? x : ~x or x >= 0 ? ~x : x to ~one_cmpl_abs (x),
3152 : but not other cases (x > -1 is equivalent of x >= 0). */
3153 25 : if (c == constm1_rtx && GET_CODE (cond) == GT)
3154 : ;
3155 1 : else if (c == const1_rtx && GET_CODE (cond) == LT)
3156 : {
3157 0 : if (one_cmpl)
3158 : return false;
3159 : }
3160 1 : else if (c == CONST0_RTX (GET_MODE (b)))
3161 : {
3162 0 : if (one_cmpl
3163 0 : && GET_CODE (cond) != GE
3164 0 : && GET_CODE (cond) != LT)
3165 : return false;
3166 : }
3167 : else
3168 : return false;
3169 :
3170 : /* Determine what sort of operation this is. */
3171 24 : switch (GET_CODE (cond))
3172 : {
3173 0 : case LT:
3174 0 : case LE:
3175 0 : case UNLT:
3176 0 : case UNLE:
3177 0 : negate = !negate;
3178 0 : break;
3179 : case GT:
3180 : case GE:
3181 : case UNGT:
3182 : case UNGE:
3183 : break;
3184 : default:
3185 : return false;
3186 : }
3187 :
3188 24 : start_sequence ();
3189 24 : if (one_cmpl)
3190 0 : target = expand_one_cmpl_abs_nojump (GET_MODE (if_info->x), b,
3191 : if_info->x);
3192 : else
3193 24 : target = expand_abs_nojump (GET_MODE (if_info->x), b, if_info->x, 1);
3194 :
3195 : /* ??? It's a quandary whether cmove would be better here, especially
3196 : for integers. Perhaps combine will clean things up. */
3197 24 : if (target && negate)
3198 : {
3199 0 : if (one_cmpl)
3200 0 : target = expand_simple_unop (GET_MODE (target), NOT, target,
3201 : if_info->x, 0);
3202 : else
3203 0 : target = expand_simple_unop (GET_MODE (target), NEG, target,
3204 : if_info->x, 0);
3205 : }
3206 :
3207 24 : if (! target)
3208 : {
3209 0 : end_sequence ();
3210 0 : return false;
3211 : }
3212 :
3213 24 : if (target != if_info->x)
3214 0 : noce_emit_move_insn (if_info->x, target);
3215 :
3216 24 : seq = end_ifcvt_sequence (if_info);
3217 24 : if (!seq)
3218 : return false;
3219 :
3220 24 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3221 24 : if_info->cond = cond;
3222 24 : if_info->cond_earliest = earliest;
3223 24 : if_info->rev_cond = NULL_RTX;
3224 24 : if_info->transform_name = "noce_try_abs";
3225 :
3226 24 : return true;
3227 : }
3228 :
3229 : /* Convert "if (m < 0) x = b; else x = 0;" to "x = (m >> C) & b;". */
3230 :
3231 : static bool
3232 111300 : noce_try_sign_mask (struct noce_if_info *if_info)
3233 : {
3234 111300 : rtx cond, t, m, c;
3235 111300 : rtx_insn *seq;
3236 111300 : machine_mode mode;
3237 111300 : enum rtx_code code;
3238 111300 : bool t_unconditional;
3239 :
3240 136912 : if (!noce_simple_bbs (if_info))
3241 : return false;
3242 :
3243 99832 : cond = if_info->cond;
3244 99832 : code = GET_CODE (cond);
3245 99832 : m = XEXP (cond, 0);
3246 99832 : c = XEXP (cond, 1);
3247 :
3248 99832 : t = NULL_RTX;
3249 99832 : if (if_info->a == const0_rtx)
3250 : {
3251 2636 : if ((code == LT && c == const0_rtx)
3252 2607 : || (code == LE && c == constm1_rtx))
3253 29 : t = if_info->b;
3254 : }
3255 97196 : else if (if_info->b == const0_rtx)
3256 : {
3257 10922 : if ((code == GE && c == const0_rtx)
3258 10922 : || (code == GT && c == constm1_rtx))
3259 : t = if_info->a;
3260 : }
3261 :
3262 95 : if (! t || side_effects_p (t))
3263 : return false;
3264 :
3265 : /* We currently don't handle different modes. */
3266 95 : mode = GET_MODE (t);
3267 95 : if (GET_MODE (m) != mode)
3268 : return false;
3269 :
3270 : /* This is only profitable if T is unconditionally executed/evaluated in the
3271 : original insn sequence or T is cheap and can't trap or fault. The former
3272 : happens if B is the non-zero (T) value and if INSN_B was taken from
3273 : TEST_BB, or there was no INSN_B which can happen for e.g. conditional
3274 : stores to memory. For the cost computation use the block TEST_BB where
3275 : the evaluation will end up after the transformation. */
3276 43 : t_unconditional
3277 86 : = (t == if_info->b
3278 43 : && (if_info->insn_b == NULL_RTX
3279 0 : || BLOCK_FOR_INSN (if_info->insn_b) == if_info->test_bb));
3280 86 : if (!(t_unconditional
3281 43 : || ((set_src_cost (t, mode, if_info->speed_p)
3282 : < COSTS_N_INSNS (2))
3283 37 : && !may_trap_or_fault_p (t))))
3284 : return false;
3285 :
3286 0 : if (!noce_can_force_operand (t))
3287 : return false;
3288 :
3289 0 : start_sequence ();
3290 : /* Use emit_store_flag to generate "m < 0 ? -1 : 0" instead of expanding
3291 : "(signed) m >> 31" directly. This benefits targets with specialized
3292 : insns to obtain the signmask, but still uses ashr_optab otherwise. */
3293 0 : m = emit_store_flag (gen_reg_rtx (mode), LT, m, const0_rtx, mode, 0, -1);
3294 0 : t = m ? expand_binop (mode, and_optab, m, t, NULL_RTX, 0, OPTAB_DIRECT)
3295 : : NULL_RTX;
3296 :
3297 0 : if (!t)
3298 : {
3299 0 : end_sequence ();
3300 0 : return false;
3301 : }
3302 :
3303 0 : noce_emit_move_insn (if_info->x, t);
3304 :
3305 0 : seq = end_ifcvt_sequence (if_info);
3306 0 : if (!seq)
3307 : return false;
3308 :
3309 0 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3310 0 : if_info->transform_name = "noce_try_sign_mask";
3311 :
3312 0 : return true;
3313 : }
3314 :
3315 : /* Helper function to return REG itself,
3316 : otherwise NULL_RTX for other RTX_CODE. */
3317 :
3318 : static rtx
3319 28654 : get_base_reg_or_constant (rtx exp)
3320 : {
3321 0 : if (REG_P (exp))
3322 : return exp;
3323 5294 : else if (SUBREG_P (exp))
3324 323 : return SUBREG_REG (exp);
3325 4971 : else if (CONST_INT_P (exp))
3326 0 : return exp;
3327 : return NULL_RTX;
3328 : }
3329 :
3330 : /* Try to covert if-then-else with conditional zero,
3331 : returning TURE on success or FALSE on failure.
3332 : IF_INFO describes the if-conversion scenario under consideration.
3333 :
3334 : It verifies the branch structure on left and transforms it into branchless
3335 : sequence on the right, with a backend provided conditional zero or orig for
3336 : operand z. If true, tmp is z, 0 otherwise (y op 0 is same as y for most op).
3337 :
3338 : if (cond) | tmp = cond ? z : 0
3339 : x = y op z | x = y op tmp
3340 : else |
3341 : x = y |
3342 :
3343 : AND is special as it needs to be handled differently.
3344 :
3345 : tmp = !cond ? y : 0
3346 : x = (y & z) | tmp
3347 : Also for AND try:
3348 : tmp = cond ? z : -1
3349 : x = y op tmp
3350 : To see if it is cheaper to produce `!cond ? y : 0`
3351 : or `cond ? z : -1`.
3352 : */
3353 :
3354 : static int
3355 156767 : noce_try_cond_arith (struct noce_if_info *if_info)
3356 : {
3357 156767 : rtx target, a, b, a_op0, a_op1;
3358 156767 : rtx cond = if_info->cond;
3359 156767 : rtx_code code = GET_CODE (cond);
3360 156767 : rtx_insn *seq;
3361 156767 : rtx_code op;
3362 156767 : machine_mode mode = GET_MODE (if_info->x);
3363 :
3364 : /* Scalar integral modes are only supported here.
3365 : Could support scalar floating point but that
3366 : would be only with -ffast-math and might
3367 : be worse than a branch. */
3368 156767 : if (!SCALAR_INT_MODE_P (mode))
3369 : return false;
3370 :
3371 166058 : if (!noce_simple_bbs (if_info))
3372 : return false;
3373 :
3374 109692 : a = copy_rtx (if_info->a);
3375 109692 : b = copy_rtx (if_info->b);
3376 :
3377 : /* Canonicalize x = y : (y op z) to x = (y op z) : y. */
3378 109692 : if (REG_P (a) && noce_cond_zero_binary_op_supported (b))
3379 : {
3380 227 : if (if_info->rev_cond)
3381 : {
3382 227 : cond = if_info->rev_cond;
3383 227 : code = GET_CODE (cond);
3384 : }
3385 : else
3386 0 : code = reversed_comparison_code (cond, if_info->jump);
3387 : std::swap (a, b);
3388 : }
3389 :
3390 : /* Check if x = (y op z) : y is supported by czero based ifcvt. */
3391 109465 : else if (!(noce_cond_zero_binary_op_supported (a) && REG_P (b)))
3392 95036 : goto fail;
3393 :
3394 14656 : if (code == UNKNOWN)
3395 0 : goto fail;
3396 :
3397 14656 : op = GET_CODE (a);
3398 :
3399 : /* Canonicalize x = (z op y) : y to x = (y op z) : y */
3400 14656 : a_op1 = get_base_reg_or_constant (XEXP (a, 1));
3401 13995 : if (a_op1 && rtx_equal_p (a_op1, b) && COMMUTATIVE_ARITH_P (a))
3402 : {
3403 10 : std::swap (XEXP (a, 0), XEXP (a, 1));
3404 10 : a_op1 = get_base_reg_or_constant (XEXP (a, 1));
3405 : }
3406 :
3407 14646 : if (a_op1 == NULL_RTX)
3408 668 : goto fail;
3409 :
3410 : /* Ensure the cond is of form: x = (y op z) : y */
3411 13988 : a_op0 = get_base_reg_or_constant (XEXP (a, 0));
3412 13983 : if (!(a_op0 && rtx_equal_p (a_op0, b)))
3413 1987 : goto fail;
3414 :
3415 12001 : start_sequence ();
3416 :
3417 12001 : if (CONST_INT_P (XEXP (a, 1)))
3418 3038 : target = gen_reg_rtx (GET_MODE (XEXP (a, 0)));
3419 : else
3420 8963 : target = gen_reg_rtx (GET_MODE (XEXP (a, op != AND)));
3421 :
3422 : /* AND requires !cond, instead we swap ops around. */
3423 12001 : target = noce_emit_cmove (if_info, target, code,
3424 : XEXP (cond, 0), XEXP (cond, 1),
3425 : op != AND ? XEXP (a, 1) : const0_rtx,
3426 : op != AND ? const0_rtx : XEXP (a, 0));
3427 12001 : if (!target)
3428 : {
3429 1223 : end_sequence ();
3430 1223 : rtx tmp = XEXP (a, op != AND);
3431 : /* If the cmove fails and this was a lowpart subreg,
3432 : then try the reg part and then putting back the lowpart
3433 : afterwards. */
3434 1223 : if (GET_CODE (tmp) != SUBREG || !subreg_lowpart_p (tmp))
3435 : return false;
3436 0 : tmp = SUBREG_REG (tmp);
3437 : /* Only handle integer scalar modes for the inner mode of
3438 : the subreg. */
3439 0 : if (!SCALAR_INT_MODE_P (GET_MODE (tmp)))
3440 : return false;
3441 :
3442 0 : start_sequence ();
3443 0 : target = gen_reg_rtx (GET_MODE (tmp));
3444 0 : target = noce_emit_cmove (if_info, target, code,
3445 : XEXP (cond, 0), XEXP (cond, 1),
3446 : op != AND ? tmp : const0_rtx,
3447 : op != AND ? const0_rtx : tmp);
3448 0 : if (!target)
3449 0 : goto end_seq_n_fail;
3450 0 : target = rtl_hooks.gen_lowpart_no_emit (GET_MODE (XEXP (a, op != AND)), target);
3451 0 : gcc_assert (target);
3452 : }
3453 :
3454 : /* For AND, try `cond ? z : -1` to see if that is cheaper or the same cost.
3455 : In some cases it will be cheaper to produce the -1 rather than the 0 case. */
3456 10778 : if (op == AND)
3457 : {
3458 85 : rtx_insn *seq0 = end_sequence ();
3459 85 : unsigned cost0 = seq_cost (seq0, if_info->speed_p);
3460 85 : if (!target)
3461 : cost0 = -1u;
3462 :
3463 : /* Produce `cond ? z : -1`. */
3464 85 : rtx targetm1;
3465 85 : start_sequence ();
3466 85 : targetm1 = gen_reg_rtx (GET_MODE (XEXP (a, 1)));
3467 85 : targetm1 = noce_emit_cmove (if_info, targetm1, code,
3468 : XEXP (cond, 0), XEXP (cond, 1),
3469 : XEXP (a, 1), constm1_rtx);
3470 85 : rtx_insn *seqm1 = end_sequence ();
3471 85 : unsigned costm1 = seq_cost (seqm1, if_info->speed_p);
3472 85 : if (!targetm1)
3473 79 : costm1 = -1u;
3474 : /* If both fails, then there is no costing to be done. */
3475 85 : if (!targetm1 && !target)
3476 : return false;
3477 :
3478 : /* If -1 is cheaper or the same cost to producing 0, then use that. */
3479 85 : if (costm1 <= cost0)
3480 : {
3481 6 : push_to_sequence (seqm1);
3482 6 : targetm1 = expand_simple_binop (mode, op, a_op0, targetm1,
3483 : if_info->x, 0, OPTAB_WIDEN);
3484 6 : if (targetm1)
3485 : {
3486 6 : target = targetm1;
3487 6 : goto success;
3488 : }
3489 0 : end_sequence ();
3490 : }
3491 79 : if (!target)
3492 : return false;
3493 : /* For 0 the produce sequence is:
3494 : tmp = !cond ? y : 0
3495 : x = (y & z) | tmp */
3496 79 : push_to_sequence (seq0);
3497 79 : rtx a_bin = gen_reg_rtx (mode);
3498 79 : noce_emit_move_insn (a_bin, a);
3499 :
3500 79 : target = expand_simple_binop (mode, IOR, a_bin, target, if_info->x, 0,
3501 : OPTAB_WIDEN);
3502 79 : if (!target)
3503 0 : goto end_seq_n_fail;
3504 79 : goto success;
3505 : }
3506 10693 : if (!target)
3507 : goto end_seq_n_fail;
3508 :
3509 10693 : target = expand_simple_binop (mode, op, a_op0, target, if_info->x, 0,
3510 : OPTAB_WIDEN);
3511 :
3512 10693 : if (!target)
3513 0 : goto end_seq_n_fail;
3514 :
3515 10693 : success:
3516 10778 : if (target != if_info->x)
3517 0 : noce_emit_move_insn (if_info->x, target);
3518 :
3519 10778 : seq = end_ifcvt_sequence (if_info);
3520 10778 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
3521 396 : goto fail;
3522 :
3523 10382 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3524 10382 : if_info->transform_name = "noce_try_cond_arith";
3525 10382 : return true;
3526 :
3527 0 : end_seq_n_fail:
3528 0 : end_sequence ();
3529 :
3530 156767 : fail:
3531 :
3532 : return false;
3533 : }
3534 :
3535 : /* Optimize away "if (x & C) x |= C" and similar bit manipulation
3536 : transformations. */
3537 :
3538 : static bool
3539 209093 : noce_try_bitop (struct noce_if_info *if_info)
3540 : {
3541 209093 : rtx cond, x, a, result;
3542 209093 : rtx_insn *seq;
3543 209093 : scalar_int_mode mode;
3544 209093 : enum rtx_code code;
3545 209093 : int bitnum;
3546 :
3547 209093 : x = if_info->x;
3548 209093 : cond = if_info->cond;
3549 209093 : code = GET_CODE (cond);
3550 :
3551 : /* Check for an integer operation. */
3552 209093 : if (!is_a <scalar_int_mode> (GET_MODE (x), &mode))
3553 : return false;
3554 :
3555 235202 : if (!noce_simple_bbs (if_info))
3556 : return false;
3557 :
3558 : /* Check for no else condition. */
3559 159179 : if (! rtx_equal_p (x, if_info->b))
3560 : return false;
3561 :
3562 : /* Check for a suitable condition. */
3563 84174 : if (code != NE && code != EQ)
3564 : return false;
3565 59840 : if (XEXP (cond, 1) != const0_rtx)
3566 : return false;
3567 41573 : cond = XEXP (cond, 0);
3568 :
3569 : /* ??? We could also handle AND here. */
3570 41573 : if (GET_CODE (cond) == ZERO_EXTRACT)
3571 : {
3572 230 : if (XEXP (cond, 1) != const1_rtx
3573 114 : || !CONST_INT_P (XEXP (cond, 2))
3574 344 : || ! rtx_equal_p (x, XEXP (cond, 0)))
3575 : return false;
3576 4 : bitnum = INTVAL (XEXP (cond, 2));
3577 4 : if (BITS_BIG_ENDIAN)
3578 : bitnum = GET_MODE_BITSIZE (mode) - 1 - bitnum;
3579 4 : if (bitnum < 0 || bitnum >= HOST_BITS_PER_WIDE_INT)
3580 : return false;
3581 : }
3582 : else
3583 : return false;
3584 :
3585 4 : a = if_info->a;
3586 4 : if (GET_CODE (a) == IOR || GET_CODE (a) == XOR)
3587 : {
3588 : /* Check for "if (X & C) x = x op C". */
3589 0 : if (! rtx_equal_p (x, XEXP (a, 0))
3590 0 : || !CONST_INT_P (XEXP (a, 1))
3591 0 : || (INTVAL (XEXP (a, 1)) & GET_MODE_MASK (mode))
3592 0 : != HOST_WIDE_INT_1U << bitnum)
3593 : return false;
3594 :
3595 : /* if ((x & C) == 0) x |= C; is transformed to x |= C. */
3596 : /* if ((x & C) != 0) x |= C; is transformed to nothing. */
3597 0 : if (GET_CODE (a) == IOR)
3598 0 : result = (code == NE) ? a : NULL_RTX;
3599 0 : else if (code == NE)
3600 : {
3601 : /* if ((x & C) == 0) x ^= C; is transformed to x |= C. */
3602 0 : result = gen_int_mode (HOST_WIDE_INT_1 << bitnum, mode);
3603 0 : result = simplify_gen_binary (IOR, mode, x, result);
3604 : }
3605 : else
3606 : {
3607 : /* if ((x & C) != 0) x ^= C; is transformed to x &= ~C. */
3608 0 : result = gen_int_mode (~(HOST_WIDE_INT_1 << bitnum), mode);
3609 0 : result = simplify_gen_binary (AND, mode, x, result);
3610 : }
3611 : }
3612 4 : else if (GET_CODE (a) == AND)
3613 : {
3614 : /* Check for "if (X & C) x &= ~C". */
3615 0 : if (! rtx_equal_p (x, XEXP (a, 0))
3616 0 : || !CONST_INT_P (XEXP (a, 1))
3617 0 : || (INTVAL (XEXP (a, 1)) & GET_MODE_MASK (mode))
3618 0 : != (~(HOST_WIDE_INT_1 << bitnum) & GET_MODE_MASK (mode)))
3619 : return false;
3620 :
3621 : /* if ((x & C) == 0) x &= ~C; is transformed to nothing. */
3622 : /* if ((x & C) != 0) x &= ~C; is transformed to x &= ~C. */
3623 0 : result = (code == EQ) ? a : NULL_RTX;
3624 : }
3625 : else
3626 : return false;
3627 :
3628 0 : if (result)
3629 : {
3630 0 : start_sequence ();
3631 0 : noce_emit_move_insn (x, result);
3632 0 : seq = end_ifcvt_sequence (if_info);
3633 0 : if (!seq)
3634 : return false;
3635 :
3636 0 : emit_insn_before_setloc (seq, if_info->jump,
3637 0 : INSN_LOCATION (if_info->insn_a));
3638 : }
3639 0 : if_info->transform_name = "noce_try_bitop";
3640 0 : return true;
3641 : }
3642 :
3643 :
3644 : /* Similar to get_condition, only the resulting condition must be
3645 : valid at JUMP, instead of at EARLIEST.
3646 :
3647 : If THEN_ELSE_REVERSED is true, the fallthrough does not go to the
3648 : THEN block of the caller, and we have to reverse the condition. */
3649 :
3650 : static rtx
3651 4395850 : noce_get_condition (rtx_insn *jump, rtx_insn **earliest,
3652 : bool then_else_reversed)
3653 : {
3654 4395850 : rtx cond, set, tmp;
3655 4395850 : bool reverse;
3656 :
3657 4395850 : if (! any_condjump_p (jump))
3658 : return NULL_RTX;
3659 :
3660 4395850 : set = pc_set (jump);
3661 :
3662 : /* If this branches to JUMP_LABEL when the condition is false,
3663 : reverse the condition. */
3664 8791700 : reverse = (GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
3665 4395850 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (jump));
3666 :
3667 : /* We may have to reverse because the caller's if block is not canonical,
3668 : i.e. the THEN block isn't the fallthrough block for the TEST block
3669 : (see find_if_header). */
3670 4395850 : if (then_else_reversed)
3671 1926312 : reverse = !reverse;
3672 :
3673 : /* If the condition variable is a register and is MODE_INT, accept it. */
3674 :
3675 4395850 : cond = XEXP (SET_SRC (set), 0);
3676 4395850 : tmp = XEXP (cond, 0);
3677 4395109 : if (REG_P (tmp) && GET_MODE_CLASS (GET_MODE (tmp)) == MODE_INT
3678 4395890 : && (GET_MODE (tmp) != BImode
3679 0 : || !targetm.small_register_classes_for_mode_p (BImode)))
3680 : {
3681 40 : *earliest = jump;
3682 :
3683 40 : if (reverse)
3684 17 : cond = gen_rtx_fmt_ee (reverse_condition (GET_CODE (cond)),
3685 : GET_MODE (cond), tmp, XEXP (cond, 1));
3686 : return cond;
3687 : }
3688 :
3689 : /* Otherwise, fall back on canonicalize_condition to do the dirty
3690 : work of manipulating MODE_CC values and COMPARE rtx codes. */
3691 4395810 : tmp = canonicalize_condition (jump, cond, reverse, earliest,
3692 : NULL_RTX, have_cbranchcc4, true);
3693 :
3694 : /* We don't handle side-effects in the condition, like handling
3695 : REG_INC notes and making sure no duplicate conditions are emitted. */
3696 4395810 : if (tmp != NULL_RTX && side_effects_p (tmp))
3697 8380 : return NULL_RTX;
3698 :
3699 : return tmp;
3700 : }
3701 :
3702 : /* Return true if OP is ok for if-then-else processing. */
3703 :
3704 : static bool
3705 7895779 : noce_operand_ok (const_rtx op)
3706 : {
3707 7895779 : if (side_effects_p (op))
3708 : return false;
3709 :
3710 : /* We special-case memories, so handle any of them with
3711 : no address side effects. */
3712 7875056 : if (MEM_P (op))
3713 1588606 : return ! side_effects_p (XEXP (op, 0));
3714 :
3715 6286450 : return ! may_trap_p (op);
3716 : }
3717 :
3718 : /* Return true iff basic block TEST_BB is valid for noce if-conversion.
3719 : The condition used in this if-conversion is in COND.
3720 : In practice, check that TEST_BB ends with a single set
3721 : x := a and all previous computations
3722 : in TEST_BB don't produce any values that are live after TEST_BB.
3723 : In other words, all the insns in TEST_BB are there only
3724 : to compute a value for x. Add the rtx cost of the insns
3725 : in TEST_BB to COST. Record whether TEST_BB is a single simple
3726 : set instruction in SIMPLE_P. */
3727 :
3728 : static bool
3729 2187692 : bb_valid_for_noce_process_p (basic_block test_bb, rtx cond,
3730 : unsigned int *cost, bool *simple_p)
3731 : {
3732 2187692 : if (!test_bb)
3733 : return false;
3734 :
3735 2187692 : rtx_insn *last_insn = last_active_insn (test_bb, false);
3736 2187692 : rtx last_set = NULL_RTX;
3737 :
3738 2187692 : rtx cc = cc_in_cond (cond);
3739 :
3740 2187692 : if (!insn_valid_noce_process_p (last_insn, cc))
3741 : return false;
3742 :
3743 : /* Punt on blocks ending with asm goto or jumps with other side-effects,
3744 : last_active_insn ignores JUMP_INSNs. */
3745 1556121 : if (JUMP_P (BB_END (test_bb)) && !onlyjump_p (BB_END (test_bb)))
3746 : return false;
3747 :
3748 1556119 : last_set = single_set (last_insn);
3749 :
3750 1556119 : rtx x = SET_DEST (last_set);
3751 1556119 : rtx_insn *first_insn = first_active_insn (test_bb);
3752 1556119 : rtx first_set = single_set (first_insn);
3753 :
3754 1556119 : if (!first_set)
3755 : return false;
3756 :
3757 : /* We have a single simple set, that's okay. */
3758 1542131 : bool speed_p = optimize_bb_for_speed_p (test_bb);
3759 :
3760 1542131 : if (first_insn == last_insn)
3761 : {
3762 562452 : *simple_p = noce_operand_ok (SET_DEST (first_set));
3763 562452 : *cost += pattern_cost (first_set, speed_p);
3764 562452 : return *simple_p;
3765 : }
3766 :
3767 979679 : rtx_insn *prev_last_insn = PREV_INSN (last_insn);
3768 979679 : gcc_assert (prev_last_insn);
3769 :
3770 : /* For now, disallow setting x multiple times in test_bb. */
3771 979679 : if (REG_P (x) && reg_set_between_p (x, first_insn, prev_last_insn))
3772 : return false;
3773 :
3774 832556 : bitmap test_bb_temps = BITMAP_ALLOC (®_obstack);
3775 :
3776 : /* The regs that are live out of test_bb. */
3777 832556 : bitmap test_bb_live_out = df_get_live_out (test_bb);
3778 :
3779 832556 : int potential_cost = pattern_cost (last_set, speed_p);
3780 832556 : rtx_insn *insn;
3781 4582842 : FOR_BB_INSNS (test_bb, insn)
3782 : {
3783 4489559 : if (insn != last_insn)
3784 : {
3785 4396276 : if (!active_insn_p (insn))
3786 2908988 : continue;
3787 :
3788 1487288 : if (!insn_valid_noce_process_p (insn, cc))
3789 229557 : goto free_bitmap_and_fail;
3790 :
3791 1257731 : rtx sset = single_set (insn);
3792 1257731 : gcc_assert (sset);
3793 1257731 : rtx dest = SET_DEST (sset);
3794 1257731 : if (SUBREG_P (dest))
3795 18537 : dest = SUBREG_REG (dest);
3796 :
3797 1257731 : if (contains_mem_rtx_p (SET_SRC (sset))
3798 887959 : || !REG_P (dest)
3799 2017156 : || reg_overlap_mentioned_p (dest, cond))
3800 509716 : goto free_bitmap_and_fail;
3801 :
3802 748015 : potential_cost += pattern_cost (sset, speed_p);
3803 748015 : bitmap_set_bit (test_bb_temps, REGNO (dest));
3804 : }
3805 : }
3806 :
3807 : /* If any of the intermediate results in test_bb are live after test_bb
3808 : then fail. */
3809 93283 : if (bitmap_intersect_p (test_bb_live_out, test_bb_temps))
3810 62594 : goto free_bitmap_and_fail;
3811 :
3812 30689 : BITMAP_FREE (test_bb_temps);
3813 30689 : *cost += potential_cost;
3814 30689 : *simple_p = false;
3815 30689 : return true;
3816 :
3817 801867 : free_bitmap_and_fail:
3818 801867 : BITMAP_FREE (test_bb_temps);
3819 801867 : return false;
3820 : }
3821 :
3822 : /* Helper function to emit a cmov sequence encapsulated in
3823 : start_sequence () and end_sequence (). If NEED_CMOV is true
3824 : we call noce_emit_cmove to create a cmove sequence. Otherwise emit
3825 : a simple move. If successful, store the first instruction of the
3826 : sequence in TEMP_DEST and the sequence costs in SEQ_COST. */
3827 :
3828 : static rtx_insn*
3829 260676 : try_emit_cmove_seq (struct noce_if_info *if_info, rtx temp,
3830 : rtx cond, rtx new_val, rtx old_val, bool need_cmov,
3831 : unsigned *cost, rtx *temp_dest,
3832 : rtx cc_cmp = NULL, rtx rev_cc_cmp = NULL)
3833 : {
3834 260676 : rtx_insn *seq = NULL;
3835 260676 : *cost = 0;
3836 :
3837 260676 : rtx x = XEXP (cond, 0);
3838 260676 : rtx y = XEXP (cond, 1);
3839 260676 : rtx_code cond_code = GET_CODE (cond);
3840 :
3841 260676 : start_sequence ();
3842 :
3843 260676 : if (need_cmov)
3844 217111 : *temp_dest = noce_emit_cmove (if_info, temp, cond_code,
3845 : x, y, new_val, old_val, cc_cmp, rev_cc_cmp);
3846 : else
3847 : {
3848 43565 : *temp_dest = temp;
3849 43565 : if (if_info->then_else_reversed)
3850 1948 : noce_emit_move_insn (temp, old_val);
3851 : else
3852 41617 : noce_emit_move_insn (temp, new_val);
3853 : }
3854 :
3855 260676 : if (*temp_dest != NULL_RTX)
3856 : {
3857 254686 : seq = get_insns ();
3858 254686 : *cost = seq_cost (seq, if_info->speed_p);
3859 : }
3860 :
3861 260676 : end_sequence ();
3862 :
3863 260676 : return seq;
3864 : }
3865 :
3866 : /* We have something like:
3867 :
3868 : if (x > y)
3869 : { i = EXPR_A; j = EXPR_B; k = EXPR_C; }
3870 :
3871 : Make it:
3872 :
3873 : tmp_i = (x > y) ? EXPR_A : i;
3874 : tmp_j = (x > y) ? EXPR_B : j;
3875 : tmp_k = (x > y) ? EXPR_C : k;
3876 : i = tmp_i;
3877 : j = tmp_j;
3878 : k = tmp_k;
3879 :
3880 : Subsequent passes are expected to clean up the extra moves.
3881 :
3882 : Look for special cases such as writes to one register which are
3883 : read back in another SET, as might occur in a swap idiom or
3884 : similar.
3885 :
3886 : These look like:
3887 :
3888 : if (x > y)
3889 : i = a;
3890 : j = i;
3891 :
3892 : Which we want to rewrite to:
3893 :
3894 : tmp_i = (x > y) ? a : i;
3895 : tmp_j = (x > y) ? tmp_i : j;
3896 : i = tmp_i;
3897 : j = tmp_j;
3898 :
3899 : We can catch these when looking at (SET x y) by keeping a list of the
3900 : registers we would have targeted before if-conversion and looking back
3901 : through it for an overlap with Y. If we find one, we rewire the
3902 : conditional set to use the temporary we introduced earlier.
3903 :
3904 : IF_INFO contains the useful information about the block structure and
3905 : jump instructions. */
3906 :
3907 : static bool
3908 30029 : noce_convert_multiple_sets (struct noce_if_info *if_info)
3909 : {
3910 30029 : basic_block test_bb = if_info->test_bb;
3911 30029 : basic_block then_bb = if_info->then_bb;
3912 30029 : basic_block join_bb = if_info->join_bb;
3913 30029 : rtx_insn *jump = if_info->jump;
3914 30029 : rtx_insn *cond_earliest;
3915 30029 : rtx_insn *insn;
3916 :
3917 30029 : start_sequence ();
3918 :
3919 : /* Decompose the condition attached to the jump. */
3920 30029 : rtx cond = noce_get_condition (jump, &cond_earliest, false);
3921 30029 : rtx x = XEXP (cond, 0);
3922 30029 : rtx y = XEXP (cond, 1);
3923 :
3924 30029 : auto_delete_vec<noce_multiple_sets_info> insn_info;
3925 30029 : init_noce_multiple_sets_info (then_bb, insn_info);
3926 :
3927 30029 : int last_needs_comparison = -1;
3928 :
3929 30029 : bool use_cond_earliest = false;
3930 :
3931 30029 : bool ok = noce_convert_multiple_sets_1
3932 30029 : (if_info, insn_info, &last_needs_comparison, &use_cond_earliest);
3933 30029 : if (!ok)
3934 : return false;
3935 :
3936 : /* Always perform a second attempt that uses information gathered in the
3937 : first. At least we can omit creating temporaries until we definitely
3938 : need them. The sequence created in the second attempt is never worse
3939 : than the first. */
3940 :
3941 29033 : end_sequence ();
3942 29033 : start_sequence ();
3943 29033 : ok = noce_convert_multiple_sets_1
3944 29033 : (if_info, insn_info, &last_needs_comparison, &use_cond_earliest);
3945 :
3946 : /* Actually we should not fail anymore if we reached here,
3947 : but better still check. */
3948 29033 : if (!ok)
3949 : return false;
3950 :
3951 : /* We must have seen some sort of insn to insert, otherwise we were
3952 : given an empty BB to convert, and we can't handle that. */
3953 29033 : gcc_assert (!insn_info.is_empty ());
3954 :
3955 : /* Now fixup the assignments.
3956 : PR116405: Iterate in reverse order and keep track of the targets so that
3957 : a move does not overwrite a subsequent value when multiple instructions
3958 : have the same target. */
3959 29033 : unsigned i;
3960 29033 : noce_multiple_sets_info *info;
3961 29033 : bitmap set_targets = BITMAP_ALLOC (®_obstack);
3962 105059 : FOR_EACH_VEC_ELT_REVERSE (insn_info, i, info)
3963 : {
3964 76026 : gcc_checking_assert (REG_P (info->target));
3965 :
3966 76026 : if (info->target != info->temporary
3967 76026 : && !bitmap_bit_p (set_targets, REGNO (info->target)))
3968 2718 : noce_emit_move_insn (info->target, info->temporary);
3969 :
3970 76026 : bitmap_set_bit (set_targets, REGNO (info->target));
3971 : }
3972 29033 : BITMAP_FREE (set_targets);
3973 :
3974 : /* Actually emit the sequence if it isn't too expensive. */
3975 29033 : rtx_insn *seq = get_insns ();
3976 :
3977 : /* If the created sequence does not use cond_earliest (but the jump
3978 : does) add its cost to the original_cost before comparing costs. */
3979 29033 : unsigned int original_cost = if_info->original_cost;
3980 29033 : if (if_info->jump != if_info->cond_earliest && !use_cond_earliest)
3981 17822 : if_info->original_cost += insn_cost (if_info->cond_earliest,
3982 : if_info->speed_p);
3983 :
3984 29033 : if (!targetm.noce_conversion_profitable_p (seq, if_info))
3985 : {
3986 3602 : end_sequence ();
3987 3602 : return false;
3988 : }
3989 :
3990 : /* Restore the original cost in case we do not succeed below. */
3991 25431 : if_info->original_cost = original_cost;
3992 :
3993 146938 : for (insn = seq; insn; insn = NEXT_INSN (insn))
3994 121507 : set_used_flags (insn);
3995 :
3996 : /* Mark all our temporaries and targets as used. */
3997 87460 : for (unsigned i = 0; i < insn_info.length (); i++)
3998 : {
3999 62029 : set_used_flags (insn_info[i]->temporary);
4000 62029 : set_used_flags (insn_info[i]->target);
4001 : }
4002 :
4003 25431 : set_used_flags (cond);
4004 25431 : set_used_flags (x);
4005 25431 : set_used_flags (y);
4006 :
4007 25431 : unshare_all_rtl_in_chain (seq);
4008 25431 : end_sequence ();
4009 :
4010 25431 : if (!seq)
4011 : return false;
4012 :
4013 146751 : for (insn = seq; insn; insn = NEXT_INSN (insn))
4014 121367 : if (JUMP_P (insn) || CALL_P (insn)
4015 121367 : || recog_memoized (insn) == -1)
4016 : return false;
4017 :
4018 25384 : emit_insn_before_setloc (seq, if_info->jump,
4019 25384 : INSN_LOCATION (insn_info.last ()->unmodified_insn));
4020 :
4021 : /* Clean up THEN_BB and the edges in and out of it. */
4022 25384 : remove_edge (find_edge (test_bb, join_bb));
4023 25384 : remove_edge (find_edge (then_bb, join_bb));
4024 25384 : redirect_edge_and_branch_force (single_succ_edge (test_bb), join_bb);
4025 25384 : delete_basic_block (then_bb);
4026 25384 : num_true_changes++;
4027 :
4028 : /* Maybe merge blocks now the jump is simple enough. */
4029 25384 : if (can_merge_blocks_p (test_bb, join_bb))
4030 : {
4031 16713 : merge_blocks (test_bb, join_bb);
4032 16713 : num_true_changes++;
4033 : }
4034 :
4035 25384 : num_updated_if_blocks++;
4036 25384 : if_info->transform_name = "noce_convert_multiple_sets";
4037 25384 : return true;
4038 30029 : }
4039 :
4040 : /* This goes through all relevant insns of IF_INFO->then_bb and tries to create
4041 : conditional moves. Information for the insns is kept in INSN_INFO. */
4042 :
4043 : static bool
4044 59062 : noce_convert_multiple_sets_1 (struct noce_if_info *if_info,
4045 : auto_delete_vec<noce_multiple_sets_info> &insn_info,
4046 : int *last_needs_comparison,
4047 : bool *use_cond_earliest)
4048 : {
4049 59062 : basic_block then_bb = if_info->then_bb;
4050 59062 : rtx_insn *jump = if_info->jump;
4051 59062 : rtx_insn *cond_earliest;
4052 :
4053 : /* Decompose the condition attached to the jump. */
4054 59062 : rtx cond = noce_get_condition (jump, &cond_earliest, false);
4055 :
4056 59062 : rtx cc_cmp = cond_exec_get_condition (jump);
4057 59062 : if (cc_cmp)
4058 59062 : cc_cmp = copy_rtx (cc_cmp);
4059 59062 : rtx rev_cc_cmp = cond_exec_get_condition (jump, /* get_reversed */ true);
4060 59062 : if (rev_cc_cmp)
4061 59062 : rev_cc_cmp = copy_rtx (rev_cc_cmp);
4062 :
4063 59062 : rtx_insn *insn;
4064 59062 : int count = 0;
4065 59062 : bool second_try = *last_needs_comparison != -1;
4066 59062 : *use_cond_earliest = false;
4067 :
4068 349763 : FOR_BB_INSNS (then_bb, insn)
4069 : {
4070 : /* Skip over non-insns. */
4071 291697 : if (!active_insn_p (insn))
4072 137200 : continue;
4073 :
4074 154497 : noce_multiple_sets_info *info = insn_info[count];
4075 :
4076 154497 : rtx set = single_set (insn);
4077 154497 : gcc_checking_assert (set);
4078 :
4079 154497 : rtx target = SET_DEST (set);
4080 154497 : rtx temp;
4081 :
4082 154497 : rtx new_val = SET_SRC (set);
4083 :
4084 154497 : int i, ii;
4085 184400 : FOR_EACH_VEC_ELT (info->rewired_src, i, ii)
4086 29903 : new_val = simplify_replace_rtx (new_val, insn_info[ii]->target,
4087 29903 : insn_info[ii]->temporary);
4088 :
4089 154497 : rtx old_val = target;
4090 :
4091 : /* As we are transforming
4092 : if (x > y)
4093 : {
4094 : a = b;
4095 : c = d;
4096 : }
4097 : into
4098 : a = (x > y) ...
4099 : c = (x > y) ...
4100 :
4101 : we potentially check x > y before every set.
4102 : Even though the check might be removed by subsequent passes, this means
4103 : that we cannot transform
4104 : if (x > y)
4105 : {
4106 : x = y;
4107 : ...
4108 : }
4109 : into
4110 : x = (x > y) ...
4111 : ...
4112 : since this would invalidate x and the following to-be-removed checks.
4113 : Therefore we introduce a temporary every time we are about to
4114 : overwrite a variable used in the check. Costing of a sequence with
4115 : these is going to be inaccurate so only use temporaries when
4116 : needed.
4117 :
4118 : If performing a second try, we know how many insns require a
4119 : temporary. For the last of these, we can omit creating one. */
4120 154497 : if (reg_overlap_mentioned_p (target, cond)
4121 154497 : && (!second_try || count < *last_needs_comparison))
4122 20722 : temp = gen_reg_rtx (GET_MODE (target));
4123 : else
4124 : temp = target;
4125 :
4126 : /* We have identified swap-style idioms before. A normal
4127 : set will need to be a cmov while the first instruction of a swap-style
4128 : idiom can be a regular move. This helps with costing. */
4129 154497 : bool need_cmov = info->need_cmov;
4130 :
4131 : /* If we had a non-canonical conditional jump (i.e. one where
4132 : the fallthrough is to the "else" case) we need to reverse
4133 : the conditional select. */
4134 154497 : if (if_info->then_else_reversed)
4135 51636 : std::swap (old_val, new_val);
4136 :
4137 : /* Try emitting a conditional move passing the backend the
4138 : canonicalized comparison. The backend is then able to
4139 : recognize expressions like
4140 :
4141 : if (x > y)
4142 : y = x;
4143 :
4144 : as min/max and emit an insn, accordingly. */
4145 154497 : unsigned cost1 = 0, cost2 = 0;
4146 154497 : rtx_insn *seq, *seq1, *seq2 = NULL;
4147 154497 : rtx temp_dest = NULL_RTX, temp_dest1 = NULL_RTX, temp_dest2 = NULL_RTX;
4148 154497 : bool read_comparison = false;
4149 :
4150 154497 : seq1 = try_emit_cmove_seq (if_info, temp, cond,
4151 : new_val, old_val, need_cmov,
4152 : &cost1, &temp_dest1);
4153 :
4154 : /* Here, we try to pass the backend a non-canonicalized cc comparison
4155 : as well. This allows the backend to emit a cmov directly without
4156 : creating an additional compare for each. If successful, costing
4157 : is easier and this sequence is usually preferred. */
4158 154497 : if (cc_cmp)
4159 : {
4160 106179 : seq2 = try_emit_cmove_seq (if_info, temp, cond,
4161 : new_val, old_val, need_cmov,
4162 : &cost2, &temp_dest2, cc_cmp, rev_cc_cmp);
4163 :
4164 : /* The if_then_else in SEQ2 may be affected when cc_cmp/rev_cc_cmp is
4165 : clobbered. We can't safely use the sequence in this case. */
4166 293492 : for (rtx_insn *iter = seq2; iter; iter = NEXT_INSN (iter))
4167 118597 : if (modified_in_p (cc_cmp, iter)
4168 118597 : || (rev_cc_cmp && modified_in_p (rev_cc_cmp, iter)))
4169 : {
4170 : seq2 = NULL;
4171 : break;
4172 : }
4173 : }
4174 :
4175 : /* The backend might have created a sequence that uses the
4176 : condition as a value. Check this. */
4177 :
4178 : /* We cannot handle anything more complex than a reg or constant. */
4179 154497 : if (!REG_P (XEXP (cond, 0)) && !CONSTANT_P (XEXP (cond, 0)))
4180 154497 : read_comparison = true;
4181 :
4182 154497 : if (!REG_P (XEXP (cond, 1)) && !CONSTANT_P (XEXP (cond, 1)))
4183 154497 : read_comparison = true;
4184 :
4185 154497 : rtx_insn *walk = seq2;
4186 154497 : int if_then_else_count = 0;
4187 232018 : while (walk && !read_comparison)
4188 : {
4189 77521 : rtx exprs_to_check[2];
4190 77521 : unsigned int exprs_count = 0;
4191 :
4192 77521 : rtx set = single_set (walk);
4193 77521 : if (set && XEXP (set, 1)
4194 77521 : && GET_CODE (XEXP (set, 1)) == IF_THEN_ELSE)
4195 : {
4196 : /* We assume that this is the cmove created by the backend that
4197 : naturally uses the condition. */
4198 49860 : exprs_to_check[exprs_count++] = XEXP (XEXP (set, 1), 1);
4199 49860 : exprs_to_check[exprs_count++] = XEXP (XEXP (set, 1), 2);
4200 49860 : if_then_else_count++;
4201 : }
4202 27661 : else if (NONDEBUG_INSN_P (walk))
4203 27661 : exprs_to_check[exprs_count++] = PATTERN (walk);
4204 :
4205 : /* Bail if we get more than one if_then_else because the assumption
4206 : above may be incorrect. */
4207 77521 : if (if_then_else_count > 1)
4208 : {
4209 0 : read_comparison = true;
4210 0 : break;
4211 : }
4212 :
4213 204902 : for (unsigned int i = 0; i < exprs_count; i++)
4214 : {
4215 127381 : subrtx_iterator::array_type array;
4216 281380 : FOR_EACH_SUBRTX (iter, array, exprs_to_check[i], NONCONST)
4217 184242 : if (*iter != NULL_RTX
4218 184242 : && (reg_overlap_mentioned_p (XEXP (cond, 0), *iter)
4219 166683 : || reg_overlap_mentioned_p (XEXP (cond, 1), *iter)))
4220 : {
4221 : read_comparison = true;
4222 : break;
4223 : }
4224 127381 : }
4225 :
4226 77521 : walk = NEXT_INSN (walk);
4227 : }
4228 :
4229 : /* Check which version is less expensive. */
4230 154497 : if (seq1 != NULL_RTX && (cost1 <= cost2 || seq2 == NULL_RTX))
4231 : {
4232 102503 : seq = seq1;
4233 102503 : temp_dest = temp_dest1;
4234 102503 : if (!second_try)
4235 52024 : *last_needs_comparison = count;
4236 : }
4237 1077 : else if (seq2 != NULL_RTX)
4238 : {
4239 51125 : seq = seq2;
4240 51125 : temp_dest = temp_dest2;
4241 51125 : if (!second_try && read_comparison)
4242 8452 : *last_needs_comparison = count;
4243 51125 : *use_cond_earliest = true;
4244 : }
4245 : else
4246 : {
4247 : /* Nothing worked, bail out. */
4248 869 : end_sequence ();
4249 996 : return false;
4250 : }
4251 :
4252 : /* Although we use temporaries if there is register overlap of COND and
4253 : TARGET, it is possible that SEQ modifies COND anyway. For example,
4254 : COND may use the flags register and if INSN clobbers flags then
4255 : we may be unable to emit a valid sequence (e.g. in x86 that would
4256 : require saving and restoring the flags register). */
4257 103149 : if (!second_try)
4258 233501 : for (rtx_insn *iter = seq; iter; iter = NEXT_INSN (iter))
4259 156026 : if (modified_in_p (cond, iter))
4260 : {
4261 127 : end_sequence ();
4262 127 : return false;
4263 : }
4264 :
4265 153501 : if (cc_cmp && seq == seq1)
4266 : {
4267 : /* Check if SEQ can clobber registers mentioned in cc_cmp/rev_cc_cmp.
4268 : If yes, we need to use only SEQ1 from that point on.
4269 : Only check when we use SEQ1 since we have already tested SEQ2. */
4270 74879 : for (rtx_insn *iter = seq; iter; iter = NEXT_INSN (iter))
4271 57435 : if (modified_in_p (cc_cmp, iter)
4272 57435 : || (rev_cc_cmp && modified_in_p (rev_cc_cmp, iter)))
4273 : {
4274 : cc_cmp = NULL_RTX;
4275 : rev_cc_cmp = NULL_RTX;
4276 : break;
4277 : }
4278 : }
4279 :
4280 : /* End the sub sequence and emit to the main sequence. */
4281 153501 : emit_insn (seq);
4282 :
4283 : /* Bookkeeping. */
4284 153501 : count++;
4285 :
4286 153501 : info->target = target;
4287 153501 : info->temporary = temp_dest;
4288 153501 : info->unmodified_insn = insn;
4289 : }
4290 :
4291 : /* Even if we did not actually need the comparison, we want to make sure
4292 : to try a second time in order to get rid of the temporaries. */
4293 58066 : if (*last_needs_comparison == -1)
4294 979 : *last_needs_comparison = 0;
4295 :
4296 : return true;
4297 : }
4298 :
4299 : /* Find local swap-style idioms in BB and mark the first insn (1)
4300 : that is only a temporary as not needing a conditional move as
4301 : it is going to be dead afterwards anyway.
4302 :
4303 : (1) int tmp = a;
4304 : a = b;
4305 : b = tmp;
4306 :
4307 : ifcvt
4308 : -->
4309 :
4310 : tmp = a;
4311 : a = cond ? b : a_old;
4312 : b = cond ? tmp : b_old;
4313 :
4314 : Additionally, store the index of insns like (2) when a subsequent
4315 : SET reads from their destination.
4316 :
4317 : (2) int c = a;
4318 : int d = c;
4319 :
4320 : ifcvt
4321 : -->
4322 :
4323 : c = cond ? a : c_old;
4324 : d = cond ? d : c; // Need to use c rather than c_old here.
4325 : */
4326 :
4327 : static void
4328 30029 : init_noce_multiple_sets_info (basic_block bb,
4329 : auto_delete_vec<noce_multiple_sets_info> &insn_info)
4330 : {
4331 30029 : rtx_insn *insn;
4332 30029 : int count = 0;
4333 30029 : auto_vec<rtx> dests;
4334 30029 : bitmap bb_live_out = df_get_live_out (bb);
4335 :
4336 : /* Iterate over all SETs, storing the destinations in DEST.
4337 : - If we encounter a previously changed register,
4338 : rewire the read to the original source.
4339 : - If we encounter a SET that writes to a destination
4340 : that is not live after this block then the register
4341 : does not need to be moved conditionally. */
4342 180879 : FOR_BB_INSNS (bb, insn)
4343 : {
4344 150850 : if (!active_insn_p (insn))
4345 71980 : continue;
4346 :
4347 78870 : noce_multiple_sets_info *info = new noce_multiple_sets_info;
4348 78870 : info->target = NULL_RTX;
4349 78870 : info->temporary = NULL_RTX;
4350 78870 : info->unmodified_insn = NULL;
4351 78870 : insn_info.safe_push (info);
4352 :
4353 78870 : rtx set = single_set (insn);
4354 78870 : gcc_checking_assert (set);
4355 :
4356 78870 : rtx src = SET_SRC (set);
4357 78870 : rtx dest = SET_DEST (set);
4358 :
4359 78870 : gcc_checking_assert (REG_P (dest) && !HARD_REGISTER_P (dest));
4360 78870 : info->need_cmov = bitmap_bit_p (bb_live_out, REGNO (dest));
4361 :
4362 : /* Check if the current SET's source mentions any previously seen
4363 : destination. Keep only the newest definition of each pseudo register.
4364 : This is quadratic but the number of insns in BB
4365 : is bounded by PARAM_MAX_RTL_IF_CONVERSION_INSNS. */
4366 78870 : auto_bitmap rewired_regs;
4367 159176 : for (int i = count - 1; i >= 0; --i)
4368 80306 : if (reg_mentioned_p (dests[i], src)
4369 80306 : && bitmap_set_bit (rewired_regs, REGNO (dests[i])))
4370 15718 : insn_info[count]->rewired_src.safe_push (i);
4371 :
4372 78870 : dests.safe_push (dest);
4373 78870 : count++;
4374 78870 : }
4375 30029 : }
4376 :
4377 : /* Return true iff basic block TEST_BB is suitable for conversion to a
4378 : series of conditional moves. Also check that we have more than one
4379 : set (other routines can handle a single set better than we would),
4380 : and fewer than PARAM_MAX_RTL_IF_CONVERSION_INSNS sets. While going
4381 : through the insns store the sum of their potential costs in COST. */
4382 :
4383 : static bool
4384 1088090 : bb_ok_for_noce_convert_multiple_sets (basic_block test_bb, unsigned *cost)
4385 : {
4386 1088090 : rtx_insn *insn;
4387 1088090 : unsigned count = 0;
4388 1088090 : unsigned param = param_max_rtl_if_conversion_insns;
4389 1088090 : bool speed_p = optimize_bb_for_speed_p (test_bb);
4390 1088090 : unsigned potential_cost = 0;
4391 :
4392 8458743 : FOR_BB_INSNS (test_bb, insn)
4393 : {
4394 : /* Skip over notes etc. */
4395 8341322 : if (!active_insn_p (insn))
4396 6975323 : continue;
4397 :
4398 : /* We only handle SET insns. */
4399 1365999 : rtx set = single_set (insn);
4400 1365999 : if (set == NULL_RTX)
4401 : return false;
4402 :
4403 1302970 : rtx dest = SET_DEST (set);
4404 1302970 : rtx src = SET_SRC (set);
4405 :
4406 : /* Dependency rewiring is keyed by register number, so restrict
4407 : destinations to pseudos. Hard-register definitions can overlap
4408 : without having the same mode. Do not handle anything involving
4409 : memory loads/stores since it might violate data-race-freedom
4410 : guarantees. Make sure we can force SRC to a register as that may
4411 : be needed in try_emit_cmove_seq. */
4412 1138748 : if (!REG_P (dest) || HARD_REGISTER_P (dest)
4413 814470 : || contains_mem_rtx_p (src)
4414 1733303 : || !noce_can_force_operand (src))
4415 : return false;
4416 :
4417 : /* Destination and source must be appropriate. */
4418 422128 : if (!noce_operand_ok (dest) || !noce_operand_ok (src))
4419 : return false;
4420 :
4421 : /* We must be able to conditionally move in this mode. */
4422 416040 : if (!can_conditionally_move_p (GET_MODE (dest)))
4423 : return false;
4424 :
4425 395330 : potential_cost += insn_cost (insn, speed_p);
4426 :
4427 395330 : count++;
4428 : }
4429 :
4430 117421 : *cost += potential_cost;
4431 :
4432 : /* If we would only put out one conditional move, the other strategies
4433 : this pass tries are better optimized and will be more appropriate.
4434 : Some targets want to strictly limit the number of conditional moves
4435 : that are emitted, they set this through PARAM, we need to respect
4436 : that. */
4437 117421 : return count > 1 && count <= param;
4438 : }
4439 :
4440 : /* Compute average of two given costs weighted by relative probabilities
4441 : of respective basic blocks in an IF-THEN-ELSE. E is the IF-THEN edge.
4442 : With P as the probability to take the IF-THEN branch, return
4443 : P * THEN_COST + (1 - P) * ELSE_COST. Evaluate this as
4444 : ELSE_COST + P * (THEN_COST - ELSE_COST) when THEN_COST >= ELSE_COST, and
4445 : THEN_COST + (1 - P) * (ELSE_COST - THEN_COST) otherwise. Both forms pass
4446 : a nonnegative value to profile_probability::apply and make its rounding
4447 : independent of the CFG arm order. */
4448 : static unsigned
4449 259934 : average_cost (unsigned then_cost, unsigned else_cost, edge e)
4450 : {
4451 259934 : if (then_cost < else_cost)
4452 6651 : return then_cost
4453 6651 : + e->probability.invert ().apply ((gcov_type) else_cost - then_cost);
4454 :
4455 253283 : return else_cost
4456 253283 : + e->probability.apply ((gcov_type) then_cost - else_cost);
4457 : }
4458 :
4459 : /* Given a simple IF-THEN-JOIN or IF-THEN-ELSE-JOIN block, attempt to convert
4460 : it without using conditional execution. Return TRUE if we were successful
4461 : at converting the block. */
4462 :
4463 : static bool
4464 1922514 : noce_process_if_block (struct noce_if_info *if_info)
4465 : {
4466 1922514 : basic_block test_bb = if_info->test_bb; /* test block */
4467 1922514 : basic_block then_bb = if_info->then_bb; /* THEN */
4468 1922514 : basic_block else_bb = if_info->else_bb; /* ELSE or NULL */
4469 1922514 : basic_block join_bb = if_info->join_bb; /* JOIN */
4470 1922514 : rtx_insn *jump = if_info->jump;
4471 1922514 : rtx cond = if_info->cond;
4472 1922514 : rtx_insn *insn_a, *insn_b;
4473 1922514 : rtx set_a, set_b;
4474 1922514 : rtx orig_x, x, a, b;
4475 :
4476 : /* We're looking for patterns of the form
4477 :
4478 : (1) if (...) x = a; else x = b;
4479 : (2) x = b; if (...) x = a;
4480 : (3) if (...) x = a; // as if with an initial x = x.
4481 : (4) if (...) { x = a; y = b; z = c; } // Like 3, for multiple SETS.
4482 : The later patterns require jumps to be more expensive.
4483 : For the if (...) x = a; else x = b; case we allow multiple insns
4484 : inside the then and else blocks as long as their only effect is
4485 : to calculate a value for x.
4486 : ??? For future expansion, further expand the "multiple X" rules. */
4487 :
4488 : /* First look for multiple SETS.
4489 : The original costs already include costs for the jump insn as well
4490 : as for a CC comparison if there is any.
4491 : If a target re-uses the existing CC comparison we keep track of that
4492 : and add the costs before default noce_conversion_profitable_p. */
4493 :
4494 1922514 : unsigned potential_cost = if_info->original_cost;
4495 1922514 : unsigned old_cost = if_info->original_cost;
4496 1922514 : if (!else_bb
4497 : && HAVE_conditional_move
4498 1922514 : && bb_ok_for_noce_convert_multiple_sets (then_bb, &potential_cost))
4499 : {
4500 : /* Temporarily set the original costs to what we estimated so
4501 : we can determine if the transformation is worth it. */
4502 30029 : if_info->original_cost = potential_cost;
4503 30029 : if (noce_convert_multiple_sets (if_info))
4504 : {
4505 25384 : if (dump_file && if_info->transform_name)
4506 1 : fprintf (dump_file, "if-conversion succeeded through %s\n",
4507 : if_info->transform_name);
4508 : return true;
4509 : }
4510 :
4511 : /* Restore the original costs. */
4512 4645 : if_info->original_cost = old_cost;
4513 : }
4514 :
4515 1897130 : bool speed_p = optimize_bb_for_speed_p (test_bb);
4516 1897130 : unsigned int then_cost = 0, else_cost = 0;
4517 1897130 : if (!bb_valid_for_noce_process_p (then_bb, cond, &then_cost,
4518 : &if_info->then_simple))
4519 : return false;
4520 :
4521 488697 : if (else_bb
4522 488697 : && !bb_valid_for_noce_process_p (else_bb, cond, &else_cost,
4523 : &if_info->else_simple))
4524 : return false;
4525 :
4526 302579 : if (speed_p)
4527 259934 : if_info->original_cost += average_cost (then_cost, else_cost,
4528 : find_edge (test_bb, then_bb));
4529 : else
4530 42645 : if_info->original_cost += then_cost + else_cost;
4531 :
4532 302579 : insn_a = last_active_insn (then_bb, false);
4533 302579 : set_a = single_set (insn_a);
4534 302579 : gcc_assert (set_a);
4535 :
4536 302579 : x = SET_DEST (set_a);
4537 302579 : a = SET_SRC (set_a);
4538 :
4539 : /* Look for the other potential set. Make sure we've got equivalent
4540 : destinations. */
4541 : /* ??? This is overconservative. Storing to two different mems is
4542 : as easy as conditionally computing the address. Storing to a
4543 : single mem merely requires a scratch memory to use as one of the
4544 : destination addresses; often the memory immediately below the
4545 : stack pointer is available for this. */
4546 302579 : set_b = NULL_RTX;
4547 302579 : if (else_bb)
4548 : {
4549 104444 : insn_b = last_active_insn (else_bb, false);
4550 104444 : set_b = single_set (insn_b);
4551 104444 : gcc_assert (set_b);
4552 :
4553 104444 : if (!rtx_interchangeable_p (x, SET_DEST (set_b)))
4554 : return false;
4555 : }
4556 : else
4557 : {
4558 198135 : insn_b = if_info->cond_earliest;
4559 536691 : do
4560 536691 : insn_b = prev_nonnote_nondebug_insn (insn_b);
4561 : while (insn_b
4562 533991 : && (BLOCK_FOR_INSN (insn_b)
4563 533991 : == BLOCK_FOR_INSN (if_info->cond_earliest))
4564 1183675 : && !modified_in_p (x, insn_b));
4565 :
4566 : /* We're going to be moving the evaluation of B down from above
4567 : COND_EARLIEST to JUMP. Make sure the relevant data is still
4568 : intact. */
4569 198135 : if (! insn_b
4570 195435 : || BLOCK_FOR_INSN (insn_b) != BLOCK_FOR_INSN (if_info->cond_earliest)
4571 110293 : || !NONJUMP_INSN_P (insn_b)
4572 105099 : || (set_b = single_set (insn_b)) == NULL_RTX
4573 103657 : || ! rtx_interchangeable_p (x, SET_DEST (set_b))
4574 89254 : || ! noce_operand_ok (SET_SRC (set_b))
4575 87279 : || reg_overlap_mentioned_p (x, SET_SRC (set_b))
4576 85533 : || modified_between_p (SET_SRC (set_b), insn_b, jump)
4577 : /* Avoid extending the lifetime of hard registers on small
4578 : register class machines. */
4579 83318 : || (REG_P (SET_SRC (set_b))
4580 20194 : && HARD_REGISTER_P (SET_SRC (set_b))
4581 3656 : && targetm.small_register_classes_for_mode_p
4582 3656 : (GET_MODE (SET_SRC (set_b))))
4583 : /* Likewise with X. In particular this can happen when
4584 : noce_get_condition looks farther back in the instruction
4585 : stream than one might expect. */
4586 79662 : || reg_overlap_mentioned_p (x, cond)
4587 66043 : || reg_overlap_mentioned_p (x, a)
4588 258512 : || modified_between_p (x, insn_b, jump))
4589 : {
4590 : insn_b = NULL;
4591 : set_b = NULL_RTX;
4592 : }
4593 : }
4594 :
4595 : /* If x has side effects then only the if-then-else form is safe to
4596 : convert. But even in that case we would need to restore any notes
4597 : (such as REG_INC) at then end. That can be tricky if
4598 : noce_emit_move_insn expands to more than one insn, so disable the
4599 : optimization entirely for now if there are side effects. */
4600 291424 : if (side_effects_p (x))
4601 : return false;
4602 :
4603 291424 : b = (set_b ? SET_SRC (set_b) : x);
4604 :
4605 : /* Only operate on register destinations, and even then avoid extending
4606 : the lifetime of hard registers on small register class machines. */
4607 291424 : orig_x = x;
4608 291424 : if_info->orig_x = orig_x;
4609 291424 : if (!REG_P (x)
4610 291424 : || (HARD_REGISTER_P (x)
4611 2 : && targetm.small_register_classes_for_mode_p (GET_MODE (x))))
4612 : {
4613 66775 : if (GET_MODE (x) == BLKmode)
4614 : return false;
4615 :
4616 66536 : if (GET_CODE (x) == ZERO_EXTRACT
4617 1 : && (!CONST_INT_P (XEXP (x, 1))
4618 1 : || !CONST_INT_P (XEXP (x, 2))))
4619 : return false;
4620 :
4621 66536 : x = gen_reg_rtx (GET_MODE (GET_CODE (x) == STRICT_LOW_PART
4622 : ? XEXP (x, 0) : x));
4623 : }
4624 :
4625 : /* Don't operate on sources that may trap or are volatile. */
4626 291185 : if (! noce_operand_ok (a) || ! noce_operand_ok (b))
4627 : return false;
4628 :
4629 318078 : retry:
4630 : /* Set up the info block for our subroutines. */
4631 318078 : if_info->insn_a = insn_a;
4632 318078 : if_info->insn_b = insn_b;
4633 318078 : if_info->x = x;
4634 318078 : if_info->a = a;
4635 318078 : if_info->b = b;
4636 :
4637 : /* Try optimizations in some approximation of a useful order. */
4638 : /* ??? Should first look to see if X is live incoming at all. If it
4639 : isn't, we don't need anything but an unconditional set. */
4640 :
4641 : /* Look and see if A and B are really the same. Avoid creating silly
4642 : cmove constructs that no one will fix up later. */
4643 318078 : if (noce_simple_bbs (if_info)
4644 295983 : && rtx_interchangeable_p (a, b))
4645 : {
4646 : /* If we have an INSN_B, we don't have to create any new rtl. Just
4647 : move the instruction that we already have. If we don't have an
4648 : INSN_B, that means that A == X, and we've got a noop move. In
4649 : that case don't do anything and let the code below delete INSN_A. */
4650 377 : if (insn_b && else_bb)
4651 : {
4652 368 : rtx note;
4653 :
4654 368 : if (else_bb && insn_b == BB_END (else_bb))
4655 257 : BB_END (else_bb) = PREV_INSN (insn_b);
4656 368 : reorder_insns (insn_b, insn_b, PREV_INSN (jump));
4657 :
4658 : /* If there was a REG_EQUAL note, delete it since it may have been
4659 : true due to this insn being after a jump. */
4660 368 : if ((note = find_reg_note (insn_b, REG_EQUAL, NULL_RTX)) != 0)
4661 0 : remove_note (insn_b, note);
4662 :
4663 377 : insn_b = NULL;
4664 : }
4665 : /* If we have "x = b; if (...) x = a;", and x has side-effects, then
4666 : x must be executed twice. */
4667 9 : else if (insn_b && side_effects_p (orig_x))
4668 : return false;
4669 :
4670 377 : x = orig_x;
4671 377 : goto success;
4672 : }
4673 :
4674 317701 : if (!set_b && MEM_P (orig_x))
4675 : /* We want to avoid store speculation to avoid cases like
4676 : if (pthread_mutex_trylock(mutex))
4677 : ++global_variable;
4678 : Rather than go to much effort here, we rely on the SSA optimizers,
4679 : which do a good enough job these days. */
4680 : return false;
4681 :
4682 252809 : if (noce_try_move (if_info))
4683 0 : goto success;
4684 252809 : if (noce_try_ifelse_collapse (if_info))
4685 17791 : goto success;
4686 235018 : if (noce_try_store_flag (if_info))
4687 25925 : goto success;
4688 209093 : if (noce_try_bitop (if_info))
4689 0 : goto success;
4690 209093 : if (noce_try_minmax (if_info))
4691 73 : goto success;
4692 209020 : if (noce_try_abs (if_info))
4693 24 : goto success;
4694 208996 : if (noce_try_inverse_constants (if_info))
4695 0 : goto success;
4696 208996 : if (!targetm.have_conditional_execution ()
4697 208996 : && noce_try_store_flag_constants (if_info))
4698 3157 : goto success;
4699 205839 : if (!targetm.have_conditional_execution ()
4700 205839 : && noce_try_shifted_store_flag (if_info))
4701 4769 : goto success;
4702 201070 : if (noce_try_sign_bit_splat (if_info))
4703 175 : goto success;
4704 200895 : if (!targetm.have_conditional_execution ()
4705 200895 : && noce_try_store_flag_logical (if_info))
4706 0 : goto success;
4707 200895 : if (HAVE_conditional_move
4708 200895 : && noce_try_cmove (if_info))
4709 42603 : goto success;
4710 158292 : if (! targetm.have_conditional_execution ())
4711 : {
4712 158292 : if (noce_try_addcc (if_info))
4713 1514 : goto success;
4714 156778 : if (noce_try_store_flag_mask (if_info))
4715 11 : goto success;
4716 156767 : if (HAVE_conditional_move
4717 156767 : && noce_try_cond_arith (if_info))
4718 10382 : goto success;
4719 146385 : if (HAVE_conditional_move
4720 146385 : && noce_try_cmove_arith (if_info))
4721 35085 : goto success;
4722 111300 : if (noce_try_sign_mask (if_info))
4723 0 : goto success;
4724 : }
4725 :
4726 111300 : if (!else_bb && set_b)
4727 : {
4728 26893 : insn_b = NULL;
4729 26893 : set_b = NULL_RTX;
4730 26893 : b = orig_x;
4731 26893 : goto retry;
4732 : }
4733 :
4734 : return false;
4735 :
4736 141886 : success:
4737 141886 : if (dump_file && if_info->transform_name)
4738 7 : fprintf (dump_file, "if-conversion succeeded through %s\n",
4739 : if_info->transform_name);
4740 :
4741 : /* If we used a temporary, fix it up now. */
4742 141886 : if (orig_x != x)
4743 : {
4744 699 : rtx_insn *seq;
4745 :
4746 699 : start_sequence ();
4747 699 : noce_emit_move_insn (orig_x, x);
4748 699 : seq = get_insns ();
4749 699 : set_used_flags (orig_x);
4750 699 : unshare_all_rtl_in_chain (seq);
4751 699 : end_sequence ();
4752 :
4753 699 : emit_insn_before_setloc (seq, BB_END (test_bb), INSN_LOCATION (insn_a));
4754 : }
4755 :
4756 : /* The original THEN and ELSE blocks may now be removed. The test block
4757 : must now jump to the join block. If the test block and the join block
4758 : can be merged, do so. */
4759 141886 : if (else_bb)
4760 : {
4761 64690 : delete_basic_block (else_bb);
4762 64690 : num_true_changes++;
4763 : }
4764 : else
4765 77196 : remove_edge (find_edge (test_bb, join_bb));
4766 :
4767 141886 : remove_edge (find_edge (then_bb, join_bb));
4768 141886 : redirect_edge_and_branch_force (single_succ_edge (test_bb), join_bb);
4769 141886 : delete_basic_block (then_bb);
4770 141886 : num_true_changes++;
4771 :
4772 141886 : if (can_merge_blocks_p (test_bb, join_bb))
4773 : {
4774 111723 : merge_blocks (test_bb, join_bb);
4775 111723 : num_true_changes++;
4776 : }
4777 :
4778 141886 : num_updated_if_blocks++;
4779 141886 : return true;
4780 : }
4781 :
4782 : /* Check whether a block is suitable for conditional move conversion.
4783 : Every insn must be a simple set of a register to a constant or a
4784 : register. For each assignment, store the value in the pointer map
4785 : VALS, keyed indexed by register pointer, then store the register
4786 : pointer in REGS. COND is the condition we will test. */
4787 :
4788 : static bool
4789 1785478 : check_cond_move_block (basic_block bb,
4790 : hash_map<rtx, rtx> *vals,
4791 : vec<rtx> *regs,
4792 : rtx cond)
4793 : {
4794 1785478 : rtx_insn *insn;
4795 1785478 : rtx cc = cc_in_cond (cond);
4796 :
4797 : /* We can only handle simple jumps at the end of the basic block.
4798 : It is almost impossible to update the CFG otherwise. */
4799 1785478 : insn = BB_END (bb);
4800 1785478 : if (JUMP_P (insn) && !onlyjump_p (insn))
4801 : return false;
4802 :
4803 10574529 : FOR_BB_INSNS (bb, insn)
4804 : {
4805 10511828 : rtx set, dest, src;
4806 :
4807 10511828 : if (!NONDEBUG_INSN_P (insn) || JUMP_P (insn))
4808 8653256 : continue;
4809 1858572 : set = single_set (insn);
4810 1858572 : if (!set)
4811 1722724 : return false;
4812 :
4813 1784922 : dest = SET_DEST (set);
4814 1784922 : src = SET_SRC (set);
4815 1784922 : if (!REG_P (dest)
4816 1784922 : || (HARD_REGISTER_P (dest)
4817 312529 : && targetm.small_register_classes_for_mode_p (GET_MODE (dest))))
4818 : return false;
4819 :
4820 1083873 : if (!CONSTANT_P (src) && !register_operand (src, VOIDmode))
4821 : return false;
4822 :
4823 158373 : if (side_effects_p (src) || side_effects_p (dest))
4824 : return false;
4825 :
4826 158373 : if (may_trap_p (src) || may_trap_p (dest))
4827 : return false;
4828 :
4829 : /* Don't try to handle this if the source register was
4830 : modified earlier in the block. */
4831 158373 : if ((REG_P (src)
4832 51833 : && vals->get (src))
4833 209953 : || (GET_CODE (src) == SUBREG && REG_P (SUBREG_REG (src))
4834 5343 : && vals->get (SUBREG_REG (src))))
4835 : return false;
4836 :
4837 : /* Don't try to handle this if the destination register was
4838 : modified earlier in the block. */
4839 158066 : if (vals->get (dest))
4840 : return false;
4841 :
4842 : /* Don't try to handle this if the condition uses the
4843 : destination register. */
4844 158066 : if (reg_overlap_mentioned_p (dest, cond))
4845 : return false;
4846 :
4847 : /* Don't try to handle this if the source register is modified
4848 : later in the block. */
4849 139123 : if (!CONSTANT_P (src)
4850 139123 : && modified_between_p (src, insn, NEXT_INSN (BB_END (bb))))
4851 : return false;
4852 :
4853 : /* Skip it if the instruction to be moved might clobber CC. */
4854 135848 : if (cc && set_of (cc, insn))
4855 : return false;
4856 :
4857 135848 : vals->put (dest, src);
4858 :
4859 135848 : regs->safe_push (dest);
4860 : }
4861 :
4862 : return true;
4863 : }
4864 :
4865 : /* Given a basic block BB suitable for conditional move conversion,
4866 : a condition COND, and pointer maps THEN_VALS and ELSE_VALS containing
4867 : the register values depending on COND, emit the insns in the block as
4868 : conditional moves. If ELSE_BLOCK is true, THEN_BB was already
4869 : processed. The caller has started a sequence for the conversion.
4870 : Return true if successful, false if something goes wrong. */
4871 :
4872 : static bool
4873 37326 : cond_move_convert_if_block (struct noce_if_info *if_infop,
4874 : basic_block bb, rtx cond,
4875 : hash_map<rtx, rtx> *then_vals,
4876 : hash_map<rtx, rtx> *else_vals,
4877 : bool else_block_p)
4878 : {
4879 37326 : enum rtx_code code;
4880 37326 : rtx_insn *insn;
4881 37326 : rtx cond_arg0, cond_arg1;
4882 :
4883 37326 : code = GET_CODE (cond);
4884 37326 : cond_arg0 = XEXP (cond, 0);
4885 37326 : cond_arg1 = XEXP (cond, 1);
4886 :
4887 129789 : FOR_BB_INSNS (bb, insn)
4888 : {
4889 106854 : rtx set, target, dest, t, e;
4890 :
4891 : /* ??? Maybe emit conditional debug insn? */
4892 106854 : if (!NONDEBUG_INSN_P (insn) || JUMP_P (insn))
4893 70423 : continue;
4894 48323 : set = single_set (insn);
4895 48323 : gcc_assert (set && REG_P (SET_DEST (set)));
4896 :
4897 48323 : dest = SET_DEST (set);
4898 :
4899 48323 : rtx *then_slot = then_vals->get (dest);
4900 48323 : rtx *else_slot = else_vals->get (dest);
4901 48323 : t = then_slot ? *then_slot : NULL_RTX;
4902 48323 : e = else_slot ? *else_slot : NULL_RTX;
4903 :
4904 48323 : if (else_block_p)
4905 : {
4906 : /* If this register was set in the then block, we already
4907 : handled this case there. */
4908 13347 : if (t)
4909 11892 : continue;
4910 1455 : t = dest;
4911 1455 : gcc_assert (e);
4912 : }
4913 : else
4914 : {
4915 34976 : gcc_assert (t);
4916 34976 : if (!e)
4917 22415 : e = dest;
4918 : }
4919 :
4920 36431 : if (if_infop->cond_inverted)
4921 0 : std::swap (t, e);
4922 :
4923 36431 : target = noce_emit_cmove (if_infop, dest, code, cond_arg0, cond_arg1,
4924 : t, e);
4925 36431 : if (!target)
4926 14391 : return false;
4927 :
4928 22040 : if (target != dest)
4929 0 : noce_emit_move_insn (dest, target);
4930 : }
4931 :
4932 : return true;
4933 : }
4934 :
4935 : /* Given a simple IF-THEN-JOIN or IF-THEN-ELSE-JOIN block, attempt to convert
4936 : it using only conditional moves. Return TRUE if we were successful at
4937 : converting the block. */
4938 :
4939 : static bool
4940 1755244 : cond_move_process_if_block (struct noce_if_info *if_info)
4941 : {
4942 1755244 : basic_block test_bb = if_info->test_bb;
4943 1755244 : basic_block then_bb = if_info->then_bb;
4944 1755244 : basic_block else_bb = if_info->else_bb;
4945 1755244 : basic_block join_bb = if_info->join_bb;
4946 1755244 : rtx_insn *jump = if_info->jump;
4947 1755244 : rtx cond = if_info->cond;
4948 1755244 : rtx_insn *seq, *loc_insn;
4949 1755244 : int c;
4950 1755244 : vec<rtx> then_regs = vNULL;
4951 1755244 : vec<rtx> else_regs = vNULL;
4952 1755244 : bool success_p = false;
4953 1755244 : int limit = param_max_rtl_if_conversion_insns;
4954 :
4955 : /* Build a mapping for each block to the value used for each
4956 : register. */
4957 1755244 : hash_map<rtx, rtx> then_vals;
4958 1755244 : hash_map<rtx, rtx> else_vals;
4959 :
4960 : /* Make sure the blocks are suitable. */
4961 1755244 : if (!check_cond_move_block (then_bb, &then_vals, &then_regs, cond)
4962 1755244 : || (else_bb
4963 30234 : && !check_cond_move_block (else_bb, &else_vals, &else_regs, cond)))
4964 1722777 : goto done;
4965 :
4966 : /* Make sure the blocks can be used together. If the same register
4967 : is set in both blocks, and is not set to a constant in both
4968 : cases, then both blocks must set it to the same register. We
4969 : have already verified that if it is set to a register, that the
4970 : source register does not change after the assignment. Also count
4971 : the number of registers set in only one of the blocks. */
4972 32467 : c = 0;
4973 133305 : for (rtx reg : then_regs)
4974 : {
4975 39344 : rtx *then_slot = then_vals.get (reg);
4976 39344 : rtx *else_slot = else_vals.get (reg);
4977 :
4978 39344 : gcc_checking_assert (then_slot);
4979 39344 : if (!else_slot)
4980 23092 : ++c;
4981 : else
4982 : {
4983 16252 : rtx then_val = *then_slot;
4984 16252 : rtx else_val = *else_slot;
4985 4602 : if (!CONSTANT_P (then_val) && !CONSTANT_P (else_val)
4986 20203 : && !rtx_equal_p (then_val, else_val))
4987 3046 : goto done;
4988 : }
4989 : }
4990 :
4991 : /* Finish off c for MAX_CONDITIONAL_EXECUTE. */
4992 61317 : for (rtx reg : else_regs)
4993 : {
4994 14642 : gcc_checking_assert (else_vals.get (reg));
4995 14642 : if (!then_vals.get (reg))
4996 1587 : ++c;
4997 : }
4998 :
4999 : /* Make sure it is reasonable to convert this block. What matters
5000 : is the number of assignments currently made in only one of the
5001 : branches, since if we convert we are going to always execute
5002 : them. */
5003 29421 : if (c > MAX_CONDITIONAL_EXECUTE
5004 29421 : || c > limit)
5005 38 : goto done;
5006 :
5007 : /* Try to emit the conditional moves. First do the then block,
5008 : then do anything left in the else blocks. */
5009 29383 : start_sequence ();
5010 29383 : if (!cond_move_convert_if_block (if_info, then_bb, cond,
5011 : &then_vals, &else_vals, false)
5012 29383 : || (else_bb
5013 7943 : && !cond_move_convert_if_block (if_info, else_bb, cond,
5014 : &then_vals, &else_vals, true)))
5015 : {
5016 14391 : end_sequence ();
5017 14391 : goto done;
5018 : }
5019 14992 : seq = end_ifcvt_sequence (if_info);
5020 14992 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
5021 11838 : goto done;
5022 :
5023 3154 : loc_insn = first_active_insn (then_bb);
5024 3154 : if (!loc_insn)
5025 : {
5026 0 : loc_insn = first_active_insn (else_bb);
5027 0 : gcc_assert (loc_insn);
5028 : }
5029 3154 : emit_insn_before_setloc (seq, jump, INSN_LOCATION (loc_insn));
5030 :
5031 3154 : if (else_bb)
5032 : {
5033 3154 : delete_basic_block (else_bb);
5034 3154 : num_true_changes++;
5035 : }
5036 : else
5037 0 : remove_edge (find_edge (test_bb, join_bb));
5038 :
5039 3154 : remove_edge (find_edge (then_bb, join_bb));
5040 3154 : redirect_edge_and_branch_force (single_succ_edge (test_bb), join_bb);
5041 3154 : delete_basic_block (then_bb);
5042 3154 : num_true_changes++;
5043 :
5044 3154 : if (can_merge_blocks_p (test_bb, join_bb))
5045 : {
5046 1256 : merge_blocks (test_bb, join_bb);
5047 1256 : num_true_changes++;
5048 : }
5049 :
5050 3154 : num_updated_if_blocks++;
5051 3154 : success_p = true;
5052 :
5053 1755244 : done:
5054 1755244 : then_regs.release ();
5055 1755244 : else_regs.release ();
5056 1755244 : return success_p;
5057 1755244 : }
5058 :
5059 :
5060 : /* Determine if a given basic block heads a simple IF-THEN-JOIN or an
5061 : IF-THEN-ELSE-JOIN block.
5062 :
5063 : If so, we'll try to convert the insns to not require the branch,
5064 : using only transformations that do not require conditional execution.
5065 :
5066 : Return TRUE if we were successful at converting the block. */
5067 :
5068 : static bool
5069 9929941 : noce_find_if_block (basic_block test_bb, edge then_edge, edge else_edge,
5070 : int pass)
5071 : {
5072 9929941 : basic_block then_bb, else_bb, join_bb;
5073 9929941 : bool then_else_reversed = false;
5074 9929941 : rtx_insn *jump;
5075 9929941 : rtx_insn *cond_earliest;
5076 9929941 : struct noce_if_info if_info;
5077 9929941 : bool speed_p = optimize_bb_for_speed_p (test_bb);
5078 :
5079 : /* We only ever should get here before reload. */
5080 9929941 : gcc_assert (!reload_completed);
5081 :
5082 : /* Recognize an IF-THEN-ELSE-JOIN block. */
5083 9929941 : if (single_pred_p (then_edge->dest)
5084 12604187 : && single_succ_p (then_edge->dest)
5085 3574540 : && single_pred_p (else_edge->dest)
5086 1872187 : && single_succ_p (else_edge->dest)
5087 11103406 : && single_succ (then_edge->dest) == single_succ (else_edge->dest))
5088 : {
5089 : then_bb = then_edge->dest;
5090 : else_bb = else_edge->dest;
5091 : join_bb = single_succ (then_bb);
5092 : }
5093 : /* Recognize an IF-THEN-JOIN block. */
5094 17100381 : else if (single_pred_p (then_edge->dest)
5095 7268379 : && single_succ_p (then_edge->dest)
5096 11703893 : && single_succ (then_edge->dest) == else_edge->dest)
5097 : {
5098 : then_bb = then_edge->dest;
5099 : else_bb = NULL_BLOCK;
5100 : join_bb = else_edge->dest;
5101 : }
5102 : /* Recognize an IF-ELSE-JOIN block. We can have those because the order
5103 : of basic blocks in cfglayout mode does not matter, so the fallthrough
5104 : edge can go to any basic block (and not just to bb->next_bb, like in
5105 : cfgrtl mode). */
5106 8070734 : else if (single_pred_p (else_edge->dest)
5107 3856886 : && single_succ_p (else_edge->dest)
5108 9474736 : && single_succ (else_edge->dest) == then_edge->dest)
5109 : {
5110 : /* The noce transformations do not apply to IF-ELSE-JOIN blocks.
5111 : To make this work, we have to invert the THEN and ELSE blocks
5112 : and reverse the jump condition. */
5113 : then_bb = else_edge->dest;
5114 : else_bb = NULL_BLOCK;
5115 : join_bb = single_succ (then_bb);
5116 : then_else_reversed = true;
5117 : }
5118 : else
5119 : /* Not a form we can handle. */
5120 : return false;
5121 :
5122 : /* The edges of the THEN and ELSE blocks cannot have complex edges. */
5123 1992090 : if (single_succ_edge (then_bb)->flags & EDGE_COMPLEX)
5124 : return false;
5125 1936579 : if (else_bb
5126 1936579 : && single_succ_edge (else_bb)->flags & EDGE_COMPLEX)
5127 : return false;
5128 :
5129 1926573 : num_possible_if_blocks++;
5130 :
5131 1926573 : if (dump_file)
5132 : {
5133 92 : fprintf (dump_file,
5134 : "\nIF-THEN%s-JOIN block found, pass %d, test %d, then %d",
5135 : (else_bb) ? "-ELSE" : "",
5136 : pass, test_bb->index, then_bb->index);
5137 :
5138 55 : if (else_bb)
5139 18 : fprintf (dump_file, ", else %d", else_bb->index);
5140 :
5141 55 : fprintf (dump_file, ", join %d\n", join_bb->index);
5142 : }
5143 :
5144 : /* If the conditional jump is more than just a conditional
5145 : jump, then we cannot do if-conversion on this block. */
5146 1926573 : jump = BB_END (test_bb);
5147 1926573 : if (! onlyjump_p (jump))
5148 : return false;
5149 :
5150 : /* Initialize an IF_INFO struct to pass around. */
5151 1926312 : memset (&if_info, 0, sizeof if_info);
5152 1926312 : if_info.test_bb = test_bb;
5153 1926312 : if_info.then_bb = then_bb;
5154 1926312 : if_info.else_bb = else_bb;
5155 1926312 : if_info.join_bb = join_bb;
5156 1926312 : if_info.cond = noce_get_condition (jump, &cond_earliest,
5157 : then_else_reversed);
5158 1926312 : rtx_insn *rev_cond_earliest;
5159 1926312 : if_info.rev_cond = noce_get_condition (jump, &rev_cond_earliest,
5160 : !then_else_reversed);
5161 1926312 : if (!if_info.cond && !if_info.rev_cond)
5162 : return false;
5163 1922514 : if (!if_info.cond)
5164 : {
5165 0 : std::swap (if_info.cond, if_info.rev_cond);
5166 0 : std::swap (cond_earliest, rev_cond_earliest);
5167 0 : if_info.cond_inverted = true;
5168 : }
5169 : /* We must be comparing objects whose modes imply the size. */
5170 1922514 : if (GET_MODE (XEXP (if_info.cond, 0)) == BLKmode)
5171 : return false;
5172 1922514 : gcc_assert (if_info.rev_cond == NULL_RTX
5173 : || rev_cond_earliest == cond_earliest);
5174 1922514 : if_info.cond_earliest = cond_earliest;
5175 1922514 : if_info.jump = jump;
5176 1922514 : if_info.then_else_reversed = then_else_reversed;
5177 1922514 : if_info.speed_p = speed_p;
5178 1922514 : if_info.max_seq_cost
5179 1922514 : = targetm.max_noce_ifcvt_seq_cost (then_edge);
5180 : /* We'll add in the cost of THEN_BB and ELSE_BB later, when we check
5181 : that they are valid to transform. We can't easily get back to the insn
5182 : for COND (and it may not exist if we had to canonicalize to get COND).
5183 : It is assumed that the costs of a jump insn are dependent on the
5184 : branch costs. */
5185 1922514 : if_info.original_cost += insn_cost (if_info.jump, if_info.speed_p);
5186 :
5187 : /* Do the real work. */
5188 :
5189 : /* ??? noce_process_if_block has not yet been updated to handle
5190 : inverted conditions. */
5191 1922514 : if (!if_info.cond_inverted && noce_process_if_block (&if_info))
5192 : return true;
5193 :
5194 1755244 : if (HAVE_conditional_move
5195 1755244 : && cond_move_process_if_block (&if_info))
5196 : return true;
5197 :
5198 : return false;
5199 : }
5200 :
5201 :
5202 : /* Merge the blocks and mark for local life update. */
5203 :
5204 : static void
5205 0 : merge_if_block (struct ce_if_block * ce_info)
5206 : {
5207 0 : basic_block test_bb = ce_info->test_bb; /* last test block */
5208 0 : basic_block then_bb = ce_info->then_bb; /* THEN */
5209 0 : basic_block else_bb = ce_info->else_bb; /* ELSE or NULL */
5210 0 : basic_block join_bb = ce_info->join_bb; /* join block */
5211 0 : basic_block combo_bb;
5212 :
5213 : /* All block merging is done into the lower block numbers. */
5214 :
5215 0 : combo_bb = test_bb;
5216 0 : df_set_bb_dirty (test_bb);
5217 :
5218 : /* Merge any basic blocks to handle && and || subtests. Each of
5219 : the blocks are on the fallthru path from the predecessor block. */
5220 0 : if (ce_info->num_multiple_test_blocks > 0)
5221 : {
5222 0 : basic_block bb = test_bb;
5223 0 : basic_block last_test_bb = ce_info->last_test_bb;
5224 0 : basic_block fallthru = block_fallthru (bb);
5225 :
5226 0 : do
5227 : {
5228 0 : bb = fallthru;
5229 0 : fallthru = block_fallthru (bb);
5230 0 : merge_blocks (combo_bb, bb);
5231 0 : num_true_changes++;
5232 : }
5233 0 : while (bb != last_test_bb);
5234 : }
5235 :
5236 : /* Merge TEST block into THEN block. Normally the THEN block won't have a
5237 : label, but it might if there were || tests. That label's count should be
5238 : zero, and it normally should be removed. */
5239 :
5240 0 : if (then_bb)
5241 : {
5242 : /* If THEN_BB has no successors, then there's a BARRIER after it.
5243 : If COMBO_BB has more than one successor (THEN_BB), then that BARRIER
5244 : is no longer needed, and in fact it is incorrect to leave it in
5245 : the insn stream. */
5246 0 : if (EDGE_COUNT (then_bb->succs) == 0
5247 0 : && EDGE_COUNT (combo_bb->succs) > 1)
5248 : {
5249 0 : rtx_insn *end = NEXT_INSN (BB_END (then_bb));
5250 0 : while (end && NOTE_P (end) && !NOTE_INSN_BASIC_BLOCK_P (end))
5251 0 : end = NEXT_INSN (end);
5252 :
5253 0 : if (end && BARRIER_P (end))
5254 0 : delete_insn (end);
5255 : }
5256 0 : merge_blocks (combo_bb, then_bb);
5257 0 : num_true_changes++;
5258 : }
5259 :
5260 : /* The ELSE block, if it existed, had a label. That label count
5261 : will almost always be zero, but odd things can happen when labels
5262 : get their addresses taken. */
5263 0 : if (else_bb)
5264 : {
5265 : /* If ELSE_BB has no successors, then there's a BARRIER after it.
5266 : If COMBO_BB has more than one successor (ELSE_BB), then that BARRIER
5267 : is no longer needed, and in fact it is incorrect to leave it in
5268 : the insn stream. */
5269 0 : if (EDGE_COUNT (else_bb->succs) == 0
5270 0 : && EDGE_COUNT (combo_bb->succs) > 1)
5271 : {
5272 0 : rtx_insn *end = NEXT_INSN (BB_END (else_bb));
5273 0 : while (end && NOTE_P (end) && !NOTE_INSN_BASIC_BLOCK_P (end))
5274 0 : end = NEXT_INSN (end);
5275 :
5276 0 : if (end && BARRIER_P (end))
5277 0 : delete_insn (end);
5278 : }
5279 0 : merge_blocks (combo_bb, else_bb);
5280 0 : num_true_changes++;
5281 : }
5282 :
5283 : /* If there was no join block reported, that means it was not adjacent
5284 : to the others, and so we cannot merge them. */
5285 :
5286 0 : if (! join_bb)
5287 : {
5288 0 : rtx_insn *last = BB_END (combo_bb);
5289 :
5290 : /* The outgoing edge for the current COMBO block should already
5291 : be correct. Verify this. */
5292 0 : if (EDGE_COUNT (combo_bb->succs) == 0)
5293 0 : gcc_assert (find_reg_note (last, REG_NORETURN, NULL)
5294 : || (NONJUMP_INSN_P (last)
5295 : && GET_CODE (PATTERN (last)) == TRAP_IF
5296 : && (TRAP_CONDITION (PATTERN (last))
5297 : == const_true_rtx)));
5298 :
5299 : else
5300 : /* There should still be something at the end of the THEN or ELSE
5301 : blocks taking us to our final destination. */
5302 0 : gcc_assert (JUMP_P (last)
5303 : || (EDGE_SUCC (combo_bb, 0)->dest
5304 : == EXIT_BLOCK_PTR_FOR_FN (cfun)
5305 : && CALL_P (last)
5306 : && SIBLING_CALL_P (last))
5307 : || ((EDGE_SUCC (combo_bb, 0)->flags & EDGE_EH)
5308 : && can_throw_internal (last)));
5309 : }
5310 :
5311 : /* The JOIN block may have had quite a number of other predecessors too.
5312 : Since we've already merged the TEST, THEN and ELSE blocks, we should
5313 : have only one remaining edge from our if-then-else diamond. If there
5314 : is more than one remaining edge, it must come from elsewhere. There
5315 : may be zero incoming edges if the THEN block didn't actually join
5316 : back up (as with a call to a non-return function). */
5317 0 : else if (EDGE_COUNT (join_bb->preds) < 2
5318 0 : && join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5319 : {
5320 : /* We can merge the JOIN cleanly and update the dataflow try
5321 : again on this pass.*/
5322 0 : merge_blocks (combo_bb, join_bb);
5323 0 : num_true_changes++;
5324 : }
5325 : else
5326 : {
5327 : /* We cannot merge the JOIN. */
5328 :
5329 : /* The outgoing edge for the current COMBO block should already
5330 : be correct. Verify this. */
5331 0 : gcc_assert (single_succ_p (combo_bb)
5332 : && single_succ (combo_bb) == join_bb);
5333 :
5334 : /* Remove the jump and cruft from the end of the COMBO block. */
5335 0 : if (join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5336 0 : tidy_fallthru_edge (single_succ_edge (combo_bb));
5337 : }
5338 :
5339 0 : num_updated_if_blocks++;
5340 0 : }
5341 :
5342 : /* Find a block ending in a simple IF condition and try to transform it
5343 : in some way. When converting a multi-block condition, put the new code
5344 : in the first such block and delete the rest. Return a pointer to this
5345 : first block if some transformation was done. Return NULL otherwise. */
5346 :
5347 : static basic_block
5348 40126121 : find_if_header (basic_block test_bb, int pass)
5349 : {
5350 40126121 : ce_if_block ce_info;
5351 40126121 : edge then_edge;
5352 40126121 : edge else_edge;
5353 :
5354 : /* The kind of block we're looking for has exactly two successors. */
5355 40126121 : if (EDGE_COUNT (test_bb->succs) != 2)
5356 : return NULL;
5357 :
5358 20647113 : then_edge = EDGE_SUCC (test_bb, 0);
5359 20647113 : else_edge = EDGE_SUCC (test_bb, 1);
5360 :
5361 20647113 : if (df_get_bb_dirty (then_edge->dest))
5362 : return NULL;
5363 20625718 : if (df_get_bb_dirty (else_edge->dest))
5364 : return NULL;
5365 :
5366 : /* Neither edge should be abnormal. */
5367 20589008 : if ((then_edge->flags & EDGE_COMPLEX)
5368 18816494 : || (else_edge->flags & EDGE_COMPLEX))
5369 : return NULL;
5370 :
5371 : /* Nor exit the loop. */
5372 18415184 : if ((then_edge->flags & EDGE_LOOP_EXIT)
5373 16929239 : || (else_edge->flags & EDGE_LOOP_EXIT))
5374 : return NULL;
5375 :
5376 : /* The THEN edge is canonically the one that falls through. */
5377 14645058 : if (then_edge->flags & EDGE_FALLTHRU)
5378 : ;
5379 7103597 : else if (else_edge->flags & EDGE_FALLTHRU)
5380 : std::swap (then_edge, else_edge);
5381 : else
5382 : /* Otherwise this must be a multiway branch of some sort. */
5383 : return NULL;
5384 :
5385 14644718 : memset (&ce_info, 0, sizeof (ce_info));
5386 14644718 : ce_info.test_bb = test_bb;
5387 14644718 : ce_info.then_bb = then_edge->dest;
5388 14644718 : ce_info.else_bb = else_edge->dest;
5389 14644718 : ce_info.pass = pass;
5390 :
5391 : #ifdef IFCVT_MACHDEP_INIT
5392 : IFCVT_MACHDEP_INIT (&ce_info);
5393 : #endif
5394 :
5395 14644718 : if (!reload_completed
5396 14644718 : && noce_find_if_block (test_bb, then_edge, else_edge, pass))
5397 170424 : goto success;
5398 :
5399 14474294 : if (reload_completed
5400 4714777 : && targetm.have_conditional_execution ()
5401 14474294 : && cond_exec_find_if_block (&ce_info))
5402 0 : goto success;
5403 :
5404 9759517 : if (!reload_completed && targetm.have_trap ()
5405 9759517 : && optab_handler (ctrap_optab, word_mode) != CODE_FOR_nothing
5406 14474294 : && find_cond_trap (test_bb, then_edge, else_edge))
5407 0 : goto success;
5408 :
5409 14474294 : if (dom_info_state (CDI_POST_DOMINATORS) >= DOM_NO_FAST_QUERY
5410 14474294 : && (reload_completed || !targetm.have_conditional_execution ()))
5411 : {
5412 14474294 : if (find_if_case_1 (test_bb, then_edge, else_edge))
5413 21836 : goto success;
5414 14452458 : if (find_if_case_2 (test_bb, then_edge, else_edge))
5415 127431 : goto success;
5416 : }
5417 :
5418 : return NULL;
5419 :
5420 319691 : success:
5421 319691 : if (dump_file)
5422 21 : fprintf (dump_file, "Conversion succeeded on pass %d.\n", pass);
5423 : /* Set this so we continue looking. */
5424 319691 : cond_exec_changed_p = true;
5425 319691 : return ce_info.test_bb;
5426 : }
5427 :
5428 : /* Return true if a block has two edges, one of which falls through to the next
5429 : block, and the other jumps to a specific block, so that we can tell if the
5430 : block is part of an && test or an || test. Returns either -1 or the number
5431 : of non-note, non-jump, non-USE/CLOBBER insns in the block. */
5432 :
5433 : static int
5434 0 : block_jumps_and_fallthru (basic_block cur_bb, basic_block target_bb)
5435 : {
5436 0 : edge cur_edge;
5437 0 : bool fallthru_p = false;
5438 0 : bool jump_p = false;
5439 0 : rtx_insn *insn;
5440 0 : rtx_insn *end;
5441 0 : int n_insns = 0;
5442 0 : edge_iterator ei;
5443 :
5444 0 : if (!cur_bb || !target_bb)
5445 : return -1;
5446 :
5447 : /* If no edges, obviously it doesn't jump or fallthru. */
5448 0 : if (EDGE_COUNT (cur_bb->succs) == 0)
5449 : return 0;
5450 :
5451 0 : FOR_EACH_EDGE (cur_edge, ei, cur_bb->succs)
5452 : {
5453 0 : if (cur_edge->flags & EDGE_COMPLEX)
5454 : /* Anything complex isn't what we want. */
5455 : return -1;
5456 :
5457 0 : else if (cur_edge->flags & EDGE_FALLTHRU)
5458 : fallthru_p = true;
5459 :
5460 0 : else if (cur_edge->dest == target_bb)
5461 : jump_p = true;
5462 :
5463 : else
5464 : return -1;
5465 : }
5466 :
5467 0 : if ((jump_p & fallthru_p) == 0)
5468 : return -1;
5469 :
5470 : /* Don't allow calls in the block, since this is used to group && and ||
5471 : together for conditional execution support. ??? we should support
5472 : conditional execution support across calls for IA-64 some day, but
5473 : for now it makes the code simpler. */
5474 0 : end = BB_END (cur_bb);
5475 0 : insn = BB_HEAD (cur_bb);
5476 :
5477 0 : while (insn != NULL_RTX)
5478 : {
5479 0 : if (CALL_P (insn))
5480 : return -1;
5481 :
5482 0 : if (INSN_P (insn)
5483 0 : && !JUMP_P (insn)
5484 0 : && !DEBUG_INSN_P (insn)
5485 0 : && GET_CODE (PATTERN (insn)) != USE
5486 0 : && GET_CODE (PATTERN (insn)) != CLOBBER)
5487 0 : n_insns++;
5488 :
5489 0 : if (insn == end)
5490 : break;
5491 :
5492 0 : insn = NEXT_INSN (insn);
5493 : }
5494 :
5495 : return n_insns;
5496 : }
5497 :
5498 : /* Determine if a given basic block heads a simple IF-THEN or IF-THEN-ELSE
5499 : block. If so, we'll try to convert the insns to not require the branch.
5500 : Return TRUE if we were successful at converting the block. */
5501 :
5502 : static bool
5503 0 : cond_exec_find_if_block (struct ce_if_block * ce_info)
5504 : {
5505 0 : basic_block test_bb = ce_info->test_bb;
5506 0 : basic_block then_bb = ce_info->then_bb;
5507 0 : basic_block else_bb = ce_info->else_bb;
5508 0 : basic_block join_bb = NULL_BLOCK;
5509 0 : edge cur_edge;
5510 0 : basic_block next;
5511 0 : edge_iterator ei;
5512 :
5513 0 : ce_info->last_test_bb = test_bb;
5514 :
5515 : /* We only ever should get here after reload,
5516 : and if we have conditional execution. */
5517 0 : gcc_assert (reload_completed && targetm.have_conditional_execution ());
5518 :
5519 : /* Discover if any fall through predecessors of the current test basic block
5520 : were && tests (which jump to the else block) or || tests (which jump to
5521 : the then block). */
5522 0 : if (single_pred_p (test_bb)
5523 0 : && single_pred_edge (test_bb)->flags == EDGE_FALLTHRU)
5524 : {
5525 0 : basic_block bb = single_pred (test_bb);
5526 0 : basic_block target_bb;
5527 0 : int max_insns = MAX_CONDITIONAL_EXECUTE;
5528 0 : int n_insns;
5529 :
5530 : /* Determine if the preceding block is an && or || block. */
5531 0 : if ((n_insns = block_jumps_and_fallthru (bb, else_bb)) >= 0)
5532 : {
5533 0 : ce_info->and_and_p = true;
5534 0 : target_bb = else_bb;
5535 : }
5536 0 : else if ((n_insns = block_jumps_and_fallthru (bb, then_bb)) >= 0)
5537 : {
5538 0 : ce_info->and_and_p = false;
5539 0 : target_bb = then_bb;
5540 : }
5541 : else
5542 : target_bb = NULL_BLOCK;
5543 :
5544 0 : if (target_bb && n_insns <= max_insns)
5545 : {
5546 0 : int total_insns = 0;
5547 0 : int blocks = 0;
5548 :
5549 0 : ce_info->last_test_bb = test_bb;
5550 :
5551 : /* Found at least one && or || block, look for more. */
5552 0 : do
5553 : {
5554 0 : ce_info->test_bb = test_bb = bb;
5555 0 : total_insns += n_insns;
5556 0 : blocks++;
5557 :
5558 0 : if (!single_pred_p (bb))
5559 : break;
5560 :
5561 0 : bb = single_pred (bb);
5562 0 : n_insns = block_jumps_and_fallthru (bb, target_bb);
5563 : }
5564 0 : while (n_insns >= 0 && (total_insns + n_insns) <= max_insns);
5565 :
5566 0 : ce_info->num_multiple_test_blocks = blocks;
5567 0 : ce_info->num_multiple_test_insns = total_insns;
5568 :
5569 0 : if (ce_info->and_and_p)
5570 0 : ce_info->num_and_and_blocks = blocks;
5571 : else
5572 0 : ce_info->num_or_or_blocks = blocks;
5573 : }
5574 : }
5575 :
5576 : /* The THEN block of an IF-THEN combo must have exactly one predecessor,
5577 : other than any || blocks which jump to the THEN block. */
5578 0 : if ((EDGE_COUNT (then_bb->preds) - ce_info->num_or_or_blocks) != 1)
5579 : return false;
5580 :
5581 : /* The edges of the THEN and ELSE blocks cannot have complex edges. */
5582 0 : FOR_EACH_EDGE (cur_edge, ei, then_bb->preds)
5583 : {
5584 0 : if (cur_edge->flags & EDGE_COMPLEX)
5585 : return false;
5586 : }
5587 :
5588 0 : FOR_EACH_EDGE (cur_edge, ei, else_bb->preds)
5589 : {
5590 0 : if (cur_edge->flags & EDGE_COMPLEX)
5591 : return false;
5592 : }
5593 :
5594 : /* The THEN block of an IF-THEN combo must have zero or one successors. */
5595 0 : if (EDGE_COUNT (then_bb->succs) > 0
5596 0 : && (!single_succ_p (then_bb)
5597 0 : || (single_succ_edge (then_bb)->flags & EDGE_COMPLEX)
5598 0 : || (epilogue_completed
5599 0 : && tablejump_p (BB_END (then_bb), NULL, NULL))))
5600 : return false;
5601 :
5602 : /* If the THEN block has no successors, conditional execution can still
5603 : make a conditional call. Don't do this unless the ELSE block has
5604 : only one incoming edge -- the CFG manipulation is too ugly otherwise.
5605 : Check for the last insn of the THEN block being an indirect jump, which
5606 : is listed as not having any successors, but confuses the rest of the CE
5607 : code processing. ??? we should fix this in the future. */
5608 0 : if (EDGE_COUNT (then_bb->succs) == 0)
5609 : {
5610 0 : if (single_pred_p (else_bb) && else_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5611 : {
5612 0 : rtx_insn *last_insn = BB_END (then_bb);
5613 :
5614 0 : while (last_insn
5615 0 : && NOTE_P (last_insn)
5616 0 : && last_insn != BB_HEAD (then_bb))
5617 0 : last_insn = PREV_INSN (last_insn);
5618 :
5619 0 : if (last_insn
5620 0 : && JUMP_P (last_insn)
5621 0 : && ! simplejump_p (last_insn))
5622 : return false;
5623 :
5624 : join_bb = else_bb;
5625 : else_bb = NULL_BLOCK;
5626 : }
5627 : else
5628 : return false;
5629 : }
5630 :
5631 : /* If the THEN block's successor is the other edge out of the TEST block,
5632 : then we have an IF-THEN combo without an ELSE. */
5633 0 : else if (single_succ (then_bb) == else_bb)
5634 : {
5635 : join_bb = else_bb;
5636 : else_bb = NULL_BLOCK;
5637 : }
5638 :
5639 : /* If the THEN and ELSE block meet in a subsequent block, and the ELSE
5640 : has exactly one predecessor and one successor, and the outgoing edge
5641 : is not complex, then we have an IF-THEN-ELSE combo. */
5642 0 : else if (single_succ_p (else_bb)
5643 0 : && single_succ (then_bb) == single_succ (else_bb)
5644 0 : && single_pred_p (else_bb)
5645 0 : && !(single_succ_edge (else_bb)->flags & EDGE_COMPLEX)
5646 0 : && !(epilogue_completed
5647 0 : && tablejump_p (BB_END (else_bb), NULL, NULL)))
5648 0 : join_bb = single_succ (else_bb);
5649 :
5650 : /* Otherwise it is not an IF-THEN or IF-THEN-ELSE combination. */
5651 : else
5652 : return false;
5653 :
5654 0 : num_possible_if_blocks++;
5655 :
5656 0 : if (dump_file)
5657 : {
5658 0 : fprintf (dump_file,
5659 : "\nIF-THEN%s block found, pass %d, start block %d "
5660 : "[insn %d], then %d [%d]",
5661 : (else_bb) ? "-ELSE" : "",
5662 : ce_info->pass,
5663 : test_bb->index,
5664 0 : BB_HEAD (test_bb) ? (int)INSN_UID (BB_HEAD (test_bb)) : -1,
5665 : then_bb->index,
5666 0 : BB_HEAD (then_bb) ? (int)INSN_UID (BB_HEAD (then_bb)) : -1);
5667 :
5668 0 : if (else_bb)
5669 0 : fprintf (dump_file, ", else %d [%d]",
5670 : else_bb->index,
5671 0 : BB_HEAD (else_bb) ? (int)INSN_UID (BB_HEAD (else_bb)) : -1);
5672 :
5673 0 : fprintf (dump_file, ", join %d [%d]",
5674 : join_bb->index,
5675 0 : BB_HEAD (join_bb) ? (int)INSN_UID (BB_HEAD (join_bb)) : -1);
5676 :
5677 0 : if (ce_info->num_multiple_test_blocks > 0)
5678 0 : fprintf (dump_file, ", %d %s block%s last test %d [%d]",
5679 : ce_info->num_multiple_test_blocks,
5680 0 : (ce_info->and_and_p) ? "&&" : "||",
5681 : (ce_info->num_multiple_test_blocks == 1) ? "" : "s",
5682 : ce_info->last_test_bb->index,
5683 0 : ((BB_HEAD (ce_info->last_test_bb))
5684 0 : ? (int)INSN_UID (BB_HEAD (ce_info->last_test_bb))
5685 : : -1));
5686 :
5687 0 : fputc ('\n', dump_file);
5688 : }
5689 :
5690 : /* Make sure IF, THEN, and ELSE, blocks are adjacent. Actually, we get the
5691 : first condition for free, since we've already asserted that there's a
5692 : fallthru edge from IF to THEN. Likewise for the && and || blocks, since
5693 : we checked the FALLTHRU flag, those are already adjacent to the last IF
5694 : block. */
5695 : /* ??? As an enhancement, move the ELSE block. Have to deal with
5696 : BLOCK notes, if by no other means than backing out the merge if they
5697 : exist. Sticky enough I don't want to think about it now. */
5698 0 : next = then_bb;
5699 0 : if (else_bb && (next = next->next_bb) != else_bb)
5700 : return false;
5701 0 : if ((next = next->next_bb) != join_bb
5702 0 : && join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5703 : {
5704 0 : if (else_bb)
5705 : join_bb = NULL;
5706 : else
5707 : return false;
5708 : }
5709 :
5710 : /* Do the real work. */
5711 :
5712 0 : ce_info->else_bb = else_bb;
5713 0 : ce_info->join_bb = join_bb;
5714 :
5715 : /* If we have && and || tests, try to first handle combining the && and ||
5716 : tests into the conditional code, and if that fails, go back and handle
5717 : it without the && and ||, which at present handles the && case if there
5718 : was no ELSE block. */
5719 0 : if (cond_exec_process_if_block (ce_info, true))
5720 : return true;
5721 :
5722 0 : if (ce_info->num_multiple_test_blocks)
5723 : {
5724 0 : cancel_changes (0);
5725 :
5726 0 : if (cond_exec_process_if_block (ce_info, false))
5727 : return true;
5728 : }
5729 :
5730 : return false;
5731 : }
5732 :
5733 : /* Convert a branch over a trap, or a branch
5734 : to a trap, into a conditional trap. */
5735 :
5736 : static bool
5737 0 : find_cond_trap (basic_block test_bb, edge then_edge, edge else_edge)
5738 : {
5739 0 : basic_block then_bb = then_edge->dest;
5740 0 : basic_block else_bb = else_edge->dest;
5741 0 : basic_block other_bb, trap_bb;
5742 0 : rtx_insn *trap, *jump;
5743 0 : rtx cond;
5744 0 : rtx_insn *cond_earliest;
5745 :
5746 : /* Locate the block with the trap instruction. */
5747 : /* ??? While we look for no successors, we really ought to allow
5748 : EH successors. Need to fix merge_if_block for that to work. */
5749 0 : if ((trap = block_has_only_trap (then_bb)) != NULL)
5750 : trap_bb = then_bb, other_bb = else_bb;
5751 0 : else if ((trap = block_has_only_trap (else_bb)) != NULL)
5752 : trap_bb = else_bb, other_bb = then_bb;
5753 : else
5754 : return false;
5755 :
5756 0 : if (dump_file)
5757 : {
5758 0 : fprintf (dump_file, "\nTRAP-IF block found, start %d, trap %d\n",
5759 : test_bb->index, trap_bb->index);
5760 : }
5761 :
5762 : /* If this is not a standard conditional jump, we can't parse it. */
5763 0 : jump = BB_END (test_bb);
5764 0 : cond = noce_get_condition (jump, &cond_earliest, then_bb == trap_bb);
5765 0 : if (! cond)
5766 : return false;
5767 :
5768 : /* If the conditional jump is more than just a conditional jump, then
5769 : we cannot do if-conversion on this block. Give up for returnjump_p,
5770 : changing a conditional return followed by unconditional trap for
5771 : conditional trap followed by unconditional return is likely not
5772 : beneficial and harder to handle. */
5773 0 : if (! onlyjump_p (jump) || returnjump_p (jump))
5774 : return false;
5775 :
5776 : /* We must be comparing objects whose modes imply the size. */
5777 0 : if (GET_MODE (XEXP (cond, 0)) == BLKmode)
5778 : return false;
5779 :
5780 : /* Attempt to generate the conditional trap. */
5781 0 : rtx_insn *seq = gen_cond_trap (GET_CODE (cond), copy_rtx (XEXP (cond, 0)),
5782 : copy_rtx (XEXP (cond, 1)),
5783 0 : TRAP_CODE (PATTERN (trap)));
5784 0 : if (seq == NULL)
5785 : return false;
5786 :
5787 : /* If that results in an invalid insn, back out. */
5788 0 : for (rtx_insn *x = seq; x; x = NEXT_INSN (x))
5789 0 : if (reload_completed
5790 0 : ? !valid_insn_p (x)
5791 0 : : recog_memoized (x) < 0)
5792 : return false;
5793 :
5794 : /* Emit the new insns before cond_earliest. */
5795 0 : emit_insn_before_setloc (seq, cond_earliest, INSN_LOCATION (trap));
5796 :
5797 : /* Delete the trap block if possible. */
5798 0 : remove_edge (trap_bb == then_bb ? then_edge : else_edge);
5799 0 : df_set_bb_dirty (test_bb);
5800 0 : df_set_bb_dirty (then_bb);
5801 0 : df_set_bb_dirty (else_bb);
5802 :
5803 0 : if (EDGE_COUNT (trap_bb->preds) == 0)
5804 : {
5805 0 : delete_basic_block (trap_bb);
5806 0 : num_true_changes++;
5807 : }
5808 :
5809 : /* Wire together the blocks again. */
5810 0 : if (current_ir_type () == IR_RTL_CFGLAYOUT)
5811 0 : single_succ_edge (test_bb)->flags |= EDGE_FALLTHRU;
5812 0 : else if (trap_bb == then_bb)
5813 : {
5814 0 : rtx lab = JUMP_LABEL (jump);
5815 0 : rtx_insn *seq = targetm.gen_jump (lab);
5816 0 : rtx_jump_insn *newjump = emit_jump_insn_after (seq, jump);
5817 0 : LABEL_NUSES (lab) += 1;
5818 0 : JUMP_LABEL (newjump) = lab;
5819 0 : emit_barrier_after (newjump);
5820 : }
5821 0 : delete_insn (jump);
5822 :
5823 0 : if (can_merge_blocks_p (test_bb, other_bb))
5824 : {
5825 0 : merge_blocks (test_bb, other_bb);
5826 0 : num_true_changes++;
5827 : }
5828 :
5829 0 : num_updated_if_blocks++;
5830 0 : return true;
5831 : }
5832 :
5833 : /* Subroutine of find_cond_trap: if BB contains only a trap insn,
5834 : return it. */
5835 :
5836 : static rtx_insn *
5837 0 : block_has_only_trap (basic_block bb)
5838 : {
5839 0 : rtx_insn *trap;
5840 :
5841 : /* We're not the exit block. */
5842 0 : if (bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
5843 : return NULL;
5844 :
5845 : /* The block must have no successors. */
5846 0 : if (EDGE_COUNT (bb->succs) > 0)
5847 : return NULL;
5848 :
5849 : /* The only instruction in the THEN block must be the trap. */
5850 0 : trap = first_active_insn (bb);
5851 0 : if (! (trap == BB_END (bb)
5852 0 : && GET_CODE (PATTERN (trap)) == TRAP_IF
5853 0 : && TRAP_CONDITION (PATTERN (trap)) == const_true_rtx))
5854 0 : return NULL;
5855 :
5856 : return trap;
5857 : }
5858 :
5859 : /* Look for IF-THEN-ELSE cases in which one of THEN or ELSE is
5860 : transformable, but not necessarily the other. There need be no
5861 : JOIN block.
5862 :
5863 : Return TRUE if we were successful at converting the block.
5864 :
5865 : Cases we'd like to look at:
5866 :
5867 : (1)
5868 : if (test) goto over; // x not live
5869 : x = a;
5870 : goto label;
5871 : over:
5872 :
5873 : becomes
5874 :
5875 : x = a;
5876 : if (! test) goto label;
5877 :
5878 : (2)
5879 : if (test) goto E; // x not live
5880 : x = big();
5881 : goto L;
5882 : E:
5883 : x = b;
5884 : goto M;
5885 :
5886 : becomes
5887 :
5888 : x = b;
5889 : if (test) goto M;
5890 : x = big();
5891 : goto L;
5892 :
5893 : (3) // This one's really only interesting for targets that can do
5894 : // multiway branching, e.g. IA-64 BBB bundles. For other targets
5895 : // it results in multiple branches on a cache line, which often
5896 : // does not sit well with predictors.
5897 :
5898 : if (test1) goto E; // predicted not taken
5899 : x = a;
5900 : if (test2) goto F;
5901 : ...
5902 : E:
5903 : x = b;
5904 : J:
5905 :
5906 : becomes
5907 :
5908 : x = a;
5909 : if (test1) goto E;
5910 : if (test2) goto F;
5911 :
5912 : Notes:
5913 :
5914 : (A) Don't do (2) if the branch is predicted against the block we're
5915 : eliminating. Do it anyway if we can eliminate a branch; this requires
5916 : that the sole successor of the eliminated block postdominate the other
5917 : side of the if.
5918 :
5919 : (B) With CE, on (3) we can steal from both sides of the if, creating
5920 :
5921 : if (test1) x = a;
5922 : if (!test1) x = b;
5923 : if (test1) goto J;
5924 : if (test2) goto F;
5925 : ...
5926 : J:
5927 :
5928 : Again, this is most useful if J postdominates.
5929 :
5930 : (C) CE substitutes for helpful life information.
5931 :
5932 : (D) These heuristics need a lot of work. */
5933 :
5934 : /* Tests for case 1 above. */
5935 :
5936 : static bool
5937 14474294 : find_if_case_1 (basic_block test_bb, edge then_edge, edge else_edge)
5938 : {
5939 14474294 : basic_block then_bb = then_edge->dest;
5940 14474294 : basic_block else_bb = else_edge->dest;
5941 14474294 : basic_block new_bb;
5942 14474294 : int then_bb_index;
5943 14474294 : profile_probability then_prob;
5944 14474294 : rtx else_target = NULL_RTX;
5945 :
5946 : /* If we are partitioning hot/cold basic blocks, we don't want to
5947 : mess up unconditional or indirect jumps that cross between hot
5948 : and cold sections.
5949 :
5950 : Basic block partitioning may result in some jumps that appear to
5951 : be optimizable (or blocks that appear to be mergeable), but which really
5952 : must be left untouched (they are required to make it safely across
5953 : partition boundaries). See the comments at the top of
5954 : bb-reorder.cc:partition_hot_cold_basic_blocks for complete details. */
5955 :
5956 14474294 : if ((BB_END (then_bb)
5957 14474294 : && JUMP_P (BB_END (then_bb))
5958 7545595 : && CROSSING_JUMP_P (BB_END (then_bb)))
5959 14235669 : || (JUMP_P (BB_END (test_bb))
5960 14235669 : && CROSSING_JUMP_P (BB_END (test_bb)))
5961 28600178 : || (BB_END (else_bb)
5962 14125884 : && JUMP_P (BB_END (else_bb))
5963 7144070 : && CROSSING_JUMP_P (BB_END (else_bb))))
5964 : return false;
5965 :
5966 : /* Verify test_bb ends in a conditional jump with no other side-effects. */
5967 14102008 : if (!onlyjump_p (BB_END (test_bb)))
5968 : return false;
5969 :
5970 : /* THEN has one successor. */
5971 14101337 : if (!single_succ_p (then_bb))
5972 : return false;
5973 :
5974 : /* THEN does not fall through, but is not strange either. */
5975 6407999 : if (single_succ_edge (then_bb)->flags & (EDGE_COMPLEX | EDGE_FALLTHRU))
5976 : return false;
5977 :
5978 : /* THEN has one predecessor. */
5979 1268211 : if (!single_pred_p (then_bb))
5980 : return false;
5981 :
5982 : /* THEN must do something. */
5983 1144351 : if (forwarder_block_p (then_bb))
5984 : return false;
5985 :
5986 723133 : num_possible_if_blocks++;
5987 723133 : if (dump_file)
5988 8 : fprintf (dump_file,
5989 : "\nIF-CASE-1 found, start %d, then %d\n",
5990 : test_bb->index, then_bb->index);
5991 :
5992 723133 : then_prob = then_edge->probability.invert ();
5993 :
5994 : /* We're speculating from the THEN path, we want to make sure the cost
5995 : of speculation is within reason. */
5996 1368148 : if (! cheap_bb_rtx_cost_p (then_bb, then_prob,
5997 1368148 : COSTS_N_INSNS (BRANCH_COST (optimize_bb_for_speed_p (then_edge->src),
5998 : predictable_edge_p (then_edge)))))
5999 : return false;
6000 :
6001 144527 : if (else_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
6002 : {
6003 0 : rtx_insn *jump = BB_END (else_edge->src);
6004 0 : gcc_assert (JUMP_P (jump));
6005 0 : else_target = JUMP_LABEL (jump);
6006 : }
6007 :
6008 : /* Registers set are dead, or are predicable. */
6009 144527 : if (! dead_or_predicable (test_bb, then_bb, else_bb,
6010 : single_succ_edge (then_bb), true))
6011 : return false;
6012 :
6013 : /* Conversion went ok, including moving the insns and fixing up the
6014 : jump. Adjust the CFG to match. */
6015 :
6016 : /* We can avoid creating a new basic block if then_bb is immediately
6017 : followed by else_bb, i.e. deleting then_bb allows test_bb to fall
6018 : through to else_bb. */
6019 :
6020 21836 : if (then_bb->next_bb == else_bb
6021 10241 : && then_bb->prev_bb == test_bb
6022 10241 : && else_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
6023 : {
6024 10241 : redirect_edge_succ (FALLTHRU_EDGE (test_bb), else_bb);
6025 10241 : new_bb = 0;
6026 : }
6027 11595 : else if (else_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
6028 0 : new_bb = force_nonfallthru_and_redirect (FALLTHRU_EDGE (test_bb),
6029 : else_bb, else_target);
6030 : else
6031 11595 : new_bb = redirect_edge_and_branch_force (FALLTHRU_EDGE (test_bb),
6032 : else_bb);
6033 :
6034 21836 : df_set_bb_dirty (test_bb);
6035 21836 : df_set_bb_dirty (else_bb);
6036 :
6037 21836 : then_bb_index = then_bb->index;
6038 21836 : delete_basic_block (then_bb);
6039 :
6040 : /* Make rest of code believe that the newly created block is the THEN_BB
6041 : block we removed. */
6042 21836 : if (new_bb)
6043 : {
6044 11595 : df_bb_replace (then_bb_index, new_bb);
6045 : /* This should have been done above via force_nonfallthru_and_redirect
6046 : (possibly called from redirect_edge_and_branch_force). */
6047 11595 : gcc_checking_assert (BB_PARTITION (new_bb) == BB_PARTITION (test_bb));
6048 : }
6049 :
6050 21836 : num_true_changes++;
6051 21836 : num_updated_if_blocks++;
6052 21836 : return true;
6053 : }
6054 :
6055 : /* Test for case 2 above. */
6056 :
6057 : static bool
6058 14452458 : find_if_case_2 (basic_block test_bb, edge then_edge, edge else_edge)
6059 : {
6060 14452458 : basic_block then_bb = then_edge->dest;
6061 14452458 : basic_block else_bb = else_edge->dest;
6062 14452458 : edge else_succ;
6063 14452458 : profile_probability then_prob, else_prob;
6064 :
6065 : /* We do not want to speculate (empty) loop latches. */
6066 14452458 : if (current_loops
6067 5630184 : && else_bb->loop_father->latch == else_bb)
6068 : return false;
6069 :
6070 : /* If we are partitioning hot/cold basic blocks, we don't want to
6071 : mess up unconditional or indirect jumps that cross between hot
6072 : and cold sections.
6073 :
6074 : Basic block partitioning may result in some jumps that appear to
6075 : be optimizable (or blocks that appear to be mergeable), but which really
6076 : must be left untouched (they are required to make it safely across
6077 : partition boundaries). See the comments at the top of
6078 : bb-reorder.cc:partition_hot_cold_basic_blocks for complete details. */
6079 :
6080 14299472 : if ((BB_END (then_bb)
6081 14299472 : && JUMP_P (BB_END (then_bb))
6082 7449210 : && CROSSING_JUMP_P (BB_END (then_bb)))
6083 14060847 : || (JUMP_P (BB_END (test_bb))
6084 14060847 : && CROSSING_JUMP_P (BB_END (test_bb)))
6085 28250534 : || (BB_END (else_bb)
6086 13951062 : && JUMP_P (BB_END (else_bb))
6087 7002965 : && CROSSING_JUMP_P (BB_END (else_bb))))
6088 : return false;
6089 :
6090 : /* Verify test_bb ends in a conditional jump with no other side-effects. */
6091 13927186 : if (!onlyjump_p (BB_END (test_bb)))
6092 : return false;
6093 :
6094 : /* ELSE has one successor. */
6095 13926516 : if (!single_succ_p (else_bb))
6096 : return false;
6097 : else
6098 5876321 : else_succ = single_succ_edge (else_bb);
6099 :
6100 : /* ELSE outgoing edge is not complex. */
6101 5876321 : if (else_succ->flags & EDGE_COMPLEX)
6102 : return false;
6103 :
6104 : /* ELSE has one predecessor. */
6105 5718177 : if (!single_pred_p (else_bb))
6106 : return false;
6107 :
6108 : /* THEN is not EXIT. */
6109 2995839 : if (then_bb->index < NUM_FIXED_BLOCKS)
6110 : return false;
6111 :
6112 2995839 : else_prob = else_edge->probability;
6113 2995839 : then_prob = else_prob.invert ();
6114 :
6115 : /* ELSE is predicted or SUCC(ELSE) postdominates THEN. */
6116 2995839 : if (else_prob > then_prob)
6117 : ;
6118 2018181 : else if (else_succ->dest->index < NUM_FIXED_BLOCKS
6119 2018181 : || dominated_by_p (CDI_POST_DOMINATORS, then_bb,
6120 : else_succ->dest))
6121 : ;
6122 : else
6123 : return false;
6124 :
6125 2372418 : num_possible_if_blocks++;
6126 2372418 : if (dump_file)
6127 44 : fprintf (dump_file,
6128 : "\nIF-CASE-2 found, start %d, else %d\n",
6129 : test_bb->index, else_bb->index);
6130 :
6131 : /* We're speculating from the ELSE path, we want to make sure the cost
6132 : of speculation is within reason. */
6133 4571579 : if (! cheap_bb_rtx_cost_p (else_bb, else_prob,
6134 4571579 : COSTS_N_INSNS (BRANCH_COST (optimize_bb_for_speed_p (else_edge->src),
6135 : predictable_edge_p (else_edge)))))
6136 : return false;
6137 :
6138 : /* Registers set are dead, or are predicable. */
6139 461867 : if (! dead_or_predicable (test_bb, else_bb, then_bb, else_succ, false))
6140 : return false;
6141 :
6142 : /* Conversion went ok, including moving the insns and fixing up the
6143 : jump. Adjust the CFG to match. */
6144 :
6145 127431 : df_set_bb_dirty (test_bb);
6146 127431 : df_set_bb_dirty (then_bb);
6147 127431 : delete_basic_block (else_bb);
6148 :
6149 127431 : num_true_changes++;
6150 127431 : num_updated_if_blocks++;
6151 :
6152 : /* ??? We may now fallthru from one of THEN's successors into a join
6153 : block. Rerun cleanup_cfg? Examine things manually? Wait? */
6154 :
6155 127431 : return true;
6156 : }
6157 :
6158 : /* Used by the code above to perform the actual rtl transformations.
6159 : Return TRUE if successful.
6160 :
6161 : TEST_BB is the block containing the conditional branch. MERGE_BB
6162 : is the block containing the code to manipulate. DEST_EDGE is an
6163 : edge representing a jump to the join block; after the conversion,
6164 : TEST_BB should be branching to its destination.
6165 : REVERSEP is true if the sense of the branch should be reversed. */
6166 :
6167 : static bool
6168 606394 : dead_or_predicable (basic_block test_bb, basic_block merge_bb,
6169 : basic_block other_bb, edge dest_edge, bool reversep)
6170 : {
6171 606394 : basic_block new_dest = dest_edge->dest;
6172 606394 : rtx_insn *head, *end, *jump;
6173 606394 : rtx_insn *earliest = NULL;
6174 606394 : rtx old_dest;
6175 606394 : bitmap merge_set = NULL;
6176 : /* Number of pending changes. */
6177 606394 : int n_validated_changes = 0;
6178 606394 : rtx new_dest_label = NULL_RTX;
6179 :
6180 606394 : jump = BB_END (test_bb);
6181 :
6182 : /* Find the extent of the real code in the merge block. */
6183 606394 : head = BB_HEAD (merge_bb);
6184 606394 : end = BB_END (merge_bb);
6185 :
6186 790693 : while (DEBUG_INSN_P (end) && end != head)
6187 184299 : end = PREV_INSN (end);
6188 :
6189 : /* If merge_bb ends with a tablejump, predicating/moving insn's
6190 : into test_bb and then deleting merge_bb will result in the jumptable
6191 : that follows merge_bb being removed along with merge_bb and then we
6192 : get an unresolved reference to the jumptable. */
6193 606394 : if (tablejump_p (end, NULL, NULL))
6194 : return false;
6195 :
6196 606394 : if (LABEL_P (head))
6197 461930 : head = NEXT_INSN (head);
6198 606394 : while (DEBUG_INSN_P (head) && head != end)
6199 0 : head = NEXT_INSN (head);
6200 606394 : if (NOTE_P (head))
6201 : {
6202 606394 : if (head == end)
6203 : {
6204 104220 : head = end = NULL;
6205 104220 : goto no_body;
6206 : }
6207 502174 : head = NEXT_INSN (head);
6208 1452996 : while (DEBUG_INSN_P (head) && head != end)
6209 448648 : head = NEXT_INSN (head);
6210 : }
6211 :
6212 502174 : if (JUMP_P (end))
6213 : {
6214 207150 : if (!onlyjump_p (end))
6215 : return false;
6216 165904 : if (head == end)
6217 : {
6218 6 : head = end = NULL;
6219 6 : goto no_body;
6220 : }
6221 165898 : end = PREV_INSN (end);
6222 440687 : while (DEBUG_INSN_P (end) && end != head)
6223 108891 : end = PREV_INSN (end);
6224 : }
6225 :
6226 : /* Don't move frame-related insn across the conditional branch. This
6227 : can lead to one of the paths of the branch having wrong unwind info. */
6228 460922 : if (epilogue_completed)
6229 : {
6230 : rtx_insn *insn = head;
6231 85030 : while (1)
6232 : {
6233 293473 : if (INSN_P (insn) && RTX_FRAME_RELATED_P (insn))
6234 : return false;
6235 286686 : if (insn == end)
6236 : break;
6237 85030 : insn = NEXT_INSN (insn);
6238 85030 : }
6239 : }
6240 :
6241 : /* Disable handling dead code by conditional execution if the machine needs
6242 : to do anything funny with the tests, etc. */
6243 : #ifndef IFCVT_MODIFY_TESTS
6244 454135 : if (targetm.have_conditional_execution ())
6245 : {
6246 : /* In the conditional execution case, we have things easy. We know
6247 : the condition is reversible. We don't have to check life info
6248 : because we're going to conditionally execute the code anyway.
6249 : All that's left is making sure the insns involved can actually
6250 : be predicated. */
6251 :
6252 0 : rtx cond;
6253 :
6254 : /* If the conditional jump is more than just a conditional jump,
6255 : then we cannot do conditional execution conversion on this block. */
6256 0 : if (!onlyjump_p (jump))
6257 0 : goto nce;
6258 :
6259 0 : cond = cond_exec_get_condition (jump);
6260 0 : if (! cond)
6261 0 : goto nce;
6262 :
6263 0 : rtx note = find_reg_note (jump, REG_BR_PROB, NULL_RTX);
6264 0 : profile_probability prob_val
6265 0 : = (note ? profile_probability::from_reg_br_prob_note (XINT (note, 0))
6266 : : profile_probability::uninitialized ());
6267 :
6268 0 : if (reversep)
6269 : {
6270 0 : enum rtx_code rev = reversed_comparison_code (cond, jump);
6271 0 : if (rev == UNKNOWN)
6272 0 : return false;
6273 0 : cond = gen_rtx_fmt_ee (rev, GET_MODE (cond), XEXP (cond, 0),
6274 : XEXP (cond, 1));
6275 0 : prob_val = prob_val.invert ();
6276 : }
6277 :
6278 0 : if (cond_exec_process_insns (NULL, head, end, cond, prob_val, false)
6279 0 : && verify_changes (0))
6280 0 : n_validated_changes = num_validated_changes ();
6281 : else
6282 0 : cancel_changes (0);
6283 :
6284 0 : earliest = jump;
6285 : }
6286 454135 : nce:
6287 : #endif
6288 :
6289 : /* If we allocated new pseudos (e.g. in the conditional move
6290 : expander called from noce_emit_cmove), we must resize the
6291 : array first. */
6292 454135 : if (max_regno < max_reg_num ())
6293 81555 : max_regno = max_reg_num ();
6294 :
6295 : /* Try the NCE path if the CE path did not result in any changes. */
6296 454135 : if (n_validated_changes == 0)
6297 : {
6298 454135 : rtx cond;
6299 454135 : rtx_insn *insn;
6300 454135 : regset live;
6301 454135 : bool success;
6302 :
6303 : /* In the non-conditional execution case, we have to verify that there
6304 : are no trapping operations, no calls, no references to memory, and
6305 : that any registers modified are dead at the branch site. */
6306 :
6307 454135 : if (!any_condjump_p (jump))
6308 : return false;
6309 :
6310 : /* Find the extent of the conditional. */
6311 454135 : cond = noce_get_condition (jump, &earliest, false);
6312 454135 : if (!cond)
6313 : return false;
6314 :
6315 453351 : live = BITMAP_ALLOC (®_obstack);
6316 453351 : simulate_backwards_to_point (merge_bb, live, end);
6317 453351 : success = can_move_insns_across (head, end, earliest, jump,
6318 : merge_bb, live,
6319 : df_get_live_in (other_bb), NULL);
6320 453351 : BITMAP_FREE (live);
6321 453351 : if (!success)
6322 : return false;
6323 :
6324 : /* Collect the set of registers set in MERGE_BB. */
6325 146715 : merge_set = BITMAP_ALLOC (®_obstack);
6326 :
6327 733527 : FOR_BB_INSNS (merge_bb, insn)
6328 586812 : if (NONDEBUG_INSN_P (insn))
6329 192930 : df_simulate_find_defs (insn, merge_set);
6330 :
6331 : /* If shrink-wrapping, disable this optimization when test_bb is
6332 : the first basic block and merge_bb exits. The idea is to not
6333 : move code setting up a return register as that may clobber a
6334 : register used to pass function parameters, which then must be
6335 : saved in caller-saved regs. A caller-saved reg requires the
6336 : prologue, killing a shrink-wrap opportunity. */
6337 146714 : if ((SHRINK_WRAPPING_ENABLED && !epilogue_completed)
6338 115574 : && ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb == test_bb
6339 19030 : && single_succ_p (new_dest)
6340 14043 : && single_succ (new_dest) == EXIT_BLOCK_PTR_FOR_FN (cfun)
6341 159735 : && bitmap_intersect_p (df_get_live_in (new_dest), merge_set))
6342 : {
6343 13020 : regset return_regs;
6344 13020 : unsigned int i;
6345 :
6346 13020 : return_regs = BITMAP_ALLOC (®_obstack);
6347 :
6348 : /* Start off with the intersection of regs used to pass
6349 : params and regs used to return values. */
6350 1223880 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
6351 1197840 : if (FUNCTION_ARG_REGNO_P (i)
6352 1197840 : && targetm.calls.function_value_regno_p (i))
6353 50910 : bitmap_set_bit (return_regs, INCOMING_REGNO (i));
6354 :
6355 13020 : bitmap_and_into (return_regs,
6356 13020 : df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
6357 13020 : bitmap_and_into (return_regs,
6358 13020 : df_get_live_in (EXIT_BLOCK_PTR_FOR_FN (cfun)));
6359 13020 : if (!bitmap_empty_p (return_regs))
6360 : {
6361 4786 : FOR_BB_INSNS_REVERSE (new_dest, insn)
6362 4239 : if (NONDEBUG_INSN_P (insn))
6363 : {
6364 2157 : df_ref def;
6365 :
6366 : /* If this insn sets any reg in return_regs, add all
6367 : reg uses to the set of regs we're interested in. */
6368 2734 : FOR_EACH_INSN_DEF (def, insn)
6369 1676 : if (bitmap_bit_p (return_regs, DF_REF_REGNO (def)))
6370 : {
6371 1099 : df_simulate_uses (insn, return_regs);
6372 1099 : break;
6373 : }
6374 : }
6375 547 : if (bitmap_intersect_p (merge_set, return_regs))
6376 : {
6377 529 : BITMAP_FREE (return_regs);
6378 529 : BITMAP_FREE (merge_set);
6379 529 : return false;
6380 : }
6381 : }
6382 12491 : BITMAP_FREE (return_regs);
6383 : }
6384 : }
6385 :
6386 250412 : no_body:
6387 : /* We don't want to use normal invert_jump or redirect_jump because
6388 : we don't want to delete_insn called. Also, we want to do our own
6389 : change group management. */
6390 :
6391 250412 : old_dest = JUMP_LABEL (jump);
6392 250412 : if (other_bb != new_dest)
6393 : {
6394 250397 : if (!any_condjump_p (jump))
6395 0 : goto cancel;
6396 :
6397 250397 : if (JUMP_P (BB_END (dest_edge->src)))
6398 22962 : new_dest_label = JUMP_LABEL (BB_END (dest_edge->src));
6399 227435 : else if (new_dest == EXIT_BLOCK_PTR_FOR_FN (cfun))
6400 101145 : new_dest_label = ret_rtx;
6401 : else
6402 126290 : new_dest_label = block_label (new_dest);
6403 :
6404 250397 : rtx_jump_insn *jump_insn = as_a <rtx_jump_insn *> (jump);
6405 250397 : if (reversep
6406 250397 : ? ! invert_jump_1 (jump_insn, new_dest_label)
6407 228561 : : ! redirect_jump_1 (jump_insn, new_dest_label))
6408 0 : goto cancel;
6409 : }
6410 :
6411 250412 : if (verify_changes (n_validated_changes))
6412 149267 : confirm_change_group ();
6413 : else
6414 101145 : goto cancel;
6415 :
6416 149267 : if (other_bb != new_dest)
6417 : {
6418 149252 : redirect_jump_2 (as_a <rtx_jump_insn *> (jump), old_dest, new_dest_label,
6419 : 0, reversep);
6420 :
6421 149252 : redirect_edge_succ (BRANCH_EDGE (test_bb), new_dest);
6422 149252 : if (reversep)
6423 : {
6424 21836 : std::swap (BRANCH_EDGE (test_bb)->probability,
6425 21836 : FALLTHRU_EDGE (test_bb)->probability);
6426 21836 : update_br_prob_note (test_bb);
6427 : }
6428 : }
6429 :
6430 : /* Move the insns out of MERGE_BB to before the branch. */
6431 149267 : if (head != NULL)
6432 : {
6433 146186 : rtx_insn *insn;
6434 :
6435 146186 : if (end == BB_END (merge_bb))
6436 112065 : BB_END (merge_bb) = PREV_INSN (head);
6437 :
6438 : /* PR 21767: when moving insns above a conditional branch, the REG_EQUAL
6439 : notes being moved might become invalid. */
6440 146186 : insn = head;
6441 176624 : do
6442 : {
6443 176624 : rtx note;
6444 :
6445 176624 : if (! INSN_P (insn))
6446 3208 : continue;
6447 173416 : note = find_reg_note (insn, REG_EQUAL, NULL_RTX);
6448 173416 : if (! note)
6449 161802 : continue;
6450 11614 : remove_note (insn, note);
6451 207062 : } while (insn != end && (insn = NEXT_INSN (insn)));
6452 :
6453 : /* PR46315: when moving insns above a conditional branch, the REG_EQUAL
6454 : notes referring to the registers being set might become invalid. */
6455 146186 : if (merge_set)
6456 : {
6457 146186 : unsigned i;
6458 146186 : bitmap_iterator bi;
6459 :
6460 330790 : EXECUTE_IF_SET_IN_BITMAP (merge_set, 0, i, bi)
6461 184604 : remove_reg_equal_equiv_notes_for_regno (i);
6462 :
6463 146186 : BITMAP_FREE (merge_set);
6464 : }
6465 :
6466 146186 : reorder_insns (head, end, PREV_INSN (earliest));
6467 : }
6468 :
6469 : /* Remove the jump and edge if we can. */
6470 149267 : if (other_bb == new_dest)
6471 : {
6472 15 : delete_insn (jump);
6473 15 : remove_edge (BRANCH_EDGE (test_bb));
6474 : /* ??? Can't merge blocks here, as then_bb is still in use.
6475 : At minimum, the merge will get done just before bb-reorder. */
6476 : }
6477 :
6478 : return true;
6479 :
6480 101145 : cancel:
6481 101145 : cancel_changes (0);
6482 :
6483 101145 : if (merge_set)
6484 0 : BITMAP_FREE (merge_set);
6485 :
6486 : return false;
6487 : }
6488 :
6489 : /* Main entry point for all if-conversion. AFTER_COMBINE is true if
6490 : we are after combine pass. */
6491 :
6492 : static void
6493 3179761 : if_convert (bool after_combine)
6494 : {
6495 3179761 : basic_block bb;
6496 3179761 : int pass;
6497 :
6498 3179761 : if (optimize == 1)
6499 : {
6500 235906 : df_live_add_problem ();
6501 235906 : df_live_set_all_dirty ();
6502 : }
6503 :
6504 : /* Record whether we are after combine pass. */
6505 3179761 : ifcvt_after_combine = after_combine;
6506 3179761 : have_cbranchcc4 = (direct_optab_handler (cbranch_optab, CCmode)
6507 3179761 : != CODE_FOR_nothing);
6508 3179761 : num_possible_if_blocks = 0;
6509 3179761 : num_updated_if_blocks = 0;
6510 3179761 : num_true_changes = 0;
6511 :
6512 3179761 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
6513 3179761 : mark_loop_exit_edges ();
6514 3179761 : loop_optimizer_finalize ();
6515 3179761 : free_dominance_info (CDI_DOMINATORS);
6516 :
6517 : /* Compute postdominators. */
6518 3179761 : calculate_dominance_info (CDI_POST_DOMINATORS);
6519 :
6520 3179761 : df_set_flags (DF_LR_RUN_DCE);
6521 :
6522 : /* Go through each of the basic blocks looking for things to convert. If we
6523 : have conditional execution, we make multiple passes to allow us to handle
6524 : IF-THEN{-ELSE} blocks within other IF-THEN{-ELSE} blocks. */
6525 3179761 : pass = 0;
6526 3345270 : do
6527 : {
6528 3345270 : df_analyze ();
6529 : /* Only need to do dce on the first pass. */
6530 3345270 : df_clear_flags (DF_LR_RUN_DCE);
6531 3345270 : cond_exec_changed_p = false;
6532 3345270 : pass++;
6533 :
6534 : #ifdef IFCVT_MULTIPLE_DUMPS
6535 3345270 : if (dump_file && pass > 1)
6536 21 : fprintf (dump_file, "\n\n========== Pass %d ==========\n", pass);
6537 : #endif
6538 :
6539 43637812 : FOR_EACH_BB_FN (bb, cfun)
6540 : {
6541 : basic_block new_bb;
6542 40612233 : while (!df_get_bb_dirty (bb)
6543 80418663 : && (new_bb = find_if_header (bb, pass)) != NULL)
6544 : bb = new_bb;
6545 : }
6546 :
6547 : #ifdef IFCVT_MULTIPLE_DUMPS
6548 3345270 : if (dump_file && cond_exec_changed_p)
6549 21 : print_rtl_with_bb (dump_file, get_insns (), dump_flags);
6550 : #endif
6551 : }
6552 : while (cond_exec_changed_p);
6553 :
6554 : #ifdef IFCVT_MULTIPLE_DUMPS
6555 3179761 : if (dump_file)
6556 87 : fprintf (dump_file, "\n\n========== no more changes\n");
6557 : #endif
6558 :
6559 3179761 : free_dominance_info (CDI_POST_DOMINATORS);
6560 :
6561 3179761 : if (dump_file)
6562 87 : fflush (dump_file);
6563 :
6564 3179761 : clear_aux_for_blocks ();
6565 :
6566 : /* If we allocated new pseudos, we must resize the array for sched1. */
6567 3179761 : if (max_regno < max_reg_num ())
6568 647616 : max_regno = max_reg_num ();
6569 :
6570 : /* Write the final stats. */
6571 3179761 : if (dump_file && num_possible_if_blocks > 0)
6572 : {
6573 61 : fprintf (dump_file,
6574 : "\n%d possible IF blocks searched.\n",
6575 : num_possible_if_blocks);
6576 61 : fprintf (dump_file,
6577 : "%d IF blocks converted.\n",
6578 : num_updated_if_blocks);
6579 61 : fprintf (dump_file,
6580 : "%d true changes made.\n\n\n",
6581 : num_true_changes);
6582 : }
6583 :
6584 3179761 : if (optimize == 1)
6585 235906 : df_remove_problem (df_live);
6586 :
6587 : /* Some non-cold blocks may now be only reachable from cold blocks.
6588 : Fix that up. */
6589 3179761 : fixup_partitions ();
6590 :
6591 3179761 : checking_verify_flow_info ();
6592 3179761 : }
6593 :
6594 : /* If-conversion and CFG cleanup. */
6595 : static void
6596 1062343 : rest_of_handle_if_conversion (void)
6597 : {
6598 1062343 : int flags = 0;
6599 :
6600 1062343 : if (flag_if_conversion)
6601 : {
6602 1059916 : if (dump_file)
6603 : {
6604 35 : dump_reg_info (dump_file);
6605 35 : dump_flow_info (dump_file, dump_flags);
6606 : }
6607 1059916 : cleanup_cfg (CLEANUP_EXPENSIVE);
6608 1059916 : if_convert (false);
6609 1059916 : if (num_updated_if_blocks)
6610 : /* Get rid of any dead CC-related instructions. */
6611 1062343 : flags |= CLEANUP_FORCE_FAST_DCE;
6612 : }
6613 :
6614 1062343 : cleanup_cfg (flags);
6615 1062343 : }
6616 :
6617 : namespace {
6618 :
6619 : const pass_data pass_data_rtl_ifcvt =
6620 : {
6621 : RTL_PASS, /* type */
6622 : "ce1", /* name */
6623 : OPTGROUP_NONE, /* optinfo_flags */
6624 : TV_IFCVT, /* tv_id */
6625 : 0, /* properties_required */
6626 : 0, /* properties_provided */
6627 : 0, /* properties_destroyed */
6628 : 0, /* todo_flags_start */
6629 : TODO_df_finish, /* todo_flags_finish */
6630 : };
6631 :
6632 : class pass_rtl_ifcvt : public rtl_opt_pass
6633 : {
6634 : public:
6635 294196 : pass_rtl_ifcvt (gcc::context *ctxt)
6636 588392 : : rtl_opt_pass (pass_data_rtl_ifcvt, ctxt)
6637 : {}
6638 :
6639 : /* opt_pass methods: */
6640 1515129 : bool gate (function *) final override
6641 : {
6642 1515129 : return (optimize > 0) && dbg_cnt (if_conversion);
6643 : }
6644 :
6645 1062343 : unsigned int execute (function *) final override
6646 : {
6647 1062343 : rest_of_handle_if_conversion ();
6648 1062343 : return 0;
6649 : }
6650 :
6651 : }; // class pass_rtl_ifcvt
6652 :
6653 : } // anon namespace
6654 :
6655 : rtl_opt_pass *
6656 294196 : make_pass_rtl_ifcvt (gcc::context *ctxt)
6657 : {
6658 294196 : return new pass_rtl_ifcvt (ctxt);
6659 : }
6660 :
6661 :
6662 : /* Rerun if-conversion, as combine may have simplified things enough
6663 : to now meet sequence length restrictions. */
6664 :
6665 : namespace {
6666 :
6667 : const pass_data pass_data_if_after_combine =
6668 : {
6669 : RTL_PASS, /* type */
6670 : "ce2", /* name */
6671 : OPTGROUP_NONE, /* optinfo_flags */
6672 : TV_IFCVT, /* tv_id */
6673 : 0, /* properties_required */
6674 : 0, /* properties_provided */
6675 : 0, /* properties_destroyed */
6676 : 0, /* todo_flags_start */
6677 : TODO_df_finish, /* todo_flags_finish */
6678 : };
6679 :
6680 : class pass_if_after_combine : public rtl_opt_pass
6681 : {
6682 : public:
6683 294196 : pass_if_after_combine (gcc::context *ctxt)
6684 588392 : : rtl_opt_pass (pass_data_if_after_combine, ctxt)
6685 : {}
6686 :
6687 : /* opt_pass methods: */
6688 1515129 : bool gate (function *) final override
6689 : {
6690 1062344 : return optimize > 0 && flag_if_conversion
6691 2575046 : && dbg_cnt (if_after_combine);
6692 : }
6693 :
6694 1059916 : unsigned int execute (function *) final override
6695 : {
6696 1059916 : if_convert (true);
6697 1059916 : return 0;
6698 : }
6699 :
6700 : }; // class pass_if_after_combine
6701 :
6702 : } // anon namespace
6703 :
6704 : rtl_opt_pass *
6705 294196 : make_pass_if_after_combine (gcc::context *ctxt)
6706 : {
6707 294196 : return new pass_if_after_combine (ctxt);
6708 : }
6709 :
6710 :
6711 : namespace {
6712 :
6713 : const pass_data pass_data_if_after_reload =
6714 : {
6715 : RTL_PASS, /* type */
6716 : "ce3", /* name */
6717 : OPTGROUP_NONE, /* optinfo_flags */
6718 : TV_IFCVT2, /* tv_id */
6719 : 0, /* properties_required */
6720 : 0, /* properties_provided */
6721 : 0, /* properties_destroyed */
6722 : 0, /* todo_flags_start */
6723 : TODO_df_finish, /* todo_flags_finish */
6724 : };
6725 :
6726 : class pass_if_after_reload : public rtl_opt_pass
6727 : {
6728 : public:
6729 294196 : pass_if_after_reload (gcc::context *ctxt)
6730 588392 : : rtl_opt_pass (pass_data_if_after_reload, ctxt)
6731 : {}
6732 :
6733 : /* opt_pass methods: */
6734 1515129 : bool gate (function *) final override
6735 : {
6736 1062344 : return optimize > 0 && flag_if_conversion2
6737 2575059 : && dbg_cnt (if_after_reload);
6738 : }
6739 :
6740 1059929 : unsigned int execute (function *) final override
6741 : {
6742 1059929 : if_convert (true);
6743 1059929 : return 0;
6744 : }
6745 :
6746 : }; // class pass_if_after_reload
6747 :
6748 : } // anon namespace
6749 :
6750 : rtl_opt_pass *
6751 294196 : make_pass_if_after_reload (gcc::context *ctxt)
6752 : {
6753 294196 : return new pass_if_after_reload (ctxt);
6754 : }
|
