Line data Source code
1 : /* If-conversion support.
2 : Copyright (C) 2000-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 : or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
14 : License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "cfghooks.h"
28 : #include "df.h"
29 : #include "memmodel.h"
30 : #include "tm_p.h"
31 : #include "expmed.h"
32 : #include "optabs.h"
33 : #include "regs.h"
34 : #include "emit-rtl.h"
35 : #include "recog.h"
36 :
37 : #include "cfgrtl.h"
38 : #include "cfganal.h"
39 : #include "cfgcleanup.h"
40 : #include "expr.h"
41 : #include "output.h"
42 : #include "cfgloop.h"
43 : #include "tree-pass.h"
44 : #include "dbgcnt.h"
45 : #include "shrink-wrap.h"
46 : #include "rtl-iter.h"
47 : #include "ifcvt.h"
48 :
49 : #ifndef MAX_CONDITIONAL_EXECUTE
50 : #define MAX_CONDITIONAL_EXECUTE \
51 : (BRANCH_COST (optimize_function_for_speed_p (cfun), false) \
52 : + 1)
53 : #endif
54 :
55 : #define IFCVT_MULTIPLE_DUMPS 1
56 :
57 : #define NULL_BLOCK ((basic_block) NULL)
58 :
59 : /* True if after combine pass. */
60 : static bool ifcvt_after_combine;
61 :
62 : /* True if the target has the cbranchcc4 optab. */
63 : static bool have_cbranchcc4;
64 :
65 : /* # of IF-THEN or IF-THEN-ELSE blocks we looked at */
66 : static int num_possible_if_blocks;
67 :
68 : /* # of IF-THEN or IF-THEN-ELSE blocks were converted to conditional
69 : execution. */
70 : static int num_updated_if_blocks;
71 :
72 : /* # of changes made. */
73 : static int num_true_changes;
74 :
75 : /* Whether conditional execution changes were made. */
76 : static bool cond_exec_changed_p;
77 :
78 : /* Forward references. */
79 : static int count_bb_insns (const_basic_block);
80 : static bool cheap_bb_rtx_cost_p (const_basic_block, profile_probability, int);
81 : static rtx_insn *first_active_insn (basic_block);
82 : static rtx_insn *last_active_insn (basic_block, bool);
83 : static rtx_insn *find_active_insn_before (basic_block, rtx_insn *);
84 : static rtx_insn *find_active_insn_after (basic_block, rtx_insn *);
85 : static basic_block block_fallthru (basic_block);
86 : static rtx cond_exec_get_condition (rtx_insn *, bool);
87 : static rtx noce_get_condition (rtx_insn *, rtx_insn **, bool);
88 : static bool noce_operand_ok (const_rtx);
89 : static void merge_if_block (ce_if_block *);
90 : static bool find_cond_trap (basic_block, edge, edge);
91 : static basic_block find_if_header (basic_block, int);
92 : static int block_jumps_and_fallthru (basic_block, basic_block);
93 : static bool noce_find_if_block (basic_block, edge, edge, int);
94 : static bool cond_exec_find_if_block (ce_if_block *);
95 : static bool find_if_case_1 (basic_block, edge, edge);
96 : static bool find_if_case_2 (basic_block, edge, edge);
97 : static bool dead_or_predicable (basic_block, basic_block, basic_block,
98 : edge, bool);
99 : static void noce_emit_move_insn (rtx, rtx);
100 : static rtx_insn *block_has_only_trap (basic_block);
101 : static void init_noce_multiple_sets_info (basic_block,
102 : auto_delete_vec<noce_multiple_sets_info> &);
103 : static bool noce_convert_multiple_sets_1 (struct noce_if_info *,
104 : auto_delete_vec<noce_multiple_sets_info> &, int *, bool *);
105 :
106 : /* Count the number of non-jump active insns in BB. */
107 :
108 : static int
109 0 : count_bb_insns (const_basic_block bb)
110 : {
111 0 : int count = 0;
112 0 : rtx_insn *insn = BB_HEAD (bb);
113 :
114 0 : while (1)
115 : {
116 0 : if (active_insn_p (insn) && !JUMP_P (insn))
117 0 : count++;
118 :
119 0 : if (insn == BB_END (bb))
120 : break;
121 0 : insn = NEXT_INSN (insn);
122 : }
123 :
124 0 : return count;
125 : }
126 :
127 : /* Determine whether the total insn_cost on non-jump insns in
128 : basic block BB is less than MAX_COST. This function returns
129 : false if the cost of any instruction could not be estimated.
130 :
131 : The cost of the non-jump insns in BB is scaled by REG_BR_PROB_BASE
132 : as those insns are being speculated. MAX_COST is scaled with SCALE
133 : plus a small fudge factor. */
134 :
135 : static bool
136 2878964 : cheap_bb_rtx_cost_p (const_basic_block bb,
137 : profile_probability prob, int max_cost)
138 : {
139 2878964 : int count = 0;
140 2878964 : rtx_insn *insn = BB_HEAD (bb);
141 2878964 : bool speed = optimize_bb_for_speed_p (bb);
142 2878964 : int scale = prob.initialized_p () ? prob.to_reg_br_prob_base ()
143 : : REG_BR_PROB_BASE;
144 :
145 : /* Set scale to REG_BR_PROB_BASE to void the identical scaling
146 : applied to insn_cost when optimizing for size. Only do
147 : this after combine because if-conversion might interfere with
148 : passes before combine.
149 :
150 : Use optimize_function_for_speed_p instead of the pre-defined
151 : variable speed to make sure it is set to same value for all
152 : basic blocks in one if-conversion transformation. */
153 2878964 : if (!optimize_function_for_speed_p (cfun) && ifcvt_after_combine)
154 : scale = REG_BR_PROB_BASE;
155 : /* Our branch probability/scaling factors are just estimates and don't
156 : account for cases where we can get speculation for free and other
157 : secondary benefits. So we fudge the scale factor to make speculating
158 : appear a little more profitable when optimizing for performance. */
159 : else
160 2827697 : scale += REG_BR_PROB_BASE / 8;
161 :
162 :
163 2878964 : max_cost *= scale;
164 :
165 11506490 : while (1)
166 : {
167 14385454 : if (NONJUMP_INSN_P (insn))
168 : {
169 : /* Inline-asm's cost is not very estimatable.
170 : It could be a costly instruction but the
171 : estimate would be the same as a non costly
172 : instruction. */
173 4277527 : if (asm_noperands (PATTERN (insn)) >= 0)
174 : return false;
175 :
176 4274770 : int cost = insn_cost (insn, speed) * REG_BR_PROB_BASE;
177 4274770 : if (cost == 0)
178 : return false;
179 :
180 : /* If this instruction is the load or set of a "stack" register,
181 : such as a floating point register on x87, then the cost of
182 : speculatively executing this insn may need to include
183 : the additional cost of popping its result off of the
184 : register stack. Unfortunately, correctly recognizing and
185 : accounting for this additional overhead is tricky, so for
186 : now we simply prohibit such speculative execution. */
187 : #ifdef STACK_REGS
188 4225428 : {
189 4225428 : rtx set = single_set (insn);
190 4225428 : if (set && STACK_REG_P (SET_DEST (set)))
191 : return false;
192 : }
193 : #endif
194 :
195 4221655 : count += cost;
196 4221655 : if (count >= max_cost)
197 : return false;
198 : }
199 10107927 : else if (CALL_P (insn))
200 : return false;
201 :
202 12067948 : if (insn == BB_END (bb))
203 : break;
204 11506490 : insn = NEXT_INSN (insn);
205 11506490 : }
206 :
207 : return true;
208 : }
209 :
210 : /* Return the first non-jump active insn in the basic block. */
211 :
212 : static rtx_insn *
213 1505672 : first_active_insn (basic_block bb)
214 : {
215 1505672 : rtx_insn *insn = BB_HEAD (bb);
216 :
217 1505672 : if (LABEL_P (insn))
218 : {
219 284680 : if (insn == BB_END (bb))
220 : return NULL;
221 284680 : insn = NEXT_INSN (insn);
222 : }
223 :
224 4696285 : while (NOTE_P (insn) || DEBUG_INSN_P (insn))
225 : {
226 3190613 : if (insn == BB_END (bb))
227 : return NULL;
228 3190613 : insn = NEXT_INSN (insn);
229 : }
230 :
231 1505672 : if (JUMP_P (insn))
232 : return NULL;
233 :
234 : return insn;
235 : }
236 :
237 : /* Return the last non-jump active (non-jump) insn in the basic block. */
238 :
239 : static rtx_insn *
240 2510188 : last_active_insn (basic_block bb, bool skip_use_p)
241 : {
242 2510188 : rtx_insn *insn = BB_END (bb);
243 2510188 : rtx_insn *head = BB_HEAD (bb);
244 :
245 2510188 : while (NOTE_P (insn)
246 : || JUMP_P (insn)
247 : || DEBUG_INSN_P (insn)
248 5570411 : || (skip_use_p
249 0 : && NONJUMP_INSN_P (insn)
250 0 : && GET_CODE (PATTERN (insn)) == USE))
251 : {
252 3060376 : if (insn == head)
253 : return NULL;
254 3060223 : insn = PREV_INSN (insn);
255 : }
256 :
257 2510035 : if (LABEL_P (insn))
258 68 : return NULL;
259 :
260 : return insn;
261 : }
262 :
263 : /* Return the active insn before INSN inside basic block CURR_BB. */
264 :
265 : static rtx_insn *
266 0 : find_active_insn_before (basic_block curr_bb, rtx_insn *insn)
267 : {
268 0 : if (!insn || insn == BB_HEAD (curr_bb))
269 : return NULL;
270 :
271 0 : while ((insn = PREV_INSN (insn)) != NULL_RTX)
272 : {
273 0 : if (NONJUMP_INSN_P (insn) || JUMP_P (insn) || CALL_P (insn))
274 : break;
275 :
276 : /* No other active insn all the way to the start of the basic block. */
277 0 : if (insn == BB_HEAD (curr_bb))
278 : return NULL;
279 : }
280 :
281 : return insn;
282 : }
283 :
284 : /* Return the active insn after INSN inside basic block CURR_BB. */
285 :
286 : static rtx_insn *
287 0 : find_active_insn_after (basic_block curr_bb, rtx_insn *insn)
288 : {
289 0 : if (!insn || insn == BB_END (curr_bb))
290 : return NULL;
291 :
292 0 : while ((insn = NEXT_INSN (insn)) != NULL_RTX)
293 : {
294 0 : if (NONJUMP_INSN_P (insn) || JUMP_P (insn) || CALL_P (insn))
295 : break;
296 :
297 : /* No other active insn all the way to the end of the basic block. */
298 0 : if (insn == BB_END (curr_bb))
299 : return NULL;
300 : }
301 :
302 : return insn;
303 : }
304 :
305 : /* Return the basic block reached by falling though the basic block BB. */
306 :
307 : static basic_block
308 0 : block_fallthru (basic_block bb)
309 : {
310 0 : edge e = find_fallthru_edge (bb->succs);
311 :
312 0 : return (e) ? e->dest : NULL_BLOCK;
313 : }
314 :
315 : /* Return true if RTXs A and B can be safely interchanged. */
316 :
317 : static bool
318 530085 : rtx_interchangeable_p (const_rtx a, const_rtx b)
319 : {
320 530085 : if (!rtx_equal_p (a, b))
321 : return false;
322 :
323 189024 : if (GET_CODE (a) != MEM)
324 : return true;
325 :
326 : /* A dead type-unsafe memory reference is legal, but a live type-unsafe memory
327 : reference is not. Interchanging a dead type-unsafe memory reference with
328 : a live type-safe one creates a live type-unsafe memory reference, in other
329 : words, it makes the program illegal.
330 : We check here conservatively whether the two memory references have equal
331 : memory attributes. */
332 :
333 1782 : return mem_attrs_eq_p (get_mem_attrs (a), get_mem_attrs (b));
334 : }
335 :
336 :
337 : /* Go through a bunch of insns, converting them to conditional
338 : execution format if possible. Return TRUE if all of the non-note
339 : insns were processed. */
340 :
341 : static bool
342 0 : cond_exec_process_insns (ce_if_block *ce_info ATTRIBUTE_UNUSED,
343 : /* if block information */rtx_insn *start,
344 : /* first insn to look at */rtx end,
345 : /* last insn to look at */rtx test,
346 : /* conditional execution test */profile_probability
347 : prob_val,
348 : /* probability of branch taken. */bool mod_ok)
349 : {
350 0 : bool must_be_last = false;
351 0 : rtx_insn *insn;
352 0 : rtx xtest;
353 0 : rtx pattern;
354 :
355 0 : if (!start || !end)
356 : return false;
357 :
358 0 : for (insn = start; ; insn = NEXT_INSN (insn))
359 : {
360 : /* dwarf2out can't cope with conditional prologues. */
361 0 : if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_PROLOGUE_END)
362 : return false;
363 :
364 0 : if (NOTE_P (insn) || DEBUG_INSN_P (insn))
365 0 : goto insn_done;
366 :
367 0 : gcc_assert (NONJUMP_INSN_P (insn) || CALL_P (insn));
368 :
369 : /* dwarf2out can't cope with conditional unwind info. */
370 0 : if (RTX_FRAME_RELATED_P (insn))
371 : return false;
372 :
373 : /* Remove USE insns that get in the way. */
374 0 : if (reload_completed && GET_CODE (PATTERN (insn)) == USE)
375 : {
376 : /* ??? Ug. Actually unlinking the thing is problematic,
377 : given what we'd have to coordinate with our callers. */
378 0 : SET_INSN_DELETED (insn);
379 0 : goto insn_done;
380 : }
381 :
382 : /* Last insn wasn't last? */
383 0 : if (must_be_last)
384 : return false;
385 :
386 0 : if (modified_in_p (test, insn))
387 : {
388 0 : if (!mod_ok)
389 : return false;
390 : must_be_last = true;
391 : }
392 :
393 : /* Now build the conditional form of the instruction. */
394 0 : pattern = PATTERN (insn);
395 0 : xtest = copy_rtx (test);
396 :
397 : /* If this is already a COND_EXEC, rewrite the test to be an AND of the
398 : two conditions. */
399 0 : if (GET_CODE (pattern) == COND_EXEC)
400 : {
401 0 : if (GET_MODE (xtest) != GET_MODE (COND_EXEC_TEST (pattern)))
402 : return false;
403 :
404 0 : xtest = gen_rtx_AND (GET_MODE (xtest), xtest,
405 : COND_EXEC_TEST (pattern));
406 0 : pattern = COND_EXEC_CODE (pattern);
407 : }
408 :
409 0 : pattern = gen_rtx_COND_EXEC (VOIDmode, xtest, pattern);
410 :
411 : /* If the machine needs to modify the insn being conditionally executed,
412 : say for example to force a constant integer operand into a temp
413 : register, do so here. */
414 : #ifdef IFCVT_MODIFY_INSN
415 : IFCVT_MODIFY_INSN (ce_info, pattern, insn);
416 : if (! pattern)
417 : return false;
418 : #endif
419 :
420 0 : validate_change (insn, &PATTERN (insn), pattern, 1);
421 :
422 0 : if (CALL_P (insn) && prob_val.initialized_p ())
423 0 : validate_change (insn, ®_NOTES (insn),
424 : gen_rtx_INT_LIST ((machine_mode) REG_BR_PROB,
425 : prob_val.to_reg_br_prob_note (),
426 : REG_NOTES (insn)), 1);
427 :
428 0 : insn_done:
429 0 : if (insn == end)
430 : break;
431 0 : }
432 :
433 : return true;
434 : }
435 :
436 : /* Return the condition for a jump. Do not do any special processing. */
437 :
438 : static rtx
439 123198 : cond_exec_get_condition (rtx_insn *jump, bool get_reversed = false)
440 : {
441 123198 : rtx test_if, cond;
442 :
443 123198 : if (any_condjump_p (jump))
444 123198 : test_if = SET_SRC (pc_set (jump));
445 : else
446 : return NULL_RTX;
447 123198 : cond = XEXP (test_if, 0);
448 :
449 : /* If this branches to JUMP_LABEL when the condition is false,
450 : reverse the condition. */
451 123198 : if (get_reversed
452 123198 : || (GET_CODE (XEXP (test_if, 2)) == LABEL_REF
453 0 : && label_ref_label (XEXP (test_if, 2))
454 0 : == JUMP_LABEL (jump)))
455 : {
456 61599 : enum rtx_code rev = reversed_comparison_code (cond, jump);
457 61599 : if (rev == UNKNOWN)
458 : return NULL_RTX;
459 :
460 61599 : cond = gen_rtx_fmt_ee (rev, GET_MODE (cond), XEXP (cond, 0),
461 : XEXP (cond, 1));
462 : }
463 :
464 : return cond;
465 : }
466 :
467 : /* Given a simple IF-THEN or IF-THEN-ELSE block, attempt to convert it
468 : to conditional execution. Return TRUE if we were successful at
469 : converting the block. */
470 :
471 : static bool
472 0 : cond_exec_process_if_block (ce_if_block * ce_info,
473 : /* if block information */bool do_multiple_p)
474 : {
475 0 : basic_block test_bb = ce_info->test_bb; /* last test block */
476 0 : basic_block then_bb = ce_info->then_bb; /* THEN */
477 0 : basic_block else_bb = ce_info->else_bb; /* ELSE or NULL */
478 0 : rtx test_expr; /* expression in IF_THEN_ELSE that is tested */
479 0 : rtx_insn *then_start; /* first insn in THEN block */
480 0 : rtx_insn *then_end; /* last insn + 1 in THEN block */
481 0 : rtx_insn *else_start = NULL; /* first insn in ELSE block or NULL */
482 0 : rtx_insn *else_end = NULL; /* last insn + 1 in ELSE block */
483 0 : int max; /* max # of insns to convert. */
484 0 : bool then_mod_ok; /* whether conditional mods are ok in THEN */
485 0 : rtx true_expr; /* test for else block insns */
486 0 : rtx false_expr; /* test for then block insns */
487 0 : profile_probability true_prob_val;/* probability of else block */
488 0 : profile_probability false_prob_val;/* probability of then block */
489 0 : rtx_insn *then_last_head = NULL; /* Last match at the head of THEN */
490 0 : rtx_insn *else_last_head = NULL; /* Last match at the head of ELSE */
491 0 : rtx_insn *then_first_tail = NULL; /* First match at the tail of THEN */
492 0 : rtx_insn *else_first_tail = NULL; /* First match at the tail of ELSE */
493 0 : int then_n_insns, else_n_insns, n_insns;
494 0 : enum rtx_code false_code;
495 0 : rtx note;
496 :
497 : /* If test is comprised of && or || elements, and we've failed at handling
498 : all of them together, just use the last test if it is the special case of
499 : && elements without an ELSE block. */
500 0 : if (!do_multiple_p && ce_info->num_multiple_test_blocks)
501 : {
502 0 : if (else_bb || ! ce_info->and_and_p)
503 : return false;
504 :
505 0 : ce_info->test_bb = test_bb = ce_info->last_test_bb;
506 0 : ce_info->num_multiple_test_blocks = 0;
507 0 : ce_info->num_and_and_blocks = 0;
508 0 : ce_info->num_or_or_blocks = 0;
509 : }
510 :
511 : /* Find the conditional jump to the ELSE or JOIN part, and isolate
512 : the test. */
513 0 : test_expr = cond_exec_get_condition (BB_END (test_bb));
514 0 : if (! test_expr)
515 : return false;
516 :
517 : /* If the conditional jump is more than just a conditional jump,
518 : then we cannot do conditional execution conversion on this block. */
519 0 : if (! onlyjump_p (BB_END (test_bb)))
520 : return false;
521 :
522 : /* Collect the bounds of where we're to search, skipping any labels, jumps
523 : and notes at the beginning and end of the block. Then count the total
524 : number of insns and see if it is small enough to convert. */
525 0 : then_start = first_active_insn (then_bb);
526 0 : then_end = last_active_insn (then_bb, true);
527 0 : then_n_insns = ce_info->num_then_insns = count_bb_insns (then_bb);
528 0 : n_insns = then_n_insns;
529 0 : max = MAX_CONDITIONAL_EXECUTE;
530 :
531 0 : if (else_bb)
532 : {
533 0 : int n_matching;
534 :
535 0 : max *= 2;
536 0 : else_start = first_active_insn (else_bb);
537 0 : else_end = last_active_insn (else_bb, true);
538 0 : else_n_insns = ce_info->num_else_insns = count_bb_insns (else_bb);
539 0 : n_insns += else_n_insns;
540 :
541 : /* Look for matching sequences at the head and tail of the two blocks,
542 : and limit the range of insns to be converted if possible. */
543 0 : n_matching = flow_find_cross_jump (then_bb, else_bb,
544 : &then_first_tail, &else_first_tail,
545 : NULL);
546 0 : if (then_first_tail == BB_HEAD (then_bb))
547 0 : then_start = then_end = NULL;
548 0 : if (else_first_tail == BB_HEAD (else_bb))
549 0 : else_start = else_end = NULL;
550 :
551 0 : if (n_matching > 0)
552 : {
553 0 : if (then_end)
554 0 : then_end = find_active_insn_before (then_bb, then_first_tail);
555 0 : if (else_end)
556 0 : else_end = find_active_insn_before (else_bb, else_first_tail);
557 0 : n_insns -= 2 * n_matching;
558 : }
559 :
560 0 : if (then_start
561 0 : && else_start
562 : && then_n_insns > n_matching
563 0 : && else_n_insns > n_matching)
564 : {
565 0 : int longest_match = MIN (then_n_insns - n_matching,
566 : else_n_insns - n_matching);
567 0 : n_matching
568 0 : = flow_find_head_matching_sequence (then_bb, else_bb,
569 : &then_last_head,
570 : &else_last_head,
571 : longest_match);
572 :
573 0 : if (n_matching > 0)
574 : {
575 0 : rtx_insn *insn;
576 :
577 : /* We won't pass the insns in the head sequence to
578 : cond_exec_process_insns, so we need to test them here
579 : to make sure that they don't clobber the condition. */
580 0 : for (insn = BB_HEAD (then_bb);
581 0 : insn != NEXT_INSN (then_last_head);
582 0 : insn = NEXT_INSN (insn))
583 0 : if (!LABEL_P (insn) && !NOTE_P (insn)
584 0 : && !DEBUG_INSN_P (insn)
585 0 : && modified_in_p (test_expr, insn))
586 : return false;
587 : }
588 :
589 0 : if (then_last_head == then_end)
590 0 : then_start = then_end = NULL;
591 0 : if (else_last_head == else_end)
592 0 : else_start = else_end = NULL;
593 :
594 0 : if (n_matching > 0)
595 : {
596 0 : if (then_start)
597 0 : then_start = find_active_insn_after (then_bb, then_last_head);
598 0 : if (else_start)
599 0 : else_start = find_active_insn_after (else_bb, else_last_head);
600 0 : n_insns -= 2 * n_matching;
601 : }
602 : }
603 : }
604 :
605 0 : if (n_insns > max)
606 : return false;
607 :
608 : /* Map test_expr/test_jump into the appropriate MD tests to use on
609 : the conditionally executed code. */
610 :
611 0 : true_expr = test_expr;
612 :
613 0 : false_code = reversed_comparison_code (true_expr, BB_END (test_bb));
614 0 : if (false_code != UNKNOWN)
615 0 : false_expr = gen_rtx_fmt_ee (false_code, GET_MODE (true_expr),
616 : XEXP (true_expr, 0), XEXP (true_expr, 1));
617 : else
618 : false_expr = NULL_RTX;
619 :
620 : #ifdef IFCVT_MODIFY_TESTS
621 : /* If the machine description needs to modify the tests, such as setting a
622 : conditional execution register from a comparison, it can do so here. */
623 : IFCVT_MODIFY_TESTS (ce_info, true_expr, false_expr);
624 :
625 : /* See if the conversion failed. */
626 : if (!true_expr || !false_expr)
627 : goto fail;
628 : #endif
629 :
630 0 : note = find_reg_note (BB_END (test_bb), REG_BR_PROB, NULL_RTX);
631 0 : if (note)
632 : {
633 0 : true_prob_val = profile_probability::from_reg_br_prob_note (XINT (note, 0));
634 0 : false_prob_val = true_prob_val.invert ();
635 : }
636 : else
637 : {
638 : true_prob_val = profile_probability::uninitialized ();
639 : false_prob_val = profile_probability::uninitialized ();
640 : }
641 :
642 : /* If we have && or || tests, do them here. These tests are in the adjacent
643 : blocks after the first block containing the test. */
644 0 : if (ce_info->num_multiple_test_blocks > 0)
645 : {
646 0 : basic_block bb = test_bb;
647 0 : basic_block last_test_bb = ce_info->last_test_bb;
648 :
649 0 : if (! false_expr)
650 0 : goto fail;
651 :
652 0 : do
653 : {
654 0 : rtx_insn *start, *end;
655 0 : rtx t, f;
656 0 : enum rtx_code f_code;
657 :
658 0 : bb = block_fallthru (bb);
659 0 : start = first_active_insn (bb);
660 0 : end = last_active_insn (bb, true);
661 0 : if (start
662 0 : && ! cond_exec_process_insns (ce_info, start, end, false_expr,
663 : false_prob_val, false))
664 0 : goto fail;
665 :
666 : /* If the conditional jump is more than just a conditional jump, then
667 : we cannot do conditional execution conversion on this block. */
668 0 : if (! onlyjump_p (BB_END (bb)))
669 0 : goto fail;
670 :
671 : /* Find the conditional jump and isolate the test. */
672 0 : t = cond_exec_get_condition (BB_END (bb));
673 0 : if (! t)
674 0 : goto fail;
675 :
676 0 : f_code = reversed_comparison_code (t, BB_END (bb));
677 0 : if (f_code == UNKNOWN)
678 0 : goto fail;
679 :
680 0 : f = gen_rtx_fmt_ee (f_code, GET_MODE (t), XEXP (t, 0), XEXP (t, 1));
681 0 : if (ce_info->and_and_p)
682 : {
683 0 : t = gen_rtx_AND (GET_MODE (t), true_expr, t);
684 0 : f = gen_rtx_IOR (GET_MODE (t), false_expr, f);
685 : }
686 : else
687 : {
688 0 : t = gen_rtx_IOR (GET_MODE (t), true_expr, t);
689 0 : f = gen_rtx_AND (GET_MODE (t), false_expr, f);
690 : }
691 :
692 : /* If the machine description needs to modify the tests, such as
693 : setting a conditional execution register from a comparison, it can
694 : do so here. */
695 : #ifdef IFCVT_MODIFY_MULTIPLE_TESTS
696 : IFCVT_MODIFY_MULTIPLE_TESTS (ce_info, bb, t, f);
697 :
698 : /* See if the conversion failed. */
699 : if (!t || !f)
700 : goto fail;
701 : #endif
702 :
703 0 : true_expr = t;
704 0 : false_expr = f;
705 : }
706 0 : while (bb != last_test_bb);
707 : }
708 :
709 : /* For IF-THEN-ELSE blocks, we don't allow modifications of the test
710 : on then THEN block. */
711 0 : then_mod_ok = (else_bb == NULL_BLOCK);
712 :
713 : /* Go through the THEN and ELSE blocks converting the insns if possible
714 : to conditional execution. */
715 :
716 0 : if (then_end
717 0 : && (! false_expr
718 0 : || ! cond_exec_process_insns (ce_info, then_start, then_end,
719 : false_expr, false_prob_val,
720 : then_mod_ok)))
721 0 : goto fail;
722 :
723 0 : if (else_bb && else_end
724 0 : && ! cond_exec_process_insns (ce_info, else_start, else_end,
725 : true_expr, true_prob_val, true))
726 0 : goto fail;
727 :
728 : /* If we cannot apply the changes, fail. Do not go through the normal fail
729 : processing, since apply_change_group will call cancel_changes. */
730 0 : if (! apply_change_group ())
731 : {
732 : #ifdef IFCVT_MODIFY_CANCEL
733 : /* Cancel any machine dependent changes. */
734 : IFCVT_MODIFY_CANCEL (ce_info);
735 : #endif
736 : return false;
737 : }
738 :
739 : #ifdef IFCVT_MODIFY_FINAL
740 : /* Do any machine dependent final modifications. */
741 : IFCVT_MODIFY_FINAL (ce_info);
742 : #endif
743 :
744 : /* Conversion succeeded. */
745 0 : if (dump_file)
746 0 : fprintf (dump_file, "%d insn%s converted to conditional execution.\n",
747 : n_insns, (n_insns == 1) ? " was" : "s were");
748 :
749 : /* Merge the blocks! If we had matching sequences, make sure to delete one
750 : copy at the appropriate location first: delete the copy in the THEN branch
751 : for a tail sequence so that the remaining one is executed last for both
752 : branches, and delete the copy in the ELSE branch for a head sequence so
753 : that the remaining one is executed first for both branches. */
754 0 : if (then_first_tail)
755 : {
756 0 : rtx_insn *from = then_first_tail;
757 0 : if (!INSN_P (from))
758 0 : from = find_active_insn_after (then_bb, from);
759 0 : delete_insn_chain (from, get_last_bb_insn (then_bb), false);
760 : }
761 0 : if (else_last_head)
762 0 : delete_insn_chain (first_active_insn (else_bb), else_last_head, false);
763 :
764 0 : merge_if_block (ce_info);
765 0 : cond_exec_changed_p = true;
766 0 : return true;
767 :
768 0 : fail:
769 : #ifdef IFCVT_MODIFY_CANCEL
770 : /* Cancel any machine dependent changes. */
771 : IFCVT_MODIFY_CANCEL (ce_info);
772 : #endif
773 :
774 0 : cancel_changes (0);
775 0 : return false;
776 : }
777 :
778 : static rtx noce_emit_store_flag (struct noce_if_info *, rtx, bool, int);
779 : static bool noce_try_move (struct noce_if_info *);
780 : static bool noce_try_ifelse_collapse (struct noce_if_info *);
781 : static bool noce_try_store_flag (struct noce_if_info *);
782 : static bool noce_try_addcc (struct noce_if_info *);
783 : static bool noce_try_store_flag_constants (struct noce_if_info *);
784 : static bool noce_try_store_flag_mask (struct noce_if_info *);
785 : static rtx noce_emit_cmove (struct noce_if_info *, rtx, enum rtx_code, rtx,
786 : rtx, rtx, rtx, rtx = NULL, rtx = NULL);
787 : static bool noce_try_cmove (struct noce_if_info *);
788 : static bool noce_try_cmove_arith (struct noce_if_info *);
789 : static rtx noce_get_alt_condition (struct noce_if_info *, rtx, rtx_insn **);
790 : static bool noce_try_minmax (struct noce_if_info *);
791 : static bool noce_try_abs (struct noce_if_info *);
792 : static bool noce_try_sign_mask (struct noce_if_info *);
793 :
794 : /* Return the comparison code for reversed condition for IF_INFO,
795 : or UNKNOWN if reversing the condition is not possible. */
796 :
797 : static inline enum rtx_code
798 322650 : noce_reversed_cond_code (struct noce_if_info *if_info)
799 : {
800 322650 : if (if_info->rev_cond)
801 322650 : return GET_CODE (if_info->rev_cond);
802 0 : return reversed_comparison_code (if_info->cond, if_info->jump);
803 : }
804 :
805 : /* Return true if SEQ is a good candidate as a replacement for the
806 : if-convertible sequence described in IF_INFO.
807 : This is the default implementation that targets can override
808 : through a target hook. */
809 :
810 : bool
811 174724 : default_noce_conversion_profitable_p (rtx_insn *seq,
812 : struct noce_if_info *if_info)
813 : {
814 174724 : bool speed_p = if_info->speed_p;
815 :
816 : /* Cost up the new sequence. */
817 174724 : unsigned int cost = seq_cost (seq, speed_p);
818 :
819 174724 : if (cost <= if_info->original_cost)
820 : return true;
821 :
822 : /* When compiling for size, we can make a reasonably accurately guess
823 : at the size growth. When compiling for speed, use the maximum. */
824 92846 : return speed_p && cost <= if_info->max_seq_cost;
825 : }
826 :
827 : /* Helper function for noce_try_store_flag*. */
828 :
829 : static rtx
830 55220 : noce_emit_store_flag (struct noce_if_info *if_info, rtx x, bool reversep,
831 : int normalize)
832 : {
833 55220 : rtx cond = if_info->cond;
834 55220 : bool cond_complex;
835 55220 : enum rtx_code code;
836 :
837 55220 : cond_complex = (! general_operand (XEXP (cond, 0), VOIDmode)
838 55220 : || ! general_operand (XEXP (cond, 1), VOIDmode));
839 :
840 : /* If earliest == jump, or when the condition is complex, try to
841 : build the store_flag insn directly. */
842 :
843 55220 : if (cond_complex)
844 : {
845 23896 : rtx set = pc_set (if_info->jump);
846 23896 : cond = XEXP (SET_SRC (set), 0);
847 23896 : if (GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
848 23896 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (if_info->jump))
849 0 : reversep = !reversep;
850 23896 : if (if_info->then_else_reversed)
851 21442 : reversep = !reversep;
852 : }
853 31324 : else if (reversep
854 14756 : && if_info->rev_cond
855 14756 : && general_operand (XEXP (if_info->rev_cond, 0), VOIDmode)
856 46080 : && general_operand (XEXP (if_info->rev_cond, 1), VOIDmode))
857 : {
858 14756 : cond = if_info->rev_cond;
859 14756 : reversep = false;
860 : }
861 :
862 55220 : if (reversep)
863 4443 : code = reversed_comparison_code (cond, if_info->jump);
864 : else
865 50777 : code = GET_CODE (cond);
866 :
867 55220 : if ((if_info->cond_earliest == if_info->jump || cond_complex)
868 23896 : && (normalize == 0 || STORE_FLAG_VALUE == normalize))
869 : {
870 18297 : rtx src = gen_rtx_fmt_ee (code, GET_MODE (x), XEXP (cond, 0),
871 : XEXP (cond, 1));
872 18297 : rtx set = gen_rtx_SET (x, src);
873 :
874 18297 : start_sequence ();
875 18297 : rtx_insn *insn = emit_insn (set);
876 :
877 18297 : if (recog_memoized (insn) >= 0)
878 : {
879 8015 : rtx_insn *seq = end_sequence ();
880 8015 : emit_insn (seq);
881 :
882 8015 : if_info->cond_earliest = if_info->jump;
883 :
884 8015 : return x;
885 : }
886 :
887 10282 : end_sequence ();
888 : }
889 :
890 : /* Don't even try if the comparison operands or the mode of X are weird. */
891 47205 : if (cond_complex || !SCALAR_INT_MODE_P (GET_MODE (x)))
892 15891 : return NULL_RTX;
893 :
894 : /* Don't try if mode of X is more than the max fixed mode size. */
895 93942 : if (known_le (MAX_FIXED_MODE_SIZE, GET_MODE_BITSIZE (GET_MODE (x))))
896 : return NULL_RTX;
897 :
898 30473 : return emit_store_flag (x, code, XEXP (cond, 0),
899 : XEXP (cond, 1), VOIDmode,
900 30473 : (code == LTU || code == LEU
901 60946 : || code == GEU || code == GTU), normalize);
902 : }
903 :
904 : /* Return true if X can be safely forced into a register by copy_to_mode_reg
905 : / force_operand. */
906 :
907 : static bool
908 1761671 : noce_can_force_operand (rtx x)
909 : {
910 1761671 : if (general_operand (x, VOIDmode))
911 : return true;
912 390953 : if (SUBREG_P (x))
913 : {
914 373 : if (!noce_can_force_operand (SUBREG_REG (x)))
915 : return false;
916 : return true;
917 : }
918 390580 : if (ARITHMETIC_P (x))
919 : {
920 333472 : if (!noce_can_force_operand (XEXP (x, 0))
921 333472 : || !noce_can_force_operand (XEXP (x, 1)))
922 3950 : return false;
923 329522 : switch (GET_CODE (x))
924 : {
925 : case MULT:
926 : case MOD:
927 : case UDIV:
928 : case UMOD:
929 : return true;
930 456 : case DIV:
931 456 : if (INTEGRAL_MODE_P (GET_MODE (x)))
932 : return true;
933 : /* FALLTHRU */
934 322397 : default:
935 322397 : auto optab = code_to_optab (GET_CODE (x));
936 322397 : if (!optab)
937 : return false;
938 320203 : return optab_handler (optab, GET_MODE (x));
939 : }
940 : }
941 57108 : if (UNARY_P (x))
942 : {
943 50350 : if (!noce_can_force_operand (XEXP (x, 0)))
944 : return false;
945 50337 : switch (GET_CODE (x))
946 : {
947 : case ZERO_EXTEND:
948 : case SIGN_EXTEND:
949 : case TRUNCATE:
950 : case FLOAT_EXTEND:
951 : case FLOAT_TRUNCATE:
952 : case FIX:
953 : case UNSIGNED_FIX:
954 : case FLOAT:
955 : case UNSIGNED_FLOAT:
956 : return true;
957 5248 : default:
958 5248 : auto optab = code_to_optab (GET_CODE (x));
959 5248 : if (!optab)
960 : return false;
961 5248 : return optab_handler (optab, GET_MODE (x));
962 : }
963 : }
964 : return false;
965 : }
966 :
967 : /* Emit instruction to move an rtx, possibly into STRICT_LOW_PART.
968 : X is the destination/target and Y is the value to copy. */
969 :
970 : static void
971 110614 : noce_emit_move_insn (rtx x, rtx y)
972 : {
973 110614 : machine_mode outmode;
974 110614 : rtx outer, inner;
975 110614 : poly_int64 bitpos;
976 :
977 110614 : if (GET_CODE (x) != STRICT_LOW_PART)
978 : {
979 110614 : rtx_insn *seq, *insn;
980 110614 : rtx target;
981 110614 : optab ot;
982 :
983 110614 : start_sequence ();
984 : /* Check that the SET_SRC is reasonable before calling emit_move_insn,
985 : otherwise construct a suitable SET pattern ourselves. */
986 110694 : insn = (OBJECT_P (y) || CONSTANT_P (y) || GET_CODE (y) == SUBREG)
987 110694 : ? emit_move_insn (x, y)
988 87952 : : emit_insn (gen_rtx_SET (x, y));
989 110614 : seq = end_sequence ();
990 :
991 110614 : if (recog_memoized (insn) <= 0)
992 : {
993 52271 : if (GET_CODE (x) == ZERO_EXTRACT)
994 : {
995 0 : rtx op = XEXP (x, 0);
996 0 : unsigned HOST_WIDE_INT size = INTVAL (XEXP (x, 1));
997 0 : unsigned HOST_WIDE_INT start = INTVAL (XEXP (x, 2));
998 :
999 : /* store_bit_field expects START to be relative to
1000 : BYTES_BIG_ENDIAN and adjusts this value for machines with
1001 : BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN. In order to be able to
1002 : invoke store_bit_field again it is necessary to have the START
1003 : value from the first call. */
1004 0 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
1005 : {
1006 : if (MEM_P (op))
1007 : start = BITS_PER_UNIT - start - size;
1008 : else
1009 : {
1010 : gcc_assert (REG_P (op));
1011 : start = BITS_PER_WORD - start - size;
1012 : }
1013 : }
1014 :
1015 0 : gcc_assert (start < (MEM_P (op) ? BITS_PER_UNIT : BITS_PER_WORD));
1016 0 : store_bit_field (op, size, start, 0, 0, GET_MODE (x), y, false,
1017 : false);
1018 0 : return;
1019 : }
1020 :
1021 52271 : switch (GET_RTX_CLASS (GET_CODE (y)))
1022 : {
1023 958 : case RTX_UNARY:
1024 958 : ot = code_to_optab (GET_CODE (y));
1025 958 : if (ot && noce_can_force_operand (XEXP (y, 0)))
1026 : {
1027 772 : start_sequence ();
1028 772 : target = expand_unop (GET_MODE (y), ot, XEXP (y, 0), x, 0);
1029 772 : if (target != NULL_RTX)
1030 : {
1031 772 : if (target != x)
1032 0 : emit_move_insn (x, target);
1033 772 : seq = get_insns ();
1034 : }
1035 772 : end_sequence ();
1036 : }
1037 : break;
1038 :
1039 20563 : case RTX_BIN_ARITH:
1040 20563 : case RTX_COMM_ARITH:
1041 20563 : ot = code_to_optab (GET_CODE (y));
1042 20563 : if (ot
1043 20563 : && noce_can_force_operand (XEXP (y, 0))
1044 41126 : && noce_can_force_operand (XEXP (y, 1)))
1045 : {
1046 20563 : start_sequence ();
1047 20563 : target = expand_binop (GET_MODE (y), ot,
1048 : XEXP (y, 0), XEXP (y, 1),
1049 : x, 0, OPTAB_DIRECT);
1050 20563 : if (target != NULL_RTX)
1051 : {
1052 20535 : if (target != x)
1053 0 : emit_move_insn (x, target);
1054 20535 : seq = get_insns ();
1055 : }
1056 20563 : end_sequence ();
1057 : }
1058 : break;
1059 :
1060 : default:
1061 : break;
1062 : }
1063 : }
1064 :
1065 110614 : emit_insn (seq);
1066 110614 : return;
1067 : }
1068 :
1069 0 : outer = XEXP (x, 0);
1070 0 : inner = XEXP (outer, 0);
1071 0 : outmode = GET_MODE (outer);
1072 0 : bitpos = SUBREG_BYTE (outer) * BITS_PER_UNIT;
1073 0 : store_bit_field (inner, GET_MODE_BITSIZE (outmode), bitpos,
1074 : 0, 0, outmode, y, false, false);
1075 : }
1076 :
1077 : /* Return the CC reg if it is used in COND. */
1078 :
1079 : static rtx
1080 4038852 : cc_in_cond (rtx cond)
1081 : {
1082 4038852 : if (have_cbranchcc4 && cond
1083 4038852 : && GET_MODE_CLASS (GET_MODE (XEXP (cond, 0))) == MODE_CC)
1084 116831 : return XEXP (cond, 0);
1085 :
1086 : return NULL_RTX;
1087 : }
1088 :
1089 : /* Return sequence of instructions generated by if conversion. This
1090 : function calls end_sequence() to end the current stream, ensures
1091 : that the instructions are unshared, recognizable non-jump insns.
1092 : On failure, this function returns a NULL_RTX. */
1093 :
1094 : static rtx_insn *
1095 245241 : end_ifcvt_sequence (struct noce_if_info *if_info)
1096 : {
1097 245241 : rtx_insn *insn;
1098 245241 : rtx_insn *seq = get_insns ();
1099 245241 : rtx cc = cc_in_cond (if_info->cond);
1100 :
1101 245241 : set_used_flags (if_info->x);
1102 245241 : set_used_flags (if_info->cond);
1103 245241 : set_used_flags (if_info->a);
1104 245241 : set_used_flags (if_info->b);
1105 :
1106 1283953 : for (insn = seq; insn; insn = NEXT_INSN (insn))
1107 793471 : set_used_flags (insn);
1108 :
1109 245241 : unshare_all_rtl_in_chain (seq);
1110 245241 : end_sequence ();
1111 :
1112 : /* Make sure that all of the instructions emitted are recognizable,
1113 : and that we haven't introduced a new jump instruction.
1114 : As an exercise for the reader, build a general mechanism that
1115 : allows proper placement of required clobbers. */
1116 1251582 : for (insn = seq; insn; insn = NEXT_INSN (insn))
1117 792951 : if (JUMP_P (insn)
1118 792951 : || recog_memoized (insn) == -1
1119 : /* Make sure new generated code does not clobber CC. */
1120 1554129 : || (cc && set_of (cc, insn)))
1121 31851 : return NULL;
1122 :
1123 : return seq;
1124 : }
1125 :
1126 : /* Return true iff the then and else basic block (if it exists)
1127 : consist of a single simple set instruction. */
1128 :
1129 : static bool
1130 3312207 : noce_simple_bbs (struct noce_if_info *if_info)
1131 : {
1132 0 : if (!if_info->then_simple)
1133 : return false;
1134 :
1135 3096747 : if (if_info->else_bb)
1136 0 : return if_info->else_simple;
1137 :
1138 : return true;
1139 : }
1140 :
1141 : /* Convert "if (a != b) x = a; else x = b" into "x = a" and
1142 : "if (a == b) x = a; else x = b" into "x = b". */
1143 :
1144 : static bool
1145 275600 : noce_try_move (struct noce_if_info *if_info)
1146 : {
1147 275600 : rtx cond = if_info->cond;
1148 275600 : enum rtx_code code = GET_CODE (cond);
1149 275600 : rtx y;
1150 275600 : rtx_insn *seq;
1151 :
1152 275600 : if (code != NE && code != EQ)
1153 : return false;
1154 :
1155 549792 : if (!noce_simple_bbs (if_info))
1156 : return false;
1157 :
1158 : /* This optimization isn't valid if either A or B could be a NaN
1159 : or a signed zero. */
1160 202948 : if (HONOR_NANS (if_info->x)
1161 202948 : || HONOR_SIGNED_ZEROS (if_info->x))
1162 52658 : return false;
1163 :
1164 : /* Check whether the operands of the comparison are A and in
1165 : either order. */
1166 150290 : if ((rtx_equal_p (if_info->a, XEXP (cond, 0))
1167 2246 : && rtx_equal_p (if_info->b, XEXP (cond, 1)))
1168 152526 : || (rtx_equal_p (if_info->a, XEXP (cond, 1))
1169 15363 : && rtx_equal_p (if_info->b, XEXP (cond, 0))))
1170 : {
1171 15 : if (!rtx_interchangeable_p (if_info->a, if_info->b))
1172 : return false;
1173 :
1174 0 : y = (code == EQ) ? if_info->a : if_info->b;
1175 :
1176 : /* Avoid generating the move if the source is the destination. */
1177 0 : if (! rtx_equal_p (if_info->x, y))
1178 : {
1179 0 : start_sequence ();
1180 0 : noce_emit_move_insn (if_info->x, y);
1181 0 : seq = end_ifcvt_sequence (if_info);
1182 0 : if (!seq)
1183 : return false;
1184 :
1185 0 : emit_insn_before_setloc (seq, if_info->jump,
1186 0 : INSN_LOCATION (if_info->insn_a));
1187 : }
1188 0 : if_info->transform_name = "noce_try_move";
1189 0 : return true;
1190 : }
1191 : return false;
1192 : }
1193 :
1194 : /* If a sign bit test is selecting across constants, we may be able
1195 : to generate efficient code utilizing the -1/0 result of a sign
1196 : bit splat idiom.
1197 :
1198 : Do this before trying the generalized conditional move as these
1199 : (when applicable) are hopefully faster than a conditional move. */
1200 :
1201 : static bool
1202 230976 : noce_try_sign_bit_splat (struct noce_if_info *if_info)
1203 : {
1204 230976 : rtx cond = if_info->cond;
1205 230976 : enum rtx_code code = GET_CODE (cond);
1206 :
1207 : /* We're looking for sign bit tests, so only a few cases are
1208 : interesting. LT/GE 0 LE/GT -1. */
1209 230976 : if (((code == LT || code == GE)
1210 10209 : && XEXP (cond, 1) == CONST0_RTX (GET_MODE (cond)))
1211 228571 : || ((code == LE || code == GT)
1212 11554 : && XEXP (cond, 1) == CONSTM1_RTX (GET_MODE (cond))))
1213 : ;
1214 : else
1215 : return false;
1216 :
1217 : /* It would be good if this could be extended since constant synthesis
1218 : on some platforms will result in blocks which fail this test. */
1219 238556 : if (!noce_simple_bbs (if_info))
1220 : return false;
1221 :
1222 : /* Only try this for constants in the true/false arms and a REG
1223 : destination. We could select between 0 and a REG pretty
1224 : easily with a logical AND. */
1225 6065 : if (!CONST_INT_P (if_info->a)
1226 1909 : || !CONST_INT_P (if_info->b)
1227 599 : || !REG_P (if_info->x))
1228 : return false;
1229 :
1230 599 : machine_mode mode = GET_MODE (if_info->x);
1231 :
1232 : /* If the mode of the destination does not match the mode of
1233 : the value we're testing, then this optimization is not valid. */
1234 599 : if (mode != GET_MODE (XEXP (cond, 0)))
1235 : return false;
1236 :
1237 206 : HOST_WIDE_INT val_a = INTVAL (if_info->a);
1238 206 : HOST_WIDE_INT val_b = INTVAL (if_info->b);
1239 :
1240 206 : rtx_insn *seq;
1241 206 : start_sequence ();
1242 :
1243 : /* We're testing the sign bit of this operand. */
1244 206 : rtx condop = XEXP (cond, 0);
1245 :
1246 : /* To splat the sign bit we arithmetically shift the
1247 : input value right by the size of the object - 1 bits.
1248 :
1249 : Note some targets do not have strong shifters, but do have
1250 : alternative ways to generate the sign bit splat. The
1251 : profitability test when we end the sequence should reject
1252 : cases when the branchy sequence is better. */
1253 206 : int splat_count = GET_MODE_BITSIZE (GET_MODE (condop)).to_constant () - 1;
1254 206 : rtx splat = GEN_INT (splat_count);
1255 :
1256 206 : rtx temp;
1257 206 : temp = expand_simple_binop (GET_MODE (XEXP (cond, 0)), ASHIFTRT,
1258 : XEXP (cond, 0), splat, NULL_RTX,
1259 : false, OPTAB_WIDEN);
1260 206 : if (!temp)
1261 0 : goto fail;
1262 :
1263 : /* IOR of anything with -1 still results in -1. So we can
1264 : IOR the other operand to generate a select between -1 and
1265 : an arbitrary constant. */
1266 206 : if (val_a == -1)
1267 : {
1268 38 : if (code == LT || code == LE)
1269 : {
1270 2 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1271 2 : if (!temp)
1272 0 : goto fail;
1273 : }
1274 :
1275 38 : temp = expand_simple_binop (mode, IOR, temp, GEN_INT (val_b),
1276 : if_info->x, false, OPTAB_WIDEN);
1277 : }
1278 : /* AND of anything with 0 is still zero. So we can AND
1279 : with the -1 operand with the a constant to select
1280 : between the constant and zero. */
1281 168 : else if (val_b == 0)
1282 : {
1283 0 : if (code == LT || code == LE)
1284 : {
1285 0 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1286 0 : if (!temp)
1287 0 : goto fail;
1288 : }
1289 :
1290 : /* Since we know the value is currently -1 or 0, some constants may
1291 : be more easily handled by shifting the value again. A right
1292 : logical shift constructs 2^n-1 constants a left shift constructs
1293 : ~(2^n-1) constants. Given some targets don't have efficient
1294 : shifts, generate the obvious RTL for both forms and select the
1295 : one with smaller cost. */
1296 0 : rtx and_form = gen_rtx_AND (mode, temp, GEN_INT (val_a));
1297 0 : rtx shift_left = gen_rtx_ASHIFT (mode, temp, GEN_INT (ctz_hwi (val_a)));
1298 0 : HOST_WIDE_INT rshift_count
1299 0 : = (clz_hwi (val_a) & (GET_MODE_PRECISION (mode).to_constant() - 1));
1300 0 : rtx shift_right = gen_rtx_LSHIFTRT (mode, temp, GEN_INT (rshift_count));
1301 0 : bool speed_p = optimize_insn_for_speed_p ();
1302 0 : if (exact_log2 (val_a + 1) >= 0
1303 0 : && (rtx_cost (shift_right, mode, SET, 1, speed_p)
1304 0 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1305 0 : temp = expand_simple_binop (mode, LSHIFTRT, temp,
1306 : GEN_INT (rshift_count),
1307 : if_info->x, false, OPTAB_WIDEN);
1308 0 : else if (exact_log2 (~val_a + 1) >= 0
1309 0 : && (rtx_cost (shift_left, mode, SET, 1, speed_p)
1310 0 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1311 0 : temp = expand_simple_binop (mode, ASHIFT, temp,
1312 0 : GEN_INT (ctz_hwi (val_a)),
1313 : if_info->x, false, OPTAB_WIDEN);
1314 : else
1315 0 : temp = expand_simple_binop (mode, AND, temp, GEN_INT (val_a),
1316 : if_info->x, false, OPTAB_WIDEN);
1317 : }
1318 : /* Same cases, but with the test or arms swapped. These
1319 : can be realized as well, though it typically costs
1320 : an extra instruction. */
1321 168 : else if (val_b == -1)
1322 : {
1323 46 : if (code != LT && code != LE)
1324 : {
1325 2 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1326 2 : if (!temp)
1327 0 : goto fail;
1328 : }
1329 :
1330 46 : temp = expand_simple_binop (mode, IOR, temp, GEN_INT (val_a),
1331 : if_info->x, false, OPTAB_WIDEN);
1332 : }
1333 122 : else if (val_a == 0)
1334 : {
1335 79 : if (code != LT && code != LE)
1336 : {
1337 9 : temp = expand_simple_unop (mode, NOT, temp, NULL_RTX, true);
1338 9 : if (!temp)
1339 0 : goto fail;
1340 : }
1341 :
1342 : /* Since we know the value is currently -1 or 0, some constants may
1343 : be more easily handled by shifting the value again. A right
1344 : logical shift constructs 2^n-1 constants a left shift constructs
1345 : ~(2^n-1) constants. Given some targets don't have efficient
1346 : shifts, generate the obvious RTL for both forms and select the
1347 : one with smaller cost. */
1348 79 : rtx and_form = gen_rtx_AND (mode, temp, GEN_INT (val_b));
1349 79 : rtx shift_left = gen_rtx_ASHIFT (mode, temp, GEN_INT (ctz_hwi (val_b)));
1350 79 : HOST_WIDE_INT rshift_count
1351 79 : = (clz_hwi (val_b) & (GET_MODE_PRECISION (mode).to_constant() - 1));
1352 79 : rtx shift_right = gen_rtx_LSHIFTRT (mode, temp, GEN_INT (rshift_count));
1353 79 : bool speed_p = optimize_insn_for_speed_p ();
1354 79 : if (exact_log2 (val_b + 1) >= 0
1355 50 : && (rtx_cost (shift_right, mode, SET, 1, speed_p)
1356 25 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1357 3 : temp = expand_simple_binop (mode, LSHIFTRT, temp,
1358 : GEN_INT (rshift_count),
1359 : if_info->x, false, OPTAB_WIDEN);
1360 76 : else if (exact_log2 (~val_b + 1) >= 0
1361 0 : && (rtx_cost (shift_left, mode, SET, 1, speed_p)
1362 0 : < rtx_cost (and_form, mode, SET, 1, speed_p)))
1363 0 : temp = expand_simple_binop (mode, ASHIFT, temp,
1364 0 : GEN_INT (ctz_hwi (val_b)),
1365 : if_info->x, false, OPTAB_WIDEN);
1366 : else
1367 76 : temp = expand_simple_binop (mode, AND, temp, GEN_INT (val_b),
1368 : if_info->x, false, OPTAB_WIDEN);
1369 : }
1370 : /* Nothing worked. */
1371 : else
1372 : temp = NULL_RTX;
1373 :
1374 163 : if (!temp)
1375 43 : goto fail;
1376 :
1377 : /* Move into the final destination if the value wasn't
1378 : constructed there. */
1379 163 : if (if_info->x != temp)
1380 0 : emit_move_insn (if_info->x, temp);
1381 :
1382 : /* This ends the sequence and tests the cost model. */
1383 163 : seq = end_ifcvt_sequence (if_info);
1384 163 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
1385 0 : return false;
1386 :
1387 : /* Everything looks good. Install the if-converted sequence. */
1388 163 : emit_insn_before_setloc (seq, if_info->jump,
1389 163 : INSN_LOCATION (if_info->insn_a));
1390 163 : if_info->transform_name = "splat_sign_bit_trivial";
1391 163 : return true;
1392 :
1393 43 : fail:
1394 43 : end_ifcvt_sequence (if_info);
1395 43 : return false;
1396 : }
1397 :
1398 :
1399 : /* Try forming an IF_THEN_ELSE (cond, b, a) and collapsing that
1400 : through simplify_rtx. Sometimes that can eliminate the IF_THEN_ELSE.
1401 : If that is the case, emit the result into x. */
1402 :
1403 : static bool
1404 275600 : noce_try_ifelse_collapse (struct noce_if_info * if_info)
1405 : {
1406 601691 : if (!noce_simple_bbs (if_info))
1407 : return false;
1408 :
1409 257849 : machine_mode mode = GET_MODE (if_info->x);
1410 257849 : rtx if_then_else = simplify_gen_ternary (IF_THEN_ELSE, mode, mode,
1411 : if_info->cond, if_info->b,
1412 : if_info->a);
1413 :
1414 257849 : if (GET_CODE (if_then_else) == IF_THEN_ELSE)
1415 : return false;
1416 :
1417 46405 : rtx_insn *seq;
1418 46405 : start_sequence ();
1419 46405 : noce_emit_move_insn (if_info->x, if_then_else);
1420 46405 : seq = end_ifcvt_sequence (if_info);
1421 46405 : if (!seq)
1422 : return false;
1423 :
1424 15655 : emit_insn_before_setloc (seq, if_info->jump,
1425 15655 : INSN_LOCATION (if_info->insn_a));
1426 :
1427 15655 : if_info->transform_name = "noce_try_ifelse_collapse";
1428 15655 : return true;
1429 : }
1430 :
1431 :
1432 : /* Convert "if (test) x = 1; else x = 0".
1433 :
1434 : Only try 0 and STORE_FLAG_VALUE here. Other combinations will be
1435 : tried in noce_try_store_flag_constants after noce_try_cmove has had
1436 : a go at the conversion. */
1437 :
1438 : static bool
1439 259945 : noce_try_store_flag (struct noce_if_info *if_info)
1440 : {
1441 259945 : bool reversep;
1442 259945 : rtx target;
1443 259945 : rtx_insn *seq;
1444 :
1445 546809 : if (!noce_simple_bbs (if_info))
1446 : return false;
1447 :
1448 242194 : if (CONST_INT_P (if_info->b)
1449 72683 : && INTVAL (if_info->b) == STORE_FLAG_VALUE
1450 13215 : && if_info->a == const0_rtx)
1451 : reversep = false;
1452 231808 : else if (if_info->b == const0_rtx
1453 30458 : && CONST_INT_P (if_info->a)
1454 20784 : && INTVAL (if_info->a) == STORE_FLAG_VALUE
1455 252173 : && noce_reversed_cond_code (if_info) != UNKNOWN)
1456 : reversep = true;
1457 : else
1458 211443 : return false;
1459 :
1460 30751 : start_sequence ();
1461 :
1462 30751 : target = noce_emit_store_flag (if_info, if_info->x, reversep, 0);
1463 30751 : if (target)
1464 : {
1465 25952 : if (target != if_info->x)
1466 2977 : noce_emit_move_insn (if_info->x, target);
1467 :
1468 25952 : seq = end_ifcvt_sequence (if_info);
1469 25952 : if (! seq)
1470 : return false;
1471 :
1472 25952 : emit_insn_before_setloc (seq, if_info->jump,
1473 25952 : INSN_LOCATION (if_info->insn_a));
1474 25952 : if_info->transform_name = "noce_try_store_flag";
1475 25952 : return true;
1476 : }
1477 : else
1478 : {
1479 4799 : end_sequence ();
1480 4799 : return false;
1481 : }
1482 : }
1483 :
1484 :
1485 : /* Convert "if (test) x = -A; else x = A" into
1486 : x = A; if (test) x = -x if the machine can do the
1487 : conditional negate form of this cheaply.
1488 : Try this before noce_try_cmove that will just load the
1489 : immediates into two registers and do a conditional select
1490 : between them. If the target has a conditional negate or
1491 : conditional invert operation we can save a potentially
1492 : expensive constant synthesis. */
1493 :
1494 : static bool
1495 233916 : noce_try_inverse_constants (struct noce_if_info *if_info)
1496 : {
1497 511083 : if (!noce_simple_bbs (if_info))
1498 : return false;
1499 :
1500 216165 : if (!CONST_INT_P (if_info->a)
1501 50449 : || !CONST_INT_P (if_info->b)
1502 21027 : || !REG_P (if_info->x))
1503 : return false;
1504 :
1505 21027 : machine_mode mode = GET_MODE (if_info->x);
1506 :
1507 21027 : HOST_WIDE_INT val_a = INTVAL (if_info->a);
1508 21027 : HOST_WIDE_INT val_b = INTVAL (if_info->b);
1509 :
1510 21027 : rtx cond = if_info->cond;
1511 :
1512 21027 : rtx x = if_info->x;
1513 21027 : rtx target;
1514 :
1515 21027 : start_sequence ();
1516 :
1517 21027 : rtx_code code;
1518 21027 : if (val_b != HOST_WIDE_INT_MIN && val_a == -val_b)
1519 : code = NEG;
1520 19823 : else if (val_a == ~val_b)
1521 : code = NOT;
1522 : else
1523 : {
1524 19755 : end_sequence ();
1525 19755 : return false;
1526 : }
1527 :
1528 1272 : rtx tmp = gen_reg_rtx (mode);
1529 1272 : noce_emit_move_insn (tmp, if_info->a);
1530 :
1531 1272 : target = emit_conditional_neg_or_complement (x, code, mode, cond, tmp, tmp);
1532 :
1533 1272 : if (target)
1534 : {
1535 0 : rtx_insn *seq = get_insns ();
1536 :
1537 0 : if (!seq)
1538 : {
1539 0 : end_sequence ();
1540 0 : return false;
1541 : }
1542 :
1543 0 : if (target != if_info->x)
1544 0 : noce_emit_move_insn (if_info->x, target);
1545 :
1546 0 : seq = end_ifcvt_sequence (if_info);
1547 :
1548 0 : if (!seq)
1549 : return false;
1550 :
1551 0 : emit_insn_before_setloc (seq, if_info->jump,
1552 0 : INSN_LOCATION (if_info->insn_a));
1553 0 : if_info->transform_name = "noce_try_inverse_constants";
1554 0 : return true;
1555 : }
1556 :
1557 1272 : end_sequence ();
1558 1272 : return false;
1559 : }
1560 :
1561 :
1562 : /* Convert "if (test) x = a; else x = b", for A and B constant.
1563 : Also allow A = y + c1, B = y + c2, with a common y between A
1564 : and B. */
1565 :
1566 : static bool
1567 233916 : noce_try_store_flag_constants (struct noce_if_info *if_info)
1568 : {
1569 233916 : rtx target;
1570 233916 : rtx_insn *seq;
1571 233916 : bool reversep;
1572 233916 : HOST_WIDE_INT itrue, ifalse, diff, tmp;
1573 233916 : int normalize;
1574 233916 : bool can_reverse;
1575 233916 : machine_mode mode = GET_MODE (if_info->x);
1576 233916 : rtx common = NULL_RTX;
1577 :
1578 233916 : rtx a = if_info->a;
1579 233916 : rtx b = if_info->b;
1580 :
1581 : /* Handle cases like x := test ? y + 3 : y + 4. */
1582 233916 : if (GET_CODE (a) == PLUS
1583 23412 : && GET_CODE (b) == PLUS
1584 1443 : && CONST_INT_P (XEXP (a, 1))
1585 589 : && CONST_INT_P (XEXP (b, 1))
1586 523 : && rtx_equal_p (XEXP (a, 0), XEXP (b, 0))
1587 : /* Allow expressions that are not using the result or plain
1588 : registers where we handle overlap below. */
1589 234366 : && (REG_P (XEXP (a, 0))
1590 10 : || (noce_operand_ok (XEXP (a, 0))
1591 10 : && ! reg_overlap_mentioned_p (if_info->x, XEXP (a, 0)))))
1592 : {
1593 450 : common = XEXP (a, 0);
1594 450 : a = XEXP (a, 1);
1595 450 : b = XEXP (b, 1);
1596 : }
1597 :
1598 294918 : if (!noce_simple_bbs (if_info))
1599 : return false;
1600 :
1601 216165 : if (CONST_INT_P (a)
1602 50887 : && CONST_INT_P (b))
1603 : {
1604 21465 : ifalse = INTVAL (a);
1605 21465 : itrue = INTVAL (b);
1606 21465 : bool subtract_flag_p = false;
1607 :
1608 21465 : diff = (unsigned HOST_WIDE_INT) itrue - ifalse;
1609 : /* Make sure we can represent the difference between the two values. */
1610 21465 : if ((diff > 0)
1611 21465 : != ((ifalse < 0) != (itrue < 0) ? ifalse < 0 : ifalse < itrue))
1612 : return false;
1613 :
1614 21345 : diff = trunc_int_for_mode (diff, mode);
1615 :
1616 21345 : can_reverse = noce_reversed_cond_code (if_info) != UNKNOWN;
1617 21345 : reversep = false;
1618 21345 : if (diff == STORE_FLAG_VALUE || diff == -STORE_FLAG_VALUE)
1619 : {
1620 11786 : normalize = 0;
1621 : /* We could collapse these cases but it is easier to follow the
1622 : diff/STORE_FLAG_VALUE combinations when they are listed
1623 : explicitly. */
1624 :
1625 : /* test ? 3 : 4
1626 : => 4 + (test != 0). */
1627 11786 : if (diff < 0 && STORE_FLAG_VALUE < 0)
1628 : reversep = false;
1629 : /* test ? 4 : 3
1630 : => can_reverse | 4 + (test == 0)
1631 : !can_reverse | 3 - (test != 0). */
1632 11786 : else if (diff > 0 && STORE_FLAG_VALUE < 0)
1633 : {
1634 : reversep = can_reverse;
1635 : subtract_flag_p = !can_reverse;
1636 : /* If we need to subtract the flag and we have PLUS-immediate
1637 : A and B then it is unlikely to be beneficial to play tricks
1638 : here. */
1639 : if (subtract_flag_p && common)
1640 : return false;
1641 : }
1642 : /* test ? 3 : 4
1643 : => can_reverse | 3 + (test == 0)
1644 : !can_reverse | 4 - (test != 0). */
1645 11786 : else if (diff < 0 && STORE_FLAG_VALUE > 0)
1646 : {
1647 6156 : reversep = can_reverse;
1648 6156 : subtract_flag_p = !can_reverse;
1649 : /* If we need to subtract the flag and we have PLUS-immediate
1650 : A and B then it is unlikely to be beneficial to play tricks
1651 : here. */
1652 6156 : if (subtract_flag_p && common)
1653 : return false;
1654 : }
1655 : /* test ? 4 : 3
1656 : => 4 + (test != 0). */
1657 : else if (diff > 0 && STORE_FLAG_VALUE > 0)
1658 : reversep = false;
1659 : else
1660 : gcc_unreachable ();
1661 : }
1662 : /* Is this (cond) ? 2^n : 0? */
1663 531 : else if (ifalse == 0 && pow2p_hwi (itrue)
1664 9559 : && STORE_FLAG_VALUE == 1)
1665 : normalize = 1;
1666 : /* Is this (cond) ? 0 : 2^n? */
1667 231328 : else if (itrue == 0 && pow2p_hwi (ifalse) && can_reverse
1668 9553 : && STORE_FLAG_VALUE == 1)
1669 : {
1670 : normalize = 1;
1671 : reversep = true;
1672 : }
1673 : /* Is this (cond) ? -1 : x? */
1674 : else if (itrue == -1
1675 : && STORE_FLAG_VALUE == -1)
1676 : normalize = -1;
1677 : /* Is this (cond) ? x : -1? */
1678 : else if (ifalse == -1 && can_reverse
1679 : && STORE_FLAG_VALUE == -1)
1680 : {
1681 : normalize = -1;
1682 : reversep = true;
1683 : }
1684 : else
1685 : return false;
1686 :
1687 6156 : if (reversep)
1688 : {
1689 6157 : std::swap (itrue, ifalse);
1690 6157 : diff = trunc_int_for_mode (-(unsigned HOST_WIDE_INT) diff, mode);
1691 : }
1692 :
1693 11794 : start_sequence ();
1694 :
1695 : /* If we have x := test ? x + 3 : x + 4 then move the original
1696 : x out of the way while we store flags. */
1697 11794 : if (common && rtx_equal_p (common, if_info->x))
1698 : {
1699 8 : common = gen_reg_rtx (mode);
1700 8 : noce_emit_move_insn (common, if_info->x);
1701 : }
1702 :
1703 11794 : target = noce_emit_store_flag (if_info, if_info->x, reversep, normalize);
1704 11794 : if (! target)
1705 : {
1706 4835 : end_sequence ();
1707 4835 : return false;
1708 : }
1709 :
1710 : /* if (test) x = 3; else x = 4;
1711 : => x = 3 + (test == 0); */
1712 6959 : if (diff == STORE_FLAG_VALUE || diff == -STORE_FLAG_VALUE)
1713 : {
1714 : /* Add the common part now. This may allow combine to merge this
1715 : with the store flag operation earlier into some sort of conditional
1716 : increment/decrement if the target allows it. */
1717 6951 : if (common)
1718 35 : target = expand_simple_binop (mode, PLUS,
1719 : target, common,
1720 : target, 0, OPTAB_WIDEN);
1721 :
1722 : /* Always use ifalse here. It should have been swapped with itrue
1723 : when appropriate when reversep is true. */
1724 13902 : target = expand_simple_binop (mode, subtract_flag_p ? MINUS : PLUS,
1725 6951 : gen_int_mode (ifalse, mode), target,
1726 : if_info->x, 0, OPTAB_WIDEN);
1727 : }
1728 : /* Other cases are not beneficial when the original A and B are PLUS
1729 : expressions. */
1730 8 : else if (common)
1731 : {
1732 0 : end_sequence ();
1733 0 : return false;
1734 : }
1735 : /* if (test) x = 8; else x = 0;
1736 : => x = (test != 0) << 3; */
1737 8 : else if (ifalse == 0 && (tmp = exact_log2 (itrue)) >= 0)
1738 : {
1739 8 : target = expand_simple_binop (mode, ASHIFT,
1740 : target, GEN_INT (tmp), if_info->x, 0,
1741 : OPTAB_WIDEN);
1742 : }
1743 :
1744 : /* if (test) x = -1; else x = b;
1745 : => x = -(test != 0) | b; */
1746 0 : else if (itrue == -1)
1747 : {
1748 0 : target = expand_simple_binop (mode, IOR,
1749 0 : target, gen_int_mode (ifalse, mode),
1750 : if_info->x, 0, OPTAB_WIDEN);
1751 : }
1752 : else
1753 : {
1754 0 : end_sequence ();
1755 0 : return false;
1756 : }
1757 :
1758 6959 : if (! target)
1759 : {
1760 0 : end_sequence ();
1761 0 : return false;
1762 : }
1763 :
1764 6959 : if (target != if_info->x)
1765 0 : noce_emit_move_insn (if_info->x, target);
1766 :
1767 6959 : seq = end_ifcvt_sequence (if_info);
1768 6959 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
1769 4019 : return false;
1770 :
1771 2940 : emit_insn_before_setloc (seq, if_info->jump,
1772 2940 : INSN_LOCATION (if_info->insn_a));
1773 2940 : if_info->transform_name = "noce_try_store_flag_constants";
1774 :
1775 2940 : return true;
1776 : }
1777 :
1778 : return false;
1779 : }
1780 :
1781 : /* This is trying to capture cases like dest = cond ? a : -1.
1782 :
1783 : The basic idea is to use a store-flag insn to generate a -1/0
1784 : value (directly or indirectly), then IOR that with the other
1785 : input. */
1786 : static bool
1787 230813 : noce_try_store_flag_logical (struct noce_if_info *if_info)
1788 : {
1789 230813 : rtx a = if_info->a;
1790 230813 : rtx b = if_info->b;
1791 230813 : rtx dest = if_info->x;
1792 230813 : machine_mode mode = GET_MODE (dest);
1793 :
1794 230813 : if (STORE_FLAG_VALUE != -1 && STORE_FLAG_VALUE != 1)
1795 : return false;
1796 :
1797 501838 : if (!noce_simple_bbs (if_info))
1798 : return false;
1799 :
1800 213062 : bool swapped = false;
1801 213062 : if (a == CONSTM1_RTX (GET_MODE (dest))
1802 1604 : && if_info->rev_cond)
1803 : {
1804 : std::swap (a, b);
1805 213062 : swapped = true;
1806 : }
1807 :
1808 213062 : if (b != CONSTM1_RTX (GET_MODE (dest)))
1809 : return false;
1810 :
1811 : /* If the other arm is not a REG/SUBREG, then punt. This is primarily to
1812 : let the target handle the constant case, which it can likely do better.
1813 : It also means we don't have to worry about non terminal expressions. */
1814 2508 : if (!REG_P (a) && !SUBREG_P (a))
1815 : return false;
1816 :
1817 : /* At this point we've got dest = cond ? a : -1. Emit the store flag and
1818 : adjust its value (if necessary) to -1/0. */
1819 972 : start_sequence ();
1820 972 : rtx temp = gen_reg_rtx (mode);
1821 972 : rtx target = noce_emit_store_flag (if_info, temp, !swapped, false);
1822 972 : if (!target)
1823 : {
1824 88 : end_sequence ();
1825 88 : return false;
1826 : }
1827 :
1828 884 : if (STORE_FLAG_VALUE == 1)
1829 : {
1830 884 : rtx x = gen_rtx_PLUS (mode, target, CONSTM1_RTX (mode));
1831 884 : emit_move_insn (target, x);
1832 : }
1833 :
1834 : /* Now we've got -1/0 in TARGET. We can just IOR with A. */
1835 884 : rtx x = gen_rtx_IOR (mode, target, a);
1836 884 : emit_move_insn (dest, x);
1837 :
1838 : /* We've generated all the RTL, make sure it recognizes and is
1839 : profitable. */
1840 884 : rtx_insn *seq = end_ifcvt_sequence (if_info);
1841 884 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
1842 884 : return false;
1843 :
1844 0 : emit_insn_before_setloc (seq, if_info->jump,
1845 0 : INSN_LOCATION (if_info->insn_a));
1846 0 : if_info->transform_name = "noce_try_store_flag_logical";
1847 0 : return true;
1848 : }
1849 :
1850 : /* Convert "if (test) foo++" into "foo += (test != 0)", and
1851 : similarly for "foo--". */
1852 :
1853 : static bool
1854 187159 : noce_try_addcc (struct noce_if_info *if_info)
1855 : {
1856 187159 : rtx target;
1857 187159 : rtx_insn *seq;
1858 187159 : bool subtract;
1859 187159 : int normalize;
1860 :
1861 393958 : if (!noce_simple_bbs (if_info))
1862 : return false;
1863 :
1864 169408 : if (GET_CODE (if_info->a) == PLUS
1865 20475 : && rtx_equal_p (XEXP (if_info->a, 0), if_info->b)
1866 186231 : && noce_reversed_cond_code (if_info) != UNKNOWN)
1867 : {
1868 16823 : rtx cond = if_info->rev_cond;
1869 16823 : enum rtx_code code;
1870 :
1871 16823 : if (cond == NULL_RTX)
1872 : {
1873 0 : cond = if_info->cond;
1874 0 : code = reversed_comparison_code (cond, if_info->jump);
1875 : }
1876 : else
1877 16823 : code = GET_CODE (cond);
1878 :
1879 : /* First try to use addcc pattern. */
1880 16823 : if (general_operand (XEXP (cond, 0), VOIDmode)
1881 16823 : && general_operand (XEXP (cond, 1), VOIDmode))
1882 : {
1883 15304 : start_sequence ();
1884 45912 : target = emit_conditional_add (if_info->x, code,
1885 : XEXP (cond, 0),
1886 : XEXP (cond, 1),
1887 : VOIDmode,
1888 : if_info->b,
1889 15304 : XEXP (if_info->a, 1),
1890 15304 : GET_MODE (if_info->x),
1891 15304 : (code == LTU || code == GEU
1892 15304 : || code == LEU || code == GTU));
1893 15304 : if (target)
1894 : {
1895 1264 : if (target != if_info->x)
1896 0 : noce_emit_move_insn (if_info->x, target);
1897 :
1898 1264 : seq = end_ifcvt_sequence (if_info);
1899 1264 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
1900 0 : return false;
1901 :
1902 1264 : emit_insn_before_setloc (seq, if_info->jump,
1903 1264 : INSN_LOCATION (if_info->insn_a));
1904 1264 : if_info->transform_name = "noce_try_addcc";
1905 :
1906 1264 : return true;
1907 : }
1908 14040 : end_sequence ();
1909 : }
1910 :
1911 : /* If that fails, construct conditional increment or decrement using
1912 : setcc. We're changing a branch and an increment to a comparison and
1913 : an ADD/SUB. */
1914 15559 : if (XEXP (if_info->a, 1) == const1_rtx
1915 10792 : || XEXP (if_info->a, 1) == constm1_rtx)
1916 : {
1917 5306 : start_sequence ();
1918 5306 : if (STORE_FLAG_VALUE == INTVAL (XEXP (if_info->a, 1)))
1919 : subtract = false, normalize = 0;
1920 539 : else if (-STORE_FLAG_VALUE == INTVAL (XEXP (if_info->a, 1)))
1921 : subtract = true, normalize = 0;
1922 : else
1923 0 : subtract = false, normalize = INTVAL (XEXP (if_info->a, 1));
1924 :
1925 :
1926 5306 : target = noce_emit_store_flag (if_info,
1927 5306 : gen_reg_rtx (GET_MODE (if_info->x)),
1928 : true, normalize);
1929 :
1930 5306 : if (target)
1931 8035 : target = expand_simple_binop (GET_MODE (if_info->x),
1932 : subtract ? MINUS : PLUS,
1933 : if_info->b, target, if_info->x,
1934 : 0, OPTAB_WIDEN);
1935 4277 : if (target)
1936 : {
1937 4277 : if (target != if_info->x)
1938 0 : noce_emit_move_insn (if_info->x, target);
1939 :
1940 4277 : seq = end_ifcvt_sequence (if_info);
1941 4277 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
1942 771 : return false;
1943 :
1944 3506 : emit_insn_before_setloc (seq, if_info->jump,
1945 3506 : INSN_LOCATION (if_info->insn_a));
1946 3506 : if_info->transform_name = "noce_try_addcc";
1947 3506 : return true;
1948 : }
1949 1029 : end_sequence ();
1950 : }
1951 : }
1952 :
1953 : return false;
1954 : }
1955 :
1956 : /* Convert "if (test) x = 0;" to "x &= -(test == 0);" */
1957 :
1958 : static bool
1959 182389 : noce_try_store_flag_mask (struct noce_if_info *if_info)
1960 : {
1961 182389 : rtx target;
1962 182389 : rtx_insn *seq;
1963 182389 : bool reversep;
1964 :
1965 389096 : if (!noce_simple_bbs (if_info))
1966 : return false;
1967 :
1968 164638 : reversep = false;
1969 :
1970 164638 : if ((if_info->a == const0_rtx
1971 2519 : && (REG_P (if_info->b) || rtx_equal_p (if_info->b, if_info->x)))
1972 166066 : || ((reversep = (noce_reversed_cond_code (if_info) != UNKNOWN))
1973 163547 : && if_info->b == const0_rtx
1974 13226 : && (REG_P (if_info->a) || rtx_equal_p (if_info->a, if_info->x))))
1975 : {
1976 1097 : start_sequence ();
1977 1097 : target = noce_emit_store_flag (if_info,
1978 1097 : gen_reg_rtx (GET_MODE (if_info->x)),
1979 : reversep, -1);
1980 1097 : if (target)
1981 318 : target = expand_simple_binop (GET_MODE (if_info->x), AND,
1982 : reversep ? if_info->a : if_info->b,
1983 : target, if_info->x, 0,
1984 : OPTAB_WIDEN);
1985 :
1986 318 : if (target)
1987 : {
1988 318 : if (target != if_info->x)
1989 0 : noce_emit_move_insn (if_info->x, target);
1990 :
1991 318 : seq = end_ifcvt_sequence (if_info);
1992 318 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
1993 316 : return false;
1994 :
1995 2 : emit_insn_before_setloc (seq, if_info->jump,
1996 2 : INSN_LOCATION (if_info->insn_a));
1997 2 : if_info->transform_name = "noce_try_store_flag_mask";
1998 :
1999 2 : return true;
2000 : }
2001 :
2002 779 : end_sequence ();
2003 : }
2004 :
2005 : return false;
2006 : }
2007 :
2008 : /* Helper function for noce_try_cmove and noce_try_cmove_arith. */
2009 :
2010 : static rtx
2011 469945 : noce_emit_cmove (struct noce_if_info *if_info, rtx x, enum rtx_code code,
2012 : rtx cmp_a, rtx cmp_b, rtx vfalse, rtx vtrue, rtx cc_cmp,
2013 : rtx rev_cc_cmp)
2014 : {
2015 469945 : rtx target ATTRIBUTE_UNUSED;
2016 469945 : bool unsignedp ATTRIBUTE_UNUSED;
2017 :
2018 : /* If earliest == jump, try to build the cmove insn directly.
2019 : This is helpful when combine has created some complex condition
2020 : (like for alpha's cmovlbs) that we can't hope to regenerate
2021 : through the normal interface. */
2022 :
2023 469945 : if (if_info->cond_earliest == if_info->jump)
2024 : {
2025 4475 : rtx cond = gen_rtx_fmt_ee (code, GET_MODE (if_info->cond), cmp_a, cmp_b);
2026 4475 : rtx if_then_else = gen_rtx_IF_THEN_ELSE (GET_MODE (x),
2027 : cond, vtrue, vfalse);
2028 4475 : rtx set = gen_rtx_SET (x, if_then_else);
2029 :
2030 4475 : start_sequence ();
2031 4475 : rtx_insn *insn = emit_insn (set);
2032 :
2033 4475 : if (recog_memoized (insn) >= 0)
2034 : {
2035 1357 : rtx_insn *seq = end_sequence ();
2036 1357 : emit_insn (seq);
2037 :
2038 1357 : return x;
2039 : }
2040 :
2041 3118 : end_sequence ();
2042 : }
2043 :
2044 937176 : unsignedp = (code == LTU || code == GEU
2045 468588 : || code == LEU || code == GTU);
2046 :
2047 468588 : if (cc_cmp != NULL_RTX && rev_cc_cmp != NULL_RTX)
2048 88801 : target = emit_conditional_move (x, cc_cmp, rev_cc_cmp,
2049 88801 : vtrue, vfalse, GET_MODE (x));
2050 : else
2051 : {
2052 : /* Don't even try if the comparison operands are weird
2053 : except that the target supports cbranchcc4. */
2054 379787 : if (! general_operand (cmp_a, GET_MODE (cmp_a))
2055 379787 : || ! general_operand (cmp_b, GET_MODE (cmp_b)))
2056 : {
2057 38330 : if (!have_cbranchcc4
2058 38330 : || GET_MODE_CLASS (GET_MODE (cmp_a)) != MODE_CC
2059 2829 : || cmp_b != const0_rtx)
2060 : return NULL_RTX;
2061 : }
2062 :
2063 344286 : target = emit_conditional_move (x, { code, cmp_a, cmp_b, VOIDmode },
2064 344286 : vtrue, vfalse, GET_MODE (x),
2065 : unsignedp);
2066 : }
2067 :
2068 433087 : if (target)
2069 : return target;
2070 :
2071 : /* We might be faced with a situation like:
2072 :
2073 : x = (reg:M TARGET)
2074 : vtrue = (subreg:M (reg:N VTRUE) BYTE)
2075 : vfalse = (subreg:M (reg:N VFALSE) BYTE)
2076 :
2077 : We can't do a conditional move in mode M, but it's possible that we
2078 : could do a conditional move in mode N instead and take a subreg of
2079 : the result.
2080 :
2081 : If we can't create new pseudos, though, don't bother. */
2082 42354 : if (reload_completed)
2083 : return NULL_RTX;
2084 :
2085 42354 : if (GET_CODE (vtrue) == SUBREG && GET_CODE (vfalse) == SUBREG)
2086 : {
2087 23 : rtx reg_vtrue = SUBREG_REG (vtrue);
2088 23 : rtx reg_vfalse = SUBREG_REG (vfalse);
2089 23 : poly_uint64 byte_vtrue = SUBREG_BYTE (vtrue);
2090 23 : poly_uint64 byte_vfalse = SUBREG_BYTE (vfalse);
2091 23 : rtx promoted_target;
2092 :
2093 23 : if (GET_MODE (reg_vtrue) != GET_MODE (reg_vfalse)
2094 23 : || maybe_ne (byte_vtrue, byte_vfalse)
2095 23 : || (SUBREG_PROMOTED_VAR_P (vtrue)
2096 23 : != SUBREG_PROMOTED_VAR_P (vfalse))
2097 23 : || (SUBREG_PROMOTED_GET (vtrue)
2098 23 : != SUBREG_PROMOTED_GET (vfalse)))
2099 : return NULL_RTX;
2100 :
2101 23 : promoted_target = gen_reg_rtx (GET_MODE (reg_vtrue));
2102 :
2103 46 : target = emit_conditional_move (promoted_target,
2104 : { code, cmp_a, cmp_b, VOIDmode },
2105 : reg_vtrue, reg_vfalse,
2106 23 : GET_MODE (reg_vtrue), unsignedp);
2107 : /* Nope, couldn't do it in that mode either. */
2108 23 : if (!target)
2109 : return NULL_RTX;
2110 :
2111 6 : target = gen_rtx_SUBREG (GET_MODE (vtrue), promoted_target, byte_vtrue);
2112 6 : SUBREG_PROMOTED_VAR_P (target) = SUBREG_PROMOTED_VAR_P (vtrue);
2113 6 : SUBREG_PROMOTED_SET (target, SUBREG_PROMOTED_GET (vtrue));
2114 6 : emit_move_insn (x, target);
2115 6 : return x;
2116 : }
2117 : else
2118 : return NULL_RTX;
2119 : }
2120 :
2121 : /* Try only simple constants and registers here. More complex cases
2122 : are handled in noce_try_cmove_arith after noce_try_store_flag_arith
2123 : has had a go at it. */
2124 :
2125 : static bool
2126 230813 : noce_try_cmove (struct noce_if_info *if_info)
2127 : {
2128 230813 : enum rtx_code code;
2129 230813 : rtx target;
2130 230813 : rtx_insn *seq;
2131 :
2132 458184 : if (!noce_simple_bbs (if_info))
2133 : return false;
2134 :
2135 155062 : if ((CONSTANT_P (if_info->a) || register_operand (if_info->a, VOIDmode))
2136 248084 : && (CONSTANT_P (if_info->b) || register_operand (if_info->b, VOIDmode)))
2137 : {
2138 86330 : start_sequence ();
2139 :
2140 86330 : code = GET_CODE (if_info->cond);
2141 86330 : target = noce_emit_cmove (if_info, if_info->x, code,
2142 : XEXP (if_info->cond, 0),
2143 : XEXP (if_info->cond, 1),
2144 : if_info->a, if_info->b);
2145 :
2146 86330 : if (target)
2147 : {
2148 58543 : if (target != if_info->x)
2149 0 : noce_emit_move_insn (if_info->x, target);
2150 :
2151 58543 : seq = end_ifcvt_sequence (if_info);
2152 58543 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
2153 14915 : return false;
2154 :
2155 43628 : emit_insn_before_setloc (seq, if_info->jump,
2156 43628 : INSN_LOCATION (if_info->insn_a));
2157 43628 : if_info->transform_name = "noce_try_cmove";
2158 :
2159 43628 : return true;
2160 : }
2161 : /* If both a and b are constants try a last-ditch transformation:
2162 : if (test) x = a; else x = b;
2163 : => x = (-(test != 0) & (b - a)) + a;
2164 : Try this only if the target-specific expansion above has failed.
2165 : The target-specific expander may want to generate sequences that
2166 : we don't know about, so give them a chance before trying this
2167 : approach. */
2168 27787 : else if (!targetm.have_conditional_execution ()
2169 27787 : && CONST_INT_P (if_info->a) && CONST_INT_P (if_info->b))
2170 : {
2171 5300 : machine_mode mode = GET_MODE (if_info->x);
2172 5300 : HOST_WIDE_INT ifalse = INTVAL (if_info->a);
2173 5300 : HOST_WIDE_INT itrue = INTVAL (if_info->b);
2174 5300 : rtx target = noce_emit_store_flag (if_info, if_info->x, false, -1);
2175 5300 : if (!target)
2176 : {
2177 5202 : end_sequence ();
2178 5202 : return false;
2179 : }
2180 :
2181 98 : HOST_WIDE_INT diff = (unsigned HOST_WIDE_INT) itrue - ifalse;
2182 : /* Make sure we can represent the difference
2183 : between the two values. */
2184 98 : if ((diff > 0)
2185 98 : != ((ifalse < 0) != (itrue < 0) ? ifalse < 0 : ifalse < itrue))
2186 : {
2187 39 : end_sequence ();
2188 39 : return false;
2189 : }
2190 :
2191 59 : diff = trunc_int_for_mode (diff, mode);
2192 59 : target = expand_simple_binop (mode, AND,
2193 59 : target, gen_int_mode (diff, mode),
2194 : if_info->x, 0, OPTAB_WIDEN);
2195 59 : if (target)
2196 59 : target = expand_simple_binop (mode, PLUS,
2197 59 : target, gen_int_mode (ifalse, mode),
2198 : if_info->x, 0, OPTAB_WIDEN);
2199 59 : if (target)
2200 : {
2201 59 : if (target != if_info->x)
2202 0 : noce_emit_move_insn (if_info->x, target);
2203 :
2204 59 : seq = end_ifcvt_sequence (if_info);
2205 59 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
2206 33 : return false;
2207 :
2208 26 : emit_insn_before_setloc (seq, if_info->jump,
2209 26 : INSN_LOCATION (if_info->insn_a));
2210 26 : if_info->transform_name = "noce_try_cmove";
2211 26 : return true;
2212 : }
2213 : else
2214 : {
2215 0 : end_sequence ();
2216 0 : return false;
2217 : }
2218 : }
2219 : else
2220 22487 : end_sequence ();
2221 : }
2222 :
2223 : return false;
2224 : }
2225 :
2226 : /* Return true if X contains a conditional code mode rtx. */
2227 :
2228 : static bool
2229 2806576 : contains_ccmode_rtx_p (rtx x)
2230 : {
2231 2806576 : subrtx_iterator::array_type array;
2232 7133072 : FOR_EACH_SUBRTX (iter, array, x, ALL)
2233 4369153 : if (GET_MODE_CLASS (GET_MODE (*iter)) == MODE_CC)
2234 42657 : return true;
2235 :
2236 2763919 : return false;
2237 2806576 : }
2238 :
2239 : /* Helper for bb_valid_for_noce_process_p. Validate that
2240 : the rtx insn INSN is a single set that does not set
2241 : the conditional register CC and is in general valid for
2242 : if-conversion. */
2243 :
2244 : static bool
2245 3486080 : insn_valid_noce_process_p (rtx_insn *insn, rtx cc)
2246 : {
2247 3486080 : if (!insn
2248 3485859 : || !NONJUMP_INSN_P (insn)
2249 6318806 : || (cc && set_of (cc, insn)))
2250 660313 : return false;
2251 :
2252 2825767 : rtx sset = single_set (insn);
2253 :
2254 : /* Currently support only simple single sets in test_bb. */
2255 2825767 : if (!sset
2256 2819714 : || !noce_operand_ok (SET_DEST (sset))
2257 2806576 : || contains_ccmode_rtx_p (SET_DEST (sset))
2258 5589686 : || !noce_operand_ok (SET_SRC (sset)))
2259 121928 : return false;
2260 :
2261 : return true;
2262 : }
2263 :
2264 :
2265 : /* Return true iff the registers that the insns in BB_A set do not get
2266 : used in BB_B. If TO_RENAME is non-NULL then it is a location that will be
2267 : renamed later by the caller and so conflicts on it should be ignored
2268 : in this function. */
2269 :
2270 : static bool
2271 55084 : bbs_ok_for_cmove_arith (basic_block bb_a, basic_block bb_b, rtx to_rename)
2272 : {
2273 55084 : rtx_insn *a_insn;
2274 55084 : bitmap bba_sets = BITMAP_ALLOC (®_obstack);
2275 :
2276 55084 : df_ref def;
2277 55084 : df_ref use;
2278 :
2279 275778 : FOR_BB_INSNS (bb_a, a_insn)
2280 : {
2281 220694 : if (!active_insn_p (a_insn))
2282 144960 : continue;
2283 :
2284 75734 : rtx sset_a = single_set (a_insn);
2285 :
2286 75734 : if (!sset_a)
2287 : {
2288 0 : BITMAP_FREE (bba_sets);
2289 0 : return false;
2290 : }
2291 : /* Record all registers that BB_A sets. */
2292 184042 : FOR_EACH_INSN_DEF (def, a_insn)
2293 108308 : if (!(to_rename && DF_REF_REG (def) == to_rename))
2294 54234 : bitmap_set_bit (bba_sets, DF_REF_REGNO (def));
2295 : }
2296 :
2297 55084 : rtx_insn *b_insn;
2298 :
2299 275840 : FOR_BB_INSNS (bb_b, b_insn)
2300 : {
2301 220850 : if (!active_insn_p (b_insn))
2302 145123 : continue;
2303 :
2304 75727 : rtx sset_b = single_set (b_insn);
2305 :
2306 75727 : if (!sset_b)
2307 : {
2308 0 : BITMAP_FREE (bba_sets);
2309 0 : return false;
2310 : }
2311 :
2312 : /* Make sure this is a REG and not some instance
2313 : of ZERO_EXTRACT or non-paradoxical SUBREG or other dangerous stuff.
2314 : If we have a memory destination then we have a pair of simple
2315 : basic blocks performing an operation of the form [addr] = c ? a : b.
2316 : bb_valid_for_noce_process_p will have ensured that these are
2317 : the only stores present. In that case [addr] should be the location
2318 : to be renamed. Assert that the callers set this up properly. */
2319 75727 : if (MEM_P (SET_DEST (sset_b)))
2320 1002 : gcc_assert (rtx_equal_p (SET_DEST (sset_b), to_rename));
2321 74725 : else if (!REG_P (SET_DEST (sset_b))
2322 74725 : && !paradoxical_subreg_p (SET_DEST (sset_b)))
2323 : {
2324 37 : BITMAP_FREE (bba_sets);
2325 37 : return false;
2326 : }
2327 :
2328 : /* If the insn uses a reg set in BB_A return false. */
2329 146876 : FOR_EACH_INSN_USE (use, b_insn)
2330 : {
2331 71243 : if (bitmap_bit_p (bba_sets, DF_REF_REGNO (use)))
2332 : {
2333 57 : BITMAP_FREE (bba_sets);
2334 57 : return false;
2335 : }
2336 : }
2337 :
2338 : }
2339 :
2340 54990 : BITMAP_FREE (bba_sets);
2341 54990 : return true;
2342 : }
2343 :
2344 : /* Emit copies of all the active instructions in BB except the last.
2345 : This is a helper for noce_try_cmove_arith. */
2346 :
2347 : static void
2348 13407 : noce_emit_all_but_last (basic_block bb)
2349 : {
2350 13407 : rtx_insn *last = last_active_insn (bb, false);
2351 13407 : rtx_insn *insn;
2352 140397 : FOR_BB_INSNS (bb, insn)
2353 : {
2354 113583 : if (insn != last && active_insn_p (insn))
2355 : {
2356 35639 : rtx_insn *to_emit = as_a <rtx_insn *> (copy_rtx (insn));
2357 :
2358 35639 : emit_insn (PATTERN (to_emit));
2359 : }
2360 : }
2361 13407 : }
2362 :
2363 : /* Helper for noce_try_cmove_arith. Emit the pattern TO_EMIT and return
2364 : the resulting insn or NULL if it's not a valid insn. */
2365 :
2366 : static rtx_insn *
2367 198744 : noce_emit_insn (rtx to_emit)
2368 : {
2369 198744 : gcc_assert (to_emit);
2370 198744 : rtx_insn *insn = emit_insn (to_emit);
2371 :
2372 198744 : if (recog_memoized (insn) < 0)
2373 202 : return NULL;
2374 :
2375 : return insn;
2376 : }
2377 :
2378 : /* Helper for noce_try_cmove_arith. Emit a copy of the insns up to
2379 : and including the penultimate one in BB if it is not simple
2380 : (as indicated by SIMPLE). Then emit LAST_INSN as the last
2381 : insn in the block. The reason for that is that LAST_INSN may
2382 : have been modified by the preparation in noce_try_cmove_arith. */
2383 :
2384 : static bool
2385 200943 : noce_emit_bb (rtx last_insn, basic_block bb, bool simple)
2386 : {
2387 200943 : if (bb && !simple)
2388 13407 : noce_emit_all_but_last (bb);
2389 :
2390 200943 : if (last_insn && !noce_emit_insn (last_insn))
2391 : return false;
2392 :
2393 : return true;
2394 : }
2395 :
2396 : /* Try more complex cases involving conditional_move. */
2397 :
2398 : static bool
2399 173941 : noce_try_cmove_arith (struct noce_if_info *if_info)
2400 : {
2401 173941 : rtx a = if_info->a;
2402 173941 : rtx b = if_info->b;
2403 173941 : rtx x = if_info->x;
2404 173941 : rtx orig_a, orig_b;
2405 173941 : rtx_insn *insn_a, *insn_b;
2406 173941 : bool a_simple = if_info->then_simple;
2407 173941 : bool b_simple = if_info->else_simple;
2408 173941 : basic_block then_bb = if_info->then_bb;
2409 173941 : basic_block else_bb = if_info->else_bb;
2410 173941 : rtx target;
2411 173941 : bool is_mem = false;
2412 173941 : enum rtx_code code;
2413 173941 : rtx cond = if_info->cond;
2414 173941 : rtx_insn *ifcvt_seq;
2415 :
2416 : /* A conditional move from two memory sources is equivalent to a
2417 : conditional on their addresses followed by a load. Don't do this
2418 : early because it'll screw alias analysis. Note that we've
2419 : already checked for no side effects. */
2420 173941 : if (cse_not_expected
2421 59824 : && MEM_P (a) && MEM_P (b)
2422 178408 : && MEM_ADDR_SPACE (a) == MEM_ADDR_SPACE (b))
2423 : {
2424 4467 : machine_mode address_mode = get_address_mode (a);
2425 :
2426 4467 : a = XEXP (a, 0);
2427 4467 : b = XEXP (b, 0);
2428 4467 : x = gen_reg_rtx (address_mode);
2429 4467 : is_mem = true;
2430 : }
2431 :
2432 : /* ??? We could handle this if we knew that a load from A or B could
2433 : not trap or fault. This is also true if we've already loaded
2434 : from the address along the path from ENTRY. */
2435 169474 : else if (may_trap_or_fault_p (a) || may_trap_or_fault_p (b))
2436 53123 : return false;
2437 :
2438 : /* if (test) x = a + b; else x = c - d;
2439 : => y = a + b;
2440 : x = c - d;
2441 : if (test)
2442 : x = y;
2443 : */
2444 :
2445 120818 : code = GET_CODE (cond);
2446 120818 : insn_a = if_info->insn_a;
2447 120818 : insn_b = if_info->insn_b;
2448 :
2449 120818 : machine_mode x_mode = GET_MODE (x);
2450 :
2451 120818 : if (!can_conditionally_move_p (x_mode))
2452 : return false;
2453 :
2454 : /* Possibly rearrange operands to make things come out more natural. */
2455 100570 : if (noce_reversed_cond_code (if_info) != UNKNOWN)
2456 : {
2457 100570 : bool reversep = false;
2458 100570 : if (rtx_equal_p (b, x))
2459 : reversep = true;
2460 56633 : else if (general_operand (b, GET_MODE (b)))
2461 : reversep = true;
2462 :
2463 : if (reversep)
2464 : {
2465 92881 : if (if_info->rev_cond)
2466 : {
2467 92881 : cond = if_info->rev_cond;
2468 92881 : code = GET_CODE (cond);
2469 : }
2470 : else
2471 0 : code = reversed_comparison_code (cond, if_info->jump);
2472 : std::swap (a, b);
2473 : std::swap (insn_a, insn_b);
2474 : std::swap (a_simple, b_simple);
2475 : std::swap (then_bb, else_bb);
2476 : }
2477 : }
2478 :
2479 28452 : if (then_bb && else_bb
2480 128159 : && (!bbs_ok_for_cmove_arith (then_bb, else_bb, if_info->orig_x)
2481 27495 : || !bbs_ok_for_cmove_arith (else_bb, then_bb, if_info->orig_x)))
2482 94 : return false;
2483 :
2484 100476 : start_sequence ();
2485 :
2486 : /* If one of the blocks is empty then the corresponding B or A value
2487 : came from the test block. The non-empty complex block that we will
2488 : emit might clobber the register used by B or A, so move it to a pseudo
2489 : first. */
2490 :
2491 100476 : rtx tmp_a = NULL_RTX;
2492 100476 : rtx tmp_b = NULL_RTX;
2493 :
2494 100476 : if (b_simple || !else_bb)
2495 89097 : tmp_b = gen_reg_rtx (x_mode);
2496 :
2497 100476 : if (a_simple || !then_bb)
2498 98448 : tmp_a = gen_reg_rtx (x_mode);
2499 :
2500 100476 : orig_a = a;
2501 100476 : orig_b = b;
2502 :
2503 100476 : rtx emit_a = NULL_RTX;
2504 100476 : rtx emit_b = NULL_RTX;
2505 100476 : rtx_insn *tmp_insn = NULL;
2506 100476 : bool modified_in_a = false;
2507 100476 : bool modified_in_b = false;
2508 : /* If either operand is complex, load it into a register first.
2509 : The best way to do this is to copy the original insn. In this
2510 : way we preserve any clobbers etc that the insn may have had.
2511 : This is of course not possible in the IS_MEM case. */
2512 :
2513 100476 : if (! general_operand (a, GET_MODE (a)) || tmp_a)
2514 : {
2515 :
2516 99887 : if (is_mem)
2517 : {
2518 4466 : rtx reg = gen_reg_rtx (GET_MODE (a));
2519 4466 : emit_a = gen_rtx_SET (reg, a);
2520 : }
2521 : else
2522 : {
2523 95421 : if (insn_a)
2524 : {
2525 51425 : a = tmp_a ? tmp_a : gen_reg_rtx (GET_MODE (a));
2526 :
2527 51425 : rtx_insn *copy_of_a = as_a <rtx_insn *> (copy_rtx (insn_a));
2528 51425 : rtx set = single_set (copy_of_a);
2529 51425 : SET_DEST (set) = a;
2530 :
2531 51425 : emit_a = PATTERN (copy_of_a);
2532 : }
2533 : else
2534 : {
2535 43996 : rtx tmp_reg = tmp_a ? tmp_a : gen_reg_rtx (GET_MODE (a));
2536 43996 : emit_a = gen_rtx_SET (tmp_reg, a);
2537 43996 : a = tmp_reg;
2538 : }
2539 : }
2540 : }
2541 :
2542 100476 : if (! general_operand (b, GET_MODE (b)) || tmp_b)
2543 : {
2544 98866 : if (is_mem)
2545 : {
2546 4466 : rtx reg = gen_reg_rtx (GET_MODE (b));
2547 4466 : emit_b = gen_rtx_SET (reg, b);
2548 : }
2549 : else
2550 : {
2551 94400 : if (insn_b)
2552 : {
2553 94208 : b = tmp_b ? tmp_b : gen_reg_rtx (GET_MODE (b));
2554 94208 : rtx_insn *copy_of_b = as_a <rtx_insn *> (copy_rtx (insn_b));
2555 94208 : rtx set = single_set (copy_of_b);
2556 :
2557 94208 : SET_DEST (set) = b;
2558 94208 : emit_b = PATTERN (copy_of_b);
2559 : }
2560 : else
2561 : {
2562 192 : rtx tmp_reg = tmp_b ? tmp_b : gen_reg_rtx (GET_MODE (b));
2563 192 : emit_b = gen_rtx_SET (tmp_reg, b);
2564 192 : b = tmp_reg;
2565 : }
2566 : }
2567 : }
2568 :
2569 100476 : modified_in_a = emit_a != NULL_RTX && modified_in_p (orig_b, emit_a);
2570 100476 : if (tmp_b && then_bb)
2571 : {
2572 80724 : FOR_BB_INSNS (then_bb, tmp_insn)
2573 : /* Don't check inside insn_a. We will have changed it to emit_a
2574 : with a destination that doesn't conflict. */
2575 60817 : if (!(insn_a && tmp_insn == insn_a)
2576 101727 : && modified_in_p (orig_b, tmp_insn))
2577 : {
2578 : modified_in_a = true;
2579 : break;
2580 : }
2581 :
2582 : }
2583 :
2584 100476 : modified_in_b = emit_b != NULL_RTX && modified_in_p (orig_a, emit_b);
2585 100476 : if (tmp_a && else_bb)
2586 : {
2587 412435 : FOR_BB_INSNS (else_bb, tmp_insn)
2588 : /* Don't check inside insn_b. We will have changed it to emit_b
2589 : with a destination that doesn't conflict. */
2590 314807 : if (!(insn_b && tmp_insn == insn_b)
2591 531986 : && modified_in_p (orig_a, tmp_insn))
2592 : {
2593 : modified_in_b = true;
2594 : break;
2595 : }
2596 : }
2597 :
2598 : /* If insn to set up A clobbers any registers B depends on, try to
2599 : swap insn that sets up A with the one that sets up B. If even
2600 : that doesn't help, punt. */
2601 100476 : if (modified_in_a && !modified_in_b)
2602 : {
2603 0 : if (!noce_emit_bb (emit_b, else_bb, b_simple))
2604 0 : goto end_seq_and_fail;
2605 :
2606 0 : if (!noce_emit_bb (emit_a, then_bb, a_simple))
2607 0 : goto end_seq_and_fail;
2608 : }
2609 100476 : else if (!modified_in_a)
2610 : {
2611 100476 : if (!noce_emit_bb (emit_a, then_bb, a_simple))
2612 9 : goto end_seq_and_fail;
2613 :
2614 100467 : if (!noce_emit_bb (emit_b, else_bb, b_simple))
2615 193 : goto end_seq_and_fail;
2616 : }
2617 : else
2618 0 : goto end_seq_and_fail;
2619 :
2620 100274 : target = noce_emit_cmove (if_info, x, code, XEXP (cond, 0), XEXP (cond, 1),
2621 : a, b);
2622 :
2623 100274 : if (! target)
2624 28316 : goto end_seq_and_fail;
2625 :
2626 : /* If we're handling a memory for above, emit the load now. */
2627 71958 : if (is_mem)
2628 : {
2629 2422 : rtx mem = gen_rtx_MEM (GET_MODE (if_info->x), target);
2630 :
2631 : /* Copy over flags as appropriate. */
2632 2422 : if (MEM_VOLATILE_P (if_info->a) || MEM_VOLATILE_P (if_info->b))
2633 0 : MEM_VOLATILE_P (mem) = 1;
2634 2422 : if (MEM_ALIAS_SET (if_info->a) == MEM_ALIAS_SET (if_info->b))
2635 2240 : set_mem_alias_set (mem, MEM_ALIAS_SET (if_info->a));
2636 4844 : set_mem_align (mem,
2637 2422 : MIN (MEM_ALIGN (if_info->a), MEM_ALIGN (if_info->b)));
2638 :
2639 2422 : gcc_assert (MEM_ADDR_SPACE (if_info->a) == MEM_ADDR_SPACE (if_info->b));
2640 2422 : set_mem_addr_space (mem, MEM_ADDR_SPACE (if_info->a));
2641 :
2642 2422 : noce_emit_move_insn (if_info->x, mem);
2643 : }
2644 69536 : else if (target != x)
2645 0 : noce_emit_move_insn (x, target);
2646 :
2647 71958 : ifcvt_seq = end_ifcvt_sequence (if_info);
2648 71958 : if (!ifcvt_seq || !targetm.noce_conversion_profitable_p (ifcvt_seq, if_info))
2649 44298 : return false;
2650 :
2651 27660 : emit_insn_before_setloc (ifcvt_seq, if_info->jump,
2652 27660 : INSN_LOCATION (if_info->insn_a));
2653 27660 : if_info->transform_name = "noce_try_cmove_arith";
2654 27660 : return true;
2655 :
2656 28518 : end_seq_and_fail:
2657 28518 : end_sequence ();
2658 28518 : return false;
2659 : }
2660 :
2661 : /* For most cases, the simplified condition we found is the best
2662 : choice, but this is not the case for the min/max/abs transforms.
2663 : For these we wish to know that it is A or B in the condition. */
2664 :
2665 : static rtx
2666 159944 : noce_get_alt_condition (struct noce_if_info *if_info, rtx target,
2667 : rtx_insn **earliest)
2668 : {
2669 159944 : rtx cond, set;
2670 159944 : rtx_insn *insn;
2671 159944 : bool reverse;
2672 :
2673 : /* If target is already mentioned in the known condition, return it. */
2674 159944 : if (reg_mentioned_p (target, if_info->cond))
2675 : {
2676 7709 : *earliest = if_info->cond_earliest;
2677 7709 : return if_info->cond;
2678 : }
2679 :
2680 152235 : set = pc_set (if_info->jump);
2681 152235 : cond = XEXP (SET_SRC (set), 0);
2682 152235 : reverse
2683 304470 : = GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
2684 152235 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (if_info->jump);
2685 152235 : if (if_info->then_else_reversed)
2686 23173 : reverse = !reverse;
2687 :
2688 : /* If we're looking for a constant, try to make the conditional
2689 : have that constant in it. There are two reasons why it may
2690 : not have the constant we want:
2691 :
2692 : 1. GCC may have needed to put the constant in a register, because
2693 : the target can't compare directly against that constant. For
2694 : this case, we look for a SET immediately before the comparison
2695 : that puts a constant in that register.
2696 :
2697 : 2. GCC may have canonicalized the conditional, for example
2698 : replacing "if x < 4" with "if x <= 3". We can undo that (or
2699 : make equivalent types of changes) to get the constants we need
2700 : if they're off by one in the right direction. */
2701 :
2702 152235 : if (CONST_INT_P (target))
2703 : {
2704 45969 : enum rtx_code code = GET_CODE (if_info->cond);
2705 45969 : rtx op_a = XEXP (if_info->cond, 0);
2706 45969 : rtx op_b = XEXP (if_info->cond, 1);
2707 45969 : rtx_insn *prev_insn;
2708 :
2709 : /* First, look to see if we put a constant in a register. */
2710 45969 : prev_insn = prev_nonnote_nondebug_insn (if_info->cond_earliest);
2711 45969 : if (prev_insn
2712 45860 : && BLOCK_FOR_INSN (prev_insn)
2713 45860 : == BLOCK_FOR_INSN (if_info->cond_earliest)
2714 36807 : && INSN_P (prev_insn)
2715 79170 : && GET_CODE (PATTERN (prev_insn)) == SET)
2716 : {
2717 25323 : rtx src = find_reg_equal_equiv_note (prev_insn);
2718 25323 : if (!src)
2719 25157 : src = SET_SRC (PATTERN (prev_insn));
2720 25323 : if (CONST_INT_P (src))
2721 : {
2722 15461 : if (rtx_equal_p (op_a, SET_DEST (PATTERN (prev_insn))))
2723 : op_a = src;
2724 14944 : else if (rtx_equal_p (op_b, SET_DEST (PATTERN (prev_insn))))
2725 1073 : op_b = src;
2726 :
2727 15461 : if (CONST_INT_P (op_a))
2728 : {
2729 517 : std::swap (op_a, op_b);
2730 517 : code = swap_condition (code);
2731 : }
2732 : }
2733 : }
2734 :
2735 : /* Now, look to see if we can get the right constant by
2736 : adjusting the conditional. */
2737 45969 : if (CONST_INT_P (op_b))
2738 : {
2739 23630 : HOST_WIDE_INT desired_val = INTVAL (target);
2740 23630 : HOST_WIDE_INT actual_val = INTVAL (op_b);
2741 :
2742 23630 : switch (code)
2743 : {
2744 1145 : case LT:
2745 1145 : if (desired_val != HOST_WIDE_INT_MAX
2746 1128 : && actual_val == desired_val + 1)
2747 : {
2748 166 : code = LE;
2749 166 : op_b = GEN_INT (desired_val);
2750 : }
2751 : break;
2752 3 : case LE:
2753 3 : if (desired_val != HOST_WIDE_INT_MIN
2754 3 : && actual_val == desired_val - 1)
2755 : {
2756 0 : code = LT;
2757 0 : op_b = GEN_INT (desired_val);
2758 : }
2759 : break;
2760 1707 : case GT:
2761 1707 : if (desired_val != HOST_WIDE_INT_MIN
2762 1707 : && actual_val == desired_val - 1)
2763 : {
2764 247 : code = GE;
2765 247 : op_b = GEN_INT (desired_val);
2766 : }
2767 : break;
2768 40 : case GE:
2769 40 : if (desired_val != HOST_WIDE_INT_MAX
2770 25 : && actual_val == desired_val + 1)
2771 : {
2772 3 : code = GT;
2773 3 : op_b = GEN_INT (desired_val);
2774 : }
2775 : break;
2776 : default:
2777 : break;
2778 : }
2779 : }
2780 :
2781 : /* If we made any changes, generate a new conditional that is
2782 : equivalent to what we started with, but has the right
2783 : constants in it. */
2784 45969 : if (code != GET_CODE (if_info->cond)
2785 45037 : || op_a != XEXP (if_info->cond, 0)
2786 45036 : || op_b != XEXP (if_info->cond, 1))
2787 : {
2788 2006 : cond = gen_rtx_fmt_ee (code, GET_MODE (cond), op_a, op_b);
2789 2006 : *earliest = if_info->cond_earliest;
2790 2006 : return cond;
2791 : }
2792 : }
2793 :
2794 150229 : cond = canonicalize_condition (if_info->jump, cond, reverse,
2795 : earliest, target, have_cbranchcc4, true);
2796 150229 : if (! cond || ! reg_mentioned_p (target, cond))
2797 150229 : return NULL;
2798 :
2799 : /* We almost certainly searched back to a different place.
2800 : Need to re-verify correct lifetimes. */
2801 :
2802 : /* X may not be mentioned in the range (cond_earliest, jump]. */
2803 0 : for (insn = if_info->jump; insn != *earliest; insn = PREV_INSN (insn))
2804 0 : if (INSN_P (insn) && reg_overlap_mentioned_p (if_info->x, PATTERN (insn)))
2805 : return NULL;
2806 :
2807 : /* A and B may not be modified in the range [cond_earliest, jump). */
2808 0 : for (insn = *earliest; insn != if_info->jump; insn = NEXT_INSN (insn))
2809 0 : if (INSN_P (insn)
2810 0 : && (modified_in_p (if_info->a, insn)
2811 0 : || modified_in_p (if_info->b, insn)))
2812 0 : return NULL;
2813 :
2814 : return cond;
2815 : }
2816 :
2817 : /* Convert "if (a < b) x = a; else x = b;" to "x = min(a, b);", etc. */
2818 :
2819 : static bool
2820 233993 : noce_try_minmax (struct noce_if_info *if_info)
2821 : {
2822 233993 : rtx cond, target;
2823 233993 : rtx_insn *earliest, *seq;
2824 233993 : enum rtx_code code, op;
2825 233993 : bool unsignedp;
2826 :
2827 511213 : if (!noce_simple_bbs (if_info))
2828 : return false;
2829 :
2830 : /* ??? Reject modes with NaNs or signed zeros since we don't know how
2831 : they will be resolved with an SMIN/SMAX. It wouldn't be too hard
2832 : to get the target to tell us... */
2833 216242 : if (HONOR_SIGNED_ZEROS (if_info->x)
2834 216242 : || HONOR_NANS (if_info->x))
2835 57299 : return false;
2836 :
2837 158943 : cond = noce_get_alt_condition (if_info, if_info->a, &earliest);
2838 158943 : if (!cond)
2839 : return false;
2840 :
2841 : /* Verify the condition is of the form we expect, and canonicalize
2842 : the comparison code. */
2843 9690 : code = GET_CODE (cond);
2844 9690 : if (rtx_equal_p (XEXP (cond, 0), if_info->a))
2845 : {
2846 2498 : if (! rtx_equal_p (XEXP (cond, 1), if_info->b))
2847 : return false;
2848 : }
2849 7192 : else if (rtx_equal_p (XEXP (cond, 1), if_info->a))
2850 : {
2851 4253 : if (! rtx_equal_p (XEXP (cond, 0), if_info->b))
2852 : return false;
2853 39 : code = swap_condition (code);
2854 : }
2855 : else
2856 : return false;
2857 :
2858 : /* Determine what sort of operation this is. Note that the code is for
2859 : a taken branch, so the code->operation mapping appears backwards. */
2860 53 : switch (code)
2861 : {
2862 : case LT:
2863 : case LE:
2864 : case UNLT:
2865 : case UNLE:
2866 : op = SMAX;
2867 : unsignedp = false;
2868 : break;
2869 8 : case GT:
2870 8 : case GE:
2871 8 : case UNGT:
2872 8 : case UNGE:
2873 8 : op = SMIN;
2874 8 : unsignedp = false;
2875 8 : break;
2876 3 : case LTU:
2877 3 : case LEU:
2878 3 : op = UMAX;
2879 3 : unsignedp = true;
2880 3 : break;
2881 21 : case GTU:
2882 21 : case GEU:
2883 21 : op = UMIN;
2884 21 : unsignedp = true;
2885 21 : break;
2886 : default:
2887 : return false;
2888 : }
2889 :
2890 53 : start_sequence ();
2891 :
2892 53 : target = expand_simple_binop (GET_MODE (if_info->x), op,
2893 : if_info->a, if_info->b,
2894 : if_info->x, unsignedp, OPTAB_WIDEN);
2895 53 : if (! target)
2896 : {
2897 0 : end_sequence ();
2898 0 : return false;
2899 : }
2900 53 : if (target != if_info->x)
2901 0 : noce_emit_move_insn (if_info->x, target);
2902 :
2903 53 : seq = end_ifcvt_sequence (if_info);
2904 53 : if (!seq)
2905 : return false;
2906 :
2907 53 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
2908 53 : if_info->cond = cond;
2909 53 : if_info->cond_earliest = earliest;
2910 53 : if_info->rev_cond = NULL_RTX;
2911 53 : if_info->transform_name = "noce_try_minmax";
2912 :
2913 53 : return true;
2914 : }
2915 :
2916 : /* Convert "if (a < 0) x = -a; else x = a;" to "x = abs(a);",
2917 : "if (a < 0) x = ~a; else x = a;" to "x = one_cmpl_abs(a);",
2918 : etc. */
2919 :
2920 : static bool
2921 233940 : noce_try_abs (struct noce_if_info *if_info)
2922 : {
2923 233940 : rtx cond, target, a, b, c;
2924 233940 : rtx_insn *earliest, *seq;
2925 233940 : bool negate;
2926 233940 : bool one_cmpl = false;
2927 :
2928 511125 : if (!noce_simple_bbs (if_info))
2929 : return false;
2930 :
2931 : /* Reject modes with signed zeros. */
2932 216189 : if (HONOR_SIGNED_ZEROS (if_info->x))
2933 : return false;
2934 :
2935 : /* Recognize A and B as constituting an ABS or NABS. The canonical
2936 : form is a branch around the negation, taken when the object is the
2937 : first operand of a comparison against 0 that evaluates to true. */
2938 158899 : a = if_info->a;
2939 158899 : b = if_info->b;
2940 158899 : if (GET_CODE (a) == NEG && rtx_equal_p (XEXP (a, 0), b))
2941 : negate = false;
2942 157928 : else if (GET_CODE (b) == NEG && rtx_equal_p (XEXP (b, 0), a))
2943 : {
2944 : std::swap (a, b);
2945 : negate = true;
2946 : }
2947 157898 : else if (GET_CODE (a) == NOT && rtx_equal_p (XEXP (a, 0), b))
2948 : {
2949 : negate = false;
2950 : one_cmpl = true;
2951 : }
2952 157898 : else if (GET_CODE (b) == NOT && rtx_equal_p (XEXP (b, 0), a))
2953 : {
2954 : std::swap (a, b);
2955 : negate = true;
2956 : one_cmpl = true;
2957 : }
2958 : else
2959 157898 : return false;
2960 :
2961 1001 : cond = noce_get_alt_condition (if_info, b, &earliest);
2962 1001 : if (!cond)
2963 : return false;
2964 :
2965 : /* Verify the condition is of the form we expect. */
2966 25 : if (rtx_equal_p (XEXP (cond, 0), b))
2967 25 : c = XEXP (cond, 1);
2968 0 : else if (rtx_equal_p (XEXP (cond, 1), b))
2969 : {
2970 0 : c = XEXP (cond, 0);
2971 0 : negate = !negate;
2972 : }
2973 : else
2974 : return false;
2975 :
2976 : /* Verify that C is zero. Search one step backward for a
2977 : REG_EQUAL note or a simple source if necessary. */
2978 25 : if (REG_P (c))
2979 : {
2980 0 : rtx set;
2981 0 : rtx_insn *insn = prev_nonnote_nondebug_insn (earliest);
2982 0 : if (insn
2983 0 : && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (earliest)
2984 0 : && (set = single_set (insn))
2985 0 : && rtx_equal_p (SET_DEST (set), c))
2986 : {
2987 0 : rtx note = find_reg_equal_equiv_note (insn);
2988 0 : if (note)
2989 0 : c = XEXP (note, 0);
2990 : else
2991 0 : c = SET_SRC (set);
2992 : }
2993 : else
2994 0 : return false;
2995 : }
2996 25 : if (MEM_P (c)
2997 0 : && GET_CODE (XEXP (c, 0)) == SYMBOL_REF
2998 25 : && CONSTANT_POOL_ADDRESS_P (XEXP (c, 0)))
2999 0 : c = get_pool_constant (XEXP (c, 0));
3000 :
3001 : /* Work around funny ideas get_condition has wrt canonicalization.
3002 : Note that these rtx constants are known to be CONST_INT, and
3003 : therefore imply integer comparisons.
3004 : The one_cmpl case is more complicated, as we want to handle
3005 : only x < 0 ? ~x : x or x >= 0 ? x : ~x to one_cmpl_abs (x)
3006 : and x < 0 ? x : ~x or x >= 0 ? ~x : x to ~one_cmpl_abs (x),
3007 : but not other cases (x > -1 is equivalent of x >= 0). */
3008 25 : if (c == constm1_rtx && GET_CODE (cond) == GT)
3009 : ;
3010 1 : else if (c == const1_rtx && GET_CODE (cond) == LT)
3011 : {
3012 0 : if (one_cmpl)
3013 : return false;
3014 : }
3015 1 : else if (c == CONST0_RTX (GET_MODE (b)))
3016 : {
3017 0 : if (one_cmpl
3018 0 : && GET_CODE (cond) != GE
3019 0 : && GET_CODE (cond) != LT)
3020 : return false;
3021 : }
3022 : else
3023 : return false;
3024 :
3025 : /* Determine what sort of operation this is. */
3026 24 : switch (GET_CODE (cond))
3027 : {
3028 0 : case LT:
3029 0 : case LE:
3030 0 : case UNLT:
3031 0 : case UNLE:
3032 0 : negate = !negate;
3033 0 : break;
3034 : case GT:
3035 : case GE:
3036 : case UNGT:
3037 : case UNGE:
3038 : break;
3039 : default:
3040 : return false;
3041 : }
3042 :
3043 24 : start_sequence ();
3044 24 : if (one_cmpl)
3045 0 : target = expand_one_cmpl_abs_nojump (GET_MODE (if_info->x), b,
3046 : if_info->x);
3047 : else
3048 24 : target = expand_abs_nojump (GET_MODE (if_info->x), b, if_info->x, 1);
3049 :
3050 : /* ??? It's a quandary whether cmove would be better here, especially
3051 : for integers. Perhaps combine will clean things up. */
3052 24 : if (target && negate)
3053 : {
3054 0 : if (one_cmpl)
3055 0 : target = expand_simple_unop (GET_MODE (target), NOT, target,
3056 : if_info->x, 0);
3057 : else
3058 0 : target = expand_simple_unop (GET_MODE (target), NEG, target,
3059 : if_info->x, 0);
3060 : }
3061 :
3062 24 : if (! target)
3063 : {
3064 0 : end_sequence ();
3065 0 : return false;
3066 : }
3067 :
3068 24 : if (target != if_info->x)
3069 0 : noce_emit_move_insn (if_info->x, target);
3070 :
3071 24 : seq = end_ifcvt_sequence (if_info);
3072 24 : if (!seq)
3073 : return false;
3074 :
3075 24 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3076 24 : if_info->cond = cond;
3077 24 : if_info->cond_earliest = earliest;
3078 24 : if_info->rev_cond = NULL_RTX;
3079 24 : if_info->transform_name = "noce_try_abs";
3080 :
3081 24 : return true;
3082 : }
3083 :
3084 : /* Convert "if (m < 0) x = b; else x = 0;" to "x = (m >> C) & b;". */
3085 :
3086 : static bool
3087 146281 : noce_try_sign_mask (struct noce_if_info *if_info)
3088 : {
3089 146281 : rtx cond, t, m, c;
3090 146281 : rtx_insn *seq;
3091 146281 : machine_mode mode;
3092 146281 : enum rtx_code code;
3093 146281 : bool t_unconditional;
3094 :
3095 306109 : if (!noce_simple_bbs (if_info))
3096 : return false;
3097 :
3098 134062 : cond = if_info->cond;
3099 134062 : code = GET_CODE (cond);
3100 134062 : m = XEXP (cond, 0);
3101 134062 : c = XEXP (cond, 1);
3102 :
3103 134062 : t = NULL_RTX;
3104 134062 : if (if_info->a == const0_rtx)
3105 : {
3106 2347 : if ((code == LT && c == const0_rtx)
3107 2318 : || (code == LE && c == constm1_rtx))
3108 29 : t = if_info->b;
3109 : }
3110 131715 : else if (if_info->b == const0_rtx)
3111 : {
3112 11004 : if ((code == GE && c == const0_rtx)
3113 11004 : || (code == GT && c == constm1_rtx))
3114 : t = if_info->a;
3115 : }
3116 :
3117 106 : if (! t || side_effects_p (t))
3118 133956 : return false;
3119 :
3120 : /* We currently don't handle different modes. */
3121 106 : mode = GET_MODE (t);
3122 106 : if (GET_MODE (m) != mode)
3123 : return false;
3124 :
3125 : /* This is only profitable if T is unconditionally executed/evaluated in the
3126 : original insn sequence or T is cheap and can't trap or fault. The former
3127 : happens if B is the non-zero (T) value and if INSN_B was taken from
3128 : TEST_BB, or there was no INSN_B which can happen for e.g. conditional
3129 : stores to memory. For the cost computation use the block TEST_BB where
3130 : the evaluation will end up after the transformation. */
3131 43 : t_unconditional
3132 86 : = (t == if_info->b
3133 43 : && (if_info->insn_b == NULL_RTX
3134 0 : || BLOCK_FOR_INSN (if_info->insn_b) == if_info->test_bb));
3135 86 : if (!(t_unconditional
3136 43 : || ((set_src_cost (t, mode, if_info->speed_p)
3137 : < COSTS_N_INSNS (2))
3138 37 : && !may_trap_or_fault_p (t))))
3139 43 : return false;
3140 :
3141 0 : if (!noce_can_force_operand (t))
3142 : return false;
3143 :
3144 0 : start_sequence ();
3145 : /* Use emit_store_flag to generate "m < 0 ? -1 : 0" instead of expanding
3146 : "(signed) m >> 31" directly. This benefits targets with specialized
3147 : insns to obtain the signmask, but still uses ashr_optab otherwise. */
3148 0 : m = emit_store_flag (gen_reg_rtx (mode), LT, m, const0_rtx, mode, 0, -1);
3149 0 : t = m ? expand_binop (mode, and_optab, m, t, NULL_RTX, 0, OPTAB_DIRECT)
3150 : : NULL_RTX;
3151 :
3152 0 : if (!t)
3153 : {
3154 0 : end_sequence ();
3155 0 : return false;
3156 : }
3157 :
3158 0 : noce_emit_move_insn (if_info->x, t);
3159 :
3160 0 : seq = end_ifcvt_sequence (if_info);
3161 0 : if (!seq)
3162 : return false;
3163 :
3164 0 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3165 0 : if_info->transform_name = "noce_try_sign_mask";
3166 :
3167 0 : return true;
3168 : }
3169 :
3170 : /* Check if OP is supported by conditional zero based if conversion,
3171 : returning TRUE if satisfied otherwise FALSE.
3172 :
3173 : OP is the operation to check. */
3174 :
3175 : static bool
3176 112538 : noce_cond_zero_binary_op_supported (rtx op)
3177 : {
3178 112538 : enum rtx_code opcode = GET_CODE (op);
3179 :
3180 0 : if (opcode == PLUS || opcode == MINUS || opcode == IOR || opcode == XOR
3181 : || opcode == ASHIFT || opcode == ASHIFTRT || opcode == LSHIFTRT
3182 : || opcode == ROTATE || opcode == ROTATERT || opcode == AND)
3183 0 : return true;
3184 :
3185 : return false;
3186 : }
3187 :
3188 : /* Helper function to return REG itself,
3189 : otherwise NULL_RTX for other RTX_CODE. */
3190 :
3191 : static rtx
3192 27226 : get_base_reg (rtx exp)
3193 : {
3194 0 : if (REG_P (exp))
3195 : return exp;
3196 7755 : else if (SUBREG_P (exp))
3197 0 : return SUBREG_REG (exp);
3198 :
3199 : return NULL_RTX;
3200 : }
3201 :
3202 : /* Try to covert if-then-else with conditional zero,
3203 : returning TURE on success or FALSE on failure.
3204 : IF_INFO describes the if-conversion scenario under consideration.
3205 :
3206 : It verifies the branch structure on left and transforms it into branchless
3207 : sequence on the right, with a backend provided conditional zero or orig for
3208 : operand z. If true, tmp is z, 0 otherwise (y op 0 is same as y for most op).
3209 :
3210 : if (cond) | tmp = cond ? z : 0
3211 : x = y op z | x = y op tmp
3212 : else |
3213 : x = y |
3214 :
3215 : AND is special as it needs to be handled differently.
3216 :
3217 : tmp = !cond ? y : 0
3218 : x = (y & z) | tmp
3219 : Also for AND try:
3220 : tmp = cond ? z : -1
3221 : x = y op tmp
3222 : To see if it is cheaper to produce `!cond ? y : 0`
3223 : or `cond ? z : -1`.
3224 : */
3225 :
3226 : static int
3227 182387 : noce_try_cond_arith (struct noce_if_info *if_info)
3228 : {
3229 182387 : rtx target, a, b, a_op0, a_op1;
3230 182387 : rtx cond = if_info->cond;
3231 182387 : rtx_code code = GET_CODE (cond);
3232 182387 : rtx_insn *seq;
3233 182387 : rtx_code op;
3234 182387 : machine_mode mode = GET_MODE (if_info->x);
3235 :
3236 : /* Scalar integral modes are only supported here.
3237 : Could support scalar floating point but that
3238 : would be only with -ffast-math and might
3239 : be worse than a branch. */
3240 182387 : if (!SCALAR_INT_MODE_P (mode))
3241 : return false;
3242 :
3243 322735 : if (!noce_simple_bbs (if_info))
3244 : return false;
3245 :
3246 111405 : a = copy_rtx (if_info->a);
3247 111405 : b = copy_rtx (if_info->b);
3248 :
3249 : /* Canonicalize x = y : (y op z) to x = (y op z) : y. */
3250 111405 : if (REG_P (a) && noce_cond_zero_binary_op_supported (b))
3251 : {
3252 259 : if (if_info->rev_cond)
3253 : {
3254 259 : cond = if_info->rev_cond;
3255 259 : code = GET_CODE (cond);
3256 : }
3257 : else
3258 0 : code = reversed_comparison_code (cond, if_info->jump);
3259 : std::swap (a, b);
3260 : }
3261 :
3262 : /* Check if x = (y op z) : y is supported by czero based ifcvt. */
3263 111146 : else if (!(noce_cond_zero_binary_op_supported (a) && REG_P (b)))
3264 94166 : goto fail;
3265 :
3266 17239 : if (code == UNKNOWN)
3267 0 : goto fail;
3268 :
3269 17239 : op = GET_CODE (a);
3270 :
3271 : /* Canonicalize x = (z op y) : y to x = (y op z) : y */
3272 17239 : a_op1 = get_base_reg (XEXP (a, 1));
3273 9986 : if (a_op1 && rtx_equal_p (a_op1, b) && COMMUTATIVE_ARITH_P (a))
3274 : {
3275 15 : std::swap (XEXP (a, 0), XEXP (a, 1));
3276 15 : a_op1 = get_base_reg (XEXP (a, 1));
3277 : }
3278 :
3279 17224 : if (a_op1 == NULL_RTX)
3280 7267 : goto fail;
3281 :
3282 : /* Ensure the cond is of form: x = (y op z) : y */
3283 9972 : a_op0 = get_base_reg (XEXP (a, 0));
3284 9963 : if (!(a_op0 && rtx_equal_p (a_op0, b)))
3285 859 : goto fail;
3286 :
3287 9113 : start_sequence ();
3288 :
3289 9113 : target = gen_reg_rtx (GET_MODE (XEXP (a, op != AND)));
3290 :
3291 : /* AND requires !cond, instead we swap ops around. */
3292 9113 : target = noce_emit_cmove (if_info, target, code,
3293 : XEXP (cond, 0), XEXP (cond, 1),
3294 : op != AND ? XEXP (a, 1) : const0_rtx,
3295 : op != AND ? const0_rtx : XEXP (a, 0));
3296 9113 : if (!target)
3297 : {
3298 386 : end_sequence ();
3299 386 : rtx tmp = XEXP (a, op != AND);
3300 : /* If the cmove fails and this was a lowpart subreg,
3301 : then try the reg part and then putting back the lowpart
3302 : afterwards. */
3303 386 : if (GET_CODE (tmp) != SUBREG || !subreg_lowpart_p (tmp))
3304 386 : return false;
3305 0 : tmp = SUBREG_REG (tmp);
3306 : /* Only handle integer scalar modes for the inner mode of
3307 : the subreg. */
3308 0 : if (!SCALAR_INT_MODE_P (GET_MODE (tmp)))
3309 : return false;
3310 :
3311 0 : start_sequence ();
3312 0 : target = gen_reg_rtx (GET_MODE (tmp));
3313 0 : target = noce_emit_cmove (if_info, target, code,
3314 : XEXP (cond, 0), XEXP (cond, 1),
3315 : op != AND ? tmp : const0_rtx,
3316 : op != AND ? const0_rtx : tmp);
3317 0 : if (!target)
3318 0 : goto end_seq_n_fail;
3319 0 : target = rtl_hooks.gen_lowpart_no_emit (GET_MODE (XEXP (a, op != AND)), target);
3320 0 : gcc_assert (target);
3321 : }
3322 :
3323 : /* For AND, try `cond ? z : -1` to see if that is cheaper or the same cost.
3324 : In some cases it will be cheaper to produce the -1 rather than the 0 case. */
3325 8727 : if (op == AND)
3326 : {
3327 6 : rtx_insn *seq0 = end_sequence ();
3328 6 : unsigned cost0 = seq_cost (seq0, if_info->speed_p);
3329 6 : if (!target)
3330 : cost0 = -1u;
3331 :
3332 : /* Produce `cond ? z : -1`. */
3333 6 : rtx targetm1;
3334 6 : start_sequence ();
3335 6 : targetm1 = gen_reg_rtx (GET_MODE (XEXP (a, 1)));
3336 6 : targetm1 = noce_emit_cmove (if_info, targetm1, code,
3337 : XEXP (cond, 0), XEXP (cond, 1),
3338 : XEXP (a, 1), constm1_rtx);
3339 6 : rtx_insn *seqm1 = end_sequence ();
3340 6 : unsigned costm1 = seq_cost (seqm1, if_info->speed_p);
3341 6 : if (!targetm1)
3342 0 : costm1 = -1u;
3343 : /* If both fails, then there is no costing to be done. */
3344 6 : if (!targetm1 && !target)
3345 : return false;
3346 :
3347 : /* If -1 is cheaper or the same cost to producing 0, then use that. */
3348 6 : if (costm1 <= cost0)
3349 : {
3350 6 : push_to_sequence (seqm1);
3351 6 : targetm1 = expand_simple_binop (mode, op, a_op0, targetm1,
3352 : if_info->x, 0, OPTAB_WIDEN);
3353 6 : if (targetm1)
3354 : {
3355 6 : target = targetm1;
3356 6 : goto success;
3357 : }
3358 0 : end_sequence ();
3359 : }
3360 0 : if (!target)
3361 : return false;
3362 : /* For 0 the produce sequence is:
3363 : tmp = !cond ? y : 0
3364 : x = (y & z) | tmp */
3365 0 : push_to_sequence (seq0);
3366 0 : rtx a_bin = gen_reg_rtx (mode);
3367 0 : noce_emit_move_insn (a_bin, a);
3368 :
3369 0 : target = expand_simple_binop (mode, IOR, a_bin, target, if_info->x, 0,
3370 : OPTAB_WIDEN);
3371 0 : if (!target)
3372 0 : goto end_seq_n_fail;
3373 0 : goto success;
3374 : }
3375 8721 : if (!target)
3376 : goto end_seq_n_fail;
3377 :
3378 8721 : target = expand_simple_binop (mode, op, a_op0, target, if_info->x, 0,
3379 : OPTAB_WIDEN);
3380 :
3381 8721 : if (!target)
3382 0 : goto end_seq_n_fail;
3383 :
3384 8721 : success:
3385 8727 : if (target != if_info->x)
3386 0 : noce_emit_move_insn (if_info->x, target);
3387 :
3388 8727 : seq = end_ifcvt_sequence (if_info);
3389 8727 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
3390 281 : goto fail;
3391 :
3392 8446 : emit_insn_before_setloc (seq, if_info->jump, INSN_LOCATION (if_info->insn_a));
3393 8446 : if_info->transform_name = "noce_try_cond_arith";
3394 8446 : return true;
3395 :
3396 0 : end_seq_n_fail:
3397 0 : end_sequence ();
3398 :
3399 : fail:
3400 :
3401 : return false;
3402 : }
3403 :
3404 : /* Optimize away "if (x & C) x |= C" and similar bit manipulation
3405 : transformations. */
3406 :
3407 : static bool
3408 233993 : noce_try_bitop (struct noce_if_info *if_info)
3409 : {
3410 233993 : rtx cond, x, a, result;
3411 233993 : rtx_insn *seq;
3412 233993 : scalar_int_mode mode;
3413 233993 : enum rtx_code code;
3414 233993 : int bitnum;
3415 :
3416 233993 : x = if_info->x;
3417 233993 : cond = if_info->cond;
3418 233993 : code = GET_CODE (cond);
3419 :
3420 : /* Check for an integer operation. */
3421 233993 : if (!is_a <scalar_int_mode> (GET_MODE (x), &mode))
3422 : return false;
3423 :
3424 447639 : if (!noce_simple_bbs (if_info))
3425 : return false;
3426 :
3427 : /* Check for no else condition. */
3428 157554 : if (! rtx_equal_p (x, if_info->b))
3429 : return false;
3430 :
3431 : /* Check for a suitable condition. */
3432 83696 : if (code != NE && code != EQ)
3433 : return false;
3434 59518 : if (XEXP (cond, 1) != const0_rtx)
3435 : return false;
3436 40823 : cond = XEXP (cond, 0);
3437 :
3438 : /* ??? We could also handle AND here. */
3439 40823 : if (GET_CODE (cond) == ZERO_EXTRACT)
3440 : {
3441 285 : if (XEXP (cond, 1) != const1_rtx
3442 169 : || !CONST_INT_P (XEXP (cond, 2))
3443 454 : || ! rtx_equal_p (x, XEXP (cond, 0)))
3444 280 : return false;
3445 5 : bitnum = INTVAL (XEXP (cond, 2));
3446 5 : if (BITS_BIG_ENDIAN)
3447 : bitnum = GET_MODE_BITSIZE (mode) - 1 - bitnum;
3448 5 : if (bitnum < 0 || bitnum >= HOST_BITS_PER_WIDE_INT)
3449 : return false;
3450 : }
3451 : else
3452 : return false;
3453 :
3454 5 : a = if_info->a;
3455 5 : if (GET_CODE (a) == IOR || GET_CODE (a) == XOR)
3456 : {
3457 : /* Check for "if (X & C) x = x op C". */
3458 0 : if (! rtx_equal_p (x, XEXP (a, 0))
3459 0 : || !CONST_INT_P (XEXP (a, 1))
3460 0 : || (INTVAL (XEXP (a, 1)) & GET_MODE_MASK (mode))
3461 0 : != HOST_WIDE_INT_1U << bitnum)
3462 : return false;
3463 :
3464 : /* if ((x & C) == 0) x |= C; is transformed to x |= C. */
3465 : /* if ((x & C) != 0) x |= C; is transformed to nothing. */
3466 0 : if (GET_CODE (a) == IOR)
3467 0 : result = (code == NE) ? a : NULL_RTX;
3468 0 : else if (code == NE)
3469 : {
3470 : /* if ((x & C) == 0) x ^= C; is transformed to x |= C. */
3471 0 : result = gen_int_mode (HOST_WIDE_INT_1 << bitnum, mode);
3472 0 : result = simplify_gen_binary (IOR, mode, x, result);
3473 : }
3474 : else
3475 : {
3476 : /* if ((x & C) != 0) x ^= C; is transformed to x &= ~C. */
3477 0 : result = gen_int_mode (~(HOST_WIDE_INT_1 << bitnum), mode);
3478 0 : result = simplify_gen_binary (AND, mode, x, result);
3479 : }
3480 : }
3481 5 : else if (GET_CODE (a) == AND)
3482 : {
3483 : /* Check for "if (X & C) x &= ~C". */
3484 0 : if (! rtx_equal_p (x, XEXP (a, 0))
3485 0 : || !CONST_INT_P (XEXP (a, 1))
3486 0 : || (INTVAL (XEXP (a, 1)) & GET_MODE_MASK (mode))
3487 0 : != (~(HOST_WIDE_INT_1 << bitnum) & GET_MODE_MASK (mode)))
3488 : return false;
3489 :
3490 : /* if ((x & C) == 0) x &= ~C; is transformed to nothing. */
3491 : /* if ((x & C) != 0) x &= ~C; is transformed to x &= ~C. */
3492 0 : result = (code == EQ) ? a : NULL_RTX;
3493 : }
3494 : else
3495 : return false;
3496 :
3497 0 : if (result)
3498 : {
3499 0 : start_sequence ();
3500 0 : noce_emit_move_insn (x, result);
3501 0 : seq = end_ifcvt_sequence (if_info);
3502 0 : if (!seq)
3503 : return false;
3504 :
3505 0 : emit_insn_before_setloc (seq, if_info->jump,
3506 0 : INSN_LOCATION (if_info->insn_a));
3507 : }
3508 0 : if_info->transform_name = "noce_try_bitop";
3509 0 : return true;
3510 : }
3511 :
3512 :
3513 : /* Similar to get_condition, only the resulting condition must be
3514 : valid at JUMP, instead of at EARLIEST.
3515 :
3516 : If THEN_ELSE_REVERSED is true, the fallthrough does not go to the
3517 : THEN block of the caller, and we have to reverse the condition. */
3518 :
3519 : static rtx
3520 4166273 : noce_get_condition (rtx_insn *jump, rtx_insn **earliest,
3521 : bool then_else_reversed)
3522 : {
3523 4166273 : rtx cond, set, tmp;
3524 4166273 : bool reverse;
3525 :
3526 4166273 : if (! any_condjump_p (jump))
3527 : return NULL_RTX;
3528 :
3529 4166273 : set = pc_set (jump);
3530 :
3531 : /* If this branches to JUMP_LABEL when the condition is false,
3532 : reverse the condition. */
3533 8332546 : reverse = (GET_CODE (XEXP (SET_SRC (set), 2)) == LABEL_REF
3534 4166273 : && label_ref_label (XEXP (SET_SRC (set), 2)) == JUMP_LABEL (jump));
3535 :
3536 : /* We may have to reverse because the caller's if block is not canonical,
3537 : i.e. the THEN block isn't the fallthrough block for the TEST block
3538 : (see find_if_header). */
3539 4166273 : if (then_else_reversed)
3540 1830241 : reverse = !reverse;
3541 :
3542 : /* If the condition variable is a register and is MODE_INT, accept it. */
3543 :
3544 4166273 : cond = XEXP (SET_SRC (set), 0);
3545 4166273 : tmp = XEXP (cond, 0);
3546 4165325 : if (REG_P (tmp) && GET_MODE_CLASS (GET_MODE (tmp)) == MODE_INT
3547 4166313 : && (GET_MODE (tmp) != BImode
3548 0 : || !targetm.small_register_classes_for_mode_p (BImode)))
3549 : {
3550 40 : *earliest = jump;
3551 :
3552 40 : if (reverse)
3553 17 : cond = gen_rtx_fmt_ee (reverse_condition (GET_CODE (cond)),
3554 : GET_MODE (cond), tmp, XEXP (cond, 1));
3555 40 : return cond;
3556 : }
3557 :
3558 : /* Otherwise, fall back on canonicalize_condition to do the dirty
3559 : work of manipulating MODE_CC values and COMPARE rtx codes. */
3560 4166233 : tmp = canonicalize_condition (jump, cond, reverse, earliest,
3561 : NULL_RTX, have_cbranchcc4, true);
3562 :
3563 : /* We don't handle side-effects in the condition, like handling
3564 : REG_INC notes and making sure no duplicate conditions are emitted. */
3565 4166233 : if (tmp != NULL_RTX && side_effects_p (tmp))
3566 : return NULL_RTX;
3567 :
3568 : return tmp;
3569 : }
3570 :
3571 : /* Return true if OP is ok for if-then-else processing. */
3572 :
3573 : static bool
3574 8795507 : noce_operand_ok (const_rtx op)
3575 : {
3576 8795507 : if (side_effects_p (op))
3577 : return false;
3578 :
3579 : /* We special-case memories, so handle any of them with
3580 : no address side effects. */
3581 8774957 : if (MEM_P (op))
3582 1547903 : return ! side_effects_p (XEXP (op, 0));
3583 :
3584 7227054 : return ! may_trap_p (op);
3585 : }
3586 :
3587 : /* Return true iff basic block TEST_BB is valid for noce if-conversion.
3588 : The condition used in this if-conversion is in COND.
3589 : In practice, check that TEST_BB ends with a single set
3590 : x := a and all previous computations
3591 : in TEST_BB don't produce any values that are live after TEST_BB.
3592 : In other words, all the insns in TEST_BB are there only
3593 : to compute a value for x. Add the rtx cost of the insns
3594 : in TEST_BB to COST. Record whether TEST_BB is a single simple
3595 : set instruction in SIMPLE_P. */
3596 :
3597 : static bool
3598 2086260 : bb_valid_for_noce_process_p (basic_block test_bb, rtx cond,
3599 : unsigned int *cost, bool *simple_p)
3600 : {
3601 2086260 : if (!test_bb)
3602 : return false;
3603 :
3604 2086260 : rtx_insn *last_insn = last_active_insn (test_bb, false);
3605 2086260 : rtx last_set = NULL_RTX;
3606 :
3607 2086260 : rtx cc = cc_in_cond (cond);
3608 :
3609 2086260 : if (!insn_valid_noce_process_p (last_insn, cc))
3610 : return false;
3611 :
3612 : /* Punt on blocks ending with asm goto or jumps with other side-effects,
3613 : last_active_insn ignores JUMP_INSNs. */
3614 1502825 : if (JUMP_P (BB_END (test_bb)) && !onlyjump_p (BB_END (test_bb)))
3615 : return false;
3616 :
3617 1502823 : last_set = single_set (last_insn);
3618 :
3619 1502823 : rtx x = SET_DEST (last_set);
3620 1502823 : rtx_insn *first_insn = first_active_insn (test_bb);
3621 1502823 : rtx first_set = single_set (first_insn);
3622 :
3623 1502823 : if (!first_set)
3624 : return false;
3625 :
3626 : /* We have a single simple set, that's okay. */
3627 1488944 : bool speed_p = optimize_bb_for_speed_p (test_bb);
3628 :
3629 1488944 : if (first_insn == last_insn)
3630 : {
3631 563116 : *simple_p = noce_operand_ok (SET_DEST (first_set));
3632 563116 : *cost += pattern_cost (first_set, speed_p);
3633 563116 : return *simple_p;
3634 : }
3635 :
3636 925828 : rtx_insn *prev_last_insn = PREV_INSN (last_insn);
3637 925828 : gcc_assert (prev_last_insn);
3638 :
3639 : /* For now, disallow setting x multiple times in test_bb. */
3640 925828 : if (REG_P (x) && reg_set_between_p (x, first_insn, prev_last_insn))
3641 : return false;
3642 :
3643 779560 : bitmap test_bb_temps = BITMAP_ALLOC (®_obstack);
3644 :
3645 : /* The regs that are live out of test_bb. */
3646 779560 : bitmap test_bb_live_out = df_get_live_out (test_bb);
3647 :
3648 779560 : int potential_cost = pattern_cost (last_set, speed_p);
3649 779560 : rtx_insn *insn;
3650 4062078 : FOR_BB_INSNS (test_bb, insn)
3651 : {
3652 3965896 : if (insn != last_insn)
3653 : {
3654 3869714 : if (!active_insn_p (insn))
3655 2469894 : continue;
3656 :
3657 1399820 : if (!insn_valid_noce_process_p (insn, cc))
3658 198806 : goto free_bitmap_and_fail;
3659 :
3660 1201014 : rtx sset = single_set (insn);
3661 1201014 : gcc_assert (sset);
3662 1201014 : rtx dest = SET_DEST (sset);
3663 1201014 : if (SUBREG_P (dest))
3664 18318 : dest = SUBREG_REG (dest);
3665 :
3666 1201014 : if (contains_mem_rtx_p (SET_SRC (sset))
3667 853066 : || !REG_P (dest)
3668 1929466 : || reg_overlap_mentioned_p (dest, cond))
3669 484572 : goto free_bitmap_and_fail;
3670 :
3671 716442 : potential_cost += pattern_cost (sset, speed_p);
3672 716442 : bitmap_set_bit (test_bb_temps, REGNO (dest));
3673 : }
3674 : }
3675 :
3676 : /* If any of the intermediate results in test_bb are live after test_bb
3677 : then fail. */
3678 96182 : if (bitmap_intersect_p (test_bb_live_out, test_bb_temps))
3679 66649 : goto free_bitmap_and_fail;
3680 :
3681 29533 : BITMAP_FREE (test_bb_temps);
3682 29533 : *cost += potential_cost;
3683 29533 : *simple_p = false;
3684 29533 : return true;
3685 :
3686 750027 : free_bitmap_and_fail:
3687 750027 : BITMAP_FREE (test_bb_temps);
3688 750027 : return false;
3689 : }
3690 :
3691 : /* Helper function to emit a cmov sequence encapsulated in
3692 : start_sequence () and end_sequence (). If NEED_CMOV is true
3693 : we call noce_emit_cmove to create a cmove sequence. Otherwise emit
3694 : a simple move. If successful, store the first instruction of the
3695 : sequence in TEMP_DEST and the sequence costs in SEQ_COST. */
3696 :
3697 : static rtx_insn*
3698 275594 : try_emit_cmove_seq (struct noce_if_info *if_info, rtx temp,
3699 : rtx cond, rtx new_val, rtx old_val, bool need_cmov,
3700 : unsigned *cost, rtx *temp_dest,
3701 : rtx cc_cmp = NULL, rtx rev_cc_cmp = NULL)
3702 : {
3703 275594 : rtx_insn *seq = NULL;
3704 275594 : *cost = 0;
3705 :
3706 275594 : rtx x = XEXP (cond, 0);
3707 275594 : rtx y = XEXP (cond, 1);
3708 275594 : rtx_code cond_code = GET_CODE (cond);
3709 :
3710 275594 : start_sequence ();
3711 :
3712 275594 : if (need_cmov)
3713 221943 : *temp_dest = noce_emit_cmove (if_info, temp, cond_code,
3714 : x, y, new_val, old_val, cc_cmp, rev_cc_cmp);
3715 : else
3716 : {
3717 53651 : *temp_dest = temp;
3718 53651 : if (if_info->then_else_reversed)
3719 1816 : noce_emit_move_insn (temp, old_val);
3720 : else
3721 51835 : noce_emit_move_insn (temp, new_val);
3722 : }
3723 :
3724 275594 : if (*temp_dest != NULL_RTX)
3725 : {
3726 269519 : seq = get_insns ();
3727 269519 : *cost = seq_cost (seq, if_info->speed_p);
3728 : }
3729 :
3730 275594 : end_sequence ();
3731 :
3732 275594 : return seq;
3733 : }
3734 :
3735 : /* We have something like:
3736 :
3737 : if (x > y)
3738 : { i = EXPR_A; j = EXPR_B; k = EXPR_C; }
3739 :
3740 : Make it:
3741 :
3742 : tmp_i = (x > y) ? EXPR_A : i;
3743 : tmp_j = (x > y) ? EXPR_B : j;
3744 : tmp_k = (x > y) ? EXPR_C : k;
3745 : i = tmp_i;
3746 : j = tmp_j;
3747 : k = tmp_k;
3748 :
3749 : Subsequent passes are expected to clean up the extra moves.
3750 :
3751 : Look for special cases such as writes to one register which are
3752 : read back in another SET, as might occur in a swap idiom or
3753 : similar.
3754 :
3755 : These look like:
3756 :
3757 : if (x > y)
3758 : i = a;
3759 : j = i;
3760 :
3761 : Which we want to rewrite to:
3762 :
3763 : tmp_i = (x > y) ? a : i;
3764 : tmp_j = (x > y) ? tmp_i : j;
3765 : i = tmp_i;
3766 : j = tmp_j;
3767 :
3768 : We can catch these when looking at (SET x y) by keeping a list of the
3769 : registers we would have targeted before if-conversion and looking back
3770 : through it for an overlap with Y. If we find one, we rewire the
3771 : conditional set to use the temporary we introduced earlier.
3772 :
3773 : IF_INFO contains the useful information about the block structure and
3774 : jump instructions. */
3775 :
3776 : static bool
3777 31346 : noce_convert_multiple_sets (struct noce_if_info *if_info)
3778 : {
3779 31346 : basic_block test_bb = if_info->test_bb;
3780 31346 : basic_block then_bb = if_info->then_bb;
3781 31346 : basic_block join_bb = if_info->join_bb;
3782 31346 : rtx_insn *jump = if_info->jump;
3783 31346 : rtx_insn *cond_earliest;
3784 31346 : rtx_insn *insn;
3785 :
3786 31346 : start_sequence ();
3787 :
3788 : /* Decompose the condition attached to the jump. */
3789 31346 : rtx cond = noce_get_condition (jump, &cond_earliest, false);
3790 31346 : rtx x = XEXP (cond, 0);
3791 31346 : rtx y = XEXP (cond, 1);
3792 :
3793 31346 : auto_delete_vec<noce_multiple_sets_info> insn_info;
3794 31346 : init_noce_multiple_sets_info (then_bb, insn_info);
3795 :
3796 31346 : int last_needs_comparison = -1;
3797 :
3798 31346 : bool use_cond_earliest = false;
3799 :
3800 31346 : bool ok = noce_convert_multiple_sets_1
3801 31346 : (if_info, insn_info, &last_needs_comparison, &use_cond_earliest);
3802 31346 : if (!ok)
3803 : return false;
3804 :
3805 : /* Always perform a second attempt that uses information gathered in the
3806 : first. At least we can omit creating temporaries until we definitely
3807 : need them. The sequence created in the second attempt is never worse
3808 : than the first. */
3809 :
3810 30253 : end_sequence ();
3811 30253 : start_sequence ();
3812 30253 : ok = noce_convert_multiple_sets_1
3813 30253 : (if_info, insn_info, &last_needs_comparison, &use_cond_earliest);
3814 :
3815 : /* Actually we should not fail anymore if we reached here,
3816 : but better still check. */
3817 30253 : if (!ok)
3818 : return false;
3819 :
3820 : /* We must have seen some sort of insn to insert, otherwise we were
3821 : given an empty BB to convert, and we can't handle that. */
3822 30253 : gcc_assert (!insn_info.is_empty ());
3823 :
3824 : /* Now fixup the assignments.
3825 : PR116405: Iterate in reverse order and keep track of the targets so that
3826 : a move does not overwrite a subsequent value when multiple instructions
3827 : have the same target. */
3828 30253 : unsigned i;
3829 30253 : noce_multiple_sets_info *info;
3830 30253 : bitmap set_targets = BITMAP_ALLOC (®_obstack);
3831 111593 : FOR_EACH_VEC_ELT_REVERSE (insn_info, i, info)
3832 : {
3833 81340 : gcc_checking_assert (REG_P (info->target));
3834 :
3835 81340 : if (info->target != info->temporary
3836 81340 : && !bitmap_bit_p (set_targets, REGNO (info->target)))
3837 3252 : noce_emit_move_insn (info->target, info->temporary);
3838 :
3839 81340 : bitmap_set_bit (set_targets, REGNO (info->target));
3840 : }
3841 30253 : BITMAP_FREE (set_targets);
3842 :
3843 : /* Actually emit the sequence if it isn't too expensive. */
3844 30253 : rtx_insn *seq = get_insns ();
3845 :
3846 : /* If the created sequence does not use cond_earliest (but the jump
3847 : does) add its cost to the original_cost before comparing costs. */
3848 30253 : unsigned int original_cost = if_info->original_cost;
3849 30253 : if (if_info->jump != if_info->cond_earliest && !use_cond_earliest)
3850 18961 : if_info->original_cost += insn_cost (if_info->cond_earliest,
3851 18961 : if_info->speed_p);
3852 :
3853 30253 : if (!targetm.noce_conversion_profitable_p (seq, if_info))
3854 : {
3855 4856 : end_sequence ();
3856 4856 : return false;
3857 : }
3858 :
3859 : /* Restore the original cost in case we do not succeed below. */
3860 25397 : if_info->original_cost = original_cost;
3861 :
3862 146983 : for (insn = seq; insn; insn = NEXT_INSN (insn))
3863 121586 : set_used_flags (insn);
3864 :
3865 : /* Mark all our temporaries and targets as used. */
3866 89810 : for (unsigned i = 0; i < insn_info.length (); i++)
3867 : {
3868 64413 : set_used_flags (insn_info[i]->temporary);
3869 64413 : set_used_flags (insn_info[i]->target);
3870 : }
3871 :
3872 25397 : set_used_flags (cond);
3873 25397 : set_used_flags (x);
3874 25397 : set_used_flags (y);
3875 :
3876 25397 : unshare_all_rtl_in_chain (seq);
3877 25397 : end_sequence ();
3878 :
3879 25397 : if (!seq)
3880 : return false;
3881 :
3882 146800 : for (insn = seq; insn; insn = NEXT_INSN (insn))
3883 121449 : if (JUMP_P (insn) || CALL_P (insn)
3884 121449 : || recog_memoized (insn) == -1)
3885 46 : return false;
3886 :
3887 25351 : emit_insn_before_setloc (seq, if_info->jump,
3888 25351 : INSN_LOCATION (insn_info.last ()->unmodified_insn));
3889 :
3890 : /* Clean up THEN_BB and the edges in and out of it. */
3891 25351 : remove_edge (find_edge (test_bb, join_bb));
3892 25351 : remove_edge (find_edge (then_bb, join_bb));
3893 25351 : redirect_edge_and_branch_force (single_succ_edge (test_bb), join_bb);
3894 25351 : delete_basic_block (then_bb);
3895 25351 : num_true_changes++;
3896 :
3897 : /* Maybe merge blocks now the jump is simple enough. */
3898 25351 : if (can_merge_blocks_p (test_bb, join_bb))
3899 : {
3900 16934 : merge_blocks (test_bb, join_bb);
3901 16934 : num_true_changes++;
3902 : }
3903 :
3904 25351 : num_updated_if_blocks++;
3905 25351 : if_info->transform_name = "noce_convert_multiple_sets";
3906 25351 : return true;
3907 31346 : }
3908 :
3909 : /* This goes through all relevant insns of IF_INFO->then_bb and tries to create
3910 : conditional moves. Information for the insns is kept in INSN_INFO. */
3911 :
3912 : static bool
3913 61599 : noce_convert_multiple_sets_1 (struct noce_if_info *if_info,
3914 : auto_delete_vec<noce_multiple_sets_info> &insn_info,
3915 : int *last_needs_comparison,
3916 : bool *use_cond_earliest)
3917 : {
3918 61599 : basic_block then_bb = if_info->then_bb;
3919 61599 : rtx_insn *jump = if_info->jump;
3920 61599 : rtx_insn *cond_earliest;
3921 :
3922 : /* Decompose the condition attached to the jump. */
3923 61599 : rtx cond = noce_get_condition (jump, &cond_earliest, false);
3924 :
3925 61599 : rtx cc_cmp = cond_exec_get_condition (jump);
3926 61599 : if (cc_cmp)
3927 61599 : cc_cmp = copy_rtx (cc_cmp);
3928 61599 : rtx rev_cc_cmp = cond_exec_get_condition (jump, /* get_reversed */ true);
3929 61599 : if (rev_cc_cmp)
3930 61599 : rev_cc_cmp = copy_rtx (rev_cc_cmp);
3931 :
3932 61599 : rtx_insn *insn;
3933 61599 : int count = 0;
3934 61599 : bool second_try = *last_needs_comparison != -1;
3935 61599 : *use_cond_earliest = false;
3936 :
3937 365127 : FOR_BB_INSNS (then_bb, insn)
3938 : {
3939 : /* Skip over non-insns. */
3940 304621 : if (!active_insn_p (insn))
3941 139355 : continue;
3942 :
3943 165266 : noce_multiple_sets_info *info = insn_info[count];
3944 :
3945 165266 : rtx set = single_set (insn);
3946 165266 : gcc_checking_assert (set);
3947 :
3948 165266 : rtx target = SET_DEST (set);
3949 165266 : rtx temp;
3950 :
3951 165266 : rtx new_val = SET_SRC (set);
3952 :
3953 165266 : int i, ii;
3954 204462 : FOR_EACH_VEC_ELT (info->rewired_src, i, ii)
3955 39196 : new_val = simplify_replace_rtx (new_val, insn_info[ii]->target,
3956 39196 : insn_info[ii]->temporary);
3957 :
3958 165266 : rtx old_val = target;
3959 :
3960 : /* As we are transforming
3961 : if (x > y)
3962 : {
3963 : a = b;
3964 : c = d;
3965 : }
3966 : into
3967 : a = (x > y) ...
3968 : c = (x > y) ...
3969 :
3970 : we potentially check x > y before every set.
3971 : Even though the check might be removed by subsequent passes, this means
3972 : that we cannot transform
3973 : if (x > y)
3974 : {
3975 : x = y;
3976 : ...
3977 : }
3978 : into
3979 : x = (x > y) ...
3980 : ...
3981 : since this would invalidate x and the following to-be-removed checks.
3982 : Therefore we introduce a temporary every time we are about to
3983 : overwrite a variable used in the check. Costing of a sequence with
3984 : these is going to be inaccurate so only use temporaries when
3985 : needed.
3986 :
3987 : If performing a second try, we know how many insns require a
3988 : temporary. For the last of these, we can omit creating one. */
3989 165266 : if (reg_overlap_mentioned_p (target, cond)
3990 165266 : && (!second_try || count < *last_needs_comparison))
3991 21280 : temp = gen_reg_rtx (GET_MODE (target));
3992 : else
3993 : temp = target;
3994 :
3995 : /* We have identified swap-style idioms before. A normal
3996 : set will need to be a cmov while the first instruction of a swap-style
3997 : idiom can be a regular move. This helps with costing. */
3998 165266 : bool need_cmov = info->need_cmov;
3999 :
4000 : /* If we had a non-canonical conditional jump (i.e. one where
4001 : the fallthrough is to the "else" case) we need to reverse
4002 : the conditional select. */
4003 165266 : if (if_info->then_else_reversed)
4004 51124 : std::swap (old_val, new_val);
4005 :
4006 : /* Try emitting a conditional move passing the backend the
4007 : canonicalized comparison. The backend is then able to
4008 : recognize expressions like
4009 :
4010 : if (x > y)
4011 : y = x;
4012 :
4013 : as min/max and emit an insn, accordingly. */
4014 165266 : unsigned cost1 = 0, cost2 = 0;
4015 165266 : rtx_insn *seq, *seq1, *seq2 = NULL;
4016 165266 : rtx temp_dest = NULL_RTX, temp_dest1 = NULL_RTX, temp_dest2 = NULL_RTX;
4017 165266 : bool read_comparison = false;
4018 :
4019 165266 : seq1 = try_emit_cmove_seq (if_info, temp, cond,
4020 : new_val, old_val, need_cmov,
4021 : &cost1, &temp_dest1);
4022 :
4023 : /* Here, we try to pass the backend a non-canonicalized cc comparison
4024 : as well. This allows the backend to emit a cmov directly without
4025 : creating an additional compare for each. If successful, costing
4026 : is easier and this sequence is usually preferred. */
4027 165266 : if (cc_cmp)
4028 : {
4029 110328 : seq2 = try_emit_cmove_seq (if_info, temp, cond,
4030 : new_val, old_val, need_cmov,
4031 : &cost2, &temp_dest2, cc_cmp, rev_cc_cmp);
4032 :
4033 : /* The if_then_else in SEQ2 may be affected when cc_cmp/rev_cc_cmp is
4034 : clobbered. We can't safely use the sequence in this case. */
4035 193718 : for (rtx_insn *iter = seq2; iter; iter = NEXT_INSN (iter))
4036 123216 : if (modified_in_p (cc_cmp, iter)
4037 123216 : || (rev_cc_cmp && modified_in_p (rev_cc_cmp, iter)))
4038 : {
4039 : seq2 = NULL;
4040 : break;
4041 : }
4042 : }
4043 :
4044 : /* The backend might have created a sequence that uses the
4045 : condition as a value. Check this. */
4046 :
4047 : /* We cannot handle anything more complex than a reg or constant. */
4048 165266 : if (!REG_P (XEXP (cond, 0)) && !CONSTANT_P (XEXP (cond, 0)))
4049 165266 : read_comparison = true;
4050 :
4051 165266 : if (!REG_P (XEXP (cond, 1)) && !CONSTANT_P (XEXP (cond, 1)))
4052 165266 : read_comparison = true;
4053 :
4054 : rtx_insn *walk = seq2;
4055 : int if_then_else_count = 0;
4056 245131 : while (walk && !read_comparison)
4057 : {
4058 79865 : rtx exprs_to_check[2];
4059 79865 : unsigned int exprs_count = 0;
4060 :
4061 79865 : rtx set = single_set (walk);
4062 79865 : if (set && XEXP (set, 1)
4063 79865 : && GET_CODE (XEXP (set, 1)) == IF_THEN_ELSE)
4064 : {
4065 : /* We assume that this is the cmove created by the backend that
4066 : naturally uses the condition. */
4067 50446 : exprs_to_check[exprs_count++] = XEXP (XEXP (set, 1), 1);
4068 50446 : exprs_to_check[exprs_count++] = XEXP (XEXP (set, 1), 2);
4069 50446 : if_then_else_count++;
4070 : }
4071 29419 : else if (NONDEBUG_INSN_P (walk))
4072 29419 : exprs_to_check[exprs_count++] = PATTERN (walk);
4073 :
4074 : /* Bail if we get more than one if_then_else because the assumption
4075 : above may be incorrect. */
4076 79865 : if (if_then_else_count > 1)
4077 : {
4078 0 : read_comparison = true;
4079 0 : break;
4080 : }
4081 :
4082 210176 : for (unsigned int i = 0; i < exprs_count; i++)
4083 : {
4084 130311 : subrtx_iterator::array_type array;
4085 291279 : FOR_EACH_SUBRTX (iter, array, exprs_to_check[i], NONCONST)
4086 191344 : if (*iter != NULL_RTX
4087 191344 : && (reg_overlap_mentioned_p (XEXP (cond, 0), *iter)
4088 173656 : || reg_overlap_mentioned_p (XEXP (cond, 1), *iter)))
4089 : {
4090 : read_comparison = true;
4091 : break;
4092 : }
4093 130311 : }
4094 :
4095 79865 : walk = NEXT_INSN (walk);
4096 : }
4097 :
4098 : /* Check which version is less expensive. */
4099 165266 : if (seq1 != NULL_RTX && (cost1 <= cost2 || seq2 == NULL_RTX))
4100 : {
4101 112711 : seq = seq1;
4102 112711 : temp_dest = temp_dest1;
4103 112711 : if (!second_try)
4104 57165 : *last_needs_comparison = count;
4105 : }
4106 1100 : else if (seq2 != NULL_RTX)
4107 : {
4108 51619 : seq = seq2;
4109 51619 : temp_dest = temp_dest2;
4110 51619 : if (!second_try && read_comparison)
4111 8448 : *last_needs_comparison = count;
4112 51619 : *use_cond_earliest = true;
4113 : }
4114 : else
4115 : {
4116 : /* Nothing worked, bail out. */
4117 936 : end_sequence ();
4118 1093 : return false;
4119 : }
4120 :
4121 : /* Although we use temporaries if there is register overlap of COND and
4122 : TARGET, it is possible that SEQ modifies COND anyway. For example,
4123 : COND may use the flags register and if INSN clobbers flags then
4124 : we may be unable to emit a valid sequence (e.g. in x86 that would
4125 : require saving and restoring the flags register). */
4126 108784 : if (!second_try)
4127 246812 : for (rtx_insn *iter = seq; iter; iter = NEXT_INSN (iter))
4128 163979 : if (modified_in_p (cond, iter))
4129 : {
4130 157 : end_sequence ();
4131 157 : return false;
4132 : }
4133 :
4134 164173 : if (cc_cmp && seq == seq1)
4135 : {
4136 : /* Check if SEQ can clobber registers mentioned in cc_cmp/rev_cc_cmp.
4137 : If yes, we need to use only SEQ1 from that point on.
4138 : Only check when we use SEQ1 since we have already tested SEQ2. */
4139 79709 : for (rtx_insn *iter = seq; iter; iter = NEXT_INSN (iter))
4140 60999 : if (modified_in_p (cc_cmp, iter)
4141 60999 : || (rev_cc_cmp && modified_in_p (rev_cc_cmp, iter)))
4142 : {
4143 : cc_cmp = NULL_RTX;
4144 : rev_cc_cmp = NULL_RTX;
4145 : break;
4146 : }
4147 : }
4148 :
4149 : /* End the sub sequence and emit to the main sequence. */
4150 164173 : emit_insn (seq);
4151 :
4152 : /* Bookkeeping. */
4153 164173 : count++;
4154 :
4155 164173 : info->target = target;
4156 164173 : info->temporary = temp_dest;
4157 164173 : info->unmodified_insn = insn;
4158 : }
4159 :
4160 : /* Even if we did not actually need the comparison, we want to make sure
4161 : to try a second time in order to get rid of the temporaries. */
4162 60506 : if (*last_needs_comparison == -1)
4163 1074 : *last_needs_comparison = 0;
4164 :
4165 : return true;
4166 : }
4167 :
4168 : /* Find local swap-style idioms in BB and mark the first insn (1)
4169 : that is only a temporary as not needing a conditional move as
4170 : it is going to be dead afterwards anyway.
4171 :
4172 : (1) int tmp = a;
4173 : a = b;
4174 : b = tmp;
4175 :
4176 : ifcvt
4177 : -->
4178 :
4179 : tmp = a;
4180 : a = cond ? b : a_old;
4181 : b = cond ? tmp : b_old;
4182 :
4183 : Additionally, store the index of insns like (2) when a subsequent
4184 : SET reads from their destination.
4185 :
4186 : (2) int c = a;
4187 : int d = c;
4188 :
4189 : ifcvt
4190 : -->
4191 :
4192 : c = cond ? a : c_old;
4193 : d = cond ? d : c; // Need to use c rather than c_old here.
4194 : */
4195 :
4196 : static void
4197 31346 : init_noce_multiple_sets_info (basic_block bb,
4198 : auto_delete_vec<noce_multiple_sets_info> &insn_info)
4199 : {
4200 31346 : rtx_insn *insn;
4201 31346 : int count = 0;
4202 31346 : auto_vec<rtx> dests;
4203 31346 : bitmap bb_live_out = df_get_live_out (bb);
4204 :
4205 : /* Iterate over all SETs, storing the destinations in DEST.
4206 : - If we encounter a previously changed register,
4207 : rewire the read to the original source.
4208 : - If we encounter a SET that writes to a destination
4209 : that is not live after this block then the register
4210 : does not need to be moved conditionally. */
4211 192241 : FOR_BB_INSNS (bb, insn)
4212 : {
4213 160895 : if (!active_insn_p (insn))
4214 76455 : continue;
4215 :
4216 84440 : noce_multiple_sets_info *info = new noce_multiple_sets_info;
4217 84440 : info->target = NULL_RTX;
4218 84440 : info->temporary = NULL_RTX;
4219 84440 : info->unmodified_insn = NULL;
4220 84440 : insn_info.safe_push (info);
4221 :
4222 84440 : rtx set = single_set (insn);
4223 84440 : gcc_checking_assert (set);
4224 :
4225 84440 : rtx src = SET_SRC (set);
4226 84440 : rtx dest = SET_DEST (set);
4227 :
4228 84440 : gcc_checking_assert (REG_P (dest));
4229 84440 : info->need_cmov = bitmap_bit_p (bb_live_out, REGNO (dest));
4230 :
4231 : /* Check if the current SET's source is the same
4232 : as any previously seen destination.
4233 : This is quadratic but the number of insns in BB
4234 : is bounded by PARAM_MAX_RTL_IF_CONVERSION_INSNS. */
4235 178159 : for (int i = count - 1; i >= 0; --i)
4236 93719 : if (reg_mentioned_p (dests[i], src))
4237 20401 : insn_info[count]->rewired_src.safe_push (i);
4238 :
4239 84440 : dests.safe_push (dest);
4240 84440 : count++;
4241 : }
4242 31346 : }
4243 :
4244 : /* Return true iff basic block TEST_BB is suitable for conversion to a
4245 : series of conditional moves. Also check that we have more than one
4246 : set (other routines can handle a single set better than we would),
4247 : and fewer than PARAM_MAX_RTL_IF_CONVERSION_INSNS sets. While going
4248 : through the insns store the sum of their potential costs in COST. */
4249 :
4250 : static bool
4251 1050328 : bb_ok_for_noce_convert_multiple_sets (basic_block test_bb, unsigned *cost)
4252 : {
4253 1050328 : rtx_insn *insn;
4254 1050328 : unsigned count = 0;
4255 1050328 : unsigned param = param_max_rtl_if_conversion_insns;
4256 1050328 : bool speed_p = optimize_bb_for_speed_p (test_bb);
4257 1050328 : unsigned potential_cost = 0;
4258 :
4259 8261099 : FOR_BB_INSNS (test_bb, insn)
4260 : {
4261 : /* Skip over notes etc. */
4262 8144083 : if (!active_insn_p (insn))
4263 6264596 : continue;
4264 :
4265 : /* We only handle SET insns. */
4266 1879487 : rtx set = single_set (insn);
4267 1879487 : if (set == NULL_RTX)
4268 : return false;
4269 :
4270 1668991 : rtx dest = SET_DEST (set);
4271 1668991 : rtx src = SET_SRC (set);
4272 :
4273 : /* Do not handle anything involving memory loads/stores since it might
4274 : violate data-race-freedom guarantees. Make sure we can force SRC
4275 : to a register as that may be needed in try_emit_cmove_seq. */
4276 1466508 : if (!REG_P (dest) || contains_mem_rtx_p (src)
4277 2672869 : || !noce_can_force_operand (src))
4278 688846 : return false;
4279 :
4280 : /* Destination and source must be appropriate. */
4281 980145 : if (!noce_operand_ok (dest) || !noce_operand_ok (src))
4282 5945 : return false;
4283 :
4284 : /* We must be able to conditionally move in this mode. */
4285 974200 : if (!can_conditionally_move_p (GET_MODE (dest)))
4286 : return false;
4287 :
4288 946175 : potential_cost += insn_cost (insn, speed_p);
4289 :
4290 946175 : count++;
4291 : }
4292 :
4293 117016 : *cost += potential_cost;
4294 :
4295 : /* If we would only put out one conditional move, the other strategies
4296 : this pass tries are better optimized and will be more appropriate.
4297 : Some targets want to strictly limit the number of conditional moves
4298 : that are emitted, they set this through PARAM, we need to respect
4299 : that. */
4300 117016 : return count > 1 && count <= param;
4301 : }
4302 :
4303 : /* Compute average of two given costs weighted by relative probabilities
4304 : of respective basic blocks in an IF-THEN-ELSE. E is the IF-THEN edge.
4305 : With P as the probability to take the IF-THEN branch, return
4306 : P * THEN_COST + (1 - P) * ELSE_COST. */
4307 : static unsigned
4308 265157 : average_cost (unsigned then_cost, unsigned else_cost, edge e)
4309 : {
4310 265157 : return else_cost + e->probability.apply ((signed) (then_cost - else_cost));
4311 : }
4312 :
4313 : /* Given a simple IF-THEN-JOIN or IF-THEN-ELSE-JOIN block, attempt to convert
4314 : it without using conditional execution. Return TRUE if we were successful
4315 : at converting the block. */
4316 :
4317 : static bool
4318 1826598 : noce_process_if_block (struct noce_if_info *if_info)
4319 : {
4320 1826598 : basic_block test_bb = if_info->test_bb; /* test block */
4321 1826598 : basic_block then_bb = if_info->then_bb; /* THEN */
4322 1826598 : basic_block else_bb = if_info->else_bb; /* ELSE or NULL */
4323 1826598 : basic_block join_bb = if_info->join_bb; /* JOIN */
4324 1826598 : rtx_insn *jump = if_info->jump;
4325 1826598 : rtx cond = if_info->cond;
4326 1826598 : rtx_insn *insn_a, *insn_b;
4327 1826598 : rtx set_a, set_b;
4328 1826598 : rtx orig_x, x, a, b;
4329 :
4330 : /* We're looking for patterns of the form
4331 :
4332 : (1) if (...) x = a; else x = b;
4333 : (2) x = b; if (...) x = a;
4334 : (3) if (...) x = a; // as if with an initial x = x.
4335 : (4) if (...) { x = a; y = b; z = c; } // Like 3, for multiple SETS.
4336 : The later patterns require jumps to be more expensive.
4337 : For the if (...) x = a; else x = b; case we allow multiple insns
4338 : inside the then and else blocks as long as their only effect is
4339 : to calculate a value for x.
4340 : ??? For future expansion, further expand the "multiple X" rules. */
4341 :
4342 : /* First look for multiple SETS.
4343 : The original costs already include costs for the jump insn as well
4344 : as for a CC comparison if there is any.
4345 : If a target re-uses the existing CC comparison we keep track of that
4346 : and add the costs before default noce_conversion_profitable_p. */
4347 :
4348 1826598 : unsigned potential_cost = if_info->original_cost;
4349 1826598 : unsigned old_cost = if_info->original_cost;
4350 1826598 : if (!else_bb
4351 : && HAVE_conditional_move
4352 1826598 : && bb_ok_for_noce_convert_multiple_sets (then_bb, &potential_cost))
4353 : {
4354 : /* Temporarily set the original costs to what we estimated so
4355 : we can determine if the transformation is worth it. */
4356 31346 : if_info->original_cost = potential_cost;
4357 31346 : if (noce_convert_multiple_sets (if_info))
4358 : {
4359 25351 : if (dump_file && if_info->transform_name)
4360 1 : fprintf (dump_file, "if-conversion succeeded through %s\n",
4361 : if_info->transform_name);
4362 25351 : return true;
4363 : }
4364 :
4365 : /* Restore the original costs. */
4366 5995 : if_info->original_cost = old_cost;
4367 : }
4368 :
4369 1801247 : bool speed_p = optimize_bb_for_speed_p (test_bb);
4370 1801247 : unsigned int then_cost = 0, else_cost = 0;
4371 1801247 : if (!bb_valid_for_noce_process_p (then_bb, cond, &then_cost,
4372 : &if_info->then_simple))
4373 : return false;
4374 :
4375 489764 : if (else_bb
4376 489764 : && !bb_valid_for_noce_process_p (else_bb, cond, &else_cost,
4377 : &if_info->else_simple))
4378 : return false;
4379 :
4380 307636 : if (speed_p)
4381 265157 : if_info->original_cost += average_cost (then_cost, else_cost,
4382 : find_edge (test_bb, then_bb));
4383 : else
4384 42479 : if_info->original_cost += then_cost + else_cost;
4385 :
4386 307636 : insn_a = last_active_insn (then_bb, false);
4387 307636 : set_a = single_set (insn_a);
4388 307636 : gcc_assert (set_a);
4389 :
4390 307636 : x = SET_DEST (set_a);
4391 307636 : a = SET_SRC (set_a);
4392 :
4393 : /* Look for the other potential set. Make sure we've got equivalent
4394 : destinations. */
4395 : /* ??? This is overconservative. Storing to two different mems is
4396 : as easy as conditionally computing the address. Storing to a
4397 : single mem merely requires a scratch memory to use as one of the
4398 : destination addresses; often the memory immediately below the
4399 : stack pointer is available for this. */
4400 307636 : set_b = NULL_RTX;
4401 307636 : if (else_bb)
4402 : {
4403 102885 : insn_b = last_active_insn (else_bb, false);
4404 102885 : set_b = single_set (insn_b);
4405 102885 : gcc_assert (set_b);
4406 :
4407 102885 : if (!rtx_interchangeable_p (x, SET_DEST (set_b)))
4408 : return false;
4409 : }
4410 : else
4411 : {
4412 204751 : insn_b = if_info->cond_earliest;
4413 546962 : do
4414 546962 : insn_b = prev_nonnote_nondebug_insn (insn_b);
4415 : while (insn_b
4416 544241 : && (BLOCK_FOR_INSN (insn_b)
4417 544241 : == BLOCK_FOR_INSN (if_info->cond_earliest))
4418 1212402 : && !modified_in_p (x, insn_b));
4419 :
4420 : /* We're going to be moving the evaluation of B down from above
4421 : COND_EARLIEST to JUMP. Make sure the relevant data is still
4422 : intact. */
4423 204751 : if (! insn_b
4424 202030 : || BLOCK_FOR_INSN (insn_b) != BLOCK_FOR_INSN (if_info->cond_earliest)
4425 118478 : || !NONJUMP_INSN_P (insn_b)
4426 113260 : || (set_b = single_set (insn_b)) == NULL_RTX
4427 111817 : || ! rtx_interchangeable_p (x, SET_DEST (set_b))
4428 96980 : || ! noce_operand_ok (SET_SRC (set_b))
4429 94965 : || reg_overlap_mentioned_p (x, SET_SRC (set_b))
4430 93061 : || modified_between_p (SET_SRC (set_b), insn_b, jump)
4431 : /* Avoid extending the lifetime of hard registers on small
4432 : register class machines. */
4433 90647 : || (REG_P (SET_SRC (set_b))
4434 26440 : && HARD_REGISTER_P (SET_SRC (set_b))
4435 3275 : && targetm.small_register_classes_for_mode_p
4436 3275 : (GET_MODE (SET_SRC (set_b))))
4437 : /* Likewise with X. In particular this can happen when
4438 : noce_get_condition looks farther back in the instruction
4439 : stream than one might expect. */
4440 87372 : || reg_overlap_mentioned_p (x, cond)
4441 74250 : || reg_overlap_mentioned_p (x, a)
4442 273139 : || modified_between_p (x, insn_b, jump))
4443 : {
4444 : insn_b = NULL;
4445 : set_b = NULL_RTX;
4446 : }
4447 : }
4448 :
4449 : /* If x has side effects then only the if-then-else form is safe to
4450 : convert. But even in that case we would need to restore any notes
4451 : (such as REG_INC) at then end. That can be tricky if
4452 : noce_emit_move_insn expands to more than one insn, so disable the
4453 : optimization entirely for now if there are side effects. */
4454 295967 : if (side_effects_p (x))
4455 : return false;
4456 :
4457 295967 : b = (set_b ? SET_SRC (set_b) : x);
4458 :
4459 : /* Only operate on register destinations, and even then avoid extending
4460 : the lifetime of hard registers on small register class machines. */
4461 295967 : orig_x = x;
4462 295967 : if_info->orig_x = orig_x;
4463 295967 : if (!REG_P (x)
4464 295967 : || (HARD_REGISTER_P (x)
4465 2 : && targetm.small_register_classes_for_mode_p (GET_MODE (x))))
4466 : {
4467 66794 : if (GET_MODE (x) == BLKmode)
4468 : return false;
4469 :
4470 66566 : if (GET_CODE (x) == ZERO_EXTRACT
4471 3 : && (!CONST_INT_P (XEXP (x, 1))
4472 3 : || !CONST_INT_P (XEXP (x, 2))))
4473 : return false;
4474 :
4475 66566 : x = gen_reg_rtx (GET_MODE (GET_CODE (x) == STRICT_LOW_PART
4476 : ? XEXP (x, 0) : x));
4477 : }
4478 :
4479 : /* Don't operate on sources that may trap or are volatile. */
4480 295739 : if (! noce_operand_ok (a) || ! noce_operand_ok (b))
4481 0 : return false;
4482 :
4483 340971 : retry:
4484 : /* Set up the info block for our subroutines. */
4485 340971 : if_info->insn_a = insn_a;
4486 340971 : if_info->insn_b = insn_b;
4487 340971 : if_info->x = x;
4488 340971 : if_info->a = a;
4489 340971 : if_info->b = b;
4490 :
4491 : /* Try optimizations in some approximation of a useful order. */
4492 : /* ??? Should first look to see if X is live incoming at all. If it
4493 : isn't, we don't need anything but an unconditional set. */
4494 :
4495 : /* Look and see if A and B are really the same. Avoid creating silly
4496 : cmove constructs that no one will fix up later. */
4497 340971 : if (noce_simple_bbs (if_info)
4498 318511 : && rtx_interchangeable_p (a, b))
4499 : {
4500 : /* If we have an INSN_B, we don't have to create any new rtl. Just
4501 : move the instruction that we already have. If we don't have an
4502 : INSN_B, that means that A == X, and we've got a noop move. In
4503 : that case don't do anything and let the code below delete INSN_A. */
4504 377 : if (insn_b && else_bb)
4505 : {
4506 365 : rtx note;
4507 :
4508 365 : if (else_bb && insn_b == BB_END (else_bb))
4509 248 : BB_END (else_bb) = PREV_INSN (insn_b);
4510 365 : reorder_insns (insn_b, insn_b, PREV_INSN (jump));
4511 :
4512 : /* If there was a REG_EQUAL note, delete it since it may have been
4513 : true due to this insn being after a jump. */
4514 365 : if ((note = find_reg_note (insn_b, REG_EQUAL, NULL_RTX)) != 0)
4515 0 : remove_note (insn_b, note);
4516 :
4517 377 : insn_b = NULL;
4518 : }
4519 : /* If we have "x = b; if (...) x = a;", and x has side-effects, then
4520 : x must be executed twice. */
4521 12 : else if (insn_b && side_effects_p (orig_x))
4522 : return false;
4523 :
4524 377 : x = orig_x;
4525 377 : goto success;
4526 : }
4527 :
4528 340594 : if (!set_b && MEM_P (orig_x))
4529 : /* We want to avoid store speculation to avoid cases like
4530 : if (pthread_mutex_trylock(mutex))
4531 : ++global_variable;
4532 : Rather than go to much effort here, we rely on the SSA optimizers,
4533 : which do a good enough job these days. */
4534 : return false;
4535 :
4536 275600 : if (noce_try_move (if_info))
4537 0 : goto success;
4538 275600 : if (noce_try_ifelse_collapse (if_info))
4539 15655 : goto success;
4540 259945 : if (noce_try_store_flag (if_info))
4541 25952 : goto success;
4542 233993 : if (noce_try_bitop (if_info))
4543 0 : goto success;
4544 233993 : if (noce_try_minmax (if_info))
4545 53 : goto success;
4546 233940 : if (noce_try_abs (if_info))
4547 24 : goto success;
4548 233916 : if (noce_try_inverse_constants (if_info))
4549 0 : goto success;
4550 233916 : if (!targetm.have_conditional_execution ()
4551 233916 : && noce_try_store_flag_constants (if_info))
4552 2940 : goto success;
4553 230976 : if (noce_try_sign_bit_splat (if_info))
4554 163 : goto success;
4555 230813 : if (!targetm.have_conditional_execution ()
4556 230813 : && noce_try_store_flag_logical (if_info))
4557 0 : goto success;
4558 230813 : if (HAVE_conditional_move
4559 230813 : && noce_try_cmove (if_info))
4560 43654 : goto success;
4561 187159 : if (! targetm.have_conditional_execution ())
4562 : {
4563 187159 : if (noce_try_addcc (if_info))
4564 4770 : goto success;
4565 182389 : if (noce_try_store_flag_mask (if_info))
4566 2 : goto success;
4567 182387 : if (HAVE_conditional_move
4568 182387 : && noce_try_cond_arith (if_info))
4569 8446 : goto success;
4570 173941 : if (HAVE_conditional_move
4571 173941 : && noce_try_cmove_arith (if_info))
4572 27660 : goto success;
4573 146281 : if (noce_try_sign_mask (if_info))
4574 0 : goto success;
4575 : }
4576 :
4577 146281 : if (!else_bb && set_b)
4578 : {
4579 45232 : insn_b = NULL;
4580 45232 : set_b = NULL_RTX;
4581 45232 : b = orig_x;
4582 45232 : goto retry;
4583 : }
4584 :
4585 : return false;
4586 :
4587 129696 : success:
4588 129696 : if (dump_file && if_info->transform_name)
4589 5 : fprintf (dump_file, "if-conversion succeeded through %s\n",
4590 : if_info->transform_name);
4591 :
4592 : /* If we used a temporary, fix it up now. */
4593 129696 : if (orig_x != x)
4594 : {
4595 627 : rtx_insn *seq;
4596 :
4597 627 : start_sequence ();
4598 627 : noce_emit_move_insn (orig_x, x);
4599 627 : seq = get_insns ();
4600 627 : set_used_flags (orig_x);
4601 627 : unshare_all_rtl_in_chain (seq);
4602 627 : end_sequence ();
4603 :
4604 627 : emit_insn_before_setloc (seq, BB_END (test_bb), INSN_LOCATION (insn_a));
4605 : }
4606 :
4607 : /* The original THEN and ELSE blocks may now be removed. The test block
4608 : must now jump to the join block. If the test block and the join block
4609 : can be merged, do so. */
4610 129696 : if (else_bb)
4611 : {
4612 61846 : delete_basic_block (else_bb);
4613 61846 : num_true_changes++;
4614 : }
4615 : else
4616 67850 : remove_edge (find_edge (test_bb, join_bb));
4617 :
4618 129696 : remove_edge (find_edge (then_bb, join_bb));
4619 129696 : redirect_edge_and_branch_force (single_succ_edge (test_bb), join_bb);
4620 129696 : delete_basic_block (then_bb);
4621 129696 : num_true_changes++;
4622 :
4623 129696 : if (can_merge_blocks_p (test_bb, join_bb))
4624 : {
4625 101196 : merge_blocks (test_bb, join_bb);
4626 101196 : num_true_changes++;
4627 : }
4628 :
4629 129696 : num_updated_if_blocks++;
4630 129696 : return true;
4631 : }
4632 :
4633 : /* Check whether a block is suitable for conditional move conversion.
4634 : Every insn must be a simple set of a register to a constant or a
4635 : register. For each assignment, store the value in the pointer map
4636 : VALS, keyed indexed by register pointer, then store the register
4637 : pointer in REGS. COND is the condition we will test. */
4638 :
4639 : static bool
4640 1707351 : check_cond_move_block (basic_block bb,
4641 : hash_map<rtx, rtx> *vals,
4642 : vec<rtx> *regs,
4643 : rtx cond)
4644 : {
4645 1707351 : rtx_insn *insn;
4646 1707351 : rtx cc = cc_in_cond (cond);
4647 :
4648 : /* We can only handle simple jumps at the end of the basic block.
4649 : It is almost impossible to update the CFG otherwise. */
4650 1707351 : insn = BB_END (bb);
4651 1707351 : if (JUMP_P (insn) && !onlyjump_p (insn))
4652 : return false;
4653 :
4654 9459274 : FOR_BB_INSNS (bb, insn)
4655 : {
4656 9385948 : rtx set, dest, src;
4657 :
4658 9385948 : if (!NONDEBUG_INSN_P (insn) || JUMP_P (insn))
4659 7591942 : continue;
4660 1794006 : set = single_set (insn);
4661 1794006 : if (!set)
4662 1633972 : return false;
4663 :
4664 1720657 : dest = SET_DEST (set);
4665 1720657 : src = SET_SRC (set);
4666 1720657 : if (!REG_P (dest)
4667 1720657 : || (HARD_REGISTER_P (dest)
4668 276416 : && targetm.small_register_classes_for_mode_p (GET_MODE (dest))))
4669 655379 : return false;
4670 :
4671 1065278 : if (!CONSTANT_P (src) && !register_operand (src, VOIDmode))
4672 : return false;
4673 :
4674 182804 : if (side_effects_p (src) || side_effects_p (dest))
4675 0 : return false;
4676 :
4677 182804 : if (may_trap_p (src) || may_trap_p (dest))
4678 0 : return false;
4679 :
4680 : /* Don't try to handle this if the source register was
4681 : modified earlier in the block. */
4682 182804 : if ((REG_P (src)
4683 51921 : && vals->get (src))
4684 234441 : || (GET_CODE (src) == SUBREG && REG_P (SUBREG_REG (src))
4685 4998 : && vals->get (SUBREG_REG (src))))
4686 335 : return false;
4687 :
4688 : /* Don't try to handle this if the destination register was
4689 : modified earlier in the block. */
4690 182469 : if (vals->get (dest))
4691 : return false;
4692 :
4693 : /* Don't try to handle this if the condition uses the
4694 : destination register. */
4695 182469 : if (reg_overlap_mentioned_p (dest, cond))
4696 : return false;
4697 :
4698 : /* Don't try to handle this if the source register is modified
4699 : later in the block. */
4700 163446 : if (!CONSTANT_P (src)
4701 163446 : && modified_between_p (src, insn, NEXT_INSN (BB_END (bb))))
4702 : return false;
4703 :
4704 : /* Skip it if the instruction to be moved might clobber CC. */
4705 160034 : if (cc && set_of (cc, insn))
4706 : return false;
4707 :
4708 160034 : vals->put (dest, src);
4709 :
4710 160034 : regs->safe_push (dest);
4711 : }
4712 :
4713 : return true;
4714 : }
4715 :
4716 : /* Given a basic block BB suitable for conditional move conversion,
4717 : a condition COND, and pointer maps THEN_VALS and ELSE_VALS containing
4718 : the register values depending on COND, emit the insns in the block as
4719 : conditional moves. If ELSE_BLOCK is true, THEN_BB was already
4720 : processed. The caller has started a sequence for the conversion.
4721 : Return true if successful, false if something goes wrong. */
4722 :
4723 : static bool
4724 47577 : cond_move_convert_if_block (struct noce_if_info *if_infop,
4725 : basic_block bb, rtx cond,
4726 : hash_map<rtx, rtx> *then_vals,
4727 : hash_map<rtx, rtx> *else_vals,
4728 : bool else_block_p)
4729 : {
4730 47577 : enum rtx_code code;
4731 47577 : rtx_insn *insn;
4732 47577 : rtx cond_arg0, cond_arg1;
4733 :
4734 47577 : code = GET_CODE (cond);
4735 47577 : cond_arg0 = XEXP (cond, 0);
4736 47577 : cond_arg1 = XEXP (cond, 1);
4737 :
4738 184969 : FOR_BB_INSNS (bb, insn)
4739 : {
4740 152677 : rtx set, target, dest, t, e;
4741 :
4742 : /* ??? Maybe emit conditional debug insn? */
4743 152677 : if (!NONDEBUG_INSN_P (insn) || JUMP_P (insn))
4744 100398 : continue;
4745 79087 : set = single_set (insn);
4746 79087 : gcc_assert (set && REG_P (SET_DEST (set)));
4747 :
4748 79087 : dest = SET_DEST (set);
4749 :
4750 79087 : rtx *then_slot = then_vals->get (dest);
4751 79087 : rtx *else_slot = else_vals->get (dest);
4752 79087 : t = then_slot ? *then_slot : NULL_RTX;
4753 79087 : e = else_slot ? *else_slot : NULL_RTX;
4754 :
4755 79087 : if (else_block_p)
4756 : {
4757 : /* If this register was set in the then block, we already
4758 : handled this case there. */
4759 28252 : if (t)
4760 26808 : continue;
4761 1444 : t = dest;
4762 1444 : gcc_assert (e);
4763 : }
4764 : else
4765 : {
4766 50835 : gcc_assert (t);
4767 50835 : if (!e)
4768 23328 : e = dest;
4769 : }
4770 :
4771 52279 : if (if_infop->cond_inverted)
4772 0 : std::swap (t, e);
4773 :
4774 52279 : target = noce_emit_cmove (if_infop, dest, code, cond_arg0, cond_arg1,
4775 : t, e);
4776 52279 : if (!target)
4777 15285 : return false;
4778 :
4779 36994 : if (target != dest)
4780 0 : noce_emit_move_insn (dest, target);
4781 : }
4782 :
4783 : return true;
4784 : }
4785 :
4786 : /* Given a simple IF-THEN-JOIN or IF-THEN-ELSE-JOIN block, attempt to convert
4787 : it using only conditional moves. Return TRUE if we were successful at
4788 : converting the block. */
4789 :
4790 : static bool
4791 1671551 : cond_move_process_if_block (struct noce_if_info *if_info)
4792 : {
4793 1671551 : basic_block test_bb = if_info->test_bb;
4794 1671551 : basic_block then_bb = if_info->then_bb;
4795 1671551 : basic_block else_bb = if_info->else_bb;
4796 1671551 : basic_block join_bb = if_info->join_bb;
4797 1671551 : rtx_insn *jump = if_info->jump;
4798 1671551 : rtx cond = if_info->cond;
4799 1671551 : rtx_insn *seq, *loc_insn;
4800 1671551 : int c;
4801 1671551 : vec<rtx> then_regs = vNULL;
4802 1671551 : vec<rtx> else_regs = vNULL;
4803 1671551 : bool success_p = false;
4804 1671551 : int limit = param_max_rtl_if_conversion_insns;
4805 :
4806 : /* Build a mapping for each block to the value used for each
4807 : register. */
4808 1671551 : hash_map<rtx, rtx> then_vals;
4809 1671551 : hash_map<rtx, rtx> else_vals;
4810 :
4811 : /* Make sure the blocks are suitable. */
4812 1671551 : if (!check_cond_move_block (then_bb, &then_vals, &then_regs, cond)
4813 1671551 : || (else_bb
4814 35800 : && !check_cond_move_block (else_bb, &else_vals, &else_regs, cond)))
4815 1634025 : goto done;
4816 :
4817 : /* Make sure the blocks can be used together. If the same register
4818 : is set in both blocks, and is not set to a constant in both
4819 : cases, then both blocks must set it to the same register. We
4820 : have already verified that if it is set to a register, that the
4821 : source register does not change after the assignment. Also count
4822 : the number of registers set in only one of the blocks. */
4823 37526 : c = 0;
4824 164549 : for (rtx reg : then_regs)
4825 : {
4826 54814 : rtx *then_slot = then_vals.get (reg);
4827 54814 : rtx *else_slot = else_vals.get (reg);
4828 :
4829 54814 : gcc_checking_assert (then_slot);
4830 54814 : if (!else_slot)
4831 24044 : ++c;
4832 : else
4833 : {
4834 30770 : rtx then_val = *then_slot;
4835 30770 : rtx else_val = *else_slot;
4836 4162 : if (!CONSTANT_P (then_val) && !CONSTANT_P (else_val)
4837 34309 : && !rtx_equal_p (then_val, else_val))
4838 2591 : goto done;
4839 : }
4840 : }
4841 :
4842 : /* Finish off c for MAX_CONDITIONAL_EXECUTE. */
4843 91391 : for (rtx reg : else_regs)
4844 : {
4845 29616 : gcc_checking_assert (else_vals.get (reg));
4846 29616 : if (!then_vals.get (reg))
4847 1584 : ++c;
4848 : }
4849 :
4850 : /* Make sure it is reasonable to convert this block. What matters
4851 : is the number of assignments currently made in only one of the
4852 : branches, since if we convert we are going to always execute
4853 : them. */
4854 34935 : if (c > MAX_CONDITIONAL_EXECUTE
4855 34935 : || c > limit)
4856 38 : goto done;
4857 :
4858 : /* Try to emit the conditional moves. First do the then block,
4859 : then do anything left in the else blocks. */
4860 34897 : start_sequence ();
4861 34897 : if (!cond_move_convert_if_block (if_info, then_bb, cond,
4862 : &then_vals, &else_vals, false)
4863 34897 : || (else_bb
4864 12680 : && !cond_move_convert_if_block (if_info, else_bb, cond,
4865 : &then_vals, &else_vals, true)))
4866 : {
4867 15285 : end_sequence ();
4868 15285 : goto done;
4869 : }
4870 19612 : seq = end_ifcvt_sequence (if_info);
4871 19612 : if (!seq || !targetm.noce_conversion_profitable_p (seq, if_info))
4872 16763 : goto done;
4873 :
4874 2849 : loc_insn = first_active_insn (then_bb);
4875 2849 : if (!loc_insn)
4876 : {
4877 0 : loc_insn = first_active_insn (else_bb);
4878 0 : gcc_assert (loc_insn);
4879 : }
4880 2849 : emit_insn_before_setloc (seq, jump, INSN_LOCATION (loc_insn));
4881 :
4882 2849 : if (else_bb)
4883 : {
4884 2849 : delete_basic_block (else_bb);
4885 2849 : num_true_changes++;
4886 : }
4887 : else
4888 0 : remove_edge (find_edge (test_bb, join_bb));
4889 :
4890 2849 : remove_edge (find_edge (then_bb, join_bb));
4891 2849 : redirect_edge_and_branch_force (single_succ_edge (test_bb), join_bb);
4892 2849 : delete_basic_block (then_bb);
4893 2849 : num_true_changes++;
4894 :
4895 2849 : if (can_merge_blocks_p (test_bb, join_bb))
4896 : {
4897 849 : merge_blocks (test_bb, join_bb);
4898 849 : num_true_changes++;
4899 : }
4900 :
4901 2849 : num_updated_if_blocks++;
4902 2849 : success_p = true;
4903 :
4904 1671551 : done:
4905 1671551 : then_regs.release ();
4906 1671551 : else_regs.release ();
4907 1671551 : return success_p;
4908 1671551 : }
4909 :
4910 :
4911 : /* Determine if a given basic block heads a simple IF-THEN-JOIN or an
4912 : IF-THEN-ELSE-JOIN block.
4913 :
4914 : If so, we'll try to convert the insns to not require the branch,
4915 : using only transformations that do not require conditional execution.
4916 :
4917 : Return TRUE if we were successful at converting the block. */
4918 :
4919 : static bool
4920 9533118 : noce_find_if_block (basic_block test_bb, edge then_edge, edge else_edge,
4921 : int pass)
4922 : {
4923 9533118 : basic_block then_bb, else_bb, join_bb;
4924 9533118 : bool then_else_reversed = false;
4925 9533118 : rtx_insn *jump;
4926 9533118 : rtx_insn *cond_earliest;
4927 9533118 : struct noce_if_info if_info;
4928 9533118 : bool speed_p = optimize_bb_for_speed_p (test_bb);
4929 :
4930 : /* We only ever should get here before reload. */
4931 9533118 : gcc_assert (!reload_completed);
4932 :
4933 : /* Recognize an IF-THEN-ELSE-JOIN block. */
4934 9533118 : if (single_pred_p (then_edge->dest)
4935 7844617 : && single_succ_p (then_edge->dest)
4936 3467061 : && single_pred_p (else_edge->dest)
4937 9797484 : && single_succ_p (else_edge->dest)
4938 10636905 : && single_succ (then_edge->dest) == single_succ (else_edge->dest))
4939 : {
4940 : then_bb = then_edge->dest;
4941 : else_bb = else_edge->dest;
4942 : join_bb = single_succ (then_bb);
4943 : }
4944 : /* Recognize an IF-THEN-JOIN block. */
4945 8693697 : else if (single_pred_p (then_edge->dest)
4946 10397934 : && single_succ_p (then_edge->dest)
4947 11321337 : && single_succ (then_edge->dest) == else_edge->dest)
4948 : {
4949 : then_bb = then_edge->dest;
4950 : else_bb = NULL_BLOCK;
4951 : join_bb = else_edge->dest;
4952 : }
4953 : /* Recognize an IF-ELSE-JOIN block. We can have those because the order
4954 : of basic blocks in cfglayout mode does not matter, so the fallthrough
4955 : edge can go to any basic block (and not just to bb->next_bb, like in
4956 : cfgrtl mode). */
4957 17145516 : else if (single_pred_p (else_edge->dest)
4958 10664632 : && single_succ_p (else_edge->dest)
4959 9059704 : && single_succ (else_edge->dest) == then_edge->dest)
4960 : {
4961 : /* The noce transformations do not apply to IF-ELSE-JOIN blocks.
4962 : To make this work, we have to invert the THEN and ELSE blocks
4963 : and reverse the jump condition. */
4964 : then_bb = else_edge->dest;
4965 : else_bb = NULL_BLOCK;
4966 : join_bb = single_succ (then_bb);
4967 : then_else_reversed = true;
4968 : }
4969 : else
4970 : /* Not a form we can handle. */
4971 : return false;
4972 :
4973 : /* The edges of the THEN and ELSE blocks cannot have complex edges. */
4974 1893283 : if (single_succ_edge (then_bb)->flags & EDGE_COMPLEX)
4975 : return false;
4976 1840562 : if (else_bb
4977 1840562 : && single_succ_edge (else_bb)->flags & EDGE_COMPLEX)
4978 : return false;
4979 :
4980 1830474 : num_possible_if_blocks++;
4981 :
4982 1830474 : if (dump_file)
4983 : {
4984 90 : fprintf (dump_file,
4985 : "\nIF-THEN%s-JOIN block found, pass %d, test %d, then %d",
4986 : (else_bb) ? "-ELSE" : "",
4987 : pass, test_bb->index, then_bb->index);
4988 :
4989 53 : if (else_bb)
4990 16 : fprintf (dump_file, ", else %d", else_bb->index);
4991 :
4992 53 : fprintf (dump_file, ", join %d\n", join_bb->index);
4993 : }
4994 :
4995 : /* If the conditional jump is more than just a conditional
4996 : jump, then we cannot do if-conversion on this block. */
4997 1830474 : jump = BB_END (test_bb);
4998 1830474 : if (! onlyjump_p (jump))
4999 : return false;
5000 :
5001 : /* Initialize an IF_INFO struct to pass around. */
5002 1830241 : memset (&if_info, 0, sizeof if_info);
5003 1830241 : if_info.test_bb = test_bb;
5004 1830241 : if_info.then_bb = then_bb;
5005 1830241 : if_info.else_bb = else_bb;
5006 1830241 : if_info.join_bb = join_bb;
5007 1830241 : if_info.cond = noce_get_condition (jump, &cond_earliest,
5008 : then_else_reversed);
5009 1830241 : rtx_insn *rev_cond_earliest;
5010 3660482 : if_info.rev_cond = noce_get_condition (jump, &rev_cond_earliest,
5011 1830241 : !then_else_reversed);
5012 1830241 : if (!if_info.cond && !if_info.rev_cond)
5013 : return false;
5014 1826598 : if (!if_info.cond)
5015 : {
5016 0 : std::swap (if_info.cond, if_info.rev_cond);
5017 0 : std::swap (cond_earliest, rev_cond_earliest);
5018 0 : if_info.cond_inverted = true;
5019 : }
5020 : /* We must be comparing objects whose modes imply the size. */
5021 1826598 : if (GET_MODE (XEXP (if_info.cond, 0)) == BLKmode)
5022 : return false;
5023 1826598 : gcc_assert (if_info.rev_cond == NULL_RTX
5024 : || rev_cond_earliest == cond_earliest);
5025 1826598 : if_info.cond_earliest = cond_earliest;
5026 1826598 : if_info.jump = jump;
5027 1826598 : if_info.then_else_reversed = then_else_reversed;
5028 1826598 : if_info.speed_p = speed_p;
5029 1826598 : if_info.max_seq_cost
5030 1826598 : = targetm.max_noce_ifcvt_seq_cost (then_edge);
5031 : /* We'll add in the cost of THEN_BB and ELSE_BB later, when we check
5032 : that they are valid to transform. We can't easily get back to the insn
5033 : for COND (and it may not exist if we had to canonicalize to get COND).
5034 : It is assumed that the costs of a jump insn are dependent on the
5035 : branch costs. */
5036 1826598 : if_info.original_cost += insn_cost (if_info.jump, if_info.speed_p);
5037 :
5038 : /* Do the real work. */
5039 :
5040 : /* ??? noce_process_if_block has not yet been updated to handle
5041 : inverted conditions. */
5042 1826598 : if (!if_info.cond_inverted && noce_process_if_block (&if_info))
5043 : return true;
5044 :
5045 1671551 : if (HAVE_conditional_move
5046 1671551 : && cond_move_process_if_block (&if_info))
5047 : return true;
5048 :
5049 : return false;
5050 : }
5051 :
5052 :
5053 : /* Merge the blocks and mark for local life update. */
5054 :
5055 : static void
5056 0 : merge_if_block (struct ce_if_block * ce_info)
5057 : {
5058 0 : basic_block test_bb = ce_info->test_bb; /* last test block */
5059 0 : basic_block then_bb = ce_info->then_bb; /* THEN */
5060 0 : basic_block else_bb = ce_info->else_bb; /* ELSE or NULL */
5061 0 : basic_block join_bb = ce_info->join_bb; /* join block */
5062 0 : basic_block combo_bb;
5063 :
5064 : /* All block merging is done into the lower block numbers. */
5065 :
5066 0 : combo_bb = test_bb;
5067 0 : df_set_bb_dirty (test_bb);
5068 :
5069 : /* Merge any basic blocks to handle && and || subtests. Each of
5070 : the blocks are on the fallthru path from the predecessor block. */
5071 0 : if (ce_info->num_multiple_test_blocks > 0)
5072 : {
5073 0 : basic_block bb = test_bb;
5074 0 : basic_block last_test_bb = ce_info->last_test_bb;
5075 0 : basic_block fallthru = block_fallthru (bb);
5076 :
5077 0 : do
5078 : {
5079 0 : bb = fallthru;
5080 0 : fallthru = block_fallthru (bb);
5081 0 : merge_blocks (combo_bb, bb);
5082 0 : num_true_changes++;
5083 : }
5084 0 : while (bb != last_test_bb);
5085 : }
5086 :
5087 : /* Merge TEST block into THEN block. Normally the THEN block won't have a
5088 : label, but it might if there were || tests. That label's count should be
5089 : zero, and it normally should be removed. */
5090 :
5091 0 : if (then_bb)
5092 : {
5093 : /* If THEN_BB has no successors, then there's a BARRIER after it.
5094 : If COMBO_BB has more than one successor (THEN_BB), then that BARRIER
5095 : is no longer needed, and in fact it is incorrect to leave it in
5096 : the insn stream. */
5097 0 : if (EDGE_COUNT (then_bb->succs) == 0
5098 0 : && EDGE_COUNT (combo_bb->succs) > 1)
5099 : {
5100 0 : rtx_insn *end = NEXT_INSN (BB_END (then_bb));
5101 0 : while (end && NOTE_P (end) && !NOTE_INSN_BASIC_BLOCK_P (end))
5102 0 : end = NEXT_INSN (end);
5103 :
5104 0 : if (end && BARRIER_P (end))
5105 0 : delete_insn (end);
5106 : }
5107 0 : merge_blocks (combo_bb, then_bb);
5108 0 : num_true_changes++;
5109 : }
5110 :
5111 : /* The ELSE block, if it existed, had a label. That label count
5112 : will almost always be zero, but odd things can happen when labels
5113 : get their addresses taken. */
5114 0 : if (else_bb)
5115 : {
5116 : /* If ELSE_BB has no successors, then there's a BARRIER after it.
5117 : If COMBO_BB has more than one successor (ELSE_BB), then that BARRIER
5118 : is no longer needed, and in fact it is incorrect to leave it in
5119 : the insn stream. */
5120 0 : if (EDGE_COUNT (else_bb->succs) == 0
5121 0 : && EDGE_COUNT (combo_bb->succs) > 1)
5122 : {
5123 0 : rtx_insn *end = NEXT_INSN (BB_END (else_bb));
5124 0 : while (end && NOTE_P (end) && !NOTE_INSN_BASIC_BLOCK_P (end))
5125 0 : end = NEXT_INSN (end);
5126 :
5127 0 : if (end && BARRIER_P (end))
5128 0 : delete_insn (end);
5129 : }
5130 0 : merge_blocks (combo_bb, else_bb);
5131 0 : num_true_changes++;
5132 : }
5133 :
5134 : /* If there was no join block reported, that means it was not adjacent
5135 : to the others, and so we cannot merge them. */
5136 :
5137 0 : if (! join_bb)
5138 : {
5139 0 : rtx_insn *last = BB_END (combo_bb);
5140 :
5141 : /* The outgoing edge for the current COMBO block should already
5142 : be correct. Verify this. */
5143 0 : if (EDGE_COUNT (combo_bb->succs) == 0)
5144 0 : gcc_assert (find_reg_note (last, REG_NORETURN, NULL)
5145 : || (NONJUMP_INSN_P (last)
5146 : && GET_CODE (PATTERN (last)) == TRAP_IF
5147 : && (TRAP_CONDITION (PATTERN (last))
5148 : == const_true_rtx)));
5149 :
5150 : else
5151 : /* There should still be something at the end of the THEN or ELSE
5152 : blocks taking us to our final destination. */
5153 0 : gcc_assert (JUMP_P (last)
5154 : || (EDGE_SUCC (combo_bb, 0)->dest
5155 : == EXIT_BLOCK_PTR_FOR_FN (cfun)
5156 : && CALL_P (last)
5157 : && SIBLING_CALL_P (last))
5158 : || ((EDGE_SUCC (combo_bb, 0)->flags & EDGE_EH)
5159 : && can_throw_internal (last)));
5160 : }
5161 :
5162 : /* The JOIN block may have had quite a number of other predecessors too.
5163 : Since we've already merged the TEST, THEN and ELSE blocks, we should
5164 : have only one remaining edge from our if-then-else diamond. If there
5165 : is more than one remaining edge, it must come from elsewhere. There
5166 : may be zero incoming edges if the THEN block didn't actually join
5167 : back up (as with a call to a non-return function). */
5168 0 : else if (EDGE_COUNT (join_bb->preds) < 2
5169 0 : && join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5170 : {
5171 : /* We can merge the JOIN cleanly and update the dataflow try
5172 : again on this pass.*/
5173 0 : merge_blocks (combo_bb, join_bb);
5174 0 : num_true_changes++;
5175 : }
5176 : else
5177 : {
5178 : /* We cannot merge the JOIN. */
5179 :
5180 : /* The outgoing edge for the current COMBO block should already
5181 : be correct. Verify this. */
5182 0 : gcc_assert (single_succ_p (combo_bb)
5183 : && single_succ (combo_bb) == join_bb);
5184 :
5185 : /* Remove the jump and cruft from the end of the COMBO block. */
5186 0 : if (join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5187 0 : tidy_fallthru_edge (single_succ_edge (combo_bb));
5188 : }
5189 :
5190 0 : num_updated_if_blocks++;
5191 0 : }
5192 :
5193 : /* Find a block ending in a simple IF condition and try to transform it
5194 : in some way. When converting a multi-block condition, put the new code
5195 : in the first such block and delete the rest. Return a pointer to this
5196 : first block if some transformation was done. Return NULL otherwise. */
5197 :
5198 : static basic_block
5199 38531131 : find_if_header (basic_block test_bb, int pass)
5200 : {
5201 38531131 : ce_if_block ce_info;
5202 38531131 : edge then_edge;
5203 38531131 : edge else_edge;
5204 :
5205 : /* The kind of block we're looking for has exactly two successors. */
5206 58095133 : if (EDGE_COUNT (test_bb->succs) != 2)
5207 : return NULL;
5208 :
5209 19865145 : then_edge = EDGE_SUCC (test_bb, 0);
5210 19865145 : else_edge = EDGE_SUCC (test_bb, 1);
5211 :
5212 19865145 : if (df_get_bb_dirty (then_edge->dest))
5213 : return NULL;
5214 19845153 : if (df_get_bb_dirty (else_edge->dest))
5215 : return NULL;
5216 :
5217 : /* Neither edge should be abnormal. */
5218 19809140 : if ((then_edge->flags & EDGE_COMPLEX)
5219 18122225 : || (else_edge->flags & EDGE_COMPLEX))
5220 : return NULL;
5221 :
5222 : /* Nor exit the loop. */
5223 17775377 : if ((then_edge->flags & EDGE_LOOP_EXIT)
5224 16334901 : || (else_edge->flags & EDGE_LOOP_EXIT))
5225 : return NULL;
5226 :
5227 : /* The THEN edge is canonically the one that falls through. */
5228 14090012 : if (then_edge->flags & EDGE_FALLTHRU)
5229 : ;
5230 6814820 : else if (else_edge->flags & EDGE_FALLTHRU)
5231 : std::swap (then_edge, else_edge);
5232 : else
5233 : /* Otherwise this must be a multiway branch of some sort. */
5234 : return NULL;
5235 :
5236 14089803 : memset (&ce_info, 0, sizeof (ce_info));
5237 14089803 : ce_info.test_bb = test_bb;
5238 14089803 : ce_info.then_bb = then_edge->dest;
5239 14089803 : ce_info.else_bb = else_edge->dest;
5240 14089803 : ce_info.pass = pass;
5241 :
5242 : #ifdef IFCVT_MACHDEP_INIT
5243 : IFCVT_MACHDEP_INIT (&ce_info);
5244 : #endif
5245 :
5246 14089803 : if (!reload_completed
5247 14089803 : && noce_find_if_block (test_bb, then_edge, else_edge, pass))
5248 157896 : goto success;
5249 :
5250 13931907 : if (reload_completed
5251 4556685 : && targetm.have_conditional_execution ()
5252 13931907 : && cond_exec_find_if_block (&ce_info))
5253 0 : goto success;
5254 :
5255 9375222 : if (!reload_completed && targetm.have_trap ()
5256 9375222 : && optab_handler (ctrap_optab, word_mode) != CODE_FOR_nothing
5257 13931907 : && find_cond_trap (test_bb, then_edge, else_edge))
5258 0 : goto success;
5259 :
5260 13931907 : if (dom_info_state (CDI_POST_DOMINATORS) >= DOM_NO_FAST_QUERY
5261 13931907 : && (reload_completed || !targetm.have_conditional_execution ()))
5262 : {
5263 13931907 : if (find_if_case_1 (test_bb, then_edge, else_edge))
5264 20584 : goto success;
5265 13911323 : if (find_if_case_2 (test_bb, then_edge, else_edge))
5266 122663 : goto success;
5267 : }
5268 :
5269 : return NULL;
5270 :
5271 301143 : success:
5272 301143 : if (dump_file)
5273 19 : fprintf (dump_file, "Conversion succeeded on pass %d.\n", pass);
5274 : /* Set this so we continue looking. */
5275 301143 : cond_exec_changed_p = true;
5276 301143 : return ce_info.test_bb;
5277 : }
5278 :
5279 : /* Return true if a block has two edges, one of which falls through to the next
5280 : block, and the other jumps to a specific block, so that we can tell if the
5281 : block is part of an && test or an || test. Returns either -1 or the number
5282 : of non-note, non-jump, non-USE/CLOBBER insns in the block. */
5283 :
5284 : static int
5285 0 : block_jumps_and_fallthru (basic_block cur_bb, basic_block target_bb)
5286 : {
5287 0 : edge cur_edge;
5288 0 : bool fallthru_p = false;
5289 0 : bool jump_p = false;
5290 0 : rtx_insn *insn;
5291 0 : rtx_insn *end;
5292 0 : int n_insns = 0;
5293 0 : edge_iterator ei;
5294 :
5295 0 : if (!cur_bb || !target_bb)
5296 : return -1;
5297 :
5298 : /* If no edges, obviously it doesn't jump or fallthru. */
5299 0 : if (EDGE_COUNT (cur_bb->succs) == 0)
5300 : return 0;
5301 :
5302 0 : FOR_EACH_EDGE (cur_edge, ei, cur_bb->succs)
5303 : {
5304 0 : if (cur_edge->flags & EDGE_COMPLEX)
5305 : /* Anything complex isn't what we want. */
5306 : return -1;
5307 :
5308 0 : else if (cur_edge->flags & EDGE_FALLTHRU)
5309 : fallthru_p = true;
5310 :
5311 0 : else if (cur_edge->dest == target_bb)
5312 : jump_p = true;
5313 :
5314 : else
5315 : return -1;
5316 : }
5317 :
5318 0 : if ((jump_p & fallthru_p) == 0)
5319 : return -1;
5320 :
5321 : /* Don't allow calls in the block, since this is used to group && and ||
5322 : together for conditional execution support. ??? we should support
5323 : conditional execution support across calls for IA-64 some day, but
5324 : for now it makes the code simpler. */
5325 0 : end = BB_END (cur_bb);
5326 0 : insn = BB_HEAD (cur_bb);
5327 :
5328 0 : while (insn != NULL_RTX)
5329 : {
5330 0 : if (CALL_P (insn))
5331 : return -1;
5332 :
5333 0 : if (INSN_P (insn)
5334 0 : && !JUMP_P (insn)
5335 0 : && !DEBUG_INSN_P (insn)
5336 0 : && GET_CODE (PATTERN (insn)) != USE
5337 0 : && GET_CODE (PATTERN (insn)) != CLOBBER)
5338 0 : n_insns++;
5339 :
5340 0 : if (insn == end)
5341 : break;
5342 :
5343 0 : insn = NEXT_INSN (insn);
5344 : }
5345 :
5346 : return n_insns;
5347 : }
5348 :
5349 : /* Determine if a given basic block heads a simple IF-THEN or IF-THEN-ELSE
5350 : block. If so, we'll try to convert the insns to not require the branch.
5351 : Return TRUE if we were successful at converting the block. */
5352 :
5353 : static bool
5354 0 : cond_exec_find_if_block (struct ce_if_block * ce_info)
5355 : {
5356 0 : basic_block test_bb = ce_info->test_bb;
5357 0 : basic_block then_bb = ce_info->then_bb;
5358 0 : basic_block else_bb = ce_info->else_bb;
5359 0 : basic_block join_bb = NULL_BLOCK;
5360 0 : edge cur_edge;
5361 0 : basic_block next;
5362 0 : edge_iterator ei;
5363 :
5364 0 : ce_info->last_test_bb = test_bb;
5365 :
5366 : /* We only ever should get here after reload,
5367 : and if we have conditional execution. */
5368 0 : gcc_assert (reload_completed && targetm.have_conditional_execution ());
5369 :
5370 : /* Discover if any fall through predecessors of the current test basic block
5371 : were && tests (which jump to the else block) or || tests (which jump to
5372 : the then block). */
5373 0 : if (single_pred_p (test_bb)
5374 0 : && single_pred_edge (test_bb)->flags == EDGE_FALLTHRU)
5375 : {
5376 0 : basic_block bb = single_pred (test_bb);
5377 0 : basic_block target_bb;
5378 0 : int max_insns = MAX_CONDITIONAL_EXECUTE;
5379 0 : int n_insns;
5380 :
5381 : /* Determine if the preceding block is an && or || block. */
5382 0 : if ((n_insns = block_jumps_and_fallthru (bb, else_bb)) >= 0)
5383 : {
5384 0 : ce_info->and_and_p = true;
5385 0 : target_bb = else_bb;
5386 : }
5387 0 : else if ((n_insns = block_jumps_and_fallthru (bb, then_bb)) >= 0)
5388 : {
5389 0 : ce_info->and_and_p = false;
5390 0 : target_bb = then_bb;
5391 : }
5392 : else
5393 : target_bb = NULL_BLOCK;
5394 :
5395 0 : if (target_bb && n_insns <= max_insns)
5396 : {
5397 0 : int total_insns = 0;
5398 0 : int blocks = 0;
5399 :
5400 0 : ce_info->last_test_bb = test_bb;
5401 :
5402 : /* Found at least one && or || block, look for more. */
5403 0 : do
5404 : {
5405 0 : ce_info->test_bb = test_bb = bb;
5406 0 : total_insns += n_insns;
5407 0 : blocks++;
5408 :
5409 0 : if (!single_pred_p (bb))
5410 : break;
5411 :
5412 0 : bb = single_pred (bb);
5413 0 : n_insns = block_jumps_and_fallthru (bb, target_bb);
5414 : }
5415 0 : while (n_insns >= 0 && (total_insns + n_insns) <= max_insns);
5416 :
5417 0 : ce_info->num_multiple_test_blocks = blocks;
5418 0 : ce_info->num_multiple_test_insns = total_insns;
5419 :
5420 0 : if (ce_info->and_and_p)
5421 0 : ce_info->num_and_and_blocks = blocks;
5422 : else
5423 0 : ce_info->num_or_or_blocks = blocks;
5424 : }
5425 : }
5426 :
5427 : /* The THEN block of an IF-THEN combo must have exactly one predecessor,
5428 : other than any || blocks which jump to the THEN block. */
5429 0 : if ((EDGE_COUNT (then_bb->preds) - ce_info->num_or_or_blocks) != 1)
5430 : return false;
5431 :
5432 : /* The edges of the THEN and ELSE blocks cannot have complex edges. */
5433 0 : FOR_EACH_EDGE (cur_edge, ei, then_bb->preds)
5434 : {
5435 0 : if (cur_edge->flags & EDGE_COMPLEX)
5436 : return false;
5437 : }
5438 :
5439 0 : FOR_EACH_EDGE (cur_edge, ei, else_bb->preds)
5440 : {
5441 0 : if (cur_edge->flags & EDGE_COMPLEX)
5442 : return false;
5443 : }
5444 :
5445 : /* The THEN block of an IF-THEN combo must have zero or one successors. */
5446 0 : if (EDGE_COUNT (then_bb->succs) > 0
5447 0 : && (!single_succ_p (then_bb)
5448 0 : || (single_succ_edge (then_bb)->flags & EDGE_COMPLEX)
5449 0 : || (epilogue_completed
5450 0 : && tablejump_p (BB_END (then_bb), NULL, NULL))))
5451 0 : return false;
5452 :
5453 : /* If the THEN block has no successors, conditional execution can still
5454 : make a conditional call. Don't do this unless the ELSE block has
5455 : only one incoming edge -- the CFG manipulation is too ugly otherwise.
5456 : Check for the last insn of the THEN block being an indirect jump, which
5457 : is listed as not having any successors, but confuses the rest of the CE
5458 : code processing. ??? we should fix this in the future. */
5459 0 : if (EDGE_COUNT (then_bb->succs) == 0)
5460 : {
5461 0 : if (single_pred_p (else_bb) && else_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5462 : {
5463 0 : rtx_insn *last_insn = BB_END (then_bb);
5464 :
5465 0 : while (last_insn
5466 0 : && NOTE_P (last_insn)
5467 0 : && last_insn != BB_HEAD (then_bb))
5468 0 : last_insn = PREV_INSN (last_insn);
5469 :
5470 0 : if (last_insn
5471 0 : && JUMP_P (last_insn)
5472 0 : && ! simplejump_p (last_insn))
5473 : return false;
5474 :
5475 : join_bb = else_bb;
5476 : else_bb = NULL_BLOCK;
5477 : }
5478 : else
5479 : return false;
5480 : }
5481 :
5482 : /* If the THEN block's successor is the other edge out of the TEST block,
5483 : then we have an IF-THEN combo without an ELSE. */
5484 0 : else if (single_succ (then_bb) == else_bb)
5485 : {
5486 : join_bb = else_bb;
5487 : else_bb = NULL_BLOCK;
5488 : }
5489 :
5490 : /* If the THEN and ELSE block meet in a subsequent block, and the ELSE
5491 : has exactly one predecessor and one successor, and the outgoing edge
5492 : is not complex, then we have an IF-THEN-ELSE combo. */
5493 0 : else if (single_succ_p (else_bb)
5494 0 : && single_succ (then_bb) == single_succ (else_bb)
5495 0 : && single_pred_p (else_bb)
5496 0 : && !(single_succ_edge (else_bb)->flags & EDGE_COMPLEX)
5497 0 : && !(epilogue_completed
5498 0 : && tablejump_p (BB_END (else_bb), NULL, NULL)))
5499 0 : join_bb = single_succ (else_bb);
5500 :
5501 : /* Otherwise it is not an IF-THEN or IF-THEN-ELSE combination. */
5502 : else
5503 0 : return false;
5504 :
5505 0 : num_possible_if_blocks++;
5506 :
5507 0 : if (dump_file)
5508 : {
5509 0 : fprintf (dump_file,
5510 : "\nIF-THEN%s block found, pass %d, start block %d "
5511 : "[insn %d], then %d [%d]",
5512 : (else_bb) ? "-ELSE" : "",
5513 : ce_info->pass,
5514 : test_bb->index,
5515 0 : BB_HEAD (test_bb) ? (int)INSN_UID (BB_HEAD (test_bb)) : -1,
5516 : then_bb->index,
5517 0 : BB_HEAD (then_bb) ? (int)INSN_UID (BB_HEAD (then_bb)) : -1);
5518 :
5519 0 : if (else_bb)
5520 0 : fprintf (dump_file, ", else %d [%d]",
5521 : else_bb->index,
5522 0 : BB_HEAD (else_bb) ? (int)INSN_UID (BB_HEAD (else_bb)) : -1);
5523 :
5524 0 : fprintf (dump_file, ", join %d [%d]",
5525 : join_bb->index,
5526 0 : BB_HEAD (join_bb) ? (int)INSN_UID (BB_HEAD (join_bb)) : -1);
5527 :
5528 0 : if (ce_info->num_multiple_test_blocks > 0)
5529 0 : fprintf (dump_file, ", %d %s block%s last test %d [%d]",
5530 : ce_info->num_multiple_test_blocks,
5531 0 : (ce_info->and_and_p) ? "&&" : "||",
5532 : (ce_info->num_multiple_test_blocks == 1) ? "" : "s",
5533 : ce_info->last_test_bb->index,
5534 0 : ((BB_HEAD (ce_info->last_test_bb))
5535 0 : ? (int)INSN_UID (BB_HEAD (ce_info->last_test_bb))
5536 : : -1));
5537 :
5538 0 : fputc ('\n', dump_file);
5539 : }
5540 :
5541 : /* Make sure IF, THEN, and ELSE, blocks are adjacent. Actually, we get the
5542 : first condition for free, since we've already asserted that there's a
5543 : fallthru edge from IF to THEN. Likewise for the && and || blocks, since
5544 : we checked the FALLTHRU flag, those are already adjacent to the last IF
5545 : block. */
5546 : /* ??? As an enhancement, move the ELSE block. Have to deal with
5547 : BLOCK notes, if by no other means than backing out the merge if they
5548 : exist. Sticky enough I don't want to think about it now. */
5549 0 : next = then_bb;
5550 0 : if (else_bb && (next = next->next_bb) != else_bb)
5551 : return false;
5552 0 : if ((next = next->next_bb) != join_bb
5553 0 : && join_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5554 : {
5555 0 : if (else_bb)
5556 : join_bb = NULL;
5557 : else
5558 : return false;
5559 : }
5560 :
5561 : /* Do the real work. */
5562 :
5563 0 : ce_info->else_bb = else_bb;
5564 0 : ce_info->join_bb = join_bb;
5565 :
5566 : /* If we have && and || tests, try to first handle combining the && and ||
5567 : tests into the conditional code, and if that fails, go back and handle
5568 : it without the && and ||, which at present handles the && case if there
5569 : was no ELSE block. */
5570 0 : if (cond_exec_process_if_block (ce_info, true))
5571 : return true;
5572 :
5573 0 : if (ce_info->num_multiple_test_blocks)
5574 : {
5575 0 : cancel_changes (0);
5576 :
5577 0 : if (cond_exec_process_if_block (ce_info, false))
5578 : return true;
5579 : }
5580 :
5581 : return false;
5582 : }
5583 :
5584 : /* Convert a branch over a trap, or a branch
5585 : to a trap, into a conditional trap. */
5586 :
5587 : static bool
5588 0 : find_cond_trap (basic_block test_bb, edge then_edge, edge else_edge)
5589 : {
5590 0 : basic_block then_bb = then_edge->dest;
5591 0 : basic_block else_bb = else_edge->dest;
5592 0 : basic_block other_bb, trap_bb;
5593 0 : rtx_insn *trap, *jump;
5594 0 : rtx cond;
5595 0 : rtx_insn *cond_earliest;
5596 :
5597 : /* Locate the block with the trap instruction. */
5598 : /* ??? While we look for no successors, we really ought to allow
5599 : EH successors. Need to fix merge_if_block for that to work. */
5600 0 : if ((trap = block_has_only_trap (then_bb)) != NULL)
5601 : trap_bb = then_bb, other_bb = else_bb;
5602 0 : else if ((trap = block_has_only_trap (else_bb)) != NULL)
5603 : trap_bb = else_bb, other_bb = then_bb;
5604 : else
5605 : return false;
5606 :
5607 0 : if (dump_file)
5608 : {
5609 0 : fprintf (dump_file, "\nTRAP-IF block found, start %d, trap %d\n",
5610 : test_bb->index, trap_bb->index);
5611 : }
5612 :
5613 : /* If this is not a standard conditional jump, we can't parse it. */
5614 0 : jump = BB_END (test_bb);
5615 0 : cond = noce_get_condition (jump, &cond_earliest, then_bb == trap_bb);
5616 0 : if (! cond)
5617 : return false;
5618 :
5619 : /* If the conditional jump is more than just a conditional jump, then
5620 : we cannot do if-conversion on this block. Give up for returnjump_p,
5621 : changing a conditional return followed by unconditional trap for
5622 : conditional trap followed by unconditional return is likely not
5623 : beneficial and harder to handle. */
5624 0 : if (! onlyjump_p (jump) || returnjump_p (jump))
5625 0 : return false;
5626 :
5627 : /* We must be comparing objects whose modes imply the size. */
5628 0 : if (GET_MODE (XEXP (cond, 0)) == BLKmode)
5629 : return false;
5630 :
5631 : /* Attempt to generate the conditional trap. */
5632 0 : rtx_insn *seq = gen_cond_trap (GET_CODE (cond), copy_rtx (XEXP (cond, 0)),
5633 : copy_rtx (XEXP (cond, 1)),
5634 0 : TRAP_CODE (PATTERN (trap)));
5635 0 : if (seq == NULL)
5636 : return false;
5637 :
5638 : /* If that results in an invalid insn, back out. */
5639 0 : for (rtx_insn *x = seq; x; x = NEXT_INSN (x))
5640 0 : if (reload_completed
5641 0 : ? !valid_insn_p (x)
5642 0 : : recog_memoized (x) < 0)
5643 : return false;
5644 :
5645 : /* Emit the new insns before cond_earliest. */
5646 0 : emit_insn_before_setloc (seq, cond_earliest, INSN_LOCATION (trap));
5647 :
5648 : /* Delete the trap block if possible. */
5649 0 : remove_edge (trap_bb == then_bb ? then_edge : else_edge);
5650 0 : df_set_bb_dirty (test_bb);
5651 0 : df_set_bb_dirty (then_bb);
5652 0 : df_set_bb_dirty (else_bb);
5653 :
5654 0 : if (EDGE_COUNT (trap_bb->preds) == 0)
5655 : {
5656 0 : delete_basic_block (trap_bb);
5657 0 : num_true_changes++;
5658 : }
5659 :
5660 : /* Wire together the blocks again. */
5661 0 : if (current_ir_type () == IR_RTL_CFGLAYOUT)
5662 0 : single_succ_edge (test_bb)->flags |= EDGE_FALLTHRU;
5663 0 : else if (trap_bb == then_bb)
5664 : {
5665 0 : rtx lab = JUMP_LABEL (jump);
5666 0 : rtx_insn *seq = targetm.gen_jump (lab);
5667 0 : rtx_jump_insn *newjump = emit_jump_insn_after (seq, jump);
5668 0 : LABEL_NUSES (lab) += 1;
5669 0 : JUMP_LABEL (newjump) = lab;
5670 0 : emit_barrier_after (newjump);
5671 : }
5672 0 : delete_insn (jump);
5673 :
5674 0 : if (can_merge_blocks_p (test_bb, other_bb))
5675 : {
5676 0 : merge_blocks (test_bb, other_bb);
5677 0 : num_true_changes++;
5678 : }
5679 :
5680 0 : num_updated_if_blocks++;
5681 0 : return true;
5682 : }
5683 :
5684 : /* Subroutine of find_cond_trap: if BB contains only a trap insn,
5685 : return it. */
5686 :
5687 : static rtx_insn *
5688 0 : block_has_only_trap (basic_block bb)
5689 : {
5690 0 : rtx_insn *trap;
5691 :
5692 : /* We're not the exit block. */
5693 0 : if (bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
5694 : return NULL;
5695 :
5696 : /* The block must have no successors. */
5697 0 : if (EDGE_COUNT (bb->succs) > 0)
5698 : return NULL;
5699 :
5700 : /* The only instruction in the THEN block must be the trap. */
5701 0 : trap = first_active_insn (bb);
5702 0 : if (! (trap == BB_END (bb)
5703 0 : && GET_CODE (PATTERN (trap)) == TRAP_IF
5704 0 : && TRAP_CONDITION (PATTERN (trap)) == const_true_rtx))
5705 0 : return NULL;
5706 :
5707 : return trap;
5708 : }
5709 :
5710 : /* Look for IF-THEN-ELSE cases in which one of THEN or ELSE is
5711 : transformable, but not necessarily the other. There need be no
5712 : JOIN block.
5713 :
5714 : Return TRUE if we were successful at converting the block.
5715 :
5716 : Cases we'd like to look at:
5717 :
5718 : (1)
5719 : if (test) goto over; // x not live
5720 : x = a;
5721 : goto label;
5722 : over:
5723 :
5724 : becomes
5725 :
5726 : x = a;
5727 : if (! test) goto label;
5728 :
5729 : (2)
5730 : if (test) goto E; // x not live
5731 : x = big();
5732 : goto L;
5733 : E:
5734 : x = b;
5735 : goto M;
5736 :
5737 : becomes
5738 :
5739 : x = b;
5740 : if (test) goto M;
5741 : x = big();
5742 : goto L;
5743 :
5744 : (3) // This one's really only interesting for targets that can do
5745 : // multiway branching, e.g. IA-64 BBB bundles. For other targets
5746 : // it results in multiple branches on a cache line, which often
5747 : // does not sit well with predictors.
5748 :
5749 : if (test1) goto E; // predicted not taken
5750 : x = a;
5751 : if (test2) goto F;
5752 : ...
5753 : E:
5754 : x = b;
5755 : J:
5756 :
5757 : becomes
5758 :
5759 : x = a;
5760 : if (test1) goto E;
5761 : if (test2) goto F;
5762 :
5763 : Notes:
5764 :
5765 : (A) Don't do (2) if the branch is predicted against the block we're
5766 : eliminating. Do it anyway if we can eliminate a branch; this requires
5767 : that the sole successor of the eliminated block postdominate the other
5768 : side of the if.
5769 :
5770 : (B) With CE, on (3) we can steal from both sides of the if, creating
5771 :
5772 : if (test1) x = a;
5773 : if (!test1) x = b;
5774 : if (test1) goto J;
5775 : if (test2) goto F;
5776 : ...
5777 : J:
5778 :
5779 : Again, this is most useful if J postdominates.
5780 :
5781 : (C) CE substitutes for helpful life information.
5782 :
5783 : (D) These heuristics need a lot of work. */
5784 :
5785 : /* Tests for case 1 above. */
5786 :
5787 : static bool
5788 13931907 : find_if_case_1 (basic_block test_bb, edge then_edge, edge else_edge)
5789 : {
5790 13931907 : basic_block then_bb = then_edge->dest;
5791 13931907 : basic_block else_bb = else_edge->dest;
5792 13931907 : basic_block new_bb;
5793 13931907 : int then_bb_index;
5794 13931907 : profile_probability then_prob;
5795 13931907 : rtx else_target = NULL_RTX;
5796 :
5797 : /* If we are partitioning hot/cold basic blocks, we don't want to
5798 : mess up unconditional or indirect jumps that cross between hot
5799 : and cold sections.
5800 :
5801 : Basic block partitioning may result in some jumps that appear to
5802 : be optimizable (or blocks that appear to be mergeable), but which really
5803 : must be left untouched (they are required to make it safely across
5804 : partition boundaries). See the comments at the top of
5805 : bb-reorder.cc:partition_hot_cold_basic_blocks for complete details. */
5806 :
5807 13931907 : if ((BB_END (then_bb)
5808 13931907 : && JUMP_P (BB_END (then_bb))
5809 7202048 : && CROSSING_JUMP_P (BB_END (then_bb)))
5810 13697914 : || (JUMP_P (BB_END (test_bb))
5811 13697914 : && CROSSING_JUMP_P (BB_END (test_bb)))
5812 27521589 : || (BB_END (else_bb)
5813 13589682 : && JUMP_P (BB_END (else_bb))
5814 6921422 : && CROSSING_JUMP_P (BB_END (else_bb))))
5815 : return false;
5816 :
5817 : /* Verify test_bb ends in a conditional jump with no other side-effects. */
5818 13561565 : if (!onlyjump_p (BB_END (test_bb)))
5819 : return false;
5820 :
5821 : /* THEN has one successor. */
5822 20135251 : if (!single_succ_p (then_bb))
5823 : return false;
5824 :
5825 : /* THEN does not fall through, but is not strange either. */
5826 6223928 : if (single_succ_edge (then_bb)->flags & (EDGE_COMPLEX | EDGE_FALLTHRU))
5827 : return false;
5828 :
5829 : /* THEN has one predecessor. */
5830 15017411 : if (!single_pred_p (then_bb))
5831 : return false;
5832 :
5833 : /* THEN must do something. */
5834 1106088 : if (forwarder_block_p (then_bb))
5835 : return false;
5836 :
5837 704279 : num_possible_if_blocks++;
5838 704279 : if (dump_file)
5839 8 : fprintf (dump_file,
5840 : "\nIF-CASE-1 found, start %d, then %d\n",
5841 : test_bb->index, then_bb->index);
5842 :
5843 704279 : then_prob = then_edge->probability.invert ();
5844 :
5845 : /* We're speculating from the THEN path, we want to make sure the cost
5846 : of speculation is within reason. */
5847 1333343 : if (! cheap_bb_rtx_cost_p (then_bb, then_prob,
5848 1333343 : COSTS_N_INSNS (BRANCH_COST (optimize_bb_for_speed_p (then_edge->src),
5849 : predictable_edge_p (then_edge)))))
5850 : return false;
5851 :
5852 140064 : if (else_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
5853 : {
5854 0 : rtx_insn *jump = BB_END (else_edge->src);
5855 0 : gcc_assert (JUMP_P (jump));
5856 0 : else_target = JUMP_LABEL (jump);
5857 : }
5858 :
5859 : /* Registers set are dead, or are predicable. */
5860 140064 : if (! dead_or_predicable (test_bb, then_bb, else_bb,
5861 : single_succ_edge (then_bb), true))
5862 : return false;
5863 :
5864 : /* Conversion went ok, including moving the insns and fixing up the
5865 : jump. Adjust the CFG to match. */
5866 :
5867 : /* We can avoid creating a new basic block if then_bb is immediately
5868 : followed by else_bb, i.e. deleting then_bb allows test_bb to fall
5869 : through to else_bb. */
5870 :
5871 20584 : if (then_bb->next_bb == else_bb
5872 9934 : && then_bb->prev_bb == test_bb
5873 9934 : && else_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
5874 : {
5875 9934 : redirect_edge_succ (FALLTHRU_EDGE (test_bb), else_bb);
5876 9934 : new_bb = 0;
5877 : }
5878 10650 : else if (else_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
5879 0 : new_bb = force_nonfallthru_and_redirect (FALLTHRU_EDGE (test_bb),
5880 : else_bb, else_target);
5881 : else
5882 10650 : new_bb = redirect_edge_and_branch_force (FALLTHRU_EDGE (test_bb),
5883 : else_bb);
5884 :
5885 20584 : df_set_bb_dirty (test_bb);
5886 20584 : df_set_bb_dirty (else_bb);
5887 :
5888 20584 : then_bb_index = then_bb->index;
5889 20584 : delete_basic_block (then_bb);
5890 :
5891 : /* Make rest of code believe that the newly created block is the THEN_BB
5892 : block we removed. */
5893 20584 : if (new_bb)
5894 : {
5895 10650 : df_bb_replace (then_bb_index, new_bb);
5896 : /* This should have been done above via force_nonfallthru_and_redirect
5897 : (possibly called from redirect_edge_and_branch_force). */
5898 10650 : gcc_checking_assert (BB_PARTITION (new_bb) == BB_PARTITION (test_bb));
5899 : }
5900 :
5901 20584 : num_true_changes++;
5902 20584 : num_updated_if_blocks++;
5903 20584 : return true;
5904 : }
5905 :
5906 : /* Test for case 2 above. */
5907 :
5908 : static bool
5909 13911323 : find_if_case_2 (basic_block test_bb, edge then_edge, edge else_edge)
5910 : {
5911 13911323 : basic_block then_bb = then_edge->dest;
5912 13911323 : basic_block else_bb = else_edge->dest;
5913 13911323 : edge else_succ;
5914 13911323 : profile_probability then_prob, else_prob;
5915 :
5916 : /* We do not want to speculate (empty) loop latches. */
5917 13911323 : if (current_loops
5918 5402900 : && else_bb->loop_father->latch == else_bb)
5919 : return false;
5920 :
5921 : /* If we are partitioning hot/cold basic blocks, we don't want to
5922 : mess up unconditional or indirect jumps that cross between hot
5923 : and cold sections.
5924 :
5925 : Basic block partitioning may result in some jumps that appear to
5926 : be optimizable (or blocks that appear to be mergeable), but which really
5927 : must be left untouched (they are required to make it safely across
5928 : partition boundaries). See the comments at the top of
5929 : bb-reorder.cc:partition_hot_cold_basic_blocks for complete details. */
5930 :
5931 13759701 : if ((BB_END (then_bb)
5932 13759701 : && JUMP_P (BB_END (then_bb))
5933 7107875 : && CROSSING_JUMP_P (BB_END (then_bb)))
5934 13525708 : || (JUMP_P (BB_END (test_bb))
5935 13525708 : && CROSSING_JUMP_P (BB_END (test_bb)))
5936 27177177 : || (BB_END (else_bb)
5937 13417476 : && JUMP_P (BB_END (else_bb))
5938 6780255 : && CROSSING_JUMP_P (BB_END (else_bb))))
5939 : return false;
5940 :
5941 : /* Verify test_bb ends in a conditional jump with no other side-effects. */
5942 13389359 : if (!onlyjump_p (BB_END (test_bb)))
5943 : return false;
5944 :
5945 : /* ELSE has one successor. */
5946 19369740 : if (!single_succ_p (else_bb))
5947 : return false;
5948 : else
5949 5581080 : else_succ = single_succ_edge (else_bb);
5950 :
5951 : /* ELSE outgoing edge is not complex. */
5952 5581080 : if (else_succ->flags & EDGE_COMPLEX)
5953 : return false;
5954 :
5955 : /* ELSE has one predecessor. */
5956 16555042 : if (!single_pred_p (else_bb))
5957 : return false;
5958 :
5959 : /* THEN is not EXIT. */
5960 2766382 : if (then_bb->index < NUM_FIXED_BLOCKS)
5961 : return false;
5962 :
5963 2766382 : else_prob = else_edge->probability;
5964 2766382 : then_prob = else_prob.invert ();
5965 :
5966 : /* ELSE is predicted or SUCC(ELSE) postdominates THEN. */
5967 2766382 : if (else_prob > then_prob)
5968 : ;
5969 1843642 : else if (else_succ->dest->index < NUM_FIXED_BLOCKS
5970 1843642 : || dominated_by_p (CDI_POST_DOMINATORS, then_bb,
5971 : else_succ->dest))
5972 : ;
5973 : else
5974 : return false;
5975 :
5976 2174685 : num_possible_if_blocks++;
5977 2174685 : if (dump_file)
5978 44 : fprintf (dump_file,
5979 : "\nIF-CASE-2 found, start %d, else %d\n",
5980 : test_bb->index, else_bb->index);
5981 :
5982 : /* We're speculating from the ELSE path, we want to make sure the cost
5983 : of speculation is within reason. */
5984 4184288 : if (! cheap_bb_rtx_cost_p (else_bb, else_prob,
5985 4184288 : COSTS_N_INSNS (BRANCH_COST (optimize_bb_for_speed_p (else_edge->src),
5986 : predictable_edge_p (else_edge)))))
5987 : return false;
5988 :
5989 : /* Registers set are dead, or are predicable. */
5990 421394 : if (! dead_or_predicable (test_bb, else_bb, then_bb, else_succ, false))
5991 : return false;
5992 :
5993 : /* Conversion went ok, including moving the insns and fixing up the
5994 : jump. Adjust the CFG to match. */
5995 :
5996 122663 : df_set_bb_dirty (test_bb);
5997 122663 : df_set_bb_dirty (then_bb);
5998 122663 : delete_basic_block (else_bb);
5999 :
6000 122663 : num_true_changes++;
6001 122663 : num_updated_if_blocks++;
6002 :
6003 : /* ??? We may now fallthru from one of THEN's successors into a join
6004 : block. Rerun cleanup_cfg? Examine things manually? Wait? */
6005 :
6006 122663 : return true;
6007 : }
6008 :
6009 : /* Used by the code above to perform the actual rtl transformations.
6010 : Return TRUE if successful.
6011 :
6012 : TEST_BB is the block containing the conditional branch. MERGE_BB
6013 : is the block containing the code to manipulate. DEST_EDGE is an
6014 : edge representing a jump to the join block; after the conversion,
6015 : TEST_BB should be branching to its destination.
6016 : REVERSEP is true if the sense of the branch should be reversed. */
6017 :
6018 : static bool
6019 561458 : dead_or_predicable (basic_block test_bb, basic_block merge_bb,
6020 : basic_block other_bb, edge dest_edge, bool reversep)
6021 : {
6022 561458 : basic_block new_dest = dest_edge->dest;
6023 561458 : rtx_insn *head, *end, *jump;
6024 561458 : rtx_insn *earliest = NULL;
6025 561458 : rtx old_dest;
6026 561458 : bitmap merge_set = NULL;
6027 : /* Number of pending changes. */
6028 561458 : int n_validated_changes = 0;
6029 561458 : rtx new_dest_label = NULL_RTX;
6030 :
6031 561458 : jump = BB_END (test_bb);
6032 :
6033 : /* Find the extent of the real code in the merge block. */
6034 561458 : head = BB_HEAD (merge_bb);
6035 561458 : end = BB_END (merge_bb);
6036 :
6037 702945 : while (DEBUG_INSN_P (end) && end != head)
6038 141487 : end = PREV_INSN (end);
6039 :
6040 : /* If merge_bb ends with a tablejump, predicating/moving insn's
6041 : into test_bb and then deleting merge_bb will result in the jumptable
6042 : that follows merge_bb being removed along with merge_bb and then we
6043 : get an unresolved reference to the jumptable. */
6044 561458 : if (tablejump_p (end, NULL, NULL))
6045 : return false;
6046 :
6047 561458 : if (LABEL_P (head))
6048 421455 : head = NEXT_INSN (head);
6049 561458 : while (DEBUG_INSN_P (head) && head != end)
6050 0 : head = NEXT_INSN (head);
6051 561458 : if (NOTE_P (head))
6052 : {
6053 561458 : if (head == end)
6054 : {
6055 101841 : head = end = NULL;
6056 101841 : goto no_body;
6057 : }
6058 459617 : head = NEXT_INSN (head);
6059 1259931 : while (DEBUG_INSN_P (head) && head != end)
6060 340697 : head = NEXT_INSN (head);
6061 : }
6062 :
6063 459617 : if (JUMP_P (end))
6064 : {
6065 193677 : if (!onlyjump_p (end))
6066 : return false;
6067 153526 : if (head == end)
6068 : {
6069 6 : head = end = NULL;
6070 6 : goto no_body;
6071 : }
6072 153520 : end = PREV_INSN (end);
6073 393983 : while (DEBUG_INSN_P (end) && end != head)
6074 86943 : end = PREV_INSN (end);
6075 : }
6076 :
6077 : /* Don't move frame-related insn across the conditional branch. This
6078 : can lead to one of the paths of the branch having wrong unwind info. */
6079 419460 : if (epilogue_completed)
6080 : {
6081 : rtx_insn *insn = head;
6082 70562 : while (1)
6083 : {
6084 265854 : if (INSN_P (insn) && RTX_FRAME_RELATED_P (insn))
6085 : return false;
6086 259240 : if (insn == end)
6087 : break;
6088 70562 : insn = NEXT_INSN (insn);
6089 70562 : }
6090 : }
6091 :
6092 : /* Disable handling dead code by conditional execution if the machine needs
6093 : to do anything funny with the tests, etc. */
6094 : #ifndef IFCVT_MODIFY_TESTS
6095 412846 : if (targetm.have_conditional_execution ())
6096 : {
6097 : /* In the conditional execution case, we have things easy. We know
6098 : the condition is reversible. We don't have to check life info
6099 : because we're going to conditionally execute the code anyway.
6100 : All that's left is making sure the insns involved can actually
6101 : be predicated. */
6102 :
6103 0 : rtx cond;
6104 :
6105 : /* If the conditional jump is more than just a conditional jump,
6106 : then we cannot do conditional execution conversion on this block. */
6107 0 : if (!onlyjump_p (jump))
6108 0 : goto nce;
6109 :
6110 0 : cond = cond_exec_get_condition (jump);
6111 0 : if (! cond)
6112 0 : goto nce;
6113 :
6114 0 : rtx note = find_reg_note (jump, REG_BR_PROB, NULL_RTX);
6115 0 : profile_probability prob_val
6116 0 : = (note ? profile_probability::from_reg_br_prob_note (XINT (note, 0))
6117 0 : : profile_probability::uninitialized ());
6118 :
6119 0 : if (reversep)
6120 : {
6121 0 : enum rtx_code rev = reversed_comparison_code (cond, jump);
6122 0 : if (rev == UNKNOWN)
6123 0 : return false;
6124 0 : cond = gen_rtx_fmt_ee (rev, GET_MODE (cond), XEXP (cond, 0),
6125 : XEXP (cond, 1));
6126 0 : prob_val = prob_val.invert ();
6127 : }
6128 :
6129 0 : if (cond_exec_process_insns (NULL, head, end, cond, prob_val, false)
6130 0 : && verify_changes (0))
6131 0 : n_validated_changes = num_validated_changes ();
6132 : else
6133 0 : cancel_changes (0);
6134 :
6135 0 : earliest = jump;
6136 : }
6137 412846 : nce:
6138 : #endif
6139 :
6140 : /* If we allocated new pseudos (e.g. in the conditional move
6141 : expander called from noce_emit_cmove), we must resize the
6142 : array first. */
6143 412846 : if (max_regno < max_reg_num ())
6144 80855 : max_regno = max_reg_num ();
6145 :
6146 : /* Try the NCE path if the CE path did not result in any changes. */
6147 412846 : if (n_validated_changes == 0)
6148 : {
6149 412846 : rtx cond;
6150 412846 : rtx_insn *insn;
6151 412846 : regset live;
6152 412846 : bool success;
6153 :
6154 : /* In the non-conditional execution case, we have to verify that there
6155 : are no trapping operations, no calls, no references to memory, and
6156 : that any registers modified are dead at the branch site. */
6157 :
6158 412846 : if (!any_condjump_p (jump))
6159 : return false;
6160 :
6161 : /* Find the extent of the conditional. */
6162 412846 : cond = noce_get_condition (jump, &earliest, false);
6163 412846 : if (!cond)
6164 : return false;
6165 :
6166 412041 : live = BITMAP_ALLOC (®_obstack);
6167 412041 : simulate_backwards_to_point (merge_bb, live, end);
6168 412041 : success = can_move_insns_across (head, end, earliest, jump,
6169 : merge_bb, live,
6170 : df_get_live_in (other_bb), NULL);
6171 412041 : BITMAP_FREE (live);
6172 412041 : if (!success)
6173 : return false;
6174 :
6175 : /* Collect the set of registers set in MERGE_BB. */
6176 140356 : merge_set = BITMAP_ALLOC (®_obstack);
6177 :
6178 667698 : FOR_BB_INSNS (merge_bb, insn)
6179 527342 : if (NONDEBUG_INSN_P (insn))
6180 183320 : df_simulate_find_defs (insn, merge_set);
6181 :
6182 : /* If shrink-wrapping, disable this optimization when test_bb is
6183 : the first basic block and merge_bb exits. The idea is to not
6184 : move code setting up a return register as that may clobber a
6185 : register used to pass function parameters, which then must be
6186 : saved in caller-saved regs. A caller-saved reg requires the
6187 : prologue, killing a shrink-wrap opportunity. */
6188 140355 : if ((SHRINK_WRAPPING_ENABLED && !epilogue_completed)
6189 110882 : && ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb == test_bb
6190 18696 : && single_succ_p (new_dest)
6191 13782 : && single_succ (new_dest) == EXIT_BLOCK_PTR_FOR_FN (cfun)
6192 153183 : && bitmap_intersect_p (df_get_live_in (new_dest), merge_set))
6193 : {
6194 12827 : regset return_regs;
6195 12827 : unsigned int i;
6196 :
6197 12827 : return_regs = BITMAP_ALLOC (®_obstack);
6198 :
6199 : /* Start off with the intersection of regs used to pass
6200 : params and regs used to return values. */
6201 1205738 : for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
6202 1180084 : if (FUNCTION_ARG_REGNO_P (i)
6203 1180084 : && targetm.calls.function_value_regno_p (i))
6204 50130 : bitmap_set_bit (return_regs, INCOMING_REGNO (i));
6205 :
6206 12827 : bitmap_and_into (return_regs,
6207 12827 : df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
6208 12827 : bitmap_and_into (return_regs,
6209 12827 : df_get_live_in (EXIT_BLOCK_PTR_FOR_FN (cfun)));
6210 12827 : if (!bitmap_empty_p (return_regs))
6211 : {
6212 4872 : FOR_BB_INSNS_REVERSE (new_dest, insn)
6213 4319 : if (NONDEBUG_INSN_P (insn))
6214 : {
6215 2188 : df_ref def;
6216 :
6217 : /* If this insn sets any reg in return_regs, add all
6218 : reg uses to the set of regs we're interested in. */
6219 2773 : FOR_EACH_INSN_DEF (def, insn)
6220 1699 : if (bitmap_bit_p (return_regs, DF_REF_REGNO (def)))
6221 : {
6222 1114 : df_simulate_uses (insn, return_regs);
6223 1114 : break;
6224 : }
6225 : }
6226 553 : if (bitmap_intersect_p (merge_set, return_regs))
6227 : {
6228 535 : BITMAP_FREE (return_regs);
6229 535 : BITMAP_FREE (merge_set);
6230 535 : return false;
6231 : }
6232 : }
6233 12292 : BITMAP_FREE (return_regs);
6234 : }
6235 : }
6236 :
6237 241668 : no_body:
6238 : /* We don't want to use normal invert_jump or redirect_jump because
6239 : we don't want to delete_insn called. Also, we want to do our own
6240 : change group management. */
6241 :
6242 241668 : old_dest = JUMP_LABEL (jump);
6243 241668 : if (other_bb != new_dest)
6244 : {
6245 241649 : if (!any_condjump_p (jump))
6246 0 : goto cancel;
6247 :
6248 241649 : if (JUMP_P (BB_END (dest_edge->src)))
6249 21547 : new_dest_label = JUMP_LABEL (BB_END (dest_edge->src));
6250 220102 : else if (new_dest == EXIT_BLOCK_PTR_FOR_FN (cfun))
6251 98421 : new_dest_label = ret_rtx;
6252 : else
6253 121681 : new_dest_label = block_label (new_dest);
6254 :
6255 241649 : rtx_jump_insn *jump_insn = as_a <rtx_jump_insn *> (jump);
6256 241649 : if (reversep
6257 241649 : ? ! invert_jump_1 (jump_insn, new_dest_label)
6258 221065 : : ! redirect_jump_1 (jump_insn, new_dest_label))
6259 0 : goto cancel;
6260 : }
6261 :
6262 241668 : if (verify_changes (n_validated_changes))
6263 143247 : confirm_change_group ();
6264 : else
6265 98421 : goto cancel;
6266 :
6267 143247 : if (other_bb != new_dest)
6268 : {
6269 143228 : redirect_jump_2 (as_a <rtx_jump_insn *> (jump), old_dest, new_dest_label,
6270 : 0, reversep);
6271 :
6272 143228 : redirect_edge_succ (BRANCH_EDGE (test_bb), new_dest);
6273 143228 : if (reversep)
6274 : {
6275 20584 : std::swap (BRANCH_EDGE (test_bb)->probability,
6276 20584 : FALLTHRU_EDGE (test_bb)->probability);
6277 20584 : update_br_prob_note (test_bb);
6278 : }
6279 : }
6280 :
6281 : /* Move the insns out of MERGE_BB to before the branch. */
6282 143247 : if (head != NULL)
6283 : {
6284 139821 : rtx_insn *insn;
6285 :
6286 139821 : if (end == BB_END (merge_bb))
6287 107575 : BB_END (merge_bb) = PREV_INSN (head);
6288 :
6289 : /* PR 21767: when moving insns above a conditional branch, the REG_EQUAL
6290 : notes being moved might become invalid. */
6291 : insn = head;
6292 167764 : do
6293 : {
6294 167764 : rtx note;
6295 :
6296 167764 : if (! INSN_P (insn))
6297 3044 : continue;
6298 164720 : note = find_reg_note (insn, REG_EQUAL, NULL_RTX);
6299 164720 : if (! note)
6300 153939 : continue;
6301 10781 : remove_note (insn, note);
6302 195707 : } while (insn != end && (insn = NEXT_INSN (insn)));
6303 :
6304 : /* PR46315: when moving insns above a conditional branch, the REG_EQUAL
6305 : notes referring to the registers being set might become invalid. */
6306 139821 : if (merge_set)
6307 : {
6308 139821 : unsigned i;
6309 139821 : bitmap_iterator bi;
6310 :
6311 315270 : EXECUTE_IF_SET_IN_BITMAP (merge_set, 0, i, bi)
6312 175449 : remove_reg_equal_equiv_notes_for_regno (i);
6313 :
6314 139821 : BITMAP_FREE (merge_set);
6315 : }
6316 :
6317 139821 : reorder_insns (head, end, PREV_INSN (earliest));
6318 : }
6319 :
6320 : /* Remove the jump and edge if we can. */
6321 143247 : if (other_bb == new_dest)
6322 : {
6323 19 : delete_insn (jump);
6324 19 : remove_edge (BRANCH_EDGE (test_bb));
6325 : /* ??? Can't merge blocks here, as then_bb is still in use.
6326 : At minimum, the merge will get done just before bb-reorder. */
6327 : }
6328 :
6329 : return true;
6330 :
6331 98421 : cancel:
6332 98421 : cancel_changes (0);
6333 :
6334 98421 : if (merge_set)
6335 0 : BITMAP_FREE (merge_set);
6336 :
6337 : return false;
6338 : }
6339 :
6340 : /* Main entry point for all if-conversion. AFTER_COMBINE is true if
6341 : we are after combine pass. */
6342 :
6343 : static void
6344 3118042 : if_convert (bool after_combine)
6345 : {
6346 3118042 : basic_block bb;
6347 3118042 : int pass;
6348 :
6349 3118042 : if (optimize == 1)
6350 : {
6351 234214 : df_live_add_problem ();
6352 234214 : df_live_set_all_dirty ();
6353 : }
6354 :
6355 : /* Record whether we are after combine pass. */
6356 3118042 : ifcvt_after_combine = after_combine;
6357 3118042 : have_cbranchcc4 = (direct_optab_handler (cbranch_optab, CCmode)
6358 3118042 : != CODE_FOR_nothing);
6359 3118042 : num_possible_if_blocks = 0;
6360 3118042 : num_updated_if_blocks = 0;
6361 3118042 : num_true_changes = 0;
6362 :
6363 3118042 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
6364 3118042 : mark_loop_exit_edges ();
6365 3118042 : loop_optimizer_finalize ();
6366 3118042 : free_dominance_info (CDI_DOMINATORS);
6367 :
6368 : /* Compute postdominators. */
6369 3118042 : calculate_dominance_info (CDI_POST_DOMINATORS);
6370 :
6371 3118042 : df_set_flags (DF_LR_RUN_DCE);
6372 :
6373 : /* Go through each of the basic blocks looking for things to convert. If we
6374 : have conditional execution, we make multiple passes to allow us to handle
6375 : IF-THEN{-ELSE} blocks within other IF-THEN{-ELSE} blocks. */
6376 3118042 : pass = 0;
6377 3277486 : do
6378 : {
6379 3277486 : df_analyze ();
6380 : /* Only need to do dce on the first pass. */
6381 3277486 : df_clear_flags (DF_LR_RUN_DCE);
6382 3277486 : cond_exec_changed_p = false;
6383 3277486 : pass++;
6384 :
6385 : #ifdef IFCVT_MULTIPLE_DUMPS
6386 3277486 : if (dump_file && pass > 1)
6387 19 : fprintf (dump_file, "\n\n========== Pass %d ==========\n", pass);
6388 : #endif
6389 :
6390 41968249 : FOR_EACH_BB_FN (bb, cfun)
6391 : {
6392 : basic_block new_bb;
6393 38991906 : while (!df_get_bb_dirty (bb)
6394 77221894 : && (new_bb = find_if_header (bb, pass)) != NULL)
6395 : bb = new_bb;
6396 : }
6397 :
6398 : #ifdef IFCVT_MULTIPLE_DUMPS
6399 3277486 : if (dump_file && cond_exec_changed_p)
6400 19 : print_rtl_with_bb (dump_file, get_insns (), dump_flags);
6401 : #endif
6402 : }
6403 : while (cond_exec_changed_p);
6404 :
6405 : #ifdef IFCVT_MULTIPLE_DUMPS
6406 3118042 : if (dump_file)
6407 85 : fprintf (dump_file, "\n\n========== no more changes\n");
6408 : #endif
6409 :
6410 3118042 : free_dominance_info (CDI_POST_DOMINATORS);
6411 :
6412 3118042 : if (dump_file)
6413 85 : fflush (dump_file);
6414 :
6415 3118042 : clear_aux_for_blocks ();
6416 :
6417 : /* If we allocated new pseudos, we must resize the array for sched1. */
6418 3118042 : if (max_regno < max_reg_num ())
6419 630286 : max_regno = max_reg_num ();
6420 :
6421 : /* Write the final stats. */
6422 3118042 : if (dump_file && num_possible_if_blocks > 0)
6423 : {
6424 59 : fprintf (dump_file,
6425 : "\n%d possible IF blocks searched.\n",
6426 : num_possible_if_blocks);
6427 59 : fprintf (dump_file,
6428 : "%d IF blocks converted.\n",
6429 : num_updated_if_blocks);
6430 59 : fprintf (dump_file,
6431 : "%d true changes made.\n\n\n",
6432 : num_true_changes);
6433 : }
6434 :
6435 3118042 : if (optimize == 1)
6436 234214 : df_remove_problem (df_live);
6437 :
6438 : /* Some non-cold blocks may now be only reachable from cold blocks.
6439 : Fix that up. */
6440 3118042 : fixup_partitions ();
6441 :
6442 3118042 : checking_verify_flow_info ();
6443 3118042 : }
6444 :
6445 : /* If-conversion and CFG cleanup. */
6446 : static void
6447 1041770 : rest_of_handle_if_conversion (void)
6448 : {
6449 1041770 : int flags = 0;
6450 :
6451 1041770 : if (flag_if_conversion)
6452 : {
6453 1039343 : if (dump_file)
6454 : {
6455 33 : dump_reg_info (dump_file);
6456 33 : dump_flow_info (dump_file, dump_flags);
6457 : }
6458 1039343 : cleanup_cfg (CLEANUP_EXPENSIVE);
6459 1039343 : if_convert (false);
6460 1039343 : if (num_updated_if_blocks)
6461 : /* Get rid of any dead CC-related instructions. */
6462 1041770 : flags |= CLEANUP_FORCE_FAST_DCE;
6463 : }
6464 :
6465 1041770 : cleanup_cfg (flags);
6466 1041770 : }
6467 :
6468 : namespace {
6469 :
6470 : const pass_data pass_data_rtl_ifcvt =
6471 : {
6472 : RTL_PASS, /* type */
6473 : "ce1", /* name */
6474 : OPTGROUP_NONE, /* optinfo_flags */
6475 : TV_IFCVT, /* tv_id */
6476 : 0, /* properties_required */
6477 : 0, /* properties_provided */
6478 : 0, /* properties_destroyed */
6479 : 0, /* todo_flags_start */
6480 : TODO_df_finish, /* todo_flags_finish */
6481 : };
6482 :
6483 : class pass_rtl_ifcvt : public rtl_opt_pass
6484 : {
6485 : public:
6486 298828 : pass_rtl_ifcvt (gcc::context *ctxt)
6487 597656 : : rtl_opt_pass (pass_data_rtl_ifcvt, ctxt)
6488 : {}
6489 :
6490 : /* opt_pass methods: */
6491 1488378 : bool gate (function *) final override
6492 : {
6493 1488378 : return (optimize > 0) && dbg_cnt (if_conversion);
6494 : }
6495 :
6496 1041770 : unsigned int execute (function *) final override
6497 : {
6498 1041770 : rest_of_handle_if_conversion ();
6499 1041770 : return 0;
6500 : }
6501 :
6502 : }; // class pass_rtl_ifcvt
6503 :
6504 : } // anon namespace
6505 :
6506 : rtl_opt_pass *
6507 298828 : make_pass_rtl_ifcvt (gcc::context *ctxt)
6508 : {
6509 298828 : return new pass_rtl_ifcvt (ctxt);
6510 : }
6511 :
6512 :
6513 : /* Rerun if-conversion, as combine may have simplified things enough
6514 : to now meet sequence length restrictions. */
6515 :
6516 : namespace {
6517 :
6518 : const pass_data pass_data_if_after_combine =
6519 : {
6520 : RTL_PASS, /* type */
6521 : "ce2", /* name */
6522 : OPTGROUP_NONE, /* optinfo_flags */
6523 : TV_IFCVT, /* tv_id */
6524 : 0, /* properties_required */
6525 : 0, /* properties_provided */
6526 : 0, /* properties_destroyed */
6527 : 0, /* todo_flags_start */
6528 : TODO_df_finish, /* todo_flags_finish */
6529 : };
6530 :
6531 : class pass_if_after_combine : public rtl_opt_pass
6532 : {
6533 : public:
6534 298828 : pass_if_after_combine (gcc::context *ctxt)
6535 597656 : : rtl_opt_pass (pass_data_if_after_combine, ctxt)
6536 : {}
6537 :
6538 : /* opt_pass methods: */
6539 1488378 : bool gate (function *) final override
6540 : {
6541 1041771 : return optimize > 0 && flag_if_conversion
6542 2527722 : && dbg_cnt (if_after_combine);
6543 : }
6544 :
6545 1039343 : unsigned int execute (function *) final override
6546 : {
6547 1039343 : if_convert (true);
6548 1039343 : return 0;
6549 : }
6550 :
6551 : }; // class pass_if_after_combine
6552 :
6553 : } // anon namespace
6554 :
6555 : rtl_opt_pass *
6556 298828 : make_pass_if_after_combine (gcc::context *ctxt)
6557 : {
6558 298828 : return new pass_if_after_combine (ctxt);
6559 : }
6560 :
6561 :
6562 : namespace {
6563 :
6564 : const pass_data pass_data_if_after_reload =
6565 : {
6566 : RTL_PASS, /* type */
6567 : "ce3", /* name */
6568 : OPTGROUP_NONE, /* optinfo_flags */
6569 : TV_IFCVT2, /* tv_id */
6570 : 0, /* properties_required */
6571 : 0, /* properties_provided */
6572 : 0, /* properties_destroyed */
6573 : 0, /* todo_flags_start */
6574 : TODO_df_finish, /* todo_flags_finish */
6575 : };
6576 :
6577 : class pass_if_after_reload : public rtl_opt_pass
6578 : {
6579 : public:
6580 298828 : pass_if_after_reload (gcc::context *ctxt)
6581 597656 : : rtl_opt_pass (pass_data_if_after_reload, ctxt)
6582 : {}
6583 :
6584 : /* opt_pass methods: */
6585 1488378 : bool gate (function *) final override
6586 : {
6587 1041771 : return optimize > 0 && flag_if_conversion2
6588 2527735 : && dbg_cnt (if_after_reload);
6589 : }
6590 :
6591 1039356 : unsigned int execute (function *) final override
6592 : {
6593 1039356 : if_convert (true);
6594 1039356 : return 0;
6595 : }
6596 :
6597 : }; // class pass_if_after_reload
6598 :
6599 : } // anon namespace
6600 :
6601 : rtl_opt_pass *
6602 298828 : make_pass_if_after_reload (gcc::context *ctxt)
6603 : {
6604 298828 : return new pass_if_after_reload (ctxt);
6605 : }
|
