Line data Source code
1 : /* Reassociation for trees.
2 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
3 : Contributed by Daniel Berlin <dan@dberlin.org>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify
8 : it under the terms of the GNU General Public License as published by
9 : the Free Software Foundation; either version 3, or (at your option)
10 : any later version.
11 :
12 : GCC is distributed in the hope that it will be useful,
13 : but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 : GNU General Public License for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "target.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "gimple.h"
29 : #include "cfghooks.h"
30 : #include "alloc-pool.h"
31 : #include "tree-pass.h"
32 : #include "memmodel.h"
33 : #include "tm_p.h"
34 : #include "ssa.h"
35 : #include "optabs-tree.h"
36 : #include "gimple-pretty-print.h"
37 : #include "diagnostic-core.h"
38 : #include "fold-const.h"
39 : #include "stor-layout.h"
40 : #include "cfganal.h"
41 : #include "gimple-iterator.h"
42 : #include "gimple-fold.h"
43 : #include "tree-eh.h"
44 : #include "gimplify-me.h"
45 : #include "tree-cfg.h"
46 : #include "tree-ssa-loop.h"
47 : #include "flags.h"
48 : #include "tree-ssa.h"
49 : #include "langhooks.h"
50 : #include "cfgloop.h"
51 : #include "builtins.h"
52 : #include "gimplify.h"
53 : #include "case-cfn-macros.h"
54 : #include "tree-ssa-reassoc.h"
55 : #include "tree-ssa-math-opts.h"
56 : #include "gimple-range.h"
57 : #include "internal-fn.h"
58 :
59 : /* This is a simple global reassociation pass. It is, in part, based
60 : on the LLVM pass of the same name (They do some things more/less
61 : than we do, in different orders, etc).
62 :
63 : It consists of five steps:
64 :
65 : 1. Breaking up subtract operations into addition + negate, where
66 : it would promote the reassociation of adds.
67 :
68 : 2. Left linearization of the expression trees, so that (A+B)+(C+D)
69 : becomes (((A+B)+C)+D), which is easier for us to rewrite later.
70 : During linearization, we place the operands of the binary
71 : expressions into a vector of operand_entry_*
72 :
73 : 3. Optimization of the operand lists, eliminating things like a +
74 : -a, a & a, etc.
75 :
76 : 3a. Combine repeated factors with the same occurrence counts
77 : into a __builtin_powi call that will later be optimized into
78 : an optimal number of multiplies.
79 :
80 : 4. Rewrite the expression trees we linearized and optimized so
81 : they are in proper rank order.
82 :
83 : 5. Repropagate negates, as nothing else will clean it up ATM.
84 :
85 : A bit of theory on #4, since nobody seems to write anything down
86 : about why it makes sense to do it the way they do it:
87 :
88 : We could do this much nicer theoretically, but don't (for reasons
89 : explained after how to do it theoretically nice :P).
90 :
91 : In order to promote the most redundancy elimination, you want
92 : binary expressions whose operands are the same rank (or
93 : preferably, the same value) exposed to the redundancy eliminator,
94 : for possible elimination.
95 :
96 : So the way to do this if we really cared, is to build the new op
97 : tree from the leaves to the roots, merging as you go, and putting the
98 : new op on the end of the worklist, until you are left with one
99 : thing on the worklist.
100 :
101 : IE if you have to rewrite the following set of operands (listed with
102 : rank in parentheses), with opcode PLUS_EXPR:
103 :
104 : a (1), b (1), c (1), d (2), e (2)
105 :
106 :
107 : We start with our merge worklist empty, and the ops list with all of
108 : those on it.
109 :
110 : You want to first merge all leaves of the same rank, as much as
111 : possible.
112 :
113 : So first build a binary op of
114 :
115 : mergetmp = a + b, and put "mergetmp" on the merge worklist.
116 :
117 : Because there is no three operand form of PLUS_EXPR, c is not going to
118 : be exposed to redundancy elimination as a rank 1 operand.
119 :
120 : So you might as well throw it on the merge worklist (you could also
121 : consider it to now be a rank two operand, and merge it with d and e,
122 : but in this case, you then have evicted e from a binary op. So at
123 : least in this situation, you can't win.)
124 :
125 : Then build a binary op of d + e
126 : mergetmp2 = d + e
127 :
128 : and put mergetmp2 on the merge worklist.
129 :
130 : so merge worklist = {mergetmp, c, mergetmp2}
131 :
132 : Continue building binary ops of these operations until you have only
133 : one operation left on the worklist.
134 :
135 : So we have
136 :
137 : build binary op
138 : mergetmp3 = mergetmp + c
139 :
140 : worklist = {mergetmp2, mergetmp3}
141 :
142 : mergetmp4 = mergetmp2 + mergetmp3
143 :
144 : worklist = {mergetmp4}
145 :
146 : because we have one operation left, we can now just set the original
147 : statement equal to the result of that operation.
148 :
149 : This will at least expose a + b and d + e to redundancy elimination
150 : as binary operations.
151 :
152 : For extra points, you can reuse the old statements to build the
153 : mergetmps, since you shouldn't run out.
154 :
155 : So why don't we do this?
156 :
157 : Because it's expensive, and rarely will help. Most trees we are
158 : reassociating have 3 or less ops. If they have 2 ops, they already
159 : will be written into a nice single binary op. If you have 3 ops, a
160 : single simple check suffices to tell you whether the first two are of the
161 : same rank. If so, you know to order it
162 :
163 : mergetmp = op1 + op2
164 : newstmt = mergetmp + op3
165 :
166 : instead of
167 : mergetmp = op2 + op3
168 : newstmt = mergetmp + op1
169 :
170 : If all three are of the same rank, you can't expose them all in a
171 : single binary operator anyway, so the above is *still* the best you
172 : can do.
173 :
174 : Thus, this is what we do. When we have three ops left, we check to see
175 : what order to put them in, and call it a day. As a nod to vector sum
176 : reduction, we check if any of the ops are really a phi node that is a
177 : destructive update for the associating op, and keep the destructive
178 : update together for vector sum reduction recognition. */
179 :
180 : /* Enable insertion of __builtin_powi calls during execute_reassoc. See
181 : point 3a in the pass header comment. */
182 : static bool reassoc_insert_powi_p;
183 :
184 : /* Enable biasing ranks of loop accumulators. We don't want this before
185 : vectorization, since it interferes with reduction chains. */
186 : static bool reassoc_bias_loop_carried_phi_ranks_p;
187 :
188 : /* Statistics */
189 : static struct
190 : {
191 : int linearized;
192 : int constants_eliminated;
193 : int ops_eliminated;
194 : int rewritten;
195 : int pows_encountered;
196 : int pows_created;
197 : } reassociate_stats;
198 :
199 :
200 : static object_allocator<operand_entry> operand_entry_pool
201 : ("operand entry pool");
202 :
203 : /* This is used to assign a unique ID to each struct operand_entry
204 : so that qsort results are identical on different hosts. */
205 : static unsigned int next_operand_entry_id;
206 :
207 : /* Starting rank number for a given basic block, so that we can rank
208 : operations using unmovable instructions in that BB based on the bb
209 : depth. */
210 : static int64_t *bb_rank;
211 :
212 : /* Operand->rank hashtable. */
213 : static hash_map<tree, int64_t> *operand_rank;
214 :
215 : /* SSA_NAMEs that are forms of loop accumulators and whose ranks need to be
216 : biased. */
217 : static auto_bitmap biased_names;
218 :
219 : /* Vector of SSA_NAMEs on which after reassociate_bb is done with
220 : all basic blocks the CFG should be adjusted - basic blocks
221 : split right after that SSA_NAME's definition statement and before
222 : the only use, which must be a bit ior. */
223 : static vec<tree> reassoc_branch_fixups;
224 :
225 : /* Forward decls. */
226 : static int64_t get_rank (tree);
227 : static bool reassoc_stmt_dominates_stmt_p (gimple *, gimple *);
228 :
229 : /* Wrapper around gsi_remove, which adjusts gimple_uid of debug stmts
230 : possibly added by gsi_remove. */
231 :
232 : static bool
233 168832 : reassoc_remove_stmt (gimple_stmt_iterator *gsi)
234 : {
235 168832 : gimple *stmt = gsi_stmt (*gsi);
236 :
237 168832 : if (!MAY_HAVE_DEBUG_BIND_STMTS || gimple_code (stmt) == GIMPLE_PHI)
238 68701 : return gsi_remove (gsi, true);
239 :
240 100131 : gimple_stmt_iterator prev = *gsi;
241 100131 : gsi_prev (&prev);
242 100131 : unsigned uid = gimple_uid (stmt);
243 100131 : basic_block bb = gimple_bb (stmt);
244 100131 : bool ret = gsi_remove (gsi, true);
245 100131 : if (!gsi_end_p (prev))
246 99921 : gsi_next (&prev);
247 : else
248 420 : prev = gsi_start_bb (bb);
249 100131 : gimple *end_stmt = gsi_stmt (*gsi);
250 104808 : while ((stmt = gsi_stmt (prev)) != end_stmt)
251 : {
252 4677 : gcc_assert (stmt && is_gimple_debug (stmt) && gimple_uid (stmt) == 0);
253 4677 : gimple_set_uid (stmt, uid);
254 4677 : gsi_next (&prev);
255 : }
256 : return ret;
257 : }
258 :
259 : /* Bias amount for loop-carried phis. We want this to be larger than
260 : the depth of any reassociation tree we can see, but not larger than
261 : the rank difference between two blocks. */
262 : #define PHI_LOOP_BIAS (1 << 15)
263 :
264 : /* Return TRUE iff PHI_LOOP_BIAS should be propagated from one of the STMT's
265 : operands to the STMT's left-hand side. The goal is to preserve bias in code
266 : like this:
267 :
268 : x_1 = phi(x_0, x_2)
269 : a = x_1 | 1
270 : b = a ^ 2
271 : .MEM = b
272 : c = b + d
273 : x_2 = c + e
274 :
275 : That is, we need to preserve bias along single-use chains originating from
276 : loop-carried phis. Only GIMPLE_ASSIGNs to SSA_NAMEs are considered to be
277 : uses, because only they participate in rank propagation. */
278 : static bool
279 7156496 : propagate_bias_p (gimple *stmt)
280 : {
281 7156496 : use_operand_p use;
282 7156496 : imm_use_iterator use_iter;
283 7156496 : gimple *single_use_stmt = NULL;
284 :
285 7156496 : if (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt)) == tcc_reference)
286 : return false;
287 :
288 12790609 : FOR_EACH_IMM_USE_FAST (use, use_iter, gimple_assign_lhs (stmt))
289 : {
290 8140029 : gimple *current_use_stmt = USE_STMT (use);
291 :
292 8140029 : if (is_gimple_assign (current_use_stmt)
293 8140029 : && TREE_CODE (gimple_assign_lhs (current_use_stmt)) == SSA_NAME)
294 : {
295 6139872 : if (single_use_stmt != NULL && single_use_stmt != current_use_stmt)
296 736480 : return false;
297 : single_use_stmt = current_use_stmt;
298 : }
299 736480 : }
300 :
301 4650580 : if (single_use_stmt == NULL)
302 : return false;
303 :
304 4650159 : if (gimple_bb (stmt)->loop_father
305 4650159 : != gimple_bb (single_use_stmt)->loop_father)
306 116579 : return false;
307 :
308 : return true;
309 : }
310 :
311 : /* Rank assigned to a phi statement. If STMT is a loop-carried phi of
312 : an innermost loop, and the phi has only a single use which is inside
313 : the loop, then the rank is the block rank of the loop latch plus an
314 : extra bias for the loop-carried dependence. This causes expressions
315 : calculated into an accumulator variable to be independent for each
316 : iteration of the loop. If STMT is some other phi, the rank is the
317 : block rank of its containing block. */
318 : static int64_t
319 1491525 : phi_rank (gimple *stmt)
320 : {
321 1491525 : basic_block bb = gimple_bb (stmt);
322 1491525 : class loop *father = bb->loop_father;
323 1491525 : tree res;
324 1491525 : unsigned i;
325 1491525 : use_operand_p use;
326 1491525 : gimple *use_stmt;
327 :
328 1491525 : if (!reassoc_bias_loop_carried_phi_ranks_p)
329 567018 : return bb_rank[bb->index];
330 :
331 : /* We only care about real loops (those with a latch). */
332 924507 : if (!father->latch)
333 1 : return bb_rank[bb->index];
334 :
335 : /* Interesting phis must be in headers of innermost loops. */
336 924506 : if (bb != father->header
337 722068 : || father->inner)
338 368297 : return bb_rank[bb->index];
339 :
340 : /* Ignore virtual SSA_NAMEs. */
341 556209 : res = gimple_phi_result (stmt);
342 1112418 : if (virtual_operand_p (res))
343 0 : return bb_rank[bb->index];
344 :
345 : /* The phi definition must have a single use, and that use must be
346 : within the loop. Otherwise this isn't an accumulator pattern. */
347 556209 : if (!single_imm_use (res, &use, &use_stmt)
348 556209 : || gimple_bb (use_stmt)->loop_father != father)
349 484716 : return bb_rank[bb->index];
350 :
351 : /* Look for phi arguments from within the loop. If found, bias this phi. */
352 81494 : for (i = 0; i < gimple_phi_num_args (stmt); i++)
353 : {
354 81221 : tree arg = gimple_phi_arg_def (stmt, i);
355 81221 : if (TREE_CODE (arg) == SSA_NAME
356 81221 : && !SSA_NAME_IS_DEFAULT_DEF (arg))
357 : {
358 75855 : gimple *def_stmt = SSA_NAME_DEF_STMT (arg);
359 75855 : if (gimple_bb (def_stmt)->loop_father == father)
360 71220 : return bb_rank[father->latch->index] + PHI_LOOP_BIAS;
361 : }
362 : }
363 :
364 : /* Must be an uninteresting phi. */
365 273 : return bb_rank[bb->index];
366 : }
367 :
368 : /* Return the maximum of RANK and the rank that should be propagated
369 : from expression OP. For most operands, this is just the rank of OP.
370 : For loop-carried phis, the value is zero to avoid undoing the bias
371 : in favor of the phi. */
372 : static int64_t
373 7792331 : propagate_rank (int64_t rank, tree op, bool *maybe_biased_p)
374 : {
375 7792331 : int64_t op_rank;
376 :
377 7792331 : op_rank = get_rank (op);
378 :
379 : /* Check whether op is biased after the get_rank () call, since it might have
380 : updated biased_names. */
381 7792331 : if (TREE_CODE (op) == SSA_NAME
382 7792331 : && bitmap_bit_p (biased_names, SSA_NAME_VERSION (op)))
383 : {
384 44713 : if (maybe_biased_p == NULL)
385 : return rank;
386 30411 : *maybe_biased_p = true;
387 : }
388 :
389 7778029 : return MAX (rank, op_rank);
390 : }
391 :
392 : /* Look up the operand rank structure for expression E. */
393 :
394 : static inline int64_t
395 14188467 : find_operand_rank (tree e)
396 : {
397 14188467 : int64_t *slot = operand_rank->get (e);
398 14188467 : return slot ? *slot : -1;
399 : }
400 :
401 : /* Insert {E,RANK} into the operand rank hashtable. */
402 :
403 : static inline void
404 15210809 : insert_operand_rank (tree e, int64_t rank)
405 : {
406 15210809 : gcc_assert (rank > 0);
407 15210809 : bool existed = operand_rank->put (e, rank);
408 15210809 : gcc_assert (!existed);
409 15210809 : }
410 :
411 : /* Given an expression E, return the rank of the expression. */
412 :
413 : static int64_t
414 17670981 : get_rank (tree e)
415 : {
416 : /* SSA_NAME's have the rank of the expression they are the result
417 : of.
418 : For globals and uninitialized values, the rank is 0.
419 : For function arguments, use the pre-setup rank.
420 : For PHI nodes, stores, asm statements, etc, we use the rank of
421 : the BB.
422 : For simple operations, the rank is the maximum rank of any of
423 : its operands, or the bb_rank, whichever is less.
424 : I make no claims that this is optimal, however, it gives good
425 : results. */
426 :
427 : /* We make an exception to the normal ranking system to break
428 : dependences of accumulator variables in loops. Suppose we
429 : have a simple one-block loop containing:
430 :
431 : x_1 = phi(x_0, x_2)
432 : b = a + x_1
433 : c = b + d
434 : x_2 = c + e
435 :
436 : As shown, each iteration of the calculation into x is fully
437 : dependent upon the iteration before it. We would prefer to
438 : see this in the form:
439 :
440 : x_1 = phi(x_0, x_2)
441 : b = a + d
442 : c = b + e
443 : x_2 = c + x_1
444 :
445 : If the loop is unrolled, the calculations of b and c from
446 : different iterations can be interleaved.
447 :
448 : To obtain this result during reassociation, we bias the rank
449 : of the phi definition x_1 upward, when it is recognized as an
450 : accumulator pattern. The artificial rank causes it to be
451 : added last, providing the desired independence. */
452 :
453 17670981 : if (TREE_CODE (e) == SSA_NAME)
454 : {
455 14188467 : ssa_op_iter iter;
456 14188467 : gimple *stmt;
457 14188467 : int64_t rank;
458 14188467 : tree op;
459 :
460 : /* If we already have a rank for this expression, use that. */
461 14188467 : rank = find_operand_rank (e);
462 14188467 : if (rank != -1)
463 : return rank;
464 :
465 8901907 : stmt = SSA_NAME_DEF_STMT (e);
466 8901907 : if (gimple_code (stmt) == GIMPLE_PHI)
467 : {
468 1491525 : rank = phi_rank (stmt);
469 1491525 : if (rank != bb_rank[gimple_bb (stmt)->index])
470 71220 : bitmap_set_bit (biased_names, SSA_NAME_VERSION (e));
471 : }
472 :
473 7410382 : else if (!is_gimple_assign (stmt))
474 253886 : rank = bb_rank[gimple_bb (stmt)->index];
475 :
476 : else
477 : {
478 7156496 : bool biased_p = false;
479 7156496 : bool *maybe_biased_p = propagate_bias_p (stmt) ? &biased_p : NULL;
480 :
481 : /* Otherwise, find the maximum rank for the operands. As an
482 : exception, remove the bias from loop-carried phis when propagating
483 : the rank so that dependent operations are not also biased. */
484 : /* Simply walk over all SSA uses - this takes advantage of the
485 : fact that non-SSA operands are is_gimple_min_invariant and
486 : thus have rank 0. */
487 7156496 : rank = 0;
488 14948827 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
489 7792331 : rank = propagate_rank (rank, op, maybe_biased_p);
490 :
491 7156496 : rank += 1;
492 7156496 : if (biased_p)
493 29116 : bitmap_set_bit (biased_names, SSA_NAME_VERSION (e));
494 : }
495 :
496 8901907 : if (dump_file && (dump_flags & TDF_DETAILS))
497 : {
498 205 : fprintf (dump_file, "Rank for ");
499 205 : print_generic_expr (dump_file, e);
500 205 : fprintf (dump_file, " is %" PRId64 "\n", rank);
501 : }
502 :
503 : /* Note the rank in the hashtable so we don't recompute it. */
504 8901907 : insert_operand_rank (e, rank);
505 8901907 : return rank;
506 : }
507 :
508 : /* Constants, globals, etc., are rank 0 */
509 : return 0;
510 : }
511 :
512 :
513 : /* We want integer ones to end up last no matter what, since they are
514 : the ones we can do the most with. */
515 : #define INTEGER_CONST_TYPE 1 << 4
516 : #define FLOAT_ONE_CONST_TYPE 1 << 3
517 : #define FLOAT_CONST_TYPE 1 << 2
518 : #define OTHER_CONST_TYPE 1 << 1
519 :
520 : /* Classify an invariant tree into integer, float, or other, so that
521 : we can sort them to be near other constants of the same type. */
522 : static inline int
523 323342 : constant_type (tree t)
524 : {
525 323342 : if (INTEGRAL_TYPE_P (TREE_TYPE (t)))
526 : return INTEGER_CONST_TYPE;
527 10598 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (t)))
528 : {
529 : /* Sort -1.0 and 1.0 constants last, while in some cases
530 : const_binop can't optimize some inexact operations, multiplication
531 : by -1.0 or 1.0 can be always merged with others. */
532 6828 : if (real_onep (t) || real_minus_onep (t))
533 816 : return FLOAT_ONE_CONST_TYPE;
534 : return FLOAT_CONST_TYPE;
535 : }
536 : else
537 : return OTHER_CONST_TYPE;
538 : }
539 :
540 : /* qsort comparison function to sort operand entries PA and PB by rank
541 : so that the sorted array is ordered by rank in decreasing order. */
542 : static int
543 23694779 : sort_by_operand_rank (const void *pa, const void *pb)
544 : {
545 23694779 : const operand_entry *oea = *(const operand_entry *const *)pa;
546 23694779 : const operand_entry *oeb = *(const operand_entry *const *)pb;
547 :
548 23694779 : if (oeb->rank != oea->rank)
549 21271160 : return oeb->rank > oea->rank ? 1 : -1;
550 :
551 : /* It's nicer for optimize_expression if constants that are likely
552 : to fold when added/multiplied/whatever are put next to each
553 : other. Since all constants have rank 0, order them by type. */
554 2423619 : if (oea->rank == 0)
555 : {
556 161613 : if (constant_type (oeb->op) != constant_type (oea->op))
557 58 : return constant_type (oea->op) - constant_type (oeb->op);
558 : else
559 : /* To make sorting result stable, we use unique IDs to determine
560 : order. */
561 161555 : return oeb->id > oea->id ? 1 : -1;
562 : }
563 :
564 2262006 : if (TREE_CODE (oea->op) != SSA_NAME)
565 : {
566 0 : if (TREE_CODE (oeb->op) != SSA_NAME)
567 0 : return oeb->id > oea->id ? 1 : -1;
568 : else
569 : return 1;
570 : }
571 2262006 : else if (TREE_CODE (oeb->op) != SSA_NAME)
572 : return -1;
573 :
574 : /* Lastly, make sure the versions that are the same go next to each
575 : other. */
576 2262006 : if (SSA_NAME_VERSION (oeb->op) != SSA_NAME_VERSION (oea->op))
577 : {
578 : /* As SSA_NAME_VERSION is assigned pretty randomly, because we reuse
579 : versions of removed SSA_NAMEs, so if possible, prefer to sort
580 : based on basic block and gimple_uid of the SSA_NAME_DEF_STMT.
581 : See PR60418. */
582 2206295 : gimple *stmta = SSA_NAME_DEF_STMT (oea->op);
583 2206295 : gimple *stmtb = SSA_NAME_DEF_STMT (oeb->op);
584 2206295 : basic_block bba = gimple_bb (stmta);
585 2206295 : basic_block bbb = gimple_bb (stmtb);
586 2206295 : if (bbb != bba)
587 : {
588 : /* One of the SSA_NAMEs can be defined in oeN->stmt_to_insert
589 : but the other might not. */
590 143751 : if (!bba)
591 : return 1;
592 139424 : if (!bbb)
593 : return -1;
594 : /* If neither is, compare bb_rank. */
595 133442 : if (bb_rank[bbb->index] != bb_rank[bba->index])
596 133442 : return (bb_rank[bbb->index] >> 16) - (bb_rank[bba->index] >> 16);
597 : }
598 :
599 2062544 : bool da = reassoc_stmt_dominates_stmt_p (stmta, stmtb);
600 2062544 : bool db = reassoc_stmt_dominates_stmt_p (stmtb, stmta);
601 2062544 : if (da != db)
602 2007151 : return da ? 1 : -1;
603 :
604 55393 : return SSA_NAME_VERSION (oeb->op) > SSA_NAME_VERSION (oea->op) ? 1 : -1;
605 : }
606 :
607 55711 : return oeb->id > oea->id ? 1 : -1;
608 : }
609 :
610 : /* Add an operand entry to *OPS for the tree operand OP. */
611 :
612 : static void
613 9878158 : add_to_ops_vec (vec<operand_entry *> *ops, tree op, gimple *stmt_to_insert = NULL)
614 : {
615 9878158 : operand_entry *oe = operand_entry_pool.allocate ();
616 :
617 9878158 : oe->op = op;
618 9878158 : oe->rank = get_rank (op);
619 9878158 : oe->id = next_operand_entry_id++;
620 9878158 : oe->count = 1;
621 9878158 : oe->stmt_to_insert = stmt_to_insert;
622 9878158 : ops->safe_push (oe);
623 9878158 : }
624 :
625 : /* Add an operand entry to *OPS for the tree operand OP with repeat
626 : count REPEAT. */
627 :
628 : static void
629 12 : add_repeat_to_ops_vec (vec<operand_entry *> *ops, tree op,
630 : HOST_WIDE_INT repeat)
631 : {
632 12 : operand_entry *oe = operand_entry_pool.allocate ();
633 :
634 12 : oe->op = op;
635 12 : oe->rank = get_rank (op);
636 12 : oe->id = next_operand_entry_id++;
637 12 : oe->count = repeat;
638 12 : oe->stmt_to_insert = NULL;
639 12 : ops->safe_push (oe);
640 :
641 12 : reassociate_stats.pows_encountered++;
642 12 : }
643 :
644 : /* Returns true if we can associate the SSA def OP. */
645 :
646 : static bool
647 32334253 : can_reassociate_op_p (tree op)
648 : {
649 32334253 : if (TREE_CODE (op) == SSA_NAME && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op))
650 : return false;
651 : /* Uninitialized variables can't participate in reassociation. */
652 32333357 : if (TREE_CODE (op) == SSA_NAME && ssa_name_maybe_undef_p (op))
653 : return false;
654 : /* Make sure asm goto outputs do not participate in reassociation since
655 : we have no way to find an insertion place after asm goto. */
656 32328489 : if (TREE_CODE (op) == SSA_NAME
657 23732333 : && gimple_code (SSA_NAME_DEF_STMT (op)) == GIMPLE_ASM
658 32351164 : && gimple_asm_nlabels (as_a <gasm *> (SSA_NAME_DEF_STMT (op))) != 0)
659 100 : return false;
660 : return true;
661 : }
662 :
663 : /* Returns true if we can reassociate operations of TYPE.
664 : That is for integral or non-saturating fixed-point types, and for
665 : floating point type when associative-math is enabled. */
666 :
667 : static bool
668 59303329 : can_reassociate_type_p (tree type)
669 : {
670 59303329 : if ((ANY_INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_WRAPS (type))
671 36807406 : || NON_SAT_FIXED_POINT_TYPE_P (type)
672 96110735 : || (flag_associative_math && FLOAT_TYPE_P (type)))
673 22887942 : return true;
674 : return false;
675 : }
676 :
677 : /* Return true if STMT is reassociable operation containing a binary
678 : operation with tree code CODE, and is inside LOOP. */
679 :
680 : static bool
681 7859404 : is_reassociable_op (gimple *stmt, enum tree_code code, class loop *loop)
682 : {
683 7859404 : basic_block bb = gimple_bb (stmt);
684 :
685 7859404 : if (gimple_bb (stmt) == NULL)
686 : return false;
687 :
688 7671410 : if (!flow_bb_inside_loop_p (loop, bb))
689 : return false;
690 :
691 7449817 : if (is_gimple_assign (stmt)
692 5922122 : && gimple_assign_rhs_code (stmt) == code
693 8280541 : && has_single_use (gimple_assign_lhs (stmt)))
694 : {
695 596116 : tree rhs1 = gimple_assign_rhs1 (stmt);
696 596116 : tree rhs2 = gimple_assign_rhs2 (stmt);
697 596116 : if (!can_reassociate_op_p (rhs1)
698 596116 : || (rhs2 && !can_reassociate_op_p (rhs2)))
699 526 : return false;
700 : return true;
701 : }
702 :
703 : return false;
704 : }
705 :
706 :
707 : /* Return true if STMT is a nop-conversion. */
708 :
709 : static bool
710 7792835 : gimple_nop_conversion_p (gimple *stmt)
711 : {
712 7792835 : if (gassign *ass = dyn_cast <gassign *> (stmt))
713 : {
714 9146294 : if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (ass))
715 6880990 : && tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (ass)),
716 1538562 : TREE_TYPE (gimple_assign_rhs1 (ass))))
717 : return true;
718 : }
719 : return false;
720 : }
721 :
722 : /* Given NAME, if NAME is defined by a unary operation OPCODE, return the
723 : operand of the negate operation. Otherwise, return NULL. */
724 :
725 : static tree
726 7701634 : get_unary_op (tree name, enum tree_code opcode)
727 : {
728 7701634 : gimple *stmt = SSA_NAME_DEF_STMT (name);
729 :
730 : /* Look through nop conversions (sign changes). */
731 7701634 : if (gimple_nop_conversion_p (stmt)
732 7701634 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME)
733 920042 : stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
734 :
735 7701634 : if (!is_gimple_assign (stmt))
736 : return NULL_TREE;
737 :
738 4891263 : if (gimple_assign_rhs_code (stmt) == opcode)
739 126064 : return gimple_assign_rhs1 (stmt);
740 : return NULL_TREE;
741 : }
742 :
743 : /* Return true if OP1 and OP2 have the same value if casted to either type. */
744 :
745 : static bool
746 46794 : ops_equal_values_p (tree op1, tree op2)
747 : {
748 46794 : if (op1 == op2)
749 : return true;
750 :
751 46563 : tree orig_op1 = op1;
752 46563 : if (TREE_CODE (op1) == SSA_NAME)
753 : {
754 46563 : gimple *stmt = SSA_NAME_DEF_STMT (op1);
755 46563 : if (gimple_nop_conversion_p (stmt))
756 : {
757 19086 : op1 = gimple_assign_rhs1 (stmt);
758 19086 : if (op1 == op2)
759 : return true;
760 : }
761 : }
762 :
763 44638 : if (TREE_CODE (op2) == SSA_NAME)
764 : {
765 44638 : gimple *stmt = SSA_NAME_DEF_STMT (op2);
766 44638 : if (gimple_nop_conversion_p (stmt))
767 : {
768 17277 : op2 = gimple_assign_rhs1 (stmt);
769 17277 : if (op1 == op2
770 17277 : || orig_op1 == op2)
771 : return true;
772 : }
773 : }
774 :
775 : return false;
776 : }
777 :
778 :
779 : /* If CURR and LAST are a pair of ops that OPCODE allows us to
780 : eliminate through equivalences, do so, remove them from OPS, and
781 : return true. Otherwise, return false. */
782 :
783 : static bool
784 9770324 : eliminate_duplicate_pair (enum tree_code opcode,
785 : vec<operand_entry *> *ops,
786 : bool *all_done,
787 : unsigned int i,
788 : operand_entry *curr,
789 : operand_entry *last)
790 : {
791 :
792 : /* If we have two of the same op, and the opcode is & |, min, or max,
793 : we can eliminate one of them.
794 : If we have two of the same op, and the opcode is ^, we can
795 : eliminate both of them. */
796 :
797 9770324 : if (last && last->op == curr->op)
798 : {
799 5591 : switch (opcode)
800 : {
801 27 : case MAX_EXPR:
802 27 : case MIN_EXPR:
803 27 : case BIT_IOR_EXPR:
804 27 : case BIT_AND_EXPR:
805 27 : if (dump_file && (dump_flags & TDF_DETAILS))
806 : {
807 1 : fprintf (dump_file, "Equivalence: ");
808 1 : print_generic_expr (dump_file, curr->op);
809 1 : fprintf (dump_file, " [&|minmax] ");
810 1 : print_generic_expr (dump_file, last->op);
811 1 : fprintf (dump_file, " -> ");
812 1 : print_generic_stmt (dump_file, last->op);
813 : }
814 :
815 27 : ops->ordered_remove (i);
816 27 : reassociate_stats.ops_eliminated ++;
817 :
818 27 : return true;
819 :
820 122 : case BIT_XOR_EXPR:
821 122 : if (dump_file && (dump_flags & TDF_DETAILS))
822 : {
823 0 : fprintf (dump_file, "Equivalence: ");
824 0 : print_generic_expr (dump_file, curr->op);
825 0 : fprintf (dump_file, " ^ ");
826 0 : print_generic_expr (dump_file, last->op);
827 0 : fprintf (dump_file, " -> nothing\n");
828 : }
829 :
830 122 : reassociate_stats.ops_eliminated += 2;
831 :
832 122 : if (ops->length () == 2)
833 : {
834 1 : ops->truncate (0);
835 1 : add_to_ops_vec (ops, build_zero_cst (TREE_TYPE (last->op)));
836 1 : *all_done = true;
837 : }
838 : else
839 : {
840 121 : ops->ordered_remove (i-1);
841 121 : ops->ordered_remove (i-1);
842 : }
843 :
844 : return true;
845 :
846 : default:
847 : break;
848 : }
849 : }
850 : return false;
851 : }
852 :
853 : static vec<tree> plus_negates;
854 :
855 : /* If OPCODE is PLUS_EXPR, CURR->OP is a negate expression or a bitwise not
856 : expression, look in OPS for a corresponding positive operation to cancel
857 : it out. If we find one, remove the other from OPS, replace
858 : OPS[CURRINDEX] with 0 or -1, respectively, and return true. Otherwise,
859 : return false. */
860 :
861 : static bool
862 9770175 : eliminate_plus_minus_pair (enum tree_code opcode,
863 : vec<operand_entry *> *ops,
864 : unsigned int currindex,
865 : operand_entry *curr)
866 : {
867 9770175 : tree negateop;
868 9770175 : tree notop;
869 9770175 : unsigned int i;
870 9770175 : operand_entry *oe;
871 :
872 9770175 : if (opcode != PLUS_EXPR || TREE_CODE (curr->op) != SSA_NAME)
873 : return false;
874 :
875 3079315 : negateop = get_unary_op (curr->op, NEGATE_EXPR);
876 3079315 : notop = get_unary_op (curr->op, BIT_NOT_EXPR);
877 3079315 : if (negateop == NULL_TREE && notop == NULL_TREE)
878 : return false;
879 :
880 : /* Any non-negated version will have a rank that is one less than
881 : the current rank. So once we hit those ranks, if we don't find
882 : one, we can stop. */
883 :
884 133655 : for (i = currindex + 1;
885 196512 : ops->iterate (i, &oe)
886 243306 : && oe->rank >= curr->rank - 1 ;
887 : i++)
888 : {
889 46794 : if (negateop
890 46794 : && ops_equal_values_p (oe->op, negateop))
891 : {
892 1579 : if (dump_file && (dump_flags & TDF_DETAILS))
893 : {
894 0 : fprintf (dump_file, "Equivalence: ");
895 0 : print_generic_expr (dump_file, negateop);
896 0 : fprintf (dump_file, " + -");
897 0 : print_generic_expr (dump_file, oe->op);
898 0 : fprintf (dump_file, " -> 0\n");
899 : }
900 :
901 1579 : ops->ordered_remove (i);
902 1579 : add_to_ops_vec (ops, build_zero_cst (TREE_TYPE (oe->op)));
903 1579 : ops->ordered_remove (currindex);
904 1579 : reassociate_stats.ops_eliminated ++;
905 :
906 1579 : return true;
907 : }
908 45215 : else if (notop
909 45215 : && ops_equal_values_p (oe->op, notop))
910 : {
911 1917 : tree op_type = TREE_TYPE (oe->op);
912 :
913 1917 : if (dump_file && (dump_flags & TDF_DETAILS))
914 : {
915 0 : fprintf (dump_file, "Equivalence: ");
916 0 : print_generic_expr (dump_file, notop);
917 0 : fprintf (dump_file, " + ~");
918 0 : print_generic_expr (dump_file, oe->op);
919 0 : fprintf (dump_file, " -> -1\n");
920 : }
921 :
922 1917 : ops->ordered_remove (i);
923 1917 : add_to_ops_vec (ops, build_all_ones_cst (op_type));
924 1917 : ops->ordered_remove (currindex);
925 1917 : reassociate_stats.ops_eliminated ++;
926 :
927 1917 : return true;
928 : }
929 : }
930 :
931 : /* If CURR->OP is a negate expr without nop conversion in a plus expr:
932 : save it for later inspection in repropagate_negates(). */
933 86861 : if (negateop != NULL_TREE
934 86861 : && gimple_assign_rhs_code (SSA_NAME_DEF_STMT (curr->op)) == NEGATE_EXPR)
935 86352 : plus_negates.safe_push (curr->op);
936 :
937 : return false;
938 : }
939 :
940 : /* If OPCODE is BIT_IOR_EXPR, BIT_AND_EXPR, and, CURR->OP is really a
941 : bitwise not expression, look in OPS for a corresponding operand to
942 : cancel it out. If we find one, remove the other from OPS, replace
943 : OPS[CURRINDEX] with 0, and return true. Otherwise, return
944 : false. */
945 :
946 : static bool
947 9770325 : eliminate_not_pairs (enum tree_code opcode,
948 : vec<operand_entry *> *ops,
949 : unsigned int currindex,
950 : operand_entry *curr)
951 : {
952 9770325 : tree notop;
953 9770325 : unsigned int i;
954 9770325 : operand_entry *oe;
955 :
956 9770325 : if ((opcode != BIT_IOR_EXPR && opcode != BIT_AND_EXPR)
957 2047180 : || TREE_CODE (curr->op) != SSA_NAME)
958 : return false;
959 :
960 1543004 : notop = get_unary_op (curr->op, BIT_NOT_EXPR);
961 1543004 : if (notop == NULL_TREE)
962 : return false;
963 :
964 : /* Any non-not version will have a rank that is one less than
965 : the current rank. So once we hit those ranks, if we don't find
966 : one, we can stop. */
967 :
968 35707 : for (i = currindex + 1;
969 9798602 : ops->iterate (i, &oe)
970 71474 : && oe->rank >= curr->rank - 1;
971 : i++)
972 : {
973 7491 : if (oe->op == notop)
974 : {
975 1 : if (dump_file && (dump_flags & TDF_DETAILS))
976 : {
977 0 : fprintf (dump_file, "Equivalence: ");
978 0 : print_generic_expr (dump_file, notop);
979 0 : if (opcode == BIT_AND_EXPR)
980 0 : fprintf (dump_file, " & ~");
981 0 : else if (opcode == BIT_IOR_EXPR)
982 0 : fprintf (dump_file, " | ~");
983 0 : print_generic_expr (dump_file, oe->op);
984 0 : if (opcode == BIT_AND_EXPR)
985 0 : fprintf (dump_file, " -> 0\n");
986 0 : else if (opcode == BIT_IOR_EXPR)
987 0 : fprintf (dump_file, " -> -1\n");
988 : }
989 :
990 1 : if (opcode == BIT_AND_EXPR)
991 1 : oe->op = build_zero_cst (TREE_TYPE (oe->op));
992 0 : else if (opcode == BIT_IOR_EXPR)
993 0 : oe->op = build_all_ones_cst (TREE_TYPE (oe->op));
994 :
995 1 : reassociate_stats.ops_eliminated += ops->length () - 1;
996 1 : ops->truncate (0);
997 1 : ops->quick_push (oe);
998 1 : return true;
999 : }
1000 : }
1001 :
1002 : return false;
1003 : }
1004 :
1005 : /* Use constant value that may be present in OPS to try to eliminate
1006 : operands. Note that this function is only really used when we've
1007 : eliminated ops for other reasons, or merged constants. Across
1008 : single statements, fold already does all of this, plus more. There
1009 : is little point in duplicating logic, so I've only included the
1010 : identities that I could ever construct testcases to trigger. */
1011 :
1012 : static void
1013 4762234 : eliminate_using_constants (enum tree_code opcode,
1014 : vec<operand_entry *> *ops)
1015 : {
1016 4762234 : operand_entry *oelast = ops->last ();
1017 4762234 : tree type = TREE_TYPE (oelast->op);
1018 :
1019 4762234 : if (oelast->rank == 0
1020 4762234 : && (ANY_INTEGRAL_TYPE_P (type) || FLOAT_TYPE_P (type)))
1021 : {
1022 3440857 : switch (opcode)
1023 : {
1024 430903 : case BIT_AND_EXPR:
1025 430903 : if (integer_zerop (oelast->op))
1026 : {
1027 0 : if (ops->length () != 1)
1028 : {
1029 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1030 0 : fprintf (dump_file, "Found & 0, removing all other ops\n");
1031 :
1032 0 : reassociate_stats.ops_eliminated += ops->length () - 1;
1033 :
1034 0 : ops->truncate (0);
1035 0 : ops->quick_push (oelast);
1036 1839 : return;
1037 : }
1038 : }
1039 430903 : else if (integer_all_onesp (oelast->op))
1040 : {
1041 4 : if (ops->length () != 1)
1042 : {
1043 4 : if (dump_file && (dump_flags & TDF_DETAILS))
1044 0 : fprintf (dump_file, "Found & -1, removing\n");
1045 4 : ops->pop ();
1046 4 : reassociate_stats.ops_eliminated++;
1047 : }
1048 : }
1049 : break;
1050 72740 : case BIT_IOR_EXPR:
1051 72740 : if (integer_all_onesp (oelast->op))
1052 : {
1053 0 : if (ops->length () != 1)
1054 : {
1055 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1056 0 : fprintf (dump_file, "Found | -1, removing all other ops\n");
1057 :
1058 0 : reassociate_stats.ops_eliminated += ops->length () - 1;
1059 :
1060 0 : ops->truncate (0);
1061 0 : ops->quick_push (oelast);
1062 0 : return;
1063 : }
1064 : }
1065 72740 : else if (integer_zerop (oelast->op))
1066 : {
1067 4 : if (ops->length () != 1)
1068 : {
1069 4 : if (dump_file && (dump_flags & TDF_DETAILS))
1070 0 : fprintf (dump_file, "Found | 0, removing\n");
1071 4 : ops->pop ();
1072 4 : reassociate_stats.ops_eliminated++;
1073 : }
1074 : }
1075 : break;
1076 962918 : case MULT_EXPR:
1077 962918 : if (integer_zerop (oelast->op)
1078 962918 : || (FLOAT_TYPE_P (type)
1079 1438 : && !HONOR_NANS (type)
1080 1343 : && !HONOR_SIGNED_ZEROS (type)
1081 1343 : && real_zerop (oelast->op)))
1082 : {
1083 0 : if (ops->length () != 1)
1084 : {
1085 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1086 0 : fprintf (dump_file, "Found * 0, removing all other ops\n");
1087 :
1088 0 : reassociate_stats.ops_eliminated += ops->length () - 1;
1089 0 : ops->truncate (0);
1090 0 : ops->quick_push (oelast);
1091 0 : return;
1092 : }
1093 : }
1094 962918 : else if (integer_onep (oelast->op)
1095 962918 : || (FLOAT_TYPE_P (type)
1096 1438 : && !HONOR_SNANS (type)
1097 1438 : && real_onep (oelast->op)))
1098 : {
1099 4 : if (ops->length () != 1)
1100 : {
1101 4 : if (dump_file && (dump_flags & TDF_DETAILS))
1102 0 : fprintf (dump_file, "Found * 1, removing\n");
1103 4 : ops->pop ();
1104 4 : reassociate_stats.ops_eliminated++;
1105 4 : return;
1106 : }
1107 : }
1108 : break;
1109 1800623 : case BIT_XOR_EXPR:
1110 1800623 : case PLUS_EXPR:
1111 1800623 : case MINUS_EXPR:
1112 1800623 : if (integer_zerop (oelast->op)
1113 1800623 : || (FLOAT_TYPE_P (type)
1114 849 : && (opcode == PLUS_EXPR || opcode == MINUS_EXPR)
1115 849 : && fold_real_zero_addition_p (type, 0, oelast->op,
1116 : opcode == MINUS_EXPR)))
1117 : {
1118 1835 : if (ops->length () != 1)
1119 : {
1120 1835 : if (dump_file && (dump_flags & TDF_DETAILS))
1121 0 : fprintf (dump_file, "Found [|^+] 0, removing\n");
1122 1835 : ops->pop ();
1123 1835 : reassociate_stats.ops_eliminated++;
1124 1835 : return;
1125 : }
1126 : }
1127 : break;
1128 : default:
1129 : break;
1130 : }
1131 : }
1132 : }
1133 :
1134 :
1135 : static void linearize_expr_tree (vec<operand_entry *> *, gimple *,
1136 : bool, bool);
1137 :
1138 : /* Structure for tracking and counting operands. */
1139 : struct oecount {
1140 : unsigned int cnt;
1141 : unsigned int id;
1142 : enum tree_code oecode;
1143 : tree op;
1144 : };
1145 :
1146 :
1147 : /* The heap for the oecount hashtable and the sorted list of operands. */
1148 : static vec<oecount> cvec;
1149 :
1150 :
1151 : /* Oecount hashtable helpers. */
1152 :
1153 : struct oecount_hasher : int_hash <int, 0, 1>
1154 : {
1155 : static inline hashval_t hash (int);
1156 : static inline bool equal (int, int);
1157 : };
1158 :
1159 : /* Hash function for oecount. */
1160 :
1161 : inline hashval_t
1162 155839 : oecount_hasher::hash (int p)
1163 : {
1164 155839 : const oecount *c = &cvec[p - 42];
1165 155839 : return htab_hash_pointer (c->op) ^ (hashval_t)c->oecode;
1166 : }
1167 :
1168 : /* Comparison function for oecount. */
1169 :
1170 : inline bool
1171 84729 : oecount_hasher::equal (int p1, int p2)
1172 : {
1173 84729 : const oecount *c1 = &cvec[p1 - 42];
1174 84729 : const oecount *c2 = &cvec[p2 - 42];
1175 84729 : return c1->oecode == c2->oecode && c1->op == c2->op;
1176 : }
1177 :
1178 : /* Comparison function for qsort sorting oecount elements by count. */
1179 :
1180 : static int
1181 601846 : oecount_cmp (const void *p1, const void *p2)
1182 : {
1183 601846 : const oecount *c1 = (const oecount *)p1;
1184 601846 : const oecount *c2 = (const oecount *)p2;
1185 601846 : if (c1->cnt != c2->cnt)
1186 8974 : return c1->cnt > c2->cnt ? 1 : -1;
1187 : else
1188 : /* If counts are identical, use unique IDs to stabilize qsort. */
1189 592872 : return c1->id > c2->id ? 1 : -1;
1190 : }
1191 :
1192 : /* Return TRUE iff STMT represents a builtin call that raises OP
1193 : to some constant integer exponent, as required by decrement_power. */
1194 :
1195 : static bool
1196 1280 : stmt_is_power_of_op (gimple *stmt, tree op)
1197 : {
1198 1280 : if (!is_gimple_call (stmt))
1199 : return false;
1200 :
1201 8 : switch (gimple_call_combined_fn (stmt))
1202 : {
1203 2 : CASE_CFN_POW:
1204 2 : {
1205 2 : tree arg1 = gimple_call_arg (stmt, 1);
1206 2 : if (TREE_CODE (arg1) != REAL_CST)
1207 : return false;
1208 1 : REAL_VALUE_TYPE c = TREE_REAL_CST (arg1);
1209 1 : REAL_VALUE_TYPE cint;
1210 1 : real_from_integer (&cint, VOIDmode, real_to_integer (&c), SIGNED);
1211 1 : if (!real_identical (&c, &cint))
1212 : return false;
1213 0 : return operand_equal_p (gimple_call_arg (stmt, 0), op, 0);
1214 : }
1215 :
1216 1 : CASE_CFN_POWI:
1217 1 : if (!tree_fits_shwi_p (gimple_call_arg (stmt, 1)))
1218 : return false;
1219 0 : return operand_equal_p (gimple_call_arg (stmt, 0), op, 0);
1220 :
1221 : default:
1222 : return false;
1223 : }
1224 : }
1225 :
1226 : /* Given STMT which is a __builtin_pow* call, decrement its exponent
1227 : in place and return the result. Assumes that stmt_is_power_of_op
1228 : was previously called for STMT and returned TRUE. */
1229 :
1230 : static HOST_WIDE_INT
1231 0 : decrement_power (gimple *stmt)
1232 : {
1233 0 : REAL_VALUE_TYPE c, cint;
1234 0 : HOST_WIDE_INT power;
1235 0 : tree arg1;
1236 :
1237 0 : switch (gimple_call_combined_fn (stmt))
1238 : {
1239 0 : CASE_CFN_POW:
1240 0 : arg1 = gimple_call_arg (stmt, 1);
1241 0 : c = TREE_REAL_CST (arg1);
1242 0 : power = real_to_integer (&c) - 1;
1243 0 : real_from_integer (&cint, VOIDmode, power, SIGNED);
1244 0 : gimple_call_set_arg (stmt, 1, build_real (TREE_TYPE (arg1), cint));
1245 0 : return power;
1246 :
1247 0 : CASE_CFN_POWI:
1248 0 : arg1 = gimple_call_arg (stmt, 1);
1249 0 : power = TREE_INT_CST_LOW (arg1) - 1;
1250 0 : gimple_call_set_arg (stmt, 1, build_int_cst (TREE_TYPE (arg1), power));
1251 0 : return power;
1252 :
1253 0 : default:
1254 0 : gcc_unreachable ();
1255 : }
1256 : }
1257 :
1258 : /* Replace SSA defined by STMT and replace all its uses with new
1259 : SSA. Also return the new SSA. */
1260 :
1261 : static tree
1262 344 : make_new_ssa_for_def (gimple *stmt, enum tree_code opcode, tree op)
1263 : {
1264 344 : gimple *use_stmt;
1265 344 : use_operand_p use;
1266 344 : imm_use_iterator iter;
1267 344 : tree new_lhs, new_debug_lhs = NULL_TREE;
1268 344 : tree lhs = gimple_get_lhs (stmt);
1269 :
1270 344 : new_lhs = make_ssa_name (TREE_TYPE (lhs));
1271 344 : gimple_set_lhs (stmt, new_lhs);
1272 :
1273 : /* Also need to update GIMPLE_DEBUGs. */
1274 745 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
1275 : {
1276 401 : tree repl = new_lhs;
1277 401 : if (is_gimple_debug (use_stmt))
1278 : {
1279 57 : if (new_debug_lhs == NULL_TREE)
1280 : {
1281 21 : new_debug_lhs = build_debug_expr_decl (TREE_TYPE (lhs));
1282 21 : gdebug *def_temp
1283 21 : = gimple_build_debug_bind (new_debug_lhs,
1284 21 : build2 (opcode, TREE_TYPE (lhs),
1285 : new_lhs, op),
1286 : stmt);
1287 21 : gimple_set_uid (def_temp, gimple_uid (stmt));
1288 21 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1289 21 : gsi_insert_after (&gsi, def_temp, GSI_SAME_STMT);
1290 : }
1291 : repl = new_debug_lhs;
1292 : }
1293 802 : FOR_EACH_IMM_USE_ON_STMT (use, iter)
1294 401 : SET_USE (use, repl);
1295 401 : update_stmt (use_stmt);
1296 344 : }
1297 344 : return new_lhs;
1298 : }
1299 :
1300 : /* Replace all SSAs defined in STMTS_TO_FIX and replace its
1301 : uses with new SSAs. Also do this for the stmt that defines DEF
1302 : if *DEF is not OP. */
1303 :
1304 : static void
1305 288 : make_new_ssa_for_all_defs (tree *def, enum tree_code opcode, tree op,
1306 : vec<gimple *> &stmts_to_fix)
1307 : {
1308 288 : unsigned i;
1309 288 : gimple *stmt;
1310 :
1311 288 : if (*def != op
1312 288 : && TREE_CODE (*def) == SSA_NAME
1313 288 : && (stmt = SSA_NAME_DEF_STMT (*def))
1314 576 : && gimple_code (stmt) != GIMPLE_NOP)
1315 288 : *def = make_new_ssa_for_def (stmt, opcode, op);
1316 :
1317 344 : FOR_EACH_VEC_ELT (stmts_to_fix, i, stmt)
1318 56 : make_new_ssa_for_def (stmt, opcode, op);
1319 288 : }
1320 :
1321 : /* Find the single immediate use of STMT's LHS, and replace it
1322 : with OP. Remove STMT. If STMT's LHS is the same as *DEF,
1323 : replace *DEF with OP as well. */
1324 :
1325 : static void
1326 886 : propagate_op_to_single_use (tree op, gimple *stmt, tree *def)
1327 : {
1328 886 : tree lhs;
1329 886 : gimple *use_stmt;
1330 886 : use_operand_p use;
1331 886 : gimple_stmt_iterator gsi;
1332 :
1333 886 : if (is_gimple_call (stmt))
1334 0 : lhs = gimple_call_lhs (stmt);
1335 : else
1336 886 : lhs = gimple_assign_lhs (stmt);
1337 :
1338 886 : gcc_assert (has_single_use (lhs));
1339 886 : single_imm_use (lhs, &use, &use_stmt);
1340 886 : if (lhs == *def)
1341 602 : *def = op;
1342 886 : SET_USE (use, op);
1343 886 : if (TREE_CODE (op) != SSA_NAME)
1344 40 : update_stmt (use_stmt);
1345 886 : gsi = gsi_for_stmt (stmt);
1346 886 : unlink_stmt_vdef (stmt);
1347 886 : reassoc_remove_stmt (&gsi);
1348 886 : release_defs (stmt);
1349 886 : }
1350 :
1351 : /* Walks the linear chain with result *DEF searching for an operation
1352 : with operand OP and code OPCODE removing that from the chain. *DEF
1353 : is updated if there is only one operand but no operation left. */
1354 :
1355 : static void
1356 890 : zero_one_operation (tree *def, enum tree_code opcode, tree op)
1357 : {
1358 890 : tree orig_def = *def;
1359 890 : gimple *stmt = SSA_NAME_DEF_STMT (*def);
1360 : /* PR72835 - Record the stmt chain that has to be updated such that
1361 : we dont use the same LHS when the values computed are different. */
1362 890 : auto_vec<gimple *, 64> stmts_to_fix;
1363 :
1364 1570 : do
1365 : {
1366 1230 : tree name;
1367 :
1368 1230 : if (opcode == MULT_EXPR)
1369 : {
1370 1228 : if (stmt_is_power_of_op (stmt, op))
1371 : {
1372 0 : if (decrement_power (stmt) == 1)
1373 : {
1374 0 : if (stmts_to_fix.length () > 0)
1375 0 : stmts_to_fix.pop ();
1376 0 : propagate_op_to_single_use (op, stmt, def);
1377 : }
1378 : break;
1379 : }
1380 1228 : else if (gimple_assign_rhs_code (stmt) == NEGATE_EXPR)
1381 : {
1382 15 : if (gimple_assign_rhs1 (stmt) == op)
1383 : {
1384 11 : tree cst = build_minus_one_cst (TREE_TYPE (op));
1385 11 : if (stmts_to_fix.length () > 0)
1386 11 : stmts_to_fix.pop ();
1387 11 : propagate_op_to_single_use (cst, stmt, def);
1388 11 : break;
1389 : }
1390 4 : else if (integer_minus_onep (op)
1391 4 : || real_minus_onep (op))
1392 : {
1393 4 : gimple_assign_set_rhs_code
1394 4 : (stmt, TREE_CODE (gimple_assign_rhs1 (stmt)));
1395 4 : break;
1396 : }
1397 : }
1398 : }
1399 :
1400 1215 : name = gimple_assign_rhs1 (stmt);
1401 :
1402 : /* If this is the operation we look for and one of the operands
1403 : is ours simply propagate the other operand into the stmts
1404 : single use. */
1405 1215 : if (gimple_assign_rhs_code (stmt) == opcode
1406 1215 : && (name == op
1407 949 : || gimple_assign_rhs2 (stmt) == op))
1408 : {
1409 875 : if (name == op)
1410 266 : name = gimple_assign_rhs2 (stmt);
1411 875 : if (stmts_to_fix.length () > 0)
1412 273 : stmts_to_fix.pop ();
1413 875 : propagate_op_to_single_use (name, stmt, def);
1414 875 : break;
1415 : }
1416 :
1417 : /* We might have a multiply of two __builtin_pow* calls, and
1418 : the operand might be hiding in the rightmost one. Likewise
1419 : this can happen for a negate. */
1420 340 : if (opcode == MULT_EXPR
1421 340 : && gimple_assign_rhs_code (stmt) == opcode
1422 340 : && TREE_CODE (gimple_assign_rhs2 (stmt)) == SSA_NAME
1423 615 : && has_single_use (gimple_assign_rhs2 (stmt)))
1424 : {
1425 52 : gimple *stmt2 = SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt));
1426 52 : if (stmt_is_power_of_op (stmt2, op))
1427 : {
1428 0 : if (decrement_power (stmt2) == 1)
1429 0 : propagate_op_to_single_use (op, stmt2, def);
1430 : else
1431 0 : stmts_to_fix.safe_push (stmt2);
1432 0 : break;
1433 : }
1434 52 : else if (is_gimple_assign (stmt2)
1435 52 : && gimple_assign_rhs_code (stmt2) == NEGATE_EXPR)
1436 : {
1437 0 : if (gimple_assign_rhs1 (stmt2) == op)
1438 : {
1439 0 : tree cst = build_minus_one_cst (TREE_TYPE (op));
1440 0 : propagate_op_to_single_use (cst, stmt2, def);
1441 0 : break;
1442 : }
1443 0 : else if (integer_minus_onep (op)
1444 0 : || real_minus_onep (op))
1445 : {
1446 0 : stmts_to_fix.safe_push (stmt2);
1447 0 : gimple_assign_set_rhs_code
1448 0 : (stmt2, TREE_CODE (gimple_assign_rhs1 (stmt2)));
1449 0 : break;
1450 : }
1451 : }
1452 : }
1453 :
1454 : /* Continue walking the chain. */
1455 340 : gcc_assert (name != op
1456 : && TREE_CODE (name) == SSA_NAME);
1457 340 : stmt = SSA_NAME_DEF_STMT (name);
1458 340 : stmts_to_fix.safe_push (stmt);
1459 340 : }
1460 : while (1);
1461 :
1462 890 : if (stmts_to_fix.length () > 0 || *def == orig_def)
1463 288 : make_new_ssa_for_all_defs (def, opcode, op, stmts_to_fix);
1464 890 : }
1465 :
1466 : /* Returns true if statement S1 dominates statement S2. Like
1467 : stmt_dominates_stmt_p, but uses stmt UIDs to optimize. */
1468 :
1469 : static bool
1470 4822474 : reassoc_stmt_dominates_stmt_p (gimple *s1, gimple *s2)
1471 : {
1472 4822474 : basic_block bb1 = gimple_bb (s1), bb2 = gimple_bb (s2);
1473 :
1474 : /* If bb1 is NULL, it should be a GIMPLE_NOP def stmt of an (D)
1475 : SSA_NAME. Assume it lives at the beginning of function and
1476 : thus dominates everything. */
1477 4822474 : if (!bb1 || s1 == s2)
1478 : return true;
1479 :
1480 : /* If bb2 is NULL, it doesn't dominate any stmt with a bb. */
1481 4819301 : if (!bb2)
1482 : return false;
1483 :
1484 4811774 : if (bb1 == bb2)
1485 : {
1486 : /* PHIs in the same basic block are assumed to be
1487 : executed all in parallel, if only one stmt is a PHI,
1488 : it dominates the other stmt in the same basic block. */
1489 4671457 : if (gimple_code (s1) == GIMPLE_PHI)
1490 : return true;
1491 :
1492 4556753 : if (gimple_code (s2) == GIMPLE_PHI)
1493 : return false;
1494 :
1495 4505018 : gcc_assert (gimple_uid (s1) && gimple_uid (s2));
1496 :
1497 4505018 : if (gimple_uid (s1) < gimple_uid (s2))
1498 : return true;
1499 :
1500 2472049 : if (gimple_uid (s1) > gimple_uid (s2))
1501 : return false;
1502 :
1503 36262 : gimple_stmt_iterator gsi = gsi_for_stmt (s1);
1504 36262 : unsigned int uid = gimple_uid (s1);
1505 78074 : for (gsi_next (&gsi); !gsi_end_p (gsi); gsi_next (&gsi))
1506 : {
1507 76212 : gimple *s = gsi_stmt (gsi);
1508 76212 : if (gimple_uid (s) != uid)
1509 : break;
1510 45071 : if (s == s2)
1511 : return true;
1512 : }
1513 :
1514 : return false;
1515 : }
1516 :
1517 140317 : return dominated_by_p (CDI_DOMINATORS, bb2, bb1);
1518 : }
1519 :
1520 : /* Insert STMT after INSERT_POINT. */
1521 :
1522 : static void
1523 38837 : insert_stmt_after (gimple *stmt, gimple *insert_point)
1524 : {
1525 38837 : gimple_stmt_iterator gsi;
1526 38837 : basic_block bb;
1527 :
1528 38837 : if (gimple_code (insert_point) == GIMPLE_PHI)
1529 30 : bb = gimple_bb (insert_point);
1530 38807 : else if (!stmt_ends_bb_p (insert_point))
1531 : {
1532 38799 : gsi = gsi_for_stmt (insert_point);
1533 38799 : gimple_set_uid (stmt, gimple_uid (insert_point));
1534 38799 : gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
1535 38799 : return;
1536 : }
1537 8 : else if (gimple_code (insert_point) == GIMPLE_ASM
1538 5 : && gimple_asm_nlabels (as_a <gasm *> (insert_point)) != 0
1539 13 : && !single_succ_p (gimple_bb (insert_point)))
1540 : /* We have no idea where to insert - it depends on where the
1541 : uses will be placed. */
1542 0 : gcc_unreachable ();
1543 : else
1544 : /* We assume INSERT_POINT is a SSA_NAME_DEF_STMT of some SSA_NAME,
1545 : thus if it must end a basic block, it should be a call that can
1546 : throw, or some assignment that can throw. If it throws, the LHS
1547 : of it will not be initialized though, so only valid places using
1548 : the SSA_NAME should be dominated by the fallthru edge. */
1549 8 : bb = find_fallthru_edge (gimple_bb (insert_point)->succs)->dest;
1550 38 : gsi = gsi_after_labels (bb);
1551 38 : if (gsi_end_p (gsi))
1552 : {
1553 0 : gimple_stmt_iterator gsi2 = gsi_last_bb (bb);
1554 0 : gimple_set_uid (stmt,
1555 0 : gsi_end_p (gsi2) ? 1 : gimple_uid (gsi_stmt (gsi2)));
1556 : }
1557 : else
1558 38 : gimple_set_uid (stmt, gimple_uid (gsi_stmt (gsi)));
1559 38 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1560 : }
1561 :
1562 : /* Builds one statement performing OP1 OPCODE OP2 using TMPVAR for
1563 : the result. Places the statement after the definition of either
1564 : OP1 or OP2. Returns the new statement. */
1565 :
1566 : static gimple *
1567 8656 : build_and_add_sum (tree type, tree op1, tree op2, enum tree_code opcode)
1568 : {
1569 8656 : gimple *op1def = NULL, *op2def = NULL;
1570 8656 : gimple_stmt_iterator gsi;
1571 8656 : tree op;
1572 8656 : gassign *sum;
1573 :
1574 : /* Create the addition statement. */
1575 8656 : op = make_ssa_name (type);
1576 8656 : sum = gimple_build_assign (op, opcode, op1, op2);
1577 :
1578 : /* Find an insertion place and insert. */
1579 8656 : if (TREE_CODE (op1) == SSA_NAME)
1580 8655 : op1def = SSA_NAME_DEF_STMT (op1);
1581 8656 : if (TREE_CODE (op2) == SSA_NAME)
1582 8365 : op2def = SSA_NAME_DEF_STMT (op2);
1583 8655 : if ((!op1def || gimple_nop_p (op1def))
1584 8742 : && (!op2def || gimple_nop_p (op2def)))
1585 : {
1586 86 : gsi = gsi_start_nondebug_after_labels_bb
1587 86 : (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
1588 86 : if (!gsi_end_p (gsi)
1589 86 : && is_gimple_call (gsi_stmt (gsi))
1590 98 : && (gimple_call_flags (gsi_stmt (gsi)) & ECF_RETURNS_TWICE))
1591 : {
1592 : /* Don't add statements before a returns_twice call at the start
1593 : of a function. */
1594 2 : split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
1595 2 : gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
1596 : }
1597 86 : if (gsi_end_p (gsi))
1598 : {
1599 2 : gimple_stmt_iterator gsi2
1600 2 : = gsi_last_bb (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
1601 2 : gimple_set_uid (sum,
1602 2 : gsi_end_p (gsi2) ? 1 : gimple_uid (gsi_stmt (gsi2)));
1603 : }
1604 : else
1605 84 : gimple_set_uid (sum, gimple_uid (gsi_stmt (gsi)));
1606 86 : gsi_insert_before (&gsi, sum, GSI_NEW_STMT);
1607 : }
1608 : else
1609 : {
1610 8570 : gimple *insert_point;
1611 8569 : if ((!op1def || gimple_nop_p (op1def))
1612 17139 : || (op2def && !gimple_nop_p (op2def)
1613 8247 : && reassoc_stmt_dominates_stmt_p (op1def, op2def)))
1614 : insert_point = op2def;
1615 : else
1616 : insert_point = op1def;
1617 8570 : insert_stmt_after (sum, insert_point);
1618 : }
1619 8656 : update_stmt (sum);
1620 :
1621 8656 : return sum;
1622 : }
1623 :
1624 : /* Perform un-distribution of divisions and multiplications.
1625 : A * X + B * X is transformed into (A + B) * X and A / X + B / X
1626 : to (A + B) / X for real X.
1627 :
1628 : The algorithm is organized as follows.
1629 :
1630 : - First we walk the addition chain *OPS looking for summands that
1631 : are defined by a multiplication or a real division. This results
1632 : in the candidates bitmap with relevant indices into *OPS.
1633 :
1634 : - Second we build the chains of multiplications or divisions for
1635 : these candidates, counting the number of occurrences of (operand, code)
1636 : pairs in all of the candidates chains.
1637 :
1638 : - Third we sort the (operand, code) pairs by number of occurrence and
1639 : process them starting with the pair with the most uses.
1640 :
1641 : * For each such pair we walk the candidates again to build a
1642 : second candidate bitmap noting all multiplication/division chains
1643 : that have at least one occurrence of (operand, code).
1644 :
1645 : * We build an alternate addition chain only covering these
1646 : candidates with one (operand, code) operation removed from their
1647 : multiplication/division chain.
1648 :
1649 : * The first candidate gets replaced by the alternate addition chain
1650 : multiplied/divided by the operand.
1651 :
1652 : * All candidate chains get disabled for further processing and
1653 : processing of (operand, code) pairs continues.
1654 :
1655 : The alternate addition chains built are re-processed by the main
1656 : reassociation algorithm which allows optimizing a * x * y + b * y * x
1657 : to (a + b ) * x * y in one invocation of the reassociation pass. */
1658 :
1659 : static bool
1660 4760050 : undistribute_ops_list (enum tree_code opcode,
1661 : vec<operand_entry *> *ops, class loop *loop)
1662 : {
1663 4760050 : unsigned int length = ops->length ();
1664 4760050 : operand_entry *oe1;
1665 4760050 : unsigned i, j;
1666 4760050 : unsigned nr_candidates, nr_candidates2;
1667 4760050 : sbitmap_iterator sbi0;
1668 4760050 : vec<operand_entry *> *subops;
1669 4760050 : bool changed = false;
1670 4760050 : unsigned int next_oecount_id = 0;
1671 :
1672 4760050 : if (length <= 1
1673 4760050 : || opcode != PLUS_EXPR)
1674 : return false;
1675 :
1676 : /* Build a list of candidates to process. */
1677 2340835 : auto_sbitmap candidates (length);
1678 2340835 : bitmap_clear (candidates);
1679 2340835 : nr_candidates = 0;
1680 7198241 : FOR_EACH_VEC_ELT (*ops, i, oe1)
1681 : {
1682 4857406 : enum tree_code dcode;
1683 4857406 : gimple *oe1def;
1684 :
1685 4857406 : if (TREE_CODE (oe1->op) != SSA_NAME)
1686 1785734 : continue;
1687 3071672 : oe1def = SSA_NAME_DEF_STMT (oe1->op);
1688 3071672 : if (!is_gimple_assign (oe1def))
1689 1116581 : continue;
1690 1955091 : dcode = gimple_assign_rhs_code (oe1def);
1691 3681535 : if ((dcode != MULT_EXPR
1692 1955091 : && dcode != RDIV_EXPR)
1693 1955091 : || !is_reassociable_op (oe1def, dcode, loop))
1694 1726444 : continue;
1695 :
1696 228647 : bitmap_set_bit (candidates, i);
1697 228647 : nr_candidates++;
1698 : }
1699 :
1700 2340835 : if (nr_candidates < 2)
1701 : return false;
1702 :
1703 16344 : if (dump_file && (dump_flags & TDF_DETAILS))
1704 : {
1705 1 : fprintf (dump_file, "searching for un-distribute opportunities ");
1706 2 : print_generic_expr (dump_file,
1707 1 : (*ops)[bitmap_first_set_bit (candidates)]->op, TDF_NONE);
1708 1 : fprintf (dump_file, " %d\n", nr_candidates);
1709 : }
1710 :
1711 : /* Build linearized sub-operand lists and the counting table. */
1712 16344 : cvec.create (0);
1713 :
1714 16344 : hash_table<oecount_hasher> ctable (15);
1715 :
1716 : /* ??? Macro arguments cannot have multi-argument template types in
1717 : them. This typedef is needed to workaround that limitation. */
1718 16344 : typedef vec<operand_entry *> vec_operand_entry_t_heap;
1719 32688 : subops = XCNEWVEC (vec_operand_entry_t_heap, ops->length ());
1720 72721 : EXECUTE_IF_SET_IN_BITMAP (candidates, 0, i, sbi0)
1721 : {
1722 40033 : gimple *oedef;
1723 40033 : enum tree_code oecode;
1724 40033 : unsigned j;
1725 :
1726 40033 : oedef = SSA_NAME_DEF_STMT ((*ops)[i]->op);
1727 40033 : oecode = gimple_assign_rhs_code (oedef);
1728 40033 : linearize_expr_tree (&subops[i], oedef,
1729 : associative_tree_code (oecode), false);
1730 :
1731 200853 : FOR_EACH_VEC_ELT (subops[i], j, oe1)
1732 : {
1733 80754 : oecount c;
1734 80754 : int *slot;
1735 80754 : int idx;
1736 80754 : c.oecode = oecode;
1737 80754 : c.cnt = 1;
1738 80754 : c.id = next_oecount_id++;
1739 80754 : c.op = oe1->op;
1740 80754 : cvec.safe_push (c);
1741 80754 : idx = cvec.length () + 41;
1742 80754 : slot = ctable.find_slot (idx, INSERT);
1743 80754 : if (!*slot)
1744 : {
1745 79738 : *slot = idx;
1746 : }
1747 : else
1748 : {
1749 1016 : cvec.pop ();
1750 1016 : cvec[*slot - 42].cnt++;
1751 : }
1752 : }
1753 : }
1754 :
1755 : /* Sort the counting table. */
1756 16344 : cvec.qsort (oecount_cmp);
1757 :
1758 16344 : if (dump_file && (dump_flags & TDF_DETAILS))
1759 : {
1760 1 : oecount *c;
1761 1 : fprintf (dump_file, "Candidates:\n");
1762 5 : FOR_EACH_VEC_ELT (cvec, j, c)
1763 : {
1764 3 : fprintf (dump_file, " %u %s: ", c->cnt,
1765 3 : c->oecode == MULT_EXPR
1766 : ? "*" : c->oecode == RDIV_EXPR ? "/" : "?");
1767 3 : print_generic_expr (dump_file, c->op);
1768 3 : fprintf (dump_file, "\n");
1769 : }
1770 : }
1771 :
1772 : /* Process the (operand, code) pairs in order of most occurrence. */
1773 16344 : auto_sbitmap candidates2 (length);
1774 33398 : while (!cvec.is_empty ())
1775 : {
1776 16984 : oecount *c = &cvec.last ();
1777 16984 : if (c->cnt < 2)
1778 : break;
1779 :
1780 : /* Now collect the operands in the outer chain that contain
1781 : the common operand in their inner chain. */
1782 710 : bitmap_clear (candidates2);
1783 710 : nr_candidates2 = 0;
1784 4718 : EXECUTE_IF_SET_IN_BITMAP (candidates, 0, i, sbi0)
1785 : {
1786 3298 : gimple *oedef;
1787 3298 : enum tree_code oecode;
1788 3298 : unsigned j;
1789 3298 : tree op = (*ops)[i]->op;
1790 :
1791 : /* If we undistributed in this chain already this may be
1792 : a constant. */
1793 3298 : if (TREE_CODE (op) != SSA_NAME)
1794 762 : continue;
1795 :
1796 2536 : oedef = SSA_NAME_DEF_STMT (op);
1797 2536 : oecode = gimple_assign_rhs_code (oedef);
1798 2536 : if (oecode != c->oecode)
1799 2 : continue;
1800 :
1801 8935 : FOR_EACH_VEC_ELT (subops[i], j, oe1)
1802 : {
1803 4588 : if (oe1->op == c->op)
1804 : {
1805 1218 : bitmap_set_bit (candidates2, i);
1806 1218 : ++nr_candidates2;
1807 1218 : break;
1808 : }
1809 : }
1810 : }
1811 :
1812 710 : if (nr_candidates2 >= 2)
1813 : {
1814 336 : operand_entry *oe1, *oe2;
1815 336 : gimple *prod;
1816 336 : int first = bitmap_first_set_bit (candidates2);
1817 :
1818 : /* Build the new addition chain. */
1819 336 : oe1 = (*ops)[first];
1820 336 : if (dump_file && (dump_flags & TDF_DETAILS))
1821 : {
1822 0 : fprintf (dump_file, "Building (");
1823 0 : print_generic_expr (dump_file, oe1->op);
1824 : }
1825 336 : zero_one_operation (&oe1->op, c->oecode, c->op);
1826 1226 : EXECUTE_IF_SET_IN_BITMAP (candidates2, first+1, i, sbi0)
1827 : {
1828 554 : gimple *sum;
1829 554 : oe2 = (*ops)[i];
1830 554 : if (dump_file && (dump_flags & TDF_DETAILS))
1831 : {
1832 0 : fprintf (dump_file, " + ");
1833 0 : print_generic_expr (dump_file, oe2->op);
1834 : }
1835 554 : zero_one_operation (&oe2->op, c->oecode, c->op);
1836 554 : sum = build_and_add_sum (TREE_TYPE (oe1->op),
1837 : oe1->op, oe2->op, opcode);
1838 554 : oe2->op = build_zero_cst (TREE_TYPE (oe2->op));
1839 554 : oe2->rank = 0;
1840 554 : oe1->op = gimple_get_lhs (sum);
1841 : }
1842 :
1843 : /* Apply the multiplication/division. */
1844 336 : prod = build_and_add_sum (TREE_TYPE (oe1->op),
1845 : oe1->op, c->op, c->oecode);
1846 336 : if (dump_file && (dump_flags & TDF_DETAILS))
1847 : {
1848 0 : fprintf (dump_file, ") %s ", c->oecode == MULT_EXPR ? "*" : "/");
1849 0 : print_generic_expr (dump_file, c->op);
1850 0 : fprintf (dump_file, "\n");
1851 : }
1852 :
1853 : /* Record it in the addition chain and disable further
1854 : undistribution with this op. */
1855 336 : oe1->op = gimple_assign_lhs (prod);
1856 336 : oe1->rank = get_rank (oe1->op);
1857 336 : subops[first].release ();
1858 :
1859 336 : changed = true;
1860 : }
1861 :
1862 710 : cvec.pop ();
1863 : }
1864 :
1865 71212 : for (i = 0; i < ops->length (); ++i)
1866 54868 : subops[i].release ();
1867 16344 : free (subops);
1868 16344 : cvec.release ();
1869 :
1870 16344 : return changed;
1871 2340835 : }
1872 :
1873 : /* Pair to hold the information of one specific VECTOR_TYPE SSA_NAME:
1874 : first: element index for each relevant BIT_FIELD_REF.
1875 : second: the index of vec ops* for each relevant BIT_FIELD_REF. */
1876 : typedef std::pair<unsigned, unsigned> v_info_elem;
1877 7995 : struct v_info {
1878 : tree vec_type;
1879 : auto_vec<v_info_elem, 32> vec;
1880 : };
1881 : typedef v_info *v_info_ptr;
1882 :
1883 : /* Comparison function for qsort on VECTOR SSA_NAME trees by machine mode. */
1884 : static int
1885 10558 : sort_by_mach_mode (const void *p_i, const void *p_j)
1886 : {
1887 10558 : const tree tr1 = *((const tree *) p_i);
1888 10558 : const tree tr2 = *((const tree *) p_j);
1889 10558 : unsigned int mode1 = TYPE_MODE (TREE_TYPE (tr1));
1890 10558 : unsigned int mode2 = TYPE_MODE (TREE_TYPE (tr2));
1891 10558 : if (mode1 > mode2)
1892 : return 1;
1893 10528 : else if (mode1 < mode2)
1894 : return -1;
1895 10491 : if (SSA_NAME_VERSION (tr1) < SSA_NAME_VERSION (tr2))
1896 : return -1;
1897 5054 : else if (SSA_NAME_VERSION (tr1) > SSA_NAME_VERSION (tr2))
1898 5054 : return 1;
1899 : return 0;
1900 : }
1901 :
1902 : /* Cleanup hash map for VECTOR information. */
1903 : static void
1904 4534112 : cleanup_vinfo_map (hash_map<tree, v_info_ptr> &info_map)
1905 : {
1906 4542107 : for (hash_map<tree, v_info_ptr>::iterator it = info_map.begin ();
1907 4542107 : it != info_map.end (); ++it)
1908 : {
1909 7995 : v_info_ptr info = (*it).second;
1910 7995 : delete info;
1911 7995 : (*it).second = NULL;
1912 : }
1913 4534112 : }
1914 :
1915 : /* Perform un-distribution of BIT_FIELD_REF on VECTOR_TYPE.
1916 : V1[0] + V1[1] + ... + V1[k] + V2[0] + V2[1] + ... + V2[k] + ... Vn[k]
1917 : is transformed to
1918 : Vs = (V1 + V2 + ... + Vn)
1919 : Vs[0] + Vs[1] + ... + Vs[k]
1920 :
1921 : The basic steps are listed below:
1922 :
1923 : 1) Check the addition chain *OPS by looking those summands coming from
1924 : VECTOR bit_field_ref on VECTOR type. Put the information into
1925 : v_info_map for each satisfied summand, using VECTOR SSA_NAME as key.
1926 :
1927 : 2) For each key (VECTOR SSA_NAME), validate all its BIT_FIELD_REFs are
1928 : continuous, they can cover the whole VECTOR perfectly without any holes.
1929 : Obtain one VECTOR list which contain candidates to be transformed.
1930 :
1931 : 3) Sort the VECTOR list by machine mode of VECTOR type, for each group of
1932 : candidates with same mode, build the addition statements for them and
1933 : generate BIT_FIELD_REFs accordingly.
1934 :
1935 : TODO:
1936 : The current implementation requires the whole VECTORs should be fully
1937 : covered, but it can be extended to support partial, checking adjacent
1938 : but not fill the whole, it may need some cost model to define the
1939 : boundary to do or not.
1940 : */
1941 : static bool
1942 4760050 : undistribute_bitref_for_vector (enum tree_code opcode,
1943 : vec<operand_entry *> *ops, struct loop *loop)
1944 : {
1945 4760050 : if (ops->length () <= 1)
1946 : return false;
1947 :
1948 4756712 : if (opcode != PLUS_EXPR
1949 4756712 : && opcode != MULT_EXPR
1950 : && opcode != BIT_XOR_EXPR
1951 1281494 : && opcode != BIT_IOR_EXPR
1952 883763 : && opcode != BIT_AND_EXPR)
1953 : return false;
1954 :
1955 4534112 : hash_map<tree, v_info_ptr> v_info_map;
1956 4534112 : operand_entry *oe1;
1957 4534112 : unsigned i;
1958 :
1959 : /* Find those summands from VECTOR BIT_FIELD_REF in addition chain, put the
1960 : information into map. */
1961 18378541 : FOR_EACH_VEC_ELT (*ops, i, oe1)
1962 : {
1963 9310317 : enum tree_code dcode;
1964 9310317 : gimple *oe1def;
1965 :
1966 9310317 : if (TREE_CODE (oe1->op) != SSA_NAME)
1967 3265644 : continue;
1968 6044673 : oe1def = SSA_NAME_DEF_STMT (oe1->op);
1969 6044673 : if (!is_gimple_assign (oe1def))
1970 1527767 : continue;
1971 4516906 : dcode = gimple_assign_rhs_code (oe1def);
1972 4516906 : if (dcode != BIT_FIELD_REF || !is_reassociable_op (oe1def, dcode, loop))
1973 4462022 : continue;
1974 :
1975 54884 : tree rhs = gimple_assign_rhs1 (oe1def);
1976 54884 : tree vec = TREE_OPERAND (rhs, 0);
1977 54884 : tree vec_type = TREE_TYPE (vec);
1978 :
1979 54884 : if (TREE_CODE (vec) != SSA_NAME || !VECTOR_TYPE_P (vec_type))
1980 30419 : continue;
1981 :
1982 : /* Ignore it if target machine can't support this VECTOR type. */
1983 24465 : if (!VECTOR_MODE_P (TYPE_MODE (vec_type)))
1984 5601 : continue;
1985 :
1986 : /* Check const vector type, constrain BIT_FIELD_REF offset and size. */
1987 18864 : if (!TYPE_VECTOR_SUBPARTS (vec_type).is_constant ())
1988 : continue;
1989 :
1990 18864 : if (VECTOR_TYPE_P (TREE_TYPE (rhs))
1991 18864 : || !is_a <scalar_mode> (TYPE_MODE (TREE_TYPE (rhs))))
1992 7810 : continue;
1993 :
1994 : /* The type of BIT_FIELD_REF might not be equal to the element type of
1995 : the vector. We want to use a vector type with element type the
1996 : same as the BIT_FIELD_REF and size the same as TREE_TYPE (vec). */
1997 11054 : if (!useless_type_conversion_p (TREE_TYPE (rhs), TREE_TYPE (vec_type)))
1998 : {
1999 1901 : machine_mode simd_mode;
2000 1901 : unsigned HOST_WIDE_INT size, nunits;
2001 1901 : unsigned HOST_WIDE_INT elem_size
2002 1901 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs)));
2003 3802 : if (!GET_MODE_BITSIZE (TYPE_MODE (vec_type)).is_constant (&size))
2004 9299697 : continue;
2005 1901 : if (size <= elem_size || (size % elem_size) != 0)
2006 0 : continue;
2007 1901 : nunits = size / elem_size;
2008 1901 : if (!mode_for_vector (SCALAR_TYPE_MODE (TREE_TYPE (rhs)),
2009 1901 : nunits).exists (&simd_mode))
2010 0 : continue;
2011 1901 : vec_type = build_vector_type_for_mode (TREE_TYPE (rhs), simd_mode);
2012 :
2013 : /* Ignore it if target machine can't support this VECTOR type. */
2014 1901 : if (!VECTOR_MODE_P (TYPE_MODE (vec_type)))
2015 0 : continue;
2016 :
2017 : /* Check const vector type, constrain BIT_FIELD_REF offset and
2018 : size. */
2019 1901 : if (!TYPE_VECTOR_SUBPARTS (vec_type).is_constant ())
2020 : continue;
2021 :
2022 3802 : if (maybe_ne (GET_MODE_SIZE (TYPE_MODE (vec_type)),
2023 3802 : GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (vec)))))
2024 0 : continue;
2025 : }
2026 :
2027 11054 : tree elem_type = TREE_TYPE (vec_type);
2028 11054 : unsigned HOST_WIDE_INT elem_size = tree_to_uhwi (TYPE_SIZE (elem_type));
2029 11054 : if (maybe_ne (bit_field_size (rhs), elem_size))
2030 0 : continue;
2031 :
2032 11054 : unsigned idx;
2033 11054 : if (!constant_multiple_p (bit_field_offset (rhs), elem_size, &idx))
2034 0 : continue;
2035 :
2036 : /* Ignore it if target machine can't support this type of VECTOR
2037 : operation. */
2038 11054 : optab op_tab = optab_for_tree_code (opcode, vec_type, optab_vector);
2039 11054 : if (optab_handler (op_tab, TYPE_MODE (vec_type)) == CODE_FOR_nothing)
2040 434 : continue;
2041 :
2042 10620 : bool existed;
2043 10620 : v_info_ptr &info = v_info_map.get_or_insert (vec, &existed);
2044 10620 : if (!existed)
2045 : {
2046 7995 : info = new v_info;
2047 7995 : info->vec_type = vec_type;
2048 : }
2049 2625 : else if (!types_compatible_p (vec_type, info->vec_type))
2050 0 : continue;
2051 10620 : info->vec.safe_push (std::make_pair (idx, i));
2052 : }
2053 :
2054 : /* At least two VECTOR to combine. */
2055 4534112 : if (v_info_map.elements () <= 1)
2056 : {
2057 4533863 : cleanup_vinfo_map (v_info_map);
2058 4533863 : return false;
2059 : }
2060 :
2061 : /* Verify all VECTOR candidates by checking two conditions:
2062 : 1) sorted offsets are adjacent, no holes.
2063 : 2) can fill the whole VECTOR perfectly.
2064 : And add the valid candidates to a vector for further handling. */
2065 249 : auto_vec<tree> valid_vecs (v_info_map.elements ());
2066 1211 : for (hash_map<tree, v_info_ptr>::iterator it = v_info_map.begin ();
2067 1211 : it != v_info_map.end (); ++it)
2068 : {
2069 962 : tree cand_vec = (*it).first;
2070 962 : v_info_ptr cand_info = (*it).second;
2071 962 : unsigned int num_elems
2072 962 : = TYPE_VECTOR_SUBPARTS (cand_info->vec_type).to_constant ();
2073 1924 : if (cand_info->vec.length () != num_elems)
2074 597 : continue;
2075 365 : sbitmap holes = sbitmap_alloc (num_elems);
2076 365 : bitmap_ones (holes);
2077 365 : bool valid = true;
2078 365 : v_info_elem *curr;
2079 2124 : FOR_EACH_VEC_ELT (cand_info->vec, i, curr)
2080 : {
2081 1394 : if (!bitmap_bit_p (holes, curr->first))
2082 : {
2083 : valid = false;
2084 : break;
2085 : }
2086 : else
2087 1394 : bitmap_clear_bit (holes, curr->first);
2088 : }
2089 365 : if (valid && bitmap_empty_p (holes))
2090 365 : valid_vecs.quick_push (cand_vec);
2091 365 : sbitmap_free (holes);
2092 : }
2093 :
2094 : /* At least two VECTOR to combine. */
2095 249 : if (valid_vecs.length () <= 1)
2096 : {
2097 213 : cleanup_vinfo_map (v_info_map);
2098 213 : return false;
2099 : }
2100 :
2101 36 : valid_vecs.qsort (sort_by_mach_mode);
2102 : /* Go through all candidates by machine mode order, query the mode_to_total
2103 : to get the total number for each mode and skip the single one. */
2104 112 : for (unsigned i = 0; i < valid_vecs.length () - 1; ++i)
2105 : {
2106 40 : tree tvec = valid_vecs[i];
2107 40 : enum machine_mode mode = TYPE_MODE (TREE_TYPE (tvec));
2108 :
2109 : /* Skip modes with only a single candidate. */
2110 40 : if (TYPE_MODE (TREE_TYPE (valid_vecs[i + 1])) != mode)
2111 0 : continue;
2112 :
2113 40 : unsigned int idx, j;
2114 40 : gimple *sum = NULL;
2115 40 : tree sum_vec = tvec;
2116 40 : v_info_ptr info_ptr = *(v_info_map.get (tvec));
2117 40 : v_info_elem *elem;
2118 40 : tree vec_type = info_ptr->vec_type;
2119 :
2120 : /* Build the sum for all candidates with same mode. */
2121 323 : do
2122 : {
2123 969 : sum = build_and_add_sum (vec_type, sum_vec,
2124 323 : valid_vecs[i + 1], opcode);
2125 : /* Update the operands only after build_and_add_sum,
2126 : so that we don't have to repeat the placement algorithm
2127 : of build_and_add_sum. */
2128 323 : if (sum_vec == tvec
2129 323 : && !useless_type_conversion_p (vec_type, TREE_TYPE (sum_vec)))
2130 : {
2131 18 : gimple_stmt_iterator gsi = gsi_for_stmt (sum);
2132 18 : tree vce = build1 (VIEW_CONVERT_EXPR, vec_type, sum_vec);
2133 18 : tree lhs = make_ssa_name (vec_type);
2134 18 : gimple *g = gimple_build_assign (lhs, VIEW_CONVERT_EXPR, vce);
2135 18 : gimple_set_uid (g, gimple_uid (sum));
2136 18 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
2137 18 : gimple_assign_set_rhs1 (sum, lhs);
2138 18 : update_stmt (sum);
2139 : }
2140 323 : if (!useless_type_conversion_p (vec_type,
2141 323 : TREE_TYPE (valid_vecs[i + 1])))
2142 : {
2143 270 : gimple_stmt_iterator gsi = gsi_for_stmt (sum);
2144 810 : tree vce = build1 (VIEW_CONVERT_EXPR, vec_type,
2145 270 : valid_vecs[i + 1]);
2146 270 : tree lhs = make_ssa_name (vec_type);
2147 270 : gimple *g = gimple_build_assign (lhs, VIEW_CONVERT_EXPR, vce);
2148 270 : gimple_set_uid (g, gimple_uid (sum));
2149 270 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
2150 270 : gimple_assign_set_rhs2 (sum, lhs);
2151 270 : update_stmt (sum);
2152 : }
2153 323 : sum_vec = gimple_get_lhs (sum);
2154 323 : info_ptr = *(v_info_map.get (valid_vecs[i + 1]));
2155 323 : gcc_assert (types_compatible_p (vec_type, info_ptr->vec_type));
2156 : /* Update those related ops of current candidate VECTOR. */
2157 1569 : FOR_EACH_VEC_ELT (info_ptr->vec, j, elem)
2158 : {
2159 1246 : idx = elem->second;
2160 1246 : gimple *def = SSA_NAME_DEF_STMT ((*ops)[idx]->op);
2161 : /* Set this then op definition will get DCEd later. */
2162 1246 : gimple_set_visited (def, true);
2163 1246 : if (opcode == PLUS_EXPR
2164 1246 : || opcode == BIT_XOR_EXPR
2165 96 : || opcode == BIT_IOR_EXPR)
2166 1186 : (*ops)[idx]->op = build_zero_cst (TREE_TYPE ((*ops)[idx]->op));
2167 60 : else if (opcode == MULT_EXPR)
2168 24 : (*ops)[idx]->op = build_one_cst (TREE_TYPE ((*ops)[idx]->op));
2169 : else
2170 : {
2171 36 : gcc_assert (opcode == BIT_AND_EXPR);
2172 36 : (*ops)[idx]->op
2173 36 : = build_all_ones_cst (TREE_TYPE ((*ops)[idx]->op));
2174 : }
2175 1246 : (*ops)[idx]->rank = 0;
2176 : }
2177 323 : if (dump_file && (dump_flags & TDF_DETAILS))
2178 : {
2179 0 : fprintf (dump_file, "Generating addition -> ");
2180 0 : print_gimple_stmt (dump_file, sum, 0);
2181 : }
2182 323 : i++;
2183 : }
2184 323 : while ((i < valid_vecs.length () - 1)
2185 363 : && TYPE_MODE (TREE_TYPE (valid_vecs[i + 1])) == mode);
2186 :
2187 : /* Referring to first valid VECTOR with this mode, generate the
2188 : BIT_FIELD_REF statements accordingly. */
2189 40 : info_ptr = *(v_info_map.get (tvec));
2190 40 : gcc_assert (sum);
2191 40 : tree elem_type = TREE_TYPE (vec_type);
2192 224 : FOR_EACH_VEC_ELT (info_ptr->vec, j, elem)
2193 : {
2194 144 : idx = elem->second;
2195 144 : tree dst = make_ssa_name (elem_type);
2196 144 : tree pos = bitsize_int (elem->first
2197 : * tree_to_uhwi (TYPE_SIZE (elem_type)));
2198 144 : tree bfr = build3 (BIT_FIELD_REF, elem_type, sum_vec,
2199 144 : TYPE_SIZE (elem_type), pos);
2200 144 : gimple *gs = gimple_build_assign (dst, BIT_FIELD_REF, bfr);
2201 144 : insert_stmt_after (gs, sum);
2202 144 : gimple *def = SSA_NAME_DEF_STMT ((*ops)[idx]->op);
2203 : /* Set this then op definition will get DCEd later. */
2204 144 : gimple_set_visited (def, true);
2205 144 : (*ops)[idx]->op = gimple_assign_lhs (gs);
2206 144 : (*ops)[idx]->rank = get_rank ((*ops)[idx]->op);
2207 144 : if (dump_file && (dump_flags & TDF_DETAILS))
2208 : {
2209 0 : fprintf (dump_file, "Generating bit_field_ref -> ");
2210 0 : print_gimple_stmt (dump_file, gs, 0);
2211 : }
2212 : }
2213 : }
2214 :
2215 36 : if (dump_file && (dump_flags & TDF_DETAILS))
2216 0 : fprintf (dump_file, "undistributiong bit_field_ref for vector done.\n");
2217 :
2218 36 : cleanup_vinfo_map (v_info_map);
2219 :
2220 36 : return true;
2221 4534361 : }
2222 :
2223 : /* If OPCODE is BIT_IOR_EXPR or BIT_AND_EXPR and CURR is a comparison
2224 : expression, examine the other OPS to see if any of them are comparisons
2225 : of the same values, which we may be able to combine or eliminate.
2226 : For example, we can rewrite (a < b) | (a == b) as (a <= b). */
2227 :
2228 : static bool
2229 9766679 : eliminate_redundant_comparison (enum tree_code opcode,
2230 : vec<operand_entry *> *ops,
2231 : unsigned int currindex,
2232 : operand_entry *curr)
2233 : {
2234 9766679 : tree op1, op2;
2235 9766679 : enum tree_code lcode, rcode;
2236 9766679 : gimple *def1, *def2;
2237 9766679 : int i;
2238 9766679 : operand_entry *oe;
2239 :
2240 9766679 : if (opcode != BIT_IOR_EXPR && opcode != BIT_AND_EXPR)
2241 : return false;
2242 :
2243 : /* Check that CURR is a comparison. */
2244 2047157 : if (TREE_CODE (curr->op) != SSA_NAME)
2245 : return false;
2246 1542981 : def1 = SSA_NAME_DEF_STMT (curr->op);
2247 1542981 : if (!is_gimple_assign (def1))
2248 : return false;
2249 1331033 : lcode = gimple_assign_rhs_code (def1);
2250 1331033 : if (TREE_CODE_CLASS (lcode) != tcc_comparison)
2251 : return false;
2252 537604 : op1 = gimple_assign_rhs1 (def1);
2253 537604 : op2 = gimple_assign_rhs2 (def1);
2254 :
2255 : /* Now look for a similar comparison in the remaining OPS. */
2256 897200 : for (i = currindex + 1; ops->iterate (i, &oe); i++)
2257 : {
2258 359830 : tree t;
2259 :
2260 359830 : if (TREE_CODE (oe->op) != SSA_NAME)
2261 41 : continue;
2262 359789 : def2 = SSA_NAME_DEF_STMT (oe->op);
2263 359789 : if (!is_gimple_assign (def2))
2264 7450 : continue;
2265 352339 : rcode = gimple_assign_rhs_code (def2);
2266 352339 : if (TREE_CODE_CLASS (rcode) != tcc_comparison)
2267 6544 : continue;
2268 :
2269 : /* If we got here, we have a match. See if we can combine the
2270 : two comparisons. */
2271 345795 : tree type = TREE_TYPE (gimple_assign_lhs (def1));
2272 345795 : if (opcode == BIT_IOR_EXPR)
2273 196778 : t = maybe_fold_or_comparisons (type, TRUTH_OR_EXPR,
2274 : lcode, op1, op2,
2275 : rcode, gimple_assign_rhs1 (def2),
2276 : gimple_assign_rhs2 (def2));
2277 : else
2278 149017 : t = maybe_fold_and_comparisons (type, TRUTH_AND_EXPR,
2279 : lcode, op1, op2,
2280 : rcode, gimple_assign_rhs1 (def2),
2281 : gimple_assign_rhs2 (def2));
2282 345795 : if (!t)
2283 345513 : continue;
2284 :
2285 : /* maybe_fold_and_comparisons and maybe_fold_or_comparisons
2286 : always give us a boolean_type_node value back. If the original
2287 : BIT_AND_EXPR or BIT_IOR_EXPR was of a wider integer type,
2288 : we need to convert. */
2289 282 : if (!useless_type_conversion_p (TREE_TYPE (curr->op), TREE_TYPE (t)))
2290 : {
2291 2 : if (!fold_convertible_p (TREE_TYPE (curr->op), t))
2292 0 : continue;
2293 2 : t = fold_convert (TREE_TYPE (curr->op), t);
2294 : }
2295 :
2296 282 : if (TREE_CODE (t) != INTEGER_CST
2297 282 : && !operand_equal_p (t, curr->op, 0))
2298 : {
2299 278 : enum tree_code subcode;
2300 278 : tree newop1, newop2;
2301 278 : if (!COMPARISON_CLASS_P (t))
2302 48 : continue;
2303 250 : extract_ops_from_tree (t, &subcode, &newop1, &newop2);
2304 250 : STRIP_USELESS_TYPE_CONVERSION (newop1);
2305 250 : STRIP_USELESS_TYPE_CONVERSION (newop2);
2306 250 : if (!is_gimple_val (newop1) || !is_gimple_val (newop2))
2307 0 : continue;
2308 250 : if (lcode == TREE_CODE (t)
2309 182 : && operand_equal_p (op1, newop1, 0)
2310 432 : && operand_equal_p (op2, newop2, 0))
2311 137 : t = curr->op;
2312 133 : else if ((TREE_CODE (newop1) == SSA_NAME
2313 113 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (newop1))
2314 206 : || (TREE_CODE (newop2) == SSA_NAME
2315 53 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (newop2)))
2316 20 : continue;
2317 : }
2318 :
2319 234 : if (dump_file && (dump_flags & TDF_DETAILS))
2320 : {
2321 6 : fprintf (dump_file, "Equivalence: ");
2322 6 : print_generic_expr (dump_file, curr->op);
2323 6 : fprintf (dump_file, " %s ", op_symbol_code (opcode));
2324 6 : print_generic_expr (dump_file, oe->op);
2325 6 : fprintf (dump_file, " -> ");
2326 6 : print_generic_expr (dump_file, t);
2327 6 : fprintf (dump_file, "\n");
2328 : }
2329 :
2330 : /* Now we can delete oe, as it has been subsumed by the new combined
2331 : expression t. */
2332 234 : ops->ordered_remove (i);
2333 234 : reassociate_stats.ops_eliminated ++;
2334 :
2335 : /* If t is the same as curr->op, we're done. Otherwise we must
2336 : replace curr->op with t. Special case is if we got a constant
2337 : back, in which case we add it to the end instead of in place of
2338 : the current entry. */
2339 234 : if (TREE_CODE (t) == INTEGER_CST)
2340 : {
2341 4 : ops->ordered_remove (currindex);
2342 4 : add_to_ops_vec (ops, t);
2343 : }
2344 230 : else if (!operand_equal_p (t, curr->op, 0))
2345 : {
2346 93 : gimple *sum;
2347 93 : enum tree_code subcode;
2348 93 : tree newop1;
2349 93 : tree newop2;
2350 93 : gcc_assert (COMPARISON_CLASS_P (t));
2351 93 : extract_ops_from_tree (t, &subcode, &newop1, &newop2);
2352 93 : STRIP_USELESS_TYPE_CONVERSION (newop1);
2353 93 : STRIP_USELESS_TYPE_CONVERSION (newop2);
2354 93 : gcc_checking_assert (is_gimple_val (newop1)
2355 : && is_gimple_val (newop2));
2356 93 : sum = build_and_add_sum (TREE_TYPE (t), newop1, newop2, subcode);
2357 93 : curr->op = gimple_get_lhs (sum);
2358 : }
2359 : return true;
2360 : }
2361 :
2362 : return false;
2363 : }
2364 :
2365 :
2366 : /* Transform repeated addition of same values into multiply with
2367 : constant. */
2368 : static bool
2369 2343977 : transform_add_to_multiply (vec<operand_entry *> *ops)
2370 : {
2371 2343977 : operand_entry *oe;
2372 2343977 : tree op = NULL_TREE;
2373 2343977 : int j;
2374 2343977 : int i, start = -1, end = 0, count = 0;
2375 2343977 : auto_vec<std::pair <int, int> > indxs;
2376 2343977 : bool changed = false;
2377 :
2378 2343977 : if (!INTEGRAL_TYPE_P (TREE_TYPE ((*ops)[0]->op))
2379 69009 : && (!SCALAR_FLOAT_TYPE_P (TREE_TYPE ((*ops)[0]->op))
2380 31859 : || !flag_unsafe_math_optimizations))
2381 : return false;
2382 :
2383 : /* Look for repeated operands. */
2384 7081557 : FOR_EACH_VEC_ELT (*ops, i, oe)
2385 : {
2386 4774791 : if (start == -1)
2387 : {
2388 2306766 : count = 1;
2389 2306766 : op = oe->op;
2390 2306766 : start = i;
2391 : }
2392 2468025 : else if (operand_equal_p (oe->op, op, 0))
2393 : {
2394 155 : count++;
2395 155 : end = i;
2396 : }
2397 : else
2398 : {
2399 2467870 : if (count > 1)
2400 47 : indxs.safe_push (std::make_pair (start, end));
2401 2467870 : count = 1;
2402 2467870 : op = oe->op;
2403 2467870 : start = i;
2404 : }
2405 : }
2406 :
2407 2306766 : if (count > 1)
2408 38 : indxs.safe_push (std::make_pair (start, end));
2409 :
2410 2306924 : for (j = indxs.length () - 1; j >= 0; --j)
2411 : {
2412 : /* Convert repeated operand addition to multiplication. */
2413 85 : start = indxs[j].first;
2414 85 : end = indxs[j].second;
2415 85 : op = (*ops)[start]->op;
2416 85 : count = end - start + 1;
2417 325 : for (i = end; i >= start; --i)
2418 240 : ops->unordered_remove (i);
2419 85 : tree tmp = make_ssa_name (TREE_TYPE (op));
2420 85 : tree cst = build_int_cst (integer_type_node, count);
2421 85 : gassign *mul_stmt
2422 85 : = gimple_build_assign (tmp, MULT_EXPR,
2423 85 : op, fold_convert (TREE_TYPE (op), cst));
2424 85 : gimple_set_visited (mul_stmt, true);
2425 85 : add_to_ops_vec (ops, tmp, mul_stmt);
2426 85 : changed = true;
2427 : }
2428 :
2429 : return changed;
2430 2343977 : }
2431 :
2432 :
2433 : /* Perform various identities and other optimizations on the list of
2434 : operand entries, stored in OPS. The tree code for the binary
2435 : operation between all the operands is OPCODE. */
2436 :
2437 : static void
2438 4760401 : optimize_ops_list (enum tree_code opcode,
2439 : vec<operand_entry *> *ops)
2440 : {
2441 4778530 : unsigned int length = ops->length ();
2442 4778530 : unsigned int i;
2443 4778530 : operand_entry *oe;
2444 9555147 : operand_entry *oelast = NULL;
2445 9555147 : bool iterate = false;
2446 :
2447 4778530 : if (length == 1)
2448 : return;
2449 :
2450 4776617 : oelast = ops->last ();
2451 :
2452 : /* If the last two are constants, pop the constants off, merge them
2453 : and try the next two. */
2454 4776617 : if (oelast->rank == 0 && is_gimple_min_invariant (oelast->op))
2455 : {
2456 3456002 : operand_entry *oelm1 = (*ops)[length - 2];
2457 :
2458 3456002 : if (oelm1->rank == 0
2459 14433 : && is_gimple_min_invariant (oelm1->op)
2460 3470435 : && useless_type_conversion_p (TREE_TYPE (oelm1->op),
2461 14433 : TREE_TYPE (oelast->op)))
2462 : {
2463 14433 : tree folded = fold_binary (opcode, TREE_TYPE (oelm1->op),
2464 : oelm1->op, oelast->op);
2465 :
2466 14433 : if (folded && is_gimple_min_invariant (folded))
2467 : {
2468 14383 : if (dump_file && (dump_flags & TDF_DETAILS))
2469 0 : fprintf (dump_file, "Merging constants\n");
2470 :
2471 14383 : ops->pop ();
2472 14383 : ops->pop ();
2473 :
2474 14383 : add_to_ops_vec (ops, folded);
2475 14383 : reassociate_stats.constants_eliminated++;
2476 :
2477 14383 : optimize_ops_list (opcode, ops);
2478 14383 : return;
2479 : }
2480 : }
2481 : }
2482 :
2483 4762234 : eliminate_using_constants (opcode, ops);
2484 4762234 : oelast = NULL;
2485 :
2486 19294791 : for (i = 0; ops->iterate (i, &oe);)
2487 : {
2488 9770325 : bool done = false;
2489 :
2490 9770325 : if (eliminate_not_pairs (opcode, ops, i, oe))
2491 2 : return;
2492 9770324 : if (eliminate_duplicate_pair (opcode, ops, &done, i, oe, oelast)
2493 9770175 : || (!done && eliminate_plus_minus_pair (opcode, ops, i, oe))
2494 19537003 : || (!done && eliminate_redundant_comparison (opcode, ops, i, oe)))
2495 : {
2496 3879 : if (done)
2497 : return;
2498 3878 : iterate = true;
2499 3878 : oelast = NULL;
2500 3878 : continue;
2501 : }
2502 9766445 : oelast = oe;
2503 9766445 : i++;
2504 : }
2505 :
2506 4762232 : if (iterate)
2507 : optimize_ops_list (opcode, ops);
2508 : }
2509 :
2510 : /* The following functions are subroutines to optimize_range_tests and allow
2511 : it to try to change a logical combination of comparisons into a range
2512 : test.
2513 :
2514 : For example, both
2515 : X == 2 || X == 5 || X == 3 || X == 4
2516 : and
2517 : X >= 2 && X <= 5
2518 : are converted to
2519 : (unsigned) (X - 2) <= 3
2520 :
2521 : For more information see comments above fold_test_range in fold-const.cc,
2522 : this implementation is for GIMPLE. */
2523 :
2524 :
2525 :
2526 : /* Dump the range entry R to FILE, skipping its expression if SKIP_EXP. */
2527 :
2528 : void
2529 141 : dump_range_entry (FILE *file, struct range_entry *r, bool skip_exp)
2530 : {
2531 141 : if (!skip_exp)
2532 59 : print_generic_expr (file, r->exp);
2533 251 : fprintf (file, " %c[", r->in_p ? '+' : '-');
2534 141 : print_generic_expr (file, r->low);
2535 141 : fputs (", ", file);
2536 141 : print_generic_expr (file, r->high);
2537 141 : fputc (']', file);
2538 141 : }
2539 :
2540 : /* Dump the range entry R to STDERR. */
2541 :
2542 : DEBUG_FUNCTION void
2543 0 : debug_range_entry (struct range_entry *r)
2544 : {
2545 0 : dump_range_entry (stderr, r, false);
2546 0 : fputc ('\n', stderr);
2547 0 : }
2548 :
2549 : /* This is similar to make_range in fold-const.cc, but on top of
2550 : GIMPLE instead of trees. If EXP is non-NULL, it should be
2551 : an SSA_NAME and STMT argument is ignored, otherwise STMT
2552 : argument should be a GIMPLE_COND. */
2553 :
2554 : void
2555 5722732 : init_range_entry (struct range_entry *r, tree exp, gimple *stmt)
2556 : {
2557 5722732 : int in_p;
2558 5722732 : tree low, high;
2559 5722732 : bool is_bool;
2560 :
2561 5722732 : r->exp = NULL_TREE;
2562 5722732 : r->in_p = false;
2563 5722732 : r->low = NULL_TREE;
2564 5722732 : r->high = NULL_TREE;
2565 5722732 : if (exp != NULL_TREE
2566 5722732 : && (TREE_CODE (exp) != SSA_NAME || !INTEGRAL_TYPE_P (TREE_TYPE (exp))))
2567 802153 : return;
2568 :
2569 : /* Start with simply saying "EXP != 0" and then look at the code of EXP
2570 : and see if we can refine the range. Some of the cases below may not
2571 : happen, but it doesn't seem worth worrying about this. We "continue"
2572 : the outer loop when we've changed something; otherwise we "break"
2573 : the switch, which will "break" the while. */
2574 5225087 : low = exp ? build_int_cst (TREE_TYPE (exp), 0) : boolean_false_node;
2575 5225087 : high = low;
2576 5225087 : in_p = 0;
2577 5225087 : is_bool = false;
2578 5225087 : if (exp == NULL_TREE)
2579 5224863 : is_bool = true;
2580 1592158 : else if (TYPE_PRECISION (TREE_TYPE (exp)) == 1)
2581 : {
2582 670181 : if (TYPE_UNSIGNED (TREE_TYPE (exp)))
2583 : is_bool = true;
2584 : else
2585 : return;
2586 : }
2587 921977 : else if (TREE_CODE (TREE_TYPE (exp)) == BOOLEAN_TYPE)
2588 4302886 : is_bool = true;
2589 :
2590 8317797 : while (1)
2591 : {
2592 8317797 : enum tree_code code;
2593 8317797 : tree arg0, arg1, exp_type;
2594 8317797 : tree nexp;
2595 8317797 : location_t loc;
2596 :
2597 8317797 : if (exp != NULL_TREE)
2598 : {
2599 4684868 : if (TREE_CODE (exp) != SSA_NAME
2600 4684868 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (exp))
2601 : break;
2602 :
2603 4684868 : stmt = SSA_NAME_DEF_STMT (exp);
2604 4684868 : if (!is_gimple_assign (stmt))
2605 : break;
2606 :
2607 2863673 : code = gimple_assign_rhs_code (stmt);
2608 2863673 : arg0 = gimple_assign_rhs1 (stmt);
2609 2863673 : arg1 = gimple_assign_rhs2 (stmt);
2610 2863673 : exp_type = TREE_TYPE (exp);
2611 : }
2612 : else
2613 : {
2614 3632929 : code = gimple_cond_code (stmt);
2615 3632929 : arg0 = gimple_cond_lhs (stmt);
2616 3632929 : arg1 = gimple_cond_rhs (stmt);
2617 3632929 : exp_type = boolean_type_node;
2618 : }
2619 :
2620 6496602 : if (TREE_CODE (arg0) != SSA_NAME
2621 5159563 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg0)
2622 11655738 : || ssa_name_maybe_undef_p (arg0))
2623 : break;
2624 5152342 : loc = gimple_location (stmt);
2625 5152342 : switch (code)
2626 : {
2627 34162 : case BIT_NOT_EXPR:
2628 34162 : if (TREE_CODE (TREE_TYPE (exp)) == BOOLEAN_TYPE
2629 : /* Ensure the range is either +[-,0], +[0,0],
2630 : -[-,0], -[0,0] or +[1,-], +[1,1], -[1,-] or
2631 : -[1,1]. If it is e.g. +[-,-] or -[-,-]
2632 : or similar expression of unconditional true or
2633 : false, it should not be negated. */
2634 34162 : && ((high && integer_zerop (high))
2635 0 : || (low && integer_onep (low))))
2636 : {
2637 6364 : in_p = !in_p;
2638 6364 : exp = arg0;
2639 6364 : continue;
2640 : }
2641 : break;
2642 1873 : case SSA_NAME:
2643 1873 : exp = arg0;
2644 1873 : continue;
2645 244651 : CASE_CONVERT:
2646 244651 : if (is_bool)
2647 : {
2648 126439 : if ((TYPE_PRECISION (exp_type) == 1
2649 119981 : || TREE_CODE (exp_type) == BOOLEAN_TYPE)
2650 126439 : && TYPE_PRECISION (TREE_TYPE (arg0)) > 1)
2651 : return;
2652 : }
2653 118212 : else if (TYPE_PRECISION (TREE_TYPE (arg0)) == 1)
2654 : {
2655 4501 : if (TYPE_UNSIGNED (TREE_TYPE (arg0)))
2656 : is_bool = true;
2657 : else
2658 : return;
2659 : }
2660 113711 : else if (TREE_CODE (TREE_TYPE (arg0)) == BOOLEAN_TYPE)
2661 128191 : is_bool = true;
2662 241902 : goto do_default;
2663 : case EQ_EXPR:
2664 : case NE_EXPR:
2665 : case LT_EXPR:
2666 : case LE_EXPR:
2667 : case GE_EXPR:
2668 : case GT_EXPR:
2669 : is_bool = true;
2670 : /* FALLTHRU */
2671 664299 : default:
2672 664299 : if (!is_bool)
2673 : return;
2674 362764 : do_default:
2675 4812023 : nexp = make_range_step (loc, code, arg0, arg1, exp_type,
2676 : &low, &high, &in_p);
2677 4812023 : if (nexp != NULL_TREE)
2678 : {
2679 3084697 : exp = nexp;
2680 3084697 : gcc_assert (TREE_CODE (exp) == SSA_NAME);
2681 3084697 : continue;
2682 : }
2683 : break;
2684 : }
2685 : break;
2686 : }
2687 4920579 : if (is_bool)
2688 : {
2689 4304638 : r->exp = exp;
2690 4304638 : r->in_p = in_p;
2691 4304638 : r->low = low;
2692 4304638 : r->high = high;
2693 : }
2694 : }
2695 :
2696 : /* Comparison function for qsort. Sort entries
2697 : without SSA_NAME exp first, then with SSA_NAMEs sorted
2698 : by increasing SSA_NAME_VERSION, and for the same SSA_NAMEs
2699 : by increasing ->low and if ->low is the same, by increasing
2700 : ->high. ->low == NULL_TREE means minimum, ->high == NULL_TREE
2701 : maximum. */
2702 :
2703 : static int
2704 6173417 : range_entry_cmp (const void *a, const void *b)
2705 : {
2706 6173417 : const struct range_entry *p = (const struct range_entry *) a;
2707 6173417 : const struct range_entry *q = (const struct range_entry *) b;
2708 :
2709 6173417 : if (p->exp != NULL_TREE && TREE_CODE (p->exp) == SSA_NAME)
2710 : {
2711 2492466 : if (q->exp != NULL_TREE && TREE_CODE (q->exp) == SSA_NAME)
2712 : {
2713 : /* Group range_entries for the same SSA_NAME together. */
2714 2440440 : if (SSA_NAME_VERSION (p->exp) < SSA_NAME_VERSION (q->exp))
2715 : return -1;
2716 1055092 : else if (SSA_NAME_VERSION (p->exp) > SSA_NAME_VERSION (q->exp))
2717 : return 1;
2718 : /* If ->low is different, NULL low goes first, then by
2719 : ascending low. */
2720 168317 : if (p->low != NULL_TREE)
2721 : {
2722 151871 : if (q->low != NULL_TREE)
2723 : {
2724 144664 : tree tem = fold_binary (LT_EXPR, boolean_type_node,
2725 : p->low, q->low);
2726 144664 : if (tem && integer_onep (tem))
2727 : return -1;
2728 70227 : tem = fold_binary (GT_EXPR, boolean_type_node,
2729 : p->low, q->low);
2730 70227 : if (tem && integer_onep (tem))
2731 : return 1;
2732 : }
2733 : else
2734 : return 1;
2735 : }
2736 16446 : else if (q->low != NULL_TREE)
2737 : return -1;
2738 : /* If ->high is different, NULL high goes last, before that by
2739 : ascending high. */
2740 30552 : if (p->high != NULL_TREE)
2741 : {
2742 30472 : if (q->high != NULL_TREE)
2743 : {
2744 30307 : tree tem = fold_binary (LT_EXPR, boolean_type_node,
2745 : p->high, q->high);
2746 30307 : if (tem && integer_onep (tem))
2747 : return -1;
2748 8976 : tem = fold_binary (GT_EXPR, boolean_type_node,
2749 : p->high, q->high);
2750 8976 : if (tem && integer_onep (tem))
2751 : return 1;
2752 : }
2753 : else
2754 : return -1;
2755 : }
2756 80 : else if (q->high != NULL_TREE)
2757 : return 1;
2758 : /* If both ranges are the same, sort below by ascending idx. */
2759 : }
2760 : else
2761 : return 1;
2762 : }
2763 3680951 : else if (q->exp != NULL_TREE && TREE_CODE (q->exp) == SSA_NAME)
2764 : return -1;
2765 :
2766 3609132 : if (p->idx < q->idx)
2767 : return -1;
2768 : else
2769 : {
2770 1821253 : gcc_checking_assert (p->idx > q->idx);
2771 : return 1;
2772 : }
2773 : }
2774 :
2775 : /* Helper function for update_range_test. Force EXPR into an SSA_NAME,
2776 : insert needed statements BEFORE or after GSI. */
2777 :
2778 : static tree
2779 25319 : force_into_ssa_name (gimple_stmt_iterator *gsi, tree expr, bool before)
2780 : {
2781 25319 : enum gsi_iterator_update m = before ? GSI_SAME_STMT : GSI_CONTINUE_LINKING;
2782 25319 : tree ret = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE, before, m);
2783 25319 : if (TREE_CODE (ret) != SSA_NAME)
2784 : {
2785 34 : gimple *g = gimple_build_assign (make_ssa_name (TREE_TYPE (ret)), ret);
2786 34 : if (before)
2787 34 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
2788 : else
2789 0 : gsi_insert_after (gsi, g, GSI_CONTINUE_LINKING);
2790 34 : ret = gimple_assign_lhs (g);
2791 : }
2792 25319 : return ret;
2793 : }
2794 :
2795 : /* Helper routine of optimize_range_test.
2796 : [EXP, IN_P, LOW, HIGH] is a merged range for
2797 : RANGE and OTHERRANGE through OTHERRANGE + COUNT - 1 ranges,
2798 : OPCODE and OPS are arguments of optimize_range_tests. If OTHERRANGE
2799 : is NULL, OTHERRANGEP should not be and then OTHERRANGEP points to
2800 : an array of COUNT pointers to other ranges. Return
2801 : true if the range merge has been successful.
2802 : If OPCODE is ERROR_MARK, this is called from within
2803 : maybe_optimize_range_tests and is performing inter-bb range optimization.
2804 : In that case, whether an op is BIT_AND_EXPR or BIT_IOR_EXPR is found in
2805 : oe->rank. */
2806 :
2807 : static bool
2808 25319 : update_range_test (struct range_entry *range, struct range_entry *otherrange,
2809 : struct range_entry **otherrangep,
2810 : unsigned int count, enum tree_code opcode,
2811 : vec<operand_entry *> *ops, tree exp, gimple_seq seq,
2812 : bool in_p, tree low, tree high)
2813 : {
2814 25319 : unsigned int idx = range->idx;
2815 25319 : struct range_entry *swap_with = NULL;
2816 25319 : basic_block rewrite_bb_first = NULL, rewrite_bb_last = NULL;
2817 25319 : if (opcode == ERROR_MARK)
2818 : {
2819 : /* For inter-bb range test optimization, pick from the range tests
2820 : the one which is tested in the earliest condition (one dominating
2821 : the others), because otherwise there could be some UB (e.g. signed
2822 : overflow) in following bbs that we'd expose which wasn't there in
2823 : the original program. See PR104196. */
2824 14515 : basic_block orig_range_bb = BASIC_BLOCK_FOR_FN (cfun, (*ops)[idx]->id);
2825 14515 : basic_block range_bb = orig_range_bb;
2826 32159 : for (unsigned int i = 0; i < count; i++)
2827 : {
2828 17644 : struct range_entry *this_range;
2829 17644 : if (otherrange)
2830 10485 : this_range = otherrange + i;
2831 : else
2832 7159 : this_range = otherrangep[i];
2833 17644 : operand_entry *oe = (*ops)[this_range->idx];
2834 17644 : basic_block this_bb = BASIC_BLOCK_FOR_FN (cfun, oe->id);
2835 17644 : if (range_bb != this_bb
2836 17644 : && dominated_by_p (CDI_DOMINATORS, range_bb, this_bb))
2837 : {
2838 6931 : swap_with = this_range;
2839 6931 : range_bb = this_bb;
2840 6931 : idx = this_range->idx;
2841 : }
2842 : }
2843 : /* If seq is non-NULL, it can contain statements that use SSA_NAMEs
2844 : only defined in later blocks. In this case we can't move the
2845 : merged comparison earlier, so instead check if there are any stmts
2846 : that might trigger signed integer overflow in between and rewrite
2847 : them. But only after we check if the optimization is possible. */
2848 14515 : if (seq && swap_with)
2849 : {
2850 3730 : rewrite_bb_first = range_bb;
2851 3730 : rewrite_bb_last = orig_range_bb;
2852 3730 : idx = range->idx;
2853 3730 : swap_with = NULL;
2854 : }
2855 : }
2856 25319 : operand_entry *oe = (*ops)[idx];
2857 25319 : tree op = oe->op;
2858 25319 : gimple *stmt = op ? SSA_NAME_DEF_STMT (op)
2859 12362 : : last_nondebug_stmt (BASIC_BLOCK_FOR_FN (cfun, oe->id));
2860 25319 : location_t loc = gimple_location (stmt);
2861 25319 : tree optype = op ? TREE_TYPE (op) : boolean_type_node;
2862 25319 : tree tem = build_range_check (loc, optype, unshare_expr (exp),
2863 : in_p, low, high);
2864 25319 : gimple_stmt_iterator gsi;
2865 25319 : unsigned int i, uid;
2866 :
2867 25319 : if (tem == NULL_TREE)
2868 : return false;
2869 :
2870 : /* If op is default def SSA_NAME, there is no place to insert the
2871 : new comparison. Give up, unless we can use OP itself as the
2872 : range test. */
2873 38276 : if (op && SSA_NAME_IS_DEFAULT_DEF (op))
2874 : {
2875 0 : if (op == range->exp
2876 0 : && ((TYPE_PRECISION (optype) == 1 && TYPE_UNSIGNED (optype))
2877 0 : || TREE_CODE (optype) == BOOLEAN_TYPE)
2878 0 : && (op == tem
2879 0 : || (TREE_CODE (tem) == EQ_EXPR
2880 0 : && TREE_OPERAND (tem, 0) == op
2881 0 : && integer_onep (TREE_OPERAND (tem, 1))))
2882 0 : && opcode != BIT_IOR_EXPR
2883 0 : && (opcode != ERROR_MARK || oe->rank != BIT_IOR_EXPR))
2884 : {
2885 : stmt = NULL;
2886 : tem = op;
2887 : }
2888 : else
2889 : return false;
2890 : }
2891 :
2892 25319 : if (swap_with)
2893 1584 : std::swap (range->idx, swap_with->idx);
2894 :
2895 25319 : if (dump_file && (dump_flags & TDF_DETAILS))
2896 : {
2897 39 : struct range_entry *r;
2898 39 : fprintf (dump_file, "Optimizing range tests ");
2899 39 : dump_range_entry (dump_file, range, false);
2900 180 : for (i = 0; i < count; i++)
2901 : {
2902 102 : if (otherrange)
2903 82 : r = otherrange + i;
2904 : else
2905 20 : r = otherrangep[i];
2906 102 : if (r->exp
2907 102 : && r->exp != range->exp
2908 20 : && TREE_CODE (r->exp) == SSA_NAME)
2909 : {
2910 20 : fprintf (dump_file, " and ");
2911 20 : dump_range_entry (dump_file, r, false);
2912 : }
2913 : else
2914 : {
2915 82 : fprintf (dump_file, " and");
2916 82 : dump_range_entry (dump_file, r, true);
2917 : }
2918 : }
2919 39 : fprintf (dump_file, "\n into ");
2920 39 : print_generic_expr (dump_file, tem);
2921 39 : fprintf (dump_file, "\n");
2922 : }
2923 :
2924 : /* In inter-bb range optimization mode, if we have a seq, we can't
2925 : move the merged comparison to the earliest bb from the comparisons
2926 : being replaced, so instead rewrite stmts that could trigger signed
2927 : integer overflow. */
2928 32348 : for (basic_block bb = rewrite_bb_last;
2929 32348 : bb != rewrite_bb_first; bb = single_pred (bb))
2930 14058 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2931 27742 : !gsi_end_p (gsi); gsi_next (&gsi))
2932 : {
2933 20713 : gimple *stmt = gsi_stmt (gsi);
2934 20713 : if (gimple_needing_rewrite_undefined (stmt))
2935 : {
2936 63 : gimple_stmt_iterator gsip = gsi;
2937 63 : gimple_stmt_iterator gsin = gsi;
2938 63 : gsi_prev (&gsip);
2939 63 : gsi_next (&gsin);
2940 63 : rewrite_to_defined_unconditional (&gsi);
2941 63 : unsigned uid = gimple_uid (stmt);
2942 63 : if (gsi_end_p (gsip))
2943 27 : gsip = gsi_after_labels (bb);
2944 : else
2945 36 : gsi_next (&gsip);
2946 253 : for (; gsi_stmt (gsip) != gsi_stmt (gsin);
2947 190 : gsi_next (&gsip))
2948 190 : gimple_set_uid (gsi_stmt (gsip), uid);
2949 : }
2950 : }
2951 :
2952 25319 : if (opcode == BIT_IOR_EXPR
2953 19055 : || (opcode == ERROR_MARK && oe->rank == BIT_IOR_EXPR))
2954 17896 : tem = invert_truthvalue_loc (loc, tem);
2955 :
2956 25319 : tem = fold_convert_loc (loc, optype, tem);
2957 25319 : if (stmt)
2958 : {
2959 25319 : gsi = gsi_for_stmt (stmt);
2960 25319 : uid = gimple_uid (stmt);
2961 : }
2962 : else
2963 : {
2964 0 : gsi = gsi_none ();
2965 0 : uid = 0;
2966 : }
2967 25319 : if (stmt == NULL)
2968 0 : gcc_checking_assert (tem == op);
2969 : /* In rare cases range->exp can be equal to lhs of stmt.
2970 : In that case we have to insert after the stmt rather then before
2971 : it. If stmt is a PHI, insert it at the start of the basic block. */
2972 25319 : else if (op != range->exp)
2973 : {
2974 25319 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
2975 25319 : tem = force_into_ssa_name (&gsi, tem, true);
2976 25319 : gsi_prev (&gsi);
2977 : }
2978 0 : else if (gimple_code (stmt) != GIMPLE_PHI)
2979 : {
2980 0 : gsi_insert_seq_after (&gsi, seq, GSI_CONTINUE_LINKING);
2981 0 : tem = force_into_ssa_name (&gsi, tem, false);
2982 : }
2983 : else
2984 : {
2985 0 : gsi = gsi_after_labels (gimple_bb (stmt));
2986 0 : if (!gsi_end_p (gsi))
2987 0 : uid = gimple_uid (gsi_stmt (gsi));
2988 : else
2989 : {
2990 0 : gsi = gsi_start_bb (gimple_bb (stmt));
2991 0 : uid = 1;
2992 0 : while (!gsi_end_p (gsi))
2993 : {
2994 0 : uid = gimple_uid (gsi_stmt (gsi));
2995 0 : gsi_next (&gsi);
2996 : }
2997 : }
2998 0 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
2999 0 : tem = force_into_ssa_name (&gsi, tem, true);
3000 0 : if (gsi_end_p (gsi))
3001 0 : gsi = gsi_last_bb (gimple_bb (stmt));
3002 : else
3003 0 : gsi_prev (&gsi);
3004 : }
3005 95324 : for (; !gsi_end_p (gsi); gsi_prev (&gsi))
3006 88020 : if (gimple_uid (gsi_stmt (gsi)))
3007 : break;
3008 : else
3009 66353 : gimple_set_uid (gsi_stmt (gsi), uid);
3010 :
3011 25319 : oe->op = tem;
3012 25319 : range->exp = exp;
3013 25319 : range->low = low;
3014 25319 : range->high = high;
3015 25319 : range->in_p = in_p;
3016 :
3017 54939 : for (i = 0; i < count; i++)
3018 : {
3019 29620 : if (otherrange)
3020 18446 : range = otherrange + i;
3021 : else
3022 11174 : range = otherrangep[i];
3023 29620 : oe = (*ops)[range->idx];
3024 : /* Now change all the other range test immediate uses, so that
3025 : those tests will be optimized away. */
3026 29620 : if (opcode == ERROR_MARK)
3027 : {
3028 17644 : if (oe->op)
3029 2341 : oe->op = build_int_cst (TREE_TYPE (oe->op),
3030 2341 : oe->rank == BIT_IOR_EXPR ? 0 : 1);
3031 : else
3032 15303 : oe->op = (oe->rank == BIT_IOR_EXPR
3033 15303 : ? boolean_false_node : boolean_true_node);
3034 : }
3035 : else
3036 11976 : oe->op = error_mark_node;
3037 29620 : range->exp = NULL_TREE;
3038 29620 : range->low = NULL_TREE;
3039 29620 : range->high = NULL_TREE;
3040 : }
3041 : return true;
3042 : }
3043 :
3044 : /* Optimize X == CST1 || X == CST2
3045 : if popcount (CST1 ^ CST2) == 1 into
3046 : (X & ~(CST1 ^ CST2)) == (CST1 & ~(CST1 ^ CST2)).
3047 : Similarly for ranges. E.g.
3048 : X != 2 && X != 3 && X != 10 && X != 11
3049 : will be transformed by the previous optimization into
3050 : !((X - 2U) <= 1U || (X - 10U) <= 1U)
3051 : and this loop can transform that into
3052 : !(((X & ~8) - 2U) <= 1U). */
3053 :
3054 : static bool
3055 23275 : optimize_range_tests_xor (enum tree_code opcode, tree type,
3056 : tree lowi, tree lowj, tree highi, tree highj,
3057 : vec<operand_entry *> *ops,
3058 : struct range_entry *rangei,
3059 : struct range_entry *rangej)
3060 : {
3061 23275 : tree lowxor, highxor, tem, exp;
3062 : /* Check lowi ^ lowj == highi ^ highj and
3063 : popcount (lowi ^ lowj) == 1. */
3064 23275 : lowxor = fold_binary (BIT_XOR_EXPR, type, lowi, lowj);
3065 23275 : if (lowxor == NULL_TREE || TREE_CODE (lowxor) != INTEGER_CST)
3066 : return false;
3067 23275 : if (!integer_pow2p (lowxor))
3068 : return false;
3069 3335 : highxor = fold_binary (BIT_XOR_EXPR, type, highi, highj);
3070 3335 : if (!tree_int_cst_equal (lowxor, highxor))
3071 : return false;
3072 :
3073 2924 : exp = rangei->exp;
3074 2924 : scalar_int_mode mode = as_a <scalar_int_mode> (TYPE_MODE (type));
3075 2924 : int prec = GET_MODE_PRECISION (mode);
3076 2924 : if (TYPE_PRECISION (type) < prec
3077 2923 : || (wi::to_wide (TYPE_MIN_VALUE (type))
3078 8770 : != wi::min_value (prec, TYPE_SIGN (type)))
3079 5847 : || (wi::to_wide (TYPE_MAX_VALUE (type))
3080 8770 : != wi::max_value (prec, TYPE_SIGN (type))))
3081 : {
3082 1 : type = build_nonstandard_integer_type (prec, TYPE_UNSIGNED (type));
3083 1 : exp = fold_convert (type, exp);
3084 1 : lowxor = fold_convert (type, lowxor);
3085 1 : lowi = fold_convert (type, lowi);
3086 1 : highi = fold_convert (type, highi);
3087 : }
3088 2924 : tem = fold_build1 (BIT_NOT_EXPR, type, lowxor);
3089 2924 : exp = fold_build2 (BIT_AND_EXPR, type, exp, tem);
3090 2924 : lowj = fold_build2 (BIT_AND_EXPR, type, lowi, tem);
3091 2924 : highj = fold_build2 (BIT_AND_EXPR, type, highi, tem);
3092 2924 : if (update_range_test (rangei, rangej, NULL, 1, opcode, ops, exp,
3093 : NULL, rangei->in_p, lowj, highj))
3094 : return true;
3095 : return false;
3096 : }
3097 :
3098 : /* Optimize X == CST1 || X == CST2
3099 : if popcount (CST2 - CST1) == 1 into
3100 : ((X - CST1) & ~(CST2 - CST1)) == 0.
3101 : Similarly for ranges. E.g.
3102 : X == 43 || X == 76 || X == 44 || X == 78 || X == 77 || X == 46
3103 : || X == 75 || X == 45
3104 : will be transformed by the previous optimization into
3105 : (X - 43U) <= 3U || (X - 75U) <= 3U
3106 : and this loop can transform that into
3107 : ((X - 43U) & ~(75U - 43U)) <= 3U. */
3108 : static bool
3109 18004 : optimize_range_tests_diff (enum tree_code opcode, tree type,
3110 : tree lowi, tree lowj, tree highi, tree highj,
3111 : vec<operand_entry *> *ops,
3112 : struct range_entry *rangei,
3113 : struct range_entry *rangej)
3114 : {
3115 18004 : tree tem1, tem2, mask;
3116 : /* Check highi - lowi == highj - lowj. */
3117 18004 : tem1 = fold_binary (MINUS_EXPR, type, highi, lowi);
3118 18004 : if (tem1 == NULL_TREE || TREE_CODE (tem1) != INTEGER_CST)
3119 : return false;
3120 18004 : tem2 = fold_binary (MINUS_EXPR, type, highj, lowj);
3121 18004 : if (!tree_int_cst_equal (tem1, tem2))
3122 : return false;
3123 : /* Check popcount (lowj - lowi) == 1. */
3124 11927 : tem1 = fold_binary (MINUS_EXPR, type, lowj, lowi);
3125 11927 : if (tem1 == NULL_TREE || TREE_CODE (tem1) != INTEGER_CST)
3126 : return false;
3127 11927 : if (!integer_pow2p (tem1))
3128 : return false;
3129 :
3130 1982 : scalar_int_mode mode = as_a <scalar_int_mode> (TYPE_MODE (type));
3131 1982 : int prec = GET_MODE_PRECISION (mode);
3132 1982 : if (TYPE_PRECISION (type) < prec
3133 1978 : || (wi::to_wide (TYPE_MIN_VALUE (type))
3134 5938 : != wi::min_value (prec, TYPE_SIGN (type)))
3135 3960 : || (wi::to_wide (TYPE_MAX_VALUE (type))
3136 5938 : != wi::max_value (prec, TYPE_SIGN (type))))
3137 4 : type = build_nonstandard_integer_type (prec, 1);
3138 : else
3139 1978 : type = unsigned_type_for (type);
3140 1982 : tem1 = fold_convert (type, tem1);
3141 1982 : tem2 = fold_convert (type, tem2);
3142 1982 : lowi = fold_convert (type, lowi);
3143 1982 : mask = fold_build1 (BIT_NOT_EXPR, type, tem1);
3144 1982 : tem1 = fold_build2 (MINUS_EXPR, type,
3145 : fold_convert (type, rangei->exp), lowi);
3146 1982 : tem1 = fold_build2 (BIT_AND_EXPR, type, tem1, mask);
3147 1982 : lowj = build_int_cst (type, 0);
3148 1982 : if (update_range_test (rangei, rangej, NULL, 1, opcode, ops, tem1,
3149 : NULL, rangei->in_p, lowj, tem2))
3150 : return true;
3151 : return false;
3152 : }
3153 :
3154 : /* It does some common checks for function optimize_range_tests_xor and
3155 : optimize_range_tests_diff.
3156 : If OPTIMIZE_XOR is TRUE, it calls optimize_range_tests_xor.
3157 : Else it calls optimize_range_tests_diff. */
3158 :
3159 : static bool
3160 2184512 : optimize_range_tests_1 (enum tree_code opcode, int first, int length,
3161 : bool optimize_xor, vec<operand_entry *> *ops,
3162 : struct range_entry *ranges)
3163 : {
3164 2184512 : int i, j;
3165 2184512 : bool any_changes = false;
3166 3815938 : for (i = first; i < length; i++)
3167 : {
3168 1631426 : tree lowi, highi, lowj, highj, type, tem;
3169 :
3170 1631426 : if (ranges[i].exp == NULL_TREE || ranges[i].in_p)
3171 1016184 : continue;
3172 615242 : type = TREE_TYPE (ranges[i].exp);
3173 615242 : if (!INTEGRAL_TYPE_P (type))
3174 49742 : continue;
3175 565500 : lowi = ranges[i].low;
3176 565500 : if (lowi == NULL_TREE)
3177 35768 : lowi = TYPE_MIN_VALUE (type);
3178 565500 : highi = ranges[i].high;
3179 565500 : if (highi == NULL_TREE)
3180 6800 : continue;
3181 947213 : for (j = i + 1; j < length && j < i + 64; j++)
3182 : {
3183 393419 : bool changes;
3184 393419 : if (ranges[i].exp != ranges[j].exp || ranges[j].in_p)
3185 352140 : continue;
3186 41279 : lowj = ranges[j].low;
3187 41279 : if (lowj == NULL_TREE)
3188 0 : continue;
3189 41279 : highj = ranges[j].high;
3190 41279 : if (highj == NULL_TREE)
3191 154 : highj = TYPE_MAX_VALUE (type);
3192 : /* Check lowj > highi. */
3193 41279 : tem = fold_binary (GT_EXPR, boolean_type_node,
3194 : lowj, highi);
3195 41279 : if (tem == NULL_TREE || !integer_onep (tem))
3196 0 : continue;
3197 41279 : if (optimize_xor)
3198 23275 : changes = optimize_range_tests_xor (opcode, type, lowi, lowj,
3199 : highi, highj, ops,
3200 : ranges + i, ranges + j);
3201 : else
3202 18004 : changes = optimize_range_tests_diff (opcode, type, lowi, lowj,
3203 : highi, highj, ops,
3204 : ranges + i, ranges + j);
3205 41279 : if (changes)
3206 : {
3207 : any_changes = true;
3208 : break;
3209 : }
3210 : }
3211 : }
3212 2184512 : return any_changes;
3213 : }
3214 :
3215 : /* Helper function of optimize_range_tests_to_bit_test. Handle a single
3216 : range, EXP, LOW, HIGH, compute bit mask of bits to test and return
3217 : EXP on success, NULL otherwise. */
3218 :
3219 : static tree
3220 170248 : extract_bit_test_mask (tree exp, int prec, tree totallow, tree low, tree high,
3221 : wide_int *mask, tree *totallowp)
3222 : {
3223 170248 : tree tem = int_const_binop (MINUS_EXPR, high, low);
3224 170248 : if (tem == NULL_TREE
3225 170248 : || TREE_CODE (tem) != INTEGER_CST
3226 170248 : || TREE_OVERFLOW (tem)
3227 159170 : || tree_int_cst_sgn (tem) == -1
3228 329418 : || compare_tree_int (tem, prec) != -1)
3229 : return NULL_TREE;
3230 :
3231 155193 : unsigned HOST_WIDE_INT max = tree_to_uhwi (tem) + 1;
3232 155193 : *mask = wi::shifted_mask (0, max, false, prec);
3233 155193 : if (TREE_CODE (exp) == BIT_AND_EXPR
3234 155193 : && TREE_CODE (TREE_OPERAND (exp, 1)) == INTEGER_CST)
3235 : {
3236 5202 : widest_int msk = wi::to_widest (TREE_OPERAND (exp, 1));
3237 5202 : msk = wi::zext (~msk, TYPE_PRECISION (TREE_TYPE (exp)));
3238 5202 : if (wi::popcount (msk) == 1
3239 5202 : && wi::ltu_p (msk, prec - max))
3240 : {
3241 4435 : *mask |= wi::shifted_mask (msk.to_uhwi (), max, false, prec);
3242 4435 : max += msk.to_uhwi ();
3243 4435 : exp = TREE_OPERAND (exp, 0);
3244 4435 : if (integer_zerop (low)
3245 2223 : && TREE_CODE (exp) == PLUS_EXPR
3246 6268 : && TREE_CODE (TREE_OPERAND (exp, 1)) == INTEGER_CST)
3247 : {
3248 1833 : tree ret = TREE_OPERAND (exp, 0);
3249 1833 : STRIP_NOPS (ret);
3250 1833 : widest_int bias
3251 1833 : = wi::neg (wi::sext (wi::to_widest (TREE_OPERAND (exp, 1)),
3252 3666 : TYPE_PRECISION (TREE_TYPE (low))));
3253 1833 : tree tbias = wide_int_to_tree (TREE_TYPE (ret), bias);
3254 1833 : if (totallowp)
3255 : {
3256 1797 : *totallowp = tbias;
3257 1797 : return ret;
3258 : }
3259 36 : else if (!tree_int_cst_lt (totallow, tbias))
3260 : return NULL_TREE;
3261 36 : bias = wi::to_widest (tbias);
3262 36 : bias -= wi::to_widest (totallow);
3263 36 : if (bias >= 0 && bias < prec - max)
3264 : {
3265 26 : *mask = wi::lshift (*mask, bias);
3266 26 : return ret;
3267 : }
3268 1833 : }
3269 : }
3270 5202 : }
3271 153370 : if (totallowp)
3272 : return exp;
3273 13838 : if (!tree_int_cst_lt (totallow, low))
3274 : return exp;
3275 13816 : tem = int_const_binop (MINUS_EXPR, low, totallow);
3276 13816 : if (tem == NULL_TREE
3277 13816 : || TREE_CODE (tem) != INTEGER_CST
3278 13816 : || TREE_OVERFLOW (tem)
3279 27475 : || compare_tree_int (tem, prec - max) == 1)
3280 : return NULL_TREE;
3281 :
3282 9800 : *mask = wi::lshift (*mask, wi::to_widest (tem));
3283 9800 : return exp;
3284 : }
3285 :
3286 : /* Attempt to optimize small range tests using bit test.
3287 : E.g.
3288 : X != 43 && X != 76 && X != 44 && X != 78 && X != 49
3289 : && X != 77 && X != 46 && X != 75 && X != 45 && X != 82
3290 : has been by earlier optimizations optimized into:
3291 : ((X - 43U) & ~32U) > 3U && X != 49 && X != 82
3292 : As all the 43 through 82 range is less than 64 numbers,
3293 : for 64-bit word targets optimize that into:
3294 : (X - 43U) > 40U && ((1 << (X - 43U)) & 0x8F0000004FULL) == 0 */
3295 :
3296 : static bool
3297 1092264 : optimize_range_tests_to_bit_test (enum tree_code opcode, int first, int length,
3298 : vec<operand_entry *> *ops,
3299 : struct range_entry *ranges)
3300 : {
3301 1092264 : int i, j;
3302 1092264 : bool any_changes = false;
3303 1092264 : int prec = GET_MODE_BITSIZE (word_mode);
3304 1092264 : auto_vec<struct range_entry *, 64> candidates;
3305 :
3306 1526298 : for (i = first; i < length - 1; i++)
3307 : {
3308 434034 : tree lowi, highi, lowj, highj, type;
3309 :
3310 434034 : if (ranges[i].exp == NULL_TREE || ranges[i].in_p)
3311 292705 : continue;
3312 175781 : type = TREE_TYPE (ranges[i].exp);
3313 175781 : if (!INTEGRAL_TYPE_P (type))
3314 17101 : continue;
3315 158680 : lowi = ranges[i].low;
3316 158680 : if (lowi == NULL_TREE)
3317 11177 : lowi = TYPE_MIN_VALUE (type);
3318 158680 : highi = ranges[i].high;
3319 158680 : if (highi == NULL_TREE)
3320 2413 : continue;
3321 156267 : wide_int mask;
3322 156267 : tree exp = extract_bit_test_mask (ranges[i].exp, prec, lowi, lowi,
3323 : highi, &mask, &lowi);
3324 156267 : if (exp == NULL_TREE)
3325 14938 : continue;
3326 141329 : candidates.truncate (0);
3327 141329 : int end = MIN (i + 64, length);
3328 309885 : for (j = i + 1; j < end; j++)
3329 : {
3330 168556 : tree exp2;
3331 168556 : if (ranges[j].exp == NULL_TREE || ranges[j].in_p)
3332 158738 : continue;
3333 100073 : if (ranges[j].exp == exp)
3334 : ;
3335 86580 : else if (TREE_CODE (ranges[j].exp) == BIT_AND_EXPR)
3336 : {
3337 1455 : exp2 = TREE_OPERAND (ranges[j].exp, 0);
3338 1455 : if (exp2 == exp)
3339 : ;
3340 1015 : else if (TREE_CODE (exp2) == PLUS_EXPR)
3341 : {
3342 814 : exp2 = TREE_OPERAND (exp2, 0);
3343 814 : STRIP_NOPS (exp2);
3344 814 : if (exp2 != exp)
3345 766 : continue;
3346 : }
3347 : else
3348 201 : continue;
3349 : }
3350 : else
3351 85125 : continue;
3352 13981 : lowj = ranges[j].low;
3353 13981 : if (lowj == NULL_TREE)
3354 0 : continue;
3355 13981 : highj = ranges[j].high;
3356 13981 : if (highj == NULL_TREE)
3357 77 : highj = TYPE_MAX_VALUE (TREE_TYPE (lowj));
3358 13981 : wide_int mask2;
3359 13981 : exp2 = extract_bit_test_mask (ranges[j].exp, prec, lowi, lowj,
3360 : highj, &mask2, NULL);
3361 13981 : if (exp2 != exp)
3362 4163 : continue;
3363 9818 : mask |= mask2;
3364 9818 : candidates.safe_push (&ranges[j]);
3365 13981 : }
3366 :
3367 : /* If every possible relative value of the expression is a valid shift
3368 : amount, then we can merge the entry test in the bit test. In this
3369 : case, if we would need otherwise 2 or more comparisons, then use
3370 : the bit test; in the other cases, the threshold is 3 comparisons. */
3371 141329 : bool entry_test_needed;
3372 141329 : int_range_max r;
3373 282658 : if (TREE_CODE (exp) == SSA_NAME
3374 281224 : && get_range_query (cfun)->range_of_expr (r, exp)
3375 140612 : && !r.undefined_p ()
3376 140612 : && !r.varying_p ()
3377 322510 : && wi::leu_p (r.upper_bound () - r.lower_bound (), prec - 1))
3378 : {
3379 4906 : wide_int min = r.lower_bound ();
3380 4906 : wide_int ilowi = wi::to_wide (lowi);
3381 4906 : if (wi::lt_p (min, ilowi, TYPE_SIGN (TREE_TYPE (lowi))))
3382 : {
3383 562 : lowi = wide_int_to_tree (TREE_TYPE (lowi), min);
3384 562 : mask = wi::lshift (mask, ilowi - min);
3385 : }
3386 4344 : else if (wi::gt_p (min, ilowi, TYPE_SIGN (TREE_TYPE (lowi))))
3387 : {
3388 1 : lowi = wide_int_to_tree (TREE_TYPE (lowi), min);
3389 1 : mask = wi::lrshift (mask, min - ilowi);
3390 : }
3391 4906 : entry_test_needed = false;
3392 4906 : }
3393 : else
3394 : entry_test_needed = true;
3395 287564 : if (candidates.length () >= (entry_test_needed ? 2 : 1))
3396 : {
3397 896 : tree high = wide_int_to_tree (TREE_TYPE (lowi),
3398 448 : wi::to_widest (lowi)
3399 1344 : + prec - 1 - wi::clz (mask));
3400 448 : operand_entry *oe = (*ops)[ranges[i].idx];
3401 448 : tree op = oe->op;
3402 448 : gimple *stmt = op ? SSA_NAME_DEF_STMT (op)
3403 50 : : last_nondebug_stmt (BASIC_BLOCK_FOR_FN
3404 398 : (cfun, oe->id));
3405 448 : location_t loc = gimple_location (stmt);
3406 448 : tree optype = op ? TREE_TYPE (op) : boolean_type_node;
3407 :
3408 : /* See if it isn't cheaper to pretend the minimum value of the
3409 : range is 0, if maximum value is small enough.
3410 : We can avoid then subtraction of the minimum value, but the
3411 : mask constant could be perhaps more expensive. */
3412 448 : if (compare_tree_int (lowi, 0) > 0
3413 371 : && compare_tree_int (high, prec) < 0
3414 833 : && (entry_test_needed || wi::ltu_p (r.upper_bound (), prec)))
3415 : {
3416 153 : int cost_diff;
3417 153 : HOST_WIDE_INT m = tree_to_uhwi (lowi);
3418 153 : rtx reg = gen_raw_REG (word_mode, 10000);
3419 153 : bool speed_p = optimize_bb_for_speed_p (gimple_bb (stmt));
3420 153 : cost_diff = set_src_cost (gen_rtx_PLUS (word_mode, reg,
3421 : GEN_INT (-m)),
3422 : word_mode, speed_p);
3423 153 : rtx r = immed_wide_int_const (mask, word_mode);
3424 153 : cost_diff += set_src_cost (gen_rtx_AND (word_mode, reg, r),
3425 : word_mode, speed_p);
3426 153 : r = immed_wide_int_const (wi::lshift (mask, m), word_mode);
3427 153 : cost_diff -= set_src_cost (gen_rtx_AND (word_mode, reg, r),
3428 : word_mode, speed_p);
3429 153 : if (cost_diff > 0)
3430 : {
3431 62 : mask = wi::lshift (mask, m);
3432 62 : lowi = build_zero_cst (TREE_TYPE (lowi));
3433 : }
3434 : }
3435 :
3436 448 : tree tem;
3437 448 : if (entry_test_needed)
3438 : {
3439 387 : tem = build_range_check (loc, optype, unshare_expr (exp),
3440 : false, lowi, high);
3441 387 : if (tem == NULL_TREE || is_gimple_val (tem))
3442 0 : continue;
3443 : }
3444 : else
3445 61 : tem = NULL_TREE;
3446 448 : tree etype = unsigned_type_for (TREE_TYPE (exp));
3447 448 : exp = fold_build2_loc (loc, MINUS_EXPR, etype,
3448 : fold_convert_loc (loc, etype, exp),
3449 : fold_convert_loc (loc, etype, lowi));
3450 448 : exp = fold_convert_loc (loc, integer_type_node, exp);
3451 448 : tree word_type = lang_hooks.types.type_for_mode (word_mode, 1);
3452 448 : exp = fold_build2_loc (loc, LSHIFT_EXPR, word_type,
3453 : build_int_cst (word_type, 1), exp);
3454 896 : exp = fold_build2_loc (loc, BIT_AND_EXPR, word_type, exp,
3455 448 : wide_int_to_tree (word_type, mask));
3456 448 : exp = fold_build2_loc (loc, EQ_EXPR, optype, exp,
3457 : build_zero_cst (word_type));
3458 448 : if (is_gimple_val (exp))
3459 0 : continue;
3460 :
3461 : /* The shift might have undefined behavior if TEM is true,
3462 : but reassociate_bb isn't prepared to have basic blocks
3463 : split when it is running. So, temporarily emit a code
3464 : with BIT_IOR_EXPR instead of &&, and fix it up in
3465 : branch_fixup. */
3466 448 : gimple_seq seq = NULL;
3467 448 : if (tem)
3468 : {
3469 387 : tem = force_gimple_operand (tem, &seq, true, NULL_TREE);
3470 387 : gcc_assert (TREE_CODE (tem) == SSA_NAME);
3471 387 : gimple_set_visited (SSA_NAME_DEF_STMT (tem), true);
3472 : }
3473 448 : gimple_seq seq2;
3474 448 : exp = force_gimple_operand (exp, &seq2, true, NULL_TREE);
3475 448 : gimple_seq_add_seq_without_update (&seq, seq2);
3476 448 : gcc_assert (TREE_CODE (exp) == SSA_NAME);
3477 448 : gimple_set_visited (SSA_NAME_DEF_STMT (exp), true);
3478 448 : if (tem)
3479 : {
3480 387 : gimple *g = gimple_build_assign (make_ssa_name (optype),
3481 : BIT_IOR_EXPR, tem, exp);
3482 387 : gimple_set_location (g, loc);
3483 387 : gimple_seq_add_stmt_without_update (&seq, g);
3484 387 : exp = gimple_assign_lhs (g);
3485 : }
3486 448 : tree val = build_zero_cst (optype);
3487 1344 : if (update_range_test (&ranges[i], NULL, candidates.address (),
3488 : candidates.length (), opcode, ops, exp,
3489 : seq, false, val, val))
3490 : {
3491 448 : any_changes = true;
3492 448 : if (tem)
3493 387 : reassoc_branch_fixups.safe_push (tem);
3494 : }
3495 : else
3496 0 : gimple_seq_discard (seq);
3497 : }
3498 156267 : }
3499 1092264 : return any_changes;
3500 1092264 : }
3501 :
3502 : /* Optimize x != 0 && y != 0 && z != 0 into (x | y | z) != 0
3503 : and similarly x != -1 && y != -1 && y != -1 into (x & y & z) != -1.
3504 : Also, handle x < C && y < C && z < C where C is power of two as
3505 : (x | y | z) < C. And also handle signed x < 0 && y < 0 && z < 0
3506 : as (x | y | z) < 0. */
3507 :
3508 : static bool
3509 1092264 : optimize_range_tests_cmp_bitwise (enum tree_code opcode, int first, int length,
3510 : vec<operand_entry *> *ops,
3511 : struct range_entry *ranges)
3512 : {
3513 1092264 : int i;
3514 1092264 : unsigned int b;
3515 1092264 : bool any_changes = false;
3516 1092264 : auto_vec<int, 128> buckets;
3517 1092264 : auto_vec<int, 32> chains;
3518 1092264 : auto_vec<struct range_entry *, 32> candidates;
3519 :
3520 1907984 : for (i = first; i < length; i++)
3521 : {
3522 815720 : int idx;
3523 :
3524 1164979 : if (ranges[i].exp == NULL_TREE
3525 795629 : || TREE_CODE (ranges[i].exp) != SSA_NAME
3526 790956 : || TYPE_PRECISION (TREE_TYPE (ranges[i].exp)) <= 1
3527 1282181 : || TREE_CODE (TREE_TYPE (ranges[i].exp)) == BOOLEAN_TYPE)
3528 349259 : continue;
3529 :
3530 466461 : if (ranges[i].low != NULL_TREE
3531 439310 : && ranges[i].high != NULL_TREE
3532 384799 : && ranges[i].in_p
3533 697384 : && tree_int_cst_equal (ranges[i].low, ranges[i].high))
3534 : {
3535 201087 : idx = !integer_zerop (ranges[i].low);
3536 201087 : if (idx && !integer_all_onesp (ranges[i].low))
3537 111943 : continue;
3538 : }
3539 265374 : else if (ranges[i].high != NULL_TREE
3540 210829 : && TREE_CODE (ranges[i].high) == INTEGER_CST
3541 210829 : && ranges[i].in_p)
3542 : {
3543 39103 : wide_int w = wi::to_wide (ranges[i].high);
3544 39103 : int prec = TYPE_PRECISION (TREE_TYPE (ranges[i].exp));
3545 39103 : int l = wi::clz (w);
3546 39103 : idx = 2;
3547 107007 : if (l <= 0
3548 39103 : || l >= prec
3549 72124 : || w != wi::mask (prec - l, false, prec))
3550 28801 : continue;
3551 10302 : if (!((TYPE_UNSIGNED (TREE_TYPE (ranges[i].exp))
3552 6492 : && ranges[i].low == NULL_TREE)
3553 10302 : || (ranges[i].low
3554 8306 : && integer_zerop (ranges[i].low))))
3555 3385 : continue;
3556 39103 : }
3557 428703 : else if (ranges[i].high == NULL_TREE
3558 54545 : && ranges[i].low != NULL_TREE
3559 : /* Perform this optimization only in the last
3560 : reassoc pass, as it interferes with the reassociation
3561 : itself or could also with VRP etc. which might not
3562 : be able to virtually undo the optimization. */
3563 54511 : && !reassoc_insert_powi_p
3564 27570 : && !TYPE_UNSIGNED (TREE_TYPE (ranges[i].exp))
3565 253500 : && integer_zerop (ranges[i].low))
3566 : idx = 3;
3567 : else
3568 202432 : continue;
3569 :
3570 119900 : b = TYPE_PRECISION (TREE_TYPE (ranges[i].exp)) * 4 + idx;
3571 119900 : if (buckets.length () <= b)
3572 105714 : buckets.safe_grow_cleared (b + 1, true);
3573 119900 : if (chains.length () <= (unsigned) i)
3574 119900 : chains.safe_grow (i + 1, true);
3575 119900 : chains[i] = buckets[b];
3576 119900 : buckets[b] = i + 1;
3577 : }
3578 :
3579 20280717 : FOR_EACH_VEC_ELT (buckets, b, i)
3580 19188453 : if (i && chains[i - 1])
3581 : {
3582 7027 : int j, k = i;
3583 7027 : if ((b % 4) == 2)
3584 : {
3585 : /* When ranges[X - 1].high + 1 is a power of two,
3586 : we need to process the same bucket up to
3587 : precision - 1 times, each time split the entries
3588 : with the same high bound into one chain and the
3589 : rest into another one to be processed later. */
3590 : int this_prev = i;
3591 : int other_prev = 0;
3592 146 : for (j = chains[i - 1]; j; j = chains[j - 1])
3593 : {
3594 79 : if (tree_int_cst_equal (ranges[i - 1].high,
3595 79 : ranges[j - 1].high))
3596 : {
3597 70 : chains[this_prev - 1] = j;
3598 70 : this_prev = j;
3599 : }
3600 9 : else if (other_prev == 0)
3601 : {
3602 7 : buckets[b] = j;
3603 7 : other_prev = j;
3604 : }
3605 : else
3606 : {
3607 2 : chains[other_prev - 1] = j;
3608 2 : other_prev = j;
3609 : }
3610 : }
3611 67 : chains[this_prev - 1] = 0;
3612 67 : if (other_prev)
3613 7 : chains[other_prev - 1] = 0;
3614 67 : if (chains[i - 1] == 0)
3615 : {
3616 5 : if (other_prev)
3617 5 : b--;
3618 5 : continue;
3619 : }
3620 : }
3621 17108 : for (j = chains[i - 1]; j; j = chains[j - 1])
3622 : {
3623 10086 : gimple *gk = SSA_NAME_DEF_STMT (ranges[k - 1].exp);
3624 10086 : gimple *gj = SSA_NAME_DEF_STMT (ranges[j - 1].exp);
3625 10086 : if (reassoc_stmt_dominates_stmt_p (gk, gj))
3626 2907 : k = j;
3627 : }
3628 7022 : tree type1 = TREE_TYPE (ranges[k - 1].exp);
3629 7022 : tree type2 = NULL_TREE;
3630 7022 : candidates.truncate (0);
3631 7022 : if (POINTER_TYPE_P (type1) || TREE_CODE (type1) == OFFSET_TYPE)
3632 704 : type1 = pointer_sized_int_node;
3633 24130 : for (j = i; j; j = chains[j - 1])
3634 : {
3635 17108 : tree type = TREE_TYPE (ranges[j - 1].exp);
3636 17108 : if (POINTER_TYPE_P (type) || TREE_CODE (type) == OFFSET_TYPE)
3637 1454 : type = pointer_sized_int_node;
3638 17108 : if ((b % 4) == 3)
3639 : {
3640 : /* For the signed < 0 cases, the types should be
3641 : really compatible (all signed with the same precision,
3642 : instead put ranges that have different in_p from
3643 : k first. */
3644 3674 : if (!useless_type_conversion_p (type1, type))
3645 0 : continue;
3646 3674 : if (ranges[j - 1].in_p != ranges[k - 1].in_p)
3647 1009 : candidates.safe_push (&ranges[j - 1]);
3648 3674 : type2 = type1;
3649 3674 : continue;
3650 : }
3651 13434 : if (j == k
3652 13434 : || useless_type_conversion_p (type1, type))
3653 : ;
3654 413 : else if (type2 == NULL_TREE
3655 413 : || useless_type_conversion_p (type2, type))
3656 : {
3657 413 : if (type2 == NULL_TREE)
3658 399 : type2 = type;
3659 413 : candidates.safe_push (&ranges[j - 1]);
3660 : }
3661 : }
3662 7022 : unsigned l = candidates.length ();
3663 24130 : for (j = i; j; j = chains[j - 1])
3664 : {
3665 17108 : tree type = TREE_TYPE (ranges[j - 1].exp);
3666 17108 : if (j == k)
3667 7022 : continue;
3668 10086 : if (POINTER_TYPE_P (type) || TREE_CODE (type) == OFFSET_TYPE)
3669 750 : type = pointer_sized_int_node;
3670 10086 : if ((b % 4) == 3)
3671 : {
3672 2034 : if (!useless_type_conversion_p (type1, type))
3673 0 : continue;
3674 2034 : if (ranges[j - 1].in_p == ranges[k - 1].in_p)
3675 1025 : candidates.safe_push (&ranges[j - 1]);
3676 2034 : continue;
3677 : }
3678 8052 : if (useless_type_conversion_p (type1, type))
3679 : ;
3680 826 : else if (type2 == NULL_TREE
3681 413 : || useless_type_conversion_p (type2, type))
3682 413 : continue;
3683 7639 : candidates.safe_push (&ranges[j - 1]);
3684 : }
3685 7022 : gimple_seq seq = NULL;
3686 7022 : tree op = NULL_TREE;
3687 7022 : unsigned int id;
3688 7022 : struct range_entry *r;
3689 7022 : candidates.safe_push (&ranges[k - 1]);
3690 24130 : FOR_EACH_VEC_ELT (candidates, id, r)
3691 : {
3692 17108 : gimple *g;
3693 17108 : enum tree_code code;
3694 17108 : if (id == 0)
3695 : {
3696 7022 : op = r->exp;
3697 7022 : continue;
3698 : }
3699 10086 : if (id == l
3700 8678 : || POINTER_TYPE_P (TREE_TYPE (op))
3701 18147 : || TREE_CODE (TREE_TYPE (op)) == OFFSET_TYPE)
3702 : {
3703 2030 : code = (b % 4) == 3 ? BIT_NOT_EXPR : NOP_EXPR;
3704 2030 : tree type3 = id >= l ? type1 : pointer_sized_int_node;
3705 2030 : if (code == BIT_NOT_EXPR
3706 2030 : && TREE_CODE (TREE_TYPE (op)) == OFFSET_TYPE)
3707 : {
3708 0 : g = gimple_build_assign (make_ssa_name (type3),
3709 : NOP_EXPR, op);
3710 0 : gimple_seq_add_stmt_without_update (&seq, g);
3711 0 : op = gimple_assign_lhs (g);
3712 : }
3713 2030 : g = gimple_build_assign (make_ssa_name (type3), code, op);
3714 2030 : gimple_seq_add_stmt_without_update (&seq, g);
3715 2030 : op = gimple_assign_lhs (g);
3716 : }
3717 10086 : tree type = TREE_TYPE (r->exp);
3718 10086 : tree exp = r->exp;
3719 10086 : if (POINTER_TYPE_P (type)
3720 9331 : || TREE_CODE (type) == OFFSET_TYPE
3721 19412 : || (id >= l && !useless_type_conversion_p (type1, type)))
3722 : {
3723 760 : tree type3 = id >= l ? type1 : pointer_sized_int_node;
3724 760 : g = gimple_build_assign (make_ssa_name (type3), NOP_EXPR, exp);
3725 760 : gimple_seq_add_stmt_without_update (&seq, g);
3726 760 : exp = gimple_assign_lhs (g);
3727 : }
3728 10086 : if ((b % 4) == 3)
3729 3358 : code = r->in_p ? BIT_IOR_EXPR : BIT_AND_EXPR;
3730 : else
3731 8052 : code = (b % 4) == 1 ? BIT_AND_EXPR : BIT_IOR_EXPR;
3732 20172 : g = gimple_build_assign (make_ssa_name (id >= l ? type1 : type2),
3733 : code, op, exp);
3734 10086 : gimple_seq_add_stmt_without_update (&seq, g);
3735 10086 : op = gimple_assign_lhs (g);
3736 : }
3737 7022 : type1 = TREE_TYPE (ranges[k - 1].exp);
3738 7022 : if (POINTER_TYPE_P (type1) || TREE_CODE (type1) == OFFSET_TYPE)
3739 : {
3740 704 : gimple *g
3741 704 : = gimple_build_assign (make_ssa_name (type1), NOP_EXPR, op);
3742 704 : gimple_seq_add_stmt_without_update (&seq, g);
3743 704 : op = gimple_assign_lhs (g);
3744 : }
3745 7022 : candidates.pop ();
3746 7022 : if (update_range_test (&ranges[k - 1], NULL, candidates.address (),
3747 : candidates.length (), opcode, ops, op,
3748 : seq, ranges[k - 1].in_p, ranges[k - 1].low,
3749 : ranges[k - 1].high))
3750 : any_changes = true;
3751 : else
3752 0 : gimple_seq_discard (seq);
3753 7084 : if ((b % 4) == 2 && buckets[b] != i)
3754 : /* There is more work to do for this bucket. */
3755 2 : b--;
3756 : }
3757 :
3758 1092264 : return any_changes;
3759 1092264 : }
3760 :
3761 : /* Attempt to optimize for signed a and b where b is known to be >= 0:
3762 : a >= 0 && a < b into (unsigned) a < (unsigned) b
3763 : a >= 0 && a <= b into (unsigned) a <= (unsigned) b */
3764 :
3765 : static bool
3766 1092264 : optimize_range_tests_var_bound (enum tree_code opcode, int first, int length,
3767 : vec<operand_entry *> *ops,
3768 : struct range_entry *ranges,
3769 : basic_block first_bb)
3770 : {
3771 1092264 : int i;
3772 1092264 : bool any_changes = false;
3773 1092264 : hash_map<tree, int> *map = NULL;
3774 :
3775 1907984 : for (i = first; i < length; i++)
3776 : {
3777 815720 : if (ranges[i].exp == NULL_TREE
3778 796186 : || TREE_CODE (ranges[i].exp) != SSA_NAME
3779 791513 : || !ranges[i].in_p)
3780 325080 : continue;
3781 :
3782 490640 : tree type = TREE_TYPE (ranges[i].exp);
3783 935943 : if (!INTEGRAL_TYPE_P (type)
3784 480816 : || TYPE_UNSIGNED (type)
3785 201601 : || ranges[i].low == NULL_TREE
3786 192334 : || !integer_zerop (ranges[i].low)
3787 564538 : || ranges[i].high != NULL_TREE)
3788 445303 : continue;
3789 : /* EXP >= 0 here. */
3790 45337 : if (map == NULL)
3791 43672 : map = new hash_map <tree, int>;
3792 45337 : map->put (ranges[i].exp, i);
3793 : }
3794 :
3795 1092264 : if (map == NULL)
3796 : return false;
3797 :
3798 134710 : for (i = 0; i < length; i++)
3799 : {
3800 91038 : bool in_p = ranges[i].in_p;
3801 91038 : if (ranges[i].low == NULL_TREE
3802 90339 : || ranges[i].high == NULL_TREE)
3803 90481 : continue;
3804 43224 : if (!integer_zerop (ranges[i].low)
3805 43224 : || !integer_zerop (ranges[i].high))
3806 : {
3807 8886 : if (ranges[i].exp
3808 4443 : && TYPE_PRECISION (TREE_TYPE (ranges[i].exp)) == 1
3809 0 : && TYPE_UNSIGNED (TREE_TYPE (ranges[i].exp))
3810 0 : && integer_onep (ranges[i].low)
3811 4443 : && integer_onep (ranges[i].high))
3812 0 : in_p = !in_p;
3813 : else
3814 4443 : continue;
3815 : }
3816 :
3817 38781 : gimple *stmt;
3818 38781 : tree_code ccode;
3819 38781 : tree rhs1, rhs2;
3820 38781 : if (ranges[i].exp)
3821 : {
3822 37691 : if (TREE_CODE (ranges[i].exp) != SSA_NAME)
3823 5 : continue;
3824 37686 : stmt = SSA_NAME_DEF_STMT (ranges[i].exp);
3825 37686 : if (!is_gimple_assign (stmt))
3826 941 : continue;
3827 36745 : ccode = gimple_assign_rhs_code (stmt);
3828 36745 : rhs1 = gimple_assign_rhs1 (stmt);
3829 36745 : rhs2 = gimple_assign_rhs2 (stmt);
3830 : }
3831 : else
3832 : {
3833 1090 : operand_entry *oe = (*ops)[ranges[i].idx];
3834 1090 : stmt = last_nondebug_stmt (BASIC_BLOCK_FOR_FN (cfun, oe->id));
3835 1090 : if (gimple_code (stmt) != GIMPLE_COND)
3836 0 : continue;
3837 1090 : ccode = gimple_cond_code (stmt);
3838 1090 : rhs1 = gimple_cond_lhs (stmt);
3839 1090 : rhs2 = gimple_cond_rhs (stmt);
3840 : }
3841 :
3842 37835 : if (TREE_CODE (rhs1) != SSA_NAME
3843 37326 : || rhs2 == NULL_TREE
3844 37223 : || TREE_CODE (rhs2) != SSA_NAME)
3845 675 : continue;
3846 :
3847 37160 : switch (ccode)
3848 : {
3849 35958 : case GT_EXPR:
3850 35958 : case GE_EXPR:
3851 35958 : case LT_EXPR:
3852 35958 : case LE_EXPR:
3853 35958 : break;
3854 1202 : default:
3855 1202 : continue;
3856 : }
3857 35958 : if (in_p)
3858 805 : ccode = invert_tree_comparison (ccode, false);
3859 35958 : switch (ccode)
3860 : {
3861 14850 : case GT_EXPR:
3862 14850 : case GE_EXPR:
3863 14850 : std::swap (rhs1, rhs2);
3864 14850 : ccode = swap_tree_comparison (ccode);
3865 14850 : break;
3866 : case LT_EXPR:
3867 : case LE_EXPR:
3868 : break;
3869 0 : default:
3870 0 : gcc_unreachable ();
3871 : }
3872 :
3873 35958 : int *idx = map->get (rhs1);
3874 35958 : if (idx == NULL)
3875 1071 : continue;
3876 :
3877 : /* maybe_optimize_range_tests allows statements without side-effects
3878 : in the basic blocks as long as they are consumed in the same bb.
3879 : Make sure rhs2's def stmt is not among them, otherwise we can't
3880 : use safely get_nonzero_bits on it. E.g. in:
3881 : # RANGE [-83, 1] NONZERO 173
3882 : # k_32 = PHI <k_47(13), k_12(9)>
3883 : ...
3884 : if (k_32 >= 0)
3885 : goto <bb 5>; [26.46%]
3886 : else
3887 : goto <bb 9>; [73.54%]
3888 :
3889 : <bb 5> [local count: 140323371]:
3890 : # RANGE [0, 1] NONZERO 1
3891 : _5 = (int) k_32;
3892 : # RANGE [0, 4] NONZERO 4
3893 : _21 = _5 << 2;
3894 : # RANGE [0, 4] NONZERO 4
3895 : iftmp.0_44 = (char) _21;
3896 : if (k_32 < iftmp.0_44)
3897 : goto <bb 6>; [84.48%]
3898 : else
3899 : goto <bb 9>; [15.52%]
3900 : the ranges on _5/_21/iftmp.0_44 are flow sensitive, assume that
3901 : k_32 >= 0. If we'd optimize k_32 >= 0 to true and k_32 < iftmp.0_44
3902 : to (unsigned) k_32 < (unsigned) iftmp.0_44, then we would execute
3903 : those stmts even for negative k_32 and the value ranges would be no
3904 : longer guaranteed and so the optimization would be invalid. */
3905 34889 : while (opcode == ERROR_MARK)
3906 : {
3907 640 : gimple *g = SSA_NAME_DEF_STMT (rhs2);
3908 640 : basic_block bb2 = gimple_bb (g);
3909 640 : if (bb2
3910 640 : && bb2 != first_bb
3911 640 : && dominated_by_p (CDI_DOMINATORS, bb2, first_bb))
3912 : {
3913 : /* As an exception, handle a few common cases. */
3914 558 : if (gimple_assign_cast_p (g)
3915 558 : && INTEGRAL_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (g))))
3916 : {
3917 42 : tree op0 = gimple_assign_rhs1 (g);
3918 42 : if (TYPE_UNSIGNED (TREE_TYPE (op0))
3919 42 : && (TYPE_PRECISION (TREE_TYPE (rhs2))
3920 12 : > TYPE_PRECISION (TREE_TYPE (op0))))
3921 : /* Zero-extension is always ok. */
3922 : break;
3923 32 : else if (TYPE_PRECISION (TREE_TYPE (rhs2))
3924 32 : == TYPE_PRECISION (TREE_TYPE (op0))
3925 32 : && TREE_CODE (op0) == SSA_NAME)
3926 : {
3927 : /* Cast from signed to unsigned or vice versa. Retry
3928 : with the op0 as new rhs2. */
3929 2 : rhs2 = op0;
3930 2 : continue;
3931 : }
3932 : }
3933 516 : else if (is_gimple_assign (g)
3934 516 : && gimple_assign_rhs_code (g) == BIT_AND_EXPR
3935 0 : && TREE_CODE (gimple_assign_rhs2 (g)) == INTEGER_CST
3936 1032 : && !wi::neg_p (wi::to_wide (gimple_assign_rhs2 (g))))
3937 : /* Masking with INTEGER_CST with MSB clear is always ok
3938 : too. */
3939 : break;
3940 : rhs2 = NULL_TREE;
3941 : }
3942 : break;
3943 : }
3944 34341 : if (rhs2 == NULL_TREE)
3945 546 : continue;
3946 :
3947 34898 : wide_int nz = get_nonzero_bits (rhs2);
3948 34341 : if (wi::neg_p (nz))
3949 33784 : continue;
3950 :
3951 : /* We have EXP < RHS2 or EXP <= RHS2 where EXP >= 0
3952 : and RHS2 is known to be RHS2 >= 0. */
3953 557 : tree utype = unsigned_type_for (TREE_TYPE (rhs1));
3954 :
3955 557 : if (dump_file && (dump_flags & TDF_DETAILS))
3956 : {
3957 7 : struct range_entry *r = &ranges[*idx];
3958 7 : fprintf (dump_file, "Optimizing range test ");
3959 7 : print_generic_expr (dump_file, r->exp);
3960 7 : fprintf (dump_file, " +[");
3961 7 : print_generic_expr (dump_file, r->low);
3962 7 : fprintf (dump_file, ", ");
3963 7 : print_generic_expr (dump_file, r->high);
3964 7 : fprintf (dump_file, "] and comparison ");
3965 7 : print_generic_expr (dump_file, rhs1);
3966 7 : fprintf (dump_file, " %s ", op_symbol_code (ccode));
3967 7 : print_generic_expr (dump_file, rhs2);
3968 7 : fprintf (dump_file, "\n into (");
3969 7 : print_generic_expr (dump_file, utype);
3970 7 : fprintf (dump_file, ") ");
3971 7 : print_generic_expr (dump_file, rhs1);
3972 7 : fprintf (dump_file, " %s (", op_symbol_code (ccode));
3973 7 : print_generic_expr (dump_file, utype);
3974 7 : fprintf (dump_file, ") ");
3975 7 : print_generic_expr (dump_file, rhs2);
3976 7 : fprintf (dump_file, "\n");
3977 : }
3978 :
3979 557 : operand_entry *oe = (*ops)[ranges[i].idx];
3980 557 : ranges[i].in_p = 0;
3981 557 : if (opcode == BIT_IOR_EXPR
3982 518 : || (opcode == ERROR_MARK && oe->rank == BIT_IOR_EXPR))
3983 : {
3984 40 : ranges[i].in_p = 1;
3985 40 : ccode = invert_tree_comparison (ccode, false);
3986 : }
3987 :
3988 557 : unsigned int uid = gimple_uid (stmt);
3989 557 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3990 557 : gimple *g = gimple_build_assign (make_ssa_name (utype), NOP_EXPR, rhs1);
3991 557 : gimple_set_uid (g, uid);
3992 557 : rhs1 = gimple_assign_lhs (g);
3993 557 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
3994 557 : if (!useless_type_conversion_p (utype, TREE_TYPE (rhs2)))
3995 : {
3996 557 : g = gimple_build_assign (make_ssa_name (utype), NOP_EXPR, rhs2);
3997 557 : gimple_set_uid (g, uid);
3998 557 : rhs2 = gimple_assign_lhs (g);
3999 557 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
4000 : }
4001 557 : if (tree_swap_operands_p (rhs1, rhs2))
4002 : {
4003 510 : std::swap (rhs1, rhs2);
4004 510 : ccode = swap_tree_comparison (ccode);
4005 : }
4006 557 : if (gimple_code (stmt) == GIMPLE_COND)
4007 : {
4008 8 : gcond *c = as_a <gcond *> (stmt);
4009 8 : gimple_cond_set_code (c, ccode);
4010 8 : gimple_cond_set_lhs (c, rhs1);
4011 8 : gimple_cond_set_rhs (c, rhs2);
4012 8 : update_stmt (stmt);
4013 : }
4014 : else
4015 : {
4016 549 : tree ctype = oe->op ? TREE_TYPE (oe->op) : boolean_type_node;
4017 549 : if (!INTEGRAL_TYPE_P (ctype)
4018 549 : || (TREE_CODE (ctype) != BOOLEAN_TYPE
4019 3 : && TYPE_PRECISION (ctype) != 1))
4020 3 : ctype = boolean_type_node;
4021 549 : g = gimple_build_assign (make_ssa_name (ctype), ccode, rhs1, rhs2);
4022 549 : gimple_set_uid (g, uid);
4023 549 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
4024 549 : if (oe->op && ctype != TREE_TYPE (oe->op))
4025 : {
4026 3 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (oe->op)),
4027 : NOP_EXPR, gimple_assign_lhs (g));
4028 3 : gimple_set_uid (g, uid);
4029 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
4030 : }
4031 549 : ranges[i].exp = gimple_assign_lhs (g);
4032 549 : oe->op = ranges[i].exp;
4033 549 : ranges[i].low = build_zero_cst (TREE_TYPE (ranges[i].exp));
4034 549 : ranges[i].high = ranges[i].low;
4035 : }
4036 557 : oe = (*ops)[ranges[*idx].idx];
4037 : /* Now change all the other range test immediate uses, so that
4038 : those tests will be optimized away. */
4039 557 : if (opcode == ERROR_MARK)
4040 : {
4041 11 : if (oe->op)
4042 3 : oe->op = build_int_cst (TREE_TYPE (oe->op),
4043 3 : oe->rank == BIT_IOR_EXPR ? 0 : 1);
4044 : else
4045 8 : oe->op = (oe->rank == BIT_IOR_EXPR
4046 8 : ? boolean_false_node : boolean_true_node);
4047 : }
4048 : else
4049 546 : oe->op = error_mark_node;
4050 557 : ranges[*idx].exp = NULL_TREE;
4051 557 : ranges[*idx].low = NULL_TREE;
4052 557 : ranges[*idx].high = NULL_TREE;
4053 557 : any_changes = true;
4054 : }
4055 :
4056 43672 : delete map;
4057 : return any_changes;
4058 : }
4059 :
4060 : /* Optimize range tests, similarly how fold_range_test optimizes
4061 : it on trees. The tree code for the binary
4062 : operation between all the operands is OPCODE.
4063 : If OPCODE is ERROR_MARK, optimize_range_tests is called from within
4064 : maybe_optimize_range_tests for inter-bb range optimization.
4065 : In that case if oe->op is NULL, oe->id is bb->index whose
4066 : GIMPLE_COND is && or ||ed into the test, and oe->rank says
4067 : the actual opcode.
4068 : FIRST_BB is the first basic block if OPCODE is ERROR_MARK. */
4069 :
4070 : static bool
4071 1092381 : optimize_range_tests (enum tree_code opcode,
4072 : vec<operand_entry *> *ops, basic_block first_bb)
4073 : {
4074 1092381 : unsigned int length = ops->length (), i, j, first;
4075 1092381 : operand_entry *oe;
4076 1092381 : struct range_entry *ranges;
4077 2184645 : bool any_changes = false;
4078 :
4079 1092381 : if (length == 1)
4080 : return false;
4081 :
4082 1092264 : ranges = XNEWVEC (struct range_entry, length);
4083 4486576 : for (i = 0; i < length; i++)
4084 : {
4085 2302048 : oe = (*ops)[i];
4086 2302048 : ranges[i].idx = i;
4087 2302048 : init_range_entry (ranges + i, oe->op,
4088 2302048 : oe->op
4089 : ? NULL
4090 270277 : : last_nondebug_stmt (BASIC_BLOCK_FOR_FN (cfun, oe->id)));
4091 : /* For | invert it now, we will invert it again before emitting
4092 : the optimized expression. */
4093 2302048 : if (opcode == BIT_IOR_EXPR
4094 1615708 : || (opcode == ERROR_MARK && oe->rank == BIT_IOR_EXPR))
4095 887270 : ranges[i].in_p = !ranges[i].in_p;
4096 : }
4097 :
4098 1092264 : qsort (ranges, length, sizeof (*ranges), range_entry_cmp);
4099 3670856 : for (i = 0; i < length; i++)
4100 1868014 : if (ranges[i].exp != NULL_TREE && TREE_CODE (ranges[i].exp) == SSA_NAME)
4101 : break;
4102 :
4103 : /* Try to merge ranges. */
4104 1894444 : for (first = i; i < length; i++)
4105 : {
4106 802180 : tree low = ranges[i].low;
4107 802180 : tree high = ranges[i].high;
4108 802180 : int in_p = ranges[i].in_p;
4109 802180 : int update_fail_count = 0;
4110 :
4111 815720 : for (j = i + 1; j < length; j++)
4112 : {
4113 434034 : if (ranges[i].exp != ranges[j].exp)
4114 : break;
4115 35069 : if (!merge_ranges (&in_p, &low, &high, in_p, low, high,
4116 35069 : ranges[j].in_p, ranges[j].low, ranges[j].high))
4117 : break;
4118 : }
4119 :
4120 802180 : if (j == i + 1)
4121 789237 : continue;
4122 :
4123 12943 : if (update_range_test (ranges + i, ranges + i + 1, NULL, j - i - 1,
4124 : opcode, ops, ranges[i].exp, NULL, in_p,
4125 : low, high))
4126 : {
4127 12943 : i = j - 1;
4128 12943 : any_changes = true;
4129 : }
4130 : /* Avoid quadratic complexity if all merge_ranges calls would succeed,
4131 : while update_range_test would fail. */
4132 : else if (update_fail_count == 64)
4133 : i = j - 1;
4134 : else
4135 12943 : ++update_fail_count;
4136 : }
4137 :
4138 1092264 : any_changes |= optimize_range_tests_1 (opcode, first, length, true,
4139 : ops, ranges);
4140 :
4141 1092264 : if (BRANCH_COST (optimize_function_for_speed_p (cfun), false) >= 2)
4142 1092248 : any_changes |= optimize_range_tests_1 (opcode, first, length, false,
4143 : ops, ranges);
4144 1092264 : if (lshift_cheap_p (optimize_function_for_speed_p (cfun)))
4145 1092264 : any_changes |= optimize_range_tests_to_bit_test (opcode, first, length,
4146 : ops, ranges);
4147 1092264 : any_changes |= optimize_range_tests_var_bound (opcode, first, length, ops,
4148 : ranges, first_bb);
4149 1092264 : any_changes |= optimize_range_tests_cmp_bitwise (opcode, first, length,
4150 : ops, ranges);
4151 :
4152 1092264 : if (any_changes && opcode != ERROR_MARK)
4153 : {
4154 : j = 0;
4155 36334 : FOR_EACH_VEC_ELT (*ops, i, oe)
4156 : {
4157 25405 : if (oe->op == error_mark_node)
4158 12522 : continue;
4159 12883 : else if (i != j)
4160 5321 : (*ops)[j] = oe;
4161 12883 : j++;
4162 : }
4163 10929 : ops->truncate (j);
4164 : }
4165 :
4166 1092264 : XDELETEVEC (ranges);
4167 1092264 : return any_changes;
4168 : }
4169 :
4170 : /* A subroutine of optimize_vec_cond_expr to extract and canonicalize
4171 : the operands of the VEC_COND_EXPR. Returns ERROR_MARK on failure,
4172 : otherwise the comparison code. TYPE is a return value that is set
4173 : to type of comparison. */
4174 :
4175 : static tree_code
4176 56776 : ovce_extract_ops (tree var, gassign **rets, bool *reti, tree *type,
4177 : tree *lhs, tree *rhs, gassign **vcond)
4178 : {
4179 56776 : if (TREE_CODE (var) != SSA_NAME)
4180 : return ERROR_MARK;
4181 :
4182 49715 : gassign *stmt = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (var));
4183 36296 : if (stmt == NULL)
4184 : return ERROR_MARK;
4185 36296 : if (vcond)
4186 36296 : *vcond = stmt;
4187 :
4188 : /* ??? If we start creating more COND_EXPR, we could perform
4189 : this same optimization with them. For now, simplify. */
4190 48635 : if (gimple_assign_rhs_code (stmt) != VEC_COND_EXPR)
4191 : return ERROR_MARK;
4192 :
4193 1375 : tree cond = gimple_assign_rhs1 (stmt);
4194 1375 : tree_code cmp = TREE_CODE (cond);
4195 1375 : if (cmp != SSA_NAME)
4196 : return ERROR_MARK;
4197 :
4198 1353 : gassign *assign = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (cond));
4199 1349 : if (assign == NULL
4200 1349 : || TREE_CODE_CLASS (gimple_assign_rhs_code (assign)) != tcc_comparison)
4201 : return ERROR_MARK;
4202 :
4203 1236 : cmp = gimple_assign_rhs_code (assign);
4204 1236 : if (lhs)
4205 1236 : *lhs = gimple_assign_rhs1 (assign);
4206 1236 : if (rhs)
4207 2472 : *rhs = gimple_assign_rhs2 (assign);
4208 :
4209 : /* ??? For now, allow only canonical true and false result vectors.
4210 : We could expand this to other constants should the need arise,
4211 : but at the moment we don't create them. */
4212 1236 : tree t = gimple_assign_rhs2 (stmt);
4213 1236 : tree f = gimple_assign_rhs3 (stmt);
4214 1236 : bool inv;
4215 1236 : if (integer_all_onesp (t))
4216 : inv = false;
4217 1116 : else if (integer_all_onesp (f))
4218 : {
4219 1 : cmp = invert_tree_comparison (cmp, false);
4220 1 : inv = true;
4221 : }
4222 : else
4223 : return ERROR_MARK;
4224 121 : if (!integer_zerop (inv ? t : f))
4225 : return ERROR_MARK;
4226 :
4227 : /* Success! */
4228 102 : if (rets)
4229 102 : *rets = assign;
4230 102 : if (reti)
4231 72 : *reti = inv;
4232 102 : if (type)
4233 72 : *type = TREE_TYPE (cond);
4234 : return cmp;
4235 : }
4236 :
4237 : /* Optimize the condition of VEC_COND_EXPRs which have been combined
4238 : with OPCODE (either BIT_AND_EXPR or BIT_IOR_EXPR). */
4239 :
4240 : static bool
4241 26638 : optimize_vec_cond_expr (tree_code opcode, vec<operand_entry *> *ops)
4242 : {
4243 26638 : unsigned int length = ops->length (), i, j;
4244 26638 : bool any_changes = false;
4245 :
4246 26638 : if (length == 1)
4247 : return false;
4248 :
4249 83366 : for (i = 0; i < length; ++i)
4250 : {
4251 56728 : tree &elt0 = (*ops)[i]->op;
4252 :
4253 56728 : gassign *stmt0, *vcond0;
4254 56728 : bool invert;
4255 56728 : tree type, lhs0, rhs0;
4256 56728 : tree_code cmp0 = ovce_extract_ops (elt0, &stmt0, &invert, &type, &lhs0,
4257 : &rhs0, &vcond0);
4258 56728 : if (cmp0 == ERROR_MARK)
4259 56656 : continue;
4260 :
4261 120 : for (j = i + 1; j < length; ++j)
4262 : {
4263 48 : tree &elt1 = (*ops)[j]->op;
4264 :
4265 48 : gassign *stmt1, *vcond1;
4266 48 : tree lhs1, rhs1;
4267 48 : tree_code cmp1 = ovce_extract_ops (elt1, &stmt1, NULL, NULL, &lhs1,
4268 : &rhs1, &vcond1);
4269 48 : if (cmp1 == ERROR_MARK)
4270 42 : continue;
4271 :
4272 30 : tree comb;
4273 30 : if (opcode == BIT_AND_EXPR)
4274 0 : comb = maybe_fold_and_comparisons (type, TRUTH_AND_EXPR,
4275 : cmp0, lhs0, rhs0,
4276 : cmp1, lhs1, rhs1);
4277 30 : else if (opcode == BIT_IOR_EXPR)
4278 30 : comb = maybe_fold_or_comparisons (type, TRUTH_OR_EXPR,
4279 : cmp0, lhs0, rhs0,
4280 : cmp1, lhs1, rhs1);
4281 : else
4282 0 : gcc_unreachable ();
4283 30 : if (comb == NULL)
4284 24 : continue;
4285 :
4286 : /* Success! */
4287 6 : if (dump_file && (dump_flags & TDF_DETAILS))
4288 : {
4289 0 : fprintf (dump_file, "Transforming ");
4290 0 : print_generic_expr (dump_file, gimple_assign_lhs (stmt0));
4291 0 : fprintf (dump_file, " %c ", opcode == BIT_AND_EXPR ? '&' : '|');
4292 0 : print_generic_expr (dump_file, gimple_assign_lhs (stmt1));
4293 0 : fprintf (dump_file, " into ");
4294 0 : print_generic_expr (dump_file, comb);
4295 0 : fputc ('\n', dump_file);
4296 : }
4297 :
4298 6 : gimple_stmt_iterator gsi = gsi_for_stmt (vcond0);
4299 6 : tree exp = force_gimple_operand_gsi (&gsi, comb, true, NULL_TREE,
4300 : true, GSI_SAME_STMT);
4301 18 : tree res = gimple_build (&gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
4302 6 : VEC_COND_EXPR, TREE_TYPE (elt0), exp,
4303 : constant_boolean_node (true,
4304 6 : TREE_TYPE (elt0)),
4305 : constant_boolean_node (false,
4306 6 : TREE_TYPE (elt0)));
4307 6 : elt0 = res;
4308 :
4309 6 : elt1 = error_mark_node;
4310 6 : any_changes = true;
4311 : }
4312 : }
4313 :
4314 26638 : if (any_changes)
4315 : {
4316 : operand_entry *oe;
4317 : j = 0;
4318 18 : FOR_EACH_VEC_ELT (*ops, i, oe)
4319 : {
4320 12 : if (oe->op == error_mark_node)
4321 6 : continue;
4322 6 : else if (i != j)
4323 0 : (*ops)[j] = oe;
4324 6 : j++;
4325 : }
4326 6 : ops->truncate (j);
4327 : }
4328 :
4329 : return any_changes;
4330 : }
4331 :
4332 : /* Return true if STMT is a cast like:
4333 : <bb N>:
4334 : ...
4335 : _123 = (int) _234;
4336 :
4337 : <bb M>:
4338 : # _345 = PHI <_123(N), 1(...), 1(...)>
4339 : where _234 has bool type, _123 has single use and
4340 : bb N has a single successor M. This is commonly used in
4341 : the last block of a range test.
4342 :
4343 : Also Return true if STMT is tcc_compare like:
4344 : <bb N>:
4345 : ...
4346 : _234 = a_2(D) == 2;
4347 :
4348 : <bb M>:
4349 : # _345 = PHI <_234(N), 1(...), 1(...)>
4350 : _346 = (int) _345;
4351 : where _234 has booltype, single use and
4352 : bb N has a single successor M. This is commonly used in
4353 : the last block of a range test. */
4354 :
4355 : static bool
4356 15917324 : final_range_test_p (gimple *stmt)
4357 : {
4358 15917324 : basic_block bb, rhs_bb, lhs_bb;
4359 15917324 : edge e;
4360 15917324 : tree lhs, rhs;
4361 15917324 : use_operand_p use_p;
4362 15917324 : gimple *use_stmt;
4363 :
4364 15917324 : if (!gimple_assign_cast_p (stmt)
4365 15917324 : && (!is_gimple_assign (stmt)
4366 5036390 : || (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt))
4367 : != tcc_comparison)))
4368 : return false;
4369 566518 : bb = gimple_bb (stmt);
4370 566518 : if (!single_succ_p (bb))
4371 : return false;
4372 566235 : e = single_succ_edge (bb);
4373 566235 : if (e->flags & EDGE_COMPLEX)
4374 : return false;
4375 :
4376 566235 : lhs = gimple_assign_lhs (stmt);
4377 566235 : rhs = gimple_assign_rhs1 (stmt);
4378 566235 : if (gimple_assign_cast_p (stmt)
4379 566235 : && (!INTEGRAL_TYPE_P (TREE_TYPE (lhs))
4380 400150 : || TREE_CODE (rhs) != SSA_NAME
4381 379148 : || TREE_CODE (TREE_TYPE (rhs)) != BOOLEAN_TYPE))
4382 : return false;
4383 :
4384 188519 : if (!gimple_assign_cast_p (stmt)
4385 188519 : && (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE))
4386 : return false;
4387 :
4388 : /* Test whether lhs is consumed only by a PHI in the only successor bb. */
4389 188041 : if (!single_imm_use (lhs, &use_p, &use_stmt))
4390 : return false;
4391 :
4392 184813 : if (gimple_code (use_stmt) != GIMPLE_PHI
4393 184813 : || gimple_bb (use_stmt) != e->dest)
4394 : return false;
4395 :
4396 : /* And that the rhs is defined in the same loop. */
4397 183038 : if (gimple_assign_cast_p (stmt))
4398 : {
4399 65276 : if (TREE_CODE (rhs) != SSA_NAME
4400 65276 : || !(rhs_bb = gimple_bb (SSA_NAME_DEF_STMT (rhs)))
4401 130546 : || !flow_bb_inside_loop_p (loop_containing_stmt (stmt), rhs_bb))
4402 53 : return false;
4403 : }
4404 : else
4405 : {
4406 117762 : if (TREE_CODE (lhs) != SSA_NAME
4407 117762 : || !(lhs_bb = gimple_bb (SSA_NAME_DEF_STMT (lhs)))
4408 235524 : || !flow_bb_inside_loop_p (loop_containing_stmt (stmt), lhs_bb))
4409 0 : return false;
4410 : }
4411 :
4412 : return true;
4413 : }
4414 :
4415 : /* Return true if BB is suitable basic block for inter-bb range test
4416 : optimization. If BACKWARD is true, BB should be the only predecessor
4417 : of TEST_BB, and *OTHER_BB is either NULL and filled by the routine,
4418 : or compared with to find a common basic block to which all conditions
4419 : branch to if true resp. false. If BACKWARD is false, TEST_BB should
4420 : be the only predecessor of BB. *TEST_SWAPPED_P is set to true if
4421 : TEST_BB is a bb ending in condition where the edge to non-*OTHER_BB
4422 : block points to an empty block that falls through into *OTHER_BB and
4423 : the phi args match that path. */
4424 :
4425 : static bool
4426 11551098 : suitable_cond_bb (basic_block bb, basic_block test_bb, basic_block *other_bb,
4427 : bool *test_swapped_p, bool backward)
4428 : {
4429 11551098 : edge_iterator ei, ei2;
4430 11551098 : edge e, e2;
4431 11551098 : gimple *stmt;
4432 11551098 : gphi_iterator gsi;
4433 11551098 : bool other_edge_seen = false;
4434 11551098 : bool is_cond;
4435 :
4436 11551098 : if (test_bb == bb)
4437 : return false;
4438 : /* Check last stmt first. */
4439 11551098 : stmt = last_nondebug_stmt (bb);
4440 11551098 : if (stmt == NULL
4441 10672586 : || (gimple_code (stmt) != GIMPLE_COND
4442 532453 : && (backward || !final_range_test_p (stmt)))
4443 10180232 : || gimple_visited_p (stmt)
4444 10120697 : || stmt_could_throw_p (cfun, stmt)
4445 21671681 : || *other_bb == bb)
4446 : return false;
4447 10120580 : is_cond = gimple_code (stmt) == GIMPLE_COND;
4448 10120580 : if (is_cond)
4449 : {
4450 : /* If last stmt is GIMPLE_COND, verify that one of the succ edges
4451 : goes to the next bb (if BACKWARD, it is TEST_BB), and the other
4452 : to *OTHER_BB (if not set yet, try to find it out). */
4453 10080595 : if (EDGE_COUNT (bb->succs) != 2)
4454 : return false;
4455 19860809 : FOR_EACH_EDGE (e, ei, bb->succs)
4456 : {
4457 16401793 : if (!(e->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE)))
4458 : return false;
4459 16401793 : if (e->dest == test_bb)
4460 : {
4461 5457504 : if (backward)
4462 5454468 : continue;
4463 : else
4464 : return false;
4465 : }
4466 10944289 : if (e->dest == bb)
4467 : return false;
4468 10785968 : if (*other_bb == NULL)
4469 : {
4470 26515614 : FOR_EACH_EDGE (e2, ei2, test_bb->succs)
4471 17677076 : if (!(e2->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE)))
4472 : return false;
4473 17677076 : else if (e->dest == e2->dest)
4474 2408999 : *other_bb = e->dest;
4475 8838538 : if (*other_bb == NULL)
4476 : return false;
4477 : }
4478 4356429 : if (e->dest == *other_bb)
4479 : other_edge_seen = true;
4480 937995 : else if (backward)
4481 : return false;
4482 : }
4483 3459016 : if (*other_bb == NULL || !other_edge_seen)
4484 : return false;
4485 : }
4486 39985 : else if (single_succ (bb) != *other_bb)
4487 : return false;
4488 :
4489 : /* Now check all PHIs of *OTHER_BB. */
4490 3457643 : e = find_edge (bb, *other_bb);
4491 3457643 : e2 = find_edge (test_bb, *other_bb);
4492 3464243 : retry:;
4493 5062133 : for (gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi); gsi_next (&gsi))
4494 : {
4495 2682003 : gphi *phi = gsi.phi ();
4496 : /* If both BB and TEST_BB end with GIMPLE_COND, all PHI arguments
4497 : corresponding to BB and TEST_BB predecessor must be the same. */
4498 2682003 : if (!operand_equal_p (gimple_phi_arg_def (phi, e->dest_idx),
4499 2682003 : gimple_phi_arg_def (phi, e2->dest_idx), 0))
4500 : {
4501 : /* Otherwise, if one of the blocks doesn't end with GIMPLE_COND,
4502 : one of the PHIs should have the lhs of the last stmt in
4503 : that block as PHI arg and that PHI should have 0 or 1
4504 : corresponding to it in all other range test basic blocks
4505 : considered. */
4506 1153637 : if (!is_cond)
4507 : {
4508 41997 : if (gimple_phi_arg_def (phi, e->dest_idx)
4509 41997 : == gimple_assign_lhs (stmt)
4510 41997 : && (integer_zerop (gimple_phi_arg_def (phi, e2->dest_idx))
4511 18042 : || integer_onep (gimple_phi_arg_def (phi,
4512 18042 : e2->dest_idx))))
4513 37609 : continue;
4514 : }
4515 : else
4516 : {
4517 1111640 : gimple *test_last = last_nondebug_stmt (test_bb);
4518 1111640 : if (gimple_code (test_last) == GIMPLE_COND)
4519 : {
4520 1075399 : if (backward ? e2->src != test_bb : e->src != bb)
4521 : return false;
4522 :
4523 : /* For last_bb, handle also:
4524 : if (x_3(D) == 3)
4525 : goto <bb 6>; [34.00%]
4526 : else
4527 : goto <bb 7>; [66.00%]
4528 :
4529 : <bb 6> [local count: 79512730]:
4530 :
4531 : <bb 7> [local count: 1073741824]:
4532 : # prephitmp_7 = PHI <1(3), 1(4), 0(5), 1(2), 1(6)>
4533 : where bb 7 is *OTHER_BB, but the PHI values from the
4534 : earlier bbs match the path through the empty bb
4535 : in between. */
4536 1070771 : edge e3;
4537 1070771 : if (backward)
4538 1477418 : e3 = EDGE_SUCC (test_bb,
4539 : e2 == EDGE_SUCC (test_bb, 0) ? 1 : 0);
4540 : else
4541 21893 : e3 = EDGE_SUCC (bb,
4542 : e == EDGE_SUCC (bb, 0) ? 1 : 0);
4543 1070771 : if (empty_block_p (e3->dest)
4544 35337 : && single_succ_p (e3->dest)
4545 35337 : && single_succ (e3->dest) == *other_bb
4546 35067 : && single_pred_p (e3->dest)
4547 1105074 : && single_succ_edge (e3->dest)->flags == EDGE_FALLTHRU)
4548 : {
4549 6600 : if (backward)
4550 5927 : e2 = single_succ_edge (e3->dest);
4551 : else
4552 673 : e = single_succ_edge (e3->dest);
4553 6600 : if (test_swapped_p)
4554 304 : *test_swapped_p = true;
4555 6600 : goto retry;
4556 : }
4557 : }
4558 36241 : else if (gimple_phi_arg_def (phi, e2->dest_idx)
4559 36241 : == gimple_assign_lhs (test_last)
4560 69036 : && (integer_zerop (gimple_phi_arg_def (phi,
4561 32795 : e->dest_idx))
4562 14978 : || integer_onep (gimple_phi_arg_def (phi,
4563 14978 : e->dest_idx))))
4564 31915 : continue;
4565 : }
4566 :
4567 : return false;
4568 : }
4569 : }
4570 : return true;
4571 : }
4572 :
4573 : /* Return true if BB doesn't have side-effects that would disallow
4574 : range test optimization, all SSA_NAMEs set in the bb are consumed
4575 : in the bb and there are no PHIs. */
4576 :
4577 : bool
4578 5487798 : no_side_effect_bb (basic_block bb)
4579 : {
4580 5487798 : gimple_stmt_iterator gsi;
4581 5487798 : gimple *last;
4582 :
4583 5487798 : if (!gimple_seq_empty_p (phi_nodes (bb)))
4584 : return false;
4585 4355885 : last = last_nondebug_stmt (bb);
4586 15459748 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4587 : {
4588 11103863 : gimple *stmt = gsi_stmt (gsi);
4589 11103863 : tree lhs;
4590 11103863 : imm_use_iterator imm_iter;
4591 11103863 : use_operand_p use_p;
4592 :
4593 11103863 : if (is_gimple_debug (stmt))
4594 4848105 : continue;
4595 6255758 : if (gimple_has_side_effects (stmt))
4596 4355885 : return false;
4597 5332561 : if (stmt == last)
4598 : return true;
4599 4238878 : if (!is_gimple_assign (stmt))
4600 : return false;
4601 3550366 : lhs = gimple_assign_lhs (stmt);
4602 3550366 : if (TREE_CODE (lhs) != SSA_NAME)
4603 : return false;
4604 3304612 : if (gimple_assign_rhs_could_trap_p (stmt))
4605 : return false;
4606 4685217 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
4607 : {
4608 2785344 : gimple *use_stmt = USE_STMT (use_p);
4609 2785344 : if (is_gimple_debug (use_stmt))
4610 217906 : continue;
4611 2567438 : if (gimple_bb (use_stmt) != bb)
4612 541138 : return false;
4613 2441011 : }
4614 : }
4615 : return false;
4616 : }
4617 :
4618 : /* If VAR is set by CODE (BIT_{AND,IOR}_EXPR) which is reassociable,
4619 : return true and fill in *OPS recursively. */
4620 :
4621 : static bool
4622 94822 : get_ops (tree var, enum tree_code code, vec<operand_entry *> *ops,
4623 : class loop *loop)
4624 : {
4625 94822 : gimple *stmt = SSA_NAME_DEF_STMT (var);
4626 94822 : tree rhs[2];
4627 94822 : int i;
4628 :
4629 94822 : if (!is_reassociable_op (stmt, code, loop))
4630 : return false;
4631 :
4632 22462 : rhs[0] = gimple_assign_rhs1 (stmt);
4633 22462 : rhs[1] = gimple_assign_rhs2 (stmt);
4634 22462 : gimple_set_visited (stmt, true);
4635 67386 : for (i = 0; i < 2; i++)
4636 44924 : if (TREE_CODE (rhs[i]) == SSA_NAME
4637 44924 : && !get_ops (rhs[i], code, ops, loop)
4638 83280 : && has_single_use (rhs[i]))
4639 : {
4640 37587 : operand_entry *oe = operand_entry_pool.allocate ();
4641 :
4642 37587 : oe->op = rhs[i];
4643 37587 : oe->rank = code;
4644 37587 : oe->id = 0;
4645 37587 : oe->count = 1;
4646 37587 : oe->stmt_to_insert = NULL;
4647 37587 : ops->safe_push (oe);
4648 : }
4649 : return true;
4650 : }
4651 :
4652 : /* Find the ops that were added by get_ops starting from VAR, see if
4653 : they were changed during update_range_test and if yes, create new
4654 : stmts. */
4655 :
4656 : static tree
4657 10797 : update_ops (tree var, enum tree_code code, const vec<operand_entry *> &ops,
4658 : unsigned int *pidx, class loop *loop)
4659 : {
4660 10797 : gimple *stmt = SSA_NAME_DEF_STMT (var);
4661 10797 : tree rhs[4];
4662 10797 : int i;
4663 :
4664 10797 : if (!is_reassociable_op (stmt, code, loop))
4665 : return NULL;
4666 :
4667 3609 : rhs[0] = gimple_assign_rhs1 (stmt);
4668 3609 : rhs[1] = gimple_assign_rhs2 (stmt);
4669 3609 : rhs[2] = rhs[0];
4670 3609 : rhs[3] = rhs[1];
4671 10827 : for (i = 0; i < 2; i++)
4672 7218 : if (TREE_CODE (rhs[i]) == SSA_NAME)
4673 : {
4674 7218 : rhs[2 + i] = update_ops (rhs[i], code, ops, pidx, loop);
4675 7218 : if (rhs[2 + i] == NULL_TREE)
4676 : {
4677 6803 : if (has_single_use (rhs[i]))
4678 6785 : rhs[2 + i] = ops[(*pidx)++]->op;
4679 : else
4680 18 : rhs[2 + i] = rhs[i];
4681 : }
4682 : }
4683 3609 : if ((rhs[2] != rhs[0] || rhs[3] != rhs[1])
4684 3360 : && (rhs[2] != rhs[1] || rhs[3] != rhs[0]))
4685 : {
4686 3360 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
4687 3360 : var = make_ssa_name (TREE_TYPE (var));
4688 3360 : gassign *g = gimple_build_assign (var, gimple_assign_rhs_code (stmt),
4689 : rhs[2], rhs[3]);
4690 3360 : gimple_set_uid (g, gimple_uid (stmt));
4691 3360 : gimple_set_visited (g, true);
4692 3360 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
4693 3360 : gimple_stmt_iterator gsi2 = gsi_for_stmt (g);
4694 3360 : if (fold_stmt_inplace (&gsi2))
4695 2190 : update_stmt (g);
4696 : }
4697 : return var;
4698 : }
4699 :
4700 : /* Structure to track the initial value passed to get_ops and
4701 : the range in the ops vector for each basic block. */
4702 :
4703 : struct inter_bb_range_test_entry
4704 : {
4705 : tree op;
4706 : unsigned int first_idx, last_idx;
4707 : };
4708 :
4709 : /* Inter-bb range test optimization.
4710 :
4711 : Returns TRUE if a gimple conditional is optimized to a true/false,
4712 : otherwise return FALSE.
4713 :
4714 : This indicates to the caller that it should run a CFG cleanup pass
4715 : once reassociation is completed. */
4716 :
4717 : static bool
4718 19325763 : maybe_optimize_range_tests (gimple *stmt)
4719 : {
4720 19325763 : basic_block first_bb = gimple_bb (stmt);
4721 19325763 : basic_block last_bb = first_bb;
4722 19325763 : basic_block other_bb = NULL;
4723 19325763 : basic_block bb;
4724 19325763 : edge_iterator ei;
4725 19325763 : edge e;
4726 19325763 : auto_vec<operand_entry *> ops;
4727 19325763 : auto_vec<inter_bb_range_test_entry> bbinfo;
4728 19325763 : bool any_changes = false;
4729 19325763 : bool cfg_cleanup_needed = false;
4730 :
4731 : /* Consider only basic blocks that end with GIMPLE_COND or
4732 : a cast statement satisfying final_range_test_p. All
4733 : but the last bb in the first_bb .. last_bb range
4734 : should end with GIMPLE_COND. */
4735 19325763 : if (gimple_code (stmt) == GIMPLE_COND)
4736 : {
4737 8892248 : if (EDGE_COUNT (first_bb->succs) != 2)
4738 : return cfg_cleanup_needed;
4739 : }
4740 10433515 : else if (final_range_test_p (stmt))
4741 73826 : other_bb = single_succ (first_bb);
4742 : else
4743 : return cfg_cleanup_needed;
4744 :
4745 8966074 : if (stmt_could_throw_p (cfun, stmt))
4746 : return cfg_cleanup_needed;
4747 :
4748 : /* As relative ordering of post-dominator sons isn't fixed,
4749 : maybe_optimize_range_tests can be called first on any
4750 : bb in the range we want to optimize. So, start searching
4751 : backwards, if first_bb can be set to a predecessor. */
4752 8968458 : while (single_pred_p (first_bb))
4753 : {
4754 5912623 : basic_block pred_bb = single_pred (first_bb);
4755 5912623 : if (!suitable_cond_bb (pred_bb, first_bb, &other_bb, NULL, true))
4756 : break;
4757 633842 : if (!no_side_effect_bb (first_bb))
4758 : break;
4759 : first_bb = pred_bb;
4760 : }
4761 : /* If first_bb is last_bb, other_bb hasn't been computed yet.
4762 : Before starting forward search in last_bb successors, find
4763 : out the other_bb. */
4764 8965906 : if (first_bb == last_bb)
4765 : {
4766 8963559 : other_bb = NULL;
4767 : /* As non-GIMPLE_COND last stmt always terminates the range,
4768 : if forward search didn't discover anything, just give up. */
4769 8963559 : if (gimple_code (stmt) != GIMPLE_COND)
4770 : return cfg_cleanup_needed;
4771 : /* Look at both successors. Either it ends with a GIMPLE_COND
4772 : and satisfies suitable_cond_bb, or ends with a cast and
4773 : other_bb is that cast's successor. */
4774 24666312 : FOR_EACH_EDGE (e, ei, first_bb->succs)
4775 17061582 : if (!(e->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE))
4776 17061582 : || e->dest == first_bb)
4777 : return cfg_cleanup_needed;
4778 25886690 : else if (single_pred_p (e->dest))
4779 : {
4780 10110367 : stmt = last_nondebug_stmt (e->dest);
4781 10110367 : if (stmt
4782 9927666 : && gimple_code (stmt) == GIMPLE_COND
4783 14639984 : && EDGE_COUNT (e->dest->succs) == 2)
4784 : {
4785 4529617 : if (suitable_cond_bb (first_bb, e->dest, &other_bb,
4786 : NULL, true))
4787 : break;
4788 : else
4789 3779823 : other_bb = NULL;
4790 : }
4791 5580750 : else if (stmt
4792 5398049 : && final_range_test_p (stmt)
4793 5649810 : && find_edge (first_bb, single_succ (e->dest)))
4794 : {
4795 38822 : other_bb = single_succ (e->dest);
4796 38822 : if (other_bb == first_bb)
4797 0 : other_bb = NULL;
4798 : }
4799 : }
4800 8354524 : if (other_bb == NULL)
4801 : return cfg_cleanup_needed;
4802 : }
4803 : /* Now do the forward search, moving last_bb to successor bbs
4804 : that aren't other_bb. */
4805 1003762 : while (EDGE_COUNT (last_bb->succs) == 2)
4806 : {
4807 1617297 : FOR_EACH_EDGE (e, ei, last_bb->succs)
4808 1617297 : if (e->dest != other_bb)
4809 : break;
4810 998296 : if (e == NULL)
4811 : break;
4812 998296 : if (!single_pred_p (e->dest))
4813 : break;
4814 959344 : if (!suitable_cond_bb (e->dest, last_bb, &other_bb, NULL, false))
4815 : break;
4816 846980 : if (!no_side_effect_bb (e->dest))
4817 : break;
4818 212799 : last_bb = e->dest;
4819 : }
4820 790963 : if (first_bb == last_bb)
4821 : return cfg_cleanup_needed;
4822 : /* Here basic blocks first_bb through last_bb's predecessor
4823 : end with GIMPLE_COND, all of them have one of the edges to
4824 : other_bb and another to another block in the range,
4825 : all blocks except first_bb don't have side-effects and
4826 : last_bb ends with either GIMPLE_COND, or cast satisfying
4827 : final_range_test_p. */
4828 215351 : for (bb = last_bb; ; bb = single_pred (bb))
4829 : {
4830 370331 : enum tree_code code;
4831 370331 : tree lhs, rhs;
4832 370331 : inter_bb_range_test_entry bb_ent;
4833 :
4834 370331 : bb_ent.op = NULL_TREE;
4835 370331 : bb_ent.first_idx = ops.length ();
4836 370331 : bb_ent.last_idx = bb_ent.first_idx;
4837 370331 : e = find_edge (bb, other_bb);
4838 370331 : stmt = last_nondebug_stmt (bb);
4839 370331 : gimple_set_visited (stmt, true);
4840 370331 : if (gimple_code (stmt) != GIMPLE_COND)
4841 : {
4842 5466 : use_operand_p use_p;
4843 5466 : gimple *phi;
4844 5466 : edge e2;
4845 5466 : unsigned int d;
4846 :
4847 5466 : lhs = gimple_assign_lhs (stmt);
4848 5466 : rhs = gimple_assign_rhs1 (stmt);
4849 5466 : gcc_assert (bb == last_bb);
4850 :
4851 : /* stmt is
4852 : _123 = (int) _234;
4853 : OR
4854 : _234 = a_2(D) == 2;
4855 :
4856 : followed by:
4857 : <bb M>:
4858 : # _345 = PHI <_123(N), 1(...), 1(...)>
4859 :
4860 : or 0 instead of 1. If it is 0, the _234
4861 : range test is anded together with all the
4862 : other range tests, if it is 1, it is ored with
4863 : them. */
4864 5466 : single_imm_use (lhs, &use_p, &phi);
4865 5466 : gcc_assert (gimple_code (phi) == GIMPLE_PHI);
4866 5466 : e2 = find_edge (first_bb, other_bb);
4867 5466 : d = e2->dest_idx;
4868 5466 : gcc_assert (gimple_phi_arg_def (phi, e->dest_idx) == lhs);
4869 5466 : if (integer_zerop (gimple_phi_arg_def (phi, d)))
4870 : code = BIT_AND_EXPR;
4871 : else
4872 : {
4873 2891 : gcc_checking_assert (integer_onep (gimple_phi_arg_def (phi, d)));
4874 : code = BIT_IOR_EXPR;
4875 : }
4876 :
4877 : /* If _234 SSA_NAME_DEF_STMT is
4878 : _234 = _567 | _789;
4879 : (or &, corresponding to 1/0 in the phi arguments,
4880 : push into ops the individual range test arguments
4881 : of the bitwise or resp. and, recursively. */
4882 5466 : if (TREE_CODE (rhs) == SSA_NAME
4883 5466 : && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt))
4884 : != tcc_comparison)
4885 2966 : && !get_ops (rhs, code, &ops,
4886 : loop_containing_stmt (stmt))
4887 8257 : && has_single_use (rhs))
4888 : {
4889 : /* Otherwise, push the _234 range test itself. */
4890 2771 : operand_entry *oe = operand_entry_pool.allocate ();
4891 :
4892 2771 : oe->op = rhs;
4893 2771 : oe->rank = code;
4894 2771 : oe->id = 0;
4895 2771 : oe->count = 1;
4896 2771 : oe->stmt_to_insert = NULL;
4897 2771 : ops.safe_push (oe);
4898 2771 : bb_ent.last_idx++;
4899 2771 : bb_ent.op = rhs;
4900 : }
4901 2695 : else if (is_gimple_assign (stmt)
4902 2695 : && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt))
4903 : == tcc_comparison)
4904 2500 : && !get_ops (lhs, code, &ops,
4905 : loop_containing_stmt (stmt))
4906 5195 : && has_single_use (lhs))
4907 : {
4908 2500 : operand_entry *oe = operand_entry_pool.allocate ();
4909 2500 : oe->op = lhs;
4910 2500 : oe->rank = code;
4911 2500 : oe->id = 0;
4912 2500 : oe->count = 1;
4913 2500 : ops.safe_push (oe);
4914 2500 : bb_ent.last_idx++;
4915 2500 : bb_ent.op = lhs;
4916 : }
4917 : else
4918 : {
4919 195 : bb_ent.last_idx = ops.length ();
4920 195 : bb_ent.op = rhs;
4921 : }
4922 5466 : bbinfo.safe_push (bb_ent);
4923 16593 : for (unsigned int i = bb_ent.first_idx; i < bb_ent.last_idx; ++i)
4924 5661 : ops[i]->id = bb->index;
4925 5466 : continue;
4926 5466 : }
4927 364865 : else if (bb == last_bb)
4928 : {
4929 : /* For last_bb, handle also:
4930 : if (x_3(D) == 3)
4931 : goto <bb 6>; [34.00%]
4932 : else
4933 : goto <bb 7>; [66.00%]
4934 :
4935 : <bb 6> [local count: 79512730]:
4936 :
4937 : <bb 7> [local count: 1073741824]:
4938 : # prephitmp_7 = PHI <1(3), 1(4), 0(5), 1(2), 1(6)>
4939 : where bb 7 is OTHER_BB, but the PHI values from the
4940 : earlier bbs match the path through the empty bb
4941 : in between. */
4942 149514 : bool test_swapped_p = false;
4943 149514 : bool ok = suitable_cond_bb (single_pred (last_bb), last_bb,
4944 : &other_bb, &test_swapped_p, true);
4945 149514 : gcc_assert (ok);
4946 149514 : if (test_swapped_p)
4947 528 : e = EDGE_SUCC (bb, e == EDGE_SUCC (bb, 0) ? 1 : 0);
4948 : }
4949 : /* Otherwise stmt is GIMPLE_COND. */
4950 364865 : code = gimple_cond_code (stmt);
4951 364865 : lhs = gimple_cond_lhs (stmt);
4952 364865 : rhs = gimple_cond_rhs (stmt);
4953 364865 : if (TREE_CODE (lhs) == SSA_NAME
4954 364801 : && INTEGRAL_TYPE_P (TREE_TYPE (lhs))
4955 669553 : && ((code != EQ_EXPR && code != NE_EXPR)
4956 229217 : || rhs != boolean_false_node
4957 : /* Either push into ops the individual bitwise
4958 : or resp. and operands, depending on which
4959 : edge is other_bb. */
4960 44432 : || !get_ops (lhs, (((e->flags & EDGE_TRUE_VALUE) == 0)
4961 44432 : ^ (code == EQ_EXPR))
4962 : ? BIT_AND_EXPR : BIT_IOR_EXPR, &ops,
4963 : loop_containing_stmt (stmt))))
4964 : {
4965 : /* Or push the GIMPLE_COND stmt itself. */
4966 288969 : operand_entry *oe = operand_entry_pool.allocate ();
4967 :
4968 288969 : oe->op = NULL;
4969 577938 : oe->rank = (e->flags & EDGE_TRUE_VALUE)
4970 288969 : ? BIT_IOR_EXPR : BIT_AND_EXPR;
4971 : /* oe->op = NULL signs that there is no SSA_NAME
4972 : for the range test, and oe->id instead is the
4973 : basic block number, at which's end the GIMPLE_COND
4974 : is. */
4975 288969 : oe->id = bb->index;
4976 288969 : oe->count = 1;
4977 288969 : oe->stmt_to_insert = NULL;
4978 288969 : ops.safe_push (oe);
4979 288969 : bb_ent.op = NULL;
4980 288969 : bb_ent.last_idx++;
4981 : }
4982 75896 : else if (ops.length () > bb_ent.first_idx)
4983 : {
4984 15647 : bb_ent.op = lhs;
4985 15647 : bb_ent.last_idx = ops.length ();
4986 : }
4987 364865 : bbinfo.safe_push (bb_ent);
4988 1055896 : for (unsigned int i = bb_ent.first_idx; i < bb_ent.last_idx; ++i)
4989 326166 : ops[i]->id = bb->index;
4990 364865 : if (bb == first_bb)
4991 : break;
4992 215351 : }
4993 19446749 : if (ops.length () > 1)
4994 120986 : any_changes = optimize_range_tests (ERROR_MARK, &ops, first_bb);
4995 120986 : if (any_changes)
4996 : {
4997 : unsigned int idx, max_idx = 0;
4998 : /* update_ops relies on has_single_use predicates returning the
4999 : same values as it did during get_ops earlier. Additionally it
5000 : never removes statements, only adds new ones and it should walk
5001 : from the single imm use and check the predicate already before
5002 : making those changes.
5003 : On the other side, the handling of GIMPLE_COND directly can turn
5004 : previously multiply used SSA_NAMEs into single use SSA_NAMEs, so
5005 : it needs to be done in a separate loop afterwards. */
5006 21714 : for (bb = last_bb, idx = 0; ; bb = single_pred (bb), idx++)
5007 : {
5008 35847 : if (bbinfo[idx].first_idx < bbinfo[idx].last_idx
5009 35847 : && bbinfo[idx].op != NULL_TREE)
5010 : {
5011 3579 : tree new_op;
5012 :
5013 3579 : max_idx = idx;
5014 3579 : stmt = last_nondebug_stmt (bb);
5015 7158 : new_op = update_ops (bbinfo[idx].op,
5016 : (enum tree_code)
5017 3579 : ops[bbinfo[idx].first_idx]->rank,
5018 3579 : ops, &bbinfo[idx].first_idx,
5019 : loop_containing_stmt (stmt));
5020 3579 : if (new_op == NULL_TREE)
5021 : {
5022 385 : gcc_assert (bb == last_bb);
5023 385 : new_op = ops[bbinfo[idx].first_idx++]->op;
5024 : }
5025 3579 : if (bbinfo[idx].op != new_op)
5026 : {
5027 3324 : imm_use_iterator iter;
5028 3324 : use_operand_p use_p;
5029 3324 : gimple *use_stmt, *cast_or_tcc_cmp_stmt = NULL;
5030 :
5031 6658 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, bbinfo[idx].op)
5032 3334 : if (is_gimple_debug (use_stmt))
5033 10 : continue;
5034 3324 : else if (gimple_code (use_stmt) == GIMPLE_COND
5035 3324 : || gimple_code (use_stmt) == GIMPLE_PHI)
5036 6080 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
5037 3040 : SET_USE (use_p, new_op);
5038 284 : else if ((is_gimple_assign (use_stmt)
5039 284 : && (TREE_CODE_CLASS
5040 : (gimple_assign_rhs_code (use_stmt))
5041 : == tcc_comparison)))
5042 : cast_or_tcc_cmp_stmt = use_stmt;
5043 284 : else if (gimple_assign_cast_p (use_stmt))
5044 : cast_or_tcc_cmp_stmt = use_stmt;
5045 : else
5046 0 : gcc_unreachable ();
5047 :
5048 3324 : if (cast_or_tcc_cmp_stmt)
5049 : {
5050 284 : gcc_assert (bb == last_bb);
5051 284 : tree lhs = gimple_assign_lhs (cast_or_tcc_cmp_stmt);
5052 284 : tree new_lhs = make_ssa_name (TREE_TYPE (lhs));
5053 284 : enum tree_code rhs_code
5054 284 : = gimple_assign_cast_p (cast_or_tcc_cmp_stmt)
5055 284 : ? gimple_assign_rhs_code (cast_or_tcc_cmp_stmt)
5056 : : CONVERT_EXPR;
5057 284 : gassign *g;
5058 284 : if (is_gimple_min_invariant (new_op))
5059 : {
5060 73 : new_op = fold_convert (TREE_TYPE (lhs), new_op);
5061 73 : g = gimple_build_assign (new_lhs, new_op);
5062 : }
5063 : else
5064 211 : g = gimple_build_assign (new_lhs, rhs_code, new_op);
5065 284 : gimple_stmt_iterator gsi
5066 284 : = gsi_for_stmt (cast_or_tcc_cmp_stmt);
5067 284 : gimple_set_uid (g, gimple_uid (cast_or_tcc_cmp_stmt));
5068 284 : gimple_set_visited (g, true);
5069 284 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5070 568 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
5071 284 : if (is_gimple_debug (use_stmt))
5072 0 : continue;
5073 284 : else if (gimple_code (use_stmt) == GIMPLE_COND
5074 284 : || gimple_code (use_stmt) == GIMPLE_PHI)
5075 568 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
5076 284 : SET_USE (use_p, new_lhs);
5077 : else
5078 284 : gcc_unreachable ();
5079 : }
5080 : }
5081 : }
5082 35847 : if (bb == first_bb)
5083 : break;
5084 21714 : }
5085 21714 : for (bb = last_bb, idx = 0; ; bb = single_pred (bb), idx++)
5086 : {
5087 35847 : if (bbinfo[idx].first_idx < bbinfo[idx].last_idx
5088 31810 : && bbinfo[idx].op == NULL_TREE
5089 67657 : && ops[bbinfo[idx].first_idx]->op != NULL_TREE)
5090 : {
5091 55346 : gcond *cond_stmt = as_a <gcond *> (*gsi_last_bb (bb));
5092 :
5093 27673 : if (idx > max_idx)
5094 : max_idx = idx;
5095 :
5096 : /* If we collapse the conditional to a true/false
5097 : condition, then bubble that knowledge up to our caller. */
5098 27673 : if (integer_zerop (ops[bbinfo[idx].first_idx]->op))
5099 : {
5100 11602 : gimple_cond_make_false (cond_stmt);
5101 11602 : cfg_cleanup_needed = true;
5102 : }
5103 16071 : else if (integer_onep (ops[bbinfo[idx].first_idx]->op))
5104 : {
5105 3713 : gimple_cond_make_true (cond_stmt);
5106 3713 : cfg_cleanup_needed = true;
5107 : }
5108 : else
5109 : {
5110 12358 : gimple_cond_set_code (cond_stmt, NE_EXPR);
5111 12358 : gimple_cond_set_lhs (cond_stmt,
5112 12358 : ops[bbinfo[idx].first_idx]->op);
5113 12358 : gimple_cond_set_rhs (cond_stmt, boolean_false_node);
5114 : }
5115 27673 : update_stmt (cond_stmt);
5116 : }
5117 35847 : if (bb == first_bb)
5118 : break;
5119 21714 : }
5120 :
5121 : /* The above changes could result in basic blocks after the first
5122 : modified one, up to and including last_bb, to be executed even if
5123 : they would not be in the original program. If the value ranges of
5124 : assignment lhs' in those bbs were dependent on the conditions
5125 : guarding those basic blocks which now can change, the VRs might
5126 : be incorrect. As no_side_effect_bb should ensure those SSA_NAMEs
5127 : are only used within the same bb, it should be not a big deal if
5128 : we just reset all the VRs in those bbs. See PR68671. */
5129 34591 : for (bb = last_bb, idx = 0; idx < max_idx; bb = single_pred (bb), idx++)
5130 20458 : reset_flow_sensitive_info_in_bb (bb);
5131 : }
5132 : return cfg_cleanup_needed;
5133 19325763 : }
5134 :
5135 : /* Remove def stmt of VAR if VAR has zero uses and recurse
5136 : on rhs1 operand if so. */
5137 :
5138 : static void
5139 71924 : remove_visited_stmt_chain (tree var)
5140 : {
5141 97006 : gimple *stmt;
5142 97006 : gimple_stmt_iterator gsi;
5143 :
5144 122088 : while (1)
5145 : {
5146 97006 : if (TREE_CODE (var) != SSA_NAME || !has_zero_uses (var))
5147 : return;
5148 36640 : stmt = SSA_NAME_DEF_STMT (var);
5149 36640 : if (is_gimple_assign (stmt) && gimple_visited_p (stmt))
5150 : {
5151 25082 : var = gimple_assign_rhs1 (stmt);
5152 25082 : gsi = gsi_for_stmt (stmt);
5153 25082 : reassoc_remove_stmt (&gsi);
5154 25082 : release_defs (stmt);
5155 : }
5156 : else
5157 : return;
5158 : }
5159 : }
5160 :
5161 : /* This function checks three consecutive operands in
5162 : passed operands vector OPS starting from OPINDEX and
5163 : swaps two operands if it is profitable for binary operation
5164 : consuming OPINDEX + 1 and OPINDEX + 2 operands.
5165 :
5166 : We pair ops with the same rank if possible. */
5167 :
5168 : static void
5169 130313 : swap_ops_for_binary_stmt (const vec<operand_entry *> &ops,
5170 : unsigned int opindex)
5171 : {
5172 130313 : operand_entry *oe1, *oe2, *oe3;
5173 :
5174 130313 : oe1 = ops[opindex];
5175 130313 : oe2 = ops[opindex + 1];
5176 130313 : oe3 = ops[opindex + 2];
5177 :
5178 130313 : if (oe1->rank == oe2->rank && oe2->rank != oe3->rank)
5179 16858 : std::swap (*oe1, *oe3);
5180 113455 : else if (oe1->rank == oe3->rank && oe2->rank != oe3->rank)
5181 363 : std::swap (*oe1, *oe2);
5182 130313 : }
5183 :
5184 : /* Return true if OP is set by a comparison in BB, which a conditional
5185 : compare can compute into the condition flags. */
5186 :
5187 : static bool
5188 2 : ccmp_comparison_p (tree op, basic_block bb)
5189 : {
5190 2 : if (TREE_CODE (op) != SSA_NAME)
5191 : return false;
5192 :
5193 2 : gimple *def = SSA_NAME_DEF_STMT (op);
5194 2 : if (!is_gimple_assign (def)
5195 2 : || gimple_bb (def) != bb
5196 4 : || TREE_CODE_CLASS (gimple_assign_rhs_code (def)) != tcc_comparison)
5197 : return false;
5198 :
5199 : /* A conditional compare sets the flags from one scalar comparison. */
5200 2 : machine_mode mode = TYPE_MODE (TREE_TYPE (gimple_assign_rhs1 (def)));
5201 2 : return SCALAR_INT_MODE_P (mode) || SCALAR_FLOAT_MODE_P (mode);
5202 : }
5203 :
5204 : /* Return how strongly OP should start a conditional compare sequence.
5205 : FOLDABLE is true if OP can be folded into the flags. */
5206 :
5207 : static ccmp_first_kind
5208 2 : ccmp_first_preference (tree op, bool foldable)
5209 : {
5210 2 : if (!foldable)
5211 : return CCMP_FIRST_NONE;
5212 :
5213 2 : gimple *def = SSA_NAME_DEF_STMT (op);
5214 2 : gcc_checking_assert (is_gimple_assign (def));
5215 :
5216 2 : tree op0 = gimple_assign_rhs1 (def);
5217 2 : tree op1 = gimple_assign_rhs2 (def);
5218 2 : ccmp_first_kind kind = targetm.ccmp_first_preference
5219 2 : (TYPE_MODE (TREE_TYPE (op0)), gimple_assign_rhs_code (def), op0, op1);
5220 2 : if (kind == CCMP_FIRST_UNSUPPORTED || !gimple_could_trap_p (def))
5221 2 : return kind;
5222 : return CCMP_FIRST_REQUIRED;
5223 : }
5224 :
5225 : /* Return true if all operands in OPS form one conditional compare sequence
5226 : for CODE in BB. Put the operand that must start the sequence last in OPS,
5227 : where rewrite_expr_tree builds the innermost pair. Every operand must be a
5228 : comparison, and at least one must be foldable into the flags. */
5229 :
5230 : static bool
5231 182493 : prepare_ccmp_chain (vec<operand_entry *> &ops, tree_code code, basic_block bb)
5232 : {
5233 182493 : if (!targetm.have_ccmp ()
5234 182493 : || (code != BIT_AND_EXPR && code != BIT_IOR_EXPR))
5235 : return false;
5236 :
5237 : bool seen_foldable = false;
5238 : int head = -1;
5239 : ccmp_first_kind head_kind = CCMP_FIRST_NONE;
5240 :
5241 2 : for (unsigned int i = 0; i < ops.length (); ++i)
5242 : {
5243 2 : const operand_entry *oe = ops[i];
5244 2 : if (!ccmp_comparison_p (oe->op, bb))
5245 : return false;
5246 :
5247 2 : bool foldable = has_single_use (oe->op);
5248 :
5249 2 : ccmp_first_kind kind
5250 2 : = ccmp_first_preference (oe->op, foldable);
5251 2 : if (kind == CCMP_FIRST_UNSUPPORTED)
5252 : return false;
5253 0 : if (kind == CCMP_FIRST_REQUIRED && head_kind == CCMP_FIRST_REQUIRED)
5254 : return false;
5255 0 : if (kind > head_kind)
5256 : {
5257 0 : head = i;
5258 0 : head_kind = kind;
5259 : }
5260 :
5261 : /* Comparison leaves can have uses outside this chain. */
5262 0 : seen_foldable |= foldable;
5263 : }
5264 :
5265 0 : if (head >= 0)
5266 0 : std::swap (*ops[head], *ops.last ());
5267 : return seen_foldable;
5268 : }
5269 :
5270 : /* If definition of RHS1 or RHS2 dominates STMT, return the later of those
5271 : two definitions, otherwise return STMT. Sets INSERT_BEFORE to indicate
5272 : whether RHS1 op RHS2 can be inserted before or needs to be inserted
5273 : after the returned stmt. */
5274 :
5275 : static inline gimple *
5276 377322 : find_insert_point (gimple *stmt, tree rhs1, tree rhs2, bool &insert_before)
5277 : {
5278 377322 : insert_before = true;
5279 377322 : if (TREE_CODE (rhs1) == SSA_NAME
5280 377322 : && reassoc_stmt_dominates_stmt_p (stmt, SSA_NAME_DEF_STMT (rhs1)))
5281 : {
5282 11084 : stmt = SSA_NAME_DEF_STMT (rhs1);
5283 11084 : insert_before = false;
5284 : }
5285 377322 : if (TREE_CODE (rhs2) == SSA_NAME
5286 377322 : && reassoc_stmt_dominates_stmt_p (stmt, SSA_NAME_DEF_STMT (rhs2)))
5287 : {
5288 20094 : stmt = SSA_NAME_DEF_STMT (rhs2);
5289 20094 : insert_before = false;
5290 : }
5291 377322 : return stmt;
5292 : }
5293 :
5294 : /* If the stmt that defines operand has to be inserted, insert it
5295 : before the use. */
5296 : static void
5297 85 : insert_stmt_before_use (gimple *stmt, gimple *stmt_to_insert)
5298 : {
5299 85 : gcc_assert (is_gimple_assign (stmt_to_insert));
5300 85 : tree rhs1 = gimple_assign_rhs1 (stmt_to_insert);
5301 85 : tree rhs2 = gimple_assign_rhs2 (stmt_to_insert);
5302 85 : bool insert_before;
5303 85 : gimple *insert_point = find_insert_point (stmt, rhs1, rhs2, insert_before);
5304 85 : gimple_stmt_iterator gsi = gsi_for_stmt (insert_point);
5305 85 : gimple_set_uid (stmt_to_insert, gimple_uid (insert_point));
5306 :
5307 : /* If the insert point is not stmt, then insert_point would be
5308 : the point where operand rhs1 or rhs2 is defined. In this case,
5309 : stmt_to_insert has to be inserted afterwards. This would
5310 : only happen when the stmt insertion point is flexible. */
5311 85 : if (insert_before)
5312 85 : gsi_insert_before (&gsi, stmt_to_insert, GSI_NEW_STMT);
5313 : else
5314 0 : insert_stmt_after (stmt_to_insert, insert_point);
5315 85 : }
5316 :
5317 :
5318 : /* Recursively rewrite our linearized statements so that the operators
5319 : match those in OPS[OPINDEX], putting the computation in rank
5320 : order. Return new lhs.
5321 : CHANGED is true if we shouldn't reuse the lhs SSA_NAME both in
5322 : the current stmt and during recursive invocations.
5323 : NEXT_CHANGED is true if we shouldn't reuse the lhs SSA_NAME in
5324 : recursive invocations. */
5325 :
5326 : static tree
5327 4973485 : rewrite_expr_tree (gimple *stmt, enum tree_code rhs_code, unsigned int opindex,
5328 : const vec<operand_entry *> &ops, bool changed,
5329 : bool next_changed)
5330 : {
5331 4973485 : tree rhs1 = gimple_assign_rhs1 (stmt);
5332 4973485 : tree rhs2 = gimple_assign_rhs2 (stmt);
5333 4973485 : tree lhs = gimple_assign_lhs (stmt);
5334 4973485 : operand_entry *oe;
5335 :
5336 : /* The final recursion case for this function is that you have
5337 : exactly two operations left.
5338 : If we had exactly one op in the entire list to start with, we
5339 : would have never called this function, and the tail recursion
5340 : rewrites them one at a time. */
5341 9946970 : if (opindex + 2 == ops.length ())
5342 : {
5343 4743498 : operand_entry *oe1, *oe2;
5344 :
5345 4743498 : oe1 = ops[opindex];
5346 4743498 : oe2 = ops[opindex + 1];
5347 4743498 : if (commutative_tree_code (rhs_code)
5348 4743498 : && tree_swap_operands_p (oe1->op, oe2->op))
5349 : std::swap (oe1, oe2);
5350 :
5351 4743498 : if (rhs1 != oe1->op || rhs2 != oe2->op)
5352 : {
5353 247408 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5354 247408 : unsigned int uid = gimple_uid (stmt);
5355 :
5356 247408 : if (dump_file && (dump_flags & TDF_DETAILS))
5357 : {
5358 31 : fprintf (dump_file, "Transforming ");
5359 31 : print_gimple_stmt (dump_file, stmt, 0);
5360 : }
5361 :
5362 : /* If the stmt that defines operand has to be inserted, insert it
5363 : before the use. */
5364 247408 : if (oe1->stmt_to_insert)
5365 34 : insert_stmt_before_use (stmt, oe1->stmt_to_insert);
5366 247408 : if (oe2->stmt_to_insert)
5367 51 : insert_stmt_before_use (stmt, oe2->stmt_to_insert);
5368 : /* Even when changed is false, reassociation could have e.g. removed
5369 : some redundant operations, so unless we are just swapping the
5370 : arguments or unless there is no change at all (then we just
5371 : return lhs), force creation of a new SSA_NAME. */
5372 247408 : if (changed || ((rhs1 != oe2->op || rhs2 != oe1->op) && opindex))
5373 : {
5374 110396 : bool insert_before;
5375 110396 : gimple *insert_point
5376 110396 : = find_insert_point (stmt, oe1->op, oe2->op, insert_before);
5377 110396 : lhs = make_ssa_name (TREE_TYPE (lhs));
5378 110396 : stmt
5379 110396 : = gimple_build_assign (lhs, rhs_code,
5380 : oe1->op, oe2->op);
5381 110396 : gimple_set_uid (stmt, uid);
5382 110396 : gimple_set_visited (stmt, true);
5383 110396 : if (insert_before)
5384 87875 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5385 : else
5386 22521 : insert_stmt_after (stmt, insert_point);
5387 110396 : }
5388 : else
5389 : {
5390 137012 : bool insert_before;
5391 137012 : gcc_checking_assert (find_insert_point (stmt, oe1->op, oe2->op,
5392 : insert_before)
5393 : == stmt);
5394 137012 : gimple_assign_set_rhs1 (stmt, oe1->op);
5395 137012 : gimple_assign_set_rhs2 (stmt, oe2->op);
5396 137012 : update_stmt (stmt);
5397 : }
5398 :
5399 247408 : if (rhs1 != oe1->op && rhs1 != oe2->op)
5400 55372 : remove_visited_stmt_chain (rhs1);
5401 :
5402 247408 : if (dump_file && (dump_flags & TDF_DETAILS))
5403 : {
5404 31 : fprintf (dump_file, " into ");
5405 31 : print_gimple_stmt (dump_file, stmt, 0);
5406 : }
5407 : }
5408 : return lhs;
5409 : }
5410 :
5411 : /* If we hit here, we should have 3 or more ops left. */
5412 229987 : gcc_assert (opindex + 2 < ops.length ());
5413 :
5414 : /* Rewrite the next operator. */
5415 229987 : oe = ops[opindex];
5416 :
5417 : /* If the stmt that defines operand has to be inserted, insert it
5418 : before the use. */
5419 229987 : if (oe->stmt_to_insert)
5420 0 : insert_stmt_before_use (stmt, oe->stmt_to_insert);
5421 :
5422 : /* Recurse on the LHS of the binary operator, which is guaranteed to
5423 : be the non-leaf side. */
5424 229987 : tree new_rhs1
5425 441015 : = rewrite_expr_tree (SSA_NAME_DEF_STMT (rhs1), rhs_code, opindex + 1, ops,
5426 211028 : changed || oe->op != rhs2 || next_changed,
5427 : false);
5428 :
5429 229987 : if (oe->op != rhs2 || new_rhs1 != rhs1)
5430 : {
5431 129829 : if (dump_file && (dump_flags & TDF_DETAILS))
5432 : {
5433 6 : fprintf (dump_file, "Transforming ");
5434 6 : print_gimple_stmt (dump_file, stmt, 0);
5435 : }
5436 :
5437 : /* If changed is false, this is either opindex == 0
5438 : or all outer rhs2's were equal to corresponding oe->op,
5439 : and powi_result is NULL.
5440 : That means lhs is equivalent before and after reassociation.
5441 : Otherwise ensure the old lhs SSA_NAME is not reused and
5442 : create a new stmt as well, so that any debug stmts will be
5443 : properly adjusted. */
5444 129829 : if (changed)
5445 : {
5446 18724 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5447 18724 : unsigned int uid = gimple_uid (stmt);
5448 18724 : bool insert_before;
5449 18724 : gimple *insert_point = find_insert_point (stmt, new_rhs1, oe->op,
5450 : insert_before);
5451 :
5452 18724 : lhs = make_ssa_name (TREE_TYPE (lhs));
5453 18724 : stmt = gimple_build_assign (lhs, rhs_code,
5454 : new_rhs1, oe->op);
5455 18724 : gimple_set_uid (stmt, uid);
5456 18724 : gimple_set_visited (stmt, true);
5457 18724 : if (insert_before)
5458 11145 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5459 : else
5460 7579 : insert_stmt_after (stmt, insert_point);
5461 : }
5462 : else
5463 : {
5464 111105 : bool insert_before;
5465 111105 : gcc_checking_assert (find_insert_point (stmt, new_rhs1, oe->op,
5466 : insert_before)
5467 : == stmt);
5468 111105 : gimple_assign_set_rhs1 (stmt, new_rhs1);
5469 111105 : gimple_assign_set_rhs2 (stmt, oe->op);
5470 111105 : update_stmt (stmt);
5471 : }
5472 :
5473 129829 : if (dump_file && (dump_flags & TDF_DETAILS))
5474 : {
5475 6 : fprintf (dump_file, " into ");
5476 6 : print_gimple_stmt (dump_file, stmt, 0);
5477 : }
5478 : }
5479 : return lhs;
5480 : }
5481 :
5482 : /* Find out how many cycles we need to compute statements chain.
5483 : OPS_NUM holds number os statements in a chain. CPU_WIDTH is a
5484 : maximum number of independent statements we may execute per cycle. */
5485 :
5486 : static int
5487 18844 : get_required_cycles (int ops_num, int cpu_width)
5488 : {
5489 18844 : int res;
5490 18844 : int elog;
5491 18844 : unsigned int rest;
5492 :
5493 : /* While we have more than 2 * cpu_width operands
5494 : we may reduce number of operands by cpu_width
5495 : per cycle. */
5496 18844 : res = ops_num / (2 * cpu_width);
5497 :
5498 : /* Remained operands count may be reduced twice per cycle
5499 : until we have only one operand. */
5500 18844 : rest = (unsigned)(ops_num - res * cpu_width);
5501 18844 : elog = exact_log2 (rest);
5502 8899 : if (elog >= 0)
5503 8899 : res += elog;
5504 : else
5505 19890 : res += floor_log2 (rest) + 1;
5506 :
5507 18844 : return res;
5508 : }
5509 :
5510 : /* Given that the target fully pipelines FMA instructions, return the latency
5511 : of MULT_EXPRs that can't be hidden by the FMAs. WIDTH is the number of
5512 : pipes. */
5513 :
5514 : static inline int
5515 0 : get_mult_latency_consider_fma (int ops_num, int mult_num, int width)
5516 : {
5517 0 : gcc_checking_assert (mult_num && mult_num <= ops_num);
5518 :
5519 : /* For each partition, if mult_num == ops_num, there's latency(MULT)*2.
5520 : e.g:
5521 :
5522 : A * B + C * D
5523 : =>
5524 : _1 = A * B;
5525 : _2 = .FMA (C, D, _1);
5526 :
5527 : Otherwise there's latency(MULT)*1 in the first FMA. */
5528 0 : return CEIL (ops_num, width) == CEIL (mult_num, width) ? 2 : 1;
5529 : }
5530 :
5531 : /* Returns an optimal number of registers to use for computation of
5532 : given statements.
5533 :
5534 : LHS is the result ssa name of OPS. MULT_NUM is number of sub-expressions
5535 : that are MULT_EXPRs, when OPS are PLUS_EXPRs or MINUS_EXPRs. */
5536 :
5537 : static int
5538 19149 : get_reassociation_width (vec<operand_entry *> *ops, int mult_num, tree lhs,
5539 : enum tree_code opc, machine_mode mode)
5540 : {
5541 19149 : int param_width = param_tree_reassoc_width;
5542 19149 : int width;
5543 19149 : int width_min;
5544 19149 : int cycles_best;
5545 19149 : int ops_num = ops->length ();
5546 :
5547 19149 : if (param_width > 0)
5548 : width = param_width;
5549 : else
5550 19105 : width = targetm.sched.reassociation_width (opc, mode);
5551 :
5552 19149 : if (width == 1)
5553 : return width;
5554 :
5555 : /* Get the minimal time required for sequence computation. */
5556 7188 : cycles_best = get_required_cycles (ops_num, width);
5557 :
5558 : /* Check if we may use less width and still compute sequence for
5559 : the same time. It will allow us to reduce registers usage.
5560 : get_required_cycles is monotonically increasing with lower width
5561 : so we can perform a binary search for the minimal width that still
5562 : results in the optimal cycle count. */
5563 7188 : width_min = 1;
5564 :
5565 : /* If the target fully pipelines FMA instruction, the multiply part can start
5566 : already if its operands are ready. Assuming symmetric pipes are used for
5567 : FMUL/FADD/FMA, then for a sequence of FMA like:
5568 :
5569 : _8 = .FMA (_2, _3, _1);
5570 : _9 = .FMA (_5, _4, _8);
5571 : _10 = .FMA (_7, _6, _9);
5572 :
5573 : , if width=1, the latency is latency(MULT) + latency(ADD)*3.
5574 : While with width=2:
5575 :
5576 : _8 = _4 * _5;
5577 : _9 = .FMA (_2, _3, _1);
5578 : _10 = .FMA (_6, _7, _8);
5579 : _11 = _9 + _10;
5580 :
5581 : , it is latency(MULT)*2 + latency(ADD)*2. Assuming latency(MULT) >=
5582 : latency(ADD), the first variant is preferred.
5583 :
5584 : Find out if we can get a smaller width considering FMA.
5585 : Assume FMUL and FMA use the same units that can also do FADD.
5586 : For other scenarios, such as when FMUL and FADD are using separated units,
5587 : the following code may not apply. */
5588 :
5589 7188 : int width_mult = targetm.sched.reassociation_width (MULT_EXPR, mode);
5590 7188 : if (width > 1 && mult_num && param_fully_pipelined_fma
5591 0 : && width_mult <= width)
5592 : {
5593 : /* Latency of MULT_EXPRs. */
5594 0 : int lat_mul
5595 0 : = get_mult_latency_consider_fma (ops_num, mult_num, width_mult);
5596 :
5597 : /* Quick search might not apply. So start from 1. */
5598 0 : for (int i = 1; i < width_mult; i++)
5599 : {
5600 0 : int lat_mul_new
5601 0 : = get_mult_latency_consider_fma (ops_num, mult_num, i);
5602 0 : int lat_add_new = get_required_cycles (ops_num, i);
5603 :
5604 : /* Assume latency(MULT) >= latency(ADD). */
5605 0 : if (lat_mul - lat_mul_new >= lat_add_new - cycles_best)
5606 : {
5607 : width = i;
5608 : break;
5609 : }
5610 : }
5611 : }
5612 : else
5613 : {
5614 16944 : while (width > width_min)
5615 : {
5616 11656 : int width_mid = (width + width_min) / 2;
5617 :
5618 11656 : if (get_required_cycles (ops_num, width_mid) == cycles_best)
5619 : width = width_mid;
5620 2097 : else if (width_min < width_mid)
5621 : width_min = width_mid;
5622 : else
5623 : break;
5624 : }
5625 : }
5626 :
5627 : /* If there's loop dependent FMA result, return width=2 to avoid it. This is
5628 : better than skipping these FMA candidates in widening_mul. */
5629 7188 : if (width == 1
5630 7188 : && maybe_le (tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (lhs))),
5631 : param_avoid_fma_max_bits))
5632 : {
5633 : /* Look for cross backedge dependency:
5634 : 1. LHS is a phi argument in the same basic block it is defined.
5635 : 2. And the result of the phi node is used in OPS. */
5636 4868 : basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (lhs));
5637 :
5638 4868 : use_operand_p use_p;
5639 4868 : imm_use_iterator iter;
5640 10943 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
5641 10310 : if (gphi *phi = dyn_cast<gphi *> (USE_STMT (use_p)))
5642 : {
5643 4235 : if (gimple_phi_arg_edge (phi, phi_arg_index_from_use (use_p))->src
5644 : != bb)
5645 0 : continue;
5646 4235 : tree phi_result = gimple_phi_result (phi);
5647 4235 : operand_entry *oe;
5648 4235 : unsigned int j;
5649 20976 : FOR_EACH_VEC_ELT (*ops, j, oe)
5650 : {
5651 12705 : if (TREE_CODE (oe->op) != SSA_NAME)
5652 0 : continue;
5653 :
5654 : /* Result of phi is operand of PLUS_EXPR. */
5655 12705 : if (oe->op == phi_result)
5656 2039 : return 2;
5657 :
5658 : /* Check is result of phi is operand of MULT_EXPR. */
5659 10666 : gimple *def_stmt = SSA_NAME_DEF_STMT (oe->op);
5660 10666 : if (is_gimple_assign (def_stmt)
5661 10666 : && gimple_assign_rhs_code (def_stmt) == NEGATE_EXPR)
5662 : {
5663 2246 : tree rhs = gimple_assign_rhs1 (def_stmt);
5664 2246 : if (TREE_CODE (rhs) == SSA_NAME)
5665 : {
5666 2246 : if (rhs == phi_result)
5667 : return 2;
5668 2246 : def_stmt = SSA_NAME_DEF_STMT (rhs);
5669 : }
5670 : }
5671 10666 : if (is_gimple_assign (def_stmt)
5672 10666 : && gimple_assign_rhs_code (def_stmt) == MULT_EXPR)
5673 : {
5674 8470 : if (gimple_assign_rhs1 (def_stmt) == phi_result
5675 8470 : || gimple_assign_rhs2 (def_stmt) == phi_result)
5676 : return 2;
5677 : }
5678 : }
5679 4868 : }
5680 : }
5681 :
5682 : return width;
5683 : }
5684 :
5685 : #define SPECIAL_BIASED_END_STMT 0 /* It is the end stmt of all ops. */
5686 : #define BIASED_END_STMT 1 /* It is the end stmt of normal or biased ops. */
5687 : #define NORMAL_END_STMT 2 /* It is the end stmt of normal ops. */
5688 :
5689 : /* Rewrite statements with dependency chain with regard the chance to generate
5690 : FMA.
5691 : For the chain with FMA: Try to keep fma opportunity as much as possible.
5692 : For the chain without FMA: Putting the computation in rank order and trying
5693 : to allow operations to be executed in parallel.
5694 : E.g.
5695 : e + f + a * b + c * d;
5696 :
5697 : ssa1 = e + a * b;
5698 : ssa2 = f + c * d;
5699 : ssa3 = ssa1 + ssa2;
5700 :
5701 : This reassociation approach preserves the chance of fma generation as much
5702 : as possible.
5703 :
5704 : Another thing is to avoid adding loop-carried ops to long chains, otherwise
5705 : the whole chain will have dependencies across the loop iteration. Just keep
5706 : loop-carried ops in a separate chain.
5707 : E.g.
5708 : x_1 = phi (x_0, x_2)
5709 : y_1 = phi (y_0, y_2)
5710 :
5711 : a + b + c + d + e + x1 + y1
5712 :
5713 : SSA1 = a + b;
5714 : SSA2 = c + d;
5715 : SSA3 = SSA1 + e;
5716 : SSA4 = SSA3 + SSA2;
5717 : SSA5 = x1 + y1;
5718 : SSA6 = SSA4 + SSA5;
5719 : */
5720 : static void
5721 1900 : rewrite_expr_tree_parallel (gassign *stmt, int width, bool has_fma,
5722 : const vec<operand_entry *> &ops)
5723 : {
5724 1900 : enum tree_code opcode = gimple_assign_rhs_code (stmt);
5725 1900 : int op_num = ops.length ();
5726 1900 : int op_normal_num = op_num;
5727 1900 : gcc_assert (op_num > 0);
5728 1900 : int stmt_num = op_num - 1;
5729 1900 : gimple **stmts = XALLOCAVEC (gimple *, stmt_num);
5730 1900 : int i = 0, j = 0;
5731 1900 : tree tmp_op[2], op1;
5732 1900 : operand_entry *oe;
5733 1900 : gimple *stmt1 = NULL;
5734 1900 : tree last_rhs1 = gimple_assign_rhs1 (stmt);
5735 1900 : int last_rhs1_stmt_index = 0, last_rhs2_stmt_index = 0;
5736 1900 : int width_active = 0, width_count = 0;
5737 1900 : bool has_biased = false, ops_changed = false;
5738 1900 : auto_vec<operand_entry *> ops_normal;
5739 1900 : auto_vec<operand_entry *> ops_biased;
5740 1900 : vec<operand_entry *> *ops1;
5741 :
5742 : /* We start expression rewriting from the top statements.
5743 : So, in this loop we create a full list of statements
5744 : we will work with. */
5745 1900 : stmts[stmt_num - 1] = stmt;
5746 9250 : for (i = stmt_num - 2; i >= 0; i--)
5747 7350 : stmts[i] = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmts[i+1]));
5748 :
5749 : /* Avoid adding loop-carried ops to long chains, first filter out the
5750 : loop-carried. But we need to make sure that the length of the remainder
5751 : is not less than 4, which is the smallest ops length we can break the
5752 : dependency. */
5753 14950 : FOR_EACH_VEC_ELT (ops, i, oe)
5754 : {
5755 11150 : if (TREE_CODE (oe->op) == SSA_NAME
5756 10984 : && bitmap_bit_p (biased_names, SSA_NAME_VERSION (oe->op))
5757 11407 : && op_normal_num > 4)
5758 : {
5759 223 : ops_biased.safe_push (oe);
5760 223 : has_biased = true;
5761 223 : op_normal_num --;
5762 : }
5763 : else
5764 10927 : ops_normal.safe_push (oe);
5765 : }
5766 :
5767 : /* Width should not be larger than ops length / 2, since we can not create
5768 : more parallel dependency chains that exceeds such value. */
5769 1900 : int width_normal = op_normal_num / 2;
5770 1900 : int width_biased = (op_num - op_normal_num) / 2;
5771 1900 : width_normal = width <= width_normal ? width : width_normal;
5772 1900 : width_biased = width <= width_biased ? width : width_biased;
5773 :
5774 1900 : ops1 = &ops_normal;
5775 1900 : width_count = width_active = width_normal;
5776 :
5777 : /* Build parallel dependency chain according to width. */
5778 11150 : for (i = 0; i < stmt_num; i++)
5779 : {
5780 9250 : if (dump_file && (dump_flags & TDF_DETAILS))
5781 : {
5782 6 : fprintf (dump_file, "Transforming ");
5783 6 : print_gimple_stmt (dump_file, stmts[i], 0);
5784 : }
5785 :
5786 : /* When the work of normal ops is over, but the loop is not over,
5787 : continue to do biased ops. */
5788 9250 : if (width_count == 0 && ops1 == &ops_normal)
5789 : {
5790 219 : ops1 = &ops_biased;
5791 219 : width_count = width_active = width_biased;
5792 219 : ops_changed = true;
5793 : }
5794 :
5795 : /* Swap the operands if no FMA in the chain. */
5796 9250 : if (ops1->length () > 2 && !has_fma)
5797 4513 : swap_ops_for_binary_stmt (*ops1, ops1->length () - 3);
5798 :
5799 9250 : if (i < width_active
5800 5254 : || (ops_changed && i <= (last_rhs1_stmt_index + width_active)))
5801 : {
5802 11997 : for (j = 0; j < 2; j++)
5803 : {
5804 7998 : oe = ops1->pop ();
5805 7998 : tmp_op[j] = oe->op;
5806 : /* If the stmt that defines operand has to be inserted, insert it
5807 : before the use. */
5808 7998 : stmt1 = oe->stmt_to_insert;
5809 7998 : if (stmt1)
5810 0 : insert_stmt_before_use (stmts[i], stmt1);
5811 7998 : stmt1 = NULL;
5812 : }
5813 3999 : stmts[i] = build_and_add_sum (TREE_TYPE (last_rhs1),
5814 : tmp_op[1],
5815 : tmp_op[0],
5816 : opcode);
5817 3999 : gimple_set_visited (stmts[i], true);
5818 :
5819 : }
5820 : else
5821 : {
5822 : /* We keep original statement only for the last one. All others are
5823 : recreated. */
5824 5251 : if (!ops1->length ())
5825 : {
5826 : /* For biased length equal to 2. */
5827 2099 : if (width_count == BIASED_END_STMT && !last_rhs2_stmt_index)
5828 1 : last_rhs2_stmt_index = i - 1;
5829 :
5830 : /* When width_count == 2 and there is no biased, just finish. */
5831 2099 : if (width_count == NORMAL_END_STMT && !has_biased)
5832 : {
5833 1681 : last_rhs1_stmt_index = i - 1;
5834 1681 : last_rhs2_stmt_index = i - 2;
5835 : }
5836 2099 : if (last_rhs1_stmt_index && (last_rhs2_stmt_index || !has_biased))
5837 : {
5838 : /* We keep original statement only for the last one. All
5839 : others are recreated. */
5840 1683 : gimple_assign_set_rhs1 (stmts[i], gimple_assign_lhs
5841 1683 : (stmts[last_rhs1_stmt_index]));
5842 1683 : gimple_assign_set_rhs2 (stmts[i], gimple_assign_lhs
5843 1683 : (stmts[last_rhs2_stmt_index]));
5844 1683 : update_stmt (stmts[i]);
5845 : }
5846 : else
5847 : {
5848 1248 : stmts[i] =
5849 416 : build_and_add_sum (TREE_TYPE (last_rhs1),
5850 416 : gimple_assign_lhs (stmts[i-width_count]),
5851 : gimple_assign_lhs
5852 416 : (stmts[i-width_count+1]),
5853 : opcode);
5854 416 : gimple_set_visited (stmts[i], true);
5855 416 : width_count--;
5856 :
5857 : /* It is the end of normal or biased ops.
5858 : last_rhs1_stmt_index used to record the last stmt index
5859 : for normal ops. last_rhs2_stmt_index used to record the
5860 : last stmt index for biased ops. */
5861 416 : if (width_count == BIASED_END_STMT)
5862 : {
5863 220 : gcc_assert (has_biased);
5864 220 : if (ops_biased.length ())
5865 : last_rhs1_stmt_index = i;
5866 : else
5867 1 : last_rhs2_stmt_index = i;
5868 : width_count--;
5869 : }
5870 : }
5871 : }
5872 : else
5873 : {
5874 : /* Attach the rest ops to the parallel dependency chain. */
5875 3152 : oe = ops1->pop ();
5876 3152 : op1 = oe->op;
5877 3152 : stmt1 = oe->stmt_to_insert;
5878 3152 : if (stmt1)
5879 0 : insert_stmt_before_use (stmts[i], stmt1);
5880 3152 : stmt1 = NULL;
5881 :
5882 : /* For only one biased ops. */
5883 3152 : if (width_count == SPECIAL_BIASED_END_STMT)
5884 : {
5885 : /* We keep original statement only for the last one. All
5886 : others are recreated. */
5887 217 : gcc_assert (has_biased);
5888 217 : gimple_assign_set_rhs1 (stmts[i], gimple_assign_lhs
5889 217 : (stmts[last_rhs1_stmt_index]));
5890 217 : gimple_assign_set_rhs2 (stmts[i], op1);
5891 217 : update_stmt (stmts[i]);
5892 : }
5893 : else
5894 : {
5895 2935 : stmts[i] = build_and_add_sum (TREE_TYPE (last_rhs1),
5896 : gimple_assign_lhs
5897 2935 : (stmts[i-width_active]),
5898 : op1,
5899 : opcode);
5900 2935 : gimple_set_visited (stmts[i], true);
5901 : }
5902 : }
5903 : }
5904 :
5905 9250 : if (dump_file && (dump_flags & TDF_DETAILS))
5906 : {
5907 6 : fprintf (dump_file, " into ");
5908 6 : print_gimple_stmt (dump_file, stmts[i], 0);
5909 : }
5910 : }
5911 :
5912 1900 : remove_visited_stmt_chain (last_rhs1);
5913 1900 : }
5914 :
5915 : /* Transform STMT, which is really (A +B) + (C + D) into the left
5916 : linear form, ((A+B)+C)+D.
5917 : Recurse on D if necessary. */
5918 :
5919 : static void
5920 2630 : linearize_expr (gimple *stmt)
5921 : {
5922 2630 : gimple_stmt_iterator gsi;
5923 2630 : gimple *binlhs = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
5924 2630 : gimple *binrhs = SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt));
5925 2630 : gimple *oldbinrhs = binrhs;
5926 2630 : enum tree_code rhscode = gimple_assign_rhs_code (stmt);
5927 2630 : gimple *newbinrhs = NULL;
5928 2630 : class loop *loop = loop_containing_stmt (stmt);
5929 2630 : tree lhs = gimple_assign_lhs (stmt);
5930 :
5931 2630 : gcc_assert (is_reassociable_op (binlhs, rhscode, loop)
5932 : && is_reassociable_op (binrhs, rhscode, loop));
5933 :
5934 2630 : gsi = gsi_for_stmt (stmt);
5935 :
5936 2630 : gimple_assign_set_rhs2 (stmt, gimple_assign_rhs1 (binrhs));
5937 2630 : binrhs = gimple_build_assign (make_ssa_name (TREE_TYPE (lhs)),
5938 : gimple_assign_rhs_code (binrhs),
5939 : gimple_assign_lhs (binlhs),
5940 : gimple_assign_rhs2 (binrhs));
5941 2630 : gimple_assign_set_rhs1 (stmt, gimple_assign_lhs (binrhs));
5942 2630 : gsi_insert_before (&gsi, binrhs, GSI_SAME_STMT);
5943 2630 : gimple_set_uid (binrhs, gimple_uid (stmt));
5944 :
5945 2630 : if (TREE_CODE (gimple_assign_rhs2 (stmt)) == SSA_NAME)
5946 2627 : newbinrhs = SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt));
5947 :
5948 2630 : if (dump_file && (dump_flags & TDF_DETAILS))
5949 : {
5950 0 : fprintf (dump_file, "Linearized: ");
5951 0 : print_gimple_stmt (dump_file, stmt, 0);
5952 : }
5953 :
5954 2630 : reassociate_stats.linearized++;
5955 2630 : update_stmt (stmt);
5956 :
5957 2630 : gsi = gsi_for_stmt (oldbinrhs);
5958 2630 : reassoc_remove_stmt (&gsi);
5959 2630 : release_defs (oldbinrhs);
5960 :
5961 2630 : gimple_set_visited (stmt, true);
5962 2630 : gimple_set_visited (binlhs, true);
5963 2630 : gimple_set_visited (binrhs, true);
5964 :
5965 : /* Tail recurse on the new rhs if it still needs reassociation. */
5966 2630 : if (newbinrhs && is_reassociable_op (newbinrhs, rhscode, loop))
5967 : /* ??? This should probably be linearize_expr (newbinrhs) but I don't
5968 : want to change the algorithm while converting to tuples. */
5969 622 : linearize_expr (stmt);
5970 2630 : }
5971 :
5972 : /* If LHS has a single immediate use that is a GIMPLE_ASSIGN statement, return
5973 : it. Otherwise, return NULL. */
5974 :
5975 : static gimple *
5976 413066 : get_single_immediate_use (tree lhs)
5977 : {
5978 413066 : use_operand_p immuse;
5979 413066 : gimple *immusestmt;
5980 :
5981 413066 : if (TREE_CODE (lhs) == SSA_NAME
5982 413066 : && single_imm_use (lhs, &immuse, &immusestmt)
5983 732347 : && is_gimple_assign (immusestmt))
5984 243057 : return immusestmt;
5985 :
5986 : return NULL;
5987 : }
5988 :
5989 : /* Recursively negate the value of TONEGATE, and return the SSA_NAME
5990 : representing the negated value. Insertions of any necessary
5991 : instructions go before GSI.
5992 : This function is recursive in that, if you hand it "a_5" as the
5993 : value to negate, and a_5 is defined by "a_5 = b_3 + b_4", it will
5994 : transform b_3 + b_4 into a_5 = -b_3 + -b_4. */
5995 :
5996 : static tree
5997 78114 : negate_value (tree tonegate, gimple_stmt_iterator *gsip)
5998 : {
5999 78114 : gimple *negatedefstmt = NULL;
6000 78114 : tree resultofnegate;
6001 78114 : gimple_stmt_iterator gsi;
6002 78114 : unsigned int uid;
6003 :
6004 : /* If we are trying to negate a name, defined by an add, negate the
6005 : add operands instead. */
6006 78114 : if (TREE_CODE (tonegate) == SSA_NAME)
6007 76254 : negatedefstmt = SSA_NAME_DEF_STMT (tonegate);
6008 78114 : if (TREE_CODE (tonegate) == SSA_NAME
6009 76254 : && is_gimple_assign (negatedefstmt)
6010 66523 : && TREE_CODE (gimple_assign_lhs (negatedefstmt)) == SSA_NAME
6011 66523 : && has_single_use (gimple_assign_lhs (negatedefstmt))
6012 127534 : && gimple_assign_rhs_code (negatedefstmt) == PLUS_EXPR)
6013 : {
6014 902 : tree rhs1 = gimple_assign_rhs1 (negatedefstmt);
6015 902 : tree rhs2 = gimple_assign_rhs2 (negatedefstmt);
6016 902 : tree lhs = gimple_assign_lhs (negatedefstmt);
6017 902 : gimple *g;
6018 :
6019 902 : gsi = gsi_for_stmt (negatedefstmt);
6020 902 : rhs1 = negate_value (rhs1, &gsi);
6021 :
6022 902 : gsi = gsi_for_stmt (negatedefstmt);
6023 902 : rhs2 = negate_value (rhs2, &gsi);
6024 :
6025 902 : gsi = gsi_for_stmt (negatedefstmt);
6026 902 : lhs = make_ssa_name (TREE_TYPE (lhs));
6027 902 : gimple_set_visited (negatedefstmt, true);
6028 902 : g = gimple_build_assign (lhs, PLUS_EXPR, rhs1, rhs2);
6029 902 : gimple_set_uid (g, gimple_uid (negatedefstmt));
6030 902 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6031 902 : return lhs;
6032 : }
6033 :
6034 77212 : tonegate = fold_build1 (NEGATE_EXPR, TREE_TYPE (tonegate), tonegate);
6035 77212 : resultofnegate = force_gimple_operand_gsi (gsip, tonegate, true,
6036 : NULL_TREE, true, GSI_SAME_STMT);
6037 77212 : gsi = *gsip;
6038 77212 : uid = gimple_uid (gsi_stmt (gsi));
6039 157135 : for (gsi_prev (&gsi); !gsi_end_p (gsi); gsi_prev (&gsi))
6040 : {
6041 147993 : gimple *stmt = gsi_stmt (gsi);
6042 147993 : if (gimple_uid (stmt) != 0)
6043 : break;
6044 75352 : gimple_set_uid (stmt, uid);
6045 : }
6046 : return resultofnegate;
6047 : }
6048 :
6049 : /* Return true if we should break up the subtract in STMT into an add
6050 : with negate. This is true when we the subtract operands are really
6051 : adds, or the subtract itself is used in an add expression. In
6052 : either case, breaking up the subtract into an add with negate
6053 : exposes the adds to reassociation. */
6054 :
6055 : static bool
6056 298387 : should_break_up_subtract (gimple *stmt)
6057 : {
6058 298387 : tree lhs = gimple_assign_lhs (stmt);
6059 298387 : tree binlhs = gimple_assign_rhs1 (stmt);
6060 298387 : tree binrhs = gimple_assign_rhs2 (stmt);
6061 298387 : gimple *immusestmt;
6062 298387 : class loop *loop = loop_containing_stmt (stmt);
6063 :
6064 298387 : if (TREE_CODE (binlhs) == SSA_NAME
6065 298387 : && is_reassociable_op (SSA_NAME_DEF_STMT (binlhs), PLUS_EXPR, loop))
6066 : return true;
6067 :
6068 280585 : if (TREE_CODE (binrhs) == SSA_NAME
6069 280585 : && is_reassociable_op (SSA_NAME_DEF_STMT (binrhs), PLUS_EXPR, loop))
6070 : return true;
6071 :
6072 279773 : if (TREE_CODE (lhs) == SSA_NAME
6073 279773 : && (immusestmt = get_single_immediate_use (lhs))
6074 131170 : && is_gimple_assign (immusestmt)
6075 410943 : && (gimple_assign_rhs_code (immusestmt) == PLUS_EXPR
6076 86791 : || (gimple_assign_rhs_code (immusestmt) == MINUS_EXPR
6077 2244 : && gimple_assign_rhs1 (immusestmt) == lhs)
6078 84557 : || gimple_assign_rhs_code (immusestmt) == MULT_EXPR))
6079 57696 : return true;
6080 : return false;
6081 : }
6082 :
6083 : /* Transform STMT from A - B into A + -B. */
6084 :
6085 : static void
6086 76310 : break_up_subtract (gimple *stmt, gimple_stmt_iterator *gsip)
6087 : {
6088 76310 : tree rhs1 = gimple_assign_rhs1 (stmt);
6089 76310 : tree rhs2 = gimple_assign_rhs2 (stmt);
6090 :
6091 76310 : if (dump_file && (dump_flags & TDF_DETAILS))
6092 : {
6093 0 : fprintf (dump_file, "Breaking up subtract ");
6094 0 : print_gimple_stmt (dump_file, stmt, 0);
6095 : }
6096 :
6097 76310 : rhs2 = negate_value (rhs2, gsip);
6098 76310 : gimple_assign_set_rhs_with_ops (gsip, PLUS_EXPR, rhs1, rhs2);
6099 76310 : update_stmt (stmt);
6100 76310 : }
6101 :
6102 : /* Determine whether STMT is a builtin call that raises an SSA name
6103 : to an integer power and has only one use. If so, and this is early
6104 : reassociation and unsafe math optimizations are permitted, place
6105 : the SSA name in *BASE and the exponent in *EXPONENT, and return TRUE.
6106 : If any of these conditions does not hold, return FALSE. */
6107 :
6108 : static bool
6109 131 : acceptable_pow_call (gcall *stmt, tree *base, HOST_WIDE_INT *exponent)
6110 : {
6111 131 : tree arg1;
6112 131 : REAL_VALUE_TYPE c, cint;
6113 :
6114 131 : switch (gimple_call_combined_fn (stmt))
6115 : {
6116 24 : CASE_CFN_POW:
6117 24 : if (flag_errno_math)
6118 : return false;
6119 :
6120 24 : *base = gimple_call_arg (stmt, 0);
6121 24 : arg1 = gimple_call_arg (stmt, 1);
6122 :
6123 24 : if (TREE_CODE (arg1) != REAL_CST)
6124 : return false;
6125 :
6126 19 : c = TREE_REAL_CST (arg1);
6127 :
6128 19 : if (REAL_EXP (&c) > HOST_BITS_PER_WIDE_INT)
6129 : return false;
6130 :
6131 19 : *exponent = real_to_integer (&c);
6132 19 : real_from_integer (&cint, VOIDmode, *exponent, SIGNED);
6133 19 : if (!real_identical (&c, &cint))
6134 : return false;
6135 :
6136 : break;
6137 :
6138 9 : CASE_CFN_POWI:
6139 9 : *base = gimple_call_arg (stmt, 0);
6140 9 : arg1 = gimple_call_arg (stmt, 1);
6141 :
6142 9 : if (!tree_fits_shwi_p (arg1))
6143 : return false;
6144 :
6145 7 : *exponent = tree_to_shwi (arg1);
6146 7 : break;
6147 :
6148 : default:
6149 : return false;
6150 : }
6151 :
6152 : /* Expanding negative exponents is generally unproductive, so we don't
6153 : complicate matters with those. Exponents of zero and one should
6154 : have been handled by expression folding. */
6155 16 : if (*exponent < 2
6156 16 : || TREE_CODE (*base) != SSA_NAME
6157 32 : || !can_reassociate_op_p (*base))
6158 : return false;
6159 :
6160 : return true;
6161 : }
6162 :
6163 : /* Try to derive and add operand entry for OP to *OPS. Return false if
6164 : unsuccessful. */
6165 :
6166 : static bool
6167 9859616 : try_special_add_to_ops (vec<operand_entry *> *ops,
6168 : enum tree_code code,
6169 : tree op, gimple* def_stmt)
6170 : {
6171 9859616 : tree base = NULL_TREE;
6172 9859616 : HOST_WIDE_INT exponent = 0;
6173 :
6174 9859616 : if (TREE_CODE (op) != SSA_NAME
6175 9859616 : || ! has_single_use (op))
6176 : return false;
6177 :
6178 3698174 : if (code == MULT_EXPR
6179 800726 : && reassoc_insert_powi_p
6180 376912 : && flag_unsafe_math_optimizations
6181 30352 : && is_gimple_call (def_stmt)
6182 3698305 : && acceptable_pow_call (as_a <gcall *> (def_stmt), &base, &exponent))
6183 : {
6184 12 : add_repeat_to_ops_vec (ops, base, exponent);
6185 12 : gimple_set_visited (def_stmt, true);
6186 12 : return true;
6187 : }
6188 3698162 : else if (code == MULT_EXPR
6189 800714 : && is_gimple_assign (def_stmt)
6190 759100 : && gimple_assign_rhs_code (def_stmt) == NEGATE_EXPR
6191 585 : && !HONOR_SNANS (TREE_TYPE (op))
6192 585 : && (!HONOR_SIGNED_ZEROS (TREE_TYPE (op))
6193 0 : || !COMPLEX_FLOAT_TYPE_P (TREE_TYPE (op)))
6194 3698747 : && (!FLOAT_TYPE_P (TREE_TYPE (op))
6195 50 : || !DECIMAL_FLOAT_MODE_P (element_mode (op))))
6196 : {
6197 578 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
6198 578 : if (!can_reassociate_op_p (rhs1))
6199 : return false;
6200 568 : tree cst = build_minus_one_cst (TREE_TYPE (op));
6201 568 : add_to_ops_vec (ops, rhs1);
6202 568 : add_to_ops_vec (ops, cst);
6203 568 : gimple_set_visited (def_stmt, true);
6204 568 : return true;
6205 : }
6206 :
6207 : return false;
6208 : }
6209 :
6210 : /* Recursively linearize a binary expression that is the RHS of STMT.
6211 : Place the operands of the expression tree in the vector named OPS. */
6212 :
6213 : static void
6214 5059567 : linearize_expr_tree (vec<operand_entry *> *ops, gimple *stmt,
6215 : bool is_associative, bool set_visited)
6216 : {
6217 5059567 : tree binlhs = gimple_assign_rhs1 (stmt);
6218 5059567 : tree binrhs = gimple_assign_rhs2 (stmt);
6219 5059567 : gimple *binlhsdef = NULL, *binrhsdef = NULL;
6220 5059567 : bool binlhsisreassoc = false;
6221 5059567 : bool binrhsisreassoc = false;
6222 5059567 : enum tree_code rhscode = gimple_assign_rhs_code (stmt);
6223 5059567 : class loop *loop = loop_containing_stmt (stmt);
6224 :
6225 5059567 : if (set_visited)
6226 5018844 : gimple_set_visited (stmt, true);
6227 :
6228 5059567 : if (TREE_CODE (binlhs) == SSA_NAME)
6229 : {
6230 5056941 : binlhsdef = SSA_NAME_DEF_STMT (binlhs);
6231 5056941 : binlhsisreassoc = (is_reassociable_op (binlhsdef, rhscode, loop)
6232 5056941 : && !stmt_could_throw_p (cfun, binlhsdef));
6233 : }
6234 :
6235 5059567 : if (TREE_CODE (binrhs) == SSA_NAME)
6236 : {
6237 1598130 : binrhsdef = SSA_NAME_DEF_STMT (binrhs);
6238 1598130 : binrhsisreassoc = (is_associative
6239 1598113 : && is_reassociable_op (binrhsdef, rhscode, loop)
6240 1664711 : && !stmt_could_throw_p (cfun, binrhsdef));
6241 : }
6242 :
6243 : /* If the LHS is not reassociable, but the RHS is, we need to swap
6244 : them. If neither is reassociable, there is nothing we can do, so
6245 : just put them in the ops vector. If the LHS is reassociable,
6246 : linearize it. If both are reassociable, then linearize the RHS
6247 : and the LHS. */
6248 :
6249 5059567 : if (!binlhsisreassoc)
6250 : {
6251 : /* If this is not a associative operation like division, give up. */
6252 4864656 : if (!is_associative)
6253 : {
6254 17 : add_to_ops_vec (ops, binrhs);
6255 17 : return;
6256 : }
6257 :
6258 4864639 : if (!binrhsisreassoc)
6259 : {
6260 4800066 : bool swap = false;
6261 4800066 : if (try_special_add_to_ops (ops, rhscode, binrhs, binrhsdef))
6262 : /* If we add ops for the rhs we expect to be able to recurse
6263 : to it via the lhs during expression rewrite so swap
6264 : operands. */
6265 : swap = true;
6266 : else
6267 4799686 : add_to_ops_vec (ops, binrhs);
6268 :
6269 4800066 : if (!try_special_add_to_ops (ops, rhscode, binlhs, binlhsdef))
6270 4799871 : add_to_ops_vec (ops, binlhs);
6271 :
6272 4800066 : if (!swap)
6273 : return;
6274 : }
6275 :
6276 64953 : if (dump_file && (dump_flags & TDF_DETAILS))
6277 : {
6278 9 : fprintf (dump_file, "swapping operands of ");
6279 9 : print_gimple_stmt (dump_file, stmt, 0);
6280 : }
6281 :
6282 64953 : swap_ssa_operands (stmt,
6283 : gimple_assign_rhs1_ptr (stmt),
6284 : gimple_assign_rhs2_ptr (stmt));
6285 64953 : update_stmt (stmt);
6286 :
6287 64953 : if (dump_file && (dump_flags & TDF_DETAILS))
6288 : {
6289 9 : fprintf (dump_file, " is now ");
6290 9 : print_gimple_stmt (dump_file, stmt, 0);
6291 : }
6292 64953 : if (!binrhsisreassoc)
6293 : return;
6294 :
6295 : /* We want to make it so the lhs is always the reassociative op,
6296 : so swap. */
6297 : std::swap (binlhs, binrhs);
6298 : }
6299 194911 : else if (binrhsisreassoc)
6300 : {
6301 2008 : linearize_expr (stmt);
6302 2008 : binlhs = gimple_assign_rhs1 (stmt);
6303 2008 : binrhs = gimple_assign_rhs2 (stmt);
6304 : }
6305 :
6306 259484 : gcc_checking_assert (TREE_CODE (binrhs) != SSA_NAME
6307 : || !is_associative
6308 : || !is_reassociable_op (SSA_NAME_DEF_STMT (binrhs),
6309 : rhscode, loop));
6310 259484 : linearize_expr_tree (ops, SSA_NAME_DEF_STMT (binlhs),
6311 : is_associative, set_visited);
6312 :
6313 259484 : if (!try_special_add_to_ops (ops, rhscode, binrhs, binrhsdef))
6314 259479 : add_to_ops_vec (ops, binrhs);
6315 : }
6316 :
6317 : /* Repropagate the negates back into subtracts, since no other pass
6318 : currently does it. */
6319 :
6320 : static void
6321 2125670 : repropagate_negates (void)
6322 : {
6323 2125670 : unsigned int i = 0;
6324 2125670 : tree negate;
6325 :
6326 2258963 : FOR_EACH_VEC_ELT (plus_negates, i, negate)
6327 : {
6328 133293 : gimple *user = get_single_immediate_use (negate);
6329 133293 : if (!user || !is_gimple_assign (user))
6330 21406 : continue;
6331 :
6332 111887 : tree negateop = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (negate));
6333 111897 : if (TREE_CODE (negateop) == SSA_NAME
6334 111887 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (negateop))
6335 10 : continue;
6336 :
6337 : /* The negate operand can be either operand of a PLUS_EXPR
6338 : (it can be the LHS if the RHS is a constant for example).
6339 :
6340 : Force the negate operand to the RHS of the PLUS_EXPR, then
6341 : transform the PLUS_EXPR into a MINUS_EXPR. */
6342 111877 : if (gimple_assign_rhs_code (user) == PLUS_EXPR)
6343 : {
6344 : /* If the negated operand appears on the LHS of the
6345 : PLUS_EXPR, exchange the operands of the PLUS_EXPR
6346 : to force the negated operand to the RHS of the PLUS_EXPR. */
6347 84462 : if (gimple_assign_rhs1 (user) == negate)
6348 : {
6349 25751 : swap_ssa_operands (user,
6350 : gimple_assign_rhs1_ptr (user),
6351 : gimple_assign_rhs2_ptr (user));
6352 : }
6353 :
6354 : /* Now transform the PLUS_EXPR into a MINUS_EXPR and replace
6355 : the RHS of the PLUS_EXPR with the operand of the NEGATE_EXPR. */
6356 84462 : if (gimple_assign_rhs2 (user) == negate)
6357 : {
6358 84462 : tree rhs1 = gimple_assign_rhs1 (user);
6359 84462 : gimple_stmt_iterator gsi = gsi_for_stmt (user);
6360 84462 : gimple_assign_set_rhs_with_ops (&gsi, MINUS_EXPR, rhs1,
6361 : negateop);
6362 84462 : update_stmt (user);
6363 : }
6364 : }
6365 27415 : else if (gimple_assign_rhs_code (user) == MINUS_EXPR)
6366 : {
6367 2052 : if (gimple_assign_rhs1 (user) == negate)
6368 : {
6369 : /* We have
6370 : x = -negateop
6371 : y = x - b
6372 : which we transform into
6373 : x = negateop + b
6374 : y = -x .
6375 : This pushes down the negate which we possibly can merge
6376 : into some other operation, hence insert it into the
6377 : plus_negates vector. */
6378 2052 : gimple *feed = SSA_NAME_DEF_STMT (negate);
6379 2052 : tree b = gimple_assign_rhs2 (user);
6380 2052 : gimple_stmt_iterator gsi = gsi_for_stmt (feed);
6381 2052 : gimple_stmt_iterator gsi2 = gsi_for_stmt (user);
6382 2052 : tree x = make_ssa_name (TREE_TYPE (gimple_assign_lhs (feed)));
6383 2052 : gimple *g = gimple_build_assign (x, PLUS_EXPR, negateop, b);
6384 2052 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
6385 2052 : gimple_assign_set_rhs_with_ops (&gsi2, NEGATE_EXPR, x);
6386 2052 : user = gsi_stmt (gsi2);
6387 2052 : update_stmt (user);
6388 2052 : reassoc_remove_stmt (&gsi);
6389 2052 : release_defs (feed);
6390 2052 : plus_negates.safe_push (gimple_assign_lhs (user));
6391 : }
6392 : else
6393 : {
6394 : /* Transform "x = -negateop; y = b - x" into "y = b + negateop",
6395 : getting rid of one operation. */
6396 0 : tree rhs1 = gimple_assign_rhs1 (user);
6397 0 : gimple_stmt_iterator gsi = gsi_for_stmt (user);
6398 0 : gimple_assign_set_rhs_with_ops (&gsi, PLUS_EXPR, rhs1, negateop);
6399 0 : update_stmt (gsi_stmt (gsi));
6400 : }
6401 : }
6402 : }
6403 2125670 : }
6404 :
6405 : /* Break up subtract operations in block BB.
6406 :
6407 : We do this top down because we don't know whether the subtract is
6408 : part of a possible chain of reassociation except at the top.
6409 :
6410 : IE given
6411 : d = f + g
6412 : c = a + e
6413 : b = c - d
6414 : q = b - r
6415 : k = t - q
6416 :
6417 : we want to break up k = t - q, but we won't until we've transformed q
6418 : = b - r, which won't be broken up until we transform b = c - d.
6419 :
6420 : En passant, clear the GIMPLE visited flag on every statement
6421 : and set UIDs within each basic block. */
6422 :
6423 : static void
6424 19961050 : break_up_subtract_bb (basic_block bb)
6425 : {
6426 19961050 : gimple_stmt_iterator gsi;
6427 19961050 : unsigned int uid = 1;
6428 :
6429 220419082 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
6430 : {
6431 180496982 : gimple *stmt = gsi_stmt (gsi);
6432 180496982 : gimple_set_visited (stmt, false);
6433 180496982 : gimple_set_uid (stmt, uid++);
6434 :
6435 180496982 : if (!is_gimple_assign (stmt)
6436 48578130 : || !can_reassociate_type_p (TREE_TYPE (gimple_assign_lhs (stmt)))
6437 198311818 : || !can_reassociate_op_p (gimple_assign_lhs (stmt)))
6438 162682515 : continue;
6439 :
6440 : /* Look for simple gimple subtract operations. */
6441 17814467 : if (gimple_assign_rhs_code (stmt) == MINUS_EXPR)
6442 : {
6443 298652 : if (!can_reassociate_op_p (gimple_assign_rhs1 (stmt))
6444 298652 : || !can_reassociate_op_p (gimple_assign_rhs2 (stmt)))
6445 265 : continue;
6446 :
6447 : /* Check for a subtract used only in an addition. If this
6448 : is the case, transform it into add of a negate for better
6449 : reassociation. IE transform C = A-B into C = A + -B if C
6450 : is only used in an addition. */
6451 298387 : if (should_break_up_subtract (stmt))
6452 76310 : break_up_subtract (stmt, &gsi);
6453 : }
6454 17515815 : else if (gimple_assign_rhs_code (stmt) == NEGATE_EXPR
6455 17515815 : && can_reassociate_op_p (gimple_assign_rhs1 (stmt)))
6456 44889 : plus_negates.safe_push (gimple_assign_lhs (stmt));
6457 : }
6458 19961050 : }
6459 :
6460 : /* Used for repeated factor analysis. */
6461 : struct repeat_factor
6462 : {
6463 : /* An SSA name that occurs in a multiply chain. */
6464 : tree factor;
6465 :
6466 : /* Cached rank of the factor. */
6467 : unsigned rank;
6468 :
6469 : /* Number of occurrences of the factor in the chain. */
6470 : HOST_WIDE_INT count;
6471 :
6472 : /* An SSA name representing the product of this factor and
6473 : all factors appearing later in the repeated factor vector. */
6474 : tree repr;
6475 : };
6476 :
6477 :
6478 : static vec<repeat_factor> repeat_factor_vec;
6479 :
6480 : /* Used for sorting the repeat factor vector. Sort primarily by
6481 : ascending occurrence count, secondarily by descending rank. */
6482 :
6483 : static int
6484 262338 : compare_repeat_factors (const void *x1, const void *x2)
6485 : {
6486 262338 : const repeat_factor *rf1 = (const repeat_factor *) x1;
6487 262338 : const repeat_factor *rf2 = (const repeat_factor *) x2;
6488 :
6489 262338 : if (rf1->count < rf2->count)
6490 : return -1;
6491 261707 : else if (rf1->count > rf2->count)
6492 : return 1;
6493 :
6494 261215 : if (rf1->rank < rf2->rank)
6495 : return 1;
6496 141948 : else if (rf1->rank > rf2->rank)
6497 118118 : return -1;
6498 :
6499 : return 0;
6500 : }
6501 :
6502 : /* Look for repeated operands in OPS in the multiply tree rooted at
6503 : STMT. Replace them with an optimal sequence of multiplies and powi
6504 : builtin calls, and remove the used operands from OPS. Return an
6505 : SSA name representing the value of the replacement sequence. */
6506 :
6507 : static tree
6508 504200 : attempt_builtin_powi (gimple *stmt, vec<operand_entry *> *ops)
6509 : {
6510 504200 : unsigned i, j, vec_len;
6511 504200 : int ii;
6512 504200 : operand_entry *oe;
6513 504200 : repeat_factor *rf1, *rf2;
6514 504200 : repeat_factor rfnew;
6515 504200 : tree result = NULL_TREE;
6516 504200 : tree target_ssa, iter_result;
6517 504200 : tree type = TREE_TYPE (gimple_get_lhs (stmt));
6518 504200 : tree powi_fndecl = mathfn_built_in (type, BUILT_IN_POWI);
6519 504200 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6520 504200 : gimple *mul_stmt, *pow_stmt;
6521 :
6522 : /* Expand powers as a series of multiplications if type is integral
6523 : or BUILT_IN_POWI doesn't exist for this type and target. We cannot
6524 : simply give up in the latter case as OPS can contain entries with
6525 : an occurrence count larger than one for __builtin_pow calls. */
6526 504200 : bool use_mults = INTEGRAL_TYPE_P (type) || !powi_fndecl;
6527 :
6528 : /* Allocate the repeated factor vector. */
6529 504200 : repeat_factor_vec.create (10);
6530 :
6531 : /* Scan the OPS vector for all SSA names in the product and build
6532 : up a vector of occurrence counts for each factor. */
6533 2021153 : FOR_EACH_VEC_ELT (*ops, i, oe)
6534 : {
6535 1012753 : if (TREE_CODE (oe->op) == SSA_NAME)
6536 : {
6537 641325 : FOR_EACH_VEC_ELT (repeat_factor_vec, j, rf1)
6538 : {
6539 71033 : if (rf1->factor == oe->op)
6540 : {
6541 2827 : rf1->count += oe->count;
6542 2827 : break;
6543 : }
6544 : }
6545 :
6546 573119 : if (j >= repeat_factor_vec.length ())
6547 : {
6548 570292 : rfnew.factor = oe->op;
6549 570292 : rfnew.rank = oe->rank;
6550 570292 : rfnew.count = oe->count;
6551 570292 : rfnew.repr = NULL_TREE;
6552 570292 : repeat_factor_vec.safe_push (rfnew);
6553 : }
6554 : }
6555 : }
6556 :
6557 : /* Sort the repeated factor vector by (a) increasing occurrence count,
6558 : and (b) decreasing rank. */
6559 504200 : repeat_factor_vec.qsort (compare_repeat_factors);
6560 :
6561 : /* It is generally best to combine as many base factors as possible
6562 : into a product before applying __builtin_powi to the result.
6563 : However, the sort order chosen for the repeated factor vector
6564 : allows us to cache partial results for the product of the base
6565 : factors for subsequent use. When we already have a cached partial
6566 : result from a previous iteration, it is best to make use of it
6567 : before looking for another __builtin_pow opportunity.
6568 :
6569 : As an example, consider x * x * y * y * y * z * z * z * z.
6570 : We want to first compose the product x * y * z, raise it to the
6571 : second power, then multiply this by y * z, and finally multiply
6572 : by z. This can be done in 5 multiplies provided we cache y * z
6573 : for use in both expressions:
6574 :
6575 : t1 = y * z
6576 : t2 = t1 * x
6577 : t3 = t2 * t2
6578 : t4 = t1 * t3
6579 : result = t4 * z
6580 :
6581 : If we instead ignored the cached y * z and first multiplied by
6582 : the __builtin_pow opportunity z * z, we would get the inferior:
6583 :
6584 : t1 = y * z
6585 : t2 = t1 * x
6586 : t3 = t2 * t2
6587 : t4 = z * z
6588 : t5 = t3 * t4
6589 : result = t5 * y */
6590 :
6591 1008400 : vec_len = repeat_factor_vec.length ();
6592 :
6593 : /* Repeatedly look for opportunities to create a builtin_powi call. */
6594 505997 : while (true)
6595 : {
6596 505997 : HOST_WIDE_INT power;
6597 :
6598 : /* First look for the largest cached product of factors from
6599 : preceding iterations. If found, create a builtin_powi for
6600 : it if the minimum occurrence count for its factors is at
6601 : least 2, or just use this cached product as our next
6602 : multiplicand if the minimum occurrence count is 1. */
6603 1078386 : FOR_EACH_VEC_ELT (repeat_factor_vec, j, rf1)
6604 : {
6605 572398 : if (rf1->repr && rf1->count > 0)
6606 : break;
6607 : }
6608 :
6609 505997 : if (j < vec_len)
6610 : {
6611 9 : power = rf1->count;
6612 :
6613 9 : if (power == 1)
6614 : {
6615 7 : iter_result = rf1->repr;
6616 :
6617 7 : if (dump_file && (dump_flags & TDF_DETAILS))
6618 : {
6619 0 : unsigned elt;
6620 0 : repeat_factor *rf;
6621 0 : fputs ("Multiplying by cached product ", dump_file);
6622 0 : for (elt = j; elt < vec_len; elt++)
6623 : {
6624 0 : rf = &repeat_factor_vec[elt];
6625 0 : print_generic_expr (dump_file, rf->factor);
6626 0 : if (elt < vec_len - 1)
6627 0 : fputs (" * ", dump_file);
6628 : }
6629 0 : fputs ("\n", dump_file);
6630 : }
6631 : }
6632 : else
6633 : {
6634 2 : if (use_mults)
6635 : {
6636 0 : gcc_assert (power > 1);
6637 0 : gimple_stmt_iterator gsip = gsi;
6638 0 : gsi_prev (&gsip);
6639 0 : iter_result = powi_as_mults (&gsi, gimple_location (stmt),
6640 : rf1->repr, power);
6641 0 : gimple_stmt_iterator gsic = gsi;
6642 0 : while (gsi_stmt (gsic) != gsi_stmt (gsip))
6643 : {
6644 0 : gimple_set_uid (gsi_stmt (gsic), gimple_uid (stmt));
6645 0 : gimple_set_visited (gsi_stmt (gsic), true);
6646 0 : gsi_prev (&gsic);
6647 : }
6648 : }
6649 : else
6650 : {
6651 2 : iter_result = make_temp_ssa_name (type, NULL, "reassocpow");
6652 2 : pow_stmt
6653 2 : = gimple_build_call (powi_fndecl, 2, rf1->repr,
6654 : build_int_cst (integer_type_node,
6655 2 : power));
6656 2 : gimple_call_set_lhs (pow_stmt, iter_result);
6657 2 : gimple_set_location (pow_stmt, gimple_location (stmt));
6658 2 : gimple_set_uid (pow_stmt, gimple_uid (stmt));
6659 2 : gsi_insert_before (&gsi, pow_stmt, GSI_SAME_STMT);
6660 : }
6661 :
6662 2 : if (dump_file && (dump_flags & TDF_DETAILS))
6663 : {
6664 0 : unsigned elt;
6665 0 : repeat_factor *rf;
6666 0 : fputs ("Building __builtin_pow call for cached product (",
6667 : dump_file);
6668 0 : for (elt = j; elt < vec_len; elt++)
6669 : {
6670 0 : rf = &repeat_factor_vec[elt];
6671 0 : print_generic_expr (dump_file, rf->factor);
6672 0 : if (elt < vec_len - 1)
6673 0 : fputs (" * ", dump_file);
6674 : }
6675 0 : fprintf (dump_file, ")^" HOST_WIDE_INT_PRINT_DEC"\n",
6676 : power);
6677 : }
6678 : }
6679 : }
6680 : else
6681 : {
6682 : /* Otherwise, find the first factor in the repeated factor
6683 : vector whose occurrence count is at least 2. If no such
6684 : factor exists, there are no builtin_powi opportunities
6685 : remaining. */
6686 1076545 : FOR_EACH_VEC_ELT (repeat_factor_vec, j, rf1)
6687 : {
6688 572345 : if (rf1->count >= 2)
6689 : break;
6690 : }
6691 :
6692 505988 : if (j >= vec_len)
6693 : break;
6694 :
6695 1788 : power = rf1->count;
6696 :
6697 1788 : if (dump_file && (dump_flags & TDF_DETAILS))
6698 : {
6699 0 : unsigned elt;
6700 0 : repeat_factor *rf;
6701 0 : fputs ("Building __builtin_pow call for (", dump_file);
6702 0 : for (elt = j; elt < vec_len; elt++)
6703 : {
6704 0 : rf = &repeat_factor_vec[elt];
6705 0 : print_generic_expr (dump_file, rf->factor);
6706 0 : if (elt < vec_len - 1)
6707 0 : fputs (" * ", dump_file);
6708 : }
6709 0 : fprintf (dump_file, ")^" HOST_WIDE_INT_PRINT_DEC"\n", power);
6710 : }
6711 :
6712 1788 : reassociate_stats.pows_created++;
6713 :
6714 : /* Visit each element of the vector in reverse order (so that
6715 : high-occurrence elements are visited first, and within the
6716 : same occurrence count, lower-ranked elements are visited
6717 : first). Form a linear product of all elements in this order
6718 : whose occurrencce count is at least that of element J.
6719 : Record the SSA name representing the product of each element
6720 : with all subsequent elements in the vector. */
6721 1788 : if (j == vec_len - 1)
6722 1767 : rf1->repr = rf1->factor;
6723 : else
6724 : {
6725 50 : for (ii = vec_len - 2; ii >= (int)j; ii--)
6726 : {
6727 29 : tree op1, op2;
6728 :
6729 29 : rf1 = &repeat_factor_vec[ii];
6730 29 : rf2 = &repeat_factor_vec[ii + 1];
6731 :
6732 : /* Init the last factor's representative to be itself. */
6733 29 : if (!rf2->repr)
6734 21 : rf2->repr = rf2->factor;
6735 :
6736 29 : op1 = rf1->factor;
6737 29 : op2 = rf2->repr;
6738 :
6739 29 : target_ssa = make_temp_ssa_name (type, NULL, "reassocpow");
6740 29 : mul_stmt = gimple_build_assign (target_ssa, MULT_EXPR,
6741 : op1, op2);
6742 29 : gimple_set_location (mul_stmt, gimple_location (stmt));
6743 29 : gimple_set_uid (mul_stmt, gimple_uid (stmt));
6744 29 : gsi_insert_before (&gsi, mul_stmt, GSI_SAME_STMT);
6745 29 : rf1->repr = target_ssa;
6746 :
6747 : /* Don't reprocess the multiply we just introduced. */
6748 29 : gimple_set_visited (mul_stmt, true);
6749 : }
6750 : }
6751 :
6752 : /* Form a call to __builtin_powi for the maximum product
6753 : just formed, raised to the power obtained earlier. */
6754 1788 : rf1 = &repeat_factor_vec[j];
6755 1788 : if (use_mults)
6756 : {
6757 1226 : gcc_assert (power > 1);
6758 1226 : gimple_stmt_iterator gsip = gsi;
6759 1226 : gsi_prev (&gsip);
6760 1226 : iter_result = powi_as_mults (&gsi, gimple_location (stmt),
6761 : rf1->repr, power);
6762 1226 : gimple_stmt_iterator gsic = gsi;
6763 3715 : while (gsi_stmt (gsic) != gsi_stmt (gsip))
6764 : {
6765 2489 : gimple_set_uid (gsi_stmt (gsic), gimple_uid (stmt));
6766 2489 : gimple_set_visited (gsi_stmt (gsic), true);
6767 2545 : gsi_prev (&gsic);
6768 : }
6769 : }
6770 : else
6771 : {
6772 562 : iter_result = make_temp_ssa_name (type, NULL, "reassocpow");
6773 562 : pow_stmt = gimple_build_call (powi_fndecl, 2, rf1->repr,
6774 : build_int_cst (integer_type_node,
6775 562 : power));
6776 562 : gimple_call_set_lhs (pow_stmt, iter_result);
6777 562 : gimple_set_location (pow_stmt, gimple_location (stmt));
6778 562 : gimple_set_uid (pow_stmt, gimple_uid (stmt));
6779 562 : gsi_insert_before (&gsi, pow_stmt, GSI_SAME_STMT);
6780 : }
6781 : }
6782 :
6783 : /* If we previously formed at least one other builtin_powi call,
6784 : form the product of this one and those others. */
6785 1797 : if (result)
6786 : {
6787 9 : tree new_result = make_temp_ssa_name (type, NULL, "reassocpow");
6788 9 : mul_stmt = gimple_build_assign (new_result, MULT_EXPR,
6789 : result, iter_result);
6790 9 : gimple_set_location (mul_stmt, gimple_location (stmt));
6791 9 : gimple_set_uid (mul_stmt, gimple_uid (stmt));
6792 9 : gsi_insert_before (&gsi, mul_stmt, GSI_SAME_STMT);
6793 9 : gimple_set_visited (mul_stmt, true);
6794 9 : result = new_result;
6795 : }
6796 : else
6797 : result = iter_result;
6798 :
6799 : /* Decrement the occurrence count of each element in the product
6800 : by the count found above, and remove this many copies of each
6801 : factor from OPS. */
6802 3628 : for (i = j; i < vec_len; i++)
6803 : {
6804 1831 : unsigned k = power;
6805 1831 : unsigned n;
6806 :
6807 1831 : rf1 = &repeat_factor_vec[i];
6808 1831 : rf1->count -= power;
6809 :
6810 8726 : FOR_EACH_VEC_ELT_REVERSE (*ops, n, oe)
6811 : {
6812 5064 : if (oe->op == rf1->factor)
6813 : {
6814 4650 : if (oe->count <= k)
6815 : {
6816 4644 : ops->ordered_remove (n);
6817 4644 : k -= oe->count;
6818 :
6819 4644 : if (k == 0)
6820 : break;
6821 : }
6822 : else
6823 : {
6824 6 : oe->count -= k;
6825 6 : break;
6826 : }
6827 : }
6828 : }
6829 : }
6830 : }
6831 :
6832 : /* At this point all elements in the repeated factor vector have a
6833 : remaining occurrence count of 0 or 1, and those with a count of 1
6834 : don't have cached representatives. Re-sort the ops vector and
6835 : clean up. */
6836 504200 : ops->qsort (sort_by_operand_rank);
6837 504200 : repeat_factor_vec.release ();
6838 :
6839 : /* Return the final product computed herein. Note that there may
6840 : still be some elements with single occurrence count left in OPS;
6841 : those will be handled by the normal reassociation logic. */
6842 504200 : return result;
6843 : }
6844 :
6845 : /* Attempt to optimize
6846 : CST1 * copysign (CST2, y) -> copysign (CST1 * CST2, y) if CST1 > 0, or
6847 : CST1 * copysign (CST2, y) -> -copysign (CST1 * CST2, y) if CST1 < 0. */
6848 :
6849 : static void
6850 1108885 : attempt_builtin_copysign (vec<operand_entry *> *ops)
6851 : {
6852 1108885 : operand_entry *oe;
6853 1108885 : unsigned int i;
6854 1108885 : unsigned int length = ops->length ();
6855 1108885 : tree cst = ops->last ()->op;
6856 :
6857 1108885 : if (length == 1 || TREE_CODE (cst) != REAL_CST)
6858 : return;
6859 :
6860 4152 : FOR_EACH_VEC_ELT (*ops, i, oe)
6861 : {
6862 2961 : if (TREE_CODE (oe->op) == SSA_NAME
6863 2961 : && has_single_use (oe->op))
6864 : {
6865 865 : gimple *def_stmt = SSA_NAME_DEF_STMT (oe->op);
6866 2999 : if (gcall *old_call = dyn_cast <gcall *> (def_stmt))
6867 : {
6868 54 : tree arg0, arg1;
6869 54 : switch (gimple_call_combined_fn (old_call))
6870 : {
6871 20 : CASE_CFN_COPYSIGN:
6872 20 : CASE_CFN_COPYSIGN_FN:
6873 20 : arg0 = gimple_call_arg (old_call, 0);
6874 20 : arg1 = gimple_call_arg (old_call, 1);
6875 : /* The first argument of copysign must be a constant,
6876 : otherwise there's nothing to do. */
6877 20 : if (TREE_CODE (arg0) == REAL_CST)
6878 : {
6879 20 : tree type = TREE_TYPE (arg0);
6880 20 : tree mul = const_binop (MULT_EXPR, type, cst, arg0);
6881 : /* If we couldn't fold to a single constant, skip it.
6882 : That happens e.g. for inexact multiplication when
6883 : -frounding-math. */
6884 20 : if (mul == NULL_TREE)
6885 : break;
6886 : /* Instead of adjusting OLD_CALL, let's build a new
6887 : call to not leak the LHS and prevent keeping bogus
6888 : debug statements. DCE will clean up the old call. */
6889 16 : gcall *new_call;
6890 16 : if (gimple_call_internal_p (old_call))
6891 0 : new_call = gimple_build_call_internal
6892 0 : (IFN_COPYSIGN, 2, mul, arg1);
6893 : else
6894 16 : new_call = gimple_build_call
6895 16 : (gimple_call_fndecl (old_call), 2, mul, arg1);
6896 16 : tree lhs = make_ssa_name (type);
6897 16 : gimple_call_set_lhs (new_call, lhs);
6898 16 : gimple_set_location (new_call,
6899 : gimple_location (old_call));
6900 16 : insert_stmt_after (new_call, old_call);
6901 : /* We've used the constant, get rid of it. */
6902 16 : ops->pop ();
6903 16 : bool cst1_neg = real_isneg (TREE_REAL_CST_PTR (cst));
6904 : /* Handle the CST1 < 0 case by negating the result. */
6905 16 : if (cst1_neg)
6906 : {
6907 7 : tree negrhs = make_ssa_name (TREE_TYPE (lhs));
6908 7 : gimple *negate_stmt
6909 7 : = gimple_build_assign (negrhs, NEGATE_EXPR, lhs);
6910 7 : insert_stmt_after (negate_stmt, new_call);
6911 7 : oe->op = negrhs;
6912 : }
6913 : else
6914 : oe->op = lhs;
6915 16 : if (dump_file && (dump_flags & TDF_DETAILS))
6916 : {
6917 14 : fprintf (dump_file, "Optimizing copysign: ");
6918 14 : print_generic_expr (dump_file, cst);
6919 14 : fprintf (dump_file, " * COPYSIGN (");
6920 14 : print_generic_expr (dump_file, arg0);
6921 14 : fprintf (dump_file, ", ");
6922 14 : print_generic_expr (dump_file, arg1);
6923 23 : fprintf (dump_file, ") into %sCOPYSIGN (",
6924 : cst1_neg ? "-" : "");
6925 14 : print_generic_expr (dump_file, mul);
6926 14 : fprintf (dump_file, ", ");
6927 14 : print_generic_expr (dump_file, arg1);
6928 14 : fprintf (dump_file, "\n");
6929 : }
6930 : return;
6931 : }
6932 : break;
6933 : default:
6934 : break;
6935 : }
6936 : }
6937 : }
6938 : }
6939 : }
6940 :
6941 : /* Transform STMT at *GSI into a copy by replacing its rhs with NEW_RHS. */
6942 :
6943 : static void
6944 14459 : transform_stmt_to_copy (gimple_stmt_iterator *gsi, gimple *stmt, tree new_rhs)
6945 : {
6946 14459 : tree rhs1;
6947 :
6948 14459 : if (dump_file && (dump_flags & TDF_DETAILS))
6949 : {
6950 28 : fprintf (dump_file, "Transforming ");
6951 28 : print_gimple_stmt (dump_file, stmt, 0);
6952 : }
6953 :
6954 14459 : rhs1 = gimple_assign_rhs1 (stmt);
6955 14459 : gimple_assign_set_rhs_from_tree (gsi, new_rhs);
6956 14459 : update_stmt (stmt);
6957 14459 : remove_visited_stmt_chain (rhs1);
6958 :
6959 14459 : if (dump_file && (dump_flags & TDF_DETAILS))
6960 : {
6961 28 : fprintf (dump_file, " into ");
6962 28 : print_gimple_stmt (dump_file, stmt, 0);
6963 : }
6964 14459 : }
6965 :
6966 : /* Transform STMT at *GSI into a multiply of RHS1 and RHS2. */
6967 :
6968 : static void
6969 193 : transform_stmt_to_multiply (gimple_stmt_iterator *gsi, gimple *stmt,
6970 : tree rhs1, tree rhs2)
6971 : {
6972 193 : if (dump_file && (dump_flags & TDF_DETAILS))
6973 : {
6974 0 : fprintf (dump_file, "Transforming ");
6975 0 : print_gimple_stmt (dump_file, stmt, 0);
6976 : }
6977 :
6978 193 : gimple_assign_set_rhs_with_ops (gsi, MULT_EXPR, rhs1, rhs2);
6979 193 : update_stmt (gsi_stmt (*gsi));
6980 193 : remove_visited_stmt_chain (rhs1);
6981 :
6982 193 : if (dump_file && (dump_flags & TDF_DETAILS))
6983 : {
6984 0 : fprintf (dump_file, " into ");
6985 0 : print_gimple_stmt (dump_file, stmt, 0);
6986 : }
6987 193 : }
6988 :
6989 : /* Rearrange ops may have more FMA when the chain may has more than 2 FMAs.
6990 : Put no-mult ops and mult ops alternately at the end of the queue, which is
6991 : conducive to generating more FMA and reducing the loss of FMA when breaking
6992 : the chain.
6993 : E.g.
6994 : a * b + c * d + e generates:
6995 :
6996 : _4 = c_9(D) * d_10(D);
6997 : _12 = .FMA (a_7(D), b_8(D), _4);
6998 : _11 = e_6(D) + _12;
6999 :
7000 : Rearrange ops to -> e + a * b + c * d generates:
7001 :
7002 : _4 = .FMA (c_7(D), d_8(D), _3);
7003 : _11 = .FMA (a_5(D), b_6(D), _4);
7004 :
7005 : Return the number of MULT_EXPRs in the chain. */
7006 : static int
7007 16826 : rank_ops_for_fma (vec<operand_entry *> *ops)
7008 : {
7009 16826 : operand_entry *oe;
7010 16826 : unsigned int i;
7011 16826 : unsigned int ops_length = ops->length ();
7012 16826 : auto_vec<operand_entry *> ops_mult;
7013 16826 : auto_vec<operand_entry *> ops_others;
7014 :
7015 73006 : FOR_EACH_VEC_ELT (*ops, i, oe)
7016 : {
7017 39354 : if (TREE_CODE (oe->op) == SSA_NAME)
7018 : {
7019 39338 : gimple *def_stmt = SSA_NAME_DEF_STMT (oe->op);
7020 39338 : if (is_gimple_assign (def_stmt))
7021 : {
7022 28093 : if (gimple_assign_rhs_code (def_stmt) == MULT_EXPR)
7023 13046 : ops_mult.safe_push (oe);
7024 : /* A negate on the multiplication leads to FNMA. */
7025 15047 : else if (gimple_assign_rhs_code (def_stmt) == NEGATE_EXPR
7026 15047 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
7027 : {
7028 2839 : gimple *neg_def_stmt
7029 2839 : = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (def_stmt));
7030 2839 : if (is_gimple_assign (neg_def_stmt)
7031 2836 : && gimple_bb (neg_def_stmt) == gimple_bb (def_stmt)
7032 5670 : && gimple_assign_rhs_code (neg_def_stmt) == MULT_EXPR)
7033 2792 : ops_mult.safe_push (oe);
7034 : else
7035 47 : ops_others.safe_push (oe);
7036 : }
7037 : else
7038 12208 : ops_others.safe_push (oe);
7039 : }
7040 : else
7041 11245 : ops_others.safe_push (oe);
7042 : }
7043 : else
7044 16 : ops_others.safe_push (oe);
7045 : }
7046 : /* 1. When ops_mult.length == 2, like the following case,
7047 :
7048 : a * b + c * d + e.
7049 :
7050 : we need to rearrange the ops.
7051 :
7052 : Putting ops that not def from mult in front can generate more FMAs.
7053 :
7054 : 2. If all ops are defined with mult, we don't need to rearrange them. */
7055 16826 : unsigned mult_num = ops_mult.length ();
7056 16826 : if (mult_num >= 2 && mult_num != ops_length)
7057 : {
7058 : /* Put no-mult ops and mult ops alternately at the end of the
7059 : queue, which is conducive to generating more FMA and reducing the
7060 : loss of FMA when breaking the chain. */
7061 5292 : ops->truncate (0);
7062 5292 : ops->splice (ops_mult);
7063 5292 : int j, opindex = ops->length ();
7064 5292 : int others_length = ops_others.length ();
7065 10589 : for (j = 0; j < others_length; j++)
7066 : {
7067 5297 : oe = ops_others.pop ();
7068 5297 : ops->quick_insert (opindex, oe);
7069 5297 : if (opindex > 0)
7070 5296 : opindex--;
7071 : }
7072 : }
7073 16826 : return mult_num;
7074 16826 : }
7075 : /* Reassociate expressions in basic block BB and its post-dominator as
7076 : children.
7077 :
7078 : Bubble up return status from maybe_optimize_range_tests. */
7079 :
7080 : static bool
7081 19960999 : reassociate_bb (basic_block bb)
7082 : {
7083 19960999 : gimple_stmt_iterator gsi;
7084 19960999 : gimple *stmt = last_nondebug_stmt (bb);
7085 19960999 : bool cfg_cleanup_needed = false;
7086 :
7087 19960999 : if (stmt && !gimple_visited_p (stmt))
7088 19325763 : cfg_cleanup_needed |= maybe_optimize_range_tests (stmt);
7089 :
7090 19960999 : bool do_prev = false;
7091 39921998 : for (gsi = gsi_last_bb (bb);
7092 381490168 : !gsi_end_p (gsi); do_prev ? gsi_prev (&gsi) : (void) 0)
7093 : {
7094 180764813 : do_prev = true;
7095 180764813 : stmt = gsi_stmt (gsi);
7096 :
7097 180764813 : if (is_gimple_assign (stmt)
7098 180764813 : && !stmt_could_throw_p (cfun, stmt))
7099 : {
7100 46814228 : tree lhs, rhs1, rhs2;
7101 46814228 : enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
7102 :
7103 : /* If this was part of an already processed statement,
7104 : we don't need to touch it again. */
7105 46814228 : if (gimple_visited_p (stmt))
7106 : {
7107 : /* This statement might have become dead because of previous
7108 : reassociations. */
7109 403995 : if (has_zero_uses (gimple_get_lhs (stmt)))
7110 : {
7111 138182 : reassoc_remove_stmt (&gsi);
7112 138182 : release_defs (stmt);
7113 : /* We might end up removing the last stmt above which
7114 : places the iterator to the end of the sequence.
7115 : Reset it to the last stmt in this case and make sure
7116 : we don't do gsi_prev in that case. */
7117 138182 : if (gsi_end_p (gsi))
7118 : {
7119 457 : gsi = gsi_last_bb (bb);
7120 457 : do_prev = false;
7121 : }
7122 : }
7123 403995 : continue;
7124 : }
7125 :
7126 : /* If this is not a gimple binary expression, there is
7127 : nothing for us to do with it. */
7128 46410233 : if (get_gimple_rhs_class (rhs_code) != GIMPLE_BINARY_RHS)
7129 34386796 : continue;
7130 :
7131 12023437 : lhs = gimple_assign_lhs (stmt);
7132 12023437 : rhs1 = gimple_assign_rhs1 (stmt);
7133 12023437 : rhs2 = gimple_assign_rhs2 (stmt);
7134 :
7135 : /* For non-bit or min/max operations we can't associate
7136 : all types. Verify that here. */
7137 17680194 : if ((rhs_code != BIT_IOR_EXPR
7138 12023437 : && rhs_code != BIT_AND_EXPR
7139 11010079 : && rhs_code != BIT_XOR_EXPR
7140 11010079 : && rhs_code != MIN_EXPR
7141 10864775 : && rhs_code != MAX_EXPR
7142 10725199 : && !can_reassociate_type_p (TREE_TYPE (lhs)))
7143 6371344 : || !can_reassociate_op_p (rhs1)
7144 18391695 : || !can_reassociate_op_p (rhs2))
7145 5656757 : continue;
7146 :
7147 6366680 : if (associative_tree_code (rhs_code))
7148 : {
7149 4807880 : auto_vec<operand_entry *> ops;
7150 4807880 : tree powi_result = NULL_TREE;
7151 4807880 : bool is_vector = VECTOR_TYPE_P (TREE_TYPE (lhs));
7152 :
7153 : /* There may be no immediate uses left by the time we
7154 : get here because we may have eliminated them all. */
7155 4807880 : if (TREE_CODE (lhs) == SSA_NAME && has_zero_uses (lhs))
7156 47830 : continue;
7157 :
7158 4760050 : gimple_set_visited (stmt, true);
7159 4760050 : linearize_expr_tree (&ops, stmt, true, true);
7160 4760050 : ops.qsort (sort_by_operand_rank);
7161 4760050 : int orig_len = ops.length ();
7162 4760050 : optimize_ops_list (rhs_code, &ops);
7163 9520100 : if (undistribute_ops_list (rhs_code, &ops,
7164 : loop_containing_stmt (stmt)))
7165 : {
7166 315 : ops.qsort (sort_by_operand_rank);
7167 315 : optimize_ops_list (rhs_code, &ops);
7168 : }
7169 9520100 : if (undistribute_bitref_for_vector (rhs_code, &ops,
7170 : loop_containing_stmt (stmt)))
7171 : {
7172 36 : ops.qsort (sort_by_operand_rank);
7173 36 : optimize_ops_list (rhs_code, &ops);
7174 : }
7175 4760050 : if (rhs_code == PLUS_EXPR
7176 4760050 : && transform_add_to_multiply (&ops))
7177 73 : ops.qsort (sort_by_operand_rank);
7178 :
7179 4760050 : if (rhs_code == BIT_IOR_EXPR || rhs_code == BIT_AND_EXPR)
7180 : {
7181 998033 : if (is_vector)
7182 26638 : optimize_vec_cond_expr (rhs_code, &ops);
7183 : else
7184 971395 : optimize_range_tests (rhs_code, &ops, NULL);
7185 : }
7186 :
7187 4760050 : if (rhs_code == MULT_EXPR && !is_vector)
7188 : {
7189 1108885 : attempt_builtin_copysign (&ops);
7190 :
7191 1108885 : if (reassoc_insert_powi_p
7192 1108885 : && (flag_unsafe_math_optimizations
7193 460767 : || (INTEGRAL_TYPE_P (TREE_TYPE (lhs)))))
7194 504200 : powi_result = attempt_builtin_powi (stmt, &ops);
7195 : }
7196 :
7197 4760050 : operand_entry *last;
7198 4760050 : bool negate_result = false;
7199 4760050 : if (ops.length () > 1
7200 4760050 : && rhs_code == MULT_EXPR)
7201 : {
7202 1132784 : last = ops.last ();
7203 1132784 : if ((integer_minus_onep (last->op)
7204 1132263 : || real_minus_onep (last->op))
7205 548 : && !HONOR_SNANS (TREE_TYPE (lhs))
7206 1133332 : && (!HONOR_SIGNED_ZEROS (TREE_TYPE (lhs))
7207 0 : || !COMPLEX_FLOAT_TYPE_P (TREE_TYPE (lhs))))
7208 : {
7209 548 : ops.pop ();
7210 548 : negate_result = true;
7211 : }
7212 : }
7213 :
7214 4760050 : tree new_lhs = lhs;
7215 : /* If the operand vector is now empty, all operands were
7216 : consumed by the __builtin_powi optimization. */
7217 4760050 : if (ops.length () == 0)
7218 1466 : transform_stmt_to_copy (&gsi, stmt, powi_result);
7219 4758584 : else if (ops.length () == 1)
7220 : {
7221 13186 : tree last_op = ops.last ()->op;
7222 :
7223 : /* If the stmt that defines operand has to be inserted, insert it
7224 : before the use. */
7225 13186 : if (ops.last ()->stmt_to_insert)
7226 0 : insert_stmt_before_use (stmt, ops.last ()->stmt_to_insert);
7227 13186 : if (powi_result)
7228 193 : transform_stmt_to_multiply (&gsi, stmt, last_op,
7229 : powi_result);
7230 : else
7231 12993 : transform_stmt_to_copy (&gsi, stmt, last_op);
7232 : }
7233 : else
7234 : {
7235 4745398 : machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
7236 4745398 : int ops_num = ops.length ();
7237 4745398 : int width = 0;
7238 4745398 : int mult_num = 0;
7239 :
7240 : /* For binary bit operations, if there are at least 3
7241 : operands and the last operand in OPS is a constant,
7242 : move it to the front. This helps ensure that we generate
7243 : (X & Y) & C rather than (X & C) & Y. The former will
7244 : often match a canonical bit test when we get to RTL. */
7245 4745398 : if (ops.length () > 2
7246 182493 : && (rhs_code == BIT_AND_EXPR
7247 : || rhs_code == BIT_IOR_EXPR
7248 159274 : || rhs_code == BIT_XOR_EXPR)
7249 4773209 : && TREE_CODE (ops.last ()->op) == INTEGER_CST)
7250 1775 : std::swap (*ops[0], *ops[ops_num - 1]);
7251 :
7252 4745398 : optimization_type opt_type = bb_optimization_type (bb);
7253 :
7254 : /* If the target support FMA, rank_ops_for_fma will detect if
7255 : the chain has fmas and rearrange the ops if so. */
7256 4745398 : if (!reassoc_insert_powi_p
7257 2928062 : && direct_internal_fn_supported_p (IFN_FMA,
7258 2928062 : TREE_TYPE (lhs),
7259 : opt_type)
7260 4780028 : && (rhs_code == PLUS_EXPR || rhs_code == MINUS_EXPR))
7261 : {
7262 16826 : mult_num = rank_ops_for_fma (&ops);
7263 : }
7264 :
7265 : /* A chain that expands to a conditional compare sequence
7266 : has to stay linear. Every conditional compare tests the
7267 : flags that the previous comparison set, so the sequence
7268 : cannot be split into independent parts. */
7269 4745398 : bool ccmp_chain = (ops_num > 2
7270 4745398 : && prepare_ccmp_chain (ops, rhs_code, bb));
7271 :
7272 : /* Only rewrite the expression tree to parallel in the
7273 : last reassoc pass to avoid useless work back-and-forth
7274 : with initial linearization. */
7275 4745398 : bool has_fma = mult_num >= 2 && mult_num != ops_num;
7276 4745398 : if (!reassoc_insert_powi_p
7277 2928062 : && !ccmp_chain
7278 2928062 : && ops.length () > 3
7279 4759259 : && (width = get_reassociation_width (&ops, mult_num, lhs,
7280 : rhs_code, mode))
7281 : > 1)
7282 : {
7283 1900 : if (dump_file && (dump_flags & TDF_DETAILS))
7284 2 : fprintf (dump_file,
7285 : "Width = %d was chosen for reassociation\n",
7286 : width);
7287 1900 : rewrite_expr_tree_parallel (as_a <gassign *> (stmt),
7288 : width,
7289 : has_fma,
7290 : ops);
7291 : }
7292 : else
7293 : {
7294 : /* When there are three operands left, we want
7295 : to make sure the ones that get the double
7296 : binary op are chosen wisely. */
7297 4743498 : int len = ops.length ();
7298 7669660 : if (!reassoc_insert_powi_p && !ccmp_chain
7299 2926162 : && len >= 3
7300 4872547 : && (!has_fma
7301 : /* width > 1 means ranking ops results in better
7302 : parallelism. Check current value to avoid
7303 : calling get_reassociation_width again. */
7304 5288 : || (width != 1
7305 5288 : && get_reassociation_width (
7306 : &ops, mult_num, lhs, rhs_code, mode)
7307 : > 1)))
7308 125800 : swap_ops_for_binary_stmt (ops, len - 3);
7309 :
7310 4743498 : new_lhs = rewrite_expr_tree (stmt, rhs_code, 0, ops,
7311 4743498 : powi_result != NULL
7312 : || negate_result,
7313 : len != orig_len);
7314 : }
7315 :
7316 : /* If we combined some repeated factors into a
7317 : __builtin_powi call, multiply that result by the
7318 : reassociated operands. */
7319 4745398 : if (powi_result)
7320 : {
7321 129 : gimple *mul_stmt, *lhs_stmt = SSA_NAME_DEF_STMT (lhs);
7322 129 : tree type = TREE_TYPE (lhs);
7323 129 : tree target_ssa = make_temp_ssa_name (type, NULL,
7324 : "reassocpow");
7325 129 : gimple_set_lhs (lhs_stmt, target_ssa);
7326 129 : update_stmt (lhs_stmt);
7327 129 : if (lhs != new_lhs)
7328 : {
7329 129 : target_ssa = new_lhs;
7330 129 : new_lhs = lhs;
7331 : }
7332 129 : mul_stmt = gimple_build_assign (lhs, MULT_EXPR,
7333 : powi_result, target_ssa);
7334 129 : gimple_set_location (mul_stmt, gimple_location (stmt));
7335 129 : gimple_set_uid (mul_stmt, gimple_uid (stmt));
7336 129 : gsi_insert_after (&gsi, mul_stmt, GSI_NEW_STMT);
7337 : }
7338 : }
7339 :
7340 4760050 : if (negate_result)
7341 : {
7342 548 : stmt = SSA_NAME_DEF_STMT (lhs);
7343 548 : tree tmp = make_ssa_name (TREE_TYPE (lhs));
7344 548 : gimple_set_lhs (stmt, tmp);
7345 548 : if (lhs != new_lhs)
7346 538 : tmp = new_lhs;
7347 548 : gassign *neg_stmt = gimple_build_assign (lhs, NEGATE_EXPR,
7348 : tmp);
7349 548 : gimple_set_uid (neg_stmt, gimple_uid (stmt));
7350 548 : gsi_insert_after (&gsi, neg_stmt, GSI_NEW_STMT);
7351 548 : update_stmt (stmt);
7352 : }
7353 4807880 : }
7354 : }
7355 : }
7356 :
7357 19960999 : return cfg_cleanup_needed;
7358 : }
7359 :
7360 : /* Add jumps around shifts for range tests turned into bit tests.
7361 : For each SSA_NAME VAR we have code like:
7362 : VAR = ...; // final stmt of range comparison
7363 : // bit test here...;
7364 : OTHERVAR = ...; // final stmt of the bit test sequence
7365 : RES = VAR | OTHERVAR;
7366 : Turn the above into:
7367 : VAR = ...;
7368 : if (VAR != 0)
7369 : goto <l3>;
7370 : else
7371 : goto <l2>;
7372 : <l2>:
7373 : // bit test here...;
7374 : OTHERVAR = ...;
7375 : <l3>:
7376 : # RES = PHI<1(l1), OTHERVAR(l2)>; */
7377 :
7378 : static void
7379 2125670 : branch_fixup (void)
7380 : {
7381 2125670 : tree var;
7382 2125670 : unsigned int i;
7383 :
7384 2126057 : FOR_EACH_VEC_ELT (reassoc_branch_fixups, i, var)
7385 : {
7386 387 : gimple *def_stmt = SSA_NAME_DEF_STMT (var);
7387 387 : gimple *use_stmt;
7388 387 : use_operand_p use;
7389 387 : bool ok = single_imm_use (var, &use, &use_stmt);
7390 387 : gcc_assert (ok
7391 : && is_gimple_assign (use_stmt)
7392 : && gimple_assign_rhs_code (use_stmt) == BIT_IOR_EXPR
7393 : && gimple_bb (def_stmt) == gimple_bb (use_stmt));
7394 :
7395 387 : basic_block cond_bb = gimple_bb (def_stmt);
7396 387 : basic_block then_bb = split_block (cond_bb, def_stmt)->dest;
7397 387 : basic_block merge_bb = split_block (then_bb, use_stmt)->dest;
7398 :
7399 387 : gimple_stmt_iterator gsi = gsi_for_stmt (def_stmt);
7400 387 : gimple *g = gimple_build_cond (NE_EXPR, var,
7401 387 : build_zero_cst (TREE_TYPE (var)),
7402 : NULL_TREE, NULL_TREE);
7403 387 : location_t loc = gimple_location (use_stmt);
7404 387 : gimple_set_location (g, loc);
7405 387 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
7406 :
7407 387 : edge etrue = make_edge (cond_bb, merge_bb, EDGE_TRUE_VALUE);
7408 387 : etrue->probability = profile_probability::even ();
7409 387 : edge efalse = find_edge (cond_bb, then_bb);
7410 387 : efalse->flags = EDGE_FALSE_VALUE;
7411 387 : efalse->probability -= etrue->probability;
7412 387 : then_bb->count -= etrue->count ();
7413 :
7414 387 : tree othervar = NULL_TREE;
7415 387 : if (gimple_assign_rhs1 (use_stmt) == var)
7416 284 : othervar = gimple_assign_rhs2 (use_stmt);
7417 103 : else if (gimple_assign_rhs2 (use_stmt) == var)
7418 : othervar = gimple_assign_rhs1 (use_stmt);
7419 : else
7420 0 : gcc_unreachable ();
7421 387 : tree lhs = gimple_assign_lhs (use_stmt);
7422 387 : gphi *phi = create_phi_node (lhs, merge_bb);
7423 387 : add_phi_arg (phi, build_one_cst (TREE_TYPE (lhs)), etrue, loc);
7424 387 : add_phi_arg (phi, othervar, single_succ_edge (then_bb), loc);
7425 387 : gsi = gsi_for_stmt (use_stmt);
7426 387 : gsi_remove (&gsi, true);
7427 :
7428 387 : set_immediate_dominator (CDI_DOMINATORS, merge_bb, cond_bb);
7429 387 : set_immediate_dominator (CDI_POST_DOMINATORS, cond_bb, merge_bb);
7430 : }
7431 2125670 : reassoc_branch_fixups.release ();
7432 2125670 : }
7433 :
7434 : void dump_ops_vector (FILE *file, vec<operand_entry *> ops);
7435 : void debug_ops_vector (vec<operand_entry *> ops);
7436 :
7437 : /* Dump the operand entry vector OPS to FILE. */
7438 :
7439 : void
7440 0 : dump_ops_vector (FILE *file, vec<operand_entry *> ops)
7441 : {
7442 0 : operand_entry *oe;
7443 0 : unsigned int i;
7444 :
7445 0 : FOR_EACH_VEC_ELT (ops, i, oe)
7446 : {
7447 0 : fprintf (file, "Op %d -> rank: %d, tree: ", i, oe->rank);
7448 0 : print_generic_expr (file, oe->op);
7449 0 : fprintf (file, "\n");
7450 : }
7451 0 : }
7452 :
7453 : /* Dump the operand entry vector OPS to STDERR. */
7454 :
7455 : DEBUG_FUNCTION void
7456 0 : debug_ops_vector (vec<operand_entry *> ops)
7457 : {
7458 0 : dump_ops_vector (stderr, ops);
7459 0 : }
7460 :
7461 : /* Bubble up return status from reassociate_bb. */
7462 :
7463 : static bool
7464 2125670 : do_reassoc ()
7465 : {
7466 2125670 : bool cfg_cleanup_needed = false;
7467 2125670 : basic_block *worklist = XNEWVEC (basic_block, n_basic_blocks_for_fn (cfun));
7468 :
7469 2125670 : unsigned sp = 0;
7470 2125670 : for (auto son = first_dom_son (CDI_DOMINATORS, ENTRY_BLOCK_PTR_FOR_FN (cfun));
7471 4251340 : son; son = next_dom_son (CDI_DOMINATORS, son))
7472 2125670 : worklist[sp++] = son;
7473 22086720 : while (sp)
7474 : {
7475 19961050 : basic_block bb = worklist[--sp];
7476 19961050 : break_up_subtract_bb (bb);
7477 19961050 : for (auto son = first_dom_son (CDI_DOMINATORS, bb);
7478 37796430 : son; son = next_dom_son (CDI_DOMINATORS, son))
7479 17835380 : worklist[sp++] = son;
7480 : }
7481 :
7482 11004736 : for (auto son = first_dom_son (CDI_POST_DOMINATORS,
7483 2125670 : EXIT_BLOCK_PTR_FOR_FN (cfun));
7484 11004736 : son; son = next_dom_son (CDI_POST_DOMINATORS, son))
7485 8879066 : worklist[sp++] = son;
7486 22086669 : while (sp)
7487 : {
7488 19960999 : basic_block bb = worklist[--sp];
7489 19960999 : cfg_cleanup_needed |= reassociate_bb (bb);
7490 19960999 : for (auto son = first_dom_son (CDI_POST_DOMINATORS, bb);
7491 31042932 : son; son = next_dom_son (CDI_POST_DOMINATORS, son))
7492 11081933 : worklist[sp++] = son;
7493 : }
7494 :
7495 2125670 : free (worklist);
7496 2125670 : return cfg_cleanup_needed;
7497 : }
7498 :
7499 : /* Initialize the reassociation pass. */
7500 :
7501 : static void
7502 2125670 : init_reassoc (void)
7503 : {
7504 2125670 : int i;
7505 2125670 : int64_t rank = 2;
7506 2125670 : int *bbs = XNEWVEC (int, n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS);
7507 :
7508 : /* Find the loops, so that we can prevent moving calculations in
7509 : them. */
7510 2125670 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
7511 :
7512 2125670 : memset (&reassociate_stats, 0, sizeof (reassociate_stats));
7513 :
7514 2125670 : next_operand_entry_id = 0;
7515 :
7516 : /* Reverse RPO (Reverse Post Order) will give us something where
7517 : deeper loops come later. */
7518 2125670 : pre_and_rev_post_order_compute (NULL, bbs, false);
7519 2125670 : bb_rank = XCNEWVEC (int64_t, last_basic_block_for_fn (cfun));
7520 2125670 : operand_rank = new hash_map<tree, int64_t>;
7521 :
7522 : /* Give each default definition a distinct rank. This includes
7523 : parameters and the static chain. Walk backwards over all
7524 : SSA names so that we get proper rank ordering according
7525 : to tree_swap_operands_p. */
7526 112809916 : for (i = num_ssa_names - 1; i > 0; --i)
7527 : {
7528 108558576 : tree name = ssa_name (i);
7529 185414706 : if (name && SSA_NAME_IS_DEFAULT_DEF (name))
7530 6308902 : insert_operand_rank (name, ++rank);
7531 : }
7532 :
7533 : /* Set up rank for each BB */
7534 22086669 : for (i = 0; i < n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS; i++)
7535 19960999 : bb_rank[bbs[i]] = ++rank << 16;
7536 :
7537 2125670 : free (bbs);
7538 2125670 : calculate_dominance_info (CDI_POST_DOMINATORS);
7539 2125670 : plus_negates = vNULL;
7540 2125670 : mark_ssa_maybe_undefs ();
7541 2125670 : }
7542 :
7543 : /* Cleanup after the reassociation pass, and print stats if
7544 : requested. */
7545 :
7546 : static void
7547 2125670 : fini_reassoc (void)
7548 : {
7549 2125670 : statistics_counter_event (cfun, "Linearized",
7550 : reassociate_stats.linearized);
7551 2125670 : statistics_counter_event (cfun, "Constants eliminated",
7552 : reassociate_stats.constants_eliminated);
7553 2125670 : statistics_counter_event (cfun, "Ops eliminated",
7554 : reassociate_stats.ops_eliminated);
7555 2125670 : statistics_counter_event (cfun, "Statements rewritten",
7556 : reassociate_stats.rewritten);
7557 2125670 : statistics_counter_event (cfun, "Built-in pow[i] calls encountered",
7558 : reassociate_stats.pows_encountered);
7559 2125670 : statistics_counter_event (cfun, "Built-in powi calls created",
7560 : reassociate_stats.pows_created);
7561 :
7562 4251340 : delete operand_rank;
7563 2125670 : bitmap_clear (biased_names);
7564 2125670 : operand_entry_pool.release ();
7565 2125670 : free (bb_rank);
7566 2125670 : plus_negates.release ();
7567 2125670 : free_dominance_info (CDI_POST_DOMINATORS);
7568 2125670 : loop_optimizer_finalize ();
7569 2125670 : }
7570 :
7571 : /* Gate and execute functions for Reassociation. If INSERT_POWI_P, enable
7572 : insertion of __builtin_powi calls.
7573 :
7574 : Returns TODO_cfg_cleanup if a CFG cleanup pass is desired due to
7575 : optimization of a gimple conditional. Otherwise returns zero. */
7576 :
7577 : static unsigned int
7578 2125670 : execute_reassoc (bool insert_powi_p, bool bias_loop_carried_phi_ranks_p)
7579 : {
7580 2125670 : reassoc_insert_powi_p = insert_powi_p;
7581 2125670 : reassoc_bias_loop_carried_phi_ranks_p = bias_loop_carried_phi_ranks_p;
7582 :
7583 2125670 : init_reassoc ();
7584 :
7585 2125670 : bool cfg_cleanup_needed;
7586 2125670 : cfg_cleanup_needed = do_reassoc ();
7587 2125670 : repropagate_negates ();
7588 2125670 : branch_fixup ();
7589 :
7590 2125670 : fini_reassoc ();
7591 2125670 : return cfg_cleanup_needed ? TODO_cleanup_cfg : 0;
7592 : }
7593 :
7594 : namespace {
7595 :
7596 : const pass_data pass_data_reassoc =
7597 : {
7598 : GIMPLE_PASS, /* type */
7599 : "reassoc", /* name */
7600 : OPTGROUP_NONE, /* optinfo_flags */
7601 : TV_TREE_REASSOC, /* tv_id */
7602 : ( PROP_cfg | PROP_ssa ), /* properties_required */
7603 : 0, /* properties_provided */
7604 : 0, /* properties_destroyed */
7605 : 0, /* todo_flags_start */
7606 : TODO_update_ssa_only_virtuals, /* todo_flags_finish */
7607 : };
7608 :
7609 : class pass_reassoc : public gimple_opt_pass
7610 : {
7611 : public:
7612 594360 : pass_reassoc (gcc::context *ctxt)
7613 1188720 : : gimple_opt_pass (pass_data_reassoc, ctxt), insert_powi_p (false)
7614 : {}
7615 :
7616 : /* opt_pass methods: */
7617 297180 : opt_pass * clone () final override { return new pass_reassoc (m_ctxt); }
7618 594360 : void set_pass_param (unsigned int n, bool param) final override
7619 : {
7620 594360 : gcc_assert (n == 0);
7621 594360 : insert_powi_p = param;
7622 594360 : bias_loop_carried_phi_ranks_p = !param;
7623 594360 : }
7624 2126036 : bool gate (function *) final override { return flag_tree_reassoc != 0; }
7625 2125670 : unsigned int execute (function *) final override
7626 : {
7627 2125670 : return execute_reassoc (insert_powi_p, bias_loop_carried_phi_ranks_p);
7628 : }
7629 :
7630 : private:
7631 : /* Enable insertion of __builtin_powi calls during execute_reassoc. See
7632 : point 3a in the pass header comment. */
7633 : bool insert_powi_p;
7634 : bool bias_loop_carried_phi_ranks_p;
7635 : }; // class pass_reassoc
7636 :
7637 : } // anon namespace
7638 :
7639 : gimple_opt_pass *
7640 297180 : make_pass_reassoc (gcc::context *ctxt)
7641 : {
7642 297180 : return new pass_reassoc (ctxt);
7643 : }
|