Line data Source code
1 : /* Reassociation for trees.
2 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
3 : Contributed by Daniel Berlin <dan@dberlin.org>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify
8 : it under the terms of the GNU General Public License as published by
9 : the Free Software Foundation; either version 3, or (at your option)
10 : any later version.
11 :
12 : GCC is distributed in the hope that it will be useful,
13 : but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 : GNU General Public License for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "target.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "gimple.h"
29 : #include "cfghooks.h"
30 : #include "alloc-pool.h"
31 : #include "tree-pass.h"
32 : #include "memmodel.h"
33 : #include "tm_p.h"
34 : #include "ssa.h"
35 : #include "optabs-tree.h"
36 : #include "gimple-pretty-print.h"
37 : #include "diagnostic-core.h"
38 : #include "fold-const.h"
39 : #include "stor-layout.h"
40 : #include "cfganal.h"
41 : #include "gimple-iterator.h"
42 : #include "gimple-fold.h"
43 : #include "tree-eh.h"
44 : #include "gimplify-me.h"
45 : #include "tree-cfg.h"
46 : #include "tree-ssa-loop.h"
47 : #include "flags.h"
48 : #include "tree-ssa.h"
49 : #include "langhooks.h"
50 : #include "cfgloop.h"
51 : #include "builtins.h"
52 : #include "gimplify.h"
53 : #include "case-cfn-macros.h"
54 : #include "tree-ssa-reassoc.h"
55 : #include "tree-ssa-math-opts.h"
56 : #include "gimple-range.h"
57 : #include "internal-fn.h"
58 :
59 : /* This is a simple global reassociation pass. It is, in part, based
60 : on the LLVM pass of the same name (They do some things more/less
61 : than we do, in different orders, etc).
62 :
63 : It consists of five steps:
64 :
65 : 1. Breaking up subtract operations into addition + negate, where
66 : it would promote the reassociation of adds.
67 :
68 : 2. Left linearization of the expression trees, so that (A+B)+(C+D)
69 : becomes (((A+B)+C)+D), which is easier for us to rewrite later.
70 : During linearization, we place the operands of the binary
71 : expressions into a vector of operand_entry_*
72 :
73 : 3. Optimization of the operand lists, eliminating things like a +
74 : -a, a & a, etc.
75 :
76 : 3a. Combine repeated factors with the same occurrence counts
77 : into a __builtin_powi call that will later be optimized into
78 : an optimal number of multiplies.
79 :
80 : 4. Rewrite the expression trees we linearized and optimized so
81 : they are in proper rank order.
82 :
83 : 5. Repropagate negates, as nothing else will clean it up ATM.
84 :
85 : A bit of theory on #4, since nobody seems to write anything down
86 : about why it makes sense to do it the way they do it:
87 :
88 : We could do this much nicer theoretically, but don't (for reasons
89 : explained after how to do it theoretically nice :P).
90 :
91 : In order to promote the most redundancy elimination, you want
92 : binary expressions whose operands are the same rank (or
93 : preferably, the same value) exposed to the redundancy eliminator,
94 : for possible elimination.
95 :
96 : So the way to do this if we really cared, is to build the new op
97 : tree from the leaves to the roots, merging as you go, and putting the
98 : new op on the end of the worklist, until you are left with one
99 : thing on the worklist.
100 :
101 : IE if you have to rewrite the following set of operands (listed with
102 : rank in parentheses), with opcode PLUS_EXPR:
103 :
104 : a (1), b (1), c (1), d (2), e (2)
105 :
106 :
107 : We start with our merge worklist empty, and the ops list with all of
108 : those on it.
109 :
110 : You want to first merge all leaves of the same rank, as much as
111 : possible.
112 :
113 : So first build a binary op of
114 :
115 : mergetmp = a + b, and put "mergetmp" on the merge worklist.
116 :
117 : Because there is no three operand form of PLUS_EXPR, c is not going to
118 : be exposed to redundancy elimination as a rank 1 operand.
119 :
120 : So you might as well throw it on the merge worklist (you could also
121 : consider it to now be a rank two operand, and merge it with d and e,
122 : but in this case, you then have evicted e from a binary op. So at
123 : least in this situation, you can't win.)
124 :
125 : Then build a binary op of d + e
126 : mergetmp2 = d + e
127 :
128 : and put mergetmp2 on the merge worklist.
129 :
130 : so merge worklist = {mergetmp, c, mergetmp2}
131 :
132 : Continue building binary ops of these operations until you have only
133 : one operation left on the worklist.
134 :
135 : So we have
136 :
137 : build binary op
138 : mergetmp3 = mergetmp + c
139 :
140 : worklist = {mergetmp2, mergetmp3}
141 :
142 : mergetmp4 = mergetmp2 + mergetmp3
143 :
144 : worklist = {mergetmp4}
145 :
146 : because we have one operation left, we can now just set the original
147 : statement equal to the result of that operation.
148 :
149 : This will at least expose a + b and d + e to redundancy elimination
150 : as binary operations.
151 :
152 : For extra points, you can reuse the old statements to build the
153 : mergetmps, since you shouldn't run out.
154 :
155 : So why don't we do this?
156 :
157 : Because it's expensive, and rarely will help. Most trees we are
158 : reassociating have 3 or less ops. If they have 2 ops, they already
159 : will be written into a nice single binary op. If you have 3 ops, a
160 : single simple check suffices to tell you whether the first two are of the
161 : same rank. If so, you know to order it
162 :
163 : mergetmp = op1 + op2
164 : newstmt = mergetmp + op3
165 :
166 : instead of
167 : mergetmp = op2 + op3
168 : newstmt = mergetmp + op1
169 :
170 : If all three are of the same rank, you can't expose them all in a
171 : single binary operator anyway, so the above is *still* the best you
172 : can do.
173 :
174 : Thus, this is what we do. When we have three ops left, we check to see
175 : what order to put them in, and call it a day. As a nod to vector sum
176 : reduction, we check if any of the ops are really a phi node that is a
177 : destructive update for the associating op, and keep the destructive
178 : update together for vector sum reduction recognition. */
179 :
180 : /* Enable insertion of __builtin_powi calls during execute_reassoc. See
181 : point 3a in the pass header comment. */
182 : static bool reassoc_insert_powi_p;
183 :
184 : /* Enable biasing ranks of loop accumulators. We don't want this before
185 : vectorization, since it interferes with reduction chains. */
186 : static bool reassoc_bias_loop_carried_phi_ranks_p;
187 :
188 : /* Statistics */
189 : static struct
190 : {
191 : int linearized;
192 : int constants_eliminated;
193 : int ops_eliminated;
194 : int rewritten;
195 : int pows_encountered;
196 : int pows_created;
197 : } reassociate_stats;
198 :
199 :
200 : static object_allocator<operand_entry> operand_entry_pool
201 : ("operand entry pool");
202 :
203 : /* This is used to assign a unique ID to each struct operand_entry
204 : so that qsort results are identical on different hosts. */
205 : static unsigned int next_operand_entry_id;
206 :
207 : /* Starting rank number for a given basic block, so that we can rank
208 : operations using unmovable instructions in that BB based on the bb
209 : depth. */
210 : static int64_t *bb_rank;
211 :
212 : /* Operand->rank hashtable. */
213 : static hash_map<tree, int64_t> *operand_rank;
214 :
215 : /* SSA_NAMEs that are forms of loop accumulators and whose ranks need to be
216 : biased. */
217 : static auto_bitmap biased_names;
218 :
219 : /* Vector of SSA_NAMEs on which after reassociate_bb is done with
220 : all basic blocks the CFG should be adjusted - basic blocks
221 : split right after that SSA_NAME's definition statement and before
222 : the only use, which must be a bit ior. */
223 : static vec<tree> reassoc_branch_fixups;
224 :
225 : /* Forward decls. */
226 : static int64_t get_rank (tree);
227 : static bool reassoc_stmt_dominates_stmt_p (gimple *, gimple *);
228 :
229 : /* Wrapper around gsi_remove, which adjusts gimple_uid of debug stmts
230 : possibly added by gsi_remove. */
231 :
232 : static bool
233 167167 : reassoc_remove_stmt (gimple_stmt_iterator *gsi)
234 : {
235 167167 : gimple *stmt = gsi_stmt (*gsi);
236 :
237 167167 : if (!MAY_HAVE_DEBUG_BIND_STMTS || gimple_code (stmt) == GIMPLE_PHI)
238 67900 : return gsi_remove (gsi, true);
239 :
240 99267 : gimple_stmt_iterator prev = *gsi;
241 99267 : gsi_prev (&prev);
242 99267 : unsigned uid = gimple_uid (stmt);
243 99267 : basic_block bb = gimple_bb (stmt);
244 99267 : bool ret = gsi_remove (gsi, true);
245 99267 : if (!gsi_end_p (prev))
246 99051 : gsi_next (&prev);
247 : else
248 432 : prev = gsi_start_bb (bb);
249 99267 : gimple *end_stmt = gsi_stmt (*gsi);
250 103907 : while ((stmt = gsi_stmt (prev)) != end_stmt)
251 : {
252 4640 : gcc_assert (stmt && is_gimple_debug (stmt) && gimple_uid (stmt) == 0);
253 4640 : gimple_set_uid (stmt, uid);
254 4640 : gsi_next (&prev);
255 : }
256 : return ret;
257 : }
258 :
259 : /* Bias amount for loop-carried phis. We want this to be larger than
260 : the depth of any reassociation tree we can see, but not larger than
261 : the rank difference between two blocks. */
262 : #define PHI_LOOP_BIAS (1 << 15)
263 :
264 : /* Return TRUE iff PHI_LOOP_BIAS should be propagated from one of the STMT's
265 : operands to the STMT's left-hand side. The goal is to preserve bias in code
266 : like this:
267 :
268 : x_1 = phi(x_0, x_2)
269 : a = x_1 | 1
270 : b = a ^ 2
271 : .MEM = b
272 : c = b + d
273 : x_2 = c + e
274 :
275 : That is, we need to preserve bias along single-use chains originating from
276 : loop-carried phis. Only GIMPLE_ASSIGNs to SSA_NAMEs are considered to be
277 : uses, because only they participate in rank propagation. */
278 : static bool
279 7175186 : propagate_bias_p (gimple *stmt)
280 : {
281 7175186 : use_operand_p use;
282 7175186 : imm_use_iterator use_iter;
283 7175186 : gimple *single_use_stmt = NULL;
284 :
285 7175186 : if (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt)) == tcc_reference)
286 : return false;
287 :
288 12791508 : FOR_EACH_IMM_USE_FAST (use, use_iter, gimple_assign_lhs (stmt))
289 : {
290 8146293 : gimple *current_use_stmt = USE_STMT (use);
291 :
292 8146293 : if (is_gimple_assign (current_use_stmt)
293 8146293 : && TREE_CODE (gimple_assign_lhs (current_use_stmt)) == SSA_NAME)
294 : {
295 6130373 : if (single_use_stmt != NULL && single_use_stmt != current_use_stmt)
296 734418 : return false;
297 : single_use_stmt = current_use_stmt;
298 : }
299 734418 : }
300 :
301 4645215 : if (single_use_stmt == NULL)
302 : return false;
303 :
304 4644794 : if (gimple_bb (stmt)->loop_father
305 4644794 : != gimple_bb (single_use_stmt)->loop_father)
306 116983 : return false;
307 :
308 : return true;
309 : }
310 :
311 : /* Rank assigned to a phi statement. If STMT is a loop-carried phi of
312 : an innermost loop, and the phi has only a single use which is inside
313 : the loop, then the rank is the block rank of the loop latch plus an
314 : extra bias for the loop-carried dependence. This causes expressions
315 : calculated into an accumulator variable to be independent for each
316 : iteration of the loop. If STMT is some other phi, the rank is the
317 : block rank of its containing block. */
318 : static int64_t
319 1494648 : phi_rank (gimple *stmt)
320 : {
321 1494648 : basic_block bb = gimple_bb (stmt);
322 1494648 : class loop *father = bb->loop_father;
323 1494648 : tree res;
324 1494648 : unsigned i;
325 1494648 : use_operand_p use;
326 1494648 : gimple *use_stmt;
327 :
328 1494648 : if (!reassoc_bias_loop_carried_phi_ranks_p)
329 569203 : return bb_rank[bb->index];
330 :
331 : /* We only care about real loops (those with a latch). */
332 925445 : if (!father->latch)
333 1 : return bb_rank[bb->index];
334 :
335 : /* Interesting phis must be in headers of innermost loops. */
336 925444 : if (bb != father->header
337 721806 : || father->inner)
338 368798 : return bb_rank[bb->index];
339 :
340 : /* Ignore virtual SSA_NAMEs. */
341 556646 : res = gimple_phi_result (stmt);
342 1113292 : if (virtual_operand_p (res))
343 0 : return bb_rank[bb->index];
344 :
345 : /* The phi definition must have a single use, and that use must be
346 : within the loop. Otherwise this isn't an accumulator pattern. */
347 556646 : if (!single_imm_use (res, &use, &use_stmt)
348 556646 : || gimple_bb (use_stmt)->loop_father != father)
349 484451 : return bb_rank[bb->index];
350 :
351 : /* Look for phi arguments from within the loop. If found, bias this phi. */
352 82047 : for (i = 0; i < gimple_phi_num_args (stmt); i++)
353 : {
354 81774 : tree arg = gimple_phi_arg_def (stmt, i);
355 81774 : if (TREE_CODE (arg) == SSA_NAME
356 81774 : && !SSA_NAME_IS_DEFAULT_DEF (arg))
357 : {
358 76514 : gimple *def_stmt = SSA_NAME_DEF_STMT (arg);
359 76514 : if (gimple_bb (def_stmt)->loop_father == father)
360 71922 : return bb_rank[father->latch->index] + PHI_LOOP_BIAS;
361 : }
362 : }
363 :
364 : /* Must be an uninteresting phi. */
365 273 : return bb_rank[bb->index];
366 : }
367 :
368 : /* Return the maximum of RANK and the rank that should be propagated
369 : from expression OP. For most operands, this is just the rank of OP.
370 : For loop-carried phis, the value is zero to avoid undoing the bias
371 : in favor of the phi. */
372 : static int64_t
373 7799521 : propagate_rank (int64_t rank, tree op, bool *maybe_biased_p)
374 : {
375 7799521 : int64_t op_rank;
376 :
377 7799521 : op_rank = get_rank (op);
378 :
379 : /* Check whether op is biased after the get_rank () call, since it might have
380 : updated biased_names. */
381 7799521 : if (TREE_CODE (op) == SSA_NAME
382 7799521 : && bitmap_bit_p (biased_names, SSA_NAME_VERSION (op)))
383 : {
384 48830 : if (maybe_biased_p == NULL)
385 : return rank;
386 34356 : *maybe_biased_p = true;
387 : }
388 :
389 7785047 : return MAX (rank, op_rank);
390 : }
391 :
392 : /* Look up the operand rank structure for expression E. */
393 :
394 : static inline int64_t
395 14211610 : find_operand_rank (tree e)
396 : {
397 14211610 : int64_t *slot = operand_rank->get (e);
398 14211610 : return slot ? *slot : -1;
399 : }
400 :
401 : /* Insert {E,RANK} into the operand rank hashtable. */
402 :
403 : static inline void
404 15257792 : insert_operand_rank (tree e, int64_t rank)
405 : {
406 15257792 : gcc_assert (rank > 0);
407 15257792 : bool existed = operand_rank->put (e, rank);
408 15257792 : gcc_assert (!existed);
409 15257792 : }
410 :
411 : /* Given an expression E, return the rank of the expression. */
412 :
413 : static int64_t
414 17711040 : get_rank (tree e)
415 : {
416 : /* SSA_NAME's have the rank of the expression they are the result
417 : of.
418 : For globals and uninitialized values, the rank is 0.
419 : For function arguments, use the pre-setup rank.
420 : For PHI nodes, stores, asm statements, etc, we use the rank of
421 : the BB.
422 : For simple operations, the rank is the maximum rank of any of
423 : its operands, or the bb_rank, whichever is less.
424 : I make no claims that this is optimal, however, it gives good
425 : results. */
426 :
427 : /* We make an exception to the normal ranking system to break
428 : dependences of accumulator variables in loops. Suppose we
429 : have a simple one-block loop containing:
430 :
431 : x_1 = phi(x_0, x_2)
432 : b = a + x_1
433 : c = b + d
434 : x_2 = c + e
435 :
436 : As shown, each iteration of the calculation into x is fully
437 : dependent upon the iteration before it. We would prefer to
438 : see this in the form:
439 :
440 : x_1 = phi(x_0, x_2)
441 : b = a + d
442 : c = b + e
443 : x_2 = c + x_1
444 :
445 : If the loop is unrolled, the calculations of b and c from
446 : different iterations can be interleaved.
447 :
448 : To obtain this result during reassociation, we bias the rank
449 : of the phi definition x_1 upward, when it is recognized as an
450 : accumulator pattern. The artificial rank causes it to be
451 : added last, providing the desired independence. */
452 :
453 17711040 : if (TREE_CODE (e) == SSA_NAME)
454 : {
455 14211610 : ssa_op_iter iter;
456 14211610 : gimple *stmt;
457 14211610 : int64_t rank;
458 14211610 : tree op;
459 :
460 : /* If we already have a rank for this expression, use that. */
461 14211610 : rank = find_operand_rank (e);
462 14211610 : if (rank != -1)
463 : return rank;
464 :
465 8925607 : stmt = SSA_NAME_DEF_STMT (e);
466 8925607 : if (gimple_code (stmt) == GIMPLE_PHI)
467 : {
468 1494648 : rank = phi_rank (stmt);
469 1494648 : if (rank != bb_rank[gimple_bb (stmt)->index])
470 71922 : bitmap_set_bit (biased_names, SSA_NAME_VERSION (e));
471 : }
472 :
473 7430959 : else if (!is_gimple_assign (stmt))
474 255773 : rank = bb_rank[gimple_bb (stmt)->index];
475 :
476 : else
477 : {
478 7175186 : bool biased_p = false;
479 7175186 : bool *maybe_biased_p = propagate_bias_p (stmt) ? &biased_p : NULL;
480 :
481 : /* Otherwise, find the maximum rank for the operands. As an
482 : exception, remove the bias from loop-carried phis when propagating
483 : the rank so that dependent operations are not also biased. */
484 : /* Simply walk over all SSA uses - this takes advantage of the
485 : fact that non-SSA operands are is_gimple_min_invariant and
486 : thus have rank 0. */
487 7175186 : rank = 0;
488 14974707 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
489 7799521 : rank = propagate_rank (rank, op, maybe_biased_p);
490 :
491 7175186 : rank += 1;
492 7175186 : if (biased_p)
493 33061 : bitmap_set_bit (biased_names, SSA_NAME_VERSION (e));
494 : }
495 :
496 8925607 : if (dump_file && (dump_flags & TDF_DETAILS))
497 : {
498 205 : fprintf (dump_file, "Rank for ");
499 205 : print_generic_expr (dump_file, e);
500 205 : fprintf (dump_file, " is %" PRId64 "\n", rank);
501 : }
502 :
503 : /* Note the rank in the hashtable so we don't recompute it. */
504 8925607 : insert_operand_rank (e, rank);
505 8925607 : return rank;
506 : }
507 :
508 : /* Constants, globals, etc., are rank 0 */
509 : return 0;
510 : }
511 :
512 :
513 : /* We want integer ones to end up last no matter what, since they are
514 : the ones we can do the most with. */
515 : #define INTEGER_CONST_TYPE 1 << 4
516 : #define FLOAT_ONE_CONST_TYPE 1 << 3
517 : #define FLOAT_CONST_TYPE 1 << 2
518 : #define OTHER_CONST_TYPE 1 << 1
519 :
520 : /* Classify an invariant tree into integer, float, or other, so that
521 : we can sort them to be near other constants of the same type. */
522 : static inline int
523 320390 : constant_type (tree t)
524 : {
525 320390 : if (INTEGRAL_TYPE_P (TREE_TYPE (t)))
526 : return INTEGER_CONST_TYPE;
527 9070 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (t)))
528 : {
529 : /* Sort -1.0 and 1.0 constants last, while in some cases
530 : const_binop can't optimize some inexact operations, multiplication
531 : by -1.0 or 1.0 can be always merged with others. */
532 6572 : if (real_onep (t) || real_minus_onep (t))
533 816 : return FLOAT_ONE_CONST_TYPE;
534 : return FLOAT_CONST_TYPE;
535 : }
536 : else
537 : return OTHER_CONST_TYPE;
538 : }
539 :
540 : /* qsort comparison function to sort operand entries PA and PB by rank
541 : so that the sorted array is ordered by rank in decreasing order. */
542 : static int
543 23727896 : sort_by_operand_rank (const void *pa, const void *pb)
544 : {
545 23727896 : const operand_entry *oea = *(const operand_entry *const *)pa;
546 23727896 : const operand_entry *oeb = *(const operand_entry *const *)pb;
547 :
548 23727896 : if (oeb->rank != oea->rank)
549 21311924 : return oeb->rank > oea->rank ? 1 : -1;
550 :
551 : /* It's nicer for optimize_expression if constants that are likely
552 : to fold when added/multiplied/whatever are put next to each
553 : other. Since all constants have rank 0, order them by type. */
554 2415972 : if (oea->rank == 0)
555 : {
556 160137 : if (constant_type (oeb->op) != constant_type (oea->op))
557 58 : return constant_type (oea->op) - constant_type (oeb->op);
558 : else
559 : /* To make sorting result stable, we use unique IDs to determine
560 : order. */
561 160079 : return oeb->id > oea->id ? 1 : -1;
562 : }
563 :
564 2255835 : if (TREE_CODE (oea->op) != SSA_NAME)
565 : {
566 0 : if (TREE_CODE (oeb->op) != SSA_NAME)
567 0 : return oeb->id > oea->id ? 1 : -1;
568 : else
569 : return 1;
570 : }
571 2255835 : else if (TREE_CODE (oeb->op) != SSA_NAME)
572 : return -1;
573 :
574 : /* Lastly, make sure the versions that are the same go next to each
575 : other. */
576 2255835 : if (SSA_NAME_VERSION (oeb->op) != SSA_NAME_VERSION (oea->op))
577 : {
578 : /* As SSA_NAME_VERSION is assigned pretty randomly, because we reuse
579 : versions of removed SSA_NAMEs, so if possible, prefer to sort
580 : based on basic block and gimple_uid of the SSA_NAME_DEF_STMT.
581 : See PR60418. */
582 2200193 : gimple *stmta = SSA_NAME_DEF_STMT (oea->op);
583 2200193 : gimple *stmtb = SSA_NAME_DEF_STMT (oeb->op);
584 2200193 : basic_block bba = gimple_bb (stmta);
585 2200193 : basic_block bbb = gimple_bb (stmtb);
586 2200193 : if (bbb != bba)
587 : {
588 : /* One of the SSA_NAMEs can be defined in oeN->stmt_to_insert
589 : but the other might not. */
590 142441 : if (!bba)
591 : return 1;
592 138109 : if (!bbb)
593 : return -1;
594 : /* If neither is, compare bb_rank. */
595 132120 : if (bb_rank[bbb->index] != bb_rank[bba->index])
596 132120 : return (bb_rank[bbb->index] >> 16) - (bb_rank[bba->index] >> 16);
597 : }
598 :
599 2057752 : bool da = reassoc_stmt_dominates_stmt_p (stmta, stmtb);
600 2057752 : bool db = reassoc_stmt_dominates_stmt_p (stmtb, stmta);
601 2057752 : if (da != db)
602 2002747 : return da ? 1 : -1;
603 :
604 55005 : return SSA_NAME_VERSION (oeb->op) > SSA_NAME_VERSION (oea->op) ? 1 : -1;
605 : }
606 :
607 55642 : return oeb->id > oea->id ? 1 : -1;
608 : }
609 :
610 : /* Add an operand entry to *OPS for the tree operand OP. */
611 :
612 : static void
613 9911021 : add_to_ops_vec (vec<operand_entry *> *ops, tree op, gimple *stmt_to_insert = NULL)
614 : {
615 9911021 : operand_entry *oe = operand_entry_pool.allocate ();
616 :
617 9911021 : oe->op = op;
618 9911021 : oe->rank = get_rank (op);
619 9911021 : oe->id = next_operand_entry_id++;
620 9911021 : oe->count = 1;
621 9911021 : oe->stmt_to_insert = stmt_to_insert;
622 9911021 : ops->safe_push (oe);
623 9911021 : }
624 :
625 : /* Add an operand entry to *OPS for the tree operand OP with repeat
626 : count REPEAT. */
627 :
628 : static void
629 18 : add_repeat_to_ops_vec (vec<operand_entry *> *ops, tree op,
630 : HOST_WIDE_INT repeat)
631 : {
632 18 : operand_entry *oe = operand_entry_pool.allocate ();
633 :
634 18 : oe->op = op;
635 18 : oe->rank = get_rank (op);
636 18 : oe->id = next_operand_entry_id++;
637 18 : oe->count = repeat;
638 18 : oe->stmt_to_insert = NULL;
639 18 : ops->safe_push (oe);
640 :
641 18 : reassociate_stats.pows_encountered++;
642 18 : }
643 :
644 : /* Returns true if we can associate the SSA def OP. */
645 :
646 : static bool
647 32334488 : can_reassociate_op_p (tree op)
648 : {
649 32334488 : if (TREE_CODE (op) == SSA_NAME && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op))
650 : return false;
651 : /* Uninitialized variables can't participate in reassociation. */
652 32333673 : if (TREE_CODE (op) == SSA_NAME && ssa_name_maybe_undef_p (op))
653 : return false;
654 : /* Make sure asm goto outputs do not participate in reassociation since
655 : we have no way to find an insertion place after asm goto. */
656 32328780 : if (TREE_CODE (op) == SSA_NAME
657 23820289 : && gimple_code (SSA_NAME_DEF_STMT (op)) == GIMPLE_ASM
658 32351403 : && gimple_asm_nlabels (as_a <gasm *> (SSA_NAME_DEF_STMT (op))) != 0)
659 90 : return false;
660 : return true;
661 : }
662 :
663 : /* Returns true if we can reassociate operations of TYPE.
664 : That is for integral or non-saturating fixed-point types, and for
665 : floating point type when associative-math is enabled. */
666 :
667 : static bool
668 59432362 : can_reassociate_type_p (tree type)
669 : {
670 59432362 : if ((ANY_INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_WRAPS (type))
671 36942003 : || NON_SAT_FIXED_POINT_TYPE_P (type)
672 96374365 : || (flag_associative_math && FLOAT_TYPE_P (type)))
673 22880674 : return true;
674 : return false;
675 : }
676 :
677 : /* Return true if STMT is reassociable operation containing a binary
678 : operation with tree code CODE, and is inside LOOP. */
679 :
680 : static bool
681 7880093 : is_reassociable_op (gimple *stmt, enum tree_code code, class loop *loop)
682 : {
683 7880093 : basic_block bb = gimple_bb (stmt);
684 :
685 7880093 : if (gimple_bb (stmt) == NULL)
686 : return false;
687 :
688 7692341 : if (!flow_bb_inside_loop_p (loop, bb))
689 : return false;
690 :
691 7470952 : if (is_gimple_assign (stmt)
692 5947292 : && gimple_assign_rhs_code (stmt) == code
693 8294569 : && has_single_use (gimple_assign_lhs (stmt)))
694 : {
695 588731 : tree rhs1 = gimple_assign_rhs1 (stmt);
696 588731 : tree rhs2 = gimple_assign_rhs2 (stmt);
697 588731 : if (!can_reassociate_op_p (rhs1)
698 588731 : || (rhs2 && !can_reassociate_op_p (rhs2)))
699 521 : return false;
700 : return true;
701 : }
702 :
703 : return false;
704 : }
705 :
706 :
707 : /* Return true if STMT is a nop-conversion. */
708 :
709 : static bool
710 7847845 : gimple_nop_conversion_p (gimple *stmt)
711 : {
712 7847845 : if (gassign *ass = dyn_cast <gassign *> (stmt))
713 : {
714 9252806 : if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (ass))
715 6951106 : && tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (ass)),
716 1549802 : TREE_TYPE (gimple_assign_rhs1 (ass))))
717 : return true;
718 : }
719 : return false;
720 : }
721 :
722 : /* Given NAME, if NAME is defined by a unary operation OPCODE, return the
723 : operand of the negate operation. Otherwise, return NULL. */
724 :
725 : static tree
726 7751584 : get_unary_op (tree name, enum tree_code opcode)
727 : {
728 7751584 : gimple *stmt = SSA_NAME_DEF_STMT (name);
729 :
730 : /* Look through nop conversions (sign changes). */
731 7751584 : if (gimple_nop_conversion_p (stmt)
732 7751584 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME)
733 932424 : stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
734 :
735 7751584 : if (!is_gimple_assign (stmt))
736 : return NULL_TREE;
737 :
738 4939602 : if (gimple_assign_rhs_code (stmt) == opcode)
739 130249 : return gimple_assign_rhs1 (stmt);
740 : return NULL_TREE;
741 : }
742 :
743 : /* Return true if OP1 and OP2 have the same value if casted to either type. */
744 :
745 : static bool
746 49304 : ops_equal_values_p (tree op1, tree op2)
747 : {
748 49304 : if (op1 == op2)
749 : return true;
750 :
751 49085 : tree orig_op1 = op1;
752 49085 : if (TREE_CODE (op1) == SSA_NAME)
753 : {
754 49085 : gimple *stmt = SSA_NAME_DEF_STMT (op1);
755 49085 : if (gimple_nop_conversion_p (stmt))
756 : {
757 21648 : op1 = gimple_assign_rhs1 (stmt);
758 21648 : if (op1 == op2)
759 : return true;
760 : }
761 : }
762 :
763 47176 : if (TREE_CODE (op2) == SSA_NAME)
764 : {
765 47176 : gimple *stmt = SSA_NAME_DEF_STMT (op2);
766 47176 : if (gimple_nop_conversion_p (stmt))
767 : {
768 19861 : op2 = gimple_assign_rhs1 (stmt);
769 19861 : if (op1 == op2
770 19861 : || orig_op1 == op2)
771 : return true;
772 : }
773 : }
774 :
775 : return false;
776 : }
777 :
778 :
779 : /* If CURR and LAST are a pair of ops that OPCODE allows us to
780 : eliminate through equivalences, do so, remove them from OPS, and
781 : return true. Otherwise, return false. */
782 :
783 : static bool
784 9803687 : eliminate_duplicate_pair (enum tree_code opcode,
785 : vec<operand_entry *> *ops,
786 : bool *all_done,
787 : unsigned int i,
788 : operand_entry *curr,
789 : operand_entry *last)
790 : {
791 :
792 : /* If we have two of the same op, and the opcode is & |, min, or max,
793 : we can eliminate one of them.
794 : If we have two of the same op, and the opcode is ^, we can
795 : eliminate both of them. */
796 :
797 9803687 : if (last && last->op == curr->op)
798 : {
799 5575 : switch (opcode)
800 : {
801 23 : case MAX_EXPR:
802 23 : case MIN_EXPR:
803 23 : case BIT_IOR_EXPR:
804 23 : case BIT_AND_EXPR:
805 23 : if (dump_file && (dump_flags & TDF_DETAILS))
806 : {
807 1 : fprintf (dump_file, "Equivalence: ");
808 1 : print_generic_expr (dump_file, curr->op);
809 1 : fprintf (dump_file, " [&|minmax] ");
810 1 : print_generic_expr (dump_file, last->op);
811 1 : fprintf (dump_file, " -> ");
812 1 : print_generic_stmt (dump_file, last->op);
813 : }
814 :
815 23 : ops->ordered_remove (i);
816 23 : reassociate_stats.ops_eliminated ++;
817 :
818 23 : return true;
819 :
820 122 : case BIT_XOR_EXPR:
821 122 : if (dump_file && (dump_flags & TDF_DETAILS))
822 : {
823 0 : fprintf (dump_file, "Equivalence: ");
824 0 : print_generic_expr (dump_file, curr->op);
825 0 : fprintf (dump_file, " ^ ");
826 0 : print_generic_expr (dump_file, last->op);
827 0 : fprintf (dump_file, " -> nothing\n");
828 : }
829 :
830 122 : reassociate_stats.ops_eliminated += 2;
831 :
832 122 : if (ops->length () == 2)
833 : {
834 1 : ops->truncate (0);
835 1 : add_to_ops_vec (ops, build_zero_cst (TREE_TYPE (last->op)));
836 1 : *all_done = true;
837 : }
838 : else
839 : {
840 121 : ops->ordered_remove (i-1);
841 121 : ops->ordered_remove (i-1);
842 : }
843 :
844 : return true;
845 :
846 : default:
847 : break;
848 : }
849 : }
850 : return false;
851 : }
852 :
853 : static vec<tree> plus_negates;
854 :
855 : /* If OPCODE is PLUS_EXPR, CURR->OP is a negate expression or a bitwise not
856 : expression, look in OPS for a corresponding positive operation to cancel
857 : it out. If we find one, remove the other from OPS, replace
858 : OPS[CURRINDEX] with 0 or -1, respectively, and return true. Otherwise,
859 : return false. */
860 :
861 : static bool
862 9803542 : eliminate_plus_minus_pair (enum tree_code opcode,
863 : vec<operand_entry *> *ops,
864 : unsigned int currindex,
865 : operand_entry *curr)
866 : {
867 9803542 : tree negateop;
868 9803542 : tree notop;
869 9803542 : unsigned int i;
870 9803542 : operand_entry *oe;
871 :
872 9803542 : if (opcode != PLUS_EXPR || TREE_CODE (curr->op) != SSA_NAME)
873 : return false;
874 :
875 3106454 : negateop = get_unary_op (curr->op, NEGATE_EXPR);
876 3106454 : notop = get_unary_op (curr->op, BIT_NOT_EXPR);
877 3106454 : if (negateop == NULL_TREE && notop == NULL_TREE)
878 : return false;
879 :
880 : /* Any non-negated version will have a rank that is one less than
881 : the current rank. So once we hit those ranks, if we don't find
882 : one, we can stop. */
883 :
884 140278 : for (i = currindex + 1;
885 204736 : ops->iterate (i, &oe)
886 254040 : && oe->rank >= curr->rank - 1 ;
887 : i++)
888 : {
889 49304 : if (negateop
890 49304 : && ops_equal_values_p (oe->op, negateop))
891 : {
892 1699 : if (dump_file && (dump_flags & TDF_DETAILS))
893 : {
894 0 : fprintf (dump_file, "Equivalence: ");
895 0 : print_generic_expr (dump_file, negateop);
896 0 : fprintf (dump_file, " + -");
897 0 : print_generic_expr (dump_file, oe->op);
898 0 : fprintf (dump_file, " -> 0\n");
899 : }
900 :
901 1699 : ops->ordered_remove (i);
902 1699 : add_to_ops_vec (ops, build_zero_cst (TREE_TYPE (oe->op)));
903 1699 : ops->ordered_remove (currindex);
904 1699 : reassociate_stats.ops_eliminated ++;
905 :
906 1699 : return true;
907 : }
908 47605 : else if (notop
909 47605 : && ops_equal_values_p (oe->op, notop))
910 : {
911 1901 : tree op_type = TREE_TYPE (oe->op);
912 :
913 1901 : if (dump_file && (dump_flags & TDF_DETAILS))
914 : {
915 0 : fprintf (dump_file, "Equivalence: ");
916 0 : print_generic_expr (dump_file, notop);
917 0 : fprintf (dump_file, " + ~");
918 0 : print_generic_expr (dump_file, oe->op);
919 0 : fprintf (dump_file, " -> -1\n");
920 : }
921 :
922 1901 : ops->ordered_remove (i);
923 1901 : add_to_ops_vec (ops, build_all_ones_cst (op_type));
924 1901 : ops->ordered_remove (currindex);
925 1901 : reassociate_stats.ops_eliminated ++;
926 :
927 1901 : return true;
928 : }
929 : }
930 :
931 : /* If CURR->OP is a negate expr without nop conversion in a plus expr:
932 : save it for later inspection in repropagate_negates(). */
933 90974 : if (negateop != NULL_TREE
934 90974 : && gimple_assign_rhs_code (SSA_NAME_DEF_STMT (curr->op)) == NEGATE_EXPR)
935 90465 : plus_negates.safe_push (curr->op);
936 :
937 : return false;
938 : }
939 :
940 : /* If OPCODE is BIT_IOR_EXPR, BIT_AND_EXPR, and, CURR->OP is really a
941 : bitwise not expression, look in OPS for a corresponding operand to
942 : cancel it out. If we find one, remove the other from OPS, replace
943 : OPS[CURRINDEX] with 0, and return true. Otherwise, return
944 : false. */
945 :
946 : static bool
947 9803688 : eliminate_not_pairs (enum tree_code opcode,
948 : vec<operand_entry *> *ops,
949 : unsigned int currindex,
950 : operand_entry *curr)
951 : {
952 9803688 : tree notop;
953 9803688 : unsigned int i;
954 9803688 : operand_entry *oe;
955 :
956 9803688 : if ((opcode != BIT_IOR_EXPR && opcode != BIT_AND_EXPR)
957 2038678 : || TREE_CODE (curr->op) != SSA_NAME)
958 : return false;
959 :
960 1538676 : notop = get_unary_op (curr->op, BIT_NOT_EXPR);
961 1538676 : if (notop == NULL_TREE)
962 : return false;
963 :
964 : /* Any non-not version will have a rank that is one less than
965 : the current rank. So once we hit those ranks, if we don't find
966 : one, we can stop. */
967 :
968 35675 : for (i = currindex + 1;
969 9831937 : ops->iterate (i, &oe)
970 71396 : && oe->rank >= curr->rank - 1;
971 : i++)
972 : {
973 7473 : if (oe->op == notop)
974 : {
975 1 : if (dump_file && (dump_flags & TDF_DETAILS))
976 : {
977 0 : fprintf (dump_file, "Equivalence: ");
978 0 : print_generic_expr (dump_file, notop);
979 0 : if (opcode == BIT_AND_EXPR)
980 0 : fprintf (dump_file, " & ~");
981 0 : else if (opcode == BIT_IOR_EXPR)
982 0 : fprintf (dump_file, " | ~");
983 0 : print_generic_expr (dump_file, oe->op);
984 0 : if (opcode == BIT_AND_EXPR)
985 0 : fprintf (dump_file, " -> 0\n");
986 0 : else if (opcode == BIT_IOR_EXPR)
987 0 : fprintf (dump_file, " -> -1\n");
988 : }
989 :
990 1 : if (opcode == BIT_AND_EXPR)
991 1 : oe->op = build_zero_cst (TREE_TYPE (oe->op));
992 0 : else if (opcode == BIT_IOR_EXPR)
993 0 : oe->op = build_all_ones_cst (TREE_TYPE (oe->op));
994 :
995 1 : reassociate_stats.ops_eliminated += ops->length () - 1;
996 1 : ops->truncate (0);
997 1 : ops->quick_push (oe);
998 1 : return true;
999 : }
1000 : }
1001 :
1002 : return false;
1003 : }
1004 :
1005 : /* Use constant value that may be present in OPS to try to eliminate
1006 : operands. Note that this function is only really used when we've
1007 : eliminated ops for other reasons, or merged constants. Across
1008 : single statements, fold already does all of this, plus more. There
1009 : is little point in duplicating logic, so I've only included the
1010 : identities that I could ever construct testcases to trigger. */
1011 :
1012 : static void
1013 4779846 : eliminate_using_constants (enum tree_code opcode,
1014 : vec<operand_entry *> *ops)
1015 : {
1016 4779846 : operand_entry *oelast = ops->last ();
1017 4779846 : tree type = TREE_TYPE (oelast->op);
1018 :
1019 4779846 : if (oelast->rank == 0
1020 4779846 : && (ANY_INTEGRAL_TYPE_P (type) || FLOAT_TYPE_P (type)))
1021 : {
1022 3458302 : switch (opcode)
1023 : {
1024 426369 : case BIT_AND_EXPR:
1025 426369 : if (integer_zerop (oelast->op))
1026 : {
1027 0 : if (ops->length () != 1)
1028 : {
1029 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1030 0 : fprintf (dump_file, "Found & 0, removing all other ops\n");
1031 :
1032 0 : reassociate_stats.ops_eliminated += ops->length () - 1;
1033 :
1034 0 : ops->truncate (0);
1035 0 : ops->quick_push (oelast);
1036 1951 : return;
1037 : }
1038 : }
1039 426369 : else if (integer_all_onesp (oelast->op))
1040 : {
1041 4 : if (ops->length () != 1)
1042 : {
1043 4 : if (dump_file && (dump_flags & TDF_DETAILS))
1044 0 : fprintf (dump_file, "Found & -1, removing\n");
1045 4 : ops->pop ();
1046 4 : reassociate_stats.ops_eliminated++;
1047 : }
1048 : }
1049 : break;
1050 73120 : case BIT_IOR_EXPR:
1051 73120 : if (integer_all_onesp (oelast->op))
1052 : {
1053 0 : if (ops->length () != 1)
1054 : {
1055 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1056 0 : fprintf (dump_file, "Found | -1, removing all other ops\n");
1057 :
1058 0 : reassociate_stats.ops_eliminated += ops->length () - 1;
1059 :
1060 0 : ops->truncate (0);
1061 0 : ops->quick_push (oelast);
1062 0 : return;
1063 : }
1064 : }
1065 73120 : else if (integer_zerop (oelast->op))
1066 : {
1067 4 : if (ops->length () != 1)
1068 : {
1069 4 : if (dump_file && (dump_flags & TDF_DETAILS))
1070 0 : fprintf (dump_file, "Found | 0, removing\n");
1071 4 : ops->pop ();
1072 4 : reassociate_stats.ops_eliminated++;
1073 : }
1074 : }
1075 : break;
1076 958109 : case MULT_EXPR:
1077 958109 : if (integer_zerop (oelast->op)
1078 958109 : || (FLOAT_TYPE_P (type)
1079 1430 : && !HONOR_NANS (type)
1080 1335 : && !HONOR_SIGNED_ZEROS (type)
1081 1335 : && real_zerop (oelast->op)))
1082 : {
1083 0 : if (ops->length () != 1)
1084 : {
1085 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1086 0 : fprintf (dump_file, "Found * 0, removing all other ops\n");
1087 :
1088 0 : reassociate_stats.ops_eliminated += ops->length () - 1;
1089 0 : ops->truncate (0);
1090 0 : ops->quick_push (oelast);
1091 0 : return;
1092 : }
1093 : }
1094 958109 : else if (integer_onep (oelast->op)
1095 958109 : || (FLOAT_TYPE_P (type)
1096 1430 : && !HONOR_SNANS (type)
1097 1430 : && real_onep (oelast->op)))
1098 : {
1099 4 : if (ops->length () != 1)
1100 : {
1101 4 : if (dump_file && (dump_flags & TDF_DETAILS))
1102 0 : fprintf (dump_file, "Found * 1, removing\n");
1103 4 : ops->pop ();
1104 4 : reassociate_stats.ops_eliminated++;
1105 4 : return;
1106 : }
1107 : }
1108 : break;
1109 1826334 : case BIT_XOR_EXPR:
1110 1826334 : case PLUS_EXPR:
1111 1826334 : case MINUS_EXPR:
1112 1826334 : if (integer_zerop (oelast->op)
1113 1826334 : || (FLOAT_TYPE_P (type)
1114 849 : && (opcode == PLUS_EXPR || opcode == MINUS_EXPR)
1115 849 : && fold_real_zero_addition_p (type, 0, oelast->op,
1116 : opcode == MINUS_EXPR)))
1117 : {
1118 1947 : if (ops->length () != 1)
1119 : {
1120 1947 : if (dump_file && (dump_flags & TDF_DETAILS))
1121 0 : fprintf (dump_file, "Found [|^+] 0, removing\n");
1122 1947 : ops->pop ();
1123 1947 : reassociate_stats.ops_eliminated++;
1124 1947 : return;
1125 : }
1126 : }
1127 : break;
1128 : default:
1129 : break;
1130 : }
1131 : }
1132 : }
1133 :
1134 :
1135 : static void linearize_expr_tree (vec<operand_entry *> *, gimple *,
1136 : bool, bool);
1137 :
1138 : /* Structure for tracking and counting operands. */
1139 : struct oecount {
1140 : unsigned int cnt;
1141 : unsigned int id;
1142 : enum tree_code oecode;
1143 : tree op;
1144 : };
1145 :
1146 :
1147 : /* The heap for the oecount hashtable and the sorted list of operands. */
1148 : static vec<oecount> cvec;
1149 :
1150 :
1151 : /* Oecount hashtable helpers. */
1152 :
1153 : struct oecount_hasher : int_hash <int, 0, 1>
1154 : {
1155 : static inline hashval_t hash (int);
1156 : static inline bool equal (int, int);
1157 : };
1158 :
1159 : /* Hash function for oecount. */
1160 :
1161 : inline hashval_t
1162 155797 : oecount_hasher::hash (int p)
1163 : {
1164 155797 : const oecount *c = &cvec[p - 42];
1165 155797 : return htab_hash_pointer (c->op) ^ (hashval_t)c->oecode;
1166 : }
1167 :
1168 : /* Comparison function for oecount. */
1169 :
1170 : inline bool
1171 84129 : oecount_hasher::equal (int p1, int p2)
1172 : {
1173 84129 : const oecount *c1 = &cvec[p1 - 42];
1174 84129 : const oecount *c2 = &cvec[p2 - 42];
1175 84129 : return c1->oecode == c2->oecode && c1->op == c2->op;
1176 : }
1177 :
1178 : /* Comparison function for qsort sorting oecount elements by count. */
1179 :
1180 : static int
1181 601697 : oecount_cmp (const void *p1, const void *p2)
1182 : {
1183 601697 : const oecount *c1 = (const oecount *)p1;
1184 601697 : const oecount *c2 = (const oecount *)p2;
1185 601697 : if (c1->cnt != c2->cnt)
1186 8956 : return c1->cnt > c2->cnt ? 1 : -1;
1187 : else
1188 : /* If counts are identical, use unique IDs to stabilize qsort. */
1189 592741 : return c1->id > c2->id ? 1 : -1;
1190 : }
1191 :
1192 : /* Return TRUE iff STMT represents a builtin call that raises OP
1193 : to some exponent. */
1194 :
1195 : static bool
1196 1286 : stmt_is_power_of_op (gimple *stmt, tree op)
1197 : {
1198 1286 : if (!is_gimple_call (stmt))
1199 : return false;
1200 :
1201 11 : switch (gimple_call_combined_fn (stmt))
1202 : {
1203 6 : CASE_CFN_POW:
1204 6 : CASE_CFN_POWI:
1205 6 : return (operand_equal_p (gimple_call_arg (stmt, 0), op, 0));
1206 :
1207 : default:
1208 : return false;
1209 : }
1210 : }
1211 :
1212 : /* Given STMT which is a __builtin_pow* call, decrement its exponent
1213 : in place and return the result. Assumes that stmt_is_power_of_op
1214 : was previously called for STMT and returned TRUE. */
1215 :
1216 : static HOST_WIDE_INT
1217 6 : decrement_power (gimple *stmt)
1218 : {
1219 6 : REAL_VALUE_TYPE c, cint;
1220 6 : HOST_WIDE_INT power;
1221 6 : tree arg1;
1222 :
1223 6 : switch (gimple_call_combined_fn (stmt))
1224 : {
1225 0 : CASE_CFN_POW:
1226 0 : arg1 = gimple_call_arg (stmt, 1);
1227 0 : c = TREE_REAL_CST (arg1);
1228 0 : power = real_to_integer (&c) - 1;
1229 0 : real_from_integer (&cint, VOIDmode, power, SIGNED);
1230 0 : gimple_call_set_arg (stmt, 1, build_real (TREE_TYPE (arg1), cint));
1231 0 : return power;
1232 :
1233 6 : CASE_CFN_POWI:
1234 6 : arg1 = gimple_call_arg (stmt, 1);
1235 6 : power = TREE_INT_CST_LOW (arg1) - 1;
1236 6 : gimple_call_set_arg (stmt, 1, build_int_cst (TREE_TYPE (arg1), power));
1237 6 : return power;
1238 :
1239 0 : default:
1240 0 : gcc_unreachable ();
1241 : }
1242 : }
1243 :
1244 : /* Replace SSA defined by STMT and replace all its uses with new
1245 : SSA. Also return the new SSA. */
1246 :
1247 : static tree
1248 352 : make_new_ssa_for_def (gimple *stmt, enum tree_code opcode, tree op)
1249 : {
1250 352 : gimple *use_stmt;
1251 352 : use_operand_p use;
1252 352 : imm_use_iterator iter;
1253 352 : tree new_lhs, new_debug_lhs = NULL_TREE;
1254 352 : tree lhs = gimple_get_lhs (stmt);
1255 :
1256 352 : new_lhs = make_ssa_name (TREE_TYPE (lhs));
1257 352 : gimple_set_lhs (stmt, new_lhs);
1258 :
1259 : /* Also need to update GIMPLE_DEBUGs. */
1260 761 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
1261 : {
1262 409 : tree repl = new_lhs;
1263 409 : if (is_gimple_debug (use_stmt))
1264 : {
1265 57 : if (new_debug_lhs == NULL_TREE)
1266 : {
1267 21 : new_debug_lhs = build_debug_expr_decl (TREE_TYPE (lhs));
1268 21 : gdebug *def_temp
1269 21 : = gimple_build_debug_bind (new_debug_lhs,
1270 21 : build2 (opcode, TREE_TYPE (lhs),
1271 : new_lhs, op),
1272 : stmt);
1273 21 : gimple_set_uid (def_temp, gimple_uid (stmt));
1274 21 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1275 21 : gsi_insert_after (&gsi, def_temp, GSI_SAME_STMT);
1276 : }
1277 : repl = new_debug_lhs;
1278 : }
1279 818 : FOR_EACH_IMM_USE_ON_STMT (use, iter)
1280 409 : SET_USE (use, repl);
1281 409 : update_stmt (use_stmt);
1282 352 : }
1283 352 : return new_lhs;
1284 : }
1285 :
1286 : /* Replace all SSAs defined in STMTS_TO_FIX and replace its
1287 : uses with new SSAs. Also do this for the stmt that defines DEF
1288 : if *DEF is not OP. */
1289 :
1290 : static void
1291 291 : make_new_ssa_for_all_defs (tree *def, enum tree_code opcode, tree op,
1292 : vec<gimple *> &stmts_to_fix)
1293 : {
1294 291 : unsigned i;
1295 291 : gimple *stmt;
1296 :
1297 291 : if (*def != op
1298 291 : && TREE_CODE (*def) == SSA_NAME
1299 291 : && (stmt = SSA_NAME_DEF_STMT (*def))
1300 582 : && gimple_code (stmt) != GIMPLE_NOP)
1301 291 : *def = make_new_ssa_for_def (stmt, opcode, op);
1302 :
1303 352 : FOR_EACH_VEC_ELT (stmts_to_fix, i, stmt)
1304 61 : make_new_ssa_for_def (stmt, opcode, op);
1305 291 : }
1306 :
1307 : /* Find the single immediate use of STMT's LHS, and replace it
1308 : with OP. Remove STMT. If STMT's LHS is the same as *DEF,
1309 : replace *DEF with OP as well. */
1310 :
1311 : static void
1312 881 : propagate_op_to_single_use (tree op, gimple *stmt, tree *def)
1313 : {
1314 881 : tree lhs;
1315 881 : gimple *use_stmt;
1316 881 : use_operand_p use;
1317 881 : gimple_stmt_iterator gsi;
1318 :
1319 881 : if (is_gimple_call (stmt))
1320 1 : lhs = gimple_call_lhs (stmt);
1321 : else
1322 880 : lhs = gimple_assign_lhs (stmt);
1323 :
1324 881 : gcc_assert (has_single_use (lhs));
1325 881 : single_imm_use (lhs, &use, &use_stmt);
1326 881 : if (lhs == *def)
1327 599 : *def = op;
1328 881 : SET_USE (use, op);
1329 881 : if (TREE_CODE (op) != SSA_NAME)
1330 40 : update_stmt (use_stmt);
1331 881 : gsi = gsi_for_stmt (stmt);
1332 881 : unlink_stmt_vdef (stmt);
1333 881 : reassoc_remove_stmt (&gsi);
1334 881 : release_defs (stmt);
1335 881 : }
1336 :
1337 : /* Walks the linear chain with result *DEF searching for an operation
1338 : with operand OP and code OPCODE removing that from the chain. *DEF
1339 : is updated if there is only one operand but no operation left. */
1340 :
1341 : static void
1342 890 : zero_one_operation (tree *def, enum tree_code opcode, tree op)
1343 : {
1344 890 : tree orig_def = *def;
1345 890 : gimple *stmt = SSA_NAME_DEF_STMT (*def);
1346 : /* PR72835 - Record the stmt chain that has to be updated such that
1347 : we dont use the same LHS when the values computed are different. */
1348 890 : auto_vec<gimple *, 64> stmts_to_fix;
1349 :
1350 1576 : do
1351 : {
1352 1233 : tree name;
1353 :
1354 1233 : if (opcode == MULT_EXPR)
1355 : {
1356 1231 : if (stmt_is_power_of_op (stmt, op))
1357 : {
1358 6 : if (decrement_power (stmt) == 1)
1359 : {
1360 1 : if (stmts_to_fix.length () > 0)
1361 1 : stmts_to_fix.pop ();
1362 1 : propagate_op_to_single_use (op, stmt, def);
1363 : }
1364 : break;
1365 : }
1366 1225 : else if (gimple_assign_rhs_code (stmt) == NEGATE_EXPR)
1367 : {
1368 15 : if (gimple_assign_rhs1 (stmt) == op)
1369 : {
1370 11 : tree cst = build_minus_one_cst (TREE_TYPE (op));
1371 11 : if (stmts_to_fix.length () > 0)
1372 11 : stmts_to_fix.pop ();
1373 11 : propagate_op_to_single_use (cst, stmt, def);
1374 11 : break;
1375 : }
1376 4 : else if (integer_minus_onep (op)
1377 4 : || real_minus_onep (op))
1378 : {
1379 4 : gimple_assign_set_rhs_code
1380 4 : (stmt, TREE_CODE (gimple_assign_rhs1 (stmt)));
1381 4 : break;
1382 : }
1383 : }
1384 : }
1385 :
1386 1212 : name = gimple_assign_rhs1 (stmt);
1387 :
1388 : /* If this is the operation we look for and one of the operands
1389 : is ours simply propagate the other operand into the stmts
1390 : single use. */
1391 1212 : if (gimple_assign_rhs_code (stmt) == opcode
1392 1212 : && (name == op
1393 952 : || gimple_assign_rhs2 (stmt) == op))
1394 : {
1395 869 : if (name == op)
1396 260 : name = gimple_assign_rhs2 (stmt);
1397 869 : if (stmts_to_fix.length () > 0)
1398 270 : stmts_to_fix.pop ();
1399 869 : propagate_op_to_single_use (name, stmt, def);
1400 869 : break;
1401 : }
1402 :
1403 : /* We might have a multiply of two __builtin_pow* calls, and
1404 : the operand might be hiding in the rightmost one. Likewise
1405 : this can happen for a negate. */
1406 343 : if (opcode == MULT_EXPR
1407 343 : && gimple_assign_rhs_code (stmt) == opcode
1408 343 : && TREE_CODE (gimple_assign_rhs2 (stmt)) == SSA_NAME
1409 621 : && has_single_use (gimple_assign_rhs2 (stmt)))
1410 : {
1411 55 : gimple *stmt2 = SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt));
1412 55 : if (stmt_is_power_of_op (stmt2, op))
1413 : {
1414 0 : if (decrement_power (stmt2) == 1)
1415 0 : propagate_op_to_single_use (op, stmt2, def);
1416 : else
1417 0 : stmts_to_fix.safe_push (stmt2);
1418 0 : break;
1419 : }
1420 55 : else if (is_gimple_assign (stmt2)
1421 55 : && gimple_assign_rhs_code (stmt2) == NEGATE_EXPR)
1422 : {
1423 0 : if (gimple_assign_rhs1 (stmt2) == op)
1424 : {
1425 0 : tree cst = build_minus_one_cst (TREE_TYPE (op));
1426 0 : propagate_op_to_single_use (cst, stmt2, def);
1427 0 : break;
1428 : }
1429 0 : else if (integer_minus_onep (op)
1430 0 : || real_minus_onep (op))
1431 : {
1432 0 : stmts_to_fix.safe_push (stmt2);
1433 0 : gimple_assign_set_rhs_code
1434 0 : (stmt2, TREE_CODE (gimple_assign_rhs1 (stmt2)));
1435 0 : break;
1436 : }
1437 : }
1438 : }
1439 :
1440 : /* Continue walking the chain. */
1441 343 : gcc_assert (name != op
1442 : && TREE_CODE (name) == SSA_NAME);
1443 343 : stmt = SSA_NAME_DEF_STMT (name);
1444 343 : stmts_to_fix.safe_push (stmt);
1445 343 : }
1446 : while (1);
1447 :
1448 890 : if (stmts_to_fix.length () > 0 || *def == orig_def)
1449 291 : make_new_ssa_for_all_defs (def, opcode, op, stmts_to_fix);
1450 890 : }
1451 :
1452 : /* Returns true if statement S1 dominates statement S2. Like
1453 : stmt_dominates_stmt_p, but uses stmt UIDs to optimize. */
1454 :
1455 : static bool
1456 4808208 : reassoc_stmt_dominates_stmt_p (gimple *s1, gimple *s2)
1457 : {
1458 4808208 : basic_block bb1 = gimple_bb (s1), bb2 = gimple_bb (s2);
1459 :
1460 : /* If bb1 is NULL, it should be a GIMPLE_NOP def stmt of an (D)
1461 : SSA_NAME. Assume it lives at the beginning of function and
1462 : thus dominates everything. */
1463 4808208 : if (!bb1 || s1 == s2)
1464 : return true;
1465 :
1466 : /* If bb2 is NULL, it doesn't dominate any stmt with a bb. */
1467 4805035 : if (!bb2)
1468 : return false;
1469 :
1470 4797479 : if (bb1 == bb2)
1471 : {
1472 : /* PHIs in the same basic block are assumed to be
1473 : executed all in parallel, if only one stmt is a PHI,
1474 : it dominates the other stmt in the same basic block. */
1475 4658499 : if (gimple_code (s1) == GIMPLE_PHI)
1476 : return true;
1477 :
1478 4544670 : if (gimple_code (s2) == GIMPLE_PHI)
1479 : return false;
1480 :
1481 4493185 : gcc_assert (gimple_uid (s1) && gimple_uid (s2));
1482 :
1483 4493185 : if (gimple_uid (s1) < gimple_uid (s2))
1484 : return true;
1485 :
1486 2464504 : if (gimple_uid (s1) > gimple_uid (s2))
1487 : return false;
1488 :
1489 37044 : gimple_stmt_iterator gsi = gsi_for_stmt (s1);
1490 37044 : unsigned int uid = gimple_uid (s1);
1491 78689 : for (gsi_next (&gsi); !gsi_end_p (gsi); gsi_next (&gsi))
1492 : {
1493 76886 : gimple *s = gsi_stmt (gsi);
1494 76886 : if (gimple_uid (s) != uid)
1495 : break;
1496 44892 : if (s == s2)
1497 : return true;
1498 : }
1499 :
1500 : return false;
1501 : }
1502 :
1503 138980 : return dominated_by_p (CDI_DOMINATORS, bb2, bb1);
1504 : }
1505 :
1506 : /* Insert STMT after INSERT_POINT. */
1507 :
1508 : static void
1509 38820 : insert_stmt_after (gimple *stmt, gimple *insert_point)
1510 : {
1511 38820 : gimple_stmt_iterator gsi;
1512 38820 : basic_block bb;
1513 :
1514 38820 : if (gimple_code (insert_point) == GIMPLE_PHI)
1515 30 : bb = gimple_bb (insert_point);
1516 38790 : else if (!stmt_ends_bb_p (insert_point))
1517 : {
1518 38782 : gsi = gsi_for_stmt (insert_point);
1519 38782 : gimple_set_uid (stmt, gimple_uid (insert_point));
1520 38782 : gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
1521 38782 : return;
1522 : }
1523 8 : else if (gimple_code (insert_point) == GIMPLE_ASM
1524 5 : && gimple_asm_nlabels (as_a <gasm *> (insert_point)) != 0
1525 13 : && !single_succ_p (gimple_bb (insert_point)))
1526 : /* We have no idea where to insert - it depends on where the
1527 : uses will be placed. */
1528 0 : gcc_unreachable ();
1529 : else
1530 : /* We assume INSERT_POINT is a SSA_NAME_DEF_STMT of some SSA_NAME,
1531 : thus if it must end a basic block, it should be a call that can
1532 : throw, or some assignment that can throw. If it throws, the LHS
1533 : of it will not be initialized though, so only valid places using
1534 : the SSA_NAME should be dominated by the fallthru edge. */
1535 8 : bb = find_fallthru_edge (gimple_bb (insert_point)->succs)->dest;
1536 38 : gsi = gsi_after_labels (bb);
1537 38 : if (gsi_end_p (gsi))
1538 : {
1539 0 : gimple_stmt_iterator gsi2 = gsi_last_bb (bb);
1540 0 : gimple_set_uid (stmt,
1541 0 : gsi_end_p (gsi2) ? 1 : gimple_uid (gsi_stmt (gsi2)));
1542 : }
1543 : else
1544 38 : gimple_set_uid (stmt, gimple_uid (gsi_stmt (gsi)));
1545 38 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1546 : }
1547 :
1548 : /* Builds one statement performing OP1 OPCODE OP2 using TMPVAR for
1549 : the result. Places the statement after the definition of either
1550 : OP1 or OP2. Returns the new statement. */
1551 :
1552 : static gimple *
1553 8568 : build_and_add_sum (tree type, tree op1, tree op2, enum tree_code opcode)
1554 : {
1555 8568 : gimple *op1def = NULL, *op2def = NULL;
1556 8568 : gimple_stmt_iterator gsi;
1557 8568 : tree op;
1558 8568 : gassign *sum;
1559 :
1560 : /* Create the addition statement. */
1561 8568 : op = make_ssa_name (type);
1562 8568 : sum = gimple_build_assign (op, opcode, op1, op2);
1563 :
1564 : /* Find an insertion place and insert. */
1565 8568 : if (TREE_CODE (op1) == SSA_NAME)
1566 8567 : op1def = SSA_NAME_DEF_STMT (op1);
1567 8568 : if (TREE_CODE (op2) == SSA_NAME)
1568 8277 : op2def = SSA_NAME_DEF_STMT (op2);
1569 8567 : if ((!op1def || gimple_nop_p (op1def))
1570 8650 : && (!op2def || gimple_nop_p (op2def)))
1571 : {
1572 82 : gsi = gsi_start_nondebug_after_labels_bb
1573 82 : (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
1574 82 : if (!gsi_end_p (gsi)
1575 82 : && is_gimple_call (gsi_stmt (gsi))
1576 94 : && (gimple_call_flags (gsi_stmt (gsi)) & ECF_RETURNS_TWICE))
1577 : {
1578 : /* Don't add statements before a returns_twice call at the start
1579 : of a function. */
1580 2 : split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
1581 2 : gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
1582 : }
1583 82 : if (gsi_end_p (gsi))
1584 : {
1585 2 : gimple_stmt_iterator gsi2
1586 2 : = gsi_last_bb (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
1587 2 : gimple_set_uid (sum,
1588 2 : gsi_end_p (gsi2) ? 1 : gimple_uid (gsi_stmt (gsi2)));
1589 : }
1590 : else
1591 80 : gimple_set_uid (sum, gimple_uid (gsi_stmt (gsi)));
1592 82 : gsi_insert_before (&gsi, sum, GSI_NEW_STMT);
1593 : }
1594 : else
1595 : {
1596 8486 : gimple *insert_point;
1597 8485 : if ((!op1def || gimple_nop_p (op1def))
1598 16971 : || (op2def && !gimple_nop_p (op2def)
1599 8167 : && reassoc_stmt_dominates_stmt_p (op1def, op2def)))
1600 : insert_point = op2def;
1601 : else
1602 : insert_point = op1def;
1603 8486 : insert_stmt_after (sum, insert_point);
1604 : }
1605 8568 : update_stmt (sum);
1606 :
1607 8568 : return sum;
1608 : }
1609 :
1610 : /* Perform un-distribution of divisions and multiplications.
1611 : A * X + B * X is transformed into (A + B) * X and A / X + B / X
1612 : to (A + B) / X for real X.
1613 :
1614 : The algorithm is organized as follows.
1615 :
1616 : - First we walk the addition chain *OPS looking for summands that
1617 : are defined by a multiplication or a real division. This results
1618 : in the candidates bitmap with relevant indices into *OPS.
1619 :
1620 : - Second we build the chains of multiplications or divisions for
1621 : these candidates, counting the number of occurrences of (operand, code)
1622 : pairs in all of the candidates chains.
1623 :
1624 : - Third we sort the (operand, code) pairs by number of occurrence and
1625 : process them starting with the pair with the most uses.
1626 :
1627 : * For each such pair we walk the candidates again to build a
1628 : second candidate bitmap noting all multiplication/division chains
1629 : that have at least one occurrence of (operand, code).
1630 :
1631 : * We build an alternate addition chain only covering these
1632 : candidates with one (operand, code) operation removed from their
1633 : multiplication/division chain.
1634 :
1635 : * The first candidate gets replaced by the alternate addition chain
1636 : multiplied/divided by the operand.
1637 :
1638 : * All candidate chains get disabled for further processing and
1639 : processing of (operand, code) pairs continues.
1640 :
1641 : The alternate addition chains built are re-processed by the main
1642 : reassociation algorithm which allows optimizing a * x * y + b * y * x
1643 : to (a + b ) * x * y in one invocation of the reassociation pass. */
1644 :
1645 : static bool
1646 4777550 : undistribute_ops_list (enum tree_code opcode,
1647 : vec<operand_entry *> *ops, class loop *loop)
1648 : {
1649 4777550 : unsigned int length = ops->length ();
1650 4777550 : operand_entry *oe1;
1651 4777550 : unsigned i, j;
1652 4777550 : unsigned nr_candidates, nr_candidates2;
1653 4777550 : sbitmap_iterator sbi0;
1654 4777550 : vec<operand_entry *> *subops;
1655 4777550 : bool changed = false;
1656 4777550 : unsigned int next_oecount_id = 0;
1657 :
1658 4777550 : if (length <= 1
1659 4777550 : || opcode != PLUS_EXPR)
1660 : return false;
1661 :
1662 : /* Build a list of candidates to process. */
1663 2368204 : auto_sbitmap candidates (length);
1664 2368204 : bitmap_clear (candidates);
1665 2368204 : nr_candidates = 0;
1666 7278717 : FOR_EACH_VEC_ELT (*ops, i, oe1)
1667 : {
1668 4910513 : enum tree_code dcode;
1669 4910513 : gimple *oe1def;
1670 :
1671 4910513 : if (TREE_CODE (oe1->op) != SSA_NAME)
1672 1812037 : continue;
1673 3098476 : oe1def = SSA_NAME_DEF_STMT (oe1->op);
1674 3098476 : if (!is_gimple_assign (oe1def))
1675 1115531 : continue;
1676 1982945 : dcode = gimple_assign_rhs_code (oe1def);
1677 3739241 : if ((dcode != MULT_EXPR
1678 1982945 : && dcode != RDIV_EXPR)
1679 1982945 : || !is_reassociable_op (oe1def, dcode, loop))
1680 1756296 : continue;
1681 :
1682 226649 : bitmap_set_bit (candidates, i);
1683 226649 : nr_candidates++;
1684 : }
1685 :
1686 2368204 : if (nr_candidates < 2)
1687 : return false;
1688 :
1689 16337 : if (dump_file && (dump_flags & TDF_DETAILS))
1690 : {
1691 1 : fprintf (dump_file, "searching for un-distribute opportunities ");
1692 2 : print_generic_expr (dump_file,
1693 1 : (*ops)[bitmap_first_set_bit (candidates)]->op, TDF_NONE);
1694 1 : fprintf (dump_file, " %d\n", nr_candidates);
1695 : }
1696 :
1697 : /* Build linearized sub-operand lists and the counting table. */
1698 16337 : cvec.create (0);
1699 :
1700 16337 : hash_table<oecount_hasher> ctable (15);
1701 :
1702 : /* ??? Macro arguments cannot have multi-argument template types in
1703 : them. This typedef is needed to workaround that limitation. */
1704 16337 : typedef vec<operand_entry *> vec_operand_entry_t_heap;
1705 32674 : subops = XCNEWVEC (vec_operand_entry_t_heap, ops->length ());
1706 72693 : EXECUTE_IF_SET_IN_BITMAP (candidates, 0, i, sbi0)
1707 : {
1708 40019 : gimple *oedef;
1709 40019 : enum tree_code oecode;
1710 40019 : unsigned j;
1711 :
1712 40019 : oedef = SSA_NAME_DEF_STMT ((*ops)[i]->op);
1713 40019 : oecode = gimple_assign_rhs_code (oedef);
1714 40019 : linearize_expr_tree (&subops[i], oedef,
1715 : associative_tree_code (oecode), false);
1716 :
1717 200780 : FOR_EACH_VEC_ELT (subops[i], j, oe1)
1718 : {
1719 80723 : oecount c;
1720 80723 : int *slot;
1721 80723 : int idx;
1722 80723 : c.oecode = oecode;
1723 80723 : c.cnt = 1;
1724 80723 : c.id = next_oecount_id++;
1725 80723 : c.op = oe1->op;
1726 80723 : cvec.safe_push (c);
1727 80723 : idx = cvec.length () + 41;
1728 80723 : slot = ctable.find_slot (idx, INSERT);
1729 80723 : if (!*slot)
1730 : {
1731 79709 : *slot = idx;
1732 : }
1733 : else
1734 : {
1735 1014 : cvec.pop ();
1736 1014 : cvec[*slot - 42].cnt++;
1737 : }
1738 : }
1739 : }
1740 :
1741 : /* Sort the counting table. */
1742 16337 : cvec.qsort (oecount_cmp);
1743 :
1744 16337 : if (dump_file && (dump_flags & TDF_DETAILS))
1745 : {
1746 1 : oecount *c;
1747 1 : fprintf (dump_file, "Candidates:\n");
1748 5 : FOR_EACH_VEC_ELT (cvec, j, c)
1749 : {
1750 3 : fprintf (dump_file, " %u %s: ", c->cnt,
1751 3 : c->oecode == MULT_EXPR
1752 : ? "*" : c->oecode == RDIV_EXPR ? "/" : "?");
1753 3 : print_generic_expr (dump_file, c->op);
1754 3 : fprintf (dump_file, "\n");
1755 : }
1756 : }
1757 :
1758 : /* Process the (operand, code) pairs in order of most occurrence. */
1759 16337 : auto_sbitmap candidates2 (length);
1760 33382 : while (!cvec.is_empty ())
1761 : {
1762 16975 : oecount *c = &cvec.last ();
1763 16975 : if (c->cnt < 2)
1764 : break;
1765 :
1766 : /* Now collect the operands in the outer chain that contain
1767 : the common operand in their inner chain. */
1768 708 : bitmap_clear (candidates2);
1769 708 : nr_candidates2 = 0;
1770 4710 : EXECUTE_IF_SET_IN_BITMAP (candidates, 0, i, sbi0)
1771 : {
1772 3294 : gimple *oedef;
1773 3294 : enum tree_code oecode;
1774 3294 : unsigned j;
1775 3294 : tree op = (*ops)[i]->op;
1776 :
1777 : /* If we undistributed in this chain already this may be
1778 : a constant. */
1779 3294 : if (TREE_CODE (op) != SSA_NAME)
1780 762 : continue;
1781 :
1782 2532 : oedef = SSA_NAME_DEF_STMT (op);
1783 2532 : oecode = gimple_assign_rhs_code (oedef);
1784 2532 : if (oecode != c->oecode)
1785 0 : continue;
1786 :
1787 8927 : FOR_EACH_VEC_ELT (subops[i], j, oe1)
1788 : {
1789 4584 : if (oe1->op == c->op)
1790 : {
1791 1216 : bitmap_set_bit (candidates2, i);
1792 1216 : ++nr_candidates2;
1793 1216 : break;
1794 : }
1795 : }
1796 : }
1797 :
1798 708 : if (nr_candidates2 >= 2)
1799 : {
1800 336 : operand_entry *oe1, *oe2;
1801 336 : gimple *prod;
1802 336 : int first = bitmap_first_set_bit (candidates2);
1803 :
1804 : /* Build the new addition chain. */
1805 336 : oe1 = (*ops)[first];
1806 336 : if (dump_file && (dump_flags & TDF_DETAILS))
1807 : {
1808 0 : fprintf (dump_file, "Building (");
1809 0 : print_generic_expr (dump_file, oe1->op);
1810 : }
1811 336 : zero_one_operation (&oe1->op, c->oecode, c->op);
1812 1226 : EXECUTE_IF_SET_IN_BITMAP (candidates2, first+1, i, sbi0)
1813 : {
1814 554 : gimple *sum;
1815 554 : oe2 = (*ops)[i];
1816 554 : if (dump_file && (dump_flags & TDF_DETAILS))
1817 : {
1818 0 : fprintf (dump_file, " + ");
1819 0 : print_generic_expr (dump_file, oe2->op);
1820 : }
1821 554 : zero_one_operation (&oe2->op, c->oecode, c->op);
1822 554 : sum = build_and_add_sum (TREE_TYPE (oe1->op),
1823 : oe1->op, oe2->op, opcode);
1824 554 : oe2->op = build_zero_cst (TREE_TYPE (oe2->op));
1825 554 : oe2->rank = 0;
1826 554 : oe1->op = gimple_get_lhs (sum);
1827 : }
1828 :
1829 : /* Apply the multiplication/division. */
1830 336 : prod = build_and_add_sum (TREE_TYPE (oe1->op),
1831 : oe1->op, c->op, c->oecode);
1832 336 : if (dump_file && (dump_flags & TDF_DETAILS))
1833 : {
1834 0 : fprintf (dump_file, ") %s ", c->oecode == MULT_EXPR ? "*" : "/");
1835 0 : print_generic_expr (dump_file, c->op);
1836 0 : fprintf (dump_file, "\n");
1837 : }
1838 :
1839 : /* Record it in the addition chain and disable further
1840 : undistribution with this op. */
1841 336 : oe1->op = gimple_assign_lhs (prod);
1842 336 : oe1->rank = get_rank (oe1->op);
1843 336 : subops[first].release ();
1844 :
1845 336 : changed = true;
1846 : }
1847 :
1848 708 : cvec.pop ();
1849 : }
1850 :
1851 71187 : for (i = 0; i < ops->length (); ++i)
1852 54850 : subops[i].release ();
1853 16337 : free (subops);
1854 16337 : cvec.release ();
1855 :
1856 16337 : return changed;
1857 2368204 : }
1858 :
1859 : /* Pair to hold the information of one specific VECTOR_TYPE SSA_NAME:
1860 : first: element index for each relevant BIT_FIELD_REF.
1861 : second: the index of vec ops* for each relevant BIT_FIELD_REF. */
1862 : typedef std::pair<unsigned, unsigned> v_info_elem;
1863 7937 : struct v_info {
1864 : tree vec_type;
1865 : auto_vec<v_info_elem, 32> vec;
1866 : };
1867 : typedef v_info *v_info_ptr;
1868 :
1869 : /* Comparison function for qsort on VECTOR SSA_NAME trees by machine mode. */
1870 : static int
1871 10571 : sort_by_mach_mode (const void *p_i, const void *p_j)
1872 : {
1873 10571 : const tree tr1 = *((const tree *) p_i);
1874 10571 : const tree tr2 = *((const tree *) p_j);
1875 10571 : unsigned int mode1 = TYPE_MODE (TREE_TYPE (tr1));
1876 10571 : unsigned int mode2 = TYPE_MODE (TREE_TYPE (tr2));
1877 10571 : if (mode1 > mode2)
1878 : return 1;
1879 10541 : else if (mode1 < mode2)
1880 : return -1;
1881 10502 : if (SSA_NAME_VERSION (tr1) < SSA_NAME_VERSION (tr2))
1882 : return -1;
1883 5049 : else if (SSA_NAME_VERSION (tr1) > SSA_NAME_VERSION (tr2))
1884 5049 : return 1;
1885 : return 0;
1886 : }
1887 :
1888 : /* Cleanup hash map for VECTOR information. */
1889 : static void
1890 4550703 : cleanup_vinfo_map (hash_map<tree, v_info_ptr> &info_map)
1891 : {
1892 4558640 : for (hash_map<tree, v_info_ptr>::iterator it = info_map.begin ();
1893 4558640 : it != info_map.end (); ++it)
1894 : {
1895 7937 : v_info_ptr info = (*it).second;
1896 7937 : delete info;
1897 7937 : (*it).second = NULL;
1898 : }
1899 4550703 : }
1900 :
1901 : /* Perform un-distribution of BIT_FIELD_REF on VECTOR_TYPE.
1902 : V1[0] + V1[1] + ... + V1[k] + V2[0] + V2[1] + ... + V2[k] + ... Vn[k]
1903 : is transformed to
1904 : Vs = (V1 + V2 + ... + Vn)
1905 : Vs[0] + Vs[1] + ... + Vs[k]
1906 :
1907 : The basic steps are listed below:
1908 :
1909 : 1) Check the addition chain *OPS by looking those summands coming from
1910 : VECTOR bit_field_ref on VECTOR type. Put the information into
1911 : v_info_map for each satisfied summand, using VECTOR SSA_NAME as key.
1912 :
1913 : 2) For each key (VECTOR SSA_NAME), validate all its BIT_FIELD_REFs are
1914 : continuous, they can cover the whole VECTOR perfectly without any holes.
1915 : Obtain one VECTOR list which contain candidates to be transformed.
1916 :
1917 : 3) Sort the VECTOR list by machine mode of VECTOR type, for each group of
1918 : candidates with same mode, build the addition statements for them and
1919 : generate BIT_FIELD_REFs accordingly.
1920 :
1921 : TODO:
1922 : The current implementation requires the whole VECTORs should be fully
1923 : covered, but it can be extended to support partial, checking adjacent
1924 : but not fill the whole, it may need some cost model to define the
1925 : boundary to do or not.
1926 : */
1927 : static bool
1928 4777550 : undistribute_bitref_for_vector (enum tree_code opcode,
1929 : vec<operand_entry *> *ops, struct loop *loop)
1930 : {
1931 4777550 : if (ops->length () <= 1)
1932 : return false;
1933 :
1934 4774135 : if (opcode != PLUS_EXPR
1935 4774135 : && opcode != MULT_EXPR
1936 : && opcode != BIT_XOR_EXPR
1937 1276667 : && opcode != BIT_IOR_EXPR
1938 879921 : && opcode != BIT_AND_EXPR)
1939 : return false;
1940 :
1941 4550703 : hash_map<tree, v_info_ptr> v_info_map;
1942 4550703 : operand_entry *oe1;
1943 4550703 : unsigned i;
1944 :
1945 : /* Find those summands from VECTOR BIT_FIELD_REF in addition chain, put the
1946 : information into map. */
1947 18443067 : FOR_EACH_VEC_ELT (*ops, i, oe1)
1948 : {
1949 9341661 : enum tree_code dcode;
1950 9341661 : gimple *oe1def;
1951 :
1952 9341661 : if (TREE_CODE (oe1->op) != SSA_NAME)
1953 3282264 : continue;
1954 6059397 : oe1def = SSA_NAME_DEF_STMT (oe1->op);
1955 6059397 : if (!is_gimple_assign (oe1def))
1956 1524238 : continue;
1957 4535159 : dcode = gimple_assign_rhs_code (oe1def);
1958 4535159 : if (dcode != BIT_FIELD_REF || !is_reassociable_op (oe1def, dcode, loop))
1959 4483372 : continue;
1960 :
1961 51787 : tree rhs = gimple_assign_rhs1 (oe1def);
1962 51787 : tree vec = TREE_OPERAND (rhs, 0);
1963 51787 : tree vec_type = TREE_TYPE (vec);
1964 :
1965 51787 : if (TREE_CODE (vec) != SSA_NAME || !VECTOR_TYPE_P (vec_type))
1966 29977 : continue;
1967 :
1968 : /* Ignore it if target machine can't support this VECTOR type. */
1969 21810 : if (!VECTOR_MODE_P (TYPE_MODE (vec_type)))
1970 5601 : continue;
1971 :
1972 : /* Check const vector type, constrain BIT_FIELD_REF offset and size. */
1973 16209 : if (!TYPE_VECTOR_SUBPARTS (vec_type).is_constant ())
1974 : continue;
1975 :
1976 16209 : if (VECTOR_TYPE_P (TREE_TYPE (rhs))
1977 16209 : || !is_a <scalar_mode> (TYPE_MODE (TREE_TYPE (rhs))))
1978 5318 : continue;
1979 :
1980 : /* The type of BIT_FIELD_REF might not be equal to the element type of
1981 : the vector. We want to use a vector type with element type the
1982 : same as the BIT_FIELD_REF and size the same as TREE_TYPE (vec). */
1983 10891 : if (!useless_type_conversion_p (TREE_TYPE (rhs), TREE_TYPE (vec_type)))
1984 : {
1985 1773 : machine_mode simd_mode;
1986 1773 : unsigned HOST_WIDE_INT size, nunits;
1987 1773 : unsigned HOST_WIDE_INT elem_size
1988 1773 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs)));
1989 3546 : if (!GET_MODE_BITSIZE (TYPE_MODE (vec_type)).is_constant (&size))
1990 9331204 : continue;
1991 1773 : if (size <= elem_size || (size % elem_size) != 0)
1992 0 : continue;
1993 1773 : nunits = size / elem_size;
1994 1773 : if (!mode_for_vector (SCALAR_TYPE_MODE (TREE_TYPE (rhs)),
1995 1773 : nunits).exists (&simd_mode))
1996 0 : continue;
1997 1773 : vec_type = build_vector_type_for_mode (TREE_TYPE (rhs), simd_mode);
1998 :
1999 : /* Ignore it if target machine can't support this VECTOR type. */
2000 1773 : if (!VECTOR_MODE_P (TYPE_MODE (vec_type)))
2001 0 : continue;
2002 :
2003 : /* Check const vector type, constrain BIT_FIELD_REF offset and
2004 : size. */
2005 1773 : if (!TYPE_VECTOR_SUBPARTS (vec_type).is_constant ())
2006 : continue;
2007 :
2008 3546 : if (maybe_ne (GET_MODE_SIZE (TYPE_MODE (vec_type)),
2009 3546 : GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (vec)))))
2010 0 : continue;
2011 : }
2012 :
2013 10891 : tree elem_type = TREE_TYPE (vec_type);
2014 10891 : unsigned HOST_WIDE_INT elem_size = tree_to_uhwi (TYPE_SIZE (elem_type));
2015 10891 : if (maybe_ne (bit_field_size (rhs), elem_size))
2016 0 : continue;
2017 :
2018 10891 : unsigned idx;
2019 10891 : if (!constant_multiple_p (bit_field_offset (rhs), elem_size, &idx))
2020 0 : continue;
2021 :
2022 : /* Ignore it if target machine can't support this type of VECTOR
2023 : operation. */
2024 10891 : optab op_tab = optab_for_tree_code (opcode, vec_type, optab_vector);
2025 10891 : if (optab_handler (op_tab, TYPE_MODE (vec_type)) == CODE_FOR_nothing)
2026 434 : continue;
2027 :
2028 10457 : bool existed;
2029 10457 : v_info_ptr &info = v_info_map.get_or_insert (vec, &existed);
2030 10457 : if (!existed)
2031 : {
2032 7937 : info = new v_info;
2033 7937 : info->vec_type = vec_type;
2034 : }
2035 2520 : else if (!types_compatible_p (vec_type, info->vec_type))
2036 0 : continue;
2037 10457 : info->vec.safe_push (std::make_pair (idx, i));
2038 : }
2039 :
2040 : /* At least two VECTOR to combine. */
2041 4550703 : if (v_info_map.elements () <= 1)
2042 : {
2043 4550458 : cleanup_vinfo_map (v_info_map);
2044 4550458 : return false;
2045 : }
2046 :
2047 : /* Verify all VECTOR candidates by checking two conditions:
2048 : 1) sorted offsets are adjacent, no holes.
2049 : 2) can fill the whole VECTOR perfectly.
2050 : And add the valid candidates to a vector for further handling. */
2051 245 : auto_vec<tree> valid_vecs (v_info_map.elements ());
2052 1199 : for (hash_map<tree, v_info_ptr>::iterator it = v_info_map.begin ();
2053 1199 : it != v_info_map.end (); ++it)
2054 : {
2055 954 : tree cand_vec = (*it).first;
2056 954 : v_info_ptr cand_info = (*it).second;
2057 954 : unsigned int num_elems
2058 954 : = TYPE_VECTOR_SUBPARTS (cand_info->vec_type).to_constant ();
2059 1908 : if (cand_info->vec.length () != num_elems)
2060 589 : continue;
2061 365 : sbitmap holes = sbitmap_alloc (num_elems);
2062 365 : bitmap_ones (holes);
2063 365 : bool valid = true;
2064 365 : v_info_elem *curr;
2065 2124 : FOR_EACH_VEC_ELT (cand_info->vec, i, curr)
2066 : {
2067 1394 : if (!bitmap_bit_p (holes, curr->first))
2068 : {
2069 : valid = false;
2070 : break;
2071 : }
2072 : else
2073 1394 : bitmap_clear_bit (holes, curr->first);
2074 : }
2075 365 : if (valid && bitmap_empty_p (holes))
2076 365 : valid_vecs.quick_push (cand_vec);
2077 365 : sbitmap_free (holes);
2078 : }
2079 :
2080 : /* At least two VECTOR to combine. */
2081 245 : if (valid_vecs.length () <= 1)
2082 : {
2083 209 : cleanup_vinfo_map (v_info_map);
2084 209 : return false;
2085 : }
2086 :
2087 36 : valid_vecs.qsort (sort_by_mach_mode);
2088 : /* Go through all candidates by machine mode order, query the mode_to_total
2089 : to get the total number for each mode and skip the single one. */
2090 112 : for (unsigned i = 0; i < valid_vecs.length () - 1; ++i)
2091 : {
2092 40 : tree tvec = valid_vecs[i];
2093 40 : enum machine_mode mode = TYPE_MODE (TREE_TYPE (tvec));
2094 :
2095 : /* Skip modes with only a single candidate. */
2096 40 : if (TYPE_MODE (TREE_TYPE (valid_vecs[i + 1])) != mode)
2097 0 : continue;
2098 :
2099 40 : unsigned int idx, j;
2100 40 : gimple *sum = NULL;
2101 40 : tree sum_vec = tvec;
2102 40 : v_info_ptr info_ptr = *(v_info_map.get (tvec));
2103 40 : v_info_elem *elem;
2104 40 : tree vec_type = info_ptr->vec_type;
2105 :
2106 : /* Build the sum for all candidates with same mode. */
2107 323 : do
2108 : {
2109 969 : sum = build_and_add_sum (vec_type, sum_vec,
2110 323 : valid_vecs[i + 1], opcode);
2111 : /* Update the operands only after build_and_add_sum,
2112 : so that we don't have to repeat the placement algorithm
2113 : of build_and_add_sum. */
2114 323 : if (sum_vec == tvec
2115 323 : && !useless_type_conversion_p (vec_type, TREE_TYPE (sum_vec)))
2116 : {
2117 18 : gimple_stmt_iterator gsi = gsi_for_stmt (sum);
2118 18 : tree vce = build1 (VIEW_CONVERT_EXPR, vec_type, sum_vec);
2119 18 : tree lhs = make_ssa_name (vec_type);
2120 18 : gimple *g = gimple_build_assign (lhs, VIEW_CONVERT_EXPR, vce);
2121 18 : gimple_set_uid (g, gimple_uid (sum));
2122 18 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
2123 18 : gimple_assign_set_rhs1 (sum, lhs);
2124 18 : update_stmt (sum);
2125 : }
2126 323 : if (!useless_type_conversion_p (vec_type,
2127 323 : TREE_TYPE (valid_vecs[i + 1])))
2128 : {
2129 270 : gimple_stmt_iterator gsi = gsi_for_stmt (sum);
2130 810 : tree vce = build1 (VIEW_CONVERT_EXPR, vec_type,
2131 270 : valid_vecs[i + 1]);
2132 270 : tree lhs = make_ssa_name (vec_type);
2133 270 : gimple *g = gimple_build_assign (lhs, VIEW_CONVERT_EXPR, vce);
2134 270 : gimple_set_uid (g, gimple_uid (sum));
2135 270 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
2136 270 : gimple_assign_set_rhs2 (sum, lhs);
2137 270 : update_stmt (sum);
2138 : }
2139 323 : sum_vec = gimple_get_lhs (sum);
2140 323 : info_ptr = *(v_info_map.get (valid_vecs[i + 1]));
2141 323 : gcc_assert (types_compatible_p (vec_type, info_ptr->vec_type));
2142 : /* Update those related ops of current candidate VECTOR. */
2143 1569 : FOR_EACH_VEC_ELT (info_ptr->vec, j, elem)
2144 : {
2145 1246 : idx = elem->second;
2146 1246 : gimple *def = SSA_NAME_DEF_STMT ((*ops)[idx]->op);
2147 : /* Set this then op definition will get DCEd later. */
2148 1246 : gimple_set_visited (def, true);
2149 1246 : if (opcode == PLUS_EXPR
2150 1246 : || opcode == BIT_XOR_EXPR
2151 96 : || opcode == BIT_IOR_EXPR)
2152 1186 : (*ops)[idx]->op = build_zero_cst (TREE_TYPE ((*ops)[idx]->op));
2153 60 : else if (opcode == MULT_EXPR)
2154 24 : (*ops)[idx]->op = build_one_cst (TREE_TYPE ((*ops)[idx]->op));
2155 : else
2156 : {
2157 36 : gcc_assert (opcode == BIT_AND_EXPR);
2158 36 : (*ops)[idx]->op
2159 36 : = build_all_ones_cst (TREE_TYPE ((*ops)[idx]->op));
2160 : }
2161 1246 : (*ops)[idx]->rank = 0;
2162 : }
2163 323 : if (dump_file && (dump_flags & TDF_DETAILS))
2164 : {
2165 0 : fprintf (dump_file, "Generating addition -> ");
2166 0 : print_gimple_stmt (dump_file, sum, 0);
2167 : }
2168 323 : i++;
2169 : }
2170 323 : while ((i < valid_vecs.length () - 1)
2171 363 : && TYPE_MODE (TREE_TYPE (valid_vecs[i + 1])) == mode);
2172 :
2173 : /* Referring to first valid VECTOR with this mode, generate the
2174 : BIT_FIELD_REF statements accordingly. */
2175 40 : info_ptr = *(v_info_map.get (tvec));
2176 40 : gcc_assert (sum);
2177 40 : tree elem_type = TREE_TYPE (vec_type);
2178 224 : FOR_EACH_VEC_ELT (info_ptr->vec, j, elem)
2179 : {
2180 144 : idx = elem->second;
2181 144 : tree dst = make_ssa_name (elem_type);
2182 144 : tree pos = bitsize_int (elem->first
2183 : * tree_to_uhwi (TYPE_SIZE (elem_type)));
2184 144 : tree bfr = build3 (BIT_FIELD_REF, elem_type, sum_vec,
2185 144 : TYPE_SIZE (elem_type), pos);
2186 144 : gimple *gs = gimple_build_assign (dst, BIT_FIELD_REF, bfr);
2187 144 : insert_stmt_after (gs, sum);
2188 144 : gimple *def = SSA_NAME_DEF_STMT ((*ops)[idx]->op);
2189 : /* Set this then op definition will get DCEd later. */
2190 144 : gimple_set_visited (def, true);
2191 144 : (*ops)[idx]->op = gimple_assign_lhs (gs);
2192 144 : (*ops)[idx]->rank = get_rank ((*ops)[idx]->op);
2193 144 : if (dump_file && (dump_flags & TDF_DETAILS))
2194 : {
2195 0 : fprintf (dump_file, "Generating bit_field_ref -> ");
2196 0 : print_gimple_stmt (dump_file, gs, 0);
2197 : }
2198 : }
2199 : }
2200 :
2201 36 : if (dump_file && (dump_flags & TDF_DETAILS))
2202 0 : fprintf (dump_file, "undistributiong bit_field_ref for vector done.\n");
2203 :
2204 36 : cleanup_vinfo_map (v_info_map);
2205 :
2206 36 : return true;
2207 4550948 : }
2208 :
2209 : /* If OPCODE is BIT_IOR_EXPR or BIT_AND_EXPR and CURR is a comparison
2210 : expression, examine the other OPS to see if any of them are comparisons
2211 : of the same values, which we may be able to combine or eliminate.
2212 : For example, we can rewrite (a < b) | (a == b) as (a <= b). */
2213 :
2214 : static bool
2215 9799942 : eliminate_redundant_comparison (enum tree_code opcode,
2216 : vec<operand_entry *> *ops,
2217 : unsigned int currindex,
2218 : operand_entry *curr)
2219 : {
2220 9799942 : tree op1, op2;
2221 9799942 : enum tree_code lcode, rcode;
2222 9799942 : gimple *def1, *def2;
2223 9799942 : int i;
2224 9799942 : operand_entry *oe;
2225 :
2226 9799942 : if (opcode != BIT_IOR_EXPR && opcode != BIT_AND_EXPR)
2227 : return false;
2228 :
2229 : /* Check that CURR is a comparison. */
2230 2038655 : if (TREE_CODE (curr->op) != SSA_NAME)
2231 : return false;
2232 1538653 : def1 = SSA_NAME_DEF_STMT (curr->op);
2233 1538653 : if (!is_gimple_assign (def1))
2234 : return false;
2235 1328767 : lcode = gimple_assign_rhs_code (def1);
2236 1328767 : if (TREE_CODE_CLASS (lcode) != tcc_comparison)
2237 : return false;
2238 538169 : op1 = gimple_assign_rhs1 (def1);
2239 538169 : op2 = gimple_assign_rhs2 (def1);
2240 :
2241 : /* Now look for a similar comparison in the remaining OPS. */
2242 898242 : for (i = currindex + 1; ops->iterate (i, &oe); i++)
2243 : {
2244 360276 : tree t;
2245 :
2246 360276 : if (TREE_CODE (oe->op) != SSA_NAME)
2247 41 : continue;
2248 360235 : def2 = SSA_NAME_DEF_STMT (oe->op);
2249 360235 : if (!is_gimple_assign (def2))
2250 7846 : continue;
2251 352389 : rcode = gimple_assign_rhs_code (def2);
2252 352389 : if (TREE_CODE_CLASS (rcode) != tcc_comparison)
2253 6527 : continue;
2254 :
2255 : /* If we got here, we have a match. See if we can combine the
2256 : two comparisons. */
2257 345862 : tree type = TREE_TYPE (gimple_assign_lhs (def1));
2258 345862 : if (opcode == BIT_IOR_EXPR)
2259 196955 : t = maybe_fold_or_comparisons (type,
2260 : lcode, op1, op2,
2261 : rcode, gimple_assign_rhs1 (def2),
2262 : gimple_assign_rhs2 (def2));
2263 : else
2264 148907 : t = maybe_fold_and_comparisons (type,
2265 : lcode, op1, op2,
2266 : rcode, gimple_assign_rhs1 (def2),
2267 : gimple_assign_rhs2 (def2));
2268 345862 : if (!t)
2269 345611 : continue;
2270 :
2271 : /* maybe_fold_and_comparisons and maybe_fold_or_comparisons
2272 : always give us a boolean_type_node value back. If the original
2273 : BIT_AND_EXPR or BIT_IOR_EXPR was of a wider integer type,
2274 : we need to convert. */
2275 251 : if (!useless_type_conversion_p (TREE_TYPE (curr->op), TREE_TYPE (t)))
2276 : {
2277 2 : if (!fold_convertible_p (TREE_TYPE (curr->op), t))
2278 0 : continue;
2279 2 : t = fold_convert (TREE_TYPE (curr->op), t);
2280 : }
2281 :
2282 251 : if (TREE_CODE (t) != INTEGER_CST
2283 251 : && !operand_equal_p (t, curr->op, 0))
2284 : {
2285 251 : enum tree_code subcode;
2286 251 : tree newop1, newop2;
2287 251 : if (!COMPARISON_CLASS_P (t))
2288 48 : continue;
2289 223 : extract_ops_from_tree (t, &subcode, &newop1, &newop2);
2290 223 : STRIP_USELESS_TYPE_CONVERSION (newop1);
2291 223 : STRIP_USELESS_TYPE_CONVERSION (newop2);
2292 223 : if (!is_gimple_val (newop1) || !is_gimple_val (newop2))
2293 0 : continue;
2294 223 : if (lcode == TREE_CODE (t)
2295 182 : && operand_equal_p (op1, newop1, 0)
2296 405 : && operand_equal_p (op2, newop2, 0))
2297 137 : t = curr->op;
2298 106 : else if ((TREE_CODE (newop1) == SSA_NAME
2299 86 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (newop1))
2300 152 : || (TREE_CODE (newop2) == SSA_NAME
2301 26 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (newop2)))
2302 20 : continue;
2303 : }
2304 :
2305 203 : if (dump_file && (dump_flags & TDF_DETAILS))
2306 : {
2307 6 : fprintf (dump_file, "Equivalence: ");
2308 6 : print_generic_expr (dump_file, curr->op);
2309 6 : fprintf (dump_file, " %s ", op_symbol_code (opcode));
2310 6 : print_generic_expr (dump_file, oe->op);
2311 6 : fprintf (dump_file, " -> ");
2312 6 : print_generic_expr (dump_file, t);
2313 6 : fprintf (dump_file, "\n");
2314 : }
2315 :
2316 : /* Now we can delete oe, as it has been subsumed by the new combined
2317 : expression t. */
2318 203 : ops->ordered_remove (i);
2319 203 : reassociate_stats.ops_eliminated ++;
2320 :
2321 : /* If t is the same as curr->op, we're done. Otherwise we must
2322 : replace curr->op with t. Special case is if we got a constant
2323 : back, in which case we add it to the end instead of in place of
2324 : the current entry. */
2325 203 : if (TREE_CODE (t) == INTEGER_CST)
2326 : {
2327 0 : ops->ordered_remove (currindex);
2328 0 : add_to_ops_vec (ops, t);
2329 : }
2330 203 : else if (!operand_equal_p (t, curr->op, 0))
2331 : {
2332 66 : gimple *sum;
2333 66 : enum tree_code subcode;
2334 66 : tree newop1;
2335 66 : tree newop2;
2336 66 : gcc_assert (COMPARISON_CLASS_P (t));
2337 66 : extract_ops_from_tree (t, &subcode, &newop1, &newop2);
2338 66 : STRIP_USELESS_TYPE_CONVERSION (newop1);
2339 66 : STRIP_USELESS_TYPE_CONVERSION (newop2);
2340 66 : gcc_checking_assert (is_gimple_val (newop1)
2341 : && is_gimple_val (newop2));
2342 66 : sum = build_and_add_sum (TREE_TYPE (t), newop1, newop2, subcode);
2343 66 : curr->op = gimple_get_lhs (sum);
2344 : }
2345 : return true;
2346 : }
2347 :
2348 : return false;
2349 : }
2350 :
2351 :
2352 : /* Transform repeated addition of same values into multiply with
2353 : constant. */
2354 : static bool
2355 2371454 : transform_add_to_multiply (vec<operand_entry *> *ops)
2356 : {
2357 2371454 : operand_entry *oe;
2358 2371454 : tree op = NULL_TREE;
2359 2371454 : int j;
2360 2371454 : int i, start = -1, end = 0, count = 0;
2361 2371454 : auto_vec<std::pair <int, int> > indxs;
2362 2371454 : bool changed = false;
2363 :
2364 2371454 : if (!INTEGRAL_TYPE_P (TREE_TYPE ((*ops)[0]->op))
2365 69401 : && (!SCALAR_FLOAT_TYPE_P (TREE_TYPE ((*ops)[0]->op))
2366 31805 : || !flag_unsafe_math_optimizations))
2367 : return false;
2368 :
2369 : /* Look for repeated operands. */
2370 7160958 : FOR_EACH_VEC_ELT (*ops, i, oe)
2371 : {
2372 4827159 : if (start == -1)
2373 : {
2374 2333799 : count = 1;
2375 2333799 : op = oe->op;
2376 2333799 : start = i;
2377 : }
2378 2493360 : else if (operand_equal_p (oe->op, op, 0))
2379 : {
2380 155 : count++;
2381 155 : end = i;
2382 : }
2383 : else
2384 : {
2385 2493205 : if (count > 1)
2386 47 : indxs.safe_push (std::make_pair (start, end));
2387 2493205 : count = 1;
2388 2493205 : op = oe->op;
2389 2493205 : start = i;
2390 : }
2391 : }
2392 :
2393 2333799 : if (count > 1)
2394 38 : indxs.safe_push (std::make_pair (start, end));
2395 :
2396 2333957 : for (j = indxs.length () - 1; j >= 0; --j)
2397 : {
2398 : /* Convert repeated operand addition to multiplication. */
2399 85 : start = indxs[j].first;
2400 85 : end = indxs[j].second;
2401 85 : op = (*ops)[start]->op;
2402 85 : count = end - start + 1;
2403 325 : for (i = end; i >= start; --i)
2404 240 : ops->unordered_remove (i);
2405 85 : tree tmp = make_ssa_name (TREE_TYPE (op));
2406 85 : tree cst = build_int_cst (integer_type_node, count);
2407 85 : gassign *mul_stmt
2408 85 : = gimple_build_assign (tmp, MULT_EXPR,
2409 85 : op, fold_convert (TREE_TYPE (op), cst));
2410 85 : gimple_set_visited (mul_stmt, true);
2411 85 : add_to_ops_vec (ops, tmp, mul_stmt);
2412 85 : changed = true;
2413 : }
2414 :
2415 : return changed;
2416 2371454 : }
2417 :
2418 :
2419 : /* Perform various identities and other optimizations on the list of
2420 : operand entries, stored in OPS. The tree code for the binary
2421 : operation between all the operands is OPCODE. */
2422 :
2423 : static void
2424 4777901 : optimize_ops_list (enum tree_code opcode,
2425 : vec<operand_entry *> *ops)
2426 : {
2427 4795868 : unsigned int length = ops->length ();
2428 4795868 : unsigned int i;
2429 4795868 : operand_entry *oe;
2430 9589866 : operand_entry *oelast = NULL;
2431 9589866 : bool iterate = false;
2432 :
2433 4795868 : if (length == 1)
2434 : return;
2435 :
2436 4793998 : oelast = ops->last ();
2437 :
2438 : /* If the last two are constants, pop the constants off, merge them
2439 : and try the next two. */
2440 4793998 : if (oelast->rank == 0 && is_gimple_min_invariant (oelast->op))
2441 : {
2442 3473200 : operand_entry *oelm1 = (*ops)[length - 2];
2443 :
2444 3473200 : if (oelm1->rank == 0
2445 14170 : && is_gimple_min_invariant (oelm1->op)
2446 3487370 : && useless_type_conversion_p (TREE_TYPE (oelm1->op),
2447 14170 : TREE_TYPE (oelast->op)))
2448 : {
2449 14170 : tree folded = fold_binary (opcode, TREE_TYPE (oelm1->op),
2450 : oelm1->op, oelast->op);
2451 :
2452 14170 : if (folded && is_gimple_min_invariant (folded))
2453 : {
2454 14152 : if (dump_file && (dump_flags & TDF_DETAILS))
2455 0 : fprintf (dump_file, "Merging constants\n");
2456 :
2457 14152 : ops->pop ();
2458 14152 : ops->pop ();
2459 :
2460 14152 : add_to_ops_vec (ops, folded);
2461 14152 : reassociate_stats.constants_eliminated++;
2462 :
2463 14152 : optimize_ops_list (opcode, ops);
2464 14152 : return;
2465 : }
2466 : }
2467 : }
2468 :
2469 4779846 : eliminate_using_constants (opcode, ops);
2470 4779846 : oelast = NULL;
2471 :
2472 19363378 : for (i = 0; ops->iterate (i, &oe);)
2473 : {
2474 9803688 : bool done = false;
2475 :
2476 9803688 : if (eliminate_not_pairs (opcode, ops, i, oe))
2477 2 : return;
2478 9803687 : if (eliminate_duplicate_pair (opcode, ops, &done, i, oe, oelast)
2479 9803542 : || (!done && eliminate_plus_minus_pair (opcode, ops, i, oe))
2480 19603629 : || (!done && eliminate_redundant_comparison (opcode, ops, i, oe)))
2481 : {
2482 3948 : if (done)
2483 : return;
2484 3947 : iterate = true;
2485 3947 : oelast = NULL;
2486 3947 : continue;
2487 : }
2488 9799739 : oelast = oe;
2489 9799739 : i++;
2490 : }
2491 :
2492 4779844 : if (iterate)
2493 : optimize_ops_list (opcode, ops);
2494 : }
2495 :
2496 : /* The following functions are subroutines to optimize_range_tests and allow
2497 : it to try to change a logical combination of comparisons into a range
2498 : test.
2499 :
2500 : For example, both
2501 : X == 2 || X == 5 || X == 3 || X == 4
2502 : and
2503 : X >= 2 && X <= 5
2504 : are converted to
2505 : (unsigned) (X - 2) <= 3
2506 :
2507 : For more information see comments above fold_test_range in fold-const.cc,
2508 : this implementation is for GIMPLE. */
2509 :
2510 :
2511 :
2512 : /* Dump the range entry R to FILE, skipping its expression if SKIP_EXP. */
2513 :
2514 : void
2515 141 : dump_range_entry (FILE *file, struct range_entry *r, bool skip_exp)
2516 : {
2517 141 : if (!skip_exp)
2518 59 : print_generic_expr (file, r->exp);
2519 251 : fprintf (file, " %c[", r->in_p ? '+' : '-');
2520 141 : print_generic_expr (file, r->low);
2521 141 : fputs (", ", file);
2522 141 : print_generic_expr (file, r->high);
2523 141 : fputc (']', file);
2524 141 : }
2525 :
2526 : /* Dump the range entry R to STDERR. */
2527 :
2528 : DEBUG_FUNCTION void
2529 0 : debug_range_entry (struct range_entry *r)
2530 : {
2531 0 : dump_range_entry (stderr, r, false);
2532 0 : fputc ('\n', stderr);
2533 0 : }
2534 :
2535 : /* This is similar to make_range in fold-const.cc, but on top of
2536 : GIMPLE instead of trees. If EXP is non-NULL, it should be
2537 : an SSA_NAME and STMT argument is ignored, otherwise STMT
2538 : argument should be a GIMPLE_COND. */
2539 :
2540 : void
2541 5728531 : init_range_entry (struct range_entry *r, tree exp, gimple *stmt)
2542 : {
2543 5728531 : int in_p;
2544 5728531 : tree low, high;
2545 5728531 : bool is_bool;
2546 :
2547 5728531 : r->exp = NULL_TREE;
2548 5728531 : r->in_p = false;
2549 5728531 : r->low = NULL_TREE;
2550 5728531 : r->high = NULL_TREE;
2551 5728531 : if (exp != NULL_TREE
2552 5728531 : && (TREE_CODE (exp) != SSA_NAME || !INTEGRAL_TYPE_P (TREE_TYPE (exp))))
2553 796838 : return;
2554 :
2555 : /* Start with simply saying "EXP != 0" and then look at the code of EXP
2556 : and see if we can refine the range. Some of the cases below may not
2557 : happen, but it doesn't seem worth worrying about this. We "continue"
2558 : the outer loop when we've changed something; otherwise we "break"
2559 : the switch, which will "break" the while. */
2560 5234723 : low = exp ? build_int_cst (TREE_TYPE (exp), 0) : boolean_false_node;
2561 5234723 : high = low;
2562 5234723 : in_p = 0;
2563 5234723 : is_bool = false;
2564 5234723 : if (exp == NULL_TREE)
2565 5234547 : is_bool = true;
2566 1587934 : else if (TYPE_PRECISION (TREE_TYPE (exp)) == 1)
2567 : {
2568 670639 : if (TYPE_UNSIGNED (TREE_TYPE (exp)))
2569 : is_bool = true;
2570 : else
2571 : return;
2572 : }
2573 917295 : else if (TREE_CODE (TREE_TYPE (exp)) == BOOLEAN_TYPE)
2574 4317252 : is_bool = true;
2575 :
2576 8340448 : while (1)
2577 : {
2578 8340448 : enum tree_code code;
2579 8340448 : tree arg0, arg1, exp_type;
2580 8340448 : tree nexp;
2581 8340448 : location_t loc;
2582 :
2583 8340448 : if (exp != NULL_TREE)
2584 : {
2585 4693659 : if (TREE_CODE (exp) != SSA_NAME
2586 4693659 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (exp))
2587 : break;
2588 :
2589 4693659 : stmt = SSA_NAME_DEF_STMT (exp);
2590 4693659 : if (!is_gimple_assign (stmt))
2591 : break;
2592 :
2593 2868976 : code = gimple_assign_rhs_code (stmt);
2594 2868976 : arg0 = gimple_assign_rhs1 (stmt);
2595 2868976 : arg1 = gimple_assign_rhs2 (stmt);
2596 2868976 : exp_type = TREE_TYPE (exp);
2597 : }
2598 : else
2599 : {
2600 3646789 : code = gimple_cond_code (stmt);
2601 3646789 : arg0 = gimple_cond_lhs (stmt);
2602 3646789 : arg1 = gimple_cond_rhs (stmt);
2603 3646789 : exp_type = boolean_type_node;
2604 : }
2605 :
2606 6515765 : if (TREE_CODE (arg0) != SSA_NAME
2607 5175856 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg0)
2608 11691200 : || ssa_name_maybe_undef_p (arg0))
2609 : break;
2610 5168635 : loc = gimple_location (stmt);
2611 5168635 : switch (code)
2612 : {
2613 34139 : case BIT_NOT_EXPR:
2614 34139 : if (TREE_CODE (TREE_TYPE (exp)) == BOOLEAN_TYPE
2615 : /* Ensure the range is either +[-,0], +[0,0],
2616 : -[-,0], -[0,0] or +[1,-], +[1,1], -[1,-] or
2617 : -[1,1]. If it is e.g. +[-,-] or -[-,-]
2618 : or similar expression of unconditional true or
2619 : false, it should not be negated. */
2620 34139 : && ((high && integer_zerop (high))
2621 0 : || (low && integer_onep (low))))
2622 : {
2623 6361 : in_p = !in_p;
2624 6361 : exp = arg0;
2625 6361 : continue;
2626 : }
2627 : break;
2628 1808 : case SSA_NAME:
2629 1808 : exp = arg0;
2630 1808 : continue;
2631 245105 : CASE_CONVERT:
2632 245105 : if (is_bool)
2633 : {
2634 126984 : if ((TYPE_PRECISION (exp_type) == 1
2635 120560 : || TREE_CODE (exp_type) == BOOLEAN_TYPE)
2636 126984 : && TYPE_PRECISION (TREE_TYPE (arg0)) > 1)
2637 : return;
2638 : }
2639 118121 : else if (TYPE_PRECISION (TREE_TYPE (arg0)) == 1)
2640 : {
2641 4549 : if (TYPE_UNSIGNED (TREE_TYPE (arg0)))
2642 : is_bool = true;
2643 : else
2644 : return;
2645 : }
2646 113572 : else if (TREE_CODE (TREE_TYPE (arg0)) == BOOLEAN_TYPE)
2647 128796 : is_bool = true;
2648 242368 : goto do_default;
2649 : case EQ_EXPR:
2650 : case NE_EXPR:
2651 : case LT_EXPR:
2652 : case LE_EXPR:
2653 : case GE_EXPR:
2654 : case GT_EXPR:
2655 : is_bool = true;
2656 : /* FALLTHRU */
2657 665606 : default:
2658 665606 : if (!is_bool)
2659 : return;
2660 365489 : do_default:
2661 4829834 : nexp = make_range_step (loc, code, arg0, arg1, exp_type,
2662 : &low, &high, &in_p);
2663 4829834 : if (nexp != NULL_TREE)
2664 : {
2665 3097732 : exp = nexp;
2666 3097732 : gcc_assert (TREE_CODE (exp) == SSA_NAME);
2667 3097732 : continue;
2668 : }
2669 : break;
2670 : }
2671 : break;
2672 : }
2673 4931693 : if (is_bool)
2674 : {
2675 4319064 : r->exp = exp;
2676 4319064 : r->in_p = in_p;
2677 4319064 : r->low = low;
2678 4319064 : r->high = high;
2679 : }
2680 : }
2681 :
2682 : /* Comparison function for qsort. Sort entries
2683 : without SSA_NAME exp first, then with SSA_NAMEs sorted
2684 : by increasing SSA_NAME_VERSION, and for the same SSA_NAMEs
2685 : by increasing ->low and if ->low is the same, by increasing
2686 : ->high. ->low == NULL_TREE means minimum, ->high == NULL_TREE
2687 : maximum. */
2688 :
2689 : static int
2690 6143702 : range_entry_cmp (const void *a, const void *b)
2691 : {
2692 6143702 : const struct range_entry *p = (const struct range_entry *) a;
2693 6143702 : const struct range_entry *q = (const struct range_entry *) b;
2694 :
2695 6143702 : if (p->exp != NULL_TREE && TREE_CODE (p->exp) == SSA_NAME)
2696 : {
2697 2483935 : if (q->exp != NULL_TREE && TREE_CODE (q->exp) == SSA_NAME)
2698 : {
2699 : /* Group range_entries for the same SSA_NAME together. */
2700 2431936 : if (SSA_NAME_VERSION (p->exp) < SSA_NAME_VERSION (q->exp))
2701 : return -1;
2702 1052483 : else if (SSA_NAME_VERSION (p->exp) > SSA_NAME_VERSION (q->exp))
2703 : return 1;
2704 : /* If ->low is different, NULL low goes first, then by
2705 : ascending low. */
2706 170026 : if (p->low != NULL_TREE)
2707 : {
2708 153577 : if (q->low != NULL_TREE)
2709 : {
2710 146369 : tree tem = fold_binary (LT_EXPR, boolean_type_node,
2711 : p->low, q->low);
2712 146369 : if (tem && integer_onep (tem))
2713 : return -1;
2714 72055 : tem = fold_binary (GT_EXPR, boolean_type_node,
2715 : p->low, q->low);
2716 72055 : if (tem && integer_onep (tem))
2717 : return 1;
2718 : }
2719 : else
2720 : return 1;
2721 : }
2722 16449 : else if (q->low != NULL_TREE)
2723 : return -1;
2724 : /* If ->high is different, NULL high goes last, before that by
2725 : ascending high. */
2726 32560 : if (p->high != NULL_TREE)
2727 : {
2728 32480 : if (q->high != NULL_TREE)
2729 : {
2730 32315 : tree tem = fold_binary (LT_EXPR, boolean_type_node,
2731 : p->high, q->high);
2732 32315 : if (tem && integer_onep (tem))
2733 : return -1;
2734 9478 : tem = fold_binary (GT_EXPR, boolean_type_node,
2735 : p->high, q->high);
2736 9478 : if (tem && integer_onep (tem))
2737 : return 1;
2738 : }
2739 : else
2740 : return -1;
2741 : }
2742 80 : else if (q->high != NULL_TREE)
2743 : return 1;
2744 : /* If both ranges are the same, sort below by ascending idx. */
2745 : }
2746 : else
2747 : return 1;
2748 : }
2749 3659767 : else if (q->exp != NULL_TREE && TREE_CODE (q->exp) == SSA_NAME)
2750 : return -1;
2751 :
2752 3587979 : if (p->idx < q->idx)
2753 : return -1;
2754 : else
2755 : {
2756 1810549 : gcc_checking_assert (p->idx > q->idx);
2757 : return 1;
2758 : }
2759 : }
2760 :
2761 : /* Helper function for update_range_test. Force EXPR into an SSA_NAME,
2762 : insert needed statements BEFORE or after GSI. */
2763 :
2764 : static tree
2765 25865 : force_into_ssa_name (gimple_stmt_iterator *gsi, tree expr, bool before)
2766 : {
2767 25865 : enum gsi_iterator_update m = before ? GSI_SAME_STMT : GSI_CONTINUE_LINKING;
2768 25865 : tree ret = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE, before, m);
2769 25865 : if (TREE_CODE (ret) != SSA_NAME)
2770 : {
2771 34 : gimple *g = gimple_build_assign (make_ssa_name (TREE_TYPE (ret)), ret);
2772 34 : if (before)
2773 34 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
2774 : else
2775 0 : gsi_insert_after (gsi, g, GSI_CONTINUE_LINKING);
2776 34 : ret = gimple_assign_lhs (g);
2777 : }
2778 25865 : return ret;
2779 : }
2780 :
2781 : /* Helper routine of optimize_range_test.
2782 : [EXP, IN_P, LOW, HIGH] is a merged range for
2783 : RANGE and OTHERRANGE through OTHERRANGE + COUNT - 1 ranges,
2784 : OPCODE and OPS are arguments of optimize_range_tests. If OTHERRANGE
2785 : is NULL, OTHERRANGEP should not be and then OTHERRANGEP points to
2786 : an array of COUNT pointers to other ranges. Return
2787 : true if the range merge has been successful.
2788 : If OPCODE is ERROR_MARK, this is called from within
2789 : maybe_optimize_range_tests and is performing inter-bb range optimization.
2790 : In that case, whether an op is BIT_AND_EXPR or BIT_IOR_EXPR is found in
2791 : oe->rank. */
2792 :
2793 : static bool
2794 25865 : update_range_test (struct range_entry *range, struct range_entry *otherrange,
2795 : struct range_entry **otherrangep,
2796 : unsigned int count, enum tree_code opcode,
2797 : vec<operand_entry *> *ops, tree exp, gimple_seq seq,
2798 : bool in_p, tree low, tree high)
2799 : {
2800 25865 : unsigned int idx = range->idx;
2801 25865 : struct range_entry *swap_with = NULL;
2802 25865 : basic_block rewrite_bb_first = NULL, rewrite_bb_last = NULL;
2803 25865 : if (opcode == ERROR_MARK)
2804 : {
2805 : /* For inter-bb range test optimization, pick from the range tests
2806 : the one which is tested in the earliest condition (one dominating
2807 : the others), because otherwise there could be some UB (e.g. signed
2808 : overflow) in following bbs that we'd expose which wasn't there in
2809 : the original program. See PR104196. */
2810 15052 : basic_block orig_range_bb = BASIC_BLOCK_FOR_FN (cfun, (*ops)[idx]->id);
2811 15052 : basic_block range_bb = orig_range_bb;
2812 33143 : for (unsigned int i = 0; i < count; i++)
2813 : {
2814 18091 : struct range_entry *this_range;
2815 18091 : if (otherrange)
2816 10970 : this_range = otherrange + i;
2817 : else
2818 7121 : this_range = otherrangep[i];
2819 18091 : operand_entry *oe = (*ops)[this_range->idx];
2820 18091 : basic_block this_bb = BASIC_BLOCK_FOR_FN (cfun, oe->id);
2821 18091 : if (range_bb != this_bb
2822 18091 : && dominated_by_p (CDI_DOMINATORS, range_bb, this_bb))
2823 : {
2824 6902 : swap_with = this_range;
2825 6902 : range_bb = this_bb;
2826 6902 : idx = this_range->idx;
2827 : }
2828 : }
2829 : /* If seq is non-NULL, it can contain statements that use SSA_NAMEs
2830 : only defined in later blocks. In this case we can't move the
2831 : merged comparison earlier, so instead check if there are any stmts
2832 : that might trigger signed integer overflow in between and rewrite
2833 : them. But only after we check if the optimization is possible. */
2834 15052 : if (seq && swap_with)
2835 : {
2836 3762 : rewrite_bb_first = range_bb;
2837 3762 : rewrite_bb_last = orig_range_bb;
2838 3762 : idx = range->idx;
2839 3762 : swap_with = NULL;
2840 : }
2841 : }
2842 25865 : operand_entry *oe = (*ops)[idx];
2843 25865 : tree op = oe->op;
2844 25865 : gimple *stmt = op ? SSA_NAME_DEF_STMT (op)
2845 12851 : : last_nondebug_stmt (BASIC_BLOCK_FOR_FN (cfun, oe->id));
2846 25865 : location_t loc = gimple_location (stmt);
2847 25865 : tree optype = op ? TREE_TYPE (op) : boolean_type_node;
2848 25865 : tree tem = build_range_check (loc, optype, unshare_expr (exp),
2849 : in_p, low, high);
2850 25865 : gimple_stmt_iterator gsi;
2851 25865 : unsigned int i, uid;
2852 :
2853 25865 : if (tem == NULL_TREE)
2854 : return false;
2855 :
2856 : /* If op is default def SSA_NAME, there is no place to insert the
2857 : new comparison. Give up, unless we can use OP itself as the
2858 : range test. */
2859 38879 : if (op && SSA_NAME_IS_DEFAULT_DEF (op))
2860 : {
2861 0 : if (op == range->exp
2862 0 : && ((TYPE_PRECISION (optype) == 1 && TYPE_UNSIGNED (optype))
2863 0 : || TREE_CODE (optype) == BOOLEAN_TYPE)
2864 0 : && (op == tem
2865 0 : || (TREE_CODE (tem) == EQ_EXPR
2866 0 : && TREE_OPERAND (tem, 0) == op
2867 0 : && integer_onep (TREE_OPERAND (tem, 1))))
2868 0 : && opcode != BIT_IOR_EXPR
2869 0 : && (opcode != ERROR_MARK || oe->rank != BIT_IOR_EXPR))
2870 : {
2871 : stmt = NULL;
2872 : tem = op;
2873 : }
2874 : else
2875 : return false;
2876 : }
2877 :
2878 25865 : if (swap_with)
2879 1579 : std::swap (range->idx, swap_with->idx);
2880 :
2881 25865 : if (dump_file && (dump_flags & TDF_DETAILS))
2882 : {
2883 39 : struct range_entry *r;
2884 39 : fprintf (dump_file, "Optimizing range tests ");
2885 39 : dump_range_entry (dump_file, range, false);
2886 180 : for (i = 0; i < count; i++)
2887 : {
2888 102 : if (otherrange)
2889 82 : r = otherrange + i;
2890 : else
2891 20 : r = otherrangep[i];
2892 102 : if (r->exp
2893 102 : && r->exp != range->exp
2894 20 : && TREE_CODE (r->exp) == SSA_NAME)
2895 : {
2896 20 : fprintf (dump_file, " and ");
2897 20 : dump_range_entry (dump_file, r, false);
2898 : }
2899 : else
2900 : {
2901 82 : fprintf (dump_file, " and");
2902 82 : dump_range_entry (dump_file, r, true);
2903 : }
2904 : }
2905 39 : fprintf (dump_file, "\n into ");
2906 39 : print_generic_expr (dump_file, tem);
2907 39 : fprintf (dump_file, "\n");
2908 : }
2909 :
2910 : /* In inter-bb range optimization mode, if we have a seq, we can't
2911 : move the merged comparison to the earliest bb from the comparisons
2912 : being replaced, so instead rewrite stmts that could trigger signed
2913 : integer overflow. */
2914 32755 : for (basic_block bb = rewrite_bb_last;
2915 32755 : bb != rewrite_bb_first; bb = single_pred (bb))
2916 13780 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2917 28709 : !gsi_end_p (gsi); gsi_next (&gsi))
2918 : {
2919 21819 : gimple *stmt = gsi_stmt (gsi);
2920 21819 : if (gimple_needing_rewrite_undefined (stmt))
2921 : {
2922 63 : gimple_stmt_iterator gsip = gsi;
2923 63 : gimple_stmt_iterator gsin = gsi;
2924 63 : gsi_prev (&gsip);
2925 63 : gsi_next (&gsin);
2926 63 : rewrite_to_defined_unconditional (&gsi);
2927 63 : unsigned uid = gimple_uid (stmt);
2928 63 : if (gsi_end_p (gsip))
2929 27 : gsip = gsi_after_labels (bb);
2930 : else
2931 36 : gsi_next (&gsip);
2932 253 : for (; gsi_stmt (gsip) != gsi_stmt (gsin);
2933 190 : gsi_next (&gsip))
2934 190 : gimple_set_uid (gsi_stmt (gsip), uid);
2935 : }
2936 : }
2937 :
2938 25865 : if (opcode == BIT_IOR_EXPR
2939 19553 : || (opcode == ERROR_MARK && oe->rank == BIT_IOR_EXPR))
2940 18362 : tem = invert_truthvalue_loc (loc, tem);
2941 :
2942 25865 : tem = fold_convert_loc (loc, optype, tem);
2943 25865 : if (stmt)
2944 : {
2945 25865 : gsi = gsi_for_stmt (stmt);
2946 25865 : uid = gimple_uid (stmt);
2947 : }
2948 : else
2949 : {
2950 0 : gsi = gsi_none ();
2951 0 : uid = 0;
2952 : }
2953 25865 : if (stmt == NULL)
2954 0 : gcc_checking_assert (tem == op);
2955 : /* In rare cases range->exp can be equal to lhs of stmt.
2956 : In that case we have to insert after the stmt rather then before
2957 : it. If stmt is a PHI, insert it at the start of the basic block. */
2958 25865 : else if (op != range->exp)
2959 : {
2960 25865 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
2961 25865 : tem = force_into_ssa_name (&gsi, tem, true);
2962 25865 : gsi_prev (&gsi);
2963 : }
2964 0 : else if (gimple_code (stmt) != GIMPLE_PHI)
2965 : {
2966 0 : gsi_insert_seq_after (&gsi, seq, GSI_CONTINUE_LINKING);
2967 0 : tem = force_into_ssa_name (&gsi, tem, false);
2968 : }
2969 : else
2970 : {
2971 0 : gsi = gsi_after_labels (gimple_bb (stmt));
2972 0 : if (!gsi_end_p (gsi))
2973 0 : uid = gimple_uid (gsi_stmt (gsi));
2974 : else
2975 : {
2976 0 : gsi = gsi_start_bb (gimple_bb (stmt));
2977 0 : uid = 1;
2978 0 : while (!gsi_end_p (gsi))
2979 : {
2980 0 : uid = gimple_uid (gsi_stmt (gsi));
2981 0 : gsi_next (&gsi);
2982 : }
2983 : }
2984 0 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
2985 0 : tem = force_into_ssa_name (&gsi, tem, true);
2986 0 : if (gsi_end_p (gsi))
2987 0 : gsi = gsi_last_bb (gimple_bb (stmt));
2988 : else
2989 0 : gsi_prev (&gsi);
2990 : }
2991 96655 : for (; !gsi_end_p (gsi); gsi_prev (&gsi))
2992 89379 : if (gimple_uid (gsi_stmt (gsi)))
2993 : break;
2994 : else
2995 67152 : gimple_set_uid (gsi_stmt (gsi), uid);
2996 :
2997 25865 : oe->op = tem;
2998 25865 : range->exp = exp;
2999 25865 : range->low = low;
3000 25865 : range->high = high;
3001 25865 : range->in_p = in_p;
3002 :
3003 55941 : for (i = 0; i < count; i++)
3004 : {
3005 30076 : if (otherrange)
3006 18943 : range = otherrange + i;
3007 : else
3008 11133 : range = otherrangep[i];
3009 30076 : oe = (*ops)[range->idx];
3010 : /* Now change all the other range test immediate uses, so that
3011 : those tests will be optimized away. */
3012 30076 : if (opcode == ERROR_MARK)
3013 : {
3014 18091 : if (oe->op)
3015 2341 : oe->op = build_int_cst (TREE_TYPE (oe->op),
3016 2341 : oe->rank == BIT_IOR_EXPR ? 0 : 1);
3017 : else
3018 15750 : oe->op = (oe->rank == BIT_IOR_EXPR
3019 15750 : ? boolean_false_node : boolean_true_node);
3020 : }
3021 : else
3022 11985 : oe->op = error_mark_node;
3023 30076 : range->exp = NULL_TREE;
3024 30076 : range->low = NULL_TREE;
3025 30076 : range->high = NULL_TREE;
3026 : }
3027 : return true;
3028 : }
3029 :
3030 : /* Optimize X == CST1 || X == CST2
3031 : if popcount (CST1 ^ CST2) == 1 into
3032 : (X & ~(CST1 ^ CST2)) == (CST1 & ~(CST1 ^ CST2)).
3033 : Similarly for ranges. E.g.
3034 : X != 2 && X != 3 && X != 10 && X != 11
3035 : will be transformed by the previous optimization into
3036 : !((X - 2U) <= 1U || (X - 10U) <= 1U)
3037 : and this loop can transform that into
3038 : !(((X & ~8) - 2U) <= 1U). */
3039 :
3040 : static bool
3041 23230 : optimize_range_tests_xor (enum tree_code opcode, tree type,
3042 : tree lowi, tree lowj, tree highi, tree highj,
3043 : vec<operand_entry *> *ops,
3044 : struct range_entry *rangei,
3045 : struct range_entry *rangej)
3046 : {
3047 23230 : tree lowxor, highxor, tem, exp;
3048 : /* Check lowi ^ lowj == highi ^ highj and
3049 : popcount (lowi ^ lowj) == 1. */
3050 23230 : lowxor = fold_binary (BIT_XOR_EXPR, type, lowi, lowj);
3051 23230 : if (lowxor == NULL_TREE || TREE_CODE (lowxor) != INTEGER_CST)
3052 : return false;
3053 23230 : if (!integer_pow2p (lowxor))
3054 : return false;
3055 3313 : highxor = fold_binary (BIT_XOR_EXPR, type, highi, highj);
3056 3313 : if (!tree_int_cst_equal (lowxor, highxor))
3057 : return false;
3058 :
3059 2902 : exp = rangei->exp;
3060 2902 : scalar_int_mode mode = as_a <scalar_int_mode> (TYPE_MODE (type));
3061 2902 : int prec = GET_MODE_PRECISION (mode);
3062 2902 : if (TYPE_PRECISION (type) < prec
3063 2901 : || (wi::to_wide (TYPE_MIN_VALUE (type))
3064 8704 : != wi::min_value (prec, TYPE_SIGN (type)))
3065 5803 : || (wi::to_wide (TYPE_MAX_VALUE (type))
3066 8704 : != wi::max_value (prec, TYPE_SIGN (type))))
3067 : {
3068 1 : type = build_nonstandard_integer_type (prec, TYPE_UNSIGNED (type));
3069 1 : exp = fold_convert (type, exp);
3070 1 : lowxor = fold_convert (type, lowxor);
3071 1 : lowi = fold_convert (type, lowi);
3072 1 : highi = fold_convert (type, highi);
3073 : }
3074 2902 : tem = fold_build1 (BIT_NOT_EXPR, type, lowxor);
3075 2902 : exp = fold_build2 (BIT_AND_EXPR, type, exp, tem);
3076 2902 : lowj = fold_build2 (BIT_AND_EXPR, type, lowi, tem);
3077 2902 : highj = fold_build2 (BIT_AND_EXPR, type, highi, tem);
3078 2902 : if (update_range_test (rangei, rangej, NULL, 1, opcode, ops, exp,
3079 : NULL, rangei->in_p, lowj, highj))
3080 : return true;
3081 : return false;
3082 : }
3083 :
3084 : /* Optimize X == CST1 || X == CST2
3085 : if popcount (CST2 - CST1) == 1 into
3086 : ((X - CST1) & ~(CST2 - CST1)) == 0.
3087 : Similarly for ranges. E.g.
3088 : X == 43 || X == 76 || X == 44 || X == 78 || X == 77 || X == 46
3089 : || X == 75 || X == 45
3090 : will be transformed by the previous optimization into
3091 : (X - 43U) <= 3U || (X - 75U) <= 3U
3092 : and this loop can transform that into
3093 : ((X - 43U) & ~(75U - 43U)) <= 3U. */
3094 : static bool
3095 17981 : optimize_range_tests_diff (enum tree_code opcode, tree type,
3096 : tree lowi, tree lowj, tree highi, tree highj,
3097 : vec<operand_entry *> *ops,
3098 : struct range_entry *rangei,
3099 : struct range_entry *rangej)
3100 : {
3101 17981 : tree tem1, tem2, mask;
3102 : /* Check highi - lowi == highj - lowj. */
3103 17981 : tem1 = fold_binary (MINUS_EXPR, type, highi, lowi);
3104 17981 : if (tem1 == NULL_TREE || TREE_CODE (tem1) != INTEGER_CST)
3105 : return false;
3106 17981 : tem2 = fold_binary (MINUS_EXPR, type, highj, lowj);
3107 17981 : if (!tree_int_cst_equal (tem1, tem2))
3108 : return false;
3109 : /* Check popcount (lowj - lowi) == 1. */
3110 11904 : tem1 = fold_binary (MINUS_EXPR, type, lowj, lowi);
3111 11904 : if (tem1 == NULL_TREE || TREE_CODE (tem1) != INTEGER_CST)
3112 : return false;
3113 11904 : if (!integer_pow2p (tem1))
3114 : return false;
3115 :
3116 1970 : scalar_int_mode mode = as_a <scalar_int_mode> (TYPE_MODE (type));
3117 1970 : int prec = GET_MODE_PRECISION (mode);
3118 1970 : if (TYPE_PRECISION (type) < prec
3119 1966 : || (wi::to_wide (TYPE_MIN_VALUE (type))
3120 5902 : != wi::min_value (prec, TYPE_SIGN (type)))
3121 3936 : || (wi::to_wide (TYPE_MAX_VALUE (type))
3122 5902 : != wi::max_value (prec, TYPE_SIGN (type))))
3123 4 : type = build_nonstandard_integer_type (prec, 1);
3124 : else
3125 1966 : type = unsigned_type_for (type);
3126 1970 : tem1 = fold_convert (type, tem1);
3127 1970 : tem2 = fold_convert (type, tem2);
3128 1970 : lowi = fold_convert (type, lowi);
3129 1970 : mask = fold_build1 (BIT_NOT_EXPR, type, tem1);
3130 1970 : tem1 = fold_build2 (MINUS_EXPR, type,
3131 : fold_convert (type, rangei->exp), lowi);
3132 1970 : tem1 = fold_build2 (BIT_AND_EXPR, type, tem1, mask);
3133 1970 : lowj = build_int_cst (type, 0);
3134 1970 : if (update_range_test (rangei, rangej, NULL, 1, opcode, ops, tem1,
3135 : NULL, rangei->in_p, lowj, tem2))
3136 : return true;
3137 : return false;
3138 : }
3139 :
3140 : /* It does some common checks for function optimize_range_tests_xor and
3141 : optimize_range_tests_diff.
3142 : If OPTIMIZE_XOR is TRUE, it calls optimize_range_tests_xor.
3143 : Else it calls optimize_range_tests_diff. */
3144 :
3145 : static bool
3146 2175774 : optimize_range_tests_1 (enum tree_code opcode, int first, int length,
3147 : bool optimize_xor, vec<operand_entry *> *ops,
3148 : struct range_entry *ranges)
3149 : {
3150 2175774 : int i, j;
3151 2175774 : bool any_changes = false;
3152 3805700 : for (i = first; i < length; i++)
3153 : {
3154 1629926 : tree lowi, highi, lowj, highj, type, tem;
3155 :
3156 1629926 : if (ranges[i].exp == NULL_TREE || ranges[i].in_p)
3157 1011600 : continue;
3158 618326 : type = TREE_TYPE (ranges[i].exp);
3159 618326 : if (!INTEGRAL_TYPE_P (type))
3160 51048 : continue;
3161 567278 : lowi = ranges[i].low;
3162 567278 : if (lowi == NULL_TREE)
3163 35726 : lowi = TYPE_MIN_VALUE (type);
3164 567278 : highi = ranges[i].high;
3165 567278 : if (highi == NULL_TREE)
3166 6800 : continue;
3167 949362 : for (j = i + 1; j < length && j < i + 64; j++)
3168 : {
3169 393756 : bool changes;
3170 393756 : if (ranges[i].exp != ranges[j].exp || ranges[j].in_p)
3171 352545 : continue;
3172 41211 : lowj = ranges[j].low;
3173 41211 : if (lowj == NULL_TREE)
3174 0 : continue;
3175 41211 : highj = ranges[j].high;
3176 41211 : if (highj == NULL_TREE)
3177 154 : highj = TYPE_MAX_VALUE (type);
3178 : /* Check lowj > highi. */
3179 41211 : tem = fold_binary (GT_EXPR, boolean_type_node,
3180 : lowj, highi);
3181 41211 : if (tem == NULL_TREE || !integer_onep (tem))
3182 0 : continue;
3183 41211 : if (optimize_xor)
3184 23230 : changes = optimize_range_tests_xor (opcode, type, lowi, lowj,
3185 : highi, highj, ops,
3186 : ranges + i, ranges + j);
3187 : else
3188 17981 : changes = optimize_range_tests_diff (opcode, type, lowi, lowj,
3189 : highi, highj, ops,
3190 : ranges + i, ranges + j);
3191 41211 : if (changes)
3192 : {
3193 : any_changes = true;
3194 : break;
3195 : }
3196 : }
3197 : }
3198 2175774 : return any_changes;
3199 : }
3200 :
3201 : /* Helper function of optimize_range_tests_to_bit_test. Handle a single
3202 : range, EXP, LOW, HIGH, compute bit mask of bits to test and return
3203 : EXP on success, NULL otherwise. */
3204 :
3205 : static tree
3206 170411 : extract_bit_test_mask (tree exp, int prec, tree totallow, tree low, tree high,
3207 : wide_int *mask, tree *totallowp)
3208 : {
3209 170411 : tree tem = int_const_binop (MINUS_EXPR, high, low);
3210 170411 : if (tem == NULL_TREE
3211 170411 : || TREE_CODE (tem) != INTEGER_CST
3212 170411 : || TREE_OVERFLOW (tem)
3213 159341 : || tree_int_cst_sgn (tem) == -1
3214 329752 : || compare_tree_int (tem, prec) != -1)
3215 : return NULL_TREE;
3216 :
3217 155359 : unsigned HOST_WIDE_INT max = tree_to_uhwi (tem) + 1;
3218 155359 : *mask = wi::shifted_mask (0, max, false, prec);
3219 155359 : if (TREE_CODE (exp) == BIT_AND_EXPR
3220 155359 : && TREE_CODE (TREE_OPERAND (exp, 1)) == INTEGER_CST)
3221 : {
3222 5168 : widest_int msk = wi::to_widest (TREE_OPERAND (exp, 1));
3223 5168 : msk = wi::zext (~msk, TYPE_PRECISION (TREE_TYPE (exp)));
3224 5168 : if (wi::popcount (msk) == 1
3225 5168 : && wi::ltu_p (msk, prec - max))
3226 : {
3227 4405 : *mask |= wi::shifted_mask (msk.to_uhwi (), max, false, prec);
3228 4405 : max += msk.to_uhwi ();
3229 4405 : exp = TREE_OPERAND (exp, 0);
3230 4405 : if (integer_zerop (low)
3231 2209 : && TREE_CODE (exp) == PLUS_EXPR
3232 6226 : && TREE_CODE (TREE_OPERAND (exp, 1)) == INTEGER_CST)
3233 : {
3234 1821 : tree ret = TREE_OPERAND (exp, 0);
3235 1821 : STRIP_NOPS (ret);
3236 1821 : widest_int bias
3237 1821 : = wi::neg (wi::sext (wi::to_widest (TREE_OPERAND (exp, 1)),
3238 3642 : TYPE_PRECISION (TREE_TYPE (low))));
3239 1821 : tree tbias = wide_int_to_tree (TREE_TYPE (ret), bias);
3240 1821 : if (totallowp)
3241 : {
3242 1785 : *totallowp = tbias;
3243 1785 : return ret;
3244 : }
3245 36 : else if (!tree_int_cst_lt (totallow, tbias))
3246 : return NULL_TREE;
3247 36 : bias = wi::to_widest (tbias);
3248 36 : bias -= wi::to_widest (totallow);
3249 36 : if (bias >= 0 && bias < prec - max)
3250 : {
3251 26 : *mask = wi::lshift (*mask, bias);
3252 26 : return ret;
3253 : }
3254 1821 : }
3255 : }
3256 5168 : }
3257 153548 : if (totallowp)
3258 : return exp;
3259 13827 : if (!tree_int_cst_lt (totallow, low))
3260 : return exp;
3261 13805 : tem = int_const_binop (MINUS_EXPR, low, totallow);
3262 13805 : if (tem == NULL_TREE
3263 13805 : || TREE_CODE (tem) != INTEGER_CST
3264 13805 : || TREE_OVERFLOW (tem)
3265 27453 : || compare_tree_int (tem, prec - max) == 1)
3266 : return NULL_TREE;
3267 :
3268 9793 : *mask = wi::lshift (*mask, wi::to_widest (tem));
3269 9793 : return exp;
3270 : }
3271 :
3272 : /* Attempt to optimize small range tests using bit test.
3273 : E.g.
3274 : X != 43 && X != 76 && X != 44 && X != 78 && X != 49
3275 : && X != 77 && X != 46 && X != 75 && X != 45 && X != 82
3276 : has been by earlier optimizations optimized into:
3277 : ((X - 43U) & ~32U) > 3U && X != 49 && X != 82
3278 : As all the 43 through 82 range is less than 64 numbers,
3279 : for 64-bit word targets optimize that into:
3280 : (X - 43U) > 40U && ((1 << (X - 43U)) & 0x8F0000004FULL) == 0 */
3281 :
3282 : static bool
3283 1087895 : optimize_range_tests_to_bit_test (enum tree_code opcode, int first, int length,
3284 : vec<operand_entry *> *ops,
3285 : struct range_entry *ranges)
3286 : {
3287 1087895 : int i, j;
3288 1087895 : bool any_changes = false;
3289 1087895 : int prec = GET_MODE_BITSIZE (word_mode);
3290 1087895 : auto_vec<struct range_entry *, 64> candidates;
3291 :
3292 1521134 : for (i = first; i < length - 1; i++)
3293 : {
3294 433239 : tree lowi, highi, lowj, highj, type;
3295 :
3296 433239 : if (ranges[i].exp == NULL_TREE || ranges[i].in_p)
3297 291733 : continue;
3298 176452 : type = TREE_TYPE (ranges[i].exp);
3299 176452 : if (!INTEGRAL_TYPE_P (type))
3300 17597 : continue;
3301 158855 : lowi = ranges[i].low;
3302 158855 : if (lowi == NULL_TREE)
3303 11169 : lowi = TYPE_MIN_VALUE (type);
3304 158855 : highi = ranges[i].high;
3305 158855 : if (highi == NULL_TREE)
3306 2414 : continue;
3307 156441 : wide_int mask;
3308 156441 : tree exp = extract_bit_test_mask (ranges[i].exp, prec, lowi, lowi,
3309 : highi, &mask, &lowi);
3310 156441 : if (exp == NULL_TREE)
3311 14935 : continue;
3312 141506 : candidates.truncate (0);
3313 141506 : int end = MIN (i + 64, length);
3314 310241 : for (j = i + 1; j < end; j++)
3315 : {
3316 168735 : tree exp2;
3317 168735 : if (ranges[j].exp == NULL_TREE || ranges[j].in_p)
3318 158924 : continue;
3319 100310 : if (ranges[j].exp == exp)
3320 : ;
3321 86828 : else if (TREE_CODE (ranges[j].exp) == BIT_AND_EXPR)
3322 : {
3323 1455 : exp2 = TREE_OPERAND (ranges[j].exp, 0);
3324 1455 : if (exp2 == exp)
3325 : ;
3326 1015 : else if (TREE_CODE (exp2) == PLUS_EXPR)
3327 : {
3328 814 : exp2 = TREE_OPERAND (exp2, 0);
3329 814 : STRIP_NOPS (exp2);
3330 814 : if (exp2 != exp)
3331 766 : continue;
3332 : }
3333 : else
3334 201 : continue;
3335 : }
3336 : else
3337 85373 : continue;
3338 13970 : lowj = ranges[j].low;
3339 13970 : if (lowj == NULL_TREE)
3340 0 : continue;
3341 13970 : highj = ranges[j].high;
3342 13970 : if (highj == NULL_TREE)
3343 77 : highj = TYPE_MAX_VALUE (TREE_TYPE (lowj));
3344 13970 : wide_int mask2;
3345 13970 : exp2 = extract_bit_test_mask (ranges[j].exp, prec, lowi, lowj,
3346 : highj, &mask2, NULL);
3347 13970 : if (exp2 != exp)
3348 4159 : continue;
3349 9811 : mask |= mask2;
3350 9811 : candidates.safe_push (&ranges[j]);
3351 13970 : }
3352 :
3353 : /* If every possible relative value of the expression is a valid shift
3354 : amount, then we can merge the entry test in the bit test. In this
3355 : case, if we would need otherwise 2 or more comparisons, then use
3356 : the bit test; in the other cases, the threshold is 3 comparisons. */
3357 141506 : bool entry_test_needed;
3358 141506 : int_range_max r;
3359 283012 : if (TREE_CODE (exp) == SSA_NAME
3360 281586 : && get_range_query (cfun)->range_of_expr (r, exp)
3361 140793 : && !r.undefined_p ()
3362 140793 : && !r.varying_p ()
3363 323252 : && wi::leu_p (r.upper_bound () - r.lower_bound (), prec - 1))
3364 : {
3365 4875 : wide_int min = r.lower_bound ();
3366 4875 : wide_int ilowi = wi::to_wide (lowi);
3367 4875 : if (wi::lt_p (min, ilowi, TYPE_SIGN (TREE_TYPE (lowi))))
3368 : {
3369 565 : lowi = wide_int_to_tree (TREE_TYPE (lowi), min);
3370 565 : mask = wi::lshift (mask, ilowi - min);
3371 : }
3372 4310 : else if (wi::gt_p (min, ilowi, TYPE_SIGN (TREE_TYPE (lowi))))
3373 : {
3374 1 : lowi = wide_int_to_tree (TREE_TYPE (lowi), min);
3375 1 : mask = wi::lrshift (mask, min - ilowi);
3376 : }
3377 4875 : entry_test_needed = false;
3378 4875 : }
3379 : else
3380 : entry_test_needed = true;
3381 287887 : if (candidates.length () >= (entry_test_needed ? 2 : 1))
3382 : {
3383 894 : tree high = wide_int_to_tree (TREE_TYPE (lowi),
3384 447 : wi::to_widest (lowi)
3385 1341 : + prec - 1 - wi::clz (mask));
3386 447 : operand_entry *oe = (*ops)[ranges[i].idx];
3387 447 : tree op = oe->op;
3388 447 : gimple *stmt = op ? SSA_NAME_DEF_STMT (op)
3389 50 : : last_nondebug_stmt (BASIC_BLOCK_FOR_FN
3390 397 : (cfun, oe->id));
3391 447 : location_t loc = gimple_location (stmt);
3392 447 : tree optype = op ? TREE_TYPE (op) : boolean_type_node;
3393 :
3394 : /* See if it isn't cheaper to pretend the minimum value of the
3395 : range is 0, if maximum value is small enough.
3396 : We can avoid then subtraction of the minimum value, but the
3397 : mask constant could be perhaps more expensive. */
3398 447 : if (compare_tree_int (lowi, 0) > 0
3399 370 : && compare_tree_int (high, prec) < 0
3400 831 : && (entry_test_needed || wi::ltu_p (r.upper_bound (), prec)))
3401 : {
3402 152 : int cost_diff;
3403 152 : HOST_WIDE_INT m = tree_to_uhwi (lowi);
3404 152 : rtx reg = gen_raw_REG (word_mode, 10000);
3405 152 : bool speed_p = optimize_bb_for_speed_p (gimple_bb (stmt));
3406 152 : cost_diff = set_src_cost (gen_rtx_PLUS (word_mode, reg,
3407 : GEN_INT (-m)),
3408 : word_mode, speed_p);
3409 152 : rtx r = immed_wide_int_const (mask, word_mode);
3410 152 : cost_diff += set_src_cost (gen_rtx_AND (word_mode, reg, r),
3411 : word_mode, speed_p);
3412 152 : r = immed_wide_int_const (wi::lshift (mask, m), word_mode);
3413 152 : cost_diff -= set_src_cost (gen_rtx_AND (word_mode, reg, r),
3414 : word_mode, speed_p);
3415 152 : if (cost_diff > 0)
3416 : {
3417 62 : mask = wi::lshift (mask, m);
3418 62 : lowi = build_zero_cst (TREE_TYPE (lowi));
3419 : }
3420 : }
3421 :
3422 447 : tree tem;
3423 447 : if (entry_test_needed)
3424 : {
3425 386 : tem = build_range_check (loc, optype, unshare_expr (exp),
3426 : false, lowi, high);
3427 386 : if (tem == NULL_TREE || is_gimple_val (tem))
3428 0 : continue;
3429 : }
3430 : else
3431 61 : tem = NULL_TREE;
3432 447 : tree etype = unsigned_type_for (TREE_TYPE (exp));
3433 447 : exp = fold_build2_loc (loc, MINUS_EXPR, etype,
3434 : fold_convert_loc (loc, etype, exp),
3435 : fold_convert_loc (loc, etype, lowi));
3436 447 : exp = fold_convert_loc (loc, integer_type_node, exp);
3437 447 : tree word_type = lang_hooks.types.type_for_mode (word_mode, 1);
3438 447 : exp = fold_build2_loc (loc, LSHIFT_EXPR, word_type,
3439 : build_int_cst (word_type, 1), exp);
3440 894 : exp = fold_build2_loc (loc, BIT_AND_EXPR, word_type, exp,
3441 447 : wide_int_to_tree (word_type, mask));
3442 447 : exp = fold_build2_loc (loc, EQ_EXPR, optype, exp,
3443 : build_zero_cst (word_type));
3444 447 : if (is_gimple_val (exp))
3445 0 : continue;
3446 :
3447 : /* The shift might have undefined behavior if TEM is true,
3448 : but reassociate_bb isn't prepared to have basic blocks
3449 : split when it is running. So, temporarily emit a code
3450 : with BIT_IOR_EXPR instead of &&, and fix it up in
3451 : branch_fixup. */
3452 447 : gimple_seq seq = NULL;
3453 447 : if (tem)
3454 : {
3455 386 : tem = force_gimple_operand (tem, &seq, true, NULL_TREE);
3456 386 : gcc_assert (TREE_CODE (tem) == SSA_NAME);
3457 386 : gimple_set_visited (SSA_NAME_DEF_STMT (tem), true);
3458 : }
3459 447 : gimple_seq seq2;
3460 447 : exp = force_gimple_operand (exp, &seq2, true, NULL_TREE);
3461 447 : gimple_seq_add_seq_without_update (&seq, seq2);
3462 447 : gcc_assert (TREE_CODE (exp) == SSA_NAME);
3463 447 : gimple_set_visited (SSA_NAME_DEF_STMT (exp), true);
3464 447 : if (tem)
3465 : {
3466 386 : gimple *g = gimple_build_assign (make_ssa_name (optype),
3467 : BIT_IOR_EXPR, tem, exp);
3468 386 : gimple_set_location (g, loc);
3469 386 : gimple_seq_add_stmt_without_update (&seq, g);
3470 386 : exp = gimple_assign_lhs (g);
3471 : }
3472 447 : tree val = build_zero_cst (optype);
3473 1341 : if (update_range_test (&ranges[i], NULL, candidates.address (),
3474 : candidates.length (), opcode, ops, exp,
3475 : seq, false, val, val))
3476 : {
3477 447 : any_changes = true;
3478 447 : if (tem)
3479 386 : reassoc_branch_fixups.safe_push (tem);
3480 : }
3481 : else
3482 0 : gimple_seq_discard (seq);
3483 : }
3484 156441 : }
3485 1087895 : return any_changes;
3486 1087895 : }
3487 :
3488 : /* Optimize x != 0 && y != 0 && z != 0 into (x | y | z) != 0
3489 : and similarly x != -1 && y != -1 && y != -1 into (x & y & z) != -1.
3490 : Also, handle x < C && y < C && z < C where C is power of two as
3491 : (x | y | z) < C. And also handle signed x < 0 && y < 0 && z < 0
3492 : as (x | y | z) < 0. */
3493 :
3494 : static bool
3495 1087895 : optimize_range_tests_cmp_bitwise (enum tree_code opcode, int first, int length,
3496 : vec<operand_entry *> *ops,
3497 : struct range_entry *ranges)
3498 : {
3499 1087895 : int i;
3500 1087895 : unsigned int b;
3501 1087895 : bool any_changes = false;
3502 1087895 : auto_vec<int, 128> buckets;
3503 1087895 : auto_vec<int, 32> chains;
3504 1087895 : auto_vec<struct range_entry *, 32> candidates;
3505 :
3506 1902865 : for (i = first; i < length; i++)
3507 : {
3508 814970 : int idx;
3509 :
3510 1164874 : if (ranges[i].exp == NULL_TREE
3511 794384 : || TREE_CODE (ranges[i].exp) != SSA_NAME
3512 789744 : || TYPE_PRECISION (TREE_TYPE (ranges[i].exp)) <= 1
3513 1280036 : || TREE_CODE (TREE_TYPE (ranges[i].exp)) == BOOLEAN_TYPE)
3514 349904 : continue;
3515 :
3516 465066 : if (ranges[i].low != NULL_TREE
3517 437941 : && ranges[i].high != NULL_TREE
3518 383194 : && ranges[i].in_p
3519 692731 : && tree_int_cst_equal (ranges[i].low, ranges[i].high))
3520 : {
3521 197262 : idx = !integer_zerop (ranges[i].low);
3522 197262 : if (idx && !integer_all_onesp (ranges[i].low))
3523 108756 : continue;
3524 : }
3525 267804 : else if (ranges[i].high != NULL_TREE
3526 213023 : && TREE_CODE (ranges[i].high) == INTEGER_CST
3527 213023 : && ranges[i].in_p)
3528 : {
3529 39665 : wide_int w = wi::to_wide (ranges[i].high);
3530 39665 : int prec = TYPE_PRECISION (TREE_TYPE (ranges[i].exp));
3531 39665 : int l = wi::clz (w);
3532 39665 : idx = 2;
3533 108703 : if (l <= 0
3534 39665 : || l >= prec
3535 73252 : || w != wi::mask (prec - l, false, prec))
3536 29373 : continue;
3537 10292 : if (!((TYPE_UNSIGNED (TREE_TYPE (ranges[i].exp))
3538 6497 : && ranges[i].low == NULL_TREE)
3539 10292 : || (ranges[i].low
3540 8299 : && integer_zerop (ranges[i].low))))
3541 3388 : continue;
3542 39665 : }
3543 432439 : else if (ranges[i].high == NULL_TREE
3544 54781 : && ranges[i].low != NULL_TREE
3545 : /* Perform this optimization only in the last
3546 : reassoc pass, as it interferes with the reassociation
3547 : itself or could also with VRP etc. which might not
3548 : be able to virtually undo the optimization. */
3549 54747 : && !reassoc_insert_powi_p
3550 27799 : && !TYPE_UNSIGNED (TREE_TYPE (ranges[i].exp))
3551 255371 : && integer_zerop (ranges[i].low))
3552 : idx = 3;
3553 : else
3554 204300 : continue;
3555 :
3556 119249 : b = TYPE_PRECISION (TREE_TYPE (ranges[i].exp)) * 4 + idx;
3557 119249 : if (buckets.length () <= b)
3558 105156 : buckets.safe_grow_cleared (b + 1, true);
3559 119249 : if (chains.length () <= (unsigned) i)
3560 119249 : chains.safe_grow (i + 1, true);
3561 119249 : chains[i] = buckets[b];
3562 119249 : buckets[b] = i + 1;
3563 : }
3564 :
3565 20154782 : FOR_EACH_VEC_ELT (buckets, b, i)
3566 19066887 : if (i && chains[i - 1])
3567 : {
3568 7063 : int j, k = i;
3569 7063 : if ((b % 4) == 2)
3570 : {
3571 : /* When ranges[X - 1].high + 1 is a power of two,
3572 : we need to process the same bucket up to
3573 : precision - 1 times, each time split the entries
3574 : with the same high bound into one chain and the
3575 : rest into another one to be processed later. */
3576 : int this_prev = i;
3577 : int other_prev = 0;
3578 146 : for (j = chains[i - 1]; j; j = chains[j - 1])
3579 : {
3580 79 : if (tree_int_cst_equal (ranges[i - 1].high,
3581 79 : ranges[j - 1].high))
3582 : {
3583 70 : chains[this_prev - 1] = j;
3584 70 : this_prev = j;
3585 : }
3586 9 : else if (other_prev == 0)
3587 : {
3588 7 : buckets[b] = j;
3589 7 : other_prev = j;
3590 : }
3591 : else
3592 : {
3593 2 : chains[other_prev - 1] = j;
3594 2 : other_prev = j;
3595 : }
3596 : }
3597 67 : chains[this_prev - 1] = 0;
3598 67 : if (other_prev)
3599 7 : chains[other_prev - 1] = 0;
3600 67 : if (chains[i - 1] == 0)
3601 : {
3602 5 : if (other_prev)
3603 5 : b--;
3604 5 : continue;
3605 : }
3606 : }
3607 17105 : for (j = chains[i - 1]; j; j = chains[j - 1])
3608 : {
3609 10047 : gimple *gk = SSA_NAME_DEF_STMT (ranges[k - 1].exp);
3610 10047 : gimple *gj = SSA_NAME_DEF_STMT (ranges[j - 1].exp);
3611 10047 : if (reassoc_stmt_dominates_stmt_p (gk, gj))
3612 2918 : k = j;
3613 : }
3614 7058 : tree type1 = TREE_TYPE (ranges[k - 1].exp);
3615 7058 : tree type2 = NULL_TREE;
3616 7058 : candidates.truncate (0);
3617 7058 : if (POINTER_TYPE_P (type1) || TREE_CODE (type1) == OFFSET_TYPE)
3618 701 : type1 = pointer_sized_int_node;
3619 24163 : for (j = i; j; j = chains[j - 1])
3620 : {
3621 17105 : tree type = TREE_TYPE (ranges[j - 1].exp);
3622 17105 : if (POINTER_TYPE_P (type) || TREE_CODE (type) == OFFSET_TYPE)
3623 1448 : type = pointer_sized_int_node;
3624 17105 : if ((b % 4) == 3)
3625 : {
3626 : /* For the signed < 0 cases, the types should be
3627 : really compatible (all signed with the same precision,
3628 : instead put ranges that have different in_p from
3629 : k first. */
3630 3688 : if (!useless_type_conversion_p (type1, type))
3631 0 : continue;
3632 3688 : if (ranges[j - 1].in_p != ranges[k - 1].in_p)
3633 1009 : candidates.safe_push (&ranges[j - 1]);
3634 3688 : type2 = type1;
3635 3688 : continue;
3636 : }
3637 13417 : if (j == k
3638 13417 : || useless_type_conversion_p (type1, type))
3639 : ;
3640 413 : else if (type2 == NULL_TREE
3641 413 : || useless_type_conversion_p (type2, type))
3642 : {
3643 413 : if (type2 == NULL_TREE)
3644 399 : type2 = type;
3645 413 : candidates.safe_push (&ranges[j - 1]);
3646 : }
3647 : }
3648 7058 : unsigned l = candidates.length ();
3649 24163 : for (j = i; j; j = chains[j - 1])
3650 : {
3651 17105 : tree type = TREE_TYPE (ranges[j - 1].exp);
3652 17105 : if (j == k)
3653 7058 : continue;
3654 10047 : if (POINTER_TYPE_P (type) || TREE_CODE (type) == OFFSET_TYPE)
3655 747 : type = pointer_sized_int_node;
3656 10047 : if ((b % 4) == 3)
3657 : {
3658 2042 : if (!useless_type_conversion_p (type1, type))
3659 0 : continue;
3660 2042 : if (ranges[j - 1].in_p == ranges[k - 1].in_p)
3661 1033 : candidates.safe_push (&ranges[j - 1]);
3662 2042 : continue;
3663 : }
3664 8005 : if (useless_type_conversion_p (type1, type))
3665 : ;
3666 826 : else if (type2 == NULL_TREE
3667 413 : || useless_type_conversion_p (type2, type))
3668 413 : continue;
3669 7592 : candidates.safe_push (&ranges[j - 1]);
3670 : }
3671 7058 : gimple_seq seq = NULL;
3672 7058 : tree op = NULL_TREE;
3673 7058 : unsigned int id;
3674 7058 : struct range_entry *r;
3675 7058 : candidates.safe_push (&ranges[k - 1]);
3676 24163 : FOR_EACH_VEC_ELT (candidates, id, r)
3677 : {
3678 17105 : gimple *g;
3679 17105 : enum tree_code code;
3680 17105 : if (id == 0)
3681 : {
3682 7058 : op = r->exp;
3683 7058 : continue;
3684 : }
3685 10047 : if (id == l
3686 8639 : || POINTER_TYPE_P (TREE_TYPE (op))
3687 18072 : || TREE_CODE (TREE_TYPE (op)) == OFFSET_TYPE)
3688 : {
3689 2027 : code = (b % 4) == 3 ? BIT_NOT_EXPR : NOP_EXPR;
3690 2027 : tree type3 = id >= l ? type1 : pointer_sized_int_node;
3691 2027 : if (code == BIT_NOT_EXPR
3692 2027 : && TREE_CODE (TREE_TYPE (op)) == OFFSET_TYPE)
3693 : {
3694 0 : g = gimple_build_assign (make_ssa_name (type3),
3695 : NOP_EXPR, op);
3696 0 : gimple_seq_add_stmt_without_update (&seq, g);
3697 0 : op = gimple_assign_lhs (g);
3698 : }
3699 2027 : g = gimple_build_assign (make_ssa_name (type3), code, op);
3700 2027 : gimple_seq_add_stmt_without_update (&seq, g);
3701 2027 : op = gimple_assign_lhs (g);
3702 : }
3703 10047 : tree type = TREE_TYPE (r->exp);
3704 10047 : tree exp = r->exp;
3705 10047 : if (POINTER_TYPE_P (type)
3706 9295 : || TREE_CODE (type) == OFFSET_TYPE
3707 19337 : || (id >= l && !useless_type_conversion_p (type1, type)))
3708 : {
3709 757 : tree type3 = id >= l ? type1 : pointer_sized_int_node;
3710 757 : g = gimple_build_assign (make_ssa_name (type3), NOP_EXPR, exp);
3711 757 : gimple_seq_add_stmt_without_update (&seq, g);
3712 757 : exp = gimple_assign_lhs (g);
3713 : }
3714 10047 : if ((b % 4) == 3)
3715 3331 : code = r->in_p ? BIT_IOR_EXPR : BIT_AND_EXPR;
3716 : else
3717 8005 : code = (b % 4) == 1 ? BIT_AND_EXPR : BIT_IOR_EXPR;
3718 20094 : g = gimple_build_assign (make_ssa_name (id >= l ? type1 : type2),
3719 : code, op, exp);
3720 10047 : gimple_seq_add_stmt_without_update (&seq, g);
3721 10047 : op = gimple_assign_lhs (g);
3722 : }
3723 7058 : type1 = TREE_TYPE (ranges[k - 1].exp);
3724 7058 : if (POINTER_TYPE_P (type1) || TREE_CODE (type1) == OFFSET_TYPE)
3725 : {
3726 701 : gimple *g
3727 701 : = gimple_build_assign (make_ssa_name (type1), NOP_EXPR, op);
3728 701 : gimple_seq_add_stmt_without_update (&seq, g);
3729 701 : op = gimple_assign_lhs (g);
3730 : }
3731 7058 : candidates.pop ();
3732 7058 : if (update_range_test (&ranges[k - 1], NULL, candidates.address (),
3733 : candidates.length (), opcode, ops, op,
3734 : seq, ranges[k - 1].in_p, ranges[k - 1].low,
3735 : ranges[k - 1].high))
3736 : any_changes = true;
3737 : else
3738 0 : gimple_seq_discard (seq);
3739 7120 : if ((b % 4) == 2 && buckets[b] != i)
3740 : /* There is more work to do for this bucket. */
3741 2 : b--;
3742 : }
3743 :
3744 1087895 : return any_changes;
3745 1087895 : }
3746 :
3747 : /* Attempt to optimize for signed a and b where b is known to be >= 0:
3748 : a >= 0 && a < b into (unsigned) a < (unsigned) b
3749 : a >= 0 && a <= b into (unsigned) a <= (unsigned) b */
3750 :
3751 : static bool
3752 1087895 : optimize_range_tests_var_bound (enum tree_code opcode, int first, int length,
3753 : vec<operand_entry *> *ops,
3754 : struct range_entry *ranges,
3755 : basic_block first_bb)
3756 : {
3757 1087895 : int i;
3758 1087895 : bool any_changes = false;
3759 1087895 : hash_map<tree, int> *map = NULL;
3760 :
3761 1902865 : for (i = first; i < length; i++)
3762 : {
3763 814970 : if (ranges[i].exp == NULL_TREE
3764 794941 : || TREE_CODE (ranges[i].exp) != SSA_NAME
3765 790301 : || !ranges[i].in_p)
3766 327125 : continue;
3767 :
3768 487845 : tree type = TREE_TYPE (ranges[i].exp);
3769 930364 : if (!INTEGRAL_TYPE_P (type)
3770 477900 : || TYPE_UNSIGNED (type)
3771 199405 : || ranges[i].low == NULL_TREE
3772 190143 : || !integer_zerop (ranges[i].low)
3773 561847 : || ranges[i].high != NULL_TREE)
3774 442519 : continue;
3775 : /* EXP >= 0 here. */
3776 45326 : if (map == NULL)
3777 43653 : map = new hash_map <tree, int>;
3778 45326 : map->put (ranges[i].exp, i);
3779 : }
3780 :
3781 1087895 : if (map == NULL)
3782 : return false;
3783 :
3784 134658 : for (i = 0; i < length; i++)
3785 : {
3786 91005 : bool in_p = ranges[i].in_p;
3787 91005 : if (ranges[i].low == NULL_TREE
3788 90306 : || ranges[i].high == NULL_TREE)
3789 90448 : continue;
3790 43202 : if (!integer_zerop (ranges[i].low)
3791 43202 : || !integer_zerop (ranges[i].high))
3792 : {
3793 8890 : if (ranges[i].exp
3794 4445 : && TYPE_PRECISION (TREE_TYPE (ranges[i].exp)) == 1
3795 0 : && TYPE_UNSIGNED (TREE_TYPE (ranges[i].exp))
3796 0 : && integer_onep (ranges[i].low)
3797 4445 : && integer_onep (ranges[i].high))
3798 0 : in_p = !in_p;
3799 : else
3800 4445 : continue;
3801 : }
3802 :
3803 38757 : gimple *stmt;
3804 38757 : tree_code ccode;
3805 38757 : tree rhs1, rhs2;
3806 38757 : if (ranges[i].exp)
3807 : {
3808 37667 : if (TREE_CODE (ranges[i].exp) != SSA_NAME)
3809 5 : continue;
3810 37662 : stmt = SSA_NAME_DEF_STMT (ranges[i].exp);
3811 37662 : if (!is_gimple_assign (stmt))
3812 925 : continue;
3813 36737 : ccode = gimple_assign_rhs_code (stmt);
3814 36737 : rhs1 = gimple_assign_rhs1 (stmt);
3815 36737 : rhs2 = gimple_assign_rhs2 (stmt);
3816 : }
3817 : else
3818 : {
3819 1090 : operand_entry *oe = (*ops)[ranges[i].idx];
3820 1090 : stmt = last_nondebug_stmt (BASIC_BLOCK_FOR_FN (cfun, oe->id));
3821 1090 : if (gimple_code (stmt) != GIMPLE_COND)
3822 0 : continue;
3823 1090 : ccode = gimple_cond_code (stmt);
3824 1090 : rhs1 = gimple_cond_lhs (stmt);
3825 1090 : rhs2 = gimple_cond_rhs (stmt);
3826 : }
3827 :
3828 37827 : if (TREE_CODE (rhs1) != SSA_NAME
3829 37315 : || rhs2 == NULL_TREE
3830 37211 : || TREE_CODE (rhs2) != SSA_NAME)
3831 679 : continue;
3832 :
3833 37148 : switch (ccode)
3834 : {
3835 35946 : case GT_EXPR:
3836 35946 : case GE_EXPR:
3837 35946 : case LT_EXPR:
3838 35946 : case LE_EXPR:
3839 35946 : break;
3840 1202 : default:
3841 1202 : continue;
3842 : }
3843 35946 : if (in_p)
3844 789 : ccode = invert_tree_comparison (ccode, false);
3845 35946 : switch (ccode)
3846 : {
3847 14803 : case GT_EXPR:
3848 14803 : case GE_EXPR:
3849 14803 : std::swap (rhs1, rhs2);
3850 14803 : ccode = swap_tree_comparison (ccode);
3851 14803 : break;
3852 : case LT_EXPR:
3853 : case LE_EXPR:
3854 : break;
3855 0 : default:
3856 0 : gcc_unreachable ();
3857 : }
3858 :
3859 35946 : int *idx = map->get (rhs1);
3860 35946 : if (idx == NULL)
3861 1084 : continue;
3862 :
3863 : /* maybe_optimize_range_tests allows statements without side-effects
3864 : in the basic blocks as long as they are consumed in the same bb.
3865 : Make sure rhs2's def stmt is not among them, otherwise we can't
3866 : use safely get_nonzero_bits on it. E.g. in:
3867 : # RANGE [-83, 1] NONZERO 173
3868 : # k_32 = PHI <k_47(13), k_12(9)>
3869 : ...
3870 : if (k_32 >= 0)
3871 : goto <bb 5>; [26.46%]
3872 : else
3873 : goto <bb 9>; [73.54%]
3874 :
3875 : <bb 5> [local count: 140323371]:
3876 : # RANGE [0, 1] NONZERO 1
3877 : _5 = (int) k_32;
3878 : # RANGE [0, 4] NONZERO 4
3879 : _21 = _5 << 2;
3880 : # RANGE [0, 4] NONZERO 4
3881 : iftmp.0_44 = (char) _21;
3882 : if (k_32 < iftmp.0_44)
3883 : goto <bb 6>; [84.48%]
3884 : else
3885 : goto <bb 9>; [15.52%]
3886 : the ranges on _5/_21/iftmp.0_44 are flow sensitive, assume that
3887 : k_32 >= 0. If we'd optimize k_32 >= 0 to true and k_32 < iftmp.0_44
3888 : to (unsigned) k_32 < (unsigned) iftmp.0_44, then we would execute
3889 : those stmts even for negative k_32 and the value ranges would be no
3890 : longer guaranteed and so the optimization would be invalid. */
3891 34864 : while (opcode == ERROR_MARK)
3892 : {
3893 640 : gimple *g = SSA_NAME_DEF_STMT (rhs2);
3894 640 : basic_block bb2 = gimple_bb (g);
3895 640 : if (bb2
3896 640 : && bb2 != first_bb
3897 640 : && dominated_by_p (CDI_DOMINATORS, bb2, first_bb))
3898 : {
3899 : /* As an exception, handle a few common cases. */
3900 558 : if (gimple_assign_cast_p (g)
3901 558 : && INTEGRAL_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (g))))
3902 : {
3903 42 : tree op0 = gimple_assign_rhs1 (g);
3904 42 : if (TYPE_UNSIGNED (TREE_TYPE (op0))
3905 42 : && (TYPE_PRECISION (TREE_TYPE (rhs2))
3906 12 : > TYPE_PRECISION (TREE_TYPE (op0))))
3907 : /* Zero-extension is always ok. */
3908 : break;
3909 32 : else if (TYPE_PRECISION (TREE_TYPE (rhs2))
3910 32 : == TYPE_PRECISION (TREE_TYPE (op0))
3911 32 : && TREE_CODE (op0) == SSA_NAME)
3912 : {
3913 : /* Cast from signed to unsigned or vice versa. Retry
3914 : with the op0 as new rhs2. */
3915 2 : rhs2 = op0;
3916 2 : continue;
3917 : }
3918 : }
3919 516 : else if (is_gimple_assign (g)
3920 516 : && gimple_assign_rhs_code (g) == BIT_AND_EXPR
3921 0 : && TREE_CODE (gimple_assign_rhs2 (g)) == INTEGER_CST
3922 1032 : && !wi::neg_p (wi::to_wide (gimple_assign_rhs2 (g))))
3923 : /* Masking with INTEGER_CST with MSB clear is always ok
3924 : too. */
3925 : break;
3926 : rhs2 = NULL_TREE;
3927 : }
3928 : break;
3929 : }
3930 34316 : if (rhs2 == NULL_TREE)
3931 546 : continue;
3932 :
3933 34873 : wide_int nz = get_nonzero_bits (rhs2);
3934 34316 : if (wi::neg_p (nz))
3935 33759 : continue;
3936 :
3937 : /* We have EXP < RHS2 or EXP <= RHS2 where EXP >= 0
3938 : and RHS2 is known to be RHS2 >= 0. */
3939 557 : tree utype = unsigned_type_for (TREE_TYPE (rhs1));
3940 :
3941 557 : if (dump_file && (dump_flags & TDF_DETAILS))
3942 : {
3943 7 : struct range_entry *r = &ranges[*idx];
3944 7 : fprintf (dump_file, "Optimizing range test ");
3945 7 : print_generic_expr (dump_file, r->exp);
3946 7 : fprintf (dump_file, " +[");
3947 7 : print_generic_expr (dump_file, r->low);
3948 7 : fprintf (dump_file, ", ");
3949 7 : print_generic_expr (dump_file, r->high);
3950 7 : fprintf (dump_file, "] and comparison ");
3951 7 : print_generic_expr (dump_file, rhs1);
3952 7 : fprintf (dump_file, " %s ", op_symbol_code (ccode));
3953 7 : print_generic_expr (dump_file, rhs2);
3954 7 : fprintf (dump_file, "\n into (");
3955 7 : print_generic_expr (dump_file, utype);
3956 7 : fprintf (dump_file, ") ");
3957 7 : print_generic_expr (dump_file, rhs1);
3958 7 : fprintf (dump_file, " %s (", op_symbol_code (ccode));
3959 7 : print_generic_expr (dump_file, utype);
3960 7 : fprintf (dump_file, ") ");
3961 7 : print_generic_expr (dump_file, rhs2);
3962 7 : fprintf (dump_file, "\n");
3963 : }
3964 :
3965 557 : operand_entry *oe = (*ops)[ranges[i].idx];
3966 557 : ranges[i].in_p = 0;
3967 557 : if (opcode == BIT_IOR_EXPR
3968 518 : || (opcode == ERROR_MARK && oe->rank == BIT_IOR_EXPR))
3969 : {
3970 40 : ranges[i].in_p = 1;
3971 40 : ccode = invert_tree_comparison (ccode, false);
3972 : }
3973 :
3974 557 : unsigned int uid = gimple_uid (stmt);
3975 557 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3976 557 : gimple *g = gimple_build_assign (make_ssa_name (utype), NOP_EXPR, rhs1);
3977 557 : gimple_set_uid (g, uid);
3978 557 : rhs1 = gimple_assign_lhs (g);
3979 557 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
3980 557 : if (!useless_type_conversion_p (utype, TREE_TYPE (rhs2)))
3981 : {
3982 557 : g = gimple_build_assign (make_ssa_name (utype), NOP_EXPR, rhs2);
3983 557 : gimple_set_uid (g, uid);
3984 557 : rhs2 = gimple_assign_lhs (g);
3985 557 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
3986 : }
3987 557 : if (tree_swap_operands_p (rhs1, rhs2))
3988 : {
3989 510 : std::swap (rhs1, rhs2);
3990 510 : ccode = swap_tree_comparison (ccode);
3991 : }
3992 557 : if (gimple_code (stmt) == GIMPLE_COND)
3993 : {
3994 8 : gcond *c = as_a <gcond *> (stmt);
3995 8 : gimple_cond_set_code (c, ccode);
3996 8 : gimple_cond_set_lhs (c, rhs1);
3997 8 : gimple_cond_set_rhs (c, rhs2);
3998 8 : update_stmt (stmt);
3999 : }
4000 : else
4001 : {
4002 549 : tree ctype = oe->op ? TREE_TYPE (oe->op) : boolean_type_node;
4003 549 : if (!INTEGRAL_TYPE_P (ctype)
4004 549 : || (TREE_CODE (ctype) != BOOLEAN_TYPE
4005 3 : && TYPE_PRECISION (ctype) != 1))
4006 3 : ctype = boolean_type_node;
4007 549 : g = gimple_build_assign (make_ssa_name (ctype), ccode, rhs1, rhs2);
4008 549 : gimple_set_uid (g, uid);
4009 549 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
4010 549 : if (oe->op && ctype != TREE_TYPE (oe->op))
4011 : {
4012 3 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (oe->op)),
4013 : NOP_EXPR, gimple_assign_lhs (g));
4014 3 : gimple_set_uid (g, uid);
4015 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
4016 : }
4017 549 : ranges[i].exp = gimple_assign_lhs (g);
4018 549 : oe->op = ranges[i].exp;
4019 549 : ranges[i].low = build_zero_cst (TREE_TYPE (ranges[i].exp));
4020 549 : ranges[i].high = ranges[i].low;
4021 : }
4022 557 : oe = (*ops)[ranges[*idx].idx];
4023 : /* Now change all the other range test immediate uses, so that
4024 : those tests will be optimized away. */
4025 557 : if (opcode == ERROR_MARK)
4026 : {
4027 11 : if (oe->op)
4028 3 : oe->op = build_int_cst (TREE_TYPE (oe->op),
4029 3 : oe->rank == BIT_IOR_EXPR ? 0 : 1);
4030 : else
4031 8 : oe->op = (oe->rank == BIT_IOR_EXPR
4032 8 : ? boolean_false_node : boolean_true_node);
4033 : }
4034 : else
4035 546 : oe->op = error_mark_node;
4036 557 : ranges[*idx].exp = NULL_TREE;
4037 557 : ranges[*idx].low = NULL_TREE;
4038 557 : ranges[*idx].high = NULL_TREE;
4039 557 : any_changes = true;
4040 : }
4041 :
4042 43653 : delete map;
4043 : return any_changes;
4044 : }
4045 :
4046 : /* Optimize range tests, similarly how fold_range_test optimizes
4047 : it on trees. The tree code for the binary
4048 : operation between all the operands is OPCODE.
4049 : If OPCODE is ERROR_MARK, optimize_range_tests is called from within
4050 : maybe_optimize_range_tests for inter-bb range optimization.
4051 : In that case if oe->op is NULL, oe->id is bb->index whose
4052 : GIMPLE_COND is && or ||ed into the test, and oe->rank says
4053 : the actual opcode.
4054 : FIRST_BB is the first basic block if OPCODE is ERROR_MARK. */
4055 :
4056 : static bool
4057 1087982 : optimize_range_tests (enum tree_code opcode,
4058 : vec<operand_entry *> *ops, basic_block first_bb)
4059 : {
4060 1087982 : unsigned int length = ops->length (), i, j, first;
4061 1087982 : operand_entry *oe;
4062 1087982 : struct range_entry *ranges;
4063 2175877 : bool any_changes = false;
4064 :
4065 1087982 : if (length == 1)
4066 : return false;
4067 :
4068 1087895 : ranges = XNEWVEC (struct range_entry, length);
4069 4467949 : for (i = 0; i < length; i++)
4070 : {
4071 2292159 : oe = (*ops)[i];
4072 2292159 : ranges[i].idx = i;
4073 2292159 : init_range_entry (ranges + i, oe->op,
4074 2292159 : oe->op
4075 : ? NULL
4076 268699 : : last_nondebug_stmt (BASIC_BLOCK_FOR_FN (cfun, oe->id)));
4077 : /* For | invert it now, we will invert it again before emitting
4078 : the optimized expression. */
4079 2292159 : if (opcode == BIT_IOR_EXPR
4080 1604745 : || (opcode == ERROR_MARK && oe->rank == BIT_IOR_EXPR))
4081 885452 : ranges[i].in_p = !ranges[i].in_p;
4082 : }
4083 :
4084 1087895 : qsort (ranges, length, sizeof (*ranges), range_entry_cmp);
4085 3652979 : for (i = 0; i < length; i++)
4086 1858920 : if (ranges[i].exp != NULL_TREE && TREE_CODE (ranges[i].exp) == SSA_NAME)
4087 : break;
4088 :
4089 : /* Try to merge ranges. */
4090 1888794 : for (first = i; i < length; i++)
4091 : {
4092 800899 : tree low = ranges[i].low;
4093 800899 : tree high = ranges[i].high;
4094 800899 : int in_p = ranges[i].in_p;
4095 800899 : int update_fail_count = 0;
4096 :
4097 814970 : for (j = i + 1; j < length; j++)
4098 : {
4099 433239 : if (ranges[i].exp != ranges[j].exp)
4100 : break;
4101 35552 : if (!merge_ranges (&in_p, &low, &high, in_p, low, high,
4102 35552 : ranges[j].in_p, ranges[j].low, ranges[j].high))
4103 : break;
4104 : }
4105 :
4106 800899 : if (j == i + 1)
4107 787411 : continue;
4108 :
4109 13488 : if (update_range_test (ranges + i, ranges + i + 1, NULL, j - i - 1,
4110 : opcode, ops, ranges[i].exp, NULL, in_p,
4111 : low, high))
4112 : {
4113 13488 : i = j - 1;
4114 13488 : any_changes = true;
4115 : }
4116 : /* Avoid quadratic complexity if all merge_ranges calls would succeed,
4117 : while update_range_test would fail. */
4118 : else if (update_fail_count == 64)
4119 : i = j - 1;
4120 : else
4121 13488 : ++update_fail_count;
4122 : }
4123 :
4124 1087895 : any_changes |= optimize_range_tests_1 (opcode, first, length, true,
4125 : ops, ranges);
4126 :
4127 1087895 : if (BRANCH_COST (optimize_function_for_speed_p (cfun), false) >= 2)
4128 1087879 : any_changes |= optimize_range_tests_1 (opcode, first, length, false,
4129 : ops, ranges);
4130 1087895 : if (lshift_cheap_p (optimize_function_for_speed_p (cfun)))
4131 1087895 : any_changes |= optimize_range_tests_to_bit_test (opcode, first, length,
4132 : ops, ranges);
4133 1087895 : any_changes |= optimize_range_tests_var_bound (opcode, first, length, ops,
4134 : ranges, first_bb);
4135 1087895 : any_changes |= optimize_range_tests_cmp_bitwise (opcode, first, length,
4136 : ops, ranges);
4137 :
4138 1087895 : if (any_changes && opcode != ERROR_MARK)
4139 : {
4140 : j = 0;
4141 36353 : FOR_EACH_VEC_ELT (*ops, i, oe)
4142 : {
4143 25415 : if (oe->op == error_mark_node)
4144 12531 : continue;
4145 12884 : else if (i != j)
4146 5347 : (*ops)[j] = oe;
4147 12884 : j++;
4148 : }
4149 10938 : ops->truncate (j);
4150 : }
4151 :
4152 1087895 : XDELETEVEC (ranges);
4153 1087895 : return any_changes;
4154 : }
4155 :
4156 : /* A subroutine of optimize_vec_cond_expr to extract and canonicalize
4157 : the operands of the VEC_COND_EXPR. Returns ERROR_MARK on failure,
4158 : otherwise the comparison code. TYPE is a return value that is set
4159 : to type of comparison. */
4160 :
4161 : static tree_code
4162 55960 : ovce_extract_ops (tree var, gassign **rets, bool *reti, tree *type,
4163 : tree *lhs, tree *rhs, gassign **vcond)
4164 : {
4165 55960 : if (TREE_CODE (var) != SSA_NAME)
4166 : return ERROR_MARK;
4167 :
4168 49248 : gassign *stmt = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (var));
4169 35835 : if (stmt == NULL)
4170 : return ERROR_MARK;
4171 35835 : if (vcond)
4172 35835 : *vcond = stmt;
4173 :
4174 : /* ??? If we start creating more COND_EXPR, we could perform
4175 : this same optimization with them. For now, simplify. */
4176 47964 : if (gimple_assign_rhs_code (stmt) != VEC_COND_EXPR)
4177 : return ERROR_MARK;
4178 :
4179 1364 : tree cond = gimple_assign_rhs1 (stmt);
4180 1364 : tree_code cmp = TREE_CODE (cond);
4181 1364 : if (cmp != SSA_NAME)
4182 : return ERROR_MARK;
4183 :
4184 1342 : gassign *assign = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (cond));
4185 1338 : if (assign == NULL
4186 1338 : || TREE_CODE_CLASS (gimple_assign_rhs_code (assign)) != tcc_comparison)
4187 : return ERROR_MARK;
4188 :
4189 1225 : cmp = gimple_assign_rhs_code (assign);
4190 1225 : if (lhs)
4191 1225 : *lhs = gimple_assign_rhs1 (assign);
4192 1225 : if (rhs)
4193 2450 : *rhs = gimple_assign_rhs2 (assign);
4194 :
4195 : /* ??? For now, allow only canonical true and false result vectors.
4196 : We could expand this to other constants should the need arise,
4197 : but at the moment we don't create them. */
4198 1225 : tree t = gimple_assign_rhs2 (stmt);
4199 1225 : tree f = gimple_assign_rhs3 (stmt);
4200 1225 : bool inv;
4201 1225 : if (integer_all_onesp (t))
4202 : inv = false;
4203 1105 : else if (integer_all_onesp (f))
4204 : {
4205 1 : cmp = invert_tree_comparison (cmp, false);
4206 1 : inv = true;
4207 : }
4208 : else
4209 : return ERROR_MARK;
4210 121 : if (!integer_zerop (inv ? t : f))
4211 : return ERROR_MARK;
4212 :
4213 : /* Success! */
4214 102 : if (rets)
4215 102 : *rets = assign;
4216 102 : if (reti)
4217 72 : *reti = inv;
4218 102 : if (type)
4219 72 : *type = TREE_TYPE (cond);
4220 : return cmp;
4221 : }
4222 :
4223 : /* Optimize the condition of VEC_COND_EXPRs which have been combined
4224 : with OPCODE (either BIT_AND_EXPR or BIT_IOR_EXPR). */
4225 :
4226 : static bool
4227 26244 : optimize_vec_cond_expr (tree_code opcode, vec<operand_entry *> *ops)
4228 : {
4229 26244 : unsigned int length = ops->length (), i, j;
4230 26244 : bool any_changes = false;
4231 :
4232 26244 : if (length == 1)
4233 : return false;
4234 :
4235 82156 : for (i = 0; i < length; ++i)
4236 : {
4237 55912 : tree &elt0 = (*ops)[i]->op;
4238 :
4239 55912 : gassign *stmt0, *vcond0;
4240 55912 : bool invert;
4241 55912 : tree type, lhs0, rhs0;
4242 55912 : tree_code cmp0 = ovce_extract_ops (elt0, &stmt0, &invert, &type, &lhs0,
4243 : &rhs0, &vcond0);
4244 55912 : if (cmp0 == ERROR_MARK)
4245 55840 : continue;
4246 :
4247 120 : for (j = i + 1; j < length; ++j)
4248 : {
4249 48 : tree &elt1 = (*ops)[j]->op;
4250 :
4251 48 : gassign *stmt1, *vcond1;
4252 48 : tree lhs1, rhs1;
4253 48 : tree_code cmp1 = ovce_extract_ops (elt1, &stmt1, NULL, NULL, &lhs1,
4254 : &rhs1, &vcond1);
4255 48 : if (cmp1 == ERROR_MARK)
4256 42 : continue;
4257 :
4258 30 : tree comb;
4259 30 : if (opcode == BIT_AND_EXPR)
4260 0 : comb = maybe_fold_and_comparisons (type, cmp0, lhs0, rhs0,
4261 : cmp1, lhs1, rhs1);
4262 30 : else if (opcode == BIT_IOR_EXPR)
4263 30 : comb = maybe_fold_or_comparisons (type, cmp0, lhs0, rhs0,
4264 : cmp1, lhs1, rhs1);
4265 : else
4266 0 : gcc_unreachable ();
4267 30 : if (comb == NULL)
4268 24 : continue;
4269 :
4270 : /* Success! */
4271 6 : if (dump_file && (dump_flags & TDF_DETAILS))
4272 : {
4273 0 : fprintf (dump_file, "Transforming ");
4274 0 : print_generic_expr (dump_file, gimple_assign_lhs (stmt0));
4275 0 : fprintf (dump_file, " %c ", opcode == BIT_AND_EXPR ? '&' : '|');
4276 0 : print_generic_expr (dump_file, gimple_assign_lhs (stmt1));
4277 0 : fprintf (dump_file, " into ");
4278 0 : print_generic_expr (dump_file, comb);
4279 0 : fputc ('\n', dump_file);
4280 : }
4281 :
4282 6 : gimple_stmt_iterator gsi = gsi_for_stmt (vcond0);
4283 6 : tree exp = force_gimple_operand_gsi (&gsi, comb, true, NULL_TREE,
4284 : true, GSI_SAME_STMT);
4285 18 : tree res = gimple_build (&gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
4286 6 : VEC_COND_EXPR, TREE_TYPE (elt0), exp,
4287 : constant_boolean_node (true,
4288 6 : TREE_TYPE (elt0)),
4289 : constant_boolean_node (false,
4290 6 : TREE_TYPE (elt0)));
4291 6 : elt0 = res;
4292 :
4293 6 : elt1 = error_mark_node;
4294 6 : any_changes = true;
4295 : }
4296 : }
4297 :
4298 26244 : if (any_changes)
4299 : {
4300 : operand_entry *oe;
4301 : j = 0;
4302 18 : FOR_EACH_VEC_ELT (*ops, i, oe)
4303 : {
4304 12 : if (oe->op == error_mark_node)
4305 6 : continue;
4306 6 : else if (i != j)
4307 0 : (*ops)[j] = oe;
4308 6 : j++;
4309 : }
4310 6 : ops->truncate (j);
4311 : }
4312 :
4313 : return any_changes;
4314 : }
4315 :
4316 : /* Return true if STMT is a cast like:
4317 : <bb N>:
4318 : ...
4319 : _123 = (int) _234;
4320 :
4321 : <bb M>:
4322 : # _345 = PHI <_123(N), 1(...), 1(...)>
4323 : where _234 has bool type, _123 has single use and
4324 : bb N has a single successor M. This is commonly used in
4325 : the last block of a range test.
4326 :
4327 : Also Return true if STMT is tcc_compare like:
4328 : <bb N>:
4329 : ...
4330 : _234 = a_2(D) == 2;
4331 :
4332 : <bb M>:
4333 : # _345 = PHI <_234(N), 1(...), 1(...)>
4334 : _346 = (int) _345;
4335 : where _234 has booltype, single use and
4336 : bb N has a single successor M. This is commonly used in
4337 : the last block of a range test. */
4338 :
4339 : static bool
4340 16154887 : final_range_test_p (gimple *stmt)
4341 : {
4342 16154887 : basic_block bb, rhs_bb, lhs_bb;
4343 16154887 : edge e;
4344 16154887 : tree lhs, rhs;
4345 16154887 : use_operand_p use_p;
4346 16154887 : gimple *use_stmt;
4347 :
4348 16154887 : if (!gimple_assign_cast_p (stmt)
4349 16154887 : && (!is_gimple_assign (stmt)
4350 5076984 : || (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt))
4351 : != tcc_comparison)))
4352 : return false;
4353 568839 : bb = gimple_bb (stmt);
4354 568839 : if (!single_succ_p (bb))
4355 : return false;
4356 568556 : e = single_succ_edge (bb);
4357 568556 : if (e->flags & EDGE_COMPLEX)
4358 : return false;
4359 :
4360 568556 : lhs = gimple_assign_lhs (stmt);
4361 568556 : rhs = gimple_assign_rhs1 (stmt);
4362 568556 : if (gimple_assign_cast_p (stmt)
4363 568556 : && (!INTEGRAL_TYPE_P (TREE_TYPE (lhs))
4364 403213 : || TREE_CODE (rhs) != SSA_NAME
4365 382322 : || TREE_CODE (TREE_TYPE (rhs)) != BOOLEAN_TYPE))
4366 : return false;
4367 :
4368 187946 : if (!gimple_assign_cast_p (stmt)
4369 187946 : && (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE))
4370 : return false;
4371 :
4372 : /* Test whether lhs is consumed only by a PHI in the only successor bb. */
4373 187620 : if (!single_imm_use (lhs, &use_p, &use_stmt))
4374 : return false;
4375 :
4376 184446 : if (gimple_code (use_stmt) != GIMPLE_PHI
4377 184446 : || gimple_bb (use_stmt) != e->dest)
4378 : return false;
4379 :
4380 : /* And that the rhs is defined in the same loop. */
4381 182756 : if (gimple_assign_cast_p (stmt))
4382 : {
4383 65301 : if (TREE_CODE (rhs) != SSA_NAME
4384 65301 : || !(rhs_bb = gimple_bb (SSA_NAME_DEF_STMT (rhs)))
4385 130596 : || !flow_bb_inside_loop_p (loop_containing_stmt (stmt), rhs_bb))
4386 53 : return false;
4387 : }
4388 : else
4389 : {
4390 117455 : if (TREE_CODE (lhs) != SSA_NAME
4391 117455 : || !(lhs_bb = gimple_bb (SSA_NAME_DEF_STMT (lhs)))
4392 234910 : || !flow_bb_inside_loop_p (loop_containing_stmt (stmt), lhs_bb))
4393 0 : return false;
4394 : }
4395 :
4396 : return true;
4397 : }
4398 :
4399 : /* Return true if BB is suitable basic block for inter-bb range test
4400 : optimization. If BACKWARD is true, BB should be the only predecessor
4401 : of TEST_BB, and *OTHER_BB is either NULL and filled by the routine,
4402 : or compared with to find a common basic block to which all conditions
4403 : branch to if true resp. false. If BACKWARD is false, TEST_BB should
4404 : be the only predecessor of BB. *TEST_SWAPPED_P is set to true if
4405 : TEST_BB is a bb ending in condition where the edge to non-*OTHER_BB
4406 : block points to an empty block that falls through into *OTHER_BB and
4407 : the phi args match that path. */
4408 :
4409 : static bool
4410 11684229 : suitable_cond_bb (basic_block bb, basic_block test_bb, basic_block *other_bb,
4411 : bool *test_swapped_p, bool backward)
4412 : {
4413 11684229 : edge_iterator ei, ei2;
4414 11684229 : edge e, e2;
4415 11684229 : gimple *stmt;
4416 11684229 : gphi_iterator gsi;
4417 11684229 : bool other_edge_seen = false;
4418 11684229 : bool is_cond;
4419 :
4420 11684229 : if (test_bb == bb)
4421 : return false;
4422 : /* Check last stmt first. */
4423 11684229 : stmt = last_nondebug_stmt (bb);
4424 11684229 : if (stmt == NULL
4425 10801296 : || (gimple_code (stmt) != GIMPLE_COND
4426 552447 : && (backward || !final_range_test_p (stmt)))
4427 10288912 : || gimple_visited_p (stmt)
4428 10230126 : || stmt_could_throw_p (cfun, stmt)
4429 21914241 : || *other_bb == bb)
4430 : return false;
4431 10230009 : is_cond = gimple_code (stmt) == GIMPLE_COND;
4432 10230009 : if (is_cond)
4433 : {
4434 : /* If last stmt is GIMPLE_COND, verify that one of the succ edges
4435 : goes to the next bb (if BACKWARD, it is TEST_BB), and the other
4436 : to *OTHER_BB (if not set yet, try to find it out). */
4437 10190060 : if (EDGE_COUNT (bb->succs) != 2)
4438 : return false;
4439 20032127 : FOR_EACH_EDGE (e, ei, bb->succs)
4440 : {
4441 16573629 : if (!(e->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE)))
4442 : return false;
4443 16573629 : if (e->dest == test_bb)
4444 : {
4445 5519670 : if (backward)
4446 5516704 : continue;
4447 : else
4448 : return false;
4449 : }
4450 11053959 : if (e->dest == bb)
4451 : return false;
4452 10897687 : if (*other_bb == NULL)
4453 : {
4454 26850849 : FOR_EACH_EDGE (e2, ei2, test_bb->succs)
4455 17900566 : if (!(e2->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE)))
4456 : return false;
4457 17900566 : else if (e->dest == e2->dest)
4458 2408468 : *other_bb = e->dest;
4459 8950283 : if (*other_bb == NULL)
4460 : return false;
4461 : }
4462 4355872 : if (e->dest == *other_bb)
4463 : other_edge_seen = true;
4464 937839 : else if (backward)
4465 : return false;
4466 : }
4467 3458498 : if (*other_bb == NULL || !other_edge_seen)
4468 : return false;
4469 : }
4470 39949 : else if (single_succ (bb) != *other_bb)
4471 : return false;
4472 :
4473 : /* Now check all PHIs of *OTHER_BB. */
4474 3457271 : e = find_edge (bb, *other_bb);
4475 3457271 : e2 = find_edge (test_bb, *other_bb);
4476 3463855 : retry:;
4477 5062816 : for (gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi); gsi_next (&gsi))
4478 : {
4479 2678345 : gphi *phi = gsi.phi ();
4480 : /* If both BB and TEST_BB end with GIMPLE_COND, all PHI arguments
4481 : corresponding to BB and TEST_BB predecessor must be the same. */
4482 2678345 : if (!operand_equal_p (gimple_phi_arg_def (phi, e->dest_idx),
4483 2678345 : gimple_phi_arg_def (phi, e2->dest_idx), 0))
4484 : {
4485 : /* Otherwise, if one of the blocks doesn't end with GIMPLE_COND,
4486 : one of the PHIs should have the lhs of the last stmt in
4487 : that block as PHI arg and that PHI should have 0 or 1
4488 : corresponding to it in all other range test basic blocks
4489 : considered. */
4490 1149080 : if (!is_cond)
4491 : {
4492 41947 : if (gimple_phi_arg_def (phi, e->dest_idx)
4493 41947 : == gimple_assign_lhs (stmt)
4494 41947 : && (integer_zerop (gimple_phi_arg_def (phi, e2->dest_idx))
4495 18038 : || integer_onep (gimple_phi_arg_def (phi,
4496 18038 : e2->dest_idx))))
4497 37673 : continue;
4498 : }
4499 : else
4500 : {
4501 1107133 : gimple *test_last = last_nondebug_stmt (test_bb);
4502 1107133 : if (gimple_code (test_last) == GIMPLE_COND)
4503 : {
4504 1070891 : if (backward ? e2->src != test_bb : e->src != bb)
4505 : return false;
4506 :
4507 : /* For last_bb, handle also:
4508 : if (x_3(D) == 3)
4509 : goto <bb 6>; [34.00%]
4510 : else
4511 : goto <bb 7>; [66.00%]
4512 :
4513 : <bb 6> [local count: 79512730]:
4514 :
4515 : <bb 7> [local count: 1073741824]:
4516 : # prephitmp_7 = PHI <1(3), 1(4), 0(5), 1(2), 1(6)>
4517 : where bb 7 is *OTHER_BB, but the PHI values from the
4518 : earlier bbs match the path through the empty bb
4519 : in between. */
4520 1066281 : edge e3;
4521 1066281 : if (backward)
4522 1473890 : e3 = EDGE_SUCC (test_bb,
4523 : e2 == EDGE_SUCC (test_bb, 0) ? 1 : 0);
4524 : else
4525 21196 : e3 = EDGE_SUCC (bb,
4526 : e == EDGE_SUCC (bb, 0) ? 1 : 0);
4527 1066281 : if (empty_block_p (e3->dest)
4528 35527 : && single_succ_p (e3->dest)
4529 35527 : && single_succ (e3->dest) == *other_bb
4530 35261 : && single_pred_p (e3->dest)
4531 1100778 : && single_succ_edge (e3->dest)->flags == EDGE_FALLTHRU)
4532 : {
4533 6584 : if (backward)
4534 5913 : e2 = single_succ_edge (e3->dest);
4535 : else
4536 671 : e = single_succ_edge (e3->dest);
4537 6584 : if (test_swapped_p)
4538 304 : *test_swapped_p = true;
4539 6584 : goto retry;
4540 : }
4541 : }
4542 36242 : else if (gimple_phi_arg_def (phi, e2->dest_idx)
4543 36242 : == gimple_assign_lhs (test_last)
4544 69129 : && (integer_zerop (gimple_phi_arg_def (phi,
4545 32887 : e->dest_idx))
4546 15072 : || integer_onep (gimple_phi_arg_def (phi,
4547 15072 : e->dest_idx))))
4548 32023 : continue;
4549 : }
4550 :
4551 : return false;
4552 : }
4553 : }
4554 : return true;
4555 : }
4556 :
4557 : /* Return true if BB doesn't have side-effects that would disallow
4558 : range test optimization, all SSA_NAMEs set in the bb are consumed
4559 : in the bb and there are no PHIs. */
4560 :
4561 : bool
4562 5506833 : no_side_effect_bb (basic_block bb)
4563 : {
4564 5506833 : gimple_stmt_iterator gsi;
4565 5506833 : gimple *last;
4566 :
4567 5506833 : if (!gimple_seq_empty_p (phi_nodes (bb)))
4568 : return false;
4569 4376979 : last = last_nondebug_stmt (bb);
4570 15618152 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4571 : {
4572 11241173 : gimple *stmt = gsi_stmt (gsi);
4573 11241173 : tree lhs;
4574 11241173 : imm_use_iterator imm_iter;
4575 11241173 : use_operand_p use_p;
4576 :
4577 11241173 : if (is_gimple_debug (stmt))
4578 4955429 : continue;
4579 6285744 : if (gimple_has_side_effects (stmt))
4580 4376979 : return false;
4581 5362787 : if (stmt == last)
4582 : return true;
4583 4264086 : if (!is_gimple_assign (stmt))
4584 : return false;
4585 3570901 : lhs = gimple_assign_lhs (stmt);
4586 3570901 : if (TREE_CODE (lhs) != SSA_NAME)
4587 : return false;
4588 3324358 : if (gimple_assign_rhs_could_trap_p (stmt))
4589 : return false;
4590 4721152 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
4591 : {
4592 2812387 : gimple *use_stmt = USE_STMT (use_p);
4593 2812387 : if (is_gimple_debug (use_stmt))
4594 224620 : continue;
4595 2587767 : if (gimple_bb (use_stmt) != bb)
4596 551439 : return false;
4597 2460204 : }
4598 : }
4599 : return false;
4600 : }
4601 :
4602 : /* If VAR is set by CODE (BIT_{AND,IOR}_EXPR) which is reassociable,
4603 : return true and fill in *OPS recursively. */
4604 :
4605 : static bool
4606 93975 : get_ops (tree var, enum tree_code code, vec<operand_entry *> *ops,
4607 : class loop *loop)
4608 : {
4609 93975 : gimple *stmt = SSA_NAME_DEF_STMT (var);
4610 93975 : tree rhs[2];
4611 93975 : int i;
4612 :
4613 93975 : if (!is_reassociable_op (stmt, code, loop))
4614 : return false;
4615 :
4616 22122 : rhs[0] = gimple_assign_rhs1 (stmt);
4617 22122 : rhs[1] = gimple_assign_rhs2 (stmt);
4618 22122 : gimple_set_visited (stmt, true);
4619 66366 : for (i = 0; i < 2; i++)
4620 44244 : if (TREE_CODE (rhs[i]) == SSA_NAME
4621 44244 : && !get_ops (rhs[i], code, ops, loop)
4622 81952 : && has_single_use (rhs[i]))
4623 : {
4624 36937 : operand_entry *oe = operand_entry_pool.allocate ();
4625 :
4626 36937 : oe->op = rhs[i];
4627 36937 : oe->rank = code;
4628 36937 : oe->id = 0;
4629 36937 : oe->count = 1;
4630 36937 : oe->stmt_to_insert = NULL;
4631 36937 : ops->safe_push (oe);
4632 : }
4633 : return true;
4634 : }
4635 :
4636 : /* Find the ops that were added by get_ops starting from VAR, see if
4637 : they were changed during update_range_test and if yes, create new
4638 : stmts. */
4639 :
4640 : static tree
4641 10847 : update_ops (tree var, enum tree_code code, const vec<operand_entry *> &ops,
4642 : unsigned int *pidx, class loop *loop)
4643 : {
4644 10847 : gimple *stmt = SSA_NAME_DEF_STMT (var);
4645 10847 : tree rhs[4];
4646 10847 : int i;
4647 :
4648 10847 : if (!is_reassociable_op (stmt, code, loop))
4649 : return NULL;
4650 :
4651 3609 : rhs[0] = gimple_assign_rhs1 (stmt);
4652 3609 : rhs[1] = gimple_assign_rhs2 (stmt);
4653 3609 : rhs[2] = rhs[0];
4654 3609 : rhs[3] = rhs[1];
4655 10827 : for (i = 0; i < 2; i++)
4656 7218 : if (TREE_CODE (rhs[i]) == SSA_NAME)
4657 : {
4658 7218 : rhs[2 + i] = update_ops (rhs[i], code, ops, pidx, loop);
4659 7218 : if (rhs[2 + i] == NULL_TREE)
4660 : {
4661 6804 : if (has_single_use (rhs[i]))
4662 6786 : rhs[2 + i] = ops[(*pidx)++]->op;
4663 : else
4664 18 : rhs[2 + i] = rhs[i];
4665 : }
4666 : }
4667 3609 : if ((rhs[2] != rhs[0] || rhs[3] != rhs[1])
4668 3360 : && (rhs[2] != rhs[1] || rhs[3] != rhs[0]))
4669 : {
4670 3360 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
4671 3360 : var = make_ssa_name (TREE_TYPE (var));
4672 3360 : gassign *g = gimple_build_assign (var, gimple_assign_rhs_code (stmt),
4673 : rhs[2], rhs[3]);
4674 3360 : gimple_set_uid (g, gimple_uid (stmt));
4675 3360 : gimple_set_visited (g, true);
4676 3360 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
4677 3360 : gimple_stmt_iterator gsi2 = gsi_for_stmt (g);
4678 3360 : if (fold_stmt_inplace (&gsi2))
4679 2219 : update_stmt (g);
4680 : }
4681 : return var;
4682 : }
4683 :
4684 : /* Structure to track the initial value passed to get_ops and
4685 : the range in the ops vector for each basic block. */
4686 :
4687 : struct inter_bb_range_test_entry
4688 : {
4689 : tree op;
4690 : unsigned int first_idx, last_idx;
4691 : };
4692 :
4693 : /* Inter-bb range test optimization.
4694 :
4695 : Returns TRUE if a gimple conditional is optimized to a true/false,
4696 : otherwise return FALSE.
4697 :
4698 : This indicates to the caller that it should run a CFG cleanup pass
4699 : once reassociation is completed. */
4700 :
4701 : static bool
4702 19589933 : maybe_optimize_range_tests (gimple *stmt)
4703 : {
4704 19589933 : basic_block first_bb = gimple_bb (stmt);
4705 19589933 : basic_block last_bb = first_bb;
4706 19589933 : basic_block other_bb = NULL;
4707 19589933 : basic_block bb;
4708 19589933 : edge_iterator ei;
4709 19589933 : edge e;
4710 19589933 : auto_vec<operand_entry *> ops;
4711 19589933 : auto_vec<inter_bb_range_test_entry> bbinfo;
4712 19589933 : bool any_changes = false;
4713 19589933 : bool cfg_cleanup_needed = false;
4714 :
4715 : /* Consider only basic blocks that end with GIMPLE_COND or
4716 : a cast statement satisfying final_range_test_p. All
4717 : but the last bb in the first_bb .. last_bb range
4718 : should end with GIMPLE_COND. */
4719 19589933 : if (gimple_code (stmt) == GIMPLE_COND)
4720 : {
4721 8999851 : if (EDGE_COUNT (first_bb->succs) != 2)
4722 : return cfg_cleanup_needed;
4723 : }
4724 10590082 : else if (final_range_test_p (stmt))
4725 73742 : other_bb = single_succ (first_bb);
4726 : else
4727 : return cfg_cleanup_needed;
4728 :
4729 9073593 : if (stmt_could_throw_p (cfun, stmt))
4730 : return cfg_cleanup_needed;
4731 :
4732 : /* As relative ordering of post-dominator sons isn't fixed,
4733 : maybe_optimize_range_tests can be called first on any
4734 : bb in the range we want to optimize. So, start searching
4735 : backwards, if first_bb can be set to a predecessor. */
4736 9075946 : while (single_pred_p (first_bb))
4737 : {
4738 5990876 : basic_block pred_bb = single_pred (first_bb);
4739 5990876 : if (!suitable_cond_bb (pred_bb, first_bb, &other_bb, NULL, true))
4740 : break;
4741 635739 : if (!no_side_effect_bb (first_bb))
4742 : break;
4743 : first_bb = pred_bb;
4744 : }
4745 : /* If first_bb is last_bb, other_bb hasn't been computed yet.
4746 : Before starting forward search in last_bb successors, find
4747 : out the other_bb. */
4748 9073425 : if (first_bb == last_bb)
4749 : {
4750 9071109 : other_bb = NULL;
4751 : /* As non-GIMPLE_COND last stmt always terminates the range,
4752 : if forward search didn't discover anything, just give up. */
4753 9071109 : if (gimple_code (stmt) != GIMPLE_COND)
4754 : return cfg_cleanup_needed;
4755 : /* Look at both successors. Either it ends with a GIMPLE_COND
4756 : and satisfies suitable_cond_bb, or ends with a cast and
4757 : other_bb is that cast's successor. */
4758 24993117 : FOR_EACH_EDGE (e, ei, first_bb->succs)
4759 17278257 : if (!(e->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE))
4760 17278257 : || e->dest == first_bb)
4761 : return cfg_cleanup_needed;
4762 26242680 : else if (single_pred_p (e->dest))
4763 : {
4764 10247186 : stmt = last_nondebug_stmt (e->dest);
4765 10247186 : if (stmt
4766 10063543 : && gimple_code (stmt) == GIMPLE_COND
4767 14831717 : && EDGE_COUNT (e->dest->succs) == 2)
4768 : {
4769 4584531 : if (suitable_cond_bb (first_bb, e->dest, &other_bb,
4770 : NULL, true))
4771 : break;
4772 : else
4773 3833113 : other_bb = NULL;
4774 : }
4775 5662655 : else if (stmt
4776 5479012 : && final_range_test_p (stmt)
4777 5731553 : && find_edge (first_bb, single_succ (e->dest)))
4778 : {
4779 38810 : other_bb = single_succ (e->dest);
4780 38810 : if (other_bb == first_bb)
4781 0 : other_bb = NULL;
4782 : }
4783 : }
4784 8466278 : if (other_bb == NULL)
4785 : return cfg_cleanup_needed;
4786 : }
4787 : /* Now do the forward search, moving last_bb to successor bbs
4788 : that aren't other_bb. */
4789 1004378 : while (EDGE_COUNT (last_bb->succs) == 2)
4790 : {
4791 1617588 : FOR_EACH_EDGE (e, ei, last_bb->succs)
4792 1617588 : if (e->dest != other_bb)
4793 : break;
4794 998949 : if (e == NULL)
4795 : break;
4796 998949 : if (!single_pred_p (e->dest))
4797 : break;
4798 959503 : if (!suitable_cond_bb (e->dest, last_bb, &other_bb, NULL, false))
4799 : break;
4800 847995 : if (!no_side_effect_bb (e->dest))
4801 : break;
4802 211834 : last_bb = e->dest;
4803 : }
4804 792544 : if (first_bb == last_bb)
4805 : return cfg_cleanup_needed;
4806 : /* Here basic blocks first_bb through last_bb's predecessor
4807 : end with GIMPLE_COND, all of them have one of the edges to
4808 : other_bb and another to another block in the range,
4809 : all blocks except first_bb don't have side-effects and
4810 : last_bb ends with either GIMPLE_COND, or cast satisfying
4811 : final_range_test_p. */
4812 214355 : for (bb = last_bb; ; bb = single_pred (bb))
4813 : {
4814 369103 : enum tree_code code;
4815 369103 : tree lhs, rhs;
4816 369103 : inter_bb_range_test_entry bb_ent;
4817 :
4818 369103 : bb_ent.op = NULL_TREE;
4819 369103 : bb_ent.first_idx = ops.length ();
4820 369103 : bb_ent.last_idx = bb_ent.first_idx;
4821 369103 : e = find_edge (bb, other_bb);
4822 369103 : stmt = last_nondebug_stmt (bb);
4823 369103 : gimple_set_visited (stmt, true);
4824 369103 : if (gimple_code (stmt) != GIMPLE_COND)
4825 : {
4826 5429 : use_operand_p use_p;
4827 5429 : gimple *phi;
4828 5429 : edge e2;
4829 5429 : unsigned int d;
4830 :
4831 5429 : lhs = gimple_assign_lhs (stmt);
4832 5429 : rhs = gimple_assign_rhs1 (stmt);
4833 5429 : gcc_assert (bb == last_bb);
4834 :
4835 : /* stmt is
4836 : _123 = (int) _234;
4837 : OR
4838 : _234 = a_2(D) == 2;
4839 :
4840 : followed by:
4841 : <bb M>:
4842 : # _345 = PHI <_123(N), 1(...), 1(...)>
4843 :
4844 : or 0 instead of 1. If it is 0, the _234
4845 : range test is anded together with all the
4846 : other range tests, if it is 1, it is ored with
4847 : them. */
4848 5429 : single_imm_use (lhs, &use_p, &phi);
4849 5429 : gcc_assert (gimple_code (phi) == GIMPLE_PHI);
4850 5429 : e2 = find_edge (first_bb, other_bb);
4851 5429 : d = e2->dest_idx;
4852 5429 : gcc_assert (gimple_phi_arg_def (phi, e->dest_idx) == lhs);
4853 5429 : if (integer_zerop (gimple_phi_arg_def (phi, d)))
4854 : code = BIT_AND_EXPR;
4855 : else
4856 : {
4857 2800 : gcc_checking_assert (integer_onep (gimple_phi_arg_def (phi, d)));
4858 : code = BIT_IOR_EXPR;
4859 : }
4860 :
4861 : /* If _234 SSA_NAME_DEF_STMT is
4862 : _234 = _567 | _789;
4863 : (or &, corresponding to 1/0 in the phi arguments,
4864 : push into ops the individual range test arguments
4865 : of the bitwise or resp. and, recursively. */
4866 5429 : if (TREE_CODE (rhs) == SSA_NAME
4867 5429 : && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt))
4868 : != tcc_comparison)
4869 2876 : && !get_ops (rhs, code, &ops,
4870 : loop_containing_stmt (stmt))
4871 8130 : && has_single_use (rhs))
4872 : {
4873 : /* Otherwise, push the _234 range test itself. */
4874 2681 : operand_entry *oe = operand_entry_pool.allocate ();
4875 :
4876 2681 : oe->op = rhs;
4877 2681 : oe->rank = code;
4878 2681 : oe->id = 0;
4879 2681 : oe->count = 1;
4880 2681 : oe->stmt_to_insert = NULL;
4881 2681 : ops.safe_push (oe);
4882 2681 : bb_ent.last_idx++;
4883 2681 : bb_ent.op = rhs;
4884 : }
4885 2748 : else if (is_gimple_assign (stmt)
4886 2748 : && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt))
4887 : == tcc_comparison)
4888 2553 : && !get_ops (lhs, code, &ops,
4889 : loop_containing_stmt (stmt))
4890 5301 : && has_single_use (lhs))
4891 : {
4892 2553 : operand_entry *oe = operand_entry_pool.allocate ();
4893 2553 : oe->op = lhs;
4894 2553 : oe->rank = code;
4895 2553 : oe->id = 0;
4896 2553 : oe->count = 1;
4897 2553 : ops.safe_push (oe);
4898 2553 : bb_ent.last_idx++;
4899 2553 : bb_ent.op = lhs;
4900 : }
4901 : else
4902 : {
4903 195 : bb_ent.last_idx = ops.length ();
4904 195 : bb_ent.op = rhs;
4905 : }
4906 5429 : bbinfo.safe_push (bb_ent);
4907 16482 : for (unsigned int i = bb_ent.first_idx; i < bb_ent.last_idx; ++i)
4908 5624 : ops[i]->id = bb->index;
4909 5429 : continue;
4910 5429 : }
4911 363674 : else if (bb == last_bb)
4912 : {
4913 : /* For last_bb, handle also:
4914 : if (x_3(D) == 3)
4915 : goto <bb 6>; [34.00%]
4916 : else
4917 : goto <bb 7>; [66.00%]
4918 :
4919 : <bb 6> [local count: 79512730]:
4920 :
4921 : <bb 7> [local count: 1073741824]:
4922 : # prephitmp_7 = PHI <1(3), 1(4), 0(5), 1(2), 1(6)>
4923 : where bb 7 is OTHER_BB, but the PHI values from the
4924 : earlier bbs match the path through the empty bb
4925 : in between. */
4926 149319 : bool test_swapped_p = false;
4927 149319 : bool ok = suitable_cond_bb (single_pred (last_bb), last_bb,
4928 : &other_bb, &test_swapped_p, true);
4929 149319 : gcc_assert (ok);
4930 149319 : if (test_swapped_p)
4931 528 : e = EDGE_SUCC (bb, e == EDGE_SUCC (bb, 0) ? 1 : 0);
4932 : }
4933 : /* Otherwise stmt is GIMPLE_COND. */
4934 363674 : code = gimple_cond_code (stmt);
4935 363674 : lhs = gimple_cond_lhs (stmt);
4936 363674 : rhs = gimple_cond_rhs (stmt);
4937 363674 : if (TREE_CODE (lhs) == SSA_NAME
4938 363610 : && INTEGRAL_TYPE_P (TREE_TYPE (lhs))
4939 666606 : && ((code != EQ_EXPR && code != NE_EXPR)
4940 226092 : || rhs != boolean_false_node
4941 : /* Either push into ops the individual bitwise
4942 : or resp. and operands, depending on which
4943 : edge is other_bb. */
4944 44302 : || !get_ops (lhs, (((e->flags & EDGE_TRUE_VALUE) == 0)
4945 44302 : ^ (code == EQ_EXPR))
4946 : ? BIT_AND_EXPR : BIT_IOR_EXPR, &ops,
4947 : loop_containing_stmt (stmt))))
4948 : {
4949 : /* Or push the GIMPLE_COND stmt itself. */
4950 287521 : operand_entry *oe = operand_entry_pool.allocate ();
4951 :
4952 287521 : oe->op = NULL;
4953 575042 : oe->rank = (e->flags & EDGE_TRUE_VALUE)
4954 287521 : ? BIT_IOR_EXPR : BIT_AND_EXPR;
4955 : /* oe->op = NULL signs that there is no SSA_NAME
4956 : for the range test, and oe->id instead is the
4957 : basic block number, at which's end the GIMPLE_COND
4958 : is. */
4959 287521 : oe->id = bb->index;
4960 287521 : oe->count = 1;
4961 287521 : oe->stmt_to_insert = NULL;
4962 287521 : ops.safe_push (oe);
4963 287521 : bb_ent.op = NULL;
4964 287521 : bb_ent.last_idx++;
4965 : }
4966 76153 : else if (ops.length () > bb_ent.first_idx)
4967 : {
4968 15339 : bb_ent.op = lhs;
4969 15339 : bb_ent.last_idx = ops.length ();
4970 : }
4971 363674 : bbinfo.safe_push (bb_ent);
4972 1051416 : for (unsigned int i = bb_ent.first_idx; i < bb_ent.last_idx; ++i)
4973 324068 : ops[i]->id = bb->index;
4974 363674 : if (bb == first_bb)
4975 : break;
4976 214355 : }
4977 19710305 : if (ops.length () > 1)
4978 120372 : any_changes = optimize_range_tests (ERROR_MARK, &ops, first_bb);
4979 120372 : if (any_changes)
4980 : {
4981 : unsigned int idx, max_idx = 0;
4982 : /* update_ops relies on has_single_use predicates returning the
4983 : same values as it did during get_ops earlier. Additionally it
4984 : never removes statements, only adds new ones and it should walk
4985 : from the single imm use and check the predicate already before
4986 : making those changes.
4987 : On the other side, the handling of GIMPLE_COND directly can turn
4988 : previously multiply used SSA_NAMEs into single use SSA_NAMEs, so
4989 : it needs to be done in a separate loop afterwards. */
4990 22125 : for (bb = last_bb, idx = 0; ; bb = single_pred (bb), idx++)
4991 : {
4992 36815 : if (bbinfo[idx].first_idx < bbinfo[idx].last_idx
4993 36815 : && bbinfo[idx].op != NULL_TREE)
4994 : {
4995 3629 : tree new_op;
4996 :
4997 3629 : max_idx = idx;
4998 3629 : stmt = last_nondebug_stmt (bb);
4999 7258 : new_op = update_ops (bbinfo[idx].op,
5000 : (enum tree_code)
5001 3629 : ops[bbinfo[idx].first_idx]->rank,
5002 3629 : ops, &bbinfo[idx].first_idx,
5003 : loop_containing_stmt (stmt));
5004 3629 : if (new_op == NULL_TREE)
5005 : {
5006 434 : gcc_assert (bb == last_bb);
5007 434 : new_op = ops[bbinfo[idx].first_idx++]->op;
5008 : }
5009 3629 : if (bbinfo[idx].op != new_op)
5010 : {
5011 3374 : imm_use_iterator iter;
5012 3374 : use_operand_p use_p;
5013 3374 : gimple *use_stmt, *cast_or_tcc_cmp_stmt = NULL;
5014 :
5015 6758 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, bbinfo[idx].op)
5016 3384 : if (is_gimple_debug (use_stmt))
5017 10 : continue;
5018 3374 : else if (gimple_code (use_stmt) == GIMPLE_COND
5019 3374 : || gimple_code (use_stmt) == GIMPLE_PHI)
5020 6194 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
5021 3097 : SET_USE (use_p, new_op);
5022 277 : else if ((is_gimple_assign (use_stmt)
5023 277 : && (TREE_CODE_CLASS
5024 : (gimple_assign_rhs_code (use_stmt))
5025 : == tcc_comparison)))
5026 : cast_or_tcc_cmp_stmt = use_stmt;
5027 277 : else if (gimple_assign_cast_p (use_stmt))
5028 : cast_or_tcc_cmp_stmt = use_stmt;
5029 : else
5030 0 : gcc_unreachable ();
5031 :
5032 3374 : if (cast_or_tcc_cmp_stmt)
5033 : {
5034 277 : gcc_assert (bb == last_bb);
5035 277 : tree lhs = gimple_assign_lhs (cast_or_tcc_cmp_stmt);
5036 277 : tree new_lhs = make_ssa_name (TREE_TYPE (lhs));
5037 277 : enum tree_code rhs_code
5038 277 : = gimple_assign_cast_p (cast_or_tcc_cmp_stmt)
5039 277 : ? gimple_assign_rhs_code (cast_or_tcc_cmp_stmt)
5040 : : CONVERT_EXPR;
5041 277 : gassign *g;
5042 277 : if (is_gimple_min_invariant (new_op))
5043 : {
5044 73 : new_op = fold_convert (TREE_TYPE (lhs), new_op);
5045 73 : g = gimple_build_assign (new_lhs, new_op);
5046 : }
5047 : else
5048 204 : g = gimple_build_assign (new_lhs, rhs_code, new_op);
5049 277 : gimple_stmt_iterator gsi
5050 277 : = gsi_for_stmt (cast_or_tcc_cmp_stmt);
5051 277 : gimple_set_uid (g, gimple_uid (cast_or_tcc_cmp_stmt));
5052 277 : gimple_set_visited (g, true);
5053 277 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5054 554 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
5055 277 : if (is_gimple_debug (use_stmt))
5056 0 : continue;
5057 277 : else if (gimple_code (use_stmt) == GIMPLE_COND
5058 277 : || gimple_code (use_stmt) == GIMPLE_PHI)
5059 554 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
5060 277 : SET_USE (use_p, new_lhs);
5061 : else
5062 277 : gcc_unreachable ();
5063 : }
5064 : }
5065 : }
5066 36815 : if (bb == first_bb)
5067 : break;
5068 22125 : }
5069 22125 : for (bb = last_bb, idx = 0; ; bb = single_pred (bb), idx++)
5070 : {
5071 36815 : if (bbinfo[idx].first_idx < bbinfo[idx].last_idx
5072 32712 : && bbinfo[idx].op == NULL_TREE
5073 69527 : && ops[bbinfo[idx].first_idx]->op != NULL_TREE)
5074 : {
5075 57218 : gcond *cond_stmt = as_a <gcond *> (*gsi_last_bb (bb));
5076 :
5077 28609 : if (idx > max_idx)
5078 : max_idx = idx;
5079 :
5080 : /* If we collapse the conditional to a true/false
5081 : condition, then bubble that knowledge up to our caller. */
5082 28609 : if (integer_zerop (ops[bbinfo[idx].first_idx]->op))
5083 : {
5084 11728 : gimple_cond_make_false (cond_stmt);
5085 11728 : cfg_cleanup_needed = true;
5086 : }
5087 16881 : else if (integer_onep (ops[bbinfo[idx].first_idx]->op))
5088 : {
5089 4034 : gimple_cond_make_true (cond_stmt);
5090 4034 : cfg_cleanup_needed = true;
5091 : }
5092 : else
5093 : {
5094 12847 : gimple_cond_set_code (cond_stmt, NE_EXPR);
5095 12847 : gimple_cond_set_lhs (cond_stmt,
5096 12847 : ops[bbinfo[idx].first_idx]->op);
5097 12847 : gimple_cond_set_rhs (cond_stmt, boolean_false_node);
5098 : }
5099 28609 : update_stmt (cond_stmt);
5100 : }
5101 36815 : if (bb == first_bb)
5102 : break;
5103 22125 : }
5104 :
5105 : /* The above changes could result in basic blocks after the first
5106 : modified one, up to and including last_bb, to be executed even if
5107 : they would not be in the original program. If the value ranges of
5108 : assignment lhs' in those bbs were dependent on the conditions
5109 : guarding those basic blocks which now can change, the VRs might
5110 : be incorrect. As no_side_effect_bb should ensure those SSA_NAMEs
5111 : are only used within the same bb, it should be not a big deal if
5112 : we just reset all the VRs in those bbs. See PR68671. */
5113 35576 : for (bb = last_bb, idx = 0; idx < max_idx; bb = single_pred (bb), idx++)
5114 20886 : reset_flow_sensitive_info_in_bb (bb);
5115 : }
5116 : return cfg_cleanup_needed;
5117 19589933 : }
5118 :
5119 : /* Remove def stmt of VAR if VAR has zero uses and recurse
5120 : on rhs1 operand if so. */
5121 :
5122 : static void
5123 71481 : remove_visited_stmt_chain (tree var)
5124 : {
5125 96379 : gimple *stmt;
5126 96379 : gimple_stmt_iterator gsi;
5127 :
5128 121277 : while (1)
5129 : {
5130 96379 : if (TREE_CODE (var) != SSA_NAME || !has_zero_uses (var))
5131 : return;
5132 36420 : stmt = SSA_NAME_DEF_STMT (var);
5133 36420 : if (is_gimple_assign (stmt) && gimple_visited_p (stmt))
5134 : {
5135 24898 : var = gimple_assign_rhs1 (stmt);
5136 24898 : gsi = gsi_for_stmt (stmt);
5137 24898 : reassoc_remove_stmt (&gsi);
5138 24898 : release_defs (stmt);
5139 : }
5140 : else
5141 : return;
5142 : }
5143 : }
5144 :
5145 : /* This function checks three consecutive operands in
5146 : passed operands vector OPS starting from OPINDEX and
5147 : swaps two operands if it is profitable for binary operation
5148 : consuming OPINDEX + 1 and OPINDEX + 2 operands.
5149 :
5150 : We pair ops with the same rank if possible. */
5151 :
5152 : static void
5153 128729 : swap_ops_for_binary_stmt (const vec<operand_entry *> &ops,
5154 : unsigned int opindex)
5155 : {
5156 128729 : operand_entry *oe1, *oe2, *oe3;
5157 :
5158 128729 : oe1 = ops[opindex];
5159 128729 : oe2 = ops[opindex + 1];
5160 128729 : oe3 = ops[opindex + 2];
5161 :
5162 128729 : if (oe1->rank == oe2->rank && oe2->rank != oe3->rank)
5163 16741 : std::swap (*oe1, *oe3);
5164 111988 : else if (oe1->rank == oe3->rank && oe2->rank != oe3->rank)
5165 369 : std::swap (*oe1, *oe2);
5166 128729 : }
5167 :
5168 : /* Return true if OP is set by a comparison in BB, which a conditional
5169 : compare can compute into the condition flags. */
5170 :
5171 : static bool
5172 6 : ccmp_comparison_p (tree op, basic_block bb)
5173 : {
5174 6 : if (TREE_CODE (op) != SSA_NAME)
5175 : return false;
5176 :
5177 6 : gimple *def = SSA_NAME_DEF_STMT (op);
5178 6 : if (!is_gimple_assign (def)
5179 6 : || gimple_bb (def) != bb
5180 12 : || TREE_CODE_CLASS (gimple_assign_rhs_code (def)) != tcc_comparison)
5181 : return false;
5182 :
5183 : /* A conditional compare sets the flags from one scalar comparison. */
5184 6 : machine_mode mode = TYPE_MODE (TREE_TYPE (gimple_assign_rhs1 (def)));
5185 2 : return ((SCALAR_INT_MODE_P (mode) || SCALAR_FLOAT_MODE_P (mode))
5186 12 : && known_le (GET_MODE_SIZE (mode), UNITS_PER_WORD));
5187 : }
5188 :
5189 : /* Return true if all operands in OPS form one conditional compare sequence
5190 : for CODE in BB. Put the operand that must start the sequence last in OPS,
5191 : where rewrite_expr_tree builds the innermost pair. Every operand must be a
5192 : comparison, and at least one must be foldable into the flags. */
5193 :
5194 : static bool
5195 180941 : prepare_ccmp_chain (vec<operand_entry *> &ops, tree_code code, basic_block bb)
5196 : {
5197 180941 : if (!targetm.have_ccmp ()
5198 180941 : || (code != BIT_AND_EXPR && code != BIT_IOR_EXPR))
5199 : return false;
5200 :
5201 : bool seen_foldable = false;
5202 : int head = -1;
5203 :
5204 8 : for (unsigned int i = 0; i < ops.length (); ++i)
5205 : {
5206 6 : const operand_entry *oe = ops[i];
5207 6 : if (!ccmp_comparison_p (oe->op, bb))
5208 : return false;
5209 :
5210 6 : bool foldable = has_single_use (oe->op);
5211 6 : if (foldable
5212 6 : && gimple_could_trap_p (SSA_NAME_DEF_STMT (oe->op)))
5213 : {
5214 0 : if (head >= 0)
5215 : return false;
5216 0 : head = i;
5217 : }
5218 :
5219 : /* Comparison leaves can have uses outside this chain. */
5220 6 : seen_foldable |= foldable;
5221 : }
5222 :
5223 2 : if (head >= 0)
5224 0 : std::swap (*ops[head], *ops.last ());
5225 : return seen_foldable;
5226 : }
5227 :
5228 : /* If definition of RHS1 or RHS2 dominates STMT, return the later of those
5229 : two definitions, otherwise return STMT. Sets INSERT_BEFORE to indicate
5230 : whether RHS1 op RHS2 can be inserted before or needs to be inserted
5231 : after the returned stmt. */
5232 :
5233 : static inline gimple *
5234 374332 : find_insert_point (gimple *stmt, tree rhs1, tree rhs2, bool &insert_before)
5235 : {
5236 374332 : insert_before = true;
5237 374332 : if (TREE_CODE (rhs1) == SSA_NAME
5238 374332 : && reassoc_stmt_dominates_stmt_p (stmt, SSA_NAME_DEF_STMT (rhs1)))
5239 : {
5240 11121 : stmt = SSA_NAME_DEF_STMT (rhs1);
5241 11121 : insert_before = false;
5242 : }
5243 374332 : if (TREE_CODE (rhs2) == SSA_NAME
5244 374332 : && reassoc_stmt_dominates_stmt_p (stmt, SSA_NAME_DEF_STMT (rhs2)))
5245 : {
5246 20122 : stmt = SSA_NAME_DEF_STMT (rhs2);
5247 20122 : insert_before = false;
5248 : }
5249 374332 : return stmt;
5250 : }
5251 :
5252 : /* If the stmt that defines operand has to be inserted, insert it
5253 : before the use. */
5254 : static void
5255 85 : insert_stmt_before_use (gimple *stmt, gimple *stmt_to_insert)
5256 : {
5257 85 : gcc_assert (is_gimple_assign (stmt_to_insert));
5258 85 : tree rhs1 = gimple_assign_rhs1 (stmt_to_insert);
5259 85 : tree rhs2 = gimple_assign_rhs2 (stmt_to_insert);
5260 85 : bool insert_before;
5261 85 : gimple *insert_point = find_insert_point (stmt, rhs1, rhs2, insert_before);
5262 85 : gimple_stmt_iterator gsi = gsi_for_stmt (insert_point);
5263 85 : gimple_set_uid (stmt_to_insert, gimple_uid (insert_point));
5264 :
5265 : /* If the insert point is not stmt, then insert_point would be
5266 : the point where operand rhs1 or rhs2 is defined. In this case,
5267 : stmt_to_insert has to be inserted afterwards. This would
5268 : only happen when the stmt insertion point is flexible. */
5269 85 : if (insert_before)
5270 85 : gsi_insert_before (&gsi, stmt_to_insert, GSI_NEW_STMT);
5271 : else
5272 0 : insert_stmt_after (stmt_to_insert, insert_point);
5273 85 : }
5274 :
5275 :
5276 : /* Recursively rewrite our linearized statements so that the operators
5277 : match those in OPS[OPINDEX], putting the computation in rank
5278 : order. Return new lhs.
5279 : CHANGED is true if we shouldn't reuse the lhs SSA_NAME both in
5280 : the current stmt and during recursive invocations.
5281 : NEXT_CHANGED is true if we shouldn't reuse the lhs SSA_NAME in
5282 : recursive invocations. */
5283 :
5284 : static tree
5285 4989129 : rewrite_expr_tree (gimple *stmt, enum tree_code rhs_code, unsigned int opindex,
5286 : const vec<operand_entry *> &ops, bool changed,
5287 : bool next_changed)
5288 : {
5289 4989129 : tree rhs1 = gimple_assign_rhs1 (stmt);
5290 4989129 : tree rhs2 = gimple_assign_rhs2 (stmt);
5291 4989129 : tree lhs = gimple_assign_lhs (stmt);
5292 4989129 : operand_entry *oe;
5293 :
5294 : /* The final recursion case for this function is that you have
5295 : exactly two operations left.
5296 : If we had exactly one op in the entire list to start with, we
5297 : would have never called this function, and the tail recursion
5298 : rewrites them one at a time. */
5299 9978258 : if (opindex + 2 == ops.length ())
5300 : {
5301 4760914 : operand_entry *oe1, *oe2;
5302 :
5303 4760914 : oe1 = ops[opindex];
5304 4760914 : oe2 = ops[opindex + 1];
5305 4760914 : if (commutative_tree_code (rhs_code)
5306 4760914 : && tree_swap_operands_p (oe1->op, oe2->op))
5307 : std::swap (oe1, oe2);
5308 :
5309 4760914 : if (rhs1 != oe1->op || rhs2 != oe2->op)
5310 : {
5311 245895 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5312 245895 : unsigned int uid = gimple_uid (stmt);
5313 :
5314 245895 : if (dump_file && (dump_flags & TDF_DETAILS))
5315 : {
5316 31 : fprintf (dump_file, "Transforming ");
5317 31 : print_gimple_stmt (dump_file, stmt, 0);
5318 : }
5319 :
5320 : /* If the stmt that defines operand has to be inserted, insert it
5321 : before the use. */
5322 245895 : if (oe1->stmt_to_insert)
5323 34 : insert_stmt_before_use (stmt, oe1->stmt_to_insert);
5324 245895 : if (oe2->stmt_to_insert)
5325 51 : insert_stmt_before_use (stmt, oe2->stmt_to_insert);
5326 : /* Even when changed is false, reassociation could have e.g. removed
5327 : some redundant operations, so unless we are just swapping the
5328 : arguments or unless there is no change at all (then we just
5329 : return lhs), force creation of a new SSA_NAME. */
5330 245895 : if (changed || ((rhs1 != oe2->op || rhs2 != oe1->op) && opindex))
5331 : {
5332 109072 : bool insert_before;
5333 109072 : gimple *insert_point
5334 109072 : = find_insert_point (stmt, oe1->op, oe2->op, insert_before);
5335 109072 : lhs = make_ssa_name (TREE_TYPE (lhs));
5336 109072 : stmt
5337 109072 : = gimple_build_assign (lhs, rhs_code,
5338 : oe1->op, oe2->op);
5339 109072 : gimple_set_uid (stmt, uid);
5340 109072 : gimple_set_visited (stmt, true);
5341 109072 : if (insert_before)
5342 86480 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5343 : else
5344 22592 : insert_stmt_after (stmt, insert_point);
5345 109072 : }
5346 : else
5347 : {
5348 136823 : bool insert_before;
5349 136823 : gcc_checking_assert (find_insert_point (stmt, oe1->op, oe2->op,
5350 : insert_before)
5351 : == stmt);
5352 136823 : gimple_assign_set_rhs1 (stmt, oe1->op);
5353 136823 : gimple_assign_set_rhs2 (stmt, oe2->op);
5354 136823 : update_stmt (stmt);
5355 : }
5356 :
5357 245895 : if (rhs1 != oe1->op && rhs1 != oe2->op)
5358 54845 : remove_visited_stmt_chain (rhs1);
5359 :
5360 245895 : if (dump_file && (dump_flags & TDF_DETAILS))
5361 : {
5362 31 : fprintf (dump_file, " into ");
5363 31 : print_gimple_stmt (dump_file, stmt, 0);
5364 : }
5365 : }
5366 : return lhs;
5367 : }
5368 :
5369 : /* If we hit here, we should have 3 or more ops left. */
5370 228215 : gcc_assert (opindex + 2 < ops.length ());
5371 :
5372 : /* Rewrite the next operator. */
5373 228215 : oe = ops[opindex];
5374 :
5375 : /* If the stmt that defines operand has to be inserted, insert it
5376 : before the use. */
5377 228215 : if (oe->stmt_to_insert)
5378 0 : insert_stmt_before_use (stmt, oe->stmt_to_insert);
5379 :
5380 : /* Recurse on the LHS of the binary operator, which is guaranteed to
5381 : be the non-leaf side. */
5382 228215 : tree new_rhs1
5383 437645 : = rewrite_expr_tree (SSA_NAME_DEF_STMT (rhs1), rhs_code, opindex + 1, ops,
5384 209430 : changed || oe->op != rhs2 || next_changed,
5385 : false);
5386 :
5387 228215 : if (oe->op != rhs2 || new_rhs1 != rhs1)
5388 : {
5389 128352 : if (dump_file && (dump_flags & TDF_DETAILS))
5390 : {
5391 6 : fprintf (dump_file, "Transforming ");
5392 6 : print_gimple_stmt (dump_file, stmt, 0);
5393 : }
5394 :
5395 : /* If changed is false, this is either opindex == 0
5396 : or all outer rhs2's were equal to corresponding oe->op,
5397 : and powi_result is NULL.
5398 : That means lhs is equivalent before and after reassociation.
5399 : Otherwise ensure the old lhs SSA_NAME is not reused and
5400 : create a new stmt as well, so that any debug stmts will be
5401 : properly adjusted. */
5402 128352 : if (changed)
5403 : {
5404 18550 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5405 18550 : unsigned int uid = gimple_uid (stmt);
5406 18550 : bool insert_before;
5407 18550 : gimple *insert_point = find_insert_point (stmt, new_rhs1, oe->op,
5408 : insert_before);
5409 :
5410 18550 : lhs = make_ssa_name (TREE_TYPE (lhs));
5411 18550 : stmt = gimple_build_assign (lhs, rhs_code,
5412 : new_rhs1, oe->op);
5413 18550 : gimple_set_uid (stmt, uid);
5414 18550 : gimple_set_visited (stmt, true);
5415 18550 : if (insert_before)
5416 10975 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5417 : else
5418 7575 : insert_stmt_after (stmt, insert_point);
5419 : }
5420 : else
5421 : {
5422 109802 : bool insert_before;
5423 109802 : gcc_checking_assert (find_insert_point (stmt, new_rhs1, oe->op,
5424 : insert_before)
5425 : == stmt);
5426 109802 : gimple_assign_set_rhs1 (stmt, new_rhs1);
5427 109802 : gimple_assign_set_rhs2 (stmt, oe->op);
5428 109802 : update_stmt (stmt);
5429 : }
5430 :
5431 128352 : if (dump_file && (dump_flags & TDF_DETAILS))
5432 : {
5433 6 : fprintf (dump_file, " into ");
5434 6 : print_gimple_stmt (dump_file, stmt, 0);
5435 : }
5436 : }
5437 : return lhs;
5438 : }
5439 :
5440 : /* Find out how many cycles we need to compute statements chain.
5441 : OPS_NUM holds number os statements in a chain. CPU_WIDTH is a
5442 : maximum number of independent statements we may execute per cycle. */
5443 :
5444 : static int
5445 18817 : get_required_cycles (int ops_num, int cpu_width)
5446 : {
5447 18817 : int res;
5448 18817 : int elog;
5449 18817 : unsigned int rest;
5450 :
5451 : /* While we have more than 2 * cpu_width operands
5452 : we may reduce number of operands by cpu_width
5453 : per cycle. */
5454 18817 : res = ops_num / (2 * cpu_width);
5455 :
5456 : /* Remained operands count may be reduced twice per cycle
5457 : until we have only one operand. */
5458 18817 : rest = (unsigned)(ops_num - res * cpu_width);
5459 18817 : elog = exact_log2 (rest);
5460 8877 : if (elog >= 0)
5461 8877 : res += elog;
5462 : else
5463 19880 : res += floor_log2 (rest) + 1;
5464 :
5465 18817 : return res;
5466 : }
5467 :
5468 : /* Given that the target fully pipelines FMA instructions, return the latency
5469 : of MULT_EXPRs that can't be hidden by the FMAs. WIDTH is the number of
5470 : pipes. */
5471 :
5472 : static inline int
5473 0 : get_mult_latency_consider_fma (int ops_num, int mult_num, int width)
5474 : {
5475 0 : gcc_checking_assert (mult_num && mult_num <= ops_num);
5476 :
5477 : /* For each partition, if mult_num == ops_num, there's latency(MULT)*2.
5478 : e.g:
5479 :
5480 : A * B + C * D
5481 : =>
5482 : _1 = A * B;
5483 : _2 = .FMA (C, D, _1);
5484 :
5485 : Otherwise there's latency(MULT)*1 in the first FMA. */
5486 0 : return CEIL (ops_num, width) == CEIL (mult_num, width) ? 2 : 1;
5487 : }
5488 :
5489 : /* Returns an optimal number of registers to use for computation of
5490 : given statements.
5491 :
5492 : LHS is the result ssa name of OPS. MULT_NUM is number of sub-expressions
5493 : that are MULT_EXPRs, when OPS are PLUS_EXPRs or MINUS_EXPRs. */
5494 :
5495 : static int
5496 18954 : get_reassociation_width (vec<operand_entry *> *ops, int mult_num, tree lhs,
5497 : enum tree_code opc, machine_mode mode)
5498 : {
5499 18954 : int param_width = param_tree_reassoc_width;
5500 18954 : int width;
5501 18954 : int width_min;
5502 18954 : int cycles_best;
5503 18954 : int ops_num = ops->length ();
5504 :
5505 18954 : if (param_width > 0)
5506 : width = param_width;
5507 : else
5508 18910 : width = targetm.sched.reassociation_width (opc, mode);
5509 :
5510 18954 : if (width == 1)
5511 : return width;
5512 :
5513 : /* Get the minimal time required for sequence computation. */
5514 7179 : cycles_best = get_required_cycles (ops_num, width);
5515 :
5516 : /* Check if we may use less width and still compute sequence for
5517 : the same time. It will allow us to reduce registers usage.
5518 : get_required_cycles is monotonically increasing with lower width
5519 : so we can perform a binary search for the minimal width that still
5520 : results in the optimal cycle count. */
5521 7179 : width_min = 1;
5522 :
5523 : /* If the target fully pipelines FMA instruction, the multiply part can start
5524 : already if its operands are ready. Assuming symmetric pipes are used for
5525 : FMUL/FADD/FMA, then for a sequence of FMA like:
5526 :
5527 : _8 = .FMA (_2, _3, _1);
5528 : _9 = .FMA (_5, _4, _8);
5529 : _10 = .FMA (_7, _6, _9);
5530 :
5531 : , if width=1, the latency is latency(MULT) + latency(ADD)*3.
5532 : While with width=2:
5533 :
5534 : _8 = _4 * _5;
5535 : _9 = .FMA (_2, _3, _1);
5536 : _10 = .FMA (_6, _7, _8);
5537 : _11 = _9 + _10;
5538 :
5539 : , it is latency(MULT)*2 + latency(ADD)*2. Assuming latency(MULT) >=
5540 : latency(ADD), the first variant is preferred.
5541 :
5542 : Find out if we can get a smaller width considering FMA.
5543 : Assume FMUL and FMA use the same units that can also do FADD.
5544 : For other scenarios, such as when FMUL and FADD are using separated units,
5545 : the following code may not apply. */
5546 :
5547 7179 : int width_mult = targetm.sched.reassociation_width (MULT_EXPR, mode);
5548 7179 : if (width > 1 && mult_num && param_fully_pipelined_fma
5549 0 : && width_mult <= width)
5550 : {
5551 : /* Latency of MULT_EXPRs. */
5552 0 : int lat_mul
5553 0 : = get_mult_latency_consider_fma (ops_num, mult_num, width_mult);
5554 :
5555 : /* Quick search might not apply. So start from 1. */
5556 0 : for (int i = 1; i < width_mult; i++)
5557 : {
5558 0 : int lat_mul_new
5559 0 : = get_mult_latency_consider_fma (ops_num, mult_num, i);
5560 0 : int lat_add_new = get_required_cycles (ops_num, i);
5561 :
5562 : /* Assume latency(MULT) >= latency(ADD). */
5563 0 : if (lat_mul - lat_mul_new >= lat_add_new - cycles_best)
5564 : {
5565 : width = i;
5566 : break;
5567 : }
5568 : }
5569 : }
5570 : else
5571 : {
5572 16926 : while (width > width_min)
5573 : {
5574 11638 : int width_mid = (width + width_min) / 2;
5575 :
5576 11638 : if (get_required_cycles (ops_num, width_mid) == cycles_best)
5577 : width = width_mid;
5578 2083 : else if (width_min < width_mid)
5579 : width_min = width_mid;
5580 : else
5581 : break;
5582 : }
5583 : }
5584 :
5585 : /* If there's loop dependent FMA result, return width=2 to avoid it. This is
5586 : better than skipping these FMA candidates in widening_mul. */
5587 7179 : if (width == 1
5588 7179 : && maybe_le (tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (lhs))),
5589 : param_avoid_fma_max_bits))
5590 : {
5591 : /* Look for cross backedge dependency:
5592 : 1. LHS is a phi argument in the same basic block it is defined.
5593 : 2. And the result of the phi node is used in OPS. */
5594 4868 : basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (lhs));
5595 :
5596 4868 : use_operand_p use_p;
5597 4868 : imm_use_iterator iter;
5598 10943 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
5599 10310 : if (gphi *phi = dyn_cast<gphi *> (USE_STMT (use_p)))
5600 : {
5601 4235 : if (gimple_phi_arg_edge (phi, phi_arg_index_from_use (use_p))->src
5602 : != bb)
5603 0 : continue;
5604 4235 : tree phi_result = gimple_phi_result (phi);
5605 4235 : operand_entry *oe;
5606 4235 : unsigned int j;
5607 20976 : FOR_EACH_VEC_ELT (*ops, j, oe)
5608 : {
5609 12705 : if (TREE_CODE (oe->op) != SSA_NAME)
5610 0 : continue;
5611 :
5612 : /* Result of phi is operand of PLUS_EXPR. */
5613 12705 : if (oe->op == phi_result)
5614 2039 : return 2;
5615 :
5616 : /* Check is result of phi is operand of MULT_EXPR. */
5617 10666 : gimple *def_stmt = SSA_NAME_DEF_STMT (oe->op);
5618 10666 : if (is_gimple_assign (def_stmt)
5619 10666 : && gimple_assign_rhs_code (def_stmt) == NEGATE_EXPR)
5620 : {
5621 2246 : tree rhs = gimple_assign_rhs1 (def_stmt);
5622 2246 : if (TREE_CODE (rhs) == SSA_NAME)
5623 : {
5624 2246 : if (rhs == phi_result)
5625 : return 2;
5626 2246 : def_stmt = SSA_NAME_DEF_STMT (rhs);
5627 : }
5628 : }
5629 10666 : if (is_gimple_assign (def_stmt)
5630 10666 : && gimple_assign_rhs_code (def_stmt) == MULT_EXPR)
5631 : {
5632 8470 : if (gimple_assign_rhs1 (def_stmt) == phi_result
5633 8470 : || gimple_assign_rhs2 (def_stmt) == phi_result)
5634 : return 2;
5635 : }
5636 : }
5637 4868 : }
5638 : }
5639 :
5640 : return width;
5641 : }
5642 :
5643 : #define SPECIAL_BIASED_END_STMT 0 /* It is the end stmt of all ops. */
5644 : #define BIASED_END_STMT 1 /* It is the end stmt of normal or biased ops. */
5645 : #define NORMAL_END_STMT 2 /* It is the end stmt of normal ops. */
5646 :
5647 : /* Rewrite statements with dependency chain with regard the chance to generate
5648 : FMA.
5649 : For the chain with FMA: Try to keep fma opportunity as much as possible.
5650 : For the chain without FMA: Putting the computation in rank order and trying
5651 : to allow operations to be executed in parallel.
5652 : E.g.
5653 : e + f + a * b + c * d;
5654 :
5655 : ssa1 = e + a * b;
5656 : ssa2 = f + c * d;
5657 : ssa3 = ssa1 + ssa2;
5658 :
5659 : This reassociation approach preserves the chance of fma generation as much
5660 : as possible.
5661 :
5662 : Another thing is to avoid adding loop-carried ops to long chains, otherwise
5663 : the whole chain will have dependencies across the loop iteration. Just keep
5664 : loop-carried ops in a separate chain.
5665 : E.g.
5666 : x_1 = phi (x_0, x_2)
5667 : y_1 = phi (y_0, y_2)
5668 :
5669 : a + b + c + d + e + x1 + y1
5670 :
5671 : SSA1 = a + b;
5672 : SSA2 = c + d;
5673 : SSA3 = SSA1 + e;
5674 : SSA4 = SSA3 + SSA2;
5675 : SSA5 = x1 + y1;
5676 : SSA6 = SSA4 + SSA5;
5677 : */
5678 : static void
5679 1891 : rewrite_expr_tree_parallel (gassign *stmt, int width, bool has_fma,
5680 : const vec<operand_entry *> &ops)
5681 : {
5682 1891 : enum tree_code opcode = gimple_assign_rhs_code (stmt);
5683 1891 : int op_num = ops.length ();
5684 1891 : int op_normal_num = op_num;
5685 1891 : gcc_assert (op_num > 0);
5686 1891 : int stmt_num = op_num - 1;
5687 1891 : gimple **stmts = XALLOCAVEC (gimple *, stmt_num);
5688 1891 : int i = 0, j = 0;
5689 1891 : tree tmp_op[2], op1;
5690 1891 : operand_entry *oe;
5691 1891 : gimple *stmt1 = NULL;
5692 1891 : tree last_rhs1 = gimple_assign_rhs1 (stmt);
5693 1891 : int last_rhs1_stmt_index = 0, last_rhs2_stmt_index = 0;
5694 1891 : int width_active = 0, width_count = 0;
5695 1891 : bool has_biased = false, ops_changed = false;
5696 1891 : auto_vec<operand_entry *> ops_normal;
5697 1891 : auto_vec<operand_entry *> ops_biased;
5698 1891 : vec<operand_entry *> *ops1;
5699 :
5700 : /* We start expression rewriting from the top statements.
5701 : So, in this loop we create a full list of statements
5702 : we will work with. */
5703 1891 : stmts[stmt_num - 1] = stmt;
5704 9180 : for (i = stmt_num - 2; i >= 0; i--)
5705 7289 : stmts[i] = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmts[i+1]));
5706 :
5707 : /* Avoid adding loop-carried ops to long chains, first filter out the
5708 : loop-carried. But we need to make sure that the length of the remainder
5709 : is not less than 4, which is the smallest ops length we can break the
5710 : dependency. */
5711 14853 : FOR_EACH_VEC_ELT (ops, i, oe)
5712 : {
5713 11071 : if (TREE_CODE (oe->op) == SSA_NAME
5714 10905 : && bitmap_bit_p (biased_names, SSA_NAME_VERSION (oe->op))
5715 11328 : && op_normal_num > 4)
5716 : {
5717 223 : ops_biased.safe_push (oe);
5718 223 : has_biased = true;
5719 223 : op_normal_num --;
5720 : }
5721 : else
5722 10848 : ops_normal.safe_push (oe);
5723 : }
5724 :
5725 : /* Width should not be larger than ops length / 2, since we can not create
5726 : more parallel dependency chains that exceeds such value. */
5727 1891 : int width_normal = op_normal_num / 2;
5728 1891 : int width_biased = (op_num - op_normal_num) / 2;
5729 1891 : width_normal = width <= width_normal ? width : width_normal;
5730 1891 : width_biased = width <= width_biased ? width : width_biased;
5731 :
5732 1891 : ops1 = &ops_normal;
5733 1891 : width_count = width_active = width_normal;
5734 :
5735 : /* Build parallel dependency chain according to width. */
5736 11071 : for (i = 0; i < stmt_num; i++)
5737 : {
5738 9180 : if (dump_file && (dump_flags & TDF_DETAILS))
5739 : {
5740 6 : fprintf (dump_file, "Transforming ");
5741 6 : print_gimple_stmt (dump_file, stmts[i], 0);
5742 : }
5743 :
5744 : /* When the work of normal ops is over, but the loop is not over,
5745 : continue to do biased ops. */
5746 9180 : if (width_count == 0 && ops1 == &ops_normal)
5747 : {
5748 219 : ops1 = &ops_biased;
5749 219 : width_count = width_active = width_biased;
5750 219 : ops_changed = true;
5751 : }
5752 :
5753 : /* Swap the operands if no FMA in the chain. */
5754 9180 : if (ops1->length () > 2 && !has_fma)
5755 4469 : swap_ops_for_binary_stmt (*ops1, ops1->length () - 3);
5756 :
5757 9180 : if (i < width_active
5758 5207 : || (ops_changed && i <= (last_rhs1_stmt_index + width_active)))
5759 : {
5760 11928 : for (j = 0; j < 2; j++)
5761 : {
5762 7952 : oe = ops1->pop ();
5763 7952 : tmp_op[j] = oe->op;
5764 : /* If the stmt that defines operand has to be inserted, insert it
5765 : before the use. */
5766 7952 : stmt1 = oe->stmt_to_insert;
5767 7952 : if (stmt1)
5768 0 : insert_stmt_before_use (stmts[i], stmt1);
5769 7952 : stmt1 = NULL;
5770 : }
5771 3976 : stmts[i] = build_and_add_sum (TREE_TYPE (last_rhs1),
5772 : tmp_op[1],
5773 : tmp_op[0],
5774 : opcode);
5775 3976 : gimple_set_visited (stmts[i], true);
5776 :
5777 : }
5778 : else
5779 : {
5780 : /* We keep original statement only for the last one. All others are
5781 : recreated. */
5782 5204 : if (!ops1->length ())
5783 : {
5784 : /* For biased length equal to 2. */
5785 2085 : if (width_count == BIASED_END_STMT && !last_rhs2_stmt_index)
5786 1 : last_rhs2_stmt_index = i - 1;
5787 :
5788 : /* When width_count == 2 and there is no biased, just finish. */
5789 2085 : if (width_count == NORMAL_END_STMT && !has_biased)
5790 : {
5791 1672 : last_rhs1_stmt_index = i - 1;
5792 1672 : last_rhs2_stmt_index = i - 2;
5793 : }
5794 2085 : if (last_rhs1_stmt_index && (last_rhs2_stmt_index || !has_biased))
5795 : {
5796 : /* We keep original statement only for the last one. All
5797 : others are recreated. */
5798 1674 : gimple_assign_set_rhs1 (stmts[i], gimple_assign_lhs
5799 1674 : (stmts[last_rhs1_stmt_index]));
5800 1674 : gimple_assign_set_rhs2 (stmts[i], gimple_assign_lhs
5801 1674 : (stmts[last_rhs2_stmt_index]));
5802 1674 : update_stmt (stmts[i]);
5803 : }
5804 : else
5805 : {
5806 1233 : stmts[i] =
5807 411 : build_and_add_sum (TREE_TYPE (last_rhs1),
5808 411 : gimple_assign_lhs (stmts[i-width_count]),
5809 : gimple_assign_lhs
5810 411 : (stmts[i-width_count+1]),
5811 : opcode);
5812 411 : gimple_set_visited (stmts[i], true);
5813 411 : width_count--;
5814 :
5815 : /* It is the end of normal or biased ops.
5816 : last_rhs1_stmt_index used to record the last stmt index
5817 : for normal ops. last_rhs2_stmt_index used to record the
5818 : last stmt index for biased ops. */
5819 411 : if (width_count == BIASED_END_STMT)
5820 : {
5821 220 : gcc_assert (has_biased);
5822 220 : if (ops_biased.length ())
5823 : last_rhs1_stmt_index = i;
5824 : else
5825 1 : last_rhs2_stmt_index = i;
5826 : width_count--;
5827 : }
5828 : }
5829 : }
5830 : else
5831 : {
5832 : /* Attach the rest ops to the parallel dependency chain. */
5833 3119 : oe = ops1->pop ();
5834 3119 : op1 = oe->op;
5835 3119 : stmt1 = oe->stmt_to_insert;
5836 3119 : if (stmt1)
5837 0 : insert_stmt_before_use (stmts[i], stmt1);
5838 3119 : stmt1 = NULL;
5839 :
5840 : /* For only one biased ops. */
5841 3119 : if (width_count == SPECIAL_BIASED_END_STMT)
5842 : {
5843 : /* We keep original statement only for the last one. All
5844 : others are recreated. */
5845 217 : gcc_assert (has_biased);
5846 217 : gimple_assign_set_rhs1 (stmts[i], gimple_assign_lhs
5847 217 : (stmts[last_rhs1_stmt_index]));
5848 217 : gimple_assign_set_rhs2 (stmts[i], op1);
5849 217 : update_stmt (stmts[i]);
5850 : }
5851 : else
5852 : {
5853 2902 : stmts[i] = build_and_add_sum (TREE_TYPE (last_rhs1),
5854 : gimple_assign_lhs
5855 2902 : (stmts[i-width_active]),
5856 : op1,
5857 : opcode);
5858 2902 : gimple_set_visited (stmts[i], true);
5859 : }
5860 : }
5861 : }
5862 :
5863 9180 : if (dump_file && (dump_flags & TDF_DETAILS))
5864 : {
5865 6 : fprintf (dump_file, " into ");
5866 6 : print_gimple_stmt (dump_file, stmts[i], 0);
5867 : }
5868 : }
5869 :
5870 1891 : remove_visited_stmt_chain (last_rhs1);
5871 1891 : }
5872 :
5873 : /* Transform STMT, which is really (A +B) + (C + D) into the left
5874 : linear form, ((A+B)+C)+D.
5875 : Recurse on D if necessary. */
5876 :
5877 : static void
5878 2614 : linearize_expr (gimple *stmt)
5879 : {
5880 2614 : gimple_stmt_iterator gsi;
5881 2614 : gimple *binlhs = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
5882 2614 : gimple *binrhs = SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt));
5883 2614 : gimple *oldbinrhs = binrhs;
5884 2614 : enum tree_code rhscode = gimple_assign_rhs_code (stmt);
5885 2614 : gimple *newbinrhs = NULL;
5886 2614 : class loop *loop = loop_containing_stmt (stmt);
5887 2614 : tree lhs = gimple_assign_lhs (stmt);
5888 :
5889 2614 : gcc_assert (is_reassociable_op (binlhs, rhscode, loop)
5890 : && is_reassociable_op (binrhs, rhscode, loop));
5891 :
5892 2614 : gsi = gsi_for_stmt (stmt);
5893 :
5894 2614 : gimple_assign_set_rhs2 (stmt, gimple_assign_rhs1 (binrhs));
5895 2614 : binrhs = gimple_build_assign (make_ssa_name (TREE_TYPE (lhs)),
5896 : gimple_assign_rhs_code (binrhs),
5897 : gimple_assign_lhs (binlhs),
5898 : gimple_assign_rhs2 (binrhs));
5899 2614 : gimple_assign_set_rhs1 (stmt, gimple_assign_lhs (binrhs));
5900 2614 : gsi_insert_before (&gsi, binrhs, GSI_SAME_STMT);
5901 2614 : gimple_set_uid (binrhs, gimple_uid (stmt));
5902 :
5903 2614 : if (TREE_CODE (gimple_assign_rhs2 (stmt)) == SSA_NAME)
5904 2611 : newbinrhs = SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt));
5905 :
5906 2614 : if (dump_file && (dump_flags & TDF_DETAILS))
5907 : {
5908 0 : fprintf (dump_file, "Linearized: ");
5909 0 : print_gimple_stmt (dump_file, stmt, 0);
5910 : }
5911 :
5912 2614 : reassociate_stats.linearized++;
5913 2614 : update_stmt (stmt);
5914 :
5915 2614 : gsi = gsi_for_stmt (oldbinrhs);
5916 2614 : reassoc_remove_stmt (&gsi);
5917 2614 : release_defs (oldbinrhs);
5918 :
5919 2614 : gimple_set_visited (stmt, true);
5920 2614 : gimple_set_visited (binlhs, true);
5921 2614 : gimple_set_visited (binrhs, true);
5922 :
5923 : /* Tail recurse on the new rhs if it still needs reassociation. */
5924 2614 : if (newbinrhs && is_reassociable_op (newbinrhs, rhscode, loop))
5925 : /* ??? This should probably be linearize_expr (newbinrhs) but I don't
5926 : want to change the algorithm while converting to tuples. */
5927 622 : linearize_expr (stmt);
5928 2614 : }
5929 :
5930 : /* If LHS has a single immediate use that is a GIMPLE_ASSIGN statement, return
5931 : it. Otherwise, return NULL. */
5932 :
5933 : static gimple *
5934 416468 : get_single_immediate_use (tree lhs)
5935 : {
5936 416468 : use_operand_p immuse;
5937 416468 : gimple *immusestmt;
5938 :
5939 416468 : if (TREE_CODE (lhs) == SSA_NAME
5940 416468 : && single_imm_use (lhs, &immuse, &immusestmt)
5941 738374 : && is_gimple_assign (immusestmt))
5942 246381 : return immusestmt;
5943 :
5944 : return NULL;
5945 : }
5946 :
5947 : /* Recursively negate the value of TONEGATE, and return the SSA_NAME
5948 : representing the negated value. Insertions of any necessary
5949 : instructions go before GSI.
5950 : This function is recursive in that, if you hand it "a_5" as the
5951 : value to negate, and a_5 is defined by "a_5 = b_3 + b_4", it will
5952 : transform b_3 + b_4 into a_5 = -b_3 + -b_4. */
5953 :
5954 : static tree
5955 82429 : negate_value (tree tonegate, gimple_stmt_iterator *gsip)
5956 : {
5957 82429 : gimple *negatedefstmt = NULL;
5958 82429 : tree resultofnegate;
5959 82429 : gimple_stmt_iterator gsi;
5960 82429 : unsigned int uid;
5961 :
5962 : /* If we are trying to negate a name, defined by an add, negate the
5963 : add operands instead. */
5964 82429 : if (TREE_CODE (tonegate) == SSA_NAME)
5965 80609 : negatedefstmt = SSA_NAME_DEF_STMT (tonegate);
5966 82429 : if (TREE_CODE (tonegate) == SSA_NAME
5967 80609 : && is_gimple_assign (negatedefstmt)
5968 70639 : && TREE_CODE (gimple_assign_lhs (negatedefstmt)) == SSA_NAME
5969 70639 : && has_single_use (gimple_assign_lhs (negatedefstmt))
5970 136180 : && gimple_assign_rhs_code (negatedefstmt) == PLUS_EXPR)
5971 : {
5972 876 : tree rhs1 = gimple_assign_rhs1 (negatedefstmt);
5973 876 : tree rhs2 = gimple_assign_rhs2 (negatedefstmt);
5974 876 : tree lhs = gimple_assign_lhs (negatedefstmt);
5975 876 : gimple *g;
5976 :
5977 876 : gsi = gsi_for_stmt (negatedefstmt);
5978 876 : rhs1 = negate_value (rhs1, &gsi);
5979 :
5980 876 : gsi = gsi_for_stmt (negatedefstmt);
5981 876 : rhs2 = negate_value (rhs2, &gsi);
5982 :
5983 876 : gsi = gsi_for_stmt (negatedefstmt);
5984 876 : lhs = make_ssa_name (TREE_TYPE (lhs));
5985 876 : gimple_set_visited (negatedefstmt, true);
5986 876 : g = gimple_build_assign (lhs, PLUS_EXPR, rhs1, rhs2);
5987 876 : gimple_set_uid (g, gimple_uid (negatedefstmt));
5988 876 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5989 876 : return lhs;
5990 : }
5991 :
5992 81553 : tonegate = fold_build1 (NEGATE_EXPR, TREE_TYPE (tonegate), tonegate);
5993 81553 : resultofnegate = force_gimple_operand_gsi (gsip, tonegate, true,
5994 : NULL_TREE, true, GSI_SAME_STMT);
5995 81553 : gsi = *gsip;
5996 81553 : uid = gimple_uid (gsi_stmt (gsi));
5997 165816 : for (gsi_prev (&gsi); !gsi_end_p (gsi); gsi_prev (&gsi))
5998 : {
5999 156756 : gimple *stmt = gsi_stmt (gsi);
6000 156756 : if (gimple_uid (stmt) != 0)
6001 : break;
6002 79733 : gimple_set_uid (stmt, uid);
6003 : }
6004 : return resultofnegate;
6005 : }
6006 :
6007 : /* Return true if we should break up the subtract in STMT into an add
6008 : with negate. This is true when we the subtract operands are really
6009 : adds, or the subtract itself is used in an add expression. In
6010 : either case, breaking up the subtract into an add with negate
6011 : exposes the adds to reassociation. */
6012 :
6013 : static bool
6014 305063 : should_break_up_subtract (gimple *stmt)
6015 : {
6016 305063 : tree lhs = gimple_assign_lhs (stmt);
6017 305063 : tree binlhs = gimple_assign_rhs1 (stmt);
6018 305063 : tree binrhs = gimple_assign_rhs2 (stmt);
6019 305063 : gimple *immusestmt;
6020 305063 : class loop *loop = loop_containing_stmt (stmt);
6021 :
6022 305063 : if (TREE_CODE (binlhs) == SSA_NAME
6023 305063 : && is_reassociable_op (SSA_NAME_DEF_STMT (binlhs), PLUS_EXPR, loop))
6024 : return true;
6025 :
6026 287158 : if (TREE_CODE (binrhs) == SSA_NAME
6027 287158 : && is_reassociable_op (SSA_NAME_DEF_STMT (binrhs), PLUS_EXPR, loop))
6028 : return true;
6029 :
6030 286372 : if (TREE_CODE (lhs) == SSA_NAME
6031 286372 : && (immusestmt = get_single_immediate_use (lhs))
6032 137534 : && is_gimple_assign (immusestmt)
6033 423906 : && (gimple_assign_rhs_code (immusestmt) == PLUS_EXPR
6034 93571 : || (gimple_assign_rhs_code (immusestmt) == MINUS_EXPR
6035 2300 : && gimple_assign_rhs1 (immusestmt) == lhs)
6036 91281 : || gimple_assign_rhs_code (immusestmt) == MULT_EXPR))
6037 61986 : return true;
6038 : return false;
6039 : }
6040 :
6041 : /* Transform STMT from A - B into A + -B. */
6042 :
6043 : static void
6044 80677 : break_up_subtract (gimple *stmt, gimple_stmt_iterator *gsip)
6045 : {
6046 80677 : tree rhs1 = gimple_assign_rhs1 (stmt);
6047 80677 : tree rhs2 = gimple_assign_rhs2 (stmt);
6048 :
6049 80677 : if (dump_file && (dump_flags & TDF_DETAILS))
6050 : {
6051 0 : fprintf (dump_file, "Breaking up subtract ");
6052 0 : print_gimple_stmt (dump_file, stmt, 0);
6053 : }
6054 :
6055 80677 : rhs2 = negate_value (rhs2, gsip);
6056 80677 : gimple_assign_set_rhs_with_ops (gsip, PLUS_EXPR, rhs1, rhs2);
6057 80677 : update_stmt (stmt);
6058 80677 : }
6059 :
6060 : /* Determine whether STMT is a builtin call that raises an SSA name
6061 : to an integer power and has only one use. If so, and this is early
6062 : reassociation and unsafe math optimizations are permitted, place
6063 : the SSA name in *BASE and the exponent in *EXPONENT, and return TRUE.
6064 : If any of these conditions does not hold, return FALSE. */
6065 :
6066 : static bool
6067 127 : acceptable_pow_call (gcall *stmt, tree *base, HOST_WIDE_INT *exponent)
6068 : {
6069 127 : tree arg1;
6070 127 : REAL_VALUE_TYPE c, cint;
6071 :
6072 127 : switch (gimple_call_combined_fn (stmt))
6073 : {
6074 19 : CASE_CFN_POW:
6075 19 : if (flag_errno_math)
6076 : return false;
6077 :
6078 19 : *base = gimple_call_arg (stmt, 0);
6079 19 : arg1 = gimple_call_arg (stmt, 1);
6080 :
6081 19 : if (TREE_CODE (arg1) != REAL_CST)
6082 : return false;
6083 :
6084 16 : c = TREE_REAL_CST (arg1);
6085 :
6086 16 : if (REAL_EXP (&c) > HOST_BITS_PER_WIDE_INT)
6087 : return false;
6088 :
6089 16 : *exponent = real_to_integer (&c);
6090 16 : real_from_integer (&cint, VOIDmode, *exponent, SIGNED);
6091 16 : if (!real_identical (&c, &cint))
6092 : return false;
6093 :
6094 : break;
6095 :
6096 10 : CASE_CFN_POWI:
6097 10 : *base = gimple_call_arg (stmt, 0);
6098 10 : arg1 = gimple_call_arg (stmt, 1);
6099 :
6100 10 : if (!tree_fits_shwi_p (arg1))
6101 : return false;
6102 :
6103 10 : *exponent = tree_to_shwi (arg1);
6104 10 : break;
6105 :
6106 : default:
6107 : return false;
6108 : }
6109 :
6110 : /* Expanding negative exponents is generally unproductive, so we don't
6111 : complicate matters with those. Exponents of zero and one should
6112 : have been handled by expression folding. */
6113 18 : if (*exponent < 2 || TREE_CODE (*base) != SSA_NAME)
6114 0 : return false;
6115 :
6116 : return true;
6117 : }
6118 :
6119 : /* Try to derive and add operand entry for OP to *OPS. Return false if
6120 : unsuccessful. */
6121 :
6122 : static bool
6123 9892618 : try_special_add_to_ops (vec<operand_entry *> *ops,
6124 : enum tree_code code,
6125 : tree op, gimple* def_stmt)
6126 : {
6127 9892618 : tree base = NULL_TREE;
6128 9892618 : HOST_WIDE_INT exponent = 0;
6129 :
6130 9892618 : if (TREE_CODE (op) != SSA_NAME
6131 9892618 : || ! has_single_use (op))
6132 : return false;
6133 :
6134 3716397 : if (code == MULT_EXPR
6135 795151 : && reassoc_insert_powi_p
6136 374588 : && flag_unsafe_math_optimizations
6137 30300 : && is_gimple_call (def_stmt)
6138 3716524 : && acceptable_pow_call (as_a <gcall *> (def_stmt), &base, &exponent))
6139 : {
6140 18 : add_repeat_to_ops_vec (ops, base, exponent);
6141 18 : gimple_set_visited (def_stmt, true);
6142 18 : return true;
6143 : }
6144 3716379 : else if (code == MULT_EXPR
6145 795133 : && is_gimple_assign (def_stmt)
6146 753439 : && gimple_assign_rhs_code (def_stmt) == NEGATE_EXPR
6147 575 : && !HONOR_SNANS (TREE_TYPE (op))
6148 575 : && (!HONOR_SIGNED_ZEROS (TREE_TYPE (op))
6149 0 : || !COMPLEX_FLOAT_TYPE_P (TREE_TYPE (op)))
6150 3716954 : && (!FLOAT_TYPE_P (TREE_TYPE (op))
6151 50 : || !DECIMAL_FLOAT_MODE_P (element_mode (op))))
6152 : {
6153 568 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
6154 568 : tree cst = build_minus_one_cst (TREE_TYPE (op));
6155 568 : add_to_ops_vec (ops, rhs1);
6156 568 : add_to_ops_vec (ops, cst);
6157 568 : gimple_set_visited (def_stmt, true);
6158 568 : return true;
6159 : }
6160 :
6161 : return false;
6162 : }
6163 :
6164 : /* Recursively linearize a binary expression that is the RHS of STMT.
6165 : Place the operands of the expression tree in the vector named OPS. */
6166 :
6167 : static void
6168 5075079 : linearize_expr_tree (vec<operand_entry *> *ops, gimple *stmt,
6169 : bool is_associative, bool set_visited)
6170 : {
6171 5075079 : tree binlhs = gimple_assign_rhs1 (stmt);
6172 5075079 : tree binrhs = gimple_assign_rhs2 (stmt);
6173 5075079 : gimple *binlhsdef = NULL, *binrhsdef = NULL;
6174 5075079 : bool binlhsisreassoc = false;
6175 5075079 : bool binrhsisreassoc = false;
6176 5075079 : enum tree_code rhscode = gimple_assign_rhs_code (stmt);
6177 5075079 : class loop *loop = loop_containing_stmt (stmt);
6178 :
6179 5075079 : if (set_visited)
6180 5034375 : gimple_set_visited (stmt, true);
6181 :
6182 5075079 : if (TREE_CODE (binlhs) == SSA_NAME)
6183 : {
6184 5072478 : binlhsdef = SSA_NAME_DEF_STMT (binlhs);
6185 5072478 : binlhsisreassoc = (is_reassociable_op (binlhsdef, rhscode, loop)
6186 5072478 : && !stmt_could_throw_p (cfun, binlhsdef));
6187 : }
6188 :
6189 5075079 : if (TREE_CODE (binrhs) == SSA_NAME)
6190 : {
6191 1596570 : binrhsdef = SSA_NAME_DEF_STMT (binrhs);
6192 1596570 : binrhsisreassoc = (is_reassociable_op (binrhsdef, rhscode, loop)
6193 1596570 : && !stmt_could_throw_p (cfun, binrhsdef));
6194 : }
6195 :
6196 : /* If the LHS is not reassociable, but the RHS is, we need to swap
6197 : them. If neither is reassociable, there is nothing we can do, so
6198 : just put them in the ops vector. If the LHS is reassociable,
6199 : linearize it. If both are reassociable, then linearize the RHS
6200 : and the LHS. */
6201 :
6202 5075079 : if (!binlhsisreassoc)
6203 : {
6204 : /* If this is not a associative operation like division, give up. */
6205 4881954 : if (!is_associative)
6206 : {
6207 15 : add_to_ops_vec (ops, binrhs);
6208 15 : return;
6209 : }
6210 :
6211 4881939 : if (!binrhsisreassoc)
6212 : {
6213 4817554 : bool swap = false;
6214 4817554 : if (try_special_add_to_ops (ops, rhscode, binrhs, binrhsdef))
6215 : /* If we add ops for the rhs we expect to be able to recurse
6216 : to it via the lhs during expression rewrite so swap
6217 : operands. */
6218 : swap = true;
6219 : else
6220 4817172 : add_to_ops_vec (ops, binrhs);
6221 :
6222 4817554 : if (!try_special_add_to_ops (ops, rhscode, binlhs, binlhsdef))
6223 4817354 : add_to_ops_vec (ops, binlhs);
6224 :
6225 4817554 : if (!swap)
6226 : return;
6227 : }
6228 :
6229 64767 : if (dump_file && (dump_flags & TDF_DETAILS))
6230 : {
6231 9 : fprintf (dump_file, "swapping operands of ");
6232 9 : print_gimple_stmt (dump_file, stmt, 0);
6233 : }
6234 :
6235 64767 : swap_ssa_operands (stmt,
6236 : gimple_assign_rhs1_ptr (stmt),
6237 : gimple_assign_rhs2_ptr (stmt));
6238 64767 : update_stmt (stmt);
6239 :
6240 64767 : if (dump_file && (dump_flags & TDF_DETAILS))
6241 : {
6242 9 : fprintf (dump_file, " is now ");
6243 9 : print_gimple_stmt (dump_file, stmt, 0);
6244 : }
6245 64767 : if (!binrhsisreassoc)
6246 : return;
6247 :
6248 : /* We want to make it so the lhs is always the reassociative op,
6249 : so swap. */
6250 : std::swap (binlhs, binrhs);
6251 : }
6252 193125 : else if (binrhsisreassoc)
6253 : {
6254 1992 : linearize_expr (stmt);
6255 1992 : binlhs = gimple_assign_rhs1 (stmt);
6256 1992 : binrhs = gimple_assign_rhs2 (stmt);
6257 : }
6258 :
6259 257510 : gcc_assert (TREE_CODE (binrhs) != SSA_NAME
6260 : || !is_reassociable_op (SSA_NAME_DEF_STMT (binrhs),
6261 : rhscode, loop));
6262 257510 : linearize_expr_tree (ops, SSA_NAME_DEF_STMT (binlhs),
6263 : is_associative, set_visited);
6264 :
6265 257510 : if (!try_special_add_to_ops (ops, rhscode, binrhs, binrhsdef))
6266 257506 : add_to_ops_vec (ops, binrhs);
6267 : }
6268 :
6269 : /* Repropagate the negates back into subtracts, since no other pass
6270 : currently does it. */
6271 :
6272 : static void
6273 2136094 : repropagate_negates (void)
6274 : {
6275 2136094 : unsigned int i = 0;
6276 2136094 : tree negate;
6277 :
6278 2266190 : FOR_EACH_VEC_ELT (plus_negates, i, negate)
6279 : {
6280 130096 : gimple *user = get_single_immediate_use (negate);
6281 130096 : if (!user || !is_gimple_assign (user))
6282 21249 : continue;
6283 :
6284 108847 : tree negateop = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (negate));
6285 108857 : if (TREE_CODE (negateop) == SSA_NAME
6286 108847 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (negateop))
6287 10 : continue;
6288 :
6289 : /* The negate operand can be either operand of a PLUS_EXPR
6290 : (it can be the LHS if the RHS is a constant for example).
6291 :
6292 : Force the negate operand to the RHS of the PLUS_EXPR, then
6293 : transform the PLUS_EXPR into a MINUS_EXPR. */
6294 108837 : if (gimple_assign_rhs_code (user) == PLUS_EXPR)
6295 : {
6296 : /* If the negated operand appears on the LHS of the
6297 : PLUS_EXPR, exchange the operands of the PLUS_EXPR
6298 : to force the negated operand to the RHS of the PLUS_EXPR. */
6299 88650 : if (gimple_assign_rhs1 (user) == negate)
6300 : {
6301 26504 : swap_ssa_operands (user,
6302 : gimple_assign_rhs1_ptr (user),
6303 : gimple_assign_rhs2_ptr (user));
6304 : }
6305 :
6306 : /* Now transform the PLUS_EXPR into a MINUS_EXPR and replace
6307 : the RHS of the PLUS_EXPR with the operand of the NEGATE_EXPR. */
6308 88650 : if (gimple_assign_rhs2 (user) == negate)
6309 : {
6310 88650 : tree rhs1 = gimple_assign_rhs1 (user);
6311 88650 : gimple_stmt_iterator gsi = gsi_for_stmt (user);
6312 88650 : gimple_assign_set_rhs_with_ops (&gsi, MINUS_EXPR, rhs1,
6313 : negateop);
6314 88650 : update_stmt (user);
6315 : }
6316 : }
6317 20187 : else if (gimple_assign_rhs_code (user) == MINUS_EXPR)
6318 : {
6319 2076 : if (gimple_assign_rhs1 (user) == negate)
6320 : {
6321 : /* We have
6322 : x = -negateop
6323 : y = x - b
6324 : which we transform into
6325 : x = negateop + b
6326 : y = -x .
6327 : This pushes down the negate which we possibly can merge
6328 : into some other operation, hence insert it into the
6329 : plus_negates vector. */
6330 2076 : gimple *feed = SSA_NAME_DEF_STMT (negate);
6331 2076 : tree b = gimple_assign_rhs2 (user);
6332 2076 : gimple_stmt_iterator gsi = gsi_for_stmt (feed);
6333 2076 : gimple_stmt_iterator gsi2 = gsi_for_stmt (user);
6334 2076 : tree x = make_ssa_name (TREE_TYPE (gimple_assign_lhs (feed)));
6335 2076 : gimple *g = gimple_build_assign (x, PLUS_EXPR, negateop, b);
6336 2076 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
6337 2076 : gimple_assign_set_rhs_with_ops (&gsi2, NEGATE_EXPR, x);
6338 2076 : user = gsi_stmt (gsi2);
6339 2076 : update_stmt (user);
6340 2076 : reassoc_remove_stmt (&gsi);
6341 2076 : release_defs (feed);
6342 2076 : plus_negates.safe_push (gimple_assign_lhs (user));
6343 : }
6344 : else
6345 : {
6346 : /* Transform "x = -negateop; y = b - x" into "y = b + negateop",
6347 : getting rid of one operation. */
6348 0 : tree rhs1 = gimple_assign_rhs1 (user);
6349 0 : gimple_stmt_iterator gsi = gsi_for_stmt (user);
6350 0 : gimple_assign_set_rhs_with_ops (&gsi, PLUS_EXPR, rhs1, negateop);
6351 0 : update_stmt (gsi_stmt (gsi));
6352 : }
6353 : }
6354 : }
6355 2136094 : }
6356 :
6357 : /* Break up subtract operations in block BB.
6358 :
6359 : We do this top down because we don't know whether the subtract is
6360 : part of a possible chain of reassociation except at the top.
6361 :
6362 : IE given
6363 : d = f + g
6364 : c = a + e
6365 : b = c - d
6366 : q = b - r
6367 : k = t - q
6368 :
6369 : we want to break up k = t - q, but we won't until we've transformed q
6370 : = b - r, which won't be broken up until we transform b = c - d.
6371 :
6372 : En passant, clear the GIMPLE visited flag on every statement
6373 : and set UIDs within each basic block. */
6374 :
6375 : static void
6376 20222348 : break_up_subtract_bb (basic_block bb)
6377 : {
6378 20222348 : gimple_stmt_iterator gsi;
6379 20222348 : unsigned int uid = 1;
6380 :
6381 225788889 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
6382 : {
6383 185344193 : gimple *stmt = gsi_stmt (gsi);
6384 185344193 : gimple_set_visited (stmt, false);
6385 185344193 : gimple_set_uid (stmt, uid++);
6386 :
6387 185344193 : if (!is_gimple_assign (stmt)
6388 48689199 : || !can_reassociate_type_p (TREE_TYPE (gimple_assign_lhs (stmt)))
6389 203128066 : || !can_reassociate_op_p (gimple_assign_lhs (stmt)))
6390 167560638 : continue;
6391 :
6392 : /* Look for simple gimple subtract operations. */
6393 17783555 : if (gimple_assign_rhs_code (stmt) == MINUS_EXPR)
6394 : {
6395 305328 : if (!can_reassociate_op_p (gimple_assign_rhs1 (stmt))
6396 305328 : || !can_reassociate_op_p (gimple_assign_rhs2 (stmt)))
6397 265 : continue;
6398 :
6399 : /* Check for a subtract used only in an addition. If this
6400 : is the case, transform it into add of a negate for better
6401 : reassociation. IE transform C = A-B into C = A + -B if C
6402 : is only used in an addition. */
6403 305063 : if (should_break_up_subtract (stmt))
6404 80677 : break_up_subtract (stmt, &gsi);
6405 : }
6406 17478227 : else if (gimple_assign_rhs_code (stmt) == NEGATE_EXPR
6407 17478227 : && can_reassociate_op_p (gimple_assign_rhs1 (stmt)))
6408 37555 : plus_negates.safe_push (gimple_assign_lhs (stmt));
6409 : }
6410 20222348 : }
6411 :
6412 : /* Used for repeated factor analysis. */
6413 : struct repeat_factor
6414 : {
6415 : /* An SSA name that occurs in a multiply chain. */
6416 : tree factor;
6417 :
6418 : /* Cached rank of the factor. */
6419 : unsigned rank;
6420 :
6421 : /* Number of occurrences of the factor in the chain. */
6422 : HOST_WIDE_INT count;
6423 :
6424 : /* An SSA name representing the product of this factor and
6425 : all factors appearing later in the repeated factor vector. */
6426 : tree repr;
6427 : };
6428 :
6429 :
6430 : static vec<repeat_factor> repeat_factor_vec;
6431 :
6432 : /* Used for sorting the repeat factor vector. Sort primarily by
6433 : ascending occurrence count, secondarily by descending rank. */
6434 :
6435 : static int
6436 247714 : compare_repeat_factors (const void *x1, const void *x2)
6437 : {
6438 247714 : const repeat_factor *rf1 = (const repeat_factor *) x1;
6439 247714 : const repeat_factor *rf2 = (const repeat_factor *) x2;
6440 :
6441 247714 : if (rf1->count < rf2->count)
6442 : return -1;
6443 247080 : else if (rf1->count > rf2->count)
6444 : return 1;
6445 :
6446 246587 : if (rf1->rank < rf2->rank)
6447 : return 1;
6448 133939 : else if (rf1->rank > rf2->rank)
6449 111499 : return -1;
6450 :
6451 : return 0;
6452 : }
6453 :
6454 : /* Look for repeated operands in OPS in the multiply tree rooted at
6455 : STMT. Replace them with an optimal sequence of multiplies and powi
6456 : builtin calls, and remove the used operands from OPS. Return an
6457 : SSA name representing the value of the replacement sequence. */
6458 :
6459 : static tree
6460 502506 : attempt_builtin_powi (gimple *stmt, vec<operand_entry *> *ops)
6461 : {
6462 502506 : unsigned i, j, vec_len;
6463 502506 : int ii;
6464 502506 : operand_entry *oe;
6465 502506 : repeat_factor *rf1, *rf2;
6466 502506 : repeat_factor rfnew;
6467 502506 : tree result = NULL_TREE;
6468 502506 : tree target_ssa, iter_result;
6469 502506 : tree type = TREE_TYPE (gimple_get_lhs (stmt));
6470 502506 : tree powi_fndecl = mathfn_built_in (type, BUILT_IN_POWI);
6471 502506 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6472 502506 : gimple *mul_stmt, *pow_stmt;
6473 :
6474 : /* Nothing to do if BUILT_IN_POWI doesn't exist for this type and
6475 : target, unless type is integral. */
6476 502506 : if (!powi_fndecl && !INTEGRAL_TYPE_P (type))
6477 : return NULL_TREE;
6478 :
6479 : /* Allocate the repeated factor vector. */
6480 498850 : repeat_factor_vec.create (10);
6481 :
6482 : /* Scan the OPS vector for all SSA names in the product and build
6483 : up a vector of occurrence counts for each factor. */
6484 1999739 : FOR_EACH_VEC_ELT (*ops, i, oe)
6485 : {
6486 1002039 : if (TREE_CODE (oe->op) == SSA_NAME)
6487 : {
6488 628429 : FOR_EACH_VEC_ELT (repeat_factor_vec, j, rf1)
6489 : {
6490 67258 : if (rf1->factor == oe->op)
6491 : {
6492 2824 : rf1->count += oe->count;
6493 2824 : break;
6494 : }
6495 : }
6496 :
6497 563995 : if (j >= repeat_factor_vec.length ())
6498 : {
6499 561171 : rfnew.factor = oe->op;
6500 561171 : rfnew.rank = oe->rank;
6501 561171 : rfnew.count = oe->count;
6502 561171 : rfnew.repr = NULL_TREE;
6503 561171 : repeat_factor_vec.safe_push (rfnew);
6504 : }
6505 : }
6506 : }
6507 :
6508 : /* Sort the repeated factor vector by (a) increasing occurrence count,
6509 : and (b) decreasing rank. */
6510 498850 : repeat_factor_vec.qsort (compare_repeat_factors);
6511 :
6512 : /* It is generally best to combine as many base factors as possible
6513 : into a product before applying __builtin_powi to the result.
6514 : However, the sort order chosen for the repeated factor vector
6515 : allows us to cache partial results for the product of the base
6516 : factors for subsequent use. When we already have a cached partial
6517 : result from a previous iteration, it is best to make use of it
6518 : before looking for another __builtin_pow opportunity.
6519 :
6520 : As an example, consider x * x * y * y * y * z * z * z * z.
6521 : We want to first compose the product x * y * z, raise it to the
6522 : second power, then multiply this by y * z, and finally multiply
6523 : by z. This can be done in 5 multiplies provided we cache y * z
6524 : for use in both expressions:
6525 :
6526 : t1 = y * z
6527 : t2 = t1 * x
6528 : t3 = t2 * t2
6529 : t4 = t1 * t3
6530 : result = t4 * z
6531 :
6532 : If we instead ignored the cached y * z and first multiplied by
6533 : the __builtin_pow opportunity z * z, we would get the inferior:
6534 :
6535 : t1 = y * z
6536 : t2 = t1 * x
6537 : t3 = t2 * t2
6538 : t4 = z * z
6539 : t5 = t3 * t4
6540 : result = t5 * y */
6541 :
6542 997700 : vec_len = repeat_factor_vec.length ();
6543 :
6544 : /* Repeatedly look for opportunities to create a builtin_powi call. */
6545 500646 : while (true)
6546 : {
6547 500646 : HOST_WIDE_INT power;
6548 :
6549 : /* First look for the largest cached product of factors from
6550 : preceding iterations. If found, create a builtin_powi for
6551 : it if the minimum occurrence count for its factors is at
6552 : least 2, or just use this cached product as our next
6553 : multiplicand if the minimum occurrence count is 1. */
6554 1063914 : FOR_EACH_VEC_ELT (repeat_factor_vec, j, rf1)
6555 : {
6556 563277 : if (rf1->repr && rf1->count > 0)
6557 : break;
6558 : }
6559 :
6560 500646 : if (j < vec_len)
6561 : {
6562 9 : power = rf1->count;
6563 :
6564 9 : if (power == 1)
6565 : {
6566 7 : iter_result = rf1->repr;
6567 :
6568 7 : if (dump_file && (dump_flags & TDF_DETAILS))
6569 : {
6570 0 : unsigned elt;
6571 0 : repeat_factor *rf;
6572 0 : fputs ("Multiplying by cached product ", dump_file);
6573 0 : for (elt = j; elt < vec_len; elt++)
6574 : {
6575 0 : rf = &repeat_factor_vec[elt];
6576 0 : print_generic_expr (dump_file, rf->factor);
6577 0 : if (elt < vec_len - 1)
6578 0 : fputs (" * ", dump_file);
6579 : }
6580 0 : fputs ("\n", dump_file);
6581 : }
6582 : }
6583 : else
6584 : {
6585 2 : if (INTEGRAL_TYPE_P (type))
6586 : {
6587 0 : gcc_assert (power > 1);
6588 0 : gimple_stmt_iterator gsip = gsi;
6589 0 : gsi_prev (&gsip);
6590 0 : iter_result = powi_as_mults (&gsi, gimple_location (stmt),
6591 : rf1->repr, power);
6592 0 : gimple_stmt_iterator gsic = gsi;
6593 0 : while (gsi_stmt (gsic) != gsi_stmt (gsip))
6594 : {
6595 0 : gimple_set_uid (gsi_stmt (gsic), gimple_uid (stmt));
6596 0 : gimple_set_visited (gsi_stmt (gsic), true);
6597 0 : gsi_prev (&gsic);
6598 : }
6599 : }
6600 : else
6601 : {
6602 2 : iter_result = make_temp_ssa_name (type, NULL, "reassocpow");
6603 2 : pow_stmt
6604 2 : = gimple_build_call (powi_fndecl, 2, rf1->repr,
6605 : build_int_cst (integer_type_node,
6606 2 : power));
6607 2 : gimple_call_set_lhs (pow_stmt, iter_result);
6608 2 : gimple_set_location (pow_stmt, gimple_location (stmt));
6609 2 : gimple_set_uid (pow_stmt, gimple_uid (stmt));
6610 2 : gsi_insert_before (&gsi, pow_stmt, GSI_SAME_STMT);
6611 : }
6612 :
6613 2 : if (dump_file && (dump_flags & TDF_DETAILS))
6614 : {
6615 0 : unsigned elt;
6616 0 : repeat_factor *rf;
6617 0 : fputs ("Building __builtin_pow call for cached product (",
6618 : dump_file);
6619 0 : for (elt = j; elt < vec_len; elt++)
6620 : {
6621 0 : rf = &repeat_factor_vec[elt];
6622 0 : print_generic_expr (dump_file, rf->factor);
6623 0 : if (elt < vec_len - 1)
6624 0 : fputs (" * ", dump_file);
6625 : }
6626 0 : fprintf (dump_file, ")^" HOST_WIDE_INT_PRINT_DEC"\n",
6627 : power);
6628 : }
6629 : }
6630 : }
6631 : else
6632 : {
6633 : /* Otherwise, find the first factor in the repeated factor
6634 : vector whose occurrence count is at least 2. If no such
6635 : factor exists, there are no builtin_powi opportunities
6636 : remaining. */
6637 1062074 : FOR_EACH_VEC_ELT (repeat_factor_vec, j, rf1)
6638 : {
6639 563224 : if (rf1->count >= 2)
6640 : break;
6641 : }
6642 :
6643 500637 : if (j >= vec_len)
6644 : break;
6645 :
6646 1787 : power = rf1->count;
6647 :
6648 1787 : if (dump_file && (dump_flags & TDF_DETAILS))
6649 : {
6650 0 : unsigned elt;
6651 0 : repeat_factor *rf;
6652 0 : fputs ("Building __builtin_pow call for (", dump_file);
6653 0 : for (elt = j; elt < vec_len; elt++)
6654 : {
6655 0 : rf = &repeat_factor_vec[elt];
6656 0 : print_generic_expr (dump_file, rf->factor);
6657 0 : if (elt < vec_len - 1)
6658 0 : fputs (" * ", dump_file);
6659 : }
6660 0 : fprintf (dump_file, ")^" HOST_WIDE_INT_PRINT_DEC"\n", power);
6661 : }
6662 :
6663 1787 : reassociate_stats.pows_created++;
6664 :
6665 : /* Visit each element of the vector in reverse order (so that
6666 : high-occurrence elements are visited first, and within the
6667 : same occurrence count, lower-ranked elements are visited
6668 : first). Form a linear product of all elements in this order
6669 : whose occurrencce count is at least that of element J.
6670 : Record the SSA name representing the product of each element
6671 : with all subsequent elements in the vector. */
6672 1787 : if (j == vec_len - 1)
6673 1766 : rf1->repr = rf1->factor;
6674 : else
6675 : {
6676 50 : for (ii = vec_len - 2; ii >= (int)j; ii--)
6677 : {
6678 29 : tree op1, op2;
6679 :
6680 29 : rf1 = &repeat_factor_vec[ii];
6681 29 : rf2 = &repeat_factor_vec[ii + 1];
6682 :
6683 : /* Init the last factor's representative to be itself. */
6684 29 : if (!rf2->repr)
6685 21 : rf2->repr = rf2->factor;
6686 :
6687 29 : op1 = rf1->factor;
6688 29 : op2 = rf2->repr;
6689 :
6690 29 : target_ssa = make_temp_ssa_name (type, NULL, "reassocpow");
6691 29 : mul_stmt = gimple_build_assign (target_ssa, MULT_EXPR,
6692 : op1, op2);
6693 29 : gimple_set_location (mul_stmt, gimple_location (stmt));
6694 29 : gimple_set_uid (mul_stmt, gimple_uid (stmt));
6695 29 : gsi_insert_before (&gsi, mul_stmt, GSI_SAME_STMT);
6696 29 : rf1->repr = target_ssa;
6697 :
6698 : /* Don't reprocess the multiply we just introduced. */
6699 29 : gimple_set_visited (mul_stmt, true);
6700 : }
6701 : }
6702 :
6703 : /* Form a call to __builtin_powi for the maximum product
6704 : just formed, raised to the power obtained earlier. */
6705 1787 : rf1 = &repeat_factor_vec[j];
6706 1787 : if (INTEGRAL_TYPE_P (type))
6707 : {
6708 1222 : gcc_assert (power > 1);
6709 1222 : gimple_stmt_iterator gsip = gsi;
6710 1222 : gsi_prev (&gsip);
6711 1222 : iter_result = powi_as_mults (&gsi, gimple_location (stmt),
6712 : rf1->repr, power);
6713 1222 : gimple_stmt_iterator gsic = gsi;
6714 3701 : while (gsi_stmt (gsic) != gsi_stmt (gsip))
6715 : {
6716 2479 : gimple_set_uid (gsi_stmt (gsic), gimple_uid (stmt));
6717 2479 : gimple_set_visited (gsi_stmt (gsic), true);
6718 2535 : gsi_prev (&gsic);
6719 : }
6720 : }
6721 : else
6722 : {
6723 565 : iter_result = make_temp_ssa_name (type, NULL, "reassocpow");
6724 565 : pow_stmt = gimple_build_call (powi_fndecl, 2, rf1->repr,
6725 : build_int_cst (integer_type_node,
6726 565 : power));
6727 565 : gimple_call_set_lhs (pow_stmt, iter_result);
6728 565 : gimple_set_location (pow_stmt, gimple_location (stmt));
6729 565 : gimple_set_uid (pow_stmt, gimple_uid (stmt));
6730 565 : gsi_insert_before (&gsi, pow_stmt, GSI_SAME_STMT);
6731 : }
6732 : }
6733 :
6734 : /* If we previously formed at least one other builtin_powi call,
6735 : form the product of this one and those others. */
6736 1796 : if (result)
6737 : {
6738 9 : tree new_result = make_temp_ssa_name (type, NULL, "reassocpow");
6739 9 : mul_stmt = gimple_build_assign (new_result, MULT_EXPR,
6740 : result, iter_result);
6741 9 : gimple_set_location (mul_stmt, gimple_location (stmt));
6742 9 : gimple_set_uid (mul_stmt, gimple_uid (stmt));
6743 9 : gsi_insert_before (&gsi, mul_stmt, GSI_SAME_STMT);
6744 9 : gimple_set_visited (mul_stmt, true);
6745 9 : result = new_result;
6746 : }
6747 : else
6748 : result = iter_result;
6749 :
6750 : /* Decrement the occurrence count of each element in the product
6751 : by the count found above, and remove this many copies of each
6752 : factor from OPS. */
6753 3626 : for (i = j; i < vec_len; i++)
6754 : {
6755 1830 : unsigned k = power;
6756 1830 : unsigned n;
6757 :
6758 1830 : rf1 = &repeat_factor_vec[i];
6759 1830 : rf1->count -= power;
6760 :
6761 8721 : FOR_EACH_VEC_ELT_REVERSE (*ops, n, oe)
6762 : {
6763 5061 : if (oe->op == rf1->factor)
6764 : {
6765 4646 : if (oe->count <= k)
6766 : {
6767 4640 : ops->ordered_remove (n);
6768 4640 : k -= oe->count;
6769 :
6770 4640 : if (k == 0)
6771 : break;
6772 : }
6773 : else
6774 : {
6775 6 : oe->count -= k;
6776 6 : break;
6777 : }
6778 : }
6779 : }
6780 : }
6781 : }
6782 :
6783 : /* At this point all elements in the repeated factor vector have a
6784 : remaining occurrence count of 0 or 1, and those with a count of 1
6785 : don't have cached representatives. Re-sort the ops vector and
6786 : clean up. */
6787 498850 : ops->qsort (sort_by_operand_rank);
6788 498850 : repeat_factor_vec.release ();
6789 :
6790 : /* Return the final product computed herein. Note that there may
6791 : still be some elements with single occurrence count left in OPS;
6792 : those will be handled by the normal reassociation logic. */
6793 498850 : return result;
6794 : }
6795 :
6796 : /* Attempt to optimize
6797 : CST1 * copysign (CST2, y) -> copysign (CST1 * CST2, y) if CST1 > 0, or
6798 : CST1 * copysign (CST2, y) -> -copysign (CST1 * CST2, y) if CST1 < 0. */
6799 :
6800 : static void
6801 1103182 : attempt_builtin_copysign (vec<operand_entry *> *ops)
6802 : {
6803 1103182 : operand_entry *oe;
6804 1103182 : unsigned int i;
6805 1103182 : unsigned int length = ops->length ();
6806 1103182 : tree cst = ops->last ()->op;
6807 :
6808 1103182 : if (length == 1 || TREE_CODE (cst) != REAL_CST)
6809 : return;
6810 :
6811 4140 : FOR_EACH_VEC_ELT (*ops, i, oe)
6812 : {
6813 2953 : if (TREE_CODE (oe->op) == SSA_NAME
6814 2953 : && has_single_use (oe->op))
6815 : {
6816 865 : gimple *def_stmt = SSA_NAME_DEF_STMT (oe->op);
6817 2991 : if (gcall *old_call = dyn_cast <gcall *> (def_stmt))
6818 : {
6819 54 : tree arg0, arg1;
6820 54 : switch (gimple_call_combined_fn (old_call))
6821 : {
6822 20 : CASE_CFN_COPYSIGN:
6823 20 : CASE_CFN_COPYSIGN_FN:
6824 20 : arg0 = gimple_call_arg (old_call, 0);
6825 20 : arg1 = gimple_call_arg (old_call, 1);
6826 : /* The first argument of copysign must be a constant,
6827 : otherwise there's nothing to do. */
6828 20 : if (TREE_CODE (arg0) == REAL_CST)
6829 : {
6830 20 : tree type = TREE_TYPE (arg0);
6831 20 : tree mul = const_binop (MULT_EXPR, type, cst, arg0);
6832 : /* If we couldn't fold to a single constant, skip it.
6833 : That happens e.g. for inexact multiplication when
6834 : -frounding-math. */
6835 20 : if (mul == NULL_TREE)
6836 : break;
6837 : /* Instead of adjusting OLD_CALL, let's build a new
6838 : call to not leak the LHS and prevent keeping bogus
6839 : debug statements. DCE will clean up the old call. */
6840 16 : gcall *new_call;
6841 16 : if (gimple_call_internal_p (old_call))
6842 0 : new_call = gimple_build_call_internal
6843 0 : (IFN_COPYSIGN, 2, mul, arg1);
6844 : else
6845 16 : new_call = gimple_build_call
6846 16 : (gimple_call_fndecl (old_call), 2, mul, arg1);
6847 16 : tree lhs = make_ssa_name (type);
6848 16 : gimple_call_set_lhs (new_call, lhs);
6849 16 : gimple_set_location (new_call,
6850 : gimple_location (old_call));
6851 16 : insert_stmt_after (new_call, old_call);
6852 : /* We've used the constant, get rid of it. */
6853 16 : ops->pop ();
6854 16 : bool cst1_neg = real_isneg (TREE_REAL_CST_PTR (cst));
6855 : /* Handle the CST1 < 0 case by negating the result. */
6856 16 : if (cst1_neg)
6857 : {
6858 7 : tree negrhs = make_ssa_name (TREE_TYPE (lhs));
6859 7 : gimple *negate_stmt
6860 7 : = gimple_build_assign (negrhs, NEGATE_EXPR, lhs);
6861 7 : insert_stmt_after (negate_stmt, new_call);
6862 7 : oe->op = negrhs;
6863 : }
6864 : else
6865 : oe->op = lhs;
6866 16 : if (dump_file && (dump_flags & TDF_DETAILS))
6867 : {
6868 14 : fprintf (dump_file, "Optimizing copysign: ");
6869 14 : print_generic_expr (dump_file, cst);
6870 14 : fprintf (dump_file, " * COPYSIGN (");
6871 14 : print_generic_expr (dump_file, arg0);
6872 14 : fprintf (dump_file, ", ");
6873 14 : print_generic_expr (dump_file, arg1);
6874 23 : fprintf (dump_file, ") into %sCOPYSIGN (",
6875 : cst1_neg ? "-" : "");
6876 14 : print_generic_expr (dump_file, mul);
6877 14 : fprintf (dump_file, ", ");
6878 14 : print_generic_expr (dump_file, arg1);
6879 14 : fprintf (dump_file, "\n");
6880 : }
6881 : return;
6882 : }
6883 : break;
6884 : default:
6885 : break;
6886 : }
6887 : }
6888 : }
6889 : }
6890 : }
6891 :
6892 : /* Transform STMT at *GSI into a copy by replacing its rhs with NEW_RHS. */
6893 :
6894 : static void
6895 14551 : transform_stmt_to_copy (gimple_stmt_iterator *gsi, gimple *stmt, tree new_rhs)
6896 : {
6897 14551 : tree rhs1;
6898 :
6899 14551 : if (dump_file && (dump_flags & TDF_DETAILS))
6900 : {
6901 28 : fprintf (dump_file, "Transforming ");
6902 28 : print_gimple_stmt (dump_file, stmt, 0);
6903 : }
6904 :
6905 14551 : rhs1 = gimple_assign_rhs1 (stmt);
6906 14551 : gimple_assign_set_rhs_from_tree (gsi, new_rhs);
6907 14551 : update_stmt (stmt);
6908 14551 : remove_visited_stmt_chain (rhs1);
6909 :
6910 14551 : if (dump_file && (dump_flags & TDF_DETAILS))
6911 : {
6912 28 : fprintf (dump_file, " into ");
6913 28 : print_gimple_stmt (dump_file, stmt, 0);
6914 : }
6915 14551 : }
6916 :
6917 : /* Transform STMT at *GSI into a multiply of RHS1 and RHS2. */
6918 :
6919 : static void
6920 194 : transform_stmt_to_multiply (gimple_stmt_iterator *gsi, gimple *stmt,
6921 : tree rhs1, tree rhs2)
6922 : {
6923 194 : if (dump_file && (dump_flags & TDF_DETAILS))
6924 : {
6925 0 : fprintf (dump_file, "Transforming ");
6926 0 : print_gimple_stmt (dump_file, stmt, 0);
6927 : }
6928 :
6929 194 : gimple_assign_set_rhs_with_ops (gsi, MULT_EXPR, rhs1, rhs2);
6930 194 : update_stmt (gsi_stmt (*gsi));
6931 194 : remove_visited_stmt_chain (rhs1);
6932 :
6933 194 : if (dump_file && (dump_flags & TDF_DETAILS))
6934 : {
6935 0 : fprintf (dump_file, " into ");
6936 0 : print_gimple_stmt (dump_file, stmt, 0);
6937 : }
6938 194 : }
6939 :
6940 : /* Rearrange ops may have more FMA when the chain may has more than 2 FMAs.
6941 : Put no-mult ops and mult ops alternately at the end of the queue, which is
6942 : conducive to generating more FMA and reducing the loss of FMA when breaking
6943 : the chain.
6944 : E.g.
6945 : a * b + c * d + e generates:
6946 :
6947 : _4 = c_9(D) * d_10(D);
6948 : _12 = .FMA (a_7(D), b_8(D), _4);
6949 : _11 = e_6(D) + _12;
6950 :
6951 : Rearrange ops to -> e + a * b + c * d generates:
6952 :
6953 : _4 = .FMA (c_7(D), d_8(D), _3);
6954 : _11 = .FMA (a_5(D), b_6(D), _4);
6955 :
6956 : Return the number of MULT_EXPRs in the chain. */
6957 : static int
6958 16824 : rank_ops_for_fma (vec<operand_entry *> *ops)
6959 : {
6960 16824 : operand_entry *oe;
6961 16824 : unsigned int i;
6962 16824 : unsigned int ops_length = ops->length ();
6963 16824 : auto_vec<operand_entry *> ops_mult;
6964 16824 : auto_vec<operand_entry *> ops_others;
6965 :
6966 72998 : FOR_EACH_VEC_ELT (*ops, i, oe)
6967 : {
6968 39350 : if (TREE_CODE (oe->op) == SSA_NAME)
6969 : {
6970 39334 : gimple *def_stmt = SSA_NAME_DEF_STMT (oe->op);
6971 39334 : if (is_gimple_assign (def_stmt))
6972 : {
6973 28093 : if (gimple_assign_rhs_code (def_stmt) == MULT_EXPR)
6974 13046 : ops_mult.safe_push (oe);
6975 : /* A negate on the multiplication leads to FNMA. */
6976 15047 : else if (gimple_assign_rhs_code (def_stmt) == NEGATE_EXPR
6977 15047 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
6978 : {
6979 2839 : gimple *neg_def_stmt
6980 2839 : = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (def_stmt));
6981 2839 : if (is_gimple_assign (neg_def_stmt)
6982 2836 : && gimple_bb (neg_def_stmt) == gimple_bb (def_stmt)
6983 5670 : && gimple_assign_rhs_code (neg_def_stmt) == MULT_EXPR)
6984 2792 : ops_mult.safe_push (oe);
6985 : else
6986 47 : ops_others.safe_push (oe);
6987 : }
6988 : else
6989 12208 : ops_others.safe_push (oe);
6990 : }
6991 : else
6992 11241 : ops_others.safe_push (oe);
6993 : }
6994 : else
6995 16 : ops_others.safe_push (oe);
6996 : }
6997 : /* 1. When ops_mult.length == 2, like the following case,
6998 :
6999 : a * b + c * d + e.
7000 :
7001 : we need to rearrange the ops.
7002 :
7003 : Putting ops that not def from mult in front can generate more FMAs.
7004 :
7005 : 2. If all ops are defined with mult, we don't need to rearrange them. */
7006 16824 : unsigned mult_num = ops_mult.length ();
7007 16824 : if (mult_num >= 2 && mult_num != ops_length)
7008 : {
7009 : /* Put no-mult ops and mult ops alternately at the end of the
7010 : queue, which is conducive to generating more FMA and reducing the
7011 : loss of FMA when breaking the chain. */
7012 5292 : ops->truncate (0);
7013 5292 : ops->splice (ops_mult);
7014 5292 : int j, opindex = ops->length ();
7015 5292 : int others_length = ops_others.length ();
7016 10589 : for (j = 0; j < others_length; j++)
7017 : {
7018 5297 : oe = ops_others.pop ();
7019 5297 : ops->quick_insert (opindex, oe);
7020 5297 : if (opindex > 0)
7021 5296 : opindex--;
7022 : }
7023 : }
7024 16824 : return mult_num;
7025 16824 : }
7026 : /* Reassociate expressions in basic block BB and its post-dominator as
7027 : children.
7028 :
7029 : Bubble up return status from maybe_optimize_range_tests. */
7030 :
7031 : static bool
7032 20222299 : reassociate_bb (basic_block bb)
7033 : {
7034 20222299 : gimple_stmt_iterator gsi;
7035 20222299 : gimple *stmt = last_nondebug_stmt (bb);
7036 20222299 : bool cfg_cleanup_needed = false;
7037 :
7038 20222299 : if (stmt && !gimple_visited_p (stmt))
7039 19589933 : cfg_cleanup_needed |= maybe_optimize_range_tests (stmt);
7040 :
7041 20222299 : bool do_prev = false;
7042 40444598 : for (gsi = gsi_last_bb (bb);
7043 391453183 : !gsi_end_p (gsi); do_prev ? gsi_prev (&gsi) : (void) 0)
7044 : {
7045 185615695 : do_prev = true;
7046 185615695 : stmt = gsi_stmt (gsi);
7047 :
7048 185615695 : if (is_gimple_assign (stmt)
7049 185615695 : && !stmt_could_throw_p (cfun, stmt))
7050 : {
7051 46929128 : tree lhs, rhs1, rhs2;
7052 46929128 : enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
7053 :
7054 : /* If this was part of an already processed statement,
7055 : we don't need to touch it again. */
7056 46929128 : if (gimple_visited_p (stmt))
7057 : {
7058 : /* This statement might have become dead because of previous
7059 : reassociations. */
7060 400267 : if (has_zero_uses (gimple_get_lhs (stmt)))
7061 : {
7062 136698 : reassoc_remove_stmt (&gsi);
7063 136698 : release_defs (stmt);
7064 : /* We might end up removing the last stmt above which
7065 : places the iterator to the end of the sequence.
7066 : Reset it to the last stmt in this case and make sure
7067 : we don't do gsi_prev in that case. */
7068 136698 : if (gsi_end_p (gsi))
7069 : {
7070 506 : gsi = gsi_last_bb (bb);
7071 506 : do_prev = false;
7072 : }
7073 : }
7074 400267 : continue;
7075 : }
7076 :
7077 : /* If this is not a gimple binary expression, there is
7078 : nothing for us to do with it. */
7079 46528861 : if (get_gimple_rhs_class (rhs_code) != GIMPLE_BINARY_RHS)
7080 34492418 : continue;
7081 :
7082 12036443 : lhs = gimple_assign_lhs (stmt);
7083 12036443 : rhs1 = gimple_assign_rhs1 (stmt);
7084 12036443 : rhs2 = gimple_assign_rhs2 (stmt);
7085 :
7086 : /* For non-bit or min/max operations we can't associate
7087 : all types. Verify that here. */
7088 17687483 : if ((rhs_code != BIT_IOR_EXPR
7089 12036443 : && rhs_code != BIT_AND_EXPR
7090 11027364 : && rhs_code != BIT_XOR_EXPR
7091 11027364 : && rhs_code != MIN_EXPR
7092 10882098 : && rhs_code != MAX_EXPR
7093 10743163 : && !can_reassociate_type_p (TREE_TYPE (lhs)))
7094 6390081 : || !can_reassociate_op_p (rhs1)
7095 18423423 : || !can_reassociate_op_p (rhs2))
7096 5651040 : continue;
7097 :
7098 6385403 : if (associative_tree_code (rhs_code))
7099 : {
7100 4825065 : auto_vec<operand_entry *> ops;
7101 4825065 : tree powi_result = NULL_TREE;
7102 4825065 : bool is_vector = VECTOR_TYPE_P (TREE_TYPE (lhs));
7103 :
7104 : /* There may be no immediate uses left by the time we
7105 : get here because we may have eliminated them all. */
7106 4825065 : if (TREE_CODE (lhs) == SSA_NAME && has_zero_uses (lhs))
7107 47515 : continue;
7108 :
7109 4777550 : gimple_set_visited (stmt, true);
7110 4777550 : linearize_expr_tree (&ops, stmt, true, true);
7111 4777550 : ops.qsort (sort_by_operand_rank);
7112 4777550 : int orig_len = ops.length ();
7113 4777550 : optimize_ops_list (rhs_code, &ops);
7114 9555100 : if (undistribute_ops_list (rhs_code, &ops,
7115 : loop_containing_stmt (stmt)))
7116 : {
7117 315 : ops.qsort (sort_by_operand_rank);
7118 315 : optimize_ops_list (rhs_code, &ops);
7119 : }
7120 9555100 : if (undistribute_bitref_for_vector (rhs_code, &ops,
7121 : loop_containing_stmt (stmt)))
7122 : {
7123 36 : ops.qsort (sort_by_operand_rank);
7124 36 : optimize_ops_list (rhs_code, &ops);
7125 : }
7126 4777550 : if (rhs_code == PLUS_EXPR
7127 4777550 : && transform_add_to_multiply (&ops))
7128 73 : ops.qsort (sort_by_operand_rank);
7129 :
7130 4777550 : if (rhs_code == BIT_IOR_EXPR || rhs_code == BIT_AND_EXPR)
7131 : {
7132 993854 : if (is_vector)
7133 26244 : optimize_vec_cond_expr (rhs_code, &ops);
7134 : else
7135 967610 : optimize_range_tests (rhs_code, &ops, NULL);
7136 : }
7137 :
7138 4777550 : if (rhs_code == MULT_EXPR && !is_vector)
7139 : {
7140 1103182 : attempt_builtin_copysign (&ops);
7141 :
7142 1103182 : if (reassoc_insert_powi_p
7143 1103182 : && (flag_unsafe_math_optimizations
7144 459111 : || (INTEGRAL_TYPE_P (TREE_TYPE (lhs)))))
7145 502506 : powi_result = attempt_builtin_powi (stmt, &ops);
7146 : }
7147 :
7148 4777550 : operand_entry *last;
7149 4777550 : bool negate_result = false;
7150 4777550 : if (ops.length () > 1
7151 4777550 : && rhs_code == MULT_EXPR)
7152 : {
7153 1127665 : last = ops.last ();
7154 1127665 : if ((integer_minus_onep (last->op)
7155 1127144 : || real_minus_onep (last->op))
7156 548 : && !HONOR_SNANS (TREE_TYPE (lhs))
7157 1128213 : && (!HONOR_SIGNED_ZEROS (TREE_TYPE (lhs))
7158 0 : || !COMPLEX_FLOAT_TYPE_P (TREE_TYPE (lhs))))
7159 : {
7160 548 : ops.pop ();
7161 548 : negate_result = true;
7162 : }
7163 : }
7164 :
7165 4777550 : tree new_lhs = lhs;
7166 : /* If the operand vector is now empty, all operands were
7167 : consumed by the __builtin_powi optimization. */
7168 4777550 : if (ops.length () == 0)
7169 1464 : transform_stmt_to_copy (&gsi, stmt, powi_result);
7170 4776086 : else if (ops.length () == 1)
7171 : {
7172 13281 : tree last_op = ops.last ()->op;
7173 :
7174 : /* If the stmt that defines operand has to be inserted, insert it
7175 : before the use. */
7176 13281 : if (ops.last ()->stmt_to_insert)
7177 0 : insert_stmt_before_use (stmt, ops.last ()->stmt_to_insert);
7178 13281 : if (powi_result)
7179 194 : transform_stmt_to_multiply (&gsi, stmt, last_op,
7180 : powi_result);
7181 : else
7182 13087 : transform_stmt_to_copy (&gsi, stmt, last_op);
7183 : }
7184 : else
7185 : {
7186 4762805 : machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
7187 4762805 : int ops_num = ops.length ();
7188 4762805 : int width = 0;
7189 4762805 : int mult_num = 0;
7190 :
7191 : /* For binary bit operations, if there are at least 3
7192 : operands and the last operand in OPS is a constant,
7193 : move it to the front. This helps ensure that we generate
7194 : (X & Y) & C rather than (X & C) & Y. The former will
7195 : often match a canonical bit test when we get to RTL. */
7196 4762805 : if (ops.length () > 2
7197 180941 : && (rhs_code == BIT_AND_EXPR
7198 : || rhs_code == BIT_IOR_EXPR
7199 157767 : || rhs_code == BIT_XOR_EXPR)
7200 4790571 : && TREE_CODE (ops.last ()->op) == INTEGER_CST)
7201 1785 : std::swap (*ops[0], *ops[ops_num - 1]);
7202 :
7203 4762805 : optimization_type opt_type = bb_optimization_type (bb);
7204 :
7205 : /* If the target support FMA, rank_ops_for_fma will detect if
7206 : the chain has fmas and rearrange the ops if so. */
7207 4762805 : if (!reassoc_insert_powi_p
7208 2933326 : && direct_internal_fn_supported_p (IFN_FMA,
7209 2933326 : TREE_TYPE (lhs),
7210 : opt_type)
7211 4797433 : && (rhs_code == PLUS_EXPR || rhs_code == MINUS_EXPR))
7212 : {
7213 16824 : mult_num = rank_ops_for_fma (&ops);
7214 : }
7215 :
7216 : /* A chain that expands to a conditional compare sequence
7217 : has to stay linear. Every conditional compare tests the
7218 : flags that the previous comparison set, so the sequence
7219 : cannot be split into independent parts. */
7220 4762805 : bool ccmp_chain = (ops_num > 2
7221 4762805 : && prepare_ccmp_chain (ops, rhs_code, bb));
7222 :
7223 : /* Only rewrite the expression tree to parallel in the
7224 : last reassoc pass to avoid useless work back-and-forth
7225 : with initial linearization. */
7226 4762805 : bool has_fma = mult_num >= 2 && mult_num != ops_num;
7227 4762805 : if (!reassoc_insert_powi_p
7228 2933326 : && !ccmp_chain
7229 2933325 : && ops.length () > 3
7230 4776471 : && (width = get_reassociation_width (&ops, mult_num, lhs,
7231 : rhs_code, mode))
7232 : > 1)
7233 : {
7234 1891 : if (dump_file && (dump_flags & TDF_DETAILS))
7235 2 : fprintf (dump_file,
7236 : "Width = %d was chosen for reassociation\n",
7237 : width);
7238 1891 : rewrite_expr_tree_parallel (as_a <gassign *> (stmt),
7239 : width,
7240 : has_fma,
7241 : ops);
7242 : }
7243 : else
7244 : {
7245 : /* When there are three operands left, we want
7246 : to make sure the ones that get the double
7247 : binary op are chosen wisely. */
7248 4760914 : int len = ops.length ();
7249 7692349 : if (!reassoc_insert_powi_p && !ccmp_chain
7250 2931435 : && len >= 3
7251 4888423 : && (!has_fma
7252 : /* width > 1 means ranking ops results in better
7253 : parallelism. Check current value to avoid
7254 : calling get_reassociation_width again. */
7255 5288 : || (width != 1
7256 5288 : && get_reassociation_width (
7257 : &ops, mult_num, lhs, rhs_code, mode)
7258 : > 1)))
7259 124260 : swap_ops_for_binary_stmt (ops, len - 3);
7260 :
7261 4760914 : new_lhs = rewrite_expr_tree (stmt, rhs_code, 0, ops,
7262 4760914 : powi_result != NULL
7263 : || negate_result,
7264 : len != orig_len);
7265 : }
7266 :
7267 : /* If we combined some repeated factors into a
7268 : __builtin_powi call, multiply that result by the
7269 : reassociated operands. */
7270 4762805 : if (powi_result)
7271 : {
7272 129 : gimple *mul_stmt, *lhs_stmt = SSA_NAME_DEF_STMT (lhs);
7273 129 : tree type = TREE_TYPE (lhs);
7274 129 : tree target_ssa = make_temp_ssa_name (type, NULL,
7275 : "reassocpow");
7276 129 : gimple_set_lhs (lhs_stmt, target_ssa);
7277 129 : update_stmt (lhs_stmt);
7278 129 : if (lhs != new_lhs)
7279 : {
7280 129 : target_ssa = new_lhs;
7281 129 : new_lhs = lhs;
7282 : }
7283 129 : mul_stmt = gimple_build_assign (lhs, MULT_EXPR,
7284 : powi_result, target_ssa);
7285 129 : gimple_set_location (mul_stmt, gimple_location (stmt));
7286 129 : gimple_set_uid (mul_stmt, gimple_uid (stmt));
7287 129 : gsi_insert_after (&gsi, mul_stmt, GSI_NEW_STMT);
7288 : }
7289 : }
7290 :
7291 4777550 : if (negate_result)
7292 : {
7293 548 : stmt = SSA_NAME_DEF_STMT (lhs);
7294 548 : tree tmp = make_ssa_name (TREE_TYPE (lhs));
7295 548 : gimple_set_lhs (stmt, tmp);
7296 548 : if (lhs != new_lhs)
7297 538 : tmp = new_lhs;
7298 548 : gassign *neg_stmt = gimple_build_assign (lhs, NEGATE_EXPR,
7299 : tmp);
7300 548 : gimple_set_uid (neg_stmt, gimple_uid (stmt));
7301 548 : gsi_insert_after (&gsi, neg_stmt, GSI_NEW_STMT);
7302 548 : update_stmt (stmt);
7303 : }
7304 4825065 : }
7305 : }
7306 : }
7307 :
7308 20222299 : return cfg_cleanup_needed;
7309 : }
7310 :
7311 : /* Add jumps around shifts for range tests turned into bit tests.
7312 : For each SSA_NAME VAR we have code like:
7313 : VAR = ...; // final stmt of range comparison
7314 : // bit test here...;
7315 : OTHERVAR = ...; // final stmt of the bit test sequence
7316 : RES = VAR | OTHERVAR;
7317 : Turn the above into:
7318 : VAR = ...;
7319 : if (VAR != 0)
7320 : goto <l3>;
7321 : else
7322 : goto <l2>;
7323 : <l2>:
7324 : // bit test here...;
7325 : OTHERVAR = ...;
7326 : <l3>:
7327 : # RES = PHI<1(l1), OTHERVAR(l2)>; */
7328 :
7329 : static void
7330 2136094 : branch_fixup (void)
7331 : {
7332 2136094 : tree var;
7333 2136094 : unsigned int i;
7334 :
7335 2136480 : FOR_EACH_VEC_ELT (reassoc_branch_fixups, i, var)
7336 : {
7337 386 : gimple *def_stmt = SSA_NAME_DEF_STMT (var);
7338 386 : gimple *use_stmt;
7339 386 : use_operand_p use;
7340 386 : bool ok = single_imm_use (var, &use, &use_stmt);
7341 386 : gcc_assert (ok
7342 : && is_gimple_assign (use_stmt)
7343 : && gimple_assign_rhs_code (use_stmt) == BIT_IOR_EXPR
7344 : && gimple_bb (def_stmt) == gimple_bb (use_stmt));
7345 :
7346 386 : basic_block cond_bb = gimple_bb (def_stmt);
7347 386 : basic_block then_bb = split_block (cond_bb, def_stmt)->dest;
7348 386 : basic_block merge_bb = split_block (then_bb, use_stmt)->dest;
7349 :
7350 386 : gimple_stmt_iterator gsi = gsi_for_stmt (def_stmt);
7351 386 : gimple *g = gimple_build_cond (NE_EXPR, var,
7352 386 : build_zero_cst (TREE_TYPE (var)),
7353 : NULL_TREE, NULL_TREE);
7354 386 : location_t loc = gimple_location (use_stmt);
7355 386 : gimple_set_location (g, loc);
7356 386 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
7357 :
7358 386 : edge etrue = make_edge (cond_bb, merge_bb, EDGE_TRUE_VALUE);
7359 386 : etrue->probability = profile_probability::even ();
7360 386 : edge efalse = find_edge (cond_bb, then_bb);
7361 386 : efalse->flags = EDGE_FALSE_VALUE;
7362 386 : efalse->probability -= etrue->probability;
7363 386 : then_bb->count -= etrue->count ();
7364 :
7365 386 : tree othervar = NULL_TREE;
7366 386 : if (gimple_assign_rhs1 (use_stmt) == var)
7367 283 : othervar = gimple_assign_rhs2 (use_stmt);
7368 103 : else if (gimple_assign_rhs2 (use_stmt) == var)
7369 : othervar = gimple_assign_rhs1 (use_stmt);
7370 : else
7371 0 : gcc_unreachable ();
7372 386 : tree lhs = gimple_assign_lhs (use_stmt);
7373 386 : gphi *phi = create_phi_node (lhs, merge_bb);
7374 386 : add_phi_arg (phi, build_one_cst (TREE_TYPE (lhs)), etrue, loc);
7375 386 : add_phi_arg (phi, othervar, single_succ_edge (then_bb), loc);
7376 386 : gsi = gsi_for_stmt (use_stmt);
7377 386 : gsi_remove (&gsi, true);
7378 :
7379 386 : set_immediate_dominator (CDI_DOMINATORS, merge_bb, cond_bb);
7380 386 : set_immediate_dominator (CDI_POST_DOMINATORS, cond_bb, merge_bb);
7381 : }
7382 2136094 : reassoc_branch_fixups.release ();
7383 2136094 : }
7384 :
7385 : void dump_ops_vector (FILE *file, vec<operand_entry *> ops);
7386 : void debug_ops_vector (vec<operand_entry *> ops);
7387 :
7388 : /* Dump the operand entry vector OPS to FILE. */
7389 :
7390 : void
7391 0 : dump_ops_vector (FILE *file, vec<operand_entry *> ops)
7392 : {
7393 0 : operand_entry *oe;
7394 0 : unsigned int i;
7395 :
7396 0 : FOR_EACH_VEC_ELT (ops, i, oe)
7397 : {
7398 0 : fprintf (file, "Op %d -> rank: %d, tree: ", i, oe->rank);
7399 0 : print_generic_expr (file, oe->op);
7400 0 : fprintf (file, "\n");
7401 : }
7402 0 : }
7403 :
7404 : /* Dump the operand entry vector OPS to STDERR. */
7405 :
7406 : DEBUG_FUNCTION void
7407 0 : debug_ops_vector (vec<operand_entry *> ops)
7408 : {
7409 0 : dump_ops_vector (stderr, ops);
7410 0 : }
7411 :
7412 : /* Bubble up return status from reassociate_bb. */
7413 :
7414 : static bool
7415 2136094 : do_reassoc ()
7416 : {
7417 2136094 : bool cfg_cleanup_needed = false;
7418 2136094 : basic_block *worklist = XNEWVEC (basic_block, n_basic_blocks_for_fn (cfun));
7419 :
7420 2136094 : unsigned sp = 0;
7421 2136094 : for (auto son = first_dom_son (CDI_DOMINATORS, ENTRY_BLOCK_PTR_FOR_FN (cfun));
7422 4272188 : son; son = next_dom_son (CDI_DOMINATORS, son))
7423 2136094 : worklist[sp++] = son;
7424 22358442 : while (sp)
7425 : {
7426 20222348 : basic_block bb = worklist[--sp];
7427 20222348 : break_up_subtract_bb (bb);
7428 20222348 : for (auto son = first_dom_son (CDI_DOMINATORS, bb);
7429 38308602 : son; son = next_dom_son (CDI_DOMINATORS, son))
7430 18086254 : worklist[sp++] = son;
7431 : }
7432 :
7433 11178868 : for (auto son = first_dom_son (CDI_POST_DOMINATORS,
7434 2136094 : EXIT_BLOCK_PTR_FOR_FN (cfun));
7435 11178868 : son; son = next_dom_son (CDI_POST_DOMINATORS, son))
7436 9042774 : worklist[sp++] = son;
7437 22358393 : while (sp)
7438 : {
7439 20222299 : basic_block bb = worklist[--sp];
7440 20222299 : cfg_cleanup_needed |= reassociate_bb (bb);
7441 20222299 : for (auto son = first_dom_son (CDI_POST_DOMINATORS, bb);
7442 31401824 : son; son = next_dom_son (CDI_POST_DOMINATORS, son))
7443 11179525 : worklist[sp++] = son;
7444 : }
7445 :
7446 2136094 : free (worklist);
7447 2136094 : return cfg_cleanup_needed;
7448 : }
7449 :
7450 : /* Initialize the reassociation pass. */
7451 :
7452 : static void
7453 2136094 : init_reassoc (void)
7454 : {
7455 2136094 : int i;
7456 2136094 : int64_t rank = 2;
7457 2136094 : int *bbs = XNEWVEC (int, n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS);
7458 :
7459 : /* Find the loops, so that we can prevent moving calculations in
7460 : them. */
7461 2136094 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
7462 :
7463 2136094 : memset (&reassociate_stats, 0, sizeof (reassociate_stats));
7464 :
7465 2136094 : next_operand_entry_id = 0;
7466 :
7467 : /* Reverse RPO (Reverse Post Order) will give us something where
7468 : deeper loops come later. */
7469 2136094 : pre_and_rev_post_order_compute (NULL, bbs, false);
7470 2136094 : bb_rank = XCNEWVEC (int64_t, last_basic_block_for_fn (cfun));
7471 2136094 : operand_rank = new hash_map<tree, int64_t>;
7472 :
7473 : /* Give each default definition a distinct rank. This includes
7474 : parameters and the static chain. Walk backwards over all
7475 : SSA names so that we get proper rank ordering according
7476 : to tree_swap_operands_p. */
7477 113518598 : for (i = num_ssa_names - 1; i > 0; --i)
7478 : {
7479 109246410 : tree name = ssa_name (i);
7480 186437699 : if (name && SSA_NAME_IS_DEFAULT_DEF (name))
7481 6332185 : insert_operand_rank (name, ++rank);
7482 : }
7483 :
7484 : /* Set up rank for each BB */
7485 22358393 : for (i = 0; i < n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS; i++)
7486 20222299 : bb_rank[bbs[i]] = ++rank << 16;
7487 :
7488 2136094 : free (bbs);
7489 2136094 : calculate_dominance_info (CDI_POST_DOMINATORS);
7490 2136094 : plus_negates = vNULL;
7491 2136094 : mark_ssa_maybe_undefs ();
7492 2136094 : }
7493 :
7494 : /* Cleanup after the reassociation pass, and print stats if
7495 : requested. */
7496 :
7497 : static void
7498 2136094 : fini_reassoc (void)
7499 : {
7500 2136094 : statistics_counter_event (cfun, "Linearized",
7501 : reassociate_stats.linearized);
7502 2136094 : statistics_counter_event (cfun, "Constants eliminated",
7503 : reassociate_stats.constants_eliminated);
7504 2136094 : statistics_counter_event (cfun, "Ops eliminated",
7505 : reassociate_stats.ops_eliminated);
7506 2136094 : statistics_counter_event (cfun, "Statements rewritten",
7507 : reassociate_stats.rewritten);
7508 2136094 : statistics_counter_event (cfun, "Built-in pow[i] calls encountered",
7509 : reassociate_stats.pows_encountered);
7510 2136094 : statistics_counter_event (cfun, "Built-in powi calls created",
7511 : reassociate_stats.pows_created);
7512 :
7513 4272188 : delete operand_rank;
7514 2136094 : bitmap_clear (biased_names);
7515 2136094 : operand_entry_pool.release ();
7516 2136094 : free (bb_rank);
7517 2136094 : plus_negates.release ();
7518 2136094 : free_dominance_info (CDI_POST_DOMINATORS);
7519 2136094 : loop_optimizer_finalize ();
7520 2136094 : }
7521 :
7522 : /* Gate and execute functions for Reassociation. If INSERT_POWI_P, enable
7523 : insertion of __builtin_powi calls.
7524 :
7525 : Returns TODO_cfg_cleanup if a CFG cleanup pass is desired due to
7526 : optimization of a gimple conditional. Otherwise returns zero. */
7527 :
7528 : static unsigned int
7529 2136094 : execute_reassoc (bool insert_powi_p, bool bias_loop_carried_phi_ranks_p)
7530 : {
7531 2136094 : reassoc_insert_powi_p = insert_powi_p;
7532 2136094 : reassoc_bias_loop_carried_phi_ranks_p = bias_loop_carried_phi_ranks_p;
7533 :
7534 2136094 : init_reassoc ();
7535 :
7536 2136094 : bool cfg_cleanup_needed;
7537 2136094 : cfg_cleanup_needed = do_reassoc ();
7538 2136094 : repropagate_negates ();
7539 2136094 : branch_fixup ();
7540 :
7541 2136094 : fini_reassoc ();
7542 2136094 : return cfg_cleanup_needed ? TODO_cleanup_cfg : 0;
7543 : }
7544 :
7545 : namespace {
7546 :
7547 : const pass_data pass_data_reassoc =
7548 : {
7549 : GIMPLE_PASS, /* type */
7550 : "reassoc", /* name */
7551 : OPTGROUP_NONE, /* optinfo_flags */
7552 : TV_TREE_REASSOC, /* tv_id */
7553 : ( PROP_cfg | PROP_ssa ), /* properties_required */
7554 : 0, /* properties_provided */
7555 : 0, /* properties_destroyed */
7556 : 0, /* todo_flags_start */
7557 : TODO_update_ssa_only_virtuals, /* todo_flags_finish */
7558 : };
7559 :
7560 : class pass_reassoc : public gimple_opt_pass
7561 : {
7562 : public:
7563 589072 : pass_reassoc (gcc::context *ctxt)
7564 1178144 : : gimple_opt_pass (pass_data_reassoc, ctxt), insert_powi_p (false)
7565 : {}
7566 :
7567 : /* opt_pass methods: */
7568 294536 : opt_pass * clone () final override { return new pass_reassoc (m_ctxt); }
7569 589072 : void set_pass_param (unsigned int n, bool param) final override
7570 : {
7571 589072 : gcc_assert (n == 0);
7572 589072 : insert_powi_p = param;
7573 589072 : bias_loop_carried_phi_ranks_p = !param;
7574 589072 : }
7575 2136438 : bool gate (function *) final override { return flag_tree_reassoc != 0; }
7576 2136094 : unsigned int execute (function *) final override
7577 : {
7578 2136094 : return execute_reassoc (insert_powi_p, bias_loop_carried_phi_ranks_p);
7579 : }
7580 :
7581 : private:
7582 : /* Enable insertion of __builtin_powi calls during execute_reassoc. See
7583 : point 3a in the pass header comment. */
7584 : bool insert_powi_p;
7585 : bool bias_loop_carried_phi_ranks_p;
7586 : }; // class pass_reassoc
7587 :
7588 : } // anon namespace
7589 :
7590 : gimple_opt_pass *
7591 294536 : make_pass_reassoc (gcc::context *ctxt)
7592 : {
7593 294536 : return new pass_reassoc (ctxt);
7594 : }
|
