Line data Source code
1 : /* Global, SSA-based optimizations using mathematical identities.
2 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by the
8 : Free Software Foundation; either version 3, or (at your option) any
9 : later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : /* Currently, the only mini-pass in this file tries to CSE reciprocal
21 : operations. These are common in sequences such as this one:
22 :
23 : modulus = sqrt(x*x + y*y + z*z);
24 : x = x / modulus;
25 : y = y / modulus;
26 : z = z / modulus;
27 :
28 : that can be optimized to
29 :
30 : modulus = sqrt(x*x + y*y + z*z);
31 : rmodulus = 1.0 / modulus;
32 : x = x * rmodulus;
33 : y = y * rmodulus;
34 : z = z * rmodulus;
35 :
36 : We do this for loop invariant divisors, and with this pass whenever
37 : we notice that a division has the same divisor multiple times.
38 :
39 : Of course, like in PRE, we don't insert a division if a dominator
40 : already has one. However, this cannot be done as an extension of
41 : PRE for several reasons.
42 :
43 : First of all, with some experiments it was found out that the
44 : transformation is not always useful if there are only two divisions
45 : by the same divisor. This is probably because modern processors
46 : can pipeline the divisions; on older, in-order processors it should
47 : still be effective to optimize two divisions by the same number.
48 : We make this a param, and it shall be called N in the remainder of
49 : this comment.
50 :
51 : Second, if trapping math is active, we have less freedom on where
52 : to insert divisions: we can only do so in basic blocks that already
53 : contain one. (If divisions don't trap, instead, we can insert
54 : divisions elsewhere, which will be in blocks that are common dominators
55 : of those that have the division).
56 :
57 : We really don't want to compute the reciprocal unless a division will
58 : be found. To do this, we won't insert the division in a basic block
59 : that has less than N divisions *post-dominating* it.
60 :
61 : The algorithm constructs a subset of the dominator tree, holding the
62 : blocks containing the divisions and the common dominators to them,
63 : and walk it twice. The first walk is in post-order, and it annotates
64 : each block with the number of divisions that post-dominate it: this
65 : gives information on where divisions can be inserted profitably.
66 : The second walk is in pre-order, and it inserts divisions as explained
67 : above, and replaces divisions by multiplications.
68 :
69 : In the best case, the cost of the pass is O(n_statements). In the
70 : worst-case, the cost is due to creating the dominator tree subset,
71 : with a cost of O(n_basic_blocks ^ 2); however this can only happen
72 : for n_statements / n_basic_blocks statements. So, the amortized cost
73 : of creating the dominator tree subset is O(n_basic_blocks) and the
74 : worst-case cost of the pass is O(n_statements * n_basic_blocks).
75 :
76 : More practically, the cost will be small because there are few
77 : divisions, and they tend to be in the same basic block, so insert_bb
78 : is called very few times.
79 :
80 : If we did this using domwalk.cc, an efficient implementation would have
81 : to work on all the variables in a single pass, because we could not
82 : work on just a subset of the dominator tree, as we do now, and the
83 : cost would also be something like O(n_statements * n_basic_blocks).
84 : The data structures would be more complex in order to work on all the
85 : variables in a single pass. */
86 :
87 : #include "config.h"
88 : #include "system.h"
89 : #include "coretypes.h"
90 : #include "backend.h"
91 : #include "target.h"
92 : #include "rtl.h"
93 : #include "tree.h"
94 : #include "gimple.h"
95 : #include "predict.h"
96 : #include "alloc-pool.h"
97 : #include "tree-pass.h"
98 : #include "ssa.h"
99 : #include "optabs-tree.h"
100 : #include "gimple-pretty-print.h"
101 : #include "alias.h"
102 : #include "fold-const.h"
103 : #include "gimple-iterator.h"
104 : #include "gimple-fold.h"
105 : #include "stor-layout.h"
106 : #include "tree-cfg.h"
107 : #include "tree-dfa.h"
108 : #include "tree-ssa.h"
109 : #include "builtins.h"
110 : #include "internal-fn.h"
111 : #include "case-cfn-macros.h"
112 : #include "optabs-libfuncs.h"
113 : #include "tree-eh.h"
114 : #include "targhooks.h"
115 : #include "domwalk.h"
116 : #include "tree-ssa-math-opts.h"
117 : #include "dbgcnt.h"
118 : #include "langhooks.h"
119 : #include "cfghooks.h"
120 :
121 : /* This structure represents one basic block that either computes a
122 : division, or is a common dominator for basic block that compute a
123 : division. */
124 : struct occurrence {
125 : /* The basic block represented by this structure. */
126 : basic_block bb = basic_block();
127 :
128 : /* If non-NULL, the SSA_NAME holding the definition for a reciprocal
129 : inserted in BB. */
130 : tree recip_def = tree();
131 :
132 : /* If non-NULL, the SSA_NAME holding the definition for a squared
133 : reciprocal inserted in BB. */
134 : tree square_recip_def = tree();
135 :
136 : /* If non-NULL, the GIMPLE_ASSIGN for a reciprocal computation that
137 : was inserted in BB. */
138 : gimple *recip_def_stmt = nullptr;
139 :
140 : /* Pointer to a list of "struct occurrence"s for blocks dominated
141 : by BB. */
142 : struct occurrence *children = nullptr;
143 :
144 : /* Pointer to the next "struct occurrence"s in the list of blocks
145 : sharing a common dominator. */
146 : struct occurrence *next = nullptr;
147 :
148 : /* The number of divisions that are in BB before compute_merit. The
149 : number of divisions that are in BB or post-dominate it after
150 : compute_merit. */
151 : int num_divisions = 0;
152 :
153 : /* True if the basic block has a division, false if it is a common
154 : dominator for basic blocks that do. If it is false and trapping
155 : math is active, BB is not a candidate for inserting a reciprocal. */
156 : bool bb_has_division = false;
157 :
158 : /* Construct a struct occurrence for basic block BB, and whose
159 : children list is headed by CHILDREN. */
160 618 : occurrence (basic_block bb, struct occurrence *children)
161 618 : : bb (bb), children (children)
162 : {
163 618 : bb->aux = this;
164 : }
165 :
166 : /* Destroy a struct occurrence and remove it from its basic block. */
167 618 : ~occurrence ()
168 : {
169 618 : bb->aux = nullptr;
170 618 : }
171 :
172 : /* Allocate memory for a struct occurrence from OCC_POOL. */
173 : static void* operator new (size_t);
174 :
175 : /* Return memory for a struct occurrence to OCC_POOL. */
176 : static void operator delete (void*, size_t);
177 : };
178 :
179 : static struct
180 : {
181 : /* Number of 1.0/X ops inserted. */
182 : int rdivs_inserted;
183 :
184 : /* Number of 1.0/FUNC ops inserted. */
185 : int rfuncs_inserted;
186 : } reciprocal_stats;
187 :
188 : static struct
189 : {
190 : /* Number of cexpi calls inserted. */
191 : int inserted;
192 :
193 : /* Number of conversions removed. */
194 : int conv_removed;
195 :
196 : } sincos_stats;
197 :
198 : static struct
199 : {
200 : /* Number of widening multiplication ops inserted. */
201 : int widen_mults_inserted;
202 :
203 : /* Number of integer multiply-and-accumulate ops inserted. */
204 : int maccs_inserted;
205 :
206 : /* Number of fp fused multiply-add ops inserted. */
207 : int fmas_inserted;
208 :
209 : /* Number of divmod calls inserted. */
210 : int divmod_calls_inserted;
211 :
212 : /* Number of highpart multiplication ops inserted. */
213 : int highpart_mults_inserted;
214 : } widen_mul_stats;
215 :
216 : /* The instance of "struct occurrence" representing the highest
217 : interesting block in the dominator tree. */
218 : static struct occurrence *occ_head;
219 :
220 : /* Allocation pool for getting instances of "struct occurrence". */
221 : static object_allocator<occurrence> *occ_pool;
222 :
223 618 : void* occurrence::operator new (size_t n)
224 : {
225 618 : gcc_assert (n == sizeof(occurrence));
226 618 : return occ_pool->allocate_raw ();
227 : }
228 :
229 618 : void occurrence::operator delete (void *occ, size_t n)
230 : {
231 618 : gcc_assert (n == sizeof(occurrence));
232 618 : occ_pool->remove_raw (occ);
233 618 : }
234 :
235 : /* Insert NEW_OCC into our subset of the dominator tree. P_HEAD points to a
236 : list of "struct occurrence"s, one per basic block, having IDOM as
237 : their common dominator.
238 :
239 : We try to insert NEW_OCC as deep as possible in the tree, and we also
240 : insert any other block that is a common dominator for BB and one
241 : block already in the tree. */
242 :
243 : static void
244 607 : insert_bb (struct occurrence *new_occ, basic_block idom,
245 : struct occurrence **p_head)
246 : {
247 612 : struct occurrence *occ, **p_occ;
248 :
249 635 : for (p_occ = p_head; (occ = *p_occ) != NULL; )
250 : {
251 28 : basic_block bb = new_occ->bb, occ_bb = occ->bb;
252 28 : basic_block dom = nearest_common_dominator (CDI_DOMINATORS, occ_bb, bb);
253 28 : if (dom == bb)
254 : {
255 : /* BB dominates OCC_BB. OCC becomes NEW_OCC's child: remove OCC
256 : from its list. */
257 6 : *p_occ = occ->next;
258 6 : occ->next = new_occ->children;
259 6 : new_occ->children = occ;
260 :
261 : /* Try the next block (it may as well be dominated by BB). */
262 : }
263 :
264 22 : else if (dom == occ_bb)
265 : {
266 : /* OCC_BB dominates BB. Tail recurse to look deeper. */
267 5 : insert_bb (new_occ, dom, &occ->children);
268 5 : return;
269 : }
270 :
271 17 : else if (dom != idom)
272 : {
273 11 : gcc_assert (!dom->aux);
274 :
275 : /* There is a dominator between IDOM and BB, add it and make
276 : two children out of NEW_OCC and OCC. First, remove OCC from
277 : its list. */
278 11 : *p_occ = occ->next;
279 11 : new_occ->next = occ;
280 11 : occ->next = NULL;
281 :
282 : /* None of the previous blocks has DOM as a dominator: if we tail
283 : recursed, we would reexamine them uselessly. Just switch BB with
284 : DOM, and go on looking for blocks dominated by DOM. */
285 11 : new_occ = new occurrence (dom, new_occ);
286 : }
287 :
288 : else
289 : {
290 : /* Nothing special, go on with the next element. */
291 6 : p_occ = &occ->next;
292 : }
293 : }
294 :
295 : /* No place was found as a child of IDOM. Make BB a sibling of IDOM. */
296 607 : new_occ->next = *p_head;
297 607 : *p_head = new_occ;
298 : }
299 :
300 : /* Register that we found a division in BB.
301 : IMPORTANCE is a measure of how much weighting to give
302 : that division. Use IMPORTANCE = 2 to register a single
303 : division. If the division is going to be found multiple
304 : times use 1 (as it is with squares). */
305 :
306 : static inline void
307 709 : register_division_in (basic_block bb, int importance)
308 : {
309 709 : struct occurrence *occ;
310 :
311 709 : occ = (struct occurrence *) bb->aux;
312 709 : if (!occ)
313 : {
314 607 : occ = new occurrence (bb, NULL);
315 607 : insert_bb (occ, ENTRY_BLOCK_PTR_FOR_FN (cfun), &occ_head);
316 : }
317 :
318 709 : occ->bb_has_division = true;
319 709 : occ->num_divisions += importance;
320 709 : }
321 :
322 :
323 : /* Compute the number of divisions that postdominate each block in OCC and
324 : its children. */
325 :
326 : static void
327 26 : compute_merit (struct occurrence *occ)
328 : {
329 26 : struct occurrence *occ_child;
330 26 : basic_block dom = occ->bb;
331 :
332 45 : for (occ_child = occ->children; occ_child; occ_child = occ_child->next)
333 : {
334 19 : basic_block bb;
335 19 : if (occ_child->children)
336 3 : compute_merit (occ_child);
337 :
338 19 : if (flag_exceptions)
339 6 : bb = single_noncomplex_succ (dom);
340 : else
341 : bb = dom;
342 :
343 19 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, occ_child->bb))
344 12 : occ->num_divisions += occ_child->num_divisions;
345 : }
346 26 : }
347 :
348 :
349 : /* Return whether USE_STMT is a floating-point division by DEF. */
350 : static inline bool
351 352854 : is_division_by (gimple *use_stmt, tree def)
352 : {
353 352854 : return is_gimple_assign (use_stmt)
354 242976 : && gimple_assign_rhs_code (use_stmt) == RDIV_EXPR
355 1251 : && gimple_assign_rhs2 (use_stmt) == def
356 : /* Do not recognize x / x as valid division, as we are getting
357 : confused later by replacing all immediate uses x in such
358 : a stmt. */
359 879 : && gimple_assign_rhs1 (use_stmt) != def
360 353733 : && !stmt_can_throw_internal (cfun, use_stmt);
361 : }
362 :
363 : /* Return TRUE if USE_STMT is a multiplication of DEF by A. */
364 : static inline bool
365 349063 : is_mult_by (gimple *use_stmt, tree def, tree a)
366 : {
367 349063 : if (gimple_code (use_stmt) == GIMPLE_ASSIGN
368 349063 : && gimple_assign_rhs_code (use_stmt) == MULT_EXPR)
369 : {
370 80029 : tree op0 = gimple_assign_rhs1 (use_stmt);
371 80029 : tree op1 = gimple_assign_rhs2 (use_stmt);
372 :
373 80029 : return (op0 == def && op1 == a)
374 80029 : || (op0 == a && op1 == def);
375 : }
376 : return 0;
377 : }
378 :
379 : /* Return whether USE_STMT is DEF * DEF. */
380 : static inline bool
381 349018 : is_square_of (gimple *use_stmt, tree def)
382 : {
383 5 : return is_mult_by (use_stmt, def, def);
384 : }
385 :
386 : /* Return whether USE_STMT is a floating-point division by
387 : DEF * DEF. */
388 : static inline bool
389 192 : is_division_by_square (gimple *use_stmt, tree def)
390 : {
391 192 : if (gimple_code (use_stmt) == GIMPLE_ASSIGN
392 185 : && gimple_assign_rhs_code (use_stmt) == RDIV_EXPR
393 7 : && gimple_assign_rhs1 (use_stmt) != gimple_assign_rhs2 (use_stmt)
394 199 : && !stmt_can_throw_internal (cfun, use_stmt))
395 : {
396 7 : tree denominator = gimple_assign_rhs2 (use_stmt);
397 7 : if (TREE_CODE (denominator) == SSA_NAME)
398 7 : return is_square_of (SSA_NAME_DEF_STMT (denominator), def);
399 : }
400 : return 0;
401 : }
402 :
403 : /* Walk the subset of the dominator tree rooted at OCC, setting the
404 : RECIP_DEF field to a definition of 1.0 / DEF that can be used in
405 : the given basic block. The field may be left NULL, of course,
406 : if it is not possible or profitable to do the optimization.
407 :
408 : DEF_BSI is an iterator pointing at the statement defining DEF.
409 : If RECIP_DEF is set, a dominator already has a computation that can
410 : be used.
411 :
412 : If should_insert_square_recip is set, then this also inserts
413 : the square of the reciprocal immediately after the definition
414 : of the reciprocal. */
415 :
416 : static void
417 42 : insert_reciprocals (gimple_stmt_iterator *def_gsi, struct occurrence *occ,
418 : tree def, tree recip_def, tree square_recip_def,
419 : int should_insert_square_recip, int threshold)
420 : {
421 42 : tree type;
422 42 : gassign *new_stmt, *new_square_stmt;
423 42 : gimple_stmt_iterator gsi;
424 42 : struct occurrence *occ_child;
425 :
426 42 : if (!recip_def
427 26 : && (occ->bb_has_division || !flag_trapping_math)
428 : /* Divide by two as all divisions are counted twice in
429 : the costing loop. */
430 25 : && occ->num_divisions / 2 >= threshold)
431 : {
432 : /* Make a variable with the replacement and substitute it. */
433 24 : type = TREE_TYPE (def);
434 24 : recip_def = create_tmp_reg (type, "reciptmp");
435 24 : new_stmt = gimple_build_assign (recip_def, RDIV_EXPR,
436 : build_one_cst (type), def);
437 :
438 24 : if (should_insert_square_recip)
439 : {
440 4 : square_recip_def = create_tmp_reg (type, "powmult_reciptmp");
441 4 : new_square_stmt = gimple_build_assign (square_recip_def, MULT_EXPR,
442 : recip_def, recip_def);
443 : }
444 :
445 24 : if (occ->bb_has_division)
446 : {
447 : /* Case 1: insert before an existing division. */
448 21 : gsi = gsi_after_labels (occ->bb);
449 212 : while (!gsi_end_p (gsi)
450 212 : && (!is_division_by (gsi_stmt (gsi), def))
451 404 : && (!is_division_by_square (gsi_stmt (gsi), def)))
452 191 : gsi_next (&gsi);
453 :
454 21 : gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
455 21 : if (should_insert_square_recip)
456 3 : gsi_insert_before (&gsi, new_square_stmt, GSI_SAME_STMT);
457 : }
458 3 : else if (def_gsi && occ->bb == gsi_bb (*def_gsi))
459 : {
460 : /* Case 2: insert right after the definition. Note that this will
461 : never happen if the definition statement can throw, because in
462 : that case the sole successor of the statement's basic block will
463 : dominate all the uses as well. */
464 2 : gsi_insert_after (def_gsi, new_stmt, GSI_NEW_STMT);
465 2 : if (should_insert_square_recip)
466 1 : gsi_insert_after (def_gsi, new_square_stmt, GSI_NEW_STMT);
467 : }
468 : else
469 : {
470 : /* Case 3: insert in a basic block not containing defs/uses. */
471 1 : gsi = gsi_after_labels (occ->bb);
472 1 : gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
473 1 : if (should_insert_square_recip)
474 0 : gsi_insert_before (&gsi, new_square_stmt, GSI_SAME_STMT);
475 : }
476 :
477 24 : reciprocal_stats.rdivs_inserted++;
478 :
479 24 : occ->recip_def_stmt = new_stmt;
480 : }
481 :
482 42 : occ->recip_def = recip_def;
483 42 : occ->square_recip_def = square_recip_def;
484 61 : for (occ_child = occ->children; occ_child; occ_child = occ_child->next)
485 19 : insert_reciprocals (def_gsi, occ_child, def, recip_def,
486 : square_recip_def, should_insert_square_recip,
487 : threshold);
488 42 : }
489 :
490 : /* Replace occurrences of expr / (x * x) with expr * ((1 / x) * (1 / x)).
491 : Take as argument the use for (x * x). */
492 : static inline void
493 4 : replace_reciprocal_squares (use_operand_p use_p)
494 : {
495 4 : gimple *use_stmt = USE_STMT (use_p);
496 4 : basic_block bb = gimple_bb (use_stmt);
497 4 : struct occurrence *occ = (struct occurrence *) bb->aux;
498 :
499 8 : if (optimize_bb_for_speed_p (bb) && occ->square_recip_def
500 8 : && occ->recip_def)
501 : {
502 4 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
503 4 : gimple_assign_set_rhs_code (use_stmt, MULT_EXPR);
504 4 : gimple_assign_set_rhs2 (use_stmt, occ->square_recip_def);
505 4 : SET_USE (use_p, occ->square_recip_def);
506 4 : fold_stmt_inplace (&gsi);
507 4 : update_stmt (use_stmt);
508 : }
509 4 : }
510 :
511 :
512 : /* Replace the division at USE_P with a multiplication by the reciprocal, if
513 : possible. */
514 :
515 : static inline void
516 105 : replace_reciprocal (use_operand_p use_p)
517 : {
518 105 : gimple *use_stmt = USE_STMT (use_p);
519 105 : basic_block bb = gimple_bb (use_stmt);
520 105 : struct occurrence *occ = (struct occurrence *) bb->aux;
521 :
522 105 : if (optimize_bb_for_speed_p (bb)
523 105 : && occ->recip_def && use_stmt != occ->recip_def_stmt)
524 : {
525 80 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
526 80 : gimple_assign_set_rhs_code (use_stmt, MULT_EXPR);
527 80 : SET_USE (use_p, occ->recip_def);
528 80 : fold_stmt_inplace (&gsi);
529 80 : update_stmt (use_stmt);
530 : }
531 105 : }
532 :
533 :
534 : /* Free OCC and return one more "struct occurrence" to be freed. */
535 :
536 : static struct occurrence *
537 618 : free_bb (struct occurrence *occ)
538 : {
539 618 : struct occurrence *child, *next;
540 :
541 : /* First get the two pointers hanging off OCC. */
542 618 : next = occ->next;
543 618 : child = occ->children;
544 618 : delete occ;
545 :
546 : /* Now ensure that we don't recurse unless it is necessary. */
547 618 : if (!child)
548 : return next;
549 : else
550 : {
551 19 : while (next)
552 0 : next = free_bb (next);
553 :
554 : return child;
555 : }
556 : }
557 :
558 : /* Transform sequences like
559 : t = sqrt (a)
560 : x = 1.0 / t;
561 : r1 = x * x;
562 : r2 = a * x;
563 : into:
564 : t = sqrt (a)
565 : r1 = 1.0 / a;
566 : r2 = t;
567 : x = r1 * r2;
568 : depending on the uses of x, r1, r2. This removes one multiplication and
569 : allows the sqrt and division operations to execute in parallel.
570 : DEF_GSI is the gsi of the initial division by sqrt that defines
571 : DEF (x in the example above). */
572 :
573 : static void
574 556 : optimize_recip_sqrt (gimple_stmt_iterator *def_gsi, tree def)
575 : {
576 556 : gimple *use_stmt;
577 556 : imm_use_iterator use_iter;
578 556 : gimple *stmt = gsi_stmt (*def_gsi);
579 556 : tree x = def;
580 556 : tree orig_sqrt_ssa_name = gimple_assign_rhs2 (stmt);
581 556 : tree div_rhs1 = gimple_assign_rhs1 (stmt);
582 :
583 556 : if (TREE_CODE (orig_sqrt_ssa_name) != SSA_NAME
584 551 : || TREE_CODE (div_rhs1) != REAL_CST
585 742 : || !real_equal (&TREE_REAL_CST (div_rhs1), &dconst1))
586 544 : return;
587 :
588 110 : gcall *sqrt_stmt
589 586 : = dyn_cast <gcall *> (SSA_NAME_DEF_STMT (orig_sqrt_ssa_name));
590 :
591 42 : if (!sqrt_stmt || !gimple_call_lhs (sqrt_stmt))
592 : return;
593 :
594 42 : switch (gimple_call_combined_fn (sqrt_stmt))
595 : {
596 31 : CASE_CFN_SQRT:
597 31 : CASE_CFN_SQRT_FN:
598 31 : break;
599 :
600 : default:
601 : return;
602 : }
603 31 : tree a = gimple_call_arg (sqrt_stmt, 0);
604 :
605 : /* We have 'a' and 'x'. Now analyze the uses of 'x'. */
606 :
607 : /* Statements that use x in x * x. */
608 43 : auto_vec<gimple *> sqr_stmts;
609 : /* Statements that use x in a * x. */
610 12 : auto_vec<gimple *> mult_stmts;
611 31 : bool has_other_use = false;
612 31 : bool mult_on_main_path = false;
613 :
614 89 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, x)
615 : {
616 58 : if (is_gimple_debug (use_stmt))
617 1 : continue;
618 57 : if (is_square_of (use_stmt, x))
619 : {
620 12 : sqr_stmts.safe_push (use_stmt);
621 12 : if (gimple_bb (use_stmt) == gimple_bb (stmt))
622 58 : mult_on_main_path = true;
623 : }
624 45 : else if (is_mult_by (use_stmt, x, a))
625 : {
626 14 : mult_stmts.safe_push (use_stmt);
627 14 : if (gimple_bb (use_stmt) == gimple_bb (stmt))
628 58 : mult_on_main_path = true;
629 : }
630 : else
631 : has_other_use = true;
632 31 : }
633 :
634 : /* In the x * x and a * x cases we just rewire stmt operands or
635 : remove multiplications. In the has_other_use case we introduce
636 : a multiplication so make sure we don't introduce a multiplication
637 : on a path where there was none. */
638 31 : if (has_other_use && !mult_on_main_path)
639 19 : return;
640 :
641 12 : if (sqr_stmts.is_empty () && mult_stmts.is_empty ())
642 : return;
643 :
644 : /* If x = 1.0 / sqrt (a) has uses other than those optimized here we want
645 : to be able to compose it from the sqr and mult cases. */
646 41 : if (has_other_use && (sqr_stmts.is_empty () || mult_stmts.is_empty ()))
647 : return;
648 :
649 12 : if (dump_file)
650 : {
651 10 : fprintf (dump_file, "Optimizing reciprocal sqrt multiplications of\n");
652 10 : print_gimple_stmt (dump_file, sqrt_stmt, 0, TDF_NONE);
653 10 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
654 10 : fprintf (dump_file, "\n");
655 : }
656 :
657 12 : bool delete_div = !has_other_use;
658 12 : tree sqr_ssa_name = NULL_TREE;
659 22 : if (!sqr_stmts.is_empty ())
660 : {
661 : /* r1 = x * x. Transform the original
662 : x = 1.0 / t
663 : into
664 : tmp1 = 1.0 / a
665 : r1 = tmp1. */
666 :
667 10 : sqr_ssa_name
668 10 : = make_temp_ssa_name (TREE_TYPE (a), NULL, "recip_sqrt_sqr");
669 :
670 10 : if (dump_file)
671 : {
672 10 : fprintf (dump_file, "Replacing original division\n");
673 10 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
674 10 : fprintf (dump_file, "with new division\n");
675 : }
676 10 : stmt
677 10 : = gimple_build_assign (sqr_ssa_name, gimple_assign_rhs_code (stmt),
678 : gimple_assign_rhs1 (stmt), a);
679 10 : gsi_insert_before (def_gsi, stmt, GSI_SAME_STMT);
680 10 : gsi_remove (def_gsi, true);
681 10 : *def_gsi = gsi_for_stmt (stmt);
682 10 : fold_stmt_inplace (def_gsi);
683 10 : update_stmt (stmt);
684 :
685 10 : if (dump_file)
686 10 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
687 :
688 10 : delete_div = false;
689 10 : gimple *sqr_stmt;
690 10 : unsigned int i;
691 20 : FOR_EACH_VEC_ELT (sqr_stmts, i, sqr_stmt)
692 : {
693 10 : gimple_stmt_iterator gsi2 = gsi_for_stmt (sqr_stmt);
694 10 : gimple_assign_set_rhs_from_tree (&gsi2, sqr_ssa_name);
695 10 : update_stmt (sqr_stmt);
696 : }
697 : }
698 24 : if (!mult_stmts.is_empty ())
699 : {
700 : /* r2 = a * x. Transform this into:
701 : r2 = t (The original sqrt (a)). */
702 : unsigned int i;
703 24 : gimple *mult_stmt = NULL;
704 24 : FOR_EACH_VEC_ELT (mult_stmts, i, mult_stmt)
705 : {
706 12 : gimple_stmt_iterator gsi2 = gsi_for_stmt (mult_stmt);
707 :
708 12 : if (dump_file)
709 : {
710 10 : fprintf (dump_file, "Replacing squaring multiplication\n");
711 10 : print_gimple_stmt (dump_file, mult_stmt, 0, TDF_NONE);
712 10 : fprintf (dump_file, "with assignment\n");
713 : }
714 12 : gimple_assign_set_rhs_from_tree (&gsi2, orig_sqrt_ssa_name);
715 12 : fold_stmt_inplace (&gsi2);
716 12 : update_stmt (mult_stmt);
717 12 : if (dump_file)
718 10 : print_gimple_stmt (dump_file, mult_stmt, 0, TDF_NONE);
719 : }
720 : }
721 :
722 12 : if (has_other_use)
723 : {
724 : /* Using the two temporaries tmp1, tmp2 from above
725 : the original x is now:
726 : x = tmp1 * tmp2. */
727 10 : gcc_assert (orig_sqrt_ssa_name);
728 10 : gcc_assert (sqr_ssa_name);
729 :
730 10 : gimple *new_stmt
731 10 : = gimple_build_assign (x, MULT_EXPR,
732 : orig_sqrt_ssa_name, sqr_ssa_name);
733 10 : gsi_insert_after (def_gsi, new_stmt, GSI_NEW_STMT);
734 10 : update_stmt (stmt);
735 : }
736 2 : else if (delete_div)
737 : {
738 : /* Remove the original division. */
739 2 : gimple_stmt_iterator gsi2 = gsi_for_stmt (stmt);
740 2 : gsi_remove (&gsi2, true);
741 2 : release_defs (stmt);
742 : }
743 : else
744 0 : release_ssa_name (x);
745 : }
746 :
747 : /* Look for floating-point divisions among DEF's uses, and try to
748 : replace them by multiplications with the reciprocal. Add
749 : as many statements computing the reciprocal as needed.
750 :
751 : DEF must be a GIMPLE register of a floating-point type. */
752 :
753 : static void
754 213262 : execute_cse_reciprocals_1 (gimple_stmt_iterator *def_gsi, tree def)
755 : {
756 213262 : use_operand_p use_p, square_use_p;
757 213262 : imm_use_iterator use_iter, square_use_iter;
758 213262 : tree square_def;
759 213262 : struct occurrence *occ;
760 213262 : int count = 0;
761 213262 : int threshold;
762 213262 : int square_recip_count = 0;
763 213262 : int sqrt_recip_count = 0;
764 :
765 213262 : gcc_assert (FLOAT_TYPE_P (TREE_TYPE (def)) && TREE_CODE (def) == SSA_NAME);
766 213262 : threshold = targetm.min_divisions_for_recip_mul (TYPE_MODE (TREE_TYPE (def)));
767 :
768 : /* If DEF is a square (x * x), count the number of divisions by x.
769 : If there are more divisions by x than by (DEF * DEF), prefer to optimize
770 : the reciprocal of x instead of DEF. This improves cases like:
771 : def = x * x
772 : t0 = a / def
773 : t1 = b / def
774 : t2 = c / x
775 : Reciprocal optimization of x results in 1 division rather than 2 or 3. */
776 213262 : gimple *def_stmt = SSA_NAME_DEF_STMT (def);
777 :
778 213262 : if (is_gimple_assign (def_stmt)
779 166784 : && gimple_assign_rhs_code (def_stmt) == MULT_EXPR
780 40571 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME
781 253756 : && gimple_assign_rhs1 (def_stmt) == gimple_assign_rhs2 (def_stmt))
782 : {
783 671 : tree op0 = gimple_assign_rhs1 (def_stmt);
784 :
785 2754 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, op0)
786 : {
787 2083 : gimple *use_stmt = USE_STMT (use_p);
788 2083 : if (is_division_by (use_stmt, op0))
789 17 : sqrt_recip_count++;
790 671 : }
791 : }
792 :
793 562211 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, def)
794 : {
795 348949 : gimple *use_stmt = USE_STMT (use_p);
796 348949 : if (is_division_by (use_stmt, def))
797 : {
798 637 : register_division_in (gimple_bb (use_stmt), 2);
799 637 : count++;
800 : }
801 :
802 348949 : if (is_square_of (use_stmt, def))
803 : {
804 1350 : square_def = gimple_assign_lhs (use_stmt);
805 2832 : FOR_EACH_IMM_USE_FAST (square_use_p, square_use_iter, square_def)
806 : {
807 1482 : gimple *square_use_stmt = USE_STMT (square_use_p);
808 1482 : if (is_division_by (square_use_stmt, square_def))
809 : {
810 : /* This is executed twice for each division by a square. */
811 72 : register_division_in (gimple_bb (square_use_stmt), 1);
812 72 : square_recip_count++;
813 : }
814 1350 : }
815 : }
816 213262 : }
817 :
818 : /* Square reciprocals were counted twice above. */
819 213262 : square_recip_count /= 2;
820 :
821 : /* If it is more profitable to optimize 1 / x, don't optimize 1 / (x * x). */
822 213262 : if (sqrt_recip_count > square_recip_count)
823 17 : goto out;
824 :
825 : /* Do the expensive part only if we can hope to optimize something. */
826 213245 : if (count + square_recip_count >= threshold && count >= 1)
827 : {
828 23 : gimple *use_stmt;
829 46 : for (occ = occ_head; occ; occ = occ->next)
830 : {
831 23 : compute_merit (occ);
832 23 : insert_reciprocals (def_gsi, occ, def, NULL, NULL,
833 : square_recip_count, threshold);
834 : }
835 :
836 143 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, def)
837 : {
838 120 : if (is_division_by (use_stmt, def))
839 : {
840 210 : FOR_EACH_IMM_USE_ON_STMT (use_p, use_iter)
841 105 : replace_reciprocal (use_p);
842 : }
843 20 : else if (square_recip_count > 0 && is_square_of (use_stmt, def))
844 : {
845 12 : FOR_EACH_IMM_USE_ON_STMT (use_p, use_iter)
846 : {
847 : /* Find all uses of the square that are divisions and
848 : * replace them by multiplications with the inverse. */
849 8 : imm_use_iterator square_iterator;
850 8 : gimple *powmult_use_stmt = USE_STMT (use_p);
851 8 : tree powmult_def_name = gimple_assign_lhs (powmult_use_stmt);
852 :
853 16 : FOR_EACH_IMM_USE_STMT (powmult_use_stmt,
854 : square_iterator, powmult_def_name)
855 16 : FOR_EACH_IMM_USE_ON_STMT (square_use_p, square_iterator)
856 : {
857 8 : gimple *powmult_use_stmt = USE_STMT (square_use_p);
858 8 : if (is_division_by (powmult_use_stmt, powmult_def_name))
859 4 : replace_reciprocal_squares (square_use_p);
860 8 : }
861 : }
862 : }
863 23 : }
864 : }
865 :
866 213222 : out:
867 213880 : for (occ = occ_head; occ; )
868 618 : occ = free_bb (occ);
869 :
870 213262 : occ_head = NULL;
871 213262 : }
872 :
873 : /* Return an internal function that implements the reciprocal of CALL,
874 : or IFN_LAST if there is no such function that the target supports. */
875 :
876 : internal_fn
877 113 : internal_fn_reciprocal (gcall *call)
878 : {
879 113 : internal_fn ifn;
880 :
881 113 : switch (gimple_call_combined_fn (call))
882 : {
883 97 : CASE_CFN_SQRT:
884 97 : CASE_CFN_SQRT_FN:
885 97 : ifn = IFN_RSQRT;
886 97 : break;
887 :
888 : default:
889 : return IFN_LAST;
890 : }
891 :
892 97 : tree_pair types = direct_internal_fn_types (ifn, call);
893 97 : if (!direct_internal_fn_supported_p (ifn, types, OPTIMIZE_FOR_SPEED))
894 44 : return IFN_LAST;
895 :
896 : return ifn;
897 : }
898 :
899 : /* Go through all the floating-point SSA_NAMEs, and call
900 : execute_cse_reciprocals_1 on each of them. */
901 : namespace {
902 :
903 : const pass_data pass_data_cse_reciprocals =
904 : {
905 : GIMPLE_PASS, /* type */
906 : "recip", /* name */
907 : OPTGROUP_NONE, /* optinfo_flags */
908 : TV_TREE_RECIP, /* tv_id */
909 : PROP_ssa, /* properties_required */
910 : 0, /* properties_provided */
911 : 0, /* properties_destroyed */
912 : 0, /* todo_flags_start */
913 : TODO_update_ssa, /* todo_flags_finish */
914 : };
915 :
916 : class pass_cse_reciprocals : public gimple_opt_pass
917 : {
918 : public:
919 294587 : pass_cse_reciprocals (gcc::context *ctxt)
920 589174 : : gimple_opt_pass (pass_data_cse_reciprocals, ctxt)
921 : {}
922 :
923 : /* opt_pass methods: */
924 1062413 : bool gate (function *) final override
925 : {
926 1062413 : return optimize && flag_reciprocal_math;
927 : }
928 : unsigned int execute (function *) final override;
929 :
930 : }; // class pass_cse_reciprocals
931 :
932 : unsigned int
933 8801 : pass_cse_reciprocals::execute (function *fun)
934 : {
935 8801 : basic_block bb;
936 8801 : tree arg;
937 :
938 8801 : occ_pool = new object_allocator<occurrence> ("dominators for recip");
939 :
940 8801 : memset (&reciprocal_stats, 0, sizeof (reciprocal_stats));
941 8801 : calculate_dominance_info (CDI_DOMINATORS);
942 8801 : calculate_dominance_info (CDI_POST_DOMINATORS);
943 :
944 8801 : if (flag_checking)
945 94425 : FOR_EACH_BB_FN (bb, fun)
946 85624 : gcc_assert (!bb->aux);
947 :
948 21826 : for (arg = DECL_ARGUMENTS (fun->decl); arg; arg = DECL_CHAIN (arg))
949 20685 : if (FLOAT_TYPE_P (TREE_TYPE (arg))
950 14112 : && is_gimple_reg (arg))
951 : {
952 6451 : tree name = ssa_default_def (fun, arg);
953 6451 : if (name)
954 5456 : execute_cse_reciprocals_1 (NULL, name);
955 : }
956 :
957 94425 : FOR_EACH_BB_FN (bb, fun)
958 : {
959 85624 : tree def;
960 :
961 196952 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
962 111328 : gsi_next (&gsi))
963 : {
964 111328 : gphi *phi = gsi.phi ();
965 111328 : def = PHI_RESULT (phi);
966 111328 : if (! virtual_operand_p (def)
967 111328 : && FLOAT_TYPE_P (TREE_TYPE (def)))
968 30805 : execute_cse_reciprocals_1 (NULL, def);
969 : }
970 :
971 1374809 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
972 1289185 : gsi_next (&gsi))
973 : {
974 1289185 : gimple *stmt = gsi_stmt (gsi);
975 :
976 2578370 : if (gimple_has_lhs (stmt)
977 809617 : && (def = SINGLE_SSA_TREE_OPERAND (stmt, SSA_OP_DEF)) != NULL
978 769382 : && FLOAT_TYPE_P (TREE_TYPE (def))
979 199920 : && TREE_CODE (def) == SSA_NAME)
980 : {
981 177001 : execute_cse_reciprocals_1 (&gsi, def);
982 177001 : stmt = gsi_stmt (gsi);
983 177001 : if (flag_unsafe_math_optimizations
984 176956 : && is_gimple_assign (stmt)
985 166741 : && gimple_assign_lhs (stmt) == def
986 166739 : && !stmt_can_throw_internal (cfun, stmt)
987 343696 : && gimple_assign_rhs_code (stmt) == RDIV_EXPR)
988 556 : optimize_recip_sqrt (&gsi, def);
989 : }
990 : }
991 :
992 85624 : if (optimize_bb_for_size_p (bb))
993 5361 : continue;
994 :
995 : /* Scan for a/func(b) and convert it to reciprocal a*rfunc(b). */
996 1347565 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
997 1267302 : gsi_next (&gsi))
998 : {
999 1267302 : gimple *stmt = gsi_stmt (gsi);
1000 :
1001 1267302 : if (is_gimple_assign (stmt)
1002 1267302 : && gimple_assign_rhs_code (stmt) == RDIV_EXPR)
1003 : {
1004 598 : tree arg1 = gimple_assign_rhs2 (stmt);
1005 598 : gimple *stmt1;
1006 :
1007 598 : if (TREE_CODE (arg1) != SSA_NAME)
1008 5 : continue;
1009 :
1010 593 : stmt1 = SSA_NAME_DEF_STMT (arg1);
1011 :
1012 593 : if (is_gimple_call (stmt1)
1013 593 : && gimple_call_lhs (stmt1))
1014 : {
1015 113 : bool fail;
1016 113 : imm_use_iterator ui;
1017 113 : use_operand_p use_p;
1018 113 : tree fndecl = NULL_TREE;
1019 :
1020 113 : gcall *call = as_a <gcall *> (stmt1);
1021 113 : internal_fn ifn = internal_fn_reciprocal (call);
1022 113 : if (ifn == IFN_LAST)
1023 : {
1024 60 : fndecl = gimple_call_fndecl (call);
1025 120 : if (!fndecl
1026 60 : || !fndecl_built_in_p (fndecl, BUILT_IN_MD))
1027 62 : continue;
1028 0 : fndecl = targetm.builtin_reciprocal (fndecl);
1029 0 : if (!fndecl)
1030 0 : continue;
1031 : }
1032 :
1033 : /* Check that all uses of the SSA name are divisions,
1034 : otherwise replacing the defining statement will do
1035 : the wrong thing. */
1036 53 : fail = false;
1037 106 : FOR_EACH_IMM_USE_FAST (use_p, ui, arg1)
1038 : {
1039 55 : gimple *stmt2 = USE_STMT (use_p);
1040 55 : if (is_gimple_debug (stmt2))
1041 0 : continue;
1042 55 : if (!is_gimple_assign (stmt2)
1043 55 : || gimple_assign_rhs_code (stmt2) != RDIV_EXPR
1044 53 : || gimple_assign_rhs1 (stmt2) == arg1
1045 108 : || gimple_assign_rhs2 (stmt2) != arg1)
1046 : {
1047 : fail = true;
1048 : break;
1049 : }
1050 53 : }
1051 53 : if (fail)
1052 2 : continue;
1053 :
1054 51 : gimple_replace_ssa_lhs (call, arg1);
1055 51 : reset_flow_sensitive_info (arg1);
1056 51 : if (gimple_call_internal_p (call) != (ifn != IFN_LAST))
1057 : {
1058 30 : auto_vec<tree, 4> args;
1059 30 : for (unsigned int i = 0;
1060 60 : i < gimple_call_num_args (call); i++)
1061 30 : args.safe_push (gimple_call_arg (call, i));
1062 30 : gcall *stmt2;
1063 30 : if (ifn == IFN_LAST)
1064 0 : stmt2 = gimple_build_call_vec (fndecl, args);
1065 : else
1066 30 : stmt2 = gimple_build_call_internal_vec (ifn, args);
1067 30 : gimple_call_set_lhs (stmt2, arg1);
1068 30 : gimple_move_vops (stmt2, call);
1069 30 : gimple_call_set_nothrow (stmt2,
1070 : gimple_call_nothrow_p (call));
1071 30 : gimple_stmt_iterator gsi2 = gsi_for_stmt (call);
1072 30 : gsi_replace (&gsi2, stmt2, true);
1073 30 : }
1074 : else
1075 : {
1076 21 : if (ifn == IFN_LAST)
1077 0 : gimple_call_set_fndecl (call, fndecl);
1078 : else
1079 21 : gimple_call_set_internal_fn (call, ifn);
1080 21 : update_stmt (call);
1081 : }
1082 51 : reciprocal_stats.rfuncs_inserted++;
1083 :
1084 102 : FOR_EACH_IMM_USE_STMT (stmt, ui, arg1)
1085 : {
1086 51 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1087 51 : gimple_assign_set_rhs_code (stmt, MULT_EXPR);
1088 51 : fold_stmt_inplace (&gsi);
1089 51 : update_stmt (stmt);
1090 51 : }
1091 : }
1092 : }
1093 : }
1094 : }
1095 :
1096 8801 : statistics_counter_event (fun, "reciprocal divs inserted",
1097 : reciprocal_stats.rdivs_inserted);
1098 8801 : statistics_counter_event (fun, "reciprocal functions inserted",
1099 : reciprocal_stats.rfuncs_inserted);
1100 :
1101 8801 : free_dominance_info (CDI_DOMINATORS);
1102 8801 : free_dominance_info (CDI_POST_DOMINATORS);
1103 17602 : delete occ_pool;
1104 8801 : return 0;
1105 : }
1106 :
1107 : } // anon namespace
1108 :
1109 : gimple_opt_pass *
1110 294587 : make_pass_cse_reciprocals (gcc::context *ctxt)
1111 : {
1112 294587 : return new pass_cse_reciprocals (ctxt);
1113 : }
1114 :
1115 : /* If NAME is the result of a type conversion, look for other
1116 : equivalent dominating or dominated conversions, and replace all
1117 : uses with the earliest dominating name, removing the redundant
1118 : conversions. Return the prevailing name. */
1119 :
1120 : static tree
1121 1069 : execute_cse_conv_1 (tree name, bool *cfg_changed)
1122 : {
1123 1069 : if (SSA_NAME_IS_DEFAULT_DEF (name)
1124 1069 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
1125 : return name;
1126 :
1127 969 : gimple *def_stmt = SSA_NAME_DEF_STMT (name);
1128 :
1129 969 : if (!gimple_assign_cast_p (def_stmt))
1130 : return name;
1131 :
1132 136 : tree src = gimple_assign_rhs1 (def_stmt);
1133 :
1134 136 : if (TREE_CODE (src) != SSA_NAME)
1135 : return name;
1136 :
1137 136 : imm_use_iterator use_iter;
1138 136 : gimple *use_stmt;
1139 :
1140 : /* Find the earliest dominating def. */
1141 521 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, src)
1142 : {
1143 763 : if (use_stmt == def_stmt
1144 385 : || !gimple_assign_cast_p (use_stmt))
1145 378 : continue;
1146 :
1147 7 : tree lhs = gimple_assign_lhs (use_stmt);
1148 :
1149 7 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
1150 14 : || (gimple_assign_rhs1 (use_stmt)
1151 7 : != gimple_assign_rhs1 (def_stmt))
1152 14 : || !types_compatible_p (TREE_TYPE (name), TREE_TYPE (lhs)))
1153 0 : continue;
1154 :
1155 7 : bool use_dominates;
1156 7 : if (gimple_bb (def_stmt) == gimple_bb (use_stmt))
1157 : {
1158 0 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
1159 0 : while (!gsi_end_p (gsi) && gsi_stmt (gsi) != def_stmt)
1160 0 : gsi_next (&gsi);
1161 0 : use_dominates = !gsi_end_p (gsi);
1162 : }
1163 7 : else if (dominated_by_p (CDI_DOMINATORS, gimple_bb (use_stmt),
1164 7 : gimple_bb (def_stmt)))
1165 : use_dominates = false;
1166 7 : else if (dominated_by_p (CDI_DOMINATORS, gimple_bb (def_stmt),
1167 7 : gimple_bb (use_stmt)))
1168 : use_dominates = true;
1169 : else
1170 4 : continue;
1171 :
1172 0 : if (use_dominates)
1173 : {
1174 : std::swap (name, lhs);
1175 : std::swap (def_stmt, use_stmt);
1176 : }
1177 136 : }
1178 :
1179 : /* Now go through all uses of SRC again, replacing the equivalent
1180 : dominated conversions. We may replace defs that were not
1181 : dominated by the then-prevailing defs when we first visited
1182 : them. */
1183 521 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, src)
1184 : {
1185 763 : if (use_stmt == def_stmt
1186 385 : || !gimple_assign_cast_p (use_stmt))
1187 378 : continue;
1188 :
1189 7 : tree lhs = gimple_assign_lhs (use_stmt);
1190 :
1191 7 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
1192 14 : || (gimple_assign_rhs1 (use_stmt)
1193 7 : != gimple_assign_rhs1 (def_stmt))
1194 14 : || !types_compatible_p (TREE_TYPE (name), TREE_TYPE (lhs)))
1195 0 : continue;
1196 :
1197 7 : basic_block use_bb = gimple_bb (use_stmt);
1198 7 : if (gimple_bb (def_stmt) == use_bb
1199 7 : || dominated_by_p (CDI_DOMINATORS, use_bb, gimple_bb (def_stmt)))
1200 : {
1201 3 : sincos_stats.conv_removed++;
1202 :
1203 3 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
1204 3 : replace_uses_by (lhs, name);
1205 3 : if (gsi_remove (&gsi, true)
1206 3 : && gimple_purge_dead_eh_edges (use_bb))
1207 3 : *cfg_changed = true;
1208 3 : release_defs (use_stmt);
1209 : }
1210 136 : }
1211 :
1212 136 : return name;
1213 : }
1214 :
1215 : /* Records an occurrence at statement USE_STMT in the vector of trees
1216 : STMTS if it is dominated by *TOP_BB or dominates it or this basic block
1217 : is not yet initialized. Returns true if the occurrence was pushed on
1218 : the vector. Adjusts *TOP_BB to be the basic block dominating all
1219 : statements in the vector. */
1220 :
1221 : static bool
1222 1290 : maybe_record_sincos (vec<gimple *> *stmts,
1223 : basic_block *top_bb, gimple *use_stmt)
1224 : {
1225 1290 : basic_block use_bb = gimple_bb (use_stmt);
1226 1290 : if (*top_bb
1227 1290 : && (*top_bb == use_bb
1228 66 : || dominated_by_p (CDI_DOMINATORS, use_bb, *top_bb)))
1229 168 : stmts->safe_push (use_stmt);
1230 1122 : else if (!*top_bb
1231 1122 : || dominated_by_p (CDI_DOMINATORS, *top_bb, use_bb))
1232 : {
1233 1102 : stmts->safe_push (use_stmt);
1234 1102 : *top_bb = use_bb;
1235 : }
1236 : else
1237 : return false;
1238 :
1239 : return true;
1240 : }
1241 :
1242 : /* Look for sin, cos and cexpi calls with the same argument NAME and
1243 : create a single call to cexpi CSEing the result in this case.
1244 : We first walk over all immediate uses of the argument collecting
1245 : statements that we can CSE in a vector and in a second pass replace
1246 : the statement rhs with a REALPART or IMAGPART expression on the
1247 : result of the cexpi call we insert before the use statement that
1248 : dominates all other candidates. */
1249 :
1250 : static bool
1251 1069 : execute_cse_sincos_1 (tree name)
1252 : {
1253 1069 : gimple_stmt_iterator gsi;
1254 1069 : imm_use_iterator use_iter;
1255 1069 : tree fndecl, res, type = NULL_TREE;
1256 1069 : gimple *def_stmt, *use_stmt, *stmt;
1257 1069 : int seen_cos = 0, seen_sin = 0, seen_cexpi = 0;
1258 1069 : auto_vec<gimple *> stmts;
1259 1069 : basic_block top_bb = NULL;
1260 1069 : int i;
1261 1069 : bool cfg_changed = false;
1262 :
1263 1069 : name = execute_cse_conv_1 (name, &cfg_changed);
1264 :
1265 4212 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, name)
1266 : {
1267 3145 : if (gimple_code (use_stmt) != GIMPLE_CALL
1268 3145 : || !gimple_call_lhs (use_stmt))
1269 1823 : continue;
1270 :
1271 1322 : switch (gimple_call_combined_fn (use_stmt))
1272 : {
1273 462 : CASE_CFN_COS:
1274 462 : seen_cos |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
1275 462 : break;
1276 :
1277 822 : CASE_CFN_SIN:
1278 822 : seen_sin |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
1279 822 : break;
1280 :
1281 6 : CASE_CFN_CEXPI:
1282 6 : seen_cexpi |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
1283 6 : break;
1284 :
1285 32 : default:;
1286 32 : continue;
1287 : }
1288 :
1289 1290 : auto stmt_cfn = gimple_call_combined_fn (use_stmt);
1290 1290 : tree t = mathfn_built_in_type (stmt_cfn);
1291 1290 : if (!t)
1292 : {
1293 : /* It is possible to get IFN_{SIN,COS} calls, for which
1294 : mathfn_built_in_type will return NULL. Those are normally only
1295 : present for vector operations. We won't be able to CSE those
1296 : at the moment. */
1297 2 : gcc_checking_assert (internal_fn_p (stmt_cfn));
1298 : return false;
1299 : }
1300 :
1301 1288 : if (!type)
1302 : {
1303 1067 : type = t;
1304 1067 : t = TREE_TYPE (name);
1305 : }
1306 : /* This checks that NAME has the right type in the first round,
1307 : and, in subsequent rounds, that the built_in type is the same
1308 : type, or a compatible type. */
1309 1288 : if (type != t && !types_compatible_p (type, t))
1310 : return false;
1311 2 : }
1312 1067 : if (seen_cos + seen_sin + seen_cexpi <= 1)
1313 : return false;
1314 :
1315 : /* Simply insert cexpi at the beginning of top_bb but not earlier than
1316 : the name def statement. */
1317 201 : fndecl = mathfn_built_in (type, BUILT_IN_CEXPI);
1318 201 : if (!fndecl)
1319 : return false;
1320 157 : stmt = gimple_build_call (fndecl, 1, name);
1321 157 : res = make_temp_ssa_name (TREE_TYPE (TREE_TYPE (fndecl)), stmt, "sincostmp");
1322 157 : gimple_call_set_lhs (stmt, res);
1323 :
1324 157 : def_stmt = SSA_NAME_DEF_STMT (name);
1325 157 : if (!SSA_NAME_IS_DEFAULT_DEF (name)
1326 127 : && gimple_code (def_stmt) != GIMPLE_PHI
1327 276 : && gimple_bb (def_stmt) == top_bb)
1328 : {
1329 119 : gsi = gsi_for_stmt (def_stmt);
1330 119 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
1331 : }
1332 : else
1333 : {
1334 38 : gsi = gsi_after_labels (top_bb);
1335 38 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1336 : }
1337 157 : sincos_stats.inserted++;
1338 :
1339 : /* And adjust the recorded old call sites. */
1340 471 : for (i = 0; stmts.iterate (i, &use_stmt); ++i)
1341 : {
1342 314 : tree rhs = NULL;
1343 :
1344 314 : switch (gimple_call_combined_fn (use_stmt))
1345 : {
1346 157 : CASE_CFN_COS:
1347 157 : rhs = fold_build1 (REALPART_EXPR, type, res);
1348 157 : break;
1349 :
1350 157 : CASE_CFN_SIN:
1351 157 : rhs = fold_build1 (IMAGPART_EXPR, type, res);
1352 157 : break;
1353 :
1354 : CASE_CFN_CEXPI:
1355 : rhs = res;
1356 : break;
1357 :
1358 0 : default:;
1359 0 : gcc_unreachable ();
1360 : }
1361 :
1362 : /* Replace call with a copy. */
1363 314 : stmt = gimple_build_assign (gimple_call_lhs (use_stmt), rhs);
1364 :
1365 314 : gsi = gsi_for_stmt (use_stmt);
1366 314 : gsi_replace (&gsi, stmt, true);
1367 314 : if (gimple_purge_dead_eh_edges (gimple_bb (stmt)))
1368 0 : cfg_changed = true;
1369 : }
1370 :
1371 157 : return cfg_changed;
1372 1069 : }
1373 :
1374 : /* To evaluate powi(x,n), the floating point value x raised to the
1375 : constant integer exponent n, we use a hybrid algorithm that
1376 : combines the "window method" with look-up tables. For an
1377 : introduction to exponentiation algorithms and "addition chains",
1378 : see section 4.6.3, "Evaluation of Powers" of Donald E. Knuth,
1379 : "Seminumerical Algorithms", Vol. 2, "The Art of Computer Programming",
1380 : 3rd Edition, 1998, and Daniel M. Gordon, "A Survey of Fast Exponentiation
1381 : Methods", Journal of Algorithms, Vol. 27, pp. 129-146, 1998. */
1382 :
1383 : /* Provide a default value for POWI_MAX_MULTS, the maximum number of
1384 : multiplications to inline before calling the system library's pow
1385 : function. powi(x,n) requires at worst 2*bits(n)-2 multiplications,
1386 : so this default never requires calling pow, powf or powl. */
1387 :
1388 : #ifndef POWI_MAX_MULTS
1389 : #define POWI_MAX_MULTS (2*HOST_BITS_PER_WIDE_INT-2)
1390 : #endif
1391 :
1392 : /* The size of the "optimal power tree" lookup table. All
1393 : exponents less than this value are simply looked up in the
1394 : powi_table below. This threshold is also used to size the
1395 : cache of pseudo registers that hold intermediate results. */
1396 : #define POWI_TABLE_SIZE 256
1397 :
1398 : /* The size, in bits of the window, used in the "window method"
1399 : exponentiation algorithm. This is equivalent to a radix of
1400 : (1<<POWI_WINDOW_SIZE) in the corresponding "m-ary method". */
1401 : #define POWI_WINDOW_SIZE 3
1402 :
1403 : /* The following table is an efficient representation of an
1404 : "optimal power tree". For each value, i, the corresponding
1405 : value, j, in the table states than an optimal evaluation
1406 : sequence for calculating pow(x,i) can be found by evaluating
1407 : pow(x,j)*pow(x,i-j). An optimal power tree for the first
1408 : 100 integers is given in Knuth's "Seminumerical algorithms". */
1409 :
1410 : static const unsigned char powi_table[POWI_TABLE_SIZE] =
1411 : {
1412 : 0, 1, 1, 2, 2, 3, 3, 4, /* 0 - 7 */
1413 : 4, 6, 5, 6, 6, 10, 7, 9, /* 8 - 15 */
1414 : 8, 16, 9, 16, 10, 12, 11, 13, /* 16 - 23 */
1415 : 12, 17, 13, 18, 14, 24, 15, 26, /* 24 - 31 */
1416 : 16, 17, 17, 19, 18, 33, 19, 26, /* 32 - 39 */
1417 : 20, 25, 21, 40, 22, 27, 23, 44, /* 40 - 47 */
1418 : 24, 32, 25, 34, 26, 29, 27, 44, /* 48 - 55 */
1419 : 28, 31, 29, 34, 30, 60, 31, 36, /* 56 - 63 */
1420 : 32, 64, 33, 34, 34, 46, 35, 37, /* 64 - 71 */
1421 : 36, 65, 37, 50, 38, 48, 39, 69, /* 72 - 79 */
1422 : 40, 49, 41, 43, 42, 51, 43, 58, /* 80 - 87 */
1423 : 44, 64, 45, 47, 46, 59, 47, 76, /* 88 - 95 */
1424 : 48, 65, 49, 66, 50, 67, 51, 66, /* 96 - 103 */
1425 : 52, 70, 53, 74, 54, 104, 55, 74, /* 104 - 111 */
1426 : 56, 64, 57, 69, 58, 78, 59, 68, /* 112 - 119 */
1427 : 60, 61, 61, 80, 62, 75, 63, 68, /* 120 - 127 */
1428 : 64, 65, 65, 128, 66, 129, 67, 90, /* 128 - 135 */
1429 : 68, 73, 69, 131, 70, 94, 71, 88, /* 136 - 143 */
1430 : 72, 128, 73, 98, 74, 132, 75, 121, /* 144 - 151 */
1431 : 76, 102, 77, 124, 78, 132, 79, 106, /* 152 - 159 */
1432 : 80, 97, 81, 160, 82, 99, 83, 134, /* 160 - 167 */
1433 : 84, 86, 85, 95, 86, 160, 87, 100, /* 168 - 175 */
1434 : 88, 113, 89, 98, 90, 107, 91, 122, /* 176 - 183 */
1435 : 92, 111, 93, 102, 94, 126, 95, 150, /* 184 - 191 */
1436 : 96, 128, 97, 130, 98, 133, 99, 195, /* 192 - 199 */
1437 : 100, 128, 101, 123, 102, 164, 103, 138, /* 200 - 207 */
1438 : 104, 145, 105, 146, 106, 109, 107, 149, /* 208 - 215 */
1439 : 108, 200, 109, 146, 110, 170, 111, 157, /* 216 - 223 */
1440 : 112, 128, 113, 130, 114, 182, 115, 132, /* 224 - 231 */
1441 : 116, 200, 117, 132, 118, 158, 119, 206, /* 232 - 239 */
1442 : 120, 240, 121, 162, 122, 147, 123, 152, /* 240 - 247 */
1443 : 124, 166, 125, 214, 126, 138, 127, 153, /* 248 - 255 */
1444 : };
1445 :
1446 :
1447 : /* Return the number of multiplications required to calculate
1448 : powi(x,n) where n is less than POWI_TABLE_SIZE. This is a
1449 : subroutine of powi_cost. CACHE is an array indicating
1450 : which exponents have already been calculated. */
1451 :
1452 : static int
1453 1120 : powi_lookup_cost (unsigned HOST_WIDE_INT n, bool *cache)
1454 : {
1455 : /* If we've already calculated this exponent, then this evaluation
1456 : doesn't require any additional multiplications. */
1457 1861 : if (cache[n])
1458 1120 : return 0;
1459 :
1460 741 : cache[n] = true;
1461 741 : return powi_lookup_cost (n - powi_table[n], cache)
1462 741 : + powi_lookup_cost (powi_table[n], cache) + 1;
1463 : }
1464 :
1465 : /* Return the number of multiplications required to calculate
1466 : powi(x,n) for an arbitrary x, given the exponent N. This
1467 : function needs to be kept in sync with powi_as_mults below. */
1468 :
1469 : static int
1470 384 : powi_cost (HOST_WIDE_INT n)
1471 : {
1472 384 : bool cache[POWI_TABLE_SIZE];
1473 384 : unsigned HOST_WIDE_INT digit;
1474 384 : unsigned HOST_WIDE_INT val;
1475 384 : int result;
1476 :
1477 384 : if (n == 0)
1478 : return 0;
1479 :
1480 : /* Ignore the reciprocal when calculating the cost. */
1481 379 : val = absu_hwi (n);
1482 :
1483 : /* Initialize the exponent cache. */
1484 379 : memset (cache, 0, POWI_TABLE_SIZE * sizeof (bool));
1485 379 : cache[1] = true;
1486 :
1487 379 : result = 0;
1488 :
1489 379 : while (val >= POWI_TABLE_SIZE)
1490 : {
1491 0 : if (val & 1)
1492 : {
1493 0 : digit = val & ((1 << POWI_WINDOW_SIZE) - 1);
1494 0 : result += powi_lookup_cost (digit, cache)
1495 0 : + POWI_WINDOW_SIZE + 1;
1496 0 : val >>= POWI_WINDOW_SIZE;
1497 : }
1498 : else
1499 : {
1500 0 : val >>= 1;
1501 0 : result++;
1502 : }
1503 : }
1504 :
1505 379 : return result + powi_lookup_cost (val, cache);
1506 : }
1507 :
1508 : /* Recursive subroutine of powi_as_mults. This function takes the
1509 : array, CACHE, of already calculated exponents and an exponent N and
1510 : returns a tree that corresponds to CACHE[1]**N, with type TYPE. */
1511 :
1512 : static tree
1513 6242 : powi_as_mults_1 (gimple_stmt_iterator *gsi, location_t loc, tree type,
1514 : unsigned HOST_WIDE_INT n, tree *cache)
1515 : {
1516 6242 : tree op0, op1, ssa_target;
1517 6242 : unsigned HOST_WIDE_INT digit;
1518 6242 : gassign *mult_stmt;
1519 :
1520 6242 : if (n < POWI_TABLE_SIZE && cache[n])
1521 : return cache[n];
1522 :
1523 2209 : ssa_target = make_temp_ssa_name (type, NULL, "powmult");
1524 :
1525 2209 : if (n < POWI_TABLE_SIZE)
1526 : {
1527 2206 : cache[n] = ssa_target;
1528 2206 : op0 = powi_as_mults_1 (gsi, loc, type, n - powi_table[n], cache);
1529 2206 : op1 = powi_as_mults_1 (gsi, loc, type, powi_table[n], cache);
1530 : }
1531 3 : else if (n & 1)
1532 : {
1533 1 : digit = n & ((1 << POWI_WINDOW_SIZE) - 1);
1534 1 : op0 = powi_as_mults_1 (gsi, loc, type, n - digit, cache);
1535 1 : op1 = powi_as_mults_1 (gsi, loc, type, digit, cache);
1536 : }
1537 : else
1538 : {
1539 2 : op0 = powi_as_mults_1 (gsi, loc, type, n >> 1, cache);
1540 2 : op1 = op0;
1541 : }
1542 :
1543 2209 : mult_stmt = gimple_build_assign (ssa_target, MULT_EXPR, op0, op1);
1544 2209 : gimple_set_location (mult_stmt, loc);
1545 2209 : gsi_insert_before (gsi, mult_stmt, GSI_SAME_STMT);
1546 :
1547 2209 : return ssa_target;
1548 : }
1549 :
1550 : /* Convert ARG0**N to a tree of multiplications of ARG0 with itself.
1551 : This function needs to be kept in sync with powi_cost above. */
1552 :
1553 : tree
1554 1826 : powi_as_mults (gimple_stmt_iterator *gsi, location_t loc,
1555 : tree arg0, HOST_WIDE_INT n)
1556 : {
1557 1826 : tree cache[POWI_TABLE_SIZE], result, type = TREE_TYPE (arg0);
1558 1826 : gassign *div_stmt;
1559 1826 : tree target;
1560 :
1561 1826 : if (n == 0)
1562 0 : return build_one_cst (type);
1563 :
1564 1826 : memset (cache, 0, sizeof (cache));
1565 1826 : cache[1] = arg0;
1566 :
1567 1826 : result = powi_as_mults_1 (gsi, loc, type, absu_hwi (n), cache);
1568 1826 : if (n >= 0)
1569 : return result;
1570 :
1571 : /* If the original exponent was negative, reciprocate the result. */
1572 8 : target = make_temp_ssa_name (type, NULL, "powmult");
1573 8 : div_stmt = gimple_build_assign (target, RDIV_EXPR,
1574 : build_real (type, dconst1), result);
1575 8 : gimple_set_location (div_stmt, loc);
1576 8 : gsi_insert_before (gsi, div_stmt, GSI_SAME_STMT);
1577 :
1578 8 : return target;
1579 : }
1580 :
1581 : /* ARG0 and N are the two arguments to a powi builtin in GSI with
1582 : location info LOC. If the arguments are appropriate, create an
1583 : equivalent sequence of statements prior to GSI using an optimal
1584 : number of multiplications, and return an expression holding the
1585 : result. */
1586 :
1587 : static tree
1588 633 : gimple_expand_builtin_powi (gimple_stmt_iterator *gsi, location_t loc,
1589 : tree arg0, HOST_WIDE_INT n)
1590 : {
1591 633 : if ((n >= -1 && n <= 2)
1592 633 : || (optimize_function_for_speed_p (cfun)
1593 351 : && powi_cost (n) <= POWI_MAX_MULTS))
1594 625 : return powi_as_mults (gsi, loc, arg0, n);
1595 :
1596 : return NULL_TREE;
1597 : }
1598 :
1599 : /* Build a gimple call statement that calls FN with argument ARG.
1600 : Set the lhs of the call statement to a fresh SSA name. Insert the
1601 : statement prior to GSI's current position, and return the fresh
1602 : SSA name. */
1603 :
1604 : static tree
1605 44 : build_and_insert_call (gimple_stmt_iterator *gsi, location_t loc,
1606 : tree fn, tree arg)
1607 : {
1608 44 : gcall *call_stmt;
1609 44 : tree ssa_target;
1610 :
1611 44 : call_stmt = gimple_build_call (fn, 1, arg);
1612 44 : ssa_target = make_temp_ssa_name (TREE_TYPE (arg), NULL, "powroot");
1613 44 : gimple_set_lhs (call_stmt, ssa_target);
1614 44 : gimple_set_location (call_stmt, loc);
1615 44 : gsi_insert_before (gsi, call_stmt, GSI_SAME_STMT);
1616 :
1617 44 : return ssa_target;
1618 : }
1619 :
1620 : /* Build a gimple binary operation with the given CODE and arguments
1621 : ARG0, ARG1, assigning the result to a new SSA name for variable
1622 : TARGET. Insert the statement prior to GSI's current position, and
1623 : return the fresh SSA name.*/
1624 :
1625 : static tree
1626 2643 : build_and_insert_binop (gimple_stmt_iterator *gsi, location_t loc,
1627 : const char *name, enum tree_code code,
1628 : tree arg0, tree arg1)
1629 : {
1630 2643 : tree result = make_temp_ssa_name (TREE_TYPE (arg0), NULL, name);
1631 2643 : gassign *stmt = gimple_build_assign (result, code, arg0, arg1);
1632 2643 : gimple_set_location (stmt, loc);
1633 2643 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1634 2643 : return result;
1635 : }
1636 :
1637 : /* Build a gimple assignment to cast VAL to TYPE. Insert the statement
1638 : prior to GSI's current position, and return the fresh SSA name. */
1639 :
1640 : static tree
1641 14571 : build_and_insert_cast (gimple_stmt_iterator *gsi, location_t loc,
1642 : tree type, tree val)
1643 : {
1644 0 : return gimple_convert (gsi, true, GSI_SAME_STMT, loc, type, val);
1645 : }
1646 :
1647 : struct pow_synth_sqrt_info
1648 : {
1649 : bool *factors;
1650 : unsigned int deepest;
1651 : unsigned int num_mults;
1652 : };
1653 :
1654 : /* Return true iff the real value C can be represented as a
1655 : sum of powers of 0.5 up to N. That is:
1656 : C == SUM<i from 1..N> (a[i]*(0.5**i)) where a[i] is either 0 or 1.
1657 : Record in INFO the various parameters of the synthesis algorithm such
1658 : as the factors a[i], the maximum 0.5 power and the number of
1659 : multiplications that will be required. */
1660 :
1661 : bool
1662 33 : representable_as_half_series_p (REAL_VALUE_TYPE c, unsigned n,
1663 : struct pow_synth_sqrt_info *info)
1664 : {
1665 33 : REAL_VALUE_TYPE factor = dconsthalf;
1666 33 : REAL_VALUE_TYPE remainder = c;
1667 :
1668 33 : info->deepest = 0;
1669 33 : info->num_mults = 0;
1670 33 : memset (info->factors, 0, n * sizeof (bool));
1671 :
1672 97 : for (unsigned i = 0; i < n; i++)
1673 : {
1674 90 : REAL_VALUE_TYPE res;
1675 :
1676 : /* If something inexact happened bail out now. */
1677 90 : if (real_arithmetic (&res, MINUS_EXPR, &remainder, &factor))
1678 26 : return false;
1679 :
1680 : /* We have hit zero. The number is representable as a sum
1681 : of powers of 0.5. */
1682 90 : if (real_equal (&res, &dconst0))
1683 : {
1684 26 : info->factors[i] = true;
1685 26 : info->deepest = i + 1;
1686 26 : return true;
1687 : }
1688 64 : else if (!REAL_VALUE_NEGATIVE (res))
1689 : {
1690 29 : remainder = res;
1691 29 : info->factors[i] = true;
1692 29 : info->num_mults++;
1693 : }
1694 : else
1695 35 : info->factors[i] = false;
1696 :
1697 64 : real_arithmetic (&factor, MULT_EXPR, &factor, &dconsthalf);
1698 : }
1699 : return false;
1700 : }
1701 :
1702 : /* Return the tree corresponding to FN being applied
1703 : to ARG N times at GSI and LOC.
1704 : Look up previous results from CACHE if need be.
1705 : cache[0] should contain just plain ARG i.e. FN applied to ARG 0 times. */
1706 :
1707 : static tree
1708 63 : get_fn_chain (tree arg, unsigned int n, gimple_stmt_iterator *gsi,
1709 : tree fn, location_t loc, tree *cache)
1710 : {
1711 63 : tree res = cache[n];
1712 63 : if (!res)
1713 : {
1714 40 : tree prev = get_fn_chain (arg, n - 1, gsi, fn, loc, cache);
1715 40 : res = build_and_insert_call (gsi, loc, fn, prev);
1716 40 : cache[n] = res;
1717 : }
1718 :
1719 63 : return res;
1720 : }
1721 :
1722 : /* Print to STREAM the repeated application of function FNAME to ARG
1723 : N times. So, for FNAME = "foo", ARG = "x", N = 2 it would print:
1724 : "foo (foo (x))". */
1725 :
1726 : static void
1727 36 : print_nested_fn (FILE* stream, const char *fname, const char* arg,
1728 : unsigned int n)
1729 : {
1730 36 : if (n == 0)
1731 10 : fprintf (stream, "%s", arg);
1732 : else
1733 : {
1734 26 : fprintf (stream, "%s (", fname);
1735 26 : print_nested_fn (stream, fname, arg, n - 1);
1736 26 : fprintf (stream, ")");
1737 : }
1738 36 : }
1739 :
1740 : /* Print to STREAM the fractional sequence of sqrt chains
1741 : applied to ARG, described by INFO. Used for the dump file. */
1742 :
1743 : static void
1744 7 : dump_fractional_sqrt_sequence (FILE *stream, const char *arg,
1745 : struct pow_synth_sqrt_info *info)
1746 : {
1747 29 : for (unsigned int i = 0; i < info->deepest; i++)
1748 : {
1749 22 : bool is_set = info->factors[i];
1750 22 : if (is_set)
1751 : {
1752 10 : print_nested_fn (stream, "sqrt", arg, i + 1);
1753 10 : if (i != info->deepest - 1)
1754 3 : fprintf (stream, " * ");
1755 : }
1756 : }
1757 7 : }
1758 :
1759 : /* Print to STREAM a representation of raising ARG to an integer
1760 : power N. Used for the dump file. */
1761 :
1762 : static void
1763 7 : dump_integer_part (FILE *stream, const char* arg, HOST_WIDE_INT n)
1764 : {
1765 7 : if (n > 1)
1766 3 : fprintf (stream, "powi (%s, " HOST_WIDE_INT_PRINT_DEC ")", arg, n);
1767 4 : else if (n == 1)
1768 3 : fprintf (stream, "%s", arg);
1769 7 : }
1770 :
1771 : /* Attempt to synthesize a POW[F] (ARG0, ARG1) call using chains of
1772 : square roots. Place at GSI and LOC. Limit the maximum depth
1773 : of the sqrt chains to MAX_DEPTH. Return the tree holding the
1774 : result of the expanded sequence or NULL_TREE if the expansion failed.
1775 :
1776 : This routine assumes that ARG1 is a real number with a fractional part
1777 : (the integer exponent case will have been handled earlier in
1778 : gimple_expand_builtin_pow).
1779 :
1780 : For ARG1 > 0.0:
1781 : * For ARG1 composed of a whole part WHOLE_PART and a fractional part
1782 : FRAC_PART i.e. WHOLE_PART == floor (ARG1) and
1783 : FRAC_PART == ARG1 - WHOLE_PART:
1784 : Produce POWI (ARG0, WHOLE_PART) * POW (ARG0, FRAC_PART) where
1785 : POW (ARG0, FRAC_PART) is expanded as a product of square root chains
1786 : if it can be expressed as such, that is if FRAC_PART satisfies:
1787 : FRAC_PART == <SUM from i = 1 until MAX_DEPTH> (a[i] * (0.5**i))
1788 : where integer a[i] is either 0 or 1.
1789 :
1790 : Example:
1791 : POW (x, 3.625) == POWI (x, 3) * POW (x, 0.625)
1792 : --> POWI (x, 3) * SQRT (x) * SQRT (SQRT (SQRT (x)))
1793 :
1794 : For ARG1 < 0.0 there are two approaches:
1795 : * (A) Expand to 1.0 / POW (ARG0, -ARG1) where POW (ARG0, -ARG1)
1796 : is calculated as above.
1797 :
1798 : Example:
1799 : POW (x, -5.625) == 1.0 / POW (x, 5.625)
1800 : --> 1.0 / (POWI (x, 5) * SQRT (x) * SQRT (SQRT (SQRT (x))))
1801 :
1802 : * (B) : WHOLE_PART := - ceil (abs (ARG1))
1803 : FRAC_PART := ARG1 - WHOLE_PART
1804 : and expand to POW (x, FRAC_PART) / POWI (x, WHOLE_PART).
1805 : Example:
1806 : POW (x, -5.875) == POW (x, 0.125) / POWI (X, 6)
1807 : --> SQRT (SQRT (SQRT (x))) / (POWI (x, 6))
1808 :
1809 : For ARG1 < 0.0 we choose between (A) and (B) depending on
1810 : how many multiplications we'd have to do.
1811 : So, for the example in (B): POW (x, -5.875), if we were to
1812 : follow algorithm (A) we would produce:
1813 : 1.0 / POWI (X, 5) * SQRT (X) * SQRT (SQRT (X)) * SQRT (SQRT (SQRT (X)))
1814 : which contains more multiplications than approach (B).
1815 :
1816 : Hopefully, this approach will eliminate potentially expensive POW library
1817 : calls when unsafe floating point math is enabled and allow the compiler to
1818 : further optimise the multiplies, square roots and divides produced by this
1819 : function. */
1820 :
1821 : static tree
1822 25 : expand_pow_as_sqrts (gimple_stmt_iterator *gsi, location_t loc,
1823 : tree arg0, tree arg1, HOST_WIDE_INT max_depth)
1824 : {
1825 25 : tree type = TREE_TYPE (arg0);
1826 25 : machine_mode mode = TYPE_MODE (type);
1827 25 : tree sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
1828 25 : bool one_over = true;
1829 :
1830 25 : if (!sqrtfn)
1831 : return NULL_TREE;
1832 :
1833 25 : if (TREE_CODE (arg1) != REAL_CST)
1834 : return NULL_TREE;
1835 :
1836 25 : REAL_VALUE_TYPE exp_init = TREE_REAL_CST (arg1);
1837 :
1838 25 : gcc_assert (max_depth > 0);
1839 25 : tree *cache = XALLOCAVEC (tree, max_depth + 1);
1840 :
1841 25 : struct pow_synth_sqrt_info synth_info;
1842 25 : synth_info.factors = XALLOCAVEC (bool, max_depth + 1);
1843 25 : synth_info.deepest = 0;
1844 25 : synth_info.num_mults = 0;
1845 :
1846 25 : bool neg_exp = REAL_VALUE_NEGATIVE (exp_init);
1847 25 : REAL_VALUE_TYPE exp = real_value_abs (&exp_init);
1848 :
1849 : /* The whole and fractional parts of exp. */
1850 25 : REAL_VALUE_TYPE whole_part;
1851 25 : REAL_VALUE_TYPE frac_part;
1852 :
1853 25 : real_floor (&whole_part, mode, &exp);
1854 25 : real_arithmetic (&frac_part, MINUS_EXPR, &exp, &whole_part);
1855 :
1856 :
1857 25 : REAL_VALUE_TYPE ceil_whole = dconst0;
1858 25 : REAL_VALUE_TYPE ceil_fract = dconst0;
1859 :
1860 25 : if (neg_exp)
1861 : {
1862 10 : real_ceil (&ceil_whole, mode, &exp);
1863 10 : real_arithmetic (&ceil_fract, MINUS_EXPR, &ceil_whole, &exp);
1864 : }
1865 :
1866 25 : if (!representable_as_half_series_p (frac_part, max_depth, &synth_info))
1867 : return NULL_TREE;
1868 :
1869 : /* Check whether it's more profitable to not use 1.0 / ... */
1870 18 : if (neg_exp)
1871 : {
1872 8 : struct pow_synth_sqrt_info alt_synth_info;
1873 8 : alt_synth_info.factors = XALLOCAVEC (bool, max_depth + 1);
1874 8 : alt_synth_info.deepest = 0;
1875 8 : alt_synth_info.num_mults = 0;
1876 :
1877 8 : if (representable_as_half_series_p (ceil_fract, max_depth,
1878 : &alt_synth_info)
1879 8 : && alt_synth_info.deepest <= synth_info.deepest
1880 16 : && alt_synth_info.num_mults < synth_info.num_mults)
1881 : {
1882 2 : whole_part = ceil_whole;
1883 2 : frac_part = ceil_fract;
1884 2 : synth_info.deepest = alt_synth_info.deepest;
1885 2 : synth_info.num_mults = alt_synth_info.num_mults;
1886 2 : memcpy (synth_info.factors, alt_synth_info.factors,
1887 : (max_depth + 1) * sizeof (bool));
1888 2 : one_over = false;
1889 : }
1890 : }
1891 :
1892 18 : HOST_WIDE_INT n = real_to_integer (&whole_part);
1893 18 : REAL_VALUE_TYPE cint;
1894 18 : real_from_integer (&cint, VOIDmode, n, SIGNED);
1895 :
1896 18 : if (!real_identical (&whole_part, &cint))
1897 : return NULL_TREE;
1898 :
1899 18 : if (powi_cost (n) + synth_info.num_mults > POWI_MAX_MULTS)
1900 : return NULL_TREE;
1901 :
1902 18 : memset (cache, 0, (max_depth + 1) * sizeof (tree));
1903 :
1904 18 : tree integer_res = n == 0 ? build_real (type, dconst1) : arg0;
1905 :
1906 : /* Calculate the integer part of the exponent. */
1907 18 : if (n > 1)
1908 : {
1909 6 : integer_res = gimple_expand_builtin_powi (gsi, loc, arg0, n);
1910 6 : if (!integer_res)
1911 : return NULL_TREE;
1912 : }
1913 :
1914 18 : if (dump_file)
1915 : {
1916 7 : char string[64];
1917 :
1918 7 : real_to_decimal (string, &exp_init, sizeof (string), 0, 1);
1919 7 : fprintf (dump_file, "synthesizing pow (x, %s) as:\n", string);
1920 :
1921 7 : if (neg_exp)
1922 : {
1923 2 : if (one_over)
1924 : {
1925 1 : fprintf (dump_file, "1.0 / (");
1926 1 : dump_integer_part (dump_file, "x", n);
1927 1 : if (n > 0)
1928 1 : fprintf (dump_file, " * ");
1929 1 : dump_fractional_sqrt_sequence (dump_file, "x", &synth_info);
1930 1 : fprintf (dump_file, ")");
1931 : }
1932 : else
1933 : {
1934 1 : dump_fractional_sqrt_sequence (dump_file, "x", &synth_info);
1935 1 : fprintf (dump_file, " / (");
1936 1 : dump_integer_part (dump_file, "x", n);
1937 1 : fprintf (dump_file, ")");
1938 : }
1939 : }
1940 : else
1941 : {
1942 5 : dump_fractional_sqrt_sequence (dump_file, "x", &synth_info);
1943 5 : if (n > 0)
1944 4 : fprintf (dump_file, " * ");
1945 5 : dump_integer_part (dump_file, "x", n);
1946 : }
1947 :
1948 7 : fprintf (dump_file, "\ndeepest sqrt chain: %d\n", synth_info.deepest);
1949 : }
1950 :
1951 :
1952 18 : tree fract_res = NULL_TREE;
1953 18 : cache[0] = arg0;
1954 :
1955 : /* Calculate the fractional part of the exponent. */
1956 58 : for (unsigned i = 0; i < synth_info.deepest; i++)
1957 : {
1958 40 : if (synth_info.factors[i])
1959 : {
1960 23 : tree sqrt_chain = get_fn_chain (arg0, i + 1, gsi, sqrtfn, loc, cache);
1961 :
1962 23 : if (!fract_res)
1963 : fract_res = sqrt_chain;
1964 :
1965 : else
1966 5 : fract_res = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
1967 : fract_res, sqrt_chain);
1968 : }
1969 : }
1970 :
1971 18 : tree res = NULL_TREE;
1972 :
1973 18 : if (neg_exp)
1974 : {
1975 8 : if (one_over)
1976 : {
1977 6 : if (n > 0)
1978 4 : res = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
1979 : fract_res, integer_res);
1980 : else
1981 : res = fract_res;
1982 :
1983 6 : res = build_and_insert_binop (gsi, loc, "powrootrecip", RDIV_EXPR,
1984 : build_real (type, dconst1), res);
1985 : }
1986 : else
1987 : {
1988 2 : res = build_and_insert_binop (gsi, loc, "powroot", RDIV_EXPR,
1989 : fract_res, integer_res);
1990 : }
1991 : }
1992 : else
1993 10 : res = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
1994 : fract_res, integer_res);
1995 : return res;
1996 : }
1997 :
1998 : /* ARG0 and ARG1 are the two arguments to a pow builtin call in GSI
1999 : with location info LOC. If possible, create an equivalent and
2000 : less expensive sequence of statements prior to GSI, and return an
2001 : expression holding the result. */
2002 :
2003 : static tree
2004 604 : gimple_expand_builtin_pow (gimple_stmt_iterator *gsi, location_t loc,
2005 : tree arg0, tree arg1)
2006 : {
2007 604 : REAL_VALUE_TYPE c, cint, dconst1_3, dconst1_4, dconst1_6;
2008 604 : REAL_VALUE_TYPE c2, dconst3;
2009 604 : HOST_WIDE_INT n;
2010 604 : tree type, sqrtfn, cbrtfn, sqrt_arg0, result, cbrt_x, powi_cbrt_x;
2011 604 : machine_mode mode;
2012 604 : bool speed_p = optimize_bb_for_speed_p (gsi_bb (*gsi));
2013 604 : bool hw_sqrt_exists, c_is_int, c2_is_int;
2014 :
2015 604 : dconst1_4 = dconst1;
2016 604 : SET_REAL_EXP (&dconst1_4, REAL_EXP (&dconst1_4) - 2);
2017 :
2018 : /* If the exponent isn't a constant, there's nothing of interest
2019 : to be done. */
2020 604 : if (TREE_CODE (arg1) != REAL_CST)
2021 : return NULL_TREE;
2022 :
2023 : /* Don't perform the operation if flag_signaling_nans is on
2024 : and the operand is a signaling NaN. */
2025 363 : if (HONOR_SNANS (TYPE_MODE (TREE_TYPE (arg1)))
2026 363 : && ((TREE_CODE (arg0) == REAL_CST
2027 0 : && REAL_VALUE_ISSIGNALING_NAN (TREE_REAL_CST (arg0)))
2028 1 : || REAL_VALUE_ISSIGNALING_NAN (TREE_REAL_CST (arg1))))
2029 : return NULL_TREE;
2030 :
2031 363 : if (flag_errno_math)
2032 : return NULL_TREE;
2033 :
2034 : /* If the exponent is equivalent to an integer, expand to an optimal
2035 : multiplication sequence when profitable. */
2036 75 : c = TREE_REAL_CST (arg1);
2037 75 : n = real_to_integer (&c);
2038 75 : real_from_integer (&cint, VOIDmode, n, SIGNED);
2039 75 : c_is_int = real_identical (&c, &cint);
2040 :
2041 75 : if (c_is_int
2042 75 : && ((n >= -1 && n <= 2)
2043 21 : || (flag_unsafe_math_optimizations
2044 11 : && speed_p
2045 11 : && powi_cost (n) <= POWI_MAX_MULTS)))
2046 30 : return gimple_expand_builtin_powi (gsi, loc, arg0, n);
2047 :
2048 : /* Attempt various optimizations using sqrt and cbrt. */
2049 45 : type = TREE_TYPE (arg0);
2050 45 : mode = TYPE_MODE (type);
2051 45 : sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
2052 :
2053 : /* Optimize pow(x,0.5) = sqrt(x). This replacement is always safe
2054 : unless signed zeros must be maintained. pow(-0,0.5) = +0, while
2055 : sqrt(-0) = -0. */
2056 45 : if (sqrtfn
2057 45 : && real_equal (&c, &dconsthalf)
2058 52 : && !HONOR_SIGNED_ZEROS (mode))
2059 0 : return build_and_insert_call (gsi, loc, sqrtfn, arg0);
2060 :
2061 45 : hw_sqrt_exists = optab_handler (sqrt_optab, mode) != CODE_FOR_nothing;
2062 :
2063 : /* Optimize pow(x,1./3.) = cbrt(x). This requires unsafe math
2064 : optimizations since 1./3. is not exactly representable. If x
2065 : is negative and finite, the correct value of pow(x,1./3.) is
2066 : a NaN with the "invalid" exception raised, because the value
2067 : of 1./3. actually has an even denominator. The correct value
2068 : of cbrt(x) is a negative real value. */
2069 45 : cbrtfn = mathfn_built_in (type, BUILT_IN_CBRT);
2070 45 : dconst1_3 = real_value_truncate (mode, dconst_third ());
2071 :
2072 45 : if (flag_unsafe_math_optimizations
2073 25 : && cbrtfn
2074 25 : && (!HONOR_NANS (mode) || tree_expr_nonnegative_p (arg0))
2075 70 : && real_equal (&c, &dconst1_3))
2076 0 : return build_and_insert_call (gsi, loc, cbrtfn, arg0);
2077 :
2078 : /* Optimize pow(x,1./6.) = cbrt(sqrt(x)). Don't do this optimization
2079 : if we don't have a hardware sqrt insn. */
2080 45 : dconst1_6 = dconst1_3;
2081 45 : SET_REAL_EXP (&dconst1_6, REAL_EXP (&dconst1_6) - 1);
2082 :
2083 45 : if (flag_unsafe_math_optimizations
2084 25 : && sqrtfn
2085 25 : && cbrtfn
2086 25 : && (!HONOR_NANS (mode) || tree_expr_nonnegative_p (arg0))
2087 : && speed_p
2088 25 : && hw_sqrt_exists
2089 70 : && real_equal (&c, &dconst1_6))
2090 : {
2091 : /* sqrt(x) */
2092 0 : sqrt_arg0 = build_and_insert_call (gsi, loc, sqrtfn, arg0);
2093 :
2094 : /* cbrt(sqrt(x)) */
2095 0 : return build_and_insert_call (gsi, loc, cbrtfn, sqrt_arg0);
2096 : }
2097 :
2098 :
2099 : /* Attempt to expand the POW as a product of square root chains.
2100 : Expand the 0.25 case even when optimising for size. */
2101 45 : if (flag_unsafe_math_optimizations
2102 25 : && sqrtfn
2103 25 : && hw_sqrt_exists
2104 25 : && (speed_p || real_equal (&c, &dconst1_4))
2105 70 : && !HONOR_SIGNED_ZEROS (mode))
2106 : {
2107 75 : unsigned int max_depth = speed_p
2108 25 : ? param_max_pow_sqrt_depth
2109 : : 2;
2110 :
2111 25 : tree expand_with_sqrts
2112 25 : = expand_pow_as_sqrts (gsi, loc, arg0, arg1, max_depth);
2113 :
2114 25 : if (expand_with_sqrts)
2115 : return expand_with_sqrts;
2116 : }
2117 :
2118 27 : real_arithmetic (&c2, MULT_EXPR, &c, &dconst2);
2119 27 : n = real_to_integer (&c2);
2120 27 : real_from_integer (&cint, VOIDmode, n, SIGNED);
2121 27 : c2_is_int = real_identical (&c2, &cint);
2122 :
2123 : /* Optimize pow(x,c), where 3c = n for some nonzero integer n, into
2124 :
2125 : powi(x, n/3) * powi(cbrt(x), n%3), n > 0;
2126 : 1.0 / (powi(x, abs(n)/3) * powi(cbrt(x), abs(n)%3)), n < 0.
2127 :
2128 : Do not calculate the first factor when n/3 = 0. As cbrt(x) is
2129 : different from pow(x, 1./3.) due to rounding and behavior with
2130 : negative x, we need to constrain this transformation to unsafe
2131 : math and positive x or finite math. */
2132 27 : real_from_integer (&dconst3, VOIDmode, 3, SIGNED);
2133 27 : real_arithmetic (&c2, MULT_EXPR, &c, &dconst3);
2134 27 : real_round (&c2, mode, &c2);
2135 27 : n = real_to_integer (&c2);
2136 27 : real_from_integer (&cint, VOIDmode, n, SIGNED);
2137 27 : real_arithmetic (&c2, RDIV_EXPR, &cint, &dconst3);
2138 27 : real_convert (&c2, mode, &c2);
2139 :
2140 27 : if (flag_unsafe_math_optimizations
2141 7 : && cbrtfn
2142 7 : && (!HONOR_NANS (mode) || tree_expr_nonnegative_p (arg0))
2143 7 : && real_identical (&c2, &c)
2144 4 : && !c2_is_int
2145 4 : && optimize_function_for_speed_p (cfun)
2146 31 : && powi_cost (n / 3) <= POWI_MAX_MULTS)
2147 : {
2148 4 : tree powi_x_ndiv3 = NULL_TREE;
2149 :
2150 : /* Attempt to fold powi(arg0, abs(n/3)) into multiplies. If not
2151 : possible or profitable, give up. Skip the degenerate case when
2152 : abs(n) < 3, where the result is always 1. */
2153 4 : if (absu_hwi (n) >= 3)
2154 : {
2155 4 : powi_x_ndiv3 = gimple_expand_builtin_powi (gsi, loc, arg0,
2156 : abs_hwi (n / 3));
2157 4 : if (!powi_x_ndiv3)
2158 : return NULL_TREE;
2159 : }
2160 :
2161 : /* Calculate powi(cbrt(x), n%3). Don't use gimple_expand_builtin_powi
2162 : as that creates an unnecessary variable. Instead, just produce
2163 : either cbrt(x) or cbrt(x) * cbrt(x). */
2164 4 : cbrt_x = build_and_insert_call (gsi, loc, cbrtfn, arg0);
2165 :
2166 4 : if (absu_hwi (n) % 3 == 1)
2167 : powi_cbrt_x = cbrt_x;
2168 : else
2169 2 : powi_cbrt_x = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
2170 : cbrt_x, cbrt_x);
2171 :
2172 : /* Multiply the two subexpressions, unless powi(x,abs(n)/3) = 1. */
2173 4 : if (absu_hwi (n) < 3)
2174 : result = powi_cbrt_x;
2175 : else
2176 4 : result = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
2177 : powi_x_ndiv3, powi_cbrt_x);
2178 :
2179 : /* If n is negative, reciprocate the result. */
2180 4 : if (n < 0)
2181 1 : result = build_and_insert_binop (gsi, loc, "powroot", RDIV_EXPR,
2182 : build_real (type, dconst1), result);
2183 :
2184 : return result;
2185 : }
2186 :
2187 : /* No optimizations succeeded. */
2188 : return NULL_TREE;
2189 : }
2190 :
2191 : /* Go through all calls to sin, cos and cexpi and call execute_cse_sincos_1
2192 : on the SSA_NAME argument of each of them. */
2193 :
2194 : namespace {
2195 :
2196 : const pass_data pass_data_cse_sincos =
2197 : {
2198 : GIMPLE_PASS, /* type */
2199 : "sincos", /* name */
2200 : OPTGROUP_NONE, /* optinfo_flags */
2201 : TV_TREE_SINCOS, /* tv_id */
2202 : PROP_ssa, /* properties_required */
2203 : 0, /* properties_provided */
2204 : 0, /* properties_destroyed */
2205 : 0, /* todo_flags_start */
2206 : TODO_update_ssa, /* todo_flags_finish */
2207 : };
2208 :
2209 : class pass_cse_sincos : public gimple_opt_pass
2210 : {
2211 : public:
2212 294587 : pass_cse_sincos (gcc::context *ctxt)
2213 589174 : : gimple_opt_pass (pass_data_cse_sincos, ctxt)
2214 : {}
2215 :
2216 : /* opt_pass methods: */
2217 1062413 : bool gate (function *) final override
2218 : {
2219 1062413 : return optimize;
2220 : }
2221 :
2222 : unsigned int execute (function *) final override;
2223 :
2224 : }; // class pass_cse_sincos
2225 :
2226 : unsigned int
2227 1062375 : pass_cse_sincos::execute (function *fun)
2228 : {
2229 1062375 : basic_block bb;
2230 1062375 : bool cfg_changed = false;
2231 :
2232 1062375 : calculate_dominance_info (CDI_DOMINATORS);
2233 1062375 : memset (&sincos_stats, 0, sizeof (sincos_stats));
2234 :
2235 11260539 : FOR_EACH_BB_FN (bb, fun)
2236 : {
2237 10198164 : gimple_stmt_iterator gsi;
2238 :
2239 100430201 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2240 : {
2241 90232037 : gimple *stmt = gsi_stmt (gsi);
2242 :
2243 90232037 : if (is_gimple_call (stmt)
2244 90232037 : && gimple_call_lhs (stmt))
2245 : {
2246 2062988 : tree arg;
2247 2062988 : switch (gimple_call_combined_fn (stmt))
2248 : {
2249 1069 : CASE_CFN_COS:
2250 1069 : CASE_CFN_SIN:
2251 1069 : CASE_CFN_CEXPI:
2252 1069 : arg = gimple_call_arg (stmt, 0);
2253 : /* Make sure we have either sincos or cexp. */
2254 1069 : if (!targetm.libc_has_function (function_c99_math_complex,
2255 1069 : TREE_TYPE (arg))
2256 1069 : && !targetm.libc_has_function (function_sincos,
2257 0 : TREE_TYPE (arg)))
2258 : break;
2259 :
2260 1069 : if (TREE_CODE (arg) == SSA_NAME)
2261 1069 : cfg_changed |= execute_cse_sincos_1 (arg);
2262 : break;
2263 : default:
2264 : break;
2265 : }
2266 : }
2267 : }
2268 : }
2269 :
2270 1062375 : statistics_counter_event (fun, "sincos statements inserted",
2271 : sincos_stats.inserted);
2272 1062375 : statistics_counter_event (fun, "conv statements removed",
2273 : sincos_stats.conv_removed);
2274 :
2275 1062375 : return cfg_changed ? TODO_cleanup_cfg : 0;
2276 : }
2277 :
2278 : } // anon namespace
2279 :
2280 : gimple_opt_pass *
2281 294587 : make_pass_cse_sincos (gcc::context *ctxt)
2282 : {
2283 294587 : return new pass_cse_sincos (ctxt);
2284 : }
2285 :
2286 : /* Expand powi(x,n) into an optimal number of multiplies, when n is a
2287 : constant. */
2288 : namespace {
2289 :
2290 : const pass_data pass_data_expand_pow =
2291 : {
2292 : GIMPLE_PASS, /* type */
2293 : "pow", /* name */
2294 : OPTGROUP_NONE, /* optinfo_flags */
2295 : TV_TREE_POW, /* tv_id */
2296 : PROP_ssa, /* properties_required */
2297 : PROP_gimple_opt_math, /* properties_provided */
2298 : 0, /* properties_destroyed */
2299 : 0, /* todo_flags_start */
2300 : TODO_update_ssa, /* todo_flags_finish */
2301 : };
2302 :
2303 : class pass_expand_pow : public gimple_opt_pass
2304 : {
2305 : public:
2306 294587 : pass_expand_pow (gcc::context *ctxt)
2307 589174 : : gimple_opt_pass (pass_data_expand_pow, ctxt)
2308 : {}
2309 :
2310 : /* opt_pass methods: */
2311 1062413 : bool gate (function *) final override
2312 : {
2313 1062413 : return optimize;
2314 : }
2315 :
2316 : unsigned int execute (function *) final override;
2317 :
2318 : }; // class pass_expand_pow
2319 :
2320 : unsigned int
2321 1062408 : pass_expand_pow::execute (function *fun)
2322 : {
2323 1062408 : basic_block bb;
2324 1062408 : bool cfg_changed = false;
2325 :
2326 1062408 : calculate_dominance_info (CDI_DOMINATORS);
2327 :
2328 11399283 : FOR_EACH_BB_FN (bb, fun)
2329 : {
2330 10336875 : gimple_stmt_iterator gsi;
2331 10336875 : bool cleanup_eh = false;
2332 :
2333 98153750 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2334 : {
2335 87816875 : gimple *stmt = gsi_stmt (gsi);
2336 :
2337 : /* Only the last stmt in a bb could throw, no need to call
2338 : gimple_purge_dead_eh_edges if we change something in the middle
2339 : of a basic block. */
2340 87816875 : cleanup_eh = false;
2341 :
2342 87816875 : if (is_gimple_call (stmt)
2343 87816875 : && gimple_call_lhs (stmt))
2344 : {
2345 2036974 : tree arg0, arg1, result;
2346 2036974 : HOST_WIDE_INT n;
2347 2036974 : location_t loc;
2348 :
2349 2036974 : switch (gimple_call_combined_fn (stmt))
2350 : {
2351 604 : CASE_CFN_POW:
2352 604 : arg0 = gimple_call_arg (stmt, 0);
2353 604 : arg1 = gimple_call_arg (stmt, 1);
2354 :
2355 604 : loc = gimple_location (stmt);
2356 604 : result = gimple_expand_builtin_pow (&gsi, loc, arg0, arg1);
2357 :
2358 604 : if (result)
2359 : {
2360 52 : tree lhs = gimple_get_lhs (stmt);
2361 52 : gassign *new_stmt = gimple_build_assign (lhs, result);
2362 52 : gimple_set_location (new_stmt, loc);
2363 52 : unlink_stmt_vdef (stmt);
2364 52 : gsi_replace (&gsi, new_stmt, true);
2365 52 : cleanup_eh = true;
2366 104 : if (gimple_vdef (stmt))
2367 0 : release_ssa_name (gimple_vdef (stmt));
2368 : }
2369 : break;
2370 :
2371 804 : CASE_CFN_POWI:
2372 804 : arg0 = gimple_call_arg (stmt, 0);
2373 804 : arg1 = gimple_call_arg (stmt, 1);
2374 804 : loc = gimple_location (stmt);
2375 :
2376 804 : if (real_minus_onep (arg0))
2377 : {
2378 0 : tree t0, t1, cond, one, minus_one;
2379 0 : gassign *stmt;
2380 :
2381 0 : t0 = TREE_TYPE (arg0);
2382 0 : t1 = TREE_TYPE (arg1);
2383 0 : one = build_real (t0, dconst1);
2384 0 : minus_one = build_real (t0, dconstm1);
2385 :
2386 0 : cond = make_temp_ssa_name (t1, NULL, "powi_cond");
2387 0 : stmt = gimple_build_assign (cond, BIT_AND_EXPR,
2388 : arg1, build_int_cst (t1, 1));
2389 0 : gimple_set_location (stmt, loc);
2390 0 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
2391 :
2392 0 : result = make_temp_ssa_name (t0, NULL, "powi");
2393 0 : stmt = gimple_build_assign (result, COND_EXPR, cond,
2394 : minus_one, one);
2395 0 : gimple_set_location (stmt, loc);
2396 0 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
2397 : }
2398 : else
2399 : {
2400 804 : if (!tree_fits_shwi_p (arg1))
2401 : break;
2402 :
2403 593 : n = tree_to_shwi (arg1);
2404 593 : result = gimple_expand_builtin_powi (&gsi, loc, arg0, n);
2405 : }
2406 :
2407 593 : if (result)
2408 : {
2409 585 : tree lhs = gimple_get_lhs (stmt);
2410 585 : gassign *new_stmt = gimple_build_assign (lhs, result);
2411 585 : gimple_set_location (new_stmt, loc);
2412 585 : unlink_stmt_vdef (stmt);
2413 585 : gsi_replace (&gsi, new_stmt, true);
2414 585 : cleanup_eh = true;
2415 87817460 : if (gimple_vdef (stmt))
2416 0 : release_ssa_name (gimple_vdef (stmt));
2417 : }
2418 : break;
2419 :
2420 211 : default:;
2421 : }
2422 : }
2423 : }
2424 10336875 : if (cleanup_eh)
2425 3 : cfg_changed |= gimple_purge_dead_eh_edges (bb);
2426 : }
2427 :
2428 1062408 : return cfg_changed ? TODO_cleanup_cfg : 0;
2429 : }
2430 :
2431 : } // anon namespace
2432 :
2433 : gimple_opt_pass *
2434 294587 : make_pass_expand_pow (gcc::context *ctxt)
2435 : {
2436 294587 : return new pass_expand_pow (ctxt);
2437 : }
2438 :
2439 : /* Return true if stmt is a type conversion operation that can be stripped
2440 : when used in a widening multiply operation. */
2441 : static bool
2442 512752 : widening_mult_conversion_strippable_p (tree result_type, gimple *stmt)
2443 : {
2444 512752 : enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
2445 :
2446 512752 : if (TREE_CODE (result_type) == INTEGER_TYPE)
2447 : {
2448 512752 : tree op_type;
2449 512752 : tree inner_op_type;
2450 :
2451 512752 : if (!CONVERT_EXPR_CODE_P (rhs_code))
2452 : return false;
2453 :
2454 200185 : op_type = TREE_TYPE (gimple_assign_lhs (stmt));
2455 :
2456 : /* If the type of OP has the same precision as the result, then
2457 : we can strip this conversion. The multiply operation will be
2458 : selected to create the correct extension as a by-product. */
2459 200185 : if (TYPE_PRECISION (result_type) == TYPE_PRECISION (op_type))
2460 : return true;
2461 :
2462 : /* We can also strip a conversion if it preserves the signed-ness of
2463 : the operation and doesn't narrow the range. */
2464 1168 : inner_op_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
2465 :
2466 : /* If the inner-most type is unsigned, then we can strip any
2467 : intermediate widening operation. If it's signed, then the
2468 : intermediate widening operation must also be signed. */
2469 1168 : if ((TYPE_UNSIGNED (inner_op_type)
2470 1164 : || TYPE_UNSIGNED (op_type) == TYPE_UNSIGNED (inner_op_type))
2471 2332 : && TYPE_PRECISION (op_type) > TYPE_PRECISION (inner_op_type))
2472 : return true;
2473 :
2474 1165 : return false;
2475 : }
2476 :
2477 0 : return rhs_code == FIXED_CONVERT_EXPR;
2478 : }
2479 :
2480 : /* Return true if RHS is a suitable operand for a widening multiplication,
2481 : assuming a target type of TYPE.
2482 : There are two cases:
2483 :
2484 : - RHS makes some value at least twice as wide. Store that value
2485 : in *NEW_RHS_OUT if so, and store its type in *TYPE_OUT.
2486 :
2487 : - RHS is an integer constant. Store that value in *NEW_RHS_OUT if so,
2488 : but leave *TYPE_OUT untouched. */
2489 :
2490 : static bool
2491 949643 : is_widening_mult_rhs_p (tree type, tree rhs, tree *type_out,
2492 : tree *new_rhs_out)
2493 : {
2494 949643 : gimple *stmt;
2495 949643 : tree type1, rhs1;
2496 :
2497 949643 : if (TREE_CODE (rhs) == SSA_NAME)
2498 : {
2499 : /* Use tree_non_zero_bits to see if this operand is zero_extended
2500 : for unsigned widening multiplications or non-negative for
2501 : signed widening multiplications. */
2502 790106 : if (TREE_CODE (type) == INTEGER_TYPE
2503 790106 : && (TYPE_PRECISION (type) & 1) == 0
2504 1580212 : && int_mode_for_size (TYPE_PRECISION (type) / 2, 1).exists ())
2505 : {
2506 784140 : unsigned int prec = TYPE_PRECISION (type);
2507 784140 : unsigned int hprec = prec / 2;
2508 784140 : wide_int bits = wide_int::from (tree_nonzero_bits (rhs), prec,
2509 1568280 : TYPE_SIGN (TREE_TYPE (rhs)));
2510 784140 : if (TYPE_UNSIGNED (type)
2511 1338111 : && wi::bit_and (bits, wi::mask (hprec, true, prec)) == 0)
2512 : {
2513 135626 : *type_out = build_nonstandard_integer_type (hprec, true);
2514 : /* X & MODE_MASK can be simplified to (T)X. */
2515 135626 : stmt = SSA_NAME_DEF_STMT (rhs);
2516 271252 : if (is_gimple_assign (stmt)
2517 120918 : && gimple_assign_rhs_code (stmt) == BIT_AND_EXPR
2518 12100 : && TREE_CODE (gimple_assign_rhs2 (stmt)) == INTEGER_CST
2519 159226 : && wide_int::from (wi::to_wide (gimple_assign_rhs2 (stmt)),
2520 11800 : prec, TYPE_SIGN (TREE_TYPE (rhs)))
2521 171026 : == wi::mask (hprec, false, prec))
2522 9858 : *new_rhs_out = gimple_assign_rhs1 (stmt);
2523 : else
2524 : *new_rhs_out = rhs;
2525 135626 : return true;
2526 : }
2527 648514 : else if (!TYPE_UNSIGNED (type)
2528 878683 : && wi::bit_and (bits, wi::mask (hprec - 1, true, prec)) == 0)
2529 : {
2530 25347 : *type_out = build_nonstandard_integer_type (hprec, false);
2531 25347 : *new_rhs_out = rhs;
2532 25347 : return true;
2533 : }
2534 784140 : }
2535 :
2536 629133 : stmt = SSA_NAME_DEF_STMT (rhs);
2537 629133 : if (is_gimple_assign (stmt))
2538 : {
2539 :
2540 512752 : if (widening_mult_conversion_strippable_p (type, stmt))
2541 : {
2542 199020 : rhs1 = gimple_assign_rhs1 (stmt);
2543 :
2544 199020 : if (TREE_CODE (rhs1) == INTEGER_CST)
2545 : {
2546 0 : *new_rhs_out = rhs1;
2547 0 : *type_out = NULL;
2548 0 : return true;
2549 : }
2550 : }
2551 : else
2552 : rhs1 = rhs;
2553 : }
2554 : else
2555 : rhs1 = rhs;
2556 :
2557 629133 : type1 = TREE_TYPE (rhs1);
2558 :
2559 629133 : if (TREE_CODE (type1) != TREE_CODE (type)
2560 629133 : || TYPE_PRECISION (type1) * 2 > TYPE_PRECISION (type))
2561 : return false;
2562 :
2563 63464 : *new_rhs_out = rhs1;
2564 63464 : *type_out = type1;
2565 63464 : return true;
2566 : }
2567 :
2568 159537 : if (TREE_CODE (rhs) == INTEGER_CST)
2569 : {
2570 159537 : *new_rhs_out = rhs;
2571 159537 : *type_out = NULL;
2572 159537 : return true;
2573 : }
2574 :
2575 : return false;
2576 : }
2577 :
2578 : /* Return true if STMT performs a widening multiplication, assuming the
2579 : output type is TYPE. If so, store the unwidened types of the operands
2580 : in *TYPE1_OUT and *TYPE2_OUT respectively. Also fill *RHS1_OUT and
2581 : *RHS2_OUT such that converting those operands to types *TYPE1_OUT
2582 : and *TYPE2_OUT would give the operands of the multiplication. */
2583 :
2584 : static bool
2585 754731 : is_widening_mult_p (gimple *stmt,
2586 : tree *type1_out, tree *rhs1_out,
2587 : tree *type2_out, tree *rhs2_out)
2588 : {
2589 754731 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
2590 :
2591 754731 : if (TREE_CODE (type) == INTEGER_TYPE)
2592 : {
2593 754731 : if (TYPE_OVERFLOW_TRAPS (type))
2594 : return false;
2595 : }
2596 0 : else if (TREE_CODE (type) != FIXED_POINT_TYPE)
2597 : return false;
2598 :
2599 754702 : if (!is_widening_mult_rhs_p (type, gimple_assign_rhs1 (stmt), type1_out,
2600 : rhs1_out))
2601 : return false;
2602 :
2603 194941 : if (!is_widening_mult_rhs_p (type, gimple_assign_rhs2 (stmt), type2_out,
2604 : rhs2_out))
2605 : return false;
2606 :
2607 189033 : if (*type1_out == NULL)
2608 : {
2609 0 : if (*type2_out == NULL || !int_fits_type_p (*rhs1_out, *type2_out))
2610 : return false;
2611 0 : *type1_out = *type2_out;
2612 : }
2613 :
2614 189033 : if (*type2_out == NULL)
2615 : {
2616 159537 : if (!int_fits_type_p (*rhs2_out, *type1_out))
2617 : return false;
2618 156330 : *type2_out = *type1_out;
2619 : }
2620 :
2621 : /* Ensure that the larger of the two operands comes first. */
2622 185826 : if (TYPE_PRECISION (*type1_out) < TYPE_PRECISION (*type2_out))
2623 : {
2624 70 : std::swap (*type1_out, *type2_out);
2625 70 : std::swap (*rhs1_out, *rhs2_out);
2626 : }
2627 :
2628 : return true;
2629 : }
2630 :
2631 : /* Check to see if the CALL statement is an invocation of copysign
2632 : with 1. being the first argument. */
2633 : static bool
2634 197277 : is_copysign_call_with_1 (gimple *call)
2635 : {
2636 197277 : gcall *c = dyn_cast <gcall *> (call);
2637 5113 : if (! c)
2638 : return false;
2639 :
2640 5113 : enum combined_fn code = gimple_call_combined_fn (c);
2641 :
2642 5113 : if (code == CFN_LAST)
2643 : return false;
2644 :
2645 4083 : if (builtin_fn_p (code))
2646 : {
2647 1030 : switch (as_builtin_fn (code))
2648 : {
2649 30 : CASE_FLT_FN (BUILT_IN_COPYSIGN):
2650 30 : CASE_FLT_FN_FLOATN_NX (BUILT_IN_COPYSIGN):
2651 30 : return real_onep (gimple_call_arg (c, 0));
2652 : default:
2653 : return false;
2654 : }
2655 : }
2656 :
2657 3053 : if (internal_fn_p (code))
2658 : {
2659 3053 : switch (as_internal_fn (code))
2660 : {
2661 24 : case IFN_COPYSIGN:
2662 24 : return real_onep (gimple_call_arg (c, 0));
2663 : default:
2664 : return false;
2665 : }
2666 : }
2667 :
2668 : return false;
2669 : }
2670 :
2671 : /* Try to expand the pattern x * copysign (1, y) into xorsign (x, y).
2672 : This only happens when the xorsign optab is defined, if the
2673 : pattern is not a xorsign pattern or if expansion fails FALSE is
2674 : returned, otherwise TRUE is returned. */
2675 : static bool
2676 745175 : convert_expand_mult_copysign (gimple *stmt, gimple_stmt_iterator *gsi)
2677 : {
2678 745175 : tree treeop0, treeop1, lhs, type;
2679 745175 : location_t loc = gimple_location (stmt);
2680 745175 : lhs = gimple_assign_lhs (stmt);
2681 745175 : treeop0 = gimple_assign_rhs1 (stmt);
2682 745175 : treeop1 = gimple_assign_rhs2 (stmt);
2683 745175 : type = TREE_TYPE (lhs);
2684 745175 : machine_mode mode = TYPE_MODE (type);
2685 :
2686 745175 : if (HONOR_SNANS (type))
2687 : return false;
2688 :
2689 744772 : if (TREE_CODE (treeop0) == SSA_NAME && TREE_CODE (treeop1) == SSA_NAME)
2690 : {
2691 238366 : gimple *call0 = SSA_NAME_DEF_STMT (treeop0);
2692 238366 : if (!has_single_use (treeop0) || !is_copysign_call_with_1 (call0))
2693 : {
2694 238340 : call0 = SSA_NAME_DEF_STMT (treeop1);
2695 238340 : if (!has_single_use (treeop1) || !is_copysign_call_with_1 (call0))
2696 : return false;
2697 :
2698 : treeop1 = treeop0;
2699 : }
2700 43 : if (optab_handler (xorsign_optab, mode) == CODE_FOR_nothing)
2701 : return false;
2702 :
2703 43 : gcall *c = as_a<gcall*> (call0);
2704 43 : treeop0 = gimple_call_arg (c, 1);
2705 :
2706 43 : gcall *call_stmt
2707 43 : = gimple_build_call_internal (IFN_XORSIGN, 2, treeop1, treeop0);
2708 43 : gimple_set_lhs (call_stmt, lhs);
2709 43 : gimple_set_location (call_stmt, loc);
2710 43 : gsi_replace (gsi, call_stmt, true);
2711 43 : return true;
2712 : }
2713 :
2714 : return false;
2715 : }
2716 :
2717 : /* Process a single gimple statement STMT, which has a MULT_EXPR as
2718 : its rhs, and try to convert it into a WIDEN_MULT_EXPR. The return
2719 : value is true iff we converted the statement. */
2720 :
2721 : static bool
2722 754025 : convert_mult_to_widen (gimple *stmt, gimple_stmt_iterator *gsi)
2723 : {
2724 754025 : tree lhs, rhs1, rhs2, type, type1, type2;
2725 754025 : enum insn_code handler;
2726 754025 : scalar_int_mode to_mode, from_mode, actual_mode;
2727 754025 : optab op;
2728 754025 : int actual_precision;
2729 754025 : location_t loc = gimple_location (stmt);
2730 754025 : bool from_unsigned1, from_unsigned2;
2731 :
2732 754025 : lhs = gimple_assign_lhs (stmt);
2733 754025 : type = TREE_TYPE (lhs);
2734 754025 : if (TREE_CODE (type) != INTEGER_TYPE)
2735 : return false;
2736 :
2737 616461 : if (!is_widening_mult_p (stmt, &type1, &rhs1, &type2, &rhs2))
2738 : return false;
2739 :
2740 : /* if any one of rhs1 and rhs2 is subject to abnormal coalescing,
2741 : avoid the transform. */
2742 153833 : if ((TREE_CODE (rhs1) == SSA_NAME
2743 153833 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1))
2744 307665 : || (TREE_CODE (rhs2) == SSA_NAME
2745 22524 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs2)))
2746 : return false;
2747 :
2748 153832 : to_mode = SCALAR_INT_TYPE_MODE (type);
2749 153832 : from_mode = SCALAR_INT_TYPE_MODE (type1);
2750 153832 : if (to_mode == from_mode)
2751 : return false;
2752 :
2753 153828 : from_unsigned1 = TYPE_UNSIGNED (type1);
2754 153828 : from_unsigned2 = TYPE_UNSIGNED (type2);
2755 :
2756 153828 : if (from_unsigned1 && from_unsigned2)
2757 : op = umul_widen_optab;
2758 60696 : else if (!from_unsigned1 && !from_unsigned2)
2759 : op = smul_widen_optab;
2760 : else
2761 1866 : op = usmul_widen_optab;
2762 :
2763 153828 : handler = find_widening_optab_handler_and_mode (op, to_mode, from_mode,
2764 : &actual_mode);
2765 :
2766 153828 : if (handler == CODE_FOR_nothing)
2767 : {
2768 144978 : if (op != smul_widen_optab)
2769 : {
2770 : /* We can use a signed multiply with unsigned types as long as
2771 : there is a wider mode to use, or it is the smaller of the two
2772 : types that is unsigned. Note that type1 >= type2, always. */
2773 87645 : if ((TYPE_UNSIGNED (type1)
2774 85971 : && TYPE_PRECISION (type1) == GET_MODE_PRECISION (from_mode))
2775 87645 : || (TYPE_UNSIGNED (type2)
2776 1674 : && TYPE_PRECISION (type2) == GET_MODE_PRECISION (from_mode)))
2777 : {
2778 87645 : if (!GET_MODE_WIDER_MODE (from_mode).exists (&from_mode)
2779 175290 : || GET_MODE_SIZE (to_mode) <= GET_MODE_SIZE (from_mode))
2780 : return false;
2781 : }
2782 :
2783 0 : op = smul_widen_optab;
2784 0 : handler = find_widening_optab_handler_and_mode (op, to_mode,
2785 : from_mode,
2786 : &actual_mode);
2787 :
2788 0 : if (handler == CODE_FOR_nothing)
2789 : return false;
2790 :
2791 : from_unsigned1 = from_unsigned2 = false;
2792 : }
2793 : else
2794 : {
2795 : /* Expand can synthesize smul_widen_optab if the target
2796 : supports umul_widen_optab. */
2797 57333 : op = umul_widen_optab;
2798 57333 : handler = find_widening_optab_handler_and_mode (op, to_mode,
2799 : from_mode,
2800 : &actual_mode);
2801 57333 : if (handler == CODE_FOR_nothing)
2802 : return false;
2803 : }
2804 : }
2805 :
2806 : /* Ensure that the inputs to the handler are in the correct precision
2807 : for the opcode. This will be the full mode size. */
2808 8850 : actual_precision = GET_MODE_PRECISION (actual_mode);
2809 8850 : if (2 * actual_precision > TYPE_PRECISION (type))
2810 : return false;
2811 8850 : if (actual_precision != TYPE_PRECISION (type1)
2812 8850 : || from_unsigned1 != TYPE_UNSIGNED (type1))
2813 : {
2814 3 : if (!useless_type_conversion_p (type1, TREE_TYPE (rhs1)))
2815 : {
2816 0 : if (TREE_CODE (rhs1) == INTEGER_CST)
2817 0 : rhs1 = fold_convert (type1, rhs1);
2818 : else
2819 0 : rhs1 = build_and_insert_cast (gsi, loc, type1, rhs1);
2820 : }
2821 3 : type1 = build_nonstandard_integer_type (actual_precision,
2822 : from_unsigned1);
2823 : }
2824 8850 : if (!useless_type_conversion_p (type1, TREE_TYPE (rhs1)))
2825 : {
2826 8113 : if (TREE_CODE (rhs1) == INTEGER_CST)
2827 0 : rhs1 = fold_convert (type1, rhs1);
2828 : else
2829 8113 : rhs1 = build_and_insert_cast (gsi, loc, type1, rhs1);
2830 : }
2831 8850 : if (actual_precision != TYPE_PRECISION (type2)
2832 8850 : || from_unsigned2 != TYPE_UNSIGNED (type2))
2833 : {
2834 3 : if (!useless_type_conversion_p (type2, TREE_TYPE (rhs2)))
2835 : {
2836 3 : if (TREE_CODE (rhs2) == INTEGER_CST)
2837 3 : rhs2 = fold_convert (type2, rhs2);
2838 : else
2839 0 : rhs2 = build_and_insert_cast (gsi, loc, type2, rhs2);
2840 : }
2841 3 : type2 = build_nonstandard_integer_type (actual_precision,
2842 : from_unsigned2);
2843 : }
2844 8850 : if (!useless_type_conversion_p (type2, TREE_TYPE (rhs2)))
2845 : {
2846 8308 : if (TREE_CODE (rhs2) == INTEGER_CST)
2847 1866 : rhs2 = fold_convert (type2, rhs2);
2848 : else
2849 6442 : rhs2 = build_and_insert_cast (gsi, loc, type2, rhs2);
2850 : }
2851 :
2852 8850 : gimple_assign_set_rhs1 (stmt, rhs1);
2853 8850 : gimple_assign_set_rhs2 (stmt, rhs2);
2854 8850 : gimple_assign_set_rhs_code (stmt, WIDEN_MULT_EXPR);
2855 8850 : update_stmt (stmt);
2856 8850 : widen_mul_stats.widen_mults_inserted++;
2857 8850 : return true;
2858 : }
2859 :
2860 : /* Process a single gimple statement STMT, which is found at the
2861 : iterator GSI and has a either a PLUS_EXPR or a MINUS_EXPR as its
2862 : rhs (given by CODE), and try to convert it into a
2863 : WIDEN_MULT_PLUS_EXPR or a WIDEN_MULT_MINUS_EXPR. The return value
2864 : is true iff we converted the statement. */
2865 :
2866 : static bool
2867 2621896 : convert_plusminus_to_widen (gimple_stmt_iterator *gsi, gimple *stmt,
2868 : enum tree_code code)
2869 : {
2870 2621896 : gimple *rhs1_stmt = NULL, *rhs2_stmt = NULL;
2871 2621896 : gimple *conv1_stmt = NULL, *conv2_stmt = NULL, *conv_stmt;
2872 2621896 : tree type, type1, type2, optype;
2873 2621896 : tree lhs, rhs1, rhs2, mult_rhs1, mult_rhs2, add_rhs;
2874 2621896 : enum tree_code rhs1_code = ERROR_MARK, rhs2_code = ERROR_MARK;
2875 2621896 : optab this_optab;
2876 2621896 : enum tree_code wmult_code;
2877 2621896 : enum insn_code handler;
2878 2621896 : scalar_mode to_mode, from_mode, actual_mode;
2879 2621896 : location_t loc = gimple_location (stmt);
2880 2621896 : int actual_precision;
2881 2621896 : bool from_unsigned1, from_unsigned2;
2882 :
2883 2621896 : lhs = gimple_assign_lhs (stmt);
2884 2621896 : type = TREE_TYPE (lhs);
2885 2621896 : if ((TREE_CODE (type) != INTEGER_TYPE
2886 402563 : && TREE_CODE (type) != FIXED_POINT_TYPE)
2887 2621896 : || !type_has_mode_precision_p (type))
2888 : return false;
2889 :
2890 2216664 : if (code == MINUS_EXPR)
2891 : wmult_code = WIDEN_MULT_MINUS_EXPR;
2892 : else
2893 1967166 : wmult_code = WIDEN_MULT_PLUS_EXPR;
2894 :
2895 2216664 : rhs1 = gimple_assign_rhs1 (stmt);
2896 2216664 : rhs2 = gimple_assign_rhs2 (stmt);
2897 :
2898 2216664 : if (TREE_CODE (rhs1) == SSA_NAME)
2899 : {
2900 2185284 : rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
2901 2185284 : if (is_gimple_assign (rhs1_stmt))
2902 1292090 : rhs1_code = gimple_assign_rhs_code (rhs1_stmt);
2903 : }
2904 :
2905 2216664 : if (TREE_CODE (rhs2) == SSA_NAME)
2906 : {
2907 801549 : rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
2908 801549 : if (is_gimple_assign (rhs2_stmt))
2909 618258 : rhs2_code = gimple_assign_rhs_code (rhs2_stmt);
2910 : }
2911 :
2912 : /* Allow for one conversion statement between the multiply
2913 : and addition/subtraction statement. If there are more than
2914 : one conversions then we assume they would invalidate this
2915 : transformation. If that's not the case then they should have
2916 : been folded before now. */
2917 2216664 : if (CONVERT_EXPR_CODE_P (rhs1_code))
2918 : {
2919 419075 : conv1_stmt = rhs1_stmt;
2920 419075 : rhs1 = gimple_assign_rhs1 (rhs1_stmt);
2921 419075 : if (TREE_CODE (rhs1) == SSA_NAME)
2922 : {
2923 349768 : rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
2924 349768 : if (is_gimple_assign (rhs1_stmt))
2925 204683 : rhs1_code = gimple_assign_rhs_code (rhs1_stmt);
2926 : }
2927 : else
2928 : return false;
2929 : }
2930 2147357 : if (CONVERT_EXPR_CODE_P (rhs2_code))
2931 : {
2932 196925 : conv2_stmt = rhs2_stmt;
2933 196925 : rhs2 = gimple_assign_rhs1 (rhs2_stmt);
2934 196925 : if (TREE_CODE (rhs2) == SSA_NAME)
2935 : {
2936 187244 : rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
2937 187244 : if (is_gimple_assign (rhs2_stmt))
2938 124795 : rhs2_code = gimple_assign_rhs_code (rhs2_stmt);
2939 : }
2940 : else
2941 : return false;
2942 : }
2943 :
2944 : /* If code is WIDEN_MULT_EXPR then it would seem unnecessary to call
2945 : is_widening_mult_p, but we still need the rhs returns.
2946 :
2947 : It might also appear that it would be sufficient to use the existing
2948 : operands of the widening multiply, but that would limit the choice of
2949 : multiply-and-accumulate instructions.
2950 :
2951 : If the widened-multiplication result has more than one uses, it is
2952 : probably wiser not to do the conversion. Also restrict this operation
2953 : to single basic block to avoid moving the multiply to a different block
2954 : with a higher execution frequency. */
2955 2137676 : if (code == PLUS_EXPR
2956 1893363 : && (rhs1_code == MULT_EXPR || rhs1_code == WIDEN_MULT_EXPR))
2957 : {
2958 144720 : if (!has_single_use (rhs1)
2959 79131 : || (conv1_stmt
2960 5166 : && !has_single_use (gimple_assign_lhs (conv1_stmt)))
2961 76418 : || gimple_bb (rhs1_stmt) != gimple_bb (stmt)
2962 211882 : || !is_widening_mult_p (rhs1_stmt, &type1, &mult_rhs1,
2963 : &type2, &mult_rhs2))
2964 : return false;
2965 : add_rhs = rhs2;
2966 : conv_stmt = conv1_stmt;
2967 : }
2968 1992956 : else if (rhs2_code == MULT_EXPR || rhs2_code == WIDEN_MULT_EXPR)
2969 : {
2970 129377 : if (!has_single_use (rhs2)
2971 79963 : || (conv2_stmt
2972 9148 : && !has_single_use (gimple_assign_lhs (conv2_stmt)))
2973 78815 : || gimple_bb (rhs2_stmt) != gimple_bb (stmt)
2974 200485 : || !is_widening_mult_p (rhs2_stmt, &type1, &mult_rhs1,
2975 : &type2, &mult_rhs2))
2976 : return false;
2977 : add_rhs = rhs1;
2978 : conv_stmt = conv2_stmt;
2979 : }
2980 : else
2981 : return false;
2982 :
2983 31993 : to_mode = SCALAR_TYPE_MODE (type);
2984 31993 : from_mode = SCALAR_TYPE_MODE (type1);
2985 31993 : if (to_mode == from_mode)
2986 : return false;
2987 :
2988 : /* For fixed point types, the mode classes could be different
2989 : so reject that case. */
2990 31992 : if (GET_MODE_CLASS (from_mode) != GET_MODE_CLASS (to_mode))
2991 : return false;
2992 :
2993 31992 : from_unsigned1 = TYPE_UNSIGNED (type1);
2994 31992 : from_unsigned2 = TYPE_UNSIGNED (type2);
2995 31992 : optype = type1;
2996 :
2997 : /* There's no such thing as a mixed sign madd yet, so use a wider mode. */
2998 31992 : if (from_unsigned1 != from_unsigned2)
2999 : {
3000 894 : if (!INTEGRAL_TYPE_P (type))
3001 : return false;
3002 : /* We can use a signed multiply with unsigned types as long as
3003 : there is a wider mode to use, or it is the smaller of the two
3004 : types that is unsigned. Note that type1 >= type2, always. */
3005 894 : if ((from_unsigned1
3006 54 : && TYPE_PRECISION (type1) == GET_MODE_PRECISION (from_mode))
3007 894 : || (from_unsigned2
3008 840 : && TYPE_PRECISION (type2) == GET_MODE_PRECISION (from_mode)))
3009 : {
3010 894 : if (!GET_MODE_WIDER_MODE (from_mode).exists (&from_mode)
3011 1788 : || GET_MODE_SIZE (from_mode) >= GET_MODE_SIZE (to_mode))
3012 : return false;
3013 : }
3014 :
3015 18 : from_unsigned1 = from_unsigned2 = false;
3016 18 : optype = build_nonstandard_integer_type (GET_MODE_PRECISION (from_mode),
3017 : false);
3018 : }
3019 :
3020 : /* If there was a conversion between the multiply and addition
3021 : then we need to make sure it fits a multiply-and-accumulate.
3022 : The should be a single mode change which does not change the
3023 : value. */
3024 31116 : if (conv_stmt)
3025 : {
3026 : /* We use the original, unmodified data types for this. */
3027 706 : tree from_type = TREE_TYPE (gimple_assign_rhs1 (conv_stmt));
3028 706 : tree to_type = TREE_TYPE (gimple_assign_lhs (conv_stmt));
3029 706 : int data_size = TYPE_PRECISION (type1) + TYPE_PRECISION (type2);
3030 706 : bool is_unsigned = TYPE_UNSIGNED (type1) && TYPE_UNSIGNED (type2);
3031 :
3032 706 : if (TYPE_PRECISION (from_type) > TYPE_PRECISION (to_type))
3033 : {
3034 : /* Conversion is a truncate. */
3035 0 : if (TYPE_PRECISION (to_type) < data_size)
3036 : return false;
3037 : }
3038 706 : else if (TYPE_PRECISION (from_type) < TYPE_PRECISION (to_type))
3039 : {
3040 : /* Conversion is an extend. Check it's the right sort. */
3041 367 : if (TYPE_UNSIGNED (from_type) != is_unsigned
3042 367 : && !(is_unsigned && TYPE_PRECISION (from_type) > data_size))
3043 : return false;
3044 : }
3045 : /* else convert is a no-op for our purposes. */
3046 : }
3047 :
3048 : /* Verify that the machine can perform a widening multiply
3049 : accumulate in this mode/signedness combination, otherwise
3050 : this transformation is likely to pessimize code. */
3051 30824 : this_optab = optab_for_tree_code (wmult_code, optype, optab_default);
3052 30824 : handler = find_widening_optab_handler_and_mode (this_optab, to_mode,
3053 : from_mode, &actual_mode);
3054 :
3055 30824 : if (handler == CODE_FOR_nothing)
3056 : return false;
3057 :
3058 : /* Ensure that the inputs to the handler are in the correct precision
3059 : for the opcode. This will be the full mode size. */
3060 0 : actual_precision = GET_MODE_PRECISION (actual_mode);
3061 0 : if (actual_precision != TYPE_PRECISION (type1)
3062 0 : || from_unsigned1 != TYPE_UNSIGNED (type1))
3063 : {
3064 0 : if (!useless_type_conversion_p (type1, TREE_TYPE (mult_rhs1)))
3065 : {
3066 0 : if (TREE_CODE (mult_rhs1) == INTEGER_CST)
3067 0 : mult_rhs1 = fold_convert (type1, mult_rhs1);
3068 : else
3069 0 : mult_rhs1 = build_and_insert_cast (gsi, loc, type1, mult_rhs1);
3070 : }
3071 0 : type1 = build_nonstandard_integer_type (actual_precision,
3072 : from_unsigned1);
3073 : }
3074 0 : if (!useless_type_conversion_p (type1, TREE_TYPE (mult_rhs1)))
3075 : {
3076 0 : if (TREE_CODE (mult_rhs1) == INTEGER_CST)
3077 0 : mult_rhs1 = fold_convert (type1, mult_rhs1);
3078 : else
3079 0 : mult_rhs1 = build_and_insert_cast (gsi, loc, type1, mult_rhs1);
3080 : }
3081 0 : if (actual_precision != TYPE_PRECISION (type2)
3082 0 : || from_unsigned2 != TYPE_UNSIGNED (type2))
3083 : {
3084 0 : if (!useless_type_conversion_p (type2, TREE_TYPE (mult_rhs2)))
3085 : {
3086 0 : if (TREE_CODE (mult_rhs2) == INTEGER_CST)
3087 0 : mult_rhs2 = fold_convert (type2, mult_rhs2);
3088 : else
3089 0 : mult_rhs2 = build_and_insert_cast (gsi, loc, type2, mult_rhs2);
3090 : }
3091 0 : type2 = build_nonstandard_integer_type (actual_precision,
3092 : from_unsigned2);
3093 : }
3094 0 : if (!useless_type_conversion_p (type2, TREE_TYPE (mult_rhs2)))
3095 : {
3096 0 : if (TREE_CODE (mult_rhs2) == INTEGER_CST)
3097 0 : mult_rhs2 = fold_convert (type2, mult_rhs2);
3098 : else
3099 0 : mult_rhs2 = build_and_insert_cast (gsi, loc, type2, mult_rhs2);
3100 : }
3101 :
3102 0 : if (!useless_type_conversion_p (type, TREE_TYPE (add_rhs)))
3103 0 : add_rhs = build_and_insert_cast (gsi, loc, type, add_rhs);
3104 :
3105 0 : gimple_assign_set_rhs_with_ops (gsi, wmult_code, mult_rhs1, mult_rhs2,
3106 : add_rhs);
3107 0 : update_stmt (gsi_stmt (*gsi));
3108 0 : widen_mul_stats.maccs_inserted++;
3109 0 : return true;
3110 : }
3111 :
3112 : /* Given a result MUL_RESULT which is a result of a multiplication of OP1 and
3113 : OP2 and which we know is used in statements that can be, together with the
3114 : multiplication, converted to FMAs, perform the transformation. */
3115 :
3116 : static void
3117 17541 : convert_mult_to_fma_1 (tree mul_result, tree op1, tree op2)
3118 : {
3119 17541 : gimple *use_stmt;
3120 17541 : imm_use_iterator imm_iter;
3121 17541 : gcall *fma_stmt;
3122 :
3123 35141 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, mul_result)
3124 : {
3125 17600 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
3126 17600 : tree addop, mulop1 = op1, result = mul_result;
3127 17600 : bool negate_p = false;
3128 17600 : gimple_seq seq = NULL;
3129 :
3130 17600 : if (is_gimple_debug (use_stmt))
3131 0 : continue;
3132 :
3133 : /* If the use is a type convert, look further into it if the operations
3134 : are the same under two's complement. */
3135 17600 : tree lhs_type;
3136 17600 : if (gimple_assign_cast_p (use_stmt)
3137 0 : && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt)))
3138 17600 : && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1)))
3139 : {
3140 0 : tree cast_lhs = gimple_get_lhs (use_stmt);
3141 0 : gimple *tmp_use;
3142 0 : use_operand_p tmp_use_p;
3143 0 : if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use))
3144 : {
3145 0 : release_defs (use_stmt);
3146 0 : use_stmt = tmp_use;
3147 0 : result = cast_lhs;
3148 0 : gsi_remove (&gsi, true);
3149 0 : gsi = gsi_for_stmt (use_stmt);
3150 : }
3151 : }
3152 :
3153 17600 : if (is_gimple_assign (use_stmt)
3154 17600 : && gimple_assign_rhs_code (use_stmt) == NEGATE_EXPR)
3155 : {
3156 700 : result = gimple_assign_lhs (use_stmt);
3157 700 : use_operand_p use_p;
3158 700 : gimple *neguse_stmt;
3159 700 : single_imm_use (gimple_assign_lhs (use_stmt), &use_p, &neguse_stmt);
3160 700 : gsi_remove (&gsi, true);
3161 700 : release_defs (use_stmt);
3162 :
3163 700 : use_stmt = neguse_stmt;
3164 700 : gsi = gsi_for_stmt (use_stmt);
3165 700 : negate_p = true;
3166 : }
3167 :
3168 17600 : tree cond, else_value, ops[3], len, bias;
3169 17600 : tree_code code;
3170 17600 : if (!can_interpret_as_conditional_op_p (use_stmt, &cond, &code,
3171 : ops, &else_value,
3172 : &len, &bias))
3173 0 : gcc_unreachable ();
3174 17600 : addop = ops[0] == result ? ops[1] : ops[0];
3175 :
3176 17600 : if (code == MINUS_EXPR)
3177 : {
3178 5839 : if (ops[0] == result)
3179 : /* a * b - c -> a * b + (-c) */
3180 2916 : addop = gimple_build (&seq, NEGATE_EXPR, TREE_TYPE (addop), addop);
3181 : else
3182 : /* a - b * c -> (-b) * c + a */
3183 2923 : negate_p = !negate_p;
3184 : }
3185 :
3186 17600 : if (negate_p)
3187 3623 : mulop1 = gimple_build (&seq, NEGATE_EXPR, TREE_TYPE (mulop1), mulop1);
3188 :
3189 17600 : if (seq)
3190 5834 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
3191 :
3192 : /* Ensure all the operands are of the same type. Use the type of the
3193 : addend as that's the statement being replaced. */
3194 17600 : op2 = gimple_convert (&gsi, true, GSI_SAME_STMT,
3195 17600 : UNKNOWN_LOCATION, TREE_TYPE (addop), op2);
3196 17600 : mulop1 = gimple_convert (&gsi, true, GSI_SAME_STMT,
3197 17600 : UNKNOWN_LOCATION, TREE_TYPE (addop), mulop1);
3198 :
3199 17600 : if (len)
3200 0 : fma_stmt
3201 0 : = gimple_build_call_internal (IFN_COND_LEN_FMA, 7, cond, mulop1, op2,
3202 : addop, else_value, len, bias);
3203 17600 : else if (cond)
3204 94 : fma_stmt = gimple_build_call_internal (IFN_COND_FMA, 5, cond, mulop1,
3205 : op2, addop, else_value);
3206 : else
3207 17506 : fma_stmt = gimple_build_call_internal (IFN_FMA, 3, mulop1, op2, addop);
3208 17600 : gimple_set_lhs (fma_stmt, gimple_get_lhs (use_stmt));
3209 17600 : gimple_call_set_nothrow (fma_stmt, !stmt_can_throw_internal (cfun,
3210 : use_stmt));
3211 17600 : gsi_replace (&gsi, fma_stmt, true);
3212 : /* Follow all SSA edges so that we generate FMS, FNMA and FNMS
3213 : regardless of where the negation occurs. */
3214 17600 : gimple *orig_stmt = gsi_stmt (gsi);
3215 17600 : if (fold_stmt (&gsi, follow_all_ssa_edges))
3216 : {
3217 5883 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, gsi_stmt (gsi)))
3218 0 : gcc_unreachable ();
3219 5883 : update_stmt (gsi_stmt (gsi));
3220 : }
3221 :
3222 17600 : if (dump_file && (dump_flags & TDF_DETAILS))
3223 : {
3224 3 : fprintf (dump_file, "Generated FMA ");
3225 3 : print_gimple_stmt (dump_file, gsi_stmt (gsi), 0, TDF_NONE);
3226 3 : fprintf (dump_file, "\n");
3227 : }
3228 :
3229 : /* If the FMA result is negated in a single use, fold the negation
3230 : too. */
3231 17600 : orig_stmt = gsi_stmt (gsi);
3232 17600 : use_operand_p use_p;
3233 17600 : gimple *neg_stmt;
3234 17600 : if (is_gimple_call (orig_stmt)
3235 17600 : && gimple_call_internal_p (orig_stmt)
3236 17600 : && gimple_call_lhs (orig_stmt)
3237 17600 : && TREE_CODE (gimple_call_lhs (orig_stmt)) == SSA_NAME
3238 17600 : && single_imm_use (gimple_call_lhs (orig_stmt), &use_p, &neg_stmt)
3239 12552 : && is_gimple_assign (neg_stmt)
3240 9888 : && gimple_assign_rhs_code (neg_stmt) == NEGATE_EXPR
3241 18953 : && !stmt_could_throw_p (cfun, neg_stmt))
3242 : {
3243 1353 : gsi = gsi_for_stmt (neg_stmt);
3244 1353 : if (fold_stmt (&gsi, follow_all_ssa_edges))
3245 : {
3246 1353 : if (maybe_clean_or_replace_eh_stmt (neg_stmt, gsi_stmt (gsi)))
3247 0 : gcc_unreachable ();
3248 1353 : update_stmt (gsi_stmt (gsi));
3249 1353 : if (dump_file && (dump_flags & TDF_DETAILS))
3250 : {
3251 0 : fprintf (dump_file, "Folded FMA negation ");
3252 0 : print_gimple_stmt (dump_file, gsi_stmt (gsi), 0, TDF_NONE);
3253 0 : fprintf (dump_file, "\n");
3254 : }
3255 : }
3256 : }
3257 :
3258 17600 : widen_mul_stats.fmas_inserted++;
3259 17541 : }
3260 17541 : }
3261 :
3262 : /* Data necessary to perform the actual transformation from a multiplication
3263 : and an addition to an FMA after decision is taken it should be done and to
3264 : then delete the multiplication statement from the function IL. */
3265 :
3266 : struct fma_transformation_info
3267 : {
3268 : gimple *mul_stmt;
3269 : tree mul_result;
3270 : tree op1;
3271 : tree op2;
3272 : };
3273 :
3274 : /* Structure containing the current state of FMA deferring, i.e. whether we are
3275 : deferring, whether to continue deferring, and all data necessary to come
3276 : back and perform all deferred transformations. */
3277 :
3278 10458165 : class fma_deferring_state
3279 : {
3280 : public:
3281 : /* Class constructor. Pass true as PERFORM_DEFERRING in order to actually
3282 : do any deferring. */
3283 :
3284 10458165 : fma_deferring_state (bool perform_deferring)
3285 10458165 : : m_candidates (), m_mul_result_set (), m_initial_phi (NULL),
3286 10458165 : m_last_result (NULL_TREE), m_deferring_p (perform_deferring) {}
3287 :
3288 : /* List of FMA candidates for which we the transformation has been determined
3289 : possible but we at this point in BB analysis we do not consider them
3290 : beneficial. */
3291 : auto_vec<fma_transformation_info, 8> m_candidates;
3292 :
3293 : /* Set of results of multiplication that are part of an already deferred FMA
3294 : candidates. */
3295 : hash_set<tree> m_mul_result_set;
3296 :
3297 : /* The PHI that supposedly feeds back result of a FMA to another over loop
3298 : boundary. */
3299 : gphi *m_initial_phi;
3300 :
3301 : /* Result of the last produced FMA candidate or NULL if there has not been
3302 : one. */
3303 : tree m_last_result;
3304 :
3305 : /* If true, deferring might still be profitable. If false, transform all
3306 : candidates and no longer defer. */
3307 : bool m_deferring_p;
3308 : };
3309 :
3310 : /* Transform all deferred FMA candidates and mark STATE as no longer
3311 : deferring. */
3312 :
3313 : static void
3314 3735398 : cancel_fma_deferring (fma_deferring_state *state)
3315 : {
3316 3735398 : if (!state->m_deferring_p)
3317 : return;
3318 :
3319 2698546 : for (unsigned i = 0; i < state->m_candidates.length (); i++)
3320 : {
3321 940 : if (dump_file && (dump_flags & TDF_DETAILS))
3322 0 : fprintf (dump_file, "Generating deferred FMA\n");
3323 :
3324 940 : const fma_transformation_info &fti = state->m_candidates[i];
3325 940 : convert_mult_to_fma_1 (fti.mul_result, fti.op1, fti.op2);
3326 :
3327 940 : gimple_stmt_iterator gsi = gsi_for_stmt (fti.mul_stmt);
3328 940 : gsi_remove (&gsi, true);
3329 940 : release_defs (fti.mul_stmt);
3330 : }
3331 2697606 : state->m_deferring_p = false;
3332 : }
3333 :
3334 : /* If OP is an SSA name defined by a PHI node, return the PHI statement.
3335 : Otherwise return NULL. */
3336 :
3337 : static gphi *
3338 5266 : result_of_phi (tree op)
3339 : {
3340 0 : if (TREE_CODE (op) != SSA_NAME)
3341 : return NULL;
3342 :
3343 5141 : return dyn_cast <gphi *> (SSA_NAME_DEF_STMT (op));
3344 : }
3345 :
3346 : /* After processing statements of a BB and recording STATE, return true if the
3347 : initial phi is fed by the last FMA candidate result ore one such result from
3348 : previously processed BBs marked in LAST_RESULT_SET. */
3349 :
3350 : static bool
3351 361 : last_fma_candidate_feeds_initial_phi (fma_deferring_state *state,
3352 : hash_set<tree> *last_result_set)
3353 : {
3354 361 : ssa_op_iter iter;
3355 361 : use_operand_p use;
3356 889 : FOR_EACH_PHI_ARG (use, state->m_initial_phi, iter, SSA_OP_USE)
3357 : {
3358 625 : tree t = USE_FROM_PTR (use);
3359 625 : if (t == state->m_last_result
3360 625 : || last_result_set->contains (t))
3361 97 : return true;
3362 : }
3363 :
3364 : return false;
3365 : }
3366 :
3367 : /* Combine the multiplication at MUL_STMT with operands MULOP1 and MULOP2
3368 : with uses in additions and subtractions to form fused multiply-add
3369 : operations. Returns true if successful and MUL_STMT should be removed.
3370 : If MUL_COND is nonnull, the multiplication in MUL_STMT is conditional
3371 : on MUL_COND, otherwise it is unconditional.
3372 :
3373 : If STATE indicates that we are deferring FMA transformation, that means
3374 : that we do not produce FMAs for basic blocks which look like:
3375 :
3376 : <bb 6>
3377 : # accumulator_111 = PHI <0.0(5), accumulator_66(6)>
3378 : _65 = _14 * _16;
3379 : accumulator_66 = _65 + accumulator_111;
3380 :
3381 : or its unrolled version, i.e. with several FMA candidates that feed result
3382 : of one into the addend of another. Instead, we add them to a list in STATE
3383 : and if we later discover an FMA candidate that is not part of such a chain,
3384 : we go back and perform all deferred past candidates. */
3385 :
3386 : static bool
3387 745261 : convert_mult_to_fma (gimple *mul_stmt, tree op1, tree op2,
3388 : fma_deferring_state *state, tree mul_cond = NULL_TREE,
3389 : tree mul_len = NULL_TREE, tree mul_bias = NULL_TREE)
3390 : {
3391 745261 : tree mul_result = gimple_get_lhs (mul_stmt);
3392 : /* If there isn't a LHS then this can't be an FMA. There can be no LHS
3393 : if the statement was left just for the side-effects. */
3394 745261 : if (!mul_result)
3395 : return false;
3396 745261 : tree type = TREE_TYPE (mul_result);
3397 745261 : gimple *use_stmt, *neguse_stmt;
3398 745261 : use_operand_p use_p;
3399 745261 : imm_use_iterator imm_iter;
3400 :
3401 647495 : if (FLOAT_TYPE_P (type)
3402 771007 : && flag_fp_contract_mode != FP_CONTRACT_FAST)
3403 : return false;
3404 :
3405 : /* We don't want to do bitfield reduction ops. */
3406 740172 : if (INTEGRAL_TYPE_P (type)
3407 740172 : && (!type_has_mode_precision_p (type) || TYPE_OVERFLOW_TRAPS (type)))
3408 : return false;
3409 :
3410 : /* If the target doesn't support it, don't generate it. We assume that
3411 : if fma isn't available then fms, fnma or fnms are not either. */
3412 739975 : optimization_type opt_type = bb_optimization_type (gimple_bb (mul_stmt));
3413 739975 : if (!direct_internal_fn_supported_p (IFN_FMA, type, opt_type))
3414 : return false;
3415 :
3416 : /* If the multiplication has zero uses, it is kept around probably because
3417 : of -fnon-call-exceptions. Don't optimize it away in that case,
3418 : it is DCE job. */
3419 23040 : if (has_zero_uses (mul_result))
3420 : return false;
3421 :
3422 23040 : bool check_defer
3423 23040 : = (state->m_deferring_p
3424 23040 : && maybe_le (tree_to_poly_int64 (TYPE_SIZE (type)),
3425 23040 : param_avoid_fma_max_bits));
3426 23040 : bool defer = check_defer;
3427 23040 : bool seen_negate_p = false;
3428 :
3429 : /* There is no numerical difference between fused and unfused integer FMAs,
3430 : and the assumption below that FMA is as cheap as addition is unlikely
3431 : to be true, especially if the multiplication occurs multiple times on
3432 : the same chain. E.g., for something like:
3433 :
3434 : (((a * b) + c) >> 1) + (a * b)
3435 :
3436 : we do not want to duplicate the a * b into two additions, not least
3437 : because the result is not a natural FMA chain. */
3438 23040 : if (ANY_INTEGRAL_TYPE_P (type)
3439 23040 : && !has_single_use (mul_result))
3440 : return false;
3441 :
3442 23040 : if (!dbg_cnt (form_fma))
3443 : return false;
3444 :
3445 : /* Make sure that the multiplication statement becomes dead after
3446 : the transformation, thus that all uses are transformed to FMAs.
3447 : This means we assume that an FMA operation has the same cost
3448 : as an addition. */
3449 41425 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, mul_result)
3450 : {
3451 23787 : tree result = mul_result;
3452 23787 : bool negate_p = false;
3453 :
3454 23787 : use_stmt = USE_STMT (use_p);
3455 :
3456 23787 : if (is_gimple_debug (use_stmt))
3457 214 : continue;
3458 :
3459 : /* If the use is a type convert, look further into it if the operations
3460 : are the same under two's complement. */
3461 23573 : tree lhs_type;
3462 23573 : if (gimple_assign_cast_p (use_stmt)
3463 243 : && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt)))
3464 23816 : && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1)))
3465 : {
3466 0 : tree cast_lhs = gimple_get_lhs (use_stmt);
3467 0 : gimple *tmp_use;
3468 0 : use_operand_p tmp_use_p;
3469 0 : if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use))
3470 0 : use_stmt = tmp_use;
3471 0 : result = cast_lhs;
3472 : }
3473 :
3474 : /* For now restrict this operations to single basic blocks. In theory
3475 : we would want to support sinking the multiplication in
3476 : m = a*b;
3477 : if ()
3478 : ma = m + c;
3479 : else
3480 : d = m;
3481 : to form a fma in the then block and sink the multiplication to the
3482 : else block. */
3483 23573 : if (gimple_bb (use_stmt) != gimple_bb (mul_stmt))
3484 5402 : return false;
3485 :
3486 : /* A negate on the multiplication leads to FNMA. */
3487 22705 : if (is_gimple_assign (use_stmt)
3488 22705 : && gimple_assign_rhs_code (use_stmt) == NEGATE_EXPR)
3489 : {
3490 706 : ssa_op_iter iter;
3491 706 : use_operand_p usep;
3492 :
3493 : /* If (due to earlier missed optimizations) we have two
3494 : negates of the same value, treat them as equivalent
3495 : to a single negate with multiple uses. */
3496 706 : if (seen_negate_p)
3497 0 : return false;
3498 :
3499 706 : result = gimple_assign_lhs (use_stmt);
3500 :
3501 : /* Make sure the negate statement becomes dead with this
3502 : single transformation. */
3503 706 : if (!single_imm_use (gimple_assign_lhs (use_stmt),
3504 : &use_p, &neguse_stmt))
3505 : return false;
3506 :
3507 : /* Make sure the multiplication isn't also used on that stmt. */
3508 2836 : FOR_EACH_PHI_OR_STMT_USE (usep, neguse_stmt, iter, SSA_OP_USE)
3509 1424 : if (USE_FROM_PTR (usep) == mul_result)
3510 : return false;
3511 :
3512 : /* Re-validate. */
3513 706 : use_stmt = neguse_stmt;
3514 706 : if (gimple_bb (use_stmt) != gimple_bb (mul_stmt))
3515 : return false;
3516 :
3517 706 : negate_p = seen_negate_p = true;
3518 : }
3519 :
3520 22705 : tree cond, else_value, ops[3], len, bias;
3521 22705 : tree_code code;
3522 22705 : if (!can_interpret_as_conditional_op_p (use_stmt, &cond, &code, ops,
3523 : &else_value, &len, &bias))
3524 : return false;
3525 :
3526 : /* The multiplication result must be one of the addition operands. */
3527 20346 : if (ops[0] != result && ops[1] != result)
3528 : return false;
3529 :
3530 19733 : switch (code)
3531 : {
3532 5845 : case MINUS_EXPR:
3533 5845 : if (ops[1] == result)
3534 2923 : negate_p = !negate_p;
3535 : break;
3536 : case PLUS_EXPR:
3537 : break;
3538 : default:
3539 : /* FMA can only be formed from PLUS and MINUS. */
3540 : return false;
3541 : }
3542 :
3543 18193 : if (len)
3544 : {
3545 : /* For COND_LEN_* operations, we may have dummpy mask which is
3546 : the all true mask. Such TREE type may be mul_cond != cond
3547 : but we still consider they are equal. */
3548 0 : if (mul_cond && cond != mul_cond
3549 0 : && !(integer_truep (mul_cond) && integer_truep (cond)))
3550 : return false;
3551 :
3552 0 : if (else_value == result)
3553 : return false;
3554 :
3555 0 : if (!direct_internal_fn_supported_p (IFN_COND_LEN_FMA, type,
3556 : opt_type))
3557 : return false;
3558 :
3559 0 : if (mul_len)
3560 : {
3561 0 : poly_int64 mul_value, value;
3562 0 : if (poly_int_tree_p (mul_len, &mul_value)
3563 0 : && poly_int_tree_p (len, &value)
3564 0 : && maybe_ne (mul_value, value))
3565 0 : return false;
3566 0 : else if (mul_len != len)
3567 : return false;
3568 :
3569 0 : if (wi::to_widest (mul_bias) != wi::to_widest (bias))
3570 : return false;
3571 : }
3572 : }
3573 : else
3574 : {
3575 18193 : if (mul_cond && cond != mul_cond)
3576 : return false;
3577 :
3578 18181 : if (cond)
3579 : {
3580 104 : if (cond == result || else_value == result)
3581 : return false;
3582 94 : if (!direct_internal_fn_supported_p (IFN_COND_FMA, type,
3583 : opt_type))
3584 : return false;
3585 : }
3586 : }
3587 :
3588 : /* If the subtrahend (OPS[1]) is computed by a MULT_EXPR that
3589 : we'll visit later, we might be able to get a more profitable
3590 : match with fnma.
3591 : OTOH, if we don't, a negate / fma pair has likely lower latency
3592 : that a mult / subtract pair. */
3593 18171 : if (code == MINUS_EXPR
3594 5839 : && !negate_p
3595 2216 : && ops[0] == result
3596 2216 : && !direct_internal_fn_supported_p (IFN_FMS, type, opt_type)
3597 0 : && direct_internal_fn_supported_p (IFN_FNMA, type, opt_type)
3598 0 : && TREE_CODE (ops[1]) == SSA_NAME
3599 18171 : && has_single_use (ops[1]))
3600 : {
3601 0 : gimple *stmt2 = SSA_NAME_DEF_STMT (ops[1]);
3602 0 : if (is_gimple_assign (stmt2)
3603 0 : && gimple_assign_rhs_code (stmt2) == MULT_EXPR)
3604 : return false;
3605 : }
3606 :
3607 : /* We can't handle a * b + a * b. */
3608 18171 : if (ops[0] == ops[1])
3609 : return false;
3610 : /* If deferring, make sure we are not looking at an instruction that
3611 : wouldn't have existed if we were not. */
3612 18171 : if (state->m_deferring_p
3613 18171 : && (state->m_mul_result_set.contains (ops[0])
3614 6461 : || state->m_mul_result_set.contains (ops[1])))
3615 : return false;
3616 :
3617 18171 : if (check_defer)
3618 : {
3619 6318 : tree use_lhs = gimple_get_lhs (use_stmt);
3620 6318 : if (state->m_last_result)
3621 : {
3622 1052 : if (ops[1] == state->m_last_result
3623 1052 : || ops[0] == state->m_last_result)
3624 : defer = true;
3625 : else
3626 6318 : defer = false;
3627 : }
3628 : else
3629 : {
3630 5266 : gcc_checking_assert (!state->m_initial_phi);
3631 5266 : gphi *phi;
3632 5266 : if (ops[0] == result)
3633 3339 : phi = result_of_phi (ops[1]);
3634 : else
3635 : {
3636 1927 : gcc_assert (ops[1] == result);
3637 1927 : phi = result_of_phi (ops[0]);
3638 : }
3639 :
3640 : if (phi)
3641 : {
3642 963 : state->m_initial_phi = phi;
3643 963 : defer = true;
3644 : }
3645 : else
3646 : defer = false;
3647 : }
3648 :
3649 6318 : state->m_last_result = use_lhs;
3650 6318 : check_defer = false;
3651 : }
3652 : else
3653 : defer = false;
3654 :
3655 : /* While it is possible to validate whether or not the exact form that
3656 : we've recognized is available in the backend, the assumption is that
3657 : if the deferring logic above did not trigger, the transformation is
3658 : never a loss. For instance, suppose the target only has the plain FMA
3659 : pattern available. Consider a*b-c -> fma(a,b,-c): we've exchanged
3660 : MUL+SUB for FMA+NEG, which is still two operations. Consider
3661 : -(a*b)-c -> fma(-a,b,-c): we still have 3 operations, but in the FMA
3662 : form the two NEGs are independent and could be run in parallel. */
3663 5402 : }
3664 :
3665 17638 : if (defer)
3666 : {
3667 1037 : fma_transformation_info fti;
3668 1037 : fti.mul_stmt = mul_stmt;
3669 1037 : fti.mul_result = mul_result;
3670 1037 : fti.op1 = op1;
3671 1037 : fti.op2 = op2;
3672 1037 : state->m_candidates.safe_push (fti);
3673 1037 : state->m_mul_result_set.add (mul_result);
3674 :
3675 1037 : if (dump_file && (dump_flags & TDF_DETAILS))
3676 : {
3677 0 : fprintf (dump_file, "Deferred generating FMA for multiplication ");
3678 0 : print_gimple_stmt (dump_file, mul_stmt, 0, TDF_NONE);
3679 0 : fprintf (dump_file, "\n");
3680 : }
3681 :
3682 1037 : return false;
3683 : }
3684 : else
3685 : {
3686 16601 : if (state->m_deferring_p)
3687 4922 : cancel_fma_deferring (state);
3688 16601 : convert_mult_to_fma_1 (mul_result, op1, op2);
3689 16601 : return true;
3690 : }
3691 : }
3692 :
3693 :
3694 : /* Helper function of match_arith_overflow. For MUL_OVERFLOW, if we have
3695 : a check for non-zero like:
3696 : _1 = x_4(D) * y_5(D);
3697 : *res_7(D) = _1;
3698 : if (x_4(D) != 0)
3699 : goto <bb 3>; [50.00%]
3700 : else
3701 : goto <bb 4>; [50.00%]
3702 :
3703 : <bb 3> [local count: 536870913]:
3704 : _2 = _1 / x_4(D);
3705 : _9 = _2 != y_5(D);
3706 : _10 = (int) _9;
3707 :
3708 : <bb 4> [local count: 1073741824]:
3709 : # iftmp.0_3 = PHI <_10(3), 0(2)>
3710 : then in addition to using .MUL_OVERFLOW (x_4(D), y_5(D)) we can also
3711 : optimize the x_4(D) != 0 condition to 1. */
3712 :
3713 : static void
3714 173 : maybe_optimize_guarding_check (vec<gimple *> &mul_stmts, gimple *cond_stmt,
3715 : gimple *div_stmt, bool *cfg_changed)
3716 : {
3717 173 : basic_block bb = gimple_bb (cond_stmt);
3718 346 : if (gimple_bb (div_stmt) != bb || !single_pred_p (bb))
3719 63 : return;
3720 173 : edge pred_edge = single_pred_edge (bb);
3721 173 : basic_block pred_bb = pred_edge->src;
3722 173 : if (EDGE_COUNT (pred_bb->succs) != 2)
3723 : return;
3724 130 : edge other_edge = EDGE_SUCC (pred_bb, EDGE_SUCC (pred_bb, 0) == pred_edge);
3725 130 : edge other_succ_edge = NULL;
3726 130 : if (gimple_code (cond_stmt) == GIMPLE_COND)
3727 : {
3728 48 : if (EDGE_COUNT (bb->succs) != 2)
3729 : return;
3730 48 : other_succ_edge = EDGE_SUCC (bb, 0);
3731 48 : if (gimple_cond_code (cond_stmt) == NE_EXPR)
3732 : {
3733 24 : if (other_succ_edge->flags & EDGE_TRUE_VALUE)
3734 24 : other_succ_edge = EDGE_SUCC (bb, 1);
3735 : }
3736 : else if (other_succ_edge->flags & EDGE_FALSE_VALUE)
3737 48 : other_succ_edge = EDGE_SUCC (bb, 0);
3738 48 : if (other_edge->dest != other_succ_edge->dest)
3739 : return;
3740 : }
3741 140 : else if (!single_succ_p (bb) || other_edge->dest != single_succ (bb))
3742 : return;
3743 311 : gcond *zero_cond = safe_dyn_cast <gcond *> (*gsi_last_bb (pred_bb));
3744 124 : if (zero_cond == NULL
3745 124 : || (gimple_cond_code (zero_cond)
3746 124 : != ((pred_edge->flags & EDGE_TRUE_VALUE) ? NE_EXPR : EQ_EXPR))
3747 124 : || !integer_zerop (gimple_cond_rhs (zero_cond)))
3748 : return;
3749 124 : tree zero_cond_lhs = gimple_cond_lhs (zero_cond);
3750 124 : if (TREE_CODE (zero_cond_lhs) != SSA_NAME)
3751 : return;
3752 124 : if (gimple_assign_rhs2 (div_stmt) != zero_cond_lhs)
3753 : {
3754 : /* Allow the divisor to be result of a same precision cast
3755 : from zero_cond_lhs. */
3756 60 : tree rhs2 = gimple_assign_rhs2 (div_stmt);
3757 60 : if (TREE_CODE (rhs2) != SSA_NAME)
3758 : return;
3759 60 : gimple *g = SSA_NAME_DEF_STMT (rhs2);
3760 60 : if (!gimple_assign_cast_p (g)
3761 53 : || gimple_assign_rhs1 (g) != gimple_cond_lhs (zero_cond)
3762 53 : || !INTEGRAL_TYPE_P (TREE_TYPE (zero_cond_lhs))
3763 113 : || (TYPE_PRECISION (TREE_TYPE (zero_cond_lhs))
3764 53 : != TYPE_PRECISION (TREE_TYPE (rhs2))))
3765 : return;
3766 : }
3767 117 : gimple_stmt_iterator gsi = gsi_after_labels (bb);
3768 117 : mul_stmts.safe_push (div_stmt);
3769 117 : if (is_gimple_debug (gsi_stmt (gsi)))
3770 0 : gsi_next_nondebug (&gsi);
3771 117 : unsigned cast_count = 0;
3772 667 : while (gsi_stmt (gsi) != cond_stmt)
3773 : {
3774 : /* If original mul_stmt has a single use, allow it in the same bb,
3775 : we are looking then just at __builtin_mul_overflow_p.
3776 : Though, in that case the original mul_stmt will be replaced
3777 : by .MUL_OVERFLOW, REALPART_EXPR and IMAGPART_EXPR stmts. */
3778 : gimple *mul_stmt;
3779 : unsigned int i;
3780 2443 : bool ok = false;
3781 2443 : FOR_EACH_VEC_ELT (mul_stmts, i, mul_stmt)
3782 : {
3783 2296 : if (gsi_stmt (gsi) == mul_stmt)
3784 : {
3785 : ok = true;
3786 : break;
3787 : }
3788 : }
3789 550 : if (!ok && gimple_assign_cast_p (gsi_stmt (gsi)) && ++cast_count < 4)
3790 : ok = true;
3791 403 : if (!ok)
3792 63 : return;
3793 550 : gsi_next_nondebug (&gsi);
3794 : }
3795 117 : if (gimple_code (cond_stmt) == GIMPLE_COND)
3796 : {
3797 47 : basic_block succ_bb = other_edge->dest;
3798 75 : for (gphi_iterator gpi = gsi_start_phis (succ_bb); !gsi_end_p (gpi);
3799 28 : gsi_next (&gpi))
3800 : {
3801 35 : gphi *phi = gpi.phi ();
3802 35 : tree v1 = gimple_phi_arg_def (phi, other_edge->dest_idx);
3803 35 : tree v2 = gimple_phi_arg_def (phi, other_succ_edge->dest_idx);
3804 35 : if (!operand_equal_p (v1, v2, 0))
3805 7 : return;
3806 : }
3807 : }
3808 : else
3809 : {
3810 70 : tree lhs = gimple_assign_lhs (cond_stmt);
3811 70 : if (!lhs || !INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
3812 : return;
3813 70 : gsi_next_nondebug (&gsi);
3814 70 : if (!gsi_end_p (gsi))
3815 : {
3816 70 : if (gimple_assign_rhs_code (cond_stmt) == COND_EXPR)
3817 : return;
3818 70 : gimple *cast_stmt = gsi_stmt (gsi);
3819 70 : if (!gimple_assign_cast_p (cast_stmt))
3820 : return;
3821 70 : tree new_lhs = gimple_assign_lhs (cast_stmt);
3822 70 : gsi_next_nondebug (&gsi);
3823 70 : if (!gsi_end_p (gsi)
3824 70 : || !new_lhs
3825 70 : || !INTEGRAL_TYPE_P (TREE_TYPE (new_lhs))
3826 140 : || TYPE_PRECISION (TREE_TYPE (new_lhs)) <= 1)
3827 : return;
3828 : lhs = new_lhs;
3829 : }
3830 70 : edge succ_edge = single_succ_edge (bb);
3831 70 : basic_block succ_bb = succ_edge->dest;
3832 70 : gsi = gsi_start_phis (succ_bb);
3833 70 : if (gsi_end_p (gsi))
3834 : return;
3835 70 : gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
3836 70 : gsi_next (&gsi);
3837 70 : if (!gsi_end_p (gsi))
3838 : return;
3839 70 : if (gimple_phi_arg_def (phi, succ_edge->dest_idx) != lhs)
3840 : return;
3841 70 : tree other_val = gimple_phi_arg_def (phi, other_edge->dest_idx);
3842 70 : if (gimple_assign_rhs_code (cond_stmt) == COND_EXPR)
3843 : {
3844 0 : tree cond = gimple_assign_rhs1 (cond_stmt);
3845 0 : if (TREE_CODE (cond) == NE_EXPR)
3846 : {
3847 0 : if (!operand_equal_p (other_val,
3848 0 : gimple_assign_rhs3 (cond_stmt), 0))
3849 : return;
3850 : }
3851 0 : else if (!operand_equal_p (other_val,
3852 0 : gimple_assign_rhs2 (cond_stmt), 0))
3853 : return;
3854 : }
3855 70 : else if (gimple_assign_rhs_code (cond_stmt) == NE_EXPR)
3856 : {
3857 41 : if (!integer_zerop (other_val))
3858 : return;
3859 : }
3860 29 : else if (!integer_onep (other_val))
3861 : return;
3862 : }
3863 110 : if (pred_edge->flags & EDGE_TRUE_VALUE)
3864 57 : gimple_cond_make_true (zero_cond);
3865 : else
3866 53 : gimple_cond_make_false (zero_cond);
3867 110 : update_stmt (zero_cond);
3868 110 : reset_flow_sensitive_info_in_bb (bb);
3869 110 : *cfg_changed = true;
3870 : }
3871 :
3872 : /* Helper function for arith_overflow_check_p. Return true
3873 : if VAL1 is equal to VAL2 cast to corresponding integral type
3874 : with other signedness or vice versa. */
3875 :
3876 : static bool
3877 382 : arith_cast_equal_p (tree val1, tree val2)
3878 : {
3879 382 : if (TREE_CODE (val1) == INTEGER_CST && TREE_CODE (val2) == INTEGER_CST)
3880 65 : return wi::eq_p (wi::to_wide (val1), wi::to_wide (val2));
3881 317 : else if (TREE_CODE (val1) != SSA_NAME || TREE_CODE (val2) != SSA_NAME)
3882 : return false;
3883 279 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (val1))
3884 279 : && gimple_assign_rhs1 (SSA_NAME_DEF_STMT (val1)) == val2)
3885 : return true;
3886 167 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (val2))
3887 167 : && gimple_assign_rhs1 (SSA_NAME_DEF_STMT (val2)) == val1)
3888 120 : return true;
3889 : return false;
3890 : }
3891 :
3892 : /* Helper function of match_arith_overflow. Return 1
3893 : if USE_STMT is unsigned overflow check ovf != 0 for
3894 : STMT, -1 if USE_STMT is unsigned overflow check ovf == 0
3895 : and 0 otherwise. */
3896 :
3897 : static int
3898 2957700 : arith_overflow_check_p (gimple *stmt, gimple *cast_stmt, gimple *&use_stmt,
3899 : tree maxval, tree *other)
3900 : {
3901 2957700 : enum tree_code ccode = ERROR_MARK;
3902 2957700 : tree crhs1 = NULL_TREE, crhs2 = NULL_TREE;
3903 2957700 : enum tree_code code = gimple_assign_rhs_code (stmt);
3904 5881437 : tree lhs = gimple_assign_lhs (cast_stmt ? cast_stmt : stmt);
3905 2957700 : tree rhs1 = gimple_assign_rhs1 (stmt);
3906 2957700 : tree rhs2 = gimple_assign_rhs2 (stmt);
3907 2957700 : tree multop = NULL_TREE, divlhs = NULL_TREE;
3908 2957700 : gimple *cur_use_stmt = use_stmt;
3909 :
3910 2957700 : if (code == MULT_EXPR)
3911 : {
3912 684318 : if (!is_gimple_assign (use_stmt))
3913 683966 : return 0;
3914 547883 : if (gimple_assign_rhs_code (use_stmt) != TRUNC_DIV_EXPR)
3915 : return 0;
3916 2217 : if (gimple_assign_rhs1 (use_stmt) != lhs)
3917 : return 0;
3918 2154 : if (cast_stmt)
3919 : {
3920 155 : if (arith_cast_equal_p (gimple_assign_rhs2 (use_stmt), rhs1))
3921 : multop = rhs2;
3922 81 : else if (arith_cast_equal_p (gimple_assign_rhs2 (use_stmt), rhs2))
3923 : multop = rhs1;
3924 : else
3925 : return 0;
3926 : }
3927 1999 : else if (gimple_assign_rhs2 (use_stmt) == rhs1)
3928 : multop = rhs2;
3929 1865 : else if (operand_equal_p (gimple_assign_rhs2 (use_stmt), rhs2, 0))
3930 : multop = rhs1;
3931 : else
3932 : return 0;
3933 356 : if (stmt_ends_bb_p (use_stmt))
3934 : return 0;
3935 356 : divlhs = gimple_assign_lhs (use_stmt);
3936 356 : if (!divlhs)
3937 : return 0;
3938 356 : use_operand_p use;
3939 356 : if (!single_imm_use (divlhs, &use, &cur_use_stmt))
3940 : return 0;
3941 352 : if (cast_stmt && gimple_assign_cast_p (cur_use_stmt))
3942 : {
3943 4 : tree cast_lhs = gimple_assign_lhs (cur_use_stmt);
3944 8 : if (INTEGRAL_TYPE_P (TREE_TYPE (cast_lhs))
3945 4 : && TYPE_UNSIGNED (TREE_TYPE (cast_lhs))
3946 4 : && (TYPE_PRECISION (TREE_TYPE (cast_lhs))
3947 4 : == TYPE_PRECISION (TREE_TYPE (divlhs)))
3948 8 : && single_imm_use (cast_lhs, &use, &cur_use_stmt))
3949 : {
3950 : cast_stmt = NULL;
3951 : divlhs = cast_lhs;
3952 : }
3953 : else
3954 : return 0;
3955 : }
3956 : }
3957 2273734 : if (gimple_code (cur_use_stmt) == GIMPLE_COND)
3958 : {
3959 575223 : ccode = gimple_cond_code (cur_use_stmt);
3960 575223 : crhs1 = gimple_cond_lhs (cur_use_stmt);
3961 575223 : crhs2 = gimple_cond_rhs (cur_use_stmt);
3962 : }
3963 1698511 : else if (is_gimple_assign (cur_use_stmt))
3964 : {
3965 802667 : if (gimple_assign_rhs_class (cur_use_stmt) == GIMPLE_BINARY_RHS)
3966 : {
3967 462732 : ccode = gimple_assign_rhs_code (cur_use_stmt);
3968 462732 : crhs1 = gimple_assign_rhs1 (cur_use_stmt);
3969 462732 : crhs2 = gimple_assign_rhs2 (cur_use_stmt);
3970 : }
3971 : else
3972 : return 0;
3973 : }
3974 : else
3975 : return 0;
3976 :
3977 1037955 : if (maxval
3978 1037955 : && ccode == RSHIFT_EXPR
3979 33 : && crhs1 == lhs
3980 17 : && TREE_CODE (crhs2) == INTEGER_CST
3981 1037972 : && wi::to_widest (crhs2) == TYPE_PRECISION (TREE_TYPE (maxval)))
3982 : {
3983 16 : tree shiftlhs = gimple_assign_lhs (use_stmt);
3984 16 : if (!shiftlhs)
3985 : return 0;
3986 16 : use_operand_p use;
3987 16 : if (!single_imm_use (shiftlhs, &use, &cur_use_stmt))
3988 : return 0;
3989 12 : if (gimple_code (cur_use_stmt) == GIMPLE_COND)
3990 : {
3991 0 : ccode = gimple_cond_code (cur_use_stmt);
3992 0 : crhs1 = gimple_cond_lhs (cur_use_stmt);
3993 0 : crhs2 = gimple_cond_rhs (cur_use_stmt);
3994 : }
3995 12 : else if (is_gimple_assign (cur_use_stmt))
3996 : {
3997 12 : if (gimple_assign_rhs_class (cur_use_stmt) == GIMPLE_BINARY_RHS)
3998 : {
3999 0 : ccode = gimple_assign_rhs_code (cur_use_stmt);
4000 0 : crhs1 = gimple_assign_rhs1 (cur_use_stmt);
4001 0 : crhs2 = gimple_assign_rhs2 (cur_use_stmt);
4002 : }
4003 12 : else if (gimple_assign_rhs_code (cur_use_stmt) == COND_EXPR)
4004 : {
4005 0 : tree cond = gimple_assign_rhs1 (cur_use_stmt);
4006 0 : if (COMPARISON_CLASS_P (cond))
4007 : {
4008 0 : ccode = TREE_CODE (cond);
4009 0 : crhs1 = TREE_OPERAND (cond, 0);
4010 0 : crhs2 = TREE_OPERAND (cond, 1);
4011 : }
4012 : else
4013 : return 0;
4014 : }
4015 : else
4016 : {
4017 12 : enum tree_code sc = gimple_assign_rhs_code (cur_use_stmt);
4018 12 : tree castlhs = gimple_assign_lhs (cur_use_stmt);
4019 12 : if (!CONVERT_EXPR_CODE_P (sc)
4020 12 : || !castlhs
4021 12 : || !INTEGRAL_TYPE_P (TREE_TYPE (castlhs))
4022 24 : || (TYPE_PRECISION (TREE_TYPE (castlhs))
4023 12 : > TYPE_PRECISION (TREE_TYPE (maxval))))
4024 0 : return 0;
4025 : return 1;
4026 : }
4027 : }
4028 : else
4029 : return 0;
4030 0 : if ((ccode != EQ_EXPR && ccode != NE_EXPR)
4031 0 : || crhs1 != shiftlhs
4032 0 : || !integer_zerop (crhs2))
4033 0 : return 0;
4034 : return 1;
4035 : }
4036 :
4037 1037939 : if (TREE_CODE_CLASS (ccode) != tcc_comparison)
4038 : return 0;
4039 :
4040 611542 : switch (ccode)
4041 : {
4042 114160 : case GT_EXPR:
4043 114160 : case LE_EXPR:
4044 114160 : if (maxval)
4045 : {
4046 : /* r = a + b; r > maxval or r <= maxval */
4047 36 : if (crhs1 == lhs
4048 35 : && TREE_CODE (crhs2) == INTEGER_CST
4049 49 : && tree_int_cst_equal (crhs2, maxval))
4050 13 : return ccode == GT_EXPR ? 1 : -1;
4051 : break;
4052 : }
4053 : /* r = a - b; r > a or r <= a
4054 : r = a + b; a > r or a <= r or b > r or b <= r. */
4055 114124 : if ((code == MINUS_EXPR && crhs1 == lhs && crhs2 == rhs1)
4056 114064 : || (code == PLUS_EXPR && (crhs1 == rhs1 || crhs1 == rhs2)
4057 7941 : && crhs2 == lhs))
4058 8001 : return ccode == GT_EXPR ? 1 : -1;
4059 : /* r = ~a; b > r or b <= r. */
4060 106123 : if (code == BIT_NOT_EXPR && crhs2 == lhs)
4061 : {
4062 232 : if (other)
4063 116 : *other = crhs1;
4064 232 : return ccode == GT_EXPR ? 1 : -1;
4065 : }
4066 : break;
4067 62433 : case LT_EXPR:
4068 62433 : case GE_EXPR:
4069 62433 : if (maxval)
4070 : break;
4071 : /* r = a - b; a < r or a >= r
4072 : r = a + b; r < a or r >= a or r < b or r >= b. */
4073 62427 : if ((code == MINUS_EXPR && crhs1 == rhs1 && crhs2 == lhs)
4074 62285 : || (code == PLUS_EXPR && crhs1 == lhs
4075 30210 : && (crhs2 == rhs1 || crhs2 == rhs2)))
4076 4155 : return ccode == LT_EXPR ? 1 : -1;
4077 : /* r = ~a; r < b or r >= b. */
4078 58272 : if (code == BIT_NOT_EXPR && crhs1 == lhs)
4079 : {
4080 263 : if (other)
4081 140 : *other = crhs2;
4082 263 : return ccode == LT_EXPR ? 1 : -1;
4083 : }
4084 : break;
4085 434949 : case EQ_EXPR:
4086 434949 : case NE_EXPR:
4087 : /* r = a * b; _1 = r / a; _1 == b
4088 : r = a * b; _1 = r / b; _1 == a
4089 : r = a * b; _1 = r / a; _1 != b
4090 : r = a * b; _1 = r / b; _1 != a. */
4091 434949 : if (code == MULT_EXPR)
4092 : {
4093 349 : if (cast_stmt)
4094 : {
4095 146 : if ((crhs1 == divlhs && arith_cast_equal_p (crhs2, multop))
4096 146 : || (crhs2 == divlhs && arith_cast_equal_p (crhs1, multop)))
4097 : {
4098 146 : use_stmt = cur_use_stmt;
4099 146 : return ccode == NE_EXPR ? 1 : -1;
4100 : }
4101 : }
4102 142 : else if ((crhs1 == divlhs && operand_equal_p (crhs2, multop, 0))
4103 203 : || (crhs2 == divlhs && crhs1 == multop))
4104 : {
4105 203 : use_stmt = cur_use_stmt;
4106 203 : return ccode == NE_EXPR ? 1 : -1;
4107 : }
4108 : }
4109 : break;
4110 : default:
4111 : break;
4112 : }
4113 : return 0;
4114 : }
4115 :
4116 : extern bool gimple_unsigned_integer_sat_add (tree, tree*, tree (*)(tree));
4117 : extern bool gimple_unsigned_integer_sat_sub (tree, tree*, tree (*)(tree));
4118 : extern bool gimple_unsigned_integer_sat_trunc (tree, tree*, tree (*)(tree));
4119 : extern bool gimple_unsigned_integer_sat_mul (tree, tree*, tree (*)(tree));
4120 : extern bool gimple_spaceship (tree, tree*, tree (*)(tree));
4121 :
4122 : extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree));
4123 : extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
4124 : extern bool gimple_signed_integer_sat_trunc (tree, tree*, tree (*)(tree));
4125 :
4126 : static bool
4127 341 : build_saturation_binary_arith_call_and_replace (gimple_stmt_iterator *gsi,
4128 : internal_fn fn, tree lhs,
4129 : tree op_0, tree op_1)
4130 : {
4131 341 : if (direct_internal_fn_supported_p (fn, TREE_TYPE (op_0), OPTIMIZE_FOR_BOTH))
4132 : {
4133 338 : gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
4134 338 : gimple_call_set_lhs (call, lhs);
4135 338 : gsi_replace (gsi, call, /* update_eh_info */ true);
4136 338 : return true;
4137 : }
4138 :
4139 : return false;
4140 : }
4141 :
4142 : static bool
4143 75 : build_saturation_binary_arith_call_and_insert (gimple_stmt_iterator *gsi,
4144 : internal_fn fn, tree lhs,
4145 : tree op_0, tree op_1)
4146 : {
4147 75 : if (!direct_internal_fn_supported_p (fn, TREE_TYPE (op_0), OPTIMIZE_FOR_BOTH))
4148 : return false;
4149 :
4150 67 : gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
4151 67 : gimple_call_set_lhs (call, lhs);
4152 67 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4153 :
4154 67 : return true;
4155 : }
4156 :
4157 : /*
4158 : * Try to match saturation unsigned add with assign.
4159 : * _7 = _4 + _6;
4160 : * _8 = _4 > _7;
4161 : * _9 = (long unsigned int) _8;
4162 : * _10 = -_9;
4163 : * _12 = _7 | _10;
4164 : * =>
4165 : * _12 = .SAT_ADD (_4, _6);
4166 : *
4167 : * Try to match IMM=-1 saturation signed add with assign.
4168 : * <bb 2> [local count: 1073741824]:
4169 : * x.0_1 = (unsigned char) x_5(D);
4170 : * _3 = -x.0_1;
4171 : * _10 = (signed char) _3;
4172 : * _8 = x_5(D) & _10;
4173 : * if (_8 < 0)
4174 : * goto <bb 4>; [1.40%]
4175 : * else
4176 : * goto <bb 3>; [98.60%]
4177 : * <bb 3> [local count: 434070867]:
4178 : * _2 = x.0_1 + 255;
4179 : * <bb 4> [local count: 1073741824]:
4180 : * # _9 = PHI <_2(3), 128(2)>
4181 : * _4 = (int8_t) _9;
4182 : * =>
4183 : * _4 = .SAT_ADD (x_5, -1);
4184 : * Return true if the statement was replaced. */
4185 :
4186 : static bool
4187 4972489 : match_saturation_add_with_assign (gimple_stmt_iterator *gsi, gassign *stmt)
4188 : {
4189 4972489 : tree ops[2];
4190 4972489 : tree lhs = gimple_assign_lhs (stmt);
4191 :
4192 4972489 : if (gimple_unsigned_integer_sat_add (lhs, ops, NULL)
4193 4972489 : || gimple_signed_integer_sat_add (lhs, ops, NULL))
4194 201 : return build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_ADD,
4195 201 : lhs, ops[0], ops[1]);
4196 :
4197 : return false;
4198 : }
4199 :
4200 : /*
4201 : * Try to match saturation add with PHI.
4202 : * For unsigned integer:
4203 : * <bb 2> :
4204 : * _1 = x_3(D) + y_4(D);
4205 : * if (_1 >= x_3(D))
4206 : * goto <bb 3>; [INV]
4207 : * else
4208 : * goto <bb 4>; [INV]
4209 : *
4210 : * <bb 3> :
4211 : *
4212 : * <bb 4> :
4213 : * # _2 = PHI <255(2), _1(3)>
4214 : * =>
4215 : * <bb 4> [local count: 1073741824]:
4216 : * _2 = .SAT_ADD (x_4(D), y_5(D));
4217 : *
4218 : * For signed integer:
4219 : * x.0_1 = (long unsigned int) x_7(D);
4220 : * y.1_2 = (long unsigned int) y_8(D);
4221 : * _3 = x.0_1 + y.1_2;
4222 : * sum_9 = (int64_t) _3;
4223 : * _4 = x_7(D) ^ y_8(D);
4224 : * _5 = x_7(D) ^ sum_9;
4225 : * _15 = ~_4;
4226 : * _16 = _5 & _15;
4227 : * if (_16 < 0)
4228 : * goto <bb 3>; [41.00%]
4229 : * else
4230 : * goto <bb 4>; [59.00%]
4231 : * _11 = x_7(D) < 0;
4232 : * _12 = (long int) _11;
4233 : * _13 = -_12;
4234 : * _14 = _13 ^ 9223372036854775807;
4235 : * # _6 = PHI <_14(3), sum_9(2)>
4236 : * =>
4237 : * _6 = .SAT_ADD (x_5(D), y_6(D)); [tail call] */
4238 :
4239 : static bool
4240 4304659 : match_saturation_add (gimple_stmt_iterator *gsi, gphi *phi)
4241 : {
4242 4304659 : if (gimple_phi_num_args (phi) != 2)
4243 : return false;
4244 :
4245 3422820 : tree ops[2];
4246 3422820 : tree phi_result = gimple_phi_result (phi);
4247 :
4248 3422820 : if (!gimple_unsigned_integer_sat_add (phi_result, ops, NULL)
4249 3422820 : && !gimple_signed_integer_sat_add (phi_result, ops, NULL))
4250 : return false;
4251 :
4252 23 : if (!TYPE_UNSIGNED (TREE_TYPE (ops[0])) && TREE_CODE (ops[1]) == INTEGER_CST)
4253 0 : ops[1] = fold_convert (TREE_TYPE (ops[0]), ops[1]);
4254 :
4255 23 : return build_saturation_binary_arith_call_and_insert (gsi, IFN_SAT_ADD,
4256 : phi_result, ops[0],
4257 23 : ops[1]);
4258 : }
4259 :
4260 : /*
4261 : * Try to match saturation unsigned sub.
4262 : * _1 = _4 >= _5;
4263 : * _3 = _4 - _5;
4264 : * _6 = _1 ? _3 : 0;
4265 : * =>
4266 : * _6 = .SAT_SUB (_4, _5);
4267 : * Return true if the statement was replaced. */
4268 :
4269 : static bool
4270 3710073 : match_unsigned_saturation_sub (gimple_stmt_iterator *gsi, gassign *stmt)
4271 : {
4272 3710073 : tree ops[2];
4273 3710073 : tree lhs = gimple_assign_lhs (stmt);
4274 :
4275 3710073 : if (gimple_unsigned_integer_sat_sub (lhs, ops, NULL))
4276 140 : return build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_SUB,
4277 140 : lhs, ops[0], ops[1]);
4278 :
4279 : return false;
4280 : }
4281 :
4282 : /*
4283 : * Try to match saturation unsigned mul.
4284 : * _1 = (unsigned int) a_6(D);
4285 : * _2 = (unsigned int) b_7(D);
4286 : * x_8 = _1 * _2;
4287 : * overflow_9 = x_8 > 255;
4288 : * _3 = (unsigned char) overflow_9;
4289 : * _4 = -_3;
4290 : * _5 = (unsigned char) x_8;
4291 : * _10 = _4 | _5;
4292 : * =>
4293 : * _10 = .SAT_SUB (a_6, b_7); */
4294 :
4295 : static void
4296 2659093 : match_unsigned_saturation_mul (gimple_stmt_iterator *gsi, gassign *stmt)
4297 : {
4298 2659093 : tree ops[2];
4299 2659093 : tree lhs = gimple_assign_lhs (stmt);
4300 :
4301 2659093 : if (gimple_unsigned_integer_sat_mul (lhs, ops, NULL))
4302 0 : build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_MUL, lhs,
4303 : ops[0], ops[1]);
4304 2659093 : }
4305 :
4306 : /* Try to match saturation unsigned mul, aka:
4307 : _6 = .MUL_OVERFLOW (a_4(D), b_5(D));
4308 : _2 = IMAGPART_EXPR <_6>;
4309 : if (_2 != 0)
4310 : goto <bb 4>; [35.00%]
4311 : else
4312 : goto <bb 3>; [65.00%]
4313 :
4314 : <bb 3> [local count: 697932184]:
4315 : _1 = REALPART_EXPR <_6>;
4316 :
4317 : <bb 4> [local count: 1073741824]:
4318 : # _3 = PHI <18446744073709551615(2), _1(3)>
4319 : =>
4320 : _3 = .SAT_MUL (a_4(D), b_5(D)); */
4321 :
4322 : static bool
4323 4304592 : match_saturation_mul (gimple_stmt_iterator *gsi, gphi *phi)
4324 : {
4325 4304592 : if (gimple_phi_num_args (phi) != 2)
4326 : return false;
4327 :
4328 3422753 : tree ops[2];
4329 3422753 : tree phi_result = gimple_phi_result (phi);
4330 :
4331 3422753 : if (!gimple_unsigned_integer_sat_mul (phi_result, ops, NULL))
4332 : return false;
4333 :
4334 0 : return build_saturation_binary_arith_call_and_insert (gsi, IFN_SAT_MUL,
4335 : phi_result, ops[0],
4336 0 : ops[1]);
4337 : }
4338 :
4339 : /* Try to match variants of spaceship operation:
4340 : <bb 2>
4341 : if (a_3(D) >= b_4(D)) -- CMP_1
4342 : goto <bb 3>;
4343 : else
4344 : goto <bb 4>;
4345 :
4346 : <bb 3>
4347 : _1 = a_3(D) > b_4(D); -- CMP_2
4348 : _5 = (int) _1;
4349 :
4350 : <bb 4>
4351 : # _2 = PHI <-1(2), _5(3)>
4352 : =>
4353 : _2 = .SPACESHIP (a_3(D), b_4(D), -1);
4354 :
4355 : All possible canonical variants of the comparison operator in CMP_1 and
4356 : CMP_2 has been included in gimple_spaceship function. */
4357 : static bool
4358 4304592 : match_spaceship (gimple_stmt_iterator *gsi, gphi *phi)
4359 : {
4360 4304592 : if (gimple_phi_num_args (phi) != 2)
4361 : return false;
4362 3422753 : tree ops[2];
4363 3422753 : tree phi_result = gimple_phi_result (phi);
4364 :
4365 3422753 : if (!gimple_spaceship (phi_result, ops, NULL))
4366 : return false;
4367 :
4368 : /* Allow different modes as long as both are integral types. */
4369 224 : if (!INTEGRAL_TYPE_P (TREE_TYPE (phi_result))
4370 224 : || !INTEGRAL_TYPE_P (TREE_TYPE (ops[0])))
4371 : return false;
4372 :
4373 112 : tree ops_type = TREE_TYPE (ops[0]);
4374 112 : machine_mode ops_mode = TYPE_MODE (ops_type);
4375 112 : machine_mode promoted_mode = ops_mode;
4376 112 : tree promoted_type = ops_type;
4377 112 : bool is_unsigned = TYPE_UNSIGNED (ops_type);
4378 :
4379 : /* Check if spaceship optab is available for the operand mode.
4380 : If not, try promoting to a wider mode that is supported. */
4381 112 : if (optab_handler (spaceship_optab, ops_mode) == CODE_FOR_nothing)
4382 : {
4383 : /* Try promoting to wider modes (e.g., QI/HI -> SI -> DI). */
4384 : machine_mode wider_mode;
4385 0 : FOR_EACH_WIDER_MODE_FROM (wider_mode, ops_mode)
4386 : {
4387 0 : if (optab_handler (spaceship_optab, wider_mode)
4388 : != CODE_FOR_nothing)
4389 : {
4390 : /* Check if we can get a type for this mode with matching
4391 : signedness. */
4392 0 : promoted_type = lang_hooks.types.type_for_mode (wider_mode,
4393 : is_unsigned);
4394 0 : if (promoted_type != NULL_TREE && INTEGRAL_TYPE_P (promoted_type))
4395 : {
4396 : promoted_mode = wider_mode;
4397 : break;
4398 : }
4399 : }
4400 : }
4401 :
4402 : // If no suitable promoted mode found, give up.
4403 0 : if (promoted_mode == ops_mode)
4404 4304592 : return false;
4405 : }
4406 :
4407 : /* If promotion is needed, insert conversion statements.
4408 : We must use GIMPLE assignments rather than fold_convert because
4409 : gimple_call arguments must be valid GIMPLE values (SSA names or
4410 : constants), not tree expressions. */
4411 112 : ops[0] = gimple_convert (gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
4412 : promoted_type, ops[0]);
4413 112 : ops[1] = gimple_convert (gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
4414 : promoted_type, ops[1]);
4415 :
4416 112 : tree spaceship_arg_3 = is_unsigned ? build_one_cst (integer_type_node)
4417 96 : : build_minus_one_cst (integer_type_node);
4418 :
4419 112 : gcall *call = gimple_build_call_internal (IFN_SPACESHIP, 3, ops[0], ops[1],
4420 : spaceship_arg_3);
4421 :
4422 : /* SPACESHIP optab always returns signed int (SI mode).
4423 : Cast to phi_result's type if needed. */
4424 112 : tree call_result_type = integer_type_node;
4425 112 : if (!types_compatible_p (TREE_TYPE (phi_result), call_result_type))
4426 : {
4427 48 : tree call_result = make_ssa_name (call_result_type);
4428 48 : gimple_call_set_lhs (call, call_result);
4429 48 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4430 48 : gassign *cast_stmt = gimple_build_assign (phi_result, NOP_EXPR,
4431 : call_result);
4432 48 : gsi_insert_before (gsi, cast_stmt, GSI_SAME_STMT);
4433 : }
4434 : else
4435 : {
4436 64 : gimple_call_set_lhs (call, phi_result);
4437 64 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4438 : }
4439 : return true;
4440 : }
4441 :
4442 :
4443 : /*
4444 : * Try to match saturation unsigned sub.
4445 : * <bb 2> [local count: 1073741824]:
4446 : * if (x_2(D) > y_3(D))
4447 : * goto <bb 3>; [50.00%]
4448 : * else
4449 : * goto <bb 4>; [50.00%]
4450 : *
4451 : * <bb 3> [local count: 536870912]:
4452 : * _4 = x_2(D) - y_3(D);
4453 : *
4454 : * <bb 4> [local count: 1073741824]:
4455 : * # _1 = PHI <0(2), _4(3)>
4456 : * =>
4457 : * <bb 4> [local count: 1073741824]:
4458 : * _1 = .SAT_SUB (x_2(D), y_3(D)); */
4459 : static bool
4460 4304640 : match_saturation_sub (gimple_stmt_iterator *gsi, gphi *phi)
4461 : {
4462 4304640 : if (gimple_phi_num_args (phi) != 2)
4463 : return false;
4464 :
4465 3422801 : tree ops[2];
4466 3422801 : tree phi_result = gimple_phi_result (phi);
4467 :
4468 3422801 : if (!gimple_unsigned_integer_sat_sub (phi_result, ops, NULL)
4469 3422801 : && !gimple_signed_integer_sat_sub (phi_result, ops, NULL))
4470 : return false;
4471 :
4472 52 : return build_saturation_binary_arith_call_and_insert (gsi, IFN_SAT_SUB,
4473 : phi_result, ops[0],
4474 52 : ops[1]);
4475 : }
4476 :
4477 : /*
4478 : * Try to match saturation unsigned sub.
4479 : * uint16_t x_4(D);
4480 : * uint8_t _6;
4481 : * overflow_5 = x_4(D) > 255;
4482 : * _1 = (unsigned char) x_4(D);
4483 : * _2 = (unsigned char) overflow_5;
4484 : * _3 = -_2;
4485 : * _6 = _1 | _3;
4486 : * =>
4487 : * _6 = .SAT_TRUNC (x_4(D));
4488 : * */
4489 : static void
4490 2659093 : match_unsigned_saturation_trunc (gimple_stmt_iterator *gsi, gassign *stmt)
4491 : {
4492 2659093 : tree ops[1];
4493 2659093 : tree lhs = gimple_assign_lhs (stmt);
4494 2659093 : tree type = TREE_TYPE (lhs);
4495 :
4496 2659093 : if (gimple_unsigned_integer_sat_trunc (lhs, ops, NULL)
4497 2659228 : && direct_internal_fn_supported_p (IFN_SAT_TRUNC,
4498 135 : tree_pair (type, TREE_TYPE (ops[0])),
4499 : OPTIMIZE_FOR_BOTH))
4500 : {
4501 108 : gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[0]);
4502 108 : gimple_call_set_lhs (call, lhs);
4503 108 : gsi_replace (gsi, call, /* update_eh_info */ true);
4504 : }
4505 2659093 : }
4506 :
4507 : /*
4508 : * Try to match saturation truncate.
4509 : * Aka:
4510 : * x.0_1 = (unsigned long) x_4(D);
4511 : * _2 = x.0_1 + 2147483648;
4512 : * if (_2 > 4294967295)
4513 : * goto <bb 4>; [50.00%]
4514 : * else
4515 : * goto <bb 3>; [50.00%]
4516 : * ;; succ: 4
4517 : * ;; 3
4518 : *
4519 : * ;; basic block 3, loop depth 0
4520 : * ;; pred: 2
4521 : * trunc_5 = (int32_t) x_4(D);
4522 : * goto <bb 5>; [100.00%]
4523 : * ;; succ: 5
4524 : *
4525 : * ;; basic block 4, loop depth 0
4526 : * ;; pred: 2
4527 : * _7 = x_4(D) < 0;
4528 : * _8 = (int) _7;
4529 : * _9 = -_8;
4530 : * _10 = _9 ^ 2147483647;
4531 : * ;; succ: 5
4532 : *
4533 : * ;; basic block 5, loop depth 0
4534 : * ;; pred: 3
4535 : * ;; 4
4536 : * # _3 = PHI <trunc_5(3), _10(4)>
4537 : * =>
4538 : * _6 = .SAT_TRUNC (x_4(D));
4539 : */
4540 :
4541 : static bool
4542 4304592 : match_saturation_trunc (gimple_stmt_iterator *gsi, gphi *phi)
4543 : {
4544 4304592 : if (gimple_phi_num_args (phi) != 2)
4545 : return false;
4546 :
4547 3422753 : tree ops[1];
4548 3422753 : tree phi_result = gimple_phi_result (phi);
4549 3422753 : tree type = TREE_TYPE (phi_result);
4550 :
4551 3422753 : if (!gimple_unsigned_integer_sat_trunc (phi_result, ops, NULL)
4552 3422753 : && !gimple_signed_integer_sat_trunc (phi_result, ops, NULL))
4553 : return false;
4554 :
4555 0 : if (!direct_internal_fn_supported_p (IFN_SAT_TRUNC,
4556 0 : tree_pair (type, TREE_TYPE (ops[0])),
4557 : OPTIMIZE_FOR_BOTH))
4558 : return false;
4559 :
4560 0 : gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[0]);
4561 0 : gimple_call_set_lhs (call, phi_result);
4562 0 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4563 :
4564 0 : return true;
4565 : }
4566 :
4567 : /* Recognize for unsigned x
4568 : x = y - z;
4569 : if (x > y)
4570 : where there are other uses of x and replace it with
4571 : _7 = .SUB_OVERFLOW (y, z);
4572 : x = REALPART_EXPR <_7>;
4573 : _8 = IMAGPART_EXPR <_7>;
4574 : if (_8)
4575 : and similarly for addition.
4576 :
4577 : Also recognize:
4578 : yc = (type) y;
4579 : zc = (type) z;
4580 : x = yc + zc;
4581 : if (x > max)
4582 : where y and z have unsigned types with maximum max
4583 : and there are other uses of x and all of those cast x
4584 : back to that unsigned type and again replace it with
4585 : _7 = .ADD_OVERFLOW (y, z);
4586 : _9 = REALPART_EXPR <_7>;
4587 : _8 = IMAGPART_EXPR <_7>;
4588 : if (_8)
4589 : and replace (utype) x with _9.
4590 : Or with x >> popcount (max) instead of x > max.
4591 :
4592 : Also recognize:
4593 : x = ~z;
4594 : if (y > x)
4595 : and replace it with
4596 : _7 = .ADD_OVERFLOW (y, z);
4597 : _8 = IMAGPART_EXPR <_7>;
4598 : if (_8)
4599 :
4600 : And also recognize:
4601 : z = x * y;
4602 : if (x != 0)
4603 : goto <bb 3>; [50.00%]
4604 : else
4605 : goto <bb 4>; [50.00%]
4606 :
4607 : <bb 3> [local count: 536870913]:
4608 : _2 = z / x;
4609 : _9 = _2 != y;
4610 : _10 = (int) _9;
4611 :
4612 : <bb 4> [local count: 1073741824]:
4613 : # iftmp.0_3 = PHI <_10(3), 0(2)>
4614 : and replace it with
4615 : _7 = .MUL_OVERFLOW (x, y);
4616 : z = IMAGPART_EXPR <_7>;
4617 : _8 = IMAGPART_EXPR <_7>;
4618 : _9 = _8 != 0;
4619 : iftmp.0_3 = (int) _9; */
4620 :
4621 : static bool
4622 3398670 : match_arith_overflow (gimple_stmt_iterator *gsi, gimple *stmt,
4623 : enum tree_code code, bool *cfg_changed)
4624 : {
4625 3398670 : tree lhs = gimple_assign_lhs (stmt);
4626 3398670 : tree type = TREE_TYPE (lhs);
4627 3398670 : use_operand_p use_p;
4628 3398670 : imm_use_iterator iter;
4629 3398670 : bool use_seen = false;
4630 3398670 : bool ovf_use_seen = false;
4631 3398670 : gimple *use_stmt;
4632 3398670 : gimple *add_stmt = NULL;
4633 3398670 : bool add_first = false;
4634 3398670 : gimple *cond_stmt = NULL;
4635 3398670 : gimple *cast_stmt = NULL;
4636 3398670 : tree cast_lhs = NULL_TREE;
4637 :
4638 3398670 : gcc_checking_assert (code == PLUS_EXPR
4639 : || code == MINUS_EXPR
4640 : || code == MULT_EXPR
4641 : || code == BIT_NOT_EXPR);
4642 3398670 : if (!INTEGRAL_TYPE_P (type)
4643 2872150 : || !TYPE_UNSIGNED (type)
4644 1977464 : || has_zero_uses (lhs)
4645 3398670 : || (code != PLUS_EXPR
4646 1976972 : && code != MULT_EXPR
4647 173660 : && optab_handler (code == MINUS_EXPR ? usubv4_optab : uaddv4_optab,
4648 147586 : TYPE_MODE (type)) == CODE_FOR_nothing))
4649 : return false;
4650 :
4651 1975162 : tree rhs1 = gimple_assign_rhs1 (stmt);
4652 1975162 : tree rhs2 = gimple_assign_rhs2 (stmt);
4653 5459260 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
4654 : {
4655 3490077 : use_stmt = USE_STMT (use_p);
4656 3490077 : if (is_gimple_debug (use_stmt))
4657 584712 : continue;
4658 :
4659 2905365 : tree other = NULL_TREE;
4660 2905365 : if (arith_overflow_check_p (stmt, NULL, use_stmt, NULL_TREE, &other))
4661 : {
4662 6554 : if (code == BIT_NOT_EXPR)
4663 : {
4664 256 : gcc_assert (other);
4665 256 : if (TREE_CODE (other) != SSA_NAME)
4666 0 : return false;
4667 256 : if (rhs2 == NULL)
4668 256 : rhs2 = other;
4669 : else
4670 : return false;
4671 256 : cond_stmt = use_stmt;
4672 : }
4673 : ovf_use_seen = true;
4674 : }
4675 : else
4676 : {
4677 2898811 : use_seen = true;
4678 2898811 : if (code == MULT_EXPR
4679 2898811 : && cast_stmt == NULL
4680 2898811 : && gimple_assign_cast_p (use_stmt))
4681 : {
4682 32815 : cast_lhs = gimple_assign_lhs (use_stmt);
4683 65630 : if (INTEGRAL_TYPE_P (TREE_TYPE (cast_lhs))
4684 32268 : && !TYPE_UNSIGNED (TREE_TYPE (cast_lhs))
4685 60312 : && (TYPE_PRECISION (TREE_TYPE (cast_lhs))
4686 27497 : == TYPE_PRECISION (TREE_TYPE (lhs))))
4687 : cast_stmt = use_stmt;
4688 : else
4689 : cast_lhs = NULL_TREE;
4690 : }
4691 : }
4692 2905365 : if (ovf_use_seen && use_seen)
4693 : break;
4694 0 : }
4695 :
4696 1975162 : if (!ovf_use_seen
4697 1975162 : && code == MULT_EXPR
4698 455945 : && cast_stmt)
4699 : {
4700 27146 : if (TREE_CODE (rhs1) != SSA_NAME
4701 27146 : || (TREE_CODE (rhs2) != SSA_NAME && TREE_CODE (rhs2) != INTEGER_CST))
4702 : return false;
4703 62752 : FOR_EACH_IMM_USE_FAST (use_p, iter, cast_lhs)
4704 : {
4705 35606 : use_stmt = USE_STMT (use_p);
4706 35606 : if (is_gimple_debug (use_stmt))
4707 1740 : continue;
4708 :
4709 33866 : if (arith_overflow_check_p (stmt, cast_stmt, use_stmt,
4710 : NULL_TREE, NULL))
4711 35606 : ovf_use_seen = true;
4712 27146 : }
4713 27146 : }
4714 : else
4715 : {
4716 : cast_stmt = NULL;
4717 : cast_lhs = NULL_TREE;
4718 : }
4719 :
4720 1975162 : tree maxval = NULL_TREE;
4721 1975162 : if (!ovf_use_seen
4722 13061 : || (code != MULT_EXPR && (code == BIT_NOT_EXPR ? use_seen : !use_seen))
4723 6166 : || (code == PLUS_EXPR
4724 5827 : && optab_handler (uaddv4_optab,
4725 5827 : TYPE_MODE (type)) == CODE_FOR_nothing)
4726 1987936 : || (code == MULT_EXPR
4727 235 : && optab_handler (cast_stmt ? mulv4_optab : umulv4_optab,
4728 155 : TYPE_MODE (type)) == CODE_FOR_nothing
4729 3 : && (use_seen
4730 3 : || cast_stmt
4731 0 : || !can_mult_highpart_p (TYPE_MODE (type), true))))
4732 : {
4733 1968844 : if (code != PLUS_EXPR)
4734 : return false;
4735 1367534 : if (TREE_CODE (rhs1) != SSA_NAME
4736 1367534 : || !gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs1)))
4737 : return false;
4738 322718 : rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs1));
4739 322718 : tree type1 = TREE_TYPE (rhs1);
4740 322718 : if (!INTEGRAL_TYPE_P (type1)
4741 168856 : || !TYPE_UNSIGNED (type1)
4742 32352 : || TYPE_PRECISION (type1) >= TYPE_PRECISION (type)
4743 336939 : || (TYPE_PRECISION (type1)
4744 336939 : != GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (type1))))
4745 : return false;
4746 9537 : if (TREE_CODE (rhs2) == INTEGER_CST)
4747 : {
4748 3924 : if (wi::ne_p (wi::rshift (wi::to_wide (rhs2),
4749 3924 : TYPE_PRECISION (type1),
4750 7848 : UNSIGNED), 0))
4751 : return false;
4752 1420 : rhs2 = fold_convert (type1, rhs2);
4753 : }
4754 : else
4755 : {
4756 5613 : if (TREE_CODE (rhs2) != SSA_NAME
4757 5613 : || !gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs2)))
4758 : return false;
4759 2410 : rhs2 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs2));
4760 2410 : tree type2 = TREE_TYPE (rhs2);
4761 2410 : if (!INTEGRAL_TYPE_P (type2)
4762 1033 : || !TYPE_UNSIGNED (type2)
4763 332 : || TYPE_PRECISION (type2) >= TYPE_PRECISION (type)
4764 2714 : || (TYPE_PRECISION (type2)
4765 2714 : != GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (type2))))
4766 : return false;
4767 : }
4768 1711 : if (TYPE_PRECISION (type1) >= TYPE_PRECISION (TREE_TYPE (rhs2)))
4769 : type = type1;
4770 : else
4771 3 : type = TREE_TYPE (rhs2);
4772 :
4773 1711 : if (TREE_CODE (type) != INTEGER_TYPE
4774 3422 : || optab_handler (uaddv4_optab,
4775 1711 : TYPE_MODE (type)) == CODE_FOR_nothing)
4776 : return false;
4777 :
4778 1711 : maxval = wide_int_to_tree (type, wi::max_value (TYPE_PRECISION (type),
4779 : UNSIGNED));
4780 1711 : ovf_use_seen = false;
4781 1711 : use_seen = false;
4782 1711 : basic_block use_bb = NULL;
4783 1914 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
4784 : {
4785 1853 : use_stmt = USE_STMT (use_p);
4786 1853 : if (is_gimple_debug (use_stmt))
4787 138 : continue;
4788 :
4789 1715 : if (arith_overflow_check_p (stmt, NULL, use_stmt, maxval, NULL))
4790 : {
4791 13 : ovf_use_seen = true;
4792 13 : use_bb = gimple_bb (use_stmt);
4793 : }
4794 : else
4795 : {
4796 1702 : if (!gimple_assign_cast_p (use_stmt)
4797 1702 : || gimple_assign_rhs_code (use_stmt) == VIEW_CONVERT_EXPR)
4798 : return false;
4799 113 : tree use_lhs = gimple_assign_lhs (use_stmt);
4800 226 : if (!INTEGRAL_TYPE_P (TREE_TYPE (use_lhs))
4801 226 : || (TYPE_PRECISION (TREE_TYPE (use_lhs))
4802 113 : > TYPE_PRECISION (type)))
4803 : return false;
4804 : use_seen = true;
4805 : }
4806 1650 : }
4807 61 : if (!ovf_use_seen)
4808 : return false;
4809 13 : if (!useless_type_conversion_p (type, TREE_TYPE (rhs1)))
4810 : {
4811 2 : if (!use_seen)
4812 : return false;
4813 2 : tree new_rhs1 = make_ssa_name (type);
4814 2 : gimple *g = gimple_build_assign (new_rhs1, NOP_EXPR, rhs1);
4815 2 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4816 2 : rhs1 = new_rhs1;
4817 : }
4818 11 : else if (!useless_type_conversion_p (type, TREE_TYPE (rhs2)))
4819 : {
4820 2 : if (!use_seen)
4821 : return false;
4822 2 : tree new_rhs2 = make_ssa_name (type);
4823 2 : gimple *g = gimple_build_assign (new_rhs2, NOP_EXPR, rhs2);
4824 2 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4825 2 : rhs2 = new_rhs2;
4826 : }
4827 9 : else if (!use_seen)
4828 : {
4829 : /* If there are no uses of the wider addition, check if
4830 : forwprop has not created a narrower addition.
4831 : Require it to be in the same bb as the overflow check. */
4832 12 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs1)
4833 : {
4834 11 : use_stmt = USE_STMT (use_p);
4835 11 : if (is_gimple_debug (use_stmt))
4836 0 : continue;
4837 :
4838 11 : if (use_stmt == stmt)
4839 0 : continue;
4840 :
4841 11 : if (!is_gimple_assign (use_stmt)
4842 11 : || gimple_bb (use_stmt) != use_bb
4843 22 : || gimple_assign_rhs_code (use_stmt) != PLUS_EXPR)
4844 3 : continue;
4845 :
4846 8 : if (gimple_assign_rhs1 (use_stmt) == rhs1)
4847 : {
4848 8 : if (!operand_equal_p (gimple_assign_rhs2 (use_stmt),
4849 : rhs2, 0))
4850 0 : continue;
4851 : }
4852 0 : else if (gimple_assign_rhs2 (use_stmt) == rhs1)
4853 : {
4854 0 : if (gimple_assign_rhs1 (use_stmt) != rhs2)
4855 0 : continue;
4856 : }
4857 : else
4858 0 : continue;
4859 :
4860 8 : add_stmt = use_stmt;
4861 8 : break;
4862 9 : }
4863 9 : if (add_stmt == NULL)
4864 : return false;
4865 :
4866 : /* If stmt and add_stmt are in the same bb, we need to find out
4867 : which one is earlier. If they are in different bbs, we've
4868 : checked add_stmt is in the same bb as one of the uses of the
4869 : stmt lhs, so stmt needs to dominate add_stmt too. */
4870 8 : if (gimple_bb (stmt) == gimple_bb (add_stmt))
4871 : {
4872 8 : gimple_stmt_iterator gsif = *gsi;
4873 8 : gimple_stmt_iterator gsib = *gsi;
4874 8 : int i;
4875 : /* Search both forward and backward from stmt and have a small
4876 : upper bound. */
4877 20 : for (i = 0; i < 128; i++)
4878 : {
4879 20 : if (!gsi_end_p (gsib))
4880 : {
4881 18 : gsi_prev_nondebug (&gsib);
4882 18 : if (gsi_stmt (gsib) == add_stmt)
4883 : {
4884 : add_first = true;
4885 : break;
4886 : }
4887 : }
4888 2 : else if (gsi_end_p (gsif))
4889 : break;
4890 18 : if (!gsi_end_p (gsif))
4891 : {
4892 18 : gsi_next_nondebug (&gsif);
4893 18 : if (gsi_stmt (gsif) == add_stmt)
4894 : break;
4895 : }
4896 : }
4897 8 : if (i == 128)
4898 0 : return false;
4899 8 : if (add_first)
4900 2 : *gsi = gsi_for_stmt (add_stmt);
4901 : }
4902 : }
4903 : }
4904 :
4905 6330 : if (code == BIT_NOT_EXPR)
4906 239 : *gsi = gsi_for_stmt (cond_stmt);
4907 :
4908 6330 : auto_vec<gimple *, 8> mul_stmts;
4909 6330 : if (code == MULT_EXPR && cast_stmt)
4910 : {
4911 75 : type = TREE_TYPE (cast_lhs);
4912 75 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
4913 75 : if (gimple_assign_cast_p (g)
4914 38 : && useless_type_conversion_p (type,
4915 38 : TREE_TYPE (gimple_assign_rhs1 (g)))
4916 113 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g)))
4917 : rhs1 = gimple_assign_rhs1 (g);
4918 : else
4919 : {
4920 37 : g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, rhs1);
4921 37 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4922 37 : rhs1 = gimple_assign_lhs (g);
4923 37 : mul_stmts.quick_push (g);
4924 : }
4925 75 : if (TREE_CODE (rhs2) == INTEGER_CST)
4926 32 : rhs2 = fold_convert (type, rhs2);
4927 : else
4928 : {
4929 43 : g = SSA_NAME_DEF_STMT (rhs2);
4930 43 : if (gimple_assign_cast_p (g)
4931 22 : && useless_type_conversion_p (type,
4932 22 : TREE_TYPE (gimple_assign_rhs1 (g)))
4933 65 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g)))
4934 : rhs2 = gimple_assign_rhs1 (g);
4935 : else
4936 : {
4937 21 : g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, rhs2);
4938 21 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4939 21 : rhs2 = gimple_assign_lhs (g);
4940 21 : mul_stmts.quick_push (g);
4941 : }
4942 : }
4943 : }
4944 6330 : internal_fn ifn = (code == MULT_EXPR
4945 6330 : ? IFN_MUL_OVERFLOW
4946 : : code != MINUS_EXPR
4947 6178 : ? IFN_ADD_OVERFLOW : IFN_SUB_OVERFLOW);
4948 : if (code != MINUS_EXPR
4949 6230 : && tree_swap_operands_p (rhs1, rhs2))
4950 : std::swap (rhs1, rhs2);
4951 :
4952 6330 : tree ctype = build_complex_type (type);
4953 6330 : gcall *g = gimple_build_call_internal (ifn, 2, rhs1, rhs2);
4954 6330 : tree ctmp = make_ssa_name (ctype);
4955 6330 : gimple_call_set_lhs (g, ctmp);
4956 6330 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4957 6330 : tree new_lhs = (maxval || cast_stmt) ? make_ssa_name (type) : lhs;
4958 6330 : gassign *g2;
4959 6330 : if (code != BIT_NOT_EXPR)
4960 : {
4961 6091 : g2 = gimple_build_assign (new_lhs, REALPART_EXPR,
4962 : build1 (REALPART_EXPR, type, ctmp));
4963 6091 : if (maxval || cast_stmt)
4964 : {
4965 87 : gsi_insert_before (gsi, g2, GSI_SAME_STMT);
4966 87 : if (add_first)
4967 2 : *gsi = gsi_for_stmt (stmt);
4968 : }
4969 : else
4970 6004 : gsi_replace (gsi, g2, true);
4971 6091 : if (code == MULT_EXPR)
4972 : {
4973 152 : mul_stmts.quick_push (g);
4974 152 : mul_stmts.quick_push (g2);
4975 152 : if (cast_stmt)
4976 : {
4977 75 : g2 = gimple_build_assign (lhs, NOP_EXPR, new_lhs);
4978 75 : gsi_replace (gsi, g2, true);
4979 75 : mul_stmts.quick_push (g2);
4980 : }
4981 : }
4982 : }
4983 6330 : tree ovf = make_ssa_name (type);
4984 6330 : g2 = gimple_build_assign (ovf, IMAGPART_EXPR,
4985 : build1 (IMAGPART_EXPR, type, ctmp));
4986 6330 : if (code != BIT_NOT_EXPR)
4987 6091 : gsi_insert_after (gsi, g2, GSI_NEW_STMT);
4988 : else
4989 239 : gsi_insert_before (gsi, g2, GSI_SAME_STMT);
4990 6330 : if (code == MULT_EXPR)
4991 152 : mul_stmts.quick_push (g2);
4992 :
4993 33623 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, cast_lhs ? cast_lhs : lhs)
4994 : {
4995 21038 : if (is_gimple_debug (use_stmt))
4996 4284 : continue;
4997 :
4998 16754 : gimple *orig_use_stmt = use_stmt;
4999 16754 : int ovf_use = arith_overflow_check_p (stmt, cast_stmt, use_stmt,
5000 : maxval, NULL);
5001 16754 : if (ovf_use == 0)
5002 : {
5003 10371 : gcc_assert (code != BIT_NOT_EXPR);
5004 10371 : if (maxval)
5005 : {
5006 4 : tree use_lhs = gimple_assign_lhs (use_stmt);
5007 4 : gimple_assign_set_rhs1 (use_stmt, new_lhs);
5008 4 : if (useless_type_conversion_p (TREE_TYPE (use_lhs),
5009 4 : TREE_TYPE (new_lhs)))
5010 4 : gimple_assign_set_rhs_code (use_stmt, SSA_NAME);
5011 4 : update_stmt (use_stmt);
5012 : }
5013 10371 : continue;
5014 10371 : }
5015 6383 : if (gimple_code (use_stmt) == GIMPLE_COND)
5016 : {
5017 4230 : gcond *cond_stmt = as_a <gcond *> (use_stmt);
5018 4230 : gimple_cond_set_lhs (cond_stmt, ovf);
5019 4230 : gimple_cond_set_rhs (cond_stmt, build_int_cst (type, 0));
5020 4460 : gimple_cond_set_code (cond_stmt, ovf_use == 1 ? NE_EXPR : EQ_EXPR);
5021 : }
5022 : else
5023 : {
5024 2153 : gcc_checking_assert (is_gimple_assign (use_stmt));
5025 2153 : if (gimple_assign_rhs_class (use_stmt) == GIMPLE_BINARY_RHS)
5026 : {
5027 2153 : if (gimple_assign_rhs_code (use_stmt) == RSHIFT_EXPR)
5028 : {
5029 6 : g2 = gimple_build_assign (make_ssa_name (boolean_type_node),
5030 : ovf_use == 1 ? NE_EXPR : EQ_EXPR,
5031 : ovf, build_int_cst (type, 0));
5032 6 : gimple_stmt_iterator gsiu = gsi_for_stmt (use_stmt);
5033 6 : gsi_insert_before (&gsiu, g2, GSI_SAME_STMT);
5034 6 : gimple_assign_set_rhs_with_ops (&gsiu, NOP_EXPR,
5035 : gimple_assign_lhs (g2));
5036 6 : update_stmt (use_stmt);
5037 6 : use_operand_p use;
5038 6 : single_imm_use (gimple_assign_lhs (use_stmt), &use,
5039 : &use_stmt);
5040 6 : if (gimple_code (use_stmt) == GIMPLE_COND)
5041 : {
5042 0 : gcond *cond_stmt = as_a <gcond *> (use_stmt);
5043 0 : gimple_cond_set_lhs (cond_stmt, ovf);
5044 0 : gimple_cond_set_rhs (cond_stmt, build_int_cst (type, 0));
5045 : }
5046 : else
5047 : {
5048 6 : gcc_checking_assert (is_gimple_assign (use_stmt));
5049 6 : if (gimple_assign_rhs_class (use_stmt)
5050 : == GIMPLE_BINARY_RHS)
5051 : {
5052 0 : gimple_assign_set_rhs1 (use_stmt, ovf);
5053 0 : gimple_assign_set_rhs2 (use_stmt,
5054 : build_int_cst (type, 0));
5055 : }
5056 6 : else if (gimple_assign_cast_p (use_stmt))
5057 6 : gimple_assign_set_rhs1 (use_stmt, ovf);
5058 : else
5059 : {
5060 0 : tree_code sc = gimple_assign_rhs_code (use_stmt);
5061 0 : gcc_checking_assert (sc == COND_EXPR);
5062 0 : tree cond = gimple_assign_rhs1 (use_stmt);
5063 0 : cond = build2 (TREE_CODE (cond),
5064 : boolean_type_node, ovf,
5065 : build_int_cst (type, 0));
5066 0 : gimple_assign_set_rhs1 (use_stmt, cond);
5067 : }
5068 : }
5069 6 : update_stmt (use_stmt);
5070 6 : gsi_remove (&gsiu, true);
5071 6 : gsiu = gsi_for_stmt (g2);
5072 6 : gsi_remove (&gsiu, true);
5073 6 : continue;
5074 6 : }
5075 : else
5076 : {
5077 2147 : gimple_assign_set_rhs1 (use_stmt, ovf);
5078 2147 : gimple_assign_set_rhs2 (use_stmt, build_int_cst (type, 0));
5079 2219 : gimple_assign_set_rhs_code (use_stmt,
5080 : ovf_use == 1
5081 : ? NE_EXPR : EQ_EXPR);
5082 : }
5083 : }
5084 : else
5085 : {
5086 0 : gcc_checking_assert (gimple_assign_rhs_code (use_stmt)
5087 : == COND_EXPR);
5088 0 : tree cond = build2 (ovf_use == 1 ? NE_EXPR : EQ_EXPR,
5089 : boolean_type_node, ovf,
5090 : build_int_cst (type, 0));
5091 0 : gimple_assign_set_rhs1 (use_stmt, cond);
5092 : }
5093 : }
5094 6377 : update_stmt (use_stmt);
5095 6377 : if (code == MULT_EXPR && use_stmt != orig_use_stmt)
5096 : {
5097 173 : gimple_stmt_iterator gsi2 = gsi_for_stmt (orig_use_stmt);
5098 173 : maybe_optimize_guarding_check (mul_stmts, use_stmt, orig_use_stmt,
5099 : cfg_changed);
5100 173 : use_operand_p use;
5101 173 : gimple *cast_stmt;
5102 173 : if (single_imm_use (gimple_assign_lhs (orig_use_stmt), &use,
5103 : &cast_stmt)
5104 173 : && gimple_assign_cast_p (cast_stmt))
5105 : {
5106 2 : gimple_stmt_iterator gsi3 = gsi_for_stmt (cast_stmt);
5107 2 : gsi_remove (&gsi3, true);
5108 2 : release_ssa_name (gimple_assign_lhs (cast_stmt));
5109 : }
5110 173 : gsi_remove (&gsi2, true);
5111 173 : release_ssa_name (gimple_assign_lhs (orig_use_stmt));
5112 : }
5113 6330 : }
5114 6330 : if (maxval)
5115 : {
5116 12 : gimple_stmt_iterator gsi2 = gsi_for_stmt (stmt);
5117 12 : gsi_remove (&gsi2, true);
5118 12 : if (add_stmt)
5119 : {
5120 8 : gimple *g = gimple_build_assign (gimple_assign_lhs (add_stmt),
5121 : new_lhs);
5122 8 : gsi2 = gsi_for_stmt (add_stmt);
5123 8 : gsi_replace (&gsi2, g, true);
5124 : }
5125 : }
5126 6318 : else if (code == BIT_NOT_EXPR)
5127 : {
5128 239 : *gsi = gsi_for_stmt (stmt);
5129 239 : gsi_remove (gsi, true);
5130 239 : release_ssa_name (lhs);
5131 239 : return true;
5132 : }
5133 : return false;
5134 6330 : }
5135 :
5136 : /* Helper of match_uaddc_usubc. Look through an integral cast
5137 : which should preserve [0, 1] range value (unless source has
5138 : 1-bit signed type) and the cast has single use. */
5139 :
5140 : static gimple *
5141 2093392 : uaddc_cast (gimple *g)
5142 : {
5143 2093392 : if (!gimple_assign_cast_p (g))
5144 : return g;
5145 500214 : tree op = gimple_assign_rhs1 (g);
5146 500214 : if (TREE_CODE (op) == SSA_NAME
5147 421474 : && INTEGRAL_TYPE_P (TREE_TYPE (op))
5148 288716 : && (TYPE_PRECISION (TREE_TYPE (op)) > 1
5149 5809 : || TYPE_UNSIGNED (TREE_TYPE (op)))
5150 788930 : && has_single_use (gimple_assign_lhs (g)))
5151 180869 : return SSA_NAME_DEF_STMT (op);
5152 : return g;
5153 : }
5154 :
5155 : /* Helper of match_uaddc_usubc. Look through a NE_EXPR
5156 : comparison with 0 which also preserves [0, 1] value range. */
5157 :
5158 : static gimple *
5159 2093552 : uaddc_ne0 (gimple *g)
5160 : {
5161 2093552 : if (is_gimple_assign (g)
5162 1286641 : && gimple_assign_rhs_code (g) == NE_EXPR
5163 60504 : && integer_zerop (gimple_assign_rhs2 (g))
5164 6485 : && TREE_CODE (gimple_assign_rhs1 (g)) == SSA_NAME
5165 2100025 : && has_single_use (gimple_assign_lhs (g)))
5166 6188 : return SSA_NAME_DEF_STMT (gimple_assign_rhs1 (g));
5167 : return g;
5168 : }
5169 :
5170 : /* Return true if G is {REAL,IMAG}PART_EXPR PART with SSA_NAME
5171 : operand. */
5172 :
5173 : static bool
5174 2094405 : uaddc_is_cplxpart (gimple *g, tree_code part)
5175 : {
5176 2094405 : return (is_gimple_assign (g)
5177 1286089 : && gimple_assign_rhs_code (g) == part
5178 2098153 : && TREE_CODE (TREE_OPERAND (gimple_assign_rhs1 (g), 0)) == SSA_NAME);
5179 : }
5180 :
5181 : /* Try to match e.g.
5182 : _29 = .ADD_OVERFLOW (_3, _4);
5183 : _30 = REALPART_EXPR <_29>;
5184 : _31 = IMAGPART_EXPR <_29>;
5185 : _32 = .ADD_OVERFLOW (_30, _38);
5186 : _33 = REALPART_EXPR <_32>;
5187 : _34 = IMAGPART_EXPR <_32>;
5188 : _35 = _31 + _34;
5189 : as
5190 : _36 = .UADDC (_3, _4, _38);
5191 : _33 = REALPART_EXPR <_36>;
5192 : _35 = IMAGPART_EXPR <_36>;
5193 : or
5194 : _22 = .SUB_OVERFLOW (_6, _5);
5195 : _23 = REALPART_EXPR <_22>;
5196 : _24 = IMAGPART_EXPR <_22>;
5197 : _25 = .SUB_OVERFLOW (_23, _37);
5198 : _26 = REALPART_EXPR <_25>;
5199 : _27 = IMAGPART_EXPR <_25>;
5200 : _28 = _24 | _27;
5201 : as
5202 : _29 = .USUBC (_6, _5, _37);
5203 : _26 = REALPART_EXPR <_29>;
5204 : _288 = IMAGPART_EXPR <_29>;
5205 : provided _38 or _37 above have [0, 1] range
5206 : and _3, _4 and _30 or _6, _5 and _23 are unsigned
5207 : integral types with the same precision. Whether + or | or ^ is
5208 : used on the IMAGPART_EXPR results doesn't matter, with one of
5209 : added or subtracted operands in [0, 1] range at most one
5210 : .ADD_OVERFLOW or .SUB_OVERFLOW will indicate overflow. */
5211 :
5212 : static bool
5213 2840259 : match_uaddc_usubc (gimple_stmt_iterator *gsi, gimple *stmt, tree_code code)
5214 : {
5215 2840259 : tree rhs[4];
5216 2840259 : rhs[0] = gimple_assign_rhs1 (stmt);
5217 2840259 : rhs[1] = gimple_assign_rhs2 (stmt);
5218 2840259 : rhs[2] = NULL_TREE;
5219 2840259 : rhs[3] = NULL_TREE;
5220 2840259 : tree type = TREE_TYPE (rhs[0]);
5221 2840259 : if (!INTEGRAL_TYPE_P (type) || !TYPE_UNSIGNED (type))
5222 : return false;
5223 :
5224 1663108 : auto_vec<gimple *, 2> temp_stmts;
5225 1663108 : if (code != BIT_IOR_EXPR && code != BIT_XOR_EXPR)
5226 : {
5227 : /* If overflow flag is ignored on the MSB limb, we can end up with
5228 : the most significant limb handled as r = op1 + op2 + ovf1 + ovf2;
5229 : or r = op1 - op2 - ovf1 - ovf2; or various equivalent expressions
5230 : thereof. Handle those like the ovf = ovf1 + ovf2; case to recognize
5231 : the limb below the MSB, but also create another .UADDC/.USUBC call
5232 : for the last limb.
5233 :
5234 : First look through assignments with the same rhs code as CODE,
5235 : with the exception that subtraction of a constant is canonicalized
5236 : into addition of its negation. rhs[0] will be minuend for
5237 : subtractions and one of addends for addition, all other assigned
5238 : rhs[i] operands will be subtrahends or other addends. */
5239 1534111 : while (TREE_CODE (rhs[0]) == SSA_NAME && !rhs[3])
5240 : {
5241 1509688 : gimple *g = SSA_NAME_DEF_STMT (rhs[0]);
5242 1509688 : if (has_single_use (rhs[0])
5243 505513 : && is_gimple_assign (g)
5244 1945723 : && (gimple_assign_rhs_code (g) == code
5245 403696 : || (code == MINUS_EXPR
5246 51603 : && gimple_assign_rhs_code (g) == PLUS_EXPR
5247 15919 : && TREE_CODE (gimple_assign_rhs2 (g)) == INTEGER_CST)))
5248 : {
5249 44780 : tree r2 = gimple_assign_rhs2 (g);
5250 44780 : if (gimple_assign_rhs_code (g) != code)
5251 : {
5252 12441 : r2 = const_unop (NEGATE_EXPR, TREE_TYPE (r2), r2);
5253 12441 : if (!r2)
5254 : break;
5255 : }
5256 44780 : rhs[0] = gimple_assign_rhs1 (g);
5257 44780 : tree &r = rhs[2] ? rhs[3] : rhs[2];
5258 44780 : r = r2;
5259 44780 : temp_stmts.quick_push (g);
5260 : }
5261 : else
5262 : break;
5263 : }
5264 4467993 : for (int i = 1; i <= 2; ++i)
5265 3020327 : while (rhs[i] && TREE_CODE (rhs[i]) == SSA_NAME && !rhs[3])
5266 : {
5267 519960 : gimple *g = SSA_NAME_DEF_STMT (rhs[i]);
5268 519960 : if (has_single_use (rhs[i])
5269 258112 : && is_gimple_assign (g)
5270 760308 : && gimple_assign_rhs_code (g) == PLUS_EXPR)
5271 : {
5272 41665 : rhs[i] = gimple_assign_rhs1 (g);
5273 41665 : if (rhs[2])
5274 8628 : rhs[3] = gimple_assign_rhs2 (g);
5275 : else
5276 33037 : rhs[2] = gimple_assign_rhs2 (g);
5277 41665 : temp_stmts.quick_push (g);
5278 : }
5279 : else
5280 : break;
5281 : }
5282 : /* If there are just 3 addends or one minuend and two subtrahends,
5283 : check for UADDC or USUBC being pattern recognized earlier.
5284 : Say r = op1 + op2 + ovf1 + ovf2; where the (ovf1 + ovf2) part
5285 : got pattern matched earlier as __imag__ .UADDC (arg1, arg2, arg3)
5286 : etc. */
5287 1489331 : if (rhs[2] && !rhs[3])
5288 : {
5289 227686 : for (int i = (code == MINUS_EXPR ? 1 : 0); i < 3; ++i)
5290 167388 : if (TREE_CODE (rhs[i]) == SSA_NAME)
5291 : {
5292 128638 : gimple *im = uaddc_cast (SSA_NAME_DEF_STMT (rhs[i]));
5293 128638 : im = uaddc_ne0 (im);
5294 128638 : if (uaddc_is_cplxpart (im, IMAGPART_EXPR))
5295 : {
5296 : /* We found one of the 3 addends or 2 subtrahends to be
5297 : __imag__ of something, verify it is .UADDC/.USUBC. */
5298 236 : tree rhs1 = gimple_assign_rhs1 (im);
5299 236 : gimple *ovf = SSA_NAME_DEF_STMT (TREE_OPERAND (rhs1, 0));
5300 236 : tree ovf_lhs = NULL_TREE;
5301 236 : tree ovf_arg1 = NULL_TREE, ovf_arg2 = NULL_TREE;
5302 256 : if (gimple_call_internal_p (ovf, code == PLUS_EXPR
5303 : ? IFN_ADD_OVERFLOW
5304 : : IFN_SUB_OVERFLOW))
5305 : {
5306 : /* Or verify it is .ADD_OVERFLOW/.SUB_OVERFLOW.
5307 : This is for the case of 2 chained .UADDC/.USUBC,
5308 : where the first one uses 0 carry-in and the second
5309 : one ignores the carry-out.
5310 : So, something like:
5311 : _16 = .ADD_OVERFLOW (_1, _2);
5312 : _17 = REALPART_EXPR <_16>;
5313 : _18 = IMAGPART_EXPR <_16>;
5314 : _15 = _3 + _4;
5315 : _12 = _15 + _18;
5316 : where the first 3 statements come from the lower
5317 : limb addition and the last 2 from the higher limb
5318 : which ignores carry-out. */
5319 201 : ovf_lhs = gimple_call_lhs (ovf);
5320 201 : tree ovf_lhs_type = TREE_TYPE (TREE_TYPE (ovf_lhs));
5321 201 : ovf_arg1 = gimple_call_arg (ovf, 0);
5322 201 : ovf_arg2 = gimple_call_arg (ovf, 1);
5323 : /* In that case we need to punt if the types don't
5324 : mismatch. */
5325 201 : if (!types_compatible_p (type, ovf_lhs_type)
5326 201 : || !types_compatible_p (type, TREE_TYPE (ovf_arg1))
5327 399 : || !types_compatible_p (type,
5328 198 : TREE_TYPE (ovf_arg2)))
5329 : ovf_lhs = NULL_TREE;
5330 : else
5331 : {
5332 498 : for (int i = (code == PLUS_EXPR ? 1 : 0);
5333 498 : i >= 0; --i)
5334 : {
5335 354 : tree r = gimple_call_arg (ovf, i);
5336 354 : if (TREE_CODE (r) != SSA_NAME)
5337 0 : continue;
5338 354 : if (uaddc_is_cplxpart (SSA_NAME_DEF_STMT (r),
5339 : REALPART_EXPR))
5340 : {
5341 : /* Punt if one of the args which isn't
5342 : subtracted isn't __real__; that could
5343 : then prevent better match later.
5344 : Consider:
5345 : _3 = .ADD_OVERFLOW (_1, _2);
5346 : _4 = REALPART_EXPR <_3>;
5347 : _5 = IMAGPART_EXPR <_3>;
5348 : _7 = .ADD_OVERFLOW (_4, _6);
5349 : _8 = REALPART_EXPR <_7>;
5350 : _9 = IMAGPART_EXPR <_7>;
5351 : _12 = _10 + _11;
5352 : _13 = _12 + _9;
5353 : _14 = _13 + _5;
5354 : We want to match this when called on
5355 : the last stmt as a pair of .UADDC calls,
5356 : but without this check we could turn
5357 : that prematurely on _13 = _12 + _9;
5358 : stmt into .UADDC with 0 carry-in just
5359 : on the second .ADD_OVERFLOW call and
5360 : another replacing the _12 and _13
5361 : additions. */
5362 : ovf_lhs = NULL_TREE;
5363 : break;
5364 : }
5365 : }
5366 : }
5367 194 : if (ovf_lhs)
5368 : {
5369 144 : use_operand_p use_p;
5370 144 : imm_use_iterator iter;
5371 144 : tree re_lhs = NULL_TREE;
5372 432 : FOR_EACH_IMM_USE_FAST (use_p, iter, ovf_lhs)
5373 : {
5374 288 : gimple *use_stmt = USE_STMT (use_p);
5375 288 : if (is_gimple_debug (use_stmt))
5376 0 : continue;
5377 288 : if (use_stmt == im)
5378 144 : continue;
5379 144 : if (!uaddc_is_cplxpart (use_stmt,
5380 : REALPART_EXPR))
5381 : {
5382 : ovf_lhs = NULL_TREE;
5383 : break;
5384 : }
5385 144 : re_lhs = gimple_assign_lhs (use_stmt);
5386 144 : }
5387 144 : if (ovf_lhs && re_lhs)
5388 : {
5389 388 : FOR_EACH_IMM_USE_FAST (use_p, iter, re_lhs)
5390 : {
5391 300 : gimple *use_stmt = USE_STMT (use_p);
5392 300 : if (is_gimple_debug (use_stmt))
5393 109 : continue;
5394 191 : internal_fn ifn
5395 191 : = gimple_call_internal_fn (ovf);
5396 : /* Punt if the __real__ of lhs is used
5397 : in the same .*_OVERFLOW call.
5398 : Consider:
5399 : _3 = .ADD_OVERFLOW (_1, _2);
5400 : _4 = REALPART_EXPR <_3>;
5401 : _5 = IMAGPART_EXPR <_3>;
5402 : _7 = .ADD_OVERFLOW (_4, _6);
5403 : _8 = REALPART_EXPR <_7>;
5404 : _9 = IMAGPART_EXPR <_7>;
5405 : _12 = _10 + _11;
5406 : _13 = _12 + _5;
5407 : _14 = _13 + _9;
5408 : We want to match this when called on
5409 : the last stmt as a pair of .UADDC calls,
5410 : but without this check we could turn
5411 : that prematurely on _13 = _12 + _5;
5412 : stmt into .UADDC with 0 carry-in just
5413 : on the first .ADD_OVERFLOW call and
5414 : another replacing the _12 and _13
5415 : additions. */
5416 191 : if (gimple_call_internal_p (use_stmt, ifn))
5417 : {
5418 : ovf_lhs = NULL_TREE;
5419 : break;
5420 : }
5421 144 : }
5422 : }
5423 : }
5424 : }
5425 144 : if ((ovf_lhs
5426 157 : || gimple_call_internal_p (ovf,
5427 : code == PLUS_EXPR
5428 : ? IFN_UADDC : IFN_USUBC))
5429 252 : && (optab_handler (code == PLUS_EXPR
5430 : ? uaddc5_optab : usubc5_optab,
5431 94 : TYPE_MODE (type))
5432 : != CODE_FOR_nothing))
5433 : {
5434 : /* And in that case build another .UADDC/.USUBC
5435 : call for the most significand limb addition.
5436 : Overflow bit is ignored here. */
5437 63 : if (i != 2)
5438 63 : std::swap (rhs[i], rhs[2]);
5439 63 : gimple *g
5440 77 : = gimple_build_call_internal (code == PLUS_EXPR
5441 : ? IFN_UADDC
5442 : : IFN_USUBC,
5443 : 3, rhs[0], rhs[1],
5444 : rhs[2]);
5445 63 : tree nlhs = make_ssa_name (build_complex_type (type));
5446 63 : gimple_call_set_lhs (g, nlhs);
5447 63 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5448 63 : tree ilhs = gimple_assign_lhs (stmt);
5449 63 : g = gimple_build_assign (ilhs, REALPART_EXPR,
5450 : build1 (REALPART_EXPR,
5451 63 : TREE_TYPE (ilhs),
5452 : nlhs));
5453 63 : gsi_replace (gsi, g, true);
5454 : /* And if it is initialized from result of __imag__
5455 : of .{ADD,SUB}_OVERFLOW call, replace that
5456 : call with .U{ADD,SUB}C call with the same arguments,
5457 : just 0 added as third argument. This isn't strictly
5458 : necessary, .ADD_OVERFLOW (x, y) and .UADDC (x, y, 0)
5459 : produce the same result, but may result in better
5460 : generated code on some targets where the backend can
5461 : better prepare in how the result will be used. */
5462 63 : if (ovf_lhs)
5463 : {
5464 57 : tree zero = build_zero_cst (type);
5465 57 : g = gimple_build_call_internal (code == PLUS_EXPR
5466 : ? IFN_UADDC
5467 : : IFN_USUBC,
5468 : 3, ovf_arg1,
5469 : ovf_arg2, zero);
5470 57 : gimple_call_set_lhs (g, ovf_lhs);
5471 57 : gimple_stmt_iterator gsi2 = gsi_for_stmt (ovf);
5472 57 : gsi_replace (&gsi2, g, true);
5473 : }
5474 : return true;
5475 : }
5476 : }
5477 : }
5478 : return false;
5479 : }
5480 1428970 : if (code == MINUS_EXPR && !rhs[2])
5481 : return false;
5482 177 : if (code == MINUS_EXPR)
5483 : /* Code below expects rhs[0] and rhs[1] to have the IMAGPART_EXPRs.
5484 : So, for MINUS_EXPR swap the single added rhs operand (others are
5485 : subtracted) to rhs[3]. */
5486 177 : std::swap (rhs[0], rhs[3]);
5487 : }
5488 : /* Walk from both operands of STMT (for +/- even sometimes from
5489 : all the 4 addends or 3 subtrahends), see through casts and != 0
5490 : statements which would preserve [0, 1] range of values and
5491 : check which is initialized from __imag__. */
5492 1495127 : gimple *im1 = NULL, *im2 = NULL;
5493 14947934 : for (int i = 0; i < (code == MINUS_EXPR ? 3 : 4); i++)
5494 5979582 : if (rhs[i] && TREE_CODE (rhs[i]) == SSA_NAME)
5495 : {
5496 1964662 : gimple *im = uaddc_cast (SSA_NAME_DEF_STMT (rhs[i]));
5497 1964662 : im = uaddc_ne0 (im);
5498 1964662 : if (uaddc_is_cplxpart (im, IMAGPART_EXPR))
5499 : {
5500 2986 : if (im1 == NULL)
5501 : {
5502 2597 : im1 = im;
5503 2597 : if (i != 0)
5504 909 : std::swap (rhs[0], rhs[i]);
5505 : }
5506 : else
5507 : {
5508 389 : im2 = im;
5509 389 : if (i != 1)
5510 22 : std::swap (rhs[1], rhs[i]);
5511 : break;
5512 : }
5513 : }
5514 : }
5515 : /* If we don't find at least two, punt. */
5516 1495127 : if (!im2)
5517 : return false;
5518 : /* Check they are __imag__ of .ADD_OVERFLOW or .SUB_OVERFLOW call results,
5519 : either both .ADD_OVERFLOW or both .SUB_OVERFLOW and that we have
5520 : uaddc5/usubc5 named pattern for the corresponding mode. */
5521 389 : gimple *ovf1
5522 389 : = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (im1), 0));
5523 389 : gimple *ovf2
5524 389 : = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (im2), 0));
5525 389 : internal_fn ifn;
5526 389 : if (!is_gimple_call (ovf1)
5527 389 : || !gimple_call_internal_p (ovf1)
5528 389 : || ((ifn = gimple_call_internal_fn (ovf1)) != IFN_ADD_OVERFLOW
5529 61 : && ifn != IFN_SUB_OVERFLOW)
5530 366 : || !gimple_call_internal_p (ovf2, ifn)
5531 396 : || optab_handler (ifn == IFN_ADD_OVERFLOW ? uaddc5_optab : usubc5_optab,
5532 362 : TYPE_MODE (type)) == CODE_FOR_nothing
5533 95 : || (rhs[2]
5534 17 : && optab_handler (code == PLUS_EXPR ? uaddc5_optab : usubc5_optab,
5535 15 : TYPE_MODE (type)) == CODE_FOR_nothing)
5536 95 : || !types_compatible_p (type,
5537 95 : TREE_TYPE (TREE_TYPE (gimple_call_lhs (ovf1))))
5538 483 : || !types_compatible_p (type,
5539 94 : TREE_TYPE (TREE_TYPE (gimple_call_lhs (ovf2)))))
5540 : return false;
5541 94 : tree arg1, arg2, arg3 = NULL_TREE;
5542 94 : gimple *re1 = NULL, *re2 = NULL;
5543 : /* On one of the two calls, one of the .ADD_OVERFLOW/.SUB_OVERFLOW arguments
5544 : should be initialized from __real__ of the other of the two calls.
5545 : Though, for .SUB_OVERFLOW, it has to be the first argument, not the
5546 : second one. */
5547 249 : for (int i = (ifn == IFN_ADD_OVERFLOW ? 1 : 0); i >= 0; --i)
5548 351 : for (gimple *ovf = ovf1; ovf; ovf = (ovf == ovf1 ? ovf2 : NULL))
5549 : {
5550 290 : tree arg = gimple_call_arg (ovf, i);
5551 290 : if (TREE_CODE (arg) != SSA_NAME)
5552 2 : continue;
5553 288 : re1 = SSA_NAME_DEF_STMT (arg);
5554 288 : if (uaddc_is_cplxpart (re1, REALPART_EXPR)
5555 382 : && (SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (re1), 0))
5556 94 : == (ovf == ovf1 ? ovf2 : ovf1)))
5557 : {
5558 94 : if (ovf == ovf1)
5559 : {
5560 : /* Make sure ovf2 is the .*_OVERFLOW call with argument
5561 : initialized from __real__ of ovf1. */
5562 20 : std::swap (rhs[0], rhs[1]);
5563 20 : std::swap (im1, im2);
5564 20 : std::swap (ovf1, ovf2);
5565 : }
5566 94 : arg3 = gimple_call_arg (ovf, 1 - i);
5567 94 : i = -1;
5568 94 : break;
5569 : }
5570 : }
5571 94 : if (!arg3)
5572 : return false;
5573 94 : arg1 = gimple_call_arg (ovf1, 0);
5574 94 : arg2 = gimple_call_arg (ovf1, 1);
5575 94 : if (!types_compatible_p (type, TREE_TYPE (arg1)))
5576 : return false;
5577 94 : int kind[2] = { 0, 0 };
5578 94 : tree arg_im[2] = { NULL_TREE, NULL_TREE };
5579 : /* At least one of arg2 and arg3 should have type compatible
5580 : with arg1/rhs[0], and the other one should have value in [0, 1]
5581 : range. If both are in [0, 1] range and type compatible with
5582 : arg1/rhs[0], try harder to find after looking through casts,
5583 : != 0 comparisons which one is initialized to __imag__ of
5584 : .{ADD,SUB}_OVERFLOW or .U{ADD,SUB}C call results. */
5585 282 : for (int i = 0; i < 2; ++i)
5586 : {
5587 188 : tree arg = i == 0 ? arg2 : arg3;
5588 188 : if (types_compatible_p (type, TREE_TYPE (arg)))
5589 163 : kind[i] = 1;
5590 376 : if (!INTEGRAL_TYPE_P (TREE_TYPE (arg))
5591 376 : || (TYPE_PRECISION (TREE_TYPE (arg)) == 1
5592 25 : && !TYPE_UNSIGNED (TREE_TYPE (arg))))
5593 0 : continue;
5594 188 : if (tree_zero_one_valued_p (arg))
5595 52 : kind[i] |= 2;
5596 188 : if (TREE_CODE (arg) == SSA_NAME)
5597 : {
5598 185 : gimple *g = SSA_NAME_DEF_STMT (arg);
5599 185 : if (gimple_assign_cast_p (g))
5600 : {
5601 30 : tree op = gimple_assign_rhs1 (g);
5602 30 : if (TREE_CODE (op) == SSA_NAME
5603 30 : && INTEGRAL_TYPE_P (TREE_TYPE (op)))
5604 30 : g = SSA_NAME_DEF_STMT (op);
5605 : }
5606 185 : g = uaddc_ne0 (g);
5607 185 : if (!uaddc_is_cplxpart (g, IMAGPART_EXPR))
5608 125 : continue;
5609 60 : arg_im[i] = gimple_assign_lhs (g);
5610 60 : g = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (g), 0));
5611 60 : if (!is_gimple_call (g) || !gimple_call_internal_p (g))
5612 0 : continue;
5613 60 : switch (gimple_call_internal_fn (g))
5614 : {
5615 60 : case IFN_ADD_OVERFLOW:
5616 60 : case IFN_SUB_OVERFLOW:
5617 60 : case IFN_UADDC:
5618 60 : case IFN_USUBC:
5619 60 : break;
5620 0 : default:
5621 0 : continue;
5622 : }
5623 60 : kind[i] |= 4;
5624 : }
5625 : }
5626 : /* Make arg2 the one with compatible type and arg3 the one
5627 : with [0, 1] range. If both is true for both operands,
5628 : prefer as arg3 result of __imag__ of some ifn. */
5629 94 : if ((kind[0] & 1) == 0 || ((kind[1] & 1) != 0 && kind[0] > kind[1]))
5630 : {
5631 1 : std::swap (arg2, arg3);
5632 1 : std::swap (kind[0], kind[1]);
5633 1 : std::swap (arg_im[0], arg_im[1]);
5634 : }
5635 94 : if ((kind[0] & 1) == 0 || (kind[1] & 6) == 0)
5636 : return false;
5637 70 : if (!has_single_use (gimple_assign_lhs (im1))
5638 68 : || !has_single_use (gimple_assign_lhs (im2))
5639 68 : || !has_single_use (gimple_assign_lhs (re1))
5640 138 : || num_imm_uses (gimple_call_lhs (ovf1)) != 2)
5641 : return false;
5642 : /* Check that ovf2's result is used in __real__ and set re2
5643 : to that statement. */
5644 68 : use_operand_p use_p;
5645 68 : imm_use_iterator iter;
5646 68 : tree lhs = gimple_call_lhs (ovf2);
5647 203 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
5648 : {
5649 135 : gimple *use_stmt = USE_STMT (use_p);
5650 135 : if (is_gimple_debug (use_stmt))
5651 0 : continue;
5652 135 : if (use_stmt == im2)
5653 68 : continue;
5654 67 : if (re2)
5655 : return false;
5656 67 : if (!uaddc_is_cplxpart (use_stmt, REALPART_EXPR))
5657 : return false;
5658 : re2 = use_stmt;
5659 0 : }
5660 : /* Build .UADDC/.USUBC call which will be placed before the stmt. */
5661 68 : gimple_stmt_iterator gsi2 = gsi_for_stmt (ovf2);
5662 68 : gimple *g;
5663 68 : if ((kind[1] & 4) != 0 && types_compatible_p (type, TREE_TYPE (arg_im[1])))
5664 : arg3 = arg_im[1];
5665 68 : if ((kind[1] & 1) == 0)
5666 : {
5667 25 : if (TREE_CODE (arg3) == INTEGER_CST)
5668 0 : arg3 = fold_convert (type, arg3);
5669 : else
5670 : {
5671 25 : g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, arg3);
5672 25 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
5673 25 : arg3 = gimple_assign_lhs (g);
5674 : }
5675 : }
5676 91 : g = gimple_build_call_internal (ifn == IFN_ADD_OVERFLOW
5677 : ? IFN_UADDC : IFN_USUBC,
5678 : 3, arg1, arg2, arg3);
5679 68 : tree nlhs = make_ssa_name (TREE_TYPE (lhs));
5680 68 : gimple_call_set_lhs (g, nlhs);
5681 68 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
5682 : /* In the case where stmt is | or ^ of two overflow flags
5683 : or addition of those, replace stmt with __imag__ of the above
5684 : added call. In case of arg1 + arg2 + (ovf1 + ovf2) or
5685 : arg1 - arg2 - (ovf1 + ovf2) just emit it before stmt. */
5686 68 : tree ilhs = rhs[2] ? make_ssa_name (type) : gimple_assign_lhs (stmt);
5687 68 : g = gimple_build_assign (ilhs, IMAGPART_EXPR,
5688 68 : build1 (IMAGPART_EXPR, TREE_TYPE (ilhs), nlhs));
5689 68 : if (rhs[2])
5690 : {
5691 15 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5692 : /* Remove some further statements which can't be kept in the IL because
5693 : they can use SSA_NAMEs whose setter is going to be removed too. */
5694 75 : for (gimple *g2 : temp_stmts)
5695 : {
5696 30 : gsi2 = gsi_for_stmt (g2);
5697 30 : gsi_remove (&gsi2, true);
5698 30 : release_defs (g2);
5699 : }
5700 : }
5701 : else
5702 53 : gsi_replace (gsi, g, true);
5703 : /* Remove some statements which can't be kept in the IL because they
5704 : use SSA_NAME whose setter is going to be removed too. */
5705 68 : tree rhs1 = rhs[1];
5706 104 : for (int i = 0; i < 2; i++)
5707 86 : if (rhs1 == gimple_assign_lhs (im2))
5708 : break;
5709 : else
5710 : {
5711 36 : g = SSA_NAME_DEF_STMT (rhs1);
5712 36 : rhs1 = gimple_assign_rhs1 (g);
5713 36 : gsi2 = gsi_for_stmt (g);
5714 36 : gsi_remove (&gsi2, true);
5715 36 : release_defs (g);
5716 : }
5717 68 : gcc_checking_assert (rhs1 == gimple_assign_lhs (im2));
5718 68 : gsi2 = gsi_for_stmt (im2);
5719 68 : gsi_remove (&gsi2, true);
5720 68 : release_defs (im2);
5721 : /* Replace the re2 statement with __real__ of the newly added
5722 : .UADDC/.USUBC call. */
5723 68 : if (re2)
5724 : {
5725 67 : gsi2 = gsi_for_stmt (re2);
5726 67 : tree rlhs = gimple_assign_lhs (re2);
5727 67 : g = gimple_build_assign (rlhs, REALPART_EXPR,
5728 67 : build1 (REALPART_EXPR, TREE_TYPE (rlhs), nlhs));
5729 67 : gsi_replace (&gsi2, g, true);
5730 : }
5731 68 : if (rhs[2])
5732 : {
5733 : /* If this is the arg1 + arg2 + (ovf1 + ovf2) or
5734 : arg1 - arg2 - (ovf1 + ovf2) case for the most significant limb,
5735 : replace stmt with __real__ of another .UADDC/.USUBC call which
5736 : handles the most significant limb. Overflow flag from this is
5737 : ignored. */
5738 17 : g = gimple_build_call_internal (code == PLUS_EXPR
5739 : ? IFN_UADDC : IFN_USUBC,
5740 : 3, rhs[3], rhs[2], ilhs);
5741 15 : nlhs = make_ssa_name (TREE_TYPE (lhs));
5742 15 : gimple_call_set_lhs (g, nlhs);
5743 15 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5744 15 : ilhs = gimple_assign_lhs (stmt);
5745 15 : g = gimple_build_assign (ilhs, REALPART_EXPR,
5746 15 : build1 (REALPART_EXPR, TREE_TYPE (ilhs), nlhs));
5747 15 : gsi_replace (gsi, g, true);
5748 : }
5749 68 : if (TREE_CODE (arg3) == SSA_NAME)
5750 : {
5751 : /* When pattern recognizing the second least significant limb
5752 : above (i.e. first pair of .{ADD,SUB}_OVERFLOW calls for one limb),
5753 : check if the [0, 1] range argument (i.e. carry in) isn't the
5754 : result of another .{ADD,SUB}_OVERFLOW call (one handling the
5755 : least significant limb). Again look through casts and != 0. */
5756 67 : gimple *im3 = SSA_NAME_DEF_STMT (arg3);
5757 92 : for (int i = 0; i < 2; ++i)
5758 : {
5759 92 : gimple *im4 = uaddc_cast (im3);
5760 92 : if (im4 == im3)
5761 : break;
5762 : else
5763 25 : im3 = im4;
5764 : }
5765 67 : im3 = uaddc_ne0 (im3);
5766 67 : if (uaddc_is_cplxpart (im3, IMAGPART_EXPR))
5767 : {
5768 60 : gimple *ovf3
5769 60 : = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (im3), 0));
5770 60 : if (gimple_call_internal_p (ovf3, ifn))
5771 : {
5772 25 : lhs = gimple_call_lhs (ovf3);
5773 25 : arg1 = gimple_call_arg (ovf3, 0);
5774 25 : arg2 = gimple_call_arg (ovf3, 1);
5775 25 : if (types_compatible_p (type, TREE_TYPE (TREE_TYPE (lhs)))
5776 25 : && types_compatible_p (type, TREE_TYPE (arg1))
5777 50 : && types_compatible_p (type, TREE_TYPE (arg2)))
5778 : {
5779 : /* And if it is initialized from result of __imag__
5780 : of .{ADD,SUB}_OVERFLOW call, replace that
5781 : call with .U{ADD,SUB}C call with the same arguments,
5782 : just 0 added as third argument. This isn't strictly
5783 : necessary, .ADD_OVERFLOW (x, y) and .UADDC (x, y, 0)
5784 : produce the same result, but may result in better
5785 : generated code on some targets where the backend can
5786 : better prepare in how the result will be used. */
5787 25 : g = gimple_build_call_internal (ifn == IFN_ADD_OVERFLOW
5788 : ? IFN_UADDC : IFN_USUBC,
5789 : 3, arg1, arg2,
5790 : build_zero_cst (type));
5791 25 : gimple_call_set_lhs (g, lhs);
5792 25 : gsi2 = gsi_for_stmt (ovf3);
5793 25 : gsi_replace (&gsi2, g, true);
5794 : }
5795 : }
5796 : }
5797 : }
5798 : return true;
5799 1663108 : }
5800 :
5801 : /* Replace .POPCOUNT (x) == 1 or .POPCOUNT (x) != 1 with
5802 : (x & (x - 1)) > x - 1 or (x & (x - 1)) <= x - 1 if .POPCOUNT
5803 : isn't a direct optab. Also handle `<=`/`>` to be
5804 : `x & (x - 1) !=/== x`. */
5805 :
5806 : static void
5807 4546610 : match_single_bit_test (gimple_stmt_iterator *gsi, gimple *stmt)
5808 : {
5809 4546610 : tree clhs, crhs;
5810 4546610 : enum tree_code code;
5811 4546610 : bool was_le = false;
5812 4546610 : if (gimple_code (stmt) == GIMPLE_COND)
5813 : {
5814 4219657 : clhs = gimple_cond_lhs (stmt);
5815 4219657 : crhs = gimple_cond_rhs (stmt);
5816 4219657 : code = gimple_cond_code (stmt);
5817 : }
5818 : else
5819 : {
5820 326953 : clhs = gimple_assign_rhs1 (stmt);
5821 326953 : crhs = gimple_assign_rhs2 (stmt);
5822 326953 : code = gimple_assign_rhs_code (stmt);
5823 : }
5824 4546610 : if (code != LE_EXPR && code != GT_EXPR
5825 4546610 : && code != EQ_EXPR && code != NE_EXPR)
5826 4546568 : return;
5827 2134790 : if (code == LE_EXPR || code == GT_EXPR)
5828 4286129 : was_le = true;
5829 4286129 : if (TREE_CODE (clhs) != SSA_NAME || !integer_onep (crhs))
5830 : return;
5831 163459 : gimple *call = SSA_NAME_DEF_STMT (clhs);
5832 163459 : combined_fn cfn = gimple_call_combined_fn (call);
5833 163459 : switch (cfn)
5834 : {
5835 51 : CASE_CFN_POPCOUNT:
5836 51 : break;
5837 : default:
5838 : return;
5839 : }
5840 51 : if (!has_single_use (clhs))
5841 : return;
5842 50 : tree arg = gimple_call_arg (call, 0);
5843 50 : tree type = TREE_TYPE (arg);
5844 50 : if (!INTEGRAL_TYPE_P (type))
5845 : return;
5846 50 : bool nonzero_arg = tree_expr_nonzero_p (arg);
5847 50 : if (direct_internal_fn_supported_p (IFN_POPCOUNT, type, OPTIMIZE_FOR_BOTH))
5848 : {
5849 : /* Tell expand_POPCOUNT the popcount result is only used in equality
5850 : comparison with one, so that it can decide based on rtx costs. */
5851 16 : gimple *g = gimple_build_call_internal (IFN_POPCOUNT, 2, arg,
5852 : was_le ? integer_minus_one_node
5853 8 : : (nonzero_arg ? integer_zero_node
5854 : : integer_one_node));
5855 8 : gimple_call_set_lhs (g, gimple_call_lhs (call));
5856 8 : gimple_stmt_iterator gsi2 = gsi_for_stmt (call);
5857 8 : gsi_replace (&gsi2, g, true);
5858 8 : return;
5859 : }
5860 42 : tree argm1 = make_ssa_name (type);
5861 42 : gimple *g = gimple_build_assign (argm1, PLUS_EXPR, arg,
5862 : build_int_cst (type, -1));
5863 42 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5864 42 : g = gimple_build_assign (make_ssa_name (type),
5865 42 : (nonzero_arg || was_le) ? BIT_AND_EXPR : BIT_XOR_EXPR,
5866 : arg, argm1);
5867 42 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5868 42 : tree_code cmpcode;
5869 42 : if (was_le)
5870 : {
5871 0 : argm1 = build_zero_cst (type);
5872 0 : cmpcode = code == LE_EXPR ? EQ_EXPR : NE_EXPR;
5873 : }
5874 42 : else if (nonzero_arg)
5875 : {
5876 2 : argm1 = build_zero_cst (type);
5877 2 : cmpcode = code;
5878 : }
5879 : else
5880 40 : cmpcode = code == EQ_EXPR ? GT_EXPR : LE_EXPR;
5881 42 : if (gcond *cond = dyn_cast <gcond *> (stmt))
5882 : {
5883 2 : gimple_cond_set_lhs (cond, gimple_assign_lhs (g));
5884 2 : gimple_cond_set_rhs (cond, argm1);
5885 2 : gimple_cond_set_code (cond, cmpcode);
5886 : }
5887 : else
5888 : {
5889 40 : gimple_assign_set_rhs1 (stmt, gimple_assign_lhs (g));
5890 40 : gimple_assign_set_rhs2 (stmt, argm1);
5891 40 : gimple_assign_set_rhs_code (stmt, cmpcode);
5892 : }
5893 42 : update_stmt (stmt);
5894 42 : gimple_stmt_iterator gsi2 = gsi_for_stmt (call);
5895 42 : gsi_remove (&gsi2, true);
5896 42 : release_defs (call);
5897 : }
5898 :
5899 : /* Return true if target has support for divmod. */
5900 :
5901 : static bool
5902 38380 : target_supports_divmod_p (optab divmod_optab, optab div_optab, machine_mode mode)
5903 : {
5904 : /* If target supports hardware divmod insn, use it for divmod. */
5905 38380 : if (optab_handler (divmod_optab, mode) != CODE_FOR_nothing)
5906 : return true;
5907 :
5908 : /* Check if libfunc for divmod is available. */
5909 2580 : rtx libfunc = optab_libfunc (divmod_optab, mode);
5910 2580 : if (libfunc != NULL_RTX)
5911 : {
5912 : /* If optab_handler exists for div_optab, perhaps in a wider mode,
5913 : we don't want to use the libfunc even if it exists for given mode. */
5914 : machine_mode div_mode;
5915 10754 : FOR_EACH_MODE_FROM (div_mode, mode)
5916 8174 : if (optab_handler (div_optab, div_mode) != CODE_FOR_nothing)
5917 : return false;
5918 :
5919 2580 : return targetm.expand_divmod_libfunc != NULL;
5920 : }
5921 :
5922 : return false;
5923 : }
5924 :
5925 : /* Check if stmt is candidate for divmod transform. */
5926 :
5927 : static bool
5928 57127 : divmod_candidate_p (gassign *stmt)
5929 : {
5930 57127 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
5931 57127 : machine_mode mode = TYPE_MODE (type);
5932 57127 : optab divmod_optab, div_optab;
5933 :
5934 57127 : if (TYPE_UNSIGNED (type))
5935 : {
5936 : divmod_optab = udivmod_optab;
5937 : div_optab = udiv_optab;
5938 : }
5939 : else
5940 : {
5941 28390 : divmod_optab = sdivmod_optab;
5942 28390 : div_optab = sdiv_optab;
5943 : }
5944 :
5945 57127 : tree op1 = gimple_assign_rhs1 (stmt);
5946 57127 : tree op2 = gimple_assign_rhs2 (stmt);
5947 :
5948 : /* Disable the transform if either is a constant, since division-by-constant
5949 : may have specialized expansion. */
5950 57127 : if (CONSTANT_CLASS_P (op1))
5951 : return false;
5952 :
5953 53178 : if (CONSTANT_CLASS_P (op2))
5954 : {
5955 17037 : if (integer_pow2p (op2))
5956 : return false;
5957 :
5958 14910 : if (element_precision (type) <= HOST_BITS_PER_WIDE_INT
5959 15991 : && element_precision (type) <= BITS_PER_WORD)
5960 : return false;
5961 :
5962 : /* If the divisor is not power of 2 and the precision wider than
5963 : HWI, expand_divmod punts on that, so in that case it is better
5964 : to use divmod optab or libfunc. Similarly if choose_multiplier
5965 : might need pre/post shifts of BITS_PER_WORD or more. */
5966 : }
5967 :
5968 : /* Exclude the case where TYPE_OVERFLOW_TRAPS (type) as that should
5969 : expand using the [su]divv optabs. */
5970 38380 : if (TYPE_OVERFLOW_TRAPS (type))
5971 : return false;
5972 :
5973 38380 : if (!target_supports_divmod_p (divmod_optab, div_optab, mode))
5974 : return false;
5975 :
5976 : return true;
5977 : }
5978 :
5979 : /* This function looks for:
5980 : t1 = a TRUNC_DIV_EXPR b;
5981 : t2 = a TRUNC_MOD_EXPR b;
5982 : and transforms it to the following sequence:
5983 : complex_tmp = DIVMOD (a, b);
5984 : t1 = REALPART_EXPR(a);
5985 : t2 = IMAGPART_EXPR(b);
5986 : For conditions enabling the transform see divmod_candidate_p().
5987 :
5988 : The pass has three parts:
5989 : 1) Find top_stmt which is trunc_div or trunc_mod stmt and dominates all
5990 : other trunc_div_expr and trunc_mod_expr stmts.
5991 : 2) Add top_stmt and all trunc_div and trunc_mod stmts dominated by top_stmt
5992 : to stmts vector.
5993 : 3) Insert DIVMOD call just before top_stmt and update entries in
5994 : stmts vector to use return value of DIMOVD (REALEXPR_PART for div,
5995 : IMAGPART_EXPR for mod). */
5996 :
5997 : static bool
5998 57146 : convert_to_divmod (gassign *stmt)
5999 : {
6000 57146 : if (stmt_can_throw_internal (cfun, stmt)
6001 57146 : || !divmod_candidate_p (stmt))
6002 : return false;
6003 :
6004 38380 : tree op1 = gimple_assign_rhs1 (stmt);
6005 38380 : tree op2 = gimple_assign_rhs2 (stmt);
6006 :
6007 38380 : imm_use_iterator use_iter;
6008 38380 : gimple *use_stmt;
6009 38380 : auto_vec<gimple *> stmts;
6010 :
6011 38380 : gimple *top_stmt = stmt;
6012 38380 : basic_block top_bb = gimple_bb (stmt);
6013 :
6014 : /* Part 1: Try to set top_stmt to "topmost" stmt that dominates
6015 : at-least stmt and possibly other trunc_div/trunc_mod stmts
6016 : having same operands as stmt. */
6017 :
6018 326672 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, op1)
6019 : {
6020 288292 : if (is_gimple_assign (use_stmt)
6021 238013 : && (gimple_assign_rhs_code (use_stmt) == TRUNC_DIV_EXPR
6022 226245 : || gimple_assign_rhs_code (use_stmt) == TRUNC_MOD_EXPR)
6023 215602 : && operand_equal_p (op1, gimple_assign_rhs1 (use_stmt), 0)
6024 503777 : && operand_equal_p (op2, gimple_assign_rhs2 (use_stmt), 0))
6025 : {
6026 50038 : if (stmt_can_throw_internal (cfun, use_stmt))
6027 0 : continue;
6028 :
6029 50038 : basic_block bb = gimple_bb (use_stmt);
6030 :
6031 50038 : if (bb == top_bb)
6032 : {
6033 49307 : if (gimple_uid (use_stmt) < gimple_uid (top_stmt))
6034 5153 : top_stmt = use_stmt;
6035 : }
6036 731 : else if (dominated_by_p (CDI_DOMINATORS, top_bb, bb))
6037 : {
6038 194 : top_bb = bb;
6039 194 : top_stmt = use_stmt;
6040 : }
6041 : }
6042 38380 : }
6043 :
6044 38380 : tree top_op1 = gimple_assign_rhs1 (top_stmt);
6045 38380 : tree top_op2 = gimple_assign_rhs2 (top_stmt);
6046 :
6047 38380 : stmts.safe_push (top_stmt);
6048 38380 : bool div_seen = (gimple_assign_rhs_code (top_stmt) == TRUNC_DIV_EXPR);
6049 :
6050 : /* Part 2: Add all trunc_div/trunc_mod statements domianted by top_bb
6051 : to stmts vector. The 2nd loop will always add stmt to stmts vector, since
6052 : gimple_bb (top_stmt) dominates gimple_bb (stmt), so the
6053 : 2nd loop ends up adding at-least single trunc_mod_expr stmt. */
6054 :
6055 326672 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, top_op1)
6056 : {
6057 288292 : if (is_gimple_assign (use_stmt)
6058 238013 : && (gimple_assign_rhs_code (use_stmt) == TRUNC_DIV_EXPR
6059 226245 : || gimple_assign_rhs_code (use_stmt) == TRUNC_MOD_EXPR)
6060 215602 : && operand_equal_p (top_op1, gimple_assign_rhs1 (use_stmt), 0)
6061 503777 : && operand_equal_p (top_op2, gimple_assign_rhs2 (use_stmt), 0))
6062 : {
6063 88508 : if (use_stmt == top_stmt
6064 11658 : || stmt_can_throw_internal (cfun, use_stmt)
6065 61696 : || !dominated_by_p (CDI_DOMINATORS, gimple_bb (use_stmt), top_bb))
6066 38470 : continue;
6067 :
6068 11568 : stmts.safe_push (use_stmt);
6069 11568 : if (gimple_assign_rhs_code (use_stmt) == TRUNC_DIV_EXPR)
6070 288292 : div_seen = true;
6071 : }
6072 38380 : }
6073 :
6074 38380 : if (!div_seen)
6075 : return false;
6076 :
6077 : /* Part 3: Create libcall to internal fn DIVMOD:
6078 : divmod_tmp = DIVMOD (op1, op2). */
6079 :
6080 11541 : gcall *call_stmt = gimple_build_call_internal (IFN_DIVMOD, 2, op1, op2);
6081 11541 : tree res = make_temp_ssa_name (build_complex_type (TREE_TYPE (op1)),
6082 : call_stmt, "divmod_tmp");
6083 11541 : gimple_call_set_lhs (call_stmt, res);
6084 : /* We rejected throwing statements above. */
6085 11541 : gimple_call_set_nothrow (call_stmt, true);
6086 :
6087 : /* Insert the call before top_stmt. */
6088 11541 : gimple_stmt_iterator top_stmt_gsi = gsi_for_stmt (top_stmt);
6089 11541 : gsi_insert_before (&top_stmt_gsi, call_stmt, GSI_SAME_STMT);
6090 :
6091 11541 : widen_mul_stats.divmod_calls_inserted++;
6092 :
6093 : /* Update all statements in stmts vector:
6094 : lhs = op1 TRUNC_DIV_EXPR op2 -> lhs = REALPART_EXPR<divmod_tmp>
6095 : lhs = op1 TRUNC_MOD_EXPR op2 -> lhs = IMAGPART_EXPR<divmod_tmp>. */
6096 :
6097 73028 : for (unsigned i = 0; stmts.iterate (i, &use_stmt); ++i)
6098 : {
6099 23107 : tree new_rhs;
6100 :
6101 23107 : switch (gimple_assign_rhs_code (use_stmt))
6102 : {
6103 11551 : case TRUNC_DIV_EXPR:
6104 11551 : new_rhs = fold_build1 (REALPART_EXPR, TREE_TYPE (op1), res);
6105 11551 : break;
6106 :
6107 11556 : case TRUNC_MOD_EXPR:
6108 11556 : new_rhs = fold_build1 (IMAGPART_EXPR, TREE_TYPE (op1), res);
6109 11556 : break;
6110 :
6111 0 : default:
6112 0 : gcc_unreachable ();
6113 : }
6114 :
6115 23107 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
6116 23107 : gimple_assign_set_rhs_from_tree (&gsi, new_rhs);
6117 23107 : update_stmt (use_stmt);
6118 : }
6119 :
6120 : return true;
6121 38380 : }
6122 :
6123 : /* Process a single gimple assignment STMT, which has a RSHIFT_EXPR as
6124 : its rhs, and try to convert it into a MULT_HIGHPART_EXPR. The return
6125 : value is true iff we converted the statement. */
6126 :
6127 : static bool
6128 173600 : convert_mult_to_highpart (gassign *stmt, gimple_stmt_iterator *gsi)
6129 : {
6130 173600 : tree lhs = gimple_assign_lhs (stmt);
6131 173600 : tree stype = TREE_TYPE (lhs);
6132 173600 : tree sarg0 = gimple_assign_rhs1 (stmt);
6133 173600 : tree sarg1 = gimple_assign_rhs2 (stmt);
6134 :
6135 173600 : if (TREE_CODE (stype) != INTEGER_TYPE
6136 166342 : || TREE_CODE (sarg1) != INTEGER_CST
6137 149457 : || TREE_CODE (sarg0) != SSA_NAME
6138 149456 : || !tree_fits_uhwi_p (sarg1)
6139 323056 : || !has_single_use (sarg0))
6140 : return false;
6141 :
6142 48396 : gassign *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (sarg0));
6143 45092 : if (!def)
6144 : return false;
6145 :
6146 45092 : enum tree_code mcode = gimple_assign_rhs_code (def);
6147 45092 : if (mcode == NOP_EXPR)
6148 : {
6149 11009 : tree tmp = gimple_assign_rhs1 (def);
6150 11009 : if (TREE_CODE (tmp) != SSA_NAME || !has_single_use (tmp))
6151 : return false;
6152 3742 : def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (tmp));
6153 3449 : if (!def)
6154 : return false;
6155 3449 : mcode = gimple_assign_rhs_code (def);
6156 : }
6157 :
6158 37532 : if (mcode != WIDEN_MULT_EXPR
6159 37532 : || gimple_bb (def) != gimple_bb (stmt))
6160 : return false;
6161 2581 : tree mtype = TREE_TYPE (gimple_assign_lhs (def));
6162 2581 : if (TREE_CODE (mtype) != INTEGER_TYPE
6163 2581 : || TYPE_PRECISION (mtype) != TYPE_PRECISION (stype))
6164 : return false;
6165 :
6166 2581 : tree mop1 = gimple_assign_rhs1 (def);
6167 2581 : tree mop2 = gimple_assign_rhs2 (def);
6168 2581 : tree optype = TREE_TYPE (mop1);
6169 2581 : bool unsignedp = TYPE_UNSIGNED (optype);
6170 2581 : unsigned int prec = TYPE_PRECISION (optype);
6171 :
6172 2581 : if (unsignedp != TYPE_UNSIGNED (mtype)
6173 2581 : || TYPE_PRECISION (mtype) != 2 * prec)
6174 : return false;
6175 :
6176 2581 : unsigned HOST_WIDE_INT bits = tree_to_uhwi (sarg1);
6177 2581 : if (bits < prec || bits >= 2 * prec)
6178 : return false;
6179 :
6180 : /* For the time being, require operands to have the same sign. */
6181 2580 : if (unsignedp != TYPE_UNSIGNED (TREE_TYPE (mop2)))
6182 : return false;
6183 :
6184 2580 : machine_mode mode = TYPE_MODE (optype);
6185 2580 : optab tab = unsignedp ? umul_highpart_optab : smul_highpart_optab;
6186 2580 : if (optab_handler (tab, mode) == CODE_FOR_nothing)
6187 : return false;
6188 :
6189 2580 : location_t loc = gimple_location (stmt);
6190 2580 : tree highpart1 = build_and_insert_binop (gsi, loc, "highparttmp",
6191 : MULT_HIGHPART_EXPR, mop1, mop2);
6192 2580 : tree highpart2 = highpart1;
6193 2580 : tree ntype = optype;
6194 :
6195 2580 : if (TYPE_UNSIGNED (stype) != TYPE_UNSIGNED (optype))
6196 : {
6197 16 : ntype = TYPE_UNSIGNED (stype) ? unsigned_type_for (optype)
6198 7 : : signed_type_for (optype);
6199 16 : highpart2 = build_and_insert_cast (gsi, loc, ntype, highpart1);
6200 : }
6201 2580 : if (bits > prec)
6202 29 : highpart2 = build_and_insert_binop (gsi, loc, "highparttmp",
6203 : RSHIFT_EXPR, highpart2,
6204 29 : build_int_cst (ntype, bits - prec));
6205 :
6206 2580 : gassign *new_stmt = gimple_build_assign (lhs, NOP_EXPR, highpart2);
6207 2580 : gsi_replace (gsi, new_stmt, true);
6208 :
6209 2580 : widen_mul_stats.highpart_mults_inserted++;
6210 2580 : return true;
6211 : }
6212 :
6213 : /* If target has spaceship<MODE>3 expander, pattern recognize
6214 : <bb 2> [local count: 1073741824]:
6215 : if (a_2(D) == b_3(D))
6216 : goto <bb 6>; [34.00%]
6217 : else
6218 : goto <bb 3>; [66.00%]
6219 :
6220 : <bb 3> [local count: 708669601]:
6221 : if (a_2(D) < b_3(D))
6222 : goto <bb 6>; [1.04%]
6223 : else
6224 : goto <bb 4>; [98.96%]
6225 :
6226 : <bb 4> [local count: 701299439]:
6227 : if (a_2(D) > b_3(D))
6228 : goto <bb 5>; [48.89%]
6229 : else
6230 : goto <bb 6>; [51.11%]
6231 :
6232 : <bb 5> [local count: 342865295]:
6233 :
6234 : <bb 6> [local count: 1073741824]:
6235 : and turn it into:
6236 : <bb 2> [local count: 1073741824]:
6237 : _1 = .SPACESHIP (a_2(D), b_3(D), 0);
6238 : if (_1 == 0)
6239 : goto <bb 6>; [34.00%]
6240 : else
6241 : goto <bb 3>; [66.00%]
6242 :
6243 : <bb 3> [local count: 708669601]:
6244 : if (_1 == -1)
6245 : goto <bb 6>; [1.04%]
6246 : else
6247 : goto <bb 4>; [98.96%]
6248 :
6249 : <bb 4> [local count: 701299439]:
6250 : if (_1 == 1)
6251 : goto <bb 5>; [48.89%]
6252 : else
6253 : goto <bb 6>; [51.11%]
6254 :
6255 : <bb 5> [local count: 342865295]:
6256 :
6257 : <bb 6> [local count: 1073741824]:
6258 : so that the backend can emit optimal comparison and
6259 : conditional jump sequence. If the
6260 : <bb 6> [local count: 1073741824]:
6261 : above has a single PHI like:
6262 : # _27 = PHI<0(2), -1(3), -128(4), 1(5)>
6263 : then replace it with effectively
6264 : _1 = .SPACESHIP (a_2(D), b_3(D), -128);
6265 : _27 = _1; */
6266 :
6267 : static void
6268 4219657 : optimize_spaceship (gcond *stmt)
6269 : {
6270 4219657 : enum tree_code code = gimple_cond_code (stmt);
6271 4219657 : if (code != EQ_EXPR && code != NE_EXPR)
6272 4219533 : return;
6273 3425386 : tree arg1 = gimple_cond_lhs (stmt);
6274 3425386 : tree arg2 = gimple_cond_rhs (stmt);
6275 3425386 : if ((!SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg1))
6276 3314269 : && !INTEGRAL_TYPE_P (TREE_TYPE (arg1)))
6277 2650379 : || optab_handler (spaceship_optab,
6278 2650379 : TYPE_MODE (TREE_TYPE (arg1))) == CODE_FOR_nothing
6279 6035294 : || operand_equal_p (arg1, arg2, 0))
6280 : return;
6281 :
6282 2608685 : basic_block bb0 = gimple_bb (stmt), bb1, bb2 = NULL;
6283 2608685 : edge em1 = NULL, e1 = NULL, e2 = NULL;
6284 2608685 : bb1 = EDGE_SUCC (bb0, 1)->dest;
6285 2608685 : if (((EDGE_SUCC (bb0, 0)->flags & EDGE_TRUE_VALUE) != 0) ^ (code == EQ_EXPR))
6286 1578482 : bb1 = EDGE_SUCC (bb0, 0)->dest;
6287 :
6288 9395156 : gcond *g = safe_dyn_cast <gcond *> (*gsi_last_bb (bb1));
6289 1138870 : if (g == NULL
6290 4826492 : || !single_pred_p (bb1)
6291 724255 : || (operand_equal_p (gimple_cond_lhs (g), arg1, 0)
6292 606959 : ? !operand_equal_p (gimple_cond_rhs (g), arg2, 0)
6293 489663 : : (!operand_equal_p (gimple_cond_lhs (g), arg2, 0)
6294 948 : || !operand_equal_p (gimple_cond_rhs (g), arg1, 0)))
6295 619320 : || !cond_only_block_p (bb1))
6296 : return;
6297 :
6298 11626 : enum tree_code ccode = (operand_equal_p (gimple_cond_lhs (g), arg1, 0)
6299 11626 : ? LT_EXPR : GT_EXPR);
6300 11626 : switch (gimple_cond_code (g))
6301 : {
6302 : case LT_EXPR:
6303 : case LE_EXPR:
6304 : break;
6305 10165 : case GT_EXPR:
6306 10165 : case GE_EXPR:
6307 10165 : ccode = ccode == LT_EXPR ? GT_EXPR : LT_EXPR;
6308 : break;
6309 : default:
6310 : return;
6311 : }
6312 :
6313 34806 : for (int i = 0; i < 2; ++i)
6314 : {
6315 : /* With NaNs, </<=/>/>= are false, so we need to look for the
6316 : third comparison on the false edge from whatever non-equality
6317 : comparison the second comparison is. */
6318 23246 : if (HONOR_NANS (TREE_TYPE (arg1))
6319 23246 : && (EDGE_SUCC (bb1, i)->flags & EDGE_TRUE_VALUE) != 0)
6320 131 : continue;
6321 :
6322 23115 : bb2 = EDGE_SUCC (bb1, i)->dest;
6323 68947 : g = safe_dyn_cast <gcond *> (*gsi_last_bb (bb2));
6324 15597 : if (g == NULL
6325 15597 : || !single_pred_p (bb2)
6326 19266 : || (operand_equal_p (gimple_cond_lhs (g), arg1, 0)
6327 11521 : ? !operand_equal_p (gimple_cond_rhs (g), arg2, 0)
6328 3776 : : (!operand_equal_p (gimple_cond_lhs (g), arg2, 0)
6329 19 : || !operand_equal_p (gimple_cond_rhs (g), arg1, 0)))
6330 66 : || !cond_only_block_p (bb2)
6331 11587 : || EDGE_SUCC (bb2, 0)->dest == EDGE_SUCC (bb2, 1)->dest)
6332 23049 : continue;
6333 :
6334 66 : enum tree_code ccode2
6335 66 : = (operand_equal_p (gimple_cond_lhs (g), arg1, 0) ? LT_EXPR : GT_EXPR);
6336 66 : switch (gimple_cond_code (g))
6337 : {
6338 : case LT_EXPR:
6339 : case LE_EXPR:
6340 : break;
6341 41 : case GT_EXPR:
6342 41 : case GE_EXPR:
6343 41 : ccode2 = ccode2 == LT_EXPR ? GT_EXPR : LT_EXPR;
6344 : break;
6345 0 : default:
6346 0 : continue;
6347 : }
6348 66 : if (HONOR_NANS (TREE_TYPE (arg1)) && ccode == ccode2)
6349 0 : continue;
6350 :
6351 132 : if ((ccode == LT_EXPR)
6352 66 : ^ ((EDGE_SUCC (bb1, i)->flags & EDGE_TRUE_VALUE) != 0))
6353 : {
6354 41 : em1 = EDGE_SUCC (bb1, 1 - i);
6355 41 : e1 = EDGE_SUCC (bb2, 0);
6356 41 : e2 = EDGE_SUCC (bb2, 1);
6357 41 : if ((ccode2 == LT_EXPR) ^ ((e1->flags & EDGE_TRUE_VALUE) == 0))
6358 0 : std::swap (e1, e2);
6359 : }
6360 : else
6361 : {
6362 25 : e1 = EDGE_SUCC (bb1, 1 - i);
6363 25 : em1 = EDGE_SUCC (bb2, 0);
6364 25 : e2 = EDGE_SUCC (bb2, 1);
6365 25 : if ((ccode2 != LT_EXPR) ^ ((em1->flags & EDGE_TRUE_VALUE) == 0))
6366 : std::swap (em1, e2);
6367 : }
6368 : break;
6369 : }
6370 :
6371 11601 : if (em1 == NULL)
6372 : {
6373 23120 : if ((ccode == LT_EXPR)
6374 11560 : ^ ((EDGE_SUCC (bb1, 0)->flags & EDGE_TRUE_VALUE) != 0))
6375 : {
6376 4656 : em1 = EDGE_SUCC (bb1, 1);
6377 4656 : e1 = EDGE_SUCC (bb1, 0);
6378 4656 : e2 = (e1->flags & EDGE_TRUE_VALUE) ? em1 : e1;
6379 : }
6380 : else
6381 : {
6382 6904 : em1 = EDGE_SUCC (bb1, 0);
6383 6904 : e1 = EDGE_SUCC (bb1, 1);
6384 6904 : e2 = (e1->flags & EDGE_TRUE_VALUE) ? em1 : e1;
6385 : }
6386 : }
6387 :
6388 : /* Check if there is a single bb into which all failed conditions
6389 : jump to (perhaps through an empty block) and if it results in
6390 : a single integral PHI which just sets it to -1, 0, 1, X
6391 : (or -1, 0, 1 when NaNs can't happen). In that case use 1 rather
6392 : than 0 as last .SPACESHIP argument to tell backends it might
6393 : consider different code generation and just cast the result
6394 : of .SPACESHIP to the PHI result. X above is some value
6395 : other than -1, 0, 1, for libstdc++ -128, for libc++ -127. */
6396 11626 : tree arg3 = integer_zero_node;
6397 11626 : edge e = EDGE_SUCC (bb0, 0);
6398 11626 : if (e->dest == bb1)
6399 8924 : e = EDGE_SUCC (bb0, 1);
6400 11626 : basic_block bbp = e->dest;
6401 11626 : gphi *phi = NULL;
6402 11626 : for (gphi_iterator psi = gsi_start_phis (bbp);
6403 13767 : !gsi_end_p (psi); gsi_next (&psi))
6404 : {
6405 3682 : gphi *gp = psi.phi ();
6406 3682 : tree res = gimple_phi_result (gp);
6407 :
6408 3682 : if (phi != NULL
6409 3345 : || virtual_operand_p (res)
6410 2429 : || !INTEGRAL_TYPE_P (TREE_TYPE (res))
6411 5956 : || TYPE_PRECISION (TREE_TYPE (res)) < 2)
6412 : {
6413 : phi = NULL;
6414 : break;
6415 : }
6416 2141 : phi = gp;
6417 : }
6418 11626 : if (phi
6419 1804 : && integer_zerop (gimple_phi_arg_def_from_edge (phi, e))
6420 12202 : && EDGE_COUNT (bbp->preds) == (HONOR_NANS (TREE_TYPE (arg1)) ? 4 : 3))
6421 : {
6422 122 : HOST_WIDE_INT argval
6423 122 : = SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg1)) ? -128 : -1;
6424 708 : for (unsigned i = 0; phi && i < EDGE_COUNT (bbp->preds) - 1; ++i)
6425 : {
6426 255 : edge e3 = i == 0 ? e1 : i == 1 ? em1 : e2;
6427 255 : if (e3->dest != bbp)
6428 : {
6429 117 : if (!empty_block_p (e3->dest)
6430 108 : || !single_succ_p (e3->dest)
6431 225 : || single_succ (e3->dest) != bbp)
6432 : {
6433 : phi = NULL;
6434 : break;
6435 : }
6436 : e3 = single_succ_edge (e3->dest);
6437 : }
6438 246 : tree a = gimple_phi_arg_def_from_edge (phi, e3);
6439 246 : if (TREE_CODE (a) != INTEGER_CST
6440 246 : || (i == 0 && !integer_onep (a))
6441 482 : || (i == 1 && !integer_all_onesp (a)))
6442 : {
6443 : phi = NULL;
6444 : break;
6445 : }
6446 236 : if (i == 2)
6447 : {
6448 30 : tree minv = TYPE_MIN_VALUE (signed_char_type_node);
6449 30 : tree maxv = TYPE_MAX_VALUE (signed_char_type_node);
6450 30 : widest_int w = widest_int::from (wi::to_wide (a), SIGNED);
6451 41 : if ((w >= -1 && w <= 1)
6452 26 : || w < wi::to_widest (minv)
6453 60 : || w > wi::to_widest (maxv))
6454 : {
6455 4 : phi = NULL;
6456 4 : break;
6457 : }
6458 26 : argval = w.to_shwi ();
6459 26 : }
6460 : }
6461 122 : if (phi)
6462 99 : arg3 = build_int_cst (integer_type_node,
6463 119 : TYPE_UNSIGNED (TREE_TYPE (arg1)) ? 1 : argval);
6464 : }
6465 :
6466 : /* For integral <=> comparisons only use .SPACESHIP if it is turned
6467 : into an integer (-1, 0, 1). */
6468 11626 : if (!SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg1)) && arg3 == integer_zero_node)
6469 : return;
6470 :
6471 223 : gcall *gc = gimple_build_call_internal (IFN_SPACESHIP, 3, arg1, arg2, arg3);
6472 223 : tree lhs = make_ssa_name (integer_type_node);
6473 223 : gimple_call_set_lhs (gc, lhs);
6474 223 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6475 223 : gsi_insert_before (&gsi, gc, GSI_SAME_STMT);
6476 :
6477 347 : wide_int wmin = wi::minus_one (TYPE_PRECISION (integer_type_node));
6478 347 : wide_int wmax = wi::one (TYPE_PRECISION (integer_type_node));
6479 223 : if (HONOR_NANS (TREE_TYPE (arg1)))
6480 : {
6481 131 : if (arg3 == integer_zero_node)
6482 105 : wmin = wi::shwi (-128, TYPE_PRECISION (integer_type_node));
6483 26 : else if (tree_int_cst_sgn (arg3) < 0)
6484 19 : wmin = wi::to_wide (arg3);
6485 : else
6486 7 : wmax = wi::to_wide (arg3);
6487 : }
6488 347 : int_range<1> vr (TREE_TYPE (lhs), wmin, wmax);
6489 223 : set_range_info (lhs, vr);
6490 :
6491 223 : if (arg3 != integer_zero_node)
6492 : {
6493 99 : tree type = TREE_TYPE (gimple_phi_result (phi));
6494 99 : if (!useless_type_conversion_p (type, integer_type_node))
6495 : {
6496 63 : tree tem = make_ssa_name (type);
6497 63 : gimple *gcv = gimple_build_assign (tem, NOP_EXPR, lhs);
6498 63 : gsi_insert_before (&gsi, gcv, GSI_SAME_STMT);
6499 63 : lhs = tem;
6500 : }
6501 99 : SET_PHI_ARG_DEF_ON_EDGE (phi, e, lhs);
6502 99 : gimple_cond_set_lhs (stmt, boolean_false_node);
6503 99 : gimple_cond_set_rhs (stmt, boolean_false_node);
6504 185 : gimple_cond_set_code (stmt, (e->flags & EDGE_TRUE_VALUE)
6505 : ? EQ_EXPR : NE_EXPR);
6506 99 : update_stmt (stmt);
6507 99 : return;
6508 : }
6509 :
6510 124 : gimple_cond_set_lhs (stmt, lhs);
6511 124 : gimple_cond_set_rhs (stmt, integer_zero_node);
6512 124 : update_stmt (stmt);
6513 :
6514 248 : gcond *cond = as_a <gcond *> (*gsi_last_bb (bb1));
6515 124 : gimple_cond_set_lhs (cond, lhs);
6516 124 : if (em1->src == bb1 && e2 != em1)
6517 : {
6518 68 : gimple_cond_set_rhs (cond, integer_minus_one_node);
6519 74 : gimple_cond_set_code (cond, (em1->flags & EDGE_TRUE_VALUE)
6520 : ? EQ_EXPR : NE_EXPR);
6521 : }
6522 : else
6523 : {
6524 56 : gcc_assert (e1->src == bb1 && e2 != e1);
6525 56 : gimple_cond_set_rhs (cond, integer_one_node);
6526 56 : gimple_cond_set_code (cond, (e1->flags & EDGE_TRUE_VALUE)
6527 : ? EQ_EXPR : NE_EXPR);
6528 : }
6529 124 : update_stmt (cond);
6530 :
6531 124 : if (e2 != e1 && e2 != em1)
6532 : {
6533 80 : cond = as_a <gcond *> (*gsi_last_bb (bb2));
6534 40 : gimple_cond_set_lhs (cond, lhs);
6535 40 : if (em1->src == bb2)
6536 25 : gimple_cond_set_rhs (cond, integer_minus_one_node);
6537 : else
6538 : {
6539 15 : gcc_assert (e1->src == bb2);
6540 15 : gimple_cond_set_rhs (cond, integer_one_node);
6541 : }
6542 40 : gimple_cond_set_code (cond,
6543 40 : (e2->flags & EDGE_TRUE_VALUE) ? NE_EXPR : EQ_EXPR);
6544 40 : update_stmt (cond);
6545 : }
6546 : }
6547 :
6548 :
6549 : /* Long-multiply inverse-lowering helper.
6550 :
6551 : The forwprop long-multiply recognizer canonicalizes a hand-written
6552 : longhand high-part multiply into a cast+mult+shift+cast chain
6553 : `(N) ((2N) a * (2N) b) >> N'. When the target lacks an expansion
6554 : path for the wide form, `lower_long_mul_high_chain' resynthesizes
6555 : the longhand at narrow precision via `build_long_mul_partials'. */
6556 :
6557 : /* Test whether the target supports an (HALF)-by-(HALF)->NARROW unsigned
6558 : widening multiply. Returns true on success, with the half-width
6559 : scalar int mode placed in *HALF_MODE. */
6560 :
6561 : static bool
6562 2231 : can_widen_to_narrow_p (scalar_int_mode narrow_mode, unsigned int half_width,
6563 : scalar_int_mode *half_mode)
6564 : {
6565 2231 : if (!int_mode_for_size (half_width, 0).exists (half_mode))
6566 0 : return false;
6567 2231 : return convert_optab_handler (umul_widen_optab, narrow_mode, *half_mode)
6568 2231 : != CODE_FOR_nothing;
6569 : }
6570 :
6571 : /* Append to *SEQ the operand split and partial products for an unsigned
6572 : long multiply of OP1 by OP2 at the precision of TREE_TYPE (OP1).
6573 : HALF_TYPE is the (N/2)-bit unsigned type; HALF_AMT is the integer-typed
6574 : shift constant equal to N/2.
6575 :
6576 : Outputs the four partial products via *LOLO, *HILO, *LOHI, *HIHI.
6577 :
6578 : USE_WIDEN selects the partial-product form:
6579 : true - cast halves to HALF_TYPE and use WIDEN_MULT_EXPR (needs
6580 : an (N/2)-by-(N/2)->N widening multiply optab).
6581 : false - mask/shift halves within the N-bit accumulator and use
6582 : plain MULT_EXPR; the halves fit in N/2 bits so the N-bit
6583 : low product is exact. */
6584 :
6585 : static void
6586 2231 : build_long_mul_partials (gimple_seq *seq, location_t loc, tree op1, tree op2,
6587 : tree half_type, tree half_amt,
6588 : tree *lolo, tree *hilo, tree *lohi, tree *hihi,
6589 : bool use_widen)
6590 : {
6591 2231 : tree acc_type = TREE_TYPE (op1);
6592 2231 : tree op1_hi = gimple_build (seq, loc, RSHIFT_EXPR, acc_type, op1, half_amt);
6593 2231 : tree op2_hi = gimple_build (seq, loc, RSHIFT_EXPR, acc_type, op2, half_amt);
6594 2231 : tree op1_lo, op2_lo;
6595 2231 : tree_code mul_code;
6596 :
6597 2231 : if (use_widen)
6598 : {
6599 2231 : op1_lo = gimple_build (seq, loc, NOP_EXPR, half_type, op1);
6600 2231 : op2_lo = gimple_build (seq, loc, NOP_EXPR, half_type, op2);
6601 2231 : op1_hi = gimple_build (seq, loc, NOP_EXPR, half_type, op1_hi);
6602 2231 : op2_hi = gimple_build (seq, loc, NOP_EXPR, half_type, op2_hi);
6603 2231 : mul_code = WIDEN_MULT_EXPR;
6604 : }
6605 : else
6606 : {
6607 0 : tree mask = wide_int_to_tree (acc_type,
6608 0 : wi::mask (TYPE_PRECISION (half_type), false,
6609 0 : TYPE_PRECISION (acc_type)));
6610 0 : op1_lo = gimple_build (seq, loc, BIT_AND_EXPR, acc_type, op1, mask);
6611 0 : op2_lo = gimple_build (seq, loc, BIT_AND_EXPR, acc_type, op2, mask);
6612 0 : mul_code = MULT_EXPR;
6613 : }
6614 :
6615 2231 : *lolo = gimple_build (seq, loc, mul_code, acc_type, op1_lo, op2_lo);
6616 2231 : *hilo = gimple_build (seq, loc, mul_code, acc_type, op1_hi, op2_lo);
6617 2231 : *lohi = gimple_build (seq, loc, mul_code, acc_type, op1_lo, op2_hi);
6618 2231 : *hihi = gimple_build (seq, loc, mul_code, acc_type, op1_hi, op2_hi);
6619 2231 : }
6620 :
6621 : /* Emit into *SEQ the high N bits of the unsigned product A * B, where A and B
6622 : are NARROW_TYPE (N-bit) values, as a longhand over (N/2)-bit partials.
6623 : Returns the high-part SSA. */
6624 :
6625 : static tree
6626 2231 : emit_long_mul_highpart (gimple_seq *seq, location_t loc, tree a, tree b,
6627 : tree narrow_type)
6628 : {
6629 2231 : scalar_int_mode narrow_mode
6630 2231 : = as_a <scalar_int_mode> (TYPE_MODE (narrow_type));
6631 2231 : unsigned int half_width = GET_MODE_PRECISION (narrow_mode) / 2;
6632 : /* Prefer (N/2)-by-(N/2)->N widening partials; fall back to plain MULT_EXPR
6633 : when the target lacks the widen optab. See build_long_mul_partials. */
6634 2231 : scalar_int_mode half_mode;
6635 2231 : bool use_widen = can_widen_to_narrow_p (narrow_mode, half_width, &half_mode);
6636 2231 : tree half_type = build_nonstandard_integer_type (half_width, 1);
6637 2231 : tree half_amt = build_int_cst (integer_type_node, half_width);
6638 2231 : tree half_mask = wide_int_to_tree (narrow_type,
6639 2231 : wi::mask (half_width, false,
6640 2231 : TYPE_PRECISION (narrow_type)));
6641 :
6642 2231 : tree lolo, hilo, lohi, hihi;
6643 2231 : build_long_mul_partials (seq, loc, a, b, half_type, half_amt,
6644 : &lolo, &hilo, &lohi, &hihi, use_widen);
6645 2231 : tree cross_sum = gimple_build (seq, loc, PLUS_EXPR, narrow_type, hilo, lohi);
6646 2231 : tree cross_lt = gimple_build (seq, loc, LT_EXPR, boolean_type_node,
6647 : cross_sum, hilo);
6648 2231 : tree cross_lt_n = gimple_build (seq, loc, NOP_EXPR, narrow_type, cross_lt);
6649 2231 : tree cross_carry = gimple_build (seq, loc, LSHIFT_EXPR, narrow_type,
6650 : cross_lt_n, half_amt);
6651 2231 : tree lolo_hi = gimple_build (seq, loc, RSHIFT_EXPR, narrow_type,
6652 : lolo, half_amt);
6653 2231 : tree cross_lo = gimple_build (seq, loc, BIT_AND_EXPR, narrow_type,
6654 : cross_sum, half_mask);
6655 2231 : tree low_accum = gimple_build (seq, loc, PLUS_EXPR, narrow_type,
6656 : lolo_hi, cross_lo);
6657 2231 : tree low_accum_hi = gimple_build (seq, loc, RSHIFT_EXPR, narrow_type,
6658 : low_accum, half_amt);
6659 2231 : tree cross_hi = gimple_build (seq, loc, RSHIFT_EXPR, narrow_type,
6660 : cross_sum, half_amt);
6661 2231 : tree t1 = gimple_build (seq, loc, PLUS_EXPR, narrow_type, hihi, cross_hi);
6662 2231 : tree t2 = gimple_build (seq, loc, PLUS_EXPR, narrow_type, t1, low_accum_hi);
6663 2231 : return gimple_build (seq, loc, PLUS_EXPR, narrow_type, t2, cross_carry);
6664 : }
6665 :
6666 : /* Emit into *SEQ the high N bits (NARROW_TYPE) of the unsigned product of two
6667 : 2N-bit values given as N-bit halves, x = L1 + H1*2^N and y = L2 + H2*2^N:
6668 : the high half of x*y is the high N bits of L1*L2, plus H1*L2 and L1*H2, all
6669 : mod 2^N. */
6670 :
6671 : static tree
6672 2231 : combine_long_mul_halves (gimple_seq *seq, location_t loc, tree l1, tree h1,
6673 : tree l2, tree h2, tree narrow_type)
6674 : {
6675 2231 : tree hh = emit_long_mul_highpart (seq, loc, l1, l2, narrow_type);
6676 2231 : tree c1 = gimple_build (seq, loc, MULT_EXPR, narrow_type, h1, l2);
6677 2231 : tree c2 = gimple_build (seq, loc, MULT_EXPR, narrow_type, l1, h2);
6678 2231 : tree s = gimple_build (seq, loc, PLUS_EXPR, narrow_type, hh, c1);
6679 2231 : return gimple_build (seq, loc, PLUS_EXPR, narrow_type, s, c2);
6680 : }
6681 :
6682 : /* True when OP fits NARROW_PREC bits as an unsigned value. Looks
6683 : through widening casts and PHIs, falling back to `tree_nonzero_bits'
6684 : otherwise. PHI_SEEN guards against cycles. */
6685 :
6686 : static bool
6687 929 : long_mul_op_fits_p (tree op, unsigned narrow_prec, bitmap phi_seen)
6688 : {
6689 1199 : if (!TYPE_UNSIGNED (TREE_TYPE (op)))
6690 : return false;
6691 1199 : if (TYPE_PRECISION (TREE_TYPE (op)) <= narrow_prec)
6692 : return true;
6693 929 : if (TREE_CODE (op) == SSA_NAME)
6694 : {
6695 357 : gimple *def = SSA_NAME_DEF_STMT (op);
6696 357 : if (is_gimple_assign (def)
6697 357 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
6698 270 : return long_mul_op_fits_p (gimple_assign_rhs1 (def), narrow_prec,
6699 270 : phi_seen);
6700 87 : if (gphi *phi = dyn_cast <gphi *> (def))
6701 69 : if (bitmap_set_bit (phi_seen, SSA_NAME_VERSION (op)))
6702 : {
6703 404 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
6704 335 : if (!long_mul_op_fits_p (gimple_phi_arg_def (phi, i),
6705 : narrow_prec, phi_seen))
6706 : return false;
6707 : return true;
6708 : }
6709 : }
6710 590 : return wi::min_precision (tree_nonzero_bits (op), UNSIGNED) <= narrow_prec;
6711 : }
6712 :
6713 : static bool long_mul_split_operand (gimple_seq *, location_t, tree, tree,
6714 : tree *, tree *);
6715 :
6716 : struct long_mul_halves
6717 : {
6718 : tree lo;
6719 : tree hi;
6720 : };
6721 :
6722 : /* Halves recorded for one run of the pass, keyed on the PHI they came from.
6723 : Several chains can reach one operand PHI, and each that splits it again
6724 : leaves another redundant pair of half PHIs behind. */
6725 :
6726 : static hash_map<tree, long_mul_halves> *long_mul_phi_halves;
6727 :
6728 : struct long_mul_arg_split
6729 : {
6730 : gimple_seq seq;
6731 : tree lo;
6732 : tree hi;
6733 : };
6734 :
6735 : /* Split the 2N-bit result of PHI into N-bit halves *LO and *HI, by splitting
6736 : each argument and merging the halves with two new PHIs. A split goes at
6737 : the end of its argument's incoming block, where the argument is available,
6738 : rather than on the edge, which could split a critical edge while the
6739 : dominator walk is still running. Returns false, having changed nothing,
6740 : when an argument cannot be split. A split that succeeds stands even if
6741 : the caller then gives up. */
6742 :
6743 : static bool
6744 0 : long_mul_split_phi (gphi *phi, tree narrow_type, tree *lo, tree *hi)
6745 : {
6746 0 : tree res = gimple_phi_result (phi);
6747 0 : if (long_mul_halves *prev = long_mul_phi_halves->get (res))
6748 : {
6749 : /* Every operand reaching a split has the 2N type, so the width is
6750 : fixed by the PHI's own type. */
6751 0 : gcc_checking_assert (types_compatible_p (TREE_TYPE (prev->lo),
6752 : narrow_type));
6753 0 : *lo = prev->lo;
6754 0 : *hi = prev->hi;
6755 0 : return true;
6756 : }
6757 :
6758 0 : unsigned int n = gimple_phi_num_args (phi);
6759 0 : location_t loc = gimple_location (phi);
6760 0 : basic_block bb = gimple_bb (phi);
6761 0 : auto_vec<long_mul_arg_split, 4> args;
6762 :
6763 0 : for (unsigned int i = 0; i < n; i++)
6764 : {
6765 0 : edge e = gimple_phi_arg_edge (phi, i);
6766 : /* A back edge could lead back to PHI and recurse forever. The entry
6767 : block cannot hold a split. */
6768 0 : if (dominated_by_p (CDI_DOMINATORS, e->src, bb)
6769 0 : || e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
6770 0 : return false;
6771 :
6772 0 : long_mul_arg_split arg = {};
6773 0 : if (!long_mul_split_operand (&arg.seq, loc, gimple_phi_arg_def (phi, i),
6774 : narrow_type, &arg.lo, &arg.hi))
6775 : return false;
6776 :
6777 0 : args.safe_push (arg);
6778 : }
6779 :
6780 : /* Every argument split, so the rewrite can be committed. */
6781 0 : gphi *lo_phi = create_phi_node (make_ssa_name (narrow_type), bb);
6782 0 : gphi *hi_phi = create_phi_node (make_ssa_name (narrow_type), bb);
6783 0 : for (unsigned int i = 0; i < n; i++)
6784 : {
6785 0 : edge e = gimple_phi_arg_edge (phi, i);
6786 0 : if (args[i].seq)
6787 : {
6788 0 : gimple_stmt_iterator gsi = gsi_last_bb (e->src);
6789 0 : if (!gsi_end_p (gsi) && stmt_ends_bb_p (gsi_stmt (gsi)))
6790 0 : gsi_insert_seq_before (&gsi, args[i].seq, GSI_SAME_STMT);
6791 : else
6792 0 : gsi_insert_seq_after (&gsi, args[i].seq, GSI_CONTINUE_LINKING);
6793 : }
6794 0 : add_phi_arg (lo_phi, args[i].lo, e, UNKNOWN_LOCATION);
6795 0 : add_phi_arg (hi_phi, args[i].hi, e, UNKNOWN_LOCATION);
6796 : }
6797 0 : *lo = gimple_phi_result (lo_phi);
6798 0 : *hi = gimple_phi_result (hi_phi);
6799 0 : long_mul_phi_halves->put (res, { *lo, *hi });
6800 :
6801 0 : if (dump_file && (dump_flags & TDF_DETAILS))
6802 0 : fprintf (dump_file, "Split long-multiply operand PHI.\n");
6803 : return true;
6804 0 : }
6805 :
6806 : /* Split the 2N-bit unsigned value OP into its low and high N bits (*LO and
6807 : *HI, both NARROW_TYPE) using only N-bit operations, as the target has no 2N
6808 : multiply or shift. A 2N product recurses on its operands, its high half
6809 : coming from combine_long_mul_halves. A value shifted down by N recurses on
6810 : the shifted value and takes its high half, rather than reading the 2N shift.
6811 : A widening cast's low half is the truncated source and its high half is what
6812 : the cast extended with, zero or the source's replicated sign bit. A value
6813 : that provably fits N bits has a zero high half. A PHI is split through its
6814 : arguments as a last resort. Returns false otherwise. */
6815 :
6816 : static bool
6817 4470 : long_mul_split_operand (gimple_seq *seq, location_t loc, tree op,
6818 : tree narrow_type, tree *lo, tree *hi)
6819 : {
6820 4470 : unsigned int narrow_prec = TYPE_PRECISION (narrow_type);
6821 4470 : if (TREE_CODE (op) == SSA_NAME)
6822 : {
6823 3943 : gimple *def = SSA_NAME_DEF_STMT (op);
6824 3943 : if (is_gimple_assign (def) && gimple_assign_rhs_code (def) == MULT_EXPR)
6825 : {
6826 13 : tree a_lo, a_hi, b_lo, b_hi;
6827 13 : if (!long_mul_split_operand (seq, loc, gimple_assign_rhs1 (def),
6828 : narrow_type, &a_lo, &a_hi)
6829 13 : || !long_mul_split_operand (seq, loc, gimple_assign_rhs2 (def),
6830 : narrow_type, &b_lo, &b_hi))
6831 : return false;
6832 13 : *lo = gimple_build (seq, loc, MULT_EXPR, narrow_type, a_lo, b_lo);
6833 13 : *hi = combine_long_mul_halves (seq, loc, a_lo, a_hi, b_lo, b_hi,
6834 : narrow_type);
6835 13 : return true;
6836 : }
6837 : /* A 2N value shifted down by N is its own high half: split the source
6838 : and use that half. Reading the shift instead leaves the 2N source
6839 : live, and the target cannot expand it. This has to come before the
6840 : widening-cast case below, which would take such a value as it
6841 : stands. */
6842 3930 : if (is_gimple_assign (def)
6843 3881 : && gimple_assign_rhs_code (def) == RSHIFT_EXPR
6844 8 : && tree_fits_uhwi_p (gimple_assign_rhs2 (def))
6845 3938 : && tree_to_uhwi (gimple_assign_rhs2 (def)) == narrow_prec)
6846 : {
6847 8 : tree src_lo, src_hi;
6848 8 : if (!long_mul_split_operand (seq, loc, gimple_assign_rhs1 (def),
6849 : narrow_type, &src_lo, &src_hi))
6850 : return false;
6851 8 : *lo = src_hi;
6852 8 : *hi = build_zero_cst (narrow_type);
6853 8 : return true;
6854 : }
6855 3922 : if (is_gimple_assign (def)
6856 3922 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
6857 : {
6858 3855 : tree src = gimple_assign_rhs1 (def);
6859 3855 : tree src_type = TREE_TYPE (src);
6860 3855 : if (INTEGRAL_TYPE_P (src_type)
6861 3855 : && TYPE_PRECISION (src_type) <= narrow_prec)
6862 : {
6863 3855 : *lo = gimple_convert (seq, loc, narrow_type, src);
6864 3855 : if (TYPE_UNSIGNED (src_type))
6865 3840 : *hi = build_zero_cst (narrow_type);
6866 : else
6867 : {
6868 : /* Sign extension: the high N bits replicate the sign bit. */
6869 15 : tree snarrow = signed_type_for (narrow_type);
6870 15 : tree s = gimple_convert (seq, loc, snarrow, *lo);
6871 15 : tree amt = build_int_cst (integer_type_node, narrow_prec - 1);
6872 15 : tree sh = gimple_build (seq, loc, RSHIFT_EXPR, snarrow, s,
6873 : amt);
6874 15 : *hi = gimple_convert (seq, loc, narrow_type, sh);
6875 : }
6876 3855 : return true;
6877 : }
6878 : }
6879 : }
6880 :
6881 : /* A value provably within N bits: its low half is the truncation to N bits
6882 : (a subreg, not a 2N shift), its high half is zero. */
6883 594 : auto_bitmap phi_seen;
6884 594 : if (long_mul_op_fits_p (op, narrow_prec, phi_seen))
6885 : {
6886 594 : *lo = gimple_convert (seq, loc, narrow_type, op);
6887 594 : *hi = build_zero_cst (narrow_type);
6888 594 : return true;
6889 : }
6890 :
6891 : /* A PHI that does not fit N bits can still be split through its arguments,
6892 : which is how a sign-extended value reaches the cast case above. */
6893 0 : if (TREE_CODE (op) == SSA_NAME)
6894 594 : if (gphi *phi = dyn_cast <gphi *> (SSA_NAME_DEF_STMT (op)))
6895 0 : return long_mul_split_phi (phi, narrow_type, lo, hi);
6896 : return false;
6897 594 : }
6898 :
6899 : /* Collect into HIGH_USES the uses of PROD forming its high half,
6900 : `PROD >> NARROW_PREC'. A use reading only the low NARROW_PREC bits is
6901 : accepted but not collected. Returns false on any other use. */
6902 :
6903 : static bool
6904 2244 : long_mul_high_half_uses (tree prod, unsigned int narrow_prec,
6905 : vec<gimple *> *high_uses)
6906 : {
6907 2244 : imm_use_iterator iui;
6908 2244 : gimple *use_stmt;
6909 4486 : FOR_EACH_IMM_USE_STMT (use_stmt, iui, prod)
6910 : {
6911 2247 : if (is_gimple_debug (use_stmt))
6912 0 : continue;
6913 2247 : if (!is_gimple_assign (use_stmt))
6914 : return false;
6915 2247 : tree_code code = gimple_assign_rhs_code (use_stmt);
6916 2247 : if (code == RSHIFT_EXPR
6917 2226 : && gimple_assign_rhs1 (use_stmt) == prod
6918 2226 : && tree_fits_uhwi_p (gimple_assign_rhs2 (use_stmt))
6919 4473 : && tree_to_uhwi (gimple_assign_rhs2 (use_stmt)) == narrow_prec)
6920 2226 : high_uses->safe_push (use_stmt);
6921 21 : else if (CONVERT_EXPR_CODE_P (code))
6922 : {
6923 0 : tree t = TREE_TYPE (gimple_assign_lhs (use_stmt));
6924 0 : if (!INTEGRAL_TYPE_P (t) || TYPE_PRECISION (t) > narrow_prec)
6925 : return false;
6926 : }
6927 21 : else if (code == BIT_AND_EXPR
6928 21 : && TREE_CODE (gimple_assign_rhs2 (use_stmt)) == INTEGER_CST)
6929 : {
6930 16 : if (wi::min_precision (wi::to_wide (gimple_assign_rhs2 (use_stmt)),
6931 : UNSIGNED) > narrow_prec)
6932 : return false;
6933 : }
6934 : else
6935 : return false;
6936 5 : }
6937 2239 : return true;
6938 : }
6939 :
6940 : /* True when every use of PROD reads only its low NARROW_PREC bits. */
6941 :
6942 : static bool
6943 2236 : long_mul_only_low_half_used_p (tree prod, unsigned int narrow_prec)
6944 : {
6945 2236 : auto_vec<gimple *, 4> high_uses;
6946 2236 : return (long_mul_high_half_uses (prod, narrow_prec, &high_uses)
6947 4472 : && high_uses.is_empty ());
6948 2236 : }
6949 :
6950 : /* True when STMT is res = a * b whose unsigned 2N-bit result is in a mode the
6951 : target cannot multiply, having neither insn nor libcall, so that expand_mult
6952 : would abort; set *NARROW_TYPE to the N-bit unsigned type its halves are
6953 : built at. A mode the target does support is left to convert_mult_to_widen
6954 : and convert_mult_to_highpart. */
6955 :
6956 : static bool
6957 88729739 : unexpandable_long_mul_p (gimple *stmt, tree *narrow_type)
6958 : {
6959 88729739 : if (!is_gimple_assign (stmt) || gimple_assign_rhs_code (stmt) != MULT_EXPR)
6960 : return false;
6961 :
6962 1479408 : tree wide_type = TREE_TYPE (gimple_assign_lhs (stmt));
6963 1479408 : scalar_int_mode wide_mode;
6964 1479408 : if (!INTEGRAL_TYPE_P (wide_type)
6965 1221844 : || !TYPE_UNSIGNED (wide_type)
6966 902558 : || !is_a <scalar_int_mode> (TYPE_MODE (wide_type), &wide_mode)
6967 2701252 : || targetm.scalar_mode_supported_p (wide_mode))
6968 : return false;
6969 :
6970 2244 : *narrow_type
6971 2244 : = build_nonstandard_integer_type (TYPE_PRECISION (wide_type) / 2, 1);
6972 2244 : return true;
6973 : }
6974 :
6975 : static bool narrow_long_mul_low_half (gimple_stmt_iterator *);
6976 :
6977 : /* OP1 and OP2 are the operands of a 2N multiply just narrowed or lowered;
6978 : that rewrite now reads each through an N-bit low-half cast. An operand
6979 : defined by another 2N multiply can thereby become low-half-only -- narrow
6980 : it too, recursing through chained wide products such as (a*b)*c. */
6981 :
6982 : static void
6983 2215 : narrow_long_mul_operands (tree op1, tree op2)
6984 : {
6985 6645 : for (tree op : { op1, op2 })
6986 4430 : if (TREE_CODE (op) == SSA_NAME)
6987 : {
6988 3903 : gimple *def = SSA_NAME_DEF_STMT (op);
6989 3903 : if (is_gimple_assign (def) && gimple_assign_rhs_code (def) == MULT_EXPR)
6990 : {
6991 5 : gimple_stmt_iterator dgsi = gsi_for_stmt (def);
6992 5 : narrow_long_mul_low_half (&dgsi);
6993 : }
6994 : }
6995 2215 : }
6996 :
6997 : /* If the statement at *GSI is res = a * b with a 2N-bit unsigned result the
6998 : target cannot multiply and every use reads only the low N bits, narrow it
6999 : to res = (2N) ((N) a * (N) b) and return true. The low N bits of a product
7000 : depend only on the low N bits of the operands, so this preserves every use;
7001 : the unused high half becomes zero. match.pd's shorten rule omits this for
7002 : MULT_EXPR. */
7003 :
7004 : static bool
7005 754046 : narrow_long_mul_low_half (gimple_stmt_iterator *gsi)
7006 : {
7007 754046 : gimple *stmt = gsi_stmt (*gsi);
7008 754046 : tree narrow_type;
7009 754046 : if (!unexpandable_long_mul_p (stmt, &narrow_type))
7010 : return false;
7011 :
7012 2236 : tree lhs = gimple_assign_lhs (stmt);
7013 2236 : if (!long_mul_only_low_half_used_p (lhs, TYPE_PRECISION (narrow_type)))
7014 : return false;
7015 :
7016 13 : tree op1 = gimple_assign_rhs1 (stmt);
7017 13 : tree op2 = gimple_assign_rhs2 (stmt);
7018 13 : location_t loc = gimple_location (stmt);
7019 13 : gimple_seq seq = NULL;
7020 13 : tree a = gimple_convert (&seq, loc, narrow_type, op1);
7021 13 : tree b = gimple_convert (&seq, loc, narrow_type, op2);
7022 13 : tree np = gimple_build (&seq, loc, MULT_EXPR, narrow_type, a, b);
7023 13 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
7024 13 : gimple *conv = gimple_build_assign (lhs, NOP_EXPR, np);
7025 13 : gimple_set_location (conv, loc);
7026 13 : gsi_replace (gsi, conv, true);
7027 :
7028 13 : if (dump_file && (dump_flags & TDF_DETAILS))
7029 0 : fprintf (dump_file, "Narrowed low-half-only long multiply.\n");
7030 :
7031 13 : narrow_long_mul_operands (op1, op2);
7032 13 : return true;
7033 : }
7034 :
7035 : /* The 2N multiply at *GSI has had its high half synthesized elsewhere, so any
7036 : use left reads only its low half: narrow it in place, or remove it when it
7037 : has no use at all. Removing it drops a use of each operand, so a 2N
7038 : multiply defining one may become low-half-only. Narrow those operands. */
7039 :
7040 : static void
7041 2218 : finish_long_mul_low_half (gimple_stmt_iterator *gsi)
7042 : {
7043 2218 : gimple *stmt = gsi_stmt (*gsi);
7044 :
7045 2218 : if (has_zero_uses (gimple_assign_lhs (stmt)))
7046 : {
7047 2202 : tree op1 = gimple_assign_rhs1 (stmt);
7048 2202 : tree op2 = gimple_assign_rhs2 (stmt);
7049 2202 : gsi_remove (gsi, true);
7050 2202 : release_defs (stmt);
7051 2202 : narrow_long_mul_operands (op1, op2);
7052 2202 : return;
7053 : }
7054 :
7055 16 : narrow_long_mul_low_half (gsi);
7056 : }
7057 :
7058 : /* Rewrite the multiply STMT into the N-bit halves its uses read, when the
7059 : target can neither multiply at 2N bits nor form an N-bit high part.
7060 : Returns true on a rewrite, which may remove STMT.
7061 :
7062 : Runs once lower_long_mul_high_chain has been applied to every statement:
7063 : both rewrite products with a high half, and this one, keyed on the
7064 : definition rather than on a consumer, would otherwise pre-empt it. What
7065 : reaches it is a product that lowering could not retire, its high half also
7066 : read as an operand of another product, or read alongside the low half. A
7067 : product read only for its low half is left to narrow_long_mul_low_half. */
7068 :
7069 : static bool
7070 87975693 : narrow_long_mul_halves (gimple *stmt)
7071 : {
7072 87975693 : tree narrow_type;
7073 87975693 : if (!unexpandable_long_mul_p (stmt, &narrow_type))
7074 : return false;
7075 :
7076 : /* Only worth lowering where the target cannot form the N-bit high part. */
7077 8 : scalar_int_mode narrow_mode;
7078 8 : if (!is_a <scalar_int_mode> (TYPE_MODE (narrow_type), &narrow_mode)
7079 8 : || can_mult_highpart_p (narrow_mode, true))
7080 : return false;
7081 :
7082 8 : tree lhs = gimple_assign_lhs (stmt);
7083 8 : auto_vec<gimple *, 4> high_uses;
7084 8 : if (!long_mul_high_half_uses (lhs, TYPE_PRECISION (narrow_type), &high_uses)
7085 16 : || high_uses.is_empty ())
7086 : return false;
7087 :
7088 8 : tree op1 = gimple_assign_rhs1 (stmt);
7089 8 : tree op2 = gimple_assign_rhs2 (stmt);
7090 8 : location_t loc = gimple_location (stmt);
7091 8 : gimple_seq seq = NULL;
7092 8 : tree l1, h1, l2, h2;
7093 8 : if (!long_mul_split_operand (&seq, loc, op1, narrow_type, &l1, &h1)
7094 8 : || !long_mul_split_operand (&seq, loc, op2, narrow_type, &l2, &h2))
7095 : return false;
7096 8 : tree hi = combine_long_mul_halves (&seq, loc, l1, h1, l2, h2, narrow_type);
7097 : /* The high part is < 2^N, so widening it back to 2N leaves every use of a
7098 : shift, full width or truncated, reading the same value. */
7099 8 : tree hi_wide = gimple_convert (&seq, loc, TREE_TYPE (lhs), hi);
7100 8 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
7101 8 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
7102 :
7103 8 : unsigned int i;
7104 8 : gimple *shift_stmt;
7105 24 : FOR_EACH_VEC_ELT (high_uses, i, shift_stmt)
7106 : {
7107 8 : gimple_stmt_iterator sgsi = gsi_for_stmt (shift_stmt);
7108 8 : gimple *conv = gimple_build_assign (gimple_assign_lhs (shift_stmt),
7109 : hi_wide);
7110 8 : gimple_set_location (conv, loc);
7111 8 : gsi_replace (&sgsi, conv, true);
7112 : }
7113 :
7114 8 : if (dump_file && (dump_flags & TDF_DETAILS))
7115 0 : fprintf (dump_file, "Narrowed high half of long multiply.\n");
7116 :
7117 8 : finish_long_mul_low_half (&gsi);
7118 8 : return true;
7119 8 : }
7120 :
7121 : /* Match.pd recognizer for the long-multiply recognizer's high-part
7122 : emit chain. */
7123 :
7124 : extern bool gimple_long_mul_high_chain (tree, tree *, tree (*)(tree));
7125 :
7126 : /* Rewrite the `long_mul_high_chain' whose tail is the statement at GSI
7127 :
7128 : wide_a = (T_2N) op1
7129 : wide_b = (T_2N) op2
7130 : wide_prod = wide_a * wide_b
7131 : hi = wide_prod >> N
7132 : lhs = (convert) hi
7133 :
7134 : to a longhand high-part synthesis at T_N precision. Never materializes
7135 : T_2N in gimple, so it covers cases where the 2N mode has no expansion path
7136 : (e.g. the high 128 bits of a 128x128 product where 2N=OImode). An operand
7137 : wider than T_N -- a shared wide product or a sign-extended cast -- is split
7138 : into T_N halves rather than truncated, so no high input bits are dropped.
7139 : Returns true on a rewrite. */
7140 :
7141 : static bool
7142 2466195 : lower_long_mul_high_chain (gimple_stmt_iterator *gsi)
7143 : {
7144 2466195 : gimple *trunc_stmt = gsi_stmt (*gsi);
7145 2466195 : if (!is_gimple_assign (trunc_stmt))
7146 : return false;
7147 :
7148 2466195 : tree narrow_lhs = gimple_assign_lhs (trunc_stmt);
7149 2466195 : tree ops[2];
7150 2466195 : if (!gimple_long_mul_high_chain (narrow_lhs, ops, NULL))
7151 : return false;
7152 :
7153 : /* Walk the matched chain back to the 2N multiply and take narrow_type at
7154 : half its precision. */
7155 2475 : gimple *shift_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (trunc_stmt));
7156 2475 : gimple *mult_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (shift_stmt));
7157 2475 : unsigned int narrow_prec
7158 2475 : = TYPE_PRECISION (TREE_TYPE (gimple_assign_lhs (mult_stmt))) / 2;
7159 2475 : tree narrow_type = build_nonstandard_integer_type (narrow_prec, /*uns=*/1);
7160 2475 : scalar_int_mode narrow_mode;
7161 2475 : if (!is_a <scalar_int_mode> (TYPE_MODE (narrow_type), &narrow_mode))
7162 : return false;
7163 :
7164 : /* Lower only when the target cannot form the N-bit high part itself. */
7165 2475 : if (can_mult_highpart_p (narrow_mode, true))
7166 : return false;
7167 :
7168 2210 : location_t loc = gimple_location (trunc_stmt);
7169 2210 : gimple_seq seq = NULL;
7170 :
7171 : /* Split each operand into N-bit halves and combine. An operand that fits
7172 : N bits yields h == 0, so its cross term folds away; with both fitting
7173 : the combine is just a plain N-bit high part. */
7174 2210 : tree l1, h1, l2, h2;
7175 2210 : if (!long_mul_split_operand (&seq, loc, gimple_assign_rhs1 (mult_stmt),
7176 : narrow_type, &l1, &h1)
7177 2210 : || !long_mul_split_operand (&seq, loc, gimple_assign_rhs2 (mult_stmt),
7178 : narrow_type, &l2, &h2))
7179 : return false;
7180 2210 : tree hi = combine_long_mul_halves (&seq, loc, l1, h1, l2, h2, narrow_type);
7181 :
7182 : /* Merging the chain's truncation with a later user cast can retarget the
7183 : outer convert to any integral type, so convert the narrow result once
7184 : here (the high part is < 2^N, so the conversion preserves it). */
7185 2210 : gimple *result_stmt;
7186 2210 : tree lhs_type = TREE_TYPE (narrow_lhs);
7187 2210 : if (useless_type_conversion_p (lhs_type, narrow_type))
7188 2204 : result_stmt = gimple_build_assign (narrow_lhs, hi);
7189 : else
7190 6 : result_stmt = gimple_build_assign (narrow_lhs, NOP_EXPR, hi);
7191 2210 : gimple_set_location (result_stmt, loc);
7192 2210 : gimple_seq_add_stmt (&seq, result_stmt);
7193 :
7194 2210 : gsi_replace_with_seq (gsi, seq, true);
7195 :
7196 : /* Clean up the shift and the 2N mult now -- LTRANS runs no DCE between
7197 : widening_mul and expand, and a dead 2N mult would abort expand_mult.
7198 : Dead upstream (T_2N) casts, if any, are harmless NOP_EXPRs and land
7199 : with normal DCE. */
7200 2210 : if (has_zero_uses (gimple_assign_lhs (shift_stmt)))
7201 : {
7202 2202 : gimple_stmt_iterator dgsi = gsi_for_stmt (shift_stmt);
7203 2202 : gsi_remove (&dgsi, true);
7204 2202 : release_defs (shift_stmt);
7205 : }
7206 :
7207 : /* The mult is either dead (low half recomputed elsewhere) or now read only
7208 : for its low half. */
7209 2210 : gimple_stmt_iterator mgsi = gsi_for_stmt (mult_stmt);
7210 2210 : finish_long_mul_low_half (&mgsi);
7211 :
7212 2210 : if (dump_file && (dump_flags & TDF_DETAILS))
7213 4 : fprintf (dump_file, "Lowered long-mul high-part chain.\n");
7214 : return true;
7215 : }
7216 :
7217 : /* True when pass_optimize_widening_mul will run. Shared with the
7218 : forwprop long-multiply recognizer so its wide-chain emit stays
7219 : paired with the lowering that rescues an unsupported 2N shape.
7220 : The -Og pipeline (pass_all_optimizations_g) does not contain
7221 : pass_optimize_widening_mul at all, so -Og -fexpensive-optimizations
7222 : must not enable the emit: the unlowered 2N multiply would reach
7223 : expand as an unexpandable mode (e.g. OImode) and ICE.
7224 : -fdisable-tree-widening_mul is not observed. */
7225 :
7226 : bool
7227 1081132 : optimize_widening_mul_active_p (void)
7228 : {
7229 1081132 : return flag_expensive_optimizations && optimize && !optimize_debug;
7230 : }
7231 :
7232 : /* Find integer multiplications where the operands are extended from
7233 : smaller types, and replace the MULT_EXPR with a WIDEN_MULT_EXPR
7234 : or MULT_HIGHPART_EXPR where appropriate. */
7235 :
7236 : namespace {
7237 :
7238 : const pass_data pass_data_optimize_widening_mul =
7239 : {
7240 : GIMPLE_PASS, /* type */
7241 : "widening_mul", /* name */
7242 : OPTGROUP_NONE, /* optinfo_flags */
7243 : TV_TREE_WIDEN_MUL, /* tv_id */
7244 : PROP_ssa, /* properties_required */
7245 : 0, /* properties_provided */
7246 : 0, /* properties_destroyed */
7247 : 0, /* todo_flags_start */
7248 : TODO_update_ssa, /* todo_flags_finish */
7249 : };
7250 :
7251 : class pass_optimize_widening_mul : public gimple_opt_pass
7252 : {
7253 : public:
7254 294587 : pass_optimize_widening_mul (gcc::context *ctxt)
7255 589174 : : gimple_opt_pass (pass_data_optimize_widening_mul, ctxt)
7256 : {}
7257 :
7258 : /* opt_pass methods: */
7259 1062413 : bool gate (function *) final override
7260 : {
7261 1062413 : return optimize_widening_mul_active_p ();
7262 : }
7263 :
7264 : unsigned int execute (function *) final override;
7265 :
7266 : }; // class pass_optimize_widening_mul
7267 :
7268 : /* Walker class to perform the transformation in reverse dominance order. */
7269 :
7270 : class math_opts_dom_walker : public dom_walker
7271 : {
7272 : public:
7273 : /* Constructor, CFG_CHANGED is a pointer to a boolean flag that will be set
7274 : if walking modidifes the CFG. */
7275 :
7276 983322 : math_opts_dom_walker (bool *cfg_changed_p)
7277 1966644 : : dom_walker (CDI_DOMINATORS), m_last_result_set (),
7278 983322 : m_cfg_changed_p (cfg_changed_p) {}
7279 :
7280 : /* The actual actions performed in the walk. */
7281 :
7282 : void after_dom_children (basic_block) final override;
7283 :
7284 : /* Set of results of chains of multiply and add statement combinations that
7285 : were not transformed into FMAs because of active deferring. */
7286 : hash_set<tree> m_last_result_set;
7287 :
7288 : /* Pointer to a flag of the user that needs to be set if CFG has been
7289 : modified. */
7290 : bool *m_cfg_changed_p;
7291 : };
7292 :
7293 : void
7294 10458165 : math_opts_dom_walker::after_dom_children (basic_block bb)
7295 : {
7296 10458165 : gimple_stmt_iterator gsi;
7297 :
7298 10458165 : fma_deferring_state fma_state (param_avoid_fma_max_bits > 0
7299 10561745 : && param_widening_mul_defer_fma);
7300 :
7301 14762824 : for (gphi_iterator psi_next, psi = gsi_start_phis (bb); !gsi_end_p (psi);
7302 4304659 : psi = psi_next)
7303 : {
7304 4304659 : psi_next = psi;
7305 4304659 : gsi_next (&psi_next);
7306 :
7307 4304659 : gimple_stmt_iterator gsi = gsi_after_labels (bb);
7308 4304659 : gphi *phi = psi.phi ();
7309 :
7310 4304659 : if (match_saturation_add (&gsi, phi)
7311 4304640 : || match_saturation_sub (&gsi, phi)
7312 4304592 : || match_saturation_trunc (&gsi, phi)
7313 4304592 : || match_saturation_mul (&gsi, phi)
7314 8609251 : || match_spaceship (&gsi, phi))
7315 179 : remove_phi_node (&psi, /* release_lhs_p */ false);
7316 : }
7317 :
7318 97340749 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi);)
7319 : {
7320 86882584 : gimple *stmt = gsi_stmt (gsi);
7321 86882584 : enum tree_code code;
7322 :
7323 86882584 : if (is_gimple_assign (stmt))
7324 : {
7325 21720069 : code = gimple_assign_rhs_code (stmt);
7326 21720069 : switch (code)
7327 : {
7328 754025 : case MULT_EXPR:
7329 754025 : if (narrow_long_mul_low_half (&gsi))
7330 : break;
7331 754025 : if (!convert_mult_to_widen (stmt, &gsi)
7332 745175 : && !convert_expand_mult_copysign (stmt, &gsi)
7333 1499157 : && convert_mult_to_fma (stmt,
7334 : gimple_assign_rhs1 (stmt),
7335 : gimple_assign_rhs2 (stmt),
7336 : &fma_state))
7337 : {
7338 16517 : gsi_remove (&gsi, true);
7339 16517 : release_defs (stmt);
7340 16517 : continue;
7341 : }
7342 737508 : match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p);
7343 737508 : match_unsigned_saturation_sub (&gsi, as_a<gassign *> (stmt));
7344 737508 : break;
7345 :
7346 2313396 : case PLUS_EXPR:
7347 2313396 : if (match_saturation_add_with_assign (&gsi,
7348 : as_a<gassign *> (stmt)))
7349 : break;
7350 : /* fall-through */
7351 2622021 : case MINUS_EXPR:
7352 2622021 : if (!match_unsigned_saturation_sub (&gsi,
7353 : as_a<gassign *> (stmt))
7354 2622021 : && !convert_plusminus_to_widen (&gsi, stmt, code))
7355 : {
7356 2621896 : match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p);
7357 2621896 : if (gsi_stmt (gsi) == stmt)
7358 2615957 : match_uaddc_usubc (&gsi, stmt, code);
7359 : }
7360 : break;
7361 :
7362 39266 : case BIT_NOT_EXPR:
7363 39266 : if (match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p))
7364 239 : continue;
7365 : break;
7366 :
7367 57146 : case TRUNC_MOD_EXPR:
7368 57146 : convert_to_divmod (as_a<gassign *> (stmt));
7369 57146 : break;
7370 :
7371 173600 : case RSHIFT_EXPR:
7372 173600 : convert_mult_to_highpart (as_a<gassign *> (stmt), &gsi);
7373 173600 : break;
7374 :
7375 192811 : case BIT_IOR_EXPR:
7376 192811 : match_unsigned_saturation_mul (&gsi, as_a<gassign *> (stmt));
7377 192811 : match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt));
7378 192811 : match_unsigned_saturation_trunc (&gsi, as_a<gassign *> (stmt));
7379 : /* fall-through */
7380 224302 : case BIT_XOR_EXPR:
7381 224302 : match_uaddc_usubc (&gsi, stmt, code);
7382 224302 : break;
7383 :
7384 326953 : case EQ_EXPR:
7385 326953 : case NE_EXPR:
7386 326953 : case LE_EXPR:
7387 326953 : case GT_EXPR:
7388 326953 : match_single_bit_test (&gsi, stmt);
7389 326953 : break;
7390 :
7391 350544 : case COND_EXPR:
7392 350544 : case BIT_AND_EXPR:
7393 350544 : match_unsigned_saturation_sub (&gsi, as_a<gassign *> (stmt));
7394 350544 : break;
7395 :
7396 2466282 : case NOP_EXPR:
7397 2466282 : match_unsigned_saturation_mul (&gsi, as_a<gassign *> (stmt));
7398 2466282 : match_unsigned_saturation_trunc (&gsi, as_a<gassign *> (stmt));
7399 2466282 : match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt));
7400 : /* fall-through */
7401 2466297 : case CONVERT_EXPR:
7402 : /* The long-multiply recognizer's high-part emit ends in an
7403 : outer convert. If the trailing cast+mult+shift+cast
7404 : chain has no expansion strategy at the 2N width, lower
7405 : the whole chain to a longhand high-part at narrow
7406 : precision. */
7407 2466297 : if (gsi_stmt (gsi) == stmt
7408 2466297 : && lower_long_mul_high_chain (&gsi))
7409 2210 : continue;
7410 : break;
7411 :
7412 193 : default:;
7413 : }
7414 : }
7415 65162515 : else if (is_gimple_call (stmt))
7416 : {
7417 4901463 : switch (gimple_call_combined_fn (stmt))
7418 : {
7419 129 : case CFN_COND_MUL:
7420 129 : if (convert_mult_to_fma (stmt,
7421 : gimple_call_arg (stmt, 1),
7422 : gimple_call_arg (stmt, 2),
7423 : &fma_state,
7424 : gimple_call_arg (stmt, 0)))
7425 :
7426 : {
7427 84 : gsi_remove (&gsi, true);
7428 84 : release_defs (stmt);
7429 84 : continue;
7430 : }
7431 : break;
7432 :
7433 0 : case CFN_COND_LEN_MUL:
7434 0 : if (convert_mult_to_fma (stmt,
7435 : gimple_call_arg (stmt, 1),
7436 : gimple_call_arg (stmt, 2),
7437 : &fma_state,
7438 : gimple_call_arg (stmt, 0),
7439 : gimple_call_arg (stmt, 4),
7440 : gimple_call_arg (stmt, 5)))
7441 :
7442 : {
7443 0 : gsi_remove (&gsi, true);
7444 0 : release_defs (stmt);
7445 0 : continue;
7446 : }
7447 : break;
7448 :
7449 3730212 : case CFN_LAST:
7450 3730212 : cancel_fma_deferring (&fma_state);
7451 3730212 : break;
7452 :
7453 : default:
7454 : break;
7455 : }
7456 : }
7457 60261052 : else if (gimple_code (stmt) == GIMPLE_COND)
7458 : {
7459 4219657 : match_single_bit_test (&gsi, stmt);
7460 4219657 : optimize_spaceship (as_a <gcond *> (stmt));
7461 : }
7462 86863534 : gsi_next (&gsi);
7463 : }
7464 10458165 : if (fma_state.m_deferring_p
7465 7657243 : && fma_state.m_initial_phi)
7466 : {
7467 361 : gcc_checking_assert (fma_state.m_last_result);
7468 361 : if (!last_fma_candidate_feeds_initial_phi (&fma_state,
7469 : &m_last_result_set))
7470 264 : cancel_fma_deferring (&fma_state);
7471 : else
7472 97 : m_last_result_set.add (fma_state.m_last_result);
7473 : }
7474 10458165 : }
7475 :
7476 :
7477 : unsigned int
7478 983322 : pass_optimize_widening_mul::execute (function *fun)
7479 : {
7480 983322 : bool cfg_changed = false;
7481 :
7482 983322 : memset (&widen_mul_stats, 0, sizeof (widen_mul_stats));
7483 983322 : calculate_dominance_info (CDI_DOMINATORS);
7484 983322 : renumber_gimple_stmt_uids (cfun);
7485 :
7486 983322 : long_mul_phi_halves = new hash_map<tree, long_mul_halves>;
7487 :
7488 983322 : math_opts_dom_walker (&cfg_changed).walk (ENTRY_BLOCK_PTR_FOR_FN (cfun));
7489 :
7490 : /* A 2N multiply the target cannot expand would abort expand_mult. Every
7491 : statement has been through the lowerings above, so one left here matched
7492 : none of them. */
7493 983322 : basic_block bb;
7494 10458165 : FOR_EACH_BB_FN (bb, fun)
7495 106925379 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
7496 : {
7497 87975693 : gimple *stmt = gsi_stmt (gsi);
7498 87975693 : gsi_next (&gsi);
7499 87975693 : narrow_long_mul_halves (stmt);
7500 : }
7501 :
7502 1966644 : delete long_mul_phi_halves;
7503 983322 : long_mul_phi_halves = NULL;
7504 :
7505 983322 : statistics_counter_event (fun, "widening multiplications inserted",
7506 : widen_mul_stats.widen_mults_inserted);
7507 983322 : statistics_counter_event (fun, "widening maccs inserted",
7508 : widen_mul_stats.maccs_inserted);
7509 983322 : statistics_counter_event (fun, "fused multiply-adds inserted",
7510 : widen_mul_stats.fmas_inserted);
7511 983322 : statistics_counter_event (fun, "divmod calls inserted",
7512 : widen_mul_stats.divmod_calls_inserted);
7513 983322 : statistics_counter_event (fun, "highpart multiplications inserted",
7514 : widen_mul_stats.highpart_mults_inserted);
7515 :
7516 983322 : return cfg_changed ? TODO_cleanup_cfg : 0;
7517 : }
7518 :
7519 : } // anon namespace
7520 :
7521 : gimple_opt_pass *
7522 294587 : make_pass_optimize_widening_mul (gcc::context *ctxt)
7523 : {
7524 294587 : return new pass_optimize_widening_mul (ctxt);
7525 : }
|
