Line data Source code
1 : /* Global, SSA-based optimizations using mathematical identities.
2 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by the
8 : Free Software Foundation; either version 3, or (at your option) any
9 : later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : /* Currently, the only mini-pass in this file tries to CSE reciprocal
21 : operations. These are common in sequences such as this one:
22 :
23 : modulus = sqrt(x*x + y*y + z*z);
24 : x = x / modulus;
25 : y = y / modulus;
26 : z = z / modulus;
27 :
28 : that can be optimized to
29 :
30 : modulus = sqrt(x*x + y*y + z*z);
31 : rmodulus = 1.0 / modulus;
32 : x = x * rmodulus;
33 : y = y * rmodulus;
34 : z = z * rmodulus;
35 :
36 : We do this for loop invariant divisors, and with this pass whenever
37 : we notice that a division has the same divisor multiple times.
38 :
39 : Of course, like in PRE, we don't insert a division if a dominator
40 : already has one. However, this cannot be done as an extension of
41 : PRE for several reasons.
42 :
43 : First of all, with some experiments it was found out that the
44 : transformation is not always useful if there are only two divisions
45 : by the same divisor. This is probably because modern processors
46 : can pipeline the divisions; on older, in-order processors it should
47 : still be effective to optimize two divisions by the same number.
48 : We make this a param, and it shall be called N in the remainder of
49 : this comment.
50 :
51 : Second, if trapping math is active, we have less freedom on where
52 : to insert divisions: we can only do so in basic blocks that already
53 : contain one. (If divisions don't trap, instead, we can insert
54 : divisions elsewhere, which will be in blocks that are common dominators
55 : of those that have the division).
56 :
57 : We really don't want to compute the reciprocal unless a division will
58 : be found. To do this, we won't insert the division in a basic block
59 : that has less than N divisions *post-dominating* it.
60 :
61 : The algorithm constructs a subset of the dominator tree, holding the
62 : blocks containing the divisions and the common dominators to them,
63 : and walk it twice. The first walk is in post-order, and it annotates
64 : each block with the number of divisions that post-dominate it: this
65 : gives information on where divisions can be inserted profitably.
66 : The second walk is in pre-order, and it inserts divisions as explained
67 : above, and replaces divisions by multiplications.
68 :
69 : In the best case, the cost of the pass is O(n_statements). In the
70 : worst-case, the cost is due to creating the dominator tree subset,
71 : with a cost of O(n_basic_blocks ^ 2); however this can only happen
72 : for n_statements / n_basic_blocks statements. So, the amortized cost
73 : of creating the dominator tree subset is O(n_basic_blocks) and the
74 : worst-case cost of the pass is O(n_statements * n_basic_blocks).
75 :
76 : More practically, the cost will be small because there are few
77 : divisions, and they tend to be in the same basic block, so insert_bb
78 : is called very few times.
79 :
80 : If we did this using domwalk.cc, an efficient implementation would have
81 : to work on all the variables in a single pass, because we could not
82 : work on just a subset of the dominator tree, as we do now, and the
83 : cost would also be something like O(n_statements * n_basic_blocks).
84 : The data structures would be more complex in order to work on all the
85 : variables in a single pass. */
86 :
87 : #include "config.h"
88 : #include "system.h"
89 : #include "coretypes.h"
90 : #include "backend.h"
91 : #include "target.h"
92 : #include "rtl.h"
93 : #include "tree.h"
94 : #include "gimple.h"
95 : #include "predict.h"
96 : #include "alloc-pool.h"
97 : #include "tree-pass.h"
98 : #include "ssa.h"
99 : #include "optabs-tree.h"
100 : #include "gimple-pretty-print.h"
101 : #include "alias.h"
102 : #include "fold-const.h"
103 : #include "gimple-iterator.h"
104 : #include "gimple-fold.h"
105 : #include "stor-layout.h"
106 : #include "tree-cfg.h"
107 : #include "tree-dfa.h"
108 : #include "tree-ssa.h"
109 : #include "builtins.h"
110 : #include "internal-fn.h"
111 : #include "case-cfn-macros.h"
112 : #include "optabs-libfuncs.h"
113 : #include "tree-eh.h"
114 : #include "targhooks.h"
115 : #include "domwalk.h"
116 : #include "tree-ssa-math-opts.h"
117 : #include "dbgcnt.h"
118 : #include "langhooks.h"
119 : #include "cfghooks.h"
120 :
121 : /* This structure represents one basic block that either computes a
122 : division, or is a common dominator for basic block that compute a
123 : division. */
124 : struct occurrence {
125 : /* The basic block represented by this structure. */
126 : basic_block bb = basic_block();
127 :
128 : /* If non-NULL, the SSA_NAME holding the definition for a reciprocal
129 : inserted in BB. */
130 : tree recip_def = tree();
131 :
132 : /* If non-NULL, the SSA_NAME holding the definition for a squared
133 : reciprocal inserted in BB. */
134 : tree square_recip_def = tree();
135 :
136 : /* If non-NULL, the GIMPLE_ASSIGN for a reciprocal computation that
137 : was inserted in BB. */
138 : gimple *recip_def_stmt = nullptr;
139 :
140 : /* Pointer to a list of "struct occurrence"s for blocks dominated
141 : by BB. */
142 : struct occurrence *children = nullptr;
143 :
144 : /* Pointer to the next "struct occurrence"s in the list of blocks
145 : sharing a common dominator. */
146 : struct occurrence *next = nullptr;
147 :
148 : /* The number of divisions that are in BB before compute_merit. The
149 : number of divisions that are in BB or post-dominate it after
150 : compute_merit. */
151 : int num_divisions = 0;
152 :
153 : /* True if the basic block has a division, false if it is a common
154 : dominator for basic blocks that do. If it is false and trapping
155 : math is active, BB is not a candidate for inserting a reciprocal. */
156 : bool bb_has_division = false;
157 :
158 : /* Construct a struct occurrence for basic block BB, and whose
159 : children list is headed by CHILDREN. */
160 618 : occurrence (basic_block bb, struct occurrence *children)
161 618 : : bb (bb), children (children)
162 : {
163 618 : bb->aux = this;
164 : }
165 :
166 : /* Destroy a struct occurrence and remove it from its basic block. */
167 618 : ~occurrence ()
168 : {
169 618 : bb->aux = nullptr;
170 618 : }
171 :
172 : /* Allocate memory for a struct occurrence from OCC_POOL. */
173 : static void* operator new (size_t);
174 :
175 : /* Return memory for a struct occurrence to OCC_POOL. */
176 : static void operator delete (void*, size_t);
177 : };
178 :
179 : static struct
180 : {
181 : /* Number of 1.0/X ops inserted. */
182 : int rdivs_inserted;
183 :
184 : /* Number of 1.0/FUNC ops inserted. */
185 : int rfuncs_inserted;
186 : } reciprocal_stats;
187 :
188 : static struct
189 : {
190 : /* Number of cexpi calls inserted. */
191 : int inserted;
192 :
193 : /* Number of conversions removed. */
194 : int conv_removed;
195 :
196 : } sincos_stats;
197 :
198 : static struct
199 : {
200 : /* Number of widening multiplication ops inserted. */
201 : int widen_mults_inserted;
202 :
203 : /* Number of integer multiply-and-accumulate ops inserted. */
204 : int maccs_inserted;
205 :
206 : /* Number of fp fused multiply-add ops inserted. */
207 : int fmas_inserted;
208 :
209 : /* Number of divmod calls inserted. */
210 : int divmod_calls_inserted;
211 :
212 : /* Number of highpart multiplication ops inserted. */
213 : int highpart_mults_inserted;
214 : } widen_mul_stats;
215 :
216 : /* The instance of "struct occurrence" representing the highest
217 : interesting block in the dominator tree. */
218 : static struct occurrence *occ_head;
219 :
220 : /* Allocation pool for getting instances of "struct occurrence". */
221 : static object_allocator<occurrence> *occ_pool;
222 :
223 618 : void* occurrence::operator new (size_t n)
224 : {
225 618 : gcc_assert (n == sizeof(occurrence));
226 618 : return occ_pool->allocate_raw ();
227 : }
228 :
229 618 : void occurrence::operator delete (void *occ, size_t n)
230 : {
231 618 : gcc_assert (n == sizeof(occurrence));
232 618 : occ_pool->remove_raw (occ);
233 618 : }
234 :
235 : /* Insert NEW_OCC into our subset of the dominator tree. P_HEAD points to a
236 : list of "struct occurrence"s, one per basic block, having IDOM as
237 : their common dominator.
238 :
239 : We try to insert NEW_OCC as deep as possible in the tree, and we also
240 : insert any other block that is a common dominator for BB and one
241 : block already in the tree. */
242 :
243 : static void
244 607 : insert_bb (struct occurrence *new_occ, basic_block idom,
245 : struct occurrence **p_head)
246 : {
247 612 : struct occurrence *occ, **p_occ;
248 :
249 635 : for (p_occ = p_head; (occ = *p_occ) != NULL; )
250 : {
251 28 : basic_block bb = new_occ->bb, occ_bb = occ->bb;
252 28 : basic_block dom = nearest_common_dominator (CDI_DOMINATORS, occ_bb, bb);
253 28 : if (dom == bb)
254 : {
255 : /* BB dominates OCC_BB. OCC becomes NEW_OCC's child: remove OCC
256 : from its list. */
257 6 : *p_occ = occ->next;
258 6 : occ->next = new_occ->children;
259 6 : new_occ->children = occ;
260 :
261 : /* Try the next block (it may as well be dominated by BB). */
262 : }
263 :
264 22 : else if (dom == occ_bb)
265 : {
266 : /* OCC_BB dominates BB. Tail recurse to look deeper. */
267 5 : insert_bb (new_occ, dom, &occ->children);
268 5 : return;
269 : }
270 :
271 17 : else if (dom != idom)
272 : {
273 11 : gcc_assert (!dom->aux);
274 :
275 : /* There is a dominator between IDOM and BB, add it and make
276 : two children out of NEW_OCC and OCC. First, remove OCC from
277 : its list. */
278 11 : *p_occ = occ->next;
279 11 : new_occ->next = occ;
280 11 : occ->next = NULL;
281 :
282 : /* None of the previous blocks has DOM as a dominator: if we tail
283 : recursed, we would reexamine them uselessly. Just switch BB with
284 : DOM, and go on looking for blocks dominated by DOM. */
285 11 : new_occ = new occurrence (dom, new_occ);
286 : }
287 :
288 : else
289 : {
290 : /* Nothing special, go on with the next element. */
291 6 : p_occ = &occ->next;
292 : }
293 : }
294 :
295 : /* No place was found as a child of IDOM. Make BB a sibling of IDOM. */
296 607 : new_occ->next = *p_head;
297 607 : *p_head = new_occ;
298 : }
299 :
300 : /* Register that we found a division in BB.
301 : IMPORTANCE is a measure of how much weighting to give
302 : that division. Use IMPORTANCE = 2 to register a single
303 : division. If the division is going to be found multiple
304 : times use 1 (as it is with squares). */
305 :
306 : static inline void
307 709 : register_division_in (basic_block bb, int importance)
308 : {
309 709 : struct occurrence *occ;
310 :
311 709 : occ = (struct occurrence *) bb->aux;
312 709 : if (!occ)
313 : {
314 607 : occ = new occurrence (bb, NULL);
315 607 : insert_bb (occ, ENTRY_BLOCK_PTR_FOR_FN (cfun), &occ_head);
316 : }
317 :
318 709 : occ->bb_has_division = true;
319 709 : occ->num_divisions += importance;
320 709 : }
321 :
322 :
323 : /* Compute the number of divisions that postdominate each block in OCC and
324 : its children. */
325 :
326 : static void
327 26 : compute_merit (struct occurrence *occ)
328 : {
329 26 : struct occurrence *occ_child;
330 26 : basic_block dom = occ->bb;
331 :
332 45 : for (occ_child = occ->children; occ_child; occ_child = occ_child->next)
333 : {
334 19 : basic_block bb;
335 19 : if (occ_child->children)
336 3 : compute_merit (occ_child);
337 :
338 19 : if (flag_exceptions)
339 6 : bb = single_noncomplex_succ (dom);
340 : else
341 : bb = dom;
342 :
343 19 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, occ_child->bb))
344 12 : occ->num_divisions += occ_child->num_divisions;
345 : }
346 26 : }
347 :
348 :
349 : /* Return whether USE_STMT is a floating-point division by DEF. */
350 : static inline bool
351 352847 : is_division_by (gimple *use_stmt, tree def)
352 : {
353 352847 : return is_gimple_assign (use_stmt)
354 242973 : && gimple_assign_rhs_code (use_stmt) == RDIV_EXPR
355 1251 : && gimple_assign_rhs2 (use_stmt) == def
356 : /* Do not recognize x / x as valid division, as we are getting
357 : confused later by replacing all immediate uses x in such
358 : a stmt. */
359 879 : && gimple_assign_rhs1 (use_stmt) != def
360 353726 : && !stmt_can_throw_internal (cfun, use_stmt);
361 : }
362 :
363 : /* Return TRUE if USE_STMT is a multiplication of DEF by A. */
364 : static inline bool
365 349056 : is_mult_by (gimple *use_stmt, tree def, tree a)
366 : {
367 349056 : if (gimple_code (use_stmt) == GIMPLE_ASSIGN
368 349056 : && gimple_assign_rhs_code (use_stmt) == MULT_EXPR)
369 : {
370 80029 : tree op0 = gimple_assign_rhs1 (use_stmt);
371 80029 : tree op1 = gimple_assign_rhs2 (use_stmt);
372 :
373 80029 : return (op0 == def && op1 == a)
374 80029 : || (op0 == a && op1 == def);
375 : }
376 : return 0;
377 : }
378 :
379 : /* Return whether USE_STMT is DEF * DEF. */
380 : static inline bool
381 349011 : is_square_of (gimple *use_stmt, tree def)
382 : {
383 5 : return is_mult_by (use_stmt, def, def);
384 : }
385 :
386 : /* Return whether USE_STMT is a floating-point division by
387 : DEF * DEF. */
388 : static inline bool
389 192 : is_division_by_square (gimple *use_stmt, tree def)
390 : {
391 192 : if (gimple_code (use_stmt) == GIMPLE_ASSIGN
392 185 : && gimple_assign_rhs_code (use_stmt) == RDIV_EXPR
393 7 : && gimple_assign_rhs1 (use_stmt) != gimple_assign_rhs2 (use_stmt)
394 199 : && !stmt_can_throw_internal (cfun, use_stmt))
395 : {
396 7 : tree denominator = gimple_assign_rhs2 (use_stmt);
397 7 : if (TREE_CODE (denominator) == SSA_NAME)
398 7 : return is_square_of (SSA_NAME_DEF_STMT (denominator), def);
399 : }
400 : return 0;
401 : }
402 :
403 : /* Walk the subset of the dominator tree rooted at OCC, setting the
404 : RECIP_DEF field to a definition of 1.0 / DEF that can be used in
405 : the given basic block. The field may be left NULL, of course,
406 : if it is not possible or profitable to do the optimization.
407 :
408 : DEF_BSI is an iterator pointing at the statement defining DEF.
409 : If RECIP_DEF is set, a dominator already has a computation that can
410 : be used.
411 :
412 : If should_insert_square_recip is set, then this also inserts
413 : the square of the reciprocal immediately after the definition
414 : of the reciprocal. */
415 :
416 : static void
417 42 : insert_reciprocals (gimple_stmt_iterator *def_gsi, struct occurrence *occ,
418 : tree def, tree recip_def, tree square_recip_def,
419 : int should_insert_square_recip, int threshold)
420 : {
421 42 : tree type;
422 42 : gassign *new_stmt, *new_square_stmt;
423 42 : gimple_stmt_iterator gsi;
424 42 : struct occurrence *occ_child;
425 :
426 42 : if (!recip_def
427 26 : && (occ->bb_has_division || !flag_trapping_math)
428 : /* Divide by two as all divisions are counted twice in
429 : the costing loop. */
430 25 : && occ->num_divisions / 2 >= threshold)
431 : {
432 : /* Make a variable with the replacement and substitute it. */
433 24 : type = TREE_TYPE (def);
434 24 : recip_def = create_tmp_reg (type, "reciptmp");
435 24 : new_stmt = gimple_build_assign (recip_def, RDIV_EXPR,
436 : build_one_cst (type), def);
437 :
438 24 : if (should_insert_square_recip)
439 : {
440 4 : square_recip_def = create_tmp_reg (type, "powmult_reciptmp");
441 4 : new_square_stmt = gimple_build_assign (square_recip_def, MULT_EXPR,
442 : recip_def, recip_def);
443 : }
444 :
445 24 : if (occ->bb_has_division)
446 : {
447 : /* Case 1: insert before an existing division. */
448 21 : gsi = gsi_after_labels (occ->bb);
449 212 : while (!gsi_end_p (gsi)
450 212 : && (!is_division_by (gsi_stmt (gsi), def))
451 404 : && (!is_division_by_square (gsi_stmt (gsi), def)))
452 191 : gsi_next (&gsi);
453 :
454 21 : gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
455 21 : if (should_insert_square_recip)
456 3 : gsi_insert_before (&gsi, new_square_stmt, GSI_SAME_STMT);
457 : }
458 3 : else if (def_gsi && occ->bb == gsi_bb (*def_gsi))
459 : {
460 : /* Case 2: insert right after the definition. Note that this will
461 : never happen if the definition statement can throw, because in
462 : that case the sole successor of the statement's basic block will
463 : dominate all the uses as well. */
464 2 : gsi_insert_after (def_gsi, new_stmt, GSI_NEW_STMT);
465 2 : if (should_insert_square_recip)
466 1 : gsi_insert_after (def_gsi, new_square_stmt, GSI_NEW_STMT);
467 : }
468 : else
469 : {
470 : /* Case 3: insert in a basic block not containing defs/uses. */
471 1 : gsi = gsi_after_labels (occ->bb);
472 1 : gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
473 1 : if (should_insert_square_recip)
474 0 : gsi_insert_before (&gsi, new_square_stmt, GSI_SAME_STMT);
475 : }
476 :
477 24 : reciprocal_stats.rdivs_inserted++;
478 :
479 24 : occ->recip_def_stmt = new_stmt;
480 : }
481 :
482 42 : occ->recip_def = recip_def;
483 42 : occ->square_recip_def = square_recip_def;
484 61 : for (occ_child = occ->children; occ_child; occ_child = occ_child->next)
485 19 : insert_reciprocals (def_gsi, occ_child, def, recip_def,
486 : square_recip_def, should_insert_square_recip,
487 : threshold);
488 42 : }
489 :
490 : /* Replace occurrences of expr / (x * x) with expr * ((1 / x) * (1 / x)).
491 : Take as argument the use for (x * x). */
492 : static inline void
493 4 : replace_reciprocal_squares (use_operand_p use_p)
494 : {
495 4 : gimple *use_stmt = USE_STMT (use_p);
496 4 : basic_block bb = gimple_bb (use_stmt);
497 4 : struct occurrence *occ = (struct occurrence *) bb->aux;
498 :
499 8 : if (optimize_bb_for_speed_p (bb) && occ->square_recip_def
500 8 : && occ->recip_def)
501 : {
502 4 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
503 4 : gimple_assign_set_rhs_code (use_stmt, MULT_EXPR);
504 4 : gimple_assign_set_rhs2 (use_stmt, occ->square_recip_def);
505 4 : SET_USE (use_p, occ->square_recip_def);
506 4 : fold_stmt_inplace (&gsi);
507 4 : update_stmt (use_stmt);
508 : }
509 4 : }
510 :
511 :
512 : /* Replace the division at USE_P with a multiplication by the reciprocal, if
513 : possible. */
514 :
515 : static inline void
516 105 : replace_reciprocal (use_operand_p use_p)
517 : {
518 105 : gimple *use_stmt = USE_STMT (use_p);
519 105 : basic_block bb = gimple_bb (use_stmt);
520 105 : struct occurrence *occ = (struct occurrence *) bb->aux;
521 :
522 105 : if (optimize_bb_for_speed_p (bb)
523 105 : && occ->recip_def && use_stmt != occ->recip_def_stmt)
524 : {
525 80 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
526 80 : gimple_assign_set_rhs_code (use_stmt, MULT_EXPR);
527 80 : SET_USE (use_p, occ->recip_def);
528 80 : fold_stmt_inplace (&gsi);
529 80 : update_stmt (use_stmt);
530 : }
531 105 : }
532 :
533 :
534 : /* Free OCC and return one more "struct occurrence" to be freed. */
535 :
536 : static struct occurrence *
537 618 : free_bb (struct occurrence *occ)
538 : {
539 618 : struct occurrence *child, *next;
540 :
541 : /* First get the two pointers hanging off OCC. */
542 618 : next = occ->next;
543 618 : child = occ->children;
544 618 : delete occ;
545 :
546 : /* Now ensure that we don't recurse unless it is necessary. */
547 618 : if (!child)
548 : return next;
549 : else
550 : {
551 19 : while (next)
552 0 : next = free_bb (next);
553 :
554 : return child;
555 : }
556 : }
557 :
558 : /* Transform sequences like
559 : t = sqrt (a)
560 : x = 1.0 / t;
561 : r1 = x * x;
562 : r2 = a * x;
563 : into:
564 : t = sqrt (a)
565 : r1 = 1.0 / a;
566 : r2 = t;
567 : x = r1 * r2;
568 : depending on the uses of x, r1, r2. This removes one multiplication and
569 : allows the sqrt and division operations to execute in parallel.
570 : DEF_GSI is the gsi of the initial division by sqrt that defines
571 : DEF (x in the example above). */
572 :
573 : static void
574 556 : optimize_recip_sqrt (gimple_stmt_iterator *def_gsi, tree def)
575 : {
576 556 : gimple *use_stmt;
577 556 : imm_use_iterator use_iter;
578 556 : gimple *stmt = gsi_stmt (*def_gsi);
579 556 : tree x = def;
580 556 : tree orig_sqrt_ssa_name = gimple_assign_rhs2 (stmt);
581 556 : tree div_rhs1 = gimple_assign_rhs1 (stmt);
582 :
583 556 : if (TREE_CODE (orig_sqrt_ssa_name) != SSA_NAME
584 551 : || TREE_CODE (div_rhs1) != REAL_CST
585 742 : || !real_equal (&TREE_REAL_CST (div_rhs1), &dconst1))
586 544 : return;
587 :
588 110 : gcall *sqrt_stmt
589 586 : = dyn_cast <gcall *> (SSA_NAME_DEF_STMT (orig_sqrt_ssa_name));
590 :
591 42 : if (!sqrt_stmt || !gimple_call_lhs (sqrt_stmt))
592 : return;
593 :
594 42 : switch (gimple_call_combined_fn (sqrt_stmt))
595 : {
596 31 : CASE_CFN_SQRT:
597 31 : CASE_CFN_SQRT_FN:
598 31 : break;
599 :
600 : default:
601 : return;
602 : }
603 31 : tree a = gimple_call_arg (sqrt_stmt, 0);
604 :
605 : /* We have 'a' and 'x'. Now analyze the uses of 'x'. */
606 :
607 : /* Statements that use x in x * x. */
608 43 : auto_vec<gimple *> sqr_stmts;
609 : /* Statements that use x in a * x. */
610 12 : auto_vec<gimple *> mult_stmts;
611 31 : bool has_other_use = false;
612 31 : bool mult_on_main_path = false;
613 :
614 89 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, x)
615 : {
616 58 : if (is_gimple_debug (use_stmt))
617 1 : continue;
618 57 : if (is_square_of (use_stmt, x))
619 : {
620 12 : sqr_stmts.safe_push (use_stmt);
621 12 : if (gimple_bb (use_stmt) == gimple_bb (stmt))
622 58 : mult_on_main_path = true;
623 : }
624 45 : else if (is_mult_by (use_stmt, x, a))
625 : {
626 14 : mult_stmts.safe_push (use_stmt);
627 14 : if (gimple_bb (use_stmt) == gimple_bb (stmt))
628 58 : mult_on_main_path = true;
629 : }
630 : else
631 : has_other_use = true;
632 31 : }
633 :
634 : /* In the x * x and a * x cases we just rewire stmt operands or
635 : remove multiplications. In the has_other_use case we introduce
636 : a multiplication so make sure we don't introduce a multiplication
637 : on a path where there was none. */
638 31 : if (has_other_use && !mult_on_main_path)
639 19 : return;
640 :
641 12 : if (sqr_stmts.is_empty () && mult_stmts.is_empty ())
642 : return;
643 :
644 : /* If x = 1.0 / sqrt (a) has uses other than those optimized here we want
645 : to be able to compose it from the sqr and mult cases. */
646 41 : if (has_other_use && (sqr_stmts.is_empty () || mult_stmts.is_empty ()))
647 : return;
648 :
649 12 : if (dump_file)
650 : {
651 10 : fprintf (dump_file, "Optimizing reciprocal sqrt multiplications of\n");
652 10 : print_gimple_stmt (dump_file, sqrt_stmt, 0, TDF_NONE);
653 10 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
654 10 : fprintf (dump_file, "\n");
655 : }
656 :
657 12 : bool delete_div = !has_other_use;
658 12 : tree sqr_ssa_name = NULL_TREE;
659 22 : if (!sqr_stmts.is_empty ())
660 : {
661 : /* r1 = x * x. Transform the original
662 : x = 1.0 / t
663 : into
664 : tmp1 = 1.0 / a
665 : r1 = tmp1. */
666 :
667 10 : sqr_ssa_name
668 10 : = make_temp_ssa_name (TREE_TYPE (a), NULL, "recip_sqrt_sqr");
669 :
670 10 : if (dump_file)
671 : {
672 10 : fprintf (dump_file, "Replacing original division\n");
673 10 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
674 10 : fprintf (dump_file, "with new division\n");
675 : }
676 10 : stmt
677 10 : = gimple_build_assign (sqr_ssa_name, gimple_assign_rhs_code (stmt),
678 : gimple_assign_rhs1 (stmt), a);
679 10 : gsi_insert_before (def_gsi, stmt, GSI_SAME_STMT);
680 10 : gsi_remove (def_gsi, true);
681 10 : *def_gsi = gsi_for_stmt (stmt);
682 10 : fold_stmt_inplace (def_gsi);
683 10 : update_stmt (stmt);
684 :
685 10 : if (dump_file)
686 10 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
687 :
688 10 : delete_div = false;
689 10 : gimple *sqr_stmt;
690 10 : unsigned int i;
691 20 : FOR_EACH_VEC_ELT (sqr_stmts, i, sqr_stmt)
692 : {
693 10 : gimple_stmt_iterator gsi2 = gsi_for_stmt (sqr_stmt);
694 10 : gimple_assign_set_rhs_from_tree (&gsi2, sqr_ssa_name);
695 10 : update_stmt (sqr_stmt);
696 : }
697 : }
698 24 : if (!mult_stmts.is_empty ())
699 : {
700 : /* r2 = a * x. Transform this into:
701 : r2 = t (The original sqrt (a)). */
702 : unsigned int i;
703 24 : gimple *mult_stmt = NULL;
704 24 : FOR_EACH_VEC_ELT (mult_stmts, i, mult_stmt)
705 : {
706 12 : gimple_stmt_iterator gsi2 = gsi_for_stmt (mult_stmt);
707 :
708 12 : if (dump_file)
709 : {
710 10 : fprintf (dump_file, "Replacing squaring multiplication\n");
711 10 : print_gimple_stmt (dump_file, mult_stmt, 0, TDF_NONE);
712 10 : fprintf (dump_file, "with assignment\n");
713 : }
714 12 : gimple_assign_set_rhs_from_tree (&gsi2, orig_sqrt_ssa_name);
715 12 : fold_stmt_inplace (&gsi2);
716 12 : update_stmt (mult_stmt);
717 12 : if (dump_file)
718 10 : print_gimple_stmt (dump_file, mult_stmt, 0, TDF_NONE);
719 : }
720 : }
721 :
722 12 : if (has_other_use)
723 : {
724 : /* Using the two temporaries tmp1, tmp2 from above
725 : the original x is now:
726 : x = tmp1 * tmp2. */
727 10 : gcc_assert (orig_sqrt_ssa_name);
728 10 : gcc_assert (sqr_ssa_name);
729 :
730 10 : gimple *new_stmt
731 10 : = gimple_build_assign (x, MULT_EXPR,
732 : orig_sqrt_ssa_name, sqr_ssa_name);
733 10 : gsi_insert_after (def_gsi, new_stmt, GSI_NEW_STMT);
734 10 : update_stmt (stmt);
735 : }
736 2 : else if (delete_div)
737 : {
738 : /* Remove the original division. */
739 2 : gimple_stmt_iterator gsi2 = gsi_for_stmt (stmt);
740 2 : gsi_remove (&gsi2, true);
741 2 : release_defs (stmt);
742 : }
743 : else
744 0 : release_ssa_name (x);
745 : }
746 :
747 : /* Look for floating-point divisions among DEF's uses, and try to
748 : replace them by multiplications with the reciprocal. Add
749 : as many statements computing the reciprocal as needed.
750 :
751 : DEF must be a GIMPLE register of a floating-point type. */
752 :
753 : static void
754 213264 : execute_cse_reciprocals_1 (gimple_stmt_iterator *def_gsi, tree def)
755 : {
756 213264 : use_operand_p use_p, square_use_p;
757 213264 : imm_use_iterator use_iter, square_use_iter;
758 213264 : tree square_def;
759 213264 : struct occurrence *occ;
760 213264 : int count = 0;
761 213264 : int threshold;
762 213264 : int square_recip_count = 0;
763 213264 : int sqrt_recip_count = 0;
764 :
765 213264 : gcc_assert (FLOAT_TYPE_P (TREE_TYPE (def)) && TREE_CODE (def) == SSA_NAME);
766 213264 : threshold = targetm.min_divisions_for_recip_mul (TYPE_MODE (TREE_TYPE (def)));
767 :
768 : /* If DEF is a square (x * x), count the number of divisions by x.
769 : If there are more divisions by x than by (DEF * DEF), prefer to optimize
770 : the reciprocal of x instead of DEF. This improves cases like:
771 : def = x * x
772 : t0 = a / def
773 : t1 = b / def
774 : t2 = c / x
775 : Reciprocal optimization of x results in 1 division rather than 2 or 3. */
776 213264 : gimple *def_stmt = SSA_NAME_DEF_STMT (def);
777 :
778 213264 : if (is_gimple_assign (def_stmt)
779 166787 : && gimple_assign_rhs_code (def_stmt) == MULT_EXPR
780 40571 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME
781 253758 : && gimple_assign_rhs1 (def_stmt) == gimple_assign_rhs2 (def_stmt))
782 : {
783 671 : tree op0 = gimple_assign_rhs1 (def_stmt);
784 :
785 2754 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, op0)
786 : {
787 2083 : gimple *use_stmt = USE_STMT (use_p);
788 2083 : if (is_division_by (use_stmt, op0))
789 17 : sqrt_recip_count++;
790 671 : }
791 : }
792 :
793 562206 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, def)
794 : {
795 348942 : gimple *use_stmt = USE_STMT (use_p);
796 348942 : if (is_division_by (use_stmt, def))
797 : {
798 637 : register_division_in (gimple_bb (use_stmt), 2);
799 637 : count++;
800 : }
801 :
802 348942 : if (is_square_of (use_stmt, def))
803 : {
804 1350 : square_def = gimple_assign_lhs (use_stmt);
805 2832 : FOR_EACH_IMM_USE_FAST (square_use_p, square_use_iter, square_def)
806 : {
807 1482 : gimple *square_use_stmt = USE_STMT (square_use_p);
808 1482 : if (is_division_by (square_use_stmt, square_def))
809 : {
810 : /* This is executed twice for each division by a square. */
811 72 : register_division_in (gimple_bb (square_use_stmt), 1);
812 72 : square_recip_count++;
813 : }
814 1350 : }
815 : }
816 213264 : }
817 :
818 : /* Square reciprocals were counted twice above. */
819 213264 : square_recip_count /= 2;
820 :
821 : /* If it is more profitable to optimize 1 / x, don't optimize 1 / (x * x). */
822 213264 : if (sqrt_recip_count > square_recip_count)
823 17 : goto out;
824 :
825 : /* Do the expensive part only if we can hope to optimize something. */
826 213247 : if (count + square_recip_count >= threshold && count >= 1)
827 : {
828 23 : gimple *use_stmt;
829 46 : for (occ = occ_head; occ; occ = occ->next)
830 : {
831 23 : compute_merit (occ);
832 23 : insert_reciprocals (def_gsi, occ, def, NULL, NULL,
833 : square_recip_count, threshold);
834 : }
835 :
836 143 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, def)
837 : {
838 120 : if (is_division_by (use_stmt, def))
839 : {
840 210 : FOR_EACH_IMM_USE_ON_STMT (use_p, use_iter)
841 105 : replace_reciprocal (use_p);
842 : }
843 20 : else if (square_recip_count > 0 && is_square_of (use_stmt, def))
844 : {
845 12 : FOR_EACH_IMM_USE_ON_STMT (use_p, use_iter)
846 : {
847 : /* Find all uses of the square that are divisions and
848 : * replace them by multiplications with the inverse. */
849 8 : imm_use_iterator square_iterator;
850 8 : gimple *powmult_use_stmt = USE_STMT (use_p);
851 8 : tree powmult_def_name = gimple_assign_lhs (powmult_use_stmt);
852 :
853 16 : FOR_EACH_IMM_USE_STMT (powmult_use_stmt,
854 : square_iterator, powmult_def_name)
855 16 : FOR_EACH_IMM_USE_ON_STMT (square_use_p, square_iterator)
856 : {
857 8 : gimple *powmult_use_stmt = USE_STMT (square_use_p);
858 8 : if (is_division_by (powmult_use_stmt, powmult_def_name))
859 4 : replace_reciprocal_squares (square_use_p);
860 8 : }
861 : }
862 : }
863 23 : }
864 : }
865 :
866 213224 : out:
867 213882 : for (occ = occ_head; occ; )
868 618 : occ = free_bb (occ);
869 :
870 213264 : occ_head = NULL;
871 213264 : }
872 :
873 : /* Return an internal function that implements the reciprocal of CALL,
874 : or IFN_LAST if there is no such function that the target supports. */
875 :
876 : internal_fn
877 113 : internal_fn_reciprocal (gcall *call)
878 : {
879 113 : internal_fn ifn;
880 :
881 113 : switch (gimple_call_combined_fn (call))
882 : {
883 97 : CASE_CFN_SQRT:
884 97 : CASE_CFN_SQRT_FN:
885 97 : ifn = IFN_RSQRT;
886 97 : break;
887 :
888 : default:
889 : return IFN_LAST;
890 : }
891 :
892 97 : tree_pair types = direct_internal_fn_types (ifn, call);
893 97 : if (!direct_internal_fn_supported_p (ifn, types, OPTIMIZE_FOR_SPEED))
894 44 : return IFN_LAST;
895 :
896 : return ifn;
897 : }
898 :
899 : /* Go through all the floating-point SSA_NAMEs, and call
900 : execute_cse_reciprocals_1 on each of them. */
901 : namespace {
902 :
903 : const pass_data pass_data_cse_reciprocals =
904 : {
905 : GIMPLE_PASS, /* type */
906 : "recip", /* name */
907 : OPTGROUP_NONE, /* optinfo_flags */
908 : TV_TREE_RECIP, /* tv_id */
909 : PROP_ssa, /* properties_required */
910 : 0, /* properties_provided */
911 : 0, /* properties_destroyed */
912 : 0, /* todo_flags_start */
913 : TODO_update_ssa, /* todo_flags_finish */
914 : };
915 :
916 : class pass_cse_reciprocals : public gimple_opt_pass
917 : {
918 : public:
919 294536 : pass_cse_reciprocals (gcc::context *ctxt)
920 589072 : : gimple_opt_pass (pass_data_cse_reciprocals, ctxt)
921 : {}
922 :
923 : /* opt_pass methods: */
924 1068219 : bool gate (function *) final override
925 : {
926 1068219 : return optimize && flag_reciprocal_math;
927 : }
928 : unsigned int execute (function *) final override;
929 :
930 : }; // class pass_cse_reciprocals
931 :
932 : unsigned int
933 8802 : pass_cse_reciprocals::execute (function *fun)
934 : {
935 8802 : basic_block bb;
936 8802 : tree arg;
937 :
938 8802 : occ_pool = new object_allocator<occurrence> ("dominators for recip");
939 :
940 8802 : memset (&reciprocal_stats, 0, sizeof (reciprocal_stats));
941 8802 : calculate_dominance_info (CDI_DOMINATORS);
942 8802 : calculate_dominance_info (CDI_POST_DOMINATORS);
943 :
944 8802 : if (flag_checking)
945 94608 : FOR_EACH_BB_FN (bb, fun)
946 85806 : gcc_assert (!bb->aux);
947 :
948 21828 : for (arg = DECL_ARGUMENTS (fun->decl); arg; arg = DECL_CHAIN (arg))
949 20687 : if (FLOAT_TYPE_P (TREE_TYPE (arg))
950 14113 : && is_gimple_reg (arg))
951 : {
952 6451 : tree name = ssa_default_def (fun, arg);
953 6451 : if (name)
954 5456 : execute_cse_reciprocals_1 (NULL, name);
955 : }
956 :
957 94608 : FOR_EACH_BB_FN (bb, fun)
958 : {
959 85806 : tree def;
960 :
961 196914 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
962 111108 : gsi_next (&gsi))
963 : {
964 111108 : gphi *phi = gsi.phi ();
965 111108 : def = PHI_RESULT (phi);
966 111108 : if (! virtual_operand_p (def)
967 111108 : && FLOAT_TYPE_P (TREE_TYPE (def)))
968 30803 : execute_cse_reciprocals_1 (NULL, def);
969 : }
970 :
971 1381031 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
972 1295225 : gsi_next (&gsi))
973 : {
974 1295225 : gimple *stmt = gsi_stmt (gsi);
975 :
976 2590450 : if (gimple_has_lhs (stmt)
977 812954 : && (def = SINGLE_SSA_TREE_OPERAND (stmt, SSA_OP_DEF)) != NULL
978 772693 : && FLOAT_TYPE_P (TREE_TYPE (def))
979 199925 : && TREE_CODE (def) == SSA_NAME)
980 : {
981 177005 : execute_cse_reciprocals_1 (&gsi, def);
982 177005 : stmt = gsi_stmt (gsi);
983 177005 : if (flag_unsafe_math_optimizations
984 176960 : && is_gimple_assign (stmt)
985 166744 : && gimple_assign_lhs (stmt) == def
986 166742 : && !stmt_can_throw_internal (cfun, stmt)
987 343703 : && gimple_assign_rhs_code (stmt) == RDIV_EXPR)
988 556 : optimize_recip_sqrt (&gsi, def);
989 : }
990 : }
991 :
992 85806 : if (optimize_bb_for_size_p (bb))
993 5371 : continue;
994 :
995 : /* Scan for a/func(b) and convert it to reciprocal a*rfunc(b). */
996 1353661 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
997 1273226 : gsi_next (&gsi))
998 : {
999 1273226 : gimple *stmt = gsi_stmt (gsi);
1000 :
1001 1273226 : if (is_gimple_assign (stmt)
1002 1273226 : && gimple_assign_rhs_code (stmt) == RDIV_EXPR)
1003 : {
1004 598 : tree arg1 = gimple_assign_rhs2 (stmt);
1005 598 : gimple *stmt1;
1006 :
1007 598 : if (TREE_CODE (arg1) != SSA_NAME)
1008 5 : continue;
1009 :
1010 593 : stmt1 = SSA_NAME_DEF_STMT (arg1);
1011 :
1012 593 : if (is_gimple_call (stmt1)
1013 593 : && gimple_call_lhs (stmt1))
1014 : {
1015 113 : bool fail;
1016 113 : imm_use_iterator ui;
1017 113 : use_operand_p use_p;
1018 113 : tree fndecl = NULL_TREE;
1019 :
1020 113 : gcall *call = as_a <gcall *> (stmt1);
1021 113 : internal_fn ifn = internal_fn_reciprocal (call);
1022 113 : if (ifn == IFN_LAST)
1023 : {
1024 60 : fndecl = gimple_call_fndecl (call);
1025 120 : if (!fndecl
1026 60 : || !fndecl_built_in_p (fndecl, BUILT_IN_MD))
1027 62 : continue;
1028 0 : fndecl = targetm.builtin_reciprocal (fndecl);
1029 0 : if (!fndecl)
1030 0 : continue;
1031 : }
1032 :
1033 : /* Check that all uses of the SSA name are divisions,
1034 : otherwise replacing the defining statement will do
1035 : the wrong thing. */
1036 53 : fail = false;
1037 106 : FOR_EACH_IMM_USE_FAST (use_p, ui, arg1)
1038 : {
1039 55 : gimple *stmt2 = USE_STMT (use_p);
1040 55 : if (is_gimple_debug (stmt2))
1041 0 : continue;
1042 55 : if (!is_gimple_assign (stmt2)
1043 55 : || gimple_assign_rhs_code (stmt2) != RDIV_EXPR
1044 53 : || gimple_assign_rhs1 (stmt2) == arg1
1045 108 : || gimple_assign_rhs2 (stmt2) != arg1)
1046 : {
1047 : fail = true;
1048 : break;
1049 : }
1050 53 : }
1051 53 : if (fail)
1052 2 : continue;
1053 :
1054 51 : gimple_replace_ssa_lhs (call, arg1);
1055 51 : reset_flow_sensitive_info (arg1);
1056 51 : if (gimple_call_internal_p (call) != (ifn != IFN_LAST))
1057 : {
1058 30 : auto_vec<tree, 4> args;
1059 30 : for (unsigned int i = 0;
1060 60 : i < gimple_call_num_args (call); i++)
1061 30 : args.safe_push (gimple_call_arg (call, i));
1062 30 : gcall *stmt2;
1063 30 : if (ifn == IFN_LAST)
1064 0 : stmt2 = gimple_build_call_vec (fndecl, args);
1065 : else
1066 30 : stmt2 = gimple_build_call_internal_vec (ifn, args);
1067 30 : gimple_call_set_lhs (stmt2, arg1);
1068 30 : gimple_move_vops (stmt2, call);
1069 30 : gimple_call_set_nothrow (stmt2,
1070 : gimple_call_nothrow_p (call));
1071 30 : gimple_stmt_iterator gsi2 = gsi_for_stmt (call);
1072 30 : gsi_replace (&gsi2, stmt2, true);
1073 30 : }
1074 : else
1075 : {
1076 21 : if (ifn == IFN_LAST)
1077 0 : gimple_call_set_fndecl (call, fndecl);
1078 : else
1079 21 : gimple_call_set_internal_fn (call, ifn);
1080 21 : update_stmt (call);
1081 : }
1082 51 : reciprocal_stats.rfuncs_inserted++;
1083 :
1084 102 : FOR_EACH_IMM_USE_STMT (stmt, ui, arg1)
1085 : {
1086 51 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1087 51 : gimple_assign_set_rhs_code (stmt, MULT_EXPR);
1088 51 : fold_stmt_inplace (&gsi);
1089 51 : update_stmt (stmt);
1090 51 : }
1091 : }
1092 : }
1093 : }
1094 : }
1095 :
1096 8802 : statistics_counter_event (fun, "reciprocal divs inserted",
1097 : reciprocal_stats.rdivs_inserted);
1098 8802 : statistics_counter_event (fun, "reciprocal functions inserted",
1099 : reciprocal_stats.rfuncs_inserted);
1100 :
1101 8802 : free_dominance_info (CDI_DOMINATORS);
1102 8802 : free_dominance_info (CDI_POST_DOMINATORS);
1103 17604 : delete occ_pool;
1104 8802 : return 0;
1105 : }
1106 :
1107 : } // anon namespace
1108 :
1109 : gimple_opt_pass *
1110 294536 : make_pass_cse_reciprocals (gcc::context *ctxt)
1111 : {
1112 294536 : return new pass_cse_reciprocals (ctxt);
1113 : }
1114 :
1115 : /* If NAME is the result of a type conversion, look for other
1116 : equivalent dominating or dominated conversions, and replace all
1117 : uses with the earliest dominating name, removing the redundant
1118 : conversions. Return the prevailing name. */
1119 :
1120 : static tree
1121 1069 : execute_cse_conv_1 (tree name, bool *cfg_changed)
1122 : {
1123 1069 : if (SSA_NAME_IS_DEFAULT_DEF (name)
1124 1069 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
1125 : return name;
1126 :
1127 969 : gimple *def_stmt = SSA_NAME_DEF_STMT (name);
1128 :
1129 969 : if (!gimple_assign_cast_p (def_stmt))
1130 : return name;
1131 :
1132 136 : tree src = gimple_assign_rhs1 (def_stmt);
1133 :
1134 136 : if (TREE_CODE (src) != SSA_NAME)
1135 : return name;
1136 :
1137 136 : imm_use_iterator use_iter;
1138 136 : gimple *use_stmt;
1139 :
1140 : /* Find the earliest dominating def. */
1141 521 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, src)
1142 : {
1143 763 : if (use_stmt == def_stmt
1144 385 : || !gimple_assign_cast_p (use_stmt))
1145 378 : continue;
1146 :
1147 7 : tree lhs = gimple_assign_lhs (use_stmt);
1148 :
1149 7 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
1150 14 : || (gimple_assign_rhs1 (use_stmt)
1151 7 : != gimple_assign_rhs1 (def_stmt))
1152 14 : || !types_compatible_p (TREE_TYPE (name), TREE_TYPE (lhs)))
1153 0 : continue;
1154 :
1155 7 : bool use_dominates;
1156 7 : if (gimple_bb (def_stmt) == gimple_bb (use_stmt))
1157 : {
1158 0 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
1159 0 : while (!gsi_end_p (gsi) && gsi_stmt (gsi) != def_stmt)
1160 0 : gsi_next (&gsi);
1161 0 : use_dominates = !gsi_end_p (gsi);
1162 : }
1163 7 : else if (dominated_by_p (CDI_DOMINATORS, gimple_bb (use_stmt),
1164 7 : gimple_bb (def_stmt)))
1165 : use_dominates = false;
1166 7 : else if (dominated_by_p (CDI_DOMINATORS, gimple_bb (def_stmt),
1167 7 : gimple_bb (use_stmt)))
1168 : use_dominates = true;
1169 : else
1170 4 : continue;
1171 :
1172 0 : if (use_dominates)
1173 : {
1174 : std::swap (name, lhs);
1175 : std::swap (def_stmt, use_stmt);
1176 : }
1177 136 : }
1178 :
1179 : /* Now go through all uses of SRC again, replacing the equivalent
1180 : dominated conversions. We may replace defs that were not
1181 : dominated by the then-prevailing defs when we first visited
1182 : them. */
1183 521 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, src)
1184 : {
1185 763 : if (use_stmt == def_stmt
1186 385 : || !gimple_assign_cast_p (use_stmt))
1187 378 : continue;
1188 :
1189 7 : tree lhs = gimple_assign_lhs (use_stmt);
1190 :
1191 7 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
1192 14 : || (gimple_assign_rhs1 (use_stmt)
1193 7 : != gimple_assign_rhs1 (def_stmt))
1194 14 : || !types_compatible_p (TREE_TYPE (name), TREE_TYPE (lhs)))
1195 0 : continue;
1196 :
1197 7 : basic_block use_bb = gimple_bb (use_stmt);
1198 7 : if (gimple_bb (def_stmt) == use_bb
1199 7 : || dominated_by_p (CDI_DOMINATORS, use_bb, gimple_bb (def_stmt)))
1200 : {
1201 3 : sincos_stats.conv_removed++;
1202 :
1203 3 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
1204 3 : replace_uses_by (lhs, name);
1205 3 : if (gsi_remove (&gsi, true)
1206 3 : && gimple_purge_dead_eh_edges (use_bb))
1207 3 : *cfg_changed = true;
1208 3 : release_defs (use_stmt);
1209 : }
1210 136 : }
1211 :
1212 136 : return name;
1213 : }
1214 :
1215 : /* Records an occurrence at statement USE_STMT in the vector of trees
1216 : STMTS if it is dominated by *TOP_BB or dominates it or this basic block
1217 : is not yet initialized. Returns true if the occurrence was pushed on
1218 : the vector. Adjusts *TOP_BB to be the basic block dominating all
1219 : statements in the vector. */
1220 :
1221 : static bool
1222 1290 : maybe_record_sincos (vec<gimple *> *stmts,
1223 : basic_block *top_bb, gimple *use_stmt)
1224 : {
1225 1290 : basic_block use_bb = gimple_bb (use_stmt);
1226 1290 : if (*top_bb
1227 1290 : && (*top_bb == use_bb
1228 66 : || dominated_by_p (CDI_DOMINATORS, use_bb, *top_bb)))
1229 168 : stmts->safe_push (use_stmt);
1230 1122 : else if (!*top_bb
1231 1122 : || dominated_by_p (CDI_DOMINATORS, *top_bb, use_bb))
1232 : {
1233 1102 : stmts->safe_push (use_stmt);
1234 1102 : *top_bb = use_bb;
1235 : }
1236 : else
1237 : return false;
1238 :
1239 : return true;
1240 : }
1241 :
1242 : /* Look for sin, cos and cexpi calls with the same argument NAME and
1243 : create a single call to cexpi CSEing the result in this case.
1244 : We first walk over all immediate uses of the argument collecting
1245 : statements that we can CSE in a vector and in a second pass replace
1246 : the statement rhs with a REALPART or IMAGPART expression on the
1247 : result of the cexpi call we insert before the use statement that
1248 : dominates all other candidates. */
1249 :
1250 : static bool
1251 1069 : execute_cse_sincos_1 (tree name)
1252 : {
1253 1069 : gimple_stmt_iterator gsi;
1254 1069 : imm_use_iterator use_iter;
1255 1069 : tree fndecl, res, type = NULL_TREE;
1256 1069 : gimple *def_stmt, *use_stmt, *stmt;
1257 1069 : int seen_cos = 0, seen_sin = 0, seen_cexpi = 0;
1258 1069 : auto_vec<gimple *> stmts;
1259 1069 : basic_block top_bb = NULL;
1260 1069 : int i;
1261 1069 : bool cfg_changed = false;
1262 :
1263 1069 : name = execute_cse_conv_1 (name, &cfg_changed);
1264 :
1265 4212 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, name)
1266 : {
1267 3145 : if (gimple_code (use_stmt) != GIMPLE_CALL
1268 3145 : || !gimple_call_lhs (use_stmt))
1269 1823 : continue;
1270 :
1271 1322 : switch (gimple_call_combined_fn (use_stmt))
1272 : {
1273 462 : CASE_CFN_COS:
1274 462 : seen_cos |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
1275 462 : break;
1276 :
1277 822 : CASE_CFN_SIN:
1278 822 : seen_sin |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
1279 822 : break;
1280 :
1281 6 : CASE_CFN_CEXPI:
1282 6 : seen_cexpi |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
1283 6 : break;
1284 :
1285 32 : default:;
1286 32 : continue;
1287 : }
1288 :
1289 1290 : auto stmt_cfn = gimple_call_combined_fn (use_stmt);
1290 1290 : tree t = mathfn_built_in_type (stmt_cfn);
1291 1290 : if (!t)
1292 : {
1293 : /* It is possible to get IFN_{SIN,COS} calls, for which
1294 : mathfn_built_in_type will return NULL. Those are normally only
1295 : present for vector operations. We won't be able to CSE those
1296 : at the moment. */
1297 2 : gcc_checking_assert (internal_fn_p (stmt_cfn));
1298 : return false;
1299 : }
1300 :
1301 1288 : if (!type)
1302 : {
1303 1067 : type = t;
1304 1067 : t = TREE_TYPE (name);
1305 : }
1306 : /* This checks that NAME has the right type in the first round,
1307 : and, in subsequent rounds, that the built_in type is the same
1308 : type, or a compatible type. */
1309 1288 : if (type != t && !types_compatible_p (type, t))
1310 : return false;
1311 2 : }
1312 1067 : if (seen_cos + seen_sin + seen_cexpi <= 1)
1313 : return false;
1314 :
1315 : /* Simply insert cexpi at the beginning of top_bb but not earlier than
1316 : the name def statement. */
1317 201 : fndecl = mathfn_built_in (type, BUILT_IN_CEXPI);
1318 201 : if (!fndecl)
1319 : return false;
1320 157 : stmt = gimple_build_call (fndecl, 1, name);
1321 157 : res = make_temp_ssa_name (TREE_TYPE (TREE_TYPE (fndecl)), stmt, "sincostmp");
1322 157 : gimple_call_set_lhs (stmt, res);
1323 :
1324 157 : def_stmt = SSA_NAME_DEF_STMT (name);
1325 157 : if (!SSA_NAME_IS_DEFAULT_DEF (name)
1326 127 : && gimple_code (def_stmt) != GIMPLE_PHI
1327 276 : && gimple_bb (def_stmt) == top_bb)
1328 : {
1329 119 : gsi = gsi_for_stmt (def_stmt);
1330 119 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
1331 : }
1332 : else
1333 : {
1334 38 : gsi = gsi_after_labels (top_bb);
1335 38 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1336 : }
1337 157 : sincos_stats.inserted++;
1338 :
1339 : /* And adjust the recorded old call sites. */
1340 471 : for (i = 0; stmts.iterate (i, &use_stmt); ++i)
1341 : {
1342 314 : tree rhs = NULL;
1343 :
1344 314 : switch (gimple_call_combined_fn (use_stmt))
1345 : {
1346 157 : CASE_CFN_COS:
1347 157 : rhs = fold_build1 (REALPART_EXPR, type, res);
1348 157 : break;
1349 :
1350 157 : CASE_CFN_SIN:
1351 157 : rhs = fold_build1 (IMAGPART_EXPR, type, res);
1352 157 : break;
1353 :
1354 : CASE_CFN_CEXPI:
1355 : rhs = res;
1356 : break;
1357 :
1358 0 : default:;
1359 0 : gcc_unreachable ();
1360 : }
1361 :
1362 : /* Replace call with a copy. */
1363 314 : stmt = gimple_build_assign (gimple_call_lhs (use_stmt), rhs);
1364 :
1365 314 : gsi = gsi_for_stmt (use_stmt);
1366 314 : gsi_replace (&gsi, stmt, true);
1367 314 : if (gimple_purge_dead_eh_edges (gimple_bb (stmt)))
1368 0 : cfg_changed = true;
1369 : }
1370 :
1371 157 : return cfg_changed;
1372 1069 : }
1373 :
1374 : /* To evaluate powi(x,n), the floating point value x raised to the
1375 : constant integer exponent n, we use a hybrid algorithm that
1376 : combines the "window method" with look-up tables. For an
1377 : introduction to exponentiation algorithms and "addition chains",
1378 : see section 4.6.3, "Evaluation of Powers" of Donald E. Knuth,
1379 : "Seminumerical Algorithms", Vol. 2, "The Art of Computer Programming",
1380 : 3rd Edition, 1998, and Daniel M. Gordon, "A Survey of Fast Exponentiation
1381 : Methods", Journal of Algorithms, Vol. 27, pp. 129-146, 1998. */
1382 :
1383 : /* Provide a default value for POWI_MAX_MULTS, the maximum number of
1384 : multiplications to inline before calling the system library's pow
1385 : function. powi(x,n) requires at worst 2*bits(n)-2 multiplications,
1386 : so this default never requires calling pow, powf or powl. */
1387 :
1388 : #ifndef POWI_MAX_MULTS
1389 : #define POWI_MAX_MULTS (2*HOST_BITS_PER_WIDE_INT-2)
1390 : #endif
1391 :
1392 : /* The size of the "optimal power tree" lookup table. All
1393 : exponents less than this value are simply looked up in the
1394 : powi_table below. This threshold is also used to size the
1395 : cache of pseudo registers that hold intermediate results. */
1396 : #define POWI_TABLE_SIZE 256
1397 :
1398 : /* The size, in bits of the window, used in the "window method"
1399 : exponentiation algorithm. This is equivalent to a radix of
1400 : (1<<POWI_WINDOW_SIZE) in the corresponding "m-ary method". */
1401 : #define POWI_WINDOW_SIZE 3
1402 :
1403 : /* The following table is an efficient representation of an
1404 : "optimal power tree". For each value, i, the corresponding
1405 : value, j, in the table states than an optimal evaluation
1406 : sequence for calculating pow(x,i) can be found by evaluating
1407 : pow(x,j)*pow(x,i-j). An optimal power tree for the first
1408 : 100 integers is given in Knuth's "Seminumerical algorithms". */
1409 :
1410 : static const unsigned char powi_table[POWI_TABLE_SIZE] =
1411 : {
1412 : 0, 1, 1, 2, 2, 3, 3, 4, /* 0 - 7 */
1413 : 4, 6, 5, 6, 6, 10, 7, 9, /* 8 - 15 */
1414 : 8, 16, 9, 16, 10, 12, 11, 13, /* 16 - 23 */
1415 : 12, 17, 13, 18, 14, 24, 15, 26, /* 24 - 31 */
1416 : 16, 17, 17, 19, 18, 33, 19, 26, /* 32 - 39 */
1417 : 20, 25, 21, 40, 22, 27, 23, 44, /* 40 - 47 */
1418 : 24, 32, 25, 34, 26, 29, 27, 44, /* 48 - 55 */
1419 : 28, 31, 29, 34, 30, 60, 31, 36, /* 56 - 63 */
1420 : 32, 64, 33, 34, 34, 46, 35, 37, /* 64 - 71 */
1421 : 36, 65, 37, 50, 38, 48, 39, 69, /* 72 - 79 */
1422 : 40, 49, 41, 43, 42, 51, 43, 58, /* 80 - 87 */
1423 : 44, 64, 45, 47, 46, 59, 47, 76, /* 88 - 95 */
1424 : 48, 65, 49, 66, 50, 67, 51, 66, /* 96 - 103 */
1425 : 52, 70, 53, 74, 54, 104, 55, 74, /* 104 - 111 */
1426 : 56, 64, 57, 69, 58, 78, 59, 68, /* 112 - 119 */
1427 : 60, 61, 61, 80, 62, 75, 63, 68, /* 120 - 127 */
1428 : 64, 65, 65, 128, 66, 129, 67, 90, /* 128 - 135 */
1429 : 68, 73, 69, 131, 70, 94, 71, 88, /* 136 - 143 */
1430 : 72, 128, 73, 98, 74, 132, 75, 121, /* 144 - 151 */
1431 : 76, 102, 77, 124, 78, 132, 79, 106, /* 152 - 159 */
1432 : 80, 97, 81, 160, 82, 99, 83, 134, /* 160 - 167 */
1433 : 84, 86, 85, 95, 86, 160, 87, 100, /* 168 - 175 */
1434 : 88, 113, 89, 98, 90, 107, 91, 122, /* 176 - 183 */
1435 : 92, 111, 93, 102, 94, 126, 95, 150, /* 184 - 191 */
1436 : 96, 128, 97, 130, 98, 133, 99, 195, /* 192 - 199 */
1437 : 100, 128, 101, 123, 102, 164, 103, 138, /* 200 - 207 */
1438 : 104, 145, 105, 146, 106, 109, 107, 149, /* 208 - 215 */
1439 : 108, 200, 109, 146, 110, 170, 111, 157, /* 216 - 223 */
1440 : 112, 128, 113, 130, 114, 182, 115, 132, /* 224 - 231 */
1441 : 116, 200, 117, 132, 118, 158, 119, 206, /* 232 - 239 */
1442 : 120, 240, 121, 162, 122, 147, 123, 152, /* 240 - 247 */
1443 : 124, 166, 125, 214, 126, 138, 127, 153, /* 248 - 255 */
1444 : };
1445 :
1446 :
1447 : /* Return the number of multiplications required to calculate
1448 : powi(x,n) where n is less than POWI_TABLE_SIZE. This is a
1449 : subroutine of powi_cost. CACHE is an array indicating
1450 : which exponents have already been calculated. */
1451 :
1452 : static int
1453 1120 : powi_lookup_cost (unsigned HOST_WIDE_INT n, bool *cache)
1454 : {
1455 : /* If we've already calculated this exponent, then this evaluation
1456 : doesn't require any additional multiplications. */
1457 1861 : if (cache[n])
1458 1120 : return 0;
1459 :
1460 741 : cache[n] = true;
1461 741 : return powi_lookup_cost (n - powi_table[n], cache)
1462 741 : + powi_lookup_cost (powi_table[n], cache) + 1;
1463 : }
1464 :
1465 : /* Return the number of multiplications required to calculate
1466 : powi(x,n) for an arbitrary x, given the exponent N. This
1467 : function needs to be kept in sync with powi_as_mults below. */
1468 :
1469 : static int
1470 384 : powi_cost (HOST_WIDE_INT n)
1471 : {
1472 384 : bool cache[POWI_TABLE_SIZE];
1473 384 : unsigned HOST_WIDE_INT digit;
1474 384 : unsigned HOST_WIDE_INT val;
1475 384 : int result;
1476 :
1477 384 : if (n == 0)
1478 : return 0;
1479 :
1480 : /* Ignore the reciprocal when calculating the cost. */
1481 379 : val = absu_hwi (n);
1482 :
1483 : /* Initialize the exponent cache. */
1484 379 : memset (cache, 0, POWI_TABLE_SIZE * sizeof (bool));
1485 379 : cache[1] = true;
1486 :
1487 379 : result = 0;
1488 :
1489 379 : while (val >= POWI_TABLE_SIZE)
1490 : {
1491 0 : if (val & 1)
1492 : {
1493 0 : digit = val & ((1 << POWI_WINDOW_SIZE) - 1);
1494 0 : result += powi_lookup_cost (digit, cache)
1495 0 : + POWI_WINDOW_SIZE + 1;
1496 0 : val >>= POWI_WINDOW_SIZE;
1497 : }
1498 : else
1499 : {
1500 0 : val >>= 1;
1501 0 : result++;
1502 : }
1503 : }
1504 :
1505 379 : return result + powi_lookup_cost (val, cache);
1506 : }
1507 :
1508 : /* Recursive subroutine of powi_as_mults. This function takes the
1509 : array, CACHE, of already calculated exponents and an exponent N and
1510 : returns a tree that corresponds to CACHE[1]**N, with type TYPE. */
1511 :
1512 : static tree
1513 6305 : powi_as_mults_1 (gimple_stmt_iterator *gsi, location_t loc, tree type,
1514 : unsigned HOST_WIDE_INT n, tree *cache)
1515 : {
1516 6305 : tree op0, op1, ssa_target;
1517 6305 : unsigned HOST_WIDE_INT digit;
1518 6305 : gassign *mult_stmt;
1519 :
1520 6305 : if (n < POWI_TABLE_SIZE && cache[n])
1521 : return cache[n];
1522 :
1523 2230 : ssa_target = make_temp_ssa_name (type, NULL, "powmult");
1524 :
1525 2230 : if (n < POWI_TABLE_SIZE)
1526 : {
1527 2227 : cache[n] = ssa_target;
1528 2227 : op0 = powi_as_mults_1 (gsi, loc, type, n - powi_table[n], cache);
1529 2227 : op1 = powi_as_mults_1 (gsi, loc, type, powi_table[n], cache);
1530 : }
1531 3 : else if (n & 1)
1532 : {
1533 1 : digit = n & ((1 << POWI_WINDOW_SIZE) - 1);
1534 1 : op0 = powi_as_mults_1 (gsi, loc, type, n - digit, cache);
1535 1 : op1 = powi_as_mults_1 (gsi, loc, type, digit, cache);
1536 : }
1537 : else
1538 : {
1539 2 : op0 = powi_as_mults_1 (gsi, loc, type, n >> 1, cache);
1540 2 : op1 = op0;
1541 : }
1542 :
1543 2230 : mult_stmt = gimple_build_assign (ssa_target, MULT_EXPR, op0, op1);
1544 2230 : gimple_set_location (mult_stmt, loc);
1545 2230 : gsi_insert_before (gsi, mult_stmt, GSI_SAME_STMT);
1546 :
1547 2230 : return ssa_target;
1548 : }
1549 :
1550 : /* Convert ARG0**N to a tree of multiplications of ARG0 with itself.
1551 : This function needs to be kept in sync with powi_cost above. */
1552 :
1553 : tree
1554 1847 : powi_as_mults (gimple_stmt_iterator *gsi, location_t loc,
1555 : tree arg0, HOST_WIDE_INT n)
1556 : {
1557 1847 : tree cache[POWI_TABLE_SIZE], result, type = TREE_TYPE (arg0);
1558 1847 : gassign *div_stmt;
1559 1847 : tree target;
1560 :
1561 1847 : if (n == 0)
1562 0 : return build_one_cst (type);
1563 :
1564 1847 : memset (cache, 0, sizeof (cache));
1565 1847 : cache[1] = arg0;
1566 :
1567 1847 : result = powi_as_mults_1 (gsi, loc, type, absu_hwi (n), cache);
1568 1847 : if (n >= 0)
1569 : return result;
1570 :
1571 : /* If the original exponent was negative, reciprocate the result. */
1572 8 : target = make_temp_ssa_name (type, NULL, "powmult");
1573 8 : div_stmt = gimple_build_assign (target, RDIV_EXPR,
1574 : build_real (type, dconst1), result);
1575 8 : gimple_set_location (div_stmt, loc);
1576 8 : gsi_insert_before (gsi, div_stmt, GSI_SAME_STMT);
1577 :
1578 8 : return target;
1579 : }
1580 :
1581 : /* ARG0 and N are the two arguments to a powi builtin in GSI with
1582 : location info LOC. If the arguments are appropriate, create an
1583 : equivalent sequence of statements prior to GSI using an optimal
1584 : number of multiplications, and return an expression holding the
1585 : result. */
1586 :
1587 : static tree
1588 633 : gimple_expand_builtin_powi (gimple_stmt_iterator *gsi, location_t loc,
1589 : tree arg0, HOST_WIDE_INT n)
1590 : {
1591 633 : if ((n >= -1 && n <= 2)
1592 633 : || (optimize_function_for_speed_p (cfun)
1593 351 : && powi_cost (n) <= POWI_MAX_MULTS))
1594 625 : return powi_as_mults (gsi, loc, arg0, n);
1595 :
1596 : return NULL_TREE;
1597 : }
1598 :
1599 : /* Build a gimple call statement that calls FN with argument ARG.
1600 : Set the lhs of the call statement to a fresh SSA name. Insert the
1601 : statement prior to GSI's current position, and return the fresh
1602 : SSA name. */
1603 :
1604 : static tree
1605 44 : build_and_insert_call (gimple_stmt_iterator *gsi, location_t loc,
1606 : tree fn, tree arg)
1607 : {
1608 44 : gcall *call_stmt;
1609 44 : tree ssa_target;
1610 :
1611 44 : call_stmt = gimple_build_call (fn, 1, arg);
1612 44 : ssa_target = make_temp_ssa_name (TREE_TYPE (arg), NULL, "powroot");
1613 44 : gimple_set_lhs (call_stmt, ssa_target);
1614 44 : gimple_set_location (call_stmt, loc);
1615 44 : gsi_insert_before (gsi, call_stmt, GSI_SAME_STMT);
1616 :
1617 44 : return ssa_target;
1618 : }
1619 :
1620 : /* Build a gimple binary operation with the given CODE and arguments
1621 : ARG0, ARG1, assigning the result to a new SSA name for variable
1622 : TARGET. Insert the statement prior to GSI's current position, and
1623 : return the fresh SSA name.*/
1624 :
1625 : static tree
1626 2644 : build_and_insert_binop (gimple_stmt_iterator *gsi, location_t loc,
1627 : const char *name, enum tree_code code,
1628 : tree arg0, tree arg1)
1629 : {
1630 2644 : tree result = make_temp_ssa_name (TREE_TYPE (arg0), NULL, name);
1631 2644 : gassign *stmt = gimple_build_assign (result, code, arg0, arg1);
1632 2644 : gimple_set_location (stmt, loc);
1633 2644 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1634 2644 : return result;
1635 : }
1636 :
1637 : /* Build a gimple assignment to cast VAL to TYPE. Insert the statement
1638 : prior to GSI's current position, and return the fresh SSA name. */
1639 :
1640 : static tree
1641 14643 : build_and_insert_cast (gimple_stmt_iterator *gsi, location_t loc,
1642 : tree type, tree val)
1643 : {
1644 0 : return gimple_convert (gsi, true, GSI_SAME_STMT, loc, type, val);
1645 : }
1646 :
1647 : struct pow_synth_sqrt_info
1648 : {
1649 : bool *factors;
1650 : unsigned int deepest;
1651 : unsigned int num_mults;
1652 : };
1653 :
1654 : /* Return true iff the real value C can be represented as a
1655 : sum of powers of 0.5 up to N. That is:
1656 : C == SUM<i from 1..N> (a[i]*(0.5**i)) where a[i] is either 0 or 1.
1657 : Record in INFO the various parameters of the synthesis algorithm such
1658 : as the factors a[i], the maximum 0.5 power and the number of
1659 : multiplications that will be required. */
1660 :
1661 : bool
1662 33 : representable_as_half_series_p (REAL_VALUE_TYPE c, unsigned n,
1663 : struct pow_synth_sqrt_info *info)
1664 : {
1665 33 : REAL_VALUE_TYPE factor = dconsthalf;
1666 33 : REAL_VALUE_TYPE remainder = c;
1667 :
1668 33 : info->deepest = 0;
1669 33 : info->num_mults = 0;
1670 33 : memset (info->factors, 0, n * sizeof (bool));
1671 :
1672 97 : for (unsigned i = 0; i < n; i++)
1673 : {
1674 90 : REAL_VALUE_TYPE res;
1675 :
1676 : /* If something inexact happened bail out now. */
1677 90 : if (real_arithmetic (&res, MINUS_EXPR, &remainder, &factor))
1678 26 : return false;
1679 :
1680 : /* We have hit zero. The number is representable as a sum
1681 : of powers of 0.5. */
1682 90 : if (real_equal (&res, &dconst0))
1683 : {
1684 26 : info->factors[i] = true;
1685 26 : info->deepest = i + 1;
1686 26 : return true;
1687 : }
1688 64 : else if (!REAL_VALUE_NEGATIVE (res))
1689 : {
1690 29 : remainder = res;
1691 29 : info->factors[i] = true;
1692 29 : info->num_mults++;
1693 : }
1694 : else
1695 35 : info->factors[i] = false;
1696 :
1697 64 : real_arithmetic (&factor, MULT_EXPR, &factor, &dconsthalf);
1698 : }
1699 : return false;
1700 : }
1701 :
1702 : /* Return the tree corresponding to FN being applied
1703 : to ARG N times at GSI and LOC.
1704 : Look up previous results from CACHE if need be.
1705 : cache[0] should contain just plain ARG i.e. FN applied to ARG 0 times. */
1706 :
1707 : static tree
1708 63 : get_fn_chain (tree arg, unsigned int n, gimple_stmt_iterator *gsi,
1709 : tree fn, location_t loc, tree *cache)
1710 : {
1711 63 : tree res = cache[n];
1712 63 : if (!res)
1713 : {
1714 40 : tree prev = get_fn_chain (arg, n - 1, gsi, fn, loc, cache);
1715 40 : res = build_and_insert_call (gsi, loc, fn, prev);
1716 40 : cache[n] = res;
1717 : }
1718 :
1719 63 : return res;
1720 : }
1721 :
1722 : /* Print to STREAM the repeated application of function FNAME to ARG
1723 : N times. So, for FNAME = "foo", ARG = "x", N = 2 it would print:
1724 : "foo (foo (x))". */
1725 :
1726 : static void
1727 36 : print_nested_fn (FILE* stream, const char *fname, const char* arg,
1728 : unsigned int n)
1729 : {
1730 36 : if (n == 0)
1731 10 : fprintf (stream, "%s", arg);
1732 : else
1733 : {
1734 26 : fprintf (stream, "%s (", fname);
1735 26 : print_nested_fn (stream, fname, arg, n - 1);
1736 26 : fprintf (stream, ")");
1737 : }
1738 36 : }
1739 :
1740 : /* Print to STREAM the fractional sequence of sqrt chains
1741 : applied to ARG, described by INFO. Used for the dump file. */
1742 :
1743 : static void
1744 7 : dump_fractional_sqrt_sequence (FILE *stream, const char *arg,
1745 : struct pow_synth_sqrt_info *info)
1746 : {
1747 29 : for (unsigned int i = 0; i < info->deepest; i++)
1748 : {
1749 22 : bool is_set = info->factors[i];
1750 22 : if (is_set)
1751 : {
1752 10 : print_nested_fn (stream, "sqrt", arg, i + 1);
1753 10 : if (i != info->deepest - 1)
1754 3 : fprintf (stream, " * ");
1755 : }
1756 : }
1757 7 : }
1758 :
1759 : /* Print to STREAM a representation of raising ARG to an integer
1760 : power N. Used for the dump file. */
1761 :
1762 : static void
1763 7 : dump_integer_part (FILE *stream, const char* arg, HOST_WIDE_INT n)
1764 : {
1765 7 : if (n > 1)
1766 3 : fprintf (stream, "powi (%s, " HOST_WIDE_INT_PRINT_DEC ")", arg, n);
1767 4 : else if (n == 1)
1768 3 : fprintf (stream, "%s", arg);
1769 7 : }
1770 :
1771 : /* Attempt to synthesize a POW[F] (ARG0, ARG1) call using chains of
1772 : square roots. Place at GSI and LOC. Limit the maximum depth
1773 : of the sqrt chains to MAX_DEPTH. Return the tree holding the
1774 : result of the expanded sequence or NULL_TREE if the expansion failed.
1775 :
1776 : This routine assumes that ARG1 is a real number with a fractional part
1777 : (the integer exponent case will have been handled earlier in
1778 : gimple_expand_builtin_pow).
1779 :
1780 : For ARG1 > 0.0:
1781 : * For ARG1 composed of a whole part WHOLE_PART and a fractional part
1782 : FRAC_PART i.e. WHOLE_PART == floor (ARG1) and
1783 : FRAC_PART == ARG1 - WHOLE_PART:
1784 : Produce POWI (ARG0, WHOLE_PART) * POW (ARG0, FRAC_PART) where
1785 : POW (ARG0, FRAC_PART) is expanded as a product of square root chains
1786 : if it can be expressed as such, that is if FRAC_PART satisfies:
1787 : FRAC_PART == <SUM from i = 1 until MAX_DEPTH> (a[i] * (0.5**i))
1788 : where integer a[i] is either 0 or 1.
1789 :
1790 : Example:
1791 : POW (x, 3.625) == POWI (x, 3) * POW (x, 0.625)
1792 : --> POWI (x, 3) * SQRT (x) * SQRT (SQRT (SQRT (x)))
1793 :
1794 : For ARG1 < 0.0 there are two approaches:
1795 : * (A) Expand to 1.0 / POW (ARG0, -ARG1) where POW (ARG0, -ARG1)
1796 : is calculated as above.
1797 :
1798 : Example:
1799 : POW (x, -5.625) == 1.0 / POW (x, 5.625)
1800 : --> 1.0 / (POWI (x, 5) * SQRT (x) * SQRT (SQRT (SQRT (x))))
1801 :
1802 : * (B) : WHOLE_PART := - ceil (abs (ARG1))
1803 : FRAC_PART := ARG1 - WHOLE_PART
1804 : and expand to POW (x, FRAC_PART) / POWI (x, WHOLE_PART).
1805 : Example:
1806 : POW (x, -5.875) == POW (x, 0.125) / POWI (X, 6)
1807 : --> SQRT (SQRT (SQRT (x))) / (POWI (x, 6))
1808 :
1809 : For ARG1 < 0.0 we choose between (A) and (B) depending on
1810 : how many multiplications we'd have to do.
1811 : So, for the example in (B): POW (x, -5.875), if we were to
1812 : follow algorithm (A) we would produce:
1813 : 1.0 / POWI (X, 5) * SQRT (X) * SQRT (SQRT (X)) * SQRT (SQRT (SQRT (X)))
1814 : which contains more multiplications than approach (B).
1815 :
1816 : Hopefully, this approach will eliminate potentially expensive POW library
1817 : calls when unsafe floating point math is enabled and allow the compiler to
1818 : further optimise the multiplies, square roots and divides produced by this
1819 : function. */
1820 :
1821 : static tree
1822 25 : expand_pow_as_sqrts (gimple_stmt_iterator *gsi, location_t loc,
1823 : tree arg0, tree arg1, HOST_WIDE_INT max_depth)
1824 : {
1825 25 : tree type = TREE_TYPE (arg0);
1826 25 : machine_mode mode = TYPE_MODE (type);
1827 25 : tree sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
1828 25 : bool one_over = true;
1829 :
1830 25 : if (!sqrtfn)
1831 : return NULL_TREE;
1832 :
1833 25 : if (TREE_CODE (arg1) != REAL_CST)
1834 : return NULL_TREE;
1835 :
1836 25 : REAL_VALUE_TYPE exp_init = TREE_REAL_CST (arg1);
1837 :
1838 25 : gcc_assert (max_depth > 0);
1839 25 : tree *cache = XALLOCAVEC (tree, max_depth + 1);
1840 :
1841 25 : struct pow_synth_sqrt_info synth_info;
1842 25 : synth_info.factors = XALLOCAVEC (bool, max_depth + 1);
1843 25 : synth_info.deepest = 0;
1844 25 : synth_info.num_mults = 0;
1845 :
1846 25 : bool neg_exp = REAL_VALUE_NEGATIVE (exp_init);
1847 25 : REAL_VALUE_TYPE exp = real_value_abs (&exp_init);
1848 :
1849 : /* The whole and fractional parts of exp. */
1850 25 : REAL_VALUE_TYPE whole_part;
1851 25 : REAL_VALUE_TYPE frac_part;
1852 :
1853 25 : real_floor (&whole_part, mode, &exp);
1854 25 : real_arithmetic (&frac_part, MINUS_EXPR, &exp, &whole_part);
1855 :
1856 :
1857 25 : REAL_VALUE_TYPE ceil_whole = dconst0;
1858 25 : REAL_VALUE_TYPE ceil_fract = dconst0;
1859 :
1860 25 : if (neg_exp)
1861 : {
1862 10 : real_ceil (&ceil_whole, mode, &exp);
1863 10 : real_arithmetic (&ceil_fract, MINUS_EXPR, &ceil_whole, &exp);
1864 : }
1865 :
1866 25 : if (!representable_as_half_series_p (frac_part, max_depth, &synth_info))
1867 : return NULL_TREE;
1868 :
1869 : /* Check whether it's more profitable to not use 1.0 / ... */
1870 18 : if (neg_exp)
1871 : {
1872 8 : struct pow_synth_sqrt_info alt_synth_info;
1873 8 : alt_synth_info.factors = XALLOCAVEC (bool, max_depth + 1);
1874 8 : alt_synth_info.deepest = 0;
1875 8 : alt_synth_info.num_mults = 0;
1876 :
1877 8 : if (representable_as_half_series_p (ceil_fract, max_depth,
1878 : &alt_synth_info)
1879 8 : && alt_synth_info.deepest <= synth_info.deepest
1880 16 : && alt_synth_info.num_mults < synth_info.num_mults)
1881 : {
1882 2 : whole_part = ceil_whole;
1883 2 : frac_part = ceil_fract;
1884 2 : synth_info.deepest = alt_synth_info.deepest;
1885 2 : synth_info.num_mults = alt_synth_info.num_mults;
1886 2 : memcpy (synth_info.factors, alt_synth_info.factors,
1887 : (max_depth + 1) * sizeof (bool));
1888 2 : one_over = false;
1889 : }
1890 : }
1891 :
1892 18 : HOST_WIDE_INT n = real_to_integer (&whole_part);
1893 18 : REAL_VALUE_TYPE cint;
1894 18 : real_from_integer (&cint, VOIDmode, n, SIGNED);
1895 :
1896 18 : if (!real_identical (&whole_part, &cint))
1897 : return NULL_TREE;
1898 :
1899 18 : if (powi_cost (n) + synth_info.num_mults > POWI_MAX_MULTS)
1900 : return NULL_TREE;
1901 :
1902 18 : memset (cache, 0, (max_depth + 1) * sizeof (tree));
1903 :
1904 18 : tree integer_res = n == 0 ? build_real (type, dconst1) : arg0;
1905 :
1906 : /* Calculate the integer part of the exponent. */
1907 18 : if (n > 1)
1908 : {
1909 6 : integer_res = gimple_expand_builtin_powi (gsi, loc, arg0, n);
1910 6 : if (!integer_res)
1911 : return NULL_TREE;
1912 : }
1913 :
1914 18 : if (dump_file)
1915 : {
1916 7 : char string[64];
1917 :
1918 7 : real_to_decimal (string, &exp_init, sizeof (string), 0, 1);
1919 7 : fprintf (dump_file, "synthesizing pow (x, %s) as:\n", string);
1920 :
1921 7 : if (neg_exp)
1922 : {
1923 2 : if (one_over)
1924 : {
1925 1 : fprintf (dump_file, "1.0 / (");
1926 1 : dump_integer_part (dump_file, "x", n);
1927 1 : if (n > 0)
1928 1 : fprintf (dump_file, " * ");
1929 1 : dump_fractional_sqrt_sequence (dump_file, "x", &synth_info);
1930 1 : fprintf (dump_file, ")");
1931 : }
1932 : else
1933 : {
1934 1 : dump_fractional_sqrt_sequence (dump_file, "x", &synth_info);
1935 1 : fprintf (dump_file, " / (");
1936 1 : dump_integer_part (dump_file, "x", n);
1937 1 : fprintf (dump_file, ")");
1938 : }
1939 : }
1940 : else
1941 : {
1942 5 : dump_fractional_sqrt_sequence (dump_file, "x", &synth_info);
1943 5 : if (n > 0)
1944 4 : fprintf (dump_file, " * ");
1945 5 : dump_integer_part (dump_file, "x", n);
1946 : }
1947 :
1948 7 : fprintf (dump_file, "\ndeepest sqrt chain: %d\n", synth_info.deepest);
1949 : }
1950 :
1951 :
1952 18 : tree fract_res = NULL_TREE;
1953 18 : cache[0] = arg0;
1954 :
1955 : /* Calculate the fractional part of the exponent. */
1956 58 : for (unsigned i = 0; i < synth_info.deepest; i++)
1957 : {
1958 40 : if (synth_info.factors[i])
1959 : {
1960 23 : tree sqrt_chain = get_fn_chain (arg0, i + 1, gsi, sqrtfn, loc, cache);
1961 :
1962 23 : if (!fract_res)
1963 : fract_res = sqrt_chain;
1964 :
1965 : else
1966 5 : fract_res = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
1967 : fract_res, sqrt_chain);
1968 : }
1969 : }
1970 :
1971 18 : tree res = NULL_TREE;
1972 :
1973 18 : if (neg_exp)
1974 : {
1975 8 : if (one_over)
1976 : {
1977 6 : if (n > 0)
1978 4 : res = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
1979 : fract_res, integer_res);
1980 : else
1981 : res = fract_res;
1982 :
1983 6 : res = build_and_insert_binop (gsi, loc, "powrootrecip", RDIV_EXPR,
1984 : build_real (type, dconst1), res);
1985 : }
1986 : else
1987 : {
1988 2 : res = build_and_insert_binop (gsi, loc, "powroot", RDIV_EXPR,
1989 : fract_res, integer_res);
1990 : }
1991 : }
1992 : else
1993 10 : res = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
1994 : fract_res, integer_res);
1995 : return res;
1996 : }
1997 :
1998 : /* ARG0 and ARG1 are the two arguments to a pow builtin call in GSI
1999 : with location info LOC. If possible, create an equivalent and
2000 : less expensive sequence of statements prior to GSI, and return an
2001 : expression holding the result. */
2002 :
2003 : static tree
2004 733 : gimple_expand_builtin_pow (gimple_stmt_iterator *gsi, location_t loc,
2005 : tree arg0, tree arg1)
2006 : {
2007 733 : REAL_VALUE_TYPE c, cint, dconst1_3, dconst1_4, dconst1_6;
2008 733 : REAL_VALUE_TYPE c2, dconst3;
2009 733 : HOST_WIDE_INT n;
2010 733 : tree type, sqrtfn, cbrtfn, sqrt_arg0, result, cbrt_x, powi_cbrt_x;
2011 733 : machine_mode mode;
2012 733 : bool speed_p = optimize_bb_for_speed_p (gsi_bb (*gsi));
2013 733 : bool hw_sqrt_exists, c_is_int, c2_is_int;
2014 :
2015 733 : dconst1_4 = dconst1;
2016 733 : SET_REAL_EXP (&dconst1_4, REAL_EXP (&dconst1_4) - 2);
2017 :
2018 : /* If the exponent isn't a constant, there's nothing of interest
2019 : to be done. */
2020 733 : if (TREE_CODE (arg1) != REAL_CST)
2021 : return NULL_TREE;
2022 :
2023 365 : if (flag_errno_math)
2024 : return NULL_TREE;
2025 :
2026 : /* Don't perform the operation if flag_signaling_nans is on
2027 : and either operand is a signaling NaN. */
2028 77 : if (tree_expr_maybe_signaling_nan_p (arg0)
2029 77 : || tree_expr_maybe_signaling_nan_p (arg1))
2030 : return NULL_TREE;
2031 :
2032 : /* If the exponent is equivalent to an integer, expand to an optimal
2033 : multiplication sequence when profitable. */
2034 77 : c = TREE_REAL_CST (arg1);
2035 77 : n = real_to_integer (&c);
2036 77 : real_from_integer (&cint, VOIDmode, n, SIGNED);
2037 77 : c_is_int = real_identical (&c, &cint);
2038 :
2039 77 : if (c_is_int
2040 77 : && ((n >= -1 && n <= 2)
2041 21 : || (flag_unsafe_math_optimizations
2042 11 : && speed_p
2043 11 : && powi_cost (n) <= POWI_MAX_MULTS)))
2044 30 : return gimple_expand_builtin_powi (gsi, loc, arg0, n);
2045 :
2046 : /* Attempt various optimizations using sqrt and cbrt. */
2047 47 : type = TREE_TYPE (arg0);
2048 47 : mode = TYPE_MODE (type);
2049 47 : sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
2050 :
2051 : /* Optimize pow(x,0.5) = sqrt(x). This replacement is safe
2052 : except for signed zeros and negative infinity.
2053 : pow(-0,0.5) = +0, while sqrt(-0) = -0.
2054 : pow(-Inf,0.5) = +Inf, while sqrt(-Inf) is NaN. */
2055 47 : if (sqrtfn
2056 47 : && real_equal (&c, &dconsthalf)
2057 9 : && !tree_expr_maybe_real_minus_zero_p (arg0)
2058 49 : && (!tree_expr_maybe_infinite_p (arg0)
2059 2 : || tree_expr_nonnegative_p (arg0)))
2060 0 : return build_and_insert_call (gsi, loc, sqrtfn, arg0);
2061 :
2062 47 : hw_sqrt_exists = optab_handler (sqrt_optab, mode) != CODE_FOR_nothing;
2063 :
2064 : /* Optimize pow(x,1./3.) = cbrt(x). This requires unsafe math
2065 : optimizations since 1./3. is not exactly representable. If x
2066 : is negative and finite, the correct value of pow(x,1./3.) is
2067 : a NaN with the "invalid" exception raised, because the value
2068 : of 1./3. actually has an even denominator. The correct value
2069 : of cbrt(x) is a negative real value. */
2070 47 : cbrtfn = mathfn_built_in (type, BUILT_IN_CBRT);
2071 47 : dconst1_3 = real_value_truncate (mode, dconst_third ());
2072 :
2073 47 : if (flag_unsafe_math_optimizations
2074 25 : && cbrtfn
2075 25 : && (!HONOR_NANS (mode) || tree_expr_nonnegative_p (arg0))
2076 72 : && real_equal (&c, &dconst1_3))
2077 0 : return build_and_insert_call (gsi, loc, cbrtfn, arg0);
2078 :
2079 : /* Optimize pow(x,1./6.) = cbrt(sqrt(x)). Don't do this optimization
2080 : if we don't have a hardware sqrt insn. */
2081 47 : dconst1_6 = dconst1_3;
2082 47 : SET_REAL_EXP (&dconst1_6, REAL_EXP (&dconst1_6) - 1);
2083 :
2084 47 : if (flag_unsafe_math_optimizations
2085 25 : && sqrtfn
2086 25 : && cbrtfn
2087 25 : && (!HONOR_NANS (mode) || tree_expr_nonnegative_p (arg0))
2088 : && speed_p
2089 25 : && hw_sqrt_exists
2090 72 : && real_equal (&c, &dconst1_6))
2091 : {
2092 : /* sqrt(x) */
2093 0 : sqrt_arg0 = build_and_insert_call (gsi, loc, sqrtfn, arg0);
2094 :
2095 : /* cbrt(sqrt(x)) */
2096 0 : return build_and_insert_call (gsi, loc, cbrtfn, sqrt_arg0);
2097 : }
2098 :
2099 :
2100 : /* Attempt to expand the POW as a product of square root chains.
2101 : Expand the 0.25 case even when optimising for size. */
2102 47 : if (flag_unsafe_math_optimizations
2103 25 : && sqrtfn
2104 25 : && hw_sqrt_exists
2105 25 : && (speed_p || real_equal (&c, &dconst1_4))
2106 72 : && !HONOR_SIGNED_ZEROS (mode))
2107 : {
2108 75 : unsigned int max_depth = speed_p
2109 25 : ? param_max_pow_sqrt_depth
2110 : : 2;
2111 :
2112 25 : tree expand_with_sqrts
2113 25 : = expand_pow_as_sqrts (gsi, loc, arg0, arg1, max_depth);
2114 :
2115 25 : if (expand_with_sqrts)
2116 : return expand_with_sqrts;
2117 : }
2118 :
2119 29 : real_arithmetic (&c2, MULT_EXPR, &c, &dconst2);
2120 29 : n = real_to_integer (&c2);
2121 29 : real_from_integer (&cint, VOIDmode, n, SIGNED);
2122 29 : c2_is_int = real_identical (&c2, &cint);
2123 :
2124 : /* Optimize pow(x,c), where 3c = n for some nonzero integer n, into
2125 :
2126 : powi(x, n/3) * powi(cbrt(x), n%3), n > 0;
2127 : 1.0 / (powi(x, abs(n)/3) * powi(cbrt(x), abs(n)%3)), n < 0.
2128 :
2129 : Do not calculate the first factor when n/3 = 0. As cbrt(x) is
2130 : different from pow(x, 1./3.) due to rounding and behavior with
2131 : negative x, we need to constrain this transformation to unsafe
2132 : math and positive x or finite math. */
2133 29 : real_from_integer (&dconst3, VOIDmode, 3, SIGNED);
2134 29 : real_arithmetic (&c2, MULT_EXPR, &c, &dconst3);
2135 29 : real_round (&c2, mode, &c2);
2136 29 : n = real_to_integer (&c2);
2137 29 : real_from_integer (&cint, VOIDmode, n, SIGNED);
2138 29 : real_arithmetic (&c2, RDIV_EXPR, &cint, &dconst3);
2139 29 : real_convert (&c2, mode, &c2);
2140 :
2141 29 : if (flag_unsafe_math_optimizations
2142 7 : && cbrtfn
2143 7 : && (!HONOR_NANS (mode) || tree_expr_nonnegative_p (arg0))
2144 7 : && real_identical (&c2, &c)
2145 4 : && !c2_is_int
2146 4 : && optimize_function_for_speed_p (cfun)
2147 33 : && powi_cost (n / 3) <= POWI_MAX_MULTS)
2148 : {
2149 4 : tree powi_x_ndiv3 = NULL_TREE;
2150 :
2151 : /* Attempt to fold powi(arg0, abs(n/3)) into multiplies. If not
2152 : possible or profitable, give up. Skip the degenerate case when
2153 : abs(n) < 3, where the result is always 1. */
2154 4 : if (absu_hwi (n) >= 3)
2155 : {
2156 4 : powi_x_ndiv3 = gimple_expand_builtin_powi (gsi, loc, arg0,
2157 : abs_hwi (n / 3));
2158 4 : if (!powi_x_ndiv3)
2159 : return NULL_TREE;
2160 : }
2161 :
2162 : /* Calculate powi(cbrt(x), n%3). Don't use gimple_expand_builtin_powi
2163 : as that creates an unnecessary variable. Instead, just produce
2164 : either cbrt(x) or cbrt(x) * cbrt(x). */
2165 4 : cbrt_x = build_and_insert_call (gsi, loc, cbrtfn, arg0);
2166 :
2167 4 : if (absu_hwi (n) % 3 == 1)
2168 : powi_cbrt_x = cbrt_x;
2169 : else
2170 2 : powi_cbrt_x = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
2171 : cbrt_x, cbrt_x);
2172 :
2173 : /* Multiply the two subexpressions, unless powi(x,abs(n)/3) = 1. */
2174 4 : if (absu_hwi (n) < 3)
2175 : result = powi_cbrt_x;
2176 : else
2177 4 : result = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
2178 : powi_x_ndiv3, powi_cbrt_x);
2179 :
2180 : /* If n is negative, reciprocate the result. */
2181 4 : if (n < 0)
2182 1 : result = build_and_insert_binop (gsi, loc, "powroot", RDIV_EXPR,
2183 : build_real (type, dconst1), result);
2184 :
2185 : return result;
2186 : }
2187 :
2188 : /* No optimizations succeeded. */
2189 : return NULL_TREE;
2190 : }
2191 :
2192 : /* Go through all calls to sin, cos and cexpi and call execute_cse_sincos_1
2193 : on the SSA_NAME argument of each of them. */
2194 :
2195 : namespace {
2196 :
2197 : const pass_data pass_data_cse_sincos =
2198 : {
2199 : GIMPLE_PASS, /* type */
2200 : "sincos", /* name */
2201 : OPTGROUP_NONE, /* optinfo_flags */
2202 : TV_TREE_SINCOS, /* tv_id */
2203 : PROP_ssa, /* properties_required */
2204 : 0, /* properties_provided */
2205 : 0, /* properties_destroyed */
2206 : 0, /* todo_flags_start */
2207 : TODO_update_ssa, /* todo_flags_finish */
2208 : };
2209 :
2210 : class pass_cse_sincos : public gimple_opt_pass
2211 : {
2212 : public:
2213 294536 : pass_cse_sincos (gcc::context *ctxt)
2214 589072 : : gimple_opt_pass (pass_data_cse_sincos, ctxt)
2215 : {}
2216 :
2217 : /* opt_pass methods: */
2218 1068219 : bool gate (function *) final override
2219 : {
2220 1068219 : return optimize;
2221 : }
2222 :
2223 : unsigned int execute (function *) final override;
2224 :
2225 : }; // class pass_cse_sincos
2226 :
2227 : unsigned int
2228 1068181 : pass_cse_sincos::execute (function *fun)
2229 : {
2230 1068181 : basic_block bb;
2231 1068181 : bool cfg_changed = false;
2232 :
2233 1068181 : calculate_dominance_info (CDI_DOMINATORS);
2234 1068181 : memset (&sincos_stats, 0, sizeof (sincos_stats));
2235 :
2236 11429256 : FOR_EACH_BB_FN (bb, fun)
2237 : {
2238 10361075 : gimple_stmt_iterator gsi;
2239 :
2240 102165693 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2241 : {
2242 91804618 : gimple *stmt = gsi_stmt (gsi);
2243 :
2244 91804618 : if (is_gimple_call (stmt)
2245 91804618 : && gimple_call_lhs (stmt))
2246 : {
2247 2084909 : tree arg;
2248 2084909 : switch (gimple_call_combined_fn (stmt))
2249 : {
2250 1069 : CASE_CFN_COS:
2251 1069 : CASE_CFN_SIN:
2252 1069 : CASE_CFN_CEXPI:
2253 1069 : arg = gimple_call_arg (stmt, 0);
2254 : /* Make sure we have either sincos or cexp. */
2255 1069 : if (!targetm.libc_has_function (function_c99_math_complex,
2256 1069 : TREE_TYPE (arg))
2257 1069 : && !targetm.libc_has_function (function_sincos,
2258 0 : TREE_TYPE (arg)))
2259 : break;
2260 :
2261 1069 : if (TREE_CODE (arg) == SSA_NAME)
2262 1069 : cfg_changed |= execute_cse_sincos_1 (arg);
2263 : break;
2264 : default:
2265 : break;
2266 : }
2267 : }
2268 : }
2269 : }
2270 :
2271 1068181 : statistics_counter_event (fun, "sincos statements inserted",
2272 : sincos_stats.inserted);
2273 1068181 : statistics_counter_event (fun, "conv statements removed",
2274 : sincos_stats.conv_removed);
2275 :
2276 1068181 : return cfg_changed ? TODO_cleanup_cfg : 0;
2277 : }
2278 :
2279 : } // anon namespace
2280 :
2281 : gimple_opt_pass *
2282 294536 : make_pass_cse_sincos (gcc::context *ctxt)
2283 : {
2284 294536 : return new pass_cse_sincos (ctxt);
2285 : }
2286 :
2287 : /* Expand powi(x,n) into an optimal number of multiplies, when n is a
2288 : constant. */
2289 : namespace {
2290 :
2291 : const pass_data pass_data_expand_pow =
2292 : {
2293 : GIMPLE_PASS, /* type */
2294 : "pow", /* name */
2295 : OPTGROUP_NONE, /* optinfo_flags */
2296 : TV_TREE_POW, /* tv_id */
2297 : PROP_ssa, /* properties_required */
2298 : PROP_gimple_opt_math, /* properties_provided */
2299 : 0, /* properties_destroyed */
2300 : 0, /* todo_flags_start */
2301 : TODO_update_ssa, /* todo_flags_finish */
2302 : };
2303 :
2304 : class pass_expand_pow : public gimple_opt_pass
2305 : {
2306 : public:
2307 294536 : pass_expand_pow (gcc::context *ctxt)
2308 589072 : : gimple_opt_pass (pass_data_expand_pow, ctxt)
2309 : {}
2310 :
2311 : /* opt_pass methods: */
2312 1068219 : bool gate (function *) final override
2313 : {
2314 1068219 : return optimize;
2315 : }
2316 :
2317 : unsigned int execute (function *) final override;
2318 :
2319 : }; // class pass_expand_pow
2320 :
2321 : unsigned int
2322 1068214 : pass_expand_pow::execute (function *fun)
2323 : {
2324 1068214 : basic_block bb;
2325 1068214 : bool cfg_changed = false;
2326 :
2327 1068214 : calculate_dominance_info (CDI_DOMINATORS);
2328 :
2329 11549051 : FOR_EACH_BB_FN (bb, fun)
2330 : {
2331 10480837 : gimple_stmt_iterator gsi;
2332 10480837 : bool cleanup_eh = false;
2333 :
2334 99943621 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2335 : {
2336 89462784 : gimple *stmt = gsi_stmt (gsi);
2337 :
2338 : /* Only the last stmt in a bb could throw, no need to call
2339 : gimple_purge_dead_eh_edges if we change something in the middle
2340 : of a basic block. */
2341 89462784 : cleanup_eh = false;
2342 :
2343 89462784 : if (is_gimple_call (stmt)
2344 89462784 : && gimple_call_lhs (stmt))
2345 : {
2346 2063489 : tree arg0, arg1, result;
2347 2063489 : HOST_WIDE_INT n;
2348 2063489 : location_t loc;
2349 :
2350 2063489 : switch (gimple_call_combined_fn (stmt))
2351 : {
2352 733 : CASE_CFN_POW:
2353 733 : arg0 = gimple_call_arg (stmt, 0);
2354 733 : arg1 = gimple_call_arg (stmt, 1);
2355 :
2356 733 : loc = gimple_location (stmt);
2357 733 : result = gimple_expand_builtin_pow (&gsi, loc, arg0, arg1);
2358 :
2359 733 : if (result)
2360 : {
2361 52 : tree lhs = gimple_get_lhs (stmt);
2362 52 : gassign *new_stmt = gimple_build_assign (lhs, result);
2363 52 : gimple_set_location (new_stmt, loc);
2364 52 : unlink_stmt_vdef (stmt);
2365 52 : gsi_replace (&gsi, new_stmt, true);
2366 52 : cleanup_eh = true;
2367 104 : if (gimple_vdef (stmt))
2368 0 : release_ssa_name (gimple_vdef (stmt));
2369 : }
2370 : break;
2371 :
2372 804 : CASE_CFN_POWI:
2373 804 : arg0 = gimple_call_arg (stmt, 0);
2374 804 : arg1 = gimple_call_arg (stmt, 1);
2375 804 : loc = gimple_location (stmt);
2376 :
2377 804 : if (real_minus_onep (arg0))
2378 : {
2379 0 : tree t0, t1, cond, one, minus_one;
2380 0 : gassign *stmt;
2381 :
2382 0 : t0 = TREE_TYPE (arg0);
2383 0 : t1 = TREE_TYPE (arg1);
2384 0 : one = build_real (t0, dconst1);
2385 0 : minus_one = build_real (t0, dconstm1);
2386 :
2387 0 : cond = make_temp_ssa_name (t1, NULL, "powi_cond");
2388 0 : stmt = gimple_build_assign (cond, BIT_AND_EXPR,
2389 : arg1, build_int_cst (t1, 1));
2390 0 : gimple_set_location (stmt, loc);
2391 0 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
2392 :
2393 0 : result = make_temp_ssa_name (t0, NULL, "powi");
2394 0 : stmt = gimple_build_assign (result, COND_EXPR, cond,
2395 : minus_one, one);
2396 0 : gimple_set_location (stmt, loc);
2397 0 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
2398 : }
2399 : else
2400 : {
2401 804 : if (!tree_fits_shwi_p (arg1))
2402 : break;
2403 :
2404 593 : n = tree_to_shwi (arg1);
2405 593 : result = gimple_expand_builtin_powi (&gsi, loc, arg0, n);
2406 : }
2407 :
2408 593 : if (result)
2409 : {
2410 585 : tree lhs = gimple_get_lhs (stmt);
2411 585 : gassign *new_stmt = gimple_build_assign (lhs, result);
2412 585 : gimple_set_location (new_stmt, loc);
2413 585 : unlink_stmt_vdef (stmt);
2414 585 : gsi_replace (&gsi, new_stmt, true);
2415 585 : cleanup_eh = true;
2416 89463369 : if (gimple_vdef (stmt))
2417 0 : release_ssa_name (gimple_vdef (stmt));
2418 : }
2419 : break;
2420 :
2421 211 : default:;
2422 : }
2423 : }
2424 : }
2425 10480837 : if (cleanup_eh)
2426 3 : cfg_changed |= gimple_purge_dead_eh_edges (bb);
2427 : }
2428 :
2429 1068214 : return cfg_changed ? TODO_cleanup_cfg : 0;
2430 : }
2431 :
2432 : } // anon namespace
2433 :
2434 : gimple_opt_pass *
2435 294536 : make_pass_expand_pow (gcc::context *ctxt)
2436 : {
2437 294536 : return new pass_expand_pow (ctxt);
2438 : }
2439 :
2440 : /* Return true if stmt is a type conversion operation that can be stripped
2441 : when used in a widening multiply operation. */
2442 : static bool
2443 516541 : widening_mult_conversion_strippable_p (tree result_type, gimple *stmt)
2444 : {
2445 516541 : enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
2446 :
2447 516541 : if (TREE_CODE (result_type) == INTEGER_TYPE)
2448 : {
2449 516541 : tree op_type;
2450 516541 : tree inner_op_type;
2451 :
2452 516541 : if (!CONVERT_EXPR_CODE_P (rhs_code))
2453 : return false;
2454 :
2455 200734 : op_type = TREE_TYPE (gimple_assign_lhs (stmt));
2456 :
2457 : /* If the type of OP has the same precision as the result, then
2458 : we can strip this conversion. The multiply operation will be
2459 : selected to create the correct extension as a by-product. */
2460 200734 : if (TYPE_PRECISION (result_type) == TYPE_PRECISION (op_type))
2461 : return true;
2462 :
2463 : /* We can also strip a conversion if it preserves the signed-ness of
2464 : the operation and doesn't narrow the range. */
2465 1167 : inner_op_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
2466 :
2467 : /* If the inner-most type is unsigned, then we can strip any
2468 : intermediate widening operation. If it's signed, then the
2469 : intermediate widening operation must also be signed. */
2470 1167 : if ((TYPE_UNSIGNED (inner_op_type)
2471 1163 : || TYPE_UNSIGNED (op_type) == TYPE_UNSIGNED (inner_op_type))
2472 2330 : && TYPE_PRECISION (op_type) > TYPE_PRECISION (inner_op_type))
2473 : return true;
2474 :
2475 1164 : return false;
2476 : }
2477 :
2478 0 : return rhs_code == FIXED_CONVERT_EXPR;
2479 : }
2480 :
2481 : /* Return true if RHS is a suitable operand for a widening multiplication,
2482 : assuming a target type of TYPE.
2483 : There are two cases:
2484 :
2485 : - RHS makes some value at least twice as wide. Store that value
2486 : in *NEW_RHS_OUT if so, and store its type in *TYPE_OUT.
2487 :
2488 : - RHS is an integer constant. Store that value in *NEW_RHS_OUT if so,
2489 : but leave *TYPE_OUT untouched. */
2490 :
2491 : static bool
2492 958250 : is_widening_mult_rhs_p (tree type, tree rhs, tree *type_out,
2493 : tree *new_rhs_out)
2494 : {
2495 958250 : gimple *stmt;
2496 958250 : tree type1, rhs1;
2497 :
2498 958250 : if (TREE_CODE (rhs) == SSA_NAME)
2499 : {
2500 : /* Use tree_non_zero_bits to see if this operand is zero_extended
2501 : for unsigned widening multiplications or non-negative for
2502 : signed widening multiplications. */
2503 796945 : if (TREE_CODE (type) == INTEGER_TYPE
2504 796945 : && (TYPE_PRECISION (type) & 1) == 0
2505 1593890 : && int_mode_for_size (TYPE_PRECISION (type) / 2, 1).exists ())
2506 : {
2507 790614 : unsigned int prec = TYPE_PRECISION (type);
2508 790614 : unsigned int hprec = prec / 2;
2509 790614 : wide_int bits = wide_int::from (tree_nonzero_bits (rhs), prec,
2510 1581228 : TYPE_SIGN (TREE_TYPE (rhs)));
2511 790614 : if (TYPE_UNSIGNED (type)
2512 1352985 : && wi::bit_and (bits, wi::mask (hprec, true, prec)) == 0)
2513 : {
2514 137517 : *type_out = build_nonstandard_integer_type (hprec, true);
2515 : /* X & MODE_MASK can be simplified to (T)X. */
2516 137517 : stmt = SSA_NAME_DEF_STMT (rhs);
2517 275034 : if (is_gimple_assign (stmt)
2518 122475 : && gimple_assign_rhs_code (stmt) == BIT_AND_EXPR
2519 12016 : && TREE_CODE (gimple_assign_rhs2 (stmt)) == INTEGER_CST
2520 160949 : && wide_int::from (wi::to_wide (gimple_assign_rhs2 (stmt)),
2521 11716 : prec, TYPE_SIGN (TREE_TYPE (rhs)))
2522 172665 : == wi::mask (hprec, false, prec))
2523 9847 : *new_rhs_out = gimple_assign_rhs1 (stmt);
2524 : else
2525 : *new_rhs_out = rhs;
2526 137517 : return true;
2527 : }
2528 653097 : else if (!TYPE_UNSIGNED (type)
2529 881340 : && wi::bit_and (bits, wi::mask (hprec - 1, true, prec)) == 0)
2530 : {
2531 25250 : *type_out = build_nonstandard_integer_type (hprec, false);
2532 25250 : *new_rhs_out = rhs;
2533 25250 : return true;
2534 : }
2535 790614 : }
2536 :
2537 634178 : stmt = SSA_NAME_DEF_STMT (rhs);
2538 634178 : if (is_gimple_assign (stmt))
2539 : {
2540 :
2541 516541 : if (widening_mult_conversion_strippable_p (type, stmt))
2542 : {
2543 199570 : rhs1 = gimple_assign_rhs1 (stmt);
2544 :
2545 199570 : if (TREE_CODE (rhs1) == INTEGER_CST)
2546 : {
2547 0 : *new_rhs_out = rhs1;
2548 0 : *type_out = NULL;
2549 0 : return true;
2550 : }
2551 : }
2552 : else
2553 : rhs1 = rhs;
2554 : }
2555 : else
2556 : rhs1 = rhs;
2557 :
2558 634178 : type1 = TREE_TYPE (rhs1);
2559 :
2560 634178 : if (TREE_CODE (type1) != TREE_CODE (type)
2561 634178 : || TYPE_PRECISION (type1) * 2 > TYPE_PRECISION (type))
2562 : return false;
2563 :
2564 63428 : *new_rhs_out = rhs1;
2565 63428 : *type_out = type1;
2566 63428 : return true;
2567 : }
2568 :
2569 161305 : if (TREE_CODE (rhs) == INTEGER_CST)
2570 : {
2571 161305 : *new_rhs_out = rhs;
2572 161305 : *type_out = NULL;
2573 161305 : return true;
2574 : }
2575 :
2576 : return false;
2577 : }
2578 :
2579 : /* Return true if STMT performs a widening multiplication, assuming the
2580 : output type is TYPE. If so, store the unwidened types of the operands
2581 : in *TYPE1_OUT and *TYPE2_OUT respectively. Also fill *RHS1_OUT and
2582 : *RHS2_OUT such that converting those operands to types *TYPE1_OUT
2583 : and *TYPE2_OUT would give the operands of the multiplication. */
2584 :
2585 : static bool
2586 761594 : is_widening_mult_p (gimple *stmt,
2587 : tree *type1_out, tree *rhs1_out,
2588 : tree *type2_out, tree *rhs2_out)
2589 : {
2590 761594 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
2591 :
2592 761594 : if (TREE_CODE (type) == INTEGER_TYPE)
2593 : {
2594 761594 : if (TYPE_OVERFLOW_TRAPS (type))
2595 : return false;
2596 : }
2597 0 : else if (TREE_CODE (type) != FIXED_POINT_TYPE)
2598 : return false;
2599 :
2600 761565 : if (!is_widening_mult_rhs_p (type, gimple_assign_rhs1 (stmt), type1_out,
2601 : rhs1_out))
2602 : return false;
2603 :
2604 196685 : if (!is_widening_mult_rhs_p (type, gimple_assign_rhs2 (stmt), type2_out,
2605 : rhs2_out))
2606 : return false;
2607 :
2608 190815 : if (*type1_out == NULL)
2609 : {
2610 0 : if (*type2_out == NULL || !int_fits_type_p (*rhs1_out, *type2_out))
2611 : return false;
2612 0 : *type1_out = *type2_out;
2613 : }
2614 :
2615 190815 : if (*type2_out == NULL)
2616 : {
2617 161305 : if (!int_fits_type_p (*rhs2_out, *type1_out))
2618 : return false;
2619 157982 : *type2_out = *type1_out;
2620 : }
2621 :
2622 : /* Ensure that the larger of the two operands comes first. */
2623 187492 : if (TYPE_PRECISION (*type1_out) < TYPE_PRECISION (*type2_out))
2624 : {
2625 70 : std::swap (*type1_out, *type2_out);
2626 70 : std::swap (*rhs1_out, *rhs2_out);
2627 : }
2628 :
2629 : return true;
2630 : }
2631 :
2632 : /* Check to see if the CALL statement is an invocation of copysign
2633 : with 1. being the first argument. */
2634 : static bool
2635 197652 : is_copysign_call_with_1 (gimple *call)
2636 : {
2637 197652 : gcall *c = dyn_cast <gcall *> (call);
2638 5135 : if (! c)
2639 : return false;
2640 :
2641 5135 : enum combined_fn code = gimple_call_combined_fn (c);
2642 :
2643 5135 : if (code == CFN_LAST)
2644 : return false;
2645 :
2646 4096 : if (builtin_fn_p (code))
2647 : {
2648 1034 : switch (as_builtin_fn (code))
2649 : {
2650 30 : CASE_FLT_FN (BUILT_IN_COPYSIGN):
2651 30 : CASE_FLT_FN_FLOATN_NX (BUILT_IN_COPYSIGN):
2652 30 : return real_onep (gimple_call_arg (c, 0));
2653 : default:
2654 : return false;
2655 : }
2656 : }
2657 :
2658 3062 : if (internal_fn_p (code))
2659 : {
2660 3062 : switch (as_internal_fn (code))
2661 : {
2662 24 : case IFN_COPYSIGN:
2663 24 : return real_onep (gimple_call_arg (c, 0));
2664 : default:
2665 : return false;
2666 : }
2667 : }
2668 :
2669 : return false;
2670 : }
2671 :
2672 : /* Try to expand the pattern x * copysign (1, y) into xorsign (x, y).
2673 : This only happens when the xorsign optab is defined, if the
2674 : pattern is not a xorsign pattern or if expansion fails FALSE is
2675 : returned, otherwise TRUE is returned. */
2676 : static bool
2677 750615 : convert_expand_mult_copysign (gimple *stmt, gimple_stmt_iterator *gsi)
2678 : {
2679 750615 : tree treeop0, treeop1, lhs, type;
2680 750615 : location_t loc = gimple_location (stmt);
2681 750615 : lhs = gimple_assign_lhs (stmt);
2682 750615 : treeop0 = gimple_assign_rhs1 (stmt);
2683 750615 : treeop1 = gimple_assign_rhs2 (stmt);
2684 750615 : type = TREE_TYPE (lhs);
2685 750615 : machine_mode mode = TYPE_MODE (type);
2686 :
2687 750615 : if (HONOR_SNANS (type))
2688 : return false;
2689 :
2690 750212 : if (TREE_CODE (treeop0) == SSA_NAME && TREE_CODE (treeop1) == SSA_NAME)
2691 : {
2692 238949 : gimple *call0 = SSA_NAME_DEF_STMT (treeop0);
2693 238949 : if (!has_single_use (treeop0) || !is_copysign_call_with_1 (call0))
2694 : {
2695 238923 : call0 = SSA_NAME_DEF_STMT (treeop1);
2696 238923 : if (!has_single_use (treeop1) || !is_copysign_call_with_1 (call0))
2697 : return false;
2698 :
2699 : treeop1 = treeop0;
2700 : }
2701 43 : if (optab_handler (xorsign_optab, mode) == CODE_FOR_nothing)
2702 : return false;
2703 :
2704 43 : gcall *c = as_a<gcall*> (call0);
2705 43 : treeop0 = gimple_call_arg (c, 1);
2706 :
2707 43 : gcall *call_stmt
2708 43 : = gimple_build_call_internal (IFN_XORSIGN, 2, treeop1, treeop0);
2709 43 : gimple_set_lhs (call_stmt, lhs);
2710 43 : gimple_set_location (call_stmt, loc);
2711 43 : gsi_replace (gsi, call_stmt, true);
2712 43 : return true;
2713 : }
2714 :
2715 : return false;
2716 : }
2717 :
2718 : /* Process a single gimple statement STMT, which has a MULT_EXPR as
2719 : its rhs, and try to convert it into a WIDEN_MULT_EXPR. The return
2720 : value is true iff we converted the statement. */
2721 :
2722 : static bool
2723 759506 : convert_mult_to_widen (gimple *stmt, gimple_stmt_iterator *gsi)
2724 : {
2725 759506 : tree lhs, rhs1, rhs2, type, type1, type2;
2726 759506 : enum insn_code handler;
2727 759506 : scalar_int_mode to_mode, from_mode, actual_mode;
2728 759506 : optab op;
2729 759506 : int actual_precision;
2730 759506 : location_t loc = gimple_location (stmt);
2731 759506 : bool from_unsigned1, from_unsigned2;
2732 :
2733 759506 : lhs = gimple_assign_lhs (stmt);
2734 759506 : type = TREE_TYPE (lhs);
2735 759506 : if (TREE_CODE (type) != INTEGER_TYPE)
2736 : return false;
2737 :
2738 620709 : if (!is_widening_mult_p (stmt, &type1, &rhs1, &type2, &rhs2))
2739 : return false;
2740 :
2741 : /* if any one of rhs1 and rhs2 is subject to abnormal coalescing,
2742 : avoid the transform. */
2743 154887 : if ((TREE_CODE (rhs1) == SSA_NAME
2744 154887 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1))
2745 309773 : || (TREE_CODE (rhs2) == SSA_NAME
2746 22545 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs2)))
2747 : return false;
2748 :
2749 154886 : to_mode = SCALAR_INT_TYPE_MODE (type);
2750 154886 : from_mode = SCALAR_INT_TYPE_MODE (type1);
2751 154886 : if (to_mode == from_mode)
2752 : return false;
2753 :
2754 154882 : from_unsigned1 = TYPE_UNSIGNED (type1);
2755 154882 : from_unsigned2 = TYPE_UNSIGNED (type2);
2756 :
2757 154882 : if (from_unsigned1 && from_unsigned2)
2758 : op = umul_widen_optab;
2759 60506 : else if (!from_unsigned1 && !from_unsigned2)
2760 : op = smul_widen_optab;
2761 : else
2762 1866 : op = usmul_widen_optab;
2763 :
2764 154882 : handler = find_widening_optab_handler_and_mode (op, to_mode, from_mode,
2765 : &actual_mode);
2766 :
2767 154882 : if (handler == CODE_FOR_nothing)
2768 : {
2769 145991 : if (op != smul_widen_optab)
2770 : {
2771 : /* We can use a signed multiply with unsigned types as long as
2772 : there is a wider mode to use, or it is the smaller of the two
2773 : types that is unsigned. Note that type1 >= type2, always. */
2774 88847 : if ((TYPE_UNSIGNED (type1)
2775 87173 : && TYPE_PRECISION (type1) == GET_MODE_PRECISION (from_mode))
2776 88847 : || (TYPE_UNSIGNED (type2)
2777 1674 : && TYPE_PRECISION (type2) == GET_MODE_PRECISION (from_mode)))
2778 : {
2779 88847 : if (!GET_MODE_WIDER_MODE (from_mode).exists (&from_mode)
2780 177694 : || GET_MODE_SIZE (to_mode) <= GET_MODE_SIZE (from_mode))
2781 : return false;
2782 : }
2783 :
2784 0 : op = smul_widen_optab;
2785 0 : handler = find_widening_optab_handler_and_mode (op, to_mode,
2786 : from_mode,
2787 : &actual_mode);
2788 :
2789 0 : if (handler == CODE_FOR_nothing)
2790 : return false;
2791 :
2792 : from_unsigned1 = from_unsigned2 = false;
2793 : }
2794 : else
2795 : {
2796 : /* Expand can synthesize smul_widen_optab if the target
2797 : supports umul_widen_optab. */
2798 57144 : op = umul_widen_optab;
2799 57144 : handler = find_widening_optab_handler_and_mode (op, to_mode,
2800 : from_mode,
2801 : &actual_mode);
2802 57144 : if (handler == CODE_FOR_nothing)
2803 : return false;
2804 : }
2805 : }
2806 :
2807 : /* Ensure that the inputs to the handler are in the correct precision
2808 : for the opcode. This will be the full mode size. */
2809 8891 : actual_precision = GET_MODE_PRECISION (actual_mode);
2810 8891 : if (2 * actual_precision > TYPE_PRECISION (type))
2811 : return false;
2812 8891 : if (actual_precision != TYPE_PRECISION (type1)
2813 8891 : || from_unsigned1 != TYPE_UNSIGNED (type1))
2814 : {
2815 3 : if (!useless_type_conversion_p (type1, TREE_TYPE (rhs1)))
2816 : {
2817 0 : if (TREE_CODE (rhs1) == INTEGER_CST)
2818 0 : rhs1 = fold_convert (type1, rhs1);
2819 : else
2820 0 : rhs1 = build_and_insert_cast (gsi, loc, type1, rhs1);
2821 : }
2822 3 : type1 = build_nonstandard_integer_type (actual_precision,
2823 : from_unsigned1);
2824 : }
2825 8891 : if (!useless_type_conversion_p (type1, TREE_TYPE (rhs1)))
2826 : {
2827 8154 : if (TREE_CODE (rhs1) == INTEGER_CST)
2828 0 : rhs1 = fold_convert (type1, rhs1);
2829 : else
2830 8154 : rhs1 = build_and_insert_cast (gsi, loc, type1, rhs1);
2831 : }
2832 8891 : if (actual_precision != TYPE_PRECISION (type2)
2833 8891 : || from_unsigned2 != TYPE_UNSIGNED (type2))
2834 : {
2835 3 : if (!useless_type_conversion_p (type2, TREE_TYPE (rhs2)))
2836 : {
2837 3 : if (TREE_CODE (rhs2) == INTEGER_CST)
2838 3 : rhs2 = fold_convert (type2, rhs2);
2839 : else
2840 0 : rhs2 = build_and_insert_cast (gsi, loc, type2, rhs2);
2841 : }
2842 3 : type2 = build_nonstandard_integer_type (actual_precision,
2843 : from_unsigned2);
2844 : }
2845 8891 : if (!useless_type_conversion_p (type2, TREE_TYPE (rhs2)))
2846 : {
2847 8349 : if (TREE_CODE (rhs2) == INTEGER_CST)
2848 1876 : rhs2 = fold_convert (type2, rhs2);
2849 : else
2850 6473 : rhs2 = build_and_insert_cast (gsi, loc, type2, rhs2);
2851 : }
2852 :
2853 8891 : gimple_assign_set_rhs1 (stmt, rhs1);
2854 8891 : gimple_assign_set_rhs2 (stmt, rhs2);
2855 8891 : gimple_assign_set_rhs_code (stmt, WIDEN_MULT_EXPR);
2856 8891 : update_stmt (stmt);
2857 8891 : widen_mul_stats.widen_mults_inserted++;
2858 8891 : return true;
2859 : }
2860 :
2861 : /* Process a single gimple statement STMT, which is found at the
2862 : iterator GSI and has a either a PLUS_EXPR or a MINUS_EXPR as its
2863 : rhs (given by CODE), and try to convert it into a
2864 : WIDEN_MULT_PLUS_EXPR or a WIDEN_MULT_MINUS_EXPR. The return value
2865 : is true iff we converted the statement. */
2866 :
2867 : static bool
2868 2646343 : convert_plusminus_to_widen (gimple_stmt_iterator *gsi, gimple *stmt,
2869 : enum tree_code code)
2870 : {
2871 2646343 : gimple *rhs1_stmt = NULL, *rhs2_stmt = NULL;
2872 2646343 : gimple *conv1_stmt = NULL, *conv2_stmt = NULL, *conv_stmt;
2873 2646343 : tree type, type1, type2, optype;
2874 2646343 : tree lhs, rhs1, rhs2, mult_rhs1, mult_rhs2, add_rhs;
2875 2646343 : enum tree_code rhs1_code = ERROR_MARK, rhs2_code = ERROR_MARK;
2876 2646343 : optab this_optab;
2877 2646343 : enum tree_code wmult_code;
2878 2646343 : enum insn_code handler;
2879 2646343 : scalar_mode to_mode, from_mode, actual_mode;
2880 2646343 : location_t loc = gimple_location (stmt);
2881 2646343 : int actual_precision;
2882 2646343 : bool from_unsigned1, from_unsigned2;
2883 :
2884 2646343 : lhs = gimple_assign_lhs (stmt);
2885 2646343 : type = TREE_TYPE (lhs);
2886 2646343 : if ((TREE_CODE (type) != INTEGER_TYPE
2887 400508 : && TREE_CODE (type) != FIXED_POINT_TYPE)
2888 2646343 : || !type_has_mode_precision_p (type))
2889 : return false;
2890 :
2891 2243104 : if (code == MINUS_EXPR)
2892 : wmult_code = WIDEN_MULT_MINUS_EXPR;
2893 : else
2894 1992877 : wmult_code = WIDEN_MULT_PLUS_EXPR;
2895 :
2896 2243104 : rhs1 = gimple_assign_rhs1 (stmt);
2897 2243104 : rhs2 = gimple_assign_rhs2 (stmt);
2898 :
2899 2243104 : if (TREE_CODE (rhs1) == SSA_NAME)
2900 : {
2901 2211810 : rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
2902 2211810 : if (is_gimple_assign (rhs1_stmt))
2903 1320816 : rhs1_code = gimple_assign_rhs_code (rhs1_stmt);
2904 : }
2905 :
2906 2243104 : if (TREE_CODE (rhs2) == SSA_NAME)
2907 : {
2908 817304 : rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
2909 817304 : if (is_gimple_assign (rhs2_stmt))
2910 627011 : rhs2_code = gimple_assign_rhs_code (rhs2_stmt);
2911 : }
2912 :
2913 : /* Allow for one conversion statement between the multiply
2914 : and addition/subtraction statement. If there are more than
2915 : one conversions then we assume they would invalidate this
2916 : transformation. If that's not the case then they should have
2917 : been folded before now. */
2918 2243104 : if (CONVERT_EXPR_CODE_P (rhs1_code))
2919 : {
2920 420091 : conv1_stmt = rhs1_stmt;
2921 420091 : rhs1 = gimple_assign_rhs1 (rhs1_stmt);
2922 420091 : if (TREE_CODE (rhs1) == SSA_NAME)
2923 : {
2924 350925 : rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
2925 350925 : if (is_gimple_assign (rhs1_stmt))
2926 207115 : rhs1_code = gimple_assign_rhs_code (rhs1_stmt);
2927 : }
2928 : else
2929 : return false;
2930 : }
2931 2173938 : if (CONVERT_EXPR_CODE_P (rhs2_code))
2932 : {
2933 189762 : conv2_stmt = rhs2_stmt;
2934 189762 : rhs2 = gimple_assign_rhs1 (rhs2_stmt);
2935 189762 : if (TREE_CODE (rhs2) == SSA_NAME)
2936 : {
2937 179794 : rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
2938 179794 : if (is_gimple_assign (rhs2_stmt))
2939 118361 : rhs2_code = gimple_assign_rhs_code (rhs2_stmt);
2940 : }
2941 : else
2942 : return false;
2943 : }
2944 :
2945 : /* If code is WIDEN_MULT_EXPR then it would seem unnecessary to call
2946 : is_widening_mult_p, but we still need the rhs returns.
2947 :
2948 : It might also appear that it would be sufficient to use the existing
2949 : operands of the widening multiply, but that would limit the choice of
2950 : multiply-and-accumulate instructions.
2951 :
2952 : If the widened-multiplication result has more than one uses, it is
2953 : probably wiser not to do the conversion. Also restrict this operation
2954 : to single basic block to avoid moving the multiply to a different block
2955 : with a higher execution frequency. */
2956 2163970 : if (code == PLUS_EXPR
2957 1918552 : && (rhs1_code == MULT_EXPR || rhs1_code == WIDEN_MULT_EXPR))
2958 : {
2959 152126 : if (!has_single_use (rhs1)
2960 81520 : || (conv1_stmt
2961 5152 : && !has_single_use (gimple_assign_lhs (conv1_stmt)))
2962 78771 : || gimple_bb (rhs1_stmt) != gimple_bb (stmt)
2963 221788 : || !is_widening_mult_p (rhs1_stmt, &type1, &mult_rhs1,
2964 : &type2, &mult_rhs2))
2965 : return false;
2966 : add_rhs = rhs2;
2967 : conv_stmt = conv1_stmt;
2968 : }
2969 2011844 : else if (rhs2_code == MULT_EXPR || rhs2_code == WIDEN_MULT_EXPR)
2970 : {
2971 125559 : if (!has_single_use (rhs2)
2972 80171 : || (conv2_stmt
2973 9149 : && !has_single_use (gimple_assign_lhs (conv2_stmt)))
2974 79006 : || gimple_bb (rhs2_stmt) != gimple_bb (stmt)
2975 196782 : || !is_widening_mult_p (rhs2_stmt, &type1, &mult_rhs1,
2976 : &type2, &mult_rhs2))
2977 : return false;
2978 : add_rhs = rhs1;
2979 : conv_stmt = conv2_stmt;
2980 : }
2981 : else
2982 : return false;
2983 :
2984 32605 : to_mode = SCALAR_TYPE_MODE (type);
2985 32605 : from_mode = SCALAR_TYPE_MODE (type1);
2986 32605 : if (to_mode == from_mode)
2987 : return false;
2988 :
2989 : /* For fixed point types, the mode classes could be different
2990 : so reject that case. */
2991 32604 : if (GET_MODE_CLASS (from_mode) != GET_MODE_CLASS (to_mode))
2992 : return false;
2993 :
2994 32604 : from_unsigned1 = TYPE_UNSIGNED (type1);
2995 32604 : from_unsigned2 = TYPE_UNSIGNED (type2);
2996 32604 : optype = type1;
2997 :
2998 : /* There's no such thing as a mixed sign madd yet, so use a wider mode. */
2999 32604 : if (from_unsigned1 != from_unsigned2)
3000 : {
3001 896 : if (!INTEGRAL_TYPE_P (type))
3002 : return false;
3003 : /* We can use a signed multiply with unsigned types as long as
3004 : there is a wider mode to use, or it is the smaller of the two
3005 : types that is unsigned. Note that type1 >= type2, always. */
3006 896 : if ((from_unsigned1
3007 54 : && TYPE_PRECISION (type1) == GET_MODE_PRECISION (from_mode))
3008 896 : || (from_unsigned2
3009 842 : && TYPE_PRECISION (type2) == GET_MODE_PRECISION (from_mode)))
3010 : {
3011 896 : if (!GET_MODE_WIDER_MODE (from_mode).exists (&from_mode)
3012 1792 : || GET_MODE_SIZE (from_mode) >= GET_MODE_SIZE (to_mode))
3013 : return false;
3014 : }
3015 :
3016 18 : from_unsigned1 = from_unsigned2 = false;
3017 18 : optype = build_nonstandard_integer_type (GET_MODE_PRECISION (from_mode),
3018 : false);
3019 : }
3020 :
3021 : /* If there was a conversion between the multiply and addition
3022 : then we need to make sure it fits a multiply-and-accumulate.
3023 : The should be a single mode change which does not change the
3024 : value. */
3025 31726 : if (conv_stmt)
3026 : {
3027 : /* We use the original, unmodified data types for this. */
3028 703 : tree from_type = TREE_TYPE (gimple_assign_rhs1 (conv_stmt));
3029 703 : tree to_type = TREE_TYPE (gimple_assign_lhs (conv_stmt));
3030 703 : int data_size = TYPE_PRECISION (type1) + TYPE_PRECISION (type2);
3031 703 : bool is_unsigned = TYPE_UNSIGNED (type1) && TYPE_UNSIGNED (type2);
3032 :
3033 703 : if (TYPE_PRECISION (from_type) > TYPE_PRECISION (to_type))
3034 : {
3035 : /* Conversion is a truncate. */
3036 0 : if (TYPE_PRECISION (to_type) < data_size)
3037 : return false;
3038 : }
3039 703 : else if (TYPE_PRECISION (from_type) < TYPE_PRECISION (to_type))
3040 : {
3041 : /* Conversion is an extend. Check it's the right sort. */
3042 367 : if (TYPE_UNSIGNED (from_type) != is_unsigned
3043 367 : && !(is_unsigned && TYPE_PRECISION (from_type) > data_size))
3044 : return false;
3045 : }
3046 : /* else convert is a no-op for our purposes. */
3047 : }
3048 :
3049 : /* Verify that the machine can perform a widening multiply
3050 : accumulate in this mode/signedness combination, otherwise
3051 : this transformation is likely to pessimize code. */
3052 31434 : this_optab = optab_for_tree_code (wmult_code, optype, optab_default);
3053 31434 : handler = find_widening_optab_handler_and_mode (this_optab, to_mode,
3054 : from_mode, &actual_mode);
3055 :
3056 31434 : if (handler == CODE_FOR_nothing)
3057 : return false;
3058 :
3059 : /* Ensure that the inputs to the handler are in the correct precision
3060 : for the opcode. This will be the full mode size. */
3061 0 : actual_precision = GET_MODE_PRECISION (actual_mode);
3062 0 : if (actual_precision != TYPE_PRECISION (type1)
3063 0 : || from_unsigned1 != TYPE_UNSIGNED (type1))
3064 : {
3065 0 : if (!useless_type_conversion_p (type1, TREE_TYPE (mult_rhs1)))
3066 : {
3067 0 : if (TREE_CODE (mult_rhs1) == INTEGER_CST)
3068 0 : mult_rhs1 = fold_convert (type1, mult_rhs1);
3069 : else
3070 0 : mult_rhs1 = build_and_insert_cast (gsi, loc, type1, mult_rhs1);
3071 : }
3072 0 : type1 = build_nonstandard_integer_type (actual_precision,
3073 : from_unsigned1);
3074 : }
3075 0 : if (!useless_type_conversion_p (type1, TREE_TYPE (mult_rhs1)))
3076 : {
3077 0 : if (TREE_CODE (mult_rhs1) == INTEGER_CST)
3078 0 : mult_rhs1 = fold_convert (type1, mult_rhs1);
3079 : else
3080 0 : mult_rhs1 = build_and_insert_cast (gsi, loc, type1, mult_rhs1);
3081 : }
3082 0 : if (actual_precision != TYPE_PRECISION (type2)
3083 0 : || from_unsigned2 != TYPE_UNSIGNED (type2))
3084 : {
3085 0 : if (!useless_type_conversion_p (type2, TREE_TYPE (mult_rhs2)))
3086 : {
3087 0 : if (TREE_CODE (mult_rhs2) == INTEGER_CST)
3088 0 : mult_rhs2 = fold_convert (type2, mult_rhs2);
3089 : else
3090 0 : mult_rhs2 = build_and_insert_cast (gsi, loc, type2, mult_rhs2);
3091 : }
3092 0 : type2 = build_nonstandard_integer_type (actual_precision,
3093 : from_unsigned2);
3094 : }
3095 0 : if (!useless_type_conversion_p (type2, TREE_TYPE (mult_rhs2)))
3096 : {
3097 0 : if (TREE_CODE (mult_rhs2) == INTEGER_CST)
3098 0 : mult_rhs2 = fold_convert (type2, mult_rhs2);
3099 : else
3100 0 : mult_rhs2 = build_and_insert_cast (gsi, loc, type2, mult_rhs2);
3101 : }
3102 :
3103 0 : if (!useless_type_conversion_p (type, TREE_TYPE (add_rhs)))
3104 0 : add_rhs = build_and_insert_cast (gsi, loc, type, add_rhs);
3105 :
3106 0 : gimple_assign_set_rhs_with_ops (gsi, wmult_code, mult_rhs1, mult_rhs2,
3107 : add_rhs);
3108 0 : update_stmt (gsi_stmt (*gsi));
3109 0 : widen_mul_stats.maccs_inserted++;
3110 0 : return true;
3111 : }
3112 :
3113 : /* Given a result MUL_RESULT which is a result of a multiplication of OP1 and
3114 : OP2 and which we know is used in statements that can be, together with the
3115 : multiplication, converted to FMAs, perform the transformation. */
3116 :
3117 : static void
3118 17553 : convert_mult_to_fma_1 (tree mul_result, tree op1, tree op2)
3119 : {
3120 17553 : gimple *use_stmt;
3121 17553 : imm_use_iterator imm_iter;
3122 17553 : gcall *fma_stmt;
3123 :
3124 35165 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, mul_result)
3125 : {
3126 17612 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
3127 17612 : tree addop, mulop1 = op1, result = mul_result;
3128 17612 : bool negate_p = false;
3129 17612 : gimple_seq seq = NULL;
3130 :
3131 17612 : if (is_gimple_debug (use_stmt))
3132 0 : continue;
3133 :
3134 : /* If the use is a type convert, look further into it if the operations
3135 : are the same under two's complement. */
3136 17612 : tree lhs_type;
3137 17612 : if (gimple_assign_cast_p (use_stmt)
3138 0 : && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt)))
3139 17612 : && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1)))
3140 : {
3141 0 : tree cast_lhs = gimple_get_lhs (use_stmt);
3142 0 : gimple *tmp_use;
3143 0 : use_operand_p tmp_use_p;
3144 0 : if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use))
3145 : {
3146 0 : release_defs (use_stmt);
3147 0 : use_stmt = tmp_use;
3148 0 : result = cast_lhs;
3149 0 : gsi_remove (&gsi, true);
3150 0 : gsi = gsi_for_stmt (use_stmt);
3151 : }
3152 : }
3153 :
3154 17612 : if (is_gimple_assign (use_stmt)
3155 17612 : && gimple_assign_rhs_code (use_stmt) == NEGATE_EXPR)
3156 : {
3157 700 : result = gimple_assign_lhs (use_stmt);
3158 700 : use_operand_p use_p;
3159 700 : gimple *neguse_stmt;
3160 700 : single_imm_use (gimple_assign_lhs (use_stmt), &use_p, &neguse_stmt);
3161 700 : gsi_remove (&gsi, true);
3162 700 : release_defs (use_stmt);
3163 :
3164 700 : use_stmt = neguse_stmt;
3165 700 : gsi = gsi_for_stmt (use_stmt);
3166 700 : negate_p = true;
3167 : }
3168 :
3169 17612 : tree cond, else_value, ops[3], len, bias;
3170 17612 : tree_code code;
3171 17612 : if (!can_interpret_as_conditional_op_p (use_stmt, &cond, &code,
3172 : ops, &else_value,
3173 : &len, &bias))
3174 0 : gcc_unreachable ();
3175 17612 : addop = ops[0] == result ? ops[1] : ops[0];
3176 :
3177 17612 : if (code == MINUS_EXPR)
3178 : {
3179 5847 : if (ops[0] == result)
3180 : /* a * b - c -> a * b + (-c) */
3181 2918 : addop = gimple_build (&seq, NEGATE_EXPR, TREE_TYPE (addop), addop);
3182 : else
3183 : /* a - b * c -> (-b) * c + a */
3184 2929 : negate_p = !negate_p;
3185 : }
3186 :
3187 17612 : if (negate_p)
3188 3629 : mulop1 = gimple_build (&seq, NEGATE_EXPR, TREE_TYPE (mulop1), mulop1);
3189 :
3190 17612 : if (seq)
3191 5842 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
3192 :
3193 : /* Ensure all the operands are of the same type. Use the type of the
3194 : addend as that's the statement being replaced. */
3195 17612 : op2 = gimple_convert (&gsi, true, GSI_SAME_STMT,
3196 17612 : UNKNOWN_LOCATION, TREE_TYPE (addop), op2);
3197 17612 : mulop1 = gimple_convert (&gsi, true, GSI_SAME_STMT,
3198 17612 : UNKNOWN_LOCATION, TREE_TYPE (addop), mulop1);
3199 :
3200 17612 : if (len)
3201 0 : fma_stmt
3202 0 : = gimple_build_call_internal (IFN_COND_LEN_FMA, 7, cond, mulop1, op2,
3203 : addop, else_value, len, bias);
3204 17612 : else if (cond)
3205 94 : fma_stmt = gimple_build_call_internal (IFN_COND_FMA, 5, cond, mulop1,
3206 : op2, addop, else_value);
3207 : else
3208 17518 : fma_stmt = gimple_build_call_internal (IFN_FMA, 3, mulop1, op2, addop);
3209 17612 : gimple_set_lhs (fma_stmt, gimple_get_lhs (use_stmt));
3210 17612 : gimple_call_set_nothrow (fma_stmt, !stmt_can_throw_internal (cfun,
3211 : use_stmt));
3212 17612 : gsi_replace (&gsi, fma_stmt, true);
3213 : /* Follow all SSA edges so that we generate FMS, FNMA and FNMS
3214 : regardless of where the negation occurs. */
3215 17612 : gimple *orig_stmt = gsi_stmt (gsi);
3216 17612 : if (fold_stmt (&gsi, follow_all_ssa_edges))
3217 : {
3218 5891 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, gsi_stmt (gsi)))
3219 0 : gcc_unreachable ();
3220 5891 : update_stmt (gsi_stmt (gsi));
3221 : }
3222 :
3223 17612 : if (dump_file && (dump_flags & TDF_DETAILS))
3224 : {
3225 3 : fprintf (dump_file, "Generated FMA ");
3226 3 : print_gimple_stmt (dump_file, gsi_stmt (gsi), 0, TDF_NONE);
3227 3 : fprintf (dump_file, "\n");
3228 : }
3229 :
3230 : /* If the FMA result is negated in a single use, fold the negation
3231 : too. */
3232 17612 : orig_stmt = gsi_stmt (gsi);
3233 17612 : use_operand_p use_p;
3234 17612 : gimple *neg_stmt;
3235 17612 : if (is_gimple_call (orig_stmt)
3236 17612 : && gimple_call_internal_p (orig_stmt)
3237 17612 : && gimple_call_lhs (orig_stmt)
3238 17612 : && TREE_CODE (gimple_call_lhs (orig_stmt)) == SSA_NAME
3239 17612 : && single_imm_use (gimple_call_lhs (orig_stmt), &use_p, &neg_stmt)
3240 12562 : && is_gimple_assign (neg_stmt)
3241 9900 : && gimple_assign_rhs_code (neg_stmt) == NEGATE_EXPR
3242 18965 : && !stmt_could_throw_p (cfun, neg_stmt))
3243 : {
3244 1353 : gsi = gsi_for_stmt (neg_stmt);
3245 1353 : if (fold_stmt (&gsi, follow_all_ssa_edges))
3246 : {
3247 1353 : if (maybe_clean_or_replace_eh_stmt (neg_stmt, gsi_stmt (gsi)))
3248 0 : gcc_unreachable ();
3249 1353 : update_stmt (gsi_stmt (gsi));
3250 1353 : if (dump_file && (dump_flags & TDF_DETAILS))
3251 : {
3252 0 : fprintf (dump_file, "Folded FMA negation ");
3253 0 : print_gimple_stmt (dump_file, gsi_stmt (gsi), 0, TDF_NONE);
3254 0 : fprintf (dump_file, "\n");
3255 : }
3256 : }
3257 : }
3258 :
3259 17612 : widen_mul_stats.fmas_inserted++;
3260 17553 : }
3261 17553 : }
3262 :
3263 : /* Data necessary to perform the actual transformation from a multiplication
3264 : and an addition to an FMA after decision is taken it should be done and to
3265 : then delete the multiplication statement from the function IL. */
3266 :
3267 : struct fma_transformation_info
3268 : {
3269 : gimple *mul_stmt;
3270 : tree mul_result;
3271 : tree op1;
3272 : tree op2;
3273 : };
3274 :
3275 : /* Structure containing the current state of FMA deferring, i.e. whether we are
3276 : deferring, whether to continue deferring, and all data necessary to come
3277 : back and perform all deferred transformations. */
3278 :
3279 10617951 : class fma_deferring_state
3280 : {
3281 : public:
3282 : /* Class constructor. Pass true as PERFORM_DEFERRING in order to actually
3283 : do any deferring. */
3284 :
3285 10617951 : fma_deferring_state (bool perform_deferring)
3286 10617951 : : m_candidates (), m_mul_result_set (), m_initial_phi (NULL),
3287 10617951 : m_last_result (NULL_TREE), m_deferring_p (perform_deferring) {}
3288 :
3289 : /* List of FMA candidates for which we the transformation has been determined
3290 : possible but we at this point in BB analysis we do not consider them
3291 : beneficial. */
3292 : auto_vec<fma_transformation_info, 8> m_candidates;
3293 :
3294 : /* Set of results of multiplication that are part of an already deferred FMA
3295 : candidates. */
3296 : hash_set<tree> m_mul_result_set;
3297 :
3298 : /* The PHI that supposedly feeds back result of a FMA to another over loop
3299 : boundary. */
3300 : gphi *m_initial_phi;
3301 :
3302 : /* Result of the last produced FMA candidate or NULL if there has not been
3303 : one. */
3304 : tree m_last_result;
3305 :
3306 : /* If true, deferring might still be profitable. If false, transform all
3307 : candidates and no longer defer. */
3308 : bool m_deferring_p;
3309 : };
3310 :
3311 : /* Transform all deferred FMA candidates and mark STATE as no longer
3312 : deferring. */
3313 :
3314 : static void
3315 3812698 : cancel_fma_deferring (fma_deferring_state *state)
3316 : {
3317 3812698 : if (!state->m_deferring_p)
3318 : return;
3319 :
3320 2759633 : for (unsigned i = 0; i < state->m_candidates.length (); i++)
3321 : {
3322 940 : if (dump_file && (dump_flags & TDF_DETAILS))
3323 0 : fprintf (dump_file, "Generating deferred FMA\n");
3324 :
3325 940 : const fma_transformation_info &fti = state->m_candidates[i];
3326 940 : convert_mult_to_fma_1 (fti.mul_result, fti.op1, fti.op2);
3327 :
3328 940 : gimple_stmt_iterator gsi = gsi_for_stmt (fti.mul_stmt);
3329 940 : gsi_remove (&gsi, true);
3330 940 : release_defs (fti.mul_stmt);
3331 : }
3332 2758693 : state->m_deferring_p = false;
3333 : }
3334 :
3335 : /* If OP is an SSA name defined by a PHI node, return the PHI statement.
3336 : Otherwise return NULL. */
3337 :
3338 : static gphi *
3339 5268 : result_of_phi (tree op)
3340 : {
3341 0 : if (TREE_CODE (op) != SSA_NAME)
3342 : return NULL;
3343 :
3344 5143 : return dyn_cast <gphi *> (SSA_NAME_DEF_STMT (op));
3345 : }
3346 :
3347 : /* After processing statements of a BB and recording STATE, return true if the
3348 : initial phi is fed by the last FMA candidate result ore one such result from
3349 : previously processed BBs marked in LAST_RESULT_SET. */
3350 :
3351 : static bool
3352 361 : last_fma_candidate_feeds_initial_phi (fma_deferring_state *state,
3353 : hash_set<tree> *last_result_set)
3354 : {
3355 361 : ssa_op_iter iter;
3356 361 : use_operand_p use;
3357 889 : FOR_EACH_PHI_ARG (use, state->m_initial_phi, iter, SSA_OP_USE)
3358 : {
3359 625 : tree t = USE_FROM_PTR (use);
3360 625 : if (t == state->m_last_result
3361 625 : || last_result_set->contains (t))
3362 97 : return true;
3363 : }
3364 :
3365 : return false;
3366 : }
3367 :
3368 : /* Combine the multiplication at MUL_STMT with operands MULOP1 and MULOP2
3369 : with uses in additions and subtractions to form fused multiply-add
3370 : operations. Returns true if successful and MUL_STMT should be removed.
3371 : If MUL_COND is nonnull, the multiplication in MUL_STMT is conditional
3372 : on MUL_COND, otherwise it is unconditional.
3373 :
3374 : If STATE indicates that we are deferring FMA transformation, that means
3375 : that we do not produce FMAs for basic blocks which look like:
3376 :
3377 : <bb 6>
3378 : # accumulator_111 = PHI <0.0(5), accumulator_66(6)>
3379 : _65 = _14 * _16;
3380 : accumulator_66 = _65 + accumulator_111;
3381 :
3382 : or its unrolled version, i.e. with several FMA candidates that feed result
3383 : of one into the addend of another. Instead, we add them to a list in STATE
3384 : and if we later discover an FMA candidate that is not part of such a chain,
3385 : we go back and perform all deferred past candidates. */
3386 :
3387 : static bool
3388 750701 : convert_mult_to_fma (gimple *mul_stmt, tree op1, tree op2,
3389 : fma_deferring_state *state, tree mul_cond = NULL_TREE,
3390 : tree mul_len = NULL_TREE, tree mul_bias = NULL_TREE)
3391 : {
3392 750701 : tree mul_result = gimple_get_lhs (mul_stmt);
3393 : /* If there isn't a LHS then this can't be an FMA. There can be no LHS
3394 : if the statement was left just for the side-effects. */
3395 750701 : if (!mul_result)
3396 : return false;
3397 750701 : tree type = TREE_TYPE (mul_result);
3398 750701 : gimple *use_stmt, *neguse_stmt;
3399 750701 : use_operand_p use_p;
3400 750701 : imm_use_iterator imm_iter;
3401 :
3402 652629 : if (FLOAT_TYPE_P (type)
3403 776429 : && flag_fp_contract_mode != FP_CONTRACT_FAST)
3404 : return false;
3405 :
3406 : /* We don't want to do bitfield reduction ops. */
3407 745612 : if (INTEGRAL_TYPE_P (type)
3408 745612 : && (!type_has_mode_precision_p (type) || TYPE_OVERFLOW_TRAPS (type)))
3409 : return false;
3410 :
3411 : /* If the target doesn't support it, don't generate it. We assume that
3412 : if fma isn't available then fms, fnma or fnms are not either. */
3413 745415 : optimization_type opt_type = bb_optimization_type (gimple_bb (mul_stmt));
3414 745415 : if (!direct_internal_fn_supported_p (IFN_FMA, type, opt_type))
3415 : return false;
3416 :
3417 : /* If the multiplication has zero uses, it is kept around probably because
3418 : of -fnon-call-exceptions. Don't optimize it away in that case,
3419 : it is DCE job. */
3420 23058 : if (has_zero_uses (mul_result))
3421 : return false;
3422 :
3423 23058 : bool check_defer
3424 23058 : = (state->m_deferring_p
3425 23058 : && maybe_le (tree_to_poly_int64 (TYPE_SIZE (type)),
3426 23058 : param_avoid_fma_max_bits));
3427 23058 : bool defer = check_defer;
3428 23058 : bool seen_negate_p = false;
3429 :
3430 : /* There is no numerical difference between fused and unfused integer FMAs,
3431 : and the assumption below that FMA is as cheap as addition is unlikely
3432 : to be true, especially if the multiplication occurs multiple times on
3433 : the same chain. E.g., for something like:
3434 :
3435 : (((a * b) + c) >> 1) + (a * b)
3436 :
3437 : we do not want to duplicate the a * b into two additions, not least
3438 : because the result is not a natural FMA chain. */
3439 23058 : if (ANY_INTEGRAL_TYPE_P (type)
3440 23058 : && !has_single_use (mul_result))
3441 : return false;
3442 :
3443 23058 : if (!dbg_cnt (form_fma))
3444 : return false;
3445 :
3446 : /* Make sure that the multiplication statement becomes dead after
3447 : the transformation, thus that all uses are transformed to FMAs.
3448 : This means we assume that an FMA operation has the same cost
3449 : as an addition. */
3450 41455 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, mul_result)
3451 : {
3452 23805 : tree result = mul_result;
3453 23805 : bool negate_p = false;
3454 :
3455 23805 : use_stmt = USE_STMT (use_p);
3456 :
3457 23805 : if (is_gimple_debug (use_stmt))
3458 214 : continue;
3459 :
3460 : /* If the use is a type convert, look further into it if the operations
3461 : are the same under two's complement. */
3462 23591 : tree lhs_type;
3463 23591 : if (gimple_assign_cast_p (use_stmt)
3464 243 : && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt)))
3465 23834 : && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1)))
3466 : {
3467 0 : tree cast_lhs = gimple_get_lhs (use_stmt);
3468 0 : gimple *tmp_use;
3469 0 : use_operand_p tmp_use_p;
3470 0 : if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use))
3471 0 : use_stmt = tmp_use;
3472 0 : result = cast_lhs;
3473 : }
3474 :
3475 : /* For now restrict this operations to single basic blocks. In theory
3476 : we would want to support sinking the multiplication in
3477 : m = a*b;
3478 : if ()
3479 : ma = m + c;
3480 : else
3481 : d = m;
3482 : to form a fma in the then block and sink the multiplication to the
3483 : else block. */
3484 23591 : if (gimple_bb (use_stmt) != gimple_bb (mul_stmt))
3485 5408 : return false;
3486 :
3487 : /* A negate on the multiplication leads to FNMA. */
3488 22723 : if (is_gimple_assign (use_stmt)
3489 22723 : && gimple_assign_rhs_code (use_stmt) == NEGATE_EXPR)
3490 : {
3491 706 : ssa_op_iter iter;
3492 706 : use_operand_p usep;
3493 :
3494 : /* If (due to earlier missed optimizations) we have two
3495 : negates of the same value, treat them as equivalent
3496 : to a single negate with multiple uses. */
3497 706 : if (seen_negate_p)
3498 0 : return false;
3499 :
3500 706 : result = gimple_assign_lhs (use_stmt);
3501 :
3502 : /* Make sure the negate statement becomes dead with this
3503 : single transformation. */
3504 706 : if (!single_imm_use (gimple_assign_lhs (use_stmt),
3505 : &use_p, &neguse_stmt))
3506 : return false;
3507 :
3508 : /* Make sure the multiplication isn't also used on that stmt. */
3509 2836 : FOR_EACH_PHI_OR_STMT_USE (usep, neguse_stmt, iter, SSA_OP_USE)
3510 1424 : if (USE_FROM_PTR (usep) == mul_result)
3511 : return false;
3512 :
3513 : /* Re-validate. */
3514 706 : use_stmt = neguse_stmt;
3515 706 : if (gimple_bb (use_stmt) != gimple_bb (mul_stmt))
3516 : return false;
3517 :
3518 706 : negate_p = seen_negate_p = true;
3519 : }
3520 :
3521 22723 : tree cond, else_value, ops[3], len, bias;
3522 22723 : tree_code code;
3523 22723 : if (!can_interpret_as_conditional_op_p (use_stmt, &cond, &code, ops,
3524 : &else_value, &len, &bias))
3525 : return false;
3526 :
3527 : /* The multiplication result must be one of the addition operands. */
3528 20364 : if (ops[0] != result && ops[1] != result)
3529 : return false;
3530 :
3531 19753 : switch (code)
3532 : {
3533 5853 : case MINUS_EXPR:
3534 5853 : if (ops[1] == result)
3535 2929 : negate_p = !negate_p;
3536 : break;
3537 : case PLUS_EXPR:
3538 : break;
3539 : default:
3540 : /* FMA can only be formed from PLUS and MINUS. */
3541 : return false;
3542 : }
3543 :
3544 18205 : if (len)
3545 : {
3546 : /* For COND_LEN_* operations, we may have dummpy mask which is
3547 : the all true mask. Such TREE type may be mul_cond != cond
3548 : but we still consider they are equal. */
3549 0 : if (mul_cond && cond != mul_cond
3550 0 : && !(integer_truep (mul_cond) && integer_truep (cond)))
3551 : return false;
3552 :
3553 0 : if (else_value == result)
3554 : return false;
3555 :
3556 0 : if (!direct_internal_fn_supported_p (IFN_COND_LEN_FMA, type,
3557 : opt_type))
3558 : return false;
3559 :
3560 0 : if (mul_len)
3561 : {
3562 0 : poly_int64 mul_value, value;
3563 0 : if (poly_int_tree_p (mul_len, &mul_value)
3564 0 : && poly_int_tree_p (len, &value)
3565 0 : && maybe_ne (mul_value, value))
3566 0 : return false;
3567 0 : else if (mul_len != len)
3568 : return false;
3569 :
3570 0 : if (wi::to_widest (mul_bias) != wi::to_widest (bias))
3571 : return false;
3572 : }
3573 : }
3574 : else
3575 : {
3576 18205 : if (mul_cond && cond != mul_cond)
3577 : return false;
3578 :
3579 18193 : if (cond)
3580 : {
3581 104 : if (cond == result || else_value == result)
3582 : return false;
3583 94 : if (!direct_internal_fn_supported_p (IFN_COND_FMA, type,
3584 : opt_type))
3585 : return false;
3586 : }
3587 : }
3588 :
3589 : /* If the subtrahend (OPS[1]) is computed by a MULT_EXPR that
3590 : we'll visit later, we might be able to get a more profitable
3591 : match with fnma.
3592 : OTOH, if we don't, a negate / fma pair has likely lower latency
3593 : that a mult / subtract pair. */
3594 18183 : if (code == MINUS_EXPR
3595 5847 : && !negate_p
3596 2218 : && ops[0] == result
3597 2218 : && !direct_internal_fn_supported_p (IFN_FMS, type, opt_type)
3598 0 : && direct_internal_fn_supported_p (IFN_FNMA, type, opt_type)
3599 0 : && TREE_CODE (ops[1]) == SSA_NAME
3600 18183 : && has_single_use (ops[1]))
3601 : {
3602 0 : gimple *stmt2 = SSA_NAME_DEF_STMT (ops[1]);
3603 0 : if (is_gimple_assign (stmt2)
3604 0 : && gimple_assign_rhs_code (stmt2) == MULT_EXPR)
3605 : return false;
3606 : }
3607 :
3608 : /* We can't handle a * b + a * b. */
3609 18183 : if (ops[0] == ops[1])
3610 : return false;
3611 : /* If deferring, make sure we are not looking at an instruction that
3612 : wouldn't have existed if we were not. */
3613 18183 : if (state->m_deferring_p
3614 18183 : && (state->m_mul_result_set.contains (ops[0])
3615 6463 : || state->m_mul_result_set.contains (ops[1])))
3616 : return false;
3617 :
3618 18183 : if (check_defer)
3619 : {
3620 6320 : tree use_lhs = gimple_get_lhs (use_stmt);
3621 6320 : if (state->m_last_result)
3622 : {
3623 1052 : if (ops[1] == state->m_last_result
3624 1052 : || ops[0] == state->m_last_result)
3625 : defer = true;
3626 : else
3627 6320 : defer = false;
3628 : }
3629 : else
3630 : {
3631 5268 : gcc_checking_assert (!state->m_initial_phi);
3632 5268 : gphi *phi;
3633 5268 : if (ops[0] == result)
3634 3325 : phi = result_of_phi (ops[1]);
3635 : else
3636 : {
3637 1943 : gcc_assert (ops[1] == result);
3638 1943 : phi = result_of_phi (ops[0]);
3639 : }
3640 :
3641 : if (phi)
3642 : {
3643 963 : state->m_initial_phi = phi;
3644 963 : defer = true;
3645 : }
3646 : else
3647 : defer = false;
3648 : }
3649 :
3650 6320 : state->m_last_result = use_lhs;
3651 6320 : check_defer = false;
3652 : }
3653 : else
3654 : defer = false;
3655 :
3656 : /* While it is possible to validate whether or not the exact form that
3657 : we've recognized is available in the backend, the assumption is that
3658 : if the deferring logic above did not trigger, the transformation is
3659 : never a loss. For instance, suppose the target only has the plain FMA
3660 : pattern available. Consider a*b-c -> fma(a,b,-c): we've exchanged
3661 : MUL+SUB for FMA+NEG, which is still two operations. Consider
3662 : -(a*b)-c -> fma(-a,b,-c): we still have 3 operations, but in the FMA
3663 : form the two NEGs are independent and could be run in parallel. */
3664 5408 : }
3665 :
3666 17650 : if (defer)
3667 : {
3668 1037 : fma_transformation_info fti;
3669 1037 : fti.mul_stmt = mul_stmt;
3670 1037 : fti.mul_result = mul_result;
3671 1037 : fti.op1 = op1;
3672 1037 : fti.op2 = op2;
3673 1037 : state->m_candidates.safe_push (fti);
3674 1037 : state->m_mul_result_set.add (mul_result);
3675 :
3676 1037 : if (dump_file && (dump_flags & TDF_DETAILS))
3677 : {
3678 0 : fprintf (dump_file, "Deferred generating FMA for multiplication ");
3679 0 : print_gimple_stmt (dump_file, mul_stmt, 0, TDF_NONE);
3680 0 : fprintf (dump_file, "\n");
3681 : }
3682 :
3683 1037 : return false;
3684 : }
3685 : else
3686 : {
3687 16613 : if (state->m_deferring_p)
3688 4924 : cancel_fma_deferring (state);
3689 16613 : convert_mult_to_fma_1 (mul_result, op1, op2);
3690 16613 : return true;
3691 : }
3692 : }
3693 :
3694 :
3695 : /* Helper function of match_arith_overflow. For MUL_OVERFLOW, if we have
3696 : a check for non-zero like:
3697 : _1 = x_4(D) * y_5(D);
3698 : *res_7(D) = _1;
3699 : if (x_4(D) != 0)
3700 : goto <bb 3>; [50.00%]
3701 : else
3702 : goto <bb 4>; [50.00%]
3703 :
3704 : <bb 3> [local count: 536870913]:
3705 : _2 = _1 / x_4(D);
3706 : _9 = _2 != y_5(D);
3707 : _10 = (int) _9;
3708 :
3709 : <bb 4> [local count: 1073741824]:
3710 : # iftmp.0_3 = PHI <_10(3), 0(2)>
3711 : then in addition to using .MUL_OVERFLOW (x_4(D), y_5(D)) we can also
3712 : optimize the x_4(D) != 0 condition to 1. */
3713 :
3714 : static void
3715 173 : maybe_optimize_guarding_check (vec<gimple *> &mul_stmts, gimple *cond_stmt,
3716 : gimple *div_stmt, bool *cfg_changed)
3717 : {
3718 173 : basic_block bb = gimple_bb (cond_stmt);
3719 346 : if (gimple_bb (div_stmt) != bb || !single_pred_p (bb))
3720 63 : return;
3721 173 : edge pred_edge = single_pred_edge (bb);
3722 173 : basic_block pred_bb = pred_edge->src;
3723 173 : if (EDGE_COUNT (pred_bb->succs) != 2)
3724 : return;
3725 130 : edge other_edge = EDGE_SUCC (pred_bb, EDGE_SUCC (pred_bb, 0) == pred_edge);
3726 130 : edge other_succ_edge = NULL;
3727 130 : if (gimple_code (cond_stmt) == GIMPLE_COND)
3728 : {
3729 48 : if (EDGE_COUNT (bb->succs) != 2)
3730 : return;
3731 48 : other_succ_edge = EDGE_SUCC (bb, 0);
3732 48 : if (gimple_cond_code (cond_stmt) == NE_EXPR)
3733 : {
3734 24 : if (other_succ_edge->flags & EDGE_TRUE_VALUE)
3735 24 : other_succ_edge = EDGE_SUCC (bb, 1);
3736 : }
3737 : else if (other_succ_edge->flags & EDGE_FALSE_VALUE)
3738 48 : other_succ_edge = EDGE_SUCC (bb, 0);
3739 48 : if (other_edge->dest != other_succ_edge->dest)
3740 : return;
3741 : }
3742 140 : else if (!single_succ_p (bb) || other_edge->dest != single_succ (bb))
3743 : return;
3744 311 : gcond *zero_cond = safe_dyn_cast <gcond *> (*gsi_last_bb (pred_bb));
3745 124 : if (zero_cond == NULL
3746 124 : || (gimple_cond_code (zero_cond)
3747 124 : != ((pred_edge->flags & EDGE_TRUE_VALUE) ? NE_EXPR : EQ_EXPR))
3748 124 : || !integer_zerop (gimple_cond_rhs (zero_cond)))
3749 : return;
3750 124 : tree zero_cond_lhs = gimple_cond_lhs (zero_cond);
3751 124 : if (TREE_CODE (zero_cond_lhs) != SSA_NAME)
3752 : return;
3753 124 : if (gimple_assign_rhs2 (div_stmt) != zero_cond_lhs)
3754 : {
3755 : /* Allow the divisor to be result of a same precision cast
3756 : from zero_cond_lhs. */
3757 60 : tree rhs2 = gimple_assign_rhs2 (div_stmt);
3758 60 : if (TREE_CODE (rhs2) != SSA_NAME)
3759 : return;
3760 60 : gimple *g = SSA_NAME_DEF_STMT (rhs2);
3761 60 : if (!gimple_assign_cast_p (g)
3762 53 : || gimple_assign_rhs1 (g) != gimple_cond_lhs (zero_cond)
3763 53 : || !INTEGRAL_TYPE_P (TREE_TYPE (zero_cond_lhs))
3764 113 : || (TYPE_PRECISION (TREE_TYPE (zero_cond_lhs))
3765 53 : != TYPE_PRECISION (TREE_TYPE (rhs2))))
3766 : return;
3767 : }
3768 117 : gimple_stmt_iterator gsi = gsi_after_labels (bb);
3769 117 : mul_stmts.safe_push (div_stmt);
3770 117 : if (is_gimple_debug (gsi_stmt (gsi)))
3771 0 : gsi_next_nondebug (&gsi);
3772 117 : unsigned cast_count = 0;
3773 667 : while (gsi_stmt (gsi) != cond_stmt)
3774 : {
3775 : /* If original mul_stmt has a single use, allow it in the same bb,
3776 : we are looking then just at __builtin_mul_overflow_p.
3777 : Though, in that case the original mul_stmt will be replaced
3778 : by .MUL_OVERFLOW, REALPART_EXPR and IMAGPART_EXPR stmts. */
3779 : gimple *mul_stmt;
3780 : unsigned int i;
3781 2443 : bool ok = false;
3782 2443 : FOR_EACH_VEC_ELT (mul_stmts, i, mul_stmt)
3783 : {
3784 2296 : if (gsi_stmt (gsi) == mul_stmt)
3785 : {
3786 : ok = true;
3787 : break;
3788 : }
3789 : }
3790 550 : if (!ok && gimple_assign_cast_p (gsi_stmt (gsi)) && ++cast_count < 4)
3791 : ok = true;
3792 403 : if (!ok)
3793 63 : return;
3794 550 : gsi_next_nondebug (&gsi);
3795 : }
3796 117 : if (gimple_code (cond_stmt) == GIMPLE_COND)
3797 : {
3798 47 : basic_block succ_bb = other_edge->dest;
3799 75 : for (gphi_iterator gpi = gsi_start_phis (succ_bb); !gsi_end_p (gpi);
3800 28 : gsi_next (&gpi))
3801 : {
3802 35 : gphi *phi = gpi.phi ();
3803 35 : tree v1 = gimple_phi_arg_def (phi, other_edge->dest_idx);
3804 35 : tree v2 = gimple_phi_arg_def (phi, other_succ_edge->dest_idx);
3805 35 : if (!operand_equal_p (v1, v2, 0))
3806 7 : return;
3807 : }
3808 : }
3809 : else
3810 : {
3811 70 : tree lhs = gimple_assign_lhs (cond_stmt);
3812 70 : if (!lhs || !INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
3813 : return;
3814 70 : gsi_next_nondebug (&gsi);
3815 70 : if (!gsi_end_p (gsi))
3816 : {
3817 70 : if (gimple_assign_rhs_code (cond_stmt) == COND_EXPR)
3818 : return;
3819 70 : gimple *cast_stmt = gsi_stmt (gsi);
3820 70 : if (!gimple_assign_cast_p (cast_stmt))
3821 : return;
3822 70 : tree new_lhs = gimple_assign_lhs (cast_stmt);
3823 70 : gsi_next_nondebug (&gsi);
3824 70 : if (!gsi_end_p (gsi)
3825 70 : || !new_lhs
3826 70 : || !INTEGRAL_TYPE_P (TREE_TYPE (new_lhs))
3827 140 : || TYPE_PRECISION (TREE_TYPE (new_lhs)) <= 1)
3828 : return;
3829 : lhs = new_lhs;
3830 : }
3831 70 : edge succ_edge = single_succ_edge (bb);
3832 70 : basic_block succ_bb = succ_edge->dest;
3833 70 : gsi = gsi_start_phis (succ_bb);
3834 70 : if (gsi_end_p (gsi))
3835 : return;
3836 70 : gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
3837 70 : gsi_next (&gsi);
3838 70 : if (!gsi_end_p (gsi))
3839 : return;
3840 70 : if (gimple_phi_arg_def (phi, succ_edge->dest_idx) != lhs)
3841 : return;
3842 70 : tree other_val = gimple_phi_arg_def (phi, other_edge->dest_idx);
3843 70 : if (gimple_assign_rhs_code (cond_stmt) == COND_EXPR)
3844 : {
3845 0 : tree cond = gimple_assign_rhs1 (cond_stmt);
3846 0 : if (TREE_CODE (cond) == NE_EXPR)
3847 : {
3848 0 : if (!operand_equal_p (other_val,
3849 0 : gimple_assign_rhs3 (cond_stmt), 0))
3850 : return;
3851 : }
3852 0 : else if (!operand_equal_p (other_val,
3853 0 : gimple_assign_rhs2 (cond_stmt), 0))
3854 : return;
3855 : }
3856 70 : else if (gimple_assign_rhs_code (cond_stmt) == NE_EXPR)
3857 : {
3858 41 : if (!integer_zerop (other_val))
3859 : return;
3860 : }
3861 29 : else if (!integer_onep (other_val))
3862 : return;
3863 : }
3864 110 : if (pred_edge->flags & EDGE_TRUE_VALUE)
3865 57 : gimple_cond_make_true (zero_cond);
3866 : else
3867 53 : gimple_cond_make_false (zero_cond);
3868 110 : update_stmt (zero_cond);
3869 110 : reset_flow_sensitive_info_in_bb (bb);
3870 110 : *cfg_changed = true;
3871 : }
3872 :
3873 : /* Helper function for arith_overflow_check_p. Return true
3874 : if VAL1 is equal to VAL2 cast to corresponding integral type
3875 : with other signedness or vice versa. */
3876 :
3877 : static bool
3878 394 : arith_cast_equal_p (tree val1, tree val2)
3879 : {
3880 394 : if (TREE_CODE (val1) == INTEGER_CST && TREE_CODE (val2) == INTEGER_CST)
3881 71 : return wi::eq_p (wi::to_wide (val1), wi::to_wide (val2));
3882 323 : else if (TREE_CODE (val1) != SSA_NAME || TREE_CODE (val2) != SSA_NAME)
3883 : return false;
3884 279 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (val1))
3885 279 : && gimple_assign_rhs1 (SSA_NAME_DEF_STMT (val1)) == val2)
3886 : return true;
3887 167 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (val2))
3888 167 : && gimple_assign_rhs1 (SSA_NAME_DEF_STMT (val2)) == val1)
3889 120 : return true;
3890 : return false;
3891 : }
3892 :
3893 : /* Helper function of match_arith_overflow. Return 1
3894 : if USE_STMT is unsigned overflow check ovf != 0 for
3895 : STMT, -1 if USE_STMT is unsigned overflow check ovf == 0
3896 : and 0 otherwise. */
3897 :
3898 : static int
3899 2977906 : arith_overflow_check_p (gimple *stmt, gimple *cast_stmt, gimple *&use_stmt,
3900 : tree maxval, tree *other)
3901 : {
3902 2977906 : enum tree_code ccode = ERROR_MARK;
3903 2977906 : tree crhs1 = NULL_TREE, crhs2 = NULL_TREE;
3904 2977906 : enum tree_code code = gimple_assign_rhs_code (stmt);
3905 5933479 : tree lhs = gimple_assign_lhs (cast_stmt ? cast_stmt : stmt);
3906 2977906 : tree rhs1 = gimple_assign_rhs1 (stmt);
3907 2977906 : tree rhs2 = gimple_assign_rhs2 (stmt);
3908 2977906 : tree multop = NULL_TREE, divlhs = NULL_TREE;
3909 2977906 : gimple *cur_use_stmt = use_stmt;
3910 :
3911 2977906 : if (code == MULT_EXPR)
3912 : {
3913 680668 : if (!is_gimple_assign (use_stmt))
3914 680316 : return 0;
3915 542486 : if (gimple_assign_rhs_code (use_stmt) != TRUNC_DIV_EXPR)
3916 : return 0;
3917 2229 : if (gimple_assign_rhs1 (use_stmt) != lhs)
3918 : return 0;
3919 2166 : if (cast_stmt)
3920 : {
3921 161 : if (arith_cast_equal_p (gimple_assign_rhs2 (use_stmt), rhs1))
3922 : multop = rhs2;
3923 87 : else if (arith_cast_equal_p (gimple_assign_rhs2 (use_stmt), rhs2))
3924 : multop = rhs1;
3925 : else
3926 : return 0;
3927 : }
3928 2005 : else if (gimple_assign_rhs2 (use_stmt) == rhs1)
3929 : multop = rhs2;
3930 1871 : else if (operand_equal_p (gimple_assign_rhs2 (use_stmt), rhs2, 0))
3931 : multop = rhs1;
3932 : else
3933 : return 0;
3934 356 : if (stmt_ends_bb_p (use_stmt))
3935 : return 0;
3936 356 : divlhs = gimple_assign_lhs (use_stmt);
3937 356 : if (!divlhs)
3938 : return 0;
3939 356 : use_operand_p use;
3940 356 : if (!single_imm_use (divlhs, &use, &cur_use_stmt))
3941 : return 0;
3942 352 : if (cast_stmt && gimple_assign_cast_p (cur_use_stmt))
3943 : {
3944 4 : tree cast_lhs = gimple_assign_lhs (cur_use_stmt);
3945 8 : if (INTEGRAL_TYPE_P (TREE_TYPE (cast_lhs))
3946 4 : && TYPE_UNSIGNED (TREE_TYPE (cast_lhs))
3947 4 : && (TYPE_PRECISION (TREE_TYPE (cast_lhs))
3948 4 : == TYPE_PRECISION (TREE_TYPE (divlhs)))
3949 8 : && single_imm_use (cast_lhs, &use, &cur_use_stmt))
3950 : {
3951 : cast_stmt = NULL;
3952 : divlhs = cast_lhs;
3953 : }
3954 : else
3955 : return 0;
3956 : }
3957 : }
3958 2297590 : if (gimple_code (cur_use_stmt) == GIMPLE_COND)
3959 : {
3960 580486 : ccode = gimple_cond_code (cur_use_stmt);
3961 580486 : crhs1 = gimple_cond_lhs (cur_use_stmt);
3962 580486 : crhs2 = gimple_cond_rhs (cur_use_stmt);
3963 : }
3964 1717104 : else if (is_gimple_assign (cur_use_stmt))
3965 : {
3966 810077 : if (gimple_assign_rhs_class (cur_use_stmt) == GIMPLE_BINARY_RHS)
3967 : {
3968 457209 : ccode = gimple_assign_rhs_code (cur_use_stmt);
3969 457209 : crhs1 = gimple_assign_rhs1 (cur_use_stmt);
3970 457209 : crhs2 = gimple_assign_rhs2 (cur_use_stmt);
3971 : }
3972 : else
3973 : return 0;
3974 : }
3975 : else
3976 : return 0;
3977 :
3978 1037695 : if (maxval
3979 1037695 : && ccode == RSHIFT_EXPR
3980 33 : && crhs1 == lhs
3981 17 : && TREE_CODE (crhs2) == INTEGER_CST
3982 1037712 : && wi::to_widest (crhs2) == TYPE_PRECISION (TREE_TYPE (maxval)))
3983 : {
3984 16 : tree shiftlhs = gimple_assign_lhs (use_stmt);
3985 16 : if (!shiftlhs)
3986 : return 0;
3987 16 : use_operand_p use;
3988 16 : if (!single_imm_use (shiftlhs, &use, &cur_use_stmt))
3989 : return 0;
3990 12 : if (gimple_code (cur_use_stmt) == GIMPLE_COND)
3991 : {
3992 0 : ccode = gimple_cond_code (cur_use_stmt);
3993 0 : crhs1 = gimple_cond_lhs (cur_use_stmt);
3994 0 : crhs2 = gimple_cond_rhs (cur_use_stmt);
3995 : }
3996 12 : else if (is_gimple_assign (cur_use_stmt))
3997 : {
3998 12 : if (gimple_assign_rhs_class (cur_use_stmt) == GIMPLE_BINARY_RHS)
3999 : {
4000 0 : ccode = gimple_assign_rhs_code (cur_use_stmt);
4001 0 : crhs1 = gimple_assign_rhs1 (cur_use_stmt);
4002 0 : crhs2 = gimple_assign_rhs2 (cur_use_stmt);
4003 : }
4004 12 : else if (gimple_assign_rhs_code (cur_use_stmt) == COND_EXPR)
4005 : {
4006 0 : tree cond = gimple_assign_rhs1 (cur_use_stmt);
4007 0 : if (COMPARISON_CLASS_P (cond))
4008 : {
4009 0 : ccode = TREE_CODE (cond);
4010 0 : crhs1 = TREE_OPERAND (cond, 0);
4011 0 : crhs2 = TREE_OPERAND (cond, 1);
4012 : }
4013 : else
4014 : return 0;
4015 : }
4016 : else
4017 : {
4018 12 : enum tree_code sc = gimple_assign_rhs_code (cur_use_stmt);
4019 12 : tree castlhs = gimple_assign_lhs (cur_use_stmt);
4020 12 : if (!CONVERT_EXPR_CODE_P (sc)
4021 12 : || !castlhs
4022 12 : || !INTEGRAL_TYPE_P (TREE_TYPE (castlhs))
4023 24 : || (TYPE_PRECISION (TREE_TYPE (castlhs))
4024 12 : > TYPE_PRECISION (TREE_TYPE (maxval))))
4025 0 : return 0;
4026 : return 1;
4027 : }
4028 : }
4029 : else
4030 : return 0;
4031 0 : if ((ccode != EQ_EXPR && ccode != NE_EXPR)
4032 0 : || crhs1 != shiftlhs
4033 0 : || !integer_zerop (crhs2))
4034 0 : return 0;
4035 : return 1;
4036 : }
4037 :
4038 1037679 : if (TREE_CODE_CLASS (ccode) != tcc_comparison)
4039 : return 0;
4040 :
4041 617183 : switch (ccode)
4042 : {
4043 114697 : case GT_EXPR:
4044 114697 : case LE_EXPR:
4045 114697 : if (maxval)
4046 : {
4047 : /* r = a + b; r > maxval or r <= maxval */
4048 40 : if (crhs1 == lhs
4049 39 : && TREE_CODE (crhs2) == INTEGER_CST
4050 57 : && tree_int_cst_equal (crhs2, maxval))
4051 13 : return ccode == GT_EXPR ? 1 : -1;
4052 : break;
4053 : }
4054 : /* r = a - b; r > a or r <= a
4055 : r = a + b; a > r or a <= r or b > r or b <= r. */
4056 114657 : if ((code == MINUS_EXPR && crhs1 == lhs && crhs2 == rhs1)
4057 114593 : || (code == PLUS_EXPR && (crhs1 == rhs1 || crhs1 == rhs2)
4058 7975 : && crhs2 == lhs))
4059 8039 : return ccode == GT_EXPR ? 1 : -1;
4060 : /* r = ~a; b > r or b <= r. */
4061 106618 : if (code == BIT_NOT_EXPR && crhs2 == lhs)
4062 : {
4063 232 : if (other)
4064 116 : *other = crhs1;
4065 232 : return ccode == GT_EXPR ? 1 : -1;
4066 : }
4067 : break;
4068 62499 : case LT_EXPR:
4069 62499 : case GE_EXPR:
4070 62499 : if (maxval)
4071 : break;
4072 : /* r = a - b; a < r or a >= r
4073 : r = a + b; r < a or r >= a or r < b or r >= b. */
4074 62493 : if ((code == MINUS_EXPR && crhs1 == rhs1 && crhs2 == lhs)
4075 62355 : || (code == PLUS_EXPR && crhs1 == lhs
4076 30157 : && (crhs2 == rhs1 || crhs2 == rhs2)))
4077 4125 : return ccode == LT_EXPR ? 1 : -1;
4078 : /* r = ~a; r < b or r >= b. */
4079 58368 : if (code == BIT_NOT_EXPR && crhs1 == lhs)
4080 : {
4081 263 : if (other)
4082 140 : *other = crhs2;
4083 263 : return ccode == LT_EXPR ? 1 : -1;
4084 : }
4085 : break;
4086 439987 : case EQ_EXPR:
4087 439987 : case NE_EXPR:
4088 : /* r = a * b; _1 = r / a; _1 == b
4089 : r = a * b; _1 = r / b; _1 == a
4090 : r = a * b; _1 = r / a; _1 != b
4091 : r = a * b; _1 = r / b; _1 != a. */
4092 439987 : if (code == MULT_EXPR)
4093 : {
4094 349 : if (cast_stmt)
4095 : {
4096 146 : if ((crhs1 == divlhs && arith_cast_equal_p (crhs2, multop))
4097 146 : || (crhs2 == divlhs && arith_cast_equal_p (crhs1, multop)))
4098 : {
4099 146 : use_stmt = cur_use_stmt;
4100 146 : return ccode == NE_EXPR ? 1 : -1;
4101 : }
4102 : }
4103 142 : else if ((crhs1 == divlhs && operand_equal_p (crhs2, multop, 0))
4104 203 : || (crhs2 == divlhs && crhs1 == multop))
4105 : {
4106 203 : use_stmt = cur_use_stmt;
4107 203 : return ccode == NE_EXPR ? 1 : -1;
4108 : }
4109 : }
4110 : break;
4111 : default:
4112 : break;
4113 : }
4114 : return 0;
4115 : }
4116 :
4117 : extern bool gimple_unsigned_integer_sat_add (tree, tree*, tree (*)(tree));
4118 : extern bool gimple_unsigned_integer_sat_sub (tree, tree*, tree (*)(tree));
4119 : extern bool gimple_unsigned_integer_sat_trunc (tree, tree*, tree (*)(tree));
4120 : extern bool gimple_unsigned_integer_sat_mul (tree, tree*, tree (*)(tree));
4121 : extern bool gimple_spaceship (tree, tree*, tree (*)(tree));
4122 :
4123 : extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree));
4124 : extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
4125 : extern bool gimple_signed_integer_sat_trunc (tree, tree*, tree (*)(tree));
4126 :
4127 : static bool
4128 351 : build_saturation_binary_arith_call_and_replace (gimple_stmt_iterator *gsi,
4129 : internal_fn fn, tree lhs,
4130 : tree op_0, tree op_1)
4131 : {
4132 351 : if (direct_internal_fn_supported_p (fn, TREE_TYPE (op_0), OPTIMIZE_FOR_BOTH))
4133 : {
4134 335 : gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
4135 335 : gimple_call_set_lhs (call, lhs);
4136 335 : gsi_replace (gsi, call, /* update_eh_info */ true);
4137 335 : return true;
4138 : }
4139 :
4140 : return false;
4141 : }
4142 :
4143 : static bool
4144 87 : build_saturation_binary_arith_call_and_insert (gimple_stmt_iterator *gsi,
4145 : internal_fn fn, tree lhs,
4146 : tree op_0, tree op_1)
4147 : {
4148 87 : if (!direct_internal_fn_supported_p (fn, TREE_TYPE (op_0), OPTIMIZE_FOR_BOTH))
4149 : return false;
4150 :
4151 67 : gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
4152 67 : gimple_call_set_lhs (call, lhs);
4153 67 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4154 :
4155 67 : return true;
4156 : }
4157 :
4158 : /*
4159 : * Try to match saturation unsigned add with assign.
4160 : * _7 = _4 + _6;
4161 : * _8 = _4 > _7;
4162 : * _9 = (long unsigned int) _8;
4163 : * _10 = -_9;
4164 : * _12 = _7 | _10;
4165 : * =>
4166 : * _12 = .SAT_ADD (_4, _6);
4167 : *
4168 : * Try to match IMM=-1 saturation signed add with assign.
4169 : * <bb 2> [local count: 1073741824]:
4170 : * x.0_1 = (unsigned char) x_5(D);
4171 : * _3 = -x.0_1;
4172 : * _10 = (signed char) _3;
4173 : * _8 = x_5(D) & _10;
4174 : * if (_8 < 0)
4175 : * goto <bb 4>; [1.40%]
4176 : * else
4177 : * goto <bb 3>; [98.60%]
4178 : * <bb 3> [local count: 434070867]:
4179 : * _2 = x.0_1 + 255;
4180 : * <bb 4> [local count: 1073741824]:
4181 : * # _9 = PHI <_2(3), 128(2)>
4182 : * _4 = (int8_t) _9;
4183 : * =>
4184 : * _4 = .SAT_ADD (x_5, -1);
4185 : * Return true if the statement was replaced. */
4186 :
4187 : static bool
4188 4990225 : match_saturation_add_with_assign (gimple_stmt_iterator *gsi, gassign *stmt)
4189 : {
4190 4990225 : tree ops[2];
4191 4990225 : tree lhs = gimple_assign_lhs (stmt);
4192 :
4193 4990225 : if (gimple_unsigned_integer_sat_add (lhs, ops, NULL)
4194 4990225 : || gimple_signed_integer_sat_add (lhs, ops, NULL))
4195 : {
4196 : /* .SAT_ADD saturates in the type of its first operand, which must be
4197 : the type of the result. */
4198 208 : gcc_checking_assert (types_compatible_p (TREE_TYPE (lhs),
4199 : TREE_TYPE (ops[0])));
4200 208 : return build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_ADD,
4201 : lhs, ops[0],
4202 208 : ops[1]);
4203 : }
4204 :
4205 : return false;
4206 : }
4207 :
4208 : /*
4209 : * Try to match saturation add with PHI.
4210 : * For unsigned integer:
4211 : * <bb 2> :
4212 : * _1 = x_3(D) + y_4(D);
4213 : * if (_1 >= x_3(D))
4214 : * goto <bb 3>; [INV]
4215 : * else
4216 : * goto <bb 4>; [INV]
4217 : *
4218 : * <bb 3> :
4219 : *
4220 : * <bb 4> :
4221 : * # _2 = PHI <255(2), _1(3)>
4222 : * =>
4223 : * <bb 4> [local count: 1073741824]:
4224 : * _2 = .SAT_ADD (x_4(D), y_5(D));
4225 : *
4226 : * For signed integer:
4227 : * x.0_1 = (long unsigned int) x_7(D);
4228 : * y.1_2 = (long unsigned int) y_8(D);
4229 : * _3 = x.0_1 + y.1_2;
4230 : * sum_9 = (int64_t) _3;
4231 : * _4 = x_7(D) ^ y_8(D);
4232 : * _5 = x_7(D) ^ sum_9;
4233 : * _15 = ~_4;
4234 : * _16 = _5 & _15;
4235 : * if (_16 < 0)
4236 : * goto <bb 3>; [41.00%]
4237 : * else
4238 : * goto <bb 4>; [59.00%]
4239 : * _11 = x_7(D) < 0;
4240 : * _12 = (long int) _11;
4241 : * _13 = -_12;
4242 : * _14 = _13 ^ 9223372036854775807;
4243 : * # _6 = PHI <_14(3), sum_9(2)>
4244 : * =>
4245 : * _6 = .SAT_ADD (x_5(D), y_6(D)); [tail call] */
4246 :
4247 : static bool
4248 4352045 : match_saturation_add (gimple_stmt_iterator *gsi, gphi *phi)
4249 : {
4250 4352045 : if (gimple_phi_num_args (phi) != 2)
4251 : return false;
4252 :
4253 3458684 : tree ops[2];
4254 3458684 : tree phi_result = gimple_phi_result (phi);
4255 :
4256 3458684 : if (!gimple_unsigned_integer_sat_add (phi_result, ops, NULL)
4257 3458684 : && !gimple_signed_integer_sat_add (phi_result, ops, NULL))
4258 : return false;
4259 :
4260 : /* .SAT_ADD saturates in the type of its first operand, which must be the
4261 : type of the result. */
4262 35 : gcc_checking_assert (types_compatible_p (TREE_TYPE (phi_result),
4263 : TREE_TYPE (ops[0])));
4264 :
4265 35 : if (!TYPE_UNSIGNED (TREE_TYPE (ops[0])) && TREE_CODE (ops[1]) == INTEGER_CST)
4266 12 : ops[1] = fold_convert (TREE_TYPE (ops[0]), ops[1]);
4267 :
4268 35 : return build_saturation_binary_arith_call_and_insert (gsi, IFN_SAT_ADD,
4269 : phi_result, ops[0],
4270 35 : ops[1]);
4271 : }
4272 :
4273 : /*
4274 : * Try to match saturation sub with assign.
4275 : * _1 = _4 >= _5;
4276 : * _3 = _4 - _5;
4277 : * _6 = _1 ? _3 : 0;
4278 : * =>
4279 : * _6 = .SAT_SUB (_4, _5);
4280 : * Return true if the statement was replaced. */
4281 :
4282 : static bool
4283 6214046 : match_saturation_sub_with_assign (gimple_stmt_iterator *gsi, gassign *stmt)
4284 : {
4285 6214046 : tree ops[2];
4286 6214046 : tree lhs = gimple_assign_lhs (stmt);
4287 :
4288 6214046 : if (gimple_unsigned_integer_sat_sub (lhs, ops, NULL)
4289 6214046 : || gimple_signed_integer_sat_sub (lhs, ops, NULL))
4290 143 : return build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_SUB,
4291 143 : lhs, ops[0], ops[1]);
4292 :
4293 : return false;
4294 : }
4295 :
4296 : /*
4297 : * Try to match saturation unsigned mul.
4298 : * _1 = (unsigned int) a_6(D);
4299 : * _2 = (unsigned int) b_7(D);
4300 : * x_8 = _1 * _2;
4301 : * overflow_9 = x_8 > 255;
4302 : * _3 = (unsigned char) overflow_9;
4303 : * _4 = -_3;
4304 : * _5 = (unsigned char) x_8;
4305 : * _10 = _4 | _5;
4306 : * =>
4307 : * _10 = .SAT_SUB (a_6, b_7); */
4308 :
4309 : static void
4310 2653679 : match_unsigned_saturation_mul (gimple_stmt_iterator *gsi, gassign *stmt)
4311 : {
4312 2653679 : tree ops[2];
4313 2653679 : tree lhs = gimple_assign_lhs (stmt);
4314 :
4315 2653679 : if (gimple_unsigned_integer_sat_mul (lhs, ops, NULL))
4316 0 : build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_MUL, lhs,
4317 : ops[0], ops[1]);
4318 2653679 : }
4319 :
4320 : /* Try to match saturation unsigned mul, aka:
4321 : _6 = .MUL_OVERFLOW (a_4(D), b_5(D));
4322 : _2 = IMAGPART_EXPR <_6>;
4323 : if (_2 != 0)
4324 : goto <bb 4>; [35.00%]
4325 : else
4326 : goto <bb 3>; [65.00%]
4327 :
4328 : <bb 3> [local count: 697932184]:
4329 : _1 = REALPART_EXPR <_6>;
4330 :
4331 : <bb 4> [local count: 1073741824]:
4332 : # _3 = PHI <18446744073709551615(2), _1(3)>
4333 : =>
4334 : _3 = .SAT_MUL (a_4(D), b_5(D)); */
4335 :
4336 : static bool
4337 4351978 : match_saturation_mul (gimple_stmt_iterator *gsi, gphi *phi)
4338 : {
4339 4351978 : if (gimple_phi_num_args (phi) != 2)
4340 : return false;
4341 :
4342 3458617 : tree ops[2];
4343 3458617 : tree phi_result = gimple_phi_result (phi);
4344 :
4345 3458617 : if (!gimple_unsigned_integer_sat_mul (phi_result, ops, NULL))
4346 : return false;
4347 :
4348 0 : return build_saturation_binary_arith_call_and_insert (gsi, IFN_SAT_MUL,
4349 : phi_result, ops[0],
4350 0 : ops[1]);
4351 : }
4352 :
4353 : /* Try to match variants of spaceship operation:
4354 : <bb 2>
4355 : if (a_3(D) >= b_4(D)) -- CMP_1
4356 : goto <bb 3>;
4357 : else
4358 : goto <bb 4>;
4359 :
4360 : <bb 3>
4361 : _1 = a_3(D) > b_4(D); -- CMP_2
4362 : _5 = (int) _1;
4363 :
4364 : <bb 4>
4365 : # _2 = PHI <-1(2), _5(3)>
4366 : =>
4367 : _2 = .SPACESHIP (a_3(D), b_4(D), -1);
4368 :
4369 : All possible canonical variants of the comparison operator in CMP_1 and
4370 : CMP_2 has been included in gimple_spaceship function. */
4371 : static bool
4372 4351978 : match_spaceship (gimple_stmt_iterator *gsi, gphi *phi)
4373 : {
4374 4351978 : if (gimple_phi_num_args (phi) != 2)
4375 : return false;
4376 3458617 : tree ops[2];
4377 3458617 : tree phi_result = gimple_phi_result (phi);
4378 :
4379 3458617 : if (!gimple_spaceship (phi_result, ops, NULL))
4380 : return false;
4381 :
4382 : /* Allow different modes as long as both are integral types. */
4383 224 : if (!INTEGRAL_TYPE_P (TREE_TYPE (phi_result))
4384 224 : || !INTEGRAL_TYPE_P (TREE_TYPE (ops[0])))
4385 : return false;
4386 :
4387 112 : tree ops_type = TREE_TYPE (ops[0]);
4388 112 : machine_mode ops_mode = TYPE_MODE (ops_type);
4389 112 : machine_mode promoted_mode = ops_mode;
4390 112 : tree promoted_type = ops_type;
4391 112 : bool is_unsigned = TYPE_UNSIGNED (ops_type);
4392 :
4393 : /* Check if spaceship optab is available for the operand mode.
4394 : If not, try promoting to a wider mode that is supported. */
4395 112 : if (optab_handler (spaceship_optab, ops_mode) == CODE_FOR_nothing)
4396 : {
4397 : /* Try promoting to wider modes (e.g., QI/HI -> SI -> DI). */
4398 : machine_mode wider_mode;
4399 0 : FOR_EACH_WIDER_MODE_FROM (wider_mode, ops_mode)
4400 : {
4401 0 : if (optab_handler (spaceship_optab, wider_mode)
4402 : != CODE_FOR_nothing)
4403 : {
4404 : /* Check if we can get a type for this mode with matching
4405 : signedness. */
4406 0 : promoted_type = lang_hooks.types.type_for_mode (wider_mode,
4407 : is_unsigned);
4408 0 : if (promoted_type != NULL_TREE && INTEGRAL_TYPE_P (promoted_type))
4409 : {
4410 : promoted_mode = wider_mode;
4411 : break;
4412 : }
4413 : }
4414 : }
4415 :
4416 : // If no suitable promoted mode found, give up.
4417 0 : if (promoted_mode == ops_mode)
4418 4351978 : return false;
4419 : }
4420 :
4421 : /* If promotion is needed, insert conversion statements.
4422 : We must use GIMPLE assignments rather than fold_convert because
4423 : gimple_call arguments must be valid GIMPLE values (SSA names or
4424 : constants), not tree expressions. */
4425 112 : ops[0] = gimple_convert (gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
4426 : promoted_type, ops[0]);
4427 112 : ops[1] = gimple_convert (gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
4428 : promoted_type, ops[1]);
4429 :
4430 112 : tree spaceship_arg_3 = is_unsigned ? build_one_cst (integer_type_node)
4431 96 : : build_minus_one_cst (integer_type_node);
4432 :
4433 112 : gcall *call = gimple_build_call_internal (IFN_SPACESHIP, 3, ops[0], ops[1],
4434 : spaceship_arg_3);
4435 :
4436 : /* SPACESHIP optab always returns signed int (SI mode).
4437 : Cast to phi_result's type if needed. */
4438 112 : tree call_result_type = integer_type_node;
4439 112 : if (!types_compatible_p (TREE_TYPE (phi_result), call_result_type))
4440 : {
4441 48 : tree call_result = make_ssa_name (call_result_type);
4442 48 : gimple_call_set_lhs (call, call_result);
4443 48 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4444 48 : gassign *cast_stmt = gimple_build_assign (phi_result, NOP_EXPR,
4445 : call_result);
4446 48 : gsi_insert_before (gsi, cast_stmt, GSI_SAME_STMT);
4447 : }
4448 : else
4449 : {
4450 64 : gimple_call_set_lhs (call, phi_result);
4451 64 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4452 : }
4453 : return true;
4454 : }
4455 :
4456 :
4457 : /*
4458 : * Try to match saturation unsigned sub.
4459 : * <bb 2> [local count: 1073741824]:
4460 : * if (x_2(D) > y_3(D))
4461 : * goto <bb 3>; [50.00%]
4462 : * else
4463 : * goto <bb 4>; [50.00%]
4464 : *
4465 : * <bb 3> [local count: 536870912]:
4466 : * _4 = x_2(D) - y_3(D);
4467 : *
4468 : * <bb 4> [local count: 1073741824]:
4469 : * # _1 = PHI <0(2), _4(3)>
4470 : * =>
4471 : * <bb 4> [local count: 1073741824]:
4472 : * _1 = .SAT_SUB (x_2(D), y_3(D)); */
4473 : static bool
4474 4352026 : match_saturation_sub (gimple_stmt_iterator *gsi, gphi *phi)
4475 : {
4476 4352026 : if (gimple_phi_num_args (phi) != 2)
4477 : return false;
4478 :
4479 3458665 : tree ops[2];
4480 3458665 : tree phi_result = gimple_phi_result (phi);
4481 :
4482 3458665 : if (!gimple_unsigned_integer_sat_sub (phi_result, ops, NULL)
4483 3458665 : && !gimple_signed_integer_sat_sub (phi_result, ops, NULL))
4484 : return false;
4485 :
4486 52 : return build_saturation_binary_arith_call_and_insert (gsi, IFN_SAT_SUB,
4487 : phi_result, ops[0],
4488 52 : ops[1]);
4489 : }
4490 :
4491 : /*
4492 : * Try to match saturation unsigned sub.
4493 : * uint16_t x_4(D);
4494 : * uint8_t _6;
4495 : * overflow_5 = x_4(D) > 255;
4496 : * _1 = (unsigned char) x_4(D);
4497 : * _2 = (unsigned char) overflow_5;
4498 : * _3 = -_2;
4499 : * _6 = _1 | _3;
4500 : * =>
4501 : * _6 = .SAT_TRUNC (x_4(D));
4502 : * */
4503 : static void
4504 2653679 : match_unsigned_saturation_trunc (gimple_stmt_iterator *gsi, gassign *stmt)
4505 : {
4506 2653679 : tree ops[1];
4507 2653679 : tree lhs = gimple_assign_lhs (stmt);
4508 2653679 : tree type = TREE_TYPE (lhs);
4509 :
4510 2653679 : if (gimple_unsigned_integer_sat_trunc (lhs, ops, NULL)
4511 2653814 : && direct_internal_fn_supported_p (IFN_SAT_TRUNC,
4512 135 : tree_pair (type, TREE_TYPE (ops[0])),
4513 : OPTIMIZE_FOR_BOTH))
4514 : {
4515 108 : gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[0]);
4516 108 : gimple_call_set_lhs (call, lhs);
4517 108 : gsi_replace (gsi, call, /* update_eh_info */ true);
4518 : }
4519 2653679 : }
4520 :
4521 : /*
4522 : * Try to match saturation truncate.
4523 : * Aka:
4524 : * x.0_1 = (unsigned long) x_4(D);
4525 : * _2 = x.0_1 + 2147483648;
4526 : * if (_2 > 4294967295)
4527 : * goto <bb 4>; [50.00%]
4528 : * else
4529 : * goto <bb 3>; [50.00%]
4530 : * ;; succ: 4
4531 : * ;; 3
4532 : *
4533 : * ;; basic block 3, loop depth 0
4534 : * ;; pred: 2
4535 : * trunc_5 = (int32_t) x_4(D);
4536 : * goto <bb 5>; [100.00%]
4537 : * ;; succ: 5
4538 : *
4539 : * ;; basic block 4, loop depth 0
4540 : * ;; pred: 2
4541 : * _7 = x_4(D) < 0;
4542 : * _8 = (int) _7;
4543 : * _9 = -_8;
4544 : * _10 = _9 ^ 2147483647;
4545 : * ;; succ: 5
4546 : *
4547 : * ;; basic block 5, loop depth 0
4548 : * ;; pred: 3
4549 : * ;; 4
4550 : * # _3 = PHI <trunc_5(3), _10(4)>
4551 : * =>
4552 : * _6 = .SAT_TRUNC (x_4(D));
4553 : */
4554 :
4555 : static bool
4556 4351978 : match_saturation_trunc (gimple_stmt_iterator *gsi, gphi *phi)
4557 : {
4558 4351978 : if (gimple_phi_num_args (phi) != 2)
4559 : return false;
4560 :
4561 3458617 : tree ops[1];
4562 3458617 : tree phi_result = gimple_phi_result (phi);
4563 3458617 : tree type = TREE_TYPE (phi_result);
4564 :
4565 3458617 : if (!gimple_unsigned_integer_sat_trunc (phi_result, ops, NULL)
4566 3458617 : && !gimple_signed_integer_sat_trunc (phi_result, ops, NULL))
4567 : return false;
4568 :
4569 99 : if (!direct_internal_fn_supported_p (IFN_SAT_TRUNC,
4570 99 : tree_pair (type, TREE_TYPE (ops[0])),
4571 : OPTIMIZE_FOR_BOTH))
4572 : return false;
4573 :
4574 0 : gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[0]);
4575 0 : gimple_call_set_lhs (call, phi_result);
4576 0 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4577 :
4578 0 : return true;
4579 : }
4580 :
4581 : /* Recognize for unsigned x
4582 : x = y - z;
4583 : if (x > y)
4584 : where there are other uses of x and replace it with
4585 : _7 = .SUB_OVERFLOW (y, z);
4586 : x = REALPART_EXPR <_7>;
4587 : _8 = IMAGPART_EXPR <_7>;
4588 : if (_8)
4589 : and similarly for addition.
4590 :
4591 : Also recognize:
4592 : yc = (type) y;
4593 : zc = (type) z;
4594 : x = yc + zc;
4595 : if (x > max)
4596 : where y and z have unsigned types with maximum max
4597 : and there are other uses of x and all of those cast x
4598 : back to that unsigned type and again replace it with
4599 : _7 = .ADD_OVERFLOW (y, z);
4600 : _9 = REALPART_EXPR <_7>;
4601 : _8 = IMAGPART_EXPR <_7>;
4602 : if (_8)
4603 : and replace (utype) x with _9.
4604 : Or with x >> popcount (max) instead of x > max.
4605 :
4606 : Also recognize:
4607 : x = ~z;
4608 : if (y > x)
4609 : and replace it with
4610 : _7 = .ADD_OVERFLOW (y, z);
4611 : _8 = IMAGPART_EXPR <_7>;
4612 : if (_8)
4613 :
4614 : And also recognize:
4615 : z = x * y;
4616 : if (x != 0)
4617 : goto <bb 3>; [50.00%]
4618 : else
4619 : goto <bb 4>; [50.00%]
4620 :
4621 : <bb 3> [local count: 536870913]:
4622 : _2 = z / x;
4623 : _9 = _2 != y;
4624 : _10 = (int) _9;
4625 :
4626 : <bb 4> [local count: 1073741824]:
4627 : # iftmp.0_3 = PHI <_10(3), 0(2)>
4628 : and replace it with
4629 : _7 = .MUL_OVERFLOW (x, y);
4630 : z = IMAGPART_EXPR <_7>;
4631 : _8 = IMAGPART_EXPR <_7>;
4632 : _9 = _8 != 0;
4633 : iftmp.0_3 = (int) _9; */
4634 :
4635 : static bool
4636 3428837 : match_arith_overflow (gimple_stmt_iterator *gsi, gimple *stmt,
4637 : enum tree_code code, bool *cfg_changed)
4638 : {
4639 3428837 : tree lhs = gimple_assign_lhs (stmt);
4640 3428837 : tree type = TREE_TYPE (lhs);
4641 3428837 : use_operand_p use_p;
4642 3428837 : imm_use_iterator iter;
4643 3428837 : bool use_seen = false;
4644 3428837 : bool ovf_use_seen = false;
4645 3428837 : gimple *use_stmt;
4646 3428837 : gimple *add_stmt = NULL;
4647 3428837 : bool add_first = false;
4648 3428837 : gimple *cond_stmt = NULL;
4649 3428837 : gimple *cast_stmt = NULL;
4650 3428837 : tree cast_lhs = NULL_TREE;
4651 :
4652 3428837 : gcc_checking_assert (code == PLUS_EXPR
4653 : || code == MINUS_EXPR
4654 : || code == MULT_EXPR
4655 : || code == BIT_NOT_EXPR);
4656 3428837 : if (!INTEGRAL_TYPE_P (type)
4657 2903211 : || !TYPE_UNSIGNED (type)
4658 1998356 : || has_zero_uses (lhs)
4659 3428837 : || (code != PLUS_EXPR
4660 1997842 : && code != MULT_EXPR
4661 174420 : && optab_handler (code == MINUS_EXPR ? usubv4_optab : uaddv4_optab,
4662 148225 : TYPE_MODE (type)) == CODE_FOR_nothing))
4663 : return false;
4664 :
4665 1996000 : tree rhs1 = gimple_assign_rhs1 (stmt);
4666 1996000 : tree rhs2 = gimple_assign_rhs2 (stmt);
4667 5530872 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
4668 : {
4669 3540855 : use_stmt = USE_STMT (use_p);
4670 3540855 : if (is_gimple_debug (use_stmt))
4671 603700 : continue;
4672 :
4673 2937155 : tree other = NULL_TREE;
4674 2937155 : if (arith_overflow_check_p (stmt, NULL, use_stmt, NULL_TREE, &other))
4675 : {
4676 6558 : if (code == BIT_NOT_EXPR)
4677 : {
4678 256 : gcc_assert (other);
4679 256 : if (TREE_CODE (other) != SSA_NAME)
4680 0 : return false;
4681 256 : if (rhs2 == NULL)
4682 256 : rhs2 = other;
4683 : else
4684 : return false;
4685 256 : cond_stmt = use_stmt;
4686 : }
4687 : ovf_use_seen = true;
4688 : }
4689 : else
4690 : {
4691 2930597 : use_seen = true;
4692 2930597 : if (code == MULT_EXPR
4693 2930597 : && cast_stmt == NULL
4694 2930597 : && gimple_assign_cast_p (use_stmt))
4695 : {
4696 23746 : cast_lhs = gimple_assign_lhs (use_stmt);
4697 47492 : if (INTEGRAL_TYPE_P (TREE_TYPE (cast_lhs))
4698 23198 : && !TYPE_UNSIGNED (TREE_TYPE (cast_lhs))
4699 41890 : && (TYPE_PRECISION (TREE_TYPE (cast_lhs))
4700 18144 : == TYPE_PRECISION (TREE_TYPE (lhs))))
4701 : cast_stmt = use_stmt;
4702 : else
4703 : cast_lhs = NULL_TREE;
4704 : }
4705 : }
4706 2937155 : if (ovf_use_seen && use_seen)
4707 : break;
4708 0 : }
4709 :
4710 1996000 : if (!ovf_use_seen
4711 1996000 : && code == MULT_EXPR
4712 461452 : && cast_stmt)
4713 : {
4714 17739 : if (TREE_CODE (rhs1) != SSA_NAME
4715 17739 : || (TREE_CODE (rhs2) != SSA_NAME && TREE_CODE (rhs2) != INTEGER_CST))
4716 : return false;
4717 41723 : FOR_EACH_IMM_USE_FAST (use_p, iter, cast_lhs)
4718 : {
4719 23984 : use_stmt = USE_STMT (use_p);
4720 23984 : if (is_gimple_debug (use_stmt))
4721 1748 : continue;
4722 :
4723 22236 : if (arith_overflow_check_p (stmt, cast_stmt, use_stmt,
4724 : NULL_TREE, NULL))
4725 23984 : ovf_use_seen = true;
4726 17739 : }
4727 17739 : }
4728 : else
4729 : {
4730 : cast_stmt = NULL;
4731 : cast_lhs = NULL_TREE;
4732 : }
4733 :
4734 1996000 : tree maxval = NULL_TREE;
4735 1996000 : if (!ovf_use_seen
4736 13069 : || (code != MULT_EXPR && (code == BIT_NOT_EXPR ? use_seen : !use_seen))
4737 6170 : || (code == PLUS_EXPR
4738 5831 : && optab_handler (uaddv4_optab,
4739 5831 : TYPE_MODE (type)) == CODE_FOR_nothing)
4740 2008782 : || (code == MULT_EXPR
4741 235 : && optab_handler (cast_stmt ? mulv4_optab : umulv4_optab,
4742 155 : TYPE_MODE (type)) == CODE_FOR_nothing
4743 3 : && (use_seen
4744 3 : || cast_stmt
4745 0 : || !can_mult_highpart_p (TYPE_MODE (type), true))))
4746 : {
4747 1989678 : if (code != PLUS_EXPR)
4748 : return false;
4749 1382254 : if (TREE_CODE (rhs1) != SSA_NAME
4750 1382254 : || !gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs1)))
4751 : return false;
4752 325310 : rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs1));
4753 325310 : tree type1 = TREE_TYPE (rhs1);
4754 325310 : if (!INTEGRAL_TYPE_P (type1)
4755 169590 : || !TYPE_UNSIGNED (type1)
4756 32235 : || TYPE_PRECISION (type1) >= TYPE_PRECISION (type)
4757 339337 : || (TYPE_PRECISION (type1)
4758 339337 : != GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (type1))))
4759 : return false;
4760 9326 : if (TREE_CODE (rhs2) == INTEGER_CST)
4761 : {
4762 3804 : if (wi::ne_p (wi::rshift (wi::to_wide (rhs2),
4763 3804 : TYPE_PRECISION (type1),
4764 7608 : UNSIGNED), 0))
4765 : return false;
4766 1273 : rhs2 = fold_convert (type1, rhs2);
4767 : }
4768 : else
4769 : {
4770 5522 : if (TREE_CODE (rhs2) != SSA_NAME
4771 5522 : || !gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs2)))
4772 : return false;
4773 2731 : rhs2 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs2));
4774 2731 : tree type2 = TREE_TYPE (rhs2);
4775 2731 : if (!INTEGRAL_TYPE_P (type2)
4776 1222 : || !TYPE_UNSIGNED (type2)
4777 515 : || TYPE_PRECISION (type2) >= TYPE_PRECISION (type)
4778 3218 : || (TYPE_PRECISION (type2)
4779 3218 : != GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (type2))))
4780 : return false;
4781 : }
4782 1747 : if (TYPE_PRECISION (type1) >= TYPE_PRECISION (TREE_TYPE (rhs2)))
4783 : type = type1;
4784 : else
4785 3 : type = TREE_TYPE (rhs2);
4786 :
4787 1747 : if (TREE_CODE (type) != INTEGER_TYPE
4788 3494 : || optab_handler (uaddv4_optab,
4789 1747 : TYPE_MODE (type)) == CODE_FOR_nothing)
4790 : return false;
4791 :
4792 1747 : maxval = wide_int_to_tree (type, wi::max_value (TYPE_PRECISION (type),
4793 : UNSIGNED));
4794 1747 : ovf_use_seen = false;
4795 1747 : use_seen = false;
4796 1747 : basic_block use_bb = NULL;
4797 1950 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
4798 : {
4799 1889 : use_stmt = USE_STMT (use_p);
4800 1889 : if (is_gimple_debug (use_stmt))
4801 138 : continue;
4802 :
4803 1751 : if (arith_overflow_check_p (stmt, NULL, use_stmt, maxval, NULL))
4804 : {
4805 13 : ovf_use_seen = true;
4806 13 : use_bb = gimple_bb (use_stmt);
4807 : }
4808 : else
4809 : {
4810 1738 : if (!gimple_assign_cast_p (use_stmt)
4811 1738 : || gimple_assign_rhs_code (use_stmt) == VIEW_CONVERT_EXPR)
4812 : return false;
4813 113 : tree use_lhs = gimple_assign_lhs (use_stmt);
4814 226 : if (!INTEGRAL_TYPE_P (TREE_TYPE (use_lhs))
4815 226 : || (TYPE_PRECISION (TREE_TYPE (use_lhs))
4816 113 : > TYPE_PRECISION (type)))
4817 : return false;
4818 : use_seen = true;
4819 : }
4820 1686 : }
4821 61 : if (!ovf_use_seen)
4822 : return false;
4823 13 : if (!useless_type_conversion_p (type, TREE_TYPE (rhs1)))
4824 : {
4825 2 : if (!use_seen)
4826 : return false;
4827 2 : tree new_rhs1 = make_ssa_name (type);
4828 2 : gimple *g = gimple_build_assign (new_rhs1, NOP_EXPR, rhs1);
4829 2 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4830 2 : rhs1 = new_rhs1;
4831 : }
4832 11 : else if (!useless_type_conversion_p (type, TREE_TYPE (rhs2)))
4833 : {
4834 2 : if (!use_seen)
4835 : return false;
4836 2 : tree new_rhs2 = make_ssa_name (type);
4837 2 : gimple *g = gimple_build_assign (new_rhs2, NOP_EXPR, rhs2);
4838 2 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4839 2 : rhs2 = new_rhs2;
4840 : }
4841 9 : else if (!use_seen)
4842 : {
4843 : /* If there are no uses of the wider addition, check if
4844 : forwprop has not created a narrower addition.
4845 : Require it to be in the same bb as the overflow check. */
4846 12 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs1)
4847 : {
4848 11 : use_stmt = USE_STMT (use_p);
4849 11 : if (is_gimple_debug (use_stmt))
4850 0 : continue;
4851 :
4852 11 : if (use_stmt == stmt)
4853 0 : continue;
4854 :
4855 11 : if (!is_gimple_assign (use_stmt)
4856 11 : || gimple_bb (use_stmt) != use_bb
4857 22 : || gimple_assign_rhs_code (use_stmt) != PLUS_EXPR)
4858 3 : continue;
4859 :
4860 8 : if (gimple_assign_rhs1 (use_stmt) == rhs1)
4861 : {
4862 8 : if (!operand_equal_p (gimple_assign_rhs2 (use_stmt),
4863 : rhs2, 0))
4864 0 : continue;
4865 : }
4866 0 : else if (gimple_assign_rhs2 (use_stmt) == rhs1)
4867 : {
4868 0 : if (gimple_assign_rhs1 (use_stmt) != rhs2)
4869 0 : continue;
4870 : }
4871 : else
4872 0 : continue;
4873 :
4874 8 : add_stmt = use_stmt;
4875 8 : break;
4876 9 : }
4877 9 : if (add_stmt == NULL)
4878 : return false;
4879 :
4880 : /* If stmt and add_stmt are in the same bb, we need to find out
4881 : which one is earlier. If they are in different bbs, we've
4882 : checked add_stmt is in the same bb as one of the uses of the
4883 : stmt lhs, so stmt needs to dominate add_stmt too. */
4884 8 : if (gimple_bb (stmt) == gimple_bb (add_stmt))
4885 : {
4886 8 : gimple_stmt_iterator gsif = *gsi;
4887 8 : gimple_stmt_iterator gsib = *gsi;
4888 8 : int i;
4889 : /* Search both forward and backward from stmt and have a small
4890 : upper bound. */
4891 20 : for (i = 0; i < 128; i++)
4892 : {
4893 20 : if (!gsi_end_p (gsib))
4894 : {
4895 18 : gsi_prev_nondebug (&gsib);
4896 18 : if (gsi_stmt (gsib) == add_stmt)
4897 : {
4898 : add_first = true;
4899 : break;
4900 : }
4901 : }
4902 2 : else if (gsi_end_p (gsif))
4903 : break;
4904 18 : if (!gsi_end_p (gsif))
4905 : {
4906 18 : gsi_next_nondebug (&gsif);
4907 18 : if (gsi_stmt (gsif) == add_stmt)
4908 : break;
4909 : }
4910 : }
4911 8 : if (i == 128)
4912 0 : return false;
4913 8 : if (add_first)
4914 2 : *gsi = gsi_for_stmt (add_stmt);
4915 : }
4916 : }
4917 : }
4918 :
4919 6334 : if (code == BIT_NOT_EXPR)
4920 239 : *gsi = gsi_for_stmt (cond_stmt);
4921 :
4922 6334 : auto_vec<gimple *, 8> mul_stmts;
4923 6334 : if (code == MULT_EXPR && cast_stmt)
4924 : {
4925 75 : type = TREE_TYPE (cast_lhs);
4926 75 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
4927 75 : if (gimple_assign_cast_p (g)
4928 38 : && useless_type_conversion_p (type,
4929 38 : TREE_TYPE (gimple_assign_rhs1 (g)))
4930 113 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g)))
4931 : rhs1 = gimple_assign_rhs1 (g);
4932 : else
4933 : {
4934 37 : g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, rhs1);
4935 37 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4936 37 : rhs1 = gimple_assign_lhs (g);
4937 37 : mul_stmts.quick_push (g);
4938 : }
4939 75 : if (TREE_CODE (rhs2) == INTEGER_CST)
4940 32 : rhs2 = fold_convert (type, rhs2);
4941 : else
4942 : {
4943 43 : g = SSA_NAME_DEF_STMT (rhs2);
4944 43 : if (gimple_assign_cast_p (g)
4945 22 : && useless_type_conversion_p (type,
4946 22 : TREE_TYPE (gimple_assign_rhs1 (g)))
4947 65 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g)))
4948 : rhs2 = gimple_assign_rhs1 (g);
4949 : else
4950 : {
4951 21 : g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, rhs2);
4952 21 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4953 21 : rhs2 = gimple_assign_lhs (g);
4954 21 : mul_stmts.quick_push (g);
4955 : }
4956 : }
4957 : }
4958 6334 : internal_fn ifn = (code == MULT_EXPR
4959 6334 : ? IFN_MUL_OVERFLOW
4960 : : code != MINUS_EXPR
4961 6182 : ? IFN_ADD_OVERFLOW : IFN_SUB_OVERFLOW);
4962 : if (code != MINUS_EXPR
4963 6234 : && tree_swap_operands_p (rhs1, rhs2))
4964 : std::swap (rhs1, rhs2);
4965 :
4966 6334 : tree ctype = build_complex_type (type);
4967 6334 : gcall *g = gimple_build_call_internal (ifn, 2, rhs1, rhs2);
4968 6334 : tree ctmp = make_ssa_name (ctype);
4969 6334 : gimple_call_set_lhs (g, ctmp);
4970 6334 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4971 6334 : tree new_lhs = (maxval || cast_stmt) ? make_ssa_name (type) : lhs;
4972 6334 : gassign *g2;
4973 6334 : if (code != BIT_NOT_EXPR)
4974 : {
4975 6095 : g2 = gimple_build_assign (new_lhs, REALPART_EXPR,
4976 : build1 (REALPART_EXPR, type, ctmp));
4977 6095 : if (maxval || cast_stmt)
4978 : {
4979 87 : gsi_insert_before (gsi, g2, GSI_SAME_STMT);
4980 87 : if (add_first)
4981 2 : *gsi = gsi_for_stmt (stmt);
4982 : }
4983 : else
4984 6008 : gsi_replace (gsi, g2, true);
4985 6095 : if (code == MULT_EXPR)
4986 : {
4987 152 : mul_stmts.quick_push (g);
4988 152 : mul_stmts.quick_push (g2);
4989 152 : if (cast_stmt)
4990 : {
4991 75 : g2 = gimple_build_assign (lhs, NOP_EXPR, new_lhs);
4992 75 : gsi_replace (gsi, g2, true);
4993 75 : mul_stmts.quick_push (g2);
4994 : }
4995 : }
4996 : }
4997 6334 : tree ovf = make_ssa_name (type);
4998 6334 : g2 = gimple_build_assign (ovf, IMAGPART_EXPR,
4999 : build1 (IMAGPART_EXPR, type, ctmp));
5000 6334 : if (code != BIT_NOT_EXPR)
5001 6095 : gsi_insert_after (gsi, g2, GSI_NEW_STMT);
5002 : else
5003 239 : gsi_insert_before (gsi, g2, GSI_SAME_STMT);
5004 6334 : if (code == MULT_EXPR)
5005 152 : mul_stmts.quick_push (g2);
5006 :
5007 33656 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, cast_lhs ? cast_lhs : lhs)
5008 : {
5009 21063 : if (is_gimple_debug (use_stmt))
5010 4299 : continue;
5011 :
5012 16764 : gimple *orig_use_stmt = use_stmt;
5013 16764 : int ovf_use = arith_overflow_check_p (stmt, cast_stmt, use_stmt,
5014 : maxval, NULL);
5015 16764 : if (ovf_use == 0)
5016 : {
5017 10377 : gcc_assert (code != BIT_NOT_EXPR);
5018 10377 : if (maxval)
5019 : {
5020 4 : tree use_lhs = gimple_assign_lhs (use_stmt);
5021 4 : gimple_assign_set_rhs1 (use_stmt, new_lhs);
5022 4 : if (useless_type_conversion_p (TREE_TYPE (use_lhs),
5023 4 : TREE_TYPE (new_lhs)))
5024 4 : gimple_assign_set_rhs_code (use_stmt, SSA_NAME);
5025 4 : update_stmt (use_stmt);
5026 : }
5027 10377 : continue;
5028 10377 : }
5029 6387 : if (gimple_code (use_stmt) == GIMPLE_COND)
5030 : {
5031 4230 : gcond *cond_stmt = as_a <gcond *> (use_stmt);
5032 4230 : gimple_cond_set_lhs (cond_stmt, ovf);
5033 4230 : gimple_cond_set_rhs (cond_stmt, build_int_cst (type, 0));
5034 4460 : gimple_cond_set_code (cond_stmt, ovf_use == 1 ? NE_EXPR : EQ_EXPR);
5035 : }
5036 : else
5037 : {
5038 2157 : gcc_checking_assert (is_gimple_assign (use_stmt));
5039 2157 : if (gimple_assign_rhs_class (use_stmt) == GIMPLE_BINARY_RHS)
5040 : {
5041 2157 : if (gimple_assign_rhs_code (use_stmt) == RSHIFT_EXPR)
5042 : {
5043 6 : g2 = gimple_build_assign (make_ssa_name (boolean_type_node),
5044 : ovf_use == 1 ? NE_EXPR : EQ_EXPR,
5045 : ovf, build_int_cst (type, 0));
5046 6 : gimple_stmt_iterator gsiu = gsi_for_stmt (use_stmt);
5047 6 : gsi_insert_before (&gsiu, g2, GSI_SAME_STMT);
5048 6 : gimple_assign_set_rhs_with_ops (&gsiu, NOP_EXPR,
5049 : gimple_assign_lhs (g2));
5050 6 : update_stmt (use_stmt);
5051 6 : use_operand_p use;
5052 6 : single_imm_use (gimple_assign_lhs (use_stmt), &use,
5053 : &use_stmt);
5054 6 : if (gimple_code (use_stmt) == GIMPLE_COND)
5055 : {
5056 0 : gcond *cond_stmt = as_a <gcond *> (use_stmt);
5057 0 : gimple_cond_set_lhs (cond_stmt, ovf);
5058 0 : gimple_cond_set_rhs (cond_stmt, build_int_cst (type, 0));
5059 : }
5060 : else
5061 : {
5062 6 : gcc_checking_assert (is_gimple_assign (use_stmt));
5063 6 : if (gimple_assign_rhs_class (use_stmt)
5064 : == GIMPLE_BINARY_RHS)
5065 : {
5066 0 : gimple_assign_set_rhs1 (use_stmt, ovf);
5067 0 : gimple_assign_set_rhs2 (use_stmt,
5068 : build_int_cst (type, 0));
5069 : }
5070 6 : else if (gimple_assign_cast_p (use_stmt))
5071 6 : gimple_assign_set_rhs1 (use_stmt, ovf);
5072 : else
5073 : {
5074 0 : tree_code sc = gimple_assign_rhs_code (use_stmt);
5075 0 : gcc_checking_assert (sc == COND_EXPR);
5076 0 : tree cond = gimple_assign_rhs1 (use_stmt);
5077 0 : cond = build2 (TREE_CODE (cond),
5078 : boolean_type_node, ovf,
5079 : build_int_cst (type, 0));
5080 0 : gimple_assign_set_rhs1 (use_stmt, cond);
5081 : }
5082 : }
5083 6 : update_stmt (use_stmt);
5084 6 : gsi_remove (&gsiu, true);
5085 6 : gsiu = gsi_for_stmt (g2);
5086 6 : gsi_remove (&gsiu, true);
5087 6 : continue;
5088 6 : }
5089 : else
5090 : {
5091 2151 : gimple_assign_set_rhs1 (use_stmt, ovf);
5092 2151 : gimple_assign_set_rhs2 (use_stmt, build_int_cst (type, 0));
5093 2223 : gimple_assign_set_rhs_code (use_stmt,
5094 : ovf_use == 1
5095 : ? NE_EXPR : EQ_EXPR);
5096 : }
5097 : }
5098 : else
5099 : {
5100 0 : gcc_checking_assert (gimple_assign_rhs_code (use_stmt)
5101 : == COND_EXPR);
5102 0 : tree cond = build2 (ovf_use == 1 ? NE_EXPR : EQ_EXPR,
5103 : boolean_type_node, ovf,
5104 : build_int_cst (type, 0));
5105 0 : gimple_assign_set_rhs1 (use_stmt, cond);
5106 : }
5107 : }
5108 6381 : update_stmt (use_stmt);
5109 6381 : if (code == MULT_EXPR && use_stmt != orig_use_stmt)
5110 : {
5111 173 : gimple_stmt_iterator gsi2 = gsi_for_stmt (orig_use_stmt);
5112 173 : maybe_optimize_guarding_check (mul_stmts, use_stmt, orig_use_stmt,
5113 : cfg_changed);
5114 173 : use_operand_p use;
5115 173 : gimple *cast_stmt;
5116 173 : if (single_imm_use (gimple_assign_lhs (orig_use_stmt), &use,
5117 : &cast_stmt)
5118 173 : && gimple_assign_cast_p (cast_stmt))
5119 : {
5120 2 : gimple_stmt_iterator gsi3 = gsi_for_stmt (cast_stmt);
5121 2 : gsi_remove (&gsi3, true);
5122 2 : release_ssa_name (gimple_assign_lhs (cast_stmt));
5123 : }
5124 173 : gsi_remove (&gsi2, true);
5125 173 : release_ssa_name (gimple_assign_lhs (orig_use_stmt));
5126 : }
5127 6334 : }
5128 6334 : if (maxval)
5129 : {
5130 12 : gimple_stmt_iterator gsi2 = gsi_for_stmt (stmt);
5131 12 : gsi_remove (&gsi2, true);
5132 12 : if (add_stmt)
5133 : {
5134 8 : gimple *g = gimple_build_assign (gimple_assign_lhs (add_stmt),
5135 : new_lhs);
5136 8 : gsi2 = gsi_for_stmt (add_stmt);
5137 8 : gsi_replace (&gsi2, g, true);
5138 : }
5139 : }
5140 6322 : else if (code == BIT_NOT_EXPR)
5141 : {
5142 239 : *gsi = gsi_for_stmt (stmt);
5143 239 : gsi_remove (gsi, true);
5144 239 : release_ssa_name (lhs);
5145 239 : return true;
5146 : }
5147 : return false;
5148 6334 : }
5149 :
5150 : /* Helper of match_uaddc_usubc. Look through an integral cast
5151 : which should preserve [0, 1] range value (unless source has
5152 : 1-bit signed type) and the cast has single use. */
5153 :
5154 : static gimple *
5155 2080737 : uaddc_cast (gimple *g)
5156 : {
5157 2080737 : if (!gimple_assign_cast_p (g))
5158 : return g;
5159 484110 : tree op = gimple_assign_rhs1 (g);
5160 484110 : if (TREE_CODE (op) == SSA_NAME
5161 405740 : && INTEGRAL_TYPE_P (TREE_TYPE (op))
5162 269925 : && (TYPE_PRECISION (TREE_TYPE (op)) > 1
5163 5842 : || TYPE_UNSIGNED (TREE_TYPE (op)))
5164 754035 : && has_single_use (gimple_assign_lhs (g)))
5165 158439 : return SSA_NAME_DEF_STMT (op);
5166 : return g;
5167 : }
5168 :
5169 : /* Helper of match_uaddc_usubc. Look through a NE_EXPR
5170 : comparison with 0 which also preserves [0, 1] value range. */
5171 :
5172 : static gimple *
5173 2080897 : uaddc_ne0 (gimple *g)
5174 : {
5175 2080897 : if (is_gimple_assign (g)
5176 1291231 : && gimple_assign_rhs_code (g) == NE_EXPR
5177 60762 : && integer_zerop (gimple_assign_rhs2 (g))
5178 6746 : && TREE_CODE (gimple_assign_rhs1 (g)) == SSA_NAME
5179 2087631 : && has_single_use (gimple_assign_lhs (g)))
5180 6442 : return SSA_NAME_DEF_STMT (gimple_assign_rhs1 (g));
5181 : return g;
5182 : }
5183 :
5184 : /* Return true if G is {REAL,IMAG}PART_EXPR PART with SSA_NAME
5185 : operand. */
5186 :
5187 : static bool
5188 2081744 : uaddc_is_cplxpart (gimple *g, tree_code part)
5189 : {
5190 2081744 : return (is_gimple_assign (g)
5191 1290599 : && gimple_assign_rhs_code (g) == part
5192 2085432 : && TREE_CODE (TREE_OPERAND (gimple_assign_rhs1 (g), 0)) == SSA_NAME);
5193 : }
5194 :
5195 : /* Try to match e.g.
5196 : _29 = .ADD_OVERFLOW (_3, _4);
5197 : _30 = REALPART_EXPR <_29>;
5198 : _31 = IMAGPART_EXPR <_29>;
5199 : _32 = .ADD_OVERFLOW (_30, _38);
5200 : _33 = REALPART_EXPR <_32>;
5201 : _34 = IMAGPART_EXPR <_32>;
5202 : _35 = _31 + _34;
5203 : as
5204 : _36 = .UADDC (_3, _4, _38);
5205 : _33 = REALPART_EXPR <_36>;
5206 : _35 = IMAGPART_EXPR <_36>;
5207 : or
5208 : _22 = .SUB_OVERFLOW (_6, _5);
5209 : _23 = REALPART_EXPR <_22>;
5210 : _24 = IMAGPART_EXPR <_22>;
5211 : _25 = .SUB_OVERFLOW (_23, _37);
5212 : _26 = REALPART_EXPR <_25>;
5213 : _27 = IMAGPART_EXPR <_25>;
5214 : _28 = _24 | _27;
5215 : as
5216 : _29 = .USUBC (_6, _5, _37);
5217 : _26 = REALPART_EXPR <_29>;
5218 : _288 = IMAGPART_EXPR <_29>;
5219 : provided _38 or _37 above have [0, 1] range
5220 : and _3, _4 and _30 or _6, _5 and _23 are unsigned
5221 : integral types with the same precision. Whether + or | or ^ is
5222 : used on the IMAGPART_EXPR results doesn't matter, with one of
5223 : added or subtracted operands in [0, 1] range at most one
5224 : .ADD_OVERFLOW or .SUB_OVERFLOW will indicate overflow. */
5225 :
5226 : static bool
5227 2857442 : match_uaddc_usubc (gimple_stmt_iterator *gsi, gimple *stmt, tree_code code)
5228 : {
5229 2857442 : tree rhs[4];
5230 2857442 : rhs[0] = gimple_assign_rhs1 (stmt);
5231 2857442 : rhs[1] = gimple_assign_rhs2 (stmt);
5232 2857442 : rhs[2] = NULL_TREE;
5233 2857442 : rhs[3] = NULL_TREE;
5234 2857442 : tree type = TREE_TYPE (rhs[0]);
5235 2857442 : if (!INTEGRAL_TYPE_P (type) || !TYPE_UNSIGNED (type))
5236 : return false;
5237 :
5238 1669806 : auto_vec<gimple *, 2> temp_stmts;
5239 1669806 : if (code != BIT_IOR_EXPR && code != BIT_XOR_EXPR)
5240 : {
5241 : /* If overflow flag is ignored on the MSB limb, we can end up with
5242 : the most significant limb handled as r = op1 + op2 + ovf1 + ovf2;
5243 : or r = op1 - op2 - ovf1 - ovf2; or various equivalent expressions
5244 : thereof. Handle those like the ovf = ovf1 + ovf2; case to recognize
5245 : the limb below the MSB, but also create another .UADDC/.USUBC call
5246 : for the last limb.
5247 :
5248 : First look through assignments with the same rhs code as CODE,
5249 : with the exception that subtraction of a constant is canonicalized
5250 : into addition of its negation. rhs[0] will be minuend for
5251 : subtractions and one of addends for addition, all other assigned
5252 : rhs[i] operands will be subtrahends or other addends. */
5253 1541662 : while (TREE_CODE (rhs[0]) == SSA_NAME && !rhs[3])
5254 : {
5255 1519961 : gimple *g = SSA_NAME_DEF_STMT (rhs[0]);
5256 1519961 : if (has_single_use (rhs[0])
5257 499441 : && is_gimple_assign (g)
5258 1953447 : && (gimple_assign_rhs_code (g) == code
5259 409073 : || (code == MINUS_EXPR
5260 51740 : && gimple_assign_rhs_code (g) == PLUS_EXPR
5261 15499 : && TREE_CODE (gimple_assign_rhs2 (g)) == INTEGER_CST)))
5262 : {
5263 37077 : tree r2 = gimple_assign_rhs2 (g);
5264 37077 : if (gimple_assign_rhs_code (g) != code)
5265 : {
5266 12664 : r2 = const_unop (NEGATE_EXPR, TREE_TYPE (r2), r2);
5267 12664 : if (!r2)
5268 : break;
5269 : }
5270 37077 : rhs[0] = gimple_assign_rhs1 (g);
5271 37077 : tree &r = rhs[2] ? rhs[3] : rhs[2];
5272 37077 : r = r2;
5273 37077 : temp_stmts.quick_push (g);
5274 : }
5275 : else
5276 : break;
5277 : }
5278 4513755 : for (int i = 1; i <= 2; ++i)
5279 3043473 : while (rhs[i] && TREE_CODE (rhs[i]) == SSA_NAME && !rhs[3])
5280 : {
5281 518874 : gimple *g = SSA_NAME_DEF_STMT (rhs[i]);
5282 518874 : if (has_single_use (rhs[i])
5283 247860 : && is_gimple_assign (g)
5284 748887 : && gimple_assign_rhs_code (g) == PLUS_EXPR)
5285 : {
5286 34303 : rhs[i] = gimple_assign_rhs1 (g);
5287 34303 : if (rhs[2])
5288 5243 : rhs[3] = gimple_assign_rhs2 (g);
5289 : else
5290 29060 : rhs[2] = gimple_assign_rhs2 (g);
5291 34303 : temp_stmts.quick_push (g);
5292 : }
5293 : else
5294 : break;
5295 : }
5296 : /* If there are just 3 addends or one minuend and two subtrahends,
5297 : check for UADDC or USUBC being pattern recognized earlier.
5298 : Say r = op1 + op2 + ovf1 + ovf2; where the (ovf1 + ovf2) part
5299 : got pattern matched earlier as __imag__ .UADDC (arg1, arg2, arg3)
5300 : etc. */
5301 1504585 : if (rhs[2] && !rhs[3])
5302 : {
5303 214050 : for (int i = (code == MINUS_EXPR ? 1 : 0); i < 3; ++i)
5304 157105 : if (TREE_CODE (rhs[i]) == SSA_NAME)
5305 : {
5306 119240 : gimple *im = uaddc_cast (SSA_NAME_DEF_STMT (rhs[i]));
5307 119240 : im = uaddc_ne0 (im);
5308 119240 : if (uaddc_is_cplxpart (im, IMAGPART_EXPR))
5309 : {
5310 : /* We found one of the 3 addends or 2 subtrahends to be
5311 : __imag__ of something, verify it is .UADDC/.USUBC. */
5312 232 : tree rhs1 = gimple_assign_rhs1 (im);
5313 232 : gimple *ovf = SSA_NAME_DEF_STMT (TREE_OPERAND (rhs1, 0));
5314 232 : tree ovf_lhs = NULL_TREE;
5315 232 : tree ovf_arg1 = NULL_TREE, ovf_arg2 = NULL_TREE;
5316 252 : if (gimple_call_internal_p (ovf, code == PLUS_EXPR
5317 : ? IFN_ADD_OVERFLOW
5318 : : IFN_SUB_OVERFLOW))
5319 : {
5320 : /* Or verify it is .ADD_OVERFLOW/.SUB_OVERFLOW.
5321 : This is for the case of 2 chained .UADDC/.USUBC,
5322 : where the first one uses 0 carry-in and the second
5323 : one ignores the carry-out.
5324 : So, something like:
5325 : _16 = .ADD_OVERFLOW (_1, _2);
5326 : _17 = REALPART_EXPR <_16>;
5327 : _18 = IMAGPART_EXPR <_16>;
5328 : _15 = _3 + _4;
5329 : _12 = _15 + _18;
5330 : where the first 3 statements come from the lower
5331 : limb addition and the last 2 from the higher limb
5332 : which ignores carry-out. */
5333 197 : ovf_lhs = gimple_call_lhs (ovf);
5334 197 : tree ovf_lhs_type = TREE_TYPE (TREE_TYPE (ovf_lhs));
5335 197 : ovf_arg1 = gimple_call_arg (ovf, 0);
5336 197 : ovf_arg2 = gimple_call_arg (ovf, 1);
5337 : /* In that case we need to punt if the types don't
5338 : mismatch. */
5339 197 : if (!types_compatible_p (type, ovf_lhs_type)
5340 197 : || !types_compatible_p (type, TREE_TYPE (ovf_arg1))
5341 393 : || !types_compatible_p (type,
5342 196 : TREE_TYPE (ovf_arg2)))
5343 : ovf_lhs = NULL_TREE;
5344 : else
5345 : {
5346 492 : for (int i = (code == PLUS_EXPR ? 1 : 0);
5347 492 : i >= 0; --i)
5348 : {
5349 350 : tree r = gimple_call_arg (ovf, i);
5350 350 : if (TREE_CODE (r) != SSA_NAME)
5351 0 : continue;
5352 350 : if (uaddc_is_cplxpart (SSA_NAME_DEF_STMT (r),
5353 : REALPART_EXPR))
5354 : {
5355 : /* Punt if one of the args which isn't
5356 : subtracted isn't __real__; that could
5357 : then prevent better match later.
5358 : Consider:
5359 : _3 = .ADD_OVERFLOW (_1, _2);
5360 : _4 = REALPART_EXPR <_3>;
5361 : _5 = IMAGPART_EXPR <_3>;
5362 : _7 = .ADD_OVERFLOW (_4, _6);
5363 : _8 = REALPART_EXPR <_7>;
5364 : _9 = IMAGPART_EXPR <_7>;
5365 : _12 = _10 + _11;
5366 : _13 = _12 + _9;
5367 : _14 = _13 + _5;
5368 : We want to match this when called on
5369 : the last stmt as a pair of .UADDC calls,
5370 : but without this check we could turn
5371 : that prematurely on _13 = _12 + _9;
5372 : stmt into .UADDC with 0 carry-in just
5373 : on the second .ADD_OVERFLOW call and
5374 : another replacing the _12 and _13
5375 : additions. */
5376 : ovf_lhs = NULL_TREE;
5377 : break;
5378 : }
5379 : }
5380 : }
5381 192 : if (ovf_lhs)
5382 : {
5383 142 : use_operand_p use_p;
5384 142 : imm_use_iterator iter;
5385 142 : tree re_lhs = NULL_TREE;
5386 426 : FOR_EACH_IMM_USE_FAST (use_p, iter, ovf_lhs)
5387 : {
5388 284 : gimple *use_stmt = USE_STMT (use_p);
5389 284 : if (is_gimple_debug (use_stmt))
5390 0 : continue;
5391 284 : if (use_stmt == im)
5392 142 : continue;
5393 142 : if (!uaddc_is_cplxpart (use_stmt,
5394 : REALPART_EXPR))
5395 : {
5396 : ovf_lhs = NULL_TREE;
5397 : break;
5398 : }
5399 142 : re_lhs = gimple_assign_lhs (use_stmt);
5400 142 : }
5401 142 : if (ovf_lhs && re_lhs)
5402 : {
5403 385 : FOR_EACH_IMM_USE_FAST (use_p, iter, re_lhs)
5404 : {
5405 297 : gimple *use_stmt = USE_STMT (use_p);
5406 297 : if (is_gimple_debug (use_stmt))
5407 108 : continue;
5408 189 : internal_fn ifn
5409 189 : = gimple_call_internal_fn (ovf);
5410 : /* Punt if the __real__ of lhs is used
5411 : in the same .*_OVERFLOW call.
5412 : Consider:
5413 : _3 = .ADD_OVERFLOW (_1, _2);
5414 : _4 = REALPART_EXPR <_3>;
5415 : _5 = IMAGPART_EXPR <_3>;
5416 : _7 = .ADD_OVERFLOW (_4, _6);
5417 : _8 = REALPART_EXPR <_7>;
5418 : _9 = IMAGPART_EXPR <_7>;
5419 : _12 = _10 + _11;
5420 : _13 = _12 + _5;
5421 : _14 = _13 + _9;
5422 : We want to match this when called on
5423 : the last stmt as a pair of .UADDC calls,
5424 : but without this check we could turn
5425 : that prematurely on _13 = _12 + _5;
5426 : stmt into .UADDC with 0 carry-in just
5427 : on the first .ADD_OVERFLOW call and
5428 : another replacing the _12 and _13
5429 : additions. */
5430 189 : if (gimple_call_internal_p (use_stmt, ifn))
5431 : {
5432 : ovf_lhs = NULL_TREE;
5433 : break;
5434 : }
5435 142 : }
5436 : }
5437 : }
5438 : }
5439 142 : if ((ovf_lhs
5440 153 : || gimple_call_internal_p (ovf,
5441 : code == PLUS_EXPR
5442 : ? IFN_UADDC : IFN_USUBC))
5443 250 : && (optab_handler (code == PLUS_EXPR
5444 : ? uaddc5_optab : usubc5_optab,
5445 94 : TYPE_MODE (type))
5446 : != CODE_FOR_nothing))
5447 : {
5448 : /* And in that case build another .UADDC/.USUBC
5449 : call for the most significand limb addition.
5450 : Overflow bit is ignored here. */
5451 63 : if (i != 2)
5452 63 : std::swap (rhs[i], rhs[2]);
5453 63 : gimple *g
5454 77 : = gimple_build_call_internal (code == PLUS_EXPR
5455 : ? IFN_UADDC
5456 : : IFN_USUBC,
5457 : 3, rhs[0], rhs[1],
5458 : rhs[2]);
5459 63 : tree nlhs = make_ssa_name (build_complex_type (type));
5460 63 : gimple_call_set_lhs (g, nlhs);
5461 63 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5462 63 : tree ilhs = gimple_assign_lhs (stmt);
5463 63 : g = gimple_build_assign (ilhs, REALPART_EXPR,
5464 : build1 (REALPART_EXPR,
5465 63 : TREE_TYPE (ilhs),
5466 : nlhs));
5467 63 : gsi_replace (gsi, g, true);
5468 : /* And if it is initialized from result of __imag__
5469 : of .{ADD,SUB}_OVERFLOW call, replace that
5470 : call with .U{ADD,SUB}C call with the same arguments,
5471 : just 0 added as third argument. This isn't strictly
5472 : necessary, .ADD_OVERFLOW (x, y) and .UADDC (x, y, 0)
5473 : produce the same result, but may result in better
5474 : generated code on some targets where the backend can
5475 : better prepare in how the result will be used. */
5476 63 : if (ovf_lhs)
5477 : {
5478 57 : tree zero = build_zero_cst (type);
5479 57 : g = gimple_build_call_internal (code == PLUS_EXPR
5480 : ? IFN_UADDC
5481 : : IFN_USUBC,
5482 : 3, ovf_arg1,
5483 : ovf_arg2, zero);
5484 57 : gimple_call_set_lhs (g, ovf_lhs);
5485 57 : gimple_stmt_iterator gsi2 = gsi_for_stmt (ovf);
5486 57 : gsi_replace (&gsi2, g, true);
5487 : }
5488 : return true;
5489 : }
5490 : }
5491 : }
5492 : return false;
5493 : }
5494 1447577 : if (code == MINUS_EXPR && !rhs[2])
5495 : return false;
5496 232 : if (code == MINUS_EXPR)
5497 : /* Code below expects rhs[0] and rhs[1] to have the IMAGPART_EXPRs.
5498 : So, for MINUS_EXPR swap the single added rhs operand (others are
5499 : subtracted) to rhs[3]. */
5500 232 : std::swap (rhs[0], rhs[3]);
5501 : }
5502 : /* Walk from both operands of STMT (for +/- even sometimes from
5503 : all the 4 addends or 3 subtrahends), see through casts and != 0
5504 : statements which would preserve [0, 1] range of values and
5505 : check which is initialized from __imag__. */
5506 1504937 : gimple *im1 = NULL, *im2 = NULL;
5507 15045828 : for (int i = 0; i < (code == MINUS_EXPR ? 3 : 4); i++)
5508 6018807 : if (rhs[i] && TREE_CODE (rhs[i]) == SSA_NAME)
5509 : {
5510 1961405 : gimple *im = uaddc_cast (SSA_NAME_DEF_STMT (rhs[i]));
5511 1961405 : im = uaddc_ne0 (im);
5512 1961405 : if (uaddc_is_cplxpart (im, IMAGPART_EXPR))
5513 : {
5514 2932 : if (im1 == NULL)
5515 : {
5516 2565 : im1 = im;
5517 2565 : if (i != 0)
5518 904 : std::swap (rhs[0], rhs[i]);
5519 : }
5520 : else
5521 : {
5522 367 : im2 = im;
5523 367 : if (i != 1)
5524 20 : std::swap (rhs[1], rhs[i]);
5525 : break;
5526 : }
5527 : }
5528 : }
5529 : /* If we don't find at least two, punt. */
5530 1504937 : if (!im2)
5531 : return false;
5532 : /* Check they are __imag__ of .ADD_OVERFLOW or .SUB_OVERFLOW call results,
5533 : either both .ADD_OVERFLOW or both .SUB_OVERFLOW and that we have
5534 : uaddc5/usubc5 named pattern for the corresponding mode. */
5535 367 : gimple *ovf1
5536 367 : = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (im1), 0));
5537 367 : gimple *ovf2
5538 367 : = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (im2), 0));
5539 367 : internal_fn ifn;
5540 367 : if (!is_gimple_call (ovf1)
5541 367 : || !gimple_call_internal_p (ovf1)
5542 367 : || ((ifn = gimple_call_internal_fn (ovf1)) != IFN_ADD_OVERFLOW
5543 41 : && ifn != IFN_SUB_OVERFLOW)
5544 364 : || !gimple_call_internal_p (ovf2, ifn)
5545 394 : || optab_handler (ifn == IFN_ADD_OVERFLOW ? uaddc5_optab : usubc5_optab,
5546 360 : TYPE_MODE (type)) == CODE_FOR_nothing
5547 95 : || (rhs[2]
5548 17 : && optab_handler (code == PLUS_EXPR ? uaddc5_optab : usubc5_optab,
5549 15 : TYPE_MODE (type)) == CODE_FOR_nothing)
5550 95 : || !types_compatible_p (type,
5551 95 : TREE_TYPE (TREE_TYPE (gimple_call_lhs (ovf1))))
5552 461 : || !types_compatible_p (type,
5553 94 : TREE_TYPE (TREE_TYPE (gimple_call_lhs (ovf2)))))
5554 : return false;
5555 94 : tree arg1, arg2, arg3 = NULL_TREE;
5556 94 : gimple *re1 = NULL, *re2 = NULL;
5557 : /* On one of the two calls, one of the .ADD_OVERFLOW/.SUB_OVERFLOW arguments
5558 : should be initialized from __real__ of the other of the two calls.
5559 : Though, for .SUB_OVERFLOW, it has to be the first argument, not the
5560 : second one. */
5561 249 : for (int i = (ifn == IFN_ADD_OVERFLOW ? 1 : 0); i >= 0; --i)
5562 351 : for (gimple *ovf = ovf1; ovf; ovf = (ovf == ovf1 ? ovf2 : NULL))
5563 : {
5564 290 : tree arg = gimple_call_arg (ovf, i);
5565 290 : if (TREE_CODE (arg) != SSA_NAME)
5566 2 : continue;
5567 288 : re1 = SSA_NAME_DEF_STMT (arg);
5568 288 : if (uaddc_is_cplxpart (re1, REALPART_EXPR)
5569 382 : && (SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (re1), 0))
5570 94 : == (ovf == ovf1 ? ovf2 : ovf1)))
5571 : {
5572 94 : if (ovf == ovf1)
5573 : {
5574 : /* Make sure ovf2 is the .*_OVERFLOW call with argument
5575 : initialized from __real__ of ovf1. */
5576 20 : std::swap (rhs[0], rhs[1]);
5577 20 : std::swap (im1, im2);
5578 20 : std::swap (ovf1, ovf2);
5579 : }
5580 94 : arg3 = gimple_call_arg (ovf, 1 - i);
5581 94 : i = -1;
5582 94 : break;
5583 : }
5584 : }
5585 94 : if (!arg3)
5586 : return false;
5587 94 : arg1 = gimple_call_arg (ovf1, 0);
5588 94 : arg2 = gimple_call_arg (ovf1, 1);
5589 94 : if (!types_compatible_p (type, TREE_TYPE (arg1)))
5590 : return false;
5591 94 : int kind[2] = { 0, 0 };
5592 94 : tree arg_im[2] = { NULL_TREE, NULL_TREE };
5593 : /* At least one of arg2 and arg3 should have type compatible
5594 : with arg1/rhs[0], and the other one should have value in [0, 1]
5595 : range. If both are in [0, 1] range and type compatible with
5596 : arg1/rhs[0], try harder to find after looking through casts,
5597 : != 0 comparisons which one is initialized to __imag__ of
5598 : .{ADD,SUB}_OVERFLOW or .U{ADD,SUB}C call results. */
5599 282 : for (int i = 0; i < 2; ++i)
5600 : {
5601 188 : tree arg = i == 0 ? arg2 : arg3;
5602 188 : if (types_compatible_p (type, TREE_TYPE (arg)))
5603 163 : kind[i] = 1;
5604 376 : if (!INTEGRAL_TYPE_P (TREE_TYPE (arg))
5605 376 : || (TYPE_PRECISION (TREE_TYPE (arg)) == 1
5606 25 : && !TYPE_UNSIGNED (TREE_TYPE (arg))))
5607 0 : continue;
5608 188 : if (tree_zero_one_valued_p (arg))
5609 52 : kind[i] |= 2;
5610 188 : if (TREE_CODE (arg) == SSA_NAME)
5611 : {
5612 185 : gimple *g = SSA_NAME_DEF_STMT (arg);
5613 185 : if (gimple_assign_cast_p (g))
5614 : {
5615 30 : tree op = gimple_assign_rhs1 (g);
5616 30 : if (TREE_CODE (op) == SSA_NAME
5617 30 : && INTEGRAL_TYPE_P (TREE_TYPE (op)))
5618 30 : g = SSA_NAME_DEF_STMT (op);
5619 : }
5620 185 : g = uaddc_ne0 (g);
5621 185 : if (!uaddc_is_cplxpart (g, IMAGPART_EXPR))
5622 125 : continue;
5623 60 : arg_im[i] = gimple_assign_lhs (g);
5624 60 : g = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (g), 0));
5625 60 : if (!is_gimple_call (g) || !gimple_call_internal_p (g))
5626 0 : continue;
5627 60 : switch (gimple_call_internal_fn (g))
5628 : {
5629 60 : case IFN_ADD_OVERFLOW:
5630 60 : case IFN_SUB_OVERFLOW:
5631 60 : case IFN_UADDC:
5632 60 : case IFN_USUBC:
5633 60 : break;
5634 0 : default:
5635 0 : continue;
5636 : }
5637 60 : kind[i] |= 4;
5638 : }
5639 : }
5640 : /* Make arg2 the one with compatible type and arg3 the one
5641 : with [0, 1] range. If both is true for both operands,
5642 : prefer as arg3 result of __imag__ of some ifn. */
5643 94 : if ((kind[0] & 1) == 0 || ((kind[1] & 1) != 0 && kind[0] > kind[1]))
5644 : {
5645 1 : std::swap (arg2, arg3);
5646 1 : std::swap (kind[0], kind[1]);
5647 1 : std::swap (arg_im[0], arg_im[1]);
5648 : }
5649 94 : if ((kind[0] & 1) == 0 || (kind[1] & 6) == 0)
5650 : return false;
5651 70 : if (!has_single_use (gimple_assign_lhs (im1))
5652 68 : || !has_single_use (gimple_assign_lhs (im2))
5653 68 : || !has_single_use (gimple_assign_lhs (re1))
5654 138 : || num_imm_uses (gimple_call_lhs (ovf1)) != 2)
5655 : return false;
5656 : /* Check that ovf2's result is used in __real__ and set re2
5657 : to that statement. */
5658 68 : use_operand_p use_p;
5659 68 : imm_use_iterator iter;
5660 68 : tree lhs = gimple_call_lhs (ovf2);
5661 203 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
5662 : {
5663 135 : gimple *use_stmt = USE_STMT (use_p);
5664 135 : if (is_gimple_debug (use_stmt))
5665 0 : continue;
5666 135 : if (use_stmt == im2)
5667 68 : continue;
5668 67 : if (re2)
5669 : return false;
5670 67 : if (!uaddc_is_cplxpart (use_stmt, REALPART_EXPR))
5671 : return false;
5672 : re2 = use_stmt;
5673 0 : }
5674 : /* Build .UADDC/.USUBC call which will be placed before the stmt. */
5675 68 : gimple_stmt_iterator gsi2 = gsi_for_stmt (ovf2);
5676 68 : gimple *g;
5677 68 : if ((kind[1] & 4) != 0 && types_compatible_p (type, TREE_TYPE (arg_im[1])))
5678 : arg3 = arg_im[1];
5679 68 : if ((kind[1] & 1) == 0)
5680 : {
5681 25 : if (TREE_CODE (arg3) == INTEGER_CST)
5682 0 : arg3 = fold_convert (type, arg3);
5683 : else
5684 : {
5685 25 : g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, arg3);
5686 25 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
5687 25 : arg3 = gimple_assign_lhs (g);
5688 : }
5689 : }
5690 91 : g = gimple_build_call_internal (ifn == IFN_ADD_OVERFLOW
5691 : ? IFN_UADDC : IFN_USUBC,
5692 : 3, arg1, arg2, arg3);
5693 68 : tree nlhs = make_ssa_name (TREE_TYPE (lhs));
5694 68 : gimple_call_set_lhs (g, nlhs);
5695 68 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
5696 : /* In the case where stmt is | or ^ of two overflow flags
5697 : or addition of those, replace stmt with __imag__ of the above
5698 : added call. In case of arg1 + arg2 + (ovf1 + ovf2) or
5699 : arg1 - arg2 - (ovf1 + ovf2) just emit it before stmt. */
5700 68 : tree ilhs = rhs[2] ? make_ssa_name (type) : gimple_assign_lhs (stmt);
5701 68 : g = gimple_build_assign (ilhs, IMAGPART_EXPR,
5702 68 : build1 (IMAGPART_EXPR, TREE_TYPE (ilhs), nlhs));
5703 68 : if (rhs[2])
5704 : {
5705 15 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5706 : /* Remove some further statements which can't be kept in the IL because
5707 : they can use SSA_NAMEs whose setter is going to be removed too. */
5708 75 : for (gimple *g2 : temp_stmts)
5709 : {
5710 30 : gsi2 = gsi_for_stmt (g2);
5711 30 : gsi_remove (&gsi2, true);
5712 30 : release_defs (g2);
5713 : }
5714 : }
5715 : else
5716 53 : gsi_replace (gsi, g, true);
5717 : /* Remove some statements which can't be kept in the IL because they
5718 : use SSA_NAME whose setter is going to be removed too. */
5719 68 : tree rhs1 = rhs[1];
5720 104 : for (int i = 0; i < 2; i++)
5721 86 : if (rhs1 == gimple_assign_lhs (im2))
5722 : break;
5723 : else
5724 : {
5725 36 : g = SSA_NAME_DEF_STMT (rhs1);
5726 36 : rhs1 = gimple_assign_rhs1 (g);
5727 36 : gsi2 = gsi_for_stmt (g);
5728 36 : gsi_remove (&gsi2, true);
5729 36 : release_defs (g);
5730 : }
5731 68 : gcc_checking_assert (rhs1 == gimple_assign_lhs (im2));
5732 68 : gsi2 = gsi_for_stmt (im2);
5733 68 : gsi_remove (&gsi2, true);
5734 68 : release_defs (im2);
5735 : /* Replace the re2 statement with __real__ of the newly added
5736 : .UADDC/.USUBC call. */
5737 68 : if (re2)
5738 : {
5739 67 : gsi2 = gsi_for_stmt (re2);
5740 67 : tree rlhs = gimple_assign_lhs (re2);
5741 67 : g = gimple_build_assign (rlhs, REALPART_EXPR,
5742 67 : build1 (REALPART_EXPR, TREE_TYPE (rlhs), nlhs));
5743 67 : gsi_replace (&gsi2, g, true);
5744 : }
5745 68 : if (rhs[2])
5746 : {
5747 : /* If this is the arg1 + arg2 + (ovf1 + ovf2) or
5748 : arg1 - arg2 - (ovf1 + ovf2) case for the most significant limb,
5749 : replace stmt with __real__ of another .UADDC/.USUBC call which
5750 : handles the most significant limb. Overflow flag from this is
5751 : ignored. */
5752 17 : g = gimple_build_call_internal (code == PLUS_EXPR
5753 : ? IFN_UADDC : IFN_USUBC,
5754 : 3, rhs[3], rhs[2], ilhs);
5755 15 : nlhs = make_ssa_name (TREE_TYPE (lhs));
5756 15 : gimple_call_set_lhs (g, nlhs);
5757 15 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5758 15 : ilhs = gimple_assign_lhs (stmt);
5759 15 : g = gimple_build_assign (ilhs, REALPART_EXPR,
5760 15 : build1 (REALPART_EXPR, TREE_TYPE (ilhs), nlhs));
5761 15 : gsi_replace (gsi, g, true);
5762 : }
5763 68 : if (TREE_CODE (arg3) == SSA_NAME)
5764 : {
5765 : /* When pattern recognizing the second least significant limb
5766 : above (i.e. first pair of .{ADD,SUB}_OVERFLOW calls for one limb),
5767 : check if the [0, 1] range argument (i.e. carry in) isn't the
5768 : result of another .{ADD,SUB}_OVERFLOW call (one handling the
5769 : least significant limb). Again look through casts and != 0. */
5770 67 : gimple *im3 = SSA_NAME_DEF_STMT (arg3);
5771 92 : for (int i = 0; i < 2; ++i)
5772 : {
5773 92 : gimple *im4 = uaddc_cast (im3);
5774 92 : if (im4 == im3)
5775 : break;
5776 : else
5777 25 : im3 = im4;
5778 : }
5779 67 : im3 = uaddc_ne0 (im3);
5780 67 : if (uaddc_is_cplxpart (im3, IMAGPART_EXPR))
5781 : {
5782 60 : gimple *ovf3
5783 60 : = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (im3), 0));
5784 60 : if (gimple_call_internal_p (ovf3, ifn))
5785 : {
5786 25 : lhs = gimple_call_lhs (ovf3);
5787 25 : arg1 = gimple_call_arg (ovf3, 0);
5788 25 : arg2 = gimple_call_arg (ovf3, 1);
5789 25 : if (types_compatible_p (type, TREE_TYPE (TREE_TYPE (lhs)))
5790 25 : && types_compatible_p (type, TREE_TYPE (arg1))
5791 50 : && types_compatible_p (type, TREE_TYPE (arg2)))
5792 : {
5793 : /* And if it is initialized from result of __imag__
5794 : of .{ADD,SUB}_OVERFLOW call, replace that
5795 : call with .U{ADD,SUB}C call with the same arguments,
5796 : just 0 added as third argument. This isn't strictly
5797 : necessary, .ADD_OVERFLOW (x, y) and .UADDC (x, y, 0)
5798 : produce the same result, but may result in better
5799 : generated code on some targets where the backend can
5800 : better prepare in how the result will be used. */
5801 25 : g = gimple_build_call_internal (ifn == IFN_ADD_OVERFLOW
5802 : ? IFN_UADDC : IFN_USUBC,
5803 : 3, arg1, arg2,
5804 : build_zero_cst (type));
5805 25 : gimple_call_set_lhs (g, lhs);
5806 25 : gsi2 = gsi_for_stmt (ovf3);
5807 25 : gsi_replace (&gsi2, g, true);
5808 : }
5809 : }
5810 : }
5811 : }
5812 : return true;
5813 1669806 : }
5814 :
5815 : /* Replace .POPCOUNT (x) == 1 or .POPCOUNT (x) != 1 with
5816 : (x & (x - 1)) > x - 1 or (x & (x - 1)) <= x - 1 if .POPCOUNT
5817 : isn't a direct optab. Also handle `<=`/`>` to be
5818 : `x & (x - 1) !=/== x`. */
5819 :
5820 : static void
5821 4585721 : match_single_bit_test (gimple_stmt_iterator *gsi, gimple *stmt)
5822 : {
5823 4585721 : tree clhs, crhs;
5824 4585721 : enum tree_code code;
5825 4585721 : bool was_le = false;
5826 4585721 : if (gimple_code (stmt) == GIMPLE_COND)
5827 : {
5828 4258847 : clhs = gimple_cond_lhs (stmt);
5829 4258847 : crhs = gimple_cond_rhs (stmt);
5830 4258847 : code = gimple_cond_code (stmt);
5831 : }
5832 : else
5833 : {
5834 326874 : clhs = gimple_assign_rhs1 (stmt);
5835 326874 : crhs = gimple_assign_rhs2 (stmt);
5836 326874 : code = gimple_assign_rhs_code (stmt);
5837 : }
5838 4585721 : if (code != LE_EXPR && code != GT_EXPR
5839 4585721 : && code != EQ_EXPR && code != NE_EXPR)
5840 4585679 : return;
5841 2150191 : if (code == LE_EXPR || code == GT_EXPR)
5842 4324538 : was_le = true;
5843 4324538 : if (TREE_CODE (clhs) != SSA_NAME || !integer_onep (crhs))
5844 : return;
5845 164855 : gimple *call = SSA_NAME_DEF_STMT (clhs);
5846 164855 : combined_fn cfn = gimple_call_combined_fn (call);
5847 164855 : switch (cfn)
5848 : {
5849 51 : CASE_CFN_POPCOUNT:
5850 51 : break;
5851 : default:
5852 : return;
5853 : }
5854 51 : if (!has_single_use (clhs))
5855 : return;
5856 50 : tree arg = gimple_call_arg (call, 0);
5857 50 : tree type = TREE_TYPE (arg);
5858 50 : if (!INTEGRAL_TYPE_P (type))
5859 : return;
5860 50 : bool nonzero_arg = tree_expr_nonzero_p (arg);
5861 50 : if (direct_internal_fn_supported_p (IFN_POPCOUNT, type, OPTIMIZE_FOR_BOTH))
5862 : {
5863 : /* Tell expand_POPCOUNT the popcount result is only used in equality
5864 : comparison with one, so that it can decide based on rtx costs. */
5865 16 : gimple *g = gimple_build_call_internal (IFN_POPCOUNT, 2, arg,
5866 : was_le ? integer_minus_one_node
5867 8 : : (nonzero_arg ? integer_zero_node
5868 : : integer_one_node));
5869 8 : gimple_call_set_lhs (g, gimple_call_lhs (call));
5870 8 : gimple_stmt_iterator gsi2 = gsi_for_stmt (call);
5871 8 : gsi_replace (&gsi2, g, true);
5872 8 : return;
5873 : }
5874 42 : tree argm1 = make_ssa_name (type);
5875 42 : gimple *g = gimple_build_assign (argm1, PLUS_EXPR, arg,
5876 : build_int_cst (type, -1));
5877 42 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5878 42 : g = gimple_build_assign (make_ssa_name (type),
5879 42 : (nonzero_arg || was_le) ? BIT_AND_EXPR : BIT_XOR_EXPR,
5880 : arg, argm1);
5881 42 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5882 42 : tree_code cmpcode;
5883 42 : if (was_le)
5884 : {
5885 0 : argm1 = build_zero_cst (type);
5886 0 : cmpcode = code == LE_EXPR ? EQ_EXPR : NE_EXPR;
5887 : }
5888 42 : else if (nonzero_arg)
5889 : {
5890 2 : argm1 = build_zero_cst (type);
5891 2 : cmpcode = code;
5892 : }
5893 : else
5894 40 : cmpcode = code == EQ_EXPR ? GT_EXPR : LE_EXPR;
5895 42 : if (gcond *cond = dyn_cast <gcond *> (stmt))
5896 : {
5897 2 : gimple_cond_set_lhs (cond, gimple_assign_lhs (g));
5898 2 : gimple_cond_set_rhs (cond, argm1);
5899 2 : gimple_cond_set_code (cond, cmpcode);
5900 : }
5901 : else
5902 : {
5903 40 : gimple_assign_set_rhs1 (stmt, gimple_assign_lhs (g));
5904 40 : gimple_assign_set_rhs2 (stmt, argm1);
5905 40 : gimple_assign_set_rhs_code (stmt, cmpcode);
5906 : }
5907 42 : update_stmt (stmt);
5908 42 : gimple_stmt_iterator gsi2 = gsi_for_stmt (call);
5909 42 : gsi_remove (&gsi2, true);
5910 42 : release_defs (call);
5911 : }
5912 :
5913 : /* Return true if target has support for divmod. */
5914 :
5915 : static bool
5916 38482 : target_supports_divmod_p (optab divmod_optab, optab div_optab, machine_mode mode)
5917 : {
5918 : /* If target supports hardware divmod insn, use it for divmod. */
5919 38482 : if (optab_handler (divmod_optab, mode) != CODE_FOR_nothing)
5920 : return true;
5921 :
5922 : /* Check if libfunc for divmod is available. */
5923 2554 : rtx libfunc = optab_libfunc (divmod_optab, mode);
5924 2554 : if (libfunc != NULL_RTX)
5925 : {
5926 : /* If optab_handler exists for div_optab, perhaps in a wider mode,
5927 : we don't want to use the libfunc even if it exists for given mode. */
5928 : machine_mode div_mode;
5929 10622 : FOR_EACH_MODE_FROM (div_mode, mode)
5930 8068 : if (optab_handler (div_optab, div_mode) != CODE_FOR_nothing)
5931 : return false;
5932 :
5933 2554 : return targetm.expand_divmod_libfunc != NULL;
5934 : }
5935 :
5936 : return false;
5937 : }
5938 :
5939 : /* Check if stmt is candidate for divmod transform. */
5940 :
5941 : static bool
5942 56695 : divmod_candidate_p (gassign *stmt)
5943 : {
5944 56695 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
5945 56695 : machine_mode mode = TYPE_MODE (type);
5946 56695 : optab divmod_optab, div_optab;
5947 :
5948 56695 : if (TYPE_UNSIGNED (type))
5949 : {
5950 : divmod_optab = udivmod_optab;
5951 : div_optab = udiv_optab;
5952 : }
5953 : else
5954 : {
5955 28388 : divmod_optab = sdivmod_optab;
5956 28388 : div_optab = sdiv_optab;
5957 : }
5958 :
5959 56695 : tree op1 = gimple_assign_rhs1 (stmt);
5960 56695 : tree op2 = gimple_assign_rhs2 (stmt);
5961 :
5962 : /* Disable the transform if either is a constant, since division-by-constant
5963 : may have specialized expansion. */
5964 56695 : if (CONSTANT_CLASS_P (op1))
5965 : return false;
5966 :
5967 52750 : if (CONSTANT_CLASS_P (op2))
5968 : {
5969 16481 : if (integer_pow2p (op2))
5970 : return false;
5971 :
5972 14388 : if (element_precision (type) <= HOST_BITS_PER_WIDE_INT
5973 15441 : && element_precision (type) <= BITS_PER_WORD)
5974 : return false;
5975 :
5976 : /* If the divisor is not power of 2 and the precision wider than
5977 : HWI, expand_divmod punts on that, so in that case it is better
5978 : to use divmod optab or libfunc. Similarly if choose_multiplier
5979 : might need pre/post shifts of BITS_PER_WORD or more. */
5980 : }
5981 :
5982 : /* Exclude the case where TYPE_OVERFLOW_TRAPS (type) as that should
5983 : expand using the [su]divv optabs. */
5984 38482 : if (TYPE_OVERFLOW_TRAPS (type))
5985 : return false;
5986 :
5987 38482 : if (!target_supports_divmod_p (divmod_optab, div_optab, mode))
5988 : return false;
5989 :
5990 : return true;
5991 : }
5992 :
5993 : /* This function looks for:
5994 : t1 = a TRUNC_DIV_EXPR b;
5995 : t2 = a TRUNC_MOD_EXPR b;
5996 : and transforms it to the following sequence:
5997 : complex_tmp = DIVMOD (a, b);
5998 : t1 = REALPART_EXPR(a);
5999 : t2 = IMAGPART_EXPR(b);
6000 : For conditions enabling the transform see divmod_candidate_p().
6001 :
6002 : The pass has three parts:
6003 : 1) Find top_stmt which is trunc_div or trunc_mod stmt and dominates all
6004 : other trunc_div_expr and trunc_mod_expr stmts.
6005 : 2) Add top_stmt and all trunc_div and trunc_mod stmts dominated by top_stmt
6006 : to stmts vector.
6007 : 3) Insert DIVMOD call just before top_stmt and update entries in
6008 : stmts vector to use return value of DIMOVD (REALEXPR_PART for div,
6009 : IMAGPART_EXPR for mod). */
6010 :
6011 : static bool
6012 56714 : convert_to_divmod (gassign *stmt)
6013 : {
6014 56714 : if (stmt_can_throw_internal (cfun, stmt)
6015 56714 : || !divmod_candidate_p (stmt))
6016 : return false;
6017 :
6018 38482 : tree op1 = gimple_assign_rhs1 (stmt);
6019 38482 : tree op2 = gimple_assign_rhs2 (stmt);
6020 :
6021 38482 : imm_use_iterator use_iter;
6022 38482 : gimple *use_stmt;
6023 38482 : auto_vec<gimple *> stmts;
6024 :
6025 38482 : gimple *top_stmt = stmt;
6026 38482 : basic_block top_bb = gimple_bb (stmt);
6027 :
6028 : /* Part 1: Try to set top_stmt to "topmost" stmt that dominates
6029 : at-least stmt and possibly other trunc_div/trunc_mod stmts
6030 : having same operands as stmt. */
6031 :
6032 327271 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, op1)
6033 : {
6034 288789 : if (is_gimple_assign (use_stmt)
6035 238416 : && (gimple_assign_rhs_code (use_stmt) == TRUNC_DIV_EXPR
6036 226674 : || gimple_assign_rhs_code (use_stmt) == TRUNC_MOD_EXPR)
6037 216250 : && operand_equal_p (op1, gimple_assign_rhs1 (use_stmt), 0)
6038 504922 : && operand_equal_p (op2, gimple_assign_rhs2 (use_stmt), 0))
6039 : {
6040 50114 : if (stmt_can_throw_internal (cfun, use_stmt))
6041 0 : continue;
6042 :
6043 50114 : basic_block bb = gimple_bb (use_stmt);
6044 :
6045 50114 : if (bb == top_bb)
6046 : {
6047 49386 : if (gimple_uid (use_stmt) < gimple_uid (top_stmt))
6048 5137 : top_stmt = use_stmt;
6049 : }
6050 728 : else if (dominated_by_p (CDI_DOMINATORS, top_bb, bb))
6051 : {
6052 194 : top_bb = bb;
6053 194 : top_stmt = use_stmt;
6054 : }
6055 : }
6056 38482 : }
6057 :
6058 38482 : tree top_op1 = gimple_assign_rhs1 (top_stmt);
6059 38482 : tree top_op2 = gimple_assign_rhs2 (top_stmt);
6060 :
6061 38482 : stmts.safe_push (top_stmt);
6062 38482 : bool div_seen = (gimple_assign_rhs_code (top_stmt) == TRUNC_DIV_EXPR);
6063 :
6064 : /* Part 2: Add all trunc_div/trunc_mod statements domianted by top_bb
6065 : to stmts vector. The 2nd loop will always add stmt to stmts vector, since
6066 : gimple_bb (top_stmt) dominates gimple_bb (stmt), so the
6067 : 2nd loop ends up adding at-least single trunc_mod_expr stmt. */
6068 :
6069 327271 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, top_op1)
6070 : {
6071 288789 : if (is_gimple_assign (use_stmt)
6072 238416 : && (gimple_assign_rhs_code (use_stmt) == TRUNC_DIV_EXPR
6073 226674 : || gimple_assign_rhs_code (use_stmt) == TRUNC_MOD_EXPR)
6074 216250 : && operand_equal_p (top_op1, gimple_assign_rhs1 (use_stmt), 0)
6075 504922 : && operand_equal_p (top_op2, gimple_assign_rhs2 (use_stmt), 0))
6076 : {
6077 88686 : if (use_stmt == top_stmt
6078 11632 : || stmt_can_throw_internal (cfun, use_stmt)
6079 61746 : || !dominated_by_p (CDI_DOMINATORS, gimple_bb (use_stmt), top_bb))
6080 38572 : continue;
6081 :
6082 11542 : stmts.safe_push (use_stmt);
6083 11542 : if (gimple_assign_rhs_code (use_stmt) == TRUNC_DIV_EXPR)
6084 288789 : div_seen = true;
6085 : }
6086 38482 : }
6087 :
6088 38482 : if (!div_seen)
6089 : return false;
6090 :
6091 : /* Part 3: Create libcall to internal fn DIVMOD:
6092 : divmod_tmp = DIVMOD (op1, op2). */
6093 :
6094 11515 : gcall *call_stmt = gimple_build_call_internal (IFN_DIVMOD, 2, op1, op2);
6095 11515 : tree res = make_temp_ssa_name (build_complex_type (TREE_TYPE (op1)),
6096 : call_stmt, "divmod_tmp");
6097 11515 : gimple_call_set_lhs (call_stmt, res);
6098 : /* We rejected throwing statements above. */
6099 11515 : gimple_call_set_nothrow (call_stmt, true);
6100 :
6101 : /* Insert the call before top_stmt. */
6102 11515 : gimple_stmt_iterator top_stmt_gsi = gsi_for_stmt (top_stmt);
6103 11515 : gsi_insert_before (&top_stmt_gsi, call_stmt, GSI_SAME_STMT);
6104 :
6105 11515 : widen_mul_stats.divmod_calls_inserted++;
6106 :
6107 : /* Update all statements in stmts vector:
6108 : lhs = op1 TRUNC_DIV_EXPR op2 -> lhs = REALPART_EXPR<divmod_tmp>
6109 : lhs = op1 TRUNC_MOD_EXPR op2 -> lhs = IMAGPART_EXPR<divmod_tmp>. */
6110 :
6111 73052 : for (unsigned i = 0; stmts.iterate (i, &use_stmt); ++i)
6112 : {
6113 23055 : tree new_rhs;
6114 :
6115 23055 : switch (gimple_assign_rhs_code (use_stmt))
6116 : {
6117 11525 : case TRUNC_DIV_EXPR:
6118 11525 : new_rhs = fold_build1 (REALPART_EXPR, TREE_TYPE (op1), res);
6119 11525 : break;
6120 :
6121 11530 : case TRUNC_MOD_EXPR:
6122 11530 : new_rhs = fold_build1 (IMAGPART_EXPR, TREE_TYPE (op1), res);
6123 11530 : break;
6124 :
6125 0 : default:
6126 0 : gcc_unreachable ();
6127 : }
6128 :
6129 23055 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
6130 23055 : gimple_assign_set_rhs_from_tree (&gsi, new_rhs);
6131 23055 : update_stmt (use_stmt);
6132 : }
6133 :
6134 : return true;
6135 38482 : }
6136 :
6137 : /* Process a single gimple assignment STMT, which has a RSHIFT_EXPR as
6138 : its rhs, and try to convert it into a MULT_HIGHPART_EXPR. The return
6139 : value is true iff we converted the statement. */
6140 :
6141 : static bool
6142 176107 : convert_mult_to_highpart (gassign *stmt, gimple_stmt_iterator *gsi)
6143 : {
6144 176107 : tree lhs = gimple_assign_lhs (stmt);
6145 176107 : tree stype = TREE_TYPE (lhs);
6146 176107 : tree sarg0 = gimple_assign_rhs1 (stmt);
6147 176107 : tree sarg1 = gimple_assign_rhs2 (stmt);
6148 :
6149 176107 : if (TREE_CODE (stype) != INTEGER_TYPE
6150 167761 : || TREE_CODE (sarg1) != INTEGER_CST
6151 150764 : || TREE_CODE (sarg0) != SSA_NAME
6152 150763 : || !tree_fits_uhwi_p (sarg1)
6153 326870 : || !has_single_use (sarg0))
6154 : return false;
6155 :
6156 49066 : gassign *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (sarg0));
6157 45901 : if (!def)
6158 : return false;
6159 :
6160 45901 : enum tree_code mcode = gimple_assign_rhs_code (def);
6161 45901 : if (mcode == NOP_EXPR)
6162 : {
6163 11025 : tree tmp = gimple_assign_rhs1 (def);
6164 11025 : if (TREE_CODE (tmp) != SSA_NAME || !has_single_use (tmp))
6165 : return false;
6166 3712 : def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (tmp));
6167 3424 : if (!def)
6168 : return false;
6169 3424 : mcode = gimple_assign_rhs_code (def);
6170 : }
6171 :
6172 38300 : if (mcode != WIDEN_MULT_EXPR
6173 38300 : || gimple_bb (def) != gimple_bb (stmt))
6174 : return false;
6175 2582 : tree mtype = TREE_TYPE (gimple_assign_lhs (def));
6176 2582 : if (TREE_CODE (mtype) != INTEGER_TYPE
6177 2582 : || TYPE_PRECISION (mtype) != TYPE_PRECISION (stype))
6178 : return false;
6179 :
6180 2582 : tree mop1 = gimple_assign_rhs1 (def);
6181 2582 : tree mop2 = gimple_assign_rhs2 (def);
6182 2582 : tree optype = TREE_TYPE (mop1);
6183 2582 : bool unsignedp = TYPE_UNSIGNED (optype);
6184 2582 : unsigned int prec = TYPE_PRECISION (optype);
6185 :
6186 2582 : if (unsignedp != TYPE_UNSIGNED (mtype)
6187 2582 : || TYPE_PRECISION (mtype) != 2 * prec)
6188 : return false;
6189 :
6190 2582 : unsigned HOST_WIDE_INT bits = tree_to_uhwi (sarg1);
6191 2582 : if (bits < prec || bits >= 2 * prec)
6192 : return false;
6193 :
6194 : /* For the time being, require operands to have the same sign. */
6195 2581 : if (unsignedp != TYPE_UNSIGNED (TREE_TYPE (mop2)))
6196 : return false;
6197 :
6198 2581 : machine_mode mode = TYPE_MODE (optype);
6199 2581 : optab tab = unsignedp ? umul_highpart_optab : smul_highpart_optab;
6200 2581 : if (optab_handler (tab, mode) == CODE_FOR_nothing)
6201 : return false;
6202 :
6203 2581 : location_t loc = gimple_location (stmt);
6204 2581 : tree highpart1 = build_and_insert_binop (gsi, loc, "highparttmp",
6205 : MULT_HIGHPART_EXPR, mop1, mop2);
6206 2581 : tree highpart2 = highpart1;
6207 2581 : tree ntype = optype;
6208 :
6209 2581 : if (TYPE_UNSIGNED (stype) != TYPE_UNSIGNED (optype))
6210 : {
6211 16 : ntype = TYPE_UNSIGNED (stype) ? unsigned_type_for (optype)
6212 7 : : signed_type_for (optype);
6213 16 : highpart2 = build_and_insert_cast (gsi, loc, ntype, highpart1);
6214 : }
6215 2581 : if (bits > prec)
6216 29 : highpart2 = build_and_insert_binop (gsi, loc, "highparttmp",
6217 : RSHIFT_EXPR, highpart2,
6218 29 : build_int_cst (ntype, bits - prec));
6219 :
6220 2581 : gassign *new_stmt = gimple_build_assign (lhs, NOP_EXPR, highpart2);
6221 2581 : gsi_replace (gsi, new_stmt, true);
6222 :
6223 2581 : widen_mul_stats.highpart_mults_inserted++;
6224 2581 : return true;
6225 : }
6226 :
6227 : /* If target has spaceship<MODE>3 expander, pattern recognize
6228 : <bb 2> [local count: 1073741824]:
6229 : if (a_2(D) == b_3(D))
6230 : goto <bb 6>; [34.00%]
6231 : else
6232 : goto <bb 3>; [66.00%]
6233 :
6234 : <bb 3> [local count: 708669601]:
6235 : if (a_2(D) < b_3(D))
6236 : goto <bb 6>; [1.04%]
6237 : else
6238 : goto <bb 4>; [98.96%]
6239 :
6240 : <bb 4> [local count: 701299439]:
6241 : if (a_2(D) > b_3(D))
6242 : goto <bb 5>; [48.89%]
6243 : else
6244 : goto <bb 6>; [51.11%]
6245 :
6246 : <bb 5> [local count: 342865295]:
6247 :
6248 : <bb 6> [local count: 1073741824]:
6249 : and turn it into:
6250 : <bb 2> [local count: 1073741824]:
6251 : _1 = .SPACESHIP (a_2(D), b_3(D), 0);
6252 : if (_1 == 0)
6253 : goto <bb 6>; [34.00%]
6254 : else
6255 : goto <bb 3>; [66.00%]
6256 :
6257 : <bb 3> [local count: 708669601]:
6258 : if (_1 == -1)
6259 : goto <bb 6>; [1.04%]
6260 : else
6261 : goto <bb 4>; [98.96%]
6262 :
6263 : <bb 4> [local count: 701299439]:
6264 : if (_1 == 1)
6265 : goto <bb 5>; [48.89%]
6266 : else
6267 : goto <bb 6>; [51.11%]
6268 :
6269 : <bb 5> [local count: 342865295]:
6270 :
6271 : <bb 6> [local count: 1073741824]:
6272 : so that the backend can emit optimal comparison and
6273 : conditional jump sequence. If the
6274 : <bb 6> [local count: 1073741824]:
6275 : above has a single PHI like:
6276 : # _27 = PHI<0(2), -1(3), -128(4), 1(5)>
6277 : then replace it with effectively
6278 : _1 = .SPACESHIP (a_2(D), b_3(D), -128);
6279 : _27 = _1; */
6280 :
6281 : static void
6282 4258847 : optimize_spaceship (gcond *stmt)
6283 : {
6284 4258847 : enum tree_code code = gimple_cond_code (stmt);
6285 4258847 : if (code != EQ_EXPR && code != NE_EXPR)
6286 4258725 : return;
6287 3465662 : tree arg1 = gimple_cond_lhs (stmt);
6288 3465662 : tree arg2 = gimple_cond_rhs (stmt);
6289 3465662 : if ((!SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg1))
6290 3354530 : && !INTEGRAL_TYPE_P (TREE_TYPE (arg1)))
6291 2671370 : || optab_handler (spaceship_optab,
6292 2671370 : TYPE_MODE (TREE_TYPE (arg1))) == CODE_FOR_nothing
6293 6096242 : || operand_equal_p (arg1, arg2, 0))
6294 : return;
6295 :
6296 2629365 : basic_block bb0 = gimple_bb (stmt), bb1, bb2 = NULL;
6297 2629365 : edge em1 = NULL, e1 = NULL, e2 = NULL;
6298 2629365 : bb1 = EDGE_SUCC (bb0, 1)->dest;
6299 2629365 : if (((EDGE_SUCC (bb0, 0)->flags & EDGE_TRUE_VALUE) != 0) ^ (code == EQ_EXPR))
6300 1553565 : bb1 = EDGE_SUCC (bb0, 0)->dest;
6301 :
6302 9475123 : gcond *g = safe_dyn_cast <gcond *> (*gsi_last_bb (bb1));
6303 1151664 : if (g == NULL
6304 4869509 : || !single_pred_p (bb1)
6305 729292 : || (operand_equal_p (gimple_cond_lhs (g), arg1, 0)
6306 610784 : ? !operand_equal_p (gimple_cond_rhs (g), arg2, 0)
6307 492276 : : (!operand_equal_p (gimple_cond_lhs (g), arg2, 0)
6308 945 : || !operand_equal_p (gimple_cond_rhs (g), arg1, 0)))
6309 623257 : || !cond_only_block_p (bb1))
6310 : return;
6311 :
6312 11750 : enum tree_code ccode = (operand_equal_p (gimple_cond_lhs (g), arg1, 0)
6313 11750 : ? LT_EXPR : GT_EXPR);
6314 11750 : switch (gimple_cond_code (g))
6315 : {
6316 : case LT_EXPR:
6317 : case LE_EXPR:
6318 : break;
6319 10231 : case GT_EXPR:
6320 10231 : case GE_EXPR:
6321 10231 : ccode = ccode == LT_EXPR ? GT_EXPR : LT_EXPR;
6322 : break;
6323 : default:
6324 : return;
6325 : }
6326 :
6327 35178 : for (int i = 0; i < 2; ++i)
6328 : {
6329 : /* With NaNs, </<=/>/>= are false, so we need to look for the
6330 : third comparison on the false edge from whatever non-equality
6331 : comparison the second comparison is. */
6332 23494 : if (HONOR_NANS (TREE_TYPE (arg1))
6333 23494 : && (EDGE_SUCC (bb1, i)->flags & EDGE_TRUE_VALUE) != 0)
6334 129 : continue;
6335 :
6336 23365 : bb2 = EDGE_SUCC (bb1, i)->dest;
6337 69696 : g = safe_dyn_cast <gcond *> (*gsi_last_bb (bb2));
6338 15780 : if (g == NULL
6339 15780 : || !single_pred_p (bb2)
6340 19539 : || (operand_equal_p (gimple_cond_lhs (g), arg1, 0)
6341 11733 : ? !operand_equal_p (gimple_cond_rhs (g), arg2, 0)
6342 3927 : : (!operand_equal_p (gimple_cond_lhs (g), arg2, 0)
6343 19 : || !operand_equal_p (gimple_cond_rhs (g), arg1, 0)))
6344 66 : || !cond_only_block_p (bb2)
6345 11799 : || EDGE_SUCC (bb2, 0)->dest == EDGE_SUCC (bb2, 1)->dest)
6346 23299 : continue;
6347 :
6348 66 : enum tree_code ccode2
6349 66 : = (operand_equal_p (gimple_cond_lhs (g), arg1, 0) ? LT_EXPR : GT_EXPR);
6350 66 : switch (gimple_cond_code (g))
6351 : {
6352 : case LT_EXPR:
6353 : case LE_EXPR:
6354 : break;
6355 41 : case GT_EXPR:
6356 41 : case GE_EXPR:
6357 41 : ccode2 = ccode2 == LT_EXPR ? GT_EXPR : LT_EXPR;
6358 : break;
6359 0 : default:
6360 0 : continue;
6361 : }
6362 66 : if (HONOR_NANS (TREE_TYPE (arg1)) && ccode == ccode2)
6363 0 : continue;
6364 :
6365 132 : if ((ccode == LT_EXPR)
6366 66 : ^ ((EDGE_SUCC (bb1, i)->flags & EDGE_TRUE_VALUE) != 0))
6367 : {
6368 41 : em1 = EDGE_SUCC (bb1, 1 - i);
6369 41 : e1 = EDGE_SUCC (bb2, 0);
6370 41 : e2 = EDGE_SUCC (bb2, 1);
6371 41 : if ((ccode2 == LT_EXPR) ^ ((e1->flags & EDGE_TRUE_VALUE) == 0))
6372 0 : std::swap (e1, e2);
6373 : }
6374 : else
6375 : {
6376 25 : e1 = EDGE_SUCC (bb1, 1 - i);
6377 25 : em1 = EDGE_SUCC (bb2, 0);
6378 25 : e2 = EDGE_SUCC (bb2, 1);
6379 25 : if ((ccode2 != LT_EXPR) ^ ((em1->flags & EDGE_TRUE_VALUE) == 0))
6380 : std::swap (em1, e2);
6381 : }
6382 : break;
6383 : }
6384 :
6385 11725 : if (em1 == NULL)
6386 : {
6387 23368 : if ((ccode == LT_EXPR)
6388 11684 : ^ ((EDGE_SUCC (bb1, 0)->flags & EDGE_TRUE_VALUE) != 0))
6389 : {
6390 4651 : em1 = EDGE_SUCC (bb1, 1);
6391 4651 : e1 = EDGE_SUCC (bb1, 0);
6392 4651 : e2 = (e1->flags & EDGE_TRUE_VALUE) ? em1 : e1;
6393 : }
6394 : else
6395 : {
6396 7033 : em1 = EDGE_SUCC (bb1, 0);
6397 7033 : e1 = EDGE_SUCC (bb1, 1);
6398 7033 : e2 = (e1->flags & EDGE_TRUE_VALUE) ? em1 : e1;
6399 : }
6400 : }
6401 :
6402 : /* Check if there is a single bb into which all failed conditions
6403 : jump to (perhaps through an empty block) and if it results in
6404 : a single integral PHI which just sets it to -1, 0, 1, X
6405 : (or -1, 0, 1 when NaNs can't happen). In that case use 1 rather
6406 : than 0 as last .SPACESHIP argument to tell backends it might
6407 : consider different code generation and just cast the result
6408 : of .SPACESHIP to the PHI result. X above is some value
6409 : other than -1, 0, 1, for libstdc++ -128, for libc++ -127. */
6410 11750 : tree arg3 = integer_zero_node;
6411 11750 : edge e = EDGE_SUCC (bb0, 0);
6412 11750 : if (e->dest == bb1)
6413 9037 : e = EDGE_SUCC (bb0, 1);
6414 11750 : basic_block bbp = e->dest;
6415 11750 : gphi *phi = NULL;
6416 11750 : for (gphi_iterator psi = gsi_start_phis (bbp);
6417 13905 : !gsi_end_p (psi); gsi_next (&psi))
6418 : {
6419 3715 : gphi *gp = psi.phi ();
6420 3715 : tree res = gimple_phi_result (gp);
6421 :
6422 3715 : if (phi != NULL
6423 3368 : || virtual_operand_p (res)
6424 2445 : || !INTEGRAL_TYPE_P (TREE_TYPE (res))
6425 6003 : || TYPE_PRECISION (TREE_TYPE (res)) < 2)
6426 : {
6427 : phi = NULL;
6428 : break;
6429 : }
6430 2155 : phi = gp;
6431 : }
6432 11750 : if (phi
6433 1808 : && integer_zerop (gimple_phi_arg_def_from_edge (phi, e))
6434 12326 : && EDGE_COUNT (bbp->preds) == (HONOR_NANS (TREE_TYPE (arg1)) ? 4 : 3))
6435 : {
6436 122 : HOST_WIDE_INT argval
6437 122 : = SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg1)) ? -128 : -1;
6438 708 : for (unsigned i = 0; phi && i < EDGE_COUNT (bbp->preds) - 1; ++i)
6439 : {
6440 255 : edge e3 = i == 0 ? e1 : i == 1 ? em1 : e2;
6441 255 : if (e3->dest != bbp)
6442 : {
6443 117 : if (!empty_block_p (e3->dest)
6444 108 : || !single_succ_p (e3->dest)
6445 225 : || single_succ (e3->dest) != bbp)
6446 : {
6447 : phi = NULL;
6448 : break;
6449 : }
6450 : e3 = single_succ_edge (e3->dest);
6451 : }
6452 246 : tree a = gimple_phi_arg_def_from_edge (phi, e3);
6453 246 : if (TREE_CODE (a) != INTEGER_CST
6454 246 : || (i == 0 && !integer_onep (a))
6455 482 : || (i == 1 && !integer_all_onesp (a)))
6456 : {
6457 : phi = NULL;
6458 : break;
6459 : }
6460 236 : if (i == 2)
6461 : {
6462 30 : tree minv = TYPE_MIN_VALUE (signed_char_type_node);
6463 30 : tree maxv = TYPE_MAX_VALUE (signed_char_type_node);
6464 30 : widest_int w = widest_int::from (wi::to_wide (a), SIGNED);
6465 41 : if ((w >= -1 && w <= 1)
6466 26 : || w < wi::to_widest (minv)
6467 60 : || w > wi::to_widest (maxv))
6468 : {
6469 4 : phi = NULL;
6470 4 : break;
6471 : }
6472 26 : argval = w.to_shwi ();
6473 26 : }
6474 : }
6475 122 : if (phi)
6476 99 : arg3 = build_int_cst (integer_type_node,
6477 119 : TYPE_UNSIGNED (TREE_TYPE (arg1)) ? 1 : argval);
6478 : }
6479 :
6480 : /* For integral <=> comparisons only use .SPACESHIP if it is turned
6481 : into an integer (-1, 0, 1). */
6482 11750 : if (!SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg1)) && arg3 == integer_zero_node)
6483 : return;
6484 :
6485 221 : gcall *gc = gimple_build_call_internal (IFN_SPACESHIP, 3, arg1, arg2, arg3);
6486 221 : tree lhs = make_ssa_name (integer_type_node);
6487 221 : gimple_call_set_lhs (gc, lhs);
6488 221 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6489 221 : gsi_insert_before (&gsi, gc, GSI_SAME_STMT);
6490 :
6491 343 : wide_int wmin = wi::minus_one (TYPE_PRECISION (integer_type_node));
6492 343 : wide_int wmax = wi::one (TYPE_PRECISION (integer_type_node));
6493 221 : if (HONOR_NANS (TREE_TYPE (arg1)))
6494 : {
6495 129 : if (arg3 == integer_zero_node)
6496 103 : wmin = wi::shwi (-128, TYPE_PRECISION (integer_type_node));
6497 26 : else if (tree_int_cst_sgn (arg3) < 0)
6498 19 : wmin = wi::to_wide (arg3);
6499 : else
6500 7 : wmax = wi::to_wide (arg3);
6501 : }
6502 343 : int_range<1> vr (TREE_TYPE (lhs), wmin, wmax);
6503 221 : set_range_info (lhs, vr);
6504 :
6505 221 : if (arg3 != integer_zero_node)
6506 : {
6507 99 : tree type = TREE_TYPE (gimple_phi_result (phi));
6508 99 : if (!useless_type_conversion_p (type, integer_type_node))
6509 : {
6510 63 : tree tem = make_ssa_name (type);
6511 63 : gimple *gcv = gimple_build_assign (tem, NOP_EXPR, lhs);
6512 63 : gsi_insert_before (&gsi, gcv, GSI_SAME_STMT);
6513 63 : lhs = tem;
6514 : }
6515 99 : SET_PHI_ARG_DEF_ON_EDGE (phi, e, lhs);
6516 99 : gimple_cond_set_lhs (stmt, boolean_false_node);
6517 99 : gimple_cond_set_rhs (stmt, boolean_false_node);
6518 185 : gimple_cond_set_code (stmt, (e->flags & EDGE_TRUE_VALUE)
6519 : ? EQ_EXPR : NE_EXPR);
6520 99 : update_stmt (stmt);
6521 99 : return;
6522 : }
6523 :
6524 122 : gimple_cond_set_lhs (stmt, lhs);
6525 122 : gimple_cond_set_rhs (stmt, integer_zero_node);
6526 122 : update_stmt (stmt);
6527 :
6528 244 : gcond *cond = as_a <gcond *> (*gsi_last_bb (bb1));
6529 122 : gimple_cond_set_lhs (cond, lhs);
6530 122 : if (em1->src == bb1 && e2 != em1)
6531 : {
6532 67 : gimple_cond_set_rhs (cond, integer_minus_one_node);
6533 73 : gimple_cond_set_code (cond, (em1->flags & EDGE_TRUE_VALUE)
6534 : ? EQ_EXPR : NE_EXPR);
6535 : }
6536 : else
6537 : {
6538 55 : gcc_assert (e1->src == bb1 && e2 != e1);
6539 55 : gimple_cond_set_rhs (cond, integer_one_node);
6540 55 : gimple_cond_set_code (cond, (e1->flags & EDGE_TRUE_VALUE)
6541 : ? EQ_EXPR : NE_EXPR);
6542 : }
6543 122 : update_stmt (cond);
6544 :
6545 122 : if (e2 != e1 && e2 != em1)
6546 : {
6547 80 : cond = as_a <gcond *> (*gsi_last_bb (bb2));
6548 40 : gimple_cond_set_lhs (cond, lhs);
6549 40 : if (em1->src == bb2)
6550 25 : gimple_cond_set_rhs (cond, integer_minus_one_node);
6551 : else
6552 : {
6553 15 : gcc_assert (e1->src == bb2);
6554 15 : gimple_cond_set_rhs (cond, integer_one_node);
6555 : }
6556 40 : gimple_cond_set_code (cond,
6557 40 : (e2->flags & EDGE_TRUE_VALUE) ? NE_EXPR : EQ_EXPR);
6558 40 : update_stmt (cond);
6559 : }
6560 : }
6561 :
6562 :
6563 : /* Long-multiply inverse-lowering helper.
6564 :
6565 : The forwprop long-multiply recognizer canonicalizes a hand-written
6566 : longhand high-part multiply into a cast+mult+shift+cast chain
6567 : `(N) ((2N) a * (2N) b) >> N'. When the target lacks an expansion
6568 : path for the wide form, `lower_long_mul_high_chain' resynthesizes
6569 : the longhand at narrow precision via `build_long_mul_partials'. */
6570 :
6571 : /* Test whether the target supports an (HALF)-by-(HALF)->NARROW unsigned
6572 : widening multiply. Returns true on success, with the half-width
6573 : scalar int mode placed in *HALF_MODE. */
6574 :
6575 : static bool
6576 2231 : can_widen_to_narrow_p (scalar_int_mode narrow_mode, unsigned int half_width,
6577 : scalar_int_mode *half_mode)
6578 : {
6579 2231 : if (!int_mode_for_size (half_width, 0).exists (half_mode))
6580 0 : return false;
6581 2231 : return convert_optab_handler (umul_widen_optab, narrow_mode, *half_mode)
6582 2231 : != CODE_FOR_nothing;
6583 : }
6584 :
6585 : static bool long_mul_op_fits_p (tree, unsigned, bitmap);
6586 :
6587 : /* Build into *SEQ the (N/2)-bit halves *LO and *HI of OP, in the form
6588 : USE_WIDEN selects. An operand provably within N/2 bits has a zero
6589 : high half: *HI is left null. */
6590 :
6591 : static void
6592 4462 : build_long_mul_partial_operand (gimple_seq *seq, location_t loc, tree op,
6593 : tree half_type, tree half_amt, bool use_widen,
6594 : tree *lo, tree *hi)
6595 : {
6596 4462 : tree acc_type = TREE_TYPE (op);
6597 4462 : unsigned int half_prec = TYPE_PRECISION (half_type);
6598 4462 : auto_bitmap phi_seen;
6599 4462 : bool fits = long_mul_op_fits_p (op, half_prec, phi_seen);
6600 :
6601 4462 : *hi = NULL_TREE;
6602 4462 : if (!fits)
6603 4264 : *hi = gimple_build (seq, loc, RSHIFT_EXPR, acc_type, op, half_amt);
6604 :
6605 4462 : if (use_widen)
6606 : {
6607 4462 : *lo = gimple_build (seq, loc, NOP_EXPR, half_type, op);
6608 4462 : if (*hi)
6609 4264 : *hi = gimple_build (seq, loc, NOP_EXPR, half_type, *hi);
6610 : }
6611 0 : else if (fits)
6612 0 : *lo = op;
6613 : else
6614 : {
6615 0 : tree mask = wide_int_to_tree (acc_type,
6616 0 : wi::mask (half_prec, false,
6617 0 : TYPE_PRECISION (acc_type)));
6618 0 : *lo = gimple_build (seq, loc, BIT_AND_EXPR, acc_type, op, mask);
6619 : }
6620 4462 : }
6621 :
6622 : /* Append to *SEQ the operand split and partial products for an unsigned
6623 : long multiply of OP1 by OP2 at the precision of TREE_TYPE (OP1).
6624 : HALF_TYPE is the (N/2)-bit unsigned type; HALF_AMT is the integer-typed
6625 : shift constant equal to N/2.
6626 :
6627 : Outputs the four partial products via *LOLO, *HILO, *LOHI, *HIHI.
6628 : An operand provably within N/2 bits contributes literal-zero high
6629 : partials.
6630 :
6631 : USE_WIDEN selects the partial-product form:
6632 : true - cast halves to HALF_TYPE and use WIDEN_MULT_EXPR (needs
6633 : an (N/2)-by-(N/2)->N widening multiply optab).
6634 : false - mask/shift halves within the N-bit accumulator and use
6635 : plain MULT_EXPR; the halves fit in N/2 bits so the N-bit
6636 : low product is exact. */
6637 :
6638 : static void
6639 2231 : build_long_mul_partials (gimple_seq *seq, location_t loc, tree op1, tree op2,
6640 : tree half_type, tree half_amt,
6641 : tree *lolo, tree *hilo, tree *lohi, tree *hihi,
6642 : bool use_widen)
6643 : {
6644 2231 : tree acc_type = TREE_TYPE (op1);
6645 2231 : tree zero = build_zero_cst (acc_type);
6646 2231 : tree_code mul_code = use_widen ? WIDEN_MULT_EXPR : MULT_EXPR;
6647 2231 : tree op1_lo, op1_hi, op2_lo, op2_hi;
6648 :
6649 2231 : build_long_mul_partial_operand (seq, loc, op1, half_type, half_amt, use_widen,
6650 : &op1_lo, &op1_hi);
6651 2231 : build_long_mul_partial_operand (seq, loc, op2, half_type, half_amt, use_widen,
6652 : &op2_lo, &op2_hi);
6653 :
6654 2231 : *lolo = gimple_build (seq, loc, mul_code, acc_type, op1_lo, op2_lo);
6655 4462 : *hilo = op1_hi
6656 2231 : ? gimple_build (seq, loc, mul_code, acc_type, op1_hi, op2_lo) : zero;
6657 4462 : *lohi = op2_hi
6658 2231 : ? gimple_build (seq, loc, mul_code, acc_type, op1_lo, op2_hi) : zero;
6659 2222 : *hihi = op1_hi && op2_hi
6660 4264 : ? gimple_build (seq, loc, mul_code, acc_type, op1_hi, op2_hi) : zero;
6661 2231 : }
6662 :
6663 : /* Emit into *SEQ the high N bits of the unsigned product A * B, where A and B
6664 : are NARROW_TYPE (N-bit) values, as a longhand over (N/2)-bit partials.
6665 : Returns the high-part SSA. */
6666 :
6667 : static tree
6668 2231 : emit_long_mul_highpart (gimple_seq *seq, location_t loc, tree a, tree b,
6669 : tree narrow_type)
6670 : {
6671 2231 : scalar_int_mode narrow_mode
6672 2231 : = as_a <scalar_int_mode> (TYPE_MODE (narrow_type));
6673 2231 : unsigned int half_width = GET_MODE_PRECISION (narrow_mode) / 2;
6674 : /* Prefer (N/2)-by-(N/2)->N widening partials; fall back to plain MULT_EXPR
6675 : when the target lacks the widen optab. See build_long_mul_partials. */
6676 2231 : scalar_int_mode half_mode;
6677 2231 : bool use_widen = can_widen_to_narrow_p (narrow_mode, half_width, &half_mode);
6678 2231 : tree half_type = build_nonstandard_integer_type (half_width, 1);
6679 2231 : tree half_amt = build_int_cst (integer_type_node, half_width);
6680 2231 : tree half_mask = wide_int_to_tree (narrow_type,
6681 2231 : wi::mask (half_width, false,
6682 2231 : TYPE_PRECISION (narrow_type)));
6683 :
6684 2231 : tree lolo, hilo, lohi, hihi;
6685 2231 : build_long_mul_partials (seq, loc, a, b, half_type, half_amt,
6686 : &lolo, &hilo, &lohi, &hihi, use_widen);
6687 2231 : tree cross_sum = gimple_build (seq, loc, PLUS_EXPR, narrow_type, hilo, lohi);
6688 2231 : tree cross_lt = gimple_build (seq, loc, LT_EXPR, boolean_type_node,
6689 : cross_sum, hilo);
6690 2231 : tree cross_lt_n = gimple_build (seq, loc, NOP_EXPR, narrow_type, cross_lt);
6691 2231 : tree cross_carry = gimple_build (seq, loc, LSHIFT_EXPR, narrow_type,
6692 : cross_lt_n, half_amt);
6693 2231 : tree lolo_hi = gimple_build (seq, loc, RSHIFT_EXPR, narrow_type,
6694 : lolo, half_amt);
6695 2231 : tree cross_lo = gimple_build (seq, loc, BIT_AND_EXPR, narrow_type,
6696 : cross_sum, half_mask);
6697 2231 : tree low_accum = gimple_build (seq, loc, PLUS_EXPR, narrow_type,
6698 : lolo_hi, cross_lo);
6699 2231 : tree low_accum_hi = gimple_build (seq, loc, RSHIFT_EXPR, narrow_type,
6700 : low_accum, half_amt);
6701 2231 : tree cross_hi = gimple_build (seq, loc, RSHIFT_EXPR, narrow_type,
6702 : cross_sum, half_amt);
6703 2231 : tree t1 = gimple_build (seq, loc, PLUS_EXPR, narrow_type, hihi, cross_hi);
6704 2231 : tree t2 = gimple_build (seq, loc, PLUS_EXPR, narrow_type, t1, low_accum_hi);
6705 2231 : return gimple_build (seq, loc, PLUS_EXPR, narrow_type, t2, cross_carry);
6706 : }
6707 :
6708 : /* Emit into *SEQ the high N bits (NARROW_TYPE) of the unsigned product of two
6709 : 2N-bit values given as N-bit halves, x = L1 + H1*2^N and y = L2 + H2*2^N:
6710 : the high half of x*y is the high N bits of L1*L2, plus H1*L2 and L1*H2, all
6711 : mod 2^N. */
6712 :
6713 : static tree
6714 2231 : combine_long_mul_halves (gimple_seq *seq, location_t loc, tree l1, tree h1,
6715 : tree l2, tree h2, tree narrow_type)
6716 : {
6717 2231 : tree hh = emit_long_mul_highpart (seq, loc, l1, l2, narrow_type);
6718 2231 : tree c1 = gimple_build (seq, loc, MULT_EXPR, narrow_type, h1, l2);
6719 2231 : tree c2 = gimple_build (seq, loc, MULT_EXPR, narrow_type, l1, h2);
6720 2231 : tree s = gimple_build (seq, loc, PLUS_EXPR, narrow_type, hh, c1);
6721 2231 : return gimple_build (seq, loc, PLUS_EXPR, narrow_type, s, c2);
6722 : }
6723 :
6724 : /* True when OP fits NARROW_PREC bits as an unsigned value. Looks
6725 : through widening casts and PHIs, falling back to `tree_nonzero_bits'
6726 : otherwise. PHI_SEEN guards against cycles. */
6727 :
6728 : static bool
6729 6132 : long_mul_op_fits_p (tree op, unsigned narrow_prec, bitmap phi_seen)
6730 : {
6731 6570 : if (!TYPE_UNSIGNED (TREE_TYPE (op)))
6732 : return false;
6733 6555 : if (TYPE_PRECISION (TREE_TYPE (op)) <= narrow_prec)
6734 : return true;
6735 6249 : if (TREE_CODE (op) == SSA_NAME)
6736 : {
6737 5064 : gimple *def = SSA_NAME_DEF_STMT (op);
6738 5064 : if (is_gimple_assign (def)
6739 5064 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
6740 438 : return long_mul_op_fits_p (gimple_assign_rhs1 (def), narrow_prec,
6741 438 : phi_seen);
6742 4626 : if (gphi *phi = dyn_cast <gphi *> (def))
6743 724 : if (bitmap_set_bit (phi_seen, SSA_NAME_VERSION (op)))
6744 : {
6745 1145 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
6746 1076 : if (!long_mul_op_fits_p (gimple_phi_arg_def (phi, i),
6747 : narrow_prec, phi_seen))
6748 : return false;
6749 : return true;
6750 : }
6751 : }
6752 5087 : return wi::min_precision (tree_nonzero_bits (op), UNSIGNED) <= narrow_prec;
6753 : }
6754 :
6755 : static bool long_mul_split_operand (gimple_seq *, location_t, tree, tree,
6756 : tree *, tree *);
6757 :
6758 : struct long_mul_halves
6759 : {
6760 : tree lo;
6761 : tree hi;
6762 : };
6763 :
6764 : /* Halves recorded for one run of the pass, keyed on the PHI they came from.
6765 : Several chains can reach one operand PHI, and each that splits it again
6766 : leaves another redundant pair of half PHIs behind. */
6767 :
6768 : static hash_map<tree, long_mul_halves> *long_mul_phi_halves;
6769 :
6770 : struct long_mul_arg_split
6771 : {
6772 : gimple_seq seq;
6773 : tree lo;
6774 : tree hi;
6775 : };
6776 :
6777 : /* Split the 2N-bit result of PHI into N-bit halves *LO and *HI, by splitting
6778 : each argument and merging the halves with two new PHIs. A split goes at
6779 : the end of its argument's incoming block, where the argument is available,
6780 : rather than on the edge, which could split a critical edge while the
6781 : dominator walk is still running. Returns false, having changed nothing,
6782 : when an argument cannot be split. A split that succeeds stands even if
6783 : the caller then gives up. */
6784 :
6785 : static bool
6786 0 : long_mul_split_phi (gphi *phi, tree narrow_type, tree *lo, tree *hi)
6787 : {
6788 0 : tree res = gimple_phi_result (phi);
6789 0 : if (long_mul_halves *prev = long_mul_phi_halves->get (res))
6790 : {
6791 : /* Every operand reaching a split has the 2N type, so the width is
6792 : fixed by the PHI's own type. */
6793 0 : gcc_checking_assert (types_compatible_p (TREE_TYPE (prev->lo),
6794 : narrow_type));
6795 0 : *lo = prev->lo;
6796 0 : *hi = prev->hi;
6797 0 : return true;
6798 : }
6799 :
6800 0 : unsigned int n = gimple_phi_num_args (phi);
6801 0 : location_t loc = gimple_location (phi);
6802 0 : basic_block bb = gimple_bb (phi);
6803 0 : auto_vec<long_mul_arg_split, 4> args;
6804 :
6805 0 : for (unsigned int i = 0; i < n; i++)
6806 : {
6807 0 : edge e = gimple_phi_arg_edge (phi, i);
6808 : /* A back edge could lead back to PHI and recurse forever. The entry
6809 : block cannot hold a split. */
6810 0 : if (dominated_by_p (CDI_DOMINATORS, e->src, bb)
6811 0 : || e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
6812 0 : return false;
6813 :
6814 0 : long_mul_arg_split arg = {};
6815 0 : if (!long_mul_split_operand (&arg.seq, loc, gimple_phi_arg_def (phi, i),
6816 : narrow_type, &arg.lo, &arg.hi))
6817 : return false;
6818 :
6819 0 : args.safe_push (arg);
6820 : }
6821 :
6822 : /* Every argument split, so the rewrite can be committed. */
6823 0 : gphi *lo_phi = create_phi_node (make_ssa_name (narrow_type), bb);
6824 0 : gphi *hi_phi = create_phi_node (make_ssa_name (narrow_type), bb);
6825 0 : for (unsigned int i = 0; i < n; i++)
6826 : {
6827 0 : edge e = gimple_phi_arg_edge (phi, i);
6828 0 : if (args[i].seq)
6829 : {
6830 0 : gimple_stmt_iterator gsi = gsi_last_bb (e->src);
6831 0 : if (!gsi_end_p (gsi) && stmt_ends_bb_p (gsi_stmt (gsi)))
6832 0 : gsi_insert_seq_before (&gsi, args[i].seq, GSI_SAME_STMT);
6833 : else
6834 0 : gsi_insert_seq_after (&gsi, args[i].seq, GSI_CONTINUE_LINKING);
6835 : }
6836 0 : add_phi_arg (lo_phi, args[i].lo, e, UNKNOWN_LOCATION);
6837 0 : add_phi_arg (hi_phi, args[i].hi, e, UNKNOWN_LOCATION);
6838 : }
6839 0 : *lo = gimple_phi_result (lo_phi);
6840 0 : *hi = gimple_phi_result (hi_phi);
6841 0 : long_mul_phi_halves->put (res, { *lo, *hi });
6842 :
6843 0 : if (dump_file && (dump_flags & TDF_DETAILS))
6844 0 : fprintf (dump_file, "Split long-multiply operand PHI.\n");
6845 : return true;
6846 0 : }
6847 :
6848 : /* Split the 2N-bit unsigned value OP into its low and high N bits (*LO and
6849 : *HI, both NARROW_TYPE) using only N-bit operations, as the target has no 2N
6850 : multiply or shift. A 2N product recurses on its operands, its high half
6851 : coming from combine_long_mul_halves. A value shifted down by N recurses on
6852 : the shifted value and takes its high half, rather than reading the 2N shift.
6853 : A widening cast's low half is the truncated source and its high half is what
6854 : the cast extended with, zero or the source's replicated sign bit. A value
6855 : that provably fits N bits has a zero high half. A PHI is split through its
6856 : arguments as a last resort. Returns false otherwise. */
6857 :
6858 : static bool
6859 4470 : long_mul_split_operand (gimple_seq *seq, location_t loc, tree op,
6860 : tree narrow_type, tree *lo, tree *hi)
6861 : {
6862 4470 : unsigned int narrow_prec = TYPE_PRECISION (narrow_type);
6863 4470 : if (TREE_CODE (op) == SSA_NAME)
6864 : {
6865 3943 : gimple *def = SSA_NAME_DEF_STMT (op);
6866 3943 : if (is_gimple_assign (def) && gimple_assign_rhs_code (def) == MULT_EXPR)
6867 : {
6868 13 : tree a_lo, a_hi, b_lo, b_hi;
6869 13 : if (!long_mul_split_operand (seq, loc, gimple_assign_rhs1 (def),
6870 : narrow_type, &a_lo, &a_hi)
6871 13 : || !long_mul_split_operand (seq, loc, gimple_assign_rhs2 (def),
6872 : narrow_type, &b_lo, &b_hi))
6873 : return false;
6874 13 : *lo = gimple_build (seq, loc, MULT_EXPR, narrow_type, a_lo, b_lo);
6875 13 : *hi = combine_long_mul_halves (seq, loc, a_lo, a_hi, b_lo, b_hi,
6876 : narrow_type);
6877 13 : return true;
6878 : }
6879 : /* A 2N value shifted down by N is its own high half: split the source
6880 : and use that half. Reading the shift instead leaves the 2N source
6881 : live, and the target cannot expand it. This has to come before the
6882 : widening-cast case below, which would take such a value as it
6883 : stands. */
6884 3930 : if (is_gimple_assign (def)
6885 3881 : && gimple_assign_rhs_code (def) == RSHIFT_EXPR
6886 8 : && tree_fits_uhwi_p (gimple_assign_rhs2 (def))
6887 3938 : && tree_to_uhwi (gimple_assign_rhs2 (def)) == narrow_prec)
6888 : {
6889 8 : tree src_lo, src_hi;
6890 8 : if (!long_mul_split_operand (seq, loc, gimple_assign_rhs1 (def),
6891 : narrow_type, &src_lo, &src_hi))
6892 : return false;
6893 8 : *lo = src_hi;
6894 8 : *hi = build_zero_cst (narrow_type);
6895 8 : return true;
6896 : }
6897 3922 : if (is_gimple_assign (def)
6898 3922 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
6899 : {
6900 3855 : tree src = gimple_assign_rhs1 (def);
6901 3855 : tree src_type = TREE_TYPE (src);
6902 3855 : if (INTEGRAL_TYPE_P (src_type)
6903 3855 : && TYPE_PRECISION (src_type) <= narrow_prec)
6904 : {
6905 3855 : *lo = gimple_convert (seq, loc, narrow_type, src);
6906 3855 : if (TYPE_UNSIGNED (src_type))
6907 3840 : *hi = build_zero_cst (narrow_type);
6908 : else
6909 : {
6910 : /* Sign extension: the high N bits replicate the sign bit. */
6911 15 : tree snarrow = signed_type_for (narrow_type);
6912 15 : tree s = gimple_convert (seq, loc, snarrow, *lo);
6913 15 : tree amt = build_int_cst (integer_type_node, narrow_prec - 1);
6914 15 : tree sh = gimple_build (seq, loc, RSHIFT_EXPR, snarrow, s,
6915 : amt);
6916 15 : *hi = gimple_convert (seq, loc, narrow_type, sh);
6917 : }
6918 3855 : return true;
6919 : }
6920 : }
6921 : }
6922 :
6923 : /* A value provably within N bits: its low half is the truncation to N bits
6924 : (a subreg, not a 2N shift), its high half is zero. */
6925 594 : auto_bitmap phi_seen;
6926 594 : if (long_mul_op_fits_p (op, narrow_prec, phi_seen))
6927 : {
6928 594 : *lo = gimple_convert (seq, loc, narrow_type, op);
6929 594 : *hi = build_zero_cst (narrow_type);
6930 594 : return true;
6931 : }
6932 :
6933 : /* A PHI that does not fit N bits can still be split through its arguments,
6934 : which is how a sign-extended value reaches the cast case above. */
6935 0 : if (TREE_CODE (op) == SSA_NAME)
6936 594 : if (gphi *phi = dyn_cast <gphi *> (SSA_NAME_DEF_STMT (op)))
6937 0 : return long_mul_split_phi (phi, narrow_type, lo, hi);
6938 : return false;
6939 594 : }
6940 :
6941 : /* Collect into HIGH_USES the uses of PROD forming its high half,
6942 : `PROD >> NARROW_PREC'. A use reading only the low NARROW_PREC bits is
6943 : accepted but not collected. Returns false on any other use. */
6944 :
6945 : static bool
6946 2244 : long_mul_high_half_uses (tree prod, unsigned int narrow_prec,
6947 : vec<gimple *> *high_uses)
6948 : {
6949 2244 : imm_use_iterator iui;
6950 2244 : gimple *use_stmt;
6951 4486 : FOR_EACH_IMM_USE_STMT (use_stmt, iui, prod)
6952 : {
6953 2247 : if (is_gimple_debug (use_stmt))
6954 0 : continue;
6955 2247 : if (!is_gimple_assign (use_stmt))
6956 : return false;
6957 2247 : tree_code code = gimple_assign_rhs_code (use_stmt);
6958 2247 : if (code == RSHIFT_EXPR
6959 2226 : && gimple_assign_rhs1 (use_stmt) == prod
6960 2226 : && tree_fits_uhwi_p (gimple_assign_rhs2 (use_stmt))
6961 4473 : && tree_to_uhwi (gimple_assign_rhs2 (use_stmt)) == narrow_prec)
6962 2226 : high_uses->safe_push (use_stmt);
6963 21 : else if (CONVERT_EXPR_CODE_P (code))
6964 : {
6965 0 : tree t = TREE_TYPE (gimple_assign_lhs (use_stmt));
6966 0 : if (!INTEGRAL_TYPE_P (t) || TYPE_PRECISION (t) > narrow_prec)
6967 : return false;
6968 : }
6969 21 : else if (code == BIT_AND_EXPR
6970 21 : && TREE_CODE (gimple_assign_rhs2 (use_stmt)) == INTEGER_CST)
6971 : {
6972 16 : if (wi::min_precision (wi::to_wide (gimple_assign_rhs2 (use_stmt)),
6973 : UNSIGNED) > narrow_prec)
6974 : return false;
6975 : }
6976 : else
6977 : return false;
6978 5 : }
6979 2239 : return true;
6980 : }
6981 :
6982 : /* True when every use of PROD reads only its low NARROW_PREC bits. */
6983 :
6984 : static bool
6985 2236 : long_mul_only_low_half_used_p (tree prod, unsigned int narrow_prec)
6986 : {
6987 2236 : auto_vec<gimple *, 4> high_uses;
6988 2236 : return (long_mul_high_half_uses (prod, narrow_prec, &high_uses)
6989 4472 : && high_uses.is_empty ());
6990 2236 : }
6991 :
6992 : /* True when STMT is res = a * b whose unsigned 2N-bit result is in a mode the
6993 : target cannot multiply, having neither insn nor libcall, so that expand_mult
6994 : would abort; set *NARROW_TYPE to the N-bit unsigned type its halves are
6995 : built at. A mode the target does support is left to convert_mult_to_widen
6996 : and convert_mult_to_highpart. */
6997 :
6998 : static bool
6999 90459443 : unexpandable_long_mul_p (gimple *stmt, tree *narrow_type)
7000 : {
7001 90459443 : if (!is_gimple_assign (stmt) || gimple_assign_rhs_code (stmt) != MULT_EXPR)
7002 : return false;
7003 :
7004 1490317 : tree wide_type = TREE_TYPE (gimple_assign_lhs (stmt));
7005 1490317 : scalar_int_mode wide_mode;
7006 1490317 : if (!INTEGRAL_TYPE_P (wide_type)
7007 1230299 : || !TYPE_UNSIGNED (wide_type)
7008 913542 : || !is_a <scalar_int_mode> (TYPE_MODE (wide_type), &wide_mode)
7009 2720616 : || targetm.scalar_mode_supported_p (wide_mode))
7010 : return false;
7011 :
7012 2244 : *narrow_type
7013 2244 : = build_nonstandard_integer_type (TYPE_PRECISION (wide_type) / 2, 1);
7014 2244 : return true;
7015 : }
7016 :
7017 : static bool narrow_long_mul_low_half (gimple_stmt_iterator *);
7018 :
7019 : /* OP1 and OP2 are the operands of a 2N multiply just narrowed or lowered;
7020 : that rewrite now reads each through an N-bit low-half cast. An operand
7021 : defined by another 2N multiply can thereby become low-half-only -- narrow
7022 : it too, recursing through chained wide products such as (a*b)*c. */
7023 :
7024 : static void
7025 2215 : narrow_long_mul_operands (tree op1, tree op2)
7026 : {
7027 6645 : for (tree op : { op1, op2 })
7028 4430 : if (TREE_CODE (op) == SSA_NAME)
7029 : {
7030 3903 : gimple *def = SSA_NAME_DEF_STMT (op);
7031 3903 : if (is_gimple_assign (def) && gimple_assign_rhs_code (def) == MULT_EXPR)
7032 : {
7033 5 : gimple_stmt_iterator dgsi = gsi_for_stmt (def);
7034 5 : narrow_long_mul_low_half (&dgsi);
7035 : }
7036 : }
7037 2215 : }
7038 :
7039 : /* If the statement at *GSI is res = a * b with a 2N-bit unsigned result the
7040 : target cannot multiply and every use reads only the low N bits, narrow it
7041 : to res = (2N) ((N) a * (N) b) and return true. The low N bits of a product
7042 : depend only on the low N bits of the operands, so this preserves every use;
7043 : the unused high half becomes zero. match.pd's shorten rule omits this for
7044 : MULT_EXPR. */
7045 :
7046 : static bool
7047 759527 : narrow_long_mul_low_half (gimple_stmt_iterator *gsi)
7048 : {
7049 759527 : gimple *stmt = gsi_stmt (*gsi);
7050 759527 : tree narrow_type;
7051 759527 : if (!unexpandable_long_mul_p (stmt, &narrow_type))
7052 : return false;
7053 :
7054 2236 : tree lhs = gimple_assign_lhs (stmt);
7055 2236 : if (!long_mul_only_low_half_used_p (lhs, TYPE_PRECISION (narrow_type)))
7056 : return false;
7057 :
7058 13 : tree op1 = gimple_assign_rhs1 (stmt);
7059 13 : tree op2 = gimple_assign_rhs2 (stmt);
7060 13 : location_t loc = gimple_location (stmt);
7061 13 : gimple_seq seq = NULL;
7062 13 : tree a = gimple_convert (&seq, loc, narrow_type, op1);
7063 13 : tree b = gimple_convert (&seq, loc, narrow_type, op2);
7064 13 : tree np = gimple_build (&seq, loc, MULT_EXPR, narrow_type, a, b);
7065 13 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
7066 13 : gimple *conv = gimple_build_assign (lhs, NOP_EXPR, np);
7067 13 : gimple_set_location (conv, loc);
7068 13 : gsi_replace (gsi, conv, true);
7069 :
7070 13 : if (dump_file && (dump_flags & TDF_DETAILS))
7071 0 : fprintf (dump_file, "Narrowed low-half-only long multiply.\n");
7072 :
7073 13 : narrow_long_mul_operands (op1, op2);
7074 13 : return true;
7075 : }
7076 :
7077 : /* The 2N multiply at *GSI has had its high half synthesized elsewhere, so any
7078 : use left reads only its low half: narrow it in place, or remove it when it
7079 : has no use at all. Removing it drops a use of each operand, so a 2N
7080 : multiply defining one may become low-half-only. Narrow those operands. */
7081 :
7082 : static void
7083 2218 : finish_long_mul_low_half (gimple_stmt_iterator *gsi)
7084 : {
7085 2218 : gimple *stmt = gsi_stmt (*gsi);
7086 :
7087 2218 : if (has_zero_uses (gimple_assign_lhs (stmt)))
7088 : {
7089 2202 : tree op1 = gimple_assign_rhs1 (stmt);
7090 2202 : tree op2 = gimple_assign_rhs2 (stmt);
7091 2202 : gsi_remove (gsi, true);
7092 2202 : release_defs (stmt);
7093 2202 : narrow_long_mul_operands (op1, op2);
7094 2202 : return;
7095 : }
7096 :
7097 16 : narrow_long_mul_low_half (gsi);
7098 : }
7099 :
7100 : /* Rewrite the multiply STMT into the N-bit halves its uses read, when the
7101 : target can neither multiply at 2N bits nor form an N-bit high part.
7102 : Returns true on a rewrite, which may remove STMT.
7103 :
7104 : Runs once lower_long_mul_high_chain has been applied to every statement:
7105 : both rewrite products with a high half, and this one, keyed on the
7106 : definition rather than on a consumer, would otherwise pre-empt it. What
7107 : reaches it is a product that lowering could not retire, its high half also
7108 : read as an operand of another product, or read alongside the low half. A
7109 : product read only for its low half is left to narrow_long_mul_low_half. */
7110 :
7111 : static bool
7112 89699916 : narrow_long_mul_halves (gimple *stmt)
7113 : {
7114 89699916 : tree narrow_type;
7115 89699916 : if (!unexpandable_long_mul_p (stmt, &narrow_type))
7116 : return false;
7117 :
7118 : /* Only worth lowering where the target cannot form the N-bit high part. */
7119 8 : scalar_int_mode narrow_mode;
7120 8 : if (!is_a <scalar_int_mode> (TYPE_MODE (narrow_type), &narrow_mode)
7121 8 : || can_mult_highpart_p (narrow_mode, true))
7122 : return false;
7123 :
7124 8 : tree lhs = gimple_assign_lhs (stmt);
7125 8 : auto_vec<gimple *, 4> high_uses;
7126 8 : if (!long_mul_high_half_uses (lhs, TYPE_PRECISION (narrow_type), &high_uses)
7127 16 : || high_uses.is_empty ())
7128 : return false;
7129 :
7130 8 : tree op1 = gimple_assign_rhs1 (stmt);
7131 8 : tree op2 = gimple_assign_rhs2 (stmt);
7132 8 : location_t loc = gimple_location (stmt);
7133 8 : gimple_seq seq = NULL;
7134 8 : tree l1, h1, l2, h2;
7135 8 : if (!long_mul_split_operand (&seq, loc, op1, narrow_type, &l1, &h1)
7136 8 : || !long_mul_split_operand (&seq, loc, op2, narrow_type, &l2, &h2))
7137 : return false;
7138 8 : tree hi = combine_long_mul_halves (&seq, loc, l1, h1, l2, h2, narrow_type);
7139 : /* The high part is < 2^N, so widening it back to 2N leaves every use of a
7140 : shift, full width or truncated, reading the same value. */
7141 8 : tree hi_wide = gimple_convert (&seq, loc, TREE_TYPE (lhs), hi);
7142 8 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
7143 8 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
7144 :
7145 8 : unsigned int i;
7146 8 : gimple *shift_stmt;
7147 24 : FOR_EACH_VEC_ELT (high_uses, i, shift_stmt)
7148 : {
7149 8 : gimple_stmt_iterator sgsi = gsi_for_stmt (shift_stmt);
7150 8 : gimple *conv = gimple_build_assign (gimple_assign_lhs (shift_stmt),
7151 : hi_wide);
7152 8 : gimple_set_location (conv, loc);
7153 8 : gsi_replace (&sgsi, conv, true);
7154 : }
7155 :
7156 8 : if (dump_file && (dump_flags & TDF_DETAILS))
7157 0 : fprintf (dump_file, "Narrowed high half of long multiply.\n");
7158 :
7159 8 : finish_long_mul_low_half (&gsi);
7160 8 : return true;
7161 8 : }
7162 :
7163 : /* Match.pd recognizer for the long-multiply recognizer's high-part
7164 : emit chain. */
7165 :
7166 : extern bool gimple_long_mul_high_chain (tree, tree *, tree (*)(tree));
7167 :
7168 : /* Rewrite the `long_mul_high_chain' whose tail is the statement at GSI
7169 :
7170 : wide_a = (T_2N) op1
7171 : wide_b = (T_2N) op2
7172 : wide_prod = wide_a * wide_b
7173 : hi = wide_prod >> N
7174 : lhs = (convert) hi
7175 :
7176 : to a longhand high-part synthesis at T_N precision. Never materializes
7177 : T_2N in gimple, so it covers cases where the 2N mode has no expansion path
7178 : (e.g. the high 128 bits of a 128x128 product where 2N=OImode). An operand
7179 : wider than T_N -- a shared wide product or a sign-extended cast -- is split
7180 : into T_N halves rather than truncated, so no high input bits are dropped.
7181 : Returns true on a rewrite. */
7182 :
7183 : static bool
7184 2469096 : lower_long_mul_high_chain (gimple_stmt_iterator *gsi)
7185 : {
7186 2469096 : gimple *trunc_stmt = gsi_stmt (*gsi);
7187 2469096 : if (!is_gimple_assign (trunc_stmt))
7188 : return false;
7189 :
7190 2469096 : tree narrow_lhs = gimple_assign_lhs (trunc_stmt);
7191 2469096 : tree ops[2];
7192 2469096 : if (!gimple_long_mul_high_chain (narrow_lhs, ops, NULL))
7193 : return false;
7194 :
7195 : /* Walk the matched chain back to the 2N multiply and take narrow_type at
7196 : half its precision. */
7197 2476 : gimple *shift_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (trunc_stmt));
7198 2476 : gimple *mult_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (shift_stmt));
7199 2476 : unsigned int narrow_prec
7200 2476 : = TYPE_PRECISION (TREE_TYPE (gimple_assign_lhs (mult_stmt))) / 2;
7201 2476 : tree narrow_type = build_nonstandard_integer_type (narrow_prec, /*uns=*/1);
7202 2476 : scalar_int_mode narrow_mode;
7203 2476 : if (!is_a <scalar_int_mode> (TYPE_MODE (narrow_type), &narrow_mode))
7204 : return false;
7205 :
7206 : /* Lower only when the target cannot form the N-bit high part itself. */
7207 2476 : if (can_mult_highpart_p (narrow_mode, true))
7208 : return false;
7209 :
7210 2210 : location_t loc = gimple_location (trunc_stmt);
7211 2210 : gimple_seq seq = NULL;
7212 :
7213 : /* Split each operand into N-bit halves and combine. An operand that fits
7214 : N bits yields h == 0, so its cross term folds away; with both fitting
7215 : the combine is just a plain N-bit high part. */
7216 2210 : tree l1, h1, l2, h2;
7217 2210 : if (!long_mul_split_operand (&seq, loc, gimple_assign_rhs1 (mult_stmt),
7218 : narrow_type, &l1, &h1)
7219 2210 : || !long_mul_split_operand (&seq, loc, gimple_assign_rhs2 (mult_stmt),
7220 : narrow_type, &l2, &h2))
7221 : return false;
7222 2210 : tree hi = combine_long_mul_halves (&seq, loc, l1, h1, l2, h2, narrow_type);
7223 :
7224 : /* Merging the chain's truncation with a later user cast can retarget the
7225 : outer convert to any integral type, so convert the narrow result once
7226 : here (the high part is < 2^N, so the conversion preserves it). */
7227 2210 : gimple *result_stmt;
7228 2210 : tree lhs_type = TREE_TYPE (narrow_lhs);
7229 2210 : if (useless_type_conversion_p (lhs_type, narrow_type))
7230 2204 : result_stmt = gimple_build_assign (narrow_lhs, hi);
7231 : else
7232 6 : result_stmt = gimple_build_assign (narrow_lhs, NOP_EXPR, hi);
7233 2210 : gimple_set_location (result_stmt, loc);
7234 2210 : gimple_seq_add_stmt (&seq, result_stmt);
7235 :
7236 2210 : gsi_replace_with_seq (gsi, seq, true);
7237 :
7238 : /* Clean up the shift and the 2N mult now -- LTRANS runs no DCE between
7239 : widening_mul and expand, and a dead 2N mult would abort expand_mult.
7240 : Dead upstream (T_2N) casts, if any, are harmless NOP_EXPRs and land
7241 : with normal DCE. */
7242 2210 : if (has_zero_uses (gimple_assign_lhs (shift_stmt)))
7243 : {
7244 2202 : gimple_stmt_iterator dgsi = gsi_for_stmt (shift_stmt);
7245 2202 : gsi_remove (&dgsi, true);
7246 2202 : release_defs (shift_stmt);
7247 : }
7248 :
7249 : /* The mult is either dead (low half recomputed elsewhere) or now read only
7250 : for its low half. */
7251 2210 : gimple_stmt_iterator mgsi = gsi_for_stmt (mult_stmt);
7252 2210 : finish_long_mul_low_half (&mgsi);
7253 :
7254 2210 : if (dump_file && (dump_flags & TDF_DETAILS))
7255 4 : fprintf (dump_file, "Lowered long-mul high-part chain.\n");
7256 : return true;
7257 : }
7258 :
7259 : /* True when pass_optimize_widening_mul will run. Shared with the
7260 : forwprop long-multiply recognizer so its wide-chain emit stays
7261 : paired with the lowering that rescues an unsupported 2N shape.
7262 : The -Og pipeline (pass_all_optimizations_g) does not contain
7263 : pass_optimize_widening_mul at all, so -Og -fexpensive-optimizations
7264 : must not enable the emit: the unlowered 2N multiply would reach
7265 : expand as an unexpandable mode (e.g. OImode) and ICE.
7266 : -fdisable-tree-widening_mul is not observed. */
7267 :
7268 : bool
7269 1086938 : optimize_widening_mul_active_p (void)
7270 : {
7271 1086938 : return flag_expensive_optimizations && optimize && !optimize_debug;
7272 : }
7273 :
7274 : /* Find integer multiplications where the operands are extended from
7275 : smaller types, and replace the MULT_EXPR with a WIDEN_MULT_EXPR
7276 : or MULT_HIGHPART_EXPR where appropriate. */
7277 :
7278 : namespace {
7279 :
7280 : const pass_data pass_data_optimize_widening_mul =
7281 : {
7282 : GIMPLE_PASS, /* type */
7283 : "widening_mul", /* name */
7284 : OPTGROUP_NONE, /* optinfo_flags */
7285 : TV_TREE_WIDEN_MUL, /* tv_id */
7286 : PROP_ssa, /* properties_required */
7287 : 0, /* properties_provided */
7288 : 0, /* properties_destroyed */
7289 : 0, /* todo_flags_start */
7290 : TODO_update_ssa, /* todo_flags_finish */
7291 : };
7292 :
7293 : class pass_optimize_widening_mul : public gimple_opt_pass
7294 : {
7295 : public:
7296 294536 : pass_optimize_widening_mul (gcc::context *ctxt)
7297 589072 : : gimple_opt_pass (pass_data_optimize_widening_mul, ctxt)
7298 : {}
7299 :
7300 : /* opt_pass methods: */
7301 1068219 : bool gate (function *) final override
7302 : {
7303 1068219 : return optimize_widening_mul_active_p ();
7304 : }
7305 :
7306 : unsigned int execute (function *) final override;
7307 :
7308 : }; // class pass_optimize_widening_mul
7309 :
7310 : /* Walker class to perform the transformation in reverse dominance order. */
7311 :
7312 : class math_opts_dom_walker : public dom_walker
7313 : {
7314 : public:
7315 : /* Constructor, CFG_CHANGED is a pointer to a boolean flag that will be set
7316 : if walking modidifes the CFG. */
7317 :
7318 988753 : math_opts_dom_walker (bool *cfg_changed_p)
7319 1977506 : : dom_walker (CDI_DOMINATORS), m_last_result_set (),
7320 988753 : m_cfg_changed_p (cfg_changed_p) {}
7321 :
7322 : /* The actual actions performed in the walk. */
7323 :
7324 : void after_dom_children (basic_block) final override;
7325 :
7326 : /* Set of results of chains of multiply and add statement combinations that
7327 : were not transformed into FMAs because of active deferring. */
7328 : hash_set<tree> m_last_result_set;
7329 :
7330 : /* Pointer to a flag of the user that needs to be set if CFG has been
7331 : modified. */
7332 : bool *m_cfg_changed_p;
7333 : };
7334 :
7335 : void
7336 10617951 : math_opts_dom_walker::after_dom_children (basic_block bb)
7337 : {
7338 10617951 : gimple_stmt_iterator gsi;
7339 :
7340 10617951 : fma_deferring_state fma_state (param_avoid_fma_max_bits > 0
7341 10721611 : && param_widening_mul_defer_fma);
7342 :
7343 14969996 : for (gphi_iterator psi_next, psi = gsi_start_phis (bb); !gsi_end_p (psi);
7344 4352045 : psi = psi_next)
7345 : {
7346 4352045 : psi_next = psi;
7347 4352045 : gsi_next (&psi_next);
7348 :
7349 4352045 : gimple_stmt_iterator gsi = gsi_after_labels (bb);
7350 4352045 : gphi *phi = psi.phi ();
7351 :
7352 4352045 : if (match_saturation_add (&gsi, phi)
7353 4352026 : || match_saturation_sub (&gsi, phi)
7354 4351978 : || match_saturation_trunc (&gsi, phi)
7355 4351978 : || match_saturation_mul (&gsi, phi)
7356 8704023 : || match_spaceship (&gsi, phi))
7357 179 : remove_phi_node (&psi, /* release_lhs_p */ false);
7358 : }
7359 :
7360 99172716 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi);)
7361 : {
7362 88554765 : gimple *stmt = gsi_stmt (gsi);
7363 88554765 : enum tree_code code;
7364 :
7365 88554765 : if (is_gimple_assign (stmt))
7366 : {
7367 21887955 : code = gimple_assign_rhs_code (stmt);
7368 21887955 : switch (code)
7369 : {
7370 759506 : case MULT_EXPR:
7371 759506 : if (narrow_long_mul_low_half (&gsi))
7372 : break;
7373 759506 : if (!convert_mult_to_widen (stmt, &gsi)
7374 750615 : && !convert_expand_mult_copysign (stmt, &gsi)
7375 1510078 : && convert_mult_to_fma (stmt,
7376 : gimple_assign_rhs1 (stmt),
7377 : gimple_assign_rhs2 (stmt),
7378 : &fma_state))
7379 : {
7380 16529 : gsi_remove (&gsi, true);
7381 16529 : release_defs (stmt);
7382 16529 : continue;
7383 : }
7384 742977 : match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p);
7385 742977 : match_saturation_sub_with_assign (&gsi, as_a<gassign *> (stmt));
7386 742977 : break;
7387 :
7388 2336546 : case PLUS_EXPR:
7389 2336546 : if (match_saturation_add_with_assign (&gsi,
7390 : as_a<gassign *> (stmt)))
7391 : break;
7392 : /* fall-through */
7393 2646465 : case MINUS_EXPR:
7394 2646465 : if (!match_saturation_sub_with_assign (&gsi,
7395 : as_a<gassign *> (stmt))
7396 2646465 : && !convert_plusminus_to_widen (&gsi, stmt, code))
7397 : {
7398 2646343 : match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p);
7399 2646343 : if (gsi_stmt (gsi) == stmt)
7400 2640400 : match_uaddc_usubc (&gsi, stmt, code);
7401 : }
7402 : break;
7403 :
7404 39517 : case BIT_NOT_EXPR:
7405 39517 : if (match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p))
7406 239 : continue;
7407 : break;
7408 :
7409 56714 : case TRUNC_MOD_EXPR:
7410 56714 : convert_to_divmod (as_a<gassign *> (stmt));
7411 56714 : break;
7412 :
7413 176107 : case RSHIFT_EXPR:
7414 176107 : convert_mult_to_highpart (as_a<gassign *> (stmt), &gsi);
7415 176107 : break;
7416 :
7417 184496 : case BIT_IOR_EXPR:
7418 184496 : match_unsigned_saturation_mul (&gsi, as_a<gassign *> (stmt));
7419 184496 : match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt));
7420 184496 : match_unsigned_saturation_trunc (&gsi, as_a<gassign *> (stmt));
7421 : /* fall-through */
7422 217042 : case BIT_XOR_EXPR:
7423 217042 : match_uaddc_usubc (&gsi, stmt, code);
7424 217042 : break;
7425 :
7426 326874 : case EQ_EXPR:
7427 326874 : case NE_EXPR:
7428 326874 : case LE_EXPR:
7429 326874 : case GT_EXPR:
7430 326874 : match_single_bit_test (&gsi, stmt);
7431 326874 : break;
7432 :
7433 355421 : case COND_EXPR:
7434 355421 : case BIT_AND_EXPR:
7435 355421 : match_saturation_sub_with_assign (&gsi, as_a<gassign *> (stmt));
7436 355421 : break;
7437 :
7438 2469183 : case NOP_EXPR:
7439 2469183 : match_unsigned_saturation_mul (&gsi, as_a<gassign *> (stmt));
7440 2469183 : match_unsigned_saturation_trunc (&gsi, as_a<gassign *> (stmt));
7441 2469183 : match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt));
7442 2469183 : match_saturation_sub_with_assign (&gsi, as_a<gassign *> (stmt));
7443 : /* fall-through */
7444 2469198 : case CONVERT_EXPR:
7445 : /* The long-multiply recognizer's high-part emit ends in an
7446 : outer convert. If the trailing cast+mult+shift+cast
7447 : chain has no expansion strategy at the 2N width, lower
7448 : the whole chain to a longhand high-part at narrow
7449 : precision. */
7450 2469198 : if (gsi_stmt (gsi) == stmt
7451 2469198 : && lower_long_mul_high_chain (&gsi))
7452 2210 : continue;
7453 : break;
7454 :
7455 193 : default:;
7456 : }
7457 : }
7458 66666810 : else if (is_gimple_call (stmt))
7459 : {
7460 4979792 : switch (gimple_call_combined_fn (stmt))
7461 : {
7462 129 : case CFN_COND_MUL:
7463 129 : if (convert_mult_to_fma (stmt,
7464 : gimple_call_arg (stmt, 1),
7465 : gimple_call_arg (stmt, 2),
7466 : &fma_state,
7467 : gimple_call_arg (stmt, 0)))
7468 :
7469 : {
7470 84 : gsi_remove (&gsi, true);
7471 84 : release_defs (stmt);
7472 84 : continue;
7473 : }
7474 : break;
7475 :
7476 0 : case CFN_COND_LEN_MUL:
7477 0 : if (convert_mult_to_fma (stmt,
7478 : gimple_call_arg (stmt, 1),
7479 : gimple_call_arg (stmt, 2),
7480 : &fma_state,
7481 : gimple_call_arg (stmt, 0),
7482 : gimple_call_arg (stmt, 4),
7483 : gimple_call_arg (stmt, 5)))
7484 :
7485 : {
7486 0 : gsi_remove (&gsi, true);
7487 0 : release_defs (stmt);
7488 0 : continue;
7489 : }
7490 : break;
7491 :
7492 3807510 : case CFN_LAST:
7493 3807510 : cancel_fma_deferring (&fma_state);
7494 3807510 : break;
7495 :
7496 : default:
7497 : break;
7498 : }
7499 : }
7500 61687018 : else if (gimple_code (stmt) == GIMPLE_COND)
7501 : {
7502 4258847 : match_single_bit_test (&gsi, stmt);
7503 4258847 : optimize_spaceship (as_a <gcond *> (stmt));
7504 : }
7505 88535703 : gsi_next (&gsi);
7506 : }
7507 10617951 : if (fma_state.m_deferring_p
7508 7755862 : && fma_state.m_initial_phi)
7509 : {
7510 361 : gcc_checking_assert (fma_state.m_last_result);
7511 361 : if (!last_fma_candidate_feeds_initial_phi (&fma_state,
7512 : &m_last_result_set))
7513 264 : cancel_fma_deferring (&fma_state);
7514 : else
7515 97 : m_last_result_set.add (fma_state.m_last_result);
7516 : }
7517 10617951 : }
7518 :
7519 :
7520 : unsigned int
7521 988753 : pass_optimize_widening_mul::execute (function *fun)
7522 : {
7523 988753 : bool cfg_changed = false;
7524 :
7525 988753 : memset (&widen_mul_stats, 0, sizeof (widen_mul_stats));
7526 988753 : calculate_dominance_info (CDI_DOMINATORS);
7527 988753 : renumber_gimple_stmt_uids (cfun);
7528 :
7529 988753 : long_mul_phi_halves = new hash_map<tree, long_mul_halves>;
7530 :
7531 988753 : math_opts_dom_walker (&cfg_changed).walk (ENTRY_BLOCK_PTR_FOR_FN (cfun));
7532 :
7533 : /* A 2N multiply the target cannot expand would abort expand_mult. Every
7534 : statement has been through the lowerings above, so one left here matched
7535 : none of them. */
7536 988753 : basic_block bb;
7537 10617951 : FOR_EACH_BB_FN (bb, fun)
7538 108958312 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
7539 : {
7540 89699916 : gimple *stmt = gsi_stmt (gsi);
7541 89699916 : gsi_next (&gsi);
7542 89699916 : narrow_long_mul_halves (stmt);
7543 : }
7544 :
7545 1977506 : delete long_mul_phi_halves;
7546 988753 : long_mul_phi_halves = NULL;
7547 :
7548 988753 : statistics_counter_event (fun, "widening multiplications inserted",
7549 : widen_mul_stats.widen_mults_inserted);
7550 988753 : statistics_counter_event (fun, "widening maccs inserted",
7551 : widen_mul_stats.maccs_inserted);
7552 988753 : statistics_counter_event (fun, "fused multiply-adds inserted",
7553 : widen_mul_stats.fmas_inserted);
7554 988753 : statistics_counter_event (fun, "divmod calls inserted",
7555 : widen_mul_stats.divmod_calls_inserted);
7556 988753 : statistics_counter_event (fun, "highpart multiplications inserted",
7557 : widen_mul_stats.highpart_mults_inserted);
7558 :
7559 988753 : return cfg_changed ? TODO_cleanup_cfg : 0;
7560 : }
7561 :
7562 : } // anon namespace
7563 :
7564 : gimple_opt_pass *
7565 294536 : make_pass_optimize_widening_mul (gcc::context *ctxt)
7566 : {
7567 294536 : return new pass_optimize_widening_mul (ctxt);
7568 : }
|
