Line data Source code
1 : /* Global, SSA-based optimizations using mathematical identities.
2 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by the
8 : Free Software Foundation; either version 3, or (at your option) any
9 : later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : /* Currently, the only mini-pass in this file tries to CSE reciprocal
21 : operations. These are common in sequences such as this one:
22 :
23 : modulus = sqrt(x*x + y*y + z*z);
24 : x = x / modulus;
25 : y = y / modulus;
26 : z = z / modulus;
27 :
28 : that can be optimized to
29 :
30 : modulus = sqrt(x*x + y*y + z*z);
31 : rmodulus = 1.0 / modulus;
32 : x = x * rmodulus;
33 : y = y * rmodulus;
34 : z = z * rmodulus;
35 :
36 : We do this for loop invariant divisors, and with this pass whenever
37 : we notice that a division has the same divisor multiple times.
38 :
39 : Of course, like in PRE, we don't insert a division if a dominator
40 : already has one. However, this cannot be done as an extension of
41 : PRE for several reasons.
42 :
43 : First of all, with some experiments it was found out that the
44 : transformation is not always useful if there are only two divisions
45 : by the same divisor. This is probably because modern processors
46 : can pipeline the divisions; on older, in-order processors it should
47 : still be effective to optimize two divisions by the same number.
48 : We make this a param, and it shall be called N in the remainder of
49 : this comment.
50 :
51 : Second, if trapping math is active, we have less freedom on where
52 : to insert divisions: we can only do so in basic blocks that already
53 : contain one. (If divisions don't trap, instead, we can insert
54 : divisions elsewhere, which will be in blocks that are common dominators
55 : of those that have the division).
56 :
57 : We really don't want to compute the reciprocal unless a division will
58 : be found. To do this, we won't insert the division in a basic block
59 : that has less than N divisions *post-dominating* it.
60 :
61 : The algorithm constructs a subset of the dominator tree, holding the
62 : blocks containing the divisions and the common dominators to them,
63 : and walk it twice. The first walk is in post-order, and it annotates
64 : each block with the number of divisions that post-dominate it: this
65 : gives information on where divisions can be inserted profitably.
66 : The second walk is in pre-order, and it inserts divisions as explained
67 : above, and replaces divisions by multiplications.
68 :
69 : In the best case, the cost of the pass is O(n_statements). In the
70 : worst-case, the cost is due to creating the dominator tree subset,
71 : with a cost of O(n_basic_blocks ^ 2); however this can only happen
72 : for n_statements / n_basic_blocks statements. So, the amortized cost
73 : of creating the dominator tree subset is O(n_basic_blocks) and the
74 : worst-case cost of the pass is O(n_statements * n_basic_blocks).
75 :
76 : More practically, the cost will be small because there are few
77 : divisions, and they tend to be in the same basic block, so insert_bb
78 : is called very few times.
79 :
80 : If we did this using domwalk.cc, an efficient implementation would have
81 : to work on all the variables in a single pass, because we could not
82 : work on just a subset of the dominator tree, as we do now, and the
83 : cost would also be something like O(n_statements * n_basic_blocks).
84 : The data structures would be more complex in order to work on all the
85 : variables in a single pass. */
86 :
87 : #include "config.h"
88 : #include "system.h"
89 : #include "coretypes.h"
90 : #include "backend.h"
91 : #include "target.h"
92 : #include "rtl.h"
93 : #include "tree.h"
94 : #include "gimple.h"
95 : #include "predict.h"
96 : #include "alloc-pool.h"
97 : #include "tree-pass.h"
98 : #include "ssa.h"
99 : #include "optabs-tree.h"
100 : #include "gimple-pretty-print.h"
101 : #include "alias.h"
102 : #include "fold-const.h"
103 : #include "gimple-iterator.h"
104 : #include "gimple-fold.h"
105 : #include "stor-layout.h"
106 : #include "tree-cfg.h"
107 : #include "tree-dfa.h"
108 : #include "tree-ssa.h"
109 : #include "builtins.h"
110 : #include "internal-fn.h"
111 : #include "case-cfn-macros.h"
112 : #include "optabs-libfuncs.h"
113 : #include "tree-eh.h"
114 : #include "targhooks.h"
115 : #include "domwalk.h"
116 : #include "tree-ssa-math-opts.h"
117 : #include "dbgcnt.h"
118 : #include "langhooks.h"
119 : #include "cfghooks.h"
120 :
121 : /* This structure represents one basic block that either computes a
122 : division, or is a common dominator for basic block that compute a
123 : division. */
124 : struct occurrence {
125 : /* The basic block represented by this structure. */
126 : basic_block bb = basic_block();
127 :
128 : /* If non-NULL, the SSA_NAME holding the definition for a reciprocal
129 : inserted in BB. */
130 : tree recip_def = tree();
131 :
132 : /* If non-NULL, the SSA_NAME holding the definition for a squared
133 : reciprocal inserted in BB. */
134 : tree square_recip_def = tree();
135 :
136 : /* If non-NULL, the GIMPLE_ASSIGN for a reciprocal computation that
137 : was inserted in BB. */
138 : gimple *recip_def_stmt = nullptr;
139 :
140 : /* Pointer to a list of "struct occurrence"s for blocks dominated
141 : by BB. */
142 : struct occurrence *children = nullptr;
143 :
144 : /* Pointer to the next "struct occurrence"s in the list of blocks
145 : sharing a common dominator. */
146 : struct occurrence *next = nullptr;
147 :
148 : /* The number of divisions that are in BB before compute_merit. The
149 : number of divisions that are in BB or post-dominate it after
150 : compute_merit. */
151 : int num_divisions = 0;
152 :
153 : /* True if the basic block has a division, false if it is a common
154 : dominator for basic blocks that do. If it is false and trapping
155 : math is active, BB is not a candidate for inserting a reciprocal. */
156 : bool bb_has_division = false;
157 :
158 : /* Construct a struct occurrence for basic block BB, and whose
159 : children list is headed by CHILDREN. */
160 618 : occurrence (basic_block bb, struct occurrence *children)
161 618 : : bb (bb), children (children)
162 : {
163 618 : bb->aux = this;
164 : }
165 :
166 : /* Destroy a struct occurrence and remove it from its basic block. */
167 618 : ~occurrence ()
168 : {
169 618 : bb->aux = nullptr;
170 618 : }
171 :
172 : /* Allocate memory for a struct occurrence from OCC_POOL. */
173 : static void* operator new (size_t);
174 :
175 : /* Return memory for a struct occurrence to OCC_POOL. */
176 : static void operator delete (void*, size_t);
177 : };
178 :
179 : static struct
180 : {
181 : /* Number of 1.0/X ops inserted. */
182 : int rdivs_inserted;
183 :
184 : /* Number of 1.0/FUNC ops inserted. */
185 : int rfuncs_inserted;
186 : } reciprocal_stats;
187 :
188 : static struct
189 : {
190 : /* Number of cexpi calls inserted. */
191 : int inserted;
192 :
193 : /* Number of conversions removed. */
194 : int conv_removed;
195 :
196 : } sincos_stats;
197 :
198 : static struct
199 : {
200 : /* Number of widening multiplication ops inserted. */
201 : int widen_mults_inserted;
202 :
203 : /* Number of integer multiply-and-accumulate ops inserted. */
204 : int maccs_inserted;
205 :
206 : /* Number of fp fused multiply-add ops inserted. */
207 : int fmas_inserted;
208 :
209 : /* Number of divmod calls inserted. */
210 : int divmod_calls_inserted;
211 :
212 : /* Number of highpart multiplication ops inserted. */
213 : int highpart_mults_inserted;
214 : } widen_mul_stats;
215 :
216 : /* The instance of "struct occurrence" representing the highest
217 : interesting block in the dominator tree. */
218 : static struct occurrence *occ_head;
219 :
220 : /* Allocation pool for getting instances of "struct occurrence". */
221 : static object_allocator<occurrence> *occ_pool;
222 :
223 618 : void* occurrence::operator new (size_t n)
224 : {
225 618 : gcc_assert (n == sizeof(occurrence));
226 618 : return occ_pool->allocate_raw ();
227 : }
228 :
229 618 : void occurrence::operator delete (void *occ, size_t n)
230 : {
231 618 : gcc_assert (n == sizeof(occurrence));
232 618 : occ_pool->remove_raw (occ);
233 618 : }
234 :
235 : /* Insert NEW_OCC into our subset of the dominator tree. P_HEAD points to a
236 : list of "struct occurrence"s, one per basic block, having IDOM as
237 : their common dominator.
238 :
239 : We try to insert NEW_OCC as deep as possible in the tree, and we also
240 : insert any other block that is a common dominator for BB and one
241 : block already in the tree. */
242 :
243 : static void
244 607 : insert_bb (struct occurrence *new_occ, basic_block idom,
245 : struct occurrence **p_head)
246 : {
247 612 : struct occurrence *occ, **p_occ;
248 :
249 635 : for (p_occ = p_head; (occ = *p_occ) != NULL; )
250 : {
251 28 : basic_block bb = new_occ->bb, occ_bb = occ->bb;
252 28 : basic_block dom = nearest_common_dominator (CDI_DOMINATORS, occ_bb, bb);
253 28 : if (dom == bb)
254 : {
255 : /* BB dominates OCC_BB. OCC becomes NEW_OCC's child: remove OCC
256 : from its list. */
257 6 : *p_occ = occ->next;
258 6 : occ->next = new_occ->children;
259 6 : new_occ->children = occ;
260 :
261 : /* Try the next block (it may as well be dominated by BB). */
262 : }
263 :
264 22 : else if (dom == occ_bb)
265 : {
266 : /* OCC_BB dominates BB. Tail recurse to look deeper. */
267 5 : insert_bb (new_occ, dom, &occ->children);
268 5 : return;
269 : }
270 :
271 17 : else if (dom != idom)
272 : {
273 11 : gcc_assert (!dom->aux);
274 :
275 : /* There is a dominator between IDOM and BB, add it and make
276 : two children out of NEW_OCC and OCC. First, remove OCC from
277 : its list. */
278 11 : *p_occ = occ->next;
279 11 : new_occ->next = occ;
280 11 : occ->next = NULL;
281 :
282 : /* None of the previous blocks has DOM as a dominator: if we tail
283 : recursed, we would reexamine them uselessly. Just switch BB with
284 : DOM, and go on looking for blocks dominated by DOM. */
285 11 : new_occ = new occurrence (dom, new_occ);
286 : }
287 :
288 : else
289 : {
290 : /* Nothing special, go on with the next element. */
291 6 : p_occ = &occ->next;
292 : }
293 : }
294 :
295 : /* No place was found as a child of IDOM. Make BB a sibling of IDOM. */
296 607 : new_occ->next = *p_head;
297 607 : *p_head = new_occ;
298 : }
299 :
300 : /* Register that we found a division in BB.
301 : IMPORTANCE is a measure of how much weighting to give
302 : that division. Use IMPORTANCE = 2 to register a single
303 : division. If the division is going to be found multiple
304 : times use 1 (as it is with squares). */
305 :
306 : static inline void
307 709 : register_division_in (basic_block bb, int importance)
308 : {
309 709 : struct occurrence *occ;
310 :
311 709 : occ = (struct occurrence *) bb->aux;
312 709 : if (!occ)
313 : {
314 607 : occ = new occurrence (bb, NULL);
315 607 : insert_bb (occ, ENTRY_BLOCK_PTR_FOR_FN (cfun), &occ_head);
316 : }
317 :
318 709 : occ->bb_has_division = true;
319 709 : occ->num_divisions += importance;
320 709 : }
321 :
322 :
323 : /* Compute the number of divisions that postdominate each block in OCC and
324 : its children. */
325 :
326 : static void
327 26 : compute_merit (struct occurrence *occ)
328 : {
329 26 : struct occurrence *occ_child;
330 26 : basic_block dom = occ->bb;
331 :
332 45 : for (occ_child = occ->children; occ_child; occ_child = occ_child->next)
333 : {
334 19 : basic_block bb;
335 19 : if (occ_child->children)
336 3 : compute_merit (occ_child);
337 :
338 19 : if (flag_exceptions)
339 6 : bb = single_noncomplex_succ (dom);
340 : else
341 : bb = dom;
342 :
343 19 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, occ_child->bb))
344 12 : occ->num_divisions += occ_child->num_divisions;
345 : }
346 26 : }
347 :
348 :
349 : /* Return whether USE_STMT is a floating-point division by DEF. */
350 : static inline bool
351 352383 : is_division_by (gimple *use_stmt, tree def)
352 : {
353 352383 : return is_gimple_assign (use_stmt)
354 242612 : && gimple_assign_rhs_code (use_stmt) == RDIV_EXPR
355 1251 : && gimple_assign_rhs2 (use_stmt) == def
356 : /* Do not recognize x / x as valid division, as we are getting
357 : confused later by replacing all immediate uses x in such
358 : a stmt. */
359 879 : && gimple_assign_rhs1 (use_stmt) != def
360 353262 : && !stmt_can_throw_internal (cfun, use_stmt);
361 : }
362 :
363 : /* Return TRUE if USE_STMT is a multiplication of DEF by A. */
364 : static inline bool
365 348592 : is_mult_by (gimple *use_stmt, tree def, tree a)
366 : {
367 348592 : if (gimple_code (use_stmt) == GIMPLE_ASSIGN
368 348592 : && gimple_assign_rhs_code (use_stmt) == MULT_EXPR)
369 : {
370 79953 : tree op0 = gimple_assign_rhs1 (use_stmt);
371 79953 : tree op1 = gimple_assign_rhs2 (use_stmt);
372 :
373 79953 : return (op0 == def && op1 == a)
374 79953 : || (op0 == a && op1 == def);
375 : }
376 : return 0;
377 : }
378 :
379 : /* Return whether USE_STMT is DEF * DEF. */
380 : static inline bool
381 348547 : is_square_of (gimple *use_stmt, tree def)
382 : {
383 5 : return is_mult_by (use_stmt, def, def);
384 : }
385 :
386 : /* Return whether USE_STMT is a floating-point division by
387 : DEF * DEF. */
388 : static inline bool
389 192 : is_division_by_square (gimple *use_stmt, tree def)
390 : {
391 192 : if (gimple_code (use_stmt) == GIMPLE_ASSIGN
392 185 : && gimple_assign_rhs_code (use_stmt) == RDIV_EXPR
393 7 : && gimple_assign_rhs1 (use_stmt) != gimple_assign_rhs2 (use_stmt)
394 199 : && !stmt_can_throw_internal (cfun, use_stmt))
395 : {
396 7 : tree denominator = gimple_assign_rhs2 (use_stmt);
397 7 : if (TREE_CODE (denominator) == SSA_NAME)
398 7 : return is_square_of (SSA_NAME_DEF_STMT (denominator), def);
399 : }
400 : return 0;
401 : }
402 :
403 : /* Walk the subset of the dominator tree rooted at OCC, setting the
404 : RECIP_DEF field to a definition of 1.0 / DEF that can be used in
405 : the given basic block. The field may be left NULL, of course,
406 : if it is not possible or profitable to do the optimization.
407 :
408 : DEF_BSI is an iterator pointing at the statement defining DEF.
409 : If RECIP_DEF is set, a dominator already has a computation that can
410 : be used.
411 :
412 : If should_insert_square_recip is set, then this also inserts
413 : the square of the reciprocal immediately after the definition
414 : of the reciprocal. */
415 :
416 : static void
417 42 : insert_reciprocals (gimple_stmt_iterator *def_gsi, struct occurrence *occ,
418 : tree def, tree recip_def, tree square_recip_def,
419 : int should_insert_square_recip, int threshold)
420 : {
421 42 : tree type;
422 42 : gassign *new_stmt, *new_square_stmt;
423 42 : gimple_stmt_iterator gsi;
424 42 : struct occurrence *occ_child;
425 :
426 42 : if (!recip_def
427 26 : && (occ->bb_has_division || !flag_trapping_math)
428 : /* Divide by two as all divisions are counted twice in
429 : the costing loop. */
430 25 : && occ->num_divisions / 2 >= threshold)
431 : {
432 : /* Make a variable with the replacement and substitute it. */
433 24 : type = TREE_TYPE (def);
434 24 : recip_def = create_tmp_reg (type, "reciptmp");
435 24 : new_stmt = gimple_build_assign (recip_def, RDIV_EXPR,
436 : build_one_cst (type), def);
437 :
438 24 : if (should_insert_square_recip)
439 : {
440 4 : square_recip_def = create_tmp_reg (type, "powmult_reciptmp");
441 4 : new_square_stmt = gimple_build_assign (square_recip_def, MULT_EXPR,
442 : recip_def, recip_def);
443 : }
444 :
445 24 : if (occ->bb_has_division)
446 : {
447 : /* Case 1: insert before an existing division. */
448 21 : gsi = gsi_after_labels (occ->bb);
449 212 : while (!gsi_end_p (gsi)
450 212 : && (!is_division_by (gsi_stmt (gsi), def))
451 404 : && (!is_division_by_square (gsi_stmt (gsi), def)))
452 191 : gsi_next (&gsi);
453 :
454 21 : gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
455 21 : if (should_insert_square_recip)
456 3 : gsi_insert_before (&gsi, new_square_stmt, GSI_SAME_STMT);
457 : }
458 3 : else if (def_gsi && occ->bb == gsi_bb (*def_gsi))
459 : {
460 : /* Case 2: insert right after the definition. Note that this will
461 : never happen if the definition statement can throw, because in
462 : that case the sole successor of the statement's basic block will
463 : dominate all the uses as well. */
464 2 : gsi_insert_after (def_gsi, new_stmt, GSI_NEW_STMT);
465 2 : if (should_insert_square_recip)
466 1 : gsi_insert_after (def_gsi, new_square_stmt, GSI_NEW_STMT);
467 : }
468 : else
469 : {
470 : /* Case 3: insert in a basic block not containing defs/uses. */
471 1 : gsi = gsi_after_labels (occ->bb);
472 1 : gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
473 1 : if (should_insert_square_recip)
474 0 : gsi_insert_before (&gsi, new_square_stmt, GSI_SAME_STMT);
475 : }
476 :
477 24 : reciprocal_stats.rdivs_inserted++;
478 :
479 24 : occ->recip_def_stmt = new_stmt;
480 : }
481 :
482 42 : occ->recip_def = recip_def;
483 42 : occ->square_recip_def = square_recip_def;
484 61 : for (occ_child = occ->children; occ_child; occ_child = occ_child->next)
485 19 : insert_reciprocals (def_gsi, occ_child, def, recip_def,
486 : square_recip_def, should_insert_square_recip,
487 : threshold);
488 42 : }
489 :
490 : /* Replace occurrences of expr / (x * x) with expr * ((1 / x) * (1 / x)).
491 : Take as argument the use for (x * x). */
492 : static inline void
493 4 : replace_reciprocal_squares (use_operand_p use_p)
494 : {
495 4 : gimple *use_stmt = USE_STMT (use_p);
496 4 : basic_block bb = gimple_bb (use_stmt);
497 4 : struct occurrence *occ = (struct occurrence *) bb->aux;
498 :
499 8 : if (optimize_bb_for_speed_p (bb) && occ->square_recip_def
500 8 : && occ->recip_def)
501 : {
502 4 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
503 4 : gimple_assign_set_rhs_code (use_stmt, MULT_EXPR);
504 4 : gimple_assign_set_rhs2 (use_stmt, occ->square_recip_def);
505 4 : SET_USE (use_p, occ->square_recip_def);
506 4 : fold_stmt_inplace (&gsi);
507 4 : update_stmt (use_stmt);
508 : }
509 4 : }
510 :
511 :
512 : /* Replace the division at USE_P with a multiplication by the reciprocal, if
513 : possible. */
514 :
515 : static inline void
516 105 : replace_reciprocal (use_operand_p use_p)
517 : {
518 105 : gimple *use_stmt = USE_STMT (use_p);
519 105 : basic_block bb = gimple_bb (use_stmt);
520 105 : struct occurrence *occ = (struct occurrence *) bb->aux;
521 :
522 105 : if (optimize_bb_for_speed_p (bb)
523 105 : && occ->recip_def && use_stmt != occ->recip_def_stmt)
524 : {
525 80 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
526 80 : gimple_assign_set_rhs_code (use_stmt, MULT_EXPR);
527 80 : SET_USE (use_p, occ->recip_def);
528 80 : fold_stmt_inplace (&gsi);
529 80 : update_stmt (use_stmt);
530 : }
531 105 : }
532 :
533 :
534 : /* Free OCC and return one more "struct occurrence" to be freed. */
535 :
536 : static struct occurrence *
537 618 : free_bb (struct occurrence *occ)
538 : {
539 618 : struct occurrence *child, *next;
540 :
541 : /* First get the two pointers hanging off OCC. */
542 618 : next = occ->next;
543 618 : child = occ->children;
544 618 : delete occ;
545 :
546 : /* Now ensure that we don't recurse unless it is necessary. */
547 618 : if (!child)
548 : return next;
549 : else
550 : {
551 19 : while (next)
552 0 : next = free_bb (next);
553 :
554 : return child;
555 : }
556 : }
557 :
558 : /* Transform sequences like
559 : t = sqrt (a)
560 : x = 1.0 / t;
561 : r1 = x * x;
562 : r2 = a * x;
563 : into:
564 : t = sqrt (a)
565 : r1 = 1.0 / a;
566 : r2 = t;
567 : x = r1 * r2;
568 : depending on the uses of x, r1, r2. This removes one multiplication and
569 : allows the sqrt and division operations to execute in parallel.
570 : DEF_GSI is the gsi of the initial division by sqrt that defines
571 : DEF (x in the example above). */
572 :
573 : static void
574 556 : optimize_recip_sqrt (gimple_stmt_iterator *def_gsi, tree def)
575 : {
576 556 : gimple *use_stmt;
577 556 : imm_use_iterator use_iter;
578 556 : gimple *stmt = gsi_stmt (*def_gsi);
579 556 : tree x = def;
580 556 : tree orig_sqrt_ssa_name = gimple_assign_rhs2 (stmt);
581 556 : tree div_rhs1 = gimple_assign_rhs1 (stmt);
582 :
583 556 : if (TREE_CODE (orig_sqrt_ssa_name) != SSA_NAME
584 551 : || TREE_CODE (div_rhs1) != REAL_CST
585 742 : || !real_equal (&TREE_REAL_CST (div_rhs1), &dconst1))
586 544 : return;
587 :
588 110 : gcall *sqrt_stmt
589 586 : = dyn_cast <gcall *> (SSA_NAME_DEF_STMT (orig_sqrt_ssa_name));
590 :
591 42 : if (!sqrt_stmt || !gimple_call_lhs (sqrt_stmt))
592 : return;
593 :
594 42 : switch (gimple_call_combined_fn (sqrt_stmt))
595 : {
596 31 : CASE_CFN_SQRT:
597 31 : CASE_CFN_SQRT_FN:
598 31 : break;
599 :
600 : default:
601 : return;
602 : }
603 31 : tree a = gimple_call_arg (sqrt_stmt, 0);
604 :
605 : /* We have 'a' and 'x'. Now analyze the uses of 'x'. */
606 :
607 : /* Statements that use x in x * x. */
608 43 : auto_vec<gimple *> sqr_stmts;
609 : /* Statements that use x in a * x. */
610 12 : auto_vec<gimple *> mult_stmts;
611 31 : bool has_other_use = false;
612 31 : bool mult_on_main_path = false;
613 :
614 89 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, x)
615 : {
616 58 : if (is_gimple_debug (use_stmt))
617 1 : continue;
618 57 : if (is_square_of (use_stmt, x))
619 : {
620 12 : sqr_stmts.safe_push (use_stmt);
621 12 : if (gimple_bb (use_stmt) == gimple_bb (stmt))
622 58 : mult_on_main_path = true;
623 : }
624 45 : else if (is_mult_by (use_stmt, x, a))
625 : {
626 14 : mult_stmts.safe_push (use_stmt);
627 14 : if (gimple_bb (use_stmt) == gimple_bb (stmt))
628 58 : mult_on_main_path = true;
629 : }
630 : else
631 : has_other_use = true;
632 31 : }
633 :
634 : /* In the x * x and a * x cases we just rewire stmt operands or
635 : remove multiplications. In the has_other_use case we introduce
636 : a multiplication so make sure we don't introduce a multiplication
637 : on a path where there was none. */
638 31 : if (has_other_use && !mult_on_main_path)
639 19 : return;
640 :
641 12 : if (sqr_stmts.is_empty () && mult_stmts.is_empty ())
642 : return;
643 :
644 : /* If x = 1.0 / sqrt (a) has uses other than those optimized here we want
645 : to be able to compose it from the sqr and mult cases. */
646 41 : if (has_other_use && (sqr_stmts.is_empty () || mult_stmts.is_empty ()))
647 : return;
648 :
649 12 : if (dump_file)
650 : {
651 10 : fprintf (dump_file, "Optimizing reciprocal sqrt multiplications of\n");
652 10 : print_gimple_stmt (dump_file, sqrt_stmt, 0, TDF_NONE);
653 10 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
654 10 : fprintf (dump_file, "\n");
655 : }
656 :
657 12 : bool delete_div = !has_other_use;
658 12 : tree sqr_ssa_name = NULL_TREE;
659 22 : if (!sqr_stmts.is_empty ())
660 : {
661 : /* r1 = x * x. Transform the original
662 : x = 1.0 / t
663 : into
664 : tmp1 = 1.0 / a
665 : r1 = tmp1. */
666 :
667 10 : sqr_ssa_name
668 10 : = make_temp_ssa_name (TREE_TYPE (a), NULL, "recip_sqrt_sqr");
669 :
670 10 : if (dump_file)
671 : {
672 10 : fprintf (dump_file, "Replacing original division\n");
673 10 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
674 10 : fprintf (dump_file, "with new division\n");
675 : }
676 10 : stmt
677 10 : = gimple_build_assign (sqr_ssa_name, gimple_assign_rhs_code (stmt),
678 : gimple_assign_rhs1 (stmt), a);
679 10 : gsi_insert_before (def_gsi, stmt, GSI_SAME_STMT);
680 10 : gsi_remove (def_gsi, true);
681 10 : *def_gsi = gsi_for_stmt (stmt);
682 10 : fold_stmt_inplace (def_gsi);
683 10 : update_stmt (stmt);
684 :
685 10 : if (dump_file)
686 10 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
687 :
688 10 : delete_div = false;
689 10 : gimple *sqr_stmt;
690 10 : unsigned int i;
691 20 : FOR_EACH_VEC_ELT (sqr_stmts, i, sqr_stmt)
692 : {
693 10 : gimple_stmt_iterator gsi2 = gsi_for_stmt (sqr_stmt);
694 10 : gimple_assign_set_rhs_from_tree (&gsi2, sqr_ssa_name);
695 10 : update_stmt (sqr_stmt);
696 : }
697 : }
698 24 : if (!mult_stmts.is_empty ())
699 : {
700 : /* r2 = a * x. Transform this into:
701 : r2 = t (The original sqrt (a)). */
702 : unsigned int i;
703 24 : gimple *mult_stmt = NULL;
704 24 : FOR_EACH_VEC_ELT (mult_stmts, i, mult_stmt)
705 : {
706 12 : gimple_stmt_iterator gsi2 = gsi_for_stmt (mult_stmt);
707 :
708 12 : if (dump_file)
709 : {
710 10 : fprintf (dump_file, "Replacing squaring multiplication\n");
711 10 : print_gimple_stmt (dump_file, mult_stmt, 0, TDF_NONE);
712 10 : fprintf (dump_file, "with assignment\n");
713 : }
714 12 : gimple_assign_set_rhs_from_tree (&gsi2, orig_sqrt_ssa_name);
715 12 : fold_stmt_inplace (&gsi2);
716 12 : update_stmt (mult_stmt);
717 12 : if (dump_file)
718 10 : print_gimple_stmt (dump_file, mult_stmt, 0, TDF_NONE);
719 : }
720 : }
721 :
722 12 : if (has_other_use)
723 : {
724 : /* Using the two temporaries tmp1, tmp2 from above
725 : the original x is now:
726 : x = tmp1 * tmp2. */
727 10 : gcc_assert (orig_sqrt_ssa_name);
728 10 : gcc_assert (sqr_ssa_name);
729 :
730 10 : gimple *new_stmt
731 10 : = gimple_build_assign (x, MULT_EXPR,
732 : orig_sqrt_ssa_name, sqr_ssa_name);
733 10 : gsi_insert_after (def_gsi, new_stmt, GSI_NEW_STMT);
734 10 : update_stmt (stmt);
735 : }
736 2 : else if (delete_div)
737 : {
738 : /* Remove the original division. */
739 2 : gimple_stmt_iterator gsi2 = gsi_for_stmt (stmt);
740 2 : gsi_remove (&gsi2, true);
741 2 : release_defs (stmt);
742 : }
743 : else
744 0 : release_ssa_name (x);
745 : }
746 :
747 : /* Look for floating-point divisions among DEF's uses, and try to
748 : replace them by multiplications with the reciprocal. Add
749 : as many statements computing the reciprocal as needed.
750 :
751 : DEF must be a GIMPLE register of a floating-point type. */
752 :
753 : static void
754 212855 : execute_cse_reciprocals_1 (gimple_stmt_iterator *def_gsi, tree def)
755 : {
756 212855 : use_operand_p use_p, square_use_p;
757 212855 : imm_use_iterator use_iter, square_use_iter;
758 212855 : tree square_def;
759 212855 : struct occurrence *occ;
760 212855 : int count = 0;
761 212855 : int threshold;
762 212855 : int square_recip_count = 0;
763 212855 : int sqrt_recip_count = 0;
764 :
765 212855 : gcc_assert (FLOAT_TYPE_P (TREE_TYPE (def)) && TREE_CODE (def) == SSA_NAME);
766 212855 : threshold = targetm.min_divisions_for_recip_mul (TYPE_MODE (TREE_TYPE (def)));
767 :
768 : /* If DEF is a square (x * x), count the number of divisions by x.
769 : If there are more divisions by x than by (DEF * DEF), prefer to optimize
770 : the reciprocal of x instead of DEF. This improves cases like:
771 : def = x * x
772 : t0 = a / def
773 : t1 = b / def
774 : t2 = c / x
775 : Reciprocal optimization of x results in 1 division rather than 2 or 3. */
776 212855 : gimple *def_stmt = SSA_NAME_DEF_STMT (def);
777 :
778 212855 : if (is_gimple_assign (def_stmt)
779 166436 : && gimple_assign_rhs_code (def_stmt) == MULT_EXPR
780 40520 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME
781 253298 : && gimple_assign_rhs1 (def_stmt) == gimple_assign_rhs2 (def_stmt))
782 : {
783 671 : tree op0 = gimple_assign_rhs1 (def_stmt);
784 :
785 2754 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, op0)
786 : {
787 2083 : gimple *use_stmt = USE_STMT (use_p);
788 2083 : if (is_division_by (use_stmt, op0))
789 17 : sqrt_recip_count++;
790 671 : }
791 : }
792 :
793 561333 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, def)
794 : {
795 348478 : gimple *use_stmt = USE_STMT (use_p);
796 348478 : if (is_division_by (use_stmt, def))
797 : {
798 637 : register_division_in (gimple_bb (use_stmt), 2);
799 637 : count++;
800 : }
801 :
802 348478 : if (is_square_of (use_stmt, def))
803 : {
804 1350 : square_def = gimple_assign_lhs (use_stmt);
805 2832 : FOR_EACH_IMM_USE_FAST (square_use_p, square_use_iter, square_def)
806 : {
807 1482 : gimple *square_use_stmt = USE_STMT (square_use_p);
808 1482 : if (is_division_by (square_use_stmt, square_def))
809 : {
810 : /* This is executed twice for each division by a square. */
811 72 : register_division_in (gimple_bb (square_use_stmt), 1);
812 72 : square_recip_count++;
813 : }
814 1350 : }
815 : }
816 212855 : }
817 :
818 : /* Square reciprocals were counted twice above. */
819 212855 : square_recip_count /= 2;
820 :
821 : /* If it is more profitable to optimize 1 / x, don't optimize 1 / (x * x). */
822 212855 : if (sqrt_recip_count > square_recip_count)
823 17 : goto out;
824 :
825 : /* Do the expensive part only if we can hope to optimize something. */
826 212838 : if (count + square_recip_count >= threshold && count >= 1)
827 : {
828 23 : gimple *use_stmt;
829 46 : for (occ = occ_head; occ; occ = occ->next)
830 : {
831 23 : compute_merit (occ);
832 23 : insert_reciprocals (def_gsi, occ, def, NULL, NULL,
833 : square_recip_count, threshold);
834 : }
835 :
836 143 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, def)
837 : {
838 120 : if (is_division_by (use_stmt, def))
839 : {
840 210 : FOR_EACH_IMM_USE_ON_STMT (use_p, use_iter)
841 105 : replace_reciprocal (use_p);
842 : }
843 20 : else if (square_recip_count > 0 && is_square_of (use_stmt, def))
844 : {
845 12 : FOR_EACH_IMM_USE_ON_STMT (use_p, use_iter)
846 : {
847 : /* Find all uses of the square that are divisions and
848 : * replace them by multiplications with the inverse. */
849 8 : imm_use_iterator square_iterator;
850 8 : gimple *powmult_use_stmt = USE_STMT (use_p);
851 8 : tree powmult_def_name = gimple_assign_lhs (powmult_use_stmt);
852 :
853 16 : FOR_EACH_IMM_USE_STMT (powmult_use_stmt,
854 : square_iterator, powmult_def_name)
855 16 : FOR_EACH_IMM_USE_ON_STMT (square_use_p, square_iterator)
856 : {
857 8 : gimple *powmult_use_stmt = USE_STMT (square_use_p);
858 8 : if (is_division_by (powmult_use_stmt, powmult_def_name))
859 4 : replace_reciprocal_squares (square_use_p);
860 8 : }
861 : }
862 : }
863 23 : }
864 : }
865 :
866 212815 : out:
867 213473 : for (occ = occ_head; occ; )
868 618 : occ = free_bb (occ);
869 :
870 212855 : occ_head = NULL;
871 212855 : }
872 :
873 : /* Return an internal function that implements the reciprocal of CALL,
874 : or IFN_LAST if there is no such function that the target supports. */
875 :
876 : internal_fn
877 113 : internal_fn_reciprocal (gcall *call)
878 : {
879 113 : internal_fn ifn;
880 :
881 113 : switch (gimple_call_combined_fn (call))
882 : {
883 97 : CASE_CFN_SQRT:
884 97 : CASE_CFN_SQRT_FN:
885 97 : ifn = IFN_RSQRT;
886 97 : break;
887 :
888 : default:
889 : return IFN_LAST;
890 : }
891 :
892 97 : tree_pair types = direct_internal_fn_types (ifn, call);
893 97 : if (!direct_internal_fn_supported_p (ifn, types, OPTIMIZE_FOR_SPEED))
894 44 : return IFN_LAST;
895 :
896 : return ifn;
897 : }
898 :
899 : /* Go through all the floating-point SSA_NAMEs, and call
900 : execute_cse_reciprocals_1 on each of them. */
901 : namespace {
902 :
903 : const pass_data pass_data_cse_reciprocals =
904 : {
905 : GIMPLE_PASS, /* type */
906 : "recip", /* name */
907 : OPTGROUP_NONE, /* optinfo_flags */
908 : TV_TREE_RECIP, /* tv_id */
909 : PROP_ssa, /* properties_required */
910 : 0, /* properties_provided */
911 : 0, /* properties_destroyed */
912 : 0, /* todo_flags_start */
913 : TODO_update_ssa, /* todo_flags_finish */
914 : };
915 :
916 : class pass_cse_reciprocals : public gimple_opt_pass
917 : {
918 : public:
919 294196 : pass_cse_reciprocals (gcc::context *ctxt)
920 588392 : : gimple_opt_pass (pass_data_cse_reciprocals, ctxt)
921 : {}
922 :
923 : /* opt_pass methods: */
924 1060389 : bool gate (function *) final override
925 : {
926 1060389 : return optimize && flag_reciprocal_math;
927 : }
928 : unsigned int execute (function *) final override;
929 :
930 : }; // class pass_cse_reciprocals
931 :
932 : unsigned int
933 8776 : pass_cse_reciprocals::execute (function *fun)
934 : {
935 8776 : basic_block bb;
936 8776 : tree arg;
937 :
938 8776 : occ_pool = new object_allocator<occurrence> ("dominators for recip");
939 :
940 8776 : memset (&reciprocal_stats, 0, sizeof (reciprocal_stats));
941 8776 : calculate_dominance_info (CDI_DOMINATORS);
942 8776 : calculate_dominance_info (CDI_POST_DOMINATORS);
943 :
944 8776 : if (flag_checking)
945 94202 : FOR_EACH_BB_FN (bb, fun)
946 85426 : gcc_assert (!bb->aux);
947 :
948 21785 : for (arg = DECL_ARGUMENTS (fun->decl); arg; arg = DECL_CHAIN (arg))
949 20657 : if (FLOAT_TYPE_P (TREE_TYPE (arg))
950 14096 : && is_gimple_reg (arg))
951 : {
952 6447 : tree name = ssa_default_def (fun, arg);
953 6447 : if (name)
954 5452 : execute_cse_reciprocals_1 (NULL, name);
955 : }
956 :
957 94202 : FOR_EACH_BB_FN (bb, fun)
958 : {
959 85426 : tree def;
960 :
961 194275 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
962 108849 : gsi_next (&gsi))
963 : {
964 108849 : gphi *phi = gsi.phi ();
965 108849 : def = PHI_RESULT (phi);
966 108849 : if (! virtual_operand_p (def)
967 108849 : && FLOAT_TYPE_P (TREE_TYPE (def)))
968 30764 : execute_cse_reciprocals_1 (NULL, def);
969 : }
970 :
971 1373846 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
972 1288420 : gsi_next (&gsi))
973 : {
974 1288420 : gimple *stmt = gsi_stmt (gsi);
975 :
976 2576840 : if (gimple_has_lhs (stmt)
977 809188 : && (def = SINGLE_SSA_TREE_OPERAND (stmt, SSA_OP_DEF)) != NULL
978 769082 : && FLOAT_TYPE_P (TREE_TYPE (def))
979 199507 : && TREE_CODE (def) == SSA_NAME)
980 : {
981 176639 : execute_cse_reciprocals_1 (&gsi, def);
982 176639 : stmt = gsi_stmt (gsi);
983 176639 : if (flag_unsafe_math_optimizations
984 176594 : && is_gimple_assign (stmt)
985 166393 : && gimple_assign_lhs (stmt) == def
986 166391 : && !stmt_can_throw_internal (cfun, stmt)
987 342986 : && gimple_assign_rhs_code (stmt) == RDIV_EXPR)
988 556 : optimize_recip_sqrt (&gsi, def);
989 : }
990 : }
991 :
992 85426 : if (optimize_bb_for_size_p (bb))
993 5353 : continue;
994 :
995 : /* Scan for a/func(b) and convert it to reciprocal a*rfunc(b). */
996 1346609 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
997 1266536 : gsi_next (&gsi))
998 : {
999 1266536 : gimple *stmt = gsi_stmt (gsi);
1000 :
1001 1266536 : if (is_gimple_assign (stmt)
1002 1266536 : && gimple_assign_rhs_code (stmt) == RDIV_EXPR)
1003 : {
1004 598 : tree arg1 = gimple_assign_rhs2 (stmt);
1005 598 : gimple *stmt1;
1006 :
1007 598 : if (TREE_CODE (arg1) != SSA_NAME)
1008 5 : continue;
1009 :
1010 593 : stmt1 = SSA_NAME_DEF_STMT (arg1);
1011 :
1012 593 : if (is_gimple_call (stmt1)
1013 593 : && gimple_call_lhs (stmt1))
1014 : {
1015 113 : bool fail;
1016 113 : imm_use_iterator ui;
1017 113 : use_operand_p use_p;
1018 113 : tree fndecl = NULL_TREE;
1019 :
1020 113 : gcall *call = as_a <gcall *> (stmt1);
1021 113 : internal_fn ifn = internal_fn_reciprocal (call);
1022 113 : if (ifn == IFN_LAST)
1023 : {
1024 60 : fndecl = gimple_call_fndecl (call);
1025 120 : if (!fndecl
1026 60 : || !fndecl_built_in_p (fndecl, BUILT_IN_MD))
1027 62 : continue;
1028 0 : fndecl = targetm.builtin_reciprocal (fndecl);
1029 0 : if (!fndecl)
1030 0 : continue;
1031 : }
1032 :
1033 : /* Check that all uses of the SSA name are divisions,
1034 : otherwise replacing the defining statement will do
1035 : the wrong thing. */
1036 53 : fail = false;
1037 106 : FOR_EACH_IMM_USE_FAST (use_p, ui, arg1)
1038 : {
1039 55 : gimple *stmt2 = USE_STMT (use_p);
1040 55 : if (is_gimple_debug (stmt2))
1041 0 : continue;
1042 55 : if (!is_gimple_assign (stmt2)
1043 55 : || gimple_assign_rhs_code (stmt2) != RDIV_EXPR
1044 53 : || gimple_assign_rhs1 (stmt2) == arg1
1045 108 : || gimple_assign_rhs2 (stmt2) != arg1)
1046 : {
1047 : fail = true;
1048 : break;
1049 : }
1050 53 : }
1051 53 : if (fail)
1052 2 : continue;
1053 :
1054 51 : gimple_replace_ssa_lhs (call, arg1);
1055 51 : reset_flow_sensitive_info (arg1);
1056 51 : if (gimple_call_internal_p (call) != (ifn != IFN_LAST))
1057 : {
1058 30 : auto_vec<tree, 4> args;
1059 30 : for (unsigned int i = 0;
1060 60 : i < gimple_call_num_args (call); i++)
1061 30 : args.safe_push (gimple_call_arg (call, i));
1062 30 : gcall *stmt2;
1063 30 : if (ifn == IFN_LAST)
1064 0 : stmt2 = gimple_build_call_vec (fndecl, args);
1065 : else
1066 30 : stmt2 = gimple_build_call_internal_vec (ifn, args);
1067 30 : gimple_call_set_lhs (stmt2, arg1);
1068 30 : gimple_move_vops (stmt2, call);
1069 30 : gimple_call_set_nothrow (stmt2,
1070 : gimple_call_nothrow_p (call));
1071 30 : gimple_stmt_iterator gsi2 = gsi_for_stmt (call);
1072 30 : gsi_replace (&gsi2, stmt2, true);
1073 30 : }
1074 : else
1075 : {
1076 21 : if (ifn == IFN_LAST)
1077 0 : gimple_call_set_fndecl (call, fndecl);
1078 : else
1079 21 : gimple_call_set_internal_fn (call, ifn);
1080 21 : update_stmt (call);
1081 : }
1082 51 : reciprocal_stats.rfuncs_inserted++;
1083 :
1084 102 : FOR_EACH_IMM_USE_STMT (stmt, ui, arg1)
1085 : {
1086 51 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1087 51 : gimple_assign_set_rhs_code (stmt, MULT_EXPR);
1088 51 : fold_stmt_inplace (&gsi);
1089 51 : update_stmt (stmt);
1090 51 : }
1091 : }
1092 : }
1093 : }
1094 : }
1095 :
1096 8776 : statistics_counter_event (fun, "reciprocal divs inserted",
1097 : reciprocal_stats.rdivs_inserted);
1098 8776 : statistics_counter_event (fun, "reciprocal functions inserted",
1099 : reciprocal_stats.rfuncs_inserted);
1100 :
1101 8776 : free_dominance_info (CDI_DOMINATORS);
1102 8776 : free_dominance_info (CDI_POST_DOMINATORS);
1103 17552 : delete occ_pool;
1104 8776 : return 0;
1105 : }
1106 :
1107 : } // anon namespace
1108 :
1109 : gimple_opt_pass *
1110 294196 : make_pass_cse_reciprocals (gcc::context *ctxt)
1111 : {
1112 294196 : return new pass_cse_reciprocals (ctxt);
1113 : }
1114 :
1115 : /* If NAME is the result of a type conversion, look for other
1116 : equivalent dominating or dominated conversions, and replace all
1117 : uses with the earliest dominating name, removing the redundant
1118 : conversions. Return the prevailing name. */
1119 :
1120 : static tree
1121 1069 : execute_cse_conv_1 (tree name, bool *cfg_changed)
1122 : {
1123 1069 : if (SSA_NAME_IS_DEFAULT_DEF (name)
1124 1069 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
1125 : return name;
1126 :
1127 969 : gimple *def_stmt = SSA_NAME_DEF_STMT (name);
1128 :
1129 969 : if (!gimple_assign_cast_p (def_stmt))
1130 : return name;
1131 :
1132 136 : tree src = gimple_assign_rhs1 (def_stmt);
1133 :
1134 136 : if (TREE_CODE (src) != SSA_NAME)
1135 : return name;
1136 :
1137 136 : imm_use_iterator use_iter;
1138 136 : gimple *use_stmt;
1139 :
1140 : /* Find the earliest dominating def. */
1141 521 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, src)
1142 : {
1143 763 : if (use_stmt == def_stmt
1144 385 : || !gimple_assign_cast_p (use_stmt))
1145 378 : continue;
1146 :
1147 7 : tree lhs = gimple_assign_lhs (use_stmt);
1148 :
1149 7 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
1150 14 : || (gimple_assign_rhs1 (use_stmt)
1151 7 : != gimple_assign_rhs1 (def_stmt))
1152 14 : || !types_compatible_p (TREE_TYPE (name), TREE_TYPE (lhs)))
1153 0 : continue;
1154 :
1155 7 : bool use_dominates;
1156 7 : if (gimple_bb (def_stmt) == gimple_bb (use_stmt))
1157 : {
1158 0 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
1159 0 : while (!gsi_end_p (gsi) && gsi_stmt (gsi) != def_stmt)
1160 0 : gsi_next (&gsi);
1161 0 : use_dominates = !gsi_end_p (gsi);
1162 : }
1163 7 : else if (dominated_by_p (CDI_DOMINATORS, gimple_bb (use_stmt),
1164 7 : gimple_bb (def_stmt)))
1165 : use_dominates = false;
1166 7 : else if (dominated_by_p (CDI_DOMINATORS, gimple_bb (def_stmt),
1167 7 : gimple_bb (use_stmt)))
1168 : use_dominates = true;
1169 : else
1170 4 : continue;
1171 :
1172 0 : if (use_dominates)
1173 : {
1174 : std::swap (name, lhs);
1175 : std::swap (def_stmt, use_stmt);
1176 : }
1177 136 : }
1178 :
1179 : /* Now go through all uses of SRC again, replacing the equivalent
1180 : dominated conversions. We may replace defs that were not
1181 : dominated by the then-prevailing defs when we first visited
1182 : them. */
1183 521 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, src)
1184 : {
1185 763 : if (use_stmt == def_stmt
1186 385 : || !gimple_assign_cast_p (use_stmt))
1187 378 : continue;
1188 :
1189 7 : tree lhs = gimple_assign_lhs (use_stmt);
1190 :
1191 7 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
1192 14 : || (gimple_assign_rhs1 (use_stmt)
1193 7 : != gimple_assign_rhs1 (def_stmt))
1194 14 : || !types_compatible_p (TREE_TYPE (name), TREE_TYPE (lhs)))
1195 0 : continue;
1196 :
1197 7 : basic_block use_bb = gimple_bb (use_stmt);
1198 7 : if (gimple_bb (def_stmt) == use_bb
1199 7 : || dominated_by_p (CDI_DOMINATORS, use_bb, gimple_bb (def_stmt)))
1200 : {
1201 3 : sincos_stats.conv_removed++;
1202 :
1203 3 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
1204 3 : replace_uses_by (lhs, name);
1205 3 : if (gsi_remove (&gsi, true)
1206 3 : && gimple_purge_dead_eh_edges (use_bb))
1207 3 : *cfg_changed = true;
1208 3 : release_defs (use_stmt);
1209 : }
1210 136 : }
1211 :
1212 136 : return name;
1213 : }
1214 :
1215 : /* Records an occurrence at statement USE_STMT in the vector of trees
1216 : STMTS if it is dominated by *TOP_BB or dominates it or this basic block
1217 : is not yet initialized. Returns true if the occurrence was pushed on
1218 : the vector. Adjusts *TOP_BB to be the basic block dominating all
1219 : statements in the vector. */
1220 :
1221 : static bool
1222 1290 : maybe_record_sincos (vec<gimple *> *stmts,
1223 : basic_block *top_bb, gimple *use_stmt)
1224 : {
1225 1290 : basic_block use_bb = gimple_bb (use_stmt);
1226 1290 : if (*top_bb
1227 1290 : && (*top_bb == use_bb
1228 66 : || dominated_by_p (CDI_DOMINATORS, use_bb, *top_bb)))
1229 168 : stmts->safe_push (use_stmt);
1230 1122 : else if (!*top_bb
1231 1122 : || dominated_by_p (CDI_DOMINATORS, *top_bb, use_bb))
1232 : {
1233 1102 : stmts->safe_push (use_stmt);
1234 1102 : *top_bb = use_bb;
1235 : }
1236 : else
1237 : return false;
1238 :
1239 : return true;
1240 : }
1241 :
1242 : /* Look for sin, cos and cexpi calls with the same argument NAME and
1243 : create a single call to cexpi CSEing the result in this case.
1244 : We first walk over all immediate uses of the argument collecting
1245 : statements that we can CSE in a vector and in a second pass replace
1246 : the statement rhs with a REALPART or IMAGPART expression on the
1247 : result of the cexpi call we insert before the use statement that
1248 : dominates all other candidates. */
1249 :
1250 : static bool
1251 1069 : execute_cse_sincos_1 (tree name)
1252 : {
1253 1069 : gimple_stmt_iterator gsi;
1254 1069 : imm_use_iterator use_iter;
1255 1069 : tree fndecl, res, type = NULL_TREE;
1256 1069 : gimple *def_stmt, *use_stmt, *stmt;
1257 1069 : int seen_cos = 0, seen_sin = 0, seen_cexpi = 0;
1258 1069 : auto_vec<gimple *> stmts;
1259 1069 : basic_block top_bb = NULL;
1260 1069 : int i;
1261 1069 : bool cfg_changed = false;
1262 :
1263 1069 : name = execute_cse_conv_1 (name, &cfg_changed);
1264 :
1265 4212 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, name)
1266 : {
1267 3145 : if (gimple_code (use_stmt) != GIMPLE_CALL
1268 3145 : || !gimple_call_lhs (use_stmt))
1269 1823 : continue;
1270 :
1271 1322 : switch (gimple_call_combined_fn (use_stmt))
1272 : {
1273 462 : CASE_CFN_COS:
1274 462 : seen_cos |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
1275 462 : break;
1276 :
1277 822 : CASE_CFN_SIN:
1278 822 : seen_sin |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
1279 822 : break;
1280 :
1281 6 : CASE_CFN_CEXPI:
1282 6 : seen_cexpi |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
1283 6 : break;
1284 :
1285 32 : default:;
1286 32 : continue;
1287 : }
1288 :
1289 1290 : auto stmt_cfn = gimple_call_combined_fn (use_stmt);
1290 1290 : tree t = mathfn_built_in_type (stmt_cfn);
1291 1290 : if (!t)
1292 : {
1293 : /* It is possible to get IFN_{SIN,COS} calls, for which
1294 : mathfn_built_in_type will return NULL. Those are normally only
1295 : present for vector operations. We won't be able to CSE those
1296 : at the moment. */
1297 2 : gcc_checking_assert (internal_fn_p (stmt_cfn));
1298 : return false;
1299 : }
1300 :
1301 1288 : if (!type)
1302 : {
1303 1067 : type = t;
1304 1067 : t = TREE_TYPE (name);
1305 : }
1306 : /* This checks that NAME has the right type in the first round,
1307 : and, in subsequent rounds, that the built_in type is the same
1308 : type, or a compatible type. */
1309 1288 : if (type != t && !types_compatible_p (type, t))
1310 : return false;
1311 2 : }
1312 1067 : if (seen_cos + seen_sin + seen_cexpi <= 1)
1313 : return false;
1314 :
1315 : /* Simply insert cexpi at the beginning of top_bb but not earlier than
1316 : the name def statement. */
1317 201 : fndecl = mathfn_built_in (type, BUILT_IN_CEXPI);
1318 201 : if (!fndecl)
1319 : return false;
1320 157 : stmt = gimple_build_call (fndecl, 1, name);
1321 157 : res = make_temp_ssa_name (TREE_TYPE (TREE_TYPE (fndecl)), stmt, "sincostmp");
1322 157 : gimple_call_set_lhs (stmt, res);
1323 :
1324 157 : def_stmt = SSA_NAME_DEF_STMT (name);
1325 157 : if (!SSA_NAME_IS_DEFAULT_DEF (name)
1326 127 : && gimple_code (def_stmt) != GIMPLE_PHI
1327 276 : && gimple_bb (def_stmt) == top_bb)
1328 : {
1329 119 : gsi = gsi_for_stmt (def_stmt);
1330 119 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
1331 : }
1332 : else
1333 : {
1334 38 : gsi = gsi_after_labels (top_bb);
1335 38 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1336 : }
1337 157 : sincos_stats.inserted++;
1338 :
1339 : /* And adjust the recorded old call sites. */
1340 471 : for (i = 0; stmts.iterate (i, &use_stmt); ++i)
1341 : {
1342 314 : tree rhs = NULL;
1343 :
1344 314 : switch (gimple_call_combined_fn (use_stmt))
1345 : {
1346 157 : CASE_CFN_COS:
1347 157 : rhs = fold_build1 (REALPART_EXPR, type, res);
1348 157 : break;
1349 :
1350 157 : CASE_CFN_SIN:
1351 157 : rhs = fold_build1 (IMAGPART_EXPR, type, res);
1352 157 : break;
1353 :
1354 : CASE_CFN_CEXPI:
1355 : rhs = res;
1356 : break;
1357 :
1358 0 : default:;
1359 0 : gcc_unreachable ();
1360 : }
1361 :
1362 : /* Replace call with a copy. */
1363 314 : stmt = gimple_build_assign (gimple_call_lhs (use_stmt), rhs);
1364 :
1365 314 : gsi = gsi_for_stmt (use_stmt);
1366 314 : gsi_replace (&gsi, stmt, true);
1367 314 : if (gimple_purge_dead_eh_edges (gimple_bb (stmt)))
1368 0 : cfg_changed = true;
1369 : }
1370 :
1371 157 : return cfg_changed;
1372 1069 : }
1373 :
1374 : /* To evaluate powi(x,n), the floating point value x raised to the
1375 : constant integer exponent n, we use a hybrid algorithm that
1376 : combines the "window method" with look-up tables. For an
1377 : introduction to exponentiation algorithms and "addition chains",
1378 : see section 4.6.3, "Evaluation of Powers" of Donald E. Knuth,
1379 : "Seminumerical Algorithms", Vol. 2, "The Art of Computer Programming",
1380 : 3rd Edition, 1998, and Daniel M. Gordon, "A Survey of Fast Exponentiation
1381 : Methods", Journal of Algorithms, Vol. 27, pp. 129-146, 1998. */
1382 :
1383 : /* Provide a default value for POWI_MAX_MULTS, the maximum number of
1384 : multiplications to inline before calling the system library's pow
1385 : function. powi(x,n) requires at worst 2*bits(n)-2 multiplications,
1386 : so this default never requires calling pow, powf or powl. */
1387 :
1388 : #ifndef POWI_MAX_MULTS
1389 : #define POWI_MAX_MULTS (2*HOST_BITS_PER_WIDE_INT-2)
1390 : #endif
1391 :
1392 : /* The size of the "optimal power tree" lookup table. All
1393 : exponents less than this value are simply looked up in the
1394 : powi_table below. This threshold is also used to size the
1395 : cache of pseudo registers that hold intermediate results. */
1396 : #define POWI_TABLE_SIZE 256
1397 :
1398 : /* The size, in bits of the window, used in the "window method"
1399 : exponentiation algorithm. This is equivalent to a radix of
1400 : (1<<POWI_WINDOW_SIZE) in the corresponding "m-ary method". */
1401 : #define POWI_WINDOW_SIZE 3
1402 :
1403 : /* The following table is an efficient representation of an
1404 : "optimal power tree". For each value, i, the corresponding
1405 : value, j, in the table states than an optimal evaluation
1406 : sequence for calculating pow(x,i) can be found by evaluating
1407 : pow(x,j)*pow(x,i-j). An optimal power tree for the first
1408 : 100 integers is given in Knuth's "Seminumerical algorithms". */
1409 :
1410 : static const unsigned char powi_table[POWI_TABLE_SIZE] =
1411 : {
1412 : 0, 1, 1, 2, 2, 3, 3, 4, /* 0 - 7 */
1413 : 4, 6, 5, 6, 6, 10, 7, 9, /* 8 - 15 */
1414 : 8, 16, 9, 16, 10, 12, 11, 13, /* 16 - 23 */
1415 : 12, 17, 13, 18, 14, 24, 15, 26, /* 24 - 31 */
1416 : 16, 17, 17, 19, 18, 33, 19, 26, /* 32 - 39 */
1417 : 20, 25, 21, 40, 22, 27, 23, 44, /* 40 - 47 */
1418 : 24, 32, 25, 34, 26, 29, 27, 44, /* 48 - 55 */
1419 : 28, 31, 29, 34, 30, 60, 31, 36, /* 56 - 63 */
1420 : 32, 64, 33, 34, 34, 46, 35, 37, /* 64 - 71 */
1421 : 36, 65, 37, 50, 38, 48, 39, 69, /* 72 - 79 */
1422 : 40, 49, 41, 43, 42, 51, 43, 58, /* 80 - 87 */
1423 : 44, 64, 45, 47, 46, 59, 47, 76, /* 88 - 95 */
1424 : 48, 65, 49, 66, 50, 67, 51, 66, /* 96 - 103 */
1425 : 52, 70, 53, 74, 54, 104, 55, 74, /* 104 - 111 */
1426 : 56, 64, 57, 69, 58, 78, 59, 68, /* 112 - 119 */
1427 : 60, 61, 61, 80, 62, 75, 63, 68, /* 120 - 127 */
1428 : 64, 65, 65, 128, 66, 129, 67, 90, /* 128 - 135 */
1429 : 68, 73, 69, 131, 70, 94, 71, 88, /* 136 - 143 */
1430 : 72, 128, 73, 98, 74, 132, 75, 121, /* 144 - 151 */
1431 : 76, 102, 77, 124, 78, 132, 79, 106, /* 152 - 159 */
1432 : 80, 97, 81, 160, 82, 99, 83, 134, /* 160 - 167 */
1433 : 84, 86, 85, 95, 86, 160, 87, 100, /* 168 - 175 */
1434 : 88, 113, 89, 98, 90, 107, 91, 122, /* 176 - 183 */
1435 : 92, 111, 93, 102, 94, 126, 95, 150, /* 184 - 191 */
1436 : 96, 128, 97, 130, 98, 133, 99, 195, /* 192 - 199 */
1437 : 100, 128, 101, 123, 102, 164, 103, 138, /* 200 - 207 */
1438 : 104, 145, 105, 146, 106, 109, 107, 149, /* 208 - 215 */
1439 : 108, 200, 109, 146, 110, 170, 111, 157, /* 216 - 223 */
1440 : 112, 128, 113, 130, 114, 182, 115, 132, /* 224 - 231 */
1441 : 116, 200, 117, 132, 118, 158, 119, 206, /* 232 - 239 */
1442 : 120, 240, 121, 162, 122, 147, 123, 152, /* 240 - 247 */
1443 : 124, 166, 125, 214, 126, 138, 127, 153, /* 248 - 255 */
1444 : };
1445 :
1446 :
1447 : /* Return the number of multiplications required to calculate
1448 : powi(x,n) where n is less than POWI_TABLE_SIZE. This is a
1449 : subroutine of powi_cost. CACHE is an array indicating
1450 : which exponents have already been calculated. */
1451 :
1452 : static int
1453 1120 : powi_lookup_cost (unsigned HOST_WIDE_INT n, bool *cache)
1454 : {
1455 : /* If we've already calculated this exponent, then this evaluation
1456 : doesn't require any additional multiplications. */
1457 1861 : if (cache[n])
1458 1120 : return 0;
1459 :
1460 741 : cache[n] = true;
1461 741 : return powi_lookup_cost (n - powi_table[n], cache)
1462 741 : + powi_lookup_cost (powi_table[n], cache) + 1;
1463 : }
1464 :
1465 : /* Return the number of multiplications required to calculate
1466 : powi(x,n) for an arbitrary x, given the exponent N. This
1467 : function needs to be kept in sync with powi_as_mults below. */
1468 :
1469 : static int
1470 384 : powi_cost (HOST_WIDE_INT n)
1471 : {
1472 384 : bool cache[POWI_TABLE_SIZE];
1473 384 : unsigned HOST_WIDE_INT digit;
1474 384 : unsigned HOST_WIDE_INT val;
1475 384 : int result;
1476 :
1477 384 : if (n == 0)
1478 : return 0;
1479 :
1480 : /* Ignore the reciprocal when calculating the cost. */
1481 379 : val = absu_hwi (n);
1482 :
1483 : /* Initialize the exponent cache. */
1484 379 : memset (cache, 0, POWI_TABLE_SIZE * sizeof (bool));
1485 379 : cache[1] = true;
1486 :
1487 379 : result = 0;
1488 :
1489 379 : while (val >= POWI_TABLE_SIZE)
1490 : {
1491 0 : if (val & 1)
1492 : {
1493 0 : digit = val & ((1 << POWI_WINDOW_SIZE) - 1);
1494 0 : result += powi_lookup_cost (digit, cache)
1495 0 : + POWI_WINDOW_SIZE + 1;
1496 0 : val >>= POWI_WINDOW_SIZE;
1497 : }
1498 : else
1499 : {
1500 0 : val >>= 1;
1501 0 : result++;
1502 : }
1503 : }
1504 :
1505 379 : return result + powi_lookup_cost (val, cache);
1506 : }
1507 :
1508 : /* Recursive subroutine of powi_as_mults. This function takes the
1509 : array, CACHE, of already calculated exponents and an exponent N and
1510 : returns a tree that corresponds to CACHE[1]**N, with type TYPE. */
1511 :
1512 : static tree
1513 6242 : powi_as_mults_1 (gimple_stmt_iterator *gsi, location_t loc, tree type,
1514 : unsigned HOST_WIDE_INT n, tree *cache)
1515 : {
1516 6242 : tree op0, op1, ssa_target;
1517 6242 : unsigned HOST_WIDE_INT digit;
1518 6242 : gassign *mult_stmt;
1519 :
1520 6242 : if (n < POWI_TABLE_SIZE && cache[n])
1521 : return cache[n];
1522 :
1523 2209 : ssa_target = make_temp_ssa_name (type, NULL, "powmult");
1524 :
1525 2209 : if (n < POWI_TABLE_SIZE)
1526 : {
1527 2206 : cache[n] = ssa_target;
1528 2206 : op0 = powi_as_mults_1 (gsi, loc, type, n - powi_table[n], cache);
1529 2206 : op1 = powi_as_mults_1 (gsi, loc, type, powi_table[n], cache);
1530 : }
1531 3 : else if (n & 1)
1532 : {
1533 1 : digit = n & ((1 << POWI_WINDOW_SIZE) - 1);
1534 1 : op0 = powi_as_mults_1 (gsi, loc, type, n - digit, cache);
1535 1 : op1 = powi_as_mults_1 (gsi, loc, type, digit, cache);
1536 : }
1537 : else
1538 : {
1539 2 : op0 = powi_as_mults_1 (gsi, loc, type, n >> 1, cache);
1540 2 : op1 = op0;
1541 : }
1542 :
1543 2209 : mult_stmt = gimple_build_assign (ssa_target, MULT_EXPR, op0, op1);
1544 2209 : gimple_set_location (mult_stmt, loc);
1545 2209 : gsi_insert_before (gsi, mult_stmt, GSI_SAME_STMT);
1546 :
1547 2209 : return ssa_target;
1548 : }
1549 :
1550 : /* Convert ARG0**N to a tree of multiplications of ARG0 with itself.
1551 : This function needs to be kept in sync with powi_cost above. */
1552 :
1553 : tree
1554 1826 : powi_as_mults (gimple_stmt_iterator *gsi, location_t loc,
1555 : tree arg0, HOST_WIDE_INT n)
1556 : {
1557 1826 : tree cache[POWI_TABLE_SIZE], result, type = TREE_TYPE (arg0);
1558 1826 : gassign *div_stmt;
1559 1826 : tree target;
1560 :
1561 1826 : if (n == 0)
1562 0 : return build_one_cst (type);
1563 :
1564 1826 : memset (cache, 0, sizeof (cache));
1565 1826 : cache[1] = arg0;
1566 :
1567 1826 : result = powi_as_mults_1 (gsi, loc, type, absu_hwi (n), cache);
1568 1826 : if (n >= 0)
1569 : return result;
1570 :
1571 : /* If the original exponent was negative, reciprocate the result. */
1572 8 : target = make_temp_ssa_name (type, NULL, "powmult");
1573 8 : div_stmt = gimple_build_assign (target, RDIV_EXPR,
1574 : build_real (type, dconst1), result);
1575 8 : gimple_set_location (div_stmt, loc);
1576 8 : gsi_insert_before (gsi, div_stmt, GSI_SAME_STMT);
1577 :
1578 8 : return target;
1579 : }
1580 :
1581 : /* ARG0 and N are the two arguments to a powi builtin in GSI with
1582 : location info LOC. If the arguments are appropriate, create an
1583 : equivalent sequence of statements prior to GSI using an optimal
1584 : number of multiplications, and return an expression holding the
1585 : result. */
1586 :
1587 : static tree
1588 633 : gimple_expand_builtin_powi (gimple_stmt_iterator *gsi, location_t loc,
1589 : tree arg0, HOST_WIDE_INT n)
1590 : {
1591 633 : if ((n >= -1 && n <= 2)
1592 633 : || (optimize_function_for_speed_p (cfun)
1593 351 : && powi_cost (n) <= POWI_MAX_MULTS))
1594 625 : return powi_as_mults (gsi, loc, arg0, n);
1595 :
1596 : return NULL_TREE;
1597 : }
1598 :
1599 : /* Build a gimple call statement that calls FN with argument ARG.
1600 : Set the lhs of the call statement to a fresh SSA name. Insert the
1601 : statement prior to GSI's current position, and return the fresh
1602 : SSA name. */
1603 :
1604 : static tree
1605 44 : build_and_insert_call (gimple_stmt_iterator *gsi, location_t loc,
1606 : tree fn, tree arg)
1607 : {
1608 44 : gcall *call_stmt;
1609 44 : tree ssa_target;
1610 :
1611 44 : call_stmt = gimple_build_call (fn, 1, arg);
1612 44 : ssa_target = make_temp_ssa_name (TREE_TYPE (arg), NULL, "powroot");
1613 44 : gimple_set_lhs (call_stmt, ssa_target);
1614 44 : gimple_set_location (call_stmt, loc);
1615 44 : gsi_insert_before (gsi, call_stmt, GSI_SAME_STMT);
1616 :
1617 44 : return ssa_target;
1618 : }
1619 :
1620 : /* Build a gimple binary operation with the given CODE and arguments
1621 : ARG0, ARG1, assigning the result to a new SSA name for variable
1622 : TARGET. Insert the statement prior to GSI's current position, and
1623 : return the fresh SSA name.*/
1624 :
1625 : static tree
1626 2643 : build_and_insert_binop (gimple_stmt_iterator *gsi, location_t loc,
1627 : const char *name, enum tree_code code,
1628 : tree arg0, tree arg1)
1629 : {
1630 2643 : tree result = make_temp_ssa_name (TREE_TYPE (arg0), NULL, name);
1631 2643 : gassign *stmt = gimple_build_assign (result, code, arg0, arg1);
1632 2643 : gimple_set_location (stmt, loc);
1633 2643 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1634 2643 : return result;
1635 : }
1636 :
1637 : /* Build a gimple assignment to cast VAL to TYPE. Insert the statement
1638 : prior to GSI's current position, and return the fresh SSA name. */
1639 :
1640 : static tree
1641 14586 : build_and_insert_cast (gimple_stmt_iterator *gsi, location_t loc,
1642 : tree type, tree val)
1643 : {
1644 0 : return gimple_convert (gsi, true, GSI_SAME_STMT, loc, type, val);
1645 : }
1646 :
1647 : struct pow_synth_sqrt_info
1648 : {
1649 : bool *factors;
1650 : unsigned int deepest;
1651 : unsigned int num_mults;
1652 : };
1653 :
1654 : /* Return true iff the real value C can be represented as a
1655 : sum of powers of 0.5 up to N. That is:
1656 : C == SUM<i from 1..N> (a[i]*(0.5**i)) where a[i] is either 0 or 1.
1657 : Record in INFO the various parameters of the synthesis algorithm such
1658 : as the factors a[i], the maximum 0.5 power and the number of
1659 : multiplications that will be required. */
1660 :
1661 : bool
1662 33 : representable_as_half_series_p (REAL_VALUE_TYPE c, unsigned n,
1663 : struct pow_synth_sqrt_info *info)
1664 : {
1665 33 : REAL_VALUE_TYPE factor = dconsthalf;
1666 33 : REAL_VALUE_TYPE remainder = c;
1667 :
1668 33 : info->deepest = 0;
1669 33 : info->num_mults = 0;
1670 33 : memset (info->factors, 0, n * sizeof (bool));
1671 :
1672 97 : for (unsigned i = 0; i < n; i++)
1673 : {
1674 90 : REAL_VALUE_TYPE res;
1675 :
1676 : /* If something inexact happened bail out now. */
1677 90 : if (real_arithmetic (&res, MINUS_EXPR, &remainder, &factor))
1678 26 : return false;
1679 :
1680 : /* We have hit zero. The number is representable as a sum
1681 : of powers of 0.5. */
1682 90 : if (real_equal (&res, &dconst0))
1683 : {
1684 26 : info->factors[i] = true;
1685 26 : info->deepest = i + 1;
1686 26 : return true;
1687 : }
1688 64 : else if (!REAL_VALUE_NEGATIVE (res))
1689 : {
1690 29 : remainder = res;
1691 29 : info->factors[i] = true;
1692 29 : info->num_mults++;
1693 : }
1694 : else
1695 35 : info->factors[i] = false;
1696 :
1697 64 : real_arithmetic (&factor, MULT_EXPR, &factor, &dconsthalf);
1698 : }
1699 : return false;
1700 : }
1701 :
1702 : /* Return the tree corresponding to FN being applied
1703 : to ARG N times at GSI and LOC.
1704 : Look up previous results from CACHE if need be.
1705 : cache[0] should contain just plain ARG i.e. FN applied to ARG 0 times. */
1706 :
1707 : static tree
1708 63 : get_fn_chain (tree arg, unsigned int n, gimple_stmt_iterator *gsi,
1709 : tree fn, location_t loc, tree *cache)
1710 : {
1711 63 : tree res = cache[n];
1712 63 : if (!res)
1713 : {
1714 40 : tree prev = get_fn_chain (arg, n - 1, gsi, fn, loc, cache);
1715 40 : res = build_and_insert_call (gsi, loc, fn, prev);
1716 40 : cache[n] = res;
1717 : }
1718 :
1719 63 : return res;
1720 : }
1721 :
1722 : /* Print to STREAM the repeated application of function FNAME to ARG
1723 : N times. So, for FNAME = "foo", ARG = "x", N = 2 it would print:
1724 : "foo (foo (x))". */
1725 :
1726 : static void
1727 36 : print_nested_fn (FILE* stream, const char *fname, const char* arg,
1728 : unsigned int n)
1729 : {
1730 36 : if (n == 0)
1731 10 : fprintf (stream, "%s", arg);
1732 : else
1733 : {
1734 26 : fprintf (stream, "%s (", fname);
1735 26 : print_nested_fn (stream, fname, arg, n - 1);
1736 26 : fprintf (stream, ")");
1737 : }
1738 36 : }
1739 :
1740 : /* Print to STREAM the fractional sequence of sqrt chains
1741 : applied to ARG, described by INFO. Used for the dump file. */
1742 :
1743 : static void
1744 7 : dump_fractional_sqrt_sequence (FILE *stream, const char *arg,
1745 : struct pow_synth_sqrt_info *info)
1746 : {
1747 29 : for (unsigned int i = 0; i < info->deepest; i++)
1748 : {
1749 22 : bool is_set = info->factors[i];
1750 22 : if (is_set)
1751 : {
1752 10 : print_nested_fn (stream, "sqrt", arg, i + 1);
1753 10 : if (i != info->deepest - 1)
1754 3 : fprintf (stream, " * ");
1755 : }
1756 : }
1757 7 : }
1758 :
1759 : /* Print to STREAM a representation of raising ARG to an integer
1760 : power N. Used for the dump file. */
1761 :
1762 : static void
1763 7 : dump_integer_part (FILE *stream, const char* arg, HOST_WIDE_INT n)
1764 : {
1765 7 : if (n > 1)
1766 3 : fprintf (stream, "powi (%s, " HOST_WIDE_INT_PRINT_DEC ")", arg, n);
1767 4 : else if (n == 1)
1768 3 : fprintf (stream, "%s", arg);
1769 7 : }
1770 :
1771 : /* Attempt to synthesize a POW[F] (ARG0, ARG1) call using chains of
1772 : square roots. Place at GSI and LOC. Limit the maximum depth
1773 : of the sqrt chains to MAX_DEPTH. Return the tree holding the
1774 : result of the expanded sequence or NULL_TREE if the expansion failed.
1775 :
1776 : This routine assumes that ARG1 is a real number with a fractional part
1777 : (the integer exponent case will have been handled earlier in
1778 : gimple_expand_builtin_pow).
1779 :
1780 : For ARG1 > 0.0:
1781 : * For ARG1 composed of a whole part WHOLE_PART and a fractional part
1782 : FRAC_PART i.e. WHOLE_PART == floor (ARG1) and
1783 : FRAC_PART == ARG1 - WHOLE_PART:
1784 : Produce POWI (ARG0, WHOLE_PART) * POW (ARG0, FRAC_PART) where
1785 : POW (ARG0, FRAC_PART) is expanded as a product of square root chains
1786 : if it can be expressed as such, that is if FRAC_PART satisfies:
1787 : FRAC_PART == <SUM from i = 1 until MAX_DEPTH> (a[i] * (0.5**i))
1788 : where integer a[i] is either 0 or 1.
1789 :
1790 : Example:
1791 : POW (x, 3.625) == POWI (x, 3) * POW (x, 0.625)
1792 : --> POWI (x, 3) * SQRT (x) * SQRT (SQRT (SQRT (x)))
1793 :
1794 : For ARG1 < 0.0 there are two approaches:
1795 : * (A) Expand to 1.0 / POW (ARG0, -ARG1) where POW (ARG0, -ARG1)
1796 : is calculated as above.
1797 :
1798 : Example:
1799 : POW (x, -5.625) == 1.0 / POW (x, 5.625)
1800 : --> 1.0 / (POWI (x, 5) * SQRT (x) * SQRT (SQRT (SQRT (x))))
1801 :
1802 : * (B) : WHOLE_PART := - ceil (abs (ARG1))
1803 : FRAC_PART := ARG1 - WHOLE_PART
1804 : and expand to POW (x, FRAC_PART) / POWI (x, WHOLE_PART).
1805 : Example:
1806 : POW (x, -5.875) == POW (x, 0.125) / POWI (X, 6)
1807 : --> SQRT (SQRT (SQRT (x))) / (POWI (x, 6))
1808 :
1809 : For ARG1 < 0.0 we choose between (A) and (B) depending on
1810 : how many multiplications we'd have to do.
1811 : So, for the example in (B): POW (x, -5.875), if we were to
1812 : follow algorithm (A) we would produce:
1813 : 1.0 / POWI (X, 5) * SQRT (X) * SQRT (SQRT (X)) * SQRT (SQRT (SQRT (X)))
1814 : which contains more multiplications than approach (B).
1815 :
1816 : Hopefully, this approach will eliminate potentially expensive POW library
1817 : calls when unsafe floating point math is enabled and allow the compiler to
1818 : further optimise the multiplies, square roots and divides produced by this
1819 : function. */
1820 :
1821 : static tree
1822 25 : expand_pow_as_sqrts (gimple_stmt_iterator *gsi, location_t loc,
1823 : tree arg0, tree arg1, HOST_WIDE_INT max_depth)
1824 : {
1825 25 : tree type = TREE_TYPE (arg0);
1826 25 : machine_mode mode = TYPE_MODE (type);
1827 25 : tree sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
1828 25 : bool one_over = true;
1829 :
1830 25 : if (!sqrtfn)
1831 : return NULL_TREE;
1832 :
1833 25 : if (TREE_CODE (arg1) != REAL_CST)
1834 : return NULL_TREE;
1835 :
1836 25 : REAL_VALUE_TYPE exp_init = TREE_REAL_CST (arg1);
1837 :
1838 25 : gcc_assert (max_depth > 0);
1839 25 : tree *cache = XALLOCAVEC (tree, max_depth + 1);
1840 :
1841 25 : struct pow_synth_sqrt_info synth_info;
1842 25 : synth_info.factors = XALLOCAVEC (bool, max_depth + 1);
1843 25 : synth_info.deepest = 0;
1844 25 : synth_info.num_mults = 0;
1845 :
1846 25 : bool neg_exp = REAL_VALUE_NEGATIVE (exp_init);
1847 25 : REAL_VALUE_TYPE exp = real_value_abs (&exp_init);
1848 :
1849 : /* The whole and fractional parts of exp. */
1850 25 : REAL_VALUE_TYPE whole_part;
1851 25 : REAL_VALUE_TYPE frac_part;
1852 :
1853 25 : real_floor (&whole_part, mode, &exp);
1854 25 : real_arithmetic (&frac_part, MINUS_EXPR, &exp, &whole_part);
1855 :
1856 :
1857 25 : REAL_VALUE_TYPE ceil_whole = dconst0;
1858 25 : REAL_VALUE_TYPE ceil_fract = dconst0;
1859 :
1860 25 : if (neg_exp)
1861 : {
1862 10 : real_ceil (&ceil_whole, mode, &exp);
1863 10 : real_arithmetic (&ceil_fract, MINUS_EXPR, &ceil_whole, &exp);
1864 : }
1865 :
1866 25 : if (!representable_as_half_series_p (frac_part, max_depth, &synth_info))
1867 : return NULL_TREE;
1868 :
1869 : /* Check whether it's more profitable to not use 1.0 / ... */
1870 18 : if (neg_exp)
1871 : {
1872 8 : struct pow_synth_sqrt_info alt_synth_info;
1873 8 : alt_synth_info.factors = XALLOCAVEC (bool, max_depth + 1);
1874 8 : alt_synth_info.deepest = 0;
1875 8 : alt_synth_info.num_mults = 0;
1876 :
1877 8 : if (representable_as_half_series_p (ceil_fract, max_depth,
1878 : &alt_synth_info)
1879 8 : && alt_synth_info.deepest <= synth_info.deepest
1880 16 : && alt_synth_info.num_mults < synth_info.num_mults)
1881 : {
1882 2 : whole_part = ceil_whole;
1883 2 : frac_part = ceil_fract;
1884 2 : synth_info.deepest = alt_synth_info.deepest;
1885 2 : synth_info.num_mults = alt_synth_info.num_mults;
1886 2 : memcpy (synth_info.factors, alt_synth_info.factors,
1887 : (max_depth + 1) * sizeof (bool));
1888 2 : one_over = false;
1889 : }
1890 : }
1891 :
1892 18 : HOST_WIDE_INT n = real_to_integer (&whole_part);
1893 18 : REAL_VALUE_TYPE cint;
1894 18 : real_from_integer (&cint, VOIDmode, n, SIGNED);
1895 :
1896 18 : if (!real_identical (&whole_part, &cint))
1897 : return NULL_TREE;
1898 :
1899 18 : if (powi_cost (n) + synth_info.num_mults > POWI_MAX_MULTS)
1900 : return NULL_TREE;
1901 :
1902 18 : memset (cache, 0, (max_depth + 1) * sizeof (tree));
1903 :
1904 18 : tree integer_res = n == 0 ? build_real (type, dconst1) : arg0;
1905 :
1906 : /* Calculate the integer part of the exponent. */
1907 18 : if (n > 1)
1908 : {
1909 6 : integer_res = gimple_expand_builtin_powi (gsi, loc, arg0, n);
1910 6 : if (!integer_res)
1911 : return NULL_TREE;
1912 : }
1913 :
1914 18 : if (dump_file)
1915 : {
1916 7 : char string[64];
1917 :
1918 7 : real_to_decimal (string, &exp_init, sizeof (string), 0, 1);
1919 7 : fprintf (dump_file, "synthesizing pow (x, %s) as:\n", string);
1920 :
1921 7 : if (neg_exp)
1922 : {
1923 2 : if (one_over)
1924 : {
1925 1 : fprintf (dump_file, "1.0 / (");
1926 1 : dump_integer_part (dump_file, "x", n);
1927 1 : if (n > 0)
1928 1 : fprintf (dump_file, " * ");
1929 1 : dump_fractional_sqrt_sequence (dump_file, "x", &synth_info);
1930 1 : fprintf (dump_file, ")");
1931 : }
1932 : else
1933 : {
1934 1 : dump_fractional_sqrt_sequence (dump_file, "x", &synth_info);
1935 1 : fprintf (dump_file, " / (");
1936 1 : dump_integer_part (dump_file, "x", n);
1937 1 : fprintf (dump_file, ")");
1938 : }
1939 : }
1940 : else
1941 : {
1942 5 : dump_fractional_sqrt_sequence (dump_file, "x", &synth_info);
1943 5 : if (n > 0)
1944 4 : fprintf (dump_file, " * ");
1945 5 : dump_integer_part (dump_file, "x", n);
1946 : }
1947 :
1948 7 : fprintf (dump_file, "\ndeepest sqrt chain: %d\n", synth_info.deepest);
1949 : }
1950 :
1951 :
1952 18 : tree fract_res = NULL_TREE;
1953 18 : cache[0] = arg0;
1954 :
1955 : /* Calculate the fractional part of the exponent. */
1956 58 : for (unsigned i = 0; i < synth_info.deepest; i++)
1957 : {
1958 40 : if (synth_info.factors[i])
1959 : {
1960 23 : tree sqrt_chain = get_fn_chain (arg0, i + 1, gsi, sqrtfn, loc, cache);
1961 :
1962 23 : if (!fract_res)
1963 : fract_res = sqrt_chain;
1964 :
1965 : else
1966 5 : fract_res = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
1967 : fract_res, sqrt_chain);
1968 : }
1969 : }
1970 :
1971 18 : tree res = NULL_TREE;
1972 :
1973 18 : if (neg_exp)
1974 : {
1975 8 : if (one_over)
1976 : {
1977 6 : if (n > 0)
1978 4 : res = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
1979 : fract_res, integer_res);
1980 : else
1981 : res = fract_res;
1982 :
1983 6 : res = build_and_insert_binop (gsi, loc, "powrootrecip", RDIV_EXPR,
1984 : build_real (type, dconst1), res);
1985 : }
1986 : else
1987 : {
1988 2 : res = build_and_insert_binop (gsi, loc, "powroot", RDIV_EXPR,
1989 : fract_res, integer_res);
1990 : }
1991 : }
1992 : else
1993 10 : res = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
1994 : fract_res, integer_res);
1995 : return res;
1996 : }
1997 :
1998 : /* ARG0 and ARG1 are the two arguments to a pow builtin call in GSI
1999 : with location info LOC. If possible, create an equivalent and
2000 : less expensive sequence of statements prior to GSI, and return an
2001 : expression holding the result. */
2002 :
2003 : static tree
2004 601 : gimple_expand_builtin_pow (gimple_stmt_iterator *gsi, location_t loc,
2005 : tree arg0, tree arg1)
2006 : {
2007 601 : REAL_VALUE_TYPE c, cint, dconst1_3, dconst1_4, dconst1_6;
2008 601 : REAL_VALUE_TYPE c2, dconst3;
2009 601 : HOST_WIDE_INT n;
2010 601 : tree type, sqrtfn, cbrtfn, sqrt_arg0, result, cbrt_x, powi_cbrt_x;
2011 601 : machine_mode mode;
2012 601 : bool speed_p = optimize_bb_for_speed_p (gsi_bb (*gsi));
2013 601 : bool hw_sqrt_exists, c_is_int, c2_is_int;
2014 :
2015 601 : dconst1_4 = dconst1;
2016 601 : SET_REAL_EXP (&dconst1_4, REAL_EXP (&dconst1_4) - 2);
2017 :
2018 : /* If the exponent isn't a constant, there's nothing of interest
2019 : to be done. */
2020 601 : if (TREE_CODE (arg1) != REAL_CST)
2021 : return NULL_TREE;
2022 :
2023 : /* Don't perform the operation if flag_signaling_nans is on
2024 : and the operand is a signaling NaN. */
2025 363 : if (HONOR_SNANS (TYPE_MODE (TREE_TYPE (arg1)))
2026 363 : && ((TREE_CODE (arg0) == REAL_CST
2027 0 : && REAL_VALUE_ISSIGNALING_NAN (TREE_REAL_CST (arg0)))
2028 1 : || REAL_VALUE_ISSIGNALING_NAN (TREE_REAL_CST (arg1))))
2029 : return NULL_TREE;
2030 :
2031 363 : if (flag_errno_math)
2032 : return NULL_TREE;
2033 :
2034 : /* If the exponent is equivalent to an integer, expand to an optimal
2035 : multiplication sequence when profitable. */
2036 75 : c = TREE_REAL_CST (arg1);
2037 75 : n = real_to_integer (&c);
2038 75 : real_from_integer (&cint, VOIDmode, n, SIGNED);
2039 75 : c_is_int = real_identical (&c, &cint);
2040 :
2041 75 : if (c_is_int
2042 75 : && ((n >= -1 && n <= 2)
2043 21 : || (flag_unsafe_math_optimizations
2044 11 : && speed_p
2045 11 : && powi_cost (n) <= POWI_MAX_MULTS)))
2046 30 : return gimple_expand_builtin_powi (gsi, loc, arg0, n);
2047 :
2048 : /* Attempt various optimizations using sqrt and cbrt. */
2049 45 : type = TREE_TYPE (arg0);
2050 45 : mode = TYPE_MODE (type);
2051 45 : sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
2052 :
2053 : /* Optimize pow(x,0.5) = sqrt(x). This replacement is always safe
2054 : unless signed zeros must be maintained. pow(-0,0.5) = +0, while
2055 : sqrt(-0) = -0. */
2056 45 : if (sqrtfn
2057 45 : && real_equal (&c, &dconsthalf)
2058 52 : && !HONOR_SIGNED_ZEROS (mode))
2059 0 : return build_and_insert_call (gsi, loc, sqrtfn, arg0);
2060 :
2061 45 : hw_sqrt_exists = optab_handler (sqrt_optab, mode) != CODE_FOR_nothing;
2062 :
2063 : /* Optimize pow(x,1./3.) = cbrt(x). This requires unsafe math
2064 : optimizations since 1./3. is not exactly representable. If x
2065 : is negative and finite, the correct value of pow(x,1./3.) is
2066 : a NaN with the "invalid" exception raised, because the value
2067 : of 1./3. actually has an even denominator. The correct value
2068 : of cbrt(x) is a negative real value. */
2069 45 : cbrtfn = mathfn_built_in (type, BUILT_IN_CBRT);
2070 45 : dconst1_3 = real_value_truncate (mode, dconst_third ());
2071 :
2072 45 : if (flag_unsafe_math_optimizations
2073 25 : && cbrtfn
2074 25 : && (!HONOR_NANS (mode) || tree_expr_nonnegative_p (arg0))
2075 70 : && real_equal (&c, &dconst1_3))
2076 0 : return build_and_insert_call (gsi, loc, cbrtfn, arg0);
2077 :
2078 : /* Optimize pow(x,1./6.) = cbrt(sqrt(x)). Don't do this optimization
2079 : if we don't have a hardware sqrt insn. */
2080 45 : dconst1_6 = dconst1_3;
2081 45 : SET_REAL_EXP (&dconst1_6, REAL_EXP (&dconst1_6) - 1);
2082 :
2083 45 : if (flag_unsafe_math_optimizations
2084 25 : && sqrtfn
2085 25 : && cbrtfn
2086 25 : && (!HONOR_NANS (mode) || tree_expr_nonnegative_p (arg0))
2087 : && speed_p
2088 25 : && hw_sqrt_exists
2089 70 : && real_equal (&c, &dconst1_6))
2090 : {
2091 : /* sqrt(x) */
2092 0 : sqrt_arg0 = build_and_insert_call (gsi, loc, sqrtfn, arg0);
2093 :
2094 : /* cbrt(sqrt(x)) */
2095 0 : return build_and_insert_call (gsi, loc, cbrtfn, sqrt_arg0);
2096 : }
2097 :
2098 :
2099 : /* Attempt to expand the POW as a product of square root chains.
2100 : Expand the 0.25 case even when optimising for size. */
2101 45 : if (flag_unsafe_math_optimizations
2102 25 : && sqrtfn
2103 25 : && hw_sqrt_exists
2104 25 : && (speed_p || real_equal (&c, &dconst1_4))
2105 70 : && !HONOR_SIGNED_ZEROS (mode))
2106 : {
2107 75 : unsigned int max_depth = speed_p
2108 25 : ? param_max_pow_sqrt_depth
2109 : : 2;
2110 :
2111 25 : tree expand_with_sqrts
2112 25 : = expand_pow_as_sqrts (gsi, loc, arg0, arg1, max_depth);
2113 :
2114 25 : if (expand_with_sqrts)
2115 : return expand_with_sqrts;
2116 : }
2117 :
2118 27 : real_arithmetic (&c2, MULT_EXPR, &c, &dconst2);
2119 27 : n = real_to_integer (&c2);
2120 27 : real_from_integer (&cint, VOIDmode, n, SIGNED);
2121 27 : c2_is_int = real_identical (&c2, &cint);
2122 :
2123 : /* Optimize pow(x,c), where 3c = n for some nonzero integer n, into
2124 :
2125 : powi(x, n/3) * powi(cbrt(x), n%3), n > 0;
2126 : 1.0 / (powi(x, abs(n)/3) * powi(cbrt(x), abs(n)%3)), n < 0.
2127 :
2128 : Do not calculate the first factor when n/3 = 0. As cbrt(x) is
2129 : different from pow(x, 1./3.) due to rounding and behavior with
2130 : negative x, we need to constrain this transformation to unsafe
2131 : math and positive x or finite math. */
2132 27 : real_from_integer (&dconst3, VOIDmode, 3, SIGNED);
2133 27 : real_arithmetic (&c2, MULT_EXPR, &c, &dconst3);
2134 27 : real_round (&c2, mode, &c2);
2135 27 : n = real_to_integer (&c2);
2136 27 : real_from_integer (&cint, VOIDmode, n, SIGNED);
2137 27 : real_arithmetic (&c2, RDIV_EXPR, &cint, &dconst3);
2138 27 : real_convert (&c2, mode, &c2);
2139 :
2140 27 : if (flag_unsafe_math_optimizations
2141 7 : && cbrtfn
2142 7 : && (!HONOR_NANS (mode) || tree_expr_nonnegative_p (arg0))
2143 7 : && real_identical (&c2, &c)
2144 4 : && !c2_is_int
2145 4 : && optimize_function_for_speed_p (cfun)
2146 31 : && powi_cost (n / 3) <= POWI_MAX_MULTS)
2147 : {
2148 4 : tree powi_x_ndiv3 = NULL_TREE;
2149 :
2150 : /* Attempt to fold powi(arg0, abs(n/3)) into multiplies. If not
2151 : possible or profitable, give up. Skip the degenerate case when
2152 : abs(n) < 3, where the result is always 1. */
2153 4 : if (absu_hwi (n) >= 3)
2154 : {
2155 4 : powi_x_ndiv3 = gimple_expand_builtin_powi (gsi, loc, arg0,
2156 : abs_hwi (n / 3));
2157 4 : if (!powi_x_ndiv3)
2158 : return NULL_TREE;
2159 : }
2160 :
2161 : /* Calculate powi(cbrt(x), n%3). Don't use gimple_expand_builtin_powi
2162 : as that creates an unnecessary variable. Instead, just produce
2163 : either cbrt(x) or cbrt(x) * cbrt(x). */
2164 4 : cbrt_x = build_and_insert_call (gsi, loc, cbrtfn, arg0);
2165 :
2166 4 : if (absu_hwi (n) % 3 == 1)
2167 : powi_cbrt_x = cbrt_x;
2168 : else
2169 2 : powi_cbrt_x = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
2170 : cbrt_x, cbrt_x);
2171 :
2172 : /* Multiply the two subexpressions, unless powi(x,abs(n)/3) = 1. */
2173 4 : if (absu_hwi (n) < 3)
2174 : result = powi_cbrt_x;
2175 : else
2176 4 : result = build_and_insert_binop (gsi, loc, "powroot", MULT_EXPR,
2177 : powi_x_ndiv3, powi_cbrt_x);
2178 :
2179 : /* If n is negative, reciprocate the result. */
2180 4 : if (n < 0)
2181 1 : result = build_and_insert_binop (gsi, loc, "powroot", RDIV_EXPR,
2182 : build_real (type, dconst1), result);
2183 :
2184 : return result;
2185 : }
2186 :
2187 : /* No optimizations succeeded. */
2188 : return NULL_TREE;
2189 : }
2190 :
2191 : /* Go through all calls to sin, cos and cexpi and call execute_cse_sincos_1
2192 : on the SSA_NAME argument of each of them. */
2193 :
2194 : namespace {
2195 :
2196 : const pass_data pass_data_cse_sincos =
2197 : {
2198 : GIMPLE_PASS, /* type */
2199 : "sincos", /* name */
2200 : OPTGROUP_NONE, /* optinfo_flags */
2201 : TV_TREE_SINCOS, /* tv_id */
2202 : PROP_ssa, /* properties_required */
2203 : 0, /* properties_provided */
2204 : 0, /* properties_destroyed */
2205 : 0, /* todo_flags_start */
2206 : TODO_update_ssa, /* todo_flags_finish */
2207 : };
2208 :
2209 : class pass_cse_sincos : public gimple_opt_pass
2210 : {
2211 : public:
2212 294196 : pass_cse_sincos (gcc::context *ctxt)
2213 588392 : : gimple_opt_pass (pass_data_cse_sincos, ctxt)
2214 : {}
2215 :
2216 : /* opt_pass methods: */
2217 1060389 : bool gate (function *) final override
2218 : {
2219 1060389 : return optimize;
2220 : }
2221 :
2222 : unsigned int execute (function *) final override;
2223 :
2224 : }; // class pass_cse_sincos
2225 :
2226 : unsigned int
2227 1060351 : pass_cse_sincos::execute (function *fun)
2228 : {
2229 1060351 : basic_block bb;
2230 1060351 : bool cfg_changed = false;
2231 :
2232 1060351 : calculate_dominance_info (CDI_DOMINATORS);
2233 1060351 : memset (&sincos_stats, 0, sizeof (sincos_stats));
2234 :
2235 11273150 : FOR_EACH_BB_FN (bb, fun)
2236 : {
2237 10212799 : gimple_stmt_iterator gsi;
2238 :
2239 100392440 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2240 : {
2241 90179641 : gimple *stmt = gsi_stmt (gsi);
2242 :
2243 90179641 : if (is_gimple_call (stmt)
2244 90179641 : && gimple_call_lhs (stmt))
2245 : {
2246 2060986 : tree arg;
2247 2060986 : switch (gimple_call_combined_fn (stmt))
2248 : {
2249 1069 : CASE_CFN_COS:
2250 1069 : CASE_CFN_SIN:
2251 1069 : CASE_CFN_CEXPI:
2252 1069 : arg = gimple_call_arg (stmt, 0);
2253 : /* Make sure we have either sincos or cexp. */
2254 1069 : if (!targetm.libc_has_function (function_c99_math_complex,
2255 1069 : TREE_TYPE (arg))
2256 1069 : && !targetm.libc_has_function (function_sincos,
2257 0 : TREE_TYPE (arg)))
2258 : break;
2259 :
2260 1069 : if (TREE_CODE (arg) == SSA_NAME)
2261 1069 : cfg_changed |= execute_cse_sincos_1 (arg);
2262 : break;
2263 : default:
2264 : break;
2265 : }
2266 : }
2267 : }
2268 : }
2269 :
2270 1060351 : statistics_counter_event (fun, "sincos statements inserted",
2271 : sincos_stats.inserted);
2272 1060351 : statistics_counter_event (fun, "conv statements removed",
2273 : sincos_stats.conv_removed);
2274 :
2275 1060351 : return cfg_changed ? TODO_cleanup_cfg : 0;
2276 : }
2277 :
2278 : } // anon namespace
2279 :
2280 : gimple_opt_pass *
2281 294196 : make_pass_cse_sincos (gcc::context *ctxt)
2282 : {
2283 294196 : return new pass_cse_sincos (ctxt);
2284 : }
2285 :
2286 : /* Expand powi(x,n) into an optimal number of multiplies, when n is a
2287 : constant. */
2288 : namespace {
2289 :
2290 : const pass_data pass_data_expand_pow =
2291 : {
2292 : GIMPLE_PASS, /* type */
2293 : "pow", /* name */
2294 : OPTGROUP_NONE, /* optinfo_flags */
2295 : TV_TREE_POW, /* tv_id */
2296 : PROP_ssa, /* properties_required */
2297 : PROP_gimple_opt_math, /* properties_provided */
2298 : 0, /* properties_destroyed */
2299 : 0, /* todo_flags_start */
2300 : TODO_update_ssa, /* todo_flags_finish */
2301 : };
2302 :
2303 : class pass_expand_pow : public gimple_opt_pass
2304 : {
2305 : public:
2306 294196 : pass_expand_pow (gcc::context *ctxt)
2307 588392 : : gimple_opt_pass (pass_data_expand_pow, ctxt)
2308 : {}
2309 :
2310 : /* opt_pass methods: */
2311 1060389 : bool gate (function *) final override
2312 : {
2313 1060389 : return optimize;
2314 : }
2315 :
2316 : unsigned int execute (function *) final override;
2317 :
2318 : }; // class pass_expand_pow
2319 :
2320 : unsigned int
2321 1060384 : pass_expand_pow::execute (function *fun)
2322 : {
2323 1060384 : basic_block bb;
2324 1060384 : bool cfg_changed = false;
2325 :
2326 1060384 : calculate_dominance_info (CDI_DOMINATORS);
2327 :
2328 10703796 : FOR_EACH_BB_FN (bb, fun)
2329 : {
2330 9643412 : gimple_stmt_iterator gsi;
2331 9643412 : bool cleanup_eh = false;
2332 :
2333 97477982 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2334 : {
2335 87834570 : gimple *stmt = gsi_stmt (gsi);
2336 :
2337 : /* Only the last stmt in a bb could throw, no need to call
2338 : gimple_purge_dead_eh_edges if we change something in the middle
2339 : of a basic block. */
2340 87834570 : cleanup_eh = false;
2341 :
2342 87834570 : if (is_gimple_call (stmt)
2343 87834570 : && gimple_call_lhs (stmt))
2344 : {
2345 2035915 : tree arg0, arg1, result;
2346 2035915 : HOST_WIDE_INT n;
2347 2035915 : location_t loc;
2348 :
2349 2035915 : switch (gimple_call_combined_fn (stmt))
2350 : {
2351 601 : CASE_CFN_POW:
2352 601 : arg0 = gimple_call_arg (stmt, 0);
2353 601 : arg1 = gimple_call_arg (stmt, 1);
2354 :
2355 601 : loc = gimple_location (stmt);
2356 601 : result = gimple_expand_builtin_pow (&gsi, loc, arg0, arg1);
2357 :
2358 601 : if (result)
2359 : {
2360 52 : tree lhs = gimple_get_lhs (stmt);
2361 52 : gassign *new_stmt = gimple_build_assign (lhs, result);
2362 52 : gimple_set_location (new_stmt, loc);
2363 52 : unlink_stmt_vdef (stmt);
2364 52 : gsi_replace (&gsi, new_stmt, true);
2365 52 : cleanup_eh = true;
2366 104 : if (gimple_vdef (stmt))
2367 0 : release_ssa_name (gimple_vdef (stmt));
2368 : }
2369 : break;
2370 :
2371 804 : CASE_CFN_POWI:
2372 804 : arg0 = gimple_call_arg (stmt, 0);
2373 804 : arg1 = gimple_call_arg (stmt, 1);
2374 804 : loc = gimple_location (stmt);
2375 :
2376 804 : if (real_minus_onep (arg0))
2377 : {
2378 0 : tree t0, t1, cond, one, minus_one;
2379 0 : gassign *stmt;
2380 :
2381 0 : t0 = TREE_TYPE (arg0);
2382 0 : t1 = TREE_TYPE (arg1);
2383 0 : one = build_real (t0, dconst1);
2384 0 : minus_one = build_real (t0, dconstm1);
2385 :
2386 0 : cond = make_temp_ssa_name (t1, NULL, "powi_cond");
2387 0 : stmt = gimple_build_assign (cond, BIT_AND_EXPR,
2388 : arg1, build_int_cst (t1, 1));
2389 0 : gimple_set_location (stmt, loc);
2390 0 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
2391 :
2392 0 : result = make_temp_ssa_name (t0, NULL, "powi");
2393 0 : stmt = gimple_build_assign (result, COND_EXPR, cond,
2394 : minus_one, one);
2395 0 : gimple_set_location (stmt, loc);
2396 0 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
2397 : }
2398 : else
2399 : {
2400 804 : if (!tree_fits_shwi_p (arg1))
2401 : break;
2402 :
2403 593 : n = tree_to_shwi (arg1);
2404 593 : result = gimple_expand_builtin_powi (&gsi, loc, arg0, n);
2405 : }
2406 :
2407 593 : if (result)
2408 : {
2409 585 : tree lhs = gimple_get_lhs (stmt);
2410 585 : gassign *new_stmt = gimple_build_assign (lhs, result);
2411 585 : gimple_set_location (new_stmt, loc);
2412 585 : unlink_stmt_vdef (stmt);
2413 585 : gsi_replace (&gsi, new_stmt, true);
2414 585 : cleanup_eh = true;
2415 87835155 : if (gimple_vdef (stmt))
2416 0 : release_ssa_name (gimple_vdef (stmt));
2417 : }
2418 : break;
2419 :
2420 211 : default:;
2421 : }
2422 : }
2423 : }
2424 9643412 : if (cleanup_eh)
2425 3 : cfg_changed |= gimple_purge_dead_eh_edges (bb);
2426 : }
2427 :
2428 1060384 : return cfg_changed ? TODO_cleanup_cfg : 0;
2429 : }
2430 :
2431 : } // anon namespace
2432 :
2433 : gimple_opt_pass *
2434 294196 : make_pass_expand_pow (gcc::context *ctxt)
2435 : {
2436 294196 : return new pass_expand_pow (ctxt);
2437 : }
2438 :
2439 : /* Return true if stmt is a type conversion operation that can be stripped
2440 : when used in a widening multiply operation. */
2441 : static bool
2442 506184 : widening_mult_conversion_strippable_p (tree result_type, gimple *stmt)
2443 : {
2444 506184 : enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
2445 :
2446 506184 : if (TREE_CODE (result_type) == INTEGER_TYPE)
2447 : {
2448 506184 : tree op_type;
2449 506184 : tree inner_op_type;
2450 :
2451 506184 : if (!CONVERT_EXPR_CODE_P (rhs_code))
2452 : return false;
2453 :
2454 193607 : op_type = TREE_TYPE (gimple_assign_lhs (stmt));
2455 :
2456 : /* If the type of OP has the same precision as the result, then
2457 : we can strip this conversion. The multiply operation will be
2458 : selected to create the correct extension as a by-product. */
2459 193607 : if (TYPE_PRECISION (result_type) == TYPE_PRECISION (op_type))
2460 : return true;
2461 :
2462 : /* We can also strip a conversion if it preserves the signed-ness of
2463 : the operation and doesn't narrow the range. */
2464 1169 : inner_op_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
2465 :
2466 : /* If the inner-most type is unsigned, then we can strip any
2467 : intermediate widening operation. If it's signed, then the
2468 : intermediate widening operation must also be signed. */
2469 1169 : if ((TYPE_UNSIGNED (inner_op_type)
2470 1164 : || TYPE_UNSIGNED (op_type) == TYPE_UNSIGNED (inner_op_type))
2471 2333 : && TYPE_PRECISION (op_type) > TYPE_PRECISION (inner_op_type))
2472 : return true;
2473 :
2474 1166 : return false;
2475 : }
2476 :
2477 0 : return rhs_code == FIXED_CONVERT_EXPR;
2478 : }
2479 :
2480 : /* Return true if RHS is a suitable operand for a widening multiplication,
2481 : assuming a target type of TYPE.
2482 : There are two cases:
2483 :
2484 : - RHS makes some value at least twice as wide. Store that value
2485 : in *NEW_RHS_OUT if so, and store its type in *TYPE_OUT.
2486 :
2487 : - RHS is an integer constant. Store that value in *NEW_RHS_OUT if so,
2488 : but leave *TYPE_OUT untouched. */
2489 :
2490 : static bool
2491 960080 : is_widening_mult_rhs_p (tree type, tree rhs, tree *type_out,
2492 : tree *new_rhs_out)
2493 : {
2494 960080 : gimple *stmt;
2495 960080 : tree type1, rhs1;
2496 :
2497 960080 : if (TREE_CODE (rhs) == SSA_NAME)
2498 : {
2499 : /* Use tree_non_zero_bits to see if this operand is zero_extended
2500 : for unsigned widening multiplications or non-negative for
2501 : signed widening multiplications. */
2502 796683 : if (TREE_CODE (type) == INTEGER_TYPE
2503 796683 : && (TYPE_PRECISION (type) & 1) == 0
2504 1593366 : && int_mode_for_size (TYPE_PRECISION (type) / 2, 1).exists ())
2505 : {
2506 790693 : unsigned int prec = TYPE_PRECISION (type);
2507 790693 : unsigned int hprec = prec / 2;
2508 790693 : wide_int bits = wide_int::from (tree_nonzero_bits (rhs), prec,
2509 1581386 : TYPE_SIGN (TREE_TYPE (rhs)));
2510 790693 : if (TYPE_UNSIGNED (type)
2511 1349755 : && wi::bit_and (bits, wi::mask (hprec, true, prec)) == 0)
2512 : {
2513 140934 : *type_out = build_nonstandard_integer_type (hprec, true);
2514 : /* X & MODE_MASK can be simplified to (T)X. */
2515 140934 : stmt = SSA_NAME_DEF_STMT (rhs);
2516 281868 : if (is_gimple_assign (stmt)
2517 122753 : && gimple_assign_rhs_code (stmt) == BIT_AND_EXPR
2518 12081 : && TREE_CODE (gimple_assign_rhs2 (stmt)) == INTEGER_CST
2519 164496 : && wide_int::from (wi::to_wide (gimple_assign_rhs2 (stmt)),
2520 11781 : prec, TYPE_SIGN (TREE_TYPE (rhs)))
2521 176277 : == wi::mask (hprec, false, prec))
2522 9864 : *new_rhs_out = gimple_assign_rhs1 (stmt);
2523 : else
2524 : *new_rhs_out = rhs;
2525 140934 : return true;
2526 : }
2527 649759 : else if (!TYPE_UNSIGNED (type)
2528 881390 : && wi::bit_and (bits, wi::mask (hprec - 1, true, prec)) == 0)
2529 : {
2530 25346 : *type_out = build_nonstandard_integer_type (hprec, false);
2531 25346 : *new_rhs_out = rhs;
2532 25346 : return true;
2533 : }
2534 790693 : }
2535 :
2536 630403 : stmt = SSA_NAME_DEF_STMT (rhs);
2537 630403 : if (is_gimple_assign (stmt))
2538 : {
2539 :
2540 506184 : if (widening_mult_conversion_strippable_p (type, stmt))
2541 : {
2542 192441 : rhs1 = gimple_assign_rhs1 (stmt);
2543 :
2544 192441 : if (TREE_CODE (rhs1) == INTEGER_CST)
2545 : {
2546 0 : *new_rhs_out = rhs1;
2547 0 : *type_out = NULL;
2548 0 : return true;
2549 : }
2550 : }
2551 : else
2552 : rhs1 = rhs;
2553 : }
2554 : else
2555 : rhs1 = rhs;
2556 :
2557 630403 : type1 = TREE_TYPE (rhs1);
2558 :
2559 630403 : if (TREE_CODE (type1) != TREE_CODE (type)
2560 630403 : || TYPE_PRECISION (type1) * 2 > TYPE_PRECISION (type))
2561 : return false;
2562 :
2563 62058 : *new_rhs_out = rhs1;
2564 62058 : *type_out = type1;
2565 62058 : return true;
2566 : }
2567 :
2568 163397 : if (TREE_CODE (rhs) == INTEGER_CST)
2569 : {
2570 163397 : *new_rhs_out = rhs;
2571 163397 : *type_out = NULL;
2572 163397 : return true;
2573 : }
2574 :
2575 : return false;
2576 : }
2577 :
2578 : /* Return true if STMT performs a widening multiplication, assuming the
2579 : output type is TYPE. If so, store the unwidened types of the operands
2580 : in *TYPE1_OUT and *TYPE2_OUT respectively. Also fill *RHS1_OUT and
2581 : *RHS2_OUT such that converting those operands to types *TYPE1_OUT
2582 : and *TYPE2_OUT would give the operands of the multiplication. */
2583 :
2584 : static bool
2585 761228 : is_widening_mult_p (gimple *stmt,
2586 : tree *type1_out, tree *rhs1_out,
2587 : tree *type2_out, tree *rhs2_out)
2588 : {
2589 761228 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
2590 :
2591 761228 : if (TREE_CODE (type) == INTEGER_TYPE)
2592 : {
2593 761228 : if (TYPE_OVERFLOW_TRAPS (type))
2594 : return false;
2595 : }
2596 0 : else if (TREE_CODE (type) != FIXED_POINT_TYPE)
2597 : return false;
2598 :
2599 761199 : if (!is_widening_mult_rhs_p (type, gimple_assign_rhs1 (stmt), type1_out,
2600 : rhs1_out))
2601 : return false;
2602 :
2603 198881 : if (!is_widening_mult_rhs_p (type, gimple_assign_rhs2 (stmt), type2_out,
2604 : rhs2_out))
2605 : return false;
2606 :
2607 192854 : if (*type1_out == NULL)
2608 : {
2609 0 : if (*type2_out == NULL || !int_fits_type_p (*rhs1_out, *type2_out))
2610 : return false;
2611 0 : *type1_out = *type2_out;
2612 : }
2613 :
2614 192854 : if (*type2_out == NULL)
2615 : {
2616 163397 : if (!int_fits_type_p (*rhs2_out, *type1_out))
2617 : return false;
2618 158869 : *type2_out = *type1_out;
2619 : }
2620 :
2621 : /* Ensure that the larger of the two operands comes first. */
2622 188326 : if (TYPE_PRECISION (*type1_out) < TYPE_PRECISION (*type2_out))
2623 : {
2624 58 : std::swap (*type1_out, *type2_out);
2625 58 : std::swap (*rhs1_out, *rhs2_out);
2626 : }
2627 :
2628 : return true;
2629 : }
2630 :
2631 : /* Check to see if the CALL statement is an invocation of copysign
2632 : with 1. being the first argument. */
2633 : static bool
2634 197583 : is_copysign_call_with_1 (gimple *call)
2635 : {
2636 197583 : gcall *c = dyn_cast <gcall *> (call);
2637 4996 : if (! c)
2638 : return false;
2639 :
2640 4996 : enum combined_fn code = gimple_call_combined_fn (c);
2641 :
2642 4996 : if (code == CFN_LAST)
2643 : return false;
2644 :
2645 4044 : if (builtin_fn_p (code))
2646 : {
2647 1012 : switch (as_builtin_fn (code))
2648 : {
2649 30 : CASE_FLT_FN (BUILT_IN_COPYSIGN):
2650 30 : CASE_FLT_FN_FLOATN_NX (BUILT_IN_COPYSIGN):
2651 30 : return real_onep (gimple_call_arg (c, 0));
2652 : default:
2653 : return false;
2654 : }
2655 : }
2656 :
2657 3032 : if (internal_fn_p (code))
2658 : {
2659 3032 : switch (as_internal_fn (code))
2660 : {
2661 23 : case IFN_COPYSIGN:
2662 23 : return real_onep (gimple_call_arg (c, 0));
2663 : default:
2664 : return false;
2665 : }
2666 : }
2667 :
2668 : return false;
2669 : }
2670 :
2671 : /* Try to expand the pattern x * copysign (1, y) into xorsign (x, y).
2672 : This only happens when the xorsign optab is defined, if the
2673 : pattern is not a xorsign pattern or if expansion fails FALSE is
2674 : returned, otherwise TRUE is returned. */
2675 : static bool
2676 744834 : convert_expand_mult_copysign (gimple *stmt, gimple_stmt_iterator *gsi)
2677 : {
2678 744834 : tree treeop0, treeop1, lhs, type;
2679 744834 : location_t loc = gimple_location (stmt);
2680 744834 : lhs = gimple_assign_lhs (stmt);
2681 744834 : treeop0 = gimple_assign_rhs1 (stmt);
2682 744834 : treeop1 = gimple_assign_rhs2 (stmt);
2683 744834 : type = TREE_TYPE (lhs);
2684 744834 : machine_mode mode = TYPE_MODE (type);
2685 :
2686 744834 : if (HONOR_SNANS (type))
2687 : return false;
2688 :
2689 744431 : if (TREE_CODE (treeop0) == SSA_NAME && TREE_CODE (treeop1) == SSA_NAME)
2690 : {
2691 238657 : gimple *call0 = SSA_NAME_DEF_STMT (treeop0);
2692 238657 : if (!has_single_use (treeop0) || !is_copysign_call_with_1 (call0))
2693 : {
2694 238631 : call0 = SSA_NAME_DEF_STMT (treeop1);
2695 238631 : if (!has_single_use (treeop1) || !is_copysign_call_with_1 (call0))
2696 : return false;
2697 :
2698 : treeop1 = treeop0;
2699 : }
2700 43 : if (optab_handler (xorsign_optab, mode) == CODE_FOR_nothing)
2701 : return false;
2702 :
2703 43 : gcall *c = as_a<gcall*> (call0);
2704 43 : treeop0 = gimple_call_arg (c, 1);
2705 :
2706 43 : gcall *call_stmt
2707 43 : = gimple_build_call_internal (IFN_XORSIGN, 2, treeop1, treeop0);
2708 43 : gimple_set_lhs (call_stmt, lhs);
2709 43 : gimple_set_location (call_stmt, loc);
2710 43 : gsi_replace (gsi, call_stmt, true);
2711 43 : return true;
2712 : }
2713 :
2714 : return false;
2715 : }
2716 :
2717 : /* Process a single gimple statement STMT, which has a MULT_EXPR as
2718 : its rhs, and try to convert it into a WIDEN_MULT_EXPR. The return
2719 : value is true iff we converted the statement. */
2720 :
2721 : static bool
2722 753715 : convert_mult_to_widen (gimple *stmt, gimple_stmt_iterator *gsi)
2723 : {
2724 753715 : tree lhs, rhs1, rhs2, type, type1, type2;
2725 753715 : enum insn_code handler;
2726 753715 : scalar_int_mode to_mode, from_mode, actual_mode;
2727 753715 : optab op;
2728 753715 : int actual_precision;
2729 753715 : location_t loc = gimple_location (stmt);
2730 753715 : bool from_unsigned1, from_unsigned2;
2731 :
2732 753715 : lhs = gimple_assign_lhs (stmt);
2733 753715 : type = TREE_TYPE (lhs);
2734 753715 : if (TREE_CODE (type) != INTEGER_TYPE)
2735 : return false;
2736 :
2737 616534 : if (!is_widening_mult_p (stmt, &type1, &rhs1, &type2, &rhs2))
2738 : return false;
2739 :
2740 : /* if any one of rhs1 and rhs2 is subject to abnormal coalescing,
2741 : avoid the transform. */
2742 155871 : if ((TREE_CODE (rhs1) == SSA_NAME
2743 155871 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1))
2744 311741 : || (TREE_CODE (rhs2) == SSA_NAME
2745 22496 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs2)))
2746 : return false;
2747 :
2748 155870 : to_mode = SCALAR_INT_TYPE_MODE (type);
2749 155870 : from_mode = SCALAR_INT_TYPE_MODE (type1);
2750 155870 : if (to_mode == from_mode)
2751 : return false;
2752 :
2753 155866 : from_unsigned1 = TYPE_UNSIGNED (type1);
2754 155866 : from_unsigned2 = TYPE_UNSIGNED (type2);
2755 :
2756 155866 : if (from_unsigned1 && from_unsigned2)
2757 : op = umul_widen_optab;
2758 59254 : else if (!from_unsigned1 && !from_unsigned2)
2759 : op = smul_widen_optab;
2760 : else
2761 1862 : op = usmul_widen_optab;
2762 :
2763 155866 : handler = find_widening_optab_handler_and_mode (op, to_mode, from_mode,
2764 : &actual_mode);
2765 :
2766 155866 : if (handler == CODE_FOR_nothing)
2767 : {
2768 146985 : if (op != smul_widen_optab)
2769 : {
2770 : /* We can use a signed multiply with unsigned types as long as
2771 : there is a wider mode to use, or it is the smaller of the two
2772 : types that is unsigned. Note that type1 >= type2, always. */
2773 91103 : if ((TYPE_UNSIGNED (type1)
2774 89433 : && TYPE_PRECISION (type1) == GET_MODE_PRECISION (from_mode))
2775 91103 : || (TYPE_UNSIGNED (type2)
2776 1670 : && TYPE_PRECISION (type2) == GET_MODE_PRECISION (from_mode)))
2777 : {
2778 91103 : if (!GET_MODE_WIDER_MODE (from_mode).exists (&from_mode)
2779 182206 : || GET_MODE_SIZE (to_mode) <= GET_MODE_SIZE (from_mode))
2780 : return false;
2781 : }
2782 :
2783 0 : op = smul_widen_optab;
2784 0 : handler = find_widening_optab_handler_and_mode (op, to_mode,
2785 : from_mode,
2786 : &actual_mode);
2787 :
2788 0 : if (handler == CODE_FOR_nothing)
2789 : return false;
2790 :
2791 : from_unsigned1 = from_unsigned2 = false;
2792 : }
2793 : else
2794 : {
2795 : /* Expand can synthesize smul_widen_optab if the target
2796 : supports umul_widen_optab. */
2797 55882 : op = umul_widen_optab;
2798 55882 : handler = find_widening_optab_handler_and_mode (op, to_mode,
2799 : from_mode,
2800 : &actual_mode);
2801 55882 : if (handler == CODE_FOR_nothing)
2802 : return false;
2803 : }
2804 : }
2805 :
2806 : /* Ensure that the inputs to the handler are in the correct precision
2807 : for the opcode. This will be the full mode size. */
2808 8881 : actual_precision = GET_MODE_PRECISION (actual_mode);
2809 8881 : if (2 * actual_precision > TYPE_PRECISION (type))
2810 : return false;
2811 8881 : if (actual_precision != TYPE_PRECISION (type1)
2812 8881 : || from_unsigned1 != TYPE_UNSIGNED (type1))
2813 : {
2814 9 : if (!useless_type_conversion_p (type1, TREE_TYPE (rhs1)))
2815 : {
2816 0 : if (TREE_CODE (rhs1) == INTEGER_CST)
2817 0 : rhs1 = fold_convert (type1, rhs1);
2818 : else
2819 0 : rhs1 = build_and_insert_cast (gsi, loc, type1, rhs1);
2820 : }
2821 9 : type1 = build_nonstandard_integer_type (actual_precision,
2822 : from_unsigned1);
2823 : }
2824 8881 : if (!useless_type_conversion_p (type1, TREE_TYPE (rhs1)))
2825 : {
2826 8143 : if (TREE_CODE (rhs1) == INTEGER_CST)
2827 0 : rhs1 = fold_convert (type1, rhs1);
2828 : else
2829 8143 : rhs1 = build_and_insert_cast (gsi, loc, type1, rhs1);
2830 : }
2831 8881 : if (actual_precision != TYPE_PRECISION (type2)
2832 8881 : || from_unsigned2 != TYPE_UNSIGNED (type2))
2833 : {
2834 9 : if (!useless_type_conversion_p (type2, TREE_TYPE (rhs2)))
2835 : {
2836 9 : if (TREE_CODE (rhs2) == INTEGER_CST)
2837 9 : rhs2 = fold_convert (type2, rhs2);
2838 : else
2839 0 : rhs2 = build_and_insert_cast (gsi, loc, type2, rhs2);
2840 : }
2841 9 : type2 = build_nonstandard_integer_type (actual_precision,
2842 : from_unsigned2);
2843 : }
2844 8881 : if (!useless_type_conversion_p (type2, TREE_TYPE (rhs2)))
2845 : {
2846 8339 : if (TREE_CODE (rhs2) == INTEGER_CST)
2847 1912 : rhs2 = fold_convert (type2, rhs2);
2848 : else
2849 6427 : rhs2 = build_and_insert_cast (gsi, loc, type2, rhs2);
2850 : }
2851 :
2852 8881 : gimple_assign_set_rhs1 (stmt, rhs1);
2853 8881 : gimple_assign_set_rhs2 (stmt, rhs2);
2854 8881 : gimple_assign_set_rhs_code (stmt, WIDEN_MULT_EXPR);
2855 8881 : update_stmt (stmt);
2856 8881 : widen_mul_stats.widen_mults_inserted++;
2857 8881 : return true;
2858 : }
2859 :
2860 : /* Process a single gimple statement STMT, which is found at the
2861 : iterator GSI and has a either a PLUS_EXPR or a MINUS_EXPR as its
2862 : rhs (given by CODE), and try to convert it into a
2863 : WIDEN_MULT_PLUS_EXPR or a WIDEN_MULT_MINUS_EXPR. The return value
2864 : is true iff we converted the statement. */
2865 :
2866 : static bool
2867 2622308 : convert_plusminus_to_widen (gimple_stmt_iterator *gsi, gimple *stmt,
2868 : enum tree_code code)
2869 : {
2870 2622308 : gimple *rhs1_stmt = NULL, *rhs2_stmt = NULL;
2871 2622308 : gimple *conv1_stmt = NULL, *conv2_stmt = NULL, *conv_stmt;
2872 2622308 : tree type, type1, type2, optype;
2873 2622308 : tree lhs, rhs1, rhs2, mult_rhs1, mult_rhs2, add_rhs;
2874 2622308 : enum tree_code rhs1_code = ERROR_MARK, rhs2_code = ERROR_MARK;
2875 2622308 : optab this_optab;
2876 2622308 : enum tree_code wmult_code;
2877 2622308 : enum insn_code handler;
2878 2622308 : scalar_mode to_mode, from_mode, actual_mode;
2879 2622308 : location_t loc = gimple_location (stmt);
2880 2622308 : int actual_precision;
2881 2622308 : bool from_unsigned1, from_unsigned2;
2882 :
2883 2622308 : lhs = gimple_assign_lhs (stmt);
2884 2622308 : type = TREE_TYPE (lhs);
2885 2622308 : if ((TREE_CODE (type) != INTEGER_TYPE
2886 400880 : && TREE_CODE (type) != FIXED_POINT_TYPE)
2887 2622308 : || !type_has_mode_precision_p (type))
2888 : return false;
2889 :
2890 2218759 : if (code == MINUS_EXPR)
2891 : wmult_code = WIDEN_MULT_MINUS_EXPR;
2892 : else
2893 1963682 : wmult_code = WIDEN_MULT_PLUS_EXPR;
2894 :
2895 2218759 : rhs1 = gimple_assign_rhs1 (stmt);
2896 2218759 : rhs2 = gimple_assign_rhs2 (stmt);
2897 :
2898 2218759 : if (TREE_CODE (rhs1) == SSA_NAME)
2899 : {
2900 2181898 : rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
2901 2181898 : if (is_gimple_assign (rhs1_stmt))
2902 1304240 : rhs1_code = gimple_assign_rhs_code (rhs1_stmt);
2903 : }
2904 :
2905 2218759 : if (TREE_CODE (rhs2) == SSA_NAME)
2906 : {
2907 815596 : rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
2908 815596 : if (is_gimple_assign (rhs2_stmt))
2909 633771 : rhs2_code = gimple_assign_rhs_code (rhs2_stmt);
2910 : }
2911 :
2912 : /* Allow for one conversion statement between the multiply
2913 : and addition/subtraction statement. If there are more than
2914 : one conversions then we assume they would invalidate this
2915 : transformation. If that's not the case then they should have
2916 : been folded before now. */
2917 2218759 : if (CONVERT_EXPR_CODE_P (rhs1_code))
2918 : {
2919 427757 : conv1_stmt = rhs1_stmt;
2920 427757 : rhs1 = gimple_assign_rhs1 (rhs1_stmt);
2921 427757 : if (TREE_CODE (rhs1) == SSA_NAME)
2922 : {
2923 359213 : rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
2924 359213 : if (is_gimple_assign (rhs1_stmt))
2925 211213 : rhs1_code = gimple_assign_rhs_code (rhs1_stmt);
2926 : }
2927 : else
2928 : return false;
2929 : }
2930 2150215 : if (CONVERT_EXPR_CODE_P (rhs2_code))
2931 : {
2932 201914 : conv2_stmt = rhs2_stmt;
2933 201914 : rhs2 = gimple_assign_rhs1 (rhs2_stmt);
2934 201914 : if (TREE_CODE (rhs2) == SSA_NAME)
2935 : {
2936 191801 : rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
2937 191801 : if (is_gimple_assign (rhs2_stmt))
2938 128142 : rhs2_code = gimple_assign_rhs_code (rhs2_stmt);
2939 : }
2940 : else
2941 : return false;
2942 : }
2943 :
2944 : /* If code is WIDEN_MULT_EXPR then it would seem unnecessary to call
2945 : is_widening_mult_p, but we still need the rhs returns.
2946 :
2947 : It might also appear that it would be sufficient to use the existing
2948 : operands of the widening multiply, but that would limit the choice of
2949 : multiply-and-accumulate instructions.
2950 :
2951 : If the widened-multiplication result has more than one uses, it is
2952 : probably wiser not to do the conversion. Also restrict this operation
2953 : to single basic block to avoid moving the multiply to a different block
2954 : with a higher execution frequency. */
2955 2140102 : if (code == PLUS_EXPR
2956 1890226 : && (rhs1_code == MULT_EXPR || rhs1_code == WIDEN_MULT_EXPR))
2957 : {
2958 144309 : if (!has_single_use (rhs1)
2959 79579 : || gimple_bb (rhs1_stmt) != gimple_bb (stmt)
2960 214129 : || !is_widening_mult_p (rhs1_stmt, &type1, &mult_rhs1,
2961 : &type2, &mult_rhs2))
2962 : return false;
2963 : add_rhs = rhs2;
2964 : conv_stmt = conv1_stmt;
2965 : }
2966 1995793 : else if (rhs2_code == MULT_EXPR || rhs2_code == WIDEN_MULT_EXPR)
2967 : {
2968 129919 : if (!has_single_use (rhs2)
2969 82234 : || gimple_bb (rhs2_stmt) != gimple_bb (stmt)
2970 204793 : || !is_widening_mult_p (rhs2_stmt, &type1, &mult_rhs1,
2971 : &type2, &mult_rhs2))
2972 : return false;
2973 : add_rhs = rhs1;
2974 : conv_stmt = conv2_stmt;
2975 : }
2976 : else
2977 : return false;
2978 :
2979 32455 : to_mode = SCALAR_TYPE_MODE (type);
2980 32455 : from_mode = SCALAR_TYPE_MODE (type1);
2981 32455 : if (to_mode == from_mode)
2982 : return false;
2983 :
2984 : /* For fixed point types, the mode classes could be different
2985 : so reject that case. */
2986 32454 : if (GET_MODE_CLASS (from_mode) != GET_MODE_CLASS (to_mode))
2987 : return false;
2988 :
2989 32454 : from_unsigned1 = TYPE_UNSIGNED (type1);
2990 32454 : from_unsigned2 = TYPE_UNSIGNED (type2);
2991 32454 : optype = type1;
2992 :
2993 : /* There's no such thing as a mixed sign madd yet, so use a wider mode. */
2994 32454 : if (from_unsigned1 != from_unsigned2)
2995 : {
2996 893 : if (!INTEGRAL_TYPE_P (type))
2997 : return false;
2998 : /* We can use a signed multiply with unsigned types as long as
2999 : there is a wider mode to use, or it is the smaller of the two
3000 : types that is unsigned. Note that type1 >= type2, always. */
3001 893 : if ((from_unsigned1
3002 54 : && TYPE_PRECISION (type1) == GET_MODE_PRECISION (from_mode))
3003 893 : || (from_unsigned2
3004 839 : && TYPE_PRECISION (type2) == GET_MODE_PRECISION (from_mode)))
3005 : {
3006 893 : if (!GET_MODE_WIDER_MODE (from_mode).exists (&from_mode)
3007 1786 : || GET_MODE_SIZE (from_mode) >= GET_MODE_SIZE (to_mode))
3008 : return false;
3009 : }
3010 :
3011 18 : from_unsigned1 = from_unsigned2 = false;
3012 18 : optype = build_nonstandard_integer_type (GET_MODE_PRECISION (from_mode),
3013 : false);
3014 : }
3015 :
3016 : /* If there was a conversion between the multiply and addition
3017 : then we need to make sure it fits a multiply-and-accumulate.
3018 : The should be a single mode change which does not change the
3019 : value. */
3020 31579 : if (conv_stmt)
3021 : {
3022 : /* We use the original, unmodified data types for this. */
3023 710 : tree from_type = TREE_TYPE (gimple_assign_rhs1 (conv_stmt));
3024 710 : tree to_type = TREE_TYPE (gimple_assign_lhs (conv_stmt));
3025 710 : int data_size = TYPE_PRECISION (type1) + TYPE_PRECISION (type2);
3026 710 : bool is_unsigned = TYPE_UNSIGNED (type1) && TYPE_UNSIGNED (type2);
3027 :
3028 710 : if (TYPE_PRECISION (from_type) > TYPE_PRECISION (to_type))
3029 : {
3030 : /* Conversion is a truncate. */
3031 0 : if (TYPE_PRECISION (to_type) < data_size)
3032 : return false;
3033 : }
3034 710 : else if (TYPE_PRECISION (from_type) < TYPE_PRECISION (to_type))
3035 : {
3036 : /* Conversion is an extend. Check it's the right sort. */
3037 313 : if (TYPE_UNSIGNED (from_type) != is_unsigned
3038 313 : && !(is_unsigned && TYPE_PRECISION (from_type) > data_size))
3039 : return false;
3040 : }
3041 : /* else convert is a no-op for our purposes. */
3042 : }
3043 :
3044 : /* Verify that the machine can perform a widening multiply
3045 : accumulate in this mode/signedness combination, otherwise
3046 : this transformation is likely to pessimize code. */
3047 31351 : this_optab = optab_for_tree_code (wmult_code, optype, optab_default);
3048 31351 : handler = find_widening_optab_handler_and_mode (this_optab, to_mode,
3049 : from_mode, &actual_mode);
3050 :
3051 31351 : if (handler == CODE_FOR_nothing)
3052 : return false;
3053 :
3054 : /* Ensure that the inputs to the handler are in the correct precision
3055 : for the opcode. This will be the full mode size. */
3056 0 : actual_precision = GET_MODE_PRECISION (actual_mode);
3057 0 : if (actual_precision != TYPE_PRECISION (type1)
3058 0 : || from_unsigned1 != TYPE_UNSIGNED (type1))
3059 : {
3060 0 : if (!useless_type_conversion_p (type1, TREE_TYPE (mult_rhs1)))
3061 : {
3062 0 : if (TREE_CODE (mult_rhs1) == INTEGER_CST)
3063 0 : mult_rhs1 = fold_convert (type1, mult_rhs1);
3064 : else
3065 0 : mult_rhs1 = build_and_insert_cast (gsi, loc, type1, mult_rhs1);
3066 : }
3067 0 : type1 = build_nonstandard_integer_type (actual_precision,
3068 : from_unsigned1);
3069 : }
3070 0 : if (!useless_type_conversion_p (type1, TREE_TYPE (mult_rhs1)))
3071 : {
3072 0 : if (TREE_CODE (mult_rhs1) == INTEGER_CST)
3073 0 : mult_rhs1 = fold_convert (type1, mult_rhs1);
3074 : else
3075 0 : mult_rhs1 = build_and_insert_cast (gsi, loc, type1, mult_rhs1);
3076 : }
3077 0 : if (actual_precision != TYPE_PRECISION (type2)
3078 0 : || from_unsigned2 != TYPE_UNSIGNED (type2))
3079 : {
3080 0 : if (!useless_type_conversion_p (type2, TREE_TYPE (mult_rhs2)))
3081 : {
3082 0 : if (TREE_CODE (mult_rhs2) == INTEGER_CST)
3083 0 : mult_rhs2 = fold_convert (type2, mult_rhs2);
3084 : else
3085 0 : mult_rhs2 = build_and_insert_cast (gsi, loc, type2, mult_rhs2);
3086 : }
3087 0 : type2 = build_nonstandard_integer_type (actual_precision,
3088 : from_unsigned2);
3089 : }
3090 0 : if (!useless_type_conversion_p (type2, TREE_TYPE (mult_rhs2)))
3091 : {
3092 0 : if (TREE_CODE (mult_rhs2) == INTEGER_CST)
3093 0 : mult_rhs2 = fold_convert (type2, mult_rhs2);
3094 : else
3095 0 : mult_rhs2 = build_and_insert_cast (gsi, loc, type2, mult_rhs2);
3096 : }
3097 :
3098 0 : if (!useless_type_conversion_p (type, TREE_TYPE (add_rhs)))
3099 0 : add_rhs = build_and_insert_cast (gsi, loc, type, add_rhs);
3100 :
3101 0 : gimple_assign_set_rhs_with_ops (gsi, wmult_code, mult_rhs1, mult_rhs2,
3102 : add_rhs);
3103 0 : update_stmt (gsi_stmt (*gsi));
3104 0 : widen_mul_stats.maccs_inserted++;
3105 0 : return true;
3106 : }
3107 :
3108 : /* Given a result MUL_RESULT which is a result of a multiplication of OP1 and
3109 : OP2 and which we know is used in statements that can be, together with the
3110 : multiplication, converted to FMAs, perform the transformation. */
3111 :
3112 : static void
3113 17529 : convert_mult_to_fma_1 (tree mul_result, tree op1, tree op2)
3114 : {
3115 17529 : gimple *use_stmt;
3116 17529 : imm_use_iterator imm_iter;
3117 17529 : gcall *fma_stmt;
3118 :
3119 35108 : FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, mul_result)
3120 : {
3121 17579 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
3122 17579 : tree addop, mulop1 = op1, result = mul_result;
3123 17579 : bool negate_p = false;
3124 17579 : gimple_seq seq = NULL;
3125 :
3126 17579 : if (is_gimple_debug (use_stmt))
3127 0 : continue;
3128 :
3129 : /* If the use is a type convert, look further into it if the operations
3130 : are the same under two's complement. */
3131 17579 : tree lhs_type;
3132 17579 : if (gimple_assign_cast_p (use_stmt)
3133 0 : && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt)))
3134 17579 : && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1)))
3135 : {
3136 0 : tree cast_lhs = gimple_get_lhs (use_stmt);
3137 0 : gimple *tmp_use;
3138 0 : use_operand_p tmp_use_p;
3139 0 : if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use))
3140 : {
3141 0 : release_defs (use_stmt);
3142 0 : use_stmt = tmp_use;
3143 0 : result = cast_lhs;
3144 0 : gsi_remove (&gsi, true);
3145 0 : gsi = gsi_for_stmt (use_stmt);
3146 : }
3147 : }
3148 :
3149 17579 : if (is_gimple_assign (use_stmt)
3150 17579 : && gimple_assign_rhs_code (use_stmt) == NEGATE_EXPR)
3151 : {
3152 700 : result = gimple_assign_lhs (use_stmt);
3153 700 : use_operand_p use_p;
3154 700 : gimple *neguse_stmt;
3155 700 : single_imm_use (gimple_assign_lhs (use_stmt), &use_p, &neguse_stmt);
3156 700 : gsi_remove (&gsi, true);
3157 700 : release_defs (use_stmt);
3158 :
3159 700 : use_stmt = neguse_stmt;
3160 700 : gsi = gsi_for_stmt (use_stmt);
3161 700 : negate_p = true;
3162 : }
3163 :
3164 17579 : tree cond, else_value, ops[3], len, bias;
3165 17579 : tree_code code;
3166 17579 : if (!can_interpret_as_conditional_op_p (use_stmt, &cond, &code,
3167 : ops, &else_value,
3168 : &len, &bias))
3169 0 : gcc_unreachable ();
3170 17579 : addop = ops[0] == result ? ops[1] : ops[0];
3171 :
3172 17579 : if (code == MINUS_EXPR)
3173 : {
3174 5828 : if (ops[0] == result)
3175 : /* a * b - c -> a * b + (-c) */
3176 2916 : addop = gimple_build (&seq, NEGATE_EXPR, TREE_TYPE (addop), addop);
3177 : else
3178 : /* a - b * c -> (-b) * c + a */
3179 2912 : negate_p = !negate_p;
3180 : }
3181 :
3182 17579 : if (negate_p)
3183 3612 : mulop1 = gimple_build (&seq, NEGATE_EXPR, TREE_TYPE (mulop1), mulop1);
3184 :
3185 17579 : if (seq)
3186 5823 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
3187 :
3188 : /* Ensure all the operands are of the same type. Use the type of the
3189 : addend as that's the statement being replaced. */
3190 17579 : op2 = gimple_convert (&gsi, true, GSI_SAME_STMT,
3191 17579 : UNKNOWN_LOCATION, TREE_TYPE (addop), op2);
3192 17579 : mulop1 = gimple_convert (&gsi, true, GSI_SAME_STMT,
3193 17579 : UNKNOWN_LOCATION, TREE_TYPE (addop), mulop1);
3194 :
3195 17579 : if (len)
3196 0 : fma_stmt
3197 0 : = gimple_build_call_internal (IFN_COND_LEN_FMA, 7, cond, mulop1, op2,
3198 : addop, else_value, len, bias);
3199 17579 : else if (cond)
3200 94 : fma_stmt = gimple_build_call_internal (IFN_COND_FMA, 5, cond, mulop1,
3201 : op2, addop, else_value);
3202 : else
3203 17485 : fma_stmt = gimple_build_call_internal (IFN_FMA, 3, mulop1, op2, addop);
3204 17579 : gimple_set_lhs (fma_stmt, gimple_get_lhs (use_stmt));
3205 17579 : gimple_call_set_nothrow (fma_stmt, !stmt_can_throw_internal (cfun,
3206 : use_stmt));
3207 17579 : gsi_replace (&gsi, fma_stmt, true);
3208 : /* Follow all SSA edges so that we generate FMS, FNMA and FNMS
3209 : regardless of where the negation occurs. */
3210 17579 : gimple *orig_stmt = gsi_stmt (gsi);
3211 17579 : if (fold_stmt (&gsi, follow_all_ssa_edges))
3212 : {
3213 5872 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, gsi_stmt (gsi)))
3214 0 : gcc_unreachable ();
3215 5872 : update_stmt (gsi_stmt (gsi));
3216 : }
3217 :
3218 17579 : if (dump_file && (dump_flags & TDF_DETAILS))
3219 : {
3220 3 : fprintf (dump_file, "Generated FMA ");
3221 3 : print_gimple_stmt (dump_file, gsi_stmt (gsi), 0, TDF_NONE);
3222 3 : fprintf (dump_file, "\n");
3223 : }
3224 :
3225 : /* If the FMA result is negated in a single use, fold the negation
3226 : too. */
3227 17579 : orig_stmt = gsi_stmt (gsi);
3228 17579 : use_operand_p use_p;
3229 17579 : gimple *neg_stmt;
3230 17579 : if (is_gimple_call (orig_stmt)
3231 17579 : && gimple_call_internal_p (orig_stmt)
3232 17579 : && gimple_call_lhs (orig_stmt)
3233 17579 : && TREE_CODE (gimple_call_lhs (orig_stmt)) == SSA_NAME
3234 17579 : && single_imm_use (gimple_call_lhs (orig_stmt), &use_p, &neg_stmt)
3235 12539 : && is_gimple_assign (neg_stmt)
3236 9923 : && gimple_assign_rhs_code (neg_stmt) == NEGATE_EXPR
3237 18932 : && !stmt_could_throw_p (cfun, neg_stmt))
3238 : {
3239 1353 : gsi = gsi_for_stmt (neg_stmt);
3240 1353 : if (fold_stmt (&gsi, follow_all_ssa_edges))
3241 : {
3242 1353 : if (maybe_clean_or_replace_eh_stmt (neg_stmt, gsi_stmt (gsi)))
3243 0 : gcc_unreachable ();
3244 1353 : update_stmt (gsi_stmt (gsi));
3245 1353 : if (dump_file && (dump_flags & TDF_DETAILS))
3246 : {
3247 0 : fprintf (dump_file, "Folded FMA negation ");
3248 0 : print_gimple_stmt (dump_file, gsi_stmt (gsi), 0, TDF_NONE);
3249 0 : fprintf (dump_file, "\n");
3250 : }
3251 : }
3252 : }
3253 :
3254 17579 : widen_mul_stats.fmas_inserted++;
3255 17529 : }
3256 17529 : }
3257 :
3258 : /* Data necessary to perform the actual transformation from a multiplication
3259 : and an addition to an FMA after decision is taken it should be done and to
3260 : then delete the multiplication statement from the function IL. */
3261 :
3262 : struct fma_transformation_info
3263 : {
3264 : gimple *mul_stmt;
3265 : tree mul_result;
3266 : tree op1;
3267 : tree op2;
3268 : };
3269 :
3270 : /* Structure containing the current state of FMA deferring, i.e. whether we are
3271 : deferring, whether to continue deferring, and all data necessary to come
3272 : back and perform all deferred transformations. */
3273 :
3274 10464990 : class fma_deferring_state
3275 : {
3276 : public:
3277 : /* Class constructor. Pass true as PERFORM_DEFERRING in order to actually
3278 : do any deferring. */
3279 :
3280 10464990 : fma_deferring_state (bool perform_deferring)
3281 10464990 : : m_candidates (), m_mul_result_set (), m_initial_phi (NULL),
3282 10464990 : m_last_result (NULL_TREE), m_deferring_p (perform_deferring) {}
3283 :
3284 : /* List of FMA candidates for which we the transformation has been determined
3285 : possible but we at this point in BB analysis we do not consider them
3286 : beneficial. */
3287 : auto_vec<fma_transformation_info, 8> m_candidates;
3288 :
3289 : /* Set of results of multiplication that are part of an already deferred FMA
3290 : candidates. */
3291 : hash_set<tree> m_mul_result_set;
3292 :
3293 : /* The PHI that supposedly feeds back result of a FMA to another over loop
3294 : boundary. */
3295 : gphi *m_initial_phi;
3296 :
3297 : /* Result of the last produced FMA candidate or NULL if there has not been
3298 : one. */
3299 : tree m_last_result;
3300 :
3301 : /* If true, deferring might still be profitable. If false, transform all
3302 : candidates and no longer defer. */
3303 : bool m_deferring_p;
3304 : };
3305 :
3306 : /* Transform all deferred FMA candidates and mark STATE as no longer
3307 : deferring. */
3308 :
3309 : static void
3310 3737868 : cancel_fma_deferring (fma_deferring_state *state)
3311 : {
3312 3737868 : if (!state->m_deferring_p)
3313 : return;
3314 :
3315 2701518 : for (unsigned i = 0; i < state->m_candidates.length (); i++)
3316 : {
3317 940 : if (dump_file && (dump_flags & TDF_DETAILS))
3318 0 : fprintf (dump_file, "Generating deferred FMA\n");
3319 :
3320 940 : const fma_transformation_info &fti = state->m_candidates[i];
3321 940 : convert_mult_to_fma_1 (fti.mul_result, fti.op1, fti.op2);
3322 :
3323 940 : gimple_stmt_iterator gsi = gsi_for_stmt (fti.mul_stmt);
3324 940 : gsi_remove (&gsi, true);
3325 940 : release_defs (fti.mul_stmt);
3326 : }
3327 2700578 : state->m_deferring_p = false;
3328 : }
3329 :
3330 : /* If OP is an SSA name defined by a PHI node, return the PHI statement.
3331 : Otherwise return NULL. */
3332 :
3333 : static gphi *
3334 5262 : result_of_phi (tree op)
3335 : {
3336 0 : if (TREE_CODE (op) != SSA_NAME)
3337 : return NULL;
3338 :
3339 5137 : return dyn_cast <gphi *> (SSA_NAME_DEF_STMT (op));
3340 : }
3341 :
3342 : /* After processing statements of a BB and recording STATE, return true if the
3343 : initial phi is fed by the last FMA candidate result ore one such result from
3344 : previously processed BBs marked in LAST_RESULT_SET. */
3345 :
3346 : static bool
3347 361 : last_fma_candidate_feeds_initial_phi (fma_deferring_state *state,
3348 : hash_set<tree> *last_result_set)
3349 : {
3350 361 : ssa_op_iter iter;
3351 361 : use_operand_p use;
3352 889 : FOR_EACH_PHI_ARG (use, state->m_initial_phi, iter, SSA_OP_USE)
3353 : {
3354 625 : tree t = USE_FROM_PTR (use);
3355 625 : if (t == state->m_last_result
3356 625 : || last_result_set->contains (t))
3357 97 : return true;
3358 : }
3359 :
3360 : return false;
3361 : }
3362 :
3363 : /* Combine the multiplication at MUL_STMT with operands MULOP1 and MULOP2
3364 : with uses in additions and subtractions to form fused multiply-add
3365 : operations. Returns true if successful and MUL_STMT should be removed.
3366 : If MUL_COND is nonnull, the multiplication in MUL_STMT is conditional
3367 : on MUL_COND, otherwise it is unconditional.
3368 :
3369 : If STATE indicates that we are deferring FMA transformation, that means
3370 : that we do not produce FMAs for basic blocks which look like:
3371 :
3372 : <bb 6>
3373 : # accumulator_111 = PHI <0.0(5), accumulator_66(6)>
3374 : _65 = _14 * _16;
3375 : accumulator_66 = _65 + accumulator_111;
3376 :
3377 : or its unrolled version, i.e. with several FMA candidates that feed result
3378 : of one into the addend of another. Instead, we add them to a list in STATE
3379 : and if we later discover an FMA candidate that is not part of such a chain,
3380 : we go back and perform all deferred past candidates. */
3381 :
3382 : static bool
3383 744920 : convert_mult_to_fma (gimple *mul_stmt, tree op1, tree op2,
3384 : fma_deferring_state *state, tree mul_cond = NULL_TREE,
3385 : tree mul_len = NULL_TREE, tree mul_bias = NULL_TREE)
3386 : {
3387 744920 : tree mul_result = gimple_get_lhs (mul_stmt);
3388 : /* If there isn't a LHS then this can't be an FMA. There can be no LHS
3389 : if the statement was left just for the side-effects. */
3390 744920 : if (!mul_result)
3391 : return false;
3392 744920 : tree type = TREE_TYPE (mul_result);
3393 744920 : gimple *use_stmt, *neguse_stmt;
3394 744920 : use_operand_p use_p;
3395 744920 : imm_use_iterator imm_iter;
3396 :
3397 647330 : if (FLOAT_TYPE_P (type)
3398 770608 : && flag_fp_contract_mode != FP_CONTRACT_FAST)
3399 : return false;
3400 :
3401 : /* We don't want to do bitfield reduction ops. */
3402 739827 : if (INTEGRAL_TYPE_P (type)
3403 739827 : && (!type_has_mode_precision_p (type) || TYPE_OVERFLOW_TRAPS (type)))
3404 : return false;
3405 :
3406 : /* If the target doesn't support it, don't generate it. We assume that
3407 : if fma isn't available then fms, fnma or fnms are not either. */
3408 739637 : optimization_type opt_type = bb_optimization_type (gimple_bb (mul_stmt));
3409 739637 : if (!direct_internal_fn_supported_p (IFN_FMA, type, opt_type))
3410 : return false;
3411 :
3412 : /* If the multiplication has zero uses, it is kept around probably because
3413 : of -fnon-call-exceptions. Don't optimize it away in that case,
3414 : it is DCE job. */
3415 23056 : if (has_zero_uses (mul_result))
3416 : return false;
3417 :
3418 23056 : bool check_defer
3419 23056 : = (state->m_deferring_p
3420 23056 : && maybe_le (tree_to_poly_int64 (TYPE_SIZE (type)),
3421 23056 : param_avoid_fma_max_bits));
3422 23056 : bool defer = check_defer;
3423 23056 : bool seen_negate_p = false;
3424 :
3425 : /* There is no numerical difference between fused and unfused integer FMAs,
3426 : and the assumption below that FMA is as cheap as addition is unlikely
3427 : to be true, especially if the multiplication occurs multiple times on
3428 : the same chain. E.g., for something like:
3429 :
3430 : (((a * b) + c) >> 1) + (a * b)
3431 :
3432 : we do not want to duplicate the a * b into two additions, not least
3433 : because the result is not a natural FMA chain. */
3434 23056 : if (ANY_INTEGRAL_TYPE_P (type)
3435 23056 : && !has_single_use (mul_result))
3436 : return false;
3437 :
3438 23056 : if (!dbg_cnt (form_fma))
3439 : return false;
3440 :
3441 : /* Make sure that the multiplication statement becomes dead after
3442 : the transformation, thus that all uses are transformed to FMAs.
3443 : This means we assume that an FMA operation has the same cost
3444 : as an addition. */
3445 41420 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, mul_result)
3446 : {
3447 23794 : tree result = mul_result;
3448 23794 : bool negate_p = false;
3449 :
3450 23794 : use_stmt = USE_STMT (use_p);
3451 :
3452 23794 : if (is_gimple_debug (use_stmt))
3453 214 : continue;
3454 :
3455 : /* If the use is a type convert, look further into it if the operations
3456 : are the same under two's complement. */
3457 23580 : tree lhs_type;
3458 23580 : if (gimple_assign_cast_p (use_stmt)
3459 229 : && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt)))
3460 23809 : && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1)))
3461 : {
3462 0 : tree cast_lhs = gimple_get_lhs (use_stmt);
3463 0 : gimple *tmp_use;
3464 0 : use_operand_p tmp_use_p;
3465 0 : if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use))
3466 0 : use_stmt = tmp_use;
3467 0 : result = cast_lhs;
3468 : }
3469 :
3470 : /* For now restrict this operations to single basic blocks. In theory
3471 : we would want to support sinking the multiplication in
3472 : m = a*b;
3473 : if ()
3474 : ma = m + c;
3475 : else
3476 : d = m;
3477 : to form a fma in the then block and sink the multiplication to the
3478 : else block. */
3479 23580 : if (gimple_bb (use_stmt) != gimple_bb (mul_stmt))
3480 5430 : return false;
3481 :
3482 : /* A negate on the multiplication leads to FNMA. */
3483 22670 : if (is_gimple_assign (use_stmt)
3484 22670 : && gimple_assign_rhs_code (use_stmt) == NEGATE_EXPR)
3485 : {
3486 706 : ssa_op_iter iter;
3487 706 : use_operand_p usep;
3488 :
3489 : /* If (due to earlier missed optimizations) we have two
3490 : negates of the same value, treat them as equivalent
3491 : to a single negate with multiple uses. */
3492 706 : if (seen_negate_p)
3493 0 : return false;
3494 :
3495 706 : result = gimple_assign_lhs (use_stmt);
3496 :
3497 : /* Make sure the negate statement becomes dead with this
3498 : single transformation. */
3499 706 : if (!single_imm_use (gimple_assign_lhs (use_stmt),
3500 : &use_p, &neguse_stmt))
3501 : return false;
3502 :
3503 : /* Make sure the multiplication isn't also used on that stmt. */
3504 2836 : FOR_EACH_PHI_OR_STMT_USE (usep, neguse_stmt, iter, SSA_OP_USE)
3505 1424 : if (USE_FROM_PTR (usep) == mul_result)
3506 : return false;
3507 :
3508 : /* Re-validate. */
3509 706 : use_stmt = neguse_stmt;
3510 706 : if (gimple_bb (use_stmt) != gimple_bb (mul_stmt))
3511 : return false;
3512 :
3513 706 : negate_p = seen_negate_p = true;
3514 : }
3515 :
3516 22670 : tree cond, else_value, ops[3], len, bias;
3517 22670 : tree_code code;
3518 22670 : if (!can_interpret_as_conditional_op_p (use_stmt, &cond, &code, ops,
3519 : &else_value, &len, &bias))
3520 : return false;
3521 :
3522 : /* The multiplication result must be one of the addition operands. */
3523 20310 : if (ops[0] != result && ops[1] != result)
3524 : return false;
3525 :
3526 19698 : switch (code)
3527 : {
3528 5834 : case MINUS_EXPR:
3529 5834 : if (ops[1] == result)
3530 2912 : negate_p = !negate_p;
3531 : break;
3532 : case PLUS_EXPR:
3533 : break;
3534 : default:
3535 : /* FMA can only be formed from PLUS and MINUS. */
3536 : return false;
3537 : }
3538 :
3539 18172 : if (len)
3540 : {
3541 : /* For COND_LEN_* operations, we may have dummpy mask which is
3542 : the all true mask. Such TREE type may be mul_cond != cond
3543 : but we still consider they are equal. */
3544 0 : if (mul_cond && cond != mul_cond
3545 0 : && !(integer_truep (mul_cond) && integer_truep (cond)))
3546 : return false;
3547 :
3548 0 : if (else_value == result)
3549 : return false;
3550 :
3551 0 : if (!direct_internal_fn_supported_p (IFN_COND_LEN_FMA, type,
3552 : opt_type))
3553 : return false;
3554 :
3555 0 : if (mul_len)
3556 : {
3557 0 : poly_int64 mul_value, value;
3558 0 : if (poly_int_tree_p (mul_len, &mul_value)
3559 0 : && poly_int_tree_p (len, &value)
3560 0 : && maybe_ne (mul_value, value))
3561 0 : return false;
3562 0 : else if (mul_len != len)
3563 : return false;
3564 :
3565 0 : if (wi::to_widest (mul_bias) != wi::to_widest (bias))
3566 : return false;
3567 : }
3568 : }
3569 : else
3570 : {
3571 18172 : if (mul_cond && cond != mul_cond)
3572 : return false;
3573 :
3574 18160 : if (cond)
3575 : {
3576 104 : if (cond == result || else_value == result)
3577 : return false;
3578 94 : if (!direct_internal_fn_supported_p (IFN_COND_FMA, type,
3579 : opt_type))
3580 : return false;
3581 : }
3582 : }
3583 :
3584 : /* If the subtrahend (OPS[1]) is computed by a MULT_EXPR that
3585 : we'll visit later, we might be able to get a more profitable
3586 : match with fnma.
3587 : OTOH, if we don't, a negate / fma pair has likely lower latency
3588 : that a mult / subtract pair. */
3589 18150 : if (code == MINUS_EXPR
3590 5828 : && !negate_p
3591 2216 : && ops[0] == result
3592 2216 : && !direct_internal_fn_supported_p (IFN_FMS, type, opt_type)
3593 0 : && direct_internal_fn_supported_p (IFN_FNMA, type, opt_type)
3594 0 : && TREE_CODE (ops[1]) == SSA_NAME
3595 18150 : && has_single_use (ops[1]))
3596 : {
3597 0 : gimple *stmt2 = SSA_NAME_DEF_STMT (ops[1]);
3598 0 : if (is_gimple_assign (stmt2)
3599 0 : && gimple_assign_rhs_code (stmt2) == MULT_EXPR)
3600 : return false;
3601 : }
3602 :
3603 : /* We can't handle a * b + a * b. */
3604 18150 : if (ops[0] == ops[1])
3605 : return false;
3606 : /* If deferring, make sure we are not looking at an instruction that
3607 : wouldn't have existed if we were not. */
3608 18150 : if (state->m_deferring_p
3609 18150 : && (state->m_mul_result_set.contains (ops[0])
3610 6452 : || state->m_mul_result_set.contains (ops[1])))
3611 : return false;
3612 :
3613 18150 : if (check_defer)
3614 : {
3615 6314 : tree use_lhs = gimple_get_lhs (use_stmt);
3616 6314 : if (state->m_last_result)
3617 : {
3618 1052 : if (ops[1] == state->m_last_result
3619 1052 : || ops[0] == state->m_last_result)
3620 : defer = true;
3621 : else
3622 6314 : defer = false;
3623 : }
3624 : else
3625 : {
3626 5262 : gcc_checking_assert (!state->m_initial_phi);
3627 5262 : gphi *phi;
3628 5262 : if (ops[0] == result)
3629 3339 : phi = result_of_phi (ops[1]);
3630 : else
3631 : {
3632 1923 : gcc_assert (ops[1] == result);
3633 1923 : phi = result_of_phi (ops[0]);
3634 : }
3635 :
3636 : if (phi)
3637 : {
3638 963 : state->m_initial_phi = phi;
3639 963 : defer = true;
3640 : }
3641 : else
3642 : defer = false;
3643 : }
3644 :
3645 6314 : state->m_last_result = use_lhs;
3646 6314 : check_defer = false;
3647 : }
3648 : else
3649 : defer = false;
3650 :
3651 : /* While it is possible to validate whether or not the exact form that
3652 : we've recognized is available in the backend, the assumption is that
3653 : if the deferring logic above did not trigger, the transformation is
3654 : never a loss. For instance, suppose the target only has the plain FMA
3655 : pattern available. Consider a*b-c -> fma(a,b,-c): we've exchanged
3656 : MUL+SUB for FMA+NEG, which is still two operations. Consider
3657 : -(a*b)-c -> fma(-a,b,-c): we still have 3 operations, but in the FMA
3658 : form the two NEGs are independent and could be run in parallel. */
3659 5430 : }
3660 :
3661 17626 : if (defer)
3662 : {
3663 1037 : fma_transformation_info fti;
3664 1037 : fti.mul_stmt = mul_stmt;
3665 1037 : fti.mul_result = mul_result;
3666 1037 : fti.op1 = op1;
3667 1037 : fti.op2 = op2;
3668 1037 : state->m_candidates.safe_push (fti);
3669 1037 : state->m_mul_result_set.add (mul_result);
3670 :
3671 1037 : if (dump_file && (dump_flags & TDF_DETAILS))
3672 : {
3673 0 : fprintf (dump_file, "Deferred generating FMA for multiplication ");
3674 0 : print_gimple_stmt (dump_file, mul_stmt, 0, TDF_NONE);
3675 0 : fprintf (dump_file, "\n");
3676 : }
3677 :
3678 1037 : return false;
3679 : }
3680 : else
3681 : {
3682 16589 : if (state->m_deferring_p)
3683 4917 : cancel_fma_deferring (state);
3684 16589 : convert_mult_to_fma_1 (mul_result, op1, op2);
3685 16589 : return true;
3686 : }
3687 : }
3688 :
3689 :
3690 : /* Helper function of match_arith_overflow. For MUL_OVERFLOW, if we have
3691 : a check for non-zero like:
3692 : _1 = x_4(D) * y_5(D);
3693 : *res_7(D) = _1;
3694 : if (x_4(D) != 0)
3695 : goto <bb 3>; [50.00%]
3696 : else
3697 : goto <bb 4>; [50.00%]
3698 :
3699 : <bb 3> [local count: 536870913]:
3700 : _2 = _1 / x_4(D);
3701 : _9 = _2 != y_5(D);
3702 : _10 = (int) _9;
3703 :
3704 : <bb 4> [local count: 1073741824]:
3705 : # iftmp.0_3 = PHI <_10(3), 0(2)>
3706 : then in addition to using .MUL_OVERFLOW (x_4(D), y_5(D)) we can also
3707 : optimize the x_4(D) != 0 condition to 1. */
3708 :
3709 : static void
3710 161 : maybe_optimize_guarding_check (vec<gimple *> &mul_stmts, gimple *cond_stmt,
3711 : gimple *div_stmt, bool *cfg_changed)
3712 : {
3713 161 : basic_block bb = gimple_bb (cond_stmt);
3714 322 : if (gimple_bb (div_stmt) != bb || !single_pred_p (bb))
3715 52 : return;
3716 161 : edge pred_edge = single_pred_edge (bb);
3717 161 : basic_block pred_bb = pred_edge->src;
3718 161 : if (EDGE_COUNT (pred_bb->succs) != 2)
3719 : return;
3720 118 : edge other_edge = EDGE_SUCC (pred_bb, EDGE_SUCC (pred_bb, 0) == pred_edge);
3721 118 : edge other_succ_edge = NULL;
3722 118 : if (gimple_code (cond_stmt) == GIMPLE_COND)
3723 : {
3724 48 : if (EDGE_COUNT (bb->succs) != 2)
3725 : return;
3726 48 : other_succ_edge = EDGE_SUCC (bb, 0);
3727 48 : if (gimple_cond_code (cond_stmt) == NE_EXPR)
3728 : {
3729 24 : if (other_succ_edge->flags & EDGE_TRUE_VALUE)
3730 24 : other_succ_edge = EDGE_SUCC (bb, 1);
3731 : }
3732 : else if (other_succ_edge->flags & EDGE_FALSE_VALUE)
3733 48 : other_succ_edge = EDGE_SUCC (bb, 0);
3734 48 : if (other_edge->dest != other_succ_edge->dest)
3735 : return;
3736 : }
3737 122 : else if (!single_succ_p (bb) || other_edge->dest != single_succ (bb))
3738 : return;
3739 286 : gcond *zero_cond = safe_dyn_cast <gcond *> (*gsi_last_bb (pred_bb));
3740 117 : if (zero_cond == NULL
3741 117 : || (gimple_cond_code (zero_cond)
3742 117 : != ((pred_edge->flags & EDGE_TRUE_VALUE) ? NE_EXPR : EQ_EXPR))
3743 117 : || !integer_zerop (gimple_cond_rhs (zero_cond)))
3744 : return;
3745 117 : tree zero_cond_lhs = gimple_cond_lhs (zero_cond);
3746 117 : if (TREE_CODE (zero_cond_lhs) != SSA_NAME)
3747 : return;
3748 117 : if (gimple_assign_rhs2 (div_stmt) != zero_cond_lhs)
3749 : {
3750 : /* Allow the divisor to be result of a same precision cast
3751 : from zero_cond_lhs. */
3752 54 : tree rhs2 = gimple_assign_rhs2 (div_stmt);
3753 54 : if (TREE_CODE (rhs2) != SSA_NAME)
3754 : return;
3755 54 : gimple *g = SSA_NAME_DEF_STMT (rhs2);
3756 54 : if (!gimple_assign_cast_p (g)
3757 53 : || gimple_assign_rhs1 (g) != gimple_cond_lhs (zero_cond)
3758 53 : || !INTEGRAL_TYPE_P (TREE_TYPE (zero_cond_lhs))
3759 107 : || (TYPE_PRECISION (TREE_TYPE (zero_cond_lhs))
3760 53 : != TYPE_PRECISION (TREE_TYPE (rhs2))))
3761 : return;
3762 : }
3763 116 : gimple_stmt_iterator gsi = gsi_after_labels (bb);
3764 116 : mul_stmts.safe_push (div_stmt);
3765 116 : if (is_gimple_debug (gsi_stmt (gsi)))
3766 0 : gsi_next_nondebug (&gsi);
3767 116 : unsigned cast_count = 0;
3768 665 : while (gsi_stmt (gsi) != cond_stmt)
3769 : {
3770 : /* If original mul_stmt has a single use, allow it in the same bb,
3771 : we are looking then just at __builtin_mul_overflow_p.
3772 : Though, in that case the original mul_stmt will be replaced
3773 : by .MUL_OVERFLOW, REALPART_EXPR and IMAGPART_EXPR stmts. */
3774 : gimple *mul_stmt;
3775 : unsigned int i;
3776 2439 : bool ok = false;
3777 2439 : FOR_EACH_VEC_ELT (mul_stmts, i, mul_stmt)
3778 : {
3779 2292 : if (gsi_stmt (gsi) == mul_stmt)
3780 : {
3781 : ok = true;
3782 : break;
3783 : }
3784 : }
3785 549 : if (!ok && gimple_assign_cast_p (gsi_stmt (gsi)) && ++cast_count < 4)
3786 : ok = true;
3787 402 : if (!ok)
3788 52 : return;
3789 549 : gsi_next_nondebug (&gsi);
3790 : }
3791 116 : if (gimple_code (cond_stmt) == GIMPLE_COND)
3792 : {
3793 47 : basic_block succ_bb = other_edge->dest;
3794 75 : for (gphi_iterator gpi = gsi_start_phis (succ_bb); !gsi_end_p (gpi);
3795 28 : gsi_next (&gpi))
3796 : {
3797 35 : gphi *phi = gpi.phi ();
3798 35 : tree v1 = gimple_phi_arg_def (phi, other_edge->dest_idx);
3799 35 : tree v2 = gimple_phi_arg_def (phi, other_succ_edge->dest_idx);
3800 35 : if (!operand_equal_p (v1, v2, 0))
3801 7 : return;
3802 : }
3803 : }
3804 : else
3805 : {
3806 69 : tree lhs = gimple_assign_lhs (cond_stmt);
3807 69 : if (!lhs || !INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
3808 : return;
3809 69 : gsi_next_nondebug (&gsi);
3810 69 : if (!gsi_end_p (gsi))
3811 : {
3812 69 : if (gimple_assign_rhs_code (cond_stmt) == COND_EXPR)
3813 : return;
3814 69 : gimple *cast_stmt = gsi_stmt (gsi);
3815 69 : if (!gimple_assign_cast_p (cast_stmt))
3816 : return;
3817 69 : tree new_lhs = gimple_assign_lhs (cast_stmt);
3818 69 : gsi_next_nondebug (&gsi);
3819 69 : if (!gsi_end_p (gsi)
3820 69 : || !new_lhs
3821 69 : || !INTEGRAL_TYPE_P (TREE_TYPE (new_lhs))
3822 138 : || TYPE_PRECISION (TREE_TYPE (new_lhs)) <= 1)
3823 : return;
3824 : lhs = new_lhs;
3825 : }
3826 69 : edge succ_edge = single_succ_edge (bb);
3827 69 : basic_block succ_bb = succ_edge->dest;
3828 69 : gsi = gsi_start_phis (succ_bb);
3829 69 : if (gsi_end_p (gsi))
3830 : return;
3831 69 : gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
3832 69 : gsi_next (&gsi);
3833 69 : if (!gsi_end_p (gsi))
3834 : return;
3835 69 : if (gimple_phi_arg_def (phi, succ_edge->dest_idx) != lhs)
3836 : return;
3837 69 : tree other_val = gimple_phi_arg_def (phi, other_edge->dest_idx);
3838 69 : if (gimple_assign_rhs_code (cond_stmt) == COND_EXPR)
3839 : {
3840 0 : tree cond = gimple_assign_rhs1 (cond_stmt);
3841 0 : if (TREE_CODE (cond) == NE_EXPR)
3842 : {
3843 0 : if (!operand_equal_p (other_val,
3844 0 : gimple_assign_rhs3 (cond_stmt), 0))
3845 : return;
3846 : }
3847 0 : else if (!operand_equal_p (other_val,
3848 0 : gimple_assign_rhs2 (cond_stmt), 0))
3849 : return;
3850 : }
3851 69 : else if (gimple_assign_rhs_code (cond_stmt) == NE_EXPR)
3852 : {
3853 40 : if (!integer_zerop (other_val))
3854 : return;
3855 : }
3856 29 : else if (!integer_onep (other_val))
3857 : return;
3858 : }
3859 109 : if (pred_edge->flags & EDGE_TRUE_VALUE)
3860 56 : gimple_cond_make_true (zero_cond);
3861 : else
3862 53 : gimple_cond_make_false (zero_cond);
3863 109 : update_stmt (zero_cond);
3864 109 : reset_flow_sensitive_info_in_bb (bb);
3865 109 : *cfg_changed = true;
3866 : }
3867 :
3868 : /* Helper function for arith_overflow_check_p. Return true
3869 : if VAL1 is equal to VAL2 cast to corresponding integral type
3870 : with other signedness or vice versa. */
3871 :
3872 : static bool
3873 382 : arith_cast_equal_p (tree val1, tree val2)
3874 : {
3875 382 : if (TREE_CODE (val1) == INTEGER_CST && TREE_CODE (val2) == INTEGER_CST)
3876 65 : return wi::eq_p (wi::to_wide (val1), wi::to_wide (val2));
3877 317 : else if (TREE_CODE (val1) != SSA_NAME || TREE_CODE (val2) != SSA_NAME)
3878 : return false;
3879 280 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (val1))
3880 280 : && gimple_assign_rhs1 (SSA_NAME_DEF_STMT (val1)) == val2)
3881 : return true;
3882 168 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (val2))
3883 168 : && gimple_assign_rhs1 (SSA_NAME_DEF_STMT (val2)) == val1)
3884 120 : return true;
3885 : return false;
3886 : }
3887 :
3888 : /* Helper function of match_arith_overflow. Return 1
3889 : if USE_STMT is unsigned overflow check ovf != 0 for
3890 : STMT, -1 if USE_STMT is unsigned overflow check ovf == 0
3891 : and 0 otherwise. */
3892 :
3893 : static int
3894 2964346 : arith_overflow_check_p (gimple *stmt, gimple *cast_stmt, gimple *&use_stmt,
3895 : tree maxval, tree *other)
3896 : {
3897 2964346 : enum tree_code ccode = ERROR_MARK;
3898 2964346 : tree crhs1 = NULL_TREE, crhs2 = NULL_TREE;
3899 2964346 : enum tree_code code = gimple_assign_rhs_code (stmt);
3900 5892321 : tree lhs = gimple_assign_lhs (cast_stmt ? cast_stmt : stmt);
3901 2964346 : tree rhs1 = gimple_assign_rhs1 (stmt);
3902 2964346 : tree rhs2 = gimple_assign_rhs2 (stmt);
3903 2964346 : tree multop = NULL_TREE, divlhs = NULL_TREE;
3904 2964346 : gimple *cur_use_stmt = use_stmt;
3905 :
3906 2964346 : if (code == MULT_EXPR)
3907 : {
3908 686356 : if (!is_gimple_assign (use_stmt))
3909 686028 : return 0;
3910 550505 : if (gimple_assign_rhs_code (use_stmt) != TRUNC_DIV_EXPR)
3911 : return 0;
3912 2193 : if (gimple_assign_rhs1 (use_stmt) != lhs)
3913 : return 0;
3914 2130 : if (cast_stmt)
3915 : {
3916 155 : if (arith_cast_equal_p (gimple_assign_rhs2 (use_stmt), rhs1))
3917 : multop = rhs2;
3918 81 : else if (arith_cast_equal_p (gimple_assign_rhs2 (use_stmt), rhs2))
3919 : multop = rhs1;
3920 : else
3921 : return 0;
3922 : }
3923 1975 : else if (gimple_assign_rhs2 (use_stmt) == rhs1)
3924 : multop = rhs2;
3925 1865 : else if (operand_equal_p (gimple_assign_rhs2 (use_stmt), rhs2, 0))
3926 : multop = rhs1;
3927 : else
3928 : return 0;
3929 332 : if (stmt_ends_bb_p (use_stmt))
3930 : return 0;
3931 332 : divlhs = gimple_assign_lhs (use_stmt);
3932 332 : if (!divlhs)
3933 : return 0;
3934 332 : use_operand_p use;
3935 332 : if (!single_imm_use (divlhs, &use, &cur_use_stmt))
3936 : return 0;
3937 328 : if (cast_stmt && gimple_assign_cast_p (cur_use_stmt))
3938 : {
3939 4 : tree cast_lhs = gimple_assign_lhs (cur_use_stmt);
3940 8 : if (INTEGRAL_TYPE_P (TREE_TYPE (cast_lhs))
3941 4 : && TYPE_UNSIGNED (TREE_TYPE (cast_lhs))
3942 4 : && (TYPE_PRECISION (TREE_TYPE (cast_lhs))
3943 4 : == TYPE_PRECISION (TREE_TYPE (divlhs)))
3944 8 : && single_imm_use (cast_lhs, &use, &cur_use_stmt))
3945 : {
3946 : cast_stmt = NULL;
3947 : divlhs = cast_lhs;
3948 : }
3949 : else
3950 : return 0;
3951 : }
3952 : }
3953 2278318 : if (gimple_code (cur_use_stmt) == GIMPLE_COND)
3954 : {
3955 579614 : ccode = gimple_cond_code (cur_use_stmt);
3956 579614 : crhs1 = gimple_cond_lhs (cur_use_stmt);
3957 579614 : crhs2 = gimple_cond_rhs (cur_use_stmt);
3958 : }
3959 1698704 : else if (is_gimple_assign (cur_use_stmt))
3960 : {
3961 819657 : if (gimple_assign_rhs_class (cur_use_stmt) == GIMPLE_BINARY_RHS)
3962 : {
3963 472987 : ccode = gimple_assign_rhs_code (cur_use_stmt);
3964 472987 : crhs1 = gimple_assign_rhs1 (cur_use_stmt);
3965 472987 : crhs2 = gimple_assign_rhs2 (cur_use_stmt);
3966 : }
3967 : else
3968 : return 0;
3969 : }
3970 : else
3971 : return 0;
3972 :
3973 1052601 : if (maxval
3974 1052601 : && ccode == RSHIFT_EXPR
3975 33 : && crhs1 == lhs
3976 17 : && TREE_CODE (crhs2) == INTEGER_CST
3977 1052618 : && wi::to_widest (crhs2) == TYPE_PRECISION (TREE_TYPE (maxval)))
3978 : {
3979 16 : tree shiftlhs = gimple_assign_lhs (use_stmt);
3980 16 : if (!shiftlhs)
3981 : return 0;
3982 16 : use_operand_p use;
3983 16 : if (!single_imm_use (shiftlhs, &use, &cur_use_stmt))
3984 : return 0;
3985 12 : if (gimple_code (cur_use_stmt) == GIMPLE_COND)
3986 : {
3987 0 : ccode = gimple_cond_code (cur_use_stmt);
3988 0 : crhs1 = gimple_cond_lhs (cur_use_stmt);
3989 0 : crhs2 = gimple_cond_rhs (cur_use_stmt);
3990 : }
3991 12 : else if (is_gimple_assign (cur_use_stmt))
3992 : {
3993 12 : if (gimple_assign_rhs_class (cur_use_stmt) == GIMPLE_BINARY_RHS)
3994 : {
3995 0 : ccode = gimple_assign_rhs_code (cur_use_stmt);
3996 0 : crhs1 = gimple_assign_rhs1 (cur_use_stmt);
3997 0 : crhs2 = gimple_assign_rhs2 (cur_use_stmt);
3998 : }
3999 12 : else if (gimple_assign_rhs_code (cur_use_stmt) == COND_EXPR)
4000 : {
4001 0 : tree cond = gimple_assign_rhs1 (cur_use_stmt);
4002 0 : if (COMPARISON_CLASS_P (cond))
4003 : {
4004 0 : ccode = TREE_CODE (cond);
4005 0 : crhs1 = TREE_OPERAND (cond, 0);
4006 0 : crhs2 = TREE_OPERAND (cond, 1);
4007 : }
4008 : else
4009 : return 0;
4010 : }
4011 : else
4012 : {
4013 12 : enum tree_code sc = gimple_assign_rhs_code (cur_use_stmt);
4014 12 : tree castlhs = gimple_assign_lhs (cur_use_stmt);
4015 12 : if (!CONVERT_EXPR_CODE_P (sc)
4016 12 : || !castlhs
4017 12 : || !INTEGRAL_TYPE_P (TREE_TYPE (castlhs))
4018 24 : || (TYPE_PRECISION (TREE_TYPE (castlhs))
4019 12 : > TYPE_PRECISION (TREE_TYPE (maxval))))
4020 0 : return 0;
4021 : return 1;
4022 : }
4023 : }
4024 : else
4025 : return 0;
4026 0 : if ((ccode != EQ_EXPR && ccode != NE_EXPR)
4027 0 : || crhs1 != shiftlhs
4028 0 : || !integer_zerop (crhs2))
4029 0 : return 0;
4030 : return 1;
4031 : }
4032 :
4033 1052585 : if (TREE_CODE_CLASS (ccode) != tcc_comparison)
4034 : return 0;
4035 :
4036 615732 : switch (ccode)
4037 : {
4038 118064 : case GT_EXPR:
4039 118064 : case LE_EXPR:
4040 118064 : if (maxval)
4041 : {
4042 : /* r = a + b; r > maxval or r <= maxval */
4043 36 : if (crhs1 == lhs
4044 35 : && TREE_CODE (crhs2) == INTEGER_CST
4045 49 : && tree_int_cst_equal (crhs2, maxval))
4046 13 : return ccode == GT_EXPR ? 1 : -1;
4047 : break;
4048 : }
4049 : /* r = a - b; r > a or r <= a
4050 : r = a + b; a > r or a <= r or b > r or b <= r. */
4051 118028 : if ((code == MINUS_EXPR && crhs1 == lhs && crhs2 == rhs1)
4052 117964 : || (code == PLUS_EXPR && (crhs1 == rhs1 || crhs1 == rhs2)
4053 7971 : && crhs2 == lhs))
4054 8035 : return ccode == GT_EXPR ? 1 : -1;
4055 : /* r = ~a; b > r or b <= r. */
4056 109993 : if (code == BIT_NOT_EXPR && crhs2 == lhs)
4057 : {
4058 190 : if (other)
4059 95 : *other = crhs1;
4060 190 : return ccode == GT_EXPR ? 1 : -1;
4061 : }
4062 : break;
4063 63014 : case LT_EXPR:
4064 63014 : case GE_EXPR:
4065 63014 : if (maxval)
4066 : break;
4067 : /* r = a - b; a < r or a >= r
4068 : r = a + b; r < a or r >= a or r < b or r >= b. */
4069 63008 : if ((code == MINUS_EXPR && crhs1 == rhs1 && crhs2 == lhs)
4070 62870 : || (code == PLUS_EXPR && crhs1 == lhs
4071 30602 : && (crhs2 == rhs1 || crhs2 == rhs2)))
4072 4127 : return ccode == LT_EXPR ? 1 : -1;
4073 : /* r = ~a; r < b or r >= b. */
4074 58881 : if (code == BIT_NOT_EXPR && crhs1 == lhs)
4075 : {
4076 167 : if (other)
4077 92 : *other = crhs2;
4078 167 : return ccode == LT_EXPR ? 1 : -1;
4079 : }
4080 : break;
4081 434654 : case EQ_EXPR:
4082 434654 : case NE_EXPR:
4083 : /* r = a * b; _1 = r / a; _1 == b
4084 : r = a * b; _1 = r / b; _1 == a
4085 : r = a * b; _1 = r / a; _1 != b
4086 : r = a * b; _1 = r / b; _1 != a. */
4087 434654 : if (code == MULT_EXPR)
4088 : {
4089 325 : if (cast_stmt)
4090 : {
4091 146 : if ((crhs1 == divlhs && arith_cast_equal_p (crhs2, multop))
4092 146 : || (crhs2 == divlhs && arith_cast_equal_p (crhs1, multop)))
4093 : {
4094 146 : use_stmt = cur_use_stmt;
4095 146 : return ccode == NE_EXPR ? 1 : -1;
4096 : }
4097 : }
4098 128 : else if ((crhs1 == divlhs && operand_equal_p (crhs2, multop, 0))
4099 179 : || (crhs2 == divlhs && crhs1 == multop))
4100 : {
4101 179 : use_stmt = cur_use_stmt;
4102 179 : return ccode == NE_EXPR ? 1 : -1;
4103 : }
4104 : }
4105 : break;
4106 : default:
4107 : break;
4108 : }
4109 : return 0;
4110 : }
4111 :
4112 : extern bool gimple_unsigned_integer_sat_add (tree, tree*, tree (*)(tree));
4113 : extern bool gimple_unsigned_integer_sat_sub (tree, tree*, tree (*)(tree));
4114 : extern bool gimple_unsigned_integer_sat_trunc (tree, tree*, tree (*)(tree));
4115 : extern bool gimple_unsigned_integer_sat_mul (tree, tree*, tree (*)(tree));
4116 : extern bool gimple_spaceship (tree, tree*, tree (*)(tree));
4117 :
4118 : extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree));
4119 : extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
4120 : extern bool gimple_signed_integer_sat_trunc (tree, tree*, tree (*)(tree));
4121 :
4122 : static void
4123 159 : build_saturation_binary_arith_call_and_replace (gimple_stmt_iterator *gsi,
4124 : internal_fn fn, tree lhs,
4125 : tree op_0, tree op_1)
4126 : {
4127 159 : if (direct_internal_fn_supported_p (fn, TREE_TYPE (op_0), OPTIMIZE_FOR_BOTH))
4128 : {
4129 157 : gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
4130 157 : gimple_call_set_lhs (call, lhs);
4131 157 : gsi_replace (gsi, call, /* update_eh_info */ true);
4132 : }
4133 159 : }
4134 :
4135 : static bool
4136 51 : build_saturation_binary_arith_call_and_insert (gimple_stmt_iterator *gsi,
4137 : internal_fn fn, tree lhs,
4138 : tree op_0, tree op_1)
4139 : {
4140 51 : if (!direct_internal_fn_supported_p (fn, TREE_TYPE (op_0), OPTIMIZE_FOR_BOTH))
4141 : return false;
4142 :
4143 43 : gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
4144 43 : gimple_call_set_lhs (call, lhs);
4145 43 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4146 :
4147 43 : return true;
4148 : }
4149 :
4150 : /*
4151 : * Try to match saturation unsigned add with assign.
4152 : * _7 = _4 + _6;
4153 : * _8 = _4 > _7;
4154 : * _9 = (long unsigned int) _8;
4155 : * _10 = -_9;
4156 : * _12 = _7 | _10;
4157 : * =>
4158 : * _12 = .SAT_ADD (_4, _6);
4159 : *
4160 : * Try to match IMM=-1 saturation signed add with assign.
4161 : * <bb 2> [local count: 1073741824]:
4162 : * x.0_1 = (unsigned char) x_5(D);
4163 : * _3 = -x.0_1;
4164 : * _10 = (signed char) _3;
4165 : * _8 = x_5(D) & _10;
4166 : * if (_8 < 0)
4167 : * goto <bb 4>; [1.40%]
4168 : * else
4169 : * goto <bb 3>; [98.60%]
4170 : * <bb 3> [local count: 434070867]:
4171 : * _2 = x.0_1 + 255;
4172 : * <bb 4> [local count: 1073741824]:
4173 : * # _9 = PHI <_2(3), 128(2)>
4174 : * _4 = (int8_t) _9;
4175 : * =>
4176 : * _4 = .SAT_ADD (x_5, -1); */
4177 :
4178 : static void
4179 4963269 : match_saturation_add_with_assign (gimple_stmt_iterator *gsi, gassign *stmt)
4180 : {
4181 4963269 : tree ops[2];
4182 4963269 : tree lhs = gimple_assign_lhs (stmt);
4183 :
4184 4963269 : if (gimple_unsigned_integer_sat_add (lhs, ops, NULL)
4185 4963269 : || gimple_signed_integer_sat_add (lhs, ops, NULL))
4186 34 : build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_ADD, lhs,
4187 : ops[0], ops[1]);
4188 4963269 : }
4189 :
4190 : /*
4191 : * Try to match saturation add with PHI.
4192 : * For unsigned integer:
4193 : * <bb 2> :
4194 : * _1 = x_3(D) + y_4(D);
4195 : * if (_1 >= x_3(D))
4196 : * goto <bb 3>; [INV]
4197 : * else
4198 : * goto <bb 4>; [INV]
4199 : *
4200 : * <bb 3> :
4201 : *
4202 : * <bb 4> :
4203 : * # _2 = PHI <255(2), _1(3)>
4204 : * =>
4205 : * <bb 4> [local count: 1073741824]:
4206 : * _2 = .SAT_ADD (x_4(D), y_5(D));
4207 : *
4208 : * For signed integer:
4209 : * x.0_1 = (long unsigned int) x_7(D);
4210 : * y.1_2 = (long unsigned int) y_8(D);
4211 : * _3 = x.0_1 + y.1_2;
4212 : * sum_9 = (int64_t) _3;
4213 : * _4 = x_7(D) ^ y_8(D);
4214 : * _5 = x_7(D) ^ sum_9;
4215 : * _15 = ~_4;
4216 : * _16 = _5 & _15;
4217 : * if (_16 < 0)
4218 : * goto <bb 3>; [41.00%]
4219 : * else
4220 : * goto <bb 4>; [59.00%]
4221 : * _11 = x_7(D) < 0;
4222 : * _12 = (long int) _11;
4223 : * _13 = -_12;
4224 : * _14 = _13 ^ 9223372036854775807;
4225 : * # _6 = PHI <_14(3), sum_9(2)>
4226 : * =>
4227 : * _6 = .SAT_ADD (x_5(D), y_6(D)); [tail call] */
4228 :
4229 : static bool
4230 4283606 : match_saturation_add (gimple_stmt_iterator *gsi, gphi *phi)
4231 : {
4232 4283606 : if (gimple_phi_num_args (phi) != 2)
4233 : return false;
4234 :
4235 3402691 : tree ops[2];
4236 3402691 : tree phi_result = gimple_phi_result (phi);
4237 :
4238 3402691 : if (!gimple_unsigned_integer_sat_add (phi_result, ops, NULL)
4239 3402691 : && !gimple_signed_integer_sat_add (phi_result, ops, NULL))
4240 : return false;
4241 :
4242 21 : if (!TYPE_UNSIGNED (TREE_TYPE (ops[0])) && TREE_CODE (ops[1]) == INTEGER_CST)
4243 0 : ops[1] = fold_convert (TREE_TYPE (ops[0]), ops[1]);
4244 :
4245 21 : return build_saturation_binary_arith_call_and_insert (gsi, IFN_SAT_ADD,
4246 : phi_result, ops[0],
4247 21 : ops[1]);
4248 : }
4249 :
4250 : /*
4251 : * Try to match saturation unsigned sub.
4252 : * _1 = _4 >= _5;
4253 : * _3 = _4 - _5;
4254 : * _6 = _1 ? _3 : 0;
4255 : * =>
4256 : * _6 = .SAT_SUB (_4, _5); */
4257 :
4258 : static void
4259 3394244 : match_unsigned_saturation_sub (gimple_stmt_iterator *gsi, gassign *stmt)
4260 : {
4261 3394244 : tree ops[2];
4262 3394244 : tree lhs = gimple_assign_lhs (stmt);
4263 :
4264 3394244 : if (gimple_unsigned_integer_sat_sub (lhs, ops, NULL))
4265 125 : build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_SUB, lhs,
4266 : ops[0], ops[1]);
4267 3394244 : }
4268 :
4269 : /*
4270 : * Try to match saturation unsigned mul.
4271 : * _1 = (unsigned int) a_6(D);
4272 : * _2 = (unsigned int) b_7(D);
4273 : * x_8 = _1 * _2;
4274 : * overflow_9 = x_8 > 255;
4275 : * _3 = (unsigned char) overflow_9;
4276 : * _4 = -_3;
4277 : * _5 = (unsigned char) x_8;
4278 : * _10 = _4 | _5;
4279 : * =>
4280 : * _10 = .SAT_SUB (a_6, b_7); */
4281 :
4282 : static void
4283 2655100 : match_unsigned_saturation_mul (gimple_stmt_iterator *gsi, gassign *stmt)
4284 : {
4285 2655100 : tree ops[2];
4286 2655100 : tree lhs = gimple_assign_lhs (stmt);
4287 :
4288 2655100 : if (gimple_unsigned_integer_sat_mul (lhs, ops, NULL))
4289 0 : build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_MUL, lhs,
4290 : ops[0], ops[1]);
4291 2655100 : }
4292 :
4293 : /* Try to match saturation unsigned mul, aka:
4294 : _6 = .MUL_OVERFLOW (a_4(D), b_5(D));
4295 : _2 = IMAGPART_EXPR <_6>;
4296 : if (_2 != 0)
4297 : goto <bb 4>; [35.00%]
4298 : else
4299 : goto <bb 3>; [65.00%]
4300 :
4301 : <bb 3> [local count: 697932184]:
4302 : _1 = REALPART_EXPR <_6>;
4303 :
4304 : <bb 4> [local count: 1073741824]:
4305 : # _3 = PHI <18446744073709551615(2), _1(3)>
4306 : =>
4307 : _3 = .SAT_MUL (a_4(D), b_5(D)); */
4308 :
4309 : static bool
4310 4283563 : match_saturation_mul (gimple_stmt_iterator *gsi, gphi *phi)
4311 : {
4312 4283563 : if (gimple_phi_num_args (phi) != 2)
4313 : return false;
4314 :
4315 3402648 : tree ops[2];
4316 3402648 : tree phi_result = gimple_phi_result (phi);
4317 :
4318 3402648 : if (!gimple_unsigned_integer_sat_mul (phi_result, ops, NULL))
4319 : return false;
4320 :
4321 0 : return build_saturation_binary_arith_call_and_insert (gsi, IFN_SAT_MUL,
4322 : phi_result, ops[0],
4323 0 : ops[1]);
4324 : }
4325 :
4326 : /* Try to match variants of spaceship operation:
4327 : <bb 2>
4328 : if (a_3(D) >= b_4(D)) -- CMP_1
4329 : goto <bb 3>;
4330 : else
4331 : goto <bb 4>;
4332 :
4333 : <bb 3>
4334 : _1 = a_3(D) > b_4(D); -- CMP_2
4335 : _5 = (int) _1;
4336 :
4337 : <bb 4>
4338 : # _2 = PHI <-1(2), _5(3)>
4339 : =>
4340 : _2 = .SPACESHIP (a_3(D), b_4(D), -1);
4341 :
4342 : All possible canonical variants of the comparison operator in CMP_1 and
4343 : CMP_2 has been included in gimple_spaceship function. */
4344 : static bool
4345 4283563 : match_spaceship (gimple_stmt_iterator *gsi, gphi *phi)
4346 : {
4347 4283563 : if (gimple_phi_num_args (phi) != 2)
4348 : return false;
4349 3402648 : tree ops[2];
4350 3402648 : tree phi_result = gimple_phi_result (phi);
4351 :
4352 3402648 : if (!gimple_spaceship (phi_result, ops, NULL))
4353 : return false;
4354 :
4355 : /* Allow different modes as long as both are integral types. */
4356 228 : if (!INTEGRAL_TYPE_P (TREE_TYPE (phi_result))
4357 228 : || !INTEGRAL_TYPE_P (TREE_TYPE (ops[0])))
4358 : return false;
4359 :
4360 114 : tree ops_type = TREE_TYPE (ops[0]);
4361 114 : machine_mode ops_mode = TYPE_MODE (ops_type);
4362 114 : machine_mode promoted_mode = ops_mode;
4363 114 : tree promoted_type = ops_type;
4364 114 : bool is_unsigned = TYPE_UNSIGNED (ops_type);
4365 :
4366 : /* Check if spaceship optab is available for the operand mode.
4367 : If not, try promoting to a wider mode that is supported. */
4368 114 : if (optab_handler (spaceship_optab, ops_mode) == CODE_FOR_nothing)
4369 : {
4370 : /* Try promoting to wider modes (e.g., QI/HI -> SI -> DI). */
4371 : machine_mode wider_mode;
4372 5 : FOR_EACH_WIDER_MODE_FROM (wider_mode, ops_mode)
4373 : {
4374 4 : if (optab_handler (spaceship_optab, wider_mode)
4375 : != CODE_FOR_nothing)
4376 : {
4377 : /* Check if we can get a type for this mode with matching
4378 : signedness. */
4379 0 : promoted_type = lang_hooks.types.type_for_mode (wider_mode,
4380 : is_unsigned);
4381 0 : if (promoted_type != NULL_TREE && INTEGRAL_TYPE_P (promoted_type))
4382 : {
4383 : promoted_mode = wider_mode;
4384 : break;
4385 : }
4386 : }
4387 : }
4388 :
4389 : // If no suitable promoted mode found, give up.
4390 1 : if (promoted_mode == ops_mode)
4391 4283563 : return false;
4392 : }
4393 :
4394 : /* If promotion is needed, insert conversion statements.
4395 : We must use GIMPLE assignments rather than fold_convert because
4396 : gimple_call arguments must be valid GIMPLE values (SSA names or
4397 : constants), not tree expressions. */
4398 113 : ops[0] = gimple_convert (gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
4399 : promoted_type, ops[0]);
4400 113 : ops[1] = gimple_convert (gsi, true, GSI_SAME_STMT, UNKNOWN_LOCATION,
4401 : promoted_type, ops[1]);
4402 :
4403 113 : tree spaceship_arg_3 = is_unsigned ? build_one_cst (integer_type_node)
4404 97 : : build_minus_one_cst (integer_type_node);
4405 :
4406 113 : gcall *call = gimple_build_call_internal (IFN_SPACESHIP, 3, ops[0], ops[1],
4407 : spaceship_arg_3);
4408 :
4409 : /* SPACESHIP optab always returns signed int (SI mode).
4410 : Cast to phi_result's type if needed. */
4411 113 : tree call_result_type = integer_type_node;
4412 113 : if (!types_compatible_p (TREE_TYPE (phi_result), call_result_type))
4413 : {
4414 49 : tree call_result = make_ssa_name (call_result_type);
4415 49 : gimple_call_set_lhs (call, call_result);
4416 49 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4417 49 : gassign *cast_stmt = gimple_build_assign (phi_result, NOP_EXPR,
4418 : call_result);
4419 49 : gsi_insert_before (gsi, cast_stmt, GSI_SAME_STMT);
4420 : }
4421 : else
4422 : {
4423 64 : gimple_call_set_lhs (call, phi_result);
4424 64 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4425 : }
4426 : return true;
4427 : }
4428 :
4429 :
4430 : /*
4431 : * Try to match saturation unsigned sub.
4432 : * <bb 2> [local count: 1073741824]:
4433 : * if (x_2(D) > y_3(D))
4434 : * goto <bb 3>; [50.00%]
4435 : * else
4436 : * goto <bb 4>; [50.00%]
4437 : *
4438 : * <bb 3> [local count: 536870912]:
4439 : * _4 = x_2(D) - y_3(D);
4440 : *
4441 : * <bb 4> [local count: 1073741824]:
4442 : * # _1 = PHI <0(2), _4(3)>
4443 : * =>
4444 : * <bb 4> [local count: 1073741824]:
4445 : * _1 = .SAT_SUB (x_2(D), y_3(D)); */
4446 : static bool
4447 4283589 : match_saturation_sub (gimple_stmt_iterator *gsi, gphi *phi)
4448 : {
4449 4283589 : if (gimple_phi_num_args (phi) != 2)
4450 : return false;
4451 :
4452 3402674 : tree ops[2];
4453 3402674 : tree phi_result = gimple_phi_result (phi);
4454 :
4455 3402674 : if (!gimple_unsigned_integer_sat_sub (phi_result, ops, NULL)
4456 3402674 : && !gimple_signed_integer_sat_sub (phi_result, ops, NULL))
4457 : return false;
4458 :
4459 30 : return build_saturation_binary_arith_call_and_insert (gsi, IFN_SAT_SUB,
4460 : phi_result, ops[0],
4461 30 : ops[1]);
4462 : }
4463 :
4464 : /*
4465 : * Try to match saturation unsigned sub.
4466 : * uint16_t x_4(D);
4467 : * uint8_t _6;
4468 : * overflow_5 = x_4(D) > 255;
4469 : * _1 = (unsigned char) x_4(D);
4470 : * _2 = (unsigned char) overflow_5;
4471 : * _3 = -_2;
4472 : * _6 = _1 | _3;
4473 : * =>
4474 : * _6 = .SAT_TRUNC (x_4(D));
4475 : * */
4476 : static void
4477 2655100 : match_unsigned_saturation_trunc (gimple_stmt_iterator *gsi, gassign *stmt)
4478 : {
4479 2655100 : tree ops[1];
4480 2655100 : tree lhs = gimple_assign_lhs (stmt);
4481 2655100 : tree type = TREE_TYPE (lhs);
4482 :
4483 2655100 : if (gimple_unsigned_integer_sat_trunc (lhs, ops, NULL)
4484 2655205 : && direct_internal_fn_supported_p (IFN_SAT_TRUNC,
4485 105 : tree_pair (type, TREE_TYPE (ops[0])),
4486 : OPTIMIZE_FOR_BOTH))
4487 : {
4488 78 : gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[0]);
4489 78 : gimple_call_set_lhs (call, lhs);
4490 78 : gsi_replace (gsi, call, /* update_eh_info */ true);
4491 : }
4492 2655100 : }
4493 :
4494 : /*
4495 : * Try to match saturation truncate.
4496 : * Aka:
4497 : * x.0_1 = (unsigned long) x_4(D);
4498 : * _2 = x.0_1 + 2147483648;
4499 : * if (_2 > 4294967295)
4500 : * goto <bb 4>; [50.00%]
4501 : * else
4502 : * goto <bb 3>; [50.00%]
4503 : * ;; succ: 4
4504 : * ;; 3
4505 : *
4506 : * ;; basic block 3, loop depth 0
4507 : * ;; pred: 2
4508 : * trunc_5 = (int32_t) x_4(D);
4509 : * goto <bb 5>; [100.00%]
4510 : * ;; succ: 5
4511 : *
4512 : * ;; basic block 4, loop depth 0
4513 : * ;; pred: 2
4514 : * _7 = x_4(D) < 0;
4515 : * _8 = (int) _7;
4516 : * _9 = -_8;
4517 : * _10 = _9 ^ 2147483647;
4518 : * ;; succ: 5
4519 : *
4520 : * ;; basic block 5, loop depth 0
4521 : * ;; pred: 3
4522 : * ;; 4
4523 : * # _3 = PHI <trunc_5(3), _10(4)>
4524 : * =>
4525 : * _6 = .SAT_TRUNC (x_4(D));
4526 : */
4527 :
4528 : static bool
4529 4283563 : match_saturation_trunc (gimple_stmt_iterator *gsi, gphi *phi)
4530 : {
4531 4283563 : if (gimple_phi_num_args (phi) != 2)
4532 : return false;
4533 :
4534 3402648 : tree ops[1];
4535 3402648 : tree phi_result = gimple_phi_result (phi);
4536 3402648 : tree type = TREE_TYPE (phi_result);
4537 :
4538 3402648 : if (!gimple_unsigned_integer_sat_trunc (phi_result, ops, NULL)
4539 3402648 : && !gimple_signed_integer_sat_trunc (phi_result, ops, NULL))
4540 : return false;
4541 :
4542 0 : if (!direct_internal_fn_supported_p (IFN_SAT_TRUNC,
4543 0 : tree_pair (type, TREE_TYPE (ops[0])),
4544 : OPTIMIZE_FOR_BOTH))
4545 : return false;
4546 :
4547 0 : gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[0]);
4548 0 : gimple_call_set_lhs (call, phi_result);
4549 0 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4550 :
4551 0 : return true;
4552 : }
4553 :
4554 : /* Recognize for unsigned x
4555 : x = y - z;
4556 : if (x > y)
4557 : where there are other uses of x and replace it with
4558 : _7 = .SUB_OVERFLOW (y, z);
4559 : x = REALPART_EXPR <_7>;
4560 : _8 = IMAGPART_EXPR <_7>;
4561 : if (_8)
4562 : and similarly for addition.
4563 :
4564 : Also recognize:
4565 : yc = (type) y;
4566 : zc = (type) z;
4567 : x = yc + zc;
4568 : if (x > max)
4569 : where y and z have unsigned types with maximum max
4570 : and there are other uses of x and all of those cast x
4571 : back to that unsigned type and again replace it with
4572 : _7 = .ADD_OVERFLOW (y, z);
4573 : _9 = REALPART_EXPR <_7>;
4574 : _8 = IMAGPART_EXPR <_7>;
4575 : if (_8)
4576 : and replace (utype) x with _9.
4577 : Or with x >> popcount (max) instead of x > max.
4578 :
4579 : Also recognize:
4580 : x = ~z;
4581 : if (y > x)
4582 : and replace it with
4583 : _7 = .ADD_OVERFLOW (y, z);
4584 : _8 = IMAGPART_EXPR <_7>;
4585 : if (_8)
4586 :
4587 : And also recognize:
4588 : z = x * y;
4589 : if (x != 0)
4590 : goto <bb 3>; [50.00%]
4591 : else
4592 : goto <bb 4>; [50.00%]
4593 :
4594 : <bb 3> [local count: 536870913]:
4595 : _2 = z / x;
4596 : _9 = _2 != y;
4597 : _10 = (int) _9;
4598 :
4599 : <bb 4> [local count: 1073741824]:
4600 : # iftmp.0_3 = PHI <_10(3), 0(2)>
4601 : and replace it with
4602 : _7 = .MUL_OVERFLOW (x, y);
4603 : z = IMAGPART_EXPR <_7>;
4604 : _8 = IMAGPART_EXPR <_7>;
4605 : _9 = _8 != 0;
4606 : iftmp.0_3 = (int) _9; */
4607 :
4608 : static bool
4609 3398409 : match_arith_overflow (gimple_stmt_iterator *gsi, gimple *stmt,
4610 : enum tree_code code, bool *cfg_changed)
4611 : {
4612 3398409 : tree lhs = gimple_assign_lhs (stmt);
4613 3398409 : tree type = TREE_TYPE (lhs);
4614 3398409 : use_operand_p use_p;
4615 3398409 : imm_use_iterator iter;
4616 3398409 : bool use_seen = false;
4617 3398409 : bool ovf_use_seen = false;
4618 3398409 : gimple *use_stmt;
4619 3398409 : gimple *add_stmt = NULL;
4620 3398409 : bool add_first = false;
4621 3398409 : gimple *cond_stmt = NULL;
4622 3398409 : gimple *cast_stmt = NULL;
4623 3398409 : tree cast_lhs = NULL_TREE;
4624 :
4625 3398409 : gcc_checking_assert (code == PLUS_EXPR
4626 : || code == MINUS_EXPR
4627 : || code == MULT_EXPR
4628 : || code == BIT_NOT_EXPR);
4629 3398409 : if (!INTEGRAL_TYPE_P (type)
4630 2873940 : || !TYPE_UNSIGNED (type)
4631 1973371 : || has_zero_uses (lhs)
4632 3398409 : || (code != PLUS_EXPR
4633 1972988 : && code != MULT_EXPR
4634 178649 : && optab_handler (code == MINUS_EXPR ? usubv4_optab : uaddv4_optab,
4635 152930 : TYPE_MODE (type)) == CODE_FOR_nothing))
4636 : return false;
4637 :
4638 1971117 : tree rhs1 = gimple_assign_rhs1 (stmt);
4639 1971117 : tree rhs2 = gimple_assign_rhs2 (stmt);
4640 5519873 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
4641 : {
4642 3554738 : use_stmt = USE_STMT (use_p);
4643 3554738 : if (is_gimple_debug (use_stmt))
4644 645090 : continue;
4645 :
4646 2909648 : tree other = NULL_TREE;
4647 2909648 : if (arith_overflow_check_p (stmt, NULL, use_stmt, NULL_TREE, &other))
4648 : {
4649 6476 : if (code == BIT_NOT_EXPR)
4650 : {
4651 187 : gcc_assert (other);
4652 187 : if (TREE_CODE (other) != SSA_NAME)
4653 0 : return false;
4654 187 : if (rhs2 == NULL)
4655 187 : rhs2 = other;
4656 : else
4657 : return false;
4658 187 : cond_stmt = use_stmt;
4659 : }
4660 : ovf_use_seen = true;
4661 : }
4662 : else
4663 : {
4664 2903172 : use_seen = true;
4665 2903172 : if (code == MULT_EXPR
4666 2903172 : && cast_stmt == NULL
4667 2903172 : && gimple_assign_cast_p (use_stmt))
4668 : {
4669 34678 : cast_lhs = gimple_assign_lhs (use_stmt);
4670 69356 : if (INTEGRAL_TYPE_P (TREE_TYPE (cast_lhs))
4671 34126 : && !TYPE_UNSIGNED (TREE_TYPE (cast_lhs))
4672 64115 : && (TYPE_PRECISION (TREE_TYPE (cast_lhs))
4673 29437 : == TYPE_PRECISION (TREE_TYPE (lhs))))
4674 : cast_stmt = use_stmt;
4675 : else
4676 : cast_lhs = NULL_TREE;
4677 : }
4678 : }
4679 2909648 : if (ovf_use_seen && use_seen)
4680 : break;
4681 0 : }
4682 :
4683 1971117 : if (!ovf_use_seen
4684 1971117 : && code == MULT_EXPR
4685 456205 : && cast_stmt)
4686 : {
4687 29079 : if (TREE_CODE (rhs1) != SSA_NAME
4688 29079 : || (TREE_CODE (rhs2) != SSA_NAME && TREE_CODE (rhs2) != INTEGER_CST))
4689 : return false;
4690 67091 : FOR_EACH_IMM_USE_FAST (use_p, iter, cast_lhs)
4691 : {
4692 38012 : use_stmt = USE_STMT (use_p);
4693 38012 : if (is_gimple_debug (use_stmt))
4694 1738 : continue;
4695 :
4696 36274 : if (arith_overflow_check_p (stmt, cast_stmt, use_stmt,
4697 : NULL_TREE, NULL))
4698 38012 : ovf_use_seen = true;
4699 29079 : }
4700 29079 : }
4701 : else
4702 : {
4703 : cast_stmt = NULL;
4704 : cast_lhs = NULL_TREE;
4705 : }
4706 :
4707 1971117 : tree maxval = NULL_TREE;
4708 1971117 : if (!ovf_use_seen
4709 12923 : || (code != MULT_EXPR && (code == BIT_NOT_EXPR ? use_seen : !use_seen))
4710 6100 : || (code == PLUS_EXPR
4711 5830 : && optab_handler (uaddv4_optab,
4712 5830 : TYPE_MODE (type)) == CODE_FOR_nothing)
4713 1983753 : || (code == MULT_EXPR
4714 223 : && optab_handler (cast_stmt ? mulv4_optab : umulv4_optab,
4715 149 : TYPE_MODE (type)) == CODE_FOR_nothing
4716 3 : && (use_seen
4717 3 : || cast_stmt
4718 0 : || !can_mult_highpart_p (TYPE_MODE (type), true))))
4719 : {
4720 1964871 : if (code != PLUS_EXPR)
4721 : return false;
4722 1357949 : if (TREE_CODE (rhs1) != SSA_NAME
4723 1357949 : || !gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs1)))
4724 : return false;
4725 328532 : rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs1));
4726 328532 : tree type1 = TREE_TYPE (rhs1);
4727 328532 : if (!INTEGRAL_TYPE_P (type1)
4728 172973 : || !TYPE_UNSIGNED (type1)
4729 33295 : || TYPE_PRECISION (type1) >= TYPE_PRECISION (type)
4730 342693 : || (TYPE_PRECISION (type1)
4731 342693 : != GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (type1))))
4732 : return false;
4733 9609 : if (TREE_CODE (rhs2) == INTEGER_CST)
4734 : {
4735 4051 : if (wi::ne_p (wi::rshift (wi::to_wide (rhs2),
4736 4051 : TYPE_PRECISION (type1),
4737 8102 : UNSIGNED), 0))
4738 : return false;
4739 1447 : rhs2 = fold_convert (type1, rhs2);
4740 : }
4741 : else
4742 : {
4743 5558 : if (TREE_CODE (rhs2) != SSA_NAME
4744 5558 : || !gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs2)))
4745 : return false;
4746 2389 : rhs2 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs2));
4747 2389 : tree type2 = TREE_TYPE (rhs2);
4748 2389 : if (!INTEGRAL_TYPE_P (type2)
4749 1006 : || !TYPE_UNSIGNED (type2)
4750 333 : || TYPE_PRECISION (type2) >= TYPE_PRECISION (type)
4751 2694 : || (TYPE_PRECISION (type2)
4752 2694 : != GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (type2))))
4753 : return false;
4754 : }
4755 1739 : if (TYPE_PRECISION (type1) >= TYPE_PRECISION (TREE_TYPE (rhs2)))
4756 : type = type1;
4757 : else
4758 5 : type = TREE_TYPE (rhs2);
4759 :
4760 1739 : if (TREE_CODE (type) != INTEGER_TYPE
4761 3478 : || optab_handler (uaddv4_optab,
4762 1739 : TYPE_MODE (type)) == CODE_FOR_nothing)
4763 : return false;
4764 :
4765 1739 : maxval = wide_int_to_tree (type, wi::max_value (TYPE_PRECISION (type),
4766 : UNSIGNED));
4767 1739 : ovf_use_seen = false;
4768 1739 : use_seen = false;
4769 1739 : basic_block use_bb = NULL;
4770 1941 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
4771 : {
4772 1880 : use_stmt = USE_STMT (use_p);
4773 1880 : if (is_gimple_debug (use_stmt))
4774 137 : continue;
4775 :
4776 1743 : if (arith_overflow_check_p (stmt, NULL, use_stmt, maxval, NULL))
4777 : {
4778 13 : ovf_use_seen = true;
4779 13 : use_bb = gimple_bb (use_stmt);
4780 : }
4781 : else
4782 : {
4783 1730 : if (!gimple_assign_cast_p (use_stmt)
4784 1730 : || gimple_assign_rhs_code (use_stmt) == VIEW_CONVERT_EXPR)
4785 : return false;
4786 113 : tree use_lhs = gimple_assign_lhs (use_stmt);
4787 226 : if (!INTEGRAL_TYPE_P (TREE_TYPE (use_lhs))
4788 226 : || (TYPE_PRECISION (TREE_TYPE (use_lhs))
4789 113 : > TYPE_PRECISION (type)))
4790 : return false;
4791 : use_seen = true;
4792 : }
4793 1678 : }
4794 61 : if (!ovf_use_seen)
4795 : return false;
4796 13 : if (!useless_type_conversion_p (type, TREE_TYPE (rhs1)))
4797 : {
4798 2 : if (!use_seen)
4799 : return false;
4800 2 : tree new_rhs1 = make_ssa_name (type);
4801 2 : gimple *g = gimple_build_assign (new_rhs1, NOP_EXPR, rhs1);
4802 2 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4803 2 : rhs1 = new_rhs1;
4804 : }
4805 11 : else if (!useless_type_conversion_p (type, TREE_TYPE (rhs2)))
4806 : {
4807 2 : if (!use_seen)
4808 : return false;
4809 2 : tree new_rhs2 = make_ssa_name (type);
4810 2 : gimple *g = gimple_build_assign (new_rhs2, NOP_EXPR, rhs2);
4811 2 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4812 2 : rhs2 = new_rhs2;
4813 : }
4814 9 : else if (!use_seen)
4815 : {
4816 : /* If there are no uses of the wider addition, check if
4817 : forwprop has not created a narrower addition.
4818 : Require it to be in the same bb as the overflow check. */
4819 12 : FOR_EACH_IMM_USE_FAST (use_p, iter, rhs1)
4820 : {
4821 11 : use_stmt = USE_STMT (use_p);
4822 11 : if (is_gimple_debug (use_stmt))
4823 0 : continue;
4824 :
4825 11 : if (use_stmt == stmt)
4826 0 : continue;
4827 :
4828 11 : if (!is_gimple_assign (use_stmt)
4829 11 : || gimple_bb (use_stmt) != use_bb
4830 22 : || gimple_assign_rhs_code (use_stmt) != PLUS_EXPR)
4831 3 : continue;
4832 :
4833 8 : if (gimple_assign_rhs1 (use_stmt) == rhs1)
4834 : {
4835 8 : if (!operand_equal_p (gimple_assign_rhs2 (use_stmt),
4836 : rhs2, 0))
4837 0 : continue;
4838 : }
4839 0 : else if (gimple_assign_rhs2 (use_stmt) == rhs1)
4840 : {
4841 0 : if (gimple_assign_rhs1 (use_stmt) != rhs2)
4842 0 : continue;
4843 : }
4844 : else
4845 0 : continue;
4846 :
4847 8 : add_stmt = use_stmt;
4848 8 : break;
4849 9 : }
4850 9 : if (add_stmt == NULL)
4851 : return false;
4852 :
4853 : /* If stmt and add_stmt are in the same bb, we need to find out
4854 : which one is earlier. If they are in different bbs, we've
4855 : checked add_stmt is in the same bb as one of the uses of the
4856 : stmt lhs, so stmt needs to dominate add_stmt too. */
4857 8 : if (gimple_bb (stmt) == gimple_bb (add_stmt))
4858 : {
4859 8 : gimple_stmt_iterator gsif = *gsi;
4860 8 : gimple_stmt_iterator gsib = *gsi;
4861 8 : int i;
4862 : /* Search both forward and backward from stmt and have a small
4863 : upper bound. */
4864 20 : for (i = 0; i < 128; i++)
4865 : {
4866 20 : if (!gsi_end_p (gsib))
4867 : {
4868 18 : gsi_prev_nondebug (&gsib);
4869 18 : if (gsi_stmt (gsib) == add_stmt)
4870 : {
4871 : add_first = true;
4872 : break;
4873 : }
4874 : }
4875 2 : else if (gsi_end_p (gsif))
4876 : break;
4877 18 : if (!gsi_end_p (gsif))
4878 : {
4879 18 : gsi_next_nondebug (&gsif);
4880 18 : if (gsi_stmt (gsif) == add_stmt)
4881 : break;
4882 : }
4883 : }
4884 8 : if (i == 128)
4885 0 : return false;
4886 8 : if (add_first)
4887 2 : *gsi = gsi_for_stmt (add_stmt);
4888 : }
4889 : }
4890 : }
4891 :
4892 6258 : if (code == BIT_NOT_EXPR)
4893 170 : *gsi = gsi_for_stmt (cond_stmt);
4894 :
4895 6258 : auto_vec<gimple *, 8> mul_stmts;
4896 6258 : if (code == MULT_EXPR && cast_stmt)
4897 : {
4898 75 : type = TREE_TYPE (cast_lhs);
4899 75 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
4900 75 : if (gimple_assign_cast_p (g)
4901 38 : && useless_type_conversion_p (type,
4902 38 : TREE_TYPE (gimple_assign_rhs1 (g)))
4903 113 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g)))
4904 : rhs1 = gimple_assign_rhs1 (g);
4905 : else
4906 : {
4907 37 : g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, rhs1);
4908 37 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4909 37 : rhs1 = gimple_assign_lhs (g);
4910 37 : mul_stmts.quick_push (g);
4911 : }
4912 75 : if (TREE_CODE (rhs2) == INTEGER_CST)
4913 32 : rhs2 = fold_convert (type, rhs2);
4914 : else
4915 : {
4916 43 : g = SSA_NAME_DEF_STMT (rhs2);
4917 43 : if (gimple_assign_cast_p (g)
4918 22 : && useless_type_conversion_p (type,
4919 22 : TREE_TYPE (gimple_assign_rhs1 (g)))
4920 65 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g)))
4921 : rhs2 = gimple_assign_rhs1 (g);
4922 : else
4923 : {
4924 21 : g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, rhs2);
4925 21 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4926 21 : rhs2 = gimple_assign_lhs (g);
4927 21 : mul_stmts.quick_push (g);
4928 : }
4929 : }
4930 : }
4931 6258 : tree ctype = build_complex_type (type);
4932 12370 : gcall *g = gimple_build_call_internal (code == MULT_EXPR
4933 : ? IFN_MUL_OVERFLOW
4934 : : code != MINUS_EXPR
4935 6112 : ? IFN_ADD_OVERFLOW : IFN_SUB_OVERFLOW,
4936 : 2, rhs1, rhs2);
4937 6258 : tree ctmp = make_ssa_name (ctype);
4938 6258 : gimple_call_set_lhs (g, ctmp);
4939 6258 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
4940 6258 : tree new_lhs = (maxval || cast_stmt) ? make_ssa_name (type) : lhs;
4941 6258 : gassign *g2;
4942 6258 : if (code != BIT_NOT_EXPR)
4943 : {
4944 6088 : g2 = gimple_build_assign (new_lhs, REALPART_EXPR,
4945 : build1 (REALPART_EXPR, type, ctmp));
4946 6088 : if (maxval || cast_stmt)
4947 : {
4948 87 : gsi_insert_before (gsi, g2, GSI_SAME_STMT);
4949 87 : if (add_first)
4950 2 : *gsi = gsi_for_stmt (stmt);
4951 : }
4952 : else
4953 6001 : gsi_replace (gsi, g2, true);
4954 6088 : if (code == MULT_EXPR)
4955 : {
4956 146 : mul_stmts.quick_push (g);
4957 146 : mul_stmts.quick_push (g2);
4958 146 : if (cast_stmt)
4959 : {
4960 75 : g2 = gimple_build_assign (lhs, NOP_EXPR, new_lhs);
4961 75 : gsi_replace (gsi, g2, true);
4962 75 : mul_stmts.quick_push (g2);
4963 : }
4964 : }
4965 : }
4966 6258 : tree ovf = make_ssa_name (type);
4967 6258 : g2 = gimple_build_assign (ovf, IMAGPART_EXPR,
4968 : build1 (IMAGPART_EXPR, type, ctmp));
4969 6258 : if (code != BIT_NOT_EXPR)
4970 6088 : gsi_insert_after (gsi, g2, GSI_NEW_STMT);
4971 : else
4972 170 : gsi_insert_before (gsi, g2, GSI_SAME_STMT);
4973 6258 : if (code == MULT_EXPR)
4974 146 : mul_stmts.quick_push (g2);
4975 :
4976 33403 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, cast_lhs ? cast_lhs : lhs)
4977 : {
4978 20962 : if (is_gimple_debug (use_stmt))
4979 4281 : continue;
4980 :
4981 16681 : gimple *orig_use_stmt = use_stmt;
4982 16681 : int ovf_use = arith_overflow_check_p (stmt, cast_stmt, use_stmt,
4983 : maxval, NULL);
4984 16681 : if (ovf_use == 0)
4985 : {
4986 10376 : gcc_assert (code != BIT_NOT_EXPR);
4987 10376 : if (maxval)
4988 : {
4989 4 : tree use_lhs = gimple_assign_lhs (use_stmt);
4990 4 : gimple_assign_set_rhs1 (use_stmt, new_lhs);
4991 4 : if (useless_type_conversion_p (TREE_TYPE (use_lhs),
4992 4 : TREE_TYPE (new_lhs)))
4993 4 : gimple_assign_set_rhs_code (use_stmt, SSA_NAME);
4994 4 : update_stmt (use_stmt);
4995 : }
4996 10376 : continue;
4997 10376 : }
4998 6305 : if (gimple_code (use_stmt) == GIMPLE_COND)
4999 : {
5000 4160 : gcond *cond_stmt = as_a <gcond *> (use_stmt);
5001 4160 : gimple_cond_set_lhs (cond_stmt, ovf);
5002 4160 : gimple_cond_set_rhs (cond_stmt, build_int_cst (type, 0));
5003 4310 : gimple_cond_set_code (cond_stmt, ovf_use == 1 ? NE_EXPR : EQ_EXPR);
5004 : }
5005 : else
5006 : {
5007 2145 : gcc_checking_assert (is_gimple_assign (use_stmt));
5008 2145 : if (gimple_assign_rhs_class (use_stmt) == GIMPLE_BINARY_RHS)
5009 : {
5010 2145 : if (gimple_assign_rhs_code (use_stmt) == RSHIFT_EXPR)
5011 : {
5012 6 : g2 = gimple_build_assign (make_ssa_name (boolean_type_node),
5013 : ovf_use == 1 ? NE_EXPR : EQ_EXPR,
5014 : ovf, build_int_cst (type, 0));
5015 6 : gimple_stmt_iterator gsiu = gsi_for_stmt (use_stmt);
5016 6 : gsi_insert_before (&gsiu, g2, GSI_SAME_STMT);
5017 6 : gimple_assign_set_rhs_with_ops (&gsiu, NOP_EXPR,
5018 : gimple_assign_lhs (g2));
5019 6 : update_stmt (use_stmt);
5020 6 : use_operand_p use;
5021 6 : single_imm_use (gimple_assign_lhs (use_stmt), &use,
5022 : &use_stmt);
5023 6 : if (gimple_code (use_stmt) == GIMPLE_COND)
5024 : {
5025 0 : gcond *cond_stmt = as_a <gcond *> (use_stmt);
5026 0 : gimple_cond_set_lhs (cond_stmt, ovf);
5027 0 : gimple_cond_set_rhs (cond_stmt, build_int_cst (type, 0));
5028 : }
5029 : else
5030 : {
5031 6 : gcc_checking_assert (is_gimple_assign (use_stmt));
5032 6 : if (gimple_assign_rhs_class (use_stmt)
5033 : == GIMPLE_BINARY_RHS)
5034 : {
5035 0 : gimple_assign_set_rhs1 (use_stmt, ovf);
5036 0 : gimple_assign_set_rhs2 (use_stmt,
5037 : build_int_cst (type, 0));
5038 : }
5039 6 : else if (gimple_assign_cast_p (use_stmt))
5040 6 : gimple_assign_set_rhs1 (use_stmt, ovf);
5041 : else
5042 : {
5043 0 : tree_code sc = gimple_assign_rhs_code (use_stmt);
5044 0 : gcc_checking_assert (sc == COND_EXPR);
5045 0 : tree cond = gimple_assign_rhs1 (use_stmt);
5046 0 : cond = build2 (TREE_CODE (cond),
5047 : boolean_type_node, ovf,
5048 : build_int_cst (type, 0));
5049 0 : gimple_assign_set_rhs1 (use_stmt, cond);
5050 : }
5051 : }
5052 6 : update_stmt (use_stmt);
5053 6 : gsi_remove (&gsiu, true);
5054 6 : gsiu = gsi_for_stmt (g2);
5055 6 : gsi_remove (&gsiu, true);
5056 6 : continue;
5057 6 : }
5058 : else
5059 : {
5060 2139 : gimple_assign_set_rhs1 (use_stmt, ovf);
5061 2139 : gimple_assign_set_rhs2 (use_stmt, build_int_cst (type, 0));
5062 2212 : gimple_assign_set_rhs_code (use_stmt,
5063 : ovf_use == 1
5064 : ? NE_EXPR : EQ_EXPR);
5065 : }
5066 : }
5067 : else
5068 : {
5069 0 : gcc_checking_assert (gimple_assign_rhs_code (use_stmt)
5070 : == COND_EXPR);
5071 0 : tree cond = build2 (ovf_use == 1 ? NE_EXPR : EQ_EXPR,
5072 : boolean_type_node, ovf,
5073 : build_int_cst (type, 0));
5074 0 : gimple_assign_set_rhs1 (use_stmt, cond);
5075 : }
5076 : }
5077 6299 : update_stmt (use_stmt);
5078 6299 : if (code == MULT_EXPR && use_stmt != orig_use_stmt)
5079 : {
5080 161 : gimple_stmt_iterator gsi2 = gsi_for_stmt (orig_use_stmt);
5081 161 : maybe_optimize_guarding_check (mul_stmts, use_stmt, orig_use_stmt,
5082 : cfg_changed);
5083 161 : use_operand_p use;
5084 161 : gimple *cast_stmt;
5085 161 : if (single_imm_use (gimple_assign_lhs (orig_use_stmt), &use,
5086 : &cast_stmt)
5087 161 : && gimple_assign_cast_p (cast_stmt))
5088 : {
5089 2 : gimple_stmt_iterator gsi3 = gsi_for_stmt (cast_stmt);
5090 2 : gsi_remove (&gsi3, true);
5091 2 : release_ssa_name (gimple_assign_lhs (cast_stmt));
5092 : }
5093 161 : gsi_remove (&gsi2, true);
5094 161 : release_ssa_name (gimple_assign_lhs (orig_use_stmt));
5095 : }
5096 6258 : }
5097 6258 : if (maxval)
5098 : {
5099 12 : gimple_stmt_iterator gsi2 = gsi_for_stmt (stmt);
5100 12 : gsi_remove (&gsi2, true);
5101 12 : if (add_stmt)
5102 : {
5103 8 : gimple *g = gimple_build_assign (gimple_assign_lhs (add_stmt),
5104 : new_lhs);
5105 8 : gsi2 = gsi_for_stmt (add_stmt);
5106 8 : gsi_replace (&gsi2, g, true);
5107 : }
5108 : }
5109 6246 : else if (code == BIT_NOT_EXPR)
5110 : {
5111 170 : *gsi = gsi_for_stmt (stmt);
5112 170 : gsi_remove (gsi, true);
5113 170 : release_ssa_name (lhs);
5114 170 : return true;
5115 : }
5116 : return false;
5117 6258 : }
5118 :
5119 : /* Helper of match_uaddc_usubc. Look through an integral cast
5120 : which should preserve [0, 1] range value (unless source has
5121 : 1-bit signed type) and the cast has single use. */
5122 :
5123 : static gimple *
5124 2083667 : uaddc_cast (gimple *g)
5125 : {
5126 2083667 : if (!gimple_assign_cast_p (g))
5127 : return g;
5128 499891 : tree op = gimple_assign_rhs1 (g);
5129 499891 : if (TREE_CODE (op) == SSA_NAME
5130 421749 : && INTEGRAL_TYPE_P (TREE_TYPE (op))
5131 286522 : && (TYPE_PRECISION (TREE_TYPE (op)) > 1
5132 5282 : || TYPE_UNSIGNED (TREE_TYPE (op)))
5133 786413 : && has_single_use (gimple_assign_lhs (g)))
5134 174565 : return SSA_NAME_DEF_STMT (op);
5135 : return g;
5136 : }
5137 :
5138 : /* Helper of match_uaddc_usubc. Look through a NE_EXPR
5139 : comparison with 0 which also preserves [0, 1] value range. */
5140 :
5141 : static gimple *
5142 2083827 : uaddc_ne0 (gimple *g)
5143 : {
5144 2083827 : if (is_gimple_assign (g)
5145 1293684 : && gimple_assign_rhs_code (g) == NE_EXPR
5146 59936 : && integer_zerop (gimple_assign_rhs2 (g))
5147 5985 : && TREE_CODE (gimple_assign_rhs1 (g)) == SSA_NAME
5148 2089800 : && has_single_use (gimple_assign_lhs (g)))
5149 5687 : return SSA_NAME_DEF_STMT (gimple_assign_rhs1 (g));
5150 : return g;
5151 : }
5152 :
5153 : /* Return true if G is {REAL,IMAG}PART_EXPR PART with SSA_NAME
5154 : operand. */
5155 :
5156 : static bool
5157 2084680 : uaddc_is_cplxpart (gimple *g, tree_code part)
5158 : {
5159 2084680 : return (is_gimple_assign (g)
5160 1293124 : && gimple_assign_rhs_code (g) == part
5161 2087144 : && TREE_CODE (TREE_OPERAND (gimple_assign_rhs1 (g), 0)) == SSA_NAME);
5162 : }
5163 :
5164 : /* Try to match e.g.
5165 : _29 = .ADD_OVERFLOW (_3, _4);
5166 : _30 = REALPART_EXPR <_29>;
5167 : _31 = IMAGPART_EXPR <_29>;
5168 : _32 = .ADD_OVERFLOW (_30, _38);
5169 : _33 = REALPART_EXPR <_32>;
5170 : _34 = IMAGPART_EXPR <_32>;
5171 : _35 = _31 + _34;
5172 : as
5173 : _36 = .UADDC (_3, _4, _38);
5174 : _33 = REALPART_EXPR <_36>;
5175 : _35 = IMAGPART_EXPR <_36>;
5176 : or
5177 : _22 = .SUB_OVERFLOW (_6, _5);
5178 : _23 = REALPART_EXPR <_22>;
5179 : _24 = IMAGPART_EXPR <_22>;
5180 : _25 = .SUB_OVERFLOW (_23, _37);
5181 : _26 = REALPART_EXPR <_25>;
5182 : _27 = IMAGPART_EXPR <_25>;
5183 : _28 = _24 | _27;
5184 : as
5185 : _29 = .USUBC (_6, _5, _37);
5186 : _26 = REALPART_EXPR <_29>;
5187 : _288 = IMAGPART_EXPR <_29>;
5188 : provided _38 or _37 above have [0, 1] range
5189 : and _3, _4 and _30 or _6, _5 and _23 are unsigned
5190 : integral types with the same precision. Whether + or | or ^ is
5191 : used on the IMAGPART_EXPR results doesn't matter, with one of
5192 : added or subtracted operands in [0, 1] range at most one
5193 : .ADD_OVERFLOW or .SUB_OVERFLOW will indicate overflow. */
5194 :
5195 : static bool
5196 2840538 : match_uaddc_usubc (gimple_stmt_iterator *gsi, gimple *stmt, tree_code code)
5197 : {
5198 2840538 : tree rhs[4];
5199 2840538 : rhs[0] = gimple_assign_rhs1 (stmt);
5200 2840538 : rhs[1] = gimple_assign_rhs2 (stmt);
5201 2840538 : rhs[2] = NULL_TREE;
5202 2840538 : rhs[3] = NULL_TREE;
5203 2840538 : tree type = TREE_TYPE (rhs[0]);
5204 2840538 : if (!INTEGRAL_TYPE_P (type) || !TYPE_UNSIGNED (type))
5205 : return false;
5206 :
5207 1659129 : auto_vec<gimple *, 2> temp_stmts;
5208 1659129 : if (code != BIT_IOR_EXPR && code != BIT_XOR_EXPR)
5209 : {
5210 : /* If overflow flag is ignored on the MSB limb, we can end up with
5211 : the most significant limb handled as r = op1 + op2 + ovf1 + ovf2;
5212 : or r = op1 - op2 - ovf1 - ovf2; or various equivalent expressions
5213 : thereof. Handle those like the ovf = ovf1 + ovf2; case to recognize
5214 : the limb below the MSB, but also create another .UADDC/.USUBC call
5215 : for the last limb.
5216 :
5217 : First look through assignments with the same rhs code as CODE,
5218 : with the exception that subtraction of a constant is canonicalized
5219 : into addition of its negation. rhs[0] will be minuend for
5220 : subtractions and one of addends for addition, all other assigned
5221 : rhs[i] operands will be subtrahends or other addends. */
5222 1528171 : while (TREE_CODE (rhs[0]) == SSA_NAME && !rhs[3])
5223 : {
5224 1498772 : gimple *g = SSA_NAME_DEF_STMT (rhs[0]);
5225 1498772 : if (has_single_use (rhs[0])
5226 503443 : && is_gimple_assign (g)
5227 1936657 : && (gimple_assign_rhs_code (g) == code
5228 407179 : || (code == MINUS_EXPR
5229 51803 : && gimple_assign_rhs_code (g) == PLUS_EXPR
5230 15792 : && TREE_CODE (gimple_assign_rhs2 (g)) == INTEGER_CST)))
5231 : {
5232 42972 : tree r2 = gimple_assign_rhs2 (g);
5233 42972 : if (gimple_assign_rhs_code (g) != code)
5234 : {
5235 12266 : r2 = const_unop (NEGATE_EXPR, TREE_TYPE (r2), r2);
5236 12266 : if (!r2)
5237 : break;
5238 : }
5239 42972 : rhs[0] = gimple_assign_rhs1 (g);
5240 42972 : tree &r = rhs[2] ? rhs[3] : rhs[2];
5241 42972 : r = r2;
5242 42972 : temp_stmts.quick_push (g);
5243 : }
5244 : else
5245 : break;
5246 : }
5247 4455597 : for (int i = 1; i <= 2; ++i)
5248 3010822 : while (rhs[i] && TREE_CODE (rhs[i]) == SSA_NAME && !rhs[3])
5249 : {
5250 527181 : gimple *g = SSA_NAME_DEF_STMT (rhs[i]);
5251 527181 : if (has_single_use (rhs[i])
5252 263377 : && is_gimple_assign (g)
5253 772752 : && gimple_assign_rhs_code (g) == PLUS_EXPR)
5254 : {
5255 40424 : rhs[i] = gimple_assign_rhs1 (g);
5256 40424 : if (rhs[2])
5257 8265 : rhs[3] = gimple_assign_rhs2 (g);
5258 : else
5259 32159 : rhs[2] = gimple_assign_rhs2 (g);
5260 40424 : temp_stmts.quick_push (g);
5261 : }
5262 : else
5263 : break;
5264 : }
5265 : /* If there are just 3 addends or one minuend and two subtrahends,
5266 : check for UADDC or USUBC being pattern recognized earlier.
5267 : Say r = op1 + op2 + ovf1 + ovf2; where the (ovf1 + ovf2) part
5268 : got pattern matched earlier as __imag__ .UADDC (arg1, arg2, arg3)
5269 : etc. */
5270 1485199 : if (rhs[2] && !rhs[3])
5271 : {
5272 222206 : for (int i = (code == MINUS_EXPR ? 1 : 0); i < 3; ++i)
5273 163331 : if (TREE_CODE (rhs[i]) == SSA_NAME)
5274 : {
5275 125544 : gimple *im = uaddc_cast (SSA_NAME_DEF_STMT (rhs[i]));
5276 125544 : im = uaddc_ne0 (im);
5277 125544 : if (uaddc_is_cplxpart (im, IMAGPART_EXPR))
5278 : {
5279 : /* We found one of the 3 addends or 2 subtrahends to be
5280 : __imag__ of something, verify it is .UADDC/.USUBC. */
5281 226 : tree rhs1 = gimple_assign_rhs1 (im);
5282 226 : gimple *ovf = SSA_NAME_DEF_STMT (TREE_OPERAND (rhs1, 0));
5283 226 : tree ovf_lhs = NULL_TREE;
5284 226 : tree ovf_arg1 = NULL_TREE, ovf_arg2 = NULL_TREE;
5285 246 : if (gimple_call_internal_p (ovf, code == PLUS_EXPR
5286 : ? IFN_ADD_OVERFLOW
5287 : : IFN_SUB_OVERFLOW))
5288 : {
5289 : /* Or verify it is .ADD_OVERFLOW/.SUB_OVERFLOW.
5290 : This is for the case of 2 chained .UADDC/.USUBC,
5291 : where the first one uses 0 carry-in and the second
5292 : one ignores the carry-out.
5293 : So, something like:
5294 : _16 = .ADD_OVERFLOW (_1, _2);
5295 : _17 = REALPART_EXPR <_16>;
5296 : _18 = IMAGPART_EXPR <_16>;
5297 : _15 = _3 + _4;
5298 : _12 = _15 + _18;
5299 : where the first 3 statements come from the lower
5300 : limb addition and the last 2 from the higher limb
5301 : which ignores carry-out. */
5302 201 : ovf_lhs = gimple_call_lhs (ovf);
5303 201 : tree ovf_lhs_type = TREE_TYPE (TREE_TYPE (ovf_lhs));
5304 201 : ovf_arg1 = gimple_call_arg (ovf, 0);
5305 201 : ovf_arg2 = gimple_call_arg (ovf, 1);
5306 : /* In that case we need to punt if the types don't
5307 : mismatch. */
5308 201 : if (!types_compatible_p (type, ovf_lhs_type)
5309 201 : || !types_compatible_p (type, TREE_TYPE (ovf_arg1))
5310 399 : || !types_compatible_p (type,
5311 198 : TREE_TYPE (ovf_arg2)))
5312 : ovf_lhs = NULL_TREE;
5313 : else
5314 : {
5315 498 : for (int i = (code == PLUS_EXPR ? 1 : 0);
5316 498 : i >= 0; --i)
5317 : {
5318 354 : tree r = gimple_call_arg (ovf, i);
5319 354 : if (TREE_CODE (r) != SSA_NAME)
5320 0 : continue;
5321 354 : if (uaddc_is_cplxpart (SSA_NAME_DEF_STMT (r),
5322 : REALPART_EXPR))
5323 : {
5324 : /* Punt if one of the args which isn't
5325 : subtracted isn't __real__; that could
5326 : then prevent better match later.
5327 : Consider:
5328 : _3 = .ADD_OVERFLOW (_1, _2);
5329 : _4 = REALPART_EXPR <_3>;
5330 : _5 = IMAGPART_EXPR <_3>;
5331 : _7 = .ADD_OVERFLOW (_4, _6);
5332 : _8 = REALPART_EXPR <_7>;
5333 : _9 = IMAGPART_EXPR <_7>;
5334 : _12 = _10 + _11;
5335 : _13 = _12 + _9;
5336 : _14 = _13 + _5;
5337 : We want to match this when called on
5338 : the last stmt as a pair of .UADDC calls,
5339 : but without this check we could turn
5340 : that prematurely on _13 = _12 + _9;
5341 : stmt into .UADDC with 0 carry-in just
5342 : on the second .ADD_OVERFLOW call and
5343 : another replacing the _12 and _13
5344 : additions. */
5345 : ovf_lhs = NULL_TREE;
5346 : break;
5347 : }
5348 : }
5349 : }
5350 194 : if (ovf_lhs)
5351 : {
5352 144 : use_operand_p use_p;
5353 144 : imm_use_iterator iter;
5354 144 : tree re_lhs = NULL_TREE;
5355 432 : FOR_EACH_IMM_USE_FAST (use_p, iter, ovf_lhs)
5356 : {
5357 288 : gimple *use_stmt = USE_STMT (use_p);
5358 288 : if (is_gimple_debug (use_stmt))
5359 0 : continue;
5360 288 : if (use_stmt == im)
5361 144 : continue;
5362 144 : if (!uaddc_is_cplxpart (use_stmt,
5363 : REALPART_EXPR))
5364 : {
5365 : ovf_lhs = NULL_TREE;
5366 : break;
5367 : }
5368 144 : re_lhs = gimple_assign_lhs (use_stmt);
5369 144 : }
5370 144 : if (ovf_lhs && re_lhs)
5371 : {
5372 388 : FOR_EACH_IMM_USE_FAST (use_p, iter, re_lhs)
5373 : {
5374 300 : gimple *use_stmt = USE_STMT (use_p);
5375 300 : if (is_gimple_debug (use_stmt))
5376 109 : continue;
5377 191 : internal_fn ifn
5378 191 : = gimple_call_internal_fn (ovf);
5379 : /* Punt if the __real__ of lhs is used
5380 : in the same .*_OVERFLOW call.
5381 : Consider:
5382 : _3 = .ADD_OVERFLOW (_1, _2);
5383 : _4 = REALPART_EXPR <_3>;
5384 : _5 = IMAGPART_EXPR <_3>;
5385 : _7 = .ADD_OVERFLOW (_4, _6);
5386 : _8 = REALPART_EXPR <_7>;
5387 : _9 = IMAGPART_EXPR <_7>;
5388 : _12 = _10 + _11;
5389 : _13 = _12 + _5;
5390 : _14 = _13 + _9;
5391 : We want to match this when called on
5392 : the last stmt as a pair of .UADDC calls,
5393 : but without this check we could turn
5394 : that prematurely on _13 = _12 + _5;
5395 : stmt into .UADDC with 0 carry-in just
5396 : on the first .ADD_OVERFLOW call and
5397 : another replacing the _12 and _13
5398 : additions. */
5399 191 : if (gimple_call_internal_p (use_stmt, ifn))
5400 : {
5401 : ovf_lhs = NULL_TREE;
5402 : break;
5403 : }
5404 144 : }
5405 : }
5406 : }
5407 : }
5408 144 : if ((ovf_lhs
5409 147 : || gimple_call_internal_p (ovf,
5410 : code == PLUS_EXPR
5411 : ? IFN_UADDC : IFN_USUBC))
5412 252 : && (optab_handler (code == PLUS_EXPR
5413 : ? uaddc5_optab : usubc5_optab,
5414 94 : TYPE_MODE (type))
5415 : != CODE_FOR_nothing))
5416 : {
5417 : /* And in that case build another .UADDC/.USUBC
5418 : call for the most significand limb addition.
5419 : Overflow bit is ignored here. */
5420 63 : if (i != 2)
5421 63 : std::swap (rhs[i], rhs[2]);
5422 63 : gimple *g
5423 77 : = gimple_build_call_internal (code == PLUS_EXPR
5424 : ? IFN_UADDC
5425 : : IFN_USUBC,
5426 : 3, rhs[0], rhs[1],
5427 : rhs[2]);
5428 63 : tree nlhs = make_ssa_name (build_complex_type (type));
5429 63 : gimple_call_set_lhs (g, nlhs);
5430 63 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5431 63 : tree ilhs = gimple_assign_lhs (stmt);
5432 63 : g = gimple_build_assign (ilhs, REALPART_EXPR,
5433 : build1 (REALPART_EXPR,
5434 63 : TREE_TYPE (ilhs),
5435 : nlhs));
5436 63 : gsi_replace (gsi, g, true);
5437 : /* And if it is initialized from result of __imag__
5438 : of .{ADD,SUB}_OVERFLOW call, replace that
5439 : call with .U{ADD,SUB}C call with the same arguments,
5440 : just 0 added as third argument. This isn't strictly
5441 : necessary, .ADD_OVERFLOW (x, y) and .UADDC (x, y, 0)
5442 : produce the same result, but may result in better
5443 : generated code on some targets where the backend can
5444 : better prepare in how the result will be used. */
5445 63 : if (ovf_lhs)
5446 : {
5447 57 : tree zero = build_zero_cst (type);
5448 57 : g = gimple_build_call_internal (code == PLUS_EXPR
5449 : ? IFN_UADDC
5450 : : IFN_USUBC,
5451 : 3, ovf_arg1,
5452 : ovf_arg2, zero);
5453 57 : gimple_call_set_lhs (g, ovf_lhs);
5454 57 : gimple_stmt_iterator gsi2 = gsi_for_stmt (ovf);
5455 57 : gsi_replace (&gsi2, g, true);
5456 : }
5457 : return true;
5458 : }
5459 : }
5460 : }
5461 : return false;
5462 : }
5463 1426261 : if (code == MINUS_EXPR && !rhs[2])
5464 : return false;
5465 182 : if (code == MINUS_EXPR)
5466 : /* Code below expects rhs[0] and rhs[1] to have the IMAGPART_EXPRs.
5467 : So, for MINUS_EXPR swap the single added rhs operand (others are
5468 : subtracted) to rhs[3]. */
5469 182 : std::swap (rhs[0], rhs[3]);
5470 : }
5471 : /* Walk from both operands of STMT (for +/- even sometimes from
5472 : all the 4 addends or 3 subtrahends), see through casts and != 0
5473 : statements which would preserve [0, 1] range of values and
5474 : check which is initialized from __imag__. */
5475 1486665 : gimple *im1 = NULL, *im2 = NULL;
5476 14863284 : for (int i = 0; i < (code == MINUS_EXPR ? 3 : 4); i++)
5477 5945729 : if (rhs[i] && TREE_CODE (rhs[i]) == SSA_NAME)
5478 : {
5479 1958031 : gimple *im = uaddc_cast (SSA_NAME_DEF_STMT (rhs[i]));
5480 1958031 : im = uaddc_ne0 (im);
5481 1958031 : if (uaddc_is_cplxpart (im, IMAGPART_EXPR))
5482 : {
5483 1712 : if (im1 == NULL)
5484 : {
5485 1323 : im1 = im;
5486 1323 : if (i != 0)
5487 414 : std::swap (rhs[0], rhs[i]);
5488 : }
5489 : else
5490 : {
5491 389 : im2 = im;
5492 389 : if (i != 1)
5493 22 : std::swap (rhs[1], rhs[i]);
5494 : break;
5495 : }
5496 : }
5497 : }
5498 : /* If we don't find at least two, punt. */
5499 1486665 : if (!im2)
5500 : return false;
5501 : /* Check they are __imag__ of .ADD_OVERFLOW or .SUB_OVERFLOW call results,
5502 : either both .ADD_OVERFLOW or both .SUB_OVERFLOW and that we have
5503 : uaddc5/usubc5 named pattern for the corresponding mode. */
5504 389 : gimple *ovf1
5505 389 : = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (im1), 0));
5506 389 : gimple *ovf2
5507 389 : = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (im2), 0));
5508 389 : internal_fn ifn;
5509 389 : if (!is_gimple_call (ovf1)
5510 389 : || !gimple_call_internal_p (ovf1)
5511 389 : || ((ifn = gimple_call_internal_fn (ovf1)) != IFN_ADD_OVERFLOW
5512 61 : && ifn != IFN_SUB_OVERFLOW)
5513 366 : || !gimple_call_internal_p (ovf2, ifn)
5514 396 : || optab_handler (ifn == IFN_ADD_OVERFLOW ? uaddc5_optab : usubc5_optab,
5515 362 : TYPE_MODE (type)) == CODE_FOR_nothing
5516 95 : || (rhs[2]
5517 17 : && optab_handler (code == PLUS_EXPR ? uaddc5_optab : usubc5_optab,
5518 15 : TYPE_MODE (type)) == CODE_FOR_nothing)
5519 95 : || !types_compatible_p (type,
5520 95 : TREE_TYPE (TREE_TYPE (gimple_call_lhs (ovf1))))
5521 483 : || !types_compatible_p (type,
5522 94 : TREE_TYPE (TREE_TYPE (gimple_call_lhs (ovf2)))))
5523 : return false;
5524 94 : tree arg1, arg2, arg3 = NULL_TREE;
5525 94 : gimple *re1 = NULL, *re2 = NULL;
5526 : /* On one of the two calls, one of the .ADD_OVERFLOW/.SUB_OVERFLOW arguments
5527 : should be initialized from __real__ of the other of the two calls.
5528 : Though, for .SUB_OVERFLOW, it has to be the first argument, not the
5529 : second one. */
5530 249 : for (int i = (ifn == IFN_ADD_OVERFLOW ? 1 : 0); i >= 0; --i)
5531 351 : for (gimple *ovf = ovf1; ovf; ovf = (ovf == ovf1 ? ovf2 : NULL))
5532 : {
5533 290 : tree arg = gimple_call_arg (ovf, i);
5534 290 : if (TREE_CODE (arg) != SSA_NAME)
5535 2 : continue;
5536 288 : re1 = SSA_NAME_DEF_STMT (arg);
5537 288 : if (uaddc_is_cplxpart (re1, REALPART_EXPR)
5538 382 : && (SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (re1), 0))
5539 94 : == (ovf == ovf1 ? ovf2 : ovf1)))
5540 : {
5541 94 : if (ovf == ovf1)
5542 : {
5543 : /* Make sure ovf2 is the .*_OVERFLOW call with argument
5544 : initialized from __real__ of ovf1. */
5545 20 : std::swap (rhs[0], rhs[1]);
5546 20 : std::swap (im1, im2);
5547 20 : std::swap (ovf1, ovf2);
5548 : }
5549 94 : arg3 = gimple_call_arg (ovf, 1 - i);
5550 94 : i = -1;
5551 94 : break;
5552 : }
5553 : }
5554 94 : if (!arg3)
5555 : return false;
5556 94 : arg1 = gimple_call_arg (ovf1, 0);
5557 94 : arg2 = gimple_call_arg (ovf1, 1);
5558 94 : if (!types_compatible_p (type, TREE_TYPE (arg1)))
5559 : return false;
5560 94 : int kind[2] = { 0, 0 };
5561 94 : tree arg_im[2] = { NULL_TREE, NULL_TREE };
5562 : /* At least one of arg2 and arg3 should have type compatible
5563 : with arg1/rhs[0], and the other one should have value in [0, 1]
5564 : range. If both are in [0, 1] range and type compatible with
5565 : arg1/rhs[0], try harder to find after looking through casts,
5566 : != 0 comparisons which one is initialized to __imag__ of
5567 : .{ADD,SUB}_OVERFLOW or .U{ADD,SUB}C call results. */
5568 282 : for (int i = 0; i < 2; ++i)
5569 : {
5570 188 : tree arg = i == 0 ? arg2 : arg3;
5571 188 : if (types_compatible_p (type, TREE_TYPE (arg)))
5572 163 : kind[i] = 1;
5573 376 : if (!INTEGRAL_TYPE_P (TREE_TYPE (arg))
5574 376 : || (TYPE_PRECISION (TREE_TYPE (arg)) == 1
5575 25 : && !TYPE_UNSIGNED (TREE_TYPE (arg))))
5576 0 : continue;
5577 188 : if (tree_zero_one_valued_p (arg))
5578 52 : kind[i] |= 2;
5579 188 : if (TREE_CODE (arg) == SSA_NAME)
5580 : {
5581 185 : gimple *g = SSA_NAME_DEF_STMT (arg);
5582 185 : if (gimple_assign_cast_p (g))
5583 : {
5584 30 : tree op = gimple_assign_rhs1 (g);
5585 30 : if (TREE_CODE (op) == SSA_NAME
5586 30 : && INTEGRAL_TYPE_P (TREE_TYPE (op)))
5587 30 : g = SSA_NAME_DEF_STMT (op);
5588 : }
5589 185 : g = uaddc_ne0 (g);
5590 185 : if (!uaddc_is_cplxpart (g, IMAGPART_EXPR))
5591 125 : continue;
5592 60 : arg_im[i] = gimple_assign_lhs (g);
5593 60 : g = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (g), 0));
5594 60 : if (!is_gimple_call (g) || !gimple_call_internal_p (g))
5595 0 : continue;
5596 60 : switch (gimple_call_internal_fn (g))
5597 : {
5598 60 : case IFN_ADD_OVERFLOW:
5599 60 : case IFN_SUB_OVERFLOW:
5600 60 : case IFN_UADDC:
5601 60 : case IFN_USUBC:
5602 60 : break;
5603 0 : default:
5604 0 : continue;
5605 : }
5606 60 : kind[i] |= 4;
5607 : }
5608 : }
5609 : /* Make arg2 the one with compatible type and arg3 the one
5610 : with [0, 1] range. If both is true for both operands,
5611 : prefer as arg3 result of __imag__ of some ifn. */
5612 94 : if ((kind[0] & 1) == 0 || ((kind[1] & 1) != 0 && kind[0] > kind[1]))
5613 : {
5614 1 : std::swap (arg2, arg3);
5615 1 : std::swap (kind[0], kind[1]);
5616 1 : std::swap (arg_im[0], arg_im[1]);
5617 : }
5618 94 : if ((kind[0] & 1) == 0 || (kind[1] & 6) == 0)
5619 : return false;
5620 70 : if (!has_single_use (gimple_assign_lhs (im1))
5621 68 : || !has_single_use (gimple_assign_lhs (im2))
5622 68 : || !has_single_use (gimple_assign_lhs (re1))
5623 138 : || num_imm_uses (gimple_call_lhs (ovf1)) != 2)
5624 : return false;
5625 : /* Check that ovf2's result is used in __real__ and set re2
5626 : to that statement. */
5627 68 : use_operand_p use_p;
5628 68 : imm_use_iterator iter;
5629 68 : tree lhs = gimple_call_lhs (ovf2);
5630 203 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
5631 : {
5632 135 : gimple *use_stmt = USE_STMT (use_p);
5633 135 : if (is_gimple_debug (use_stmt))
5634 0 : continue;
5635 135 : if (use_stmt == im2)
5636 68 : continue;
5637 67 : if (re2)
5638 : return false;
5639 67 : if (!uaddc_is_cplxpart (use_stmt, REALPART_EXPR))
5640 : return false;
5641 : re2 = use_stmt;
5642 0 : }
5643 : /* Build .UADDC/.USUBC call which will be placed before the stmt. */
5644 68 : gimple_stmt_iterator gsi2 = gsi_for_stmt (ovf2);
5645 68 : gimple *g;
5646 68 : if ((kind[1] & 4) != 0 && types_compatible_p (type, TREE_TYPE (arg_im[1])))
5647 : arg3 = arg_im[1];
5648 68 : if ((kind[1] & 1) == 0)
5649 : {
5650 25 : if (TREE_CODE (arg3) == INTEGER_CST)
5651 0 : arg3 = fold_convert (type, arg3);
5652 : else
5653 : {
5654 25 : g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, arg3);
5655 25 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
5656 25 : arg3 = gimple_assign_lhs (g);
5657 : }
5658 : }
5659 91 : g = gimple_build_call_internal (ifn == IFN_ADD_OVERFLOW
5660 : ? IFN_UADDC : IFN_USUBC,
5661 : 3, arg1, arg2, arg3);
5662 68 : tree nlhs = make_ssa_name (TREE_TYPE (lhs));
5663 68 : gimple_call_set_lhs (g, nlhs);
5664 68 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
5665 : /* In the case where stmt is | or ^ of two overflow flags
5666 : or addition of those, replace stmt with __imag__ of the above
5667 : added call. In case of arg1 + arg2 + (ovf1 + ovf2) or
5668 : arg1 - arg2 - (ovf1 + ovf2) just emit it before stmt. */
5669 68 : tree ilhs = rhs[2] ? make_ssa_name (type) : gimple_assign_lhs (stmt);
5670 68 : g = gimple_build_assign (ilhs, IMAGPART_EXPR,
5671 68 : build1 (IMAGPART_EXPR, TREE_TYPE (ilhs), nlhs));
5672 68 : if (rhs[2])
5673 : {
5674 15 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5675 : /* Remove some further statements which can't be kept in the IL because
5676 : they can use SSA_NAMEs whose setter is going to be removed too. */
5677 75 : for (gimple *g2 : temp_stmts)
5678 : {
5679 30 : gsi2 = gsi_for_stmt (g2);
5680 30 : gsi_remove (&gsi2, true);
5681 30 : release_defs (g2);
5682 : }
5683 : }
5684 : else
5685 53 : gsi_replace (gsi, g, true);
5686 : /* Remove some statements which can't be kept in the IL because they
5687 : use SSA_NAME whose setter is going to be removed too. */
5688 68 : tree rhs1 = rhs[1];
5689 104 : for (int i = 0; i < 2; i++)
5690 86 : if (rhs1 == gimple_assign_lhs (im2))
5691 : break;
5692 : else
5693 : {
5694 36 : g = SSA_NAME_DEF_STMT (rhs1);
5695 36 : rhs1 = gimple_assign_rhs1 (g);
5696 36 : gsi2 = gsi_for_stmt (g);
5697 36 : gsi_remove (&gsi2, true);
5698 36 : release_defs (g);
5699 : }
5700 68 : gcc_checking_assert (rhs1 == gimple_assign_lhs (im2));
5701 68 : gsi2 = gsi_for_stmt (im2);
5702 68 : gsi_remove (&gsi2, true);
5703 68 : release_defs (im2);
5704 : /* Replace the re2 statement with __real__ of the newly added
5705 : .UADDC/.USUBC call. */
5706 68 : if (re2)
5707 : {
5708 67 : gsi2 = gsi_for_stmt (re2);
5709 67 : tree rlhs = gimple_assign_lhs (re2);
5710 67 : g = gimple_build_assign (rlhs, REALPART_EXPR,
5711 67 : build1 (REALPART_EXPR, TREE_TYPE (rlhs), nlhs));
5712 67 : gsi_replace (&gsi2, g, true);
5713 : }
5714 68 : if (rhs[2])
5715 : {
5716 : /* If this is the arg1 + arg2 + (ovf1 + ovf2) or
5717 : arg1 - arg2 - (ovf1 + ovf2) case for the most significant limb,
5718 : replace stmt with __real__ of another .UADDC/.USUBC call which
5719 : handles the most significant limb. Overflow flag from this is
5720 : ignored. */
5721 17 : g = gimple_build_call_internal (code == PLUS_EXPR
5722 : ? IFN_UADDC : IFN_USUBC,
5723 : 3, rhs[3], rhs[2], ilhs);
5724 15 : nlhs = make_ssa_name (TREE_TYPE (lhs));
5725 15 : gimple_call_set_lhs (g, nlhs);
5726 15 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5727 15 : ilhs = gimple_assign_lhs (stmt);
5728 15 : g = gimple_build_assign (ilhs, REALPART_EXPR,
5729 15 : build1 (REALPART_EXPR, TREE_TYPE (ilhs), nlhs));
5730 15 : gsi_replace (gsi, g, true);
5731 : }
5732 68 : if (TREE_CODE (arg3) == SSA_NAME)
5733 : {
5734 : /* When pattern recognizing the second least significant limb
5735 : above (i.e. first pair of .{ADD,SUB}_OVERFLOW calls for one limb),
5736 : check if the [0, 1] range argument (i.e. carry in) isn't the
5737 : result of another .{ADD,SUB}_OVERFLOW call (one handling the
5738 : least significant limb). Again look through casts and != 0. */
5739 67 : gimple *im3 = SSA_NAME_DEF_STMT (arg3);
5740 92 : for (int i = 0; i < 2; ++i)
5741 : {
5742 92 : gimple *im4 = uaddc_cast (im3);
5743 92 : if (im4 == im3)
5744 : break;
5745 : else
5746 25 : im3 = im4;
5747 : }
5748 67 : im3 = uaddc_ne0 (im3);
5749 67 : if (uaddc_is_cplxpart (im3, IMAGPART_EXPR))
5750 : {
5751 60 : gimple *ovf3
5752 60 : = SSA_NAME_DEF_STMT (TREE_OPERAND (gimple_assign_rhs1 (im3), 0));
5753 60 : if (gimple_call_internal_p (ovf3, ifn))
5754 : {
5755 25 : lhs = gimple_call_lhs (ovf3);
5756 25 : arg1 = gimple_call_arg (ovf3, 0);
5757 25 : arg2 = gimple_call_arg (ovf3, 1);
5758 25 : if (types_compatible_p (type, TREE_TYPE (TREE_TYPE (lhs)))
5759 25 : && types_compatible_p (type, TREE_TYPE (arg1))
5760 50 : && types_compatible_p (type, TREE_TYPE (arg2)))
5761 : {
5762 : /* And if it is initialized from result of __imag__
5763 : of .{ADD,SUB}_OVERFLOW call, replace that
5764 : call with .U{ADD,SUB}C call with the same arguments,
5765 : just 0 added as third argument. This isn't strictly
5766 : necessary, .ADD_OVERFLOW (x, y) and .UADDC (x, y, 0)
5767 : produce the same result, but may result in better
5768 : generated code on some targets where the backend can
5769 : better prepare in how the result will be used. */
5770 25 : g = gimple_build_call_internal (ifn == IFN_ADD_OVERFLOW
5771 : ? IFN_UADDC : IFN_USUBC,
5772 : 3, arg1, arg2,
5773 : build_zero_cst (type));
5774 25 : gimple_call_set_lhs (g, lhs);
5775 25 : gsi2 = gsi_for_stmt (ovf3);
5776 25 : gsi_replace (&gsi2, g, true);
5777 : }
5778 : }
5779 : }
5780 : }
5781 : return true;
5782 1659129 : }
5783 :
5784 : /* Replace .POPCOUNT (x) == 1 or .POPCOUNT (x) != 1 with
5785 : (x & (x - 1)) > x - 1 or (x & (x - 1)) <= x - 1 if .POPCOUNT
5786 : isn't a direct optab. Also handle `<=`/`>` to be
5787 : `x & (x - 1) !=/== x`. */
5788 :
5789 : static void
5790 4549452 : match_single_bit_test (gimple_stmt_iterator *gsi, gimple *stmt)
5791 : {
5792 4549452 : tree clhs, crhs;
5793 4549452 : enum tree_code code;
5794 4549452 : bool was_le = false;
5795 4549452 : if (gimple_code (stmt) == GIMPLE_COND)
5796 : {
5797 4220378 : clhs = gimple_cond_lhs (stmt);
5798 4220378 : crhs = gimple_cond_rhs (stmt);
5799 4220378 : code = gimple_cond_code (stmt);
5800 : }
5801 : else
5802 : {
5803 329074 : clhs = gimple_assign_rhs1 (stmt);
5804 329074 : crhs = gimple_assign_rhs2 (stmt);
5805 329074 : code = gimple_assign_rhs_code (stmt);
5806 : }
5807 4549452 : if (code != LE_EXPR && code != GT_EXPR
5808 4549452 : && code != EQ_EXPR && code != NE_EXPR)
5809 4549410 : return;
5810 2137828 : if (code == LE_EXPR || code == GT_EXPR)
5811 4287800 : was_le = true;
5812 4287800 : if (TREE_CODE (clhs) != SSA_NAME || !integer_onep (crhs))
5813 : return;
5814 163281 : gimple *call = SSA_NAME_DEF_STMT (clhs);
5815 163281 : combined_fn cfn = gimple_call_combined_fn (call);
5816 163281 : switch (cfn)
5817 : {
5818 51 : CASE_CFN_POPCOUNT:
5819 51 : break;
5820 : default:
5821 : return;
5822 : }
5823 51 : if (!has_single_use (clhs))
5824 : return;
5825 50 : tree arg = gimple_call_arg (call, 0);
5826 50 : tree type = TREE_TYPE (arg);
5827 50 : if (!INTEGRAL_TYPE_P (type))
5828 : return;
5829 50 : bool nonzero_arg = tree_expr_nonzero_p (arg);
5830 50 : if (direct_internal_fn_supported_p (IFN_POPCOUNT, type, OPTIMIZE_FOR_BOTH))
5831 : {
5832 : /* Tell expand_POPCOUNT the popcount result is only used in equality
5833 : comparison with one, so that it can decide based on rtx costs. */
5834 16 : gimple *g = gimple_build_call_internal (IFN_POPCOUNT, 2, arg,
5835 : was_le ? integer_minus_one_node
5836 8 : : (nonzero_arg ? integer_zero_node
5837 : : integer_one_node));
5838 8 : gimple_call_set_lhs (g, gimple_call_lhs (call));
5839 8 : gimple_stmt_iterator gsi2 = gsi_for_stmt (call);
5840 8 : gsi_replace (&gsi2, g, true);
5841 8 : return;
5842 : }
5843 42 : tree argm1 = make_ssa_name (type);
5844 42 : gimple *g = gimple_build_assign (argm1, PLUS_EXPR, arg,
5845 : build_int_cst (type, -1));
5846 42 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5847 42 : g = gimple_build_assign (make_ssa_name (type),
5848 42 : (nonzero_arg || was_le) ? BIT_AND_EXPR : BIT_XOR_EXPR,
5849 : arg, argm1);
5850 42 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5851 42 : tree_code cmpcode;
5852 42 : if (was_le)
5853 : {
5854 0 : argm1 = build_zero_cst (type);
5855 0 : cmpcode = code == LE_EXPR ? EQ_EXPR : NE_EXPR;
5856 : }
5857 42 : else if (nonzero_arg)
5858 : {
5859 2 : argm1 = build_zero_cst (type);
5860 2 : cmpcode = code;
5861 : }
5862 : else
5863 40 : cmpcode = code == EQ_EXPR ? GT_EXPR : LE_EXPR;
5864 42 : if (gcond *cond = dyn_cast <gcond *> (stmt))
5865 : {
5866 2 : gimple_cond_set_lhs (cond, gimple_assign_lhs (g));
5867 2 : gimple_cond_set_rhs (cond, argm1);
5868 2 : gimple_cond_set_code (cond, cmpcode);
5869 : }
5870 : else
5871 : {
5872 40 : gimple_assign_set_rhs1 (stmt, gimple_assign_lhs (g));
5873 40 : gimple_assign_set_rhs2 (stmt, argm1);
5874 40 : gimple_assign_set_rhs_code (stmt, cmpcode);
5875 : }
5876 42 : update_stmt (stmt);
5877 42 : gimple_stmt_iterator gsi2 = gsi_for_stmt (call);
5878 42 : gsi_remove (&gsi2, true);
5879 42 : release_defs (call);
5880 : }
5881 :
5882 : /* Return true if target has support for divmod. */
5883 :
5884 : static bool
5885 38276 : target_supports_divmod_p (optab divmod_optab, optab div_optab, machine_mode mode)
5886 : {
5887 : /* If target supports hardware divmod insn, use it for divmod. */
5888 38276 : if (optab_handler (divmod_optab, mode) != CODE_FOR_nothing)
5889 : return true;
5890 :
5891 : /* Check if libfunc for divmod is available. */
5892 2572 : rtx libfunc = optab_libfunc (divmod_optab, mode);
5893 2572 : if (libfunc != NULL_RTX)
5894 : {
5895 : /* If optab_handler exists for div_optab, perhaps in a wider mode,
5896 : we don't want to use the libfunc even if it exists for given mode. */
5897 : machine_mode div_mode;
5898 10719 : FOR_EACH_MODE_FROM (div_mode, mode)
5899 8147 : if (optab_handler (div_optab, div_mode) != CODE_FOR_nothing)
5900 : return false;
5901 :
5902 2572 : return targetm.expand_divmod_libfunc != NULL;
5903 : }
5904 :
5905 : return false;
5906 : }
5907 :
5908 : /* Check if stmt is candidate for divmod transform. */
5909 :
5910 : static bool
5911 57014 : divmod_candidate_p (gassign *stmt)
5912 : {
5913 57014 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
5914 57014 : machine_mode mode = TYPE_MODE (type);
5915 57014 : optab divmod_optab, div_optab;
5916 :
5917 57014 : if (TYPE_UNSIGNED (type))
5918 : {
5919 : divmod_optab = udivmod_optab;
5920 : div_optab = udiv_optab;
5921 : }
5922 : else
5923 : {
5924 28311 : divmod_optab = sdivmod_optab;
5925 28311 : div_optab = sdiv_optab;
5926 : }
5927 :
5928 57014 : tree op1 = gimple_assign_rhs1 (stmt);
5929 57014 : tree op2 = gimple_assign_rhs2 (stmt);
5930 :
5931 : /* Disable the transform if either is a constant, since division-by-constant
5932 : may have specialized expansion. */
5933 57014 : if (CONSTANT_CLASS_P (op1))
5934 : return false;
5935 :
5936 53086 : if (CONSTANT_CLASS_P (op2))
5937 : {
5938 17041 : if (integer_pow2p (op2))
5939 : return false;
5940 :
5941 14917 : if (element_precision (type) <= HOST_BITS_PER_WIDE_INT
5942 16002 : && element_precision (type) <= BITS_PER_WORD)
5943 : return false;
5944 :
5945 : /* If the divisor is not power of 2 and the precision wider than
5946 : HWI, expand_divmod punts on that, so in that case it is better
5947 : to use divmod optab or libfunc. Similarly if choose_multiplier
5948 : might need pre/post shifts of BITS_PER_WORD or more. */
5949 : }
5950 :
5951 : /* Exclude the case where TYPE_OVERFLOW_TRAPS (type) as that should
5952 : expand using the [su]divv optabs. */
5953 38276 : if (TYPE_OVERFLOW_TRAPS (type))
5954 : return false;
5955 :
5956 38276 : if (!target_supports_divmod_p (divmod_optab, div_optab, mode))
5957 : return false;
5958 :
5959 : return true;
5960 : }
5961 :
5962 : /* This function looks for:
5963 : t1 = a TRUNC_DIV_EXPR b;
5964 : t2 = a TRUNC_MOD_EXPR b;
5965 : and transforms it to the following sequence:
5966 : complex_tmp = DIVMOD (a, b);
5967 : t1 = REALPART_EXPR(a);
5968 : t2 = IMAGPART_EXPR(b);
5969 : For conditions enabling the transform see divmod_candidate_p().
5970 :
5971 : The pass has three parts:
5972 : 1) Find top_stmt which is trunc_div or trunc_mod stmt and dominates all
5973 : other trunc_div_expr and trunc_mod_expr stmts.
5974 : 2) Add top_stmt and all trunc_div and trunc_mod stmts dominated by top_stmt
5975 : to stmts vector.
5976 : 3) Insert DIVMOD call just before top_stmt and update entries in
5977 : stmts vector to use return value of DIMOVD (REALEXPR_PART for div,
5978 : IMAGPART_EXPR for mod). */
5979 :
5980 : static bool
5981 57033 : convert_to_divmod (gassign *stmt)
5982 : {
5983 57033 : if (stmt_can_throw_internal (cfun, stmt)
5984 57033 : || !divmod_candidate_p (stmt))
5985 : return false;
5986 :
5987 38276 : tree op1 = gimple_assign_rhs1 (stmt);
5988 38276 : tree op2 = gimple_assign_rhs2 (stmt);
5989 :
5990 38276 : imm_use_iterator use_iter;
5991 38276 : gimple *use_stmt;
5992 38276 : auto_vec<gimple *> stmts;
5993 :
5994 38276 : gimple *top_stmt = stmt;
5995 38276 : basic_block top_bb = gimple_bb (stmt);
5996 :
5997 : /* Part 1: Try to set top_stmt to "topmost" stmt that dominates
5998 : at-least stmt and possibly other trunc_div/trunc_mod stmts
5999 : having same operands as stmt. */
6000 :
6001 325690 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, op1)
6002 : {
6003 287414 : if (is_gimple_assign (use_stmt)
6004 237300 : && (gimple_assign_rhs_code (use_stmt) == TRUNC_DIV_EXPR
6005 225536 : || gimple_assign_rhs_code (use_stmt) == TRUNC_MOD_EXPR)
6006 214954 : && operand_equal_p (op1, gimple_assign_rhs1 (use_stmt), 0)
6007 502251 : && operand_equal_p (op2, gimple_assign_rhs2 (use_stmt), 0))
6008 : {
6009 49930 : if (stmt_can_throw_internal (cfun, use_stmt))
6010 0 : continue;
6011 :
6012 49930 : basic_block bb = gimple_bb (use_stmt);
6013 :
6014 49930 : if (bb == top_bb)
6015 : {
6016 49204 : if (gimple_uid (use_stmt) < gimple_uid (top_stmt))
6017 5154 : top_stmt = use_stmt;
6018 : }
6019 726 : else if (dominated_by_p (CDI_DOMINATORS, top_bb, bb))
6020 : {
6021 194 : top_bb = bb;
6022 194 : top_stmt = use_stmt;
6023 : }
6024 : }
6025 38276 : }
6026 :
6027 38276 : tree top_op1 = gimple_assign_rhs1 (top_stmt);
6028 38276 : tree top_op2 = gimple_assign_rhs2 (top_stmt);
6029 :
6030 38276 : stmts.safe_push (top_stmt);
6031 38276 : bool div_seen = (gimple_assign_rhs_code (top_stmt) == TRUNC_DIV_EXPR);
6032 :
6033 : /* Part 2: Add all trunc_div/trunc_mod statements domianted by top_bb
6034 : to stmts vector. The 2nd loop will always add stmt to stmts vector, since
6035 : gimple_bb (top_stmt) dominates gimple_bb (stmt), so the
6036 : 2nd loop ends up adding at-least single trunc_mod_expr stmt. */
6037 :
6038 325690 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, top_op1)
6039 : {
6040 287414 : if (is_gimple_assign (use_stmt)
6041 237300 : && (gimple_assign_rhs_code (use_stmt) == TRUNC_DIV_EXPR
6042 225536 : || gimple_assign_rhs_code (use_stmt) == TRUNC_MOD_EXPR)
6043 214954 : && operand_equal_p (top_op1, gimple_assign_rhs1 (use_stmt), 0)
6044 502251 : && operand_equal_p (top_op2, gimple_assign_rhs2 (use_stmt), 0))
6045 : {
6046 88294 : if (use_stmt == top_stmt
6047 11654 : || stmt_can_throw_internal (cfun, use_stmt)
6048 61584 : || !dominated_by_p (CDI_DOMINATORS, gimple_bb (use_stmt), top_bb))
6049 38364 : continue;
6050 :
6051 11566 : stmts.safe_push (use_stmt);
6052 11566 : if (gimple_assign_rhs_code (use_stmt) == TRUNC_DIV_EXPR)
6053 287414 : div_seen = true;
6054 : }
6055 38276 : }
6056 :
6057 38276 : if (!div_seen)
6058 : return false;
6059 :
6060 : /* Part 3: Create libcall to internal fn DIVMOD:
6061 : divmod_tmp = DIVMOD (op1, op2). */
6062 :
6063 11539 : gcall *call_stmt = gimple_build_call_internal (IFN_DIVMOD, 2, op1, op2);
6064 11539 : tree res = make_temp_ssa_name (build_complex_type (TREE_TYPE (op1)),
6065 : call_stmt, "divmod_tmp");
6066 11539 : gimple_call_set_lhs (call_stmt, res);
6067 : /* We rejected throwing statements above. */
6068 11539 : gimple_call_set_nothrow (call_stmt, true);
6069 :
6070 : /* Insert the call before top_stmt. */
6071 11539 : gimple_stmt_iterator top_stmt_gsi = gsi_for_stmt (top_stmt);
6072 11539 : gsi_insert_before (&top_stmt_gsi, call_stmt, GSI_SAME_STMT);
6073 :
6074 11539 : widen_mul_stats.divmod_calls_inserted++;
6075 :
6076 : /* Update all statements in stmts vector:
6077 : lhs = op1 TRUNC_DIV_EXPR op2 -> lhs = REALPART_EXPR<divmod_tmp>
6078 : lhs = op1 TRUNC_MOD_EXPR op2 -> lhs = IMAGPART_EXPR<divmod_tmp>. */
6079 :
6080 72918 : for (unsigned i = 0; stmts.iterate (i, &use_stmt); ++i)
6081 : {
6082 23103 : tree new_rhs;
6083 :
6084 23103 : switch (gimple_assign_rhs_code (use_stmt))
6085 : {
6086 11549 : case TRUNC_DIV_EXPR:
6087 11549 : new_rhs = fold_build1 (REALPART_EXPR, TREE_TYPE (op1), res);
6088 11549 : break;
6089 :
6090 11554 : case TRUNC_MOD_EXPR:
6091 11554 : new_rhs = fold_build1 (IMAGPART_EXPR, TREE_TYPE (op1), res);
6092 11554 : break;
6093 :
6094 0 : default:
6095 0 : gcc_unreachable ();
6096 : }
6097 :
6098 23103 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
6099 23103 : gimple_assign_set_rhs_from_tree (&gsi, new_rhs);
6100 23103 : update_stmt (use_stmt);
6101 : }
6102 :
6103 : return true;
6104 38276 : }
6105 :
6106 : /* Process a single gimple assignment STMT, which has a RSHIFT_EXPR as
6107 : its rhs, and try to convert it into a MULT_HIGHPART_EXPR. The return
6108 : value is true iff we converted the statement. */
6109 :
6110 : static bool
6111 174356 : convert_mult_to_highpart (gassign *stmt, gimple_stmt_iterator *gsi)
6112 : {
6113 174356 : tree lhs = gimple_assign_lhs (stmt);
6114 174356 : tree stype = TREE_TYPE (lhs);
6115 174356 : tree sarg0 = gimple_assign_rhs1 (stmt);
6116 174356 : tree sarg1 = gimple_assign_rhs2 (stmt);
6117 :
6118 174356 : if (TREE_CODE (stype) != INTEGER_TYPE
6119 167211 : || TREE_CODE (sarg1) != INTEGER_CST
6120 150330 : || TREE_CODE (sarg0) != SSA_NAME
6121 150329 : || !tree_fits_uhwi_p (sarg1)
6122 324685 : || !has_single_use (sarg0))
6123 : return false;
6124 :
6125 48823 : gassign *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (sarg0));
6126 45505 : if (!def)
6127 : return false;
6128 :
6129 45505 : enum tree_code mcode = gimple_assign_rhs_code (def);
6130 45505 : if (mcode == NOP_EXPR)
6131 : {
6132 11120 : tree tmp = gimple_assign_rhs1 (def);
6133 11120 : if (TREE_CODE (tmp) != SSA_NAME || !has_single_use (tmp))
6134 : return false;
6135 3741 : def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (tmp));
6136 3448 : if (!def)
6137 : return false;
6138 3448 : mcode = gimple_assign_rhs_code (def);
6139 : }
6140 :
6141 37833 : if (mcode != WIDEN_MULT_EXPR
6142 37833 : || gimple_bb (def) != gimple_bb (stmt))
6143 : return false;
6144 2581 : tree mtype = TREE_TYPE (gimple_assign_lhs (def));
6145 2581 : if (TREE_CODE (mtype) != INTEGER_TYPE
6146 2581 : || TYPE_PRECISION (mtype) != TYPE_PRECISION (stype))
6147 : return false;
6148 :
6149 2581 : tree mop1 = gimple_assign_rhs1 (def);
6150 2581 : tree mop2 = gimple_assign_rhs2 (def);
6151 2581 : tree optype = TREE_TYPE (mop1);
6152 2581 : bool unsignedp = TYPE_UNSIGNED (optype);
6153 2581 : unsigned int prec = TYPE_PRECISION (optype);
6154 :
6155 2581 : if (unsignedp != TYPE_UNSIGNED (mtype)
6156 2581 : || TYPE_PRECISION (mtype) != 2 * prec)
6157 : return false;
6158 :
6159 2581 : unsigned HOST_WIDE_INT bits = tree_to_uhwi (sarg1);
6160 2581 : if (bits < prec || bits >= 2 * prec)
6161 : return false;
6162 :
6163 : /* For the time being, require operands to have the same sign. */
6164 2580 : if (unsignedp != TYPE_UNSIGNED (TREE_TYPE (mop2)))
6165 : return false;
6166 :
6167 2580 : machine_mode mode = TYPE_MODE (optype);
6168 2580 : optab tab = unsignedp ? umul_highpart_optab : smul_highpart_optab;
6169 2580 : if (optab_handler (tab, mode) == CODE_FOR_nothing)
6170 : return false;
6171 :
6172 2580 : location_t loc = gimple_location (stmt);
6173 2580 : tree highpart1 = build_and_insert_binop (gsi, loc, "highparttmp",
6174 : MULT_HIGHPART_EXPR, mop1, mop2);
6175 2580 : tree highpart2 = highpart1;
6176 2580 : tree ntype = optype;
6177 :
6178 2580 : if (TYPE_UNSIGNED (stype) != TYPE_UNSIGNED (optype))
6179 : {
6180 16 : ntype = TYPE_UNSIGNED (stype) ? unsigned_type_for (optype)
6181 7 : : signed_type_for (optype);
6182 16 : highpart2 = build_and_insert_cast (gsi, loc, ntype, highpart1);
6183 : }
6184 2580 : if (bits > prec)
6185 29 : highpart2 = build_and_insert_binop (gsi, loc, "highparttmp",
6186 : RSHIFT_EXPR, highpart2,
6187 29 : build_int_cst (ntype, bits - prec));
6188 :
6189 2580 : gassign *new_stmt = gimple_build_assign (lhs, NOP_EXPR, highpart2);
6190 2580 : gsi_replace (gsi, new_stmt, true);
6191 :
6192 2580 : widen_mul_stats.highpart_mults_inserted++;
6193 2580 : return true;
6194 : }
6195 :
6196 : /* If target has spaceship<MODE>3 expander, pattern recognize
6197 : <bb 2> [local count: 1073741824]:
6198 : if (a_2(D) == b_3(D))
6199 : goto <bb 6>; [34.00%]
6200 : else
6201 : goto <bb 3>; [66.00%]
6202 :
6203 : <bb 3> [local count: 708669601]:
6204 : if (a_2(D) < b_3(D))
6205 : goto <bb 6>; [1.04%]
6206 : else
6207 : goto <bb 4>; [98.96%]
6208 :
6209 : <bb 4> [local count: 701299439]:
6210 : if (a_2(D) > b_3(D))
6211 : goto <bb 5>; [48.89%]
6212 : else
6213 : goto <bb 6>; [51.11%]
6214 :
6215 : <bb 5> [local count: 342865295]:
6216 :
6217 : <bb 6> [local count: 1073741824]:
6218 : and turn it into:
6219 : <bb 2> [local count: 1073741824]:
6220 : _1 = .SPACESHIP (a_2(D), b_3(D), 0);
6221 : if (_1 == 0)
6222 : goto <bb 6>; [34.00%]
6223 : else
6224 : goto <bb 3>; [66.00%]
6225 :
6226 : <bb 3> [local count: 708669601]:
6227 : if (_1 == -1)
6228 : goto <bb 6>; [1.04%]
6229 : else
6230 : goto <bb 4>; [98.96%]
6231 :
6232 : <bb 4> [local count: 701299439]:
6233 : if (_1 == 1)
6234 : goto <bb 5>; [48.89%]
6235 : else
6236 : goto <bb 6>; [51.11%]
6237 :
6238 : <bb 5> [local count: 342865295]:
6239 :
6240 : <bb 6> [local count: 1073741824]:
6241 : so that the backend can emit optimal comparison and
6242 : conditional jump sequence. If the
6243 : <bb 6> [local count: 1073741824]:
6244 : above has a single PHI like:
6245 : # _27 = PHI<0(2), -1(3), -128(4), 1(5)>
6246 : then replace it with effectively
6247 : _1 = .SPACESHIP (a_2(D), b_3(D), -128);
6248 : _27 = _1; */
6249 :
6250 : static void
6251 4220378 : optimize_spaceship (gcond *stmt)
6252 : {
6253 4220378 : enum tree_code code = gimple_cond_code (stmt);
6254 4220378 : if (code != EQ_EXPR && code != NE_EXPR)
6255 4220254 : return;
6256 3422802 : tree arg1 = gimple_cond_lhs (stmt);
6257 3422802 : tree arg2 = gimple_cond_rhs (stmt);
6258 3422802 : if ((!SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg1))
6259 3311810 : && !INTEGRAL_TYPE_P (TREE_TYPE (arg1)))
6260 2646672 : || optab_handler (spaceship_optab,
6261 2646672 : TYPE_MODE (TREE_TYPE (arg1))) == CODE_FOR_nothing
6262 6028998 : || operand_equal_p (arg1, arg2, 0))
6263 : return;
6264 :
6265 2604992 : basic_block bb0 = gimple_bb (stmt), bb1, bb2 = NULL;
6266 2604992 : edge em1 = NULL, e1 = NULL, e2 = NULL;
6267 2604992 : bb1 = EDGE_SUCC (bb0, 1)->dest;
6268 2604992 : if (((EDGE_SUCC (bb0, 0)->flags & EDGE_TRUE_VALUE) != 0) ^ (code == EQ_EXPR))
6269 1574514 : bb1 = EDGE_SUCC (bb0, 0)->dest;
6270 :
6271 9389055 : gcond *g = safe_dyn_cast <gcond *> (*gsi_last_bb (bb1));
6272 1134383 : if (g == NULL
6273 4825899 : || !single_pred_p (bb1)
6274 720932 : || (operand_equal_p (gimple_cond_lhs (g), arg1, 0)
6275 605645 : ? !operand_equal_p (gimple_cond_rhs (g), arg2, 0)
6276 490358 : : (!operand_equal_p (gimple_cond_lhs (g), arg2, 0)
6277 974 : || !operand_equal_p (gimple_cond_rhs (g), arg1, 0)))
6278 616464 : || !cond_only_block_p (bb1))
6279 : return;
6280 :
6281 10092 : enum tree_code ccode = (operand_equal_p (gimple_cond_lhs (g), arg1, 0)
6282 10092 : ? LT_EXPR : GT_EXPR);
6283 10092 : switch (gimple_cond_code (g))
6284 : {
6285 : case LT_EXPR:
6286 : case LE_EXPR:
6287 : break;
6288 8643 : case GT_EXPR:
6289 8643 : case GE_EXPR:
6290 8643 : ccode = ccode == LT_EXPR ? GT_EXPR : LT_EXPR;
6291 : break;
6292 : default:
6293 : return;
6294 : }
6295 :
6296 30204 : for (int i = 0; i < 2; ++i)
6297 : {
6298 : /* With NaNs, </<=/>/>= are false, so we need to look for the
6299 : third comparison on the false edge from whatever non-equality
6300 : comparison the second comparison is. */
6301 20178 : if (HONOR_NANS (TREE_TYPE (arg1))
6302 20178 : && (EDGE_SUCC (bb1, i)->flags & EDGE_TRUE_VALUE) != 0)
6303 131 : continue;
6304 :
6305 20047 : bb2 = EDGE_SUCC (bb1, i)->dest;
6306 59743 : g = safe_dyn_cast <gcond *> (*gsi_last_bb (bb2));
6307 14039 : if (g == NULL
6308 14039 : || !single_pred_p (bb2)
6309 19226 : || (operand_equal_p (gimple_cond_lhs (g), arg1, 0)
6310 11499 : ? !operand_equal_p (gimple_cond_rhs (g), arg2, 0)
6311 3772 : : (!operand_equal_p (gimple_cond_lhs (g), arg2, 0)
6312 15 : || !operand_equal_p (gimple_cond_rhs (g), arg1, 0)))
6313 68 : || !cond_only_block_p (bb2)
6314 11567 : || EDGE_SUCC (bb2, 0)->dest == EDGE_SUCC (bb2, 1)->dest)
6315 19979 : continue;
6316 :
6317 68 : enum tree_code ccode2
6318 68 : = (operand_equal_p (gimple_cond_lhs (g), arg1, 0) ? LT_EXPR : GT_EXPR);
6319 68 : switch (gimple_cond_code (g))
6320 : {
6321 : case LT_EXPR:
6322 : case LE_EXPR:
6323 : break;
6324 41 : case GT_EXPR:
6325 41 : case GE_EXPR:
6326 41 : ccode2 = ccode2 == LT_EXPR ? GT_EXPR : LT_EXPR;
6327 : break;
6328 2 : default:
6329 2 : continue;
6330 : }
6331 66 : if (HONOR_NANS (TREE_TYPE (arg1)) && ccode == ccode2)
6332 0 : continue;
6333 :
6334 132 : if ((ccode == LT_EXPR)
6335 66 : ^ ((EDGE_SUCC (bb1, i)->flags & EDGE_TRUE_VALUE) != 0))
6336 : {
6337 41 : em1 = EDGE_SUCC (bb1, 1 - i);
6338 41 : e1 = EDGE_SUCC (bb2, 0);
6339 41 : e2 = EDGE_SUCC (bb2, 1);
6340 41 : if ((ccode2 == LT_EXPR) ^ ((e1->flags & EDGE_TRUE_VALUE) == 0))
6341 0 : std::swap (e1, e2);
6342 : }
6343 : else
6344 : {
6345 25 : e1 = EDGE_SUCC (bb1, 1 - i);
6346 25 : em1 = EDGE_SUCC (bb2, 0);
6347 25 : e2 = EDGE_SUCC (bb2, 1);
6348 25 : if ((ccode2 != LT_EXPR) ^ ((em1->flags & EDGE_TRUE_VALUE) == 0))
6349 : std::swap (em1, e2);
6350 : }
6351 : break;
6352 : }
6353 :
6354 10067 : if (em1 == NULL)
6355 : {
6356 20052 : if ((ccode == LT_EXPR)
6357 10026 : ^ ((EDGE_SUCC (bb1, 0)->flags & EDGE_TRUE_VALUE) != 0))
6358 : {
6359 3149 : em1 = EDGE_SUCC (bb1, 1);
6360 3149 : e1 = EDGE_SUCC (bb1, 0);
6361 3149 : e2 = (e1->flags & EDGE_TRUE_VALUE) ? em1 : e1;
6362 : }
6363 : else
6364 : {
6365 6877 : em1 = EDGE_SUCC (bb1, 0);
6366 6877 : e1 = EDGE_SUCC (bb1, 1);
6367 6877 : e2 = (e1->flags & EDGE_TRUE_VALUE) ? em1 : e1;
6368 : }
6369 : }
6370 :
6371 : /* Check if there is a single bb into which all failed conditions
6372 : jump to (perhaps through an empty block) and if it results in
6373 : a single integral PHI which just sets it to -1, 0, 1, X
6374 : (or -1, 0, 1 when NaNs can't happen). In that case use 1 rather
6375 : than 0 as last .SPACESHIP argument to tell backends it might
6376 : consider different code generation and just cast the result
6377 : of .SPACESHIP to the PHI result. X above is some value
6378 : other than -1, 0, 1, for libstdc++ -128, for libc++ -127. */
6379 10092 : tree arg3 = integer_zero_node;
6380 10092 : edge e = EDGE_SUCC (bb0, 0);
6381 10092 : if (e->dest == bb1)
6382 7396 : e = EDGE_SUCC (bb0, 1);
6383 10092 : basic_block bbp = e->dest;
6384 10092 : gphi *phi = NULL;
6385 10092 : for (gphi_iterator psi = gsi_start_phis (bbp);
6386 12246 : !gsi_end_p (psi); gsi_next (&psi))
6387 : {
6388 3689 : gphi *gp = psi.phi ();
6389 3689 : tree res = gimple_phi_result (gp);
6390 :
6391 3689 : if (phi != NULL
6392 3339 : || virtual_operand_p (res)
6393 2435 : || !INTEGRAL_TYPE_P (TREE_TYPE (res))
6394 5974 : || TYPE_PRECISION (TREE_TYPE (res)) < 2)
6395 : {
6396 : phi = NULL;
6397 : break;
6398 : }
6399 2154 : phi = gp;
6400 : }
6401 10092 : if (phi
6402 1804 : && integer_zerop (gimple_phi_arg_def_from_edge (phi, e))
6403 10668 : && EDGE_COUNT (bbp->preds) == (HONOR_NANS (TREE_TYPE (arg1)) ? 4 : 3))
6404 : {
6405 122 : HOST_WIDE_INT argval
6406 122 : = SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg1)) ? -128 : -1;
6407 724 : for (unsigned i = 0; phi && i < EDGE_COUNT (bbp->preds) - 1; ++i)
6408 : {
6409 259 : edge e3 = i == 0 ? e1 : i == 1 ? em1 : e2;
6410 259 : if (e3->dest != bbp)
6411 : {
6412 121 : if (!empty_block_p (e3->dest)
6413 112 : || !single_succ_p (e3->dest)
6414 233 : || single_succ (e3->dest) != bbp)
6415 : {
6416 : phi = NULL;
6417 : break;
6418 : }
6419 : e3 = single_succ_edge (e3->dest);
6420 : }
6421 250 : tree a = gimple_phi_arg_def_from_edge (phi, e3);
6422 250 : if (TREE_CODE (a) != INTEGER_CST
6423 250 : || (i == 0 && !integer_onep (a))
6424 494 : || (i == 1 && !integer_all_onesp (a)))
6425 : {
6426 : phi = NULL;
6427 : break;
6428 : }
6429 244 : if (i == 2)
6430 : {
6431 30 : tree minv = TYPE_MIN_VALUE (signed_char_type_node);
6432 30 : tree maxv = TYPE_MAX_VALUE (signed_char_type_node);
6433 30 : widest_int w = widest_int::from (wi::to_wide (a), SIGNED);
6434 41 : if ((w >= -1 && w <= 1)
6435 26 : || w < wi::to_widest (minv)
6436 60 : || w > wi::to_widest (maxv))
6437 : {
6438 4 : phi = NULL;
6439 4 : break;
6440 : }
6441 26 : argval = w.to_shwi ();
6442 26 : }
6443 : }
6444 122 : if (phi)
6445 103 : arg3 = build_int_cst (integer_type_node,
6446 127 : TYPE_UNSIGNED (TREE_TYPE (arg1)) ? 1 : argval);
6447 : }
6448 :
6449 : /* For integral <=> comparisons only use .SPACESHIP if it is turned
6450 : into an integer (-1, 0, 1). */
6451 10092 : if (!SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg1)) && arg3 == integer_zero_node)
6452 : return;
6453 :
6454 227 : gcall *gc = gimple_build_call_internal (IFN_SPACESHIP, 3, arg1, arg2, arg3);
6455 227 : tree lhs = make_ssa_name (integer_type_node);
6456 227 : gimple_call_set_lhs (gc, lhs);
6457 227 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6458 227 : gsi_insert_before (&gsi, gc, GSI_SAME_STMT);
6459 :
6460 351 : wide_int wmin = wi::minus_one (TYPE_PRECISION (integer_type_node));
6461 351 : wide_int wmax = wi::one (TYPE_PRECISION (integer_type_node));
6462 227 : if (HONOR_NANS (TREE_TYPE (arg1)))
6463 : {
6464 131 : if (arg3 == integer_zero_node)
6465 105 : wmin = wi::shwi (-128, TYPE_PRECISION (integer_type_node));
6466 26 : else if (tree_int_cst_sgn (arg3) < 0)
6467 19 : wmin = wi::to_wide (arg3);
6468 : else
6469 7 : wmax = wi::to_wide (arg3);
6470 : }
6471 351 : int_range<1> vr (TREE_TYPE (lhs), wmin, wmax);
6472 227 : set_range_info (lhs, vr);
6473 :
6474 227 : if (arg3 != integer_zero_node)
6475 : {
6476 103 : tree type = TREE_TYPE (gimple_phi_result (phi));
6477 103 : if (!useless_type_conversion_p (type, integer_type_node))
6478 : {
6479 63 : tree tem = make_ssa_name (type);
6480 63 : gimple *gcv = gimple_build_assign (tem, NOP_EXPR, lhs);
6481 63 : gsi_insert_before (&gsi, gcv, GSI_SAME_STMT);
6482 63 : lhs = tem;
6483 : }
6484 103 : SET_PHI_ARG_DEF_ON_EDGE (phi, e, lhs);
6485 103 : gimple_cond_set_lhs (stmt, boolean_false_node);
6486 103 : gimple_cond_set_rhs (stmt, boolean_false_node);
6487 193 : gimple_cond_set_code (stmt, (e->flags & EDGE_TRUE_VALUE)
6488 : ? EQ_EXPR : NE_EXPR);
6489 103 : update_stmt (stmt);
6490 103 : return;
6491 : }
6492 :
6493 124 : gimple_cond_set_lhs (stmt, lhs);
6494 124 : gimple_cond_set_rhs (stmt, integer_zero_node);
6495 124 : update_stmt (stmt);
6496 :
6497 248 : gcond *cond = as_a <gcond *> (*gsi_last_bb (bb1));
6498 124 : gimple_cond_set_lhs (cond, lhs);
6499 124 : if (em1->src == bb1 && e2 != em1)
6500 : {
6501 68 : gimple_cond_set_rhs (cond, integer_minus_one_node);
6502 74 : gimple_cond_set_code (cond, (em1->flags & EDGE_TRUE_VALUE)
6503 : ? EQ_EXPR : NE_EXPR);
6504 : }
6505 : else
6506 : {
6507 56 : gcc_assert (e1->src == bb1 && e2 != e1);
6508 56 : gimple_cond_set_rhs (cond, integer_one_node);
6509 56 : gimple_cond_set_code (cond, (e1->flags & EDGE_TRUE_VALUE)
6510 : ? EQ_EXPR : NE_EXPR);
6511 : }
6512 124 : update_stmt (cond);
6513 :
6514 124 : if (e2 != e1 && e2 != em1)
6515 : {
6516 80 : cond = as_a <gcond *> (*gsi_last_bb (bb2));
6517 40 : gimple_cond_set_lhs (cond, lhs);
6518 40 : if (em1->src == bb2)
6519 25 : gimple_cond_set_rhs (cond, integer_minus_one_node);
6520 : else
6521 : {
6522 15 : gcc_assert (e1->src == bb2);
6523 15 : gimple_cond_set_rhs (cond, integer_one_node);
6524 : }
6525 40 : gimple_cond_set_code (cond,
6526 40 : (e2->flags & EDGE_TRUE_VALUE) ? NE_EXPR : EQ_EXPR);
6527 40 : update_stmt (cond);
6528 : }
6529 : }
6530 :
6531 :
6532 : /* Long-multiply inverse-lowering helper.
6533 :
6534 : The forwprop long-multiply recognizer canonicalizes a hand-written
6535 : longhand high-part multiply into a cast+mult+shift+cast chain
6536 : `(N) ((2N) a * (2N) b) >> N'. When the target lacks an expansion
6537 : path for the wide form, `lower_long_mul_high_chain' resynthesizes
6538 : the longhand at narrow precision via `build_long_mul_partials'. */
6539 :
6540 : /* Test whether the target supports an (HALF)-by-(HALF)->NARROW unsigned
6541 : widening multiply. Returns true on success, with the half-width
6542 : scalar int mode placed in *HALF_MODE. */
6543 :
6544 : static bool
6545 2231 : can_widen_to_narrow_p (scalar_int_mode narrow_mode, unsigned int half_width,
6546 : scalar_int_mode *half_mode)
6547 : {
6548 2231 : if (!int_mode_for_size (half_width, 0).exists (half_mode))
6549 0 : return false;
6550 2231 : return convert_optab_handler (umul_widen_optab, narrow_mode, *half_mode)
6551 2231 : != CODE_FOR_nothing;
6552 : }
6553 :
6554 : /* Append to *SEQ the operand split and partial products for an unsigned
6555 : long multiply of OP1 by OP2 at the precision of TREE_TYPE (OP1).
6556 : HALF_TYPE is the (N/2)-bit unsigned type; HALF_AMT is the integer-typed
6557 : shift constant equal to N/2.
6558 :
6559 : Outputs the four partial products via *LOLO, *HILO, *LOHI, *HIHI.
6560 :
6561 : USE_WIDEN selects the partial-product form:
6562 : true - cast halves to HALF_TYPE and use WIDEN_MULT_EXPR (needs
6563 : an (N/2)-by-(N/2)->N widening multiply optab).
6564 : false - mask/shift halves within the N-bit accumulator and use
6565 : plain MULT_EXPR; the halves fit in N/2 bits so the N-bit
6566 : low product is exact. */
6567 :
6568 : static void
6569 2231 : build_long_mul_partials (gimple_seq *seq, location_t loc, tree op1, tree op2,
6570 : tree half_type, tree half_amt,
6571 : tree *lolo, tree *hilo, tree *lohi, tree *hihi,
6572 : bool use_widen)
6573 : {
6574 2231 : tree acc_type = TREE_TYPE (op1);
6575 2231 : tree op1_hi = gimple_build (seq, loc, RSHIFT_EXPR, acc_type, op1, half_amt);
6576 2231 : tree op2_hi = gimple_build (seq, loc, RSHIFT_EXPR, acc_type, op2, half_amt);
6577 2231 : tree op1_lo, op2_lo;
6578 2231 : tree_code mul_code;
6579 :
6580 2231 : if (use_widen)
6581 : {
6582 2231 : op1_lo = gimple_build (seq, loc, NOP_EXPR, half_type, op1);
6583 2231 : op2_lo = gimple_build (seq, loc, NOP_EXPR, half_type, op2);
6584 2231 : op1_hi = gimple_build (seq, loc, NOP_EXPR, half_type, op1_hi);
6585 2231 : op2_hi = gimple_build (seq, loc, NOP_EXPR, half_type, op2_hi);
6586 2231 : mul_code = WIDEN_MULT_EXPR;
6587 : }
6588 : else
6589 : {
6590 0 : tree mask = wide_int_to_tree (acc_type,
6591 0 : wi::mask (TYPE_PRECISION (half_type), false,
6592 0 : TYPE_PRECISION (acc_type)));
6593 0 : op1_lo = gimple_build (seq, loc, BIT_AND_EXPR, acc_type, op1, mask);
6594 0 : op2_lo = gimple_build (seq, loc, BIT_AND_EXPR, acc_type, op2, mask);
6595 0 : mul_code = MULT_EXPR;
6596 : }
6597 :
6598 2231 : *lolo = gimple_build (seq, loc, mul_code, acc_type, op1_lo, op2_lo);
6599 2231 : *hilo = gimple_build (seq, loc, mul_code, acc_type, op1_hi, op2_lo);
6600 2231 : *lohi = gimple_build (seq, loc, mul_code, acc_type, op1_lo, op2_hi);
6601 2231 : *hihi = gimple_build (seq, loc, mul_code, acc_type, op1_hi, op2_hi);
6602 2231 : }
6603 :
6604 : /* Emit into *SEQ the high N bits of the unsigned product A * B, where A and B
6605 : are NARROW_TYPE (N-bit) values, as a longhand over (N/2)-bit partials.
6606 : Returns the high-part SSA. */
6607 :
6608 : static tree
6609 2231 : emit_long_mul_highpart (gimple_seq *seq, location_t loc, tree a, tree b,
6610 : tree narrow_type)
6611 : {
6612 2231 : scalar_int_mode narrow_mode
6613 2231 : = as_a <scalar_int_mode> (TYPE_MODE (narrow_type));
6614 2231 : unsigned int half_width = GET_MODE_PRECISION (narrow_mode) / 2;
6615 : /* Prefer (N/2)-by-(N/2)->N widening partials; fall back to plain MULT_EXPR
6616 : when the target lacks the widen optab. See build_long_mul_partials. */
6617 2231 : scalar_int_mode half_mode;
6618 2231 : bool use_widen = can_widen_to_narrow_p (narrow_mode, half_width, &half_mode);
6619 2231 : tree half_type = build_nonstandard_integer_type (half_width, 1);
6620 2231 : tree half_amt = build_int_cst (integer_type_node, half_width);
6621 2231 : tree half_mask = wide_int_to_tree (narrow_type,
6622 2231 : wi::mask (half_width, false,
6623 2231 : TYPE_PRECISION (narrow_type)));
6624 :
6625 2231 : tree lolo, hilo, lohi, hihi;
6626 2231 : build_long_mul_partials (seq, loc, a, b, half_type, half_amt,
6627 : &lolo, &hilo, &lohi, &hihi, use_widen);
6628 2231 : tree cross_sum = gimple_build (seq, loc, PLUS_EXPR, narrow_type, hilo, lohi);
6629 2231 : tree cross_lt = gimple_build (seq, loc, LT_EXPR, boolean_type_node,
6630 : cross_sum, hilo);
6631 2231 : tree cross_lt_n = gimple_build (seq, loc, NOP_EXPR, narrow_type, cross_lt);
6632 2231 : tree cross_carry = gimple_build (seq, loc, LSHIFT_EXPR, narrow_type,
6633 : cross_lt_n, half_amt);
6634 2231 : tree lolo_hi = gimple_build (seq, loc, RSHIFT_EXPR, narrow_type,
6635 : lolo, half_amt);
6636 2231 : tree cross_lo = gimple_build (seq, loc, BIT_AND_EXPR, narrow_type,
6637 : cross_sum, half_mask);
6638 2231 : tree low_accum = gimple_build (seq, loc, PLUS_EXPR, narrow_type,
6639 : lolo_hi, cross_lo);
6640 2231 : tree low_accum_hi = gimple_build (seq, loc, RSHIFT_EXPR, narrow_type,
6641 : low_accum, half_amt);
6642 2231 : tree cross_hi = gimple_build (seq, loc, RSHIFT_EXPR, narrow_type,
6643 : cross_sum, half_amt);
6644 2231 : tree t1 = gimple_build (seq, loc, PLUS_EXPR, narrow_type, hihi, cross_hi);
6645 2231 : tree t2 = gimple_build (seq, loc, PLUS_EXPR, narrow_type, t1, low_accum_hi);
6646 2231 : return gimple_build (seq, loc, PLUS_EXPR, narrow_type, t2, cross_carry);
6647 : }
6648 :
6649 : /* Emit into *SEQ the high N bits (NARROW_TYPE) of the unsigned product of two
6650 : 2N-bit values given as N-bit halves, x = L1 + H1*2^N and y = L2 + H2*2^N:
6651 : the high half of x*y is the high N bits of L1*L2, plus H1*L2 and L1*H2, all
6652 : mod 2^N. */
6653 :
6654 : static tree
6655 2231 : combine_long_mul_halves (gimple_seq *seq, location_t loc, tree l1, tree h1,
6656 : tree l2, tree h2, tree narrow_type)
6657 : {
6658 2231 : tree hh = emit_long_mul_highpart (seq, loc, l1, l2, narrow_type);
6659 2231 : tree c1 = gimple_build (seq, loc, MULT_EXPR, narrow_type, h1, l2);
6660 2231 : tree c2 = gimple_build (seq, loc, MULT_EXPR, narrow_type, l1, h2);
6661 2231 : tree s = gimple_build (seq, loc, PLUS_EXPR, narrow_type, hh, c1);
6662 2231 : return gimple_build (seq, loc, PLUS_EXPR, narrow_type, s, c2);
6663 : }
6664 :
6665 : /* True when OP fits NARROW_PREC bits as an unsigned value. Looks
6666 : through widening casts and PHIs, falling back to `tree_nonzero_bits'
6667 : otherwise. PHI_SEEN guards against cycles. */
6668 :
6669 : static bool
6670 929 : long_mul_op_fits_p (tree op, unsigned narrow_prec, bitmap phi_seen)
6671 : {
6672 1199 : if (!TYPE_UNSIGNED (TREE_TYPE (op)))
6673 : return false;
6674 1199 : if (TYPE_PRECISION (TREE_TYPE (op)) <= narrow_prec)
6675 : return true;
6676 929 : if (TREE_CODE (op) == SSA_NAME)
6677 : {
6678 357 : gimple *def = SSA_NAME_DEF_STMT (op);
6679 357 : if (is_gimple_assign (def)
6680 357 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
6681 270 : return long_mul_op_fits_p (gimple_assign_rhs1 (def), narrow_prec,
6682 270 : phi_seen);
6683 87 : if (gphi *phi = dyn_cast <gphi *> (def))
6684 69 : if (bitmap_set_bit (phi_seen, SSA_NAME_VERSION (op)))
6685 : {
6686 404 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
6687 335 : if (!long_mul_op_fits_p (gimple_phi_arg_def (phi, i),
6688 : narrow_prec, phi_seen))
6689 : return false;
6690 : return true;
6691 : }
6692 : }
6693 590 : return wi::min_precision (tree_nonzero_bits (op), UNSIGNED) <= narrow_prec;
6694 : }
6695 :
6696 : static bool long_mul_split_operand (gimple_seq *, location_t, tree, tree,
6697 : tree *, tree *);
6698 :
6699 : struct long_mul_halves
6700 : {
6701 : tree lo;
6702 : tree hi;
6703 : };
6704 :
6705 : /* Halves recorded for one run of the pass, keyed on the PHI they came from.
6706 : Several chains can reach one operand PHI, and each that splits it again
6707 : leaves another redundant pair of half PHIs behind. */
6708 :
6709 : static hash_map<tree, long_mul_halves> *long_mul_phi_halves;
6710 :
6711 : struct long_mul_arg_split
6712 : {
6713 : gimple_seq seq;
6714 : tree lo;
6715 : tree hi;
6716 : };
6717 :
6718 : /* Split the 2N-bit result of PHI into N-bit halves *LO and *HI, by splitting
6719 : each argument and merging the halves with two new PHIs. A split goes at
6720 : the end of its argument's incoming block, where the argument is available,
6721 : rather than on the edge, which could split a critical edge while the
6722 : dominator walk is still running. Returns false, having changed nothing,
6723 : when an argument cannot be split. A split that succeeds stands even if
6724 : the caller then gives up. */
6725 :
6726 : static bool
6727 0 : long_mul_split_phi (gphi *phi, tree narrow_type, tree *lo, tree *hi)
6728 : {
6729 0 : tree res = gimple_phi_result (phi);
6730 0 : if (long_mul_halves *prev = long_mul_phi_halves->get (res))
6731 : {
6732 : /* Every operand reaching a split has the 2N type, so the width is
6733 : fixed by the PHI's own type. */
6734 0 : gcc_checking_assert (types_compatible_p (TREE_TYPE (prev->lo),
6735 : narrow_type));
6736 0 : *lo = prev->lo;
6737 0 : *hi = prev->hi;
6738 0 : return true;
6739 : }
6740 :
6741 0 : unsigned int n = gimple_phi_num_args (phi);
6742 0 : location_t loc = gimple_location (phi);
6743 0 : basic_block bb = gimple_bb (phi);
6744 0 : auto_vec<long_mul_arg_split, 4> args;
6745 :
6746 0 : for (unsigned int i = 0; i < n; i++)
6747 : {
6748 0 : edge e = gimple_phi_arg_edge (phi, i);
6749 : /* A back edge could lead back to PHI and recurse forever. The entry
6750 : block cannot hold a split. */
6751 0 : if (dominated_by_p (CDI_DOMINATORS, e->src, bb)
6752 0 : || e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
6753 0 : return false;
6754 :
6755 0 : long_mul_arg_split arg = {};
6756 0 : if (!long_mul_split_operand (&arg.seq, loc, gimple_phi_arg_def (phi, i),
6757 : narrow_type, &arg.lo, &arg.hi))
6758 : return false;
6759 :
6760 0 : args.safe_push (arg);
6761 : }
6762 :
6763 : /* Every argument split, so the rewrite can be committed. */
6764 0 : gphi *lo_phi = create_phi_node (make_ssa_name (narrow_type), bb);
6765 0 : gphi *hi_phi = create_phi_node (make_ssa_name (narrow_type), bb);
6766 0 : for (unsigned int i = 0; i < n; i++)
6767 : {
6768 0 : edge e = gimple_phi_arg_edge (phi, i);
6769 0 : if (args[i].seq)
6770 : {
6771 0 : gimple_stmt_iterator gsi = gsi_last_bb (e->src);
6772 0 : if (!gsi_end_p (gsi) && stmt_ends_bb_p (gsi_stmt (gsi)))
6773 0 : gsi_insert_seq_before (&gsi, args[i].seq, GSI_SAME_STMT);
6774 : else
6775 0 : gsi_insert_seq_after (&gsi, args[i].seq, GSI_CONTINUE_LINKING);
6776 : }
6777 0 : add_phi_arg (lo_phi, args[i].lo, e, UNKNOWN_LOCATION);
6778 0 : add_phi_arg (hi_phi, args[i].hi, e, UNKNOWN_LOCATION);
6779 : }
6780 0 : *lo = gimple_phi_result (lo_phi);
6781 0 : *hi = gimple_phi_result (hi_phi);
6782 0 : long_mul_phi_halves->put (res, { *lo, *hi });
6783 :
6784 0 : if (dump_file && (dump_flags & TDF_DETAILS))
6785 0 : fprintf (dump_file, "Split long-multiply operand PHI.\n");
6786 : return true;
6787 0 : }
6788 :
6789 : /* Split the 2N-bit unsigned value OP into its low and high N bits (*LO and
6790 : *HI, both NARROW_TYPE) using only N-bit operations, as the target has no 2N
6791 : multiply or shift. A 2N product recurses on its operands, its high half
6792 : coming from combine_long_mul_halves. A value shifted down by N recurses on
6793 : the shifted value and takes its high half, rather than reading the 2N shift.
6794 : A widening cast's low half is the truncated source and its high half is what
6795 : the cast extended with, zero or the source's replicated sign bit. A value
6796 : that provably fits N bits has a zero high half. A PHI is split through its
6797 : arguments as a last resort. Returns false otherwise. */
6798 :
6799 : static bool
6800 4470 : long_mul_split_operand (gimple_seq *seq, location_t loc, tree op,
6801 : tree narrow_type, tree *lo, tree *hi)
6802 : {
6803 4470 : unsigned int narrow_prec = TYPE_PRECISION (narrow_type);
6804 4470 : if (TREE_CODE (op) == SSA_NAME)
6805 : {
6806 3943 : gimple *def = SSA_NAME_DEF_STMT (op);
6807 3943 : if (is_gimple_assign (def) && gimple_assign_rhs_code (def) == MULT_EXPR)
6808 : {
6809 13 : tree a_lo, a_hi, b_lo, b_hi;
6810 13 : if (!long_mul_split_operand (seq, loc, gimple_assign_rhs1 (def),
6811 : narrow_type, &a_lo, &a_hi)
6812 13 : || !long_mul_split_operand (seq, loc, gimple_assign_rhs2 (def),
6813 : narrow_type, &b_lo, &b_hi))
6814 : return false;
6815 13 : *lo = gimple_build (seq, loc, MULT_EXPR, narrow_type, a_lo, b_lo);
6816 13 : *hi = combine_long_mul_halves (seq, loc, a_lo, a_hi, b_lo, b_hi,
6817 : narrow_type);
6818 13 : return true;
6819 : }
6820 : /* A 2N value shifted down by N is its own high half: split the source
6821 : and use that half. Reading the shift instead leaves the 2N source
6822 : live, and the target cannot expand it. This has to come before the
6823 : widening-cast case below, which would take such a value as it
6824 : stands. */
6825 3930 : if (is_gimple_assign (def)
6826 3881 : && gimple_assign_rhs_code (def) == RSHIFT_EXPR
6827 8 : && tree_fits_uhwi_p (gimple_assign_rhs2 (def))
6828 3938 : && tree_to_uhwi (gimple_assign_rhs2 (def)) == narrow_prec)
6829 : {
6830 8 : tree src_lo, src_hi;
6831 8 : if (!long_mul_split_operand (seq, loc, gimple_assign_rhs1 (def),
6832 : narrow_type, &src_lo, &src_hi))
6833 : return false;
6834 8 : *lo = src_hi;
6835 8 : *hi = build_zero_cst (narrow_type);
6836 8 : return true;
6837 : }
6838 3922 : if (is_gimple_assign (def)
6839 3922 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
6840 : {
6841 3855 : tree src = gimple_assign_rhs1 (def);
6842 3855 : tree src_type = TREE_TYPE (src);
6843 3855 : if (INTEGRAL_TYPE_P (src_type)
6844 3855 : && TYPE_PRECISION (src_type) <= narrow_prec)
6845 : {
6846 3855 : *lo = gimple_convert (seq, loc, narrow_type, src);
6847 3855 : if (TYPE_UNSIGNED (src_type))
6848 3840 : *hi = build_zero_cst (narrow_type);
6849 : else
6850 : {
6851 : /* Sign extension: the high N bits replicate the sign bit. */
6852 15 : tree snarrow = signed_type_for (narrow_type);
6853 15 : tree s = gimple_convert (seq, loc, snarrow, *lo);
6854 15 : tree amt = build_int_cst (integer_type_node, narrow_prec - 1);
6855 15 : tree sh = gimple_build (seq, loc, RSHIFT_EXPR, snarrow, s,
6856 : amt);
6857 15 : *hi = gimple_convert (seq, loc, narrow_type, sh);
6858 : }
6859 3855 : return true;
6860 : }
6861 : }
6862 : }
6863 :
6864 : /* A value provably within N bits: its low half is the truncation to N bits
6865 : (a subreg, not a 2N shift), its high half is zero. */
6866 594 : auto_bitmap phi_seen;
6867 594 : if (long_mul_op_fits_p (op, narrow_prec, phi_seen))
6868 : {
6869 594 : *lo = gimple_convert (seq, loc, narrow_type, op);
6870 594 : *hi = build_zero_cst (narrow_type);
6871 594 : return true;
6872 : }
6873 :
6874 : /* A PHI that does not fit N bits can still be split through its arguments,
6875 : which is how a sign-extended value reaches the cast case above. */
6876 0 : if (TREE_CODE (op) == SSA_NAME)
6877 594 : if (gphi *phi = dyn_cast <gphi *> (SSA_NAME_DEF_STMT (op)))
6878 0 : return long_mul_split_phi (phi, narrow_type, lo, hi);
6879 : return false;
6880 594 : }
6881 :
6882 : /* Collect into HIGH_USES the uses of PROD forming its high half,
6883 : `PROD >> NARROW_PREC'. A use reading only the low NARROW_PREC bits is
6884 : accepted but not collected. Returns false on any other use. */
6885 :
6886 : static bool
6887 2244 : long_mul_high_half_uses (tree prod, unsigned int narrow_prec,
6888 : vec<gimple *> *high_uses)
6889 : {
6890 2244 : imm_use_iterator iui;
6891 2244 : gimple *use_stmt;
6892 4486 : FOR_EACH_IMM_USE_STMT (use_stmt, iui, prod)
6893 : {
6894 2247 : if (is_gimple_debug (use_stmt))
6895 0 : continue;
6896 2247 : if (!is_gimple_assign (use_stmt))
6897 : return false;
6898 2247 : tree_code code = gimple_assign_rhs_code (use_stmt);
6899 2247 : if (code == RSHIFT_EXPR
6900 2226 : && gimple_assign_rhs1 (use_stmt) == prod
6901 2226 : && tree_fits_uhwi_p (gimple_assign_rhs2 (use_stmt))
6902 4473 : && tree_to_uhwi (gimple_assign_rhs2 (use_stmt)) == narrow_prec)
6903 2226 : high_uses->safe_push (use_stmt);
6904 21 : else if (CONVERT_EXPR_CODE_P (code))
6905 : {
6906 0 : tree t = TREE_TYPE (gimple_assign_lhs (use_stmt));
6907 0 : if (!INTEGRAL_TYPE_P (t) || TYPE_PRECISION (t) > narrow_prec)
6908 : return false;
6909 : }
6910 21 : else if (code == BIT_AND_EXPR
6911 21 : && TREE_CODE (gimple_assign_rhs2 (use_stmt)) == INTEGER_CST)
6912 : {
6913 16 : if (wi::min_precision (wi::to_wide (gimple_assign_rhs2 (use_stmt)),
6914 : UNSIGNED) > narrow_prec)
6915 : return false;
6916 : }
6917 : else
6918 : return false;
6919 5 : }
6920 2239 : return true;
6921 : }
6922 :
6923 : /* True when every use of PROD reads only its low NARROW_PREC bits. */
6924 :
6925 : static bool
6926 2236 : long_mul_only_low_half_used_p (tree prod, unsigned int narrow_prec)
6927 : {
6928 2236 : auto_vec<gimple *, 4> high_uses;
6929 2236 : return (long_mul_high_half_uses (prod, narrow_prec, &high_uses)
6930 4472 : && high_uses.is_empty ());
6931 2236 : }
6932 :
6933 : /* True when STMT is res = a * b whose unsigned 2N-bit result is in a mode the
6934 : target cannot multiply, having neither insn nor libcall, so that expand_mult
6935 : would abort; set *NARROW_TYPE to the N-bit unsigned type its halves are
6936 : built at. A mode the target does support is left to convert_mult_to_widen
6937 : and convert_mult_to_highpart. */
6938 :
6939 : static bool
6940 88733899 : unexpandable_long_mul_p (gimple *stmt, tree *narrow_type)
6941 : {
6942 88733899 : if (!is_gimple_assign (stmt) || gimple_assign_rhs_code (stmt) != MULT_EXPR)
6943 : return false;
6944 :
6945 1478775 : tree wide_type = TREE_TYPE (gimple_assign_lhs (stmt));
6946 1478775 : scalar_int_mode wide_mode;
6947 1478775 : if (!INTEGRAL_TYPE_P (wide_type)
6948 1221961 : || !TYPE_UNSIGNED (wide_type)
6949 903054 : || !is_a <scalar_int_mode> (TYPE_MODE (wide_type), &wide_mode)
6950 2700736 : || targetm.scalar_mode_supported_p (wide_mode))
6951 : return false;
6952 :
6953 2244 : *narrow_type
6954 2244 : = build_nonstandard_integer_type (TYPE_PRECISION (wide_type) / 2, 1);
6955 2244 : return true;
6956 : }
6957 :
6958 : static bool narrow_long_mul_low_half (gimple_stmt_iterator *);
6959 :
6960 : /* OP1 and OP2 are the operands of a 2N multiply just narrowed or lowered;
6961 : that rewrite now reads each through an N-bit low-half cast. An operand
6962 : defined by another 2N multiply can thereby become low-half-only -- narrow
6963 : it too, recursing through chained wide products such as (a*b)*c. */
6964 :
6965 : static void
6966 2215 : narrow_long_mul_operands (tree op1, tree op2)
6967 : {
6968 6645 : for (tree op : { op1, op2 })
6969 4430 : if (TREE_CODE (op) == SSA_NAME)
6970 : {
6971 3903 : gimple *def = SSA_NAME_DEF_STMT (op);
6972 3903 : if (is_gimple_assign (def) && gimple_assign_rhs_code (def) == MULT_EXPR)
6973 : {
6974 5 : gimple_stmt_iterator dgsi = gsi_for_stmt (def);
6975 5 : narrow_long_mul_low_half (&dgsi);
6976 : }
6977 : }
6978 2215 : }
6979 :
6980 : /* If the statement at *GSI is res = a * b with a 2N-bit unsigned result the
6981 : target cannot multiply and every use reads only the low N bits, narrow it
6982 : to res = (2N) ((N) a * (N) b) and return true. The low N bits of a product
6983 : depend only on the low N bits of the operands, so this preserves every use;
6984 : the unused high half becomes zero. match.pd's shorten rule omits this for
6985 : MULT_EXPR. */
6986 :
6987 : static bool
6988 753736 : narrow_long_mul_low_half (gimple_stmt_iterator *gsi)
6989 : {
6990 753736 : gimple *stmt = gsi_stmt (*gsi);
6991 753736 : tree narrow_type;
6992 753736 : if (!unexpandable_long_mul_p (stmt, &narrow_type))
6993 : return false;
6994 :
6995 2236 : tree lhs = gimple_assign_lhs (stmt);
6996 2236 : if (!long_mul_only_low_half_used_p (lhs, TYPE_PRECISION (narrow_type)))
6997 : return false;
6998 :
6999 13 : tree op1 = gimple_assign_rhs1 (stmt);
7000 13 : tree op2 = gimple_assign_rhs2 (stmt);
7001 13 : location_t loc = gimple_location (stmt);
7002 13 : gimple_seq seq = NULL;
7003 13 : tree a = gimple_convert (&seq, loc, narrow_type, op1);
7004 13 : tree b = gimple_convert (&seq, loc, narrow_type, op2);
7005 13 : tree np = gimple_build (&seq, loc, MULT_EXPR, narrow_type, a, b);
7006 13 : gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
7007 13 : gimple *conv = gimple_build_assign (lhs, NOP_EXPR, np);
7008 13 : gimple_set_location (conv, loc);
7009 13 : gsi_replace (gsi, conv, true);
7010 :
7011 13 : if (dump_file && (dump_flags & TDF_DETAILS))
7012 0 : fprintf (dump_file, "Narrowed low-half-only long multiply.\n");
7013 :
7014 13 : narrow_long_mul_operands (op1, op2);
7015 13 : return true;
7016 : }
7017 :
7018 : /* The 2N multiply at *GSI has had its high half synthesized elsewhere, so any
7019 : use left reads only its low half: narrow it in place, or remove it when it
7020 : has no use at all. Removing it drops a use of each operand, so a 2N
7021 : multiply defining one may become low-half-only. Narrow those operands. */
7022 :
7023 : static void
7024 2218 : finish_long_mul_low_half (gimple_stmt_iterator *gsi)
7025 : {
7026 2218 : gimple *stmt = gsi_stmt (*gsi);
7027 :
7028 2218 : if (has_zero_uses (gimple_assign_lhs (stmt)))
7029 : {
7030 2202 : tree op1 = gimple_assign_rhs1 (stmt);
7031 2202 : tree op2 = gimple_assign_rhs2 (stmt);
7032 2202 : gsi_remove (gsi, true);
7033 2202 : release_defs (stmt);
7034 2202 : narrow_long_mul_operands (op1, op2);
7035 2202 : return;
7036 : }
7037 :
7038 16 : narrow_long_mul_low_half (gsi);
7039 : }
7040 :
7041 : /* Rewrite the multiply STMT into the N-bit halves its uses read, when the
7042 : target can neither multiply at 2N bits nor form an N-bit high part.
7043 : Returns true on a rewrite, which may remove STMT.
7044 :
7045 : Runs once lower_long_mul_high_chain has been applied to every statement:
7046 : both rewrite products with a high half, and this one, keyed on the
7047 : definition rather than on a consumer, would otherwise pre-empt it. What
7048 : reaches it is a product that lowering could not retire, its high half also
7049 : read as an operand of another product, or read alongside the low half. A
7050 : product read only for its low half is left to narrow_long_mul_low_half. */
7051 :
7052 : static bool
7053 87980163 : narrow_long_mul_halves (gimple *stmt)
7054 : {
7055 87980163 : tree narrow_type;
7056 87980163 : if (!unexpandable_long_mul_p (stmt, &narrow_type))
7057 : return false;
7058 :
7059 : /* Only worth lowering where the target cannot form the N-bit high part. */
7060 8 : scalar_int_mode narrow_mode;
7061 8 : if (!is_a <scalar_int_mode> (TYPE_MODE (narrow_type), &narrow_mode)
7062 8 : || can_mult_highpart_p (narrow_mode, true))
7063 : return false;
7064 :
7065 8 : tree lhs = gimple_assign_lhs (stmt);
7066 8 : auto_vec<gimple *, 4> high_uses;
7067 8 : if (!long_mul_high_half_uses (lhs, TYPE_PRECISION (narrow_type), &high_uses)
7068 16 : || high_uses.is_empty ())
7069 : return false;
7070 :
7071 8 : tree op1 = gimple_assign_rhs1 (stmt);
7072 8 : tree op2 = gimple_assign_rhs2 (stmt);
7073 8 : location_t loc = gimple_location (stmt);
7074 8 : gimple_seq seq = NULL;
7075 8 : tree l1, h1, l2, h2;
7076 8 : if (!long_mul_split_operand (&seq, loc, op1, narrow_type, &l1, &h1)
7077 8 : || !long_mul_split_operand (&seq, loc, op2, narrow_type, &l2, &h2))
7078 : return false;
7079 8 : tree hi = combine_long_mul_halves (&seq, loc, l1, h1, l2, h2, narrow_type);
7080 : /* The high part is < 2^N, so widening it back to 2N leaves every use of a
7081 : shift, full width or truncated, reading the same value. */
7082 8 : tree hi_wide = gimple_convert (&seq, loc, TREE_TYPE (lhs), hi);
7083 8 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
7084 8 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
7085 :
7086 8 : unsigned int i;
7087 8 : gimple *shift_stmt;
7088 24 : FOR_EACH_VEC_ELT (high_uses, i, shift_stmt)
7089 : {
7090 8 : gimple_stmt_iterator sgsi = gsi_for_stmt (shift_stmt);
7091 8 : gimple *conv = gimple_build_assign (gimple_assign_lhs (shift_stmt),
7092 : hi_wide);
7093 8 : gimple_set_location (conv, loc);
7094 8 : gsi_replace (&sgsi, conv, true);
7095 : }
7096 :
7097 8 : if (dump_file && (dump_flags & TDF_DETAILS))
7098 0 : fprintf (dump_file, "Narrowed high half of long multiply.\n");
7099 :
7100 8 : finish_long_mul_low_half (&gsi);
7101 8 : return true;
7102 8 : }
7103 :
7104 : /* Match.pd recognizer for the long-multiply recognizer's high-part
7105 : emit chain. */
7106 :
7107 : extern bool gimple_long_mul_high_chain (tree, tree *, tree (*)(tree));
7108 :
7109 : /* Rewrite the `long_mul_high_chain' whose tail is the statement at GSI
7110 :
7111 : wide_a = (T_2N) op1
7112 : wide_b = (T_2N) op2
7113 : wide_prod = wide_a * wide_b
7114 : hi = wide_prod >> N
7115 : lhs = (convert) hi
7116 :
7117 : to a longhand high-part synthesis at T_N precision. Never materializes
7118 : T_2N in gimple, so it covers cases where the 2N mode has no expansion path
7119 : (e.g. the high 128 bits of a 128x128 product where 2N=OImode). An operand
7120 : wider than T_N -- a shared wide product or a sign-extended cast -- is split
7121 : into T_N halves rather than truncated, so no high input bits are dropped.
7122 : Returns true on a rewrite. */
7123 :
7124 : static bool
7125 2462289 : lower_long_mul_high_chain (gimple_stmt_iterator *gsi)
7126 : {
7127 2462289 : gimple *trunc_stmt = gsi_stmt (*gsi);
7128 2462289 : if (!is_gimple_assign (trunc_stmt))
7129 : return false;
7130 :
7131 2462289 : tree narrow_lhs = gimple_assign_lhs (trunc_stmt);
7132 2462289 : tree ops[2];
7133 2462289 : if (!gimple_long_mul_high_chain (narrow_lhs, ops, NULL))
7134 : return false;
7135 :
7136 : /* Walk the matched chain back to the 2N multiply and take narrow_type at
7137 : half its precision. */
7138 2475 : gimple *shift_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (trunc_stmt));
7139 2475 : gimple *mult_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (shift_stmt));
7140 2475 : unsigned int narrow_prec
7141 2475 : = TYPE_PRECISION (TREE_TYPE (gimple_assign_lhs (mult_stmt))) / 2;
7142 2475 : tree narrow_type = build_nonstandard_integer_type (narrow_prec, /*uns=*/1);
7143 2475 : scalar_int_mode narrow_mode;
7144 2475 : if (!is_a <scalar_int_mode> (TYPE_MODE (narrow_type), &narrow_mode))
7145 : return false;
7146 :
7147 : /* Lower only when the target cannot form the N-bit high part itself. */
7148 2475 : if (can_mult_highpart_p (narrow_mode, true))
7149 : return false;
7150 :
7151 2210 : location_t loc = gimple_location (trunc_stmt);
7152 2210 : gimple_seq seq = NULL;
7153 :
7154 : /* Split each operand into N-bit halves and combine. An operand that fits
7155 : N bits yields h == 0, so its cross term folds away; with both fitting
7156 : the combine is just a plain N-bit high part. */
7157 2210 : tree l1, h1, l2, h2;
7158 2210 : if (!long_mul_split_operand (&seq, loc, gimple_assign_rhs1 (mult_stmt),
7159 : narrow_type, &l1, &h1)
7160 2210 : || !long_mul_split_operand (&seq, loc, gimple_assign_rhs2 (mult_stmt),
7161 : narrow_type, &l2, &h2))
7162 : return false;
7163 2210 : tree hi = combine_long_mul_halves (&seq, loc, l1, h1, l2, h2, narrow_type);
7164 :
7165 : /* Merging the chain's truncation with a later user cast can retarget the
7166 : outer convert to any integral type, so convert the narrow result once
7167 : here (the high part is < 2^N, so the conversion preserves it). */
7168 2210 : gimple *result_stmt;
7169 2210 : tree lhs_type = TREE_TYPE (narrow_lhs);
7170 2210 : if (useless_type_conversion_p (lhs_type, narrow_type))
7171 2204 : result_stmt = gimple_build_assign (narrow_lhs, hi);
7172 : else
7173 6 : result_stmt = gimple_build_assign (narrow_lhs, NOP_EXPR, hi);
7174 2210 : gimple_set_location (result_stmt, loc);
7175 2210 : gimple_seq_add_stmt (&seq, result_stmt);
7176 :
7177 2210 : gsi_replace_with_seq (gsi, seq, true);
7178 :
7179 : /* Clean up the shift and the 2N mult now -- LTRANS runs no DCE between
7180 : widening_mul and expand, and a dead 2N mult would abort expand_mult.
7181 : Dead upstream (T_2N) casts, if any, are harmless NOP_EXPRs and land
7182 : with normal DCE. */
7183 2210 : if (has_zero_uses (gimple_assign_lhs (shift_stmt)))
7184 : {
7185 2202 : gimple_stmt_iterator dgsi = gsi_for_stmt (shift_stmt);
7186 2202 : gsi_remove (&dgsi, true);
7187 2202 : release_defs (shift_stmt);
7188 : }
7189 :
7190 : /* The mult is either dead (low half recomputed elsewhere) or now read only
7191 : for its low half. */
7192 2210 : gimple_stmt_iterator mgsi = gsi_for_stmt (mult_stmt);
7193 2210 : finish_long_mul_low_half (&mgsi);
7194 :
7195 2210 : if (dump_file && (dump_flags & TDF_DETAILS))
7196 4 : fprintf (dump_file, "Lowered long-mul high-part chain.\n");
7197 : return true;
7198 : }
7199 :
7200 : /* True when pass_optimize_widening_mul will run. Shared with the
7201 : forwprop long-multiply recognizer so its wide-chain emit stays
7202 : paired with the lowering that rescues an unsupported 2N shape.
7203 : The -Og pipeline (pass_all_optimizations_g) does not contain
7204 : pass_optimize_widening_mul at all, so -Og -fexpensive-optimizations
7205 : must not enable the emit: the unlowered 2N multiply would reach
7206 : expand as an unexpandable mode (e.g. OImode) and ICE.
7207 : -fdisable-tree-widening_mul is not observed. */
7208 :
7209 : bool
7210 1079108 : optimize_widening_mul_active_p (void)
7211 : {
7212 1079108 : return flag_expensive_optimizations && optimize && !optimize_debug;
7213 : }
7214 :
7215 : /* Find integer multiplications where the operands are extended from
7216 : smaller types, and replace the MULT_EXPR with a WIDEN_MULT_EXPR
7217 : or MULT_HIGHPART_EXPR where appropriate. */
7218 :
7219 : namespace {
7220 :
7221 : const pass_data pass_data_optimize_widening_mul =
7222 : {
7223 : GIMPLE_PASS, /* type */
7224 : "widening_mul", /* name */
7225 : OPTGROUP_NONE, /* optinfo_flags */
7226 : TV_TREE_WIDEN_MUL, /* tv_id */
7227 : PROP_ssa, /* properties_required */
7228 : 0, /* properties_provided */
7229 : 0, /* properties_destroyed */
7230 : 0, /* todo_flags_start */
7231 : TODO_update_ssa, /* todo_flags_finish */
7232 : };
7233 :
7234 : class pass_optimize_widening_mul : public gimple_opt_pass
7235 : {
7236 : public:
7237 294196 : pass_optimize_widening_mul (gcc::context *ctxt)
7238 588392 : : gimple_opt_pass (pass_data_optimize_widening_mul, ctxt)
7239 : {}
7240 :
7241 : /* opt_pass methods: */
7242 1060389 : bool gate (function *) final override
7243 : {
7244 1060389 : return optimize_widening_mul_active_p ();
7245 : }
7246 :
7247 : unsigned int execute (function *) final override;
7248 :
7249 : }; // class pass_optimize_widening_mul
7250 :
7251 : /* Walker class to perform the transformation in reverse dominance order. */
7252 :
7253 : class math_opts_dom_walker : public dom_walker
7254 : {
7255 : public:
7256 : /* Constructor, CFG_CHANGED is a pointer to a boolean flag that will be set
7257 : if walking modidifes the CFG. */
7258 :
7259 981533 : math_opts_dom_walker (bool *cfg_changed_p)
7260 1963066 : : dom_walker (CDI_DOMINATORS), m_last_result_set (),
7261 981533 : m_cfg_changed_p (cfg_changed_p) {}
7262 :
7263 : /* The actual actions performed in the walk. */
7264 :
7265 : void after_dom_children (basic_block) final override;
7266 :
7267 : /* Set of results of chains of multiply and add statement combinations that
7268 : were not transformed into FMAs because of active deferring. */
7269 : hash_set<tree> m_last_result_set;
7270 :
7271 : /* Pointer to a flag of the user that needs to be set if CFG has been
7272 : modified. */
7273 : bool *m_cfg_changed_p;
7274 : };
7275 :
7276 : void
7277 10464990 : math_opts_dom_walker::after_dom_children (basic_block bb)
7278 : {
7279 10464990 : gimple_stmt_iterator gsi;
7280 :
7281 10464990 : fma_deferring_state fma_state (param_avoid_fma_max_bits > 0
7282 10568275 : && param_widening_mul_defer_fma);
7283 :
7284 14748596 : for (gphi_iterator psi_next, psi = gsi_start_phis (bb); !gsi_end_p (psi);
7285 4283606 : psi = psi_next)
7286 : {
7287 4283606 : psi_next = psi;
7288 4283606 : gsi_next (&psi_next);
7289 :
7290 4283606 : gimple_stmt_iterator gsi = gsi_after_labels (bb);
7291 4283606 : gphi *phi = psi.phi ();
7292 :
7293 4283606 : if (match_saturation_add (&gsi, phi)
7294 4283589 : || match_saturation_sub (&gsi, phi)
7295 4283563 : || match_saturation_trunc (&gsi, phi)
7296 4283563 : || match_saturation_mul (&gsi, phi)
7297 8567169 : || match_spaceship (&gsi, phi))
7298 156 : remove_phi_node (&psi, /* release_lhs_p */ false);
7299 : }
7300 :
7301 97350386 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi);)
7302 : {
7303 86885396 : gimple *stmt = gsi_stmt (gsi);
7304 86885396 : enum tree_code code;
7305 :
7306 86885396 : if (is_gimple_assign (stmt))
7307 : {
7308 21701144 : code = gimple_assign_rhs_code (stmt);
7309 21701144 : switch (code)
7310 : {
7311 753715 : case MULT_EXPR:
7312 753715 : if (narrow_long_mul_low_half (&gsi))
7313 : break;
7314 753715 : if (!convert_mult_to_widen (stmt, &gsi)
7315 744834 : && !convert_expand_mult_copysign (stmt, &gsi)
7316 1498506 : && convert_mult_to_fma (stmt,
7317 : gimple_assign_rhs1 (stmt),
7318 : gimple_assign_rhs2 (stmt),
7319 : &fma_state))
7320 : {
7321 16505 : gsi_remove (&gsi, true);
7322 16505 : release_defs (stmt);
7323 16505 : continue;
7324 : }
7325 737210 : match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p);
7326 737210 : match_unsigned_saturation_sub (&gsi, as_a<gassign *> (stmt));
7327 737210 : break;
7328 :
7329 2308169 : case PLUS_EXPR:
7330 2308169 : match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt));
7331 2308169 : match_unsigned_saturation_sub (&gsi, as_a<gassign *> (stmt));
7332 : /* fall-through */
7333 2622308 : case MINUS_EXPR:
7334 2622308 : if (!convert_plusminus_to_widen (&gsi, stmt, code))
7335 : {
7336 2622308 : match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p);
7337 2622308 : if (gsi_stmt (gsi) == stmt)
7338 2616229 : match_uaddc_usubc (&gsi, stmt, code);
7339 : }
7340 : break;
7341 :
7342 38891 : case BIT_NOT_EXPR:
7343 38891 : if (match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p))
7344 170 : continue;
7345 : break;
7346 :
7347 57033 : case TRUNC_MOD_EXPR:
7348 57033 : convert_to_divmod (as_a<gassign *> (stmt));
7349 57033 : break;
7350 :
7351 174356 : case RSHIFT_EXPR:
7352 174356 : convert_mult_to_highpart (as_a<gassign *> (stmt), &gsi);
7353 174356 : break;
7354 :
7355 192753 : case BIT_IOR_EXPR:
7356 192753 : match_unsigned_saturation_mul (&gsi, as_a<gassign *> (stmt));
7357 192753 : match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt));
7358 192753 : match_unsigned_saturation_trunc (&gsi, as_a<gassign *> (stmt));
7359 : /* fall-through */
7360 224309 : case BIT_XOR_EXPR:
7361 224309 : match_uaddc_usubc (&gsi, stmt, code);
7362 224309 : break;
7363 :
7364 329074 : case EQ_EXPR:
7365 329074 : case NE_EXPR:
7366 329074 : case LE_EXPR:
7367 329074 : case GT_EXPR:
7368 329074 : match_single_bit_test (&gsi, stmt);
7369 329074 : break;
7370 :
7371 348865 : case COND_EXPR:
7372 348865 : case BIT_AND_EXPR:
7373 348865 : match_unsigned_saturation_sub (&gsi, as_a<gassign *> (stmt));
7374 348865 : break;
7375 :
7376 2462347 : case NOP_EXPR:
7377 2462347 : match_unsigned_saturation_mul (&gsi, as_a<gassign *> (stmt));
7378 2462347 : match_unsigned_saturation_trunc (&gsi, as_a<gassign *> (stmt));
7379 2462347 : match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt));
7380 : /* fall-through */
7381 2462361 : case CONVERT_EXPR:
7382 : /* The long-multiply recognizer's high-part emit ends in an
7383 : outer convert. If the trailing cast+mult+shift+cast
7384 : chain has no expansion strategy at the 2N width, lower
7385 : the whole chain to a longhand high-part at narrow
7386 : precision. */
7387 2462361 : if (gsi_stmt (gsi) == stmt
7388 2462361 : && lower_long_mul_high_chain (&gsi))
7389 2210 : continue;
7390 : break;
7391 :
7392 0 : default:;
7393 : }
7394 : }
7395 65184252 : else if (is_gimple_call (stmt))
7396 : {
7397 4902410 : switch (gimple_call_combined_fn (stmt))
7398 : {
7399 129 : case CFN_COND_MUL:
7400 129 : if (convert_mult_to_fma (stmt,
7401 : gimple_call_arg (stmt, 1),
7402 : gimple_call_arg (stmt, 2),
7403 : &fma_state,
7404 : gimple_call_arg (stmt, 0)))
7405 :
7406 : {
7407 84 : gsi_remove (&gsi, true);
7408 84 : release_defs (stmt);
7409 84 : continue;
7410 : }
7411 : break;
7412 :
7413 0 : case CFN_COND_LEN_MUL:
7414 0 : if (convert_mult_to_fma (stmt,
7415 : gimple_call_arg (stmt, 1),
7416 : gimple_call_arg (stmt, 2),
7417 : &fma_state,
7418 : gimple_call_arg (stmt, 0),
7419 : gimple_call_arg (stmt, 4),
7420 : gimple_call_arg (stmt, 5)))
7421 :
7422 : {
7423 0 : gsi_remove (&gsi, true);
7424 0 : release_defs (stmt);
7425 0 : continue;
7426 : }
7427 : break;
7428 :
7429 3732687 : case CFN_LAST:
7430 3732687 : cancel_fma_deferring (&fma_state);
7431 3732687 : break;
7432 :
7433 : default:
7434 : break;
7435 : }
7436 : }
7437 60281842 : else if (gimple_code (stmt) == GIMPLE_COND)
7438 : {
7439 4220378 : match_single_bit_test (&gsi, stmt);
7440 4220378 : optimize_spaceship (as_a <gcond *> (stmt));
7441 : }
7442 86866427 : gsi_next (&gsi);
7443 : }
7444 10464990 : if (fma_state.m_deferring_p
7445 7661391 : && fma_state.m_initial_phi)
7446 : {
7447 361 : gcc_checking_assert (fma_state.m_last_result);
7448 361 : if (!last_fma_candidate_feeds_initial_phi (&fma_state,
7449 : &m_last_result_set))
7450 264 : cancel_fma_deferring (&fma_state);
7451 : else
7452 97 : m_last_result_set.add (fma_state.m_last_result);
7453 : }
7454 10464990 : }
7455 :
7456 :
7457 : unsigned int
7458 981533 : pass_optimize_widening_mul::execute (function *fun)
7459 : {
7460 981533 : bool cfg_changed = false;
7461 :
7462 981533 : memset (&widen_mul_stats, 0, sizeof (widen_mul_stats));
7463 981533 : calculate_dominance_info (CDI_DOMINATORS);
7464 981533 : renumber_gimple_stmt_uids (cfun);
7465 :
7466 981533 : long_mul_phi_halves = new hash_map<tree, long_mul_halves>;
7467 :
7468 981533 : math_opts_dom_walker (&cfg_changed).walk (ENTRY_BLOCK_PTR_FOR_FN (cfun));
7469 :
7470 : /* A 2N multiply the target cannot expand would abort expand_mult. Every
7471 : statement has been through the lowerings above, so one left here matched
7472 : none of them. */
7473 981533 : basic_block bb;
7474 10464990 : FOR_EACH_BB_FN (bb, fun)
7475 106947077 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
7476 : {
7477 87980163 : gimple *stmt = gsi_stmt (gsi);
7478 87980163 : gsi_next (&gsi);
7479 87980163 : narrow_long_mul_halves (stmt);
7480 : }
7481 :
7482 1963066 : delete long_mul_phi_halves;
7483 981533 : long_mul_phi_halves = NULL;
7484 :
7485 981533 : statistics_counter_event (fun, "widening multiplications inserted",
7486 : widen_mul_stats.widen_mults_inserted);
7487 981533 : statistics_counter_event (fun, "widening maccs inserted",
7488 : widen_mul_stats.maccs_inserted);
7489 981533 : statistics_counter_event (fun, "fused multiply-adds inserted",
7490 : widen_mul_stats.fmas_inserted);
7491 981533 : statistics_counter_event (fun, "divmod calls inserted",
7492 : widen_mul_stats.divmod_calls_inserted);
7493 981533 : statistics_counter_event (fun, "highpart multiplications inserted",
7494 : widen_mul_stats.highpart_mults_inserted);
7495 :
7496 981533 : return cfg_changed ? TODO_cleanup_cfg : 0;
7497 : }
7498 :
7499 : } // anon namespace
7500 :
7501 : gimple_opt_pass *
7502 294196 : make_pass_optimize_widening_mul (gcc::context *ctxt)
7503 : {
7504 294196 : return new pass_optimize_widening_mul (ctxt);
7505 : }
|
