Line data Source code
1 : /* Forward propagation of expressions for single use variables.
2 : Copyright (C) 2004-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify
7 : it under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful,
12 : but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : GNU General Public License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "rtl.h"
25 : #include "tree.h"
26 : #include "gimple.h"
27 : #include "cfghooks.h"
28 : #include "tree-pass.h"
29 : #include "ssa.h"
30 : #include "expmed.h"
31 : #include "optabs-query.h"
32 : #include "gimple-pretty-print.h"
33 : #include "fold-const.h"
34 : #include "stor-layout.h"
35 : #include "gimple-iterator.h"
36 : #include "gimple-fold.h"
37 : #include "tree-eh.h"
38 : #include "gimplify.h"
39 : #include "gimplify-me.h"
40 : #include "tree-cfg.h"
41 : #include "expr.h"
42 : #include "tree-dfa.h"
43 : #include "tree-ssa-propagate.h"
44 : #include "tree-ssa-dom.h"
45 : #include "tree-ssa-strlen.h"
46 : #include "builtins.h"
47 : #include "tree-cfgcleanup.h"
48 : #include "cfganal.h"
49 : #include "optabs-tree.h"
50 : #include "insn-config.h"
51 : #include "recog.h"
52 : #include "cfgloop.h"
53 : #include "tree-vectorizer.h"
54 : #include "tree-vector-builder.h"
55 : #include "vec-perm-indices.h"
56 : #include "internal-fn.h"
57 : #include "cgraph.h"
58 : #include "tree-ssa.h"
59 : #include "gimple-range.h"
60 : #include "tree-ssa-dce.h"
61 : #include "tree-ssa-math-opts.h"
62 :
63 : /* This pass propagates the RHS of assignment statements into use
64 : sites of the LHS of the assignment. It's basically a specialized
65 : form of tree combination. It is hoped all of this can disappear
66 : when we have a generalized tree combiner.
67 :
68 : One class of common cases we handle is forward propagating a single use
69 : variable into a COND_EXPR.
70 :
71 : bb0:
72 : x = a COND b;
73 : if (x) goto ... else goto ...
74 :
75 : Will be transformed into:
76 :
77 : bb0:
78 : if (a COND b) goto ... else goto ...
79 :
80 : Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
81 :
82 : Or (assuming c1 and c2 are constants):
83 :
84 : bb0:
85 : x = a + c1;
86 : if (x EQ/NEQ c2) goto ... else goto ...
87 :
88 : Will be transformed into:
89 :
90 : bb0:
91 : if (a EQ/NEQ (c2 - c1)) goto ... else goto ...
92 :
93 : Similarly for x = a - c1.
94 :
95 : Or
96 :
97 : bb0:
98 : x = !a
99 : if (x) goto ... else goto ...
100 :
101 : Will be transformed into:
102 :
103 : bb0:
104 : if (a == 0) goto ... else goto ...
105 :
106 : Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
107 : For these cases, we propagate A into all, possibly more than one,
108 : COND_EXPRs that use X.
109 :
110 : Or
111 :
112 : bb0:
113 : x = (typecast) a
114 : if (x) goto ... else goto ...
115 :
116 : Will be transformed into:
117 :
118 : bb0:
119 : if (a != 0) goto ... else goto ...
120 :
121 : (Assuming a is an integral type and x is a boolean or x is an
122 : integral and a is a boolean.)
123 :
124 : Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
125 : For these cases, we propagate A into all, possibly more than one,
126 : COND_EXPRs that use X.
127 :
128 : In addition to eliminating the variable and the statement which assigns
129 : a value to the variable, we may be able to later thread the jump without
130 : adding insane complexity in the dominator optimizer.
131 :
132 : Also note these transformations can cascade. We handle this by having
133 : a worklist of COND_EXPR statements to examine. As we make a change to
134 : a statement, we put it back on the worklist to examine on the next
135 : iteration of the main loop.
136 :
137 : A second class of propagation opportunities arises for ADDR_EXPR
138 : nodes.
139 :
140 : ptr = &x->y->z;
141 : res = *ptr;
142 :
143 : Will get turned into
144 :
145 : res = x->y->z;
146 :
147 : Or
148 : ptr = (type1*)&type2var;
149 : res = *ptr
150 :
151 : Will get turned into (if type1 and type2 are the same size
152 : and neither have volatile on them):
153 : res = VIEW_CONVERT_EXPR<type1>(type2var)
154 :
155 : Or
156 :
157 : ptr = &x[0];
158 : ptr2 = ptr + <constant>;
159 :
160 : Will get turned into
161 :
162 : ptr2 = &x[constant/elementsize];
163 :
164 : Or
165 :
166 : ptr = &x[0];
167 : offset = index * element_size;
168 : offset_p = (pointer) offset;
169 : ptr2 = ptr + offset_p
170 :
171 : Will get turned into:
172 :
173 : ptr2 = &x[index];
174 :
175 : Or
176 : ssa = (int) decl
177 : res = ssa & 1
178 :
179 : Provided that decl has known alignment >= 2, will get turned into
180 :
181 : res = 0
182 :
183 : We also propagate casts into SWITCH_EXPR and COND_EXPR conditions to
184 : allow us to remove the cast and {NOT_EXPR,NEG_EXPR} into a subsequent
185 : {NOT_EXPR,NEG_EXPR}.
186 :
187 : This will (of course) be extended as other needs arise. */
188 :
189 : /* Data structure that contains simplifiable vectorized permute sequences.
190 : See recognise_vec_perm_simplify_seq () for a description of the sequence. */
191 :
192 : struct _vec_perm_simplify_seq
193 : {
194 : /* Defining stmts of vectors in the sequence. */
195 : gassign *v_1_stmt;
196 : gassign *v_2_stmt;
197 : gassign *v_x_stmt;
198 : gassign *v_y_stmt;
199 : /* Final permute statement. */
200 : gassign *stmt;
201 : /* New selector indices for stmt. */
202 : tree new_sel;
203 : /* Elements of each vector and selector. */
204 : unsigned int nelts;
205 : };
206 : typedef struct _vec_perm_simplify_seq *vec_perm_simplify_seq;
207 :
208 : static bool forward_propagate_addr_expr (tree, tree, bool);
209 :
210 : /* Set to true if we delete dead edges during the optimization. */
211 : static bool cfg_changed;
212 :
213 : static tree rhs_to_tree (tree type, gimple *stmt);
214 :
215 : static bitmap to_purge;
216 :
217 : /* Const-and-copy lattice. */
218 : static vec<tree> lattice;
219 :
220 : /* Set the lattice entry for NAME to VAL. */
221 : static void
222 33526974 : fwprop_set_lattice_val (tree name, tree val)
223 : {
224 33526974 : if (TREE_CODE (name) == SSA_NAME)
225 : {
226 33526974 : if (SSA_NAME_VERSION (name) >= lattice.length ())
227 : {
228 31518 : lattice.reserve (num_ssa_names - lattice.length ());
229 21012 : lattice.quick_grow_cleared (num_ssa_names);
230 : }
231 33526974 : lattice[SSA_NAME_VERSION (name)] = val;
232 : /* As this now constitutes a copy duplicate points-to
233 : and range info appropriately. */
234 33526974 : if (TREE_CODE (val) == SSA_NAME)
235 33030107 : maybe_duplicate_ssa_info_at_copy (name, val);
236 : }
237 33526974 : }
238 :
239 : /* Invalidate the lattice entry for NAME, done when releasing SSA names. */
240 : static void
241 941733 : fwprop_invalidate_lattice (tree name)
242 : {
243 941733 : if (name
244 939192 : && TREE_CODE (name) == SSA_NAME
245 1880797 : && SSA_NAME_VERSION (name) < lattice.length ())
246 939033 : lattice[SSA_NAME_VERSION (name)] = NULL_TREE;
247 941733 : }
248 :
249 : /* Get the statement we can propagate from into NAME skipping
250 : trivial copies. Returns the statement which defines the
251 : propagation source or NULL_TREE if there is no such one.
252 : If SINGLE_USE_ONLY is set considers only sources which have
253 : a single use chain up to NAME. If SINGLE_USE_P is non-null,
254 : it is set to whether the chain to NAME is a single use chain
255 : or not. SINGLE_USE_P is not written to if SINGLE_USE_ONLY is set. */
256 :
257 : static gimple *
258 28484311 : get_prop_source_stmt (tree name, bool single_use_only, bool *single_use_p)
259 : {
260 28484311 : bool single_use = true;
261 :
262 28485329 : do {
263 28484820 : gimple *def_stmt = SSA_NAME_DEF_STMT (name);
264 :
265 28484820 : if (!has_single_use (name))
266 : {
267 15521247 : single_use = false;
268 15521247 : if (single_use_only)
269 : return NULL;
270 : }
271 :
272 : /* If name is defined by a PHI node or is the default def, bail out. */
273 28483262 : if (!is_gimple_assign (def_stmt))
274 : return NULL;
275 :
276 : /* If def_stmt is a simple copy, continue looking. */
277 20055048 : if (gimple_assign_rhs_code (def_stmt) == SSA_NAME)
278 509 : name = gimple_assign_rhs1 (def_stmt);
279 : else
280 : {
281 20054539 : if (!single_use_only && single_use_p)
282 19726959 : *single_use_p = single_use;
283 :
284 : return def_stmt;
285 : }
286 509 : } while (1);
287 : }
288 :
289 : /* Checks if the destination ssa name in DEF_STMT can be used as
290 : propagation source. Returns true if so, otherwise false. */
291 :
292 : static bool
293 28166711 : can_propagate_from (gimple *def_stmt)
294 : {
295 28166711 : gcc_assert (is_gimple_assign (def_stmt));
296 :
297 : /* If the rhs has side-effects we cannot propagate from it. */
298 28166711 : if (gimple_has_volatile_ops (def_stmt))
299 : return false;
300 :
301 : /* If the rhs is a load we cannot propagate from it. */
302 27571738 : if (TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_reference
303 27571738 : || TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_declaration)
304 : return false;
305 :
306 : /* Constants can be always propagated. */
307 13569555 : if (gimple_assign_single_p (def_stmt)
308 13569555 : && is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
309 : return true;
310 :
311 : /* We cannot propagate ssa names that occur in abnormal phi nodes. */
312 13569555 : if (stmt_references_abnormal_ssa_name (def_stmt))
313 : return false;
314 :
315 : /* If the definition is a conversion of a pointer to a function type,
316 : then we cannot apply optimizations as some targets require
317 : function pointers to be canonicalized and in this case this
318 : optimization could eliminate a necessary canonicalization. */
319 13568865 : if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def_stmt)))
320 : {
321 3285643 : tree rhs = gimple_assign_rhs1 (def_stmt);
322 3285643 : if (FUNCTION_POINTER_TYPE_P (TREE_TYPE (rhs)))
323 818 : return false;
324 : }
325 :
326 : return true;
327 : }
328 :
329 : /* Remove a chain of dead statements starting at the definition of
330 : NAME. The chain is linked via the first operand of the defining statements.
331 : If NAME was replaced in its only use then this function can be used
332 : to clean up dead stmts. The function handles already released SSA
333 : names gracefully. */
334 :
335 : static void
336 233091 : remove_prop_source_from_use (tree name)
337 : {
338 295401 : gimple_stmt_iterator gsi;
339 295401 : gimple *stmt;
340 :
341 295401 : do {
342 295401 : basic_block bb;
343 :
344 295401 : if (SSA_NAME_IN_FREE_LIST (name)
345 295358 : || SSA_NAME_IS_DEFAULT_DEF (name)
346 589486 : || !has_zero_uses (name))
347 : break;
348 :
349 62770 : stmt = SSA_NAME_DEF_STMT (name);
350 62770 : if (gimple_code (stmt) == GIMPLE_PHI
351 62770 : || gimple_has_side_effects (stmt))
352 : break;
353 :
354 62770 : bb = gimple_bb (stmt);
355 62770 : gsi = gsi_for_stmt (stmt);
356 62770 : unlink_stmt_vdef (stmt);
357 62770 : if (gsi_remove (&gsi, true))
358 6 : bitmap_set_bit (to_purge, bb->index);
359 62770 : fwprop_invalidate_lattice (gimple_get_lhs (stmt));
360 62770 : release_defs (stmt);
361 :
362 62770 : name = is_gimple_assign (stmt) ? gimple_assign_rhs1 (stmt) : NULL_TREE;
363 62770 : } while (name && TREE_CODE (name) == SSA_NAME);
364 :
365 233091 : }
366 :
367 : /* Return the rhs of a gassign *STMT in a form of a single tree,
368 : converted to type TYPE.
369 :
370 : This should disappear, but is needed so we can combine expressions and use
371 : the fold() interfaces. Long term, we need to develop folding and combine
372 : routines that deal with gimple exclusively . */
373 :
374 : static tree
375 7394938 : rhs_to_tree (tree type, gimple *stmt)
376 : {
377 7394938 : location_t loc = gimple_location (stmt);
378 7394938 : enum tree_code code = gimple_assign_rhs_code (stmt);
379 7394938 : switch (get_gimple_rhs_class (code))
380 : {
381 13290 : case GIMPLE_TERNARY_RHS:
382 13290 : return fold_build3_loc (loc, code, type, gimple_assign_rhs1 (stmt),
383 : gimple_assign_rhs2 (stmt),
384 13290 : gimple_assign_rhs3 (stmt));
385 5038226 : case GIMPLE_BINARY_RHS:
386 5038226 : return fold_build2_loc (loc, code, type, gimple_assign_rhs1 (stmt),
387 5038226 : gimple_assign_rhs2 (stmt));
388 2076168 : case GIMPLE_UNARY_RHS:
389 2076168 : return build1 (code, type, gimple_assign_rhs1 (stmt));
390 267254 : case GIMPLE_SINGLE_RHS:
391 267254 : return gimple_assign_rhs1 (stmt);
392 0 : default:
393 0 : gcc_unreachable ();
394 : }
395 : }
396 :
397 : /* Combine OP0 CODE OP1 in the context of a COND_EXPR. Returns
398 : the folded result in a form suitable for COND_EXPR_COND or
399 : NULL_TREE, if there is no suitable simplified form. If
400 : INVARIANT_ONLY is true only gimple_min_invariant results are
401 : considered simplified. */
402 :
403 : static tree
404 8331384 : combine_cond_expr_cond (gimple *stmt, enum tree_code code, tree type,
405 : tree op0, tree op1, bool invariant_only)
406 : {
407 8331384 : tree t;
408 :
409 8331384 : gcc_assert (TREE_CODE_CLASS (code) == tcc_comparison);
410 :
411 8331384 : t = fold_binary_loc (gimple_location (stmt), code, type, op0, op1);
412 8331384 : if (!t)
413 : return NULL_TREE;
414 :
415 : /* Require that we got a boolean type out if we put one in. */
416 3646653 : gcc_assert (TREE_CODE (TREE_TYPE (t)) == TREE_CODE (type));
417 :
418 : /* Canonicalize the combined condition for use in a COND_EXPR. */
419 3646653 : t = canonicalize_cond_expr_cond (t);
420 :
421 : /* Bail out if we required an invariant but didn't get one. */
422 3646653 : if (!t || (invariant_only && !is_gimple_min_invariant (t)))
423 3416026 : return NULL_TREE;
424 :
425 : return t;
426 : }
427 :
428 : /* Combine the comparison OP0 CODE OP1 at LOC with the defining statements
429 : of its operand. Return a new comparison tree or NULL_TREE if there
430 : were no simplifying combines. */
431 :
432 : static tree
433 22432887 : forward_propagate_into_comparison_1 (gimple *stmt,
434 : enum tree_code code, tree type,
435 : tree op0, tree op1)
436 : {
437 22432887 : tree tmp = NULL_TREE;
438 22432887 : tree rhs0 = NULL_TREE, rhs1 = NULL_TREE;
439 22432887 : bool single_use0_p = false, single_use1_p = false;
440 :
441 : /* For comparisons use the first operand, that is likely to
442 : simplify comparisons against constants. */
443 22432887 : if (TREE_CODE (op0) == SSA_NAME)
444 : {
445 22362689 : gimple *def_stmt = get_prop_source_stmt (op0, false, &single_use0_p);
446 22362689 : if (def_stmt && can_propagate_from (def_stmt))
447 : {
448 5606267 : enum tree_code def_code = gimple_assign_rhs_code (def_stmt);
449 5606267 : bool invariant_only_p = !single_use0_p;
450 :
451 5606267 : rhs0 = rhs_to_tree (TREE_TYPE (op1), def_stmt);
452 :
453 : /* Always combine comparisons or conversions from booleans. */
454 5606267 : if (TREE_CODE (op1) == INTEGER_CST
455 5606267 : && ((CONVERT_EXPR_CODE_P (def_code)
456 896858 : && TREE_CODE (TREE_TYPE (TREE_OPERAND (rhs0, 0)))
457 : == BOOLEAN_TYPE)
458 3583135 : || TREE_CODE_CLASS (def_code) == tcc_comparison))
459 : invariant_only_p = false;
460 :
461 5606267 : tmp = combine_cond_expr_cond (stmt, code, type,
462 : rhs0, op1, invariant_only_p);
463 5606267 : if (tmp)
464 : return tmp;
465 : }
466 : }
467 :
468 : /* If that wasn't successful, try the second operand. */
469 22210891 : if (TREE_CODE (op1) == SSA_NAME)
470 : {
471 5522820 : gimple *def_stmt = get_prop_source_stmt (op1, false, &single_use1_p);
472 5522820 : if (def_stmt && can_propagate_from (def_stmt))
473 : {
474 1788671 : rhs1 = rhs_to_tree (TREE_TYPE (op0), def_stmt);
475 1788671 : tmp = combine_cond_expr_cond (stmt, code, type,
476 : op0, rhs1, !single_use1_p);
477 1788671 : if (tmp)
478 : return tmp;
479 : }
480 : }
481 :
482 : /* If that wasn't successful either, try both operands. */
483 22204091 : if (rhs0 != NULL_TREE
484 22204091 : && rhs1 != NULL_TREE)
485 936446 : tmp = combine_cond_expr_cond (stmt, code, type,
486 : rhs0, rhs1,
487 936446 : !(single_use0_p && single_use1_p));
488 :
489 : return tmp;
490 : }
491 :
492 : /* Propagate from the ssa name definition statements of the assignment
493 : from a comparison at *GSI into the conditional if that simplifies it.
494 : Returns true if the stmt was modified. */
495 :
496 : static bool
497 2640354 : forward_propagate_into_comparison (gimple_stmt_iterator *gsi)
498 : {
499 2640354 : gimple *stmt = gsi_stmt (*gsi);
500 2640354 : tree tmp;
501 2640354 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
502 2640354 : tree rhs1 = gimple_assign_rhs1 (stmt);
503 2640354 : tree rhs2 = gimple_assign_rhs2 (stmt);
504 :
505 : /* Combine the comparison with defining statements. */
506 2640354 : tmp = forward_propagate_into_comparison_1 (stmt,
507 : gimple_assign_rhs_code (stmt),
508 : type, rhs1, rhs2);
509 2640354 : if (tmp && useless_type_conversion_p (type, TREE_TYPE (tmp)))
510 : {
511 7360 : if (dump_file)
512 : {
513 0 : fprintf (dump_file, " Replaced '");
514 0 : print_gimple_expr (dump_file, stmt, 0);
515 0 : fprintf (dump_file, "' with '");
516 0 : print_generic_expr (dump_file, tmp);
517 0 : fprintf (dump_file, "'\n");
518 : }
519 7360 : gimple_assign_set_rhs_from_tree (gsi, tmp);
520 7360 : fold_stmt (gsi);
521 7360 : update_stmt (gsi_stmt (*gsi));
522 :
523 7360 : if (TREE_CODE (rhs1) == SSA_NAME)
524 7360 : remove_prop_source_from_use (rhs1);
525 7360 : if (TREE_CODE (rhs2) == SSA_NAME)
526 3123 : remove_prop_source_from_use (rhs2);
527 : return true;
528 : }
529 :
530 : return false;
531 : }
532 :
533 : /* Propagate from the ssa name definition statements of COND_EXPR
534 : in GIMPLE_COND statement STMT into the conditional if that simplifies it.
535 : Returns zero if no statement was changed, one if there were
536 : changes and two if cfg_cleanup needs to run. */
537 :
538 : static int
539 19792533 : forward_propagate_into_gimple_cond (gcond *stmt)
540 : {
541 19792533 : tree tmp;
542 19792533 : enum tree_code code = gimple_cond_code (stmt);
543 19792533 : tree rhs1 = gimple_cond_lhs (stmt);
544 19792533 : tree rhs2 = gimple_cond_rhs (stmt);
545 :
546 : /* GIMPLE_COND will always be a comparison. */
547 19792533 : gcc_assert (TREE_CODE_CLASS (gimple_cond_code (stmt)) == tcc_comparison);
548 :
549 19792533 : tmp = forward_propagate_into_comparison_1 (stmt, code,
550 : boolean_type_node,
551 : rhs1, rhs2);
552 19792533 : if (tmp
553 19792533 : && is_gimple_condexpr_for_cond (tmp))
554 : {
555 216804 : if (dump_file)
556 : {
557 9 : fprintf (dump_file, " Replaced '");
558 9 : print_gimple_expr (dump_file, stmt, 0);
559 9 : fprintf (dump_file, "' with '");
560 9 : print_generic_expr (dump_file, tmp);
561 9 : fprintf (dump_file, "'\n");
562 : }
563 :
564 216804 : gimple_cond_set_condition_from_tree (stmt, unshare_expr (tmp));
565 216804 : update_stmt (stmt);
566 :
567 216804 : if (TREE_CODE (rhs1) == SSA_NAME)
568 216804 : remove_prop_source_from_use (rhs1);
569 216804 : if (TREE_CODE (rhs2) == SSA_NAME)
570 5803 : remove_prop_source_from_use (rhs2);
571 216804 : return is_gimple_min_invariant (tmp) ? 2 : 1;
572 : }
573 :
574 19575729 : if (canonicalize_bool_cond (stmt, gimple_bb (stmt)))
575 : return 1;
576 :
577 : return 0;
578 : }
579 :
580 : /* We've just substituted an ADDR_EXPR into stmt. Update all the
581 : relevant data structures to match. */
582 :
583 : static void
584 2005377 : tidy_after_forward_propagate_addr (gimple *stmt)
585 : {
586 : /* We may have turned a trapping insn into a non-trapping insn. */
587 2005377 : if (maybe_clean_or_replace_eh_stmt (stmt, stmt))
588 39 : bitmap_set_bit (to_purge, gimple_bb (stmt)->index);
589 :
590 2005377 : if (TREE_CODE (gimple_assign_rhs1 (stmt)) == ADDR_EXPR)
591 250930 : recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
592 2005377 : }
593 :
594 : /* NAME is a SSA_NAME representing DEF_RHS which is of the form
595 : ADDR_EXPR <whatever>.
596 :
597 : Try to forward propagate the ADDR_EXPR into the use USE_STMT.
598 : Often this will allow for removal of an ADDR_EXPR and INDIRECT_REF
599 : node or for recovery of array indexing from pointer arithmetic.
600 :
601 : Return true if the propagation was successful (the propagation can
602 : be not totally successful, yet things may have been changed). */
603 :
604 : static bool
605 2863108 : forward_propagate_addr_expr_1 (tree name, tree def_rhs,
606 : gimple_stmt_iterator *use_stmt_gsi,
607 : bool single_use_p)
608 : {
609 2863108 : tree lhs, rhs, rhs2, array_ref;
610 2863108 : gimple *use_stmt = gsi_stmt (*use_stmt_gsi);
611 2863108 : enum tree_code rhs_code;
612 2863108 : bool res = true;
613 :
614 2863108 : gcc_assert (TREE_CODE (def_rhs) == ADDR_EXPR);
615 :
616 2863108 : lhs = gimple_assign_lhs (use_stmt);
617 2863108 : rhs_code = gimple_assign_rhs_code (use_stmt);
618 2863108 : rhs = gimple_assign_rhs1 (use_stmt);
619 :
620 : /* Do not perform copy-propagation but recurse through copy chains. */
621 2863108 : if (TREE_CODE (lhs) == SSA_NAME
622 1424276 : && rhs_code == SSA_NAME)
623 8807 : return forward_propagate_addr_expr (lhs, def_rhs, single_use_p);
624 :
625 : /* The use statement could be a conversion. Recurse to the uses of the
626 : lhs as copyprop does not copy through pointer to integer to pointer
627 : conversions and FRE does not catch all cases either.
628 : Treat the case of a single-use name and
629 : a conversion to def_rhs type separate, though. */
630 2854301 : if (TREE_CODE (lhs) == SSA_NAME
631 1415469 : && CONVERT_EXPR_CODE_P (rhs_code))
632 : {
633 : /* If there is a point in a conversion chain where the types match
634 : so we can remove a conversion re-materialize the address here
635 : and stop. */
636 25065 : if (single_use_p
637 25065 : && useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (def_rhs)))
638 : {
639 2 : gimple_assign_set_rhs1 (use_stmt, unshare_expr (def_rhs));
640 2 : gimple_assign_set_rhs_code (use_stmt, TREE_CODE (def_rhs));
641 2 : return true;
642 : }
643 :
644 : /* Else recurse if the conversion preserves the address value. */
645 50126 : if ((INTEGRAL_TYPE_P (TREE_TYPE (lhs))
646 2 : || POINTER_TYPE_P (TREE_TYPE (lhs)))
647 50126 : && (TYPE_PRECISION (TREE_TYPE (lhs))
648 25063 : >= TYPE_PRECISION (TREE_TYPE (def_rhs))))
649 24996 : return forward_propagate_addr_expr (lhs, def_rhs, single_use_p);
650 :
651 : return false;
652 : }
653 :
654 : /* If this isn't a conversion chain from this on we only can propagate
655 : into compatible pointer contexts. */
656 2829236 : if (!types_compatible_p (TREE_TYPE (name), TREE_TYPE (def_rhs)))
657 : return false;
658 :
659 : /* Propagate through constant pointer adjustments. */
660 2807832 : if (TREE_CODE (lhs) == SSA_NAME
661 1370786 : && rhs_code == POINTER_PLUS_EXPR
662 1370786 : && rhs == name
663 2978851 : && TREE_CODE (gimple_assign_rhs2 (use_stmt)) == INTEGER_CST)
664 : {
665 120214 : tree new_def_rhs;
666 : /* As we come here with non-invariant addresses in def_rhs we need
667 : to make sure we can build a valid constant offsetted address
668 : for further propagation. Simply rely on fold building that
669 : and check after the fact. */
670 120214 : new_def_rhs = fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (rhs)),
671 : def_rhs,
672 : fold_convert (ptr_type_node,
673 : gimple_assign_rhs2 (use_stmt)));
674 120214 : if (TREE_CODE (new_def_rhs) == MEM_REF
675 120214 : && !is_gimple_mem_ref_addr (TREE_OPERAND (new_def_rhs, 0)))
676 : return false;
677 116098 : new_def_rhs = build1 (ADDR_EXPR, TREE_TYPE (rhs), new_def_rhs);
678 :
679 : /* Recurse. If we could propagate into all uses of lhs do not
680 : bother to replace into the current use but just pretend we did. */
681 116098 : if (forward_propagate_addr_expr (lhs, new_def_rhs, single_use_p))
682 : return true;
683 :
684 39951 : if (useless_type_conversion_p (TREE_TYPE (lhs),
685 39951 : TREE_TYPE (new_def_rhs)))
686 39951 : gimple_assign_set_rhs_with_ops (use_stmt_gsi, TREE_CODE (new_def_rhs),
687 : new_def_rhs);
688 0 : else if (is_gimple_min_invariant (new_def_rhs))
689 0 : gimple_assign_set_rhs_with_ops (use_stmt_gsi, NOP_EXPR, new_def_rhs);
690 : else
691 : return false;
692 39951 : gcc_assert (gsi_stmt (*use_stmt_gsi) == use_stmt);
693 39951 : update_stmt (use_stmt);
694 39951 : return true;
695 : }
696 :
697 : /* Now strip away any outer COMPONENT_REF/ARRAY_REF nodes from the LHS.
698 : ADDR_EXPR will not appear on the LHS. */
699 2687618 : tree *lhsp = gimple_assign_lhs_ptr (use_stmt);
700 4078602 : while (handled_component_p (*lhsp))
701 1390984 : lhsp = &TREE_OPERAND (*lhsp, 0);
702 2687618 : lhs = *lhsp;
703 :
704 : /* Now see if the LHS node is a MEM_REF using NAME. If so,
705 : propagate the ADDR_EXPR into the use of NAME and fold the result. */
706 2687618 : if (TREE_CODE (lhs) == MEM_REF
707 2687618 : && TREE_OPERAND (lhs, 0) == name)
708 : {
709 903098 : tree def_rhs_base;
710 903098 : poly_int64 def_rhs_offset;
711 : /* If the address is invariant we can always fold it. */
712 903098 : if ((def_rhs_base = get_addr_base_and_unit_offset (TREE_OPERAND (def_rhs, 0),
713 : &def_rhs_offset)))
714 : {
715 854029 : poly_offset_int off = mem_ref_offset (lhs);
716 854029 : tree new_ptr;
717 854029 : off += def_rhs_offset;
718 854029 : if (TREE_CODE (def_rhs_base) == MEM_REF)
719 : {
720 832864 : off += mem_ref_offset (def_rhs_base);
721 832864 : new_ptr = TREE_OPERAND (def_rhs_base, 0);
722 : }
723 : else
724 21165 : new_ptr = build_fold_addr_expr (def_rhs_base);
725 854029 : TREE_OPERAND (lhs, 0) = new_ptr;
726 854029 : TREE_OPERAND (lhs, 1)
727 854029 : = wide_int_to_tree (TREE_TYPE (TREE_OPERAND (lhs, 1)), off);
728 854029 : tidy_after_forward_propagate_addr (use_stmt);
729 : /* Continue propagating into the RHS if this was not the only use. */
730 854029 : if (single_use_p)
731 229508 : return true;
732 : }
733 : /* If the LHS is a plain dereference and the value type is the same as
734 : that of the pointed-to type of the address we can put the
735 : dereferenced address on the LHS preserving the original alias-type. */
736 49069 : else if (integer_zerop (TREE_OPERAND (lhs, 1))
737 19049 : && ((gimple_assign_lhs (use_stmt) == lhs
738 15431 : && useless_type_conversion_p
739 15431 : (TREE_TYPE (TREE_OPERAND (def_rhs, 0)),
740 15431 : TREE_TYPE (gimple_assign_rhs1 (use_stmt))))
741 14103 : || types_compatible_p (TREE_TYPE (lhs),
742 14103 : TREE_TYPE (TREE_OPERAND (def_rhs, 0))))
743 : /* Don't forward anything into clobber stmts if it would result
744 : in the lhs no longer being a MEM_REF. */
745 57270 : && (!gimple_clobber_p (use_stmt)
746 166 : || TREE_CODE (TREE_OPERAND (def_rhs, 0)) == MEM_REF))
747 : {
748 8035 : tree *def_rhs_basep = &TREE_OPERAND (def_rhs, 0);
749 8035 : tree new_offset, new_base, saved, new_lhs;
750 29143 : while (handled_component_p (*def_rhs_basep))
751 13073 : def_rhs_basep = &TREE_OPERAND (*def_rhs_basep, 0);
752 8035 : saved = *def_rhs_basep;
753 8035 : if (TREE_CODE (*def_rhs_basep) == MEM_REF)
754 : {
755 3944 : new_base = TREE_OPERAND (*def_rhs_basep, 0);
756 3944 : new_offset = fold_convert (TREE_TYPE (TREE_OPERAND (lhs, 1)),
757 : TREE_OPERAND (*def_rhs_basep, 1));
758 : }
759 : else
760 : {
761 4091 : new_base = build_fold_addr_expr (*def_rhs_basep);
762 4091 : new_offset = TREE_OPERAND (lhs, 1);
763 : }
764 8035 : tree atype = TREE_TYPE (*def_rhs_basep);
765 8035 : if (TYPE_ALIGN (TREE_TYPE (lhs)) < TYPE_ALIGN (atype))
766 312 : atype = build_aligned_type (atype, TYPE_ALIGN (TREE_TYPE (lhs)));
767 8035 : *def_rhs_basep = build2 (MEM_REF, atype, new_base, new_offset);
768 8035 : TREE_THIS_VOLATILE (*def_rhs_basep) = TREE_THIS_VOLATILE (lhs);
769 8035 : TREE_SIDE_EFFECTS (*def_rhs_basep) = TREE_SIDE_EFFECTS (lhs);
770 8035 : TREE_THIS_NOTRAP (*def_rhs_basep) = TREE_THIS_NOTRAP (lhs);
771 8035 : new_lhs = unshare_expr (TREE_OPERAND (def_rhs, 0));
772 8035 : *lhsp = new_lhs;
773 8035 : TREE_THIS_VOLATILE (new_lhs) = TREE_THIS_VOLATILE (lhs);
774 8035 : TREE_SIDE_EFFECTS (new_lhs) = TREE_SIDE_EFFECTS (lhs);
775 8035 : *def_rhs_basep = saved;
776 8035 : tidy_after_forward_propagate_addr (use_stmt);
777 : /* Continue propagating into the RHS if this was not the
778 : only use. */
779 8035 : if (single_use_p)
780 : return true;
781 : }
782 : else
783 : /* We can have a struct assignment dereferencing our name twice.
784 : Note that we didn't propagate into the lhs to not falsely
785 : claim we did when propagating into the rhs. */
786 : res = false;
787 : }
788 :
789 : /* Strip away any outer COMPONENT_REF, ARRAY_REF or ADDR_EXPR
790 : nodes from the RHS. */
791 2453899 : tree *rhsp = gimple_assign_rhs1_ptr (use_stmt);
792 2453899 : if (TREE_CODE (*rhsp) == ADDR_EXPR)
793 239291 : rhsp = &TREE_OPERAND (*rhsp, 0);
794 3473934 : while (handled_component_p (*rhsp))
795 1020035 : rhsp = &TREE_OPERAND (*rhsp, 0);
796 2453899 : rhs = *rhsp;
797 :
798 : /* Now see if the RHS node is a MEM_REF using NAME. If so,
799 : propagate the ADDR_EXPR into the use of NAME and fold the result. */
800 2453899 : if (TREE_CODE (rhs) == MEM_REF
801 2453899 : && TREE_OPERAND (rhs, 0) == name)
802 : {
803 1165732 : tree def_rhs_base;
804 1165732 : poly_int64 def_rhs_offset;
805 1165732 : if ((def_rhs_base = get_addr_base_and_unit_offset (TREE_OPERAND (def_rhs, 0),
806 : &def_rhs_offset)))
807 : {
808 1127916 : poly_offset_int off = mem_ref_offset (rhs);
809 1127916 : tree new_ptr;
810 1127916 : off += def_rhs_offset;
811 1127916 : if (TREE_CODE (def_rhs_base) == MEM_REF)
812 : {
813 1100807 : off += mem_ref_offset (def_rhs_base);
814 1100807 : new_ptr = TREE_OPERAND (def_rhs_base, 0);
815 : }
816 : else
817 27109 : new_ptr = build_fold_addr_expr (def_rhs_base);
818 1127916 : TREE_OPERAND (rhs, 0) = new_ptr;
819 1127916 : TREE_OPERAND (rhs, 1)
820 1127916 : = wide_int_to_tree (TREE_TYPE (TREE_OPERAND (rhs, 1)), off);
821 1127916 : fold_stmt_inplace (use_stmt_gsi);
822 1127916 : tidy_after_forward_propagate_addr (use_stmt);
823 1127916 : return res;
824 : }
825 : /* If the RHS is a plain dereference and the value type is the same as
826 : that of the pointed-to type of the address we can put the
827 : dereferenced address on the RHS preserving the original alias-type. */
828 37816 : else if (integer_zerop (TREE_OPERAND (rhs, 1))
829 37816 : && ((gimple_assign_rhs1 (use_stmt) == rhs
830 20495 : && useless_type_conversion_p
831 20495 : (TREE_TYPE (gimple_assign_lhs (use_stmt)),
832 20495 : TREE_TYPE (TREE_OPERAND (def_rhs, 0))))
833 22710 : || types_compatible_p (TREE_TYPE (rhs),
834 22710 : TREE_TYPE (TREE_OPERAND (def_rhs, 0)))))
835 : {
836 15397 : tree *def_rhs_basep = &TREE_OPERAND (def_rhs, 0);
837 15397 : tree new_offset, new_base, saved, new_rhs;
838 54760 : while (handled_component_p (*def_rhs_basep))
839 23966 : def_rhs_basep = &TREE_OPERAND (*def_rhs_basep, 0);
840 15397 : saved = *def_rhs_basep;
841 15397 : if (TREE_CODE (*def_rhs_basep) == MEM_REF)
842 : {
843 7415 : new_base = TREE_OPERAND (*def_rhs_basep, 0);
844 7415 : new_offset = fold_convert (TREE_TYPE (TREE_OPERAND (rhs, 1)),
845 : TREE_OPERAND (*def_rhs_basep, 1));
846 : }
847 : else
848 : {
849 7982 : new_base = build_fold_addr_expr (*def_rhs_basep);
850 7982 : new_offset = TREE_OPERAND (rhs, 1);
851 : }
852 15397 : tree atype = TREE_TYPE (*def_rhs_basep);
853 15397 : if (TYPE_ALIGN (TREE_TYPE (rhs)) < TYPE_ALIGN (atype))
854 526 : atype = build_aligned_type (atype, TYPE_ALIGN (TREE_TYPE (rhs)));
855 15397 : *def_rhs_basep = build2 (MEM_REF, atype, new_base, new_offset);
856 15397 : TREE_THIS_VOLATILE (*def_rhs_basep) = TREE_THIS_VOLATILE (rhs);
857 15397 : TREE_SIDE_EFFECTS (*def_rhs_basep) = TREE_SIDE_EFFECTS (rhs);
858 15397 : TREE_THIS_NOTRAP (*def_rhs_basep) = TREE_THIS_NOTRAP (rhs);
859 15397 : new_rhs = unshare_expr (TREE_OPERAND (def_rhs, 0));
860 15397 : *rhsp = new_rhs;
861 15397 : TREE_THIS_VOLATILE (new_rhs) = TREE_THIS_VOLATILE (rhs);
862 15397 : TREE_SIDE_EFFECTS (new_rhs) = TREE_SIDE_EFFECTS (rhs);
863 15397 : *def_rhs_basep = saved;
864 15397 : fold_stmt_inplace (use_stmt_gsi);
865 15397 : tidy_after_forward_propagate_addr (use_stmt);
866 15397 : return res;
867 : }
868 : }
869 :
870 : /* If the use of the ADDR_EXPR is not a POINTER_PLUS_EXPR, there
871 : is nothing to do. */
872 1310586 : if (gimple_assign_rhs_code (use_stmt) != POINTER_PLUS_EXPR
873 1310586 : || gimple_assign_rhs1 (use_stmt) != name)
874 : return false;
875 :
876 : /* The remaining cases are all for turning pointer arithmetic into
877 : array indexing. They only apply when we have the address of
878 : element zero in an array. If that is not the case then there
879 : is nothing to do. */
880 50805 : array_ref = TREE_OPERAND (def_rhs, 0);
881 50805 : if ((TREE_CODE (array_ref) != ARRAY_REF
882 4669 : || TREE_CODE (TREE_TYPE (TREE_OPERAND (array_ref, 0))) != ARRAY_TYPE
883 4669 : || TREE_CODE (TREE_OPERAND (array_ref, 1)) != INTEGER_CST)
884 52323 : && TREE_CODE (TREE_TYPE (array_ref)) != ARRAY_TYPE)
885 : return false;
886 :
887 27322 : rhs2 = gimple_assign_rhs2 (use_stmt);
888 : /* Optimize &x[C1] p+ C2 to &x p+ C3 with C3 = C1 * element_size + C2. */
889 27322 : if (TREE_CODE (rhs2) == INTEGER_CST)
890 : {
891 0 : tree new_rhs = build1_loc (gimple_location (use_stmt),
892 0 : ADDR_EXPR, TREE_TYPE (def_rhs),
893 0 : fold_build2 (MEM_REF,
894 : TREE_TYPE (TREE_TYPE (def_rhs)),
895 : unshare_expr (def_rhs),
896 : fold_convert (ptr_type_node,
897 : rhs2)));
898 0 : gimple_assign_set_rhs_from_tree (use_stmt_gsi, new_rhs);
899 0 : use_stmt = gsi_stmt (*use_stmt_gsi);
900 0 : update_stmt (use_stmt);
901 0 : tidy_after_forward_propagate_addr (use_stmt);
902 0 : return true;
903 : }
904 :
905 : return false;
906 : }
907 :
908 : /* STMT is a statement of the form SSA_NAME = ADDR_EXPR <whatever>.
909 :
910 : Try to forward propagate the ADDR_EXPR into all uses of the SSA_NAME.
911 : Often this will allow for removal of an ADDR_EXPR and INDIRECT_REF
912 : node or for recovery of array indexing from pointer arithmetic.
913 :
914 : PARENT_SINGLE_USE_P tells if, when in a recursive invocation, NAME was
915 : the single use in the previous invocation. Pass true when calling
916 : this as toplevel.
917 :
918 : Returns true, if all uses have been propagated into. */
919 :
920 : static bool
921 3318841 : forward_propagate_addr_expr (tree name, tree rhs, bool parent_single_use_p)
922 : {
923 3318841 : bool all = true;
924 3318841 : bool single_use_p = parent_single_use_p && has_single_use (name);
925 :
926 17519381 : for (gimple *use_stmt : gather_imm_use_stmts (name))
927 : {
928 7562858 : bool result;
929 7562858 : tree use_rhs;
930 :
931 : /* If the use is not in a simple assignment statement, then
932 : there is nothing we can do. */
933 7562858 : if (!is_gimple_assign (use_stmt))
934 : {
935 4699750 : if (!is_gimple_debug (use_stmt))
936 1932735 : all = false;
937 4699750 : continue;
938 : }
939 :
940 2863108 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
941 2863108 : result = forward_propagate_addr_expr_1 (name, rhs, &gsi,
942 : single_use_p);
943 : /* If the use has moved to a different statement adjust
944 : the update machinery for the old statement too. */
945 2863108 : if (use_stmt != gsi_stmt (gsi))
946 : {
947 0 : update_stmt (use_stmt);
948 0 : use_stmt = gsi_stmt (gsi);
949 : }
950 2863108 : update_stmt (use_stmt);
951 2863108 : all &= result;
952 :
953 : /* Remove intermediate now unused copy and conversion chains. */
954 2863108 : use_rhs = gimple_assign_rhs1 (use_stmt);
955 2863108 : if (result
956 1498394 : && TREE_CODE (gimple_assign_lhs (use_stmt)) == SSA_NAME
957 1251739 : && TREE_CODE (use_rhs) == SSA_NAME
958 2944517 : && has_zero_uses (gimple_assign_lhs (use_stmt)))
959 : {
960 81409 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
961 81409 : fwprop_invalidate_lattice (gimple_get_lhs (use_stmt));
962 81409 : release_defs (use_stmt);
963 81409 : gsi_remove (&gsi, true);
964 : }
965 3318841 : }
966 :
967 3318841 : return all && has_zero_uses (name);
968 : }
969 :
970 :
971 : /* Helper function for simplify_gimple_switch. Remove case labels that
972 : have values outside the range of the new type. */
973 :
974 : static void
975 11802 : simplify_gimple_switch_label_vec (gswitch *stmt, tree index_type,
976 : vec<std::pair<int, int> > &edges_to_remove)
977 : {
978 11802 : unsigned int branch_num = gimple_switch_num_labels (stmt);
979 11802 : auto_vec<tree> labels (branch_num);
980 11802 : unsigned int i, len;
981 :
982 : /* Collect the existing case labels in a VEC, and preprocess it as if
983 : we are gimplifying a GENERIC SWITCH_EXPR. */
984 75206 : for (i = 1; i < branch_num; i++)
985 51602 : labels.quick_push (gimple_switch_label (stmt, i));
986 11802 : preprocess_case_label_vec_for_gimple (labels, index_type, NULL);
987 :
988 : /* If any labels were removed, replace the existing case labels
989 : in the GIMPLE_SWITCH statement with the correct ones.
990 : Note that the type updates were done in-place on the case labels,
991 : so we only have to replace the case labels in the GIMPLE_SWITCH
992 : if the number of labels changed. */
993 11802 : len = labels.length ();
994 11802 : if (len < branch_num - 1)
995 : {
996 0 : bitmap target_blocks;
997 0 : edge_iterator ei;
998 0 : edge e;
999 :
1000 : /* Corner case: *all* case labels have been removed as being
1001 : out-of-range for INDEX_TYPE. Push one label and let the
1002 : CFG cleanups deal with this further. */
1003 0 : if (len == 0)
1004 : {
1005 0 : tree label, elt;
1006 :
1007 0 : label = CASE_LABEL (gimple_switch_default_label (stmt));
1008 0 : elt = build_case_label (build_int_cst (index_type, 0), NULL, label);
1009 0 : labels.quick_push (elt);
1010 0 : len = 1;
1011 : }
1012 :
1013 0 : for (i = 0; i < labels.length (); i++)
1014 0 : gimple_switch_set_label (stmt, i + 1, labels[i]);
1015 0 : for (i++ ; i < branch_num; i++)
1016 0 : gimple_switch_set_label (stmt, i, NULL_TREE);
1017 0 : gimple_switch_set_num_labels (stmt, len + 1);
1018 :
1019 : /* Cleanup any edges that are now dead. */
1020 0 : target_blocks = BITMAP_ALLOC (NULL);
1021 0 : for (i = 0; i < gimple_switch_num_labels (stmt); i++)
1022 : {
1023 0 : tree elt = gimple_switch_label (stmt, i);
1024 0 : basic_block target = label_to_block (cfun, CASE_LABEL (elt));
1025 0 : bitmap_set_bit (target_blocks, target->index);
1026 : }
1027 0 : for (ei = ei_start (gimple_bb (stmt)->succs); (e = ei_safe_edge (ei)); )
1028 : {
1029 0 : if (! bitmap_bit_p (target_blocks, e->dest->index))
1030 0 : edges_to_remove.safe_push (std::make_pair (e->src->index,
1031 0 : e->dest->index));
1032 : else
1033 0 : ei_next (&ei);
1034 : }
1035 0 : BITMAP_FREE (target_blocks);
1036 : }
1037 11802 : }
1038 :
1039 : /* STMT is a SWITCH_EXPR for which we attempt to find equivalent forms of
1040 : the condition which we may be able to optimize better. */
1041 :
1042 : static bool
1043 111600 : simplify_gimple_switch (gswitch *stmt,
1044 : vec<std::pair<int, int> > &edges_to_remove,
1045 : bitmap simple_dce_worklist)
1046 : {
1047 : /* The optimization that we really care about is removing unnecessary
1048 : casts. That will let us do much better in propagating the inferred
1049 : constant at the switch target. */
1050 111600 : tree cond = gimple_switch_index (stmt);
1051 111600 : if (TREE_CODE (cond) == SSA_NAME)
1052 : {
1053 111599 : gimple *def_stmt = SSA_NAME_DEF_STMT (cond);
1054 111599 : if (gimple_assign_cast_p (def_stmt))
1055 : {
1056 12779 : tree def = gimple_assign_rhs1 (def_stmt);
1057 12779 : if (TREE_CODE (def) != SSA_NAME)
1058 : return false;
1059 :
1060 : /* If we have an extension or sign-change that preserves the
1061 : values we check against then we can copy the source value into
1062 : the switch. */
1063 12779 : tree ti = TREE_TYPE (def);
1064 12779 : if (INTEGRAL_TYPE_P (ti)
1065 12779 : && TYPE_PRECISION (ti) <= TYPE_PRECISION (TREE_TYPE (cond)))
1066 : {
1067 12509 : size_t n = gimple_switch_num_labels (stmt);
1068 12509 : tree min = NULL_TREE, max = NULL_TREE;
1069 12509 : if (n > 1)
1070 : {
1071 12509 : min = CASE_LOW (gimple_switch_label (stmt, 1));
1072 12509 : if (CASE_HIGH (gimple_switch_label (stmt, n - 1)))
1073 186 : max = CASE_HIGH (gimple_switch_label (stmt, n - 1));
1074 : else
1075 12323 : max = CASE_LOW (gimple_switch_label (stmt, n - 1));
1076 : }
1077 12509 : if ((!min || int_fits_type_p (min, ti))
1078 12505 : && (!max || int_fits_type_p (max, ti)))
1079 : {
1080 11802 : bitmap_set_bit (simple_dce_worklist,
1081 11802 : SSA_NAME_VERSION (cond));
1082 11802 : gimple_switch_set_index (stmt, def);
1083 11802 : simplify_gimple_switch_label_vec (stmt, ti,
1084 : edges_to_remove);
1085 11802 : update_stmt (stmt);
1086 11802 : return true;
1087 : }
1088 : }
1089 : }
1090 : }
1091 :
1092 : return false;
1093 : }
1094 :
1095 : /* For pointers p2 and p1 return p2 - p1 if the
1096 : difference is known and constant, otherwise return NULL. */
1097 :
1098 : static tree
1099 5505 : constant_pointer_difference (tree p1, tree p2)
1100 : {
1101 5505 : int i, j;
1102 : #define CPD_ITERATIONS 5
1103 5505 : tree exps[2][CPD_ITERATIONS];
1104 5505 : tree offs[2][CPD_ITERATIONS];
1105 5505 : int cnt[2];
1106 :
1107 16515 : for (i = 0; i < 2; i++)
1108 : {
1109 11010 : tree p = i ? p1 : p2;
1110 11010 : tree off = size_zero_node;
1111 11010 : gimple *stmt;
1112 11010 : enum tree_code code;
1113 :
1114 : /* For each of p1 and p2 we need to iterate at least
1115 : twice, to handle ADDR_EXPR directly in p1/p2,
1116 : SSA_NAME with ADDR_EXPR or POINTER_PLUS_EXPR etc.
1117 : on definition's stmt RHS. Iterate a few extra times. */
1118 11010 : j = 0;
1119 12800 : do
1120 : {
1121 12800 : if (!POINTER_TYPE_P (TREE_TYPE (p)))
1122 : break;
1123 12794 : if (TREE_CODE (p) == ADDR_EXPR)
1124 : {
1125 9647 : tree q = TREE_OPERAND (p, 0);
1126 9647 : poly_int64 offset;
1127 9647 : tree base = get_addr_base_and_unit_offset (q, &offset);
1128 9647 : if (base)
1129 : {
1130 8855 : q = base;
1131 8855 : if (maybe_ne (offset, 0))
1132 3778 : off = size_binop (PLUS_EXPR, off, size_int (offset));
1133 : }
1134 9647 : if (TREE_CODE (q) == MEM_REF
1135 9647 : && TREE_CODE (TREE_OPERAND (q, 0)) == SSA_NAME)
1136 : {
1137 155 : p = TREE_OPERAND (q, 0);
1138 155 : off = size_binop (PLUS_EXPR, off,
1139 : wide_int_to_tree (sizetype,
1140 : mem_ref_offset (q)));
1141 : }
1142 : else
1143 : {
1144 9492 : exps[i][j] = q;
1145 9492 : offs[i][j++] = off;
1146 9492 : break;
1147 : }
1148 : }
1149 3302 : if (TREE_CODE (p) != SSA_NAME)
1150 : break;
1151 3302 : exps[i][j] = p;
1152 3302 : offs[i][j++] = off;
1153 3302 : if (j == CPD_ITERATIONS)
1154 : break;
1155 3302 : stmt = SSA_NAME_DEF_STMT (p);
1156 3302 : if (!is_gimple_assign (stmt) || gimple_assign_lhs (stmt) != p)
1157 : break;
1158 2643 : code = gimple_assign_rhs_code (stmt);
1159 2643 : if (code == POINTER_PLUS_EXPR)
1160 : {
1161 1390 : if (TREE_CODE (gimple_assign_rhs2 (stmt)) != INTEGER_CST)
1162 : break;
1163 863 : off = size_binop (PLUS_EXPR, off, gimple_assign_rhs2 (stmt));
1164 863 : p = gimple_assign_rhs1 (stmt);
1165 : }
1166 : else if (code == ADDR_EXPR || CONVERT_EXPR_CODE_P (code))
1167 927 : p = gimple_assign_rhs1 (stmt);
1168 : else
1169 : break;
1170 : }
1171 : while (1);
1172 11010 : cnt[i] = j;
1173 : }
1174 :
1175 7489 : for (i = 0; i < cnt[0]; i++)
1176 9744 : for (j = 0; j < cnt[1]; j++)
1177 7760 : if (exps[0][i] == exps[1][j])
1178 4620 : return size_binop (MINUS_EXPR, offs[0][i], offs[1][j]);
1179 :
1180 : return NULL_TREE;
1181 : }
1182 :
1183 : /* Helper function for optimize_aggr_zeroprop.
1184 : Props the zeroing (memset, VAL) that was done in DEST+OFFSET:LEN
1185 : (DEFSTMT) into the STMT. Returns true if the STMT was updated. */
1186 : static void
1187 22640795 : optimize_aggr_zeroprop_1 (gimple *defstmt, gimple *stmt,
1188 : tree dest, poly_int64 offset, tree val,
1189 : poly_offset_int len)
1190 : {
1191 22640795 : tree src2;
1192 22640795 : tree len2 = NULL_TREE;
1193 22640795 : poly_int64 offset2;
1194 :
1195 22640795 : if (gimple_call_builtin_p (stmt, BUILT_IN_MEMCPY)
1196 22324 : && TREE_CODE (gimple_call_arg (stmt, 1)) == ADDR_EXPR
1197 22654968 : && poly_int_tree_p (gimple_call_arg (stmt, 2)))
1198 : {
1199 13056 : src2 = TREE_OPERAND (gimple_call_arg (stmt, 1), 0);
1200 13056 : len2 = gimple_call_arg (stmt, 2);
1201 : }
1202 22627739 : else if (gimple_assign_load_p (stmt) && gimple_store_p (stmt))
1203 : {
1204 1914741 : src2 = gimple_assign_rhs1 (stmt);
1205 1914741 : len2 = (TREE_CODE (src2) == COMPONENT_REF
1206 1914741 : ? DECL_SIZE_UNIT (TREE_OPERAND (src2, 1))
1207 1746391 : : TYPE_SIZE_UNIT (TREE_TYPE (src2)));
1208 : /* Can only handle zero memsets. */
1209 1914741 : if (!integer_zerop (val))
1210 22619537 : return;
1211 : }
1212 : else
1213 : return;
1214 :
1215 1926674 : if (len2 == NULL_TREE
1216 1926674 : || !poly_int_tree_p (len2))
1217 : return;
1218 :
1219 1926674 : src2 = get_addr_base_and_unit_offset (src2, &offset2);
1220 1926674 : if (src2 == NULL_TREE
1221 1926674 : || maybe_lt (offset2, offset))
1222 : return;
1223 :
1224 877339 : if (!operand_equal_p (dest, src2, 0))
1225 : return;
1226 :
1227 : /* [ dest + offset, dest + offset + len - 1 ] is set to val.
1228 : Make sure that
1229 : [ dest + offset2, dest + offset2 + len2 - 1 ] is a subset of that. */
1230 131196 : if (maybe_gt (wi::to_poly_offset (len2) + (offset2 - offset),
1231 : len))
1232 : return;
1233 :
1234 21258 : if (dump_file && (dump_flags & TDF_DETAILS))
1235 : {
1236 32 : fprintf (dump_file, "Simplified\n ");
1237 32 : print_gimple_stmt (dump_file, stmt, 0, dump_flags);
1238 32 : fprintf (dump_file, "after previous\n ");
1239 32 : print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
1240 : }
1241 21258 : gimple *orig_stmt = stmt;
1242 : /* For simplicity, don't change the kind of the stmt,
1243 : turn dest = src; into dest = {}; and memcpy (&dest, &src, len);
1244 : into memset (&dest, val, len);
1245 : In theory we could change dest = src into memset if dest
1246 : is addressable (maybe beneficial if val is not 0), or
1247 : memcpy (&dest, &src, len) into dest = {} if len is the size
1248 : of dest, dest isn't volatile. */
1249 21258 : if (is_gimple_assign (stmt))
1250 : {
1251 21253 : tree ctor_type = TREE_TYPE (gimple_assign_lhs (stmt));
1252 21253 : tree ctor = build_constructor (ctor_type, NULL);
1253 21253 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1254 21253 : gimple_assign_set_rhs_from_tree (&gsi, ctor);
1255 21253 : update_stmt (stmt);
1256 21253 : statistics_counter_event (cfun, "copy zeroing propagation of aggregate", 1);
1257 : }
1258 : else /* If stmt is memcpy, transform it into memset. */
1259 : {
1260 5 : gcall *call = as_a <gcall *> (stmt);
1261 5 : tree fndecl = builtin_decl_implicit (BUILT_IN_MEMSET);
1262 5 : gimple_call_set_fndecl (call, fndecl);
1263 5 : gimple_call_set_fntype (call, TREE_TYPE (fndecl));
1264 5 : gimple_call_set_arg (call, 1, val);
1265 5 : update_stmt (stmt);
1266 5 : statistics_counter_event (cfun, "memcpy to memset changed", 1);
1267 : }
1268 :
1269 21258 : if (dump_file && (dump_flags & TDF_DETAILS))
1270 : {
1271 32 : fprintf (dump_file, "into\n ");
1272 32 : print_gimple_stmt (dump_file, stmt, 0, dump_flags);
1273 : }
1274 :
1275 : /* Mark the bb for eh cleanup if needed. */
1276 21258 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
1277 6 : bitmap_set_bit (to_purge, gimple_bb (stmt)->index);
1278 : }
1279 :
1280 : /* Optimize
1281 : a = {}; // DEST = value ;; LEN(nullptr)
1282 : b = a;
1283 : into
1284 : a = {};
1285 : b = {};
1286 : Similarly for memset (&a, ..., sizeof (a)); instead of a = {};
1287 : and/or memcpy (&b, &a, sizeof (a)); instead of b = a; */
1288 :
1289 : static void
1290 31975584 : optimize_aggr_zeroprop (gimple *stmt, bool full_walk)
1291 : {
1292 31975584 : ao_ref read;
1293 63951168 : if (gimple_has_volatile_ops (stmt))
1294 27907779 : return;
1295 :
1296 30818560 : tree dest = NULL_TREE;
1297 30818560 : tree val = integer_zero_node;
1298 30818560 : tree len = NULL_TREE;
1299 30818560 : bool can_use_tbba = true;
1300 :
1301 30818560 : if (gimple_call_builtin_p (stmt, BUILT_IN_MEMSET)
1302 113245 : && TREE_CODE (gimple_call_arg (stmt, 0)) == ADDR_EXPR
1303 58030 : && TREE_CODE (gimple_call_arg (stmt, 1)) == INTEGER_CST
1304 30874287 : && poly_int_tree_p (gimple_call_arg (stmt, 2)))
1305 : {
1306 52960 : dest = TREE_OPERAND (gimple_call_arg (stmt, 0), 0);
1307 52960 : len = gimple_call_arg (stmt, 2);
1308 52960 : val = gimple_call_arg (stmt, 1);
1309 52960 : ao_ref_init_from_ptr_and_size (&read, gimple_call_arg (stmt, 0), len);
1310 52960 : can_use_tbba = false;
1311 : }
1312 30765600 : else if (gimple_store_p (stmt)
1313 30705155 : && gimple_assign_single_p (stmt)
1314 61470755 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == STRING_CST)
1315 : {
1316 31000 : tree str = gimple_assign_rhs1 (stmt);
1317 31000 : dest = gimple_assign_lhs (stmt);
1318 31000 : ao_ref_init (&read, dest);
1319 : /* The string must contain all null char's for now. */
1320 67086 : for (int i = 0; i < TREE_STRING_LENGTH (str); i++)
1321 : {
1322 33699 : if (TREE_STRING_POINTER (str)[i] != 0)
1323 : {
1324 : dest = NULL_TREE;
1325 : break;
1326 : }
1327 : }
1328 : }
1329 : /* A store of integer (scalar, vector or complex) zeros is
1330 : a zero store. */
1331 30734600 : else if (gimple_store_p (stmt)
1332 30674155 : && gimple_assign_single_p (stmt)
1333 61408755 : && integer_zerop (gimple_assign_rhs1 (stmt)))
1334 : {
1335 3577264 : tree rhs = gimple_assign_rhs1 (stmt);
1336 3577264 : tree type = TREE_TYPE (rhs);
1337 3577264 : dest = gimple_assign_lhs (stmt);
1338 3577264 : ao_ref_init (&read, dest);
1339 : /* For integral types, the type precision needs to be a multiply of BITS_PER_UNIT. */
1340 3577264 : if (INTEGRAL_TYPE_P (type)
1341 3577264 : && (TYPE_PRECISION (type) % BITS_PER_UNIT) != 0)
1342 : dest = NULL_TREE;
1343 : }
1344 27157336 : else if (gimple_store_p (stmt)
1345 27096891 : && gimple_assign_single_p (stmt)
1346 27096891 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == CONSTRUCTOR
1347 27891958 : && !gimple_clobber_p (stmt))
1348 : {
1349 734622 : dest = gimple_assign_lhs (stmt);
1350 734622 : ao_ref_init (&read, dest);
1351 : }
1352 :
1353 4178584 : if (dest == NULL_TREE)
1354 : return;
1355 :
1356 4149971 : if (len == NULL_TREE)
1357 4097011 : len = (TREE_CODE (dest) == COMPONENT_REF
1358 4097011 : ? DECL_SIZE_UNIT (TREE_OPERAND (dest, 1))
1359 1788033 : : TYPE_SIZE_UNIT (TREE_TYPE (dest)));
1360 4097011 : if (len == NULL_TREE
1361 4149971 : || !poly_int_tree_p (len))
1362 : return;
1363 :
1364 : /* Sometimes memset can have no vdef due to invalid declaration of memset (const, etc.). */
1365 36207697 : if (!gimple_vdef (stmt))
1366 : return;
1367 :
1368 : /* This store needs to be on the byte boundary and pointing to an object. */
1369 4149947 : poly_int64 offset;
1370 4149947 : tree dest_base = get_addr_base_and_unit_offset (dest, &offset);
1371 4149947 : if (dest_base == NULL_TREE)
1372 : return;
1373 :
1374 : /* Setup the worklist. */
1375 4067805 : auto_vec<std::pair<tree, unsigned>> worklist;
1376 4067805 : unsigned limit = full_walk ? param_sccvn_max_alias_queries_per_access : 0;
1377 8135610 : worklist.safe_push (std::make_pair (gimple_vdef (stmt), limit));
1378 :
1379 28240890 : while (!worklist.is_empty ())
1380 : {
1381 20105280 : std::pair<tree, unsigned> top = worklist.pop ();
1382 20105280 : tree vdef = top.first;
1383 20105280 : limit = top.second;
1384 20105280 : gimple *use_stmt;
1385 20105280 : imm_use_iterator iter;
1386 44794284 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
1387 : {
1388 : /* Handling PHI nodes might not be worth it so don't. */
1389 24689004 : if (is_a <gphi*> (use_stmt))
1390 2048209 : continue;
1391 :
1392 : /* If this statement does not clobber add the vdef stmt to the
1393 : worklist.
1394 : After hitting the limit, allow clobbers to able to pass through. */
1395 2059137 : if ((limit != 0 || gimple_clobber_p (use_stmt))
1396 20618171 : && gimple_vdef (use_stmt)
1397 40123648 : && !stmt_may_clobber_ref_p_1 (use_stmt, &read,
1398 : /* tbaa_p = */ can_use_tbba))
1399 : {
1400 16037475 : unsigned new_limit = limit == 0 ? 0 : limit - 1;
1401 32074950 : worklist.safe_push (std::make_pair (gimple_vdef (use_stmt),
1402 : new_limit));
1403 : }
1404 :
1405 22640795 : optimize_aggr_zeroprop_1 (stmt, use_stmt, dest_base, offset,
1406 22640795 : val, wi::to_poly_offset (len));
1407 20105280 : }
1408 : }
1409 :
1410 4067805 : }
1411 :
1412 : /* Returns the pointer to the base of the object of the
1413 : reference EXPR and extracts the information about
1414 : the offset of the access, storing it to PBYTESIZE,
1415 : PBYTEPOS and PREVERSEP.
1416 : If the access is not a byte sized or position is not
1417 : on the byte, return NULL. */
1418 : static tree
1419 5366798 : split_core_and_offset_size (tree expr,
1420 : poly_int64 *pbytesize, poly_int64 *pbytepos,
1421 : tree *poffset, int *preversep)
1422 : {
1423 5366798 : tree core;
1424 5366798 : machine_mode mode;
1425 5366798 : int unsignedp, volatilep;
1426 5366798 : poly_int64 bitsize;
1427 5366798 : poly_int64 bitpos;
1428 5366798 : location_t loc = EXPR_LOCATION (expr);
1429 :
1430 5366798 : core = get_inner_reference (expr, &bitsize, &bitpos,
1431 : poffset, &mode, &unsignedp, preversep,
1432 : &volatilep);
1433 10733596 : if (!multiple_p (bitsize, BITS_PER_UNIT, pbytesize))
1434 : return NULL_TREE;
1435 5366798 : if (!multiple_p (bitpos, BITS_PER_UNIT, pbytepos))
1436 : return NULL_TREE;
1437 : /* If we are left with MEM[a + CST] strip that and add it to the
1438 : pbytepos and return a. */
1439 5366798 : if (TREE_CODE (core) == MEM_REF)
1440 : {
1441 1261545 : poly_offset_int tem;
1442 1261545 : tem = wi::to_poly_offset (TREE_OPERAND (core, 1));
1443 1261545 : tem += *pbytepos;
1444 1261545 : if (tem.to_shwi (pbytepos))
1445 1259306 : return TREE_OPERAND (core, 0);
1446 : }
1447 4107492 : core = build_fold_addr_expr_loc (loc, core);
1448 4107492 : STRIP_NOPS (core);
1449 4107492 : return core;
1450 : }
1451 :
1452 : /* Returns a new src based on the
1453 : copy `DEST = SRC` and for the old SRC2.
1454 : Returns null if SRC2 is not related to DEST. */
1455 :
1456 : static tree
1457 1282157 : new_src_based_on_copy (tree src2, tree dest, tree src)
1458 : {
1459 : /* If the second src is not exactly the same as dest,
1460 : try to handle it separately; see it is address/size equivalent.
1461 : Handles `a` and `a.b` and `MEM<char[N]>(&a)` which all have
1462 : the same size and offsets as address/size equivalent.
1463 : This allows copying over a memcpy and also one for copying
1464 : where one field is the same size as the whole struct. */
1465 1282157 : if (operand_equal_p (dest, src2))
1466 : return src;
1467 : /* if both dest and src2 are decls, then we know these 2
1468 : accesses can't be the same. */
1469 731307 : if (DECL_P (dest) && DECL_P (src2))
1470 : return NULL_TREE;
1471 : /* A VCE can't be used with imag/real or BFR so reject them early. */
1472 383284 : if (TREE_CODE (src) == IMAGPART_EXPR
1473 383284 : || TREE_CODE (src) == REALPART_EXPR
1474 383284 : || TREE_CODE (src) == BIT_FIELD_REF)
1475 : return NULL_TREE;
1476 383284 : tree core1, core2;
1477 383284 : poly_int64 bytepos1, bytepos2;
1478 383284 : poly_int64 bytesize1, bytesize2;
1479 383284 : tree toffset1, toffset2;
1480 383284 : int reversep1 = 0;
1481 383284 : int reversep2 = 0;
1482 383284 : poly_int64 diff = 0;
1483 383284 : core1 = split_core_and_offset_size (dest, &bytesize1, &bytepos1,
1484 : &toffset1, &reversep1);
1485 383284 : core2 = split_core_and_offset_size (src2, &bytesize2, &bytepos2,
1486 : &toffset2, &reversep2);
1487 383284 : if (!core1 || !core2)
1488 : return NULL_TREE;
1489 383284 : if (reversep1 != reversep2)
1490 : return NULL_TREE;
1491 : /* The sizes of the 2 accesses need to be the same. */
1492 383284 : if (!known_eq (bytesize1, bytesize2))
1493 : return NULL_TREE;
1494 175875 : if (!operand_equal_p (core1, core2, 0))
1495 : return NULL_TREE;
1496 :
1497 23612 : if (toffset1 && toffset2)
1498 : {
1499 2 : tree type = TREE_TYPE (toffset1);
1500 2 : if (type != TREE_TYPE (toffset2))
1501 0 : toffset2 = fold_convert (type, toffset2);
1502 :
1503 2 : tree tdiff = fold_build2 (MINUS_EXPR, type, toffset1, toffset2);
1504 2 : if (!cst_and_fits_in_hwi (tdiff))
1505 : return NULL_TREE;
1506 :
1507 0 : diff = int_cst_value (tdiff);
1508 0 : }
1509 23610 : else if (toffset1 || toffset2)
1510 : {
1511 : /* If only one of the offsets is non-constant, the difference cannot
1512 : be a constant. */
1513 : return NULL_TREE;
1514 : }
1515 23578 : diff += bytepos1 - bytepos2;
1516 : /* The offset between the 2 need to be 0. */
1517 23578 : if (!known_eq (diff, 0))
1518 : return NULL_TREE;
1519 22740 : return fold_build1 (VIEW_CONVERT_EXPR,TREE_TYPE (src2), src);
1520 : }
1521 :
1522 : /* Returns true if SRC and DEST are the same address such that
1523 : `SRC == DEST;` is considered a nop. This is more than an
1524 : operand_equal_p check as it needs to be similar to
1525 : new_src_based_on_copy. */
1526 :
1527 : static bool
1528 4553242 : same_for_assignment (tree src, tree dest)
1529 : {
1530 4553242 : if (operand_equal_p (dest, src, 0))
1531 : return true;
1532 : /* if both dest and src2 are decls, then we know these 2
1533 : accesses can't be the same. */
1534 4550333 : if (DECL_P (dest) && DECL_P (src))
1535 : return false;
1536 :
1537 2300115 : tree core1, core2;
1538 2300115 : poly_int64 bytepos1, bytepos2;
1539 2300115 : poly_int64 bytesize1, bytesize2;
1540 2300115 : tree toffset1, toffset2;
1541 2300115 : int reversep1 = 0;
1542 2300115 : int reversep2 = 0;
1543 2300115 : poly_int64 diff = 0;
1544 2300115 : core1 = split_core_and_offset_size (dest, &bytesize1, &bytepos1,
1545 : &toffset1, &reversep1);
1546 2300115 : core2 = split_core_and_offset_size (src, &bytesize2, &bytepos2,
1547 : &toffset2, &reversep2);
1548 2300115 : if (!core1 || !core2)
1549 : return false;
1550 2300115 : if (reversep1 != reversep2)
1551 : return false;
1552 : /* The sizes of the 2 accesses need to be the same. */
1553 2300115 : if (!known_eq (bytesize1, bytesize2))
1554 : return false;
1555 2299206 : if (!operand_equal_p (core1, core2, 0))
1556 : return false;
1557 6194 : if (toffset1 && toffset2)
1558 : {
1559 313 : tree type = TREE_TYPE (toffset1);
1560 313 : if (type != TREE_TYPE (toffset2))
1561 0 : toffset2 = fold_convert (type, toffset2);
1562 :
1563 313 : tree tdiff = fold_build2 (MINUS_EXPR, type, toffset1, toffset2);
1564 313 : if (!cst_and_fits_in_hwi (tdiff))
1565 : return false;
1566 :
1567 0 : diff = int_cst_value (tdiff);
1568 0 : }
1569 5881 : else if (toffset1 || toffset2)
1570 : {
1571 : /* If only one of the offsets is non-constant, the difference cannot
1572 : be a constant. */
1573 : return false;
1574 : }
1575 5881 : diff += bytepos1 - bytepos2;
1576 : /* The offset between the 2 need to be 0. */
1577 5881 : if (!known_eq (diff, 0))
1578 5635 : return false;
1579 : return true;
1580 : }
1581 :
1582 : /* Helper function for optimize_agr_copyprop.
1583 : For aggregate copies in USE_STMT, see if DEST
1584 : is on the lhs of USE_STMT and replace it with SRC. */
1585 : static void
1586 1036597 : optimize_agr_copyprop_1 (gimple *stmt, gimple *use_stmt,
1587 : tree dest, tree src)
1588 : {
1589 1036597 : gcc_assert (gimple_assign_load_p (use_stmt)
1590 : && gimple_store_p (use_stmt));
1591 2073194 : if (gimple_has_volatile_ops (use_stmt))
1592 621053 : return;
1593 1036596 : tree dest2 = gimple_assign_lhs (use_stmt);
1594 1036596 : tree src2 = gimple_assign_rhs1 (use_stmt);
1595 : /* If the new store is `src2 = src2;` skip over it. */
1596 1036596 : if (same_for_assignment (src2, dest2))
1597 : return;
1598 1036033 : src = new_src_based_on_copy (src2, dest, src);
1599 1036033 : if (!src)
1600 : return;
1601 : /* For 2 memory references and using a temporary to do the copy,
1602 : don't remove the temporary as the 2 memory references might overlap.
1603 : Note t does not need to be decl as it could be field.
1604 : See PR 22237 for full details.
1605 : E.g.
1606 : t = *a; #DEST = SRC;
1607 : *b = t; #DEST2 = SRC2;
1608 : Cannot be convert into
1609 : t = *a;
1610 : *b = *a;
1611 : Though the following is allowed to be done:
1612 : t = *a;
1613 : *a = t;
1614 : And convert it into:
1615 : t = *a;
1616 : *a = *a;
1617 : */
1618 444545 : if (!operand_equal_p (dest2, src, 0)
1619 444545 : && !DECL_P (dest2) && !DECL_P (src))
1620 : {
1621 : /* If *a and *b have the same base see if
1622 : the offset between the two is greater than
1623 : or equal to the size of the type. */
1624 32379 : poly_int64 offset1, offset2;
1625 32379 : tree len = TYPE_SIZE_UNIT (TREE_TYPE (src));
1626 32379 : if (len == NULL_TREE
1627 32379 : || !tree_fits_poly_int64_p (len))
1628 29001 : return;
1629 32379 : tree base1 = get_addr_base_and_unit_offset (dest2, &offset1);
1630 32379 : tree base2 = get_addr_base_and_unit_offset (src, &offset2);
1631 32379 : poly_int64 size = tree_to_poly_int64 (len);
1632 : /* If the bases are 2 different decls,
1633 : then there can be no overlapping. */
1634 32379 : if (base1 && base2
1635 31448 : && DECL_P (base1) && DECL_P (base2)
1636 1874 : && base1 != base2)
1637 : ;
1638 : /* If we can't figure out the base or the bases are
1639 : not equal then fall back to an alignment check. */
1640 30734 : else if (!base1
1641 30734 : || !base2
1642 30734 : || !operand_equal_p (base1, base2))
1643 : {
1644 30362 : unsigned int align1 = get_object_alignment (src);
1645 30362 : unsigned int align2 = get_object_alignment (dest2);
1646 30362 : align1 /= BITS_PER_UNIT;
1647 30362 : align2 /= BITS_PER_UNIT;
1648 : /* If the alignment of either object is less
1649 : than the size then there is a possibility
1650 : of overlapping. */
1651 30362 : if (maybe_lt (align1, size)
1652 30362 : || maybe_lt (align2, size))
1653 29001 : return;
1654 : }
1655 : /* Make sure [offset1, offset1 + len - 1] does
1656 : not overlap with [offset2, offset2 + len - 1],
1657 : it is ok if they are at the same location though. */
1658 372 : else if (ranges_maybe_overlap_p (offset1, size, offset2, size)
1659 372 : && !known_eq (offset2, offset1))
1660 : return;
1661 : }
1662 :
1663 415544 : if (dump_file && (dump_flags & TDF_DETAILS))
1664 : {
1665 11 : fprintf (dump_file, "Simplified\n ");
1666 11 : print_gimple_stmt (dump_file, use_stmt, 0, dump_flags);
1667 11 : fprintf (dump_file, "after previous\n ");
1668 11 : print_gimple_stmt (dump_file, stmt, 0, dump_flags);
1669 : }
1670 415544 : gimple *orig_stmt = use_stmt;
1671 415544 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
1672 415544 : gimple_assign_set_rhs_from_tree (&gsi, unshare_expr (src));
1673 415544 : update_stmt (use_stmt);
1674 :
1675 415544 : if (dump_file && (dump_flags & TDF_DETAILS))
1676 : {
1677 11 : fprintf (dump_file, "into\n ");
1678 11 : print_gimple_stmt (dump_file, use_stmt, 0, dump_flags);
1679 : }
1680 415544 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, use_stmt))
1681 0 : bitmap_set_bit (to_purge, gimple_bb (use_stmt)->index);
1682 415544 : statistics_counter_event (cfun, "copy prop for aggregate", 1);
1683 : }
1684 :
1685 : /* Helper function for optimize_agr_copyprop_1, propagate aggregates
1686 : into the arguments of USE_STMT if the argument matches with DEST;
1687 : replacing it with SRC. */
1688 : static void
1689 715805 : optimize_agr_copyprop_arg (gimple *defstmt, gcall *call,
1690 : tree dest, tree src)
1691 : {
1692 715805 : bool changed = false;
1693 2398056 : for (unsigned arg = 0; arg < gimple_call_num_args (call); arg++)
1694 : {
1695 1682251 : tree *argptr = gimple_call_arg_ptr (call, arg);
1696 3170989 : if (TREE_CODE (*argptr) == SSA_NAME
1697 960295 : || is_gimple_min_invariant (*argptr)
1698 1875764 : || TYPE_VOLATILE (TREE_TYPE (*argptr)))
1699 1488738 : continue;
1700 193513 : tree newsrc = new_src_based_on_copy (*argptr, dest, src);
1701 193513 : if (!newsrc)
1702 116953 : continue;
1703 :
1704 76560 : if (dump_file && (dump_flags & TDF_DETAILS))
1705 : {
1706 9 : fprintf (dump_file, "Simplified\n ");
1707 9 : print_gimple_stmt (dump_file, call, 0, dump_flags);
1708 9 : fprintf (dump_file, "after previous\n ");
1709 9 : print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
1710 : }
1711 76560 : *argptr = unshare_expr (newsrc);
1712 76560 : changed = true;
1713 76560 : if (dump_file && (dump_flags & TDF_DETAILS))
1714 : {
1715 9 : fprintf (dump_file, "into\n ");
1716 9 : print_gimple_stmt (dump_file, call, 0, dump_flags);
1717 : }
1718 : }
1719 715805 : if (changed)
1720 76382 : update_stmt (call);
1721 715805 : }
1722 :
1723 : /* Helper function for optimize_agr_copyprop, propagate aggregates
1724 : into the return stmt USE if the operand of the return matches DEST;
1725 : replacing it with SRC. */
1726 : static void
1727 129353 : optimize_agr_copyprop_return (gimple *defstmt, greturn *use,
1728 : tree dest, tree src)
1729 : {
1730 129353 : tree rvalue = gimple_return_retval (use);
1731 129353 : if (!rvalue
1732 83523 : || TREE_CODE (rvalue) == SSA_NAME
1733 74577 : || is_gimple_min_invariant (rvalue)
1734 203507 : || TYPE_VOLATILE (TREE_TYPE (rvalue)))
1735 : return;
1736 :
1737 : /* `return <retval>;` is already the best it could be.
1738 : Likewise `return *<retval>_N(D)`. */
1739 74153 : if (TREE_CODE (rvalue) == RESULT_DECL
1740 74153 : || (TREE_CODE (rvalue) == MEM_REF
1741 0 : && TREE_CODE (TREE_OPERAND (rvalue, 0)) == SSA_NAME
1742 0 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (rvalue, 0)))
1743 : == RESULT_DECL))
1744 : return;
1745 52611 : tree newsrc = new_src_based_on_copy (rvalue, dest, src);
1746 52611 : if (!newsrc)
1747 : return;
1748 : /* Currently only support non-global vars.
1749 : See PR 124099 on enumtls not supporting expanding for GIMPLE_RETURN.
1750 : FIXME: could support VCEs too? */
1751 52485 : if (!VAR_P (newsrc) || is_global_var (newsrc))
1752 : return;
1753 26685 : if (dump_file && (dump_flags & TDF_DETAILS))
1754 : {
1755 1 : fprintf (dump_file, "Simplified\n ");
1756 1 : print_gimple_stmt (dump_file, use, 0, dump_flags);
1757 1 : fprintf (dump_file, "after previous\n ");
1758 1 : print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
1759 : }
1760 26685 : gimple_return_set_retval (use, newsrc);
1761 26685 : if (dump_file && (dump_flags & TDF_DETAILS))
1762 : {
1763 1 : fprintf (dump_file, "into\n ");
1764 1 : print_gimple_stmt (dump_file, use, 0, dump_flags);
1765 : }
1766 26685 : update_stmt (use);
1767 : }
1768 :
1769 : /* Optimizes
1770 : DEST = SRC;
1771 : DEST2 = DEST; # DEST2 = SRC2;
1772 : into
1773 : DEST = SRC;
1774 : DEST2 = SRC;
1775 : STMT is the first statement and SRC is the common
1776 : between the statements.
1777 :
1778 : Also optimizes:
1779 : DEST = SRC;
1780 : call_func(..., DEST, ...);
1781 : into:
1782 : DEST = SRC;
1783 : call_func(..., SRC, ...);
1784 :
1785 : */
1786 : static void
1787 3925834 : optimize_agr_copyprop (gimple *stmt)
1788 : {
1789 7851668 : if (gimple_has_volatile_ops (stmt))
1790 411780 : return;
1791 :
1792 : /* Can't prop if the statement could throw. */
1793 3924623 : if (stmt_could_throw_p (cfun, stmt))
1794 : return;
1795 :
1796 3516646 : tree dest = gimple_assign_lhs (stmt);
1797 3516646 : tree src = gimple_assign_rhs1 (stmt);
1798 : /* If the statement is `src = src;` then ignore it. */
1799 3516646 : if (same_for_assignment (dest, src))
1800 : return;
1801 :
1802 3514054 : tree vdef = gimple_vdef (stmt);
1803 3514054 : imm_use_iterator iter;
1804 3514054 : gimple *use_stmt;
1805 10133364 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
1806 : {
1807 6619310 : if (gimple_assign_load_p (use_stmt)
1808 6619310 : && gimple_store_p (use_stmt))
1809 1036597 : optimize_agr_copyprop_1 (stmt, use_stmt, dest, src);
1810 5582713 : else if (is_gimple_call (use_stmt))
1811 715805 : optimize_agr_copyprop_arg (stmt, as_a<gcall*>(use_stmt), dest, src);
1812 4866908 : else if (is_a<greturn*> (use_stmt))
1813 129353 : optimize_agr_copyprop_return (stmt, as_a<greturn*>(use_stmt), dest, src);
1814 3514054 : }
1815 : }
1816 :
1817 : /* Simple DSE of the lhs from a clobber STMT.
1818 : This is used mostly to clean up from optimize_agr_copyprop and
1819 : to remove (exactly one) extra copy that might later on confuse SRA.
1820 : An example is:
1821 : ;; write to a and such.
1822 : b = a; // This statement is to be removed
1823 : b = {CLOBBER};
1824 : SRA will totally scalarize b (which means also a) here for the extra copy
1825 : which is not something welcomed. So removing the copy will
1826 : allow SRA to move the scalarization of a further down or not at all.
1827 : */
1828 : static void
1829 7511610 : do_simple_agr_dse (gassign *stmt, bool full_walk)
1830 : {
1831 : /* Don't do this while in -Og as we want to keep around the copy
1832 : for debuggability. */
1833 7511610 : if (optimize_debug)
1834 5193770 : return;
1835 7508065 : ao_ref read;
1836 7508065 : basic_block bb = gimple_bb (stmt);
1837 7508065 : tree lhs = gimple_assign_lhs (stmt);
1838 : /* Only handle clobbers of a full decl. */
1839 7508065 : if (!DECL_P (lhs))
1840 : return;
1841 6781565 : ao_ref_init (&read, lhs);
1842 6781565 : tree vuse = gimple_vuse (stmt);
1843 6781565 : unsigned limit = full_walk ? param_sccvn_max_alias_queries_per_access : 4;
1844 17517203 : while (limit)
1845 : {
1846 17503942 : gimple *ostmt = SSA_NAME_DEF_STMT (vuse);
1847 : /* Don't handle phis, just declare to be done. */
1848 17503942 : if (is_a<gphi*>(ostmt) || gimple_nop_p (ostmt))
1849 : break;
1850 15199363 : basic_block obb = gimple_bb (ostmt);
1851 : /* If the clobber is not fully dominating the statement define,
1852 : then it is not "simple" to detect if the define is fully clobbered. */
1853 15199363 : if (obb != bb && !dominated_by_p (CDI_DOMINATORS, bb, obb))
1854 4463725 : return;
1855 15199363 : gimple *use_stmt;
1856 15199363 : imm_use_iterator iter;
1857 46414128 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, gimple_vdef (ostmt))
1858 : {
1859 18020450 : basic_block ubb = gimple_bb (use_stmt);
1860 18020450 : if (stmt == use_stmt)
1861 5217095 : continue;
1862 : /* If the use is a clobber for lhs,
1863 : then it can be safely skipped; this happens with eh
1864 : and sometimes jump threading. */
1865 12803355 : if (gimple_clobber_p (use_stmt)
1866 12803355 : && lhs == gimple_assign_lhs (use_stmt))
1867 176480 : continue;
1868 : /* If the use is a phi and it is single use then check if that single use
1869 : is a clobber and lhs is the same. */
1870 12626875 : if (gphi *use_phi = dyn_cast<gphi*>(use_stmt))
1871 : {
1872 378785 : use_operand_p ou;
1873 378785 : gimple *ostmt;
1874 378785 : if (single_imm_use (gimple_phi_result (use_phi), &ou, &ostmt)
1875 316627 : && gimple_clobber_p (ostmt)
1876 631759 : && lhs == gimple_assign_lhs (ostmt))
1877 69906 : continue;
1878 : /* A phi node will never be dominating the clobber. */
1879 308879 : return;
1880 : }
1881 : /* The use needs to be dominating the clobber. */
1882 1502122 : if ((ubb != bb && !dominated_by_p (CDI_DOMINATORS, bb, ubb))
1883 13003021 : || ref_maybe_used_by_stmt_p (use_stmt, &read, false))
1884 : return;
1885 : /* Count the above alias lookup towards the limit. */
1886 11024826 : limit--;
1887 11024826 : if (limit == 0)
1888 : return;
1889 2005048 : }
1890 13194315 : vuse = gimple_vuse (ostmt);
1891 : /* This is a call with an assignment to the clobber decl,
1892 : remove the lhs or the whole stmt if it was pure/const. */
1893 13194315 : if (is_a <gcall*>(ostmt)
1894 13194315 : && lhs == gimple_call_lhs (ostmt))
1895 : {
1896 : /* Don't remove stores/statements that are needed for non-call
1897 : eh to work. */
1898 4050 : if (stmt_unremovable_because_of_non_call_eh_p (cfun, ostmt))
1899 : return;
1900 : /* If we delete a stmt that could throw, mark the block
1901 : in to_purge to cleanup afterwards. */
1902 4044 : if (stmt_could_throw_p (cfun, ostmt))
1903 1035 : bitmap_set_bit (to_purge, obb->index);
1904 4044 : int flags = gimple_call_flags (ostmt);
1905 4044 : if ((flags & (ECF_PURE|ECF_CONST|ECF_NOVOPS))
1906 205 : && !(flags & (ECF_LOOPING_CONST_OR_PURE)))
1907 : {
1908 121 : gimple_stmt_iterator gsi = gsi_for_stmt (ostmt);
1909 121 : if (dump_file && (dump_flags & TDF_DETAILS))
1910 : {
1911 14 : fprintf (dump_file, "Removing dead call store stmt ");
1912 14 : print_gimple_stmt (dump_file, ostmt, 0);
1913 14 : fprintf (dump_file, "\n");
1914 : }
1915 121 : unlink_stmt_vdef (ostmt);
1916 121 : release_defs (ostmt);
1917 121 : gsi_remove (&gsi, true);
1918 121 : statistics_counter_event (cfun, "delete call dead store", 1);
1919 : /* Only remove the first store previous statement. */
1920 121 : return;
1921 : }
1922 : /* Make sure we do not remove a return slot we cannot reconstruct
1923 : later. */
1924 3923 : if (gimple_call_return_slot_opt_p (as_a <gcall *>(ostmt))
1925 3923 : && (TREE_ADDRESSABLE (TREE_TYPE (gimple_call_fntype (ostmt)))
1926 553 : || !poly_int_tree_p
1927 553 : (TYPE_SIZE (TREE_TYPE (gimple_call_fntype (ostmt))))))
1928 : return;
1929 690 : if (dump_file && (dump_flags & TDF_DETAILS))
1930 : {
1931 6 : fprintf (dump_file, "Removing lhs of call stmt ");
1932 6 : print_gimple_stmt (dump_file, ostmt, 0);
1933 6 : fprintf (dump_file, "\n");
1934 : }
1935 690 : gimple_call_set_lhs (ostmt, NULL_TREE);
1936 690 : update_stmt (ostmt);
1937 690 : statistics_counter_event (cfun, "removed lhs call", 1);
1938 690 : return;
1939 : }
1940 : /* This an assignment store to the clobbered decl,
1941 : then maybe remove it. */
1942 13190265 : if (is_a <gassign*>(ostmt)
1943 11091550 : && gimple_store_p (ostmt)
1944 11091550 : && !gimple_clobber_p (ostmt)
1945 16494261 : && lhs == gimple_assign_lhs (ostmt))
1946 : {
1947 : /* Don't remove stores/statements that are needed for non-call
1948 : eh to work. */
1949 169220 : if (stmt_unremovable_because_of_non_call_eh_p (cfun, ostmt))
1950 : return;
1951 : /* If we delete a stmt that could throw, mark the block
1952 : in to_purge to cleanup afterwards. */
1953 164138 : if (stmt_could_throw_p (cfun, ostmt))
1954 0 : bitmap_set_bit (to_purge, obb->index);
1955 164138 : gimple_stmt_iterator gsi = gsi_for_stmt (ostmt);
1956 164138 : if (dump_file && (dump_flags & TDF_DETAILS))
1957 : {
1958 12 : fprintf (dump_file, "Removing dead store stmt ");
1959 12 : print_gimple_stmt (dump_file, ostmt, 0);
1960 12 : fprintf (dump_file, "\n");
1961 : }
1962 164138 : unlink_stmt_vdef (ostmt);
1963 164138 : release_defs (ostmt);
1964 164138 : gsi_remove (&gsi, true);
1965 164138 : statistics_counter_event (cfun, "delete dead store", 1);
1966 : /* Only remove the first store previous statement. */
1967 164138 : return;
1968 : }
1969 : /* If the statement uses or maybe writes to the decl,
1970 : then nothing is to be removed. Don't know if the write
1971 : to the decl is partial write or a full one so the need
1972 : to stop.
1973 : e.g.
1974 : b.c = a;
1975 : Easier to stop here rather than do a full partial
1976 : dse of this statement.
1977 : b = {CLOBBER}; */
1978 13021045 : if (stmt_may_clobber_ref_p_1 (ostmt, &read, false)
1979 13021045 : || ref_maybe_used_by_stmt_p (ostmt, &read, false))
1980 : return;
1981 10735638 : limit--;
1982 : }
1983 : }
1984 :
1985 : /* Optimizes builtin memcmps for small constant sizes.
1986 : GSI_P is the GSI for the call. STMT is the call itself.
1987 : */
1988 :
1989 : static bool
1990 467804 : simplify_builtin_memcmp (gimple_stmt_iterator *gsi_p, gcall *stmt)
1991 : {
1992 : /* Make sure memcmp arguments are the correct type. */
1993 467804 : if (gimple_call_num_args (stmt) != 3)
1994 : return false;
1995 467804 : tree arg1 = gimple_call_arg (stmt, 0);
1996 467804 : tree arg2 = gimple_call_arg (stmt, 1);
1997 467804 : tree len = gimple_call_arg (stmt, 2);
1998 :
1999 467804 : if (!POINTER_TYPE_P (TREE_TYPE (arg1)))
2000 : return false;
2001 467804 : if (!POINTER_TYPE_P (TREE_TYPE (arg2)))
2002 : return false;
2003 467804 : if (!INTEGRAL_TYPE_P (TREE_TYPE (len)))
2004 : return false;
2005 :
2006 : /* The return value of the memcmp has to be used
2007 : equality comparison to zero. */
2008 467804 : tree res = gimple_call_lhs (stmt);
2009 :
2010 467804 : if (!res || !use_in_zero_equality (res))
2011 : return false;
2012 :
2013 452008 : unsigned HOST_WIDE_INT leni;
2014 :
2015 452008 : if (tree_fits_uhwi_p (len)
2016 629774 : && (leni = tree_to_uhwi (len)) <= GET_MODE_SIZE (word_mode)
2017 532375 : && pow2p_hwi (leni))
2018 : {
2019 19100 : leni *= CHAR_TYPE_SIZE;
2020 19100 : unsigned align1 = get_pointer_alignment (arg1);
2021 19100 : unsigned align2 = get_pointer_alignment (arg2);
2022 19100 : unsigned align = MIN (align1, align2);
2023 19100 : scalar_int_mode mode;
2024 38200 : if (int_mode_for_size (leni, 1).exists (&mode)
2025 19100 : && (align >= leni || !targetm.slow_unaligned_access (mode, align)))
2026 : {
2027 19100 : location_t loc = gimple_location (stmt);
2028 19100 : tree type, off;
2029 19100 : type = build_nonstandard_integer_type (leni, 1);
2030 38200 : gcc_assert (known_eq (GET_MODE_BITSIZE (TYPE_MODE (type)), leni));
2031 19100 : tree ptrtype = build_pointer_type_for_mode (char_type_node,
2032 : ptr_mode, true);
2033 19100 : off = build_int_cst (ptrtype, 0);
2034 :
2035 : /* Create unaligned types if needed. */
2036 19100 : tree type1 = type, type2 = type;
2037 19100 : if (TYPE_ALIGN (type1) > align1)
2038 7784 : type1 = build_aligned_type (type1, align1);
2039 19100 : if (TYPE_ALIGN (type2) > align2)
2040 8279 : type2 = build_aligned_type (type2, align2);
2041 :
2042 19100 : arg1 = build2_loc (loc, MEM_REF, type1, arg1, off);
2043 19100 : arg2 = build2_loc (loc, MEM_REF, type2, arg2, off);
2044 19100 : tree tem1 = fold_const_aggregate_ref (arg1);
2045 19100 : if (tem1)
2046 242 : arg1 = tem1;
2047 19100 : tree tem2 = fold_const_aggregate_ref (arg2);
2048 19100 : if (tem2)
2049 7408 : arg2 = tem2;
2050 19100 : res = fold_convert_loc (loc, TREE_TYPE (res),
2051 : fold_build2_loc (loc, NE_EXPR,
2052 : boolean_type_node,
2053 : arg1, arg2));
2054 19100 : gimplify_and_update_call_from_tree (gsi_p, res);
2055 19100 : return true;
2056 : }
2057 : }
2058 :
2059 : /* Replace memcmp with memcmp_eq if the above fails. */
2060 432908 : if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) == BUILT_IN_MEMCMP_EQ)
2061 : return false;
2062 342927 : if (!fold_before_rtl_expansion_p ())
2063 : return false;
2064 89981 : gimple_call_set_fndecl (stmt, builtin_decl_explicit (BUILT_IN_MEMCMP_EQ));
2065 89981 : update_stmt (stmt);
2066 89981 : return true;
2067 : }
2068 :
2069 : /* Optimizes builtin memchrs for small constant sizes with a const string.
2070 : GSI_P is the GSI for the call. STMT is the call itself.
2071 : */
2072 :
2073 : static bool
2074 11536 : simplify_builtin_memchr (gimple_stmt_iterator *gsi_p, gcall *stmt)
2075 : {
2076 11536 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
2077 : return false;
2078 :
2079 11536 : if (gimple_call_num_args (stmt) != 3)
2080 : return false;
2081 :
2082 11536 : tree res = gimple_call_lhs (stmt);
2083 11536 : if (!res || !use_in_zero_equality (res))
2084 : return false;
2085 :
2086 1405 : tree ptr = gimple_call_arg (stmt, 0);
2087 1405 : if (TREE_CODE (ptr) != ADDR_EXPR
2088 1405 : || TREE_CODE (TREE_OPERAND (ptr, 0)) != STRING_CST)
2089 : return false;
2090 :
2091 398 : unsigned HOST_WIDE_INT slen
2092 398 : = TREE_STRING_LENGTH (TREE_OPERAND (ptr, 0));
2093 : /* It must be a non-empty string constant. */
2094 398 : if (slen < 2)
2095 : return false;
2096 :
2097 : /* For -Os, only simplify strings with a single character. */
2098 394 : if (!optimize_bb_for_speed_p (gimple_bb (stmt))
2099 394 : && slen > 2)
2100 : return false;
2101 :
2102 378 : tree size = gimple_call_arg (stmt, 2);
2103 : /* Size must be a constant which is <= UNITS_PER_WORD and
2104 : <= the string length. */
2105 378 : if (!tree_fits_uhwi_p (size))
2106 : return false;
2107 :
2108 378 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (size);
2109 379 : if (sz == 0 || sz > UNITS_PER_WORD || sz >= slen)
2110 : return false;
2111 :
2112 326 : tree ch = gimple_call_arg (stmt, 1);
2113 326 : location_t loc = gimple_location (stmt);
2114 326 : if (!useless_type_conversion_p (char_type_node,
2115 326 : TREE_TYPE (ch)))
2116 326 : ch = fold_convert_loc (loc, char_type_node, ch);
2117 326 : const char *p = TREE_STRING_POINTER (TREE_OPERAND (ptr, 0));
2118 326 : unsigned int isize = sz;
2119 326 : tree *op = XALLOCAVEC (tree, isize);
2120 1153 : for (unsigned int i = 0; i < isize; i++)
2121 : {
2122 827 : op[i] = build_int_cst (char_type_node, p[i]);
2123 827 : op[i] = fold_build2_loc (loc, EQ_EXPR, boolean_type_node,
2124 : op[i], ch);
2125 : }
2126 827 : for (unsigned int i = isize - 1; i >= 1; i--)
2127 501 : op[i - 1] = fold_convert_loc (loc, boolean_type_node,
2128 : fold_build2_loc (loc,
2129 : BIT_IOR_EXPR,
2130 : boolean_type_node,
2131 501 : op[i - 1],
2132 501 : op[i]));
2133 326 : res = fold_convert_loc (loc, TREE_TYPE (res), op[0]);
2134 326 : gimplify_and_update_call_from_tree (gsi_p, res);
2135 326 : return true;
2136 : }
2137 :
2138 : /* *GSI_P is a GIMPLE_CALL to a builtin function.
2139 : Optimize
2140 : memcpy (p, "abcd", 4); // STMT1
2141 : memset (p + 4, ' ', 3); // STMT2
2142 : into
2143 : memcpy (p, "abcd ", 7);
2144 : call if the latter can be stored by pieces during expansion.
2145 : */
2146 :
2147 : static bool
2148 113405 : simplify_builtin_memcpy_memset (gimple_stmt_iterator *gsi_p, gcall *stmt2)
2149 : {
2150 113405 : if (gimple_call_num_args (stmt2) != 3
2151 113405 : || gimple_call_lhs (stmt2)
2152 : || CHAR_BIT != 8
2153 113405 : || BITS_PER_UNIT != 8)
2154 : return false;
2155 :
2156 107228 : tree vuse = gimple_vuse (stmt2);
2157 107228 : if (vuse == NULL)
2158 : return false;
2159 107212 : gimple *stmt1 = SSA_NAME_DEF_STMT (vuse);
2160 :
2161 107212 : tree callee1;
2162 107212 : tree ptr1, src1, str1, off1, len1, lhs1;
2163 107212 : tree ptr2 = gimple_call_arg (stmt2, 0);
2164 107212 : tree val2 = gimple_call_arg (stmt2, 1);
2165 107212 : tree len2 = gimple_call_arg (stmt2, 2);
2166 107212 : tree diff, vdef, new_str_cst;
2167 107212 : gimple *use_stmt;
2168 107212 : unsigned int ptr1_align;
2169 107212 : unsigned HOST_WIDE_INT src_len;
2170 107212 : char *src_buf;
2171 107212 : use_operand_p use_p;
2172 :
2173 107212 : if (!tree_fits_shwi_p (val2)
2174 102725 : || !tree_fits_uhwi_p (len2)
2175 172414 : || compare_tree_int (len2, 1024) == 1)
2176 : return false;
2177 :
2178 59623 : if (is_gimple_call (stmt1))
2179 : {
2180 : /* If first stmt is a call, it needs to be memcpy
2181 : or mempcpy, with string literal as second argument and
2182 : constant length. */
2183 31015 : callee1 = gimple_call_fndecl (stmt1);
2184 31015 : if (callee1 == NULL_TREE
2185 30899 : || !fndecl_built_in_p (callee1, BUILT_IN_NORMAL)
2186 58109 : || gimple_call_num_args (stmt1) != 3)
2187 : return false;
2188 25743 : if (DECL_FUNCTION_CODE (callee1) != BUILT_IN_MEMCPY
2189 25743 : && DECL_FUNCTION_CODE (callee1) != BUILT_IN_MEMPCPY)
2190 : return false;
2191 11196 : ptr1 = gimple_call_arg (stmt1, 0);
2192 11196 : src1 = gimple_call_arg (stmt1, 1);
2193 11196 : len1 = gimple_call_arg (stmt1, 2);
2194 11196 : lhs1 = gimple_call_lhs (stmt1);
2195 11196 : if (!tree_fits_uhwi_p (len1))
2196 : return false;
2197 11109 : str1 = string_constant (src1, &off1, NULL, NULL);
2198 11109 : if (str1 == NULL_TREE)
2199 : return false;
2200 5222 : if (!tree_fits_uhwi_p (off1)
2201 5222 : || compare_tree_int (off1, TREE_STRING_LENGTH (str1) - 1) > 0
2202 5222 : || compare_tree_int (len1, TREE_STRING_LENGTH (str1)
2203 5222 : - tree_to_uhwi (off1)) > 0
2204 5155 : || TREE_CODE (TREE_TYPE (str1)) != ARRAY_TYPE
2205 15532 : || TYPE_MODE (TREE_TYPE (TREE_TYPE (str1)))
2206 5155 : != TYPE_MODE (char_type_node))
2207 : return false;
2208 : }
2209 28608 : else if (gimple_assign_single_p (stmt1))
2210 : {
2211 : /* Otherwise look for length 1 memcpy optimized into
2212 : assignment. */
2213 17434 : ptr1 = gimple_assign_lhs (stmt1);
2214 17434 : src1 = gimple_assign_rhs1 (stmt1);
2215 17434 : if (TREE_CODE (ptr1) != MEM_REF
2216 3484 : || TYPE_MODE (TREE_TYPE (ptr1)) != TYPE_MODE (char_type_node)
2217 18423 : || !tree_fits_shwi_p (src1))
2218 : return false;
2219 343 : ptr1 = build_fold_addr_expr (ptr1);
2220 343 : STRIP_USELESS_TYPE_CONVERSION (ptr1);
2221 343 : callee1 = NULL_TREE;
2222 343 : len1 = size_one_node;
2223 343 : lhs1 = NULL_TREE;
2224 343 : off1 = size_zero_node;
2225 343 : str1 = NULL_TREE;
2226 : }
2227 : else
2228 : return false;
2229 :
2230 5498 : diff = constant_pointer_difference (ptr1, ptr2);
2231 5498 : if (diff == NULL && lhs1 != NULL)
2232 : {
2233 7 : diff = constant_pointer_difference (lhs1, ptr2);
2234 7 : if (DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY
2235 7 : && diff != NULL)
2236 7 : diff = size_binop (PLUS_EXPR, diff,
2237 : fold_convert (sizetype, len1));
2238 : }
2239 : /* If the difference between the second and first destination pointer
2240 : is not constant, or is bigger than memcpy length, bail out. */
2241 5498 : if (diff == NULL
2242 4620 : || !tree_fits_uhwi_p (diff)
2243 4620 : || tree_int_cst_lt (len1, diff)
2244 9862 : || compare_tree_int (diff, 1024) == 1)
2245 : return false;
2246 :
2247 : /* Use maximum of difference plus memset length and memcpy length
2248 : as the new memcpy length, if it is too big, bail out. */
2249 4364 : src_len = tree_to_uhwi (diff);
2250 4364 : src_len += tree_to_uhwi (len2);
2251 4364 : if (src_len < tree_to_uhwi (len1))
2252 : src_len = tree_to_uhwi (len1);
2253 4364 : if (src_len > 1024)
2254 : return false;
2255 :
2256 : /* If mempcpy value is used elsewhere, bail out, as mempcpy
2257 : with bigger length will return different result. */
2258 4364 : if (lhs1 != NULL_TREE
2259 64 : && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY
2260 4371 : && (TREE_CODE (lhs1) != SSA_NAME
2261 7 : || !single_imm_use (lhs1, &use_p, &use_stmt)
2262 7 : || use_stmt != stmt2))
2263 : return false;
2264 :
2265 : /* If anything reads memory in between memcpy and memset
2266 : call, the modified memcpy call might change it. */
2267 4364 : vdef = gimple_vdef (stmt1);
2268 4364 : if (vdef != NULL
2269 4364 : && (!single_imm_use (vdef, &use_p, &use_stmt)
2270 3645 : || use_stmt != stmt2))
2271 : return false;
2272 :
2273 3645 : ptr1_align = get_pointer_alignment (ptr1);
2274 : /* Construct the new source string literal. */
2275 3645 : src_buf = XALLOCAVEC (char, src_len + 1);
2276 3645 : if (callee1)
2277 3479 : memcpy (src_buf,
2278 3479 : TREE_STRING_POINTER (str1) + tree_to_uhwi (off1),
2279 : tree_to_uhwi (len1));
2280 : else
2281 166 : src_buf[0] = tree_to_shwi (src1);
2282 3645 : memset (src_buf + tree_to_uhwi (diff),
2283 3645 : tree_to_shwi (val2), tree_to_uhwi (len2));
2284 3645 : src_buf[src_len] = '\0';
2285 : /* Neither builtin_strncpy_read_str nor builtin_memcpy_read_str
2286 : handle embedded '\0's. */
2287 3645 : if (strlen (src_buf) != src_len)
2288 : return false;
2289 3551 : rtl_profile_for_bb (gimple_bb (stmt2));
2290 : /* If the new memcpy wouldn't be emitted by storing the literal
2291 : by pieces, this optimization might enlarge .rodata too much,
2292 : as commonly used string literals couldn't be shared any
2293 : longer. */
2294 3551 : if (!can_store_by_pieces (src_len,
2295 : builtin_strncpy_read_str,
2296 : src_buf, ptr1_align, false))
2297 : return false;
2298 :
2299 2669 : new_str_cst = build_string_literal (src_len, src_buf);
2300 2669 : if (callee1)
2301 : {
2302 : /* If STMT1 is a mem{,p}cpy call, adjust it and remove
2303 : memset call. */
2304 2541 : if (lhs1 && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY)
2305 7 : gimple_call_set_lhs (stmt1, NULL_TREE);
2306 2541 : gimple_call_set_arg (stmt1, 1, new_str_cst);
2307 2541 : gimple_call_set_arg (stmt1, 2,
2308 2541 : build_int_cst (TREE_TYPE (len1), src_len));
2309 2541 : update_stmt (stmt1);
2310 2541 : unlink_stmt_vdef (stmt2);
2311 2541 : gsi_replace (gsi_p, gimple_build_nop (), false);
2312 2541 : fwprop_invalidate_lattice (gimple_get_lhs (stmt2));
2313 2541 : release_defs (stmt2);
2314 2541 : if (lhs1 && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY)
2315 : {
2316 7 : fwprop_invalidate_lattice (lhs1);
2317 7 : release_ssa_name (lhs1);
2318 : }
2319 : return true;
2320 : }
2321 : else
2322 : {
2323 : /* Otherwise, if STMT1 is length 1 memcpy optimized into
2324 : assignment, remove STMT1 and change memset call into
2325 : memcpy call. */
2326 128 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt1);
2327 :
2328 128 : if (!is_gimple_val (ptr1))
2329 12 : ptr1 = force_gimple_operand_gsi (gsi_p, ptr1, true, NULL_TREE,
2330 : true, GSI_SAME_STMT);
2331 128 : tree fndecl = builtin_decl_explicit (BUILT_IN_MEMCPY);
2332 128 : gimple_call_set_fndecl (stmt2, fndecl);
2333 128 : gimple_call_set_fntype (stmt2,
2334 128 : TREE_TYPE (fndecl));
2335 128 : gimple_call_set_arg (stmt2, 0, ptr1);
2336 128 : gimple_call_set_arg (stmt2, 1, new_str_cst);
2337 128 : gimple_call_set_arg (stmt2, 2,
2338 128 : build_int_cst (TREE_TYPE (len2), src_len));
2339 128 : unlink_stmt_vdef (stmt1);
2340 128 : gsi_remove (&gsi, true);
2341 128 : fwprop_invalidate_lattice (gimple_get_lhs (stmt1));
2342 128 : release_defs (stmt1);
2343 128 : update_stmt (stmt2);
2344 128 : return false;
2345 : }
2346 : }
2347 :
2348 :
2349 : /* Try to optimize out __builtin_stack_restore. Optimize it out
2350 : if there is another __builtin_stack_restore in the same basic
2351 : block and no calls or ASM_EXPRs are in between, or if this block's
2352 : only outgoing edge is to EXIT_BLOCK and there are no calls or
2353 : ASM_EXPRs after this __builtin_stack_restore.
2354 : Note restore right before a noreturn function is not needed.
2355 : And skip some cheap calls that will most likely become an instruction.
2356 : Restoring the stack before a call is important to be able to keep
2357 : stack usage down so that call does not run out of stack. */
2358 :
2359 :
2360 : static bool
2361 10629 : optimize_stack_restore (gimple_stmt_iterator *gsi, gimple *call)
2362 : {
2363 10629 : if (!fold_before_rtl_expansion_p ())
2364 : return false;
2365 2592 : tree callee;
2366 2592 : gimple *stmt;
2367 :
2368 2592 : basic_block bb = gsi_bb (*gsi);
2369 :
2370 2592 : if (gimple_call_num_args (call) != 1
2371 2592 : || TREE_CODE (gimple_call_arg (call, 0)) != SSA_NAME
2372 5184 : || !POINTER_TYPE_P (TREE_TYPE (gimple_call_arg (call, 0))))
2373 : return false;
2374 :
2375 2592 : gimple_stmt_iterator i = *gsi;
2376 6492 : for (gsi_next (&i); !gsi_end_p (i); gsi_next (&i))
2377 : {
2378 4369 : stmt = gsi_stmt (i);
2379 4369 : if (is_a<gasm*> (stmt))
2380 : return false;
2381 4368 : gcall *call = dyn_cast<gcall*>(stmt);
2382 4368 : if (!call)
2383 3685 : continue;
2384 :
2385 : /* We can remove the restore in front of noreturn
2386 : calls. Since the restore will happen either
2387 : via an unwind/longjmp or not at all. */
2388 683 : if (gimple_call_noreturn_p (call))
2389 : break;
2390 :
2391 : /* Internal calls are ok, to bypass
2392 : check first since fndecl will be null. */
2393 667 : if (gimple_call_internal_p (call))
2394 1 : continue;
2395 :
2396 666 : callee = gimple_call_fndecl (call);
2397 : /* Non-builtin calls are not ok. */
2398 666 : if (!callee
2399 666 : || !fndecl_built_in_p (callee))
2400 : return false;
2401 :
2402 : /* Do not remove stack updates before strub leave. */
2403 586 : if (fndecl_built_in_p (callee, BUILT_IN___STRUB_LEAVE)
2404 : /* Alloca calls are not ok either. */
2405 586 : || fndecl_builtin_alloc_p (callee))
2406 : return false;
2407 :
2408 371 : if (fndecl_built_in_p (callee, BUILT_IN_STACK_RESTORE))
2409 57 : goto second_stack_restore;
2410 :
2411 : /* If not a simple or inexpensive builtin, then it is not ok either. */
2412 314 : if (!is_simple_builtin (callee)
2413 314 : && !is_inexpensive_builtin (callee))
2414 : return false;
2415 : }
2416 :
2417 : /* Allow one successor of the exit block, or zero successors. */
2418 2139 : switch (EDGE_COUNT (bb->succs))
2419 : {
2420 : case 0:
2421 : break;
2422 2049 : case 1:
2423 2049 : if (single_succ_edge (bb)->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
2424 : return false;
2425 : break;
2426 : default:
2427 : return false;
2428 : }
2429 1779 : second_stack_restore:
2430 :
2431 : /* If there's exactly one use, then zap the call to __builtin_stack_save.
2432 : If there are multiple uses, then the last one should remove the call.
2433 : In any case, whether the call to __builtin_stack_save can be removed
2434 : or not is irrelevant to removing the call to __builtin_stack_restore. */
2435 1779 : if (has_single_use (gimple_call_arg (call, 0)))
2436 : {
2437 1607 : gimple *stack_save = SSA_NAME_DEF_STMT (gimple_call_arg (call, 0));
2438 1607 : if (is_gimple_call (stack_save))
2439 : {
2440 1605 : callee = gimple_call_fndecl (stack_save);
2441 1605 : if (callee && fndecl_built_in_p (callee, BUILT_IN_STACK_SAVE))
2442 : {
2443 1605 : gimple_stmt_iterator stack_save_gsi;
2444 1605 : tree rhs;
2445 :
2446 1605 : stack_save_gsi = gsi_for_stmt (stack_save);
2447 1605 : rhs = build_int_cst (TREE_TYPE (gimple_call_arg (call, 0)), 0);
2448 1605 : replace_call_with_value (&stack_save_gsi, rhs);
2449 : }
2450 : }
2451 : }
2452 :
2453 : /* No effect, so the statement will be deleted. */
2454 1779 : replace_call_with_value (gsi, NULL_TREE);
2455 1779 : return true;
2456 : }
2457 :
2458 : /* Optimizes strlen (s) ==/!= 0 to *s ==/!= 0. */
2459 : static bool
2460 61082 : optimize_strlen_comp (gimple_stmt_iterator *gsi, gimple *call)
2461 : {
2462 61082 : if (!fold_before_rtl_expansion_p ())
2463 : return false;
2464 :
2465 13145 : tree lhs = gimple_call_lhs (call);
2466 13145 : if (lhs == NULL_TREE || use_in_zero_equality (lhs, true) == NULL)
2467 : return false;
2468 :
2469 : /* The string passed to strlen. */
2470 108 : tree ptr = gimple_call_arg (call, 0);
2471 :
2472 : /* Dereference the string. */
2473 108 : tree deref = fold_build2 (MEM_REF, char_type_node, ptr,
2474 : build_zero_cst (ptr_type_node));
2475 :
2476 : /* Perform a type conversion. */
2477 108 : deref = fold_convert_loc (gimple_location (call),
2478 108 : TREE_TYPE (lhs),
2479 : deref);
2480 :
2481 : /* Replace the original call to strlen with the dereference we just built. */
2482 108 : gimplify_and_update_call_from_tree (gsi, deref);
2483 :
2484 108 : return true;
2485 : }
2486 :
2487 : /* *GSI_P is a GIMPLE_CALL to a builtin function.
2488 : Optimize
2489 : memcpy (p, "abcd", 4);
2490 : memset (p + 4, ' ', 3);
2491 : into
2492 : memcpy (p, "abcd ", 7);
2493 : call if the latter can be stored by pieces during expansion.
2494 :
2495 : Optimize
2496 : memchr ("abcd", a, 4) == 0;
2497 : or
2498 : memchr ("abcd", a, 4) != 0;
2499 : to
2500 : (a == 'a' || a == 'b' || a == 'c' || a == 'd') == 0
2501 : or
2502 : (a == 'a' || a == 'b' || a == 'c' || a == 'd') != 0
2503 :
2504 : Also canonicalize __atomic_fetch_op (p, x, y) op x
2505 : to __atomic_op_fetch (p, x, y) or
2506 : __atomic_op_fetch (p, x, y) iop x
2507 : to __atomic_fetch_op (p, x, y) when possible (also __sync). */
2508 :
2509 : static bool
2510 6319213 : simplify_builtin_call (gimple_stmt_iterator *gsi_p, tree callee2, bool full_walk)
2511 : {
2512 6319213 : gimple *stmt2 = gsi_stmt (*gsi_p);
2513 6319213 : enum built_in_function other_atomic = END_BUILTINS;
2514 6319213 : enum tree_code atomic_op = ERROR_MARK;
2515 :
2516 6319213 : switch (DECL_FUNCTION_CODE (callee2))
2517 : {
2518 61082 : case BUILT_IN_STRLEN:
2519 61082 : return optimize_strlen_comp (gsi_p, as_a<gcall*>(stmt2));
2520 10629 : case BUILT_IN_STACK_RESTORE:
2521 10629 : return optimize_stack_restore (gsi_p, as_a<gcall*>(stmt2));
2522 467804 : case BUILT_IN_MEMCMP:
2523 467804 : case BUILT_IN_MEMCMP_EQ:
2524 467804 : return simplify_builtin_memcmp (gsi_p, as_a<gcall*>(stmt2));
2525 11536 : case BUILT_IN_MEMCHR:
2526 11536 : return simplify_builtin_memchr (gsi_p, as_a<gcall*>(stmt2));
2527 :
2528 113405 : case BUILT_IN_MEMSET:
2529 113405 : if (gimple_call_num_args (stmt2) == 3)
2530 : {
2531 : /* Try to prop the zeroing/value of the memset to memcpy
2532 : if the dest is an address and the value is a constant. */
2533 113405 : optimize_aggr_zeroprop (stmt2, full_walk);
2534 : }
2535 113405 : return simplify_builtin_memcpy_memset (gsi_p, as_a<gcall*>(stmt2));
2536 :
2537 : #define CASE_ATOMIC(NAME, OTHER, OP) \
2538 : case BUILT_IN_##NAME##_1: \
2539 : case BUILT_IN_##NAME##_2: \
2540 : case BUILT_IN_##NAME##_4: \
2541 : case BUILT_IN_##NAME##_8: \
2542 : case BUILT_IN_##NAME##_16: \
2543 : atomic_op = OP; \
2544 : other_atomic \
2545 : = (enum built_in_function) (BUILT_IN_##OTHER##_1 \
2546 : + (DECL_FUNCTION_CODE (callee2) \
2547 : - BUILT_IN_##NAME##_1)); \
2548 : goto handle_atomic_fetch_op;
2549 :
2550 49277 : CASE_ATOMIC (ATOMIC_FETCH_ADD, ATOMIC_ADD_FETCH, PLUS_EXPR)
2551 7261 : CASE_ATOMIC (ATOMIC_FETCH_SUB, ATOMIC_SUB_FETCH, MINUS_EXPR)
2552 3084 : CASE_ATOMIC (ATOMIC_FETCH_AND, ATOMIC_AND_FETCH, BIT_AND_EXPR)
2553 3063 : CASE_ATOMIC (ATOMIC_FETCH_XOR, ATOMIC_XOR_FETCH, BIT_XOR_EXPR)
2554 4202 : CASE_ATOMIC (ATOMIC_FETCH_OR, ATOMIC_OR_FETCH, BIT_IOR_EXPR)
2555 :
2556 2373 : CASE_ATOMIC (SYNC_FETCH_AND_ADD, SYNC_ADD_AND_FETCH, PLUS_EXPR)
2557 2012 : CASE_ATOMIC (SYNC_FETCH_AND_SUB, SYNC_SUB_AND_FETCH, MINUS_EXPR)
2558 1876 : CASE_ATOMIC (SYNC_FETCH_AND_AND, SYNC_AND_AND_FETCH, BIT_AND_EXPR)
2559 2144 : CASE_ATOMIC (SYNC_FETCH_AND_XOR, SYNC_XOR_AND_FETCH, BIT_XOR_EXPR)
2560 1987 : CASE_ATOMIC (SYNC_FETCH_AND_OR, SYNC_OR_AND_FETCH, BIT_IOR_EXPR)
2561 :
2562 14461 : CASE_ATOMIC (ATOMIC_ADD_FETCH, ATOMIC_FETCH_ADD, MINUS_EXPR)
2563 8580 : CASE_ATOMIC (ATOMIC_SUB_FETCH, ATOMIC_FETCH_SUB, PLUS_EXPR)
2564 2428 : CASE_ATOMIC (ATOMIC_XOR_FETCH, ATOMIC_FETCH_XOR, BIT_XOR_EXPR)
2565 :
2566 854 : CASE_ATOMIC (SYNC_ADD_AND_FETCH, SYNC_FETCH_AND_ADD, MINUS_EXPR)
2567 740 : CASE_ATOMIC (SYNC_SUB_AND_FETCH, SYNC_FETCH_AND_SUB, PLUS_EXPR)
2568 800 : CASE_ATOMIC (SYNC_XOR_AND_FETCH, SYNC_FETCH_AND_XOR, BIT_XOR_EXPR)
2569 :
2570 : #undef CASE_ATOMIC
2571 :
2572 105142 : handle_atomic_fetch_op:
2573 105142 : if (gimple_call_num_args (stmt2) >= 2 && gimple_call_lhs (stmt2))
2574 : {
2575 61074 : tree lhs2 = gimple_call_lhs (stmt2), lhsc = lhs2;
2576 61074 : tree arg = gimple_call_arg (stmt2, 1);
2577 61074 : gimple *use_stmt, *cast_stmt = NULL;
2578 61074 : use_operand_p use_p;
2579 61074 : tree ndecl = builtin_decl_explicit (other_atomic);
2580 :
2581 61074 : if (ndecl == NULL_TREE || !single_imm_use (lhs2, &use_p, &use_stmt))
2582 : break;
2583 :
2584 59957 : if (gimple_assign_cast_p (use_stmt))
2585 : {
2586 31573 : cast_stmt = use_stmt;
2587 31573 : lhsc = gimple_assign_lhs (cast_stmt);
2588 31573 : if (lhsc == NULL_TREE
2589 31573 : || !INTEGRAL_TYPE_P (TREE_TYPE (lhsc))
2590 31022 : || (TYPE_PRECISION (TREE_TYPE (lhsc))
2591 31022 : != TYPE_PRECISION (TREE_TYPE (lhs2)))
2592 60977 : || !single_imm_use (lhsc, &use_p, &use_stmt))
2593 : {
2594 2697 : use_stmt = cast_stmt;
2595 2697 : cast_stmt = NULL;
2596 2697 : lhsc = lhs2;
2597 : }
2598 : }
2599 :
2600 59957 : bool ok = false;
2601 59957 : tree oarg = NULL_TREE;
2602 59957 : enum tree_code ccode = ERROR_MARK;
2603 59957 : tree crhs1 = NULL_TREE, crhs2 = NULL_TREE;
2604 59957 : if (is_gimple_assign (use_stmt)
2605 59957 : && gimple_assign_rhs_code (use_stmt) == atomic_op)
2606 : {
2607 1416 : if (gimple_assign_rhs1 (use_stmt) == lhsc)
2608 1016 : oarg = gimple_assign_rhs2 (use_stmt);
2609 400 : else if (atomic_op != MINUS_EXPR)
2610 : oarg = gimple_assign_rhs1 (use_stmt);
2611 : }
2612 58541 : else if (atomic_op == MINUS_EXPR
2613 13423 : && is_gimple_assign (use_stmt)
2614 3744 : && gimple_assign_rhs_code (use_stmt) == PLUS_EXPR
2615 199 : && TREE_CODE (arg) == INTEGER_CST
2616 58740 : && (TREE_CODE (gimple_assign_rhs2 (use_stmt))
2617 : == INTEGER_CST))
2618 : {
2619 183 : tree a = fold_convert (TREE_TYPE (lhs2), arg);
2620 183 : tree o = fold_convert (TREE_TYPE (lhs2),
2621 : gimple_assign_rhs2 (use_stmt));
2622 183 : if (wi::to_wide (a) == wi::neg (wi::to_wide (o)))
2623 : ok = true;
2624 : }
2625 58358 : else if (atomic_op == BIT_AND_EXPR || atomic_op == BIT_IOR_EXPR)
2626 : ;
2627 52708 : else if (gimple_code (use_stmt) == GIMPLE_COND)
2628 : {
2629 19730 : ccode = gimple_cond_code (use_stmt);
2630 19730 : crhs1 = gimple_cond_lhs (use_stmt);
2631 19730 : crhs2 = gimple_cond_rhs (use_stmt);
2632 : }
2633 32978 : else if (is_gimple_assign (use_stmt))
2634 : {
2635 9853 : if (gimple_assign_rhs_class (use_stmt) == GIMPLE_BINARY_RHS)
2636 : {
2637 4185 : ccode = gimple_assign_rhs_code (use_stmt);
2638 4185 : crhs1 = gimple_assign_rhs1 (use_stmt);
2639 4185 : crhs2 = gimple_assign_rhs2 (use_stmt);
2640 : }
2641 : }
2642 24931 : if (ccode == EQ_EXPR || ccode == NE_EXPR)
2643 : {
2644 : /* Deal with x - y == 0 or x ^ y == 0
2645 : being optimized into x == y and x + cst == 0
2646 : into x == -cst. */
2647 22717 : tree o = NULL_TREE;
2648 22717 : if (crhs1 == lhsc)
2649 : o = crhs2;
2650 133 : else if (crhs2 == lhsc)
2651 133 : o = crhs1;
2652 22717 : if (o && atomic_op != PLUS_EXPR)
2653 : oarg = o;
2654 10271 : else if (o
2655 10271 : && TREE_CODE (o) == INTEGER_CST
2656 10271 : && TREE_CODE (arg) == INTEGER_CST)
2657 : {
2658 9561 : tree a = fold_convert (TREE_TYPE (lhs2), arg);
2659 9561 : o = fold_convert (TREE_TYPE (lhs2), o);
2660 9561 : if (wi::to_wide (a) == wi::neg (wi::to_wide (o)))
2661 59957 : ok = true;
2662 : }
2663 : }
2664 59957 : if (oarg && !ok)
2665 : {
2666 13862 : if (operand_equal_p (arg, oarg, 0))
2667 : ok = true;
2668 12521 : else if (TREE_CODE (arg) == SSA_NAME
2669 2215 : && TREE_CODE (oarg) == SSA_NAME)
2670 : {
2671 745 : tree oarg2 = oarg;
2672 745 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (oarg)))
2673 : {
2674 104 : gimple *g = SSA_NAME_DEF_STMT (oarg);
2675 104 : oarg2 = gimple_assign_rhs1 (g);
2676 104 : if (TREE_CODE (oarg2) != SSA_NAME
2677 104 : || !INTEGRAL_TYPE_P (TREE_TYPE (oarg2))
2678 208 : || (TYPE_PRECISION (TREE_TYPE (oarg2))
2679 104 : != TYPE_PRECISION (TREE_TYPE (oarg))))
2680 : oarg2 = oarg;
2681 : }
2682 745 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (arg)))
2683 : {
2684 544 : gimple *g = SSA_NAME_DEF_STMT (arg);
2685 544 : tree rhs1 = gimple_assign_rhs1 (g);
2686 : /* Handle e.g.
2687 : x.0_1 = (long unsigned int) x_4(D);
2688 : _2 = __atomic_fetch_add_8 (&vlong, x.0_1, 0);
2689 : _3 = (long int) _2;
2690 : _7 = x_4(D) + _3; */
2691 544 : if (rhs1 == oarg || rhs1 == oarg2)
2692 : ok = true;
2693 : /* Handle e.g.
2694 : x.18_1 = (short unsigned int) x_5(D);
2695 : _2 = (int) x.18_1;
2696 : _3 = __atomic_fetch_xor_2 (&vshort, _2, 0);
2697 : _4 = (short int) _3;
2698 : _8 = x_5(D) ^ _4;
2699 : This happens only for char/short. */
2700 160 : else if (TREE_CODE (rhs1) == SSA_NAME
2701 160 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
2702 320 : && (TYPE_PRECISION (TREE_TYPE (rhs1))
2703 160 : == TYPE_PRECISION (TREE_TYPE (lhs2))))
2704 : {
2705 160 : g = SSA_NAME_DEF_STMT (rhs1);
2706 160 : if (gimple_assign_cast_p (g)
2707 160 : && (gimple_assign_rhs1 (g) == oarg
2708 0 : || gimple_assign_rhs1 (g) == oarg2))
2709 : ok = true;
2710 : }
2711 : }
2712 745 : if (!ok && arg == oarg2)
2713 : /* Handle e.g.
2714 : _1 = __sync_fetch_and_add_4 (&v, x_5(D));
2715 : _2 = (int) _1;
2716 : x.0_3 = (int) x_5(D);
2717 : _7 = _2 + x.0_3; */
2718 : ok = true;
2719 : }
2720 : }
2721 :
2722 58616 : if (ok)
2723 : {
2724 2622 : tree new_lhs = make_ssa_name (TREE_TYPE (lhs2));
2725 2622 : gimple_call_set_lhs (stmt2, new_lhs);
2726 2622 : gimple_call_set_fndecl (stmt2, ndecl);
2727 2622 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
2728 2622 : if (ccode == ERROR_MARK)
2729 2000 : gimple_assign_set_rhs_with_ops (&gsi, cast_stmt
2730 : ? NOP_EXPR : SSA_NAME,
2731 : new_lhs);
2732 : else
2733 : {
2734 1399 : crhs1 = new_lhs;
2735 1399 : crhs2 = build_zero_cst (TREE_TYPE (lhs2));
2736 1399 : if (gimple_code (use_stmt) == GIMPLE_COND)
2737 : {
2738 1054 : gcond *cond_stmt = as_a <gcond *> (use_stmt);
2739 1054 : gimple_cond_set_lhs (cond_stmt, crhs1);
2740 1054 : gimple_cond_set_rhs (cond_stmt, crhs2);
2741 : }
2742 345 : else if (gimple_assign_rhs_class (use_stmt)
2743 : == GIMPLE_BINARY_RHS)
2744 : {
2745 345 : gimple_assign_set_rhs1 (use_stmt, crhs1);
2746 345 : gimple_assign_set_rhs2 (use_stmt, crhs2);
2747 : }
2748 : }
2749 2622 : update_stmt (use_stmt);
2750 2622 : if (atomic_op != BIT_AND_EXPR
2751 2622 : && atomic_op != BIT_IOR_EXPR
2752 2622 : && !stmt_ends_bb_p (stmt2))
2753 : {
2754 : /* For the benefit of debug stmts, emit stmt(s) to set
2755 : lhs2 to the value it had from the new builtin.
2756 : E.g. if it was previously:
2757 : lhs2 = __atomic_fetch_add_8 (ptr, arg, 0);
2758 : emit:
2759 : new_lhs = __atomic_add_fetch_8 (ptr, arg, 0);
2760 : lhs2 = new_lhs - arg;
2761 : We also keep cast_stmt if any in the IL for
2762 : the same reasons.
2763 : These stmts will be DCEd later and proper debug info
2764 : will be emitted.
2765 : This is only possible for reversible operations
2766 : (+/-/^) and without -fnon-call-exceptions. */
2767 2281 : gsi = gsi_for_stmt (stmt2);
2768 2281 : tree type = TREE_TYPE (lhs2);
2769 2281 : if (TREE_CODE (arg) == INTEGER_CST)
2770 1699 : arg = fold_convert (type, arg);
2771 582 : else if (!useless_type_conversion_p (type, TREE_TYPE (arg)))
2772 : {
2773 0 : tree narg = make_ssa_name (type);
2774 0 : gimple *g = gimple_build_assign (narg, NOP_EXPR, arg);
2775 0 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
2776 0 : arg = narg;
2777 : }
2778 2281 : enum tree_code rcode;
2779 2281 : switch (atomic_op)
2780 : {
2781 : case PLUS_EXPR: rcode = MINUS_EXPR; break;
2782 733 : case MINUS_EXPR: rcode = PLUS_EXPR; break;
2783 498 : case BIT_XOR_EXPR: rcode = atomic_op; break;
2784 : default: gcc_unreachable ();
2785 : }
2786 2281 : gimple *g = gimple_build_assign (lhs2, rcode, new_lhs, arg);
2787 2281 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
2788 2281 : update_stmt (stmt2);
2789 : }
2790 : else
2791 : {
2792 : /* For e.g.
2793 : lhs2 = __atomic_fetch_or_8 (ptr, arg, 0);
2794 : after we change it to
2795 : new_lhs = __atomic_or_fetch_8 (ptr, arg, 0);
2796 : there is no way to find out the lhs2 value (i.e.
2797 : what the atomic memory contained before the operation),
2798 : values of some bits are lost. We have checked earlier
2799 : that we don't have any non-debug users except for what
2800 : we are already changing, so we need to reset the
2801 : debug stmts and remove the cast_stmt if any. */
2802 341 : imm_use_iterator iter;
2803 676 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs2)
2804 335 : if (use_stmt != cast_stmt)
2805 : {
2806 168 : gcc_assert (is_gimple_debug (use_stmt));
2807 168 : gimple_debug_bind_reset_value (use_stmt);
2808 168 : update_stmt (use_stmt);
2809 341 : }
2810 341 : if (cast_stmt)
2811 : {
2812 167 : gsi = gsi_for_stmt (cast_stmt);
2813 167 : gsi_remove (&gsi, true);
2814 : }
2815 341 : update_stmt (stmt2);
2816 341 : release_ssa_name (lhs2);
2817 : }
2818 : }
2819 : }
2820 : break;
2821 :
2822 : default:
2823 : break;
2824 : }
2825 : return false;
2826 : }
2827 :
2828 : /* Given a ssa_name in NAME see if it was defined by an assignment and
2829 : set CODE to be the code and ARG1 to the first operand on the rhs and ARG2
2830 : to the second operand on the rhs. */
2831 :
2832 : static inline void
2833 17727550 : defcodefor_name (tree name, enum tree_code *code, tree *arg1, tree *arg2)
2834 : {
2835 17727550 : gimple *def;
2836 17727550 : enum tree_code code1;
2837 17727550 : tree arg11;
2838 17727550 : tree arg21;
2839 17727550 : tree arg31;
2840 17727550 : enum gimple_rhs_class grhs_class;
2841 :
2842 17727550 : code1 = TREE_CODE (name);
2843 17727550 : arg11 = name;
2844 17727550 : arg21 = NULL_TREE;
2845 17727550 : arg31 = NULL_TREE;
2846 17727550 : grhs_class = get_gimple_rhs_class (code1);
2847 :
2848 17727550 : if (code1 == SSA_NAME)
2849 : {
2850 11786005 : def = SSA_NAME_DEF_STMT (name);
2851 :
2852 11786005 : if (def && is_gimple_assign (def)
2853 19088784 : && can_propagate_from (def))
2854 : {
2855 5010454 : code1 = gimple_assign_rhs_code (def);
2856 5010454 : arg11 = gimple_assign_rhs1 (def);
2857 5010454 : arg21 = gimple_assign_rhs2 (def);
2858 5010454 : arg31 = gimple_assign_rhs3 (def);
2859 : }
2860 : }
2861 5941545 : else if (grhs_class != GIMPLE_SINGLE_RHS)
2862 0 : code1 = ERROR_MARK;
2863 :
2864 17727550 : *code = code1;
2865 17727550 : *arg1 = arg11;
2866 17727550 : if (arg2)
2867 17710291 : *arg2 = arg21;
2868 17727550 : if (arg31)
2869 2715 : *code = ERROR_MARK;
2870 17727550 : }
2871 :
2872 :
2873 : /* Recognize rotation patterns. Return true if a transformation
2874 : applied, otherwise return false.
2875 :
2876 : We are looking for X with unsigned type T with bitsize B, OP being
2877 : +, | or ^, some type T2 wider than T. For:
2878 : (X << CNT1) OP (X >> CNT2) iff CNT1 + CNT2 == B
2879 : ((T) ((T2) X << CNT1)) OP ((T) ((T2) X >> CNT2)) iff CNT1 + CNT2 == B
2880 :
2881 : transform these into:
2882 : X r<< CNT1
2883 :
2884 : Or for:
2885 : (X << Y) OP (X >> (B - Y))
2886 : (X << (int) Y) OP (X >> (int) (B - Y))
2887 : ((T) ((T2) X << Y)) OP ((T) ((T2) X >> (B - Y)))
2888 : ((T) ((T2) X << (int) Y)) OP ((T) ((T2) X >> (int) (B - Y)))
2889 : (X << Y) | (X >> ((-Y) & (B - 1)))
2890 : (X << (int) Y) | (X >> (int) ((-Y) & (B - 1)))
2891 : ((T) ((T2) X << Y)) | ((T) ((T2) X >> ((-Y) & (B - 1))))
2892 : ((T) ((T2) X << (int) Y)) | ((T) ((T2) X >> (int) ((-Y) & (B - 1))))
2893 :
2894 : transform these into (last 2 only if ranger can prove Y < B
2895 : or Y = N * B):
2896 : X r<< Y
2897 : or
2898 : X r<< (& & (B - 1))
2899 : The latter for the forms with T2 wider than T if ranger can't prove Y < B.
2900 :
2901 : Or for:
2902 : (X << (Y & (B - 1))) | (X >> ((-Y) & (B - 1)))
2903 : (X << (int) (Y & (B - 1))) | (X >> (int) ((-Y) & (B - 1)))
2904 : ((T) ((T2) X << (Y & (B - 1)))) | ((T) ((T2) X >> ((-Y) & (B - 1))))
2905 : ((T) ((T2) X << (int) (Y & (B - 1)))) \
2906 : | ((T) ((T2) X >> (int) ((-Y) & (B - 1))))
2907 :
2908 : transform these into:
2909 : X r<< (Y & (B - 1))
2910 :
2911 : Note, in the patterns with T2 type, the type of OP operands
2912 : might be even a signed type, but should have precision B.
2913 : Expressions with & (B - 1) should be recognized only if B is
2914 : a power of 2. */
2915 :
2916 : static bool
2917 10383032 : simplify_rotate (gimple_stmt_iterator *gsi)
2918 : {
2919 10383032 : gimple *stmt = gsi_stmt (*gsi);
2920 10383032 : tree arg[2], rtype, rotcnt = NULL_TREE;
2921 10383032 : tree def_arg1[2], def_arg2[2];
2922 10383032 : enum tree_code def_code[2];
2923 10383032 : tree lhs;
2924 10383032 : int i;
2925 10383032 : bool swapped_p = false;
2926 10383032 : gimple *g;
2927 10383032 : gimple *def_arg_stmt[2] = { NULL, NULL };
2928 10383032 : int wider_prec = 0;
2929 10383032 : bool add_masking = false;
2930 :
2931 10383032 : arg[0] = gimple_assign_rhs1 (stmt);
2932 10383032 : arg[1] = gimple_assign_rhs2 (stmt);
2933 10383032 : rtype = TREE_TYPE (arg[0]);
2934 :
2935 : /* Only create rotates in complete modes. Other cases are not
2936 : expanded properly. */
2937 10383032 : if (!INTEGRAL_TYPE_P (rtype)
2938 10383032 : || !type_has_mode_precision_p (rtype))
2939 : return false;
2940 :
2941 26446941 : for (i = 0; i < 2; i++)
2942 : {
2943 17631294 : defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
2944 17631294 : if (TREE_CODE (arg[i]) == SSA_NAME)
2945 11689749 : def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
2946 : }
2947 :
2948 : /* Look through narrowing (or same precision) conversions. */
2949 7851182 : if (CONVERT_EXPR_CODE_P (def_code[0])
2950 964465 : && CONVERT_EXPR_CODE_P (def_code[1])
2951 141932 : && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[0]))
2952 117639 : && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[1]))
2953 109670 : && TYPE_PRECISION (TREE_TYPE (def_arg1[0]))
2954 109670 : == TYPE_PRECISION (TREE_TYPE (def_arg1[1]))
2955 63478 : && TYPE_PRECISION (TREE_TYPE (def_arg1[0])) >= TYPE_PRECISION (rtype)
2956 44404 : && has_single_use (arg[0])
2957 8847607 : && has_single_use (arg[1]))
2958 : {
2959 26806 : wider_prec = TYPE_PRECISION (TREE_TYPE (def_arg1[0]));
2960 80418 : for (i = 0; i < 2; i++)
2961 : {
2962 53612 : arg[i] = def_arg1[i];
2963 53612 : defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
2964 53612 : if (TREE_CODE (arg[i]) == SSA_NAME)
2965 53612 : def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
2966 : }
2967 : }
2968 : else
2969 : {
2970 : /* Handle signed rotate; the RSHIFT_EXPR has to be done
2971 : in unsigned type but LSHIFT_EXPR could be signed. */
2972 8788841 : i = (def_code[0] == LSHIFT_EXPR || def_code[0] == RSHIFT_EXPR);
2973 7833427 : if (CONVERT_EXPR_CODE_P (def_code[i])
2974 955414 : && (def_code[1 - i] == LSHIFT_EXPR || def_code[1 - i] == RSHIFT_EXPR)
2975 32375 : && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[i]))
2976 31109 : && TYPE_PRECISION (rtype) == TYPE_PRECISION (TREE_TYPE (def_arg1[i]))
2977 8793820 : && has_single_use (arg[i]))
2978 : {
2979 2120 : arg[i] = def_arg1[i];
2980 2120 : defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
2981 2120 : if (TREE_CODE (arg[i]) == SSA_NAME)
2982 2120 : def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
2983 : }
2984 : }
2985 :
2986 : /* One operand has to be LSHIFT_EXPR and one RSHIFT_EXPR. */
2987 9005437 : for (i = 0; i < 2; i++)
2988 8980105 : if (def_code[i] != LSHIFT_EXPR && def_code[i] != RSHIFT_EXPR)
2989 : return false;
2990 231122 : else if (!has_single_use (arg[i]))
2991 : return false;
2992 25332 : if (def_code[0] == def_code[1])
2993 : return false;
2994 :
2995 : /* If we've looked through narrowing conversions before, look through
2996 : widening conversions from unsigned type with the same precision
2997 : as rtype here. */
2998 20968 : if (TYPE_PRECISION (TREE_TYPE (def_arg1[0])) != TYPE_PRECISION (rtype))
2999 19345 : for (i = 0; i < 2; i++)
3000 : {
3001 12897 : tree tem;
3002 12897 : enum tree_code code;
3003 12897 : defcodefor_name (def_arg1[i], &code, &tem, NULL);
3004 1 : if (!CONVERT_EXPR_CODE_P (code)
3005 12896 : || !INTEGRAL_TYPE_P (TREE_TYPE (tem))
3006 25793 : || TYPE_PRECISION (TREE_TYPE (tem)) != TYPE_PRECISION (rtype))
3007 1 : return false;
3008 12896 : def_arg1[i] = tem;
3009 : }
3010 : /* Both shifts have to use the same first operand. */
3011 20967 : if (!operand_equal_for_phi_arg_p (def_arg1[0], def_arg1[1])
3012 33263 : || !types_compatible_p (TREE_TYPE (def_arg1[0]),
3013 12296 : TREE_TYPE (def_arg1[1])))
3014 : {
3015 8671 : if ((TYPE_PRECISION (TREE_TYPE (def_arg1[0]))
3016 8671 : != TYPE_PRECISION (TREE_TYPE (def_arg1[1])))
3017 8671 : || (TYPE_UNSIGNED (TREE_TYPE (def_arg1[0]))
3018 8671 : == TYPE_UNSIGNED (TREE_TYPE (def_arg1[1]))))
3019 8647 : return false;
3020 :
3021 : /* Handle signed rotate; the RSHIFT_EXPR has to be done
3022 : in unsigned type but LSHIFT_EXPR could be signed. */
3023 543 : i = def_code[0] != RSHIFT_EXPR;
3024 543 : if (!TYPE_UNSIGNED (TREE_TYPE (def_arg1[i])))
3025 : return false;
3026 :
3027 506 : tree tem;
3028 506 : enum tree_code code;
3029 506 : defcodefor_name (def_arg1[i], &code, &tem, NULL);
3030 303 : if (!CONVERT_EXPR_CODE_P (code)
3031 203 : || !INTEGRAL_TYPE_P (TREE_TYPE (tem))
3032 709 : || TYPE_PRECISION (TREE_TYPE (tem)) != TYPE_PRECISION (rtype))
3033 : return false;
3034 194 : def_arg1[i] = tem;
3035 194 : if (!operand_equal_for_phi_arg_p (def_arg1[0], def_arg1[1])
3036 218 : || !types_compatible_p (TREE_TYPE (def_arg1[0]),
3037 24 : TREE_TYPE (def_arg1[1])))
3038 : return false;
3039 : }
3040 12296 : else if (!TYPE_UNSIGNED (TREE_TYPE (def_arg1[0])))
3041 : return false;
3042 :
3043 : /* CNT1 + CNT2 == B case above. */
3044 11065 : if (tree_fits_uhwi_p (def_arg2[0])
3045 1568 : && tree_fits_uhwi_p (def_arg2[1])
3046 11065 : && tree_to_uhwi (def_arg2[0])
3047 1568 : + tree_to_uhwi (def_arg2[1]) == TYPE_PRECISION (rtype))
3048 : rotcnt = def_arg2[0];
3049 10142 : else if (TREE_CODE (def_arg2[0]) != SSA_NAME
3050 9497 : || TREE_CODE (def_arg2[1]) != SSA_NAME)
3051 : return false;
3052 : else
3053 : {
3054 9497 : tree cdef_arg1[2], cdef_arg2[2], def_arg2_alt[2];
3055 9497 : enum tree_code cdef_code[2];
3056 9497 : gimple *def_arg_alt_stmt[2] = { NULL, NULL };
3057 9497 : int check_range = 0;
3058 9497 : gimple *check_range_stmt = NULL;
3059 : /* Look through conversion of the shift count argument.
3060 : The C/C++ FE cast any shift count argument to integer_type_node.
3061 : The only problem might be if the shift count type maximum value
3062 : is equal or smaller than number of bits in rtype. */
3063 28491 : for (i = 0; i < 2; i++)
3064 : {
3065 18994 : def_arg2_alt[i] = def_arg2[i];
3066 18994 : defcodefor_name (def_arg2[i], &cdef_code[i],
3067 : &cdef_arg1[i], &cdef_arg2[i]);
3068 14723 : if (CONVERT_EXPR_CODE_P (cdef_code[i])
3069 4271 : && INTEGRAL_TYPE_P (TREE_TYPE (cdef_arg1[i]))
3070 4271 : && TYPE_PRECISION (TREE_TYPE (cdef_arg1[i]))
3071 4271 : > floor_log2 (TYPE_PRECISION (rtype))
3072 23265 : && type_has_mode_precision_p (TREE_TYPE (cdef_arg1[i])))
3073 : {
3074 4271 : def_arg2_alt[i] = cdef_arg1[i];
3075 4271 : if (TREE_CODE (def_arg2[i]) == SSA_NAME)
3076 4271 : def_arg_alt_stmt[i] = SSA_NAME_DEF_STMT (def_arg2[i]);
3077 4271 : defcodefor_name (def_arg2_alt[i], &cdef_code[i],
3078 : &cdef_arg1[i], &cdef_arg2[i]);
3079 : }
3080 : else
3081 14723 : def_arg_alt_stmt[i] = def_arg_stmt[i];
3082 : }
3083 25816 : for (i = 0; i < 2; i++)
3084 : /* Check for one shift count being Y and the other B - Y,
3085 : with optional casts. */
3086 18643 : if (cdef_code[i] == MINUS_EXPR
3087 862 : && tree_fits_shwi_p (cdef_arg1[i])
3088 862 : && tree_to_shwi (cdef_arg1[i]) == TYPE_PRECISION (rtype)
3089 19465 : && TREE_CODE (cdef_arg2[i]) == SSA_NAME)
3090 : {
3091 822 : tree tem;
3092 822 : enum tree_code code;
3093 :
3094 822 : if (cdef_arg2[i] == def_arg2[1 - i]
3095 472 : || cdef_arg2[i] == def_arg2_alt[1 - i])
3096 : {
3097 350 : rotcnt = cdef_arg2[i];
3098 350 : check_range = -1;
3099 350 : if (cdef_arg2[i] == def_arg2[1 - i])
3100 350 : check_range_stmt = def_arg_stmt[1 - i];
3101 : else
3102 0 : check_range_stmt = def_arg_alt_stmt[1 - i];
3103 806 : break;
3104 : }
3105 472 : defcodefor_name (cdef_arg2[i], &code, &tem, NULL);
3106 16 : if (CONVERT_EXPR_CODE_P (code)
3107 456 : && INTEGRAL_TYPE_P (TREE_TYPE (tem))
3108 456 : && TYPE_PRECISION (TREE_TYPE (tem))
3109 456 : > floor_log2 (TYPE_PRECISION (rtype))
3110 456 : && type_has_mode_precision_p (TREE_TYPE (tem))
3111 928 : && (tem == def_arg2[1 - i]
3112 288 : || tem == def_arg2_alt[1 - i]))
3113 : {
3114 456 : rotcnt = tem;
3115 456 : check_range = -1;
3116 456 : if (tem == def_arg2[1 - i])
3117 168 : check_range_stmt = def_arg_stmt[1 - i];
3118 : else
3119 288 : check_range_stmt = def_arg_alt_stmt[1 - i];
3120 : break;
3121 : }
3122 : }
3123 : /* The above sequence isn't safe for Y being 0,
3124 : because then one of the shifts triggers undefined behavior.
3125 : This alternative is safe even for rotation count of 0.
3126 : One shift count is Y and the other (-Y) & (B - 1).
3127 : Or one shift count is Y & (B - 1) and the other (-Y) & (B - 1). */
3128 17821 : else if (cdef_code[i] == BIT_AND_EXPR
3129 28752 : && pow2p_hwi (TYPE_PRECISION (rtype))
3130 12433 : && tree_fits_shwi_p (cdef_arg2[i])
3131 24866 : && tree_to_shwi (cdef_arg2[i])
3132 12433 : == TYPE_PRECISION (rtype) - 1
3133 12373 : && TREE_CODE (cdef_arg1[i]) == SSA_NAME
3134 30194 : && gimple_assign_rhs_code (stmt) == BIT_IOR_EXPR)
3135 : {
3136 2313 : tree tem;
3137 2313 : enum tree_code code;
3138 :
3139 2313 : defcodefor_name (cdef_arg1[i], &code, &tem, NULL);
3140 2114 : if (CONVERT_EXPR_CODE_P (code)
3141 199 : && INTEGRAL_TYPE_P (TREE_TYPE (tem))
3142 199 : && TYPE_PRECISION (TREE_TYPE (tem))
3143 199 : > floor_log2 (TYPE_PRECISION (rtype))
3144 2512 : && type_has_mode_precision_p (TREE_TYPE (tem)))
3145 199 : defcodefor_name (tem, &code, &tem, NULL);
3146 :
3147 2313 : if (code == NEGATE_EXPR)
3148 : {
3149 1532 : if (tem == def_arg2[1 - i] || tem == def_arg2_alt[1 - i])
3150 : {
3151 853 : rotcnt = tem;
3152 853 : check_range = 1;
3153 853 : if (tem == def_arg2[1 - i])
3154 845 : check_range_stmt = def_arg_stmt[1 - i];
3155 : else
3156 8 : check_range_stmt = def_arg_alt_stmt[1 - i];
3157 1518 : break;
3158 : }
3159 679 : tree tem2;
3160 679 : defcodefor_name (tem, &code, &tem2, NULL);
3161 237 : if (CONVERT_EXPR_CODE_P (code)
3162 442 : && INTEGRAL_TYPE_P (TREE_TYPE (tem2))
3163 442 : && TYPE_PRECISION (TREE_TYPE (tem2))
3164 442 : > floor_log2 (TYPE_PRECISION (rtype))
3165 1121 : && type_has_mode_precision_p (TREE_TYPE (tem2)))
3166 : {
3167 442 : if (tem2 == def_arg2[1 - i]
3168 442 : || tem2 == def_arg2_alt[1 - i])
3169 : {
3170 228 : rotcnt = tem2;
3171 228 : check_range = 1;
3172 228 : if (tem2 == def_arg2[1 - i])
3173 0 : check_range_stmt = def_arg_stmt[1 - i];
3174 : else
3175 228 : check_range_stmt = def_arg_alt_stmt[1 - i];
3176 : break;
3177 : }
3178 : }
3179 : else
3180 237 : tem2 = NULL_TREE;
3181 :
3182 451 : if (cdef_code[1 - i] == BIT_AND_EXPR
3183 438 : && tree_fits_shwi_p (cdef_arg2[1 - i])
3184 876 : && tree_to_shwi (cdef_arg2[1 - i])
3185 438 : == TYPE_PRECISION (rtype) - 1
3186 889 : && TREE_CODE (cdef_arg1[1 - i]) == SSA_NAME)
3187 : {
3188 438 : if (tem == cdef_arg1[1 - i]
3189 213 : || tem2 == cdef_arg1[1 - i])
3190 : {
3191 : rotcnt = def_arg2[1 - i];
3192 437 : break;
3193 : }
3194 193 : tree tem3;
3195 193 : defcodefor_name (cdef_arg1[1 - i], &code, &tem3, NULL);
3196 0 : if (CONVERT_EXPR_CODE_P (code)
3197 193 : && INTEGRAL_TYPE_P (TREE_TYPE (tem3))
3198 193 : && TYPE_PRECISION (TREE_TYPE (tem3))
3199 193 : > floor_log2 (TYPE_PRECISION (rtype))
3200 386 : && type_has_mode_precision_p (TREE_TYPE (tem3)))
3201 : {
3202 193 : if (tem == tem3 || tem2 == tem3)
3203 : {
3204 : rotcnt = def_arg2[1 - i];
3205 : break;
3206 : }
3207 : }
3208 : }
3209 : }
3210 : }
3211 2324 : if (check_range && wider_prec > TYPE_PRECISION (rtype))
3212 : {
3213 1533 : if (TREE_CODE (rotcnt) != SSA_NAME)
3214 573 : return false;
3215 1533 : int_range_max r;
3216 1533 : range_query *q = get_range_query (cfun);
3217 1533 : if (q == get_global_range_query ())
3218 1522 : q = enable_ranger (cfun);
3219 1533 : if (!q->range_of_expr (r, rotcnt, check_range_stmt))
3220 : {
3221 0 : if (check_range > 0)
3222 : return false;
3223 0 : r.set_varying (TREE_TYPE (rotcnt));
3224 : }
3225 1533 : int prec = TYPE_PRECISION (TREE_TYPE (rotcnt));
3226 1533 : signop sign = TYPE_SIGN (TREE_TYPE (rotcnt));
3227 1533 : wide_int min = wide_int::from (TYPE_PRECISION (rtype), prec, sign);
3228 1533 : wide_int max = wide_int::from (wider_prec - 1, prec, sign);
3229 1533 : if (check_range < 0)
3230 616 : max = min;
3231 1533 : int_range<1> r2 (TREE_TYPE (rotcnt), min, max);
3232 1533 : r.intersect (r2);
3233 1533 : if (!r.undefined_p ())
3234 : {
3235 1181 : if (check_range > 0)
3236 : {
3237 589 : int_range_max r3;
3238 1844 : for (int i = TYPE_PRECISION (rtype) + 1; i < wider_prec;
3239 1255 : i += TYPE_PRECISION (rtype))
3240 : {
3241 1255 : int j = i + TYPE_PRECISION (rtype) - 2;
3242 1255 : min = wide_int::from (i, prec, sign);
3243 1255 : max = wide_int::from (MIN (j, wider_prec - 1),
3244 1255 : prec, sign);
3245 1255 : int_range<1> r4 (TREE_TYPE (rotcnt), min, max);
3246 1255 : r3.union_ (r4);
3247 1255 : }
3248 589 : r.intersect (r3);
3249 589 : if (!r.undefined_p ())
3250 573 : return false;
3251 589 : }
3252 : add_masking = true;
3253 : }
3254 1533 : }
3255 8924 : if (rotcnt == NULL_TREE)
3256 : return false;
3257 1751 : swapped_p = i != 1;
3258 : }
3259 :
3260 2674 : if (!useless_type_conversion_p (TREE_TYPE (def_arg2[0]),
3261 2674 : TREE_TYPE (rotcnt)))
3262 : {
3263 496 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (def_arg2[0])),
3264 : NOP_EXPR, rotcnt);
3265 496 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3266 496 : rotcnt = gimple_assign_lhs (g);
3267 : }
3268 2674 : if (add_masking)
3269 : {
3270 608 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rotcnt)),
3271 : BIT_AND_EXPR, rotcnt,
3272 608 : build_int_cst (TREE_TYPE (rotcnt),
3273 608 : TYPE_PRECISION (rtype) - 1));
3274 608 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3275 608 : rotcnt = gimple_assign_lhs (g);
3276 : }
3277 2674 : lhs = gimple_assign_lhs (stmt);
3278 2674 : if (!useless_type_conversion_p (rtype, TREE_TYPE (def_arg1[0])))
3279 1010 : lhs = make_ssa_name (TREE_TYPE (def_arg1[0]));
3280 2674 : g = gimple_build_assign (lhs,
3281 2674 : ((def_code[0] == LSHIFT_EXPR) ^ swapped_p)
3282 : ? LROTATE_EXPR : RROTATE_EXPR, def_arg1[0], rotcnt);
3283 2674 : if (!useless_type_conversion_p (rtype, TREE_TYPE (def_arg1[0])))
3284 : {
3285 1010 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3286 1010 : g = gimple_build_assign (gimple_assign_lhs (stmt), NOP_EXPR, lhs);
3287 : }
3288 2674 : gsi_replace (gsi, g, false);
3289 2674 : return true;
3290 : }
3291 :
3292 :
3293 : /* Check whether an array contains a valid table according to VALIDATE_FN. */
3294 : template<typename ValidateFn>
3295 : static bool
3296 55 : check_table_array (tree ctor, HOST_WIDE_INT &zero_val, unsigned bits,
3297 : ValidateFn validate_fn)
3298 : {
3299 : tree elt, idx;
3300 55 : unsigned HOST_WIDE_INT i, raw_idx = 0;
3301 55 : unsigned matched = 0;
3302 :
3303 55 : zero_val = 0;
3304 :
3305 2608 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ctor), i, idx, elt)
3306 : {
3307 2608 : if (!tree_fits_shwi_p (idx))
3308 : return false;
3309 2608 : if (!tree_fits_shwi_p (elt) && TREE_CODE (elt) != RAW_DATA_CST)
3310 : return false;
3311 :
3312 2608 : unsigned HOST_WIDE_INT index = tree_to_shwi (idx);
3313 : HOST_WIDE_INT val;
3314 :
3315 2608 : if (TREE_CODE (elt) == INTEGER_CST)
3316 2544 : val = tree_to_shwi (elt);
3317 : else
3318 : {
3319 64 : if (raw_idx == (unsigned) RAW_DATA_LENGTH (elt))
3320 : {
3321 0 : raw_idx = 0;
3322 0 : continue;
3323 : }
3324 64 : if (TYPE_UNSIGNED (TREE_TYPE (elt)))
3325 0 : val = RAW_DATA_UCHAR_ELT (elt, raw_idx);
3326 : else
3327 64 : val = RAW_DATA_SCHAR_ELT (elt, raw_idx);
3328 64 : index += raw_idx;
3329 64 : raw_idx++;
3330 64 : i--;
3331 : }
3332 :
3333 2608 : if (index > bits * 2)
3334 : return false;
3335 :
3336 2608 : if (index == 0)
3337 : {
3338 55 : zero_val = val;
3339 55 : matched++;
3340 : }
3341 :
3342 2608 : if (val >= 0 && val < bits && validate_fn (val, index))
3343 1888 : matched++;
3344 :
3345 2608 : if (matched > bits)
3346 : return true;
3347 : }
3348 :
3349 : return false;
3350 : }
3351 :
3352 : /* Check whether a string contains a valid table according to VALIDATE_FN. */
3353 : template<typename ValidateFn>
3354 : static bool
3355 39 : check_table_string (tree string, HOST_WIDE_INT &zero_val,unsigned bits,
3356 : ValidateFn validate_fn)
3357 : {
3358 39 : unsigned HOST_WIDE_INT len = TREE_STRING_LENGTH (string);
3359 39 : unsigned matched = 0;
3360 39 : const unsigned char *p = (const unsigned char *) TREE_STRING_POINTER (string);
3361 :
3362 39 : if (len < bits || len > bits * 2)
3363 : return false;
3364 :
3365 39 : zero_val = p[0];
3366 :
3367 1703 : for (unsigned i = 0; i < len; i++)
3368 1664 : if (p[i] < bits && validate_fn (p[i], i))
3369 1664 : matched++;
3370 :
3371 39 : return matched == bits;
3372 : }
3373 :
3374 : /* Check whether CTOR contains a valid table according to VALIDATE_FN. */
3375 : template<typename ValidateFn>
3376 : static bool
3377 102 : check_table (tree ctor, tree type, HOST_WIDE_INT &zero_val, unsigned bits,
3378 : ValidateFn validate_fn)
3379 : {
3380 102 : if (TREE_CODE (ctor) == CONSTRUCTOR)
3381 55 : return check_table_array (ctor, zero_val, bits, validate_fn);
3382 : else if (TREE_CODE (ctor) == STRING_CST
3383 47 : && TYPE_PRECISION (type) == CHAR_TYPE_SIZE)
3384 39 : return check_table_string (ctor, zero_val, bits, validate_fn);
3385 : return false;
3386 : }
3387 :
3388 : /* Match.pd function to match the ctz expression. */
3389 : extern bool gimple_ctz_table_index (tree, tree *, tree (*)(tree));
3390 : extern bool gimple_clz_table_index (tree, tree *, tree (*)(tree));
3391 : extern bool gimple_clz_msb_iso_table_index (tree, tree *, tree (*)(tree));
3392 :
3393 : /* Recognize count leading and trailing zeroes idioms.
3394 : The canonical form is array[((x & -x) * C) >> SHIFT] where C is a magic
3395 : constant which when multiplied by a power of 2 creates a unique value
3396 : in the top 5 or 6 bits. This is then indexed into a table which maps it
3397 : to the number of trailing zeroes. Array[0] is returned so the caller can
3398 : emit an appropriate sequence depending on whether ctz (0) is defined on
3399 : the target. */
3400 :
3401 : static bool
3402 2020771 : simplify_count_zeroes (gimple_stmt_iterator *gsi)
3403 : {
3404 2020771 : gimple *stmt = gsi_stmt (*gsi);
3405 2020771 : tree array_ref = gimple_assign_rhs1 (stmt);
3406 2020771 : tree res_ops[3];
3407 :
3408 2020771 : gcc_checking_assert (TREE_CODE (array_ref) == ARRAY_REF);
3409 :
3410 2020771 : internal_fn fn = IFN_LAST;
3411 : /* When true, the matched idiom is a CLZ using DeBruijn CTZ on the
3412 : isolated MSB -- see clz_msb_iso_table_index in match.pd. The
3413 : table stores MSB positions and must satisfy the direct CTZ
3414 : DeBruijn property, so we validate it with the CTZ checkfn even
3415 : though we emit IFN_CLZ code. */
3416 2020771 : bool clz_via_ctz = false;
3417 : /* For CTZ we recognize ((x & -x) * C) >> SHIFT where the array data
3418 : represents the number of trailing zeros. */
3419 2020771 : if (gimple_ctz_table_index (TREE_OPERAND (array_ref, 1), &res_ops[0], NULL))
3420 : fn = IFN_CTZ;
3421 : /* For CLZ we recognize
3422 : x |= x >> 1;
3423 : x |= x >> 2;
3424 : x |= x >> 4;
3425 : x |= x >> 8;
3426 : x |= x >> 16;
3427 : (x * C) >> SHIFT
3428 : where 31 minus the array data represents the number of leading zeros. */
3429 2020705 : else if (gimple_clz_table_index (TREE_OPERAND (array_ref, 1), &res_ops[0],
3430 : NULL))
3431 : fn = IFN_CLZ;
3432 : /* Variant CLZ idiom: after the OR-cascade sets all bits from 0 to
3433 : the original MSB, (value - (value >> 1)) isolates the MSB as a
3434 : power of two (2^k), and the subsequent DeBruijn multiply-and-shift
3435 : is a CTZ-style lookup on 2^k. The table stores MSB positions
3436 : directly. */
3437 2020665 : else if (gimple_clz_msb_iso_table_index (TREE_OPERAND (array_ref, 1),
3438 : &res_ops[0], NULL))
3439 : {
3440 : fn = IFN_CLZ;
3441 : clz_via_ctz = true;
3442 : }
3443 : else
3444 : return false;
3445 :
3446 107 : HOST_WIDE_INT zero_val;
3447 107 : tree type = TREE_TYPE (array_ref);
3448 107 : tree array = TREE_OPERAND (array_ref, 0);
3449 107 : tree input_type = TREE_TYPE (res_ops[0]);
3450 107 : unsigned input_bits = tree_to_shwi (TYPE_SIZE (input_type));
3451 :
3452 : /* Check the array element type is integral and not wider than 64 bits,
3453 : and the input is an unsigned 32-bit or 64-bit type. The table values
3454 : are bit positions in [0, input_bits - 1], so any integer element type
3455 : with at least 6 bits of precision suffices; the cap is just to keep
3456 : the transformation simple. */
3457 107 : if (!INTEGRAL_TYPE_P (type) || TYPE_PRECISION (type) > 64
3458 214 : || !TYPE_UNSIGNED (input_type))
3459 : return false;
3460 103 : if (input_bits != 32 && input_bits != 64)
3461 : return false;
3462 :
3463 103 : if (!direct_internal_fn_supported_p (fn, input_type, OPTIMIZE_FOR_BOTH))
3464 : return false;
3465 :
3466 : /* Check the lower bound of the array is zero. */
3467 103 : tree low = array_ref_low_bound (array_ref);
3468 103 : if (!low || !integer_zerop (low))
3469 : return false;
3470 :
3471 : /* Check the shift extracts the top 5..7 bits. */
3472 103 : unsigned shiftval = tree_to_shwi (res_ops[2]);
3473 103 : if (shiftval < input_bits - 7 || shiftval > input_bits - 5)
3474 : return false;
3475 :
3476 102 : tree ctor = ctor_for_folding (array);
3477 102 : if (!ctor)
3478 : return false;
3479 102 : unsigned HOST_WIDE_INT mulval = tree_to_uhwi (res_ops[1]);
3480 : /* CTZ and the MSB-isolation CLZ variant both use the direct CTZ
3481 : DeBruijn check (table[(magic << data) >> shift] == data). */
3482 102 : if (fn == IFN_CTZ || clz_via_ctz)
3483 : {
3484 2782 : auto checkfn = [&](unsigned data, unsigned i) -> bool
3485 : {
3486 2720 : unsigned HOST_WIDE_INT mask
3487 2720 : = ((HOST_WIDE_INT_1U << (input_bits - shiftval)) - 1) << shiftval;
3488 2720 : return (((mulval << data) & mask) >> shiftval) == i;
3489 62 : };
3490 62 : if (!check_table (ctor, type, zero_val, input_bits, checkfn))
3491 8 : return false;
3492 : }
3493 40 : else if (fn == IFN_CLZ)
3494 : {
3495 1569 : auto checkfn = [&](unsigned data, unsigned i) -> bool
3496 : {
3497 1529 : unsigned HOST_WIDE_INT mask
3498 1529 : = ((HOST_WIDE_INT_1U << (input_bits - shiftval)) - 1) << shiftval;
3499 : /* The OR-cascade produces a value with all bits from 0 to the
3500 : original MSB set. Compute (1 << (data + 1)) - 1 to simulate
3501 : that value. When data + 1 equals HOST_BITS_PER_WIDE_INT
3502 : (i.e. data is the MSB position of a 64-bit input) the shift
3503 : is undefined behavior, so handle that case explicitly using
3504 : all-ones. Without this, any well-formed 64-bit DeBruijn CLZ
3505 : table is rejected because its entry for the all-ones input
3506 : correctly maps to the MSB (e.g. table[...] == 63).
3507 : PR tree-optimization/122569. */
3508 3057 : unsigned HOST_WIDE_INT all_bits_below
3509 : = (data + 1 == HOST_BITS_PER_WIDE_INT)
3510 1529 : ? HOST_WIDE_INT_M1U
3511 1528 : : ((HOST_WIDE_INT_1U << (data + 1)) - 1);
3512 1529 : return (((all_bits_below * mulval) & mask) >> shiftval) == i;
3513 40 : };
3514 40 : if (!check_table (ctor, type, zero_val, input_bits, checkfn))
3515 0 : return false;
3516 : }
3517 :
3518 : /* The sequence below computes the result in int, for a CLZ as
3519 : INPUT_BITS - 1 minus the count. If the table entry for zero does not
3520 : survive that, select it in TYPE after the conversion instead. */
3521 94 : HOST_WIDE_INT table_zero = zero_val;
3522 94 : bool late_select
3523 94 : = (!wi::fits_to_tree_p (zero_val, integer_type_node)
3524 94 : || (fn == IFN_CLZ
3525 34 : && !wi::fits_to_tree_p (input_bits - 1 - zero_val,
3526 34 : integer_type_node)));
3527 :
3528 94 : HOST_WIDE_INT ctz_val = -1;
3529 94 : bool zero_ok;
3530 94 : if (fn == IFN_CTZ)
3531 : {
3532 53 : ctz_val = 0;
3533 106 : zero_ok = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (input_type),
3534 : ctz_val) == 2;
3535 : }
3536 41 : else if (fn == IFN_CLZ)
3537 : {
3538 41 : ctz_val = 32;
3539 41 : zero_ok = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (input_type),
3540 : ctz_val) == 2;
3541 41 : if (!late_select)
3542 34 : zero_val = input_bits - 1 - zero_val;
3543 : }
3544 : /* Pass the value at zero only if the target defines it; otherwise the
3545 : COND_EXPR below supplies the result for zero. */
3546 94 : int nargs = zero_ok ? 2 : 1;
3547 :
3548 : /* If the input value can't be zero, don't special case ctz (0). */
3549 94 : range_query *q = get_range_query (cfun);
3550 94 : if (q == get_global_range_query ())
3551 94 : q = enable_ranger (cfun);
3552 94 : int_range_max vr;
3553 94 : if (q->range_of_expr (vr, res_ops[0], stmt)
3554 94 : && !range_includes_zero_p (vr))
3555 : {
3556 24 : zero_ok = true;
3557 24 : zero_val = 0;
3558 24 : ctz_val = 0;
3559 24 : nargs = 1;
3560 24 : late_select = false;
3561 : }
3562 :
3563 94 : gimple_seq seq = NULL;
3564 94 : gimple *g;
3565 102 : gcall *call = gimple_build_call_internal (fn, nargs, res_ops[0],
3566 : nargs == 1 ? NULL_TREE
3567 102 : : build_int_cst (integer_type_node,
3568 8 : ctz_val));
3569 94 : gimple_set_location (call, gimple_location (stmt));
3570 94 : gimple_set_lhs (call, make_ssa_name (integer_type_node));
3571 94 : gimple_seq_add_stmt (&seq, call);
3572 :
3573 94 : tree prev_lhs = gimple_call_lhs (call);
3574 :
3575 94 : if (late_select || (zero_ok && zero_val == ctz_val))
3576 : ;
3577 : /* Emit ctz (x) & 31 if ctz (0) is 32 but we need to return 0. */
3578 6 : else if (zero_ok && zero_val == 0 && ctz_val == input_bits)
3579 : {
3580 5 : g = gimple_build_assign (make_ssa_name (integer_type_node),
3581 : BIT_AND_EXPR, prev_lhs,
3582 : build_int_cst (integer_type_node,
3583 5 : input_bits - 1));
3584 5 : gimple_set_location (g, gimple_location (stmt));
3585 5 : gimple_seq_add_stmt (&seq, g);
3586 5 : prev_lhs = gimple_assign_lhs (g);
3587 : }
3588 : /* As fallback emit a conditional move. */
3589 : else
3590 : {
3591 42 : g = gimple_build_assign (make_ssa_name (boolean_type_node), EQ_EXPR,
3592 : res_ops[0], build_zero_cst (input_type));
3593 42 : gimple_set_location (g, gimple_location (stmt));
3594 42 : gimple_seq_add_stmt (&seq, g);
3595 42 : tree cond = gimple_assign_lhs (g);
3596 42 : g = gimple_build_assign (make_ssa_name (integer_type_node),
3597 : COND_EXPR, cond,
3598 42 : build_int_cst (integer_type_node, zero_val),
3599 : prev_lhs);
3600 42 : gimple_set_location (g, gimple_location (stmt));
3601 42 : gimple_seq_add_stmt (&seq, g);
3602 42 : prev_lhs = gimple_assign_lhs (g);
3603 : }
3604 :
3605 94 : if (fn == IFN_CLZ)
3606 : {
3607 41 : g = gimple_build_assign (make_ssa_name (integer_type_node),
3608 : MINUS_EXPR,
3609 : build_int_cst (integer_type_node,
3610 41 : input_bits - 1),
3611 : prev_lhs);
3612 41 : gimple_set_location (g, gimple_location (stmt));
3613 41 : gimple_seq_add_stmt (&seq, g);
3614 41 : prev_lhs = gimple_assign_lhs (g);
3615 : }
3616 :
3617 94 : if (late_select)
3618 : {
3619 21 : g = gimple_build_assign (make_ssa_name (type), NOP_EXPR, prev_lhs);
3620 21 : gimple_set_location (g, gimple_location (stmt));
3621 21 : gimple_seq_add_stmt (&seq, g);
3622 21 : prev_lhs = gimple_assign_lhs (g);
3623 21 : g = gimple_build_assign (make_ssa_name (boolean_type_node), EQ_EXPR,
3624 : res_ops[0], build_zero_cst (input_type));
3625 21 : gimple_set_location (g, gimple_location (stmt));
3626 21 : gimple_seq_add_stmt (&seq, g);
3627 21 : g = gimple_build_assign (gimple_assign_lhs (stmt), COND_EXPR,
3628 : gimple_assign_lhs (g),
3629 21 : build_int_cst (type, table_zero), prev_lhs);
3630 : }
3631 : else
3632 73 : g = gimple_build_assign (gimple_assign_lhs (stmt), NOP_EXPR, prev_lhs);
3633 94 : gimple_seq_add_stmt (&seq, g);
3634 94 : gsi_replace_with_seq (gsi, seq, true);
3635 94 : return true;
3636 94 : }
3637 :
3638 : /* Long-multiply fold framework.
3639 :
3640 : Walks the outer addition or bit_ior chain on a candidate statement,
3641 : classifies each summand against the atom match patterns from
3642 : match.pd, and looks the resulting multiset of (kind, extract) tuples
3643 : up in a table. On a hit, three cross-summand consistency checks
3644 : decide whether the wide multiply is emitted. */
3645 :
3646 : /* Match.pd recognizers for the conditional carry-add pattern. */
3647 :
3648 : extern bool gimple_cond_carry_add (tree, tree *, tree (*)(tree));
3649 :
3650 : /* Match.pd functions to match long multiplication. */
3651 :
3652 : extern bool gimple_mul_hi (tree, tree *, tree (*)(tree));
3653 : extern bool gimple_mul_lo (tree, tree *, tree (*)(tree));
3654 : extern bool gimple_mul_hilo (tree, tree *, tree (*)(tree));
3655 : extern bool gimple_mul_lolo (tree, tree *, tree (*)(tree));
3656 : extern bool gimple_mul_hihi (tree, tree *, tree (*)(tree));
3657 : extern bool gimple_mul_cross_sum (tree, tree *, tree (*)(tree));
3658 : extern bool gimple_mul_low_sum (tree, tree *, tree (*)(tree));
3659 : extern bool gimple_mul_low_accum (tree, tree *, tree (*)(tree));
3660 : extern bool gimple_mul_carry_cross_sum (tree, tree *, tree (*)(tree));
3661 : extern bool gimple_mul_carry_low_sum (tree, tree *, tree (*)(tree));
3662 : extern bool gimple_mul_carry_low (tree, tree *, tree (*)(tree));
3663 : extern bool gimple_mul_ladder_sum1 (tree, tree *, tree (*)(tree));
3664 : extern bool gimple_mul_ladder_sum2 (tree, tree *, tree (*)(tree));
3665 : extern bool gimple_mul_ladder_sum3 (tree, tree *, tree (*)(tree));
3666 : extern bool gimple_mul_ladder_part_sum (tree, tree *, tree (*)(tree));
3667 :
3668 : /* Append to SEQ statements assigning DEST the high-part multiply of
3669 : OP1 and OP2, emitted as
3670 : (N)(((2N) op1 * (2N) op2) >> N).
3671 : pass_optimize_widening_mul's convert_mult_to_widen and
3672 : convert_mult_to_highpart later rewrite this to a single
3673 : WIDEN_MULT_EXPR or MULT_HIGHPART_EXPR when the target supports it,
3674 : otherwise the 2N multiply expands directly. Emitting the canonical
3675 : widening shape keeps target-capability decisions in the layer that
3676 : already owns them. */
3677 :
3678 : static void
3679 1926 : build_mul_high_seq (tree op1, tree op2, tree dest, location_t loc,
3680 : gimple_seq *seq)
3681 : {
3682 1926 : tree op_type = TREE_TYPE (op1);
3683 1926 : unsigned int width = TYPE_PRECISION (op_type);
3684 1926 : tree wide_type = build_nonstandard_integer_type (width * 2, 1);
3685 :
3686 1926 : tree wide_a = gimple_convert (seq, loc, wide_type, op1);
3687 1926 : tree wide_b = gimple_convert (seq, loc, wide_type, op2);
3688 1926 : tree wide_prod = gimple_build (seq, loc, MULT_EXPR, wide_type,
3689 : wide_a, wide_b);
3690 1926 : tree hi = gimple_build (seq, loc, RSHIFT_EXPR, wide_type, wide_prod,
3691 1926 : build_int_cst (integer_type_node, width));
3692 :
3693 1926 : gimple *prod = gimple_build_assign (dest, NOP_EXPR, hi);
3694 1926 : gimple_set_location (prod, loc);
3695 1926 : gimple_seq_add_stmt (seq, prod);
3696 1926 : }
3697 :
3698 : /* Append to SEQ statements combining ACC with each of EXTRAS under
3699 : OUTER, the last one assigning to STMT's lhs. EXTRAS are leaves of
3700 : STMT's own chain, so any combining order is valid. */
3701 :
3702 : static void
3703 361 : long_mul_apply_extras (tree acc, const vec<tree> &extras, tree_code outer,
3704 : gassign *stmt, gimple_seq *seq)
3705 : {
3706 361 : location_t loc = gimple_location (stmt);
3707 361 : tree lhs = gimple_assign_lhs (stmt);
3708 388 : for (unsigned i = 0; i + 1 < extras.length (); i++)
3709 27 : acc = gimple_build (seq, loc, outer, TREE_TYPE (lhs), acc, extras[i]);
3710 361 : gimple *last = gimple_build_assign (lhs, outer, acc, extras.last ());
3711 361 : gimple_set_location (last, loc);
3712 361 : gimple_seq_add_stmt (seq, last);
3713 361 : }
3714 :
3715 : /* Replace STMT with a high-part multiply of OP1 and OP2, combining any
3716 : EXTRAS back on top under OUTER. */
3717 :
3718 : static void
3719 1912 : create_mul_high_seq (tree op1, tree op2, gassign *stmt,
3720 : const vec<tree> &extras, tree_code outer)
3721 : {
3722 1912 : gimple_seq seq = NULL;
3723 1912 : tree lhs = gimple_assign_lhs (stmt);
3724 2273 : tree dest = extras.is_empty () ? lhs : make_ssa_name (TREE_TYPE (lhs));
3725 1912 : build_mul_high_seq (op1, op2, dest, gimple_location (stmt), &seq);
3726 1912 : if (!extras.is_empty ())
3727 361 : long_mul_apply_extras (dest, extras, outer, stmt, &seq);
3728 1912 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3729 1912 : gsi_replace_with_seq (&gsi, seq, true);
3730 1912 : }
3731 :
3732 : /* Replace STMT with a low-part multiply of OP1 and OP2, combining any
3733 : EXTRAS back on top under OUTER. */
3734 :
3735 : static void
3736 34 : create_mul_low_seq (tree op1, tree op2, gassign *stmt,
3737 : const vec<tree> &extras, tree_code outer)
3738 : {
3739 34 : gimple_seq seq = NULL;
3740 34 : tree lhs = gimple_assign_lhs (stmt);
3741 34 : tree dest = extras.is_empty () ? lhs : make_ssa_name (TREE_TYPE (lhs));
3742 34 : gimple *prod = gimple_build_assign (dest, MULT_EXPR, op1, op2);
3743 34 : gimple_set_location (prod, gimple_location (stmt));
3744 34 : gimple_seq_add_stmt (&seq, prod);
3745 34 : if (!extras.is_empty ())
3746 0 : long_mul_apply_extras (dest, extras, outer, stmt, &seq);
3747 34 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3748 34 : gsi_replace_with_seq (&gsi, seq, true);
3749 34 : }
3750 :
3751 : /* Widest match.pd atom (mul_carry_low_sum) takes 7 captures; round up
3752 : to 8 for the scratch buffers below. */
3753 : static constexpr unsigned LONG_MUL_MAX_CAPTURES = 8;
3754 :
3755 : /* Longest variant in long_mul_table has 4 summands. */
3756 : static constexpr unsigned LONG_MUL_MAX_SUMMANDS = 4;
3757 :
3758 : /* Cap on the leaves set aside as not part of the idiom, so an
3759 : arbitrarily long unrelated chain still bails early. */
3760 : static constexpr unsigned LONG_MUL_MAX_EXTRAS = 4;
3761 :
3762 : namespace {
3763 :
3764 : enum long_mul_kind : unsigned char {
3765 : LMK_INVALID,
3766 : LMK_MUL_HIHI,
3767 : LMK_MUL_LOLO,
3768 : LMK_MUL_HILO,
3769 : LMK_CROSS_SUM,
3770 : LMK_LOW_ACCUM,
3771 : LMK_LOW_SUM,
3772 : LMK_LADDER_SUM1,
3773 : LMK_LADDER_SUM2,
3774 : LMK_LADDER_SUM3,
3775 : LMK_LADDER_PART_SUM,
3776 : LMK_CARRY_LOW,
3777 : LMK_CARRY_CROSS_SUM,
3778 : LMK_CARRY_LOW_SUM,
3779 : };
3780 :
3781 : /* How the leaf wraps its inner kind. Carry kinds use LMX_NONE: their
3782 : match.pd pattern bakes the lshift in, so the leaf is already the
3783 : complete carry expression. */
3784 :
3785 : enum long_mul_extract : unsigned char {
3786 : LMX_NONE,
3787 : LMX_HI,
3788 : LMX_LO,
3789 : LMX_SHL_N,
3790 : };
3791 :
3792 : struct long_mul_summand {
3793 6990872 : long_mul_summand () {}
3794 : long_mul_summand (tree);
3795 : long_mul_summand (const long_mul_summand &) = default;
3796 : long_mul_kind kind = LMK_INVALID;
3797 : long_mul_extract extract = LMX_NONE;
3798 : tree op0 = NULL_TREE, op1 = NULL_TREE;
3799 : tree hilo[3] = { NULL_TREE, NULL_TREE, NULL_TREE };
3800 : tree carry_a = NULL_TREE, carry_b = NULL_TREE;
3801 : unsigned HOST_WIDE_INT shift = 0;
3802 6971639 : operator bool () { return kind != LMK_INVALID; }
3803 : private:
3804 : bool classify_carry (tree leaf);
3805 : bool classify_hi_extract (tree, unsigned HOST_WIDE_INT);
3806 : bool classify_lo_extract (tree);
3807 : bool classify_shl_extract (tree, unsigned HOST_WIDE_INT);
3808 : bool classify_plus_kinds (tree);
3809 : bool classify_bare (tree);
3810 : void set (long_mul_kind kind, const tree *res_ops);
3811 : };
3812 :
3813 : }
3814 :
3815 : /* Walk the OUTER addition or BIT_IOR chain rooted at STMT and collect
3816 : the leaf operands into LEAVES. Descends through single-use
3817 : intermediate stmts of the same code. Returns false once the leaf
3818 : count exceeds LONG_MUL_MAX_SUMMANDS + LONG_MUL_MAX_EXTRAS, so an
3819 : overlong chain bails mid-walk instead of after a full traversal.
3820 :
3821 : If SHARED_DEF_OUT is non-NULL, record there the first inner stmt that
3822 : shares the outer code but has more than one use -- descending into it
3823 : would change semantics, so it stays as a leaf. Such a leaf often
3824 : classifies as something no row matches, silently disabling the fold;
3825 : the caller surfaces this as a dump-file hint. */
3826 :
3827 : static bool
3828 3405795 : long_mul_linearize_chain (gimple *stmt, tree_code outer, vec<tree> &leaves,
3829 : gimple **shared_def_out = NULL)
3830 : {
3831 3405795 : auto_vec<tree, 8> stack;
3832 3405795 : stack.safe_push (gimple_assign_rhs2 (stmt));
3833 3405795 : stack.safe_push (gimple_assign_rhs1 (stmt));
3834 :
3835 10566324 : while (!stack.is_empty ())
3836 : {
3837 7161324 : tree t = stack.pop ();
3838 7161324 : if (TREE_CODE (t) == SSA_NAME)
3839 : {
3840 4704415 : gimple *def = SSA_NAME_DEF_STMT (t);
3841 4704415 : if (def
3842 4704415 : && is_gimple_assign (def)
3843 7946130 : && gimple_assign_rhs_code (def) == outer)
3844 : {
3845 340181 : if (has_single_use (t))
3846 : {
3847 177212 : stack.safe_push (gimple_assign_rhs2 (def));
3848 177212 : stack.safe_push (gimple_assign_rhs1 (def));
3849 177212 : continue;
3850 : }
3851 162969 : if (shared_def_out && !*shared_def_out)
3852 147894 : *shared_def_out = def;
3853 : }
3854 : }
3855 6984112 : leaves.safe_push (t);
3856 6984112 : if (leaves.length () > LONG_MUL_MAX_SUMMANDS + LONG_MUL_MAX_EXTRAS)
3857 795 : return false;
3858 : }
3859 6810000 : return !leaves.is_empty ();
3860 3405795 : }
3861 :
3862 : /* If EXPR is defined by LSHIFT_EXPR with a uhwi-valued amount, return
3863 : the shifted input via *INNER_OUT and the amount via *SHIFT_OUT. */
3864 :
3865 : static bool
3866 6911593 : long_mul_is_lshift_def (tree expr, tree *inner_out,
3867 : unsigned HOST_WIDE_INT *shift_out)
3868 : {
3869 6911593 : if (TREE_CODE (expr) != SSA_NAME)
3870 : return false;
3871 4455170 : gimple *def = SSA_NAME_DEF_STMT (expr);
3872 4455170 : if (!def || !is_gimple_assign (def)
3873 7448401 : || gimple_assign_rhs_code (def) != LSHIFT_EXPR)
3874 : return false;
3875 85503 : tree amount = gimple_assign_rhs2 (def);
3876 85503 : if (!tree_fits_uhwi_p (amount))
3877 : return false;
3878 59208 : *inner_out = gimple_assign_rhs1 (def);
3879 59208 : *shift_out = tree_to_uhwi (amount);
3880 59208 : return true;
3881 : }
3882 :
3883 : /* Fill THIS's kind plus the captures from RES_OPS that the kind requires.
3884 : The kind itself determines how many (op0, op1) and hilo captures to
3885 : pick up from RES_OPS, and whether a baked-in shift is present. */
3886 :
3887 : void
3888 55611 : long_mul_summand::set (long_mul_kind kind, const tree *res_ops)
3889 : {
3890 55611 : this->kind = kind;
3891 55611 : unsigned n_ops = 0;
3892 55611 : unsigned n_hilos = 0;
3893 55611 : int shift_idx = -1;
3894 53164 : switch (kind)
3895 : {
3896 0 : default:
3897 0 : gcc_unreachable ();
3898 : case LMK_MUL_HIHI:
3899 : case LMK_MUL_LOLO:
3900 : case LMK_MUL_HILO:
3901 : n_ops = 2;
3902 : break;
3903 5750 : case LMK_CROSS_SUM:
3904 5750 : n_hilos = 2;
3905 5750 : break;
3906 18606 : case LMK_LOW_ACCUM:
3907 18606 : case LMK_LOW_SUM:
3908 18606 : case LMK_LADDER_SUM1:
3909 18606 : case LMK_LADDER_SUM2:
3910 18606 : case LMK_LADDER_SUM3:
3911 18606 : n_ops = 2;
3912 18606 : n_hilos = 2;
3913 18606 : break;
3914 148 : case LMK_LADDER_PART_SUM:
3915 148 : n_ops = 2;
3916 148 : n_hilos = 1;
3917 148 : break;
3918 : case LMK_CARRY_CROSS_SUM:
3919 : n_hilos = 3;
3920 : shift_idx = 3;
3921 : break;
3922 106 : case LMK_CARRY_LOW_SUM:
3923 106 : n_ops = 2;
3924 106 : n_hilos = 3;
3925 106 : shift_idx = 5;
3926 106 : break;
3927 2447 : case LMK_CARRY_LOW:
3928 2447 : this->carry_a = res_ops[0];
3929 2447 : this->carry_b = res_ops[1];
3930 0 : return;
3931 : }
3932 24610 : if (n_ops >= 1)
3933 47240 : this->op0 = res_ops[0];
3934 47240 : if (n_ops >= 2)
3935 47240 : this->op1 = res_ops[1];
3936 53164 : gcc_checking_assert (n_hilos <= 3);
3937 102864 : for (unsigned i = 0; i < n_hilos; i++)
3938 49700 : this->hilo[i] = res_ops[n_ops + i];
3939 53164 : if (shift_idx >= 0)
3940 : /* The carry atoms (mul_carry_cross_sum, mul_carry_low_sum) capture the
3941 : shift since it is always less than TYPE_PRECISION, using tree_to_uhwi is safe. */
3942 280 : this->shift = tree_to_uhwi (res_ops[shift_idx]);
3943 : }
3944 :
3945 : /* Classify LEAF as a carry-kind summand. The lshift amount is baked
3946 : into mul_carry_cross_sum / mul_carry_low_sum, so they're tried before
3947 : any branch that looks for a generic (X >> N) or (X << N) wrapper. */
3948 :
3949 : bool
3950 6971639 : long_mul_summand::classify_carry (tree leaf)
3951 : {
3952 6971639 : tree res_ops[LONG_MUL_MAX_CAPTURES];
3953 : /* mul_carry_low_sum's inner is constrained to mul_low_sum (cross_sum
3954 : + mul_hi(mul_lolo)); mul_carry_cross_sum's inner is just
3955 : mul_cross_sum (any plus); mul_carry_low matches gt:c (@0, plus(@0,
3956 : @1)) without a baked-in shift. Most specific first, so the
3957 : less-constrained pattern doesn't shadow the more-constrained one. */
3958 6971639 : if (gimple_mul_carry_low_sum (leaf, res_ops, NULL))
3959 : {
3960 106 : set (LMK_CARRY_LOW_SUM, res_ops);
3961 106 : return true;
3962 : }
3963 6971533 : if (gimple_mul_carry_cross_sum (leaf, res_ops, NULL))
3964 : {
3965 174 : set (LMK_CARRY_CROSS_SUM, res_ops);
3966 174 : return true;
3967 : }
3968 6971359 : if (gimple_mul_carry_low (leaf, res_ops, NULL))
3969 : {
3970 2447 : set (LMK_CARRY_LOW, res_ops);
3971 2447 : return true;
3972 : }
3973 : return false;
3974 : }
3975 :
3976 : /* Plus-based summand kinds shared by the (X >> SHIFT) and (X << SHIFT)
3977 : classifiers. Order is by specificity: mul_low_sum's first arm is any
3978 : plus, so mul_ladder_sum1/3 (which constrain that arm to a plus
3979 : containing a mul_lo) and mul_low_accum (which constrains both arms)
3980 : shadow it and must come first. */
3981 :
3982 : bool
3983 85550 : long_mul_summand::classify_plus_kinds (tree inner)
3984 : {
3985 85550 : tree res_ops[LONG_MUL_MAX_CAPTURES];
3986 85550 : if (gimple_mul_low_accum (inner, res_ops, NULL))
3987 : {
3988 28 : set (LMK_LOW_ACCUM, res_ops);
3989 28 : return true;
3990 : }
3991 85522 : if (gimple_mul_ladder_sum3 (inner, res_ops, NULL))
3992 : {
3993 15 : set (LMK_LADDER_SUM3, res_ops);
3994 15 : return true;
3995 : }
3996 85507 : if (gimple_mul_ladder_sum1 (inner, res_ops, NULL))
3997 : {
3998 18230 : set (LMK_LADDER_SUM1, res_ops);
3999 18230 : return true;
4000 : }
4001 67277 : if (gimple_mul_low_sum (inner, res_ops, NULL))
4002 : {
4003 230 : set (LMK_LOW_SUM, res_ops);
4004 230 : return true;
4005 : }
4006 67047 : if (gimple_mul_ladder_sum2 (inner, res_ops, NULL))
4007 : {
4008 103 : set (LMK_LADDER_SUM2, res_ops);
4009 103 : return true;
4010 : }
4011 : return false;
4012 : }
4013 :
4014 : /* Classify INNER -- already unwrapped from an outer (X >> SHIFT) -- as
4015 : a high-half-extracted summand. mul_hilo (mult-shape) is orthogonal
4016 : to the plus-based kinds and is tried first; ladder_part_sum (one arm
4017 : unconstrained) and mul_cross_sum (any plus) are the fallbacks after
4018 : the shared plus-based ladder. */
4019 :
4020 : bool
4021 35612 : long_mul_summand::classify_hi_extract (tree inner, unsigned HOST_WIDE_INT shift)
4022 : {
4023 35612 : tree res_ops[LONG_MUL_MAX_CAPTURES];
4024 35612 : this->extract = LMX_HI;
4025 35612 : this->shift = shift;
4026 35612 : if (gimple_mul_hilo (inner, res_ops, NULL))
4027 : {
4028 9258 : set (LMK_MUL_HILO, res_ops);
4029 9258 : return true;
4030 : }
4031 26354 : if (classify_plus_kinds (inner))
4032 : return true;
4033 16919 : if (gimple_mul_ladder_part_sum (inner, res_ops, NULL))
4034 : {
4035 148 : set (LMK_LADDER_PART_SUM, res_ops);
4036 148 : return true;
4037 : }
4038 16771 : if (gimple_mul_cross_sum (inner, res_ops, NULL))
4039 : {
4040 2821 : set (LMK_CROSS_SUM, res_ops);
4041 2821 : return true;
4042 : }
4043 : return false;
4044 : }
4045 :
4046 : /* Classify INNER -- already unwrapped from an outer (X & MASK) -- as
4047 : a low-half-masked summand. */
4048 :
4049 : bool
4050 21721 : long_mul_summand::classify_lo_extract (tree inner)
4051 : {
4052 21721 : tree res_ops[LONG_MUL_MAX_CAPTURES];
4053 21721 : this->extract = LMX_LO;
4054 21721 : if (gimple_mul_lolo (inner, res_ops, NULL))
4055 : {
4056 9331 : set (LMK_MUL_LOLO, res_ops);
4057 9331 : return true;
4058 : }
4059 : return false;
4060 : }
4061 :
4062 : /* Classify INNER -- already unwrapped from an outer (X << SHIFT) -- as
4063 : a left-shifted summand. No mul_hilo / ladder_part_sum here -- those
4064 : shapes appear only under (X >> SHIFT). */
4065 :
4066 : bool
4067 59196 : long_mul_summand::classify_shl_extract (tree inner,
4068 : unsigned HOST_WIDE_INT shift)
4069 : {
4070 59196 : tree res_ops[LONG_MUL_MAX_CAPTURES];
4071 59196 : this->extract = LMX_SHL_N;
4072 59196 : this->shift = shift;
4073 59196 : if (classify_plus_kinds (inner))
4074 : return true;
4075 50025 : if (gimple_mul_cross_sum (inner, res_ops, NULL))
4076 : {
4077 2929 : set (LMK_CROSS_SUM, res_ops);
4078 2929 : return true;
4079 : }
4080 : return false;
4081 : }
4082 :
4083 : /* Classify LEAF as one of the bare-kind summands (no extraction
4084 : wrapper): mul_hihi or mul_lolo standing on their own. */
4085 :
4086 : bool
4087 6852383 : long_mul_summand::classify_bare (tree leaf)
4088 : {
4089 6852383 : tree res_ops[LONG_MUL_MAX_CAPTURES];
4090 6852383 : if (gimple_mul_hihi (leaf, res_ops, NULL))
4091 : {
4092 9728 : set (LMK_MUL_HIHI, res_ops);
4093 9728 : return true;
4094 : }
4095 6842655 : if (gimple_mul_lolo (leaf, res_ops, NULL))
4096 : {
4097 63 : set (LMK_MUL_LOLO, res_ops);
4098 63 : return true;
4099 : }
4100 : return false;
4101 : }
4102 :
4103 : /* Classify LEAF as one of the long-multiply summand shapes. On success,
4104 : fill *INFO with the kind, extract, captured operands and shift.
4105 : Dispatches to per-extract helpers; the order matters because the
4106 : carry kinds bake an lshift into the pattern and would otherwise be
4107 : misread by the (X << N) branch. */
4108 :
4109 6971639 : long_mul_summand::long_mul_summand (tree leaf)
4110 6971639 : : long_mul_summand()
4111 : {
4112 6971639 : tree res_ops[LONG_MUL_MAX_CAPTURES];
4113 :
4114 6971639 : if (classify_carry (leaf))
4115 119256 : return;
4116 :
4117 6968912 : if (gimple_mul_hi (leaf, res_ops, NULL))
4118 : {
4119 35612 : classify_hi_extract (res_ops[0], tree_to_uhwi (res_ops[1]));
4120 35612 : return;
4121 : }
4122 :
4123 6933300 : if (gimple_mul_lo (leaf, res_ops, NULL))
4124 : {
4125 21721 : classify_lo_extract (res_ops[0]);
4126 21721 : return;
4127 : }
4128 :
4129 6911579 : tree inner;
4130 6911579 : unsigned HOST_WIDE_INT shift;
4131 6911579 : if (long_mul_is_lshift_def (leaf, &inner, &shift))
4132 : {
4133 59196 : classify_shl_extract (inner, shift);
4134 59196 : return;
4135 : }
4136 :
4137 6852383 : classify_bare (leaf);
4138 : }
4139 :
4140 : /* qsort comparator: sort summands by (kind, extract) to put a multiset
4141 : into canonical order for table lookup. Unstable sort within a tie is
4142 : harmless: no row in long_mul_table pairs distinct subterms under the
4143 : same (kind, extract), and long_mul_check_consistency cross-validates
4144 : that matching summands share one canonical (op0, op1). */
4145 :
4146 : static int
4147 140020 : long_mul_summand_compare (const void *a, const void *b)
4148 : {
4149 140020 : const long_mul_summand *sa = (const long_mul_summand *) a;
4150 140020 : const long_mul_summand *sb = (const long_mul_summand *) b;
4151 140020 : gcc_checking_assert (sa->kind != LMK_INVALID);
4152 140020 : gcc_checking_assert (sb->kind != LMK_INVALID);
4153 140020 : if (sa->kind != sb->kind)
4154 139157 : return (int) sa->kind - (int) sb->kind;
4155 863 : return (int) sa->extract - (int) sb->extract;
4156 : }
4157 :
4158 : /* One row of the long-multiply variant table. COUNT is how many entries
4159 : of SIG carry the row's signature (2 to LONG_MUL_MAX_SUMMANDS); a row
4160 : with fewer summands leaves the remaining SIG entries zero-initialized.
4161 : Those zeros are not a terminator -- {LMK_MUL_HIHI, LMX_NONE} is itself a
4162 : valid signature -- so long_mul_signature_matches is bounded by COUNT,
4163 : never by a sentinel entry. */
4164 :
4165 : struct long_mul_row {
4166 : enum long_mul_row_part { HIGH_PART, LOW_PART } part;
4167 : tree_code outer;
4168 : unsigned char count;
4169 : struct {
4170 : long_mul_kind kind;
4171 : long_mul_extract extract;
4172 : } sig[LONG_MUL_MAX_SUMMANDS];
4173 : bool (*extra_check) (const vec<long_mul_summand> &, gimple *);
4174 : };
4175 :
4176 : /* True if (A, B) is the same pair as (OP0, OP1) in either order. */
4177 :
4178 : static inline bool
4179 23337 : long_mul_same_ops (tree a, tree b, tree op0, tree op1)
4180 : {
4181 32740 : return (a == op0 && b == op1) || (a == op1 && b == op0);
4182 : }
4183 :
4184 : /* True if H is a cross-half product of (OP0, OP1) -- gimple_mul_hilo
4185 : recognizes it and its captured operands match the pair. */
4186 :
4187 : static bool
4188 4473 : long_mul_is_cross_half (tree h, tree op0, tree op1)
4189 : {
4190 4473 : tree scratch[LONG_MUL_MAX_CAPTURES];
4191 4473 : return gimple_mul_hilo (h, scratch, NULL)
4192 4473 : && long_mul_same_ops (scratch[0], scratch[1], op0, op1);
4193 : }
4194 :
4195 : /* Orientation of the mul_hilo capture H relative to (OP0, OP1):
4196 : returns 0 for high(OP0)*low(OP1), 1 for high(OP1)*low(OP0), or -1
4197 : if H does not decompose that way. A cross-sum of two mul_hilos must
4198 : see one of each orientation -- otherwise a doubled factor would fold
4199 : to the wrong value. (In a self-multiply the two orientations
4200 : coincide; see the OP0 == OP1 bypass in long_mul_check_consistency.) */
4201 :
4202 : static int
4203 6085 : long_mul_hilo_orientation (tree h, tree op0, tree op1)
4204 : {
4205 6085 : tree scratch[LONG_MUL_MAX_CAPTURES];
4206 6085 : if (!gimple_mul_hilo (h, scratch, NULL))
4207 : return -1;
4208 6085 : if (scratch[0] == op0 && scratch[1] == op1)
4209 : return 0;
4210 2135 : if (scratch[0] == op1 && scratch[1] == op0)
4211 2135 : return 1;
4212 : return -1;
4213 : }
4214 :
4215 : /* Find the first summand that carries operand captures, and return its
4216 : (op0, op1) pair in *OP0_OUT / *OP1_OUT. Returns false if no summand
4217 : provides them. */
4218 :
4219 : static bool
4220 8136 : long_mul_canonical_ops (const vec<long_mul_summand> &summands,
4221 : tree *op0_out, tree *op1_out)
4222 : {
4223 24408 : for (const long_mul_summand &s : summands)
4224 8136 : if (s.op0)
4225 : {
4226 8136 : *op0_out = s.op0;
4227 8136 : *op1_out = s.op1;
4228 8136 : return true;
4229 : }
4230 : return false;
4231 : }
4232 :
4233 : /* Return the first summand in SUMMANDS whose kind matches KIND, or NULL. */
4234 :
4235 : static const long_mul_summand *
4236 3738 : long_mul_find_summand (const vec<long_mul_summand> &summands,
4237 : long_mul_kind kind)
4238 : {
4239 3738 : gcc_checking_assert (kind != LMK_INVALID);
4240 16827 : for (const long_mul_summand &s : summands)
4241 9351 : if (s.kind == kind)
4242 : return &s;
4243 : return NULL;
4244 : }
4245 :
4246 : /* Run the cross-summand validation invariants and return the canonical
4247 : (op0, op1). Returns false unless all summands that carry operands use
4248 : the same (op0, op1) pair (in either order), every LMX_HI/LMX_SHL_N shift
4249 : equals halfwidth, every captured hilo is a true cross-half product of
4250 : (op0, op1), and every cross-half pair (both those inside a single
4251 : mul_cross_sum-bearing summand and those spread across separate
4252 : LMK_MUL_HILO summands) contains one of each orientation. */
4253 :
4254 : static bool
4255 8124 : long_mul_check_consistency (const vec<long_mul_summand> &summands,
4256 : tree *op0_out, tree *op1_out)
4257 : {
4258 8124 : tree op0, op1;
4259 8124 : if (!long_mul_canonical_ops (summands, &op0, &op1))
4260 : return false;
4261 :
4262 8124 : tree op_type = TREE_TYPE (op0);
4263 8124 : if (!INTEGRAL_TYPE_P (op_type)
4264 8124 : || TYPE_PRECISION (op_type) % 2 != 0)
4265 : return false;
4266 8124 : unsigned int halfwidth = TYPE_PRECISION (op_type) / 2;
4267 :
4268 : /* Self-multiply (x*x) collapses the two cross-halves onto one value,
4269 : so the complementarity constraint is a trivial no-op there. */
4270 8124 : bool need_orient = op0 != op1;
4271 8124 : int mul_hilo_orient[2] = { 0, 0 };
4272 :
4273 37249 : for (const long_mul_summand &s : summands)
4274 : {
4275 18981 : if (s.op0 && !long_mul_same_ops (s.op0, s.op1, op0, op1))
4276 : return false;
4277 12903 : if ((s.extract == LMX_HI || s.extract == LMX_SHL_N)
4278 4690 : && s.shift != halfwidth)
4279 : return false;
4280 51558 : for (tree h : s.hilo)
4281 38681 : if (h && !long_mul_is_cross_half (h, op0, op1))
4282 : return false;
4283 :
4284 12877 : if (!need_orient)
4285 8 : continue;
4286 :
4287 : /* The two cross-sum operands are the last two non-null hilos:
4288 : (hilo[1], hilo[2]) for the CARRY_*_SUM kinds, (hilo[0], hilo[1]) for
4289 : the CROSS_SUM / SUM / ACCUM / LADDER_SUM kinds, none for the
4290 : rest. */
4291 12869 : tree a = NULL_TREE;
4292 12869 : tree b = NULL_TREE;
4293 12869 : if (s.hilo[2])
4294 : {
4295 77 : a = s.hilo[1];
4296 77 : b = s.hilo[2];
4297 : }
4298 12792 : else if (s.hilo[1])
4299 : {
4300 2034 : a = s.hilo[0];
4301 2034 : b = s.hilo[1];
4302 : }
4303 12869 : if (a && b
4304 14980 : && (long_mul_hilo_orientation (a, op0, op1)
4305 2111 : == long_mul_hilo_orientation (b, op0, op1)))
4306 : return false;
4307 :
4308 : /* Two LMK_MUL_HILO summands (the two-hilos ladder form) stand for
4309 : the two cross-halves separately; count orientations and require
4310 : the pair to be complementary. s.op0/op1 is already validated to
4311 : match (op0, op1) in some order above. */
4312 12869 : if (s.kind == LMK_MUL_HILO && s.op0)
4313 2532 : mul_hilo_orient[s.op0 == op1]++;
4314 : }
4315 :
4316 2020 : if (mul_hilo_orient[0] + mul_hilo_orient[1] >= 2
4317 10 : && (mul_hilo_orient[0] == 0 || mul_hilo_orient[1] == 0))
4318 : return false;
4319 :
4320 2020 : *op0_out = op0;
4321 2020 : *op1_out = op1;
4322 2020 : return true;
4323 : }
4324 :
4325 : /* Compare the (already-sorted) SUMMANDS multiset against ROW.sig. */
4326 :
4327 : static bool
4328 124705 : long_mul_signature_matches (const vec<long_mul_summand> &summands,
4329 : const long_mul_row &row)
4330 : {
4331 249410 : if (row.count != summands.length ())
4332 : return false;
4333 75726 : for (unsigned i = 0; i < row.count; i++)
4334 67602 : if (summands[i].kind != row.sig[i].kind
4335 67602 : || summands[i].extract != row.sig[i].extract)
4336 : return false;
4337 : return true;
4338 : }
4339 :
4340 : /* Extra check for the two-carries high-part row: the LMK_CARRY_LOW summand's
4341 : two operands (carry_a, carry_b) must be a (cross_shifted, mul_lolo) pair
4342 : consistent with the multiset's canonical (op0, op1). */
4343 :
4344 : static bool
4345 12 : long_mul_check_two_carries (const vec<long_mul_summand> &summands,
4346 : gimple *)
4347 : {
4348 12 : tree op0, op1;
4349 12 : if (!long_mul_canonical_ops (summands, &op0, &op1))
4350 : return false;
4351 12 : unsigned int halfwidth = TYPE_PRECISION (TREE_TYPE (op0)) / 2;
4352 :
4353 12 : const long_mul_summand *cl = long_mul_find_summand (summands, LMK_CARRY_LOW);
4354 12 : if (!cl)
4355 : return false;
4356 :
4357 : /* The two carry_low operands must be (cross_shifted, mul_lolo) in either
4358 : order. cross_shifted = LSHIFT_EXPR (mul_cross_sum, halfwidth). */
4359 12 : tree cs = cl->carry_a, lolo = cl->carry_b;
4360 12 : tree inner;
4361 12 : unsigned HOST_WIDE_INT shift;
4362 12 : if (!long_mul_is_lshift_def (cs, &inner, &shift))
4363 : {
4364 2 : std::swap (cs, lolo);
4365 2 : if (!long_mul_is_lshift_def (cs, &inner, &shift))
4366 : return false;
4367 : }
4368 12 : if (shift != halfwidth)
4369 : return false;
4370 :
4371 12 : tree scratch[LONG_MUL_MAX_CAPTURES];
4372 12 : if (!gimple_mul_cross_sum (inner, scratch, NULL))
4373 : return false;
4374 36 : for (int i = 0; i < 2; i++)
4375 24 : if (!long_mul_is_cross_half (scratch[i], op0, op1))
4376 : return false;
4377 12 : if (!gimple_mul_lolo (lolo, scratch, NULL)
4378 12 : || !long_mul_same_ops (scratch[0], scratch[1], op0, op1))
4379 : return false;
4380 :
4381 : return true;
4382 : }
4383 :
4384 : /* Extra check for xh*yh + (hilo >> N) + (ladder_sum1 >> N). hilo must be
4385 : the product that ladder_sum1 masks, since the high half of the unmasked
4386 : one is already in the sum. */
4387 :
4388 : static bool
4389 1769 : long_mul_check_ladder_sum1 (const vec<long_mul_summand> &summands,
4390 : gimple *)
4391 : {
4392 1769 : const long_mul_summand *hilo
4393 1769 : = long_mul_find_summand (summands, LMK_MUL_HILO);
4394 1769 : const long_mul_summand *sum
4395 1769 : = long_mul_find_summand (summands, LMK_LADDER_SUM1);
4396 1769 : return long_mul_hilo_orientation (sum->hilo[0], hilo->op0, hilo->op1) == 0;
4397 : }
4398 :
4399 : /* Likewise for xh*yh + (ladder_sum2 >> N) + (ladder_part_sum >> N):
4400 : ladder_part_sum must hold the product that ladder_sum2 masks. */
4401 :
4402 : static bool
4403 94 : long_mul_check_ladder_sum2 (const vec<long_mul_summand> &summands,
4404 : gimple *)
4405 : {
4406 94 : const long_mul_summand *part
4407 94 : = long_mul_find_summand (summands, LMK_LADDER_PART_SUM);
4408 94 : const long_mul_summand *sum
4409 94 : = long_mul_find_summand (summands, LMK_LADDER_SUM2);
4410 94 : tree masked[LONG_MUL_MAX_CAPTURES];
4411 94 : return (gimple_mul_hilo (sum->hilo[0], masked, NULL)
4412 94 : && (long_mul_hilo_orientation (part->hilo[0], masked[0], masked[1])
4413 94 : == 0));
4414 : }
4415 :
4416 : /* The lolo + cross_shifted shape is also the low half of a two-carry
4417 : long-multiply, where an unsigned overflow compare against one of
4418 : the PLUS operands is the low-carry term consumed by the matching
4419 : high-part fold. Folding to mul_lo here destroys cross_shifted,
4420 : which both the compare and the high-part match still need; defer
4421 : so the high-part fold runs first. After it does, the compare is
4422 : dead and the surviving lolo + cross_shifted is picked up by this
4423 : row in the next forwprop instance. Returns false to defer. */
4424 :
4425 : static bool
4426 51 : long_mul_check_low_plus_defer (const vec<long_mul_summand> &, gimple *stmt)
4427 : {
4428 : /* The PHI entry passes its gphi as the candidate but commits only to
4429 : HIGH_PART rows, so a LOW_PART row never folds from there. Guard the
4430 : gimple_assign accessors regardless, so this stays correct if a future
4431 : PLUS-shaped row reachable from the PHI path uses it. */
4432 51 : if (!is_gimple_assign (stmt))
4433 : return false;
4434 :
4435 51 : tree lhs = gimple_assign_lhs (stmt);
4436 51 : tree rhs1 = gimple_assign_rhs1 (stmt);
4437 51 : tree rhs2 = gimple_assign_rhs2 (stmt);
4438 :
4439 51 : imm_use_iterator iter;
4440 51 : gimple *use_stmt;
4441 79 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
4442 : {
4443 64 : tree cmp_op1 = NULL_TREE, cmp_op2 = NULL_TREE;
4444 64 : enum tree_code use_code = ERROR_MARK;
4445 64 : if (is_gimple_assign (use_stmt))
4446 : {
4447 42 : use_code = gimple_assign_rhs_code (use_stmt);
4448 42 : cmp_op1 = gimple_assign_rhs1 (use_stmt);
4449 42 : cmp_op2 = gimple_assign_rhs2 (use_stmt);
4450 : }
4451 22 : else if (gcond *cond = dyn_cast<gcond *> (use_stmt))
4452 : {
4453 9 : use_code = gimple_cond_code (cond);
4454 9 : cmp_op1 = gimple_cond_lhs (cond);
4455 9 : cmp_op2 = gimple_cond_rhs (cond);
4456 : }
4457 64 : if (use_code == GT_EXPR || use_code == LT_EXPR
4458 28 : || use_code == GE_EXPR || use_code == LE_EXPR)
4459 : {
4460 36 : tree other = (cmp_op1 == lhs) ? cmp_op2
4461 23 : : (cmp_op2 == lhs) ? cmp_op1 : NULL_TREE;
4462 36 : if (other && (other == rhs1 || other == rhs2))
4463 36 : return false;
4464 : }
4465 36 : }
4466 15 : return true;
4467 : }
4468 :
4469 : /* Long-multiply variant table. Each row enumerates the multiset of
4470 : (kind, extract) summands that compose one long-multiply form. Rows
4471 : are sorted by long_mul_summand_compare, matching the input summands'
4472 : sort order, so a plain element-wise compare suffices. Rows describe
4473 : unsigned schoolbook expansions on an even-width 2N-bit type split at
4474 : half-width N; EXTRA_CHECK carries invariants the (kind, extract)
4475 : signature cannot express.
4476 :
4477 : The formula on each row uses xh, xl, yh, yl for the half-width pieces
4478 : of x and y, cross_sum for xh*yl + xl*yh, and hilo for either cross-half
4479 : product (consumers validate the operand shape). */
4480 :
4481 : static const long_mul_row long_mul_table[] = {
4482 : /* HIGH-PART folds. */
4483 : /* xh*yh + (low_sum >> N) + ((hilo > low_sum) << N),
4484 : low_sum = cross_sum + (xl*yl >> N). */
4485 : { long_mul_row::HIGH_PART, PLUS_EXPR, 3,
4486 : { { LMK_MUL_HIHI, LMX_NONE },
4487 : { LMK_LOW_SUM, LMX_HI },
4488 : { LMK_CARRY_LOW_SUM, LMX_NONE } },
4489 : NULL },
4490 : /* xh*yh + (low_accum >> N) + (cross_sum >> N) + ((hilo > cross_sum) << N),
4491 : low_accum = (xl*yl >> N) + (cross_sum & mask). */
4492 : { long_mul_row::HIGH_PART, PLUS_EXPR, 4,
4493 : { { LMK_MUL_HIHI, LMX_NONE },
4494 : { LMK_CROSS_SUM, LMX_HI },
4495 : { LMK_LOW_ACCUM, LMX_HI },
4496 : { LMK_CARRY_CROSS_SUM, LMX_NONE } },
4497 : NULL },
4498 : /* xh*yh + (cross_sum >> N) + carry_low + ((hilo > cross_sum) << N),
4499 : carry_low = (xl*yl + (cross_sum << N)) < (cross_sum << N). */
4500 : { long_mul_row::HIGH_PART, PLUS_EXPR, 4,
4501 : { { LMK_MUL_HIHI, LMX_NONE },
4502 : { LMK_CROSS_SUM, LMX_HI },
4503 : { LMK_CARRY_LOW, LMX_NONE },
4504 : { LMK_CARRY_CROSS_SUM, LMX_NONE } },
4505 : long_mul_check_two_carries },
4506 : /* xh*yh + (hilo >> N) + (ladder_sum1 >> N),
4507 : ladder_sum1 = (hilo & mask) + hilo' + (xl*yl >> N),
4508 : hilo, hilo' the two cross-half products. */
4509 : { long_mul_row::HIGH_PART, PLUS_EXPR, 3,
4510 : { { LMK_MUL_HIHI, LMX_NONE },
4511 : { LMK_MUL_HILO, LMX_HI },
4512 : { LMK_LADDER_SUM1, LMX_HI } },
4513 : long_mul_check_ladder_sum1 },
4514 : /* xh*yh + (ladder_sum2 >> N) + (ladder_part_sum >> N),
4515 : ladder_part_sum = (xl*yl >> N) + hilo,
4516 : ladder_sum2 = (ladder_part_sum & mask) + hilo'. */
4517 : { long_mul_row::HIGH_PART, PLUS_EXPR, 3,
4518 : { { LMK_MUL_HIHI, LMX_NONE },
4519 : { LMK_LADDER_SUM2, LMX_HI },
4520 : { LMK_LADDER_PART_SUM, LMX_HI } },
4521 : long_mul_check_ladder_sum2 },
4522 : /* xh*yh + (hilo >> N) + (hilo' >> N) + (ladder_sum3 >> N),
4523 : ladder_sum3 = (hilo & mask) + (hilo' & mask) + (xl*yl >> N). */
4524 : { long_mul_row::HIGH_PART, PLUS_EXPR, 4,
4525 : { { LMK_MUL_HIHI, LMX_NONE },
4526 : { LMK_MUL_HILO, LMX_HI },
4527 : { LMK_MUL_HILO, LMX_HI },
4528 : { LMK_LADDER_SUM3, LMX_HI } },
4529 : NULL },
4530 : /* LOW-PART folds. Recover the lower 2N bits from xl*yl plus a
4531 : shifted cross-half term. */
4532 : /* xl*yl + (cross_sum << N). */
4533 : { long_mul_row::LOW_PART, PLUS_EXPR, 2,
4534 : { { LMK_MUL_LOLO, LMX_NONE },
4535 : { LMK_CROSS_SUM, LMX_SHL_N } },
4536 : long_mul_check_low_plus_defer },
4537 : /* (xl*yl & mask) | (low_accum << N),
4538 : low_accum = (xl*yl >> N) + (cross_sum & mask). */
4539 : { long_mul_row::LOW_PART, BIT_IOR_EXPR, 2,
4540 : { { LMK_MUL_LOLO, LMX_LO },
4541 : { LMK_LOW_ACCUM, LMX_SHL_N } },
4542 : NULL },
4543 : /* (xl*yl & mask) | (low_sum << N),
4544 : low_sum = cross_sum + (xl*yl >> N). */
4545 : { long_mul_row::LOW_PART, BIT_IOR_EXPR, 2,
4546 : { { LMK_MUL_LOLO, LMX_LO },
4547 : { LMK_LOW_SUM, LMX_SHL_N } },
4548 : NULL },
4549 : /* (xl*yl & mask) | (ladder_sum1 << N),
4550 : ladder_sum1 as in the high ladder row above. */
4551 : { long_mul_row::LOW_PART, BIT_IOR_EXPR, 2,
4552 : { { LMK_MUL_LOLO, LMX_LO },
4553 : { LMK_LADDER_SUM1, LMX_SHL_N } },
4554 : NULL },
4555 : /* (xl*yl & mask) | (ladder_sum2 << N),
4556 : ladder_sum2 as in the high ladder row above. */
4557 : { long_mul_row::LOW_PART, BIT_IOR_EXPR, 2,
4558 : { { LMK_MUL_LOLO, LMX_LO },
4559 : { LMK_LADDER_SUM2, LMX_SHL_N } },
4560 : NULL },
4561 : /* (xl*yl & mask) | (ladder_sum3 << N),
4562 : ladder_sum3 as in the high ladder-long row above. */
4563 : { long_mul_row::LOW_PART, BIT_IOR_EXPR, 2,
4564 : { { LMK_MUL_LOLO, LMX_LO },
4565 : { LMK_LADDER_SUM3, LMX_SHL_N } },
4566 : NULL },
4567 : };
4568 :
4569 : /* If a multi-used inner addition (sharing the chain's outer code) blocked
4570 : linearization of a long-mul candidate, emit a dump-file hint pointing
4571 : at it. */
4572 :
4573 : static void
4574 21370 : long_mul_hint_shared_intermediate (gimple *shared_def)
4575 : {
4576 21370 : if (!shared_def || !dump_file || !(dump_flags & TDF_DETAILS))
4577 : return;
4578 0 : fprintf (dump_file, "long-mul fold rejected: shared intermediate at ");
4579 0 : print_gimple_stmt (dump_file, shared_def, 0, TDF_SLIM);
4580 : }
4581 :
4582 : /* Search long_mul_table for a row whose multiset matches SUMMANDS for
4583 : outer kind OUTER on a result of type LHS_TYPE. CANDIDATE_STMT is
4584 : passed to per-row extra_check predicates. On a hit, returns the
4585 : matching row and writes the half-width operands via OUT_OP0/OUT_OP1.
4586 : No IR mutation. */
4587 :
4588 : static const long_mul_row *
4589 18763 : long_mul_classify_match (const vec<long_mul_summand> &summands,
4590 : tree lhs_type, tree_code outer,
4591 : gimple *candidate_stmt,
4592 : tree *out_op0, tree *out_op1)
4593 : {
4594 : /* HIGH_PART rows emit a 2N-bit multiply that pass_optimize_widening_mul
4595 : consumes -- either via WIDEN_MULT_EXPR / MULT_HIGHPART conversion when
4596 : the target has a native 2N multiply, or via lower_long_mul_high_chain
4597 : when it does not. LOW_PART rows emit a plain MULT_EXPR. Emission
4598 : needs a 2N mode to exist in the mode table AND the widening_mul pass
4599 : to be active: without the pass, the emit could reach RTL expand as an
4600 : unexpandable 2N multiply (e.g. OImode). BITINT_TYPE is
4601 : refused -- the long_mul_high_chain atom excludes it. */
4602 18763 : scalar_int_mode mode, wide_mode;
4603 18763 : bool can_emit_high
4604 18763 : = optimize_widening_mul_active_p ()
4605 18672 : && TREE_CODE (lhs_type) != BITINT_TYPE
4606 37344 : && is_a <scalar_int_mode> (TYPE_MODE (lhs_type), &mode)
4607 37435 : && GET_MODE_2XWIDER_MODE (mode).exists (&wide_mode);
4608 :
4609 226350 : for (const long_mul_row &row : long_mul_table)
4610 : {
4611 410970 : if (row.outer != outer
4612 125251 : || (row.part == long_mul_row::HIGH_PART && !can_emit_high)
4613 334252 : || !long_mul_signature_matches (summands, row))
4614 207587 : continue;
4615 :
4616 8124 : tree op0, op1;
4617 8124 : if (!long_mul_check_consistency (summands, &op0, &op1))
4618 6104 : continue;
4619 :
4620 : /* Do not emit the wide chain when an operand is subject to
4621 : abnormal coalescing: the widening_mul-side consumers refuse
4622 : such operands (see convert_mult_to_widen), which would leave
4623 : the chain without a consumer. */
4624 2020 : if (row.part == long_mul_row::HIGH_PART
4625 2020 : && ((TREE_CODE (op0) == SSA_NAME
4626 1950 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op0))
4627 1950 : || (TREE_CODE (op1) == SSA_NAME
4628 1950 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op1))))
4629 0 : continue;
4630 :
4631 2020 : if (row.extra_check && !row.extra_check (summands, candidate_stmt))
4632 60 : continue;
4633 :
4634 1960 : *out_op0 = op0;
4635 1960 : *out_op1 = op1;
4636 1960 : return &row;
4637 : }
4638 : return NULL;
4639 : }
4640 :
4641 : /* Walk STMT's outer chain (kind OUTER), classify each leaf as a
4642 : long-multiply summand, optionally add the already-classified EXTRA,
4643 : and look the multiset up in long_mul_table for a result of type
4644 : LHS_TYPE. CANDIDATE is passed to per-row extra_check predicates.
4645 :
4646 : If EXTRAS_OUT is non-NULL, leaves matching no summand are set aside
4647 : there instead of failing the match, and the caller must re-apply
4648 : them on top of the folded multiply. A leaf that does match is
4649 : always consumed: if that makes the signature miss every row the
4650 : match fails, rather than retrying with the leaf demoted to an extra
4651 : (subset search would be exponential).
4652 :
4653 : Returns the matched row and the half-width operands via
4654 : OUT_OP0/OUT_OP1, or NULL on a miss. No IR mutation. */
4655 :
4656 : static const long_mul_row *
4657 3405795 : long_mul_classify_chain (gimple *stmt, tree_code outer, tree lhs_type,
4658 : gimple *candidate, const long_mul_summand *extra,
4659 : vec<tree> *extras_out,
4660 : tree *out_op0, tree *out_op1)
4661 : {
4662 3405795 : auto_vec<tree, LONG_MUL_MAX_SUMMANDS + LONG_MUL_MAX_EXTRAS> leaves;
4663 3405795 : gimple *shared_def = NULL;
4664 3405795 : if (!long_mul_linearize_chain (stmt, outer, leaves, &shared_def))
4665 : return NULL;
4666 :
4667 3405000 : auto_vec<long_mul_summand,
4668 3405000 : LONG_MUL_MAX_SUMMANDS + LONG_MUL_MAX_EXTRAS + 1> summands;
4669 17182072 : for (tree leaf : leaves)
4670 : {
4671 6971639 : if (long_mul_summand s{leaf})
4672 : {
4673 55611 : gcc_checking_assert (s.kind != LMK_INVALID);
4674 55611 : summands.quick_push (s);
4675 : }
4676 13831780 : else if (extras_out && extras_out->length () < LONG_MUL_MAX_EXTRAS)
4677 6911461 : extras_out->safe_push (leaf);
4678 : else
4679 : {
4680 4567 : long_mul_hint_shared_intermediate (shared_def);
4681 4567 : return NULL;
4682 : }
4683 : }
4684 3400433 : if (extra)
4685 : {
4686 354 : gcc_checking_assert (extra->kind != LMK_INVALID);
4687 354 : summands.quick_push (*extra);
4688 : }
4689 6805433 : if (summands.length () < 2
4690 3400433 : || summands.length () > LONG_MUL_MAX_SUMMANDS)
4691 : return NULL;
4692 18763 : summands.qsort (long_mul_summand_compare);
4693 :
4694 18763 : const long_mul_row *row
4695 18763 : = long_mul_classify_match (summands, lhs_type, outer, candidate,
4696 : out_op0, out_op1);
4697 18763 : if (!row)
4698 16803 : long_mul_hint_shared_intermediate (shared_def);
4699 : return row;
4700 3405000 : }
4701 :
4702 : /* Top-level entry for long-multiply folding. Walks STMT's outer
4703 : addition or BIT_IOR chain, classifies the summands, and dispatches
4704 : to create_mul_high_seq / create_mul_low_seq if the multiset matches
4705 : a known long-multiply form. Returns true on success. */
4706 :
4707 : static bool
4708 10235596 : match_long_mul (gassign *stmt)
4709 : {
4710 10235596 : tree_code outer = gimple_assign_rhs_code (stmt);
4711 10235596 : if (outer != PLUS_EXPR && outer != BIT_IOR_EXPR)
4712 : return false;
4713 :
4714 : /* Skip non-candidate adds (signed, pointer, odd-width) before walking the
4715 : chain. No legitimate long-mul leaf has a type the atoms would reject.
4716 : This just avoids the linearize/classify work on every other PLUS/IOR. */
4717 10235596 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
4718 10235596 : if (!INTEGRAL_TYPE_P (lhs_type)
4719 8944543 : || !TYPE_UNSIGNED (lhs_type)
4720 14044510 : || TYPE_PRECISION (lhs_type) % 2 != 0)
4721 : return false;
4722 :
4723 : /* Only start at the end of a chain: a consumer with the same code
4724 : linearizes through this statement anyway, so starting here is
4725 : redundant. A consumer in another block does not count -- folding
4726 : at the later use could sink a loop-invariant multiply into a
4727 : loop. */
4728 3572924 : use_operand_p use_p;
4729 3572924 : gimple *use_stmt;
4730 3572924 : if (single_imm_use (gimple_assign_lhs (stmt), &use_p, &use_stmt)
4731 2525115 : && is_gimple_assign (use_stmt)
4732 1383742 : && gimple_assign_rhs_code (use_stmt) == outer
4733 3747553 : && gimple_bb (use_stmt) == gimple_bb (stmt))
4734 : return false;
4735 :
4736 3405165 : auto_vec<tree, LONG_MUL_MAX_EXTRAS> extras;
4737 3405165 : tree op0, op1;
4738 3405165 : const long_mul_row *row
4739 3405165 : = long_mul_classify_chain (stmt, outer, lhs_type, stmt, NULL, &extras,
4740 : &op0, &op1);
4741 3405165 : if (!row)
4742 : return false;
4743 :
4744 1946 : if (row->part == long_mul_row::HIGH_PART)
4745 : {
4746 1912 : create_mul_high_seq (op0, op1, stmt, extras, outer);
4747 1912 : if (dump_file && (dump_flags & TDF_DETAILS))
4748 35 : fprintf (dump_file, "Long multiplication high part folded.\n");
4749 : return true;
4750 : }
4751 34 : create_mul_low_seq (op0, op1, stmt, extras, outer);
4752 34 : if (dump_file && (dump_flags & TDF_DETAILS))
4753 10 : fprintf (dump_file, "Long multiplication low part folded.\n");
4754 : return true;
4755 3405165 : }
4756 :
4757 : /* PHI-driven entry for long-multiply folding. When PHI's value
4758 : flattens to base + (carry << N), probe sum to classify the carry
4759 : kind, linearize base for the remaining high-part summands, and run
4760 : the long-multiply table. On a hit, emit a 2N-bit multiply at the
4761 : top of the join block with PHI_RES as its LHS and remove the PHI.
4762 : Otherwise leave the IR untouched. Only HIGH_PART rows are
4763 : reachable. LOW_PART rows are BIT_IOR-shaped and never produce a
4764 : carry PHI. */
4765 :
4766 : static bool
4767 8856254 : match_long_mul_phi (gphi *phi)
4768 : {
4769 8856254 : tree phi_res = gimple_phi_result (phi);
4770 8856254 : tree lhs_type = TREE_TYPE (phi_res);
4771 6266098 : if (!INTEGRAL_TYPE_P (lhs_type) || !TYPE_UNSIGNED (lhs_type)
4772 11633169 : || TYPE_PRECISION (lhs_type) % 2 != 0)
4773 : return false;
4774 :
4775 2046507 : tree cca_ops[4];
4776 2046507 : if (!gimple_cond_carry_add (phi_res, cca_ops, NULL))
4777 : return false;
4778 19233 : tree cmp_lhs = cca_ops[0];
4779 19233 : tree sum = cca_ops[1];
4780 19233 : tree base = cca_ops[2];
4781 :
4782 : /* Classify sum and populate the carry summand directly. Most
4783 : specific first, mirroring long_mul_classify_carry's order. */
4784 19233 : long_mul_summand carry{};
4785 19233 : tree sum_ops[LONG_MUL_MAX_CAPTURES];
4786 19233 : unsigned HOST_WIDE_INT shift_amt
4787 19233 : = wi::exact_log2 (wi::to_wide (cca_ops[3]));
4788 19233 : unsigned HOST_WIDE_INT halfwidth = TYPE_PRECISION (lhs_type) / 2;
4789 19233 : carry.shift = shift_amt;
4790 :
4791 19233 : if (shift_amt == halfwidth
4792 19233 : && gimple_mul_low_sum (sum, sum_ops, NULL))
4793 : {
4794 : /* mul_carry_low_sum's flat form ties the outer lshift amount to
4795 : the inner mul_hi's INTEGER_CST@0 via match.pd capture re-use;
4796 : the PHI form has no such tie, so gate on shift_amt explicitly. */
4797 14 : carry.kind = LMK_CARRY_LOW_SUM;
4798 14 : carry.op0 = sum_ops[0];
4799 14 : carry.op1 = sum_ops[1];
4800 14 : carry.hilo[0] = cmp_lhs;
4801 14 : carry.hilo[1] = sum_ops[2];
4802 14 : carry.hilo[2] = sum_ops[3];
4803 : }
4804 19219 : else if (shift_amt == halfwidth
4805 19219 : && gimple_mul_cross_sum (sum, sum_ops, NULL))
4806 : {
4807 : /* mul_cross_sum is just (plus:c @0 @1) with no half-width
4808 : constraint. Gate here to mirror mul_carry_cross_sum;
4809 : a mismatch falls through to the LMK_CARRY_LOW branch. */
4810 1 : carry.kind = LMK_CARRY_CROSS_SUM;
4811 1 : carry.hilo[0] = cmp_lhs;
4812 1 : carry.hilo[1] = sum_ops[0];
4813 1 : carry.hilo[2] = sum_ops[1];
4814 : }
4815 19218 : else if (shift_amt == 0 && TREE_CODE (sum) == SSA_NAME)
4816 : {
4817 18070 : gimple *def = SSA_NAME_DEF_STMT (sum);
4818 18070 : if (!is_gimple_assign (def)
4819 18070 : || gimple_assign_rhs_code (def) != PLUS_EXPR)
4820 : return false;
4821 4744 : tree p1 = gimple_assign_rhs1 (def);
4822 4744 : tree p2 = gimple_assign_rhs2 (def);
4823 4744 : if (p1 != cmp_lhs && p2 != cmp_lhs)
4824 : return false;
4825 4009 : carry.kind = LMK_CARRY_LOW;
4826 4009 : carry.carry_a = cmp_lhs;
4827 4009 : carry.carry_b = p1 == cmp_lhs ? p2 : p1;
4828 4009 : }
4829 : else
4830 : return false;
4831 :
4832 : /* Linearize base, the rest of the high-part chain. */
4833 4024 : if (TREE_CODE (base) != SSA_NAME)
4834 : return false;
4835 4024 : gimple *base_def = SSA_NAME_DEF_STMT (base);
4836 4024 : if (!is_gimple_assign (base_def)
4837 4024 : || gimple_assign_rhs_code (base_def) != PLUS_EXPR)
4838 : return false;
4839 :
4840 630 : tree op0, op1;
4841 630 : const long_mul_row *row
4842 630 : = long_mul_classify_chain (base_def, PLUS_EXPR, lhs_type, phi, &carry,
4843 : NULL, &op0, &op1);
4844 630 : if (!row || row->part != long_mul_row::HIGH_PART)
4845 : return false;
4846 :
4847 14 : gimple_seq seq = NULL;
4848 14 : build_mul_high_seq (op0, op1, phi_res, gimple_location (phi), &seq);
4849 14 : gimple_stmt_iterator gsi = gsi_after_labels (gimple_bb (phi));
4850 14 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
4851 14 : gimple_stmt_iterator psi = gsi_for_stmt (phi);
4852 14 : remove_phi_node (&psi, false);
4853 14 : if (dump_file && (dump_flags & TDF_DETAILS))
4854 4 : fprintf (dump_file,
4855 : "Long multiplication high part folded (carry PHI).\n");
4856 : return true;
4857 : }
4858 :
4859 : /* Verify if we have the following structure:
4860 :
4861 : iftmp1 = PHI <pow2a, pow2b, pow2c, ...>
4862 : _ssa1 = _ssa2 MOD|DIV iftmp1;
4863 : _ssa3 = _ssa1 EQ|NE 0;
4864 :
4865 : And, if the right conditions are met, change the PHI args
4866 : and "_ssa1" stmt to a cheaper alternative.
4867 :
4868 : - for MOD, if either "_ssa2" is known to be positive or
4869 : "_ssa1" is used just in zero comparisons:
4870 :
4871 : iftmp1 = PHI <(pow2a - 1), (pow2b - 1), (pow2c - 1), ...>
4872 : _ssa1 = _ssa2 & iftmp1;
4873 :
4874 : - for DIV, if "_ssa2" is known to be positive:
4875 :
4876 : iftmp1 = PHI <log2 (pow2a), log2 (pow2b), log2 (pow2c), ...>
4877 : _ssa1 = _ssa2 >> iftmp1; */
4878 : static bool
4879 134686 : simplify_phi_result_movdiv (gimple *stmt, tree_code code)
4880 : {
4881 134686 : tree rhs1 = gimple_assign_rhs1 (stmt);
4882 134686 : tree_code new_code;
4883 134686 : bool needs_reset = false;
4884 :
4885 : /* Skip complex types (PR127163) */
4886 134686 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1)))
4887 : return false;
4888 :
4889 68081 : switch (code)
4890 : {
4891 228 : case TRUNC_MOD_EXPR:
4892 228 : case FLOOR_MOD_EXPR:
4893 228 : if (!tree_expr_nonnegative_p (rhs1))
4894 : {
4895 : /* For a negative X, the truncating X % 2^k and X & (2^k - 1)
4896 : differ (e.g. -3 % 4 is -3 but -3 & 3 is 1), though both are
4897 : zero for the same X. That is enough when the result is only
4898 : compared with zero, but the range recorded for the MOD result
4899 : no longer holds for the AND, so reset it. */
4900 143 : needs_reset = true;
4901 143 : if (!use_in_zero_equality (gimple_assign_lhs (stmt), true))
4902 : return false;
4903 : }
4904 :
4905 : new_code = BIT_AND_EXPR;
4906 : break;
4907 :
4908 723 : case TRUNC_DIV_EXPR:
4909 723 : case FLOOR_DIV_EXPR:
4910 723 : case EXACT_DIV_EXPR:
4911 723 : if (!tree_expr_nonnegative_p (rhs1))
4912 : return false;
4913 :
4914 : new_code = RSHIFT_EXPR;
4915 : break;
4916 :
4917 : default:
4918 : return false;
4919 : }
4920 :
4921 572 : gphi *phi = as_a<gphi *> (SSA_NAME_DEF_STMT (gimple_assign_rhs2 (stmt)));
4922 :
4923 1199 : for (unsigned int i = 0; i < gimple_phi_num_args (phi); i++)
4924 614 : if (!integer_pow2p (gimple_phi_arg_def (phi, i))
4925 614 : || tree_int_cst_sgn (gimple_phi_arg_def (phi, i)) < 0)
4926 : return false;
4927 :
4928 13 : tree type = TREE_TYPE (gimple_phi_result (phi));
4929 13 : tree new_phires = make_ssa_name (type);
4930 13 : gphi *new_phi = create_phi_node (new_phires, phi->bb);
4931 :
4932 54 : for (unsigned int i = 0; i < gimple_phi_num_args (phi); i++)
4933 : {
4934 28 : tree phi_arg = gimple_phi_arg_def (phi, i);
4935 28 : edge e = gimple_phi_arg_edge (phi, i);
4936 28 : tree arg;
4937 :
4938 28 : if (new_code == RSHIFT_EXPR)
4939 4 : arg = wide_int_to_tree (type, wi::exact_log2 (wi::to_wide (phi_arg)));
4940 : else
4941 24 : arg = wide_int_to_tree (type, wi::to_wide (phi_arg) - 1);
4942 :
4943 28 : add_phi_arg (new_phi, arg, e,
4944 28 : gimple_phi_arg_location (phi, e->dest_idx));
4945 : }
4946 :
4947 : /* Add a gimple_convert to integer_type_node for new_phires
4948 : since it might be a long long which we want to convert
4949 : into an integer or a bit_int that we want to convert into
4950 : an integer. */
4951 13 : gimple_stmt_iterator gsi;
4952 13 : if (new_code == RSHIFT_EXPR)
4953 : {
4954 2 : gsi = gsi_for_stmt (stmt);
4955 2 : new_phires = gimple_convert (&gsi, true, GSI_SAME_STMT,
4956 : gimple_location (stmt),
4957 : integer_type_node, new_phires);
4958 : }
4959 :
4960 13 : gimple_assign_set_rhs2 (stmt, new_phires);
4961 13 : gimple_assign_set_rhs_code (stmt, new_code);
4962 13 : if (needs_reset)
4963 8 : reset_flow_sensitive_info (gimple_assign_lhs (stmt));
4964 13 : update_stmt (stmt);
4965 :
4966 13 : gsi = gsi_for_phi (phi);
4967 13 : remove_phi_node (&gsi, true);
4968 :
4969 13 : return true;
4970 : }
4971 :
4972 : /* Determine whether applying the 2 permutations (mask1 then mask2)
4973 : gives back one of the input. */
4974 :
4975 : static int
4976 46 : is_combined_permutation_identity (tree mask1, tree mask2)
4977 : {
4978 46 : tree mask;
4979 46 : unsigned HOST_WIDE_INT nelts, i, j;
4980 46 : bool maybe_identity1 = true;
4981 46 : bool maybe_identity2 = true;
4982 :
4983 46 : gcc_checking_assert (TREE_CODE (mask1) == VECTOR_CST
4984 : && TREE_CODE (mask2) == VECTOR_CST);
4985 :
4986 : /* For VLA masks, check for the following pattern:
4987 : v1 = VEC_PERM_EXPR (v0, ..., mask1)
4988 : v2 = VEC_PERM_EXPR (v1, ..., mask2)
4989 : -->
4990 : v2 = v0
4991 : if mask1 == mask2 == {nelts - 1, nelts - 2, ...}. */
4992 :
4993 46 : if (operand_equal_p (mask1, mask2, 0)
4994 46 : && !VECTOR_CST_NELTS (mask1).is_constant ())
4995 : {
4996 : vec_perm_builder builder;
4997 : if (tree_to_vec_perm_builder (&builder, mask1))
4998 : {
4999 : poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask1));
5000 : vec_perm_indices sel (builder, 1, nelts);
5001 : if (sel.series_p (0, 1, nelts - 1, -1))
5002 : return 1;
5003 : }
5004 : }
5005 :
5006 46 : mask = fold_ternary (VEC_PERM_EXPR, TREE_TYPE (mask1), mask1, mask1, mask2);
5007 46 : if (mask == NULL_TREE || TREE_CODE (mask) != VECTOR_CST)
5008 : return 0;
5009 :
5010 46 : if (!VECTOR_CST_NELTS (mask).is_constant (&nelts))
5011 : return 0;
5012 80 : for (i = 0; i < nelts; i++)
5013 : {
5014 80 : tree val = VECTOR_CST_ELT (mask, i);
5015 80 : gcc_assert (TREE_CODE (val) == INTEGER_CST);
5016 80 : j = TREE_INT_CST_LOW (val) & (2 * nelts - 1);
5017 80 : if (j == i)
5018 : maybe_identity2 = false;
5019 63 : else if (j == i + nelts)
5020 : maybe_identity1 = false;
5021 : else
5022 : return 0;
5023 : }
5024 0 : return maybe_identity1 ? 1 : maybe_identity2 ? 2 : 0;
5025 : }
5026 :
5027 : /* Combine a shuffle with its arguments. Returns true if there were any
5028 : changes made. */
5029 :
5030 : static bool
5031 193226 : simplify_permutation (gimple_stmt_iterator *gsi)
5032 : {
5033 193226 : gimple *stmt = gsi_stmt (*gsi);
5034 193226 : gimple *def_stmt = NULL;
5035 193226 : tree op0, op1, op2, op3, arg0, arg1;
5036 193226 : enum tree_code code, code2 = ERROR_MARK;
5037 193226 : bool single_use_op0 = false;
5038 :
5039 193226 : gcc_checking_assert (gimple_assign_rhs_code (stmt) == VEC_PERM_EXPR);
5040 :
5041 193226 : op0 = gimple_assign_rhs1 (stmt);
5042 193226 : op1 = gimple_assign_rhs2 (stmt);
5043 193226 : op2 = gimple_assign_rhs3 (stmt);
5044 :
5045 193226 : if (TREE_CODE (op2) != VECTOR_CST)
5046 : return false;
5047 :
5048 190434 : if (TREE_CODE (op0) == VECTOR_CST)
5049 : {
5050 : code = VECTOR_CST;
5051 : arg0 = op0;
5052 : }
5053 188265 : else if (TREE_CODE (op0) == SSA_NAME)
5054 : {
5055 188265 : def_stmt = get_prop_source_stmt (op0, false, &single_use_op0);
5056 188265 : if (!def_stmt)
5057 : return false;
5058 179756 : code = gimple_assign_rhs_code (def_stmt);
5059 179756 : if (code == VIEW_CONVERT_EXPR)
5060 : {
5061 1636 : tree rhs = gimple_assign_rhs1 (def_stmt);
5062 1636 : tree name = TREE_OPERAND (rhs, 0);
5063 1636 : if (TREE_CODE (name) != SSA_NAME)
5064 : return false;
5065 1636 : if (!has_single_use (name))
5066 254 : single_use_op0 = false;
5067 : /* Here we update the def_stmt through this VIEW_CONVERT_EXPR,
5068 : but still keep the code to indicate it comes from
5069 : VIEW_CONVERT_EXPR. */
5070 1636 : def_stmt = SSA_NAME_DEF_STMT (name);
5071 1636 : if (!def_stmt || !is_gimple_assign (def_stmt))
5072 : return false;
5073 849 : if (gimple_assign_rhs_code (def_stmt) != CONSTRUCTOR)
5074 : return false;
5075 : }
5076 178386 : if (!can_propagate_from (def_stmt))
5077 : return false;
5078 24663 : arg0 = gimple_assign_rhs1 (def_stmt);
5079 : }
5080 : else
5081 : return false;
5082 :
5083 : /* Two consecutive shuffles. */
5084 24663 : if (code == VEC_PERM_EXPR)
5085 : {
5086 6757 : tree orig;
5087 6757 : int ident;
5088 :
5089 6757 : if (op0 != op1)
5090 : return false;
5091 46 : op3 = gimple_assign_rhs3 (def_stmt);
5092 46 : if (TREE_CODE (op3) != VECTOR_CST)
5093 : return false;
5094 : /* is_combined_permutation_identity assumes that both permutes and
5095 : their inputs have the same number of elements. Mixed widths are
5096 : left to the match.pd VEC_PERM merge patterns. */
5097 46 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op2)),
5098 46 : TYPE_VECTOR_SUBPARTS (TREE_TYPE (op3)))
5099 92 : || maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op0)),
5100 : TYPE_VECTOR_SUBPARTS
5101 46 : (TREE_TYPE (gimple_assign_rhs1 (def_stmt)))))
5102 : return false;
5103 46 : ident = is_combined_permutation_identity (op3, op2);
5104 46 : if (!ident)
5105 : return false;
5106 0 : orig = (ident == 1) ? gimple_assign_rhs1 (def_stmt)
5107 0 : : gimple_assign_rhs2 (def_stmt);
5108 0 : gimple_assign_set_rhs1 (stmt, unshare_expr (orig));
5109 0 : gimple_assign_set_rhs_code (stmt, TREE_CODE (orig));
5110 0 : gimple_set_num_ops (stmt, 2);
5111 0 : update_stmt (stmt);
5112 0 : remove_prop_source_from_use (op0);
5113 0 : return true;
5114 : }
5115 20075 : else if (code == CONSTRUCTOR
5116 20075 : || code == VECTOR_CST
5117 : || code == VIEW_CONVERT_EXPR)
5118 : {
5119 5010 : if (op0 != op1)
5120 : {
5121 4813 : if (TREE_CODE (op0) == SSA_NAME && !single_use_op0)
5122 : return false;
5123 :
5124 4130 : if (TREE_CODE (op1) == VECTOR_CST)
5125 : arg1 = op1;
5126 3497 : else if (TREE_CODE (op1) == SSA_NAME)
5127 : {
5128 3497 : gimple *def_stmt2 = get_prop_source_stmt (op1, true, NULL);
5129 3497 : if (!def_stmt2)
5130 : return false;
5131 1698 : code2 = gimple_assign_rhs_code (def_stmt2);
5132 1698 : if (code2 == VIEW_CONVERT_EXPR)
5133 : {
5134 0 : tree rhs = gimple_assign_rhs1 (def_stmt2);
5135 0 : tree name = TREE_OPERAND (rhs, 0);
5136 0 : if (TREE_CODE (name) != SSA_NAME)
5137 : return false;
5138 0 : if (!has_single_use (name))
5139 : return false;
5140 0 : def_stmt2 = SSA_NAME_DEF_STMT (name);
5141 0 : if (!def_stmt2 || !is_gimple_assign (def_stmt2))
5142 : return false;
5143 0 : if (gimple_assign_rhs_code (def_stmt2) != CONSTRUCTOR)
5144 : return false;
5145 : }
5146 1698 : else if (code2 != CONSTRUCTOR && code2 != VECTOR_CST)
5147 : return false;
5148 1485 : if (!can_propagate_from (def_stmt2))
5149 : return false;
5150 1485 : arg1 = gimple_assign_rhs1 (def_stmt2);
5151 : }
5152 : else
5153 : return false;
5154 : }
5155 : else
5156 : {
5157 : /* Already used twice in this statement. */
5158 197 : if (TREE_CODE (op0) == SSA_NAME && num_imm_uses (op0) > 2)
5159 : return false;
5160 : arg1 = arg0;
5161 : }
5162 :
5163 : /* If there are any VIEW_CONVERT_EXPRs found when finding permutation
5164 : operands source, check whether it's valid to transform and prepare
5165 : the required new operands. */
5166 2247 : if (code == VIEW_CONVERT_EXPR || code2 == VIEW_CONVERT_EXPR)
5167 : {
5168 : /* Figure out the target vector type to which operands should be
5169 : converted. If both are CONSTRUCTOR, the types should be the
5170 : same, otherwise, use the one of CONSTRUCTOR. */
5171 24 : tree tgt_type = NULL_TREE;
5172 24 : if (code == VIEW_CONVERT_EXPR)
5173 : {
5174 24 : gcc_assert (gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR);
5175 24 : code = CONSTRUCTOR;
5176 24 : tgt_type = TREE_TYPE (arg0);
5177 : }
5178 24 : if (code2 == VIEW_CONVERT_EXPR)
5179 : {
5180 0 : tree arg1_type = TREE_TYPE (arg1);
5181 0 : if (tgt_type == NULL_TREE)
5182 : tgt_type = arg1_type;
5183 0 : else if (tgt_type != arg1_type)
5184 23 : return false;
5185 : }
5186 :
5187 24 : if (!VECTOR_TYPE_P (tgt_type))
5188 : return false;
5189 24 : tree op2_type = TREE_TYPE (op2);
5190 :
5191 : /* Figure out the shrunk factor. */
5192 24 : poly_uint64 tgt_units = TYPE_VECTOR_SUBPARTS (tgt_type);
5193 24 : poly_uint64 op2_units = TYPE_VECTOR_SUBPARTS (op2_type);
5194 24 : if (maybe_gt (tgt_units, op2_units))
5195 : return false;
5196 24 : unsigned int factor;
5197 47 : if (!constant_multiple_p (op2_units, tgt_units, &factor))
5198 : return false;
5199 :
5200 : /* Build the new permutation control vector as target vector. */
5201 24 : vec_perm_builder builder;
5202 24 : if (!tree_to_vec_perm_builder (&builder, op2))
5203 : return false;
5204 24 : vec_perm_indices indices (builder, 2, op2_units);
5205 24 : vec_perm_indices new_indices;
5206 24 : if (new_indices.new_shrunk_vector (indices, factor))
5207 : {
5208 1 : tree mask_type = tgt_type;
5209 1 : if (!VECTOR_INTEGER_TYPE_P (mask_type))
5210 : {
5211 0 : tree elem_type = TREE_TYPE (mask_type);
5212 0 : unsigned elem_size = TREE_INT_CST_LOW (TYPE_SIZE (elem_type));
5213 0 : tree int_type = build_nonstandard_integer_type (elem_size, 0);
5214 0 : mask_type = build_vector_type (int_type, tgt_units);
5215 : }
5216 1 : op2 = vec_perm_indices_to_tree (mask_type, new_indices);
5217 : }
5218 : else
5219 23 : return false;
5220 :
5221 : /* Convert the VECTOR_CST to the appropriate vector type. */
5222 1 : if (tgt_type != TREE_TYPE (arg0))
5223 0 : arg0 = fold_build1 (VIEW_CONVERT_EXPR, tgt_type, arg0);
5224 1 : else if (tgt_type != TREE_TYPE (arg1))
5225 0 : arg1 = fold_build1 (VIEW_CONVERT_EXPR, tgt_type, arg1);
5226 47 : }
5227 :
5228 : /* VIEW_CONVERT_EXPR should be updated to CONSTRUCTOR before. */
5229 2224 : gcc_assert (code == CONSTRUCTOR || code == VECTOR_CST);
5230 :
5231 : /* Shuffle of a constructor. */
5232 2224 : tree res_type
5233 2224 : = build_vector_type (TREE_TYPE (TREE_TYPE (arg0)),
5234 2224 : TYPE_VECTOR_SUBPARTS (TREE_TYPE (op2)));
5235 2224 : tree opt = fold_ternary (VEC_PERM_EXPR, res_type, arg0, arg1, op2);
5236 2224 : if (!opt
5237 280 : || (TREE_CODE (opt) != CONSTRUCTOR && TREE_CODE (opt) != VECTOR_CST))
5238 : return false;
5239 : /* Found VIEW_CONVERT_EXPR before, need one explicit conversion. */
5240 280 : if (res_type != TREE_TYPE (op0))
5241 : {
5242 1 : tree name = make_ssa_name (TREE_TYPE (opt));
5243 1 : gimple *ass_stmt = gimple_build_assign (name, opt);
5244 1 : gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
5245 1 : opt = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (op0), name);
5246 : }
5247 280 : gimple_assign_set_rhs_from_tree (gsi, opt);
5248 280 : update_stmt (gsi_stmt (*gsi));
5249 280 : if (TREE_CODE (op0) == SSA_NAME)
5250 1 : remove_prop_source_from_use (op0);
5251 280 : if (op0 != op1 && TREE_CODE (op1) == SSA_NAME)
5252 0 : remove_prop_source_from_use (op1);
5253 : return true;
5254 : }
5255 :
5256 : return false;
5257 : }
5258 :
5259 : /* Get the BIT_FIELD_REF definition of VAL, if any, looking through
5260 : conversions with code CONV_CODE or update it if still ERROR_MARK.
5261 : Return NULL_TREE if no such matching def was found. */
5262 :
5263 : static tree
5264 437871 : get_bit_field_ref_def (tree val, enum tree_code &conv_code)
5265 : {
5266 437871 : if (TREE_CODE (val) != SSA_NAME)
5267 : return NULL_TREE ;
5268 407040 : gimple *def_stmt = get_prop_source_stmt (val, false, NULL);
5269 407040 : if (!def_stmt)
5270 : return NULL_TREE;
5271 325882 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
5272 325882 : if (code == FLOAT_EXPR
5273 325882 : || code == FIX_TRUNC_EXPR
5274 : || CONVERT_EXPR_CODE_P (code))
5275 : {
5276 188476 : tree op1 = gimple_assign_rhs1 (def_stmt);
5277 188476 : if (conv_code == ERROR_MARK)
5278 90240 : conv_code = code;
5279 98236 : else if (conv_code != code)
5280 : return NULL_TREE;
5281 188451 : if (TREE_CODE (op1) != SSA_NAME)
5282 : return NULL_TREE;
5283 79950 : def_stmt = SSA_NAME_DEF_STMT (op1);
5284 79950 : if (! is_gimple_assign (def_stmt))
5285 : return NULL_TREE;
5286 63754 : code = gimple_assign_rhs_code (def_stmt);
5287 : }
5288 201160 : if (code != BIT_FIELD_REF)
5289 : return NULL_TREE;
5290 23822 : return gimple_assign_rhs1 (def_stmt);
5291 : }
5292 :
5293 : /* Recognize a VEC_PERM_EXPR. Returns true if there were any changes. */
5294 :
5295 : static bool
5296 168113 : simplify_vector_constructor (gimple_stmt_iterator *gsi)
5297 : {
5298 168113 : gimple *stmt = gsi_stmt (*gsi);
5299 168113 : tree op, orig[2], type;
5300 168113 : unsigned i;
5301 168113 : unsigned HOST_WIDE_INT nelts;
5302 168113 : unsigned HOST_WIDE_INT refnelts;
5303 168113 : enum tree_code conv_code;
5304 168113 : constructor_elt *elt;
5305 :
5306 168113 : op = gimple_assign_rhs1 (stmt);
5307 168113 : type = TREE_TYPE (op);
5308 168113 : gcc_checking_assert (TREE_CODE (op) == CONSTRUCTOR
5309 : && TREE_CODE (type) == VECTOR_TYPE);
5310 :
5311 168113 : if (!TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
5312 : return false;
5313 :
5314 168113 : orig[0] = NULL;
5315 168113 : orig[1] = NULL;
5316 168113 : tree orig_elem_type[2] = {};
5317 168113 : conv_code = ERROR_MARK;
5318 168113 : bool maybe_ident = true;
5319 168113 : bool maybe_blend[2] = { true, true };
5320 168113 : tree one_constant = NULL_TREE;
5321 168113 : tree one_nonconstant = NULL_TREE;
5322 168113 : tree subelt;
5323 168113 : auto_vec<tree> constants;
5324 168113 : constants.safe_grow_cleared (nelts, true);
5325 168113 : auto_vec<std::pair<unsigned, unsigned>, 64> elts;
5326 168113 : unsigned int tsubelts = 0;
5327 471489 : FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (op), i, elt)
5328 : {
5329 437871 : tree ref, op1;
5330 437871 : unsigned int elem, src_elem_size;
5331 437871 : unsigned HOST_WIDE_INT nsubelts = 1;
5332 :
5333 437871 : if (i >= nelts)
5334 168113 : return false;
5335 :
5336 : /* Look for elements extracted and possibly converted from
5337 : another vector. */
5338 437871 : op1 = get_bit_field_ref_def (elt->value, conv_code);
5339 442196 : if (op1
5340 23822 : && TREE_CODE ((ref = TREE_OPERAND (op1, 0))) == SSA_NAME
5341 : /* Do not extend the live range of an SSA name that occurs in an
5342 : abnormal PHI. */
5343 5070 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ref)
5344 5020 : && VECTOR_TYPE_P (TREE_TYPE (ref))
5345 5005 : && (tree_nop_conversion_p (TREE_TYPE (op1),
5346 5005 : TREE_TYPE (TREE_TYPE (ref)))
5347 815 : || (VECTOR_TYPE_P (TREE_TYPE (op1))
5348 135 : && tree_nop_conversion_p (TREE_TYPE (TREE_TYPE (op1)),
5349 135 : TREE_TYPE (TREE_TYPE (ref)))
5350 135 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (op1))
5351 135 : .is_constant (&nsubelts)))
5352 4325 : && constant_multiple_p (bit_field_size (op1), nsubelts,
5353 : &src_elem_size)
5354 442196 : && constant_multiple_p (bit_field_offset (op1), src_elem_size, &elem)
5355 442196 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (ref)).is_constant (&refnelts))
5356 : {
5357 : unsigned int j;
5358 4685 : for (j = 0; j < 2; ++j)
5359 : {
5360 4660 : if (!orig[j])
5361 : {
5362 1884 : if (j == 0
5363 2084 : || useless_type_conversion_p (TREE_TYPE (orig[0]),
5364 200 : TREE_TYPE (ref)))
5365 : break;
5366 : }
5367 2776 : else if (ref == orig[j])
5368 : break;
5369 : }
5370 : /* Found a suitable vector element. */
5371 4325 : if (j < 2)
5372 : {
5373 4300 : orig[j] = ref;
5374 : /* Track what element type was actually extracted (which may
5375 : differ in signedness from the vector's element type due to
5376 : tree_nop_conversion_p). */
5377 4300 : if (!orig_elem_type[j])
5378 1880 : orig_elem_type[j] = TREE_TYPE (op1);
5379 4300 : if (elem != i || j != 0)
5380 2241 : maybe_ident = false;
5381 4300 : if (elem != i)
5382 2180 : maybe_blend[j] = false;
5383 8861 : for (unsigned int k = 0; k < nsubelts; ++k)
5384 4561 : elts.safe_push (std::make_pair (j, elem + k));
5385 4300 : tsubelts += nsubelts;
5386 4300 : continue;
5387 4300 : }
5388 : /* Else fallthru. */
5389 : }
5390 : /* Handle elements not extracted from a vector.
5391 : 1. constants by permuting with constant vector
5392 : 2. a unique non-constant element by permuting with a splat vector */
5393 433571 : if (orig[1]
5394 266311 : && orig[1] != error_mark_node)
5395 : return false;
5396 433546 : orig[1] = error_mark_node;
5397 433546 : if (VECTOR_TYPE_P (TREE_TYPE (elt->value))
5398 433546 : && !TYPE_VECTOR_SUBPARTS (TREE_TYPE (elt->value))
5399 5803 : .is_constant (&nsubelts))
5400 : return false;
5401 433546 : if (CONSTANT_CLASS_P (elt->value))
5402 : {
5403 30827 : if (one_nonconstant)
5404 : return false;
5405 21859 : if (!one_constant)
5406 9792 : one_constant = TREE_CODE (elt->value) == VECTOR_CST
5407 9792 : ? VECTOR_CST_ELT (elt->value, 0)
5408 : : elt->value;
5409 21859 : if (TREE_CODE (elt->value) == VECTOR_CST)
5410 : {
5411 777 : for (unsigned int k = 0; k < nsubelts; k++)
5412 587 : constants[tsubelts + k] = VECTOR_CST_ELT (elt->value, k);
5413 : }
5414 : else
5415 21669 : constants[tsubelts] = elt->value;
5416 : }
5417 : else
5418 : {
5419 402719 : if (one_constant)
5420 : return false;
5421 393448 : subelt = VECTOR_TYPE_P (TREE_TYPE (elt->value))
5422 393448 : ? ssa_uniform_vector_p (elt->value)
5423 : : elt->value;
5424 393448 : if (!subelt
5425 : /* The splat of the element of a uniform vector is a new use
5426 : of that element. Do not extend the live range of an SSA
5427 : name that occurs in an abnormal PHI. */
5428 393448 : || (VECTOR_TYPE_P (TREE_TYPE (elt->value))
5429 123 : && TREE_CODE (subelt) == SSA_NAME
5430 123 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (subelt)))
5431 : return false;
5432 388087 : if (!one_nonconstant)
5433 : one_nonconstant = subelt;
5434 235980 : else if (!operand_equal_p (one_nonconstant, subelt, 0))
5435 : return false;
5436 : }
5437 598551 : for (unsigned int k = 0; k < nsubelts; ++k)
5438 299475 : elts.safe_push (std::make_pair (1, tsubelts + k));
5439 299076 : tsubelts += nsubelts;
5440 299076 : maybe_ident = false;
5441 : }
5442 :
5443 67236 : if (elts.length () < nelts)
5444 : return false;
5445 :
5446 32446 : if (! orig[0]
5447 32446 : || ! VECTOR_TYPE_P (TREE_TYPE (orig[0])))
5448 : return false;
5449 1199 : refnelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (orig[0])).to_constant ();
5450 : /* We currently do not handle larger destination vectors. */
5451 1199 : if (refnelts < nelts)
5452 : return false;
5453 :
5454 : /* Determine the element type for the conversion source.
5455 : As orig_elem_type keeps track of the original type, check
5456 : if we need to perform a sign swap after permuting.
5457 : We need to be able to construct a vector type from the element
5458 : type which is not possible for e.g. BitInt or pointers
5459 : so pun with an integer type if needed. */
5460 985 : tree perm_eltype = TREE_TYPE (TREE_TYPE (orig[0]));
5461 985 : bool sign_change_p = false;
5462 985 : if (conv_code != ERROR_MARK
5463 213 : && orig_elem_type[0]
5464 1198 : && TYPE_SIGN (orig_elem_type[0]) != TYPE_SIGN (perm_eltype))
5465 : {
5466 35 : perm_eltype = signed_or_unsigned_type_for
5467 35 : (TYPE_UNSIGNED (orig_elem_type[0]), perm_eltype);
5468 35 : sign_change_p = true;
5469 : }
5470 985 : tree conv_src_type = build_vector_type (perm_eltype, nelts);
5471 :
5472 985 : if (maybe_ident)
5473 : {
5474 : /* When there is no conversion, use the target type directly. */
5475 196 : if (conv_code == ERROR_MARK && nelts != refnelts)
5476 196 : conv_src_type = type;
5477 196 : if (conv_code != ERROR_MARK
5478 196 : && !supportable_convert_operation (conv_code, type, conv_src_type))
5479 : {
5480 : /* Only few targets implement direct conversion patterns so try
5481 : some simple special cases via VEC_[UN]PACK[_FLOAT]_LO_EXPR. */
5482 103 : optab optab;
5483 103 : insn_code icode;
5484 103 : tree halfvectype, dblvectype;
5485 103 : enum tree_code unpack_op;
5486 :
5487 103 : if (!BYTES_BIG_ENDIAN)
5488 183 : unpack_op = (FLOAT_TYPE_P (TREE_TYPE (type))
5489 103 : ? VEC_UNPACK_FLOAT_LO_EXPR
5490 : : VEC_UNPACK_LO_EXPR);
5491 : else
5492 : unpack_op = (FLOAT_TYPE_P (TREE_TYPE (type))
5493 : ? VEC_UNPACK_FLOAT_HI_EXPR
5494 : : VEC_UNPACK_HI_EXPR);
5495 :
5496 : /* Conversions between DFP and FP have no special tree code
5497 : but we cannot handle those since all relevant vector conversion
5498 : optabs only have a single mode. */
5499 15 : if (CONVERT_EXPR_CODE_P (conv_code)
5500 88 : && FLOAT_TYPE_P (TREE_TYPE (type))
5501 119 : && (DECIMAL_FLOAT_TYPE_P (TREE_TYPE (type))
5502 8 : != DECIMAL_FLOAT_TYPE_P (TREE_TYPE (conv_src_type))))
5503 : return false;
5504 :
5505 15 : if (CONVERT_EXPR_CODE_P (conv_code)
5506 87 : && (2 * TYPE_PRECISION (TREE_TYPE (TREE_TYPE (orig[0])))
5507 87 : == TYPE_PRECISION (TREE_TYPE (type)))
5508 6 : && orig_elem_type[0]
5509 6 : && useless_type_conversion_p (orig_elem_type[0],
5510 6 : TREE_TYPE (TREE_TYPE (orig[0])))
5511 6 : && mode_for_vector (as_a <scalar_mode>
5512 6 : (TYPE_MODE (TREE_TYPE (TREE_TYPE (orig[0])))),
5513 12 : nelts * 2).exists ()
5514 6 : && (dblvectype
5515 6 : = build_vector_type (TREE_TYPE (TREE_TYPE (orig[0])),
5516 6 : nelts * 2))
5517 : /* Only use it for vector modes or for vector booleans
5518 : represented as scalar bitmasks. See PR95528. */
5519 6 : && (VECTOR_MODE_P (TYPE_MODE (dblvectype))
5520 0 : || VECTOR_BOOLEAN_TYPE_P (dblvectype))
5521 6 : && (optab = optab_for_tree_code (unpack_op,
5522 : dblvectype,
5523 : optab_default))
5524 6 : && ((icode = optab_handler (optab, TYPE_MODE (dblvectype)))
5525 : != CODE_FOR_nothing)
5526 102 : && (insn_data[icode].operand[0].mode == TYPE_MODE (type)))
5527 : {
5528 0 : gimple_seq stmts = NULL;
5529 0 : tree dbl;
5530 0 : if (refnelts == nelts)
5531 : {
5532 : /* ??? Paradoxical subregs don't exist, so insert into
5533 : the lower half of a wider zero vector. */
5534 0 : dbl = gimple_build (&stmts, BIT_INSERT_EXPR, dblvectype,
5535 : build_zero_cst (dblvectype), orig[0],
5536 0 : bitsize_zero_node);
5537 : }
5538 0 : else if (refnelts == 2 * nelts)
5539 : dbl = orig[0];
5540 : else
5541 0 : dbl = gimple_build (&stmts, BIT_FIELD_REF, dblvectype,
5542 0 : orig[0], TYPE_SIZE (dblvectype),
5543 0 : bitsize_zero_node);
5544 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
5545 0 : gimple_assign_set_rhs_with_ops (gsi, unpack_op, dbl);
5546 : }
5547 15 : else if (CONVERT_EXPR_CODE_P (conv_code)
5548 87 : && (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (orig[0])))
5549 87 : == 2 * TYPE_PRECISION (TREE_TYPE (type)))
5550 5 : && orig_elem_type[0]
5551 5 : && useless_type_conversion_p (orig_elem_type[0],
5552 5 : TREE_TYPE (TREE_TYPE (orig[0])))
5553 1 : && mode_for_vector (as_a <scalar_mode>
5554 1 : (TYPE_MODE
5555 : (TREE_TYPE (TREE_TYPE (orig[0])))),
5556 2 : nelts / 2).exists ()
5557 1 : && (halfvectype
5558 1 : = build_vector_type (TREE_TYPE (TREE_TYPE (orig[0])),
5559 1 : nelts / 2))
5560 : /* Only use it for vector modes or for vector booleans
5561 : represented as scalar bitmasks. See PR95528. */
5562 1 : && (VECTOR_MODE_P (TYPE_MODE (halfvectype))
5563 0 : || VECTOR_BOOLEAN_TYPE_P (halfvectype))
5564 1 : && (optab = optab_for_tree_code (VEC_PACK_TRUNC_EXPR,
5565 : halfvectype,
5566 : optab_default))
5567 1 : && ((icode = optab_handler (optab, TYPE_MODE (halfvectype)))
5568 : != CODE_FOR_nothing)
5569 103 : && (insn_data[icode].operand[0].mode == TYPE_MODE (type)))
5570 : {
5571 0 : gimple_seq stmts = NULL;
5572 0 : tree low = gimple_build (&stmts, BIT_FIELD_REF, halfvectype,
5573 0 : orig[0], TYPE_SIZE (halfvectype),
5574 0 : bitsize_zero_node);
5575 0 : tree hig = gimple_build (&stmts, BIT_FIELD_REF, halfvectype,
5576 0 : orig[0], TYPE_SIZE (halfvectype),
5577 0 : TYPE_SIZE (halfvectype));
5578 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
5579 0 : gimple_assign_set_rhs_with_ops (gsi, VEC_PACK_TRUNC_EXPR,
5580 : low, hig);
5581 : }
5582 : else
5583 102 : return false;
5584 0 : update_stmt (gsi_stmt (*gsi));
5585 0 : return true;
5586 : }
5587 93 : if (nelts != refnelts)
5588 : {
5589 15 : gassign *lowpart
5590 15 : = gimple_build_assign (make_ssa_name (conv_src_type),
5591 : build3 (BIT_FIELD_REF, conv_src_type,
5592 15 : orig[0], TYPE_SIZE (conv_src_type),
5593 : bitsize_zero_node));
5594 15 : gsi_insert_before (gsi, lowpart, GSI_SAME_STMT);
5595 15 : orig[0] = gimple_assign_lhs (lowpart);
5596 : }
5597 78 : else if (sign_change_p)
5598 : {
5599 0 : gassign *conv
5600 0 : = gimple_build_assign (make_ssa_name (conv_src_type),
5601 : build1 (VIEW_CONVERT_EXPR, conv_src_type,
5602 : orig[0]));
5603 0 : gsi_insert_before (gsi, conv, GSI_SAME_STMT);
5604 0 : orig[0] = gimple_assign_lhs (conv);
5605 : }
5606 93 : if (conv_code == ERROR_MARK)
5607 : {
5608 76 : tree src_type = TREE_TYPE (orig[0]);
5609 76 : if (!useless_type_conversion_p (type, src_type))
5610 : {
5611 0 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (type),
5612 : TYPE_VECTOR_SUBPARTS (src_type))
5613 : && tree_nop_conversion_p (TREE_TYPE (type),
5614 : TREE_TYPE (src_type)));
5615 0 : tree rhs = build1 (VIEW_CONVERT_EXPR, type, orig[0]);
5616 0 : orig[0] = make_ssa_name (type);
5617 0 : gassign *assign = gimple_build_assign (orig[0], rhs);
5618 0 : gsi_insert_before (gsi, assign, GSI_SAME_STMT);
5619 : }
5620 76 : gimple_assign_set_rhs_from_tree (gsi, orig[0]);
5621 : }
5622 : else
5623 17 : gimple_assign_set_rhs_with_ops (gsi, conv_code, orig[0],
5624 : NULL_TREE, NULL_TREE);
5625 : }
5626 : else
5627 : {
5628 : /* If we combine a vector with a non-vector avoid cases where
5629 : we'll obviously end up with more GIMPLE stmts which is when
5630 : we'll later not fold this to a single insert into the vector
5631 : and we had a single extract originally. See PR92819. */
5632 789 : if (nelts == 2
5633 335 : && refnelts > 2
5634 123 : && orig[1] == error_mark_node
5635 31 : && !maybe_blend[0])
5636 92 : return false;
5637 764 : tree mask_type, perm_type;
5638 764 : perm_type = TREE_TYPE (orig[0]);
5639 764 : if (conv_code != ERROR_MARK
5640 764 : && !supportable_convert_operation (conv_code, type, conv_src_type))
5641 : return false;
5642 :
5643 : /* Now that we know the number of elements of the source build the
5644 : permute vector.
5645 : ??? When the second vector has constant values we can shuffle
5646 : it and its source indexes to make the permutation supported.
5647 : For now it mimics a blend. */
5648 754 : vec_perm_builder sel (refnelts, refnelts, 1);
5649 754 : bool all_same_p = true;
5650 8416 : for (i = 0; i < elts.length (); ++i)
5651 : {
5652 3454 : sel.quick_push (elts[i].second + elts[i].first * refnelts);
5653 3454 : all_same_p &= known_eq (sel[i], sel[0]);
5654 : }
5655 : /* And fill the tail with "something". It's really don't care,
5656 : and ideally we'd allow VEC_PERM to have a smaller destination
5657 : vector. As a heuristic:
5658 :
5659 : (a) if what we have so far duplicates a single element, make the
5660 : tail do the same
5661 :
5662 : (b) otherwise preserve a uniform orig[0]. This facilitates
5663 : later pattern-matching of VEC_PERM_EXPR to a BIT_INSERT_EXPR. */
5664 1372 : for (; i < refnelts; ++i)
5665 1236 : sel.quick_push (all_same_p
5666 1854 : ? sel[0]
5667 156 : : (elts[0].second == 0 && elts[0].first == 0
5668 920 : ? 0 : refnelts) + i);
5669 953 : vec_perm_indices indices (sel, orig[1] ? 2 : 1, refnelts);
5670 754 : machine_mode vmode = TYPE_MODE (perm_type);
5671 754 : if ((cfun->curr_properties & PROP_gimple_lvec)
5672 754 : && !can_vec_perm_const_p (vmode, vmode, indices))
5673 : return false;
5674 697 : mask_type = build_vector_type (ssizetype, refnelts);
5675 697 : tree op2 = vec_perm_indices_to_tree (mask_type, indices);
5676 697 : bool converted_orig1 = false;
5677 697 : gimple_seq stmts = NULL;
5678 697 : if (!orig[1])
5679 174 : orig[1] = orig[0];
5680 523 : else if (orig[1] == error_mark_node
5681 401 : && one_nonconstant)
5682 : {
5683 : /* ??? We can see if we can safely convert to the original
5684 : element type. */
5685 91 : converted_orig1 = conv_code != ERROR_MARK;
5686 91 : tree target_type = converted_orig1 ? type : perm_type;
5687 91 : tree nonconstant_for_splat = one_nonconstant;
5688 : /* If there's a nop conversion between the target element type and
5689 : the nonconstant's type, convert it. */
5690 91 : if (!useless_type_conversion_p (TREE_TYPE (target_type),
5691 91 : TREE_TYPE (one_nonconstant)))
5692 0 : nonconstant_for_splat
5693 0 : = gimple_build (&stmts, NOP_EXPR, TREE_TYPE (target_type),
5694 : one_nonconstant);
5695 91 : orig[1] = gimple_build_vector_from_val (&stmts, UNKNOWN_LOCATION,
5696 : target_type,
5697 : nonconstant_for_splat);
5698 91 : }
5699 432 : else if (orig[1] == error_mark_node)
5700 : {
5701 : /* ??? See if we can convert the vector to the original type. */
5702 310 : converted_orig1 = conv_code != ERROR_MARK;
5703 310 : unsigned n = converted_orig1 ? nelts : refnelts;
5704 293 : tree target_type = converted_orig1 ? type : perm_type;
5705 310 : tree_vector_builder vec (target_type, n, 1);
5706 2508 : for (unsigned i = 0; i < n; ++i)
5707 3652 : if (i < nelts && constants[i])
5708 : {
5709 1056 : tree constant = constants[i];
5710 : /* If there's a nop conversion, convert the constant. */
5711 1056 : if (!useless_type_conversion_p (TREE_TYPE (target_type),
5712 1056 : TREE_TYPE (constant)))
5713 2 : constant = fold_convert (TREE_TYPE (target_type), constant);
5714 1056 : vec.quick_push (constant);
5715 : }
5716 : else
5717 : {
5718 : /* ??? Push a don't-care value. */
5719 832 : tree constant = one_constant;
5720 832 : if (!useless_type_conversion_p (TREE_TYPE (target_type),
5721 832 : TREE_TYPE (constant)))
5722 2 : constant = fold_convert (TREE_TYPE (target_type), constant);
5723 832 : vec.quick_push (constant);
5724 : }
5725 310 : orig[1] = vec.build ();
5726 310 : }
5727 575 : tree blend_op2 = NULL_TREE;
5728 575 : if (converted_orig1)
5729 : {
5730 : /* Make sure we can do a blend in the target type. */
5731 34 : vec_perm_builder sel (nelts, nelts, 1);
5732 136 : for (i = 0; i < elts.length (); ++i)
5733 102 : sel.quick_push (elts[i].first
5734 102 : ? elts[i].second + nelts : i);
5735 34 : vec_perm_indices indices (sel, 2, nelts);
5736 34 : machine_mode vmode = TYPE_MODE (type);
5737 34 : if ((cfun->curr_properties & PROP_gimple_lvec)
5738 34 : && !can_vec_perm_const_p (vmode, vmode, indices))
5739 0 : return false;
5740 34 : mask_type = build_vector_type (ssizetype, nelts);
5741 34 : blend_op2 = vec_perm_indices_to_tree (mask_type, indices);
5742 34 : }
5743 :
5744 : /* For a real orig[1] (no splat, constant etc.) we might need to
5745 : nop-convert it. Do so here. */
5746 697 : if (orig[1] && orig[1] != error_mark_node
5747 697 : && !converted_orig1
5748 663 : && !useless_type_conversion_p (perm_type, TREE_TYPE (orig[1]))
5749 697 : && tree_nop_conversion_p (TREE_TYPE (perm_type),
5750 0 : TREE_TYPE (TREE_TYPE (orig[1]))))
5751 0 : orig[1] = gimple_build (&stmts, VIEW_CONVERT_EXPR, perm_type,
5752 : orig[1]);
5753 :
5754 697 : tree orig1_for_perm
5755 697 : = converted_orig1 ? build_zero_cst (perm_type) : orig[1];
5756 697 : tree res = gimple_build (&stmts, VEC_PERM_EXPR, perm_type,
5757 : orig[0], orig1_for_perm, op2);
5758 : /* If we're building a smaller vector, extract the element
5759 : with the proper type. */
5760 697 : if (nelts != refnelts)
5761 246 : res = gimple_build (&stmts, BIT_FIELD_REF,
5762 : conv_code != ERROR_MARK ? conv_src_type : type,
5763 : res,
5764 123 : TYPE_SIZE (conv_code != ERROR_MARK ? conv_src_type
5765 : : type),
5766 123 : bitsize_zero_node);
5767 : /* Otherwise, we can still have an intermediate sign change.
5768 : ??? In that case we have two subsequent conversions.
5769 : We should be able to merge them. */
5770 574 : else if (sign_change_p)
5771 15 : res = gimple_build (&stmts, VIEW_CONVERT_EXPR, conv_src_type, res);
5772 : /* Finally, apply the conversion. */
5773 697 : if (conv_code != ERROR_MARK)
5774 68 : res = gimple_build (&stmts, conv_code, type, res);
5775 629 : else if (!useless_type_conversion_p (type, TREE_TYPE (res)))
5776 : {
5777 3 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (type),
5778 : TYPE_VECTOR_SUBPARTS (perm_type))
5779 : && tree_nop_conversion_p (TREE_TYPE (type),
5780 : TREE_TYPE (perm_type)));
5781 3 : res = gimple_build (&stmts, VIEW_CONVERT_EXPR, type, res);
5782 : }
5783 : /* Blend in the actual constant. */
5784 697 : if (converted_orig1)
5785 34 : res = gimple_build (&stmts, VEC_PERM_EXPR, type,
5786 34 : res, orig[1], blend_op2);
5787 697 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
5788 697 : gimple_assign_set_rhs_with_ops (gsi, SSA_NAME, res);
5789 754 : }
5790 790 : update_stmt (gsi_stmt (*gsi));
5791 790 : return true;
5792 168113 : }
5793 :
5794 : /* Prepare a TARGET_MEM_REF ref so that it can be subsetted as
5795 : lvalue. This splits out an address computation stmt before *GSI
5796 : and returns a MEM_REF wrapping the address. */
5797 :
5798 : static tree
5799 1143 : prepare_target_mem_ref_lvalue (tree ref, gimple_stmt_iterator *gsi)
5800 : {
5801 1143 : if (TREE_CODE (TREE_OPERAND (ref, 0)) == ADDR_EXPR)
5802 189 : mark_addressable (TREE_OPERAND (TREE_OPERAND (ref, 0), 0));
5803 1143 : tree ptrtype = build_pointer_type (TREE_TYPE (ref));
5804 1143 : tree tem = make_ssa_name (ptrtype);
5805 1143 : gimple *new_stmt
5806 1143 : = gimple_build_assign (tem, build1 (ADDR_EXPR, TREE_TYPE (tem),
5807 : unshare_expr (ref)));
5808 1143 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
5809 2286 : ref = build2_loc (EXPR_LOCATION (ref),
5810 1143 : MEM_REF, TREE_TYPE (ref), tem,
5811 1143 : build_int_cst (TREE_TYPE (TREE_OPERAND (ref, 1)), 0));
5812 1143 : return ref;
5813 : }
5814 :
5815 : /* Rewrite the vector load at *GSI to component-wise loads if the load
5816 : is only used in BIT_FIELD_REF extractions with eventual intermediate
5817 : widening. */
5818 :
5819 : static void
5820 282909 : optimize_vector_load (gimple_stmt_iterator *gsi)
5821 : {
5822 282909 : gimple *stmt = gsi_stmt (*gsi);
5823 282909 : tree lhs = gimple_assign_lhs (stmt);
5824 282909 : tree rhs = gimple_assign_rhs1 (stmt);
5825 282909 : tree vuse = gimple_vuse (stmt);
5826 :
5827 : /* Gather BIT_FIELD_REFs to rewrite, looking through
5828 : VEC_UNPACK_{LO,HI}_EXPR. */
5829 282909 : use_operand_p use_p;
5830 282909 : imm_use_iterator iter;
5831 282909 : bool rewrite = true;
5832 282909 : bool scalar_use = false;
5833 282909 : bool unpack_use = false;
5834 282909 : auto_vec<gimple *, 8> bf_stmts;
5835 282909 : auto_vec<tree, 8> worklist;
5836 282909 : worklist.quick_push (lhs);
5837 284740 : do
5838 : {
5839 284740 : tree def = worklist.pop ();
5840 284740 : unsigned HOST_WIDE_INT def_eltsize
5841 284740 : = TREE_INT_CST_LOW (TYPE_SIZE (TREE_TYPE (TREE_TYPE (def))));
5842 364165 : FOR_EACH_IMM_USE_FAST (use_p, iter, def)
5843 : {
5844 342941 : gimple *use_stmt = USE_STMT (use_p);
5845 342941 : if (is_gimple_debug (use_stmt))
5846 79425 : continue;
5847 341297 : tree use_lhs;
5848 341297 : if (!is_gimple_assign (use_stmt)
5849 : /* For alias reasons we move the use to the place of the
5850 : load. Avoid this when abnormals are involved. */
5851 341297 : || ((TREE_CODE ((use_lhs = gimple_assign_lhs (use_stmt)))
5852 : == SSA_NAME)
5853 240853 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs)))
5854 : {
5855 : rewrite = false;
5856 263516 : break;
5857 : }
5858 311188 : enum tree_code use_code = gimple_assign_rhs_code (use_stmt);
5859 311188 : tree use_rhs = gimple_assign_rhs1 (use_stmt);
5860 385324 : if (use_code == BIT_FIELD_REF
5861 74137 : && TREE_OPERAND (use_rhs, 0) == def
5862 : /* If its on the VEC_UNPACK_{HI,LO}_EXPR
5863 : def need to verify it is element aligned. */
5864 385325 : && (def == lhs
5865 153 : || (known_eq (bit_field_size (use_rhs), def_eltsize)
5866 74289 : && constant_multiple_p (bit_field_offset (use_rhs),
5867 : def_eltsize)
5868 : /* We can simulate the VEC_UNPACK_{HI,LO}_EXPR
5869 : via a NOP_EXPR only for integral types.
5870 : ??? Support VEC_UNPACK_FLOAT_{HI,LO}_EXPR. */
5871 153 : && INTEGRAL_TYPE_P (TREE_TYPE (use_rhs)))))
5872 : {
5873 74136 : if (!VECTOR_TYPE_P (TREE_TYPE (gimple_assign_lhs (use_stmt))))
5874 71739 : scalar_use = true;
5875 74136 : bf_stmts.safe_push (use_stmt);
5876 74136 : continue;
5877 : }
5878 : /* Walk through one level of VEC_UNPACK_{LO,HI}_EXPR. */
5879 237052 : if (def == lhs
5880 235302 : && (use_code == VEC_UNPACK_HI_EXPR
5881 235302 : || use_code == VEC_UNPACK_LO_EXPR)
5882 3645 : && use_rhs == lhs)
5883 : {
5884 3645 : unpack_use = true;
5885 3645 : worklist.safe_push (gimple_assign_lhs (use_stmt));
5886 3645 : continue;
5887 : }
5888 : rewrite = false;
5889 : break;
5890 284740 : }
5891 284740 : if (!rewrite)
5892 : break;
5893 : }
5894 42448 : while (!worklist.is_empty ());
5895 :
5896 282909 : rewrite = rewrite && (scalar_use
5897 19393 : || unpack_use
5898 674 : || !can_implement_p (mov_optab,
5899 674 : TYPE_MODE (TREE_TYPE (lhs))));
5900 263730 : if (!rewrite)
5901 : {
5902 263730 : gsi_next (gsi);
5903 263730 : return;
5904 : }
5905 : /* We now have all ultimate uses of the load to rewrite in bf_stmts. */
5906 :
5907 : /* Prepare the original ref to be wrapped in adjusted BIT_FIELD_REFs.
5908 : For TARGET_MEM_REFs we have to separate the LEA from the reference. */
5909 19179 : tree load_rhs = rhs;
5910 19179 : if (TREE_CODE (load_rhs) == TARGET_MEM_REF)
5911 1142 : load_rhs = prepare_target_mem_ref_lvalue (load_rhs, gsi);
5912 :
5913 : /* Rewrite the BIT_FIELD_REFs to be actual loads, re-emitting them at
5914 : the place of the original load. */
5915 127409 : for (gimple *use_stmt : bf_stmts)
5916 : {
5917 69872 : tree bfr = gimple_assign_rhs1 (use_stmt);
5918 69872 : tree new_rhs = unshare_expr (load_rhs);
5919 69872 : if (TREE_OPERAND (bfr, 0) != lhs)
5920 : {
5921 : /* When the BIT_FIELD_REF is on the promoted vector we have to
5922 : adjust it and emit a conversion afterwards. */
5923 152 : gimple *def_stmt
5924 152 : = SSA_NAME_DEF_STMT (TREE_OPERAND (bfr, 0));
5925 152 : enum tree_code def_code
5926 152 : = gimple_assign_rhs_code (def_stmt);
5927 :
5928 : /* The adjusted BIT_FIELD_REF is of the promotion source
5929 : vector size and at half of the offset... */
5930 152 : new_rhs = fold_build3 (BIT_FIELD_REF,
5931 : TREE_TYPE (TREE_TYPE (lhs)),
5932 : new_rhs,
5933 : TYPE_SIZE (TREE_TYPE (TREE_TYPE (lhs))),
5934 : size_binop (EXACT_DIV_EXPR,
5935 : TREE_OPERAND (bfr, 2),
5936 : bitsize_int (2)));
5937 : /* ... and offsetted by half of the vector if VEC_UNPACK_HI_EXPR. */
5938 152 : if (def_code == (!BYTES_BIG_ENDIAN
5939 : ? VEC_UNPACK_HI_EXPR : VEC_UNPACK_LO_EXPR))
5940 76 : TREE_OPERAND (new_rhs, 2)
5941 152 : = size_binop (PLUS_EXPR, TREE_OPERAND (new_rhs, 2),
5942 : size_binop (EXACT_DIV_EXPR,
5943 : TYPE_SIZE (TREE_TYPE (lhs)),
5944 : bitsize_int (2)));
5945 152 : tree tem = make_ssa_name (TREE_TYPE (TREE_TYPE (lhs)));
5946 152 : gimple *new_stmt = gimple_build_assign (tem, new_rhs);
5947 152 : location_t loc = gimple_location (use_stmt);
5948 152 : gimple_set_location (new_stmt, loc);
5949 152 : gimple_set_vuse (new_stmt, vuse);
5950 152 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
5951 : /* Perform scalar promotion. */
5952 152 : new_stmt = gimple_build_assign (gimple_assign_lhs (use_stmt),
5953 : NOP_EXPR, tem);
5954 152 : gimple_set_location (new_stmt, loc);
5955 152 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
5956 : }
5957 : else
5958 : {
5959 : /* When the BIT_FIELD_REF is on the original load result
5960 : we can just wrap that. */
5961 69720 : tree new_rhs = fold_build3 (BIT_FIELD_REF, TREE_TYPE (bfr),
5962 : unshare_expr (load_rhs),
5963 : TREE_OPERAND (bfr, 1),
5964 : TREE_OPERAND (bfr, 2));
5965 69720 : gimple *new_stmt = gimple_build_assign (gimple_assign_lhs (use_stmt),
5966 : new_rhs);
5967 69720 : location_t loc = gimple_location (use_stmt);
5968 69720 : gimple_set_location (new_stmt, loc);
5969 69720 : gimple_set_vuse (new_stmt, vuse);
5970 69720 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
5971 : }
5972 69872 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5973 69872 : unlink_stmt_vdef (use_stmt);
5974 69872 : gsi_remove (&gsi2, true);
5975 : }
5976 :
5977 : /* Finally get rid of the intermediate stmts. */
5978 19179 : gimple *use_stmt;
5979 19915 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
5980 : {
5981 736 : if (is_gimple_debug (use_stmt))
5982 : {
5983 674 : if (gimple_debug_bind_p (use_stmt))
5984 : {
5985 674 : gimple_debug_bind_reset_value (use_stmt);
5986 674 : update_stmt (use_stmt);
5987 : }
5988 674 : continue;
5989 : }
5990 62 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5991 62 : unlink_stmt_vdef (use_stmt);
5992 62 : release_defs (use_stmt);
5993 62 : gsi_remove (&gsi2, true);
5994 19179 : }
5995 : /* And the original load. */
5996 19179 : release_defs (stmt);
5997 19179 : gsi_remove (gsi, true);
5998 282909 : }
5999 :
6000 :
6001 : /* Primitive "lattice" function for gimple_simplify. */
6002 :
6003 : static tree
6004 1785901107 : fwprop_ssa_val (tree name)
6005 : {
6006 : /* First valueize NAME. */
6007 1785901107 : if (TREE_CODE (name) == SSA_NAME
6008 1785901107 : && SSA_NAME_VERSION (name) < lattice.length ())
6009 : {
6010 1784973014 : tree val = lattice[SSA_NAME_VERSION (name)];
6011 1784973014 : if (val)
6012 1785901107 : name = val;
6013 : }
6014 : /* We continue matching along SSA use-def edges for SSA names
6015 : that are not single-use. Currently there are no patterns
6016 : that would cause any issues with that. */
6017 1785901107 : return name;
6018 : }
6019 :
6020 : /* Search for opportunities to free half of the lanes in the following pattern:
6021 :
6022 : v_in = {e0, e1, e2, e3}
6023 : v_1 = VEC_PERM <v_in, v_in, {0, 2, 0, 2}>
6024 : // v_1 = {e0, e2, e0, e2}
6025 : v_2 = VEC_PERM <v_in, v_in, {1, 3, 1, 3}>
6026 : // v_2 = {e1, e3, e1, e3}
6027 :
6028 : v_x = v_1 + v_2
6029 : // v_x = {e0+e1, e2+e3, e0+e1, e2+e3}
6030 : v_y = v_1 - v_2
6031 : // v_y = {e0-e1, e2-e3, e0-e1, e2-e3}
6032 :
6033 : v_out = VEC_PERM <v_x, v_y, {0, 1, 6, 7}>
6034 : // v_out = {e0+e1, e2+e3, e0-e1, e2-e3}
6035 :
6036 : The last statement could be simplified to:
6037 : v_out' = VEC_PERM <v_x, v_y, {0, 1, 4, 5}>
6038 : // v_out' = {e0+e1, e2+e3, e0-e1, e2-e3}
6039 :
6040 : Characteristic properties:
6041 : - v_1 and v_2 are created from the same input vector v_in and introduce the
6042 : lane duplication (in the selection operand) that we can eliminate.
6043 : - v_x and v_y are results from lane-preserving operations that use v_1 and
6044 : v_2 as inputs.
6045 : - v_out is created by selecting from duplicated lanes. */
6046 :
6047 : static bool
6048 190450 : recognise_vec_perm_simplify_seq (gassign *stmt, vec_perm_simplify_seq *seq)
6049 : {
6050 190450 : unsigned HOST_WIDE_INT nelts;
6051 :
6052 190450 : gcc_checking_assert (stmt);
6053 190450 : gcc_checking_assert (gimple_assign_rhs_code (stmt) == VEC_PERM_EXPR);
6054 190450 : basic_block bb = gimple_bb (stmt);
6055 :
6056 : /* Decompose the final vec permute statement. */
6057 190450 : tree v_x = gimple_assign_rhs1 (stmt);
6058 190450 : tree v_y = gimple_assign_rhs2 (stmt);
6059 190450 : tree sel = gimple_assign_rhs3 (stmt);
6060 :
6061 190450 : if (TREE_CODE (sel) != VECTOR_CST
6062 265192 : || !VECTOR_CST_NELTS (sel).is_constant (&nelts)
6063 187658 : || TREE_CODE (v_x) != SSA_NAME
6064 185568 : || TREE_CODE (v_y) != SSA_NAME
6065 180091 : || !has_single_use (v_x)
6066 303133 : || !has_single_use (v_y))
6067 : return false;
6068 :
6069 : /* Don't analyse sequences with many lanes. */
6070 110826 : if (nelts > 4)
6071 : return false;
6072 :
6073 : /* Lookup the definition of v_x and v_y. */
6074 106597 : gassign *v_x_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x));
6075 106597 : gassign *v_y_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_y));
6076 106208 : if (!v_x_stmt || gimple_bb (v_x_stmt) != bb
6077 212805 : || !v_y_stmt || gimple_bb (v_y_stmt) != bb)
6078 : return false;
6079 :
6080 : /* Check the operations that define v_x and v_y. */
6081 106201 : if (TREE_CODE_CLASS (gimple_assign_rhs_code (v_x_stmt)) != tcc_binary
6082 108306 : || TREE_CODE_CLASS (gimple_assign_rhs_code (v_y_stmt)) != tcc_binary)
6083 : return false;
6084 :
6085 2105 : tree v_x_1 = gimple_assign_rhs1 (v_x_stmt);
6086 2105 : tree v_x_2 = gimple_assign_rhs2 (v_x_stmt);
6087 2105 : tree v_y_1 = gimple_assign_rhs1 (v_y_stmt);
6088 2105 : tree v_y_2 = gimple_assign_rhs2 (v_y_stmt);
6089 :
6090 2105 : if (v_x_stmt == v_y_stmt
6091 2105 : || TREE_CODE (v_x_1) != SSA_NAME
6092 2102 : || TREE_CODE (v_x_2) != SSA_NAME
6093 2078 : || num_imm_uses (v_x_1) != 2
6094 4025 : || num_imm_uses (v_x_2) != 2)
6095 : return false;
6096 :
6097 1882 : if (v_x_1 != v_y_1 || v_x_2 != v_y_2)
6098 : {
6099 : /* Allow operands of commutative operators to swap. */
6100 656 : if (commutative_tree_code (gimple_assign_rhs_code (v_x_stmt)))
6101 : {
6102 : /* Keep v_x_1 the first operand for non-commutative operators. */
6103 253 : std::swap (v_x_1, v_x_2);
6104 253 : if (v_x_1 != v_y_1 || v_x_2 != v_y_2)
6105 : return false;
6106 : }
6107 403 : else if (commutative_tree_code (gimple_assign_rhs_code (v_y_stmt)))
6108 : {
6109 403 : if (v_x_1 != v_y_2 || v_x_2 != v_y_1)
6110 : return false;
6111 : }
6112 : else
6113 : return false;
6114 : }
6115 1882 : gassign *v_1_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x_1));
6116 1882 : gassign *v_2_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x_2));
6117 1818 : if (!v_1_stmt || gimple_bb (v_1_stmt) != bb
6118 3700 : || !v_2_stmt || gimple_bb (v_2_stmt) != bb)
6119 : return false;
6120 :
6121 1812 : if (gimple_assign_rhs_code (v_1_stmt) != VEC_PERM_EXPR
6122 1938 : || gimple_assign_rhs_code (v_2_stmt) != VEC_PERM_EXPR)
6123 : return false;
6124 :
6125 : /* Decompose initial VEC_PERM_EXPRs. */
6126 112 : tree v_in = gimple_assign_rhs1 (v_1_stmt);
6127 112 : tree v_1_sel = gimple_assign_rhs3 (v_1_stmt);
6128 112 : tree v_2_sel = gimple_assign_rhs3 (v_2_stmt);
6129 112 : if (v_in != gimple_assign_rhs2 (v_1_stmt)
6130 107 : || v_in != gimple_assign_rhs1 (v_2_stmt)
6131 217 : || v_in != gimple_assign_rhs2 (v_2_stmt))
6132 : return false;
6133 :
6134 105 : unsigned HOST_WIDE_INT v_1_nelts, v_2_nelts;
6135 105 : if (TREE_CODE (v_1_sel) != VECTOR_CST
6136 105 : || !VECTOR_CST_NELTS (v_1_sel).is_constant (&v_1_nelts)
6137 105 : || TREE_CODE (v_2_sel) != VECTOR_CST
6138 210 : || !VECTOR_CST_NELTS (v_2_sel).is_constant (&v_2_nelts))
6139 : return false;
6140 :
6141 105 : if (nelts != v_1_nelts || nelts != v_2_nelts)
6142 : return false;
6143 :
6144 : /* Create the new selector. */
6145 105 : vec_perm_builder new_sel_perm (nelts, nelts, 1);
6146 105 : auto_vec<bool> lanes (nelts);
6147 105 : lanes.quick_grow_cleared (nelts);
6148 630 : for (unsigned int i = 0; i < nelts; i++)
6149 : {
6150 : /* Extract the i-th value from the selector. */
6151 420 : unsigned int sel_cst = TREE_INT_CST_LOW (VECTOR_CST_ELT (sel, i));
6152 420 : unsigned int lane = sel_cst % nelts;
6153 420 : unsigned int offs = sel_cst / nelts;
6154 :
6155 : /* Check what's in the lane. */
6156 420 : unsigned int e_1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (v_1_sel, lane));
6157 420 : unsigned int e_2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (v_2_sel, lane));
6158 :
6159 : /* Reuse previous lane (if any). */
6160 420 : unsigned int l = 0;
6161 711 : for (; l < lane; l++)
6162 : {
6163 489 : if ((TREE_INT_CST_LOW (VECTOR_CST_ELT (v_1_sel, l)) == e_1)
6164 489 : && (TREE_INT_CST_LOW (VECTOR_CST_ELT (v_2_sel, l)) == e_2))
6165 : break;
6166 : }
6167 :
6168 : /* Add to narrowed selector. */
6169 420 : new_sel_perm.quick_push (l + offs * nelts);
6170 :
6171 : /* Mark lane as used. */
6172 420 : lanes[l] = true;
6173 : }
6174 :
6175 : /* Count how many lanes are need. */
6176 : unsigned int cnt = 0;
6177 525 : for (unsigned int i = 0; i < nelts; i++)
6178 420 : cnt += lanes[i];
6179 :
6180 : /* If more than (nelts/2) lanes are needed, skip the sequence. */
6181 105 : if (cnt > nelts / 2)
6182 : return false;
6183 :
6184 : /* Check if the resulting permutation is cheap. */
6185 105 : vec_perm_indices new_indices (new_sel_perm, 2, nelts);
6186 105 : tree vectype = TREE_TYPE (gimple_assign_lhs (stmt));
6187 105 : machine_mode vmode = TYPE_MODE (vectype);
6188 105 : if (!can_vec_perm_const_p (vmode, vmode, new_indices, false))
6189 : return false;
6190 :
6191 105 : *seq = XNEW (struct _vec_perm_simplify_seq);
6192 105 : (*seq)->stmt = stmt;
6193 105 : (*seq)->v_1_stmt = v_1_stmt;
6194 105 : (*seq)->v_2_stmt = v_2_stmt;
6195 105 : (*seq)->v_x_stmt = v_x_stmt;
6196 105 : (*seq)->v_y_stmt = v_y_stmt;
6197 105 : (*seq)->nelts = nelts;
6198 105 : (*seq)->new_sel = vect_gen_perm_mask_checked (vectype, new_indices);
6199 :
6200 105 : if (dump_file)
6201 : {
6202 32 : fprintf (dump_file, "Found vec perm simplify sequence ending with:\n\t");
6203 32 : print_gimple_stmt (dump_file, stmt, 0);
6204 :
6205 32 : if (dump_flags & TDF_DETAILS)
6206 : {
6207 32 : fprintf (dump_file, "\tNarrowed vec_perm selector: ");
6208 32 : print_generic_expr (dump_file, (*seq)->new_sel);
6209 32 : fprintf (dump_file, "\n");
6210 : }
6211 : }
6212 :
6213 : return true;
6214 210 : }
6215 :
6216 : /* Reduce the lane consumption of a simplifiable vec perm sequence. */
6217 :
6218 : static void
6219 78 : narrow_vec_perm_simplify_seq (const vec_perm_simplify_seq &seq)
6220 : {
6221 78 : gassign *stmt = seq->stmt;
6222 78 : if (dump_file && (dump_flags & TDF_DETAILS))
6223 : {
6224 26 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
6225 26 : fprintf (dump_file, "Old stmt: ");
6226 26 : print_gimple_stmt (dump_file, stmt, 0);
6227 : }
6228 :
6229 : /* Update the last VEC_PERM statement. */
6230 78 : gimple_assign_set_rhs3 (stmt, seq->new_sel);
6231 78 : update_stmt (stmt);
6232 :
6233 78 : if (dump_file && (dump_flags & TDF_DETAILS))
6234 : {
6235 26 : fprintf (dump_file, "New stmt: ");
6236 26 : print_gimple_stmt (dump_file, stmt, 0);
6237 : }
6238 78 : }
6239 :
6240 : /* Test if we can blend two simplifiable vec permute sequences.
6241 : NEED_SWAP will be set, if sequences must be swapped for blending. */
6242 :
6243 : static bool
6244 49 : can_blend_vec_perm_simplify_seqs_p (vec_perm_simplify_seq seq1,
6245 : vec_perm_simplify_seq seq2,
6246 : bool *need_swap)
6247 : {
6248 49 : unsigned int nelts = seq1->nelts;
6249 49 : basic_block bb = gimple_bb (seq1->stmt);
6250 :
6251 49 : gcc_assert (gimple_bb (seq2->stmt) == bb);
6252 :
6253 : /* BBs and number of elements must be equal. */
6254 49 : if (gimple_bb (seq2->stmt) != bb || seq2->nelts != nelts)
6255 : return false;
6256 :
6257 : /* We need vectors of the same type. */
6258 49 : if (TREE_TYPE (gimple_assign_lhs (seq1->stmt))
6259 49 : != TREE_TYPE (gimple_assign_lhs (seq2->stmt)))
6260 : return false;
6261 :
6262 : /* We require isomorphic operators. */
6263 43 : if (((gimple_assign_rhs_code (seq1->v_x_stmt)
6264 43 : != gimple_assign_rhs_code (seq2->v_x_stmt))
6265 43 : || (gimple_assign_rhs_code (seq1->v_y_stmt)
6266 43 : != gimple_assign_rhs_code (seq2->v_y_stmt))))
6267 : return false;
6268 :
6269 : /* We cannot have any dependencies between the sequences.
6270 :
6271 : For merging, we will reuse seq1->v_1_stmt and seq1->v_2_stmt.
6272 : seq1's v_in is defined before these statements, but we need
6273 : to check if seq2's v_in is defined before them as well.
6274 :
6275 : Further, we will reuse seq2->stmt. We need to ensure that
6276 : seq1->v_x_stmt and seq1->v_y_stmt are before it.
6277 :
6278 : Note, that we don't need to check the BBs here, because all
6279 : statements of both sequences have to be in the same BB. */
6280 :
6281 43 : tree seq2_v_in = gimple_assign_rhs1 (seq2->v_1_stmt);
6282 43 : if (TREE_CODE (seq2_v_in) != SSA_NAME)
6283 : return false;
6284 :
6285 43 : gassign *seq2_v_in_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (seq2_v_in));
6286 43 : if (!seq2_v_in_stmt || gimple_bb (seq2_v_in_stmt) != bb
6287 43 : || (gimple_uid (seq2_v_in_stmt) > gimple_uid (seq1->v_1_stmt))
6288 39 : || (gimple_uid (seq1->v_x_stmt) > gimple_uid (seq2->stmt))
6289 39 : || (gimple_uid (seq1->v_y_stmt) > gimple_uid (seq2->stmt)))
6290 : {
6291 4 : tree seq1_v_in = gimple_assign_rhs1 (seq1->v_1_stmt);
6292 4 : if (TREE_CODE (seq1_v_in) != SSA_NAME)
6293 : return false;
6294 :
6295 4 : gassign *seq1_v_in_stmt
6296 4 : = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (seq1_v_in));
6297 : /* Let's try to see if we succeed when swapping the sequences. */
6298 4 : if (!seq1_v_in_stmt || gimple_bb (seq1_v_in_stmt)
6299 0 : || (gimple_uid (seq1_v_in_stmt) > gimple_uid (seq2->v_1_stmt))
6300 0 : || (gimple_uid (seq2->v_x_stmt) > gimple_uid (seq1->stmt))
6301 0 : || (gimple_uid (seq2->v_y_stmt) > gimple_uid (seq1->stmt)))
6302 : return false;
6303 0 : *need_swap = true;
6304 : }
6305 : else
6306 39 : *need_swap = false;
6307 :
6308 39 : if (dump_file && (dump_flags & TDF_DETAILS))
6309 13 : fprintf (dump_file, "Found vec perm simplify sequence pair.\n");
6310 :
6311 : return true;
6312 : }
6313 :
6314 : /* Calculate the permutations for blending the two given vec permute
6315 : sequences. This may fail if the resulting permutation is not
6316 : supported. */
6317 :
6318 : static bool
6319 39 : calc_perm_vec_perm_simplify_seqs (vec_perm_simplify_seq seq1,
6320 : vec_perm_simplify_seq seq2,
6321 : vec_perm_indices *seq2_stmt_indices,
6322 : vec_perm_indices *seq1_v_1_stmt_indices,
6323 : vec_perm_indices *seq1_v_2_stmt_indices)
6324 : {
6325 39 : unsigned int i;
6326 39 : unsigned int nelts = seq1->nelts;
6327 39 : auto_vec<unsigned int> lane_assignment;
6328 39 : lane_assignment.create (nelts);
6329 :
6330 : /* Mark all lanes as free. */
6331 39 : lane_assignment.quick_grow_cleared (nelts);
6332 :
6333 : /* Allocate lanes for seq1. */
6334 234 : for (i = 0; i < nelts; i++)
6335 : {
6336 156 : unsigned int l = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq1->new_sel, i));
6337 156 : l %= nelts;
6338 156 : lane_assignment[l] = 1;
6339 : }
6340 :
6341 : /* Allocate lanes for seq2 and calculate selector for seq2->stmt. */
6342 39 : vec_perm_builder seq2_stmt_sel_perm (nelts, nelts, 1);
6343 195 : for (i = 0; i < nelts; i++)
6344 : {
6345 156 : unsigned int sel = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq2->new_sel, i));
6346 156 : unsigned int lane = sel % nelts;
6347 156 : unsigned int offs = sel / nelts;
6348 156 : unsigned int new_sel;
6349 :
6350 : /* Check if we already allocated the lane for seq2. */
6351 156 : unsigned int j = 0;
6352 279 : for (; j < i; j++)
6353 : {
6354 201 : unsigned int sel_old;
6355 201 : sel_old = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq2->new_sel, j));
6356 201 : unsigned int lane_old = sel_old % nelts;
6357 201 : if (lane == lane_old)
6358 : {
6359 78 : new_sel = seq2_stmt_sel_perm[j].to_constant ();
6360 78 : new_sel = (new_sel % nelts) + offs * nelts;
6361 78 : break;
6362 : }
6363 : }
6364 :
6365 : /* If the lane is not allocated, we need to do that now. */
6366 156 : if (j == i)
6367 : {
6368 : unsigned int l_orig = lane;
6369 194 : while (lane_assignment[lane] != 0)
6370 : {
6371 116 : lane = (lane + 1) % nelts;
6372 :
6373 : /* This should not happen if both sequences utilize no more than
6374 : half of the lanes. Test anyway to guarantee termination. */
6375 116 : if (lane == l_orig)
6376 39 : return false;
6377 : }
6378 :
6379 : /* Allocate lane. */
6380 78 : lane_assignment[lane] = 2 + l_orig;
6381 78 : new_sel = lane + offs * nelts;
6382 : }
6383 :
6384 156 : seq2_stmt_sel_perm.quick_push (new_sel);
6385 : }
6386 :
6387 : /* Check if the resulting permutation is cheap. */
6388 39 : seq2_stmt_indices->new_vector (seq2_stmt_sel_perm, 2, nelts);
6389 39 : tree vectype = TREE_TYPE (gimple_assign_lhs (seq2->stmt));
6390 39 : machine_mode vmode = TYPE_MODE (vectype);
6391 39 : if (!can_vec_perm_const_p (vmode, vmode, *seq2_stmt_indices, false))
6392 : return false;
6393 :
6394 : /* Calculate selectors for seq1->v_1_stmt and seq1->v_2_stmt. */
6395 39 : vec_perm_builder seq1_v_1_stmt_sel_perm (nelts, nelts, 1);
6396 39 : vec_perm_builder seq1_v_2_stmt_sel_perm (nelts, nelts, 1);
6397 195 : for (i = 0; i < nelts; i++)
6398 : {
6399 156 : bool use_seq1 = lane_assignment[i] < 2;
6400 156 : unsigned int l1, l2;
6401 :
6402 156 : if (use_seq1)
6403 : {
6404 : /* Just reuse the selector indices. */
6405 78 : tree s1 = gimple_assign_rhs3 (seq1->v_1_stmt);
6406 78 : tree s2 = gimple_assign_rhs3 (seq1->v_2_stmt);
6407 78 : l1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s1, i));
6408 78 : l2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s2, i));
6409 : }
6410 : else
6411 : {
6412 : /* We moved the lanes for seq2, so we need to adjust for that. */
6413 78 : tree s1 = gimple_assign_rhs3 (seq2->v_1_stmt);
6414 78 : tree s2 = gimple_assign_rhs3 (seq2->v_2_stmt);
6415 78 : l1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s1, lane_assignment[i] - 2));
6416 78 : l2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s2, lane_assignment[i] - 2));
6417 : }
6418 :
6419 156 : l1 %= nelts;
6420 156 : l2 %= nelts;
6421 234 : seq1_v_1_stmt_sel_perm.quick_push (l1 + (use_seq1 ? 0 : nelts));
6422 156 : seq1_v_2_stmt_sel_perm.quick_push (l2 + (use_seq1 ? 0 : nelts));
6423 : }
6424 :
6425 39 : seq1_v_1_stmt_indices->new_vector (seq1_v_1_stmt_sel_perm, 2, nelts);
6426 39 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_1_stmt));
6427 39 : vmode = TYPE_MODE (vectype);
6428 39 : if (!can_vec_perm_const_p (vmode, vmode, *seq1_v_1_stmt_indices, false))
6429 : return false;
6430 :
6431 39 : seq1_v_2_stmt_indices->new_vector (seq1_v_2_stmt_sel_perm, 2, nelts);
6432 39 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_2_stmt));
6433 39 : vmode = TYPE_MODE (vectype);
6434 39 : if (!can_vec_perm_const_p (vmode, vmode, *seq1_v_2_stmt_indices, false))
6435 : return false;
6436 :
6437 : return true;
6438 78 : }
6439 :
6440 : /* Blend the two given simplifiable vec permute sequences using the
6441 : given permutations. */
6442 :
6443 : static void
6444 39 : blend_vec_perm_simplify_seqs (vec_perm_simplify_seq seq1,
6445 : vec_perm_simplify_seq seq2,
6446 : const vec_perm_indices &seq2_stmt_indices,
6447 : const vec_perm_indices &seq1_v_1_stmt_indices,
6448 : const vec_perm_indices &seq1_v_2_stmt_indices)
6449 : {
6450 : /* We don't need to adjust seq1->stmt because its lanes consumption
6451 : was already narrowed before entering this function. */
6452 :
6453 : /* Adjust seq2->stmt: copy RHS1/RHS2 from seq1->stmt and set new sel. */
6454 39 : if (dump_file && (dump_flags & TDF_DETAILS))
6455 : {
6456 13 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
6457 13 : fprintf (dump_file, "Old stmt: ");
6458 13 : print_gimple_stmt (dump_file, seq2->stmt, 0);
6459 : }
6460 :
6461 39 : gimple_assign_set_rhs1 (seq2->stmt, gimple_assign_rhs1 (seq1->stmt));
6462 78 : gimple_assign_set_rhs2 (seq2->stmt, gimple_assign_rhs2 (seq1->stmt));
6463 39 : tree vectype = TREE_TYPE (gimple_assign_lhs (seq2->stmt));
6464 39 : tree sel = vect_gen_perm_mask_checked (vectype, seq2_stmt_indices);
6465 39 : gimple_assign_set_rhs3 (seq2->stmt, sel);
6466 39 : update_stmt (seq2->stmt);
6467 :
6468 39 : if (dump_file && (dump_flags & TDF_DETAILS))
6469 : {
6470 13 : fprintf (dump_file, "New stmt: ");
6471 13 : print_gimple_stmt (dump_file, seq2->stmt, 0);
6472 : }
6473 :
6474 : /* Adjust seq1->v_1_stmt: copy RHS2 from seq2->v_1_stmt and set new sel. */
6475 39 : if (dump_file && (dump_flags & TDF_DETAILS))
6476 : {
6477 13 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
6478 13 : fprintf (dump_file, "Old stmt: ");
6479 13 : print_gimple_stmt (dump_file, seq1->v_1_stmt, 0);
6480 : }
6481 :
6482 39 : gimple_assign_set_rhs2 (seq1->v_1_stmt, gimple_assign_rhs1 (seq2->v_1_stmt));
6483 39 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_1_stmt));
6484 39 : sel = vect_gen_perm_mask_checked (vectype, seq1_v_1_stmt_indices);
6485 39 : gimple_assign_set_rhs3 (seq1->v_1_stmt, sel);
6486 39 : update_stmt (seq1->v_1_stmt);
6487 :
6488 39 : if (dump_file && (dump_flags & TDF_DETAILS))
6489 : {
6490 13 : fprintf (dump_file, "New stmt: ");
6491 13 : print_gimple_stmt (dump_file, seq1->v_1_stmt, 0);
6492 : }
6493 :
6494 : /* Adjust seq1->v_2_stmt: copy RHS2 from seq2->v_2_stmt and set new sel. */
6495 39 : if (dump_file && (dump_flags & TDF_DETAILS))
6496 : {
6497 13 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
6498 13 : fprintf (dump_file, "Old stmt: ");
6499 13 : print_gimple_stmt (dump_file, seq1->v_2_stmt, 0);
6500 : }
6501 :
6502 39 : gimple_assign_set_rhs2 (seq1->v_2_stmt, gimple_assign_rhs1 (seq2->v_2_stmt));
6503 39 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_2_stmt));
6504 39 : sel = vect_gen_perm_mask_checked (vectype, seq1_v_2_stmt_indices);
6505 39 : gimple_assign_set_rhs3 (seq1->v_2_stmt, sel);
6506 39 : update_stmt (seq1->v_2_stmt);
6507 :
6508 39 : if (dump_file && (dump_flags & TDF_DETAILS))
6509 : {
6510 13 : fprintf (dump_file, "New stmt: ");
6511 13 : print_gimple_stmt (dump_file, seq1->v_2_stmt, 0);
6512 : }
6513 :
6514 : /* At this point, we have four unmodified seq2 stmts, which will be
6515 : eliminated by DCE. */
6516 :
6517 39 : if (dump_file)
6518 13 : fprintf (dump_file, "Vec perm simplify sequences have been blended.\n\n");
6519 39 : }
6520 :
6521 : /* Try to blend narrowed vec_perm_simplify_seqs pairwise.
6522 : The provided list will be empty after this call. */
6523 :
6524 : static void
6525 139509286 : process_vec_perm_simplify_seq_list (vec<vec_perm_simplify_seq> *l)
6526 : {
6527 139509286 : unsigned int i, j;
6528 139509286 : vec_perm_simplify_seq seq1, seq2;
6529 :
6530 139509286 : if (l->is_empty ())
6531 139509239 : return;
6532 :
6533 47 : if (dump_file && (dump_flags & TDF_DETAILS))
6534 15 : fprintf (dump_file, "\nProcessing %u vec perm simplify sequences.\n",
6535 : l->length ());
6536 :
6537 160 : FOR_EACH_VEC_ELT (*l, i, seq1)
6538 : {
6539 66 : if (i + 1 < l->length ())
6540 : {
6541 119 : FOR_EACH_VEC_ELT_FROM (*l, j, seq2, i + 1)
6542 : {
6543 49 : bool swap = false;
6544 49 : if (can_blend_vec_perm_simplify_seqs_p (seq1, seq2, &swap))
6545 : {
6546 39 : vec_perm_indices seq2_stmt_indices;
6547 39 : vec_perm_indices seq1_v_1_stmt_indices;
6548 39 : vec_perm_indices seq1_v_2_stmt_indices;
6549 117 : if (calc_perm_vec_perm_simplify_seqs (swap ? seq2 : seq1,
6550 : swap ? seq1 : seq2,
6551 : &seq2_stmt_indices,
6552 : &seq1_v_1_stmt_indices,
6553 : &seq1_v_2_stmt_indices))
6554 : {
6555 : /* Narrow lane usage. */
6556 39 : narrow_vec_perm_simplify_seq (seq1);
6557 39 : narrow_vec_perm_simplify_seq (seq2);
6558 :
6559 : /* Blend sequences. */
6560 39 : blend_vec_perm_simplify_seqs (swap ? seq2 : seq1,
6561 : swap ? seq1 : seq2,
6562 : seq2_stmt_indices,
6563 : seq1_v_1_stmt_indices,
6564 : seq1_v_2_stmt_indices);
6565 :
6566 : /* We can use unordered_remove as we break the loop. */
6567 39 : l->unordered_remove (j);
6568 39 : XDELETE (seq2);
6569 39 : break;
6570 : }
6571 39 : }
6572 : }
6573 : }
6574 :
6575 : /* We don't need to call l->remove for seq1. */
6576 66 : XDELETE (seq1);
6577 : }
6578 :
6579 47 : l->truncate (0);
6580 : }
6581 :
6582 : static void
6583 105 : append_vec_perm_simplify_seq_list (vec<vec_perm_simplify_seq> *l,
6584 : const vec_perm_simplify_seq &seq)
6585 : {
6586 : /* If no space on list left, then process the list. */
6587 105 : if (!l->space (1))
6588 0 : process_vec_perm_simplify_seq_list (l);
6589 :
6590 105 : l->quick_push (seq);
6591 105 : }
6592 :
6593 : /* Main entry point for the forward propagation and statement combine
6594 : optimizer. */
6595 :
6596 : namespace {
6597 :
6598 : const pass_data pass_data_forwprop =
6599 : {
6600 : GIMPLE_PASS, /* type */
6601 : "forwprop", /* name */
6602 : OPTGROUP_NONE, /* optinfo_flags */
6603 : TV_TREE_FORWPROP, /* tv_id */
6604 : ( PROP_cfg | PROP_ssa ), /* properties_required */
6605 : 0, /* properties_provided */
6606 : 0, /* properties_destroyed */
6607 : 0, /* todo_flags_start */
6608 : 0, /* todo_flags_finish */
6609 : };
6610 :
6611 : class pass_forwprop : public gimple_opt_pass
6612 : {
6613 : public:
6614 1485900 : pass_forwprop (gcc::context *ctxt)
6615 2971800 : : gimple_opt_pass (pass_data_forwprop, ctxt), last_p (false)
6616 : {}
6617 :
6618 : /* opt_pass methods: */
6619 1188720 : opt_pass * clone () final override { return new pass_forwprop (m_ctxt); }
6620 1783080 : void set_pass_param (unsigned int n, bool param) final override
6621 : {
6622 1783080 : switch (n)
6623 : {
6624 1188720 : case 0:
6625 1188720 : m_full_walk = param;
6626 1188720 : break;
6627 594360 : case 1:
6628 594360 : last_p = param;
6629 594360 : break;
6630 0 : default:
6631 0 : gcc_unreachable();
6632 : }
6633 1783080 : }
6634 5716205 : bool gate (function *) final override { return flag_tree_forwprop; }
6635 : unsigned int execute (function *) final override;
6636 :
6637 : private:
6638 : /* Determines whether the pass instance should set PROP_last_full_fold. */
6639 : bool last_p;
6640 :
6641 : /* True if the aggregate props are doing a full walk or not. */
6642 : bool m_full_walk = false;
6643 : }; // class pass_forwprop
6644 :
6645 : /* Attempt to make the BB block of __builtin_unreachable unreachable by changing
6646 : the incoming jumps. Return true if at least one jump was changed. */
6647 :
6648 : static bool
6649 1236 : optimize_unreachable (basic_block bb)
6650 : {
6651 1236 : gimple_stmt_iterator gsi;
6652 1236 : gimple *stmt;
6653 1236 : edge_iterator ei;
6654 1236 : edge e;
6655 1236 : bool ret;
6656 :
6657 1236 : ret = false;
6658 2554 : FOR_EACH_EDGE (e, ei, bb->preds)
6659 : {
6660 1318 : gsi = gsi_last_bb (e->src);
6661 1318 : if (gsi_end_p (gsi))
6662 327 : continue;
6663 :
6664 991 : stmt = gsi_stmt (gsi);
6665 991 : if (gcond *cond_stmt = dyn_cast <gcond *> (stmt))
6666 : {
6667 : /* If the condition is already true/false
6668 : ignore it. This can happen during copy prop of forwprop. */
6669 699 : if (gimple_cond_true_p (cond_stmt)
6670 691 : || gimple_cond_false_p (cond_stmt))
6671 8 : continue;
6672 683 : else if (e->flags & EDGE_TRUE_VALUE)
6673 599 : gimple_cond_make_false (cond_stmt);
6674 84 : else if (e->flags & EDGE_FALSE_VALUE)
6675 84 : gimple_cond_make_true (cond_stmt);
6676 : else
6677 0 : gcc_unreachable ();
6678 683 : update_stmt (cond_stmt);
6679 : }
6680 : else
6681 : {
6682 : /* Todo: handle other cases. e.g. switch. */
6683 300 : continue;
6684 : }
6685 :
6686 683 : ret = true;
6687 : }
6688 :
6689 1236 : return ret;
6690 : }
6691 :
6692 : unsigned int
6693 5713356 : pass_forwprop::execute (function *fun)
6694 : {
6695 5713356 : unsigned int todoflags = 0;
6696 : /* Handle a full walk only when expensive optimizations are on. */
6697 5713356 : bool full_walk = m_full_walk && flag_expensive_optimizations;
6698 :
6699 5713356 : cfg_changed = false;
6700 5713356 : if (last_p)
6701 1064909 : fun->curr_properties |= PROP_last_full_fold;
6702 :
6703 5713356 : calculate_dominance_info (CDI_DOMINATORS);
6704 :
6705 : /* Combine stmts with the stmts defining their operands. Do that
6706 : in an order that guarantees visiting SSA defs before SSA uses. */
6707 11426712 : lattice.create (num_ssa_names);
6708 11426712 : lattice.quick_grow_cleared (num_ssa_names);
6709 5713356 : int *postorder = XNEWVEC (int, n_basic_blocks_for_fn (fun));
6710 5713356 : int postorder_num = pre_and_rev_post_order_compute_fn (fun, NULL,
6711 : postorder, false);
6712 5713356 : int *bb_to_rpo = XNEWVEC (int, last_basic_block_for_fn (fun));
6713 57482102 : for (int i = 0; i < postorder_num; ++i)
6714 : {
6715 46055390 : bb_to_rpo[postorder[i]] = i;
6716 46055390 : edge_iterator ei;
6717 46055390 : edge e;
6718 110858816 : FOR_EACH_EDGE (e, ei, BASIC_BLOCK_FOR_FN (fun, postorder[i])->succs)
6719 64803426 : e->flags &= ~EDGE_EXECUTABLE;
6720 : }
6721 5713356 : single_succ_edge (BASIC_BLOCK_FOR_FN (fun, ENTRY_BLOCK))->flags
6722 5713356 : |= EDGE_EXECUTABLE;
6723 5713356 : auto_vec<gimple *, 4> to_fixup;
6724 5713356 : auto_vec<gimple *, 32> to_remove;
6725 5713356 : auto_vec<unsigned, 32> to_remove_defs;
6726 5713356 : auto_vec<std::pair<int, int>, 10> edges_to_remove;
6727 5713356 : auto_bitmap simple_dce_worklist;
6728 5713356 : auto_bitmap need_ab_cleanup;
6729 5713356 : to_purge = BITMAP_ALLOC (NULL);
6730 5713356 : auto_vec<vec_perm_simplify_seq, 8> vec_perm_simplify_seq_list;
6731 51768746 : for (int i = 0; i < postorder_num; ++i)
6732 : {
6733 46055390 : gimple_stmt_iterator gsi;
6734 46055390 : basic_block bb = BASIC_BLOCK_FOR_FN (fun, postorder[i]);
6735 46055390 : edge_iterator ei;
6736 46055390 : edge e;
6737 :
6738 : /* Skip processing not executable blocks. We could improve
6739 : single_use tracking by at least unlinking uses from unreachable
6740 : blocks but since blocks with uses are not processed in a
6741 : meaningful order this is probably not worth it. */
6742 46055390 : bool any = false;
6743 47267718 : FOR_EACH_EDGE (e, ei, bb->preds)
6744 : {
6745 47240888 : if ((e->flags & EDGE_EXECUTABLE)
6746 : /* We can handle backedges in natural loops correctly but
6747 : for irreducible regions we have to take all backedges
6748 : conservatively when we did not visit the source yet. */
6749 47240888 : || (bb_to_rpo[e->src->index] > i
6750 727420 : && !dominated_by_p (CDI_DOMINATORS, e->src, e->dest)))
6751 : {
6752 : any = true;
6753 : break;
6754 : }
6755 : }
6756 46055390 : if (!any)
6757 27475 : continue;
6758 :
6759 : /* Remove conditions that go directly to unreachable when this is the last forwprop. */
6760 46028560 : if (last_p
6761 10138628 : && !(flag_sanitize & SANITIZE_UNREACHABLE))
6762 : {
6763 10133656 : gimple_stmt_iterator gsi;
6764 10133656 : gsi = gsi_start_nondebug_after_labels_bb (bb);
6765 10134301 : if (!gsi_end_p (gsi)
6766 9165272 : && gimple_call_builtin_p (*gsi, BUILT_IN_UNREACHABLE)
6767 10134892 : && optimize_unreachable (bb))
6768 : {
6769 645 : cfg_changed = true;
6770 645 : continue;
6771 : }
6772 : }
6773 :
6774 : /* Fold PHI-form long-multiply carries and record degenerate
6775 : PHIs in the lattice. Iterator advanced up front so a folded
6776 : PHI can be removed in-flight; a long-mul carry PHI is never
6777 : degenerate, so the two cases are disjoint. */
6778 62324210 : for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);)
6779 : {
6780 16296295 : gphi *phi = si.phi ();
6781 16296295 : gsi_next (&si);
6782 16296295 : tree res = gimple_phi_result (phi);
6783 32592590 : if (virtual_operand_p (res))
6784 7440055 : continue;
6785 8856254 : if (match_long_mul_phi (phi))
6786 14 : continue;
6787 :
6788 8856240 : tree first = NULL_TREE;
6789 8856240 : bool all_same = true;
6790 8856240 : edge_iterator ei;
6791 8856240 : edge e;
6792 18228692 : FOR_EACH_EDGE (e, ei, bb->preds)
6793 : {
6794 : /* Ignore not executable forward edges. */
6795 17974861 : if (!(e->flags & EDGE_EXECUTABLE))
6796 : {
6797 4182173 : if (bb_to_rpo[e->src->index] < i)
6798 17268 : continue;
6799 : /* Avoid equivalences from backedges - while we might
6800 : be able to make irreducible regions reducible and
6801 : thus turning a back into a forward edge we do not
6802 : want to deal with the intermediate SSA issues that
6803 : exposes. */
6804 : all_same = false;
6805 : }
6806 17957593 : tree use = PHI_ARG_DEF_FROM_EDGE (phi, e);
6807 17957593 : if (use == res)
6808 : /* The PHI result can also appear on a backedge, if so
6809 : we can ignore this case for the purpose of determining
6810 : the singular value. */
6811 : ;
6812 17951683 : else if (! first)
6813 : first = use;
6814 9095443 : else if (! operand_equal_p (first, use, 0))
6815 : {
6816 : all_same = false;
6817 : break;
6818 : }
6819 : }
6820 8856240 : if (all_same)
6821 : {
6822 247961 : if (may_propagate_copy (res, first))
6823 247290 : to_remove_defs.safe_push (SSA_NAME_VERSION (res));
6824 247961 : fwprop_set_lattice_val (res, first);
6825 : }
6826 : }
6827 :
6828 : /* Apply forward propagation to all stmts in the basic-block.
6829 : Note we update GSI within the loop as necessary. */
6830 46027915 : unsigned int uid = 1;
6831 450069200 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); )
6832 : {
6833 358013370 : gimple *stmt = gsi_stmt (gsi);
6834 358013370 : tree lhs, rhs;
6835 358013370 : enum tree_code code;
6836 :
6837 358013370 : if (is_gimple_debug (stmt))
6838 : {
6839 195977768 : gsi_next (&gsi);
6840 195977768 : continue;
6841 : }
6842 :
6843 162035602 : gimple_set_uid (stmt, uid++);
6844 :
6845 162035602 : if (!is_gimple_assign (stmt))
6846 : {
6847 53476210 : process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
6848 53476210 : gsi_next (&gsi);
6849 53476210 : continue;
6850 : }
6851 :
6852 108559392 : lhs = gimple_assign_lhs (stmt);
6853 108559392 : rhs = gimple_assign_rhs1 (stmt);
6854 108559392 : code = gimple_assign_rhs_code (stmt);
6855 :
6856 148564553 : if (TREE_CODE (lhs) != SSA_NAME
6857 108559392 : || has_zero_uses (lhs))
6858 : {
6859 40005161 : process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
6860 40005161 : gsi_next (&gsi);
6861 40005161 : continue;
6862 : }
6863 :
6864 : /* If this statement sets an SSA_NAME to an address,
6865 : try to propagate the address into the uses of the SSA_NAME. */
6866 68554231 : if ((code == ADDR_EXPR
6867 : /* Handle pointer conversions on invariant addresses
6868 : as well, as this is valid gimple. */
6869 66222683 : || (CONVERT_EXPR_CODE_P (code)
6870 9165536 : && TREE_CODE (rhs) == ADDR_EXPR
6871 363787 : && POINTER_TYPE_P (TREE_TYPE (lhs))))
6872 68554455 : && TREE_CODE (TREE_OPERAND (rhs, 0)) != TARGET_MEM_REF)
6873 : {
6874 2331153 : tree base = get_base_address (TREE_OPERAND (rhs, 0));
6875 2331153 : if ((!base
6876 2331153 : || !DECL_P (base)
6877 133137 : || decl_address_invariant_p (base))
6878 2331153 : && !stmt_references_abnormal_ssa_name (stmt)
6879 4662290 : && forward_propagate_addr_expr (lhs, rhs, true))
6880 : {
6881 477056 : fwprop_invalidate_lattice (gimple_get_lhs (stmt));
6882 477056 : release_defs (stmt);
6883 477056 : gsi_remove (&gsi, true);
6884 : }
6885 : else
6886 1854097 : gsi_next (&gsi);
6887 : }
6888 66223078 : else if (code == POINTER_PLUS_EXPR)
6889 : {
6890 3697060 : tree off = gimple_assign_rhs2 (stmt);
6891 3697060 : if (TREE_CODE (off) == INTEGER_CST
6892 1136858 : && can_propagate_from (stmt)
6893 1136507 : && !simple_iv_increment_p (stmt)
6894 : /* ??? Better adjust the interface to that function
6895 : instead of building new trees here. */
6896 4534863 : && forward_propagate_addr_expr
6897 2513409 : (lhs,
6898 : build1_loc (gimple_location (stmt),
6899 837803 : ADDR_EXPR, TREE_TYPE (rhs),
6900 837803 : fold_build2 (MEM_REF,
6901 : TREE_TYPE (TREE_TYPE (rhs)),
6902 : rhs,
6903 : fold_convert (ptr_type_node,
6904 : off))), true))
6905 : {
6906 317822 : fwprop_invalidate_lattice (gimple_get_lhs (stmt));
6907 317822 : release_defs (stmt);
6908 317822 : gsi_remove (&gsi, true);
6909 : }
6910 3379238 : else if (is_gimple_min_invariant (rhs))
6911 : {
6912 : /* Make sure to fold &a[0] + off_1 here. */
6913 416265 : fold_stmt_inplace (&gsi);
6914 416265 : update_stmt (stmt);
6915 416265 : if (gimple_assign_rhs_code (stmt) == POINTER_PLUS_EXPR)
6916 416241 : gsi_next (&gsi);
6917 : }
6918 : else
6919 2962973 : gsi_next (&gsi);
6920 : }
6921 62526018 : else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE
6922 214531 : && gimple_assign_load_p (stmt)
6923 135192 : && !gimple_has_volatile_ops (stmt)
6924 41117 : && TREE_CODE (rhs) != TARGET_MEM_REF
6925 41087 : && TREE_CODE (rhs) != BIT_FIELD_REF
6926 62567101 : && !stmt_can_throw_internal (fun, stmt))
6927 : {
6928 : /* Rewrite loads used only in real/imagpart extractions to
6929 : component-wise loads. */
6930 40965 : use_operand_p use_p;
6931 40965 : imm_use_iterator iter;
6932 40965 : tree vuse = gimple_vuse (stmt);
6933 40965 : bool rewrite = true;
6934 46151 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
6935 : {
6936 44022 : gimple *use_stmt = USE_STMT (use_p);
6937 44022 : if (is_gimple_debug (use_stmt))
6938 1011 : continue;
6939 43011 : if (!is_gimple_assign (use_stmt)
6940 28301 : || (gimple_assign_rhs_code (use_stmt) != REALPART_EXPR
6941 26186 : && gimple_assign_rhs_code (use_stmt) != IMAGPART_EXPR)
6942 4186 : || TREE_OPERAND (gimple_assign_rhs1 (use_stmt), 0) != lhs
6943 : /* We are moving the def, so avoid abnormals. */
6944 47197 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI
6945 : (gimple_assign_lhs (use_stmt)))
6946 : {
6947 : rewrite = false;
6948 : break;
6949 : }
6950 40965 : }
6951 40965 : if (rewrite)
6952 : {
6953 2129 : gimple *use_stmt;
6954 6769 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
6955 : {
6956 4640 : if (is_gimple_debug (use_stmt))
6957 : {
6958 498 : if (gimple_debug_bind_p (use_stmt))
6959 : {
6960 498 : gimple_debug_bind_reset_value (use_stmt);
6961 498 : update_stmt (use_stmt);
6962 : }
6963 498 : continue;
6964 : }
6965 :
6966 8284 : tree new_rhs = build1 (gimple_assign_rhs_code (use_stmt),
6967 4142 : TREE_TYPE (TREE_TYPE (rhs)),
6968 : unshare_expr (rhs));
6969 4142 : gimple *new_stmt
6970 4142 : = gimple_build_assign (gimple_assign_lhs (use_stmt),
6971 : new_rhs);
6972 :
6973 4142 : location_t loc = gimple_location (use_stmt);
6974 4142 : gimple_set_location (new_stmt, loc);
6975 4142 : gimple_set_vuse (new_stmt, vuse);
6976 4142 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
6977 4142 : unlink_stmt_vdef (use_stmt);
6978 4142 : gsi_remove (&gsi2, true);
6979 :
6980 4142 : gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
6981 2129 : }
6982 :
6983 2129 : release_defs (stmt);
6984 2129 : gsi_remove (&gsi, true);
6985 : }
6986 : else
6987 38836 : gsi_next (&gsi);
6988 : }
6989 62485053 : else if (TREE_CODE (TREE_TYPE (lhs)) == VECTOR_TYPE
6990 1816617 : && (TYPE_MODE (TREE_TYPE (lhs)) == BLKmode
6991 : /* After vector lowering rewrite all loads, but
6992 : initially do not since this conflicts with
6993 : vector CONSTRUCTOR to shuffle optimization. */
6994 1788774 : || (fun->curr_properties & PROP_gimple_lvec))
6995 922747 : && gimple_assign_load_p (stmt)
6996 298124 : && !gimple_has_volatile_ops (stmt)
6997 283412 : && !stmt_can_throw_internal (fun, stmt)
6998 62768465 : && (!VAR_P (rhs) || !DECL_HARD_REGISTER (rhs)))
6999 282909 : optimize_vector_load (&gsi);
7000 :
7001 62202144 : else if (code == COMPLEX_EXPR)
7002 : {
7003 : /* Rewrite stores of a single-use complex build expression
7004 : to component-wise stores. */
7005 37954 : use_operand_p use_p;
7006 37954 : gimple *use_stmt, *def1, *def2;
7007 37954 : tree rhs2;
7008 37954 : if (single_imm_use (lhs, &use_p, &use_stmt)
7009 35771 : && gimple_store_p (use_stmt)
7010 42098 : && !gimple_has_volatile_ops (use_stmt)
7011 3133 : && is_gimple_assign (use_stmt)
7012 3129 : && (TREE_CODE (TREE_TYPE (gimple_assign_lhs (use_stmt)))
7013 : == COMPLEX_TYPE)
7014 41078 : && (TREE_CODE (gimple_assign_lhs (use_stmt))
7015 : != TARGET_MEM_REF))
7016 : {
7017 3120 : tree use_lhs = gimple_assign_lhs (use_stmt);
7018 3120 : if (auto_var_p (use_lhs))
7019 601 : DECL_NOT_GIMPLE_REG_P (use_lhs) = 1;
7020 6240 : tree new_lhs = build1 (REALPART_EXPR,
7021 3120 : TREE_TYPE (TREE_TYPE (use_lhs)),
7022 : unshare_expr (use_lhs));
7023 3120 : gimple *new_stmt = gimple_build_assign (new_lhs, rhs);
7024 3120 : location_t loc = gimple_location (use_stmt);
7025 3120 : gimple_set_location (new_stmt, loc);
7026 6240 : gimple_set_vuse (new_stmt, gimple_vuse (use_stmt));
7027 3120 : gimple_set_vdef (new_stmt, make_ssa_name (gimple_vop (fun)));
7028 6240 : SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
7029 6240 : gimple_set_vuse (use_stmt, gimple_vdef (new_stmt));
7030 3120 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
7031 3120 : gsi_insert_before (&gsi2, new_stmt, GSI_SAME_STMT);
7032 :
7033 6240 : new_lhs = build1 (IMAGPART_EXPR,
7034 3120 : TREE_TYPE (TREE_TYPE (use_lhs)),
7035 : unshare_expr (use_lhs));
7036 3120 : gimple_assign_set_lhs (use_stmt, new_lhs);
7037 3120 : gimple_assign_set_rhs1 (use_stmt, gimple_assign_rhs2 (stmt));
7038 3120 : update_stmt (use_stmt);
7039 :
7040 3120 : release_defs (stmt);
7041 3120 : gsi_remove (&gsi, true);
7042 : }
7043 : /* Rewrite a component-wise load of a complex to a complex
7044 : load if the components are not used separately. */
7045 34834 : else if (TREE_CODE (rhs) == SSA_NAME
7046 34368 : && has_single_use (rhs)
7047 30818 : && ((rhs2 = gimple_assign_rhs2 (stmt)), true)
7048 30818 : && TREE_CODE (rhs2) == SSA_NAME
7049 28997 : && has_single_use (rhs2)
7050 28577 : && (def1 = SSA_NAME_DEF_STMT (rhs),
7051 28577 : gimple_assign_load_p (def1))
7052 1093 : && (def2 = SSA_NAME_DEF_STMT (rhs2),
7053 1093 : gimple_assign_load_p (def2))
7054 1598 : && (gimple_vuse (def1) == gimple_vuse (def2))
7055 796 : && !gimple_has_volatile_ops (def1)
7056 796 : && !gimple_has_volatile_ops (def2)
7057 796 : && !stmt_can_throw_internal (fun, def1)
7058 796 : && !stmt_can_throw_internal (fun, def2)
7059 796 : && gimple_assign_rhs_code (def1) == REALPART_EXPR
7060 550 : && gimple_assign_rhs_code (def2) == IMAGPART_EXPR
7061 35384 : && operand_equal_p (TREE_OPERAND (gimple_assign_rhs1
7062 : (def1), 0),
7063 550 : TREE_OPERAND (gimple_assign_rhs1
7064 : (def2), 0)))
7065 : {
7066 550 : tree cl = TREE_OPERAND (gimple_assign_rhs1 (def1), 0);
7067 550 : gimple_assign_set_rhs_from_tree (&gsi, unshare_expr (cl));
7068 550 : gcc_assert (gsi_stmt (gsi) == stmt);
7069 1100 : gimple_set_vuse (stmt, gimple_vuse (def1));
7070 550 : gimple_set_modified (stmt, true);
7071 550 : gimple_stmt_iterator gsi2 = gsi_for_stmt (def1);
7072 550 : gsi_remove (&gsi, false);
7073 550 : gsi_insert_after (&gsi2, stmt, GSI_SAME_STMT);
7074 : }
7075 : else
7076 34284 : gsi_next (&gsi);
7077 : }
7078 62164190 : else if (code == CONSTRUCTOR
7079 170514 : && VECTOR_TYPE_P (TREE_TYPE (rhs))
7080 170514 : && TYPE_MODE (TREE_TYPE (rhs)) == BLKmode
7081 4749 : && CONSTRUCTOR_NELTS (rhs) > 0
7082 62168939 : && (!VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
7083 2104 : || (TYPE_MODE (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
7084 : != BLKmode)))
7085 : {
7086 : /* Rewrite stores of a single-use vector constructors
7087 : to component-wise stores if the mode isn't supported. */
7088 4392 : use_operand_p use_p;
7089 4392 : gimple *use_stmt;
7090 4392 : if (single_imm_use (lhs, &use_p, &use_stmt)
7091 3928 : && gimple_store_p (use_stmt)
7092 3428 : && !gimple_has_volatile_ops (use_stmt)
7093 1708 : && !stmt_can_throw_internal (fun, use_stmt)
7094 6093 : && is_gimple_assign (use_stmt))
7095 : {
7096 1701 : tree elt_t = TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value);
7097 1701 : unsigned HOST_WIDE_INT elt_w
7098 1701 : = tree_to_uhwi (TYPE_SIZE (elt_t));
7099 1701 : unsigned HOST_WIDE_INT n
7100 1701 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs)));
7101 1701 : tree use_lhs = gimple_assign_lhs (use_stmt);
7102 1701 : if (auto_var_p (use_lhs))
7103 575 : DECL_NOT_GIMPLE_REG_P (use_lhs) = 1;
7104 1126 : else if (TREE_CODE (use_lhs) == TARGET_MEM_REF)
7105 : {
7106 1 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
7107 1 : use_lhs = prepare_target_mem_ref_lvalue (use_lhs, &gsi2);
7108 : }
7109 43684 : for (unsigned HOST_WIDE_INT bi = 0; bi < n; bi += elt_w)
7110 : {
7111 41983 : unsigned HOST_WIDE_INT ci = bi / elt_w;
7112 41983 : tree new_rhs;
7113 41983 : if (ci < CONSTRUCTOR_NELTS (rhs))
7114 41365 : new_rhs = CONSTRUCTOR_ELT (rhs, ci)->value;
7115 : else
7116 618 : new_rhs = build_zero_cst (elt_t);
7117 41983 : tree new_lhs = build3 (BIT_FIELD_REF,
7118 : elt_t,
7119 : unshare_expr (use_lhs),
7120 41983 : bitsize_int (elt_w),
7121 41983 : bitsize_int (bi));
7122 41983 : gimple *new_stmt = gimple_build_assign (new_lhs, new_rhs);
7123 41983 : location_t loc = gimple_location (use_stmt);
7124 41983 : gimple_set_location (new_stmt, loc);
7125 83966 : gimple_set_vuse (new_stmt, gimple_vuse (use_stmt));
7126 41983 : gimple_set_vdef (new_stmt,
7127 : make_ssa_name (gimple_vop (fun)));
7128 83966 : SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
7129 83966 : gimple_set_vuse (use_stmt, gimple_vdef (new_stmt));
7130 41983 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
7131 41983 : gsi_insert_before (&gsi2, new_stmt, GSI_SAME_STMT);
7132 : }
7133 1701 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
7134 1701 : unlink_stmt_vdef (use_stmt);
7135 1701 : release_defs (use_stmt);
7136 1701 : gsi_remove (&gsi2, true);
7137 1701 : release_defs (stmt);
7138 1701 : gsi_remove (&gsi, true);
7139 : }
7140 : else
7141 2691 : gsi_next (&gsi);
7142 : }
7143 62159798 : else if (code == VEC_PERM_EXPR)
7144 : {
7145 : /* Find vectorized sequences, where we can reduce the lane
7146 : utilization. The narrowing will be donw later and only
7147 : if we find a pair of sequences that can be blended. */
7148 190450 : gassign *assign = dyn_cast <gassign *> (stmt);
7149 190450 : vec_perm_simplify_seq seq;
7150 190450 : if (recognise_vec_perm_simplify_seq (assign, &seq))
7151 105 : append_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list,
7152 : seq);
7153 :
7154 190450 : gsi_next (&gsi);
7155 : }
7156 : else
7157 61969348 : gsi_next (&gsi);
7158 : }
7159 :
7160 46027915 : process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
7161 :
7162 : /* Combine stmts with the stmts defining their operands.
7163 : Note we update GSI within the loop as necessary. */
7164 449740972 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7165 : {
7166 357685142 : gimple *stmt = gsi_stmt (gsi);
7167 :
7168 : /* Mark stmt as potentially needing revisiting. */
7169 357685142 : gimple_set_plf (stmt, GF_PLF_1, false);
7170 :
7171 357685142 : bool can_make_abnormal_goto = (is_gimple_call (stmt)
7172 357685142 : && stmt_can_make_abnormal_goto (stmt));
7173 :
7174 : /* Substitute from our lattice. We need to do so only once. */
7175 357685142 : bool substituted_p = false;
7176 357685142 : use_operand_p usep;
7177 357685142 : ssa_op_iter iter;
7178 524308322 : FOR_EACH_SSA_USE_OPERAND (usep, stmt, iter, SSA_OP_USE)
7179 : {
7180 166623180 : tree use = USE_FROM_PTR (usep);
7181 166623180 : tree val = fwprop_ssa_val (use);
7182 166623180 : if (val && val != use)
7183 : {
7184 1971858 : if (!is_gimple_debug (stmt))
7185 1637733 : bitmap_set_bit (simple_dce_worklist, SSA_NAME_VERSION (use));
7186 1971858 : if (may_propagate_copy (use, val))
7187 : {
7188 1968448 : propagate_value (usep, val);
7189 1968448 : substituted_p = true;
7190 : }
7191 : }
7192 : }
7193 357685142 : if (substituted_p)
7194 1911946 : update_stmt (stmt);
7195 1911946 : if (substituted_p
7196 1911946 : && is_gimple_assign (stmt)
7197 1159521 : && gimple_assign_rhs_code (stmt) == ADDR_EXPR)
7198 21058 : recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
7199 357685142 : if (substituted_p
7200 357685142 : && can_make_abnormal_goto
7201 357685142 : && !stmt_can_make_abnormal_goto (stmt))
7202 3 : bitmap_set_bit (need_ab_cleanup, bb->index);
7203 :
7204 360612202 : bool changed;
7205 721224404 : do
7206 : {
7207 360612202 : gimple *orig_stmt = stmt = gsi_stmt (gsi);
7208 360612202 : bool was_call = is_gimple_call (stmt);
7209 360612202 : bool was_noreturn = (was_call
7210 360612202 : && gimple_call_noreturn_p (stmt));
7211 360612202 : changed = false;
7212 :
7213 360612202 : auto_vec<tree, 8> uses;
7214 530272370 : FOR_EACH_SSA_USE_OPERAND (usep, stmt, iter, SSA_OP_USE)
7215 338925978 : if (uses.space (1))
7216 169265810 : uses.quick_push (USE_FROM_PTR (usep));
7217 :
7218 360612202 : if (fold_stmt (&gsi, fwprop_ssa_val, simple_dce_worklist))
7219 : {
7220 2612725 : changed = true;
7221 : /* There is no updating of the address
7222 : taken after the last forwprop so update
7223 : the addresses when a folding happened to a call.
7224 : The va_* builtins can remove taking of the address so
7225 : can the sincos->cexpi transformation. See PR 39643 and PR 20983. */
7226 2612725 : if (was_call && last_p)
7227 2612725 : todoflags |= TODO_update_address_taken;
7228 2612725 : stmt = gsi_stmt (gsi);
7229 : /* Cleanup the CFG if we simplified a condition to
7230 : true or false. */
7231 2612725 : if (gcond *cond = dyn_cast <gcond *> (stmt))
7232 1048216 : if (gimple_cond_true_p (cond)
7233 1048216 : || gimple_cond_false_p (cond))
7234 30895 : cfg_changed = true;
7235 : /* Queue old uses for simple DCE if not debug statement. */
7236 2612725 : if (!is_gimple_debug (stmt))
7237 11009465 : for (tree use : uses)
7238 3192929 : if (TREE_CODE (use) == SSA_NAME
7239 3192929 : && !SSA_NAME_IS_DEFAULT_DEF (use))
7240 2994498 : bitmap_set_bit (simple_dce_worklist,
7241 2994498 : SSA_NAME_VERSION (use));
7242 2612725 : update_stmt (stmt);
7243 : }
7244 :
7245 360612202 : switch (gimple_code (stmt))
7246 : {
7247 109632307 : case GIMPLE_ASSIGN:
7248 109632307 : {
7249 109632307 : tree rhs1 = gimple_assign_rhs1 (stmt);
7250 109632307 : enum tree_code code = gimple_assign_rhs_code (stmt);
7251 109632307 : if (gimple_clobber_p (stmt))
7252 7511610 : do_simple_agr_dse (as_a<gassign*>(stmt), full_walk);
7253 102120697 : else if (gimple_store_p (stmt))
7254 : {
7255 31862179 : optimize_aggr_zeroprop (stmt, full_walk);
7256 31862179 : if (gimple_assign_load_p (stmt))
7257 3925834 : optimize_agr_copyprop (stmt);
7258 : }
7259 70258518 : else if (TREE_CODE_CLASS (code) == tcc_comparison)
7260 2640354 : changed |= forward_propagate_into_comparison (&gsi);
7261 67618164 : else if ((code == PLUS_EXPR || code == BIT_IOR_EXPR))
7262 : {
7263 10235596 : bool folded = match_long_mul (as_a <gassign *> (stmt));
7264 10235596 : if (!folded)
7265 10233650 : folded = simplify_rotate (&gsi);
7266 10235596 : changed |= folded;
7267 : }
7268 : else if (code == BIT_XOR_EXPR)
7269 149382 : changed |= simplify_rotate (&gsi);
7270 : else if (code == VEC_PERM_EXPR)
7271 193226 : changed |= simplify_permutation (&gsi);
7272 : else if (code == CONSTRUCTOR
7273 168113 : && TREE_CODE (TREE_TYPE (rhs1)) == VECTOR_TYPE)
7274 168113 : changed |= simplify_vector_constructor (&gsi);
7275 56871847 : else if (code == ARRAY_REF)
7276 2020771 : changed |= simplify_count_zeroes (&gsi);
7277 54851076 : else if (get_gimple_rhs_class (code) == GIMPLE_BINARY_RHS
7278 12694182 : && TREE_CODE (
7279 : gimple_assign_rhs2 (stmt)) == SSA_NAME
7280 6723490 : && has_single_use (gimple_assign_rhs2 (stmt))
7281 58989305 : && is_a<gphi*> (SSA_NAME_DEF_STMT (
7282 : gimple_assign_rhs2 (stmt))))
7283 134686 : changed |= simplify_phi_result_movdiv (stmt, code);
7284 : break;
7285 : }
7286 :
7287 111600 : case GIMPLE_SWITCH:
7288 111600 : changed |= simplify_gimple_switch (as_a <gswitch *> (stmt),
7289 : edges_to_remove,
7290 : simple_dce_worklist);
7291 111600 : break;
7292 :
7293 19792533 : case GIMPLE_COND:
7294 19792533 : {
7295 19792533 : int did_something = forward_propagate_into_gimple_cond
7296 19792533 : (as_a <gcond *> (stmt));
7297 19792533 : if (did_something == 2)
7298 1618 : cfg_changed = true;
7299 19792533 : changed |= did_something != 0;
7300 19792533 : break;
7301 : }
7302 :
7303 23968931 : case GIMPLE_CALL:
7304 23968931 : {
7305 23968931 : tree callee = gimple_call_fndecl (stmt);
7306 23968931 : if (callee != NULL_TREE
7307 23968931 : && fndecl_built_in_p (callee, BUILT_IN_NORMAL))
7308 6319213 : changed |= simplify_builtin_call (&gsi, callee, full_walk);
7309 : break;
7310 : }
7311 :
7312 360612202 : default:;
7313 : }
7314 :
7315 360612202 : if (changed || substituted_p)
7316 : {
7317 4267822 : substituted_p = false;
7318 4267822 : stmt = gsi_stmt (gsi);
7319 4267822 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
7320 70 : bitmap_set_bit (to_purge, bb->index);
7321 4267822 : if (!was_noreturn
7322 4267822 : && is_gimple_call (stmt) && gimple_call_noreturn_p (stmt))
7323 12 : to_fixup.safe_push (stmt);
7324 : }
7325 4267822 : if (changed)
7326 : {
7327 : /* If the stmt changed then re-visit it and the statements
7328 : inserted before it. */
7329 6301255 : for (; !gsi_end_p (gsi); gsi_prev (&gsi))
7330 5834843 : if (gimple_plf (gsi_stmt (gsi), GF_PLF_1))
7331 : break;
7332 2927060 : if (gsi_end_p (gsi))
7333 466412 : gsi = gsi_start_bb (bb);
7334 : else
7335 2693854 : gsi_next (&gsi);
7336 : }
7337 360612202 : }
7338 : while (changed);
7339 :
7340 : /* Stmt no longer needs to be revisited. */
7341 357685142 : stmt = gsi_stmt (gsi);
7342 357685142 : gcc_checking_assert (!gimple_plf (stmt, GF_PLF_1));
7343 357685142 : gimple_set_plf (stmt, GF_PLF_1, true);
7344 :
7345 : /* Fill up the lattice. */
7346 357685142 : if (gimple_assign_single_p (stmt))
7347 : {
7348 72639799 : tree lhs = gimple_assign_lhs (stmt);
7349 72639799 : tree rhs = gimple_assign_rhs1 (stmt);
7350 72639799 : if (TREE_CODE (lhs) == SSA_NAME)
7351 : {
7352 33279013 : tree val = lhs;
7353 33279013 : if (TREE_CODE (rhs) == SSA_NAME)
7354 785302 : val = fwprop_ssa_val (rhs);
7355 32493711 : else if (is_gimple_min_invariant (rhs))
7356 472932 : val = rhs;
7357 : /* If we can propagate the lattice-value mark the
7358 : stmt for removal. */
7359 33279013 : if (val != lhs
7360 33279013 : && may_propagate_copy (lhs, val))
7361 1254522 : to_remove_defs.safe_push (SSA_NAME_VERSION (lhs));
7362 33279013 : fwprop_set_lattice_val (lhs, val);
7363 : }
7364 : }
7365 285045343 : else if (gimple_nop_p (stmt))
7366 102685 : to_remove.safe_push (stmt);
7367 : }
7368 :
7369 : /* Substitute in destination PHI arguments. */
7370 110816285 : FOR_EACH_EDGE (e, ei, bb->succs)
7371 64788370 : for (gphi_iterator gsi = gsi_start_phis (e->dest);
7372 107616178 : !gsi_end_p (gsi); gsi_next (&gsi))
7373 : {
7374 42827808 : gphi *phi = gsi.phi ();
7375 42827808 : use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
7376 42827808 : tree arg = USE_FROM_PTR (use_p);
7377 70656276 : if (TREE_CODE (arg) != SSA_NAME
7378 42827808 : || virtual_operand_p (arg))
7379 27828468 : continue;
7380 14999340 : tree val = fwprop_ssa_val (arg);
7381 14999340 : if (val != arg
7382 14999340 : && may_propagate_copy (arg, val, !(e->flags & EDGE_ABNORMAL)))
7383 259609 : propagate_value (use_p, val);
7384 : }
7385 :
7386 : /* Mark outgoing executable edges. */
7387 46027915 : if (edge e = find_taken_edge (bb, NULL))
7388 : {
7389 19641894 : e->flags |= EDGE_EXECUTABLE;
7390 46064732 : if (EDGE_COUNT (bb->succs) > 1)
7391 36817 : cfg_changed = true;
7392 : }
7393 : else
7394 : {
7395 71495679 : FOR_EACH_EDGE (e, ei, bb->succs)
7396 45109658 : e->flags |= EDGE_EXECUTABLE;
7397 : }
7398 : }
7399 5713356 : free (postorder);
7400 5713356 : free (bb_to_rpo);
7401 5713356 : lattice.release ();
7402 :
7403 : /* First remove chains of stmts where we check no uses remain. */
7404 5713356 : simple_dce_from_worklist (simple_dce_worklist, to_purge);
7405 :
7406 6078383 : auto remove = [](gimple *stmt)
7407 : {
7408 365027 : if (dump_file && (dump_flags & TDF_DETAILS))
7409 : {
7410 1 : fprintf (dump_file, "Removing dead stmt ");
7411 1 : print_gimple_stmt (dump_file, stmt, 0);
7412 1 : fprintf (dump_file, "\n");
7413 : }
7414 365027 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
7415 365027 : if (gimple_code (stmt) == GIMPLE_PHI)
7416 95299 : remove_phi_node (&gsi, true);
7417 : else
7418 : {
7419 269728 : unlink_stmt_vdef (stmt);
7420 269728 : gsi_remove (&gsi, true);
7421 269728 : release_defs (stmt);
7422 : }
7423 365027 : };
7424 :
7425 : /* Then remove stmts we know we can remove even though we did not
7426 : substitute in dead code regions, so uses can remain. Do so in reverse
7427 : order to make debug stmt creation possible. */
7428 12928524 : while (!to_remove_defs.is_empty())
7429 : {
7430 1501812 : tree def = ssa_name (to_remove_defs.pop ());
7431 : /* For example remove_prop_source_from_use can remove stmts queued
7432 : for removal. Deal with this gracefully. */
7433 1501812 : if (!def)
7434 1239470 : continue;
7435 262342 : gimple *stmt = SSA_NAME_DEF_STMT (def);
7436 262342 : remove (stmt);
7437 : }
7438 :
7439 : /* Wipe other queued stmts that do not have SSA defs. */
7440 5816041 : while (!to_remove.is_empty())
7441 : {
7442 102685 : gimple *stmt = to_remove.pop ();
7443 102685 : remove (stmt);
7444 : }
7445 :
7446 : /* Fixup stmts that became noreturn calls. This may require splitting
7447 : blocks and thus isn't possible during the walk. Do this
7448 : in reverse order so we don't inadvertently remove a stmt we want to
7449 : fixup by visiting a dominating now noreturn call first. */
7450 5713368 : while (!to_fixup.is_empty ())
7451 : {
7452 12 : gimple *stmt = to_fixup.pop ();
7453 12 : if (dump_file && dump_flags & TDF_DETAILS)
7454 : {
7455 0 : fprintf (dump_file, "Fixing up noreturn call ");
7456 0 : print_gimple_stmt (dump_file, stmt, 0);
7457 0 : fprintf (dump_file, "\n");
7458 : }
7459 12 : cfg_changed |= fixup_noreturn_call (stmt);
7460 : }
7461 :
7462 5713356 : cfg_changed |= gimple_purge_all_dead_eh_edges (to_purge);
7463 5713356 : cfg_changed |= gimple_purge_all_dead_abnormal_call_edges (need_ab_cleanup);
7464 5713356 : BITMAP_FREE (to_purge);
7465 :
7466 : /* Remove edges queued from switch stmt simplification. */
7467 17140068 : for (auto ep : edges_to_remove)
7468 : {
7469 0 : basic_block src = BASIC_BLOCK_FOR_FN (fun, ep.first);
7470 0 : basic_block dest = BASIC_BLOCK_FOR_FN (fun, ep.second);
7471 0 : edge e;
7472 0 : if (src && dest && (e = find_edge (src, dest)))
7473 : {
7474 0 : free_dominance_info (CDI_DOMINATORS);
7475 0 : remove_edge (e);
7476 0 : cfg_changed = true;
7477 : }
7478 : }
7479 :
7480 5713359 : if (get_range_query (fun) != get_global_range_query ())
7481 1616 : disable_ranger (fun);
7482 :
7483 5713356 : if (cfg_changed)
7484 13063 : todoflags |= TODO_cleanup_cfg;
7485 :
7486 5713356 : return todoflags;
7487 5713356 : }
7488 :
7489 : } // anon namespace
7490 :
7491 : gimple_opt_pass *
7492 297180 : make_pass_forwprop (gcc::context *ctxt)
7493 : {
7494 297180 : return new pass_forwprop (ctxt);
7495 : }
|