Line data Source code
1 : /* Forward propagation of expressions for single use variables.
2 : Copyright (C) 2004-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify
7 : it under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful,
12 : but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : GNU General Public License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "rtl.h"
25 : #include "tree.h"
26 : #include "gimple.h"
27 : #include "cfghooks.h"
28 : #include "tree-pass.h"
29 : #include "ssa.h"
30 : #include "expmed.h"
31 : #include "optabs-query.h"
32 : #include "gimple-pretty-print.h"
33 : #include "fold-const.h"
34 : #include "stor-layout.h"
35 : #include "gimple-iterator.h"
36 : #include "gimple-fold.h"
37 : #include "tree-eh.h"
38 : #include "gimplify.h"
39 : #include "gimplify-me.h"
40 : #include "tree-cfg.h"
41 : #include "expr.h"
42 : #include "tree-dfa.h"
43 : #include "tree-ssa-propagate.h"
44 : #include "tree-ssa-dom.h"
45 : #include "tree-ssa-strlen.h"
46 : #include "builtins.h"
47 : #include "tree-cfgcleanup.h"
48 : #include "cfganal.h"
49 : #include "optabs-tree.h"
50 : #include "insn-config.h"
51 : #include "recog.h"
52 : #include "cfgloop.h"
53 : #include "tree-vectorizer.h"
54 : #include "tree-vector-builder.h"
55 : #include "vec-perm-indices.h"
56 : #include "internal-fn.h"
57 : #include "cgraph.h"
58 : #include "tree-ssa.h"
59 : #include "gimple-range.h"
60 : #include "tree-ssa-dce.h"
61 :
62 : /* This pass propagates the RHS of assignment statements into use
63 : sites of the LHS of the assignment. It's basically a specialized
64 : form of tree combination. It is hoped all of this can disappear
65 : when we have a generalized tree combiner.
66 :
67 : One class of common cases we handle is forward propagating a single use
68 : variable into a COND_EXPR.
69 :
70 : bb0:
71 : x = a COND b;
72 : if (x) goto ... else goto ...
73 :
74 : Will be transformed into:
75 :
76 : bb0:
77 : if (a COND b) goto ... else goto ...
78 :
79 : Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
80 :
81 : Or (assuming c1 and c2 are constants):
82 :
83 : bb0:
84 : x = a + c1;
85 : if (x EQ/NEQ c2) goto ... else goto ...
86 :
87 : Will be transformed into:
88 :
89 : bb0:
90 : if (a EQ/NEQ (c2 - c1)) goto ... else goto ...
91 :
92 : Similarly for x = a - c1.
93 :
94 : Or
95 :
96 : bb0:
97 : x = !a
98 : if (x) goto ... else goto ...
99 :
100 : Will be transformed into:
101 :
102 : bb0:
103 : if (a == 0) goto ... else goto ...
104 :
105 : Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
106 : For these cases, we propagate A into all, possibly more than one,
107 : COND_EXPRs that use X.
108 :
109 : Or
110 :
111 : bb0:
112 : x = (typecast) a
113 : if (x) goto ... else goto ...
114 :
115 : Will be transformed into:
116 :
117 : bb0:
118 : if (a != 0) goto ... else goto ...
119 :
120 : (Assuming a is an integral type and x is a boolean or x is an
121 : integral and a is a boolean.)
122 :
123 : Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
124 : For these cases, we propagate A into all, possibly more than one,
125 : COND_EXPRs that use X.
126 :
127 : In addition to eliminating the variable and the statement which assigns
128 : a value to the variable, we may be able to later thread the jump without
129 : adding insane complexity in the dominator optimizer.
130 :
131 : Also note these transformations can cascade. We handle this by having
132 : a worklist of COND_EXPR statements to examine. As we make a change to
133 : a statement, we put it back on the worklist to examine on the next
134 : iteration of the main loop.
135 :
136 : A second class of propagation opportunities arises for ADDR_EXPR
137 : nodes.
138 :
139 : ptr = &x->y->z;
140 : res = *ptr;
141 :
142 : Will get turned into
143 :
144 : res = x->y->z;
145 :
146 : Or
147 : ptr = (type1*)&type2var;
148 : res = *ptr
149 :
150 : Will get turned into (if type1 and type2 are the same size
151 : and neither have volatile on them):
152 : res = VIEW_CONVERT_EXPR<type1>(type2var)
153 :
154 : Or
155 :
156 : ptr = &x[0];
157 : ptr2 = ptr + <constant>;
158 :
159 : Will get turned into
160 :
161 : ptr2 = &x[constant/elementsize];
162 :
163 : Or
164 :
165 : ptr = &x[0];
166 : offset = index * element_size;
167 : offset_p = (pointer) offset;
168 : ptr2 = ptr + offset_p
169 :
170 : Will get turned into:
171 :
172 : ptr2 = &x[index];
173 :
174 : Or
175 : ssa = (int) decl
176 : res = ssa & 1
177 :
178 : Provided that decl has known alignment >= 2, will get turned into
179 :
180 : res = 0
181 :
182 : We also propagate casts into SWITCH_EXPR and COND_EXPR conditions to
183 : allow us to remove the cast and {NOT_EXPR,NEG_EXPR} into a subsequent
184 : {NOT_EXPR,NEG_EXPR}.
185 :
186 : This will (of course) be extended as other needs arise. */
187 :
188 : /* Data structure that contains simplifiable vectorized permute sequences.
189 : See recognise_vec_perm_simplify_seq () for a description of the sequence. */
190 :
191 : struct _vec_perm_simplify_seq
192 : {
193 : /* Defining stmts of vectors in the sequence. */
194 : gassign *v_1_stmt;
195 : gassign *v_2_stmt;
196 : gassign *v_x_stmt;
197 : gassign *v_y_stmt;
198 : /* Final permute statement. */
199 : gassign *stmt;
200 : /* New selector indices for stmt. */
201 : tree new_sel;
202 : /* Elements of each vector and selector. */
203 : unsigned int nelts;
204 : };
205 : typedef struct _vec_perm_simplify_seq *vec_perm_simplify_seq;
206 :
207 : static bool forward_propagate_addr_expr (tree, tree, bool);
208 :
209 : /* Set to true if we delete dead edges during the optimization. */
210 : static bool cfg_changed;
211 :
212 : static tree rhs_to_tree (tree type, gimple *stmt);
213 :
214 : static bitmap to_purge;
215 :
216 : /* Const-and-copy lattice. */
217 : static vec<tree> lattice;
218 :
219 : /* Set the lattice entry for NAME to VAL. */
220 : static void
221 32907714 : fwprop_set_lattice_val (tree name, tree val)
222 : {
223 32907714 : if (TREE_CODE (name) == SSA_NAME)
224 : {
225 32907714 : if (SSA_NAME_VERSION (name) >= lattice.length ())
226 : {
227 32211 : lattice.reserve (num_ssa_names - lattice.length ());
228 21474 : lattice.quick_grow_cleared (num_ssa_names);
229 : }
230 32907714 : lattice[SSA_NAME_VERSION (name)] = val;
231 : /* As this now constitutes a copy duplicate points-to
232 : and range info appropriately. */
233 32907714 : if (TREE_CODE (val) == SSA_NAME)
234 32457635 : maybe_duplicate_ssa_info_at_copy (name, val);
235 : }
236 32907714 : }
237 :
238 : /* Invalidate the lattice entry for NAME, done when releasing SSA names. */
239 : static void
240 928932 : fwprop_invalidate_lattice (tree name)
241 : {
242 928932 : if (name
243 926428 : && TREE_CODE (name) == SSA_NAME
244 1855232 : && SSA_NAME_VERSION (name) < lattice.length ())
245 926267 : lattice[SSA_NAME_VERSION (name)] = NULL_TREE;
246 928932 : }
247 :
248 : /* Get the statement we can propagate from into NAME skipping
249 : trivial copies. Returns the statement which defines the
250 : propagation source or NULL_TREE if there is no such one.
251 : If SINGLE_USE_ONLY is set considers only sources which have
252 : a single use chain up to NAME. If SINGLE_USE_P is non-null,
253 : it is set to whether the chain to NAME is a single use chain
254 : or not. SINGLE_USE_P is not written to if SINGLE_USE_ONLY is set. */
255 :
256 : static gimple *
257 28204945 : get_prop_source_stmt (tree name, bool single_use_only, bool *single_use_p)
258 : {
259 28204945 : bool single_use = true;
260 :
261 28205933 : do {
262 28205439 : gimple *def_stmt = SSA_NAME_DEF_STMT (name);
263 :
264 28205439 : if (!has_single_use (name))
265 : {
266 15372510 : single_use = false;
267 15372510 : if (single_use_only)
268 : return NULL;
269 : }
270 :
271 : /* If name is defined by a PHI node or is the default def, bail out. */
272 28203956 : if (!is_gimple_assign (def_stmt))
273 : return NULL;
274 :
275 : /* If def_stmt is a simple copy, continue looking. */
276 19899796 : if (gimple_assign_rhs_code (def_stmt) == SSA_NAME)
277 494 : name = gimple_assign_rhs1 (def_stmt);
278 : else
279 : {
280 19899302 : if (!single_use_only && single_use_p)
281 19574734 : *single_use_p = single_use;
282 :
283 19899302 : return def_stmt;
284 : }
285 494 : } while (1);
286 : }
287 :
288 : /* Checks if the destination ssa name in DEF_STMT can be used as
289 : propagation source. Returns true if so, otherwise false. */
290 :
291 : static bool
292 27965137 : can_propagate_from (gimple *def_stmt)
293 : {
294 27965137 : gcc_assert (is_gimple_assign (def_stmt));
295 :
296 : /* If the rhs has side-effects we cannot propagate from it. */
297 27965137 : if (gimple_has_volatile_ops (def_stmt))
298 : return false;
299 :
300 : /* If the rhs is a load we cannot propagate from it. */
301 27372458 : if (TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_reference
302 27372458 : || TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_declaration)
303 : return false;
304 :
305 : /* Constants can be always propagated. */
306 13535810 : if (gimple_assign_single_p (def_stmt)
307 13535810 : && is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
308 : return true;
309 :
310 : /* We cannot propagate ssa names that occur in abnormal phi nodes. */
311 13535810 : if (stmt_references_abnormal_ssa_name (def_stmt))
312 : return false;
313 :
314 : /* If the definition is a conversion of a pointer to a function type,
315 : then we cannot apply optimizations as some targets require
316 : function pointers to be canonicalized and in this case this
317 : optimization could eliminate a necessary canonicalization. */
318 13535119 : if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def_stmt)))
319 : {
320 3251093 : tree rhs = gimple_assign_rhs1 (def_stmt);
321 3251093 : if (FUNCTION_POINTER_TYPE_P (TREE_TYPE (rhs)))
322 : return false;
323 : }
324 :
325 : return true;
326 : }
327 :
328 : /* Remove a chain of dead statements starting at the definition of
329 : NAME. The chain is linked via the first operand of the defining statements.
330 : If NAME was replaced in its only use then this function can be used
331 : to clean up dead stmts. The function handles already released SSA
332 : names gracefully. */
333 :
334 : static void
335 241473 : remove_prop_source_from_use (tree name)
336 : {
337 302067 : gimple_stmt_iterator gsi;
338 302067 : gimple *stmt;
339 :
340 302067 : do {
341 302067 : basic_block bb;
342 :
343 302067 : if (SSA_NAME_IN_FREE_LIST (name)
344 302024 : || SSA_NAME_IS_DEFAULT_DEF (name)
345 602651 : || !has_zero_uses (name))
346 : break;
347 :
348 61051 : stmt = SSA_NAME_DEF_STMT (name);
349 61051 : if (gimple_code (stmt) == GIMPLE_PHI
350 61051 : || gimple_has_side_effects (stmt))
351 : break;
352 :
353 61051 : bb = gimple_bb (stmt);
354 61051 : gsi = gsi_for_stmt (stmt);
355 61051 : unlink_stmt_vdef (stmt);
356 61051 : if (gsi_remove (&gsi, true))
357 6 : bitmap_set_bit (to_purge, bb->index);
358 61051 : fwprop_invalidate_lattice (gimple_get_lhs (stmt));
359 61051 : release_defs (stmt);
360 :
361 61051 : name = is_gimple_assign (stmt) ? gimple_assign_rhs1 (stmt) : NULL_TREE;
362 61051 : } while (name && TREE_CODE (name) == SSA_NAME);
363 :
364 241473 : }
365 :
366 : /* Return the rhs of a gassign *STMT in a form of a single tree,
367 : converted to type TYPE.
368 :
369 : This should disappear, but is needed so we can combine expressions and use
370 : the fold() interfaces. Long term, we need to develop folding and combine
371 : routines that deal with gimple exclusively . */
372 :
373 : static tree
374 7380306 : rhs_to_tree (tree type, gimple *stmt)
375 : {
376 7380306 : location_t loc = gimple_location (stmt);
377 7380306 : enum tree_code code = gimple_assign_rhs_code (stmt);
378 7380306 : switch (get_gimple_rhs_class (code))
379 : {
380 13296 : case GIMPLE_TERNARY_RHS:
381 13296 : return fold_build3_loc (loc, code, type, gimple_assign_rhs1 (stmt),
382 : gimple_assign_rhs2 (stmt),
383 13296 : gimple_assign_rhs3 (stmt));
384 5050969 : case GIMPLE_BINARY_RHS:
385 5050969 : return fold_build2_loc (loc, code, type, gimple_assign_rhs1 (stmt),
386 5050969 : gimple_assign_rhs2 (stmt));
387 2049510 : case GIMPLE_UNARY_RHS:
388 2049510 : return build1 (code, type, gimple_assign_rhs1 (stmt));
389 266531 : case GIMPLE_SINGLE_RHS:
390 266531 : return gimple_assign_rhs1 (stmt);
391 0 : default:
392 0 : gcc_unreachable ();
393 : }
394 : }
395 :
396 : /* Combine OP0 CODE OP1 in the context of a COND_EXPR. Returns
397 : the folded result in a form suitable for COND_EXPR_COND or
398 : NULL_TREE, if there is no suitable simplified form. If
399 : INVARIANT_ONLY is true only gimple_min_invariant results are
400 : considered simplified. */
401 :
402 : static tree
403 8306496 : combine_cond_expr_cond (gimple *stmt, enum tree_code code, tree type,
404 : tree op0, tree op1, bool invariant_only)
405 : {
406 8306496 : tree t;
407 :
408 8306496 : gcc_assert (TREE_CODE_CLASS (code) == tcc_comparison);
409 :
410 8306496 : t = fold_binary_loc (gimple_location (stmt), code, type, op0, op1);
411 8306496 : if (!t)
412 : return NULL_TREE;
413 :
414 : /* Require that we got a boolean type out if we put one in. */
415 3614337 : gcc_assert (TREE_CODE (TREE_TYPE (t)) == TREE_CODE (type));
416 :
417 : /* Canonicalize the combined condition for use in a COND_EXPR. */
418 3614337 : t = canonicalize_cond_expr_cond (t);
419 :
420 : /* Bail out if we required an invariant but didn't get one. */
421 3614337 : if (!t || (invariant_only && !is_gimple_min_invariant (t)))
422 3375247 : return NULL_TREE;
423 :
424 : return t;
425 : }
426 :
427 : /* Combine the comparison OP0 CODE OP1 at LOC with the defining statements
428 : of its operand. Return a new comparison tree or NULL_TREE if there
429 : were no simplifying combines. */
430 :
431 : static tree
432 22179094 : forward_propagate_into_comparison_1 (gimple *stmt,
433 : enum tree_code code, tree type,
434 : tree op0, tree op1)
435 : {
436 22179094 : tree tmp = NULL_TREE;
437 22179094 : tree rhs0 = NULL_TREE, rhs1 = NULL_TREE;
438 22179094 : bool single_use0_p = false, single_use1_p = false;
439 :
440 : /* For comparisons use the first operand, that is likely to
441 : simplify comparisons against constants. */
442 22179094 : if (TREE_CODE (op0) == SSA_NAME)
443 : {
444 22142143 : gimple *def_stmt = get_prop_source_stmt (op0, false, &single_use0_p);
445 22142143 : if (def_stmt && can_propagate_from (def_stmt))
446 : {
447 5609828 : enum tree_code def_code = gimple_assign_rhs_code (def_stmt);
448 5609828 : bool invariant_only_p = !single_use0_p;
449 :
450 5609828 : rhs0 = rhs_to_tree (TREE_TYPE (op1), def_stmt);
451 :
452 : /* Always combine comparisons or conversions from booleans. */
453 5609828 : if (TREE_CODE (op1) == INTEGER_CST
454 5609828 : && ((CONVERT_EXPR_CODE_P (def_code)
455 892841 : && TREE_CODE (TREE_TYPE (TREE_OPERAND (rhs0, 0)))
456 : == BOOLEAN_TYPE)
457 3601721 : || TREE_CODE_CLASS (def_code) == tcc_comparison))
458 : invariant_only_p = false;
459 :
460 5609828 : tmp = combine_cond_expr_cond (stmt, code, type,
461 : rhs0, op1, invariant_only_p);
462 5609828 : if (tmp)
463 : return tmp;
464 : }
465 : }
466 :
467 : /* If that wasn't successful, try the second operand. */
468 21948425 : if (TREE_CODE (op1) == SSA_NAME)
469 : {
470 5470806 : gimple *def_stmt = get_prop_source_stmt (op1, false, &single_use1_p);
471 5470806 : if (def_stmt && can_propagate_from (def_stmt))
472 : {
473 1770478 : rhs1 = rhs_to_tree (TREE_TYPE (op0), def_stmt);
474 3540956 : tmp = combine_cond_expr_cond (stmt, code, type,
475 1770478 : op0, rhs1, !single_use1_p);
476 1770478 : if (tmp)
477 : return tmp;
478 : }
479 : }
480 :
481 : /* If that wasn't successful either, try both operands. */
482 21941919 : if (rhs0 != NULL_TREE
483 21941919 : && rhs1 != NULL_TREE)
484 926190 : tmp = combine_cond_expr_cond (stmt, code, type,
485 : rhs0, rhs1,
486 926190 : !(single_use0_p && single_use1_p));
487 :
488 : return tmp;
489 : }
490 :
491 : /* Propagate from the ssa name definition statements of the assignment
492 : from a comparison at *GSI into the conditional if that simplifies it.
493 : Returns true if the stmt was modified. */
494 :
495 : static bool
496 2654692 : forward_propagate_into_comparison (gimple_stmt_iterator *gsi)
497 : {
498 2654692 : gimple *stmt = gsi_stmt (*gsi);
499 2654692 : tree tmp;
500 2654692 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
501 2654692 : tree rhs1 = gimple_assign_rhs1 (stmt);
502 2654692 : tree rhs2 = gimple_assign_rhs2 (stmt);
503 :
504 : /* Combine the comparison with defining statements. */
505 2654692 : tmp = forward_propagate_into_comparison_1 (stmt,
506 : gimple_assign_rhs_code (stmt),
507 : type, rhs1, rhs2);
508 2654692 : if (tmp && useless_type_conversion_p (type, TREE_TYPE (tmp)))
509 : {
510 7218 : if (dump_file)
511 : {
512 0 : fprintf (dump_file, " Replaced '");
513 0 : print_gimple_expr (dump_file, stmt, 0);
514 0 : fprintf (dump_file, "' with '");
515 0 : print_generic_expr (dump_file, tmp);
516 0 : fprintf (dump_file, "'\n");
517 : }
518 7218 : gimple_assign_set_rhs_from_tree (gsi, tmp);
519 7218 : fold_stmt (gsi);
520 7218 : update_stmt (gsi_stmt (*gsi));
521 :
522 7218 : if (TREE_CODE (rhs1) == SSA_NAME)
523 7218 : remove_prop_source_from_use (rhs1);
524 7218 : if (TREE_CODE (rhs2) == SSA_NAME)
525 3024 : remove_prop_source_from_use (rhs2);
526 7218 : return true;
527 : }
528 :
529 : return false;
530 : }
531 :
532 : /* Propagate from the ssa name definition statements of COND_EXPR
533 : in GIMPLE_COND statement STMT into the conditional if that simplifies it.
534 : Returns zero if no statement was changed, one if there were
535 : changes and two if cfg_cleanup needs to run. */
536 :
537 : static int
538 19524402 : forward_propagate_into_gimple_cond (gcond *stmt)
539 : {
540 19524402 : tree tmp;
541 19524402 : enum tree_code code = gimple_cond_code (stmt);
542 19524402 : tree rhs1 = gimple_cond_lhs (stmt);
543 19524402 : tree rhs2 = gimple_cond_rhs (stmt);
544 :
545 : /* GIMPLE_COND will always be a comparison. */
546 19524402 : gcc_assert (TREE_CODE_CLASS (gimple_cond_code (stmt)) == tcc_comparison);
547 :
548 19524402 : tmp = forward_propagate_into_comparison_1 (stmt, code,
549 : boolean_type_node,
550 : rhs1, rhs2);
551 19524402 : if (tmp
552 19524402 : && is_gimple_condexpr_for_cond (tmp))
553 : {
554 225547 : if (dump_file)
555 : {
556 9 : fprintf (dump_file, " Replaced '");
557 9 : print_gimple_expr (dump_file, stmt, 0);
558 9 : fprintf (dump_file, "' with '");
559 9 : print_generic_expr (dump_file, tmp);
560 9 : fprintf (dump_file, "'\n");
561 : }
562 :
563 225547 : gimple_cond_set_condition_from_tree (stmt, unshare_expr (tmp));
564 225547 : update_stmt (stmt);
565 :
566 225547 : if (TREE_CODE (rhs1) == SSA_NAME)
567 225547 : remove_prop_source_from_use (rhs1);
568 225547 : if (TREE_CODE (rhs2) == SSA_NAME)
569 5683 : remove_prop_source_from_use (rhs2);
570 225547 : return is_gimple_min_invariant (tmp) ? 2 : 1;
571 : }
572 :
573 19298855 : if (canonicalize_bool_cond (stmt, gimple_bb (stmt)))
574 : return 1;
575 :
576 : return 0;
577 : }
578 :
579 : /* We've just substituted an ADDR_EXPR into stmt. Update all the
580 : relevant data structures to match. */
581 :
582 : static void
583 1981781 : tidy_after_forward_propagate_addr (gimple *stmt)
584 : {
585 : /* We may have turned a trapping insn into a non-trapping insn. */
586 1981781 : if (maybe_clean_or_replace_eh_stmt (stmt, stmt))
587 131 : bitmap_set_bit (to_purge, gimple_bb (stmt)->index);
588 :
589 1981781 : if (TREE_CODE (gimple_assign_rhs1 (stmt)) == ADDR_EXPR)
590 246659 : recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
591 1981781 : }
592 :
593 : /* NAME is a SSA_NAME representing DEF_RHS which is of the form
594 : ADDR_EXPR <whatever>.
595 :
596 : Try to forward propagate the ADDR_EXPR into the use USE_STMT.
597 : Often this will allow for removal of an ADDR_EXPR and INDIRECT_REF
598 : node or for recovery of array indexing from pointer arithmetic.
599 :
600 : Return true if the propagation was successful (the propagation can
601 : be not totally successful, yet things may have been changed). */
602 :
603 : static bool
604 2817252 : forward_propagate_addr_expr_1 (tree name, tree def_rhs,
605 : gimple_stmt_iterator *use_stmt_gsi,
606 : bool single_use_p)
607 : {
608 2817252 : tree lhs, rhs, rhs2, array_ref;
609 2817252 : gimple *use_stmt = gsi_stmt (*use_stmt_gsi);
610 2817252 : enum tree_code rhs_code;
611 2817252 : bool res = true;
612 :
613 2817252 : gcc_assert (TREE_CODE (def_rhs) == ADDR_EXPR);
614 :
615 2817252 : lhs = gimple_assign_lhs (use_stmt);
616 2817252 : rhs_code = gimple_assign_rhs_code (use_stmt);
617 2817252 : rhs = gimple_assign_rhs1 (use_stmt);
618 :
619 : /* Do not perform copy-propagation but recurse through copy chains. */
620 2817252 : if (TREE_CODE (lhs) == SSA_NAME
621 1399360 : && rhs_code == SSA_NAME)
622 8617 : return forward_propagate_addr_expr (lhs, def_rhs, single_use_p);
623 :
624 : /* The use statement could be a conversion. Recurse to the uses of the
625 : lhs as copyprop does not copy through pointer to integer to pointer
626 : conversions and FRE does not catch all cases either.
627 : Treat the case of a single-use name and
628 : a conversion to def_rhs type separate, though. */
629 2808635 : if (TREE_CODE (lhs) == SSA_NAME
630 1390743 : && CONVERT_EXPR_CODE_P (rhs_code))
631 : {
632 : /* If there is a point in a conversion chain where the types match
633 : so we can remove a conversion re-materialize the address here
634 : and stop. */
635 23947 : if (single_use_p
636 23947 : && useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (def_rhs)))
637 : {
638 1 : gimple_assign_set_rhs1 (use_stmt, unshare_expr (def_rhs));
639 1 : gimple_assign_set_rhs_code (use_stmt, TREE_CODE (def_rhs));
640 1 : return true;
641 : }
642 :
643 : /* Else recurse if the conversion preserves the address value. */
644 47892 : if ((INTEGRAL_TYPE_P (TREE_TYPE (lhs))
645 2 : || POINTER_TYPE_P (TREE_TYPE (lhs)))
646 47892 : && (TYPE_PRECISION (TREE_TYPE (lhs))
647 23946 : >= TYPE_PRECISION (TREE_TYPE (def_rhs))))
648 23879 : return forward_propagate_addr_expr (lhs, def_rhs, single_use_p);
649 :
650 : return false;
651 : }
652 :
653 : /* If this isn't a conversion chain from this on we only can propagate
654 : into compatible pointer contexts. */
655 2784688 : if (!types_compatible_p (TREE_TYPE (name), TREE_TYPE (def_rhs)))
656 : return false;
657 :
658 : /* Propagate through constant pointer adjustments. */
659 2764082 : if (TREE_CODE (lhs) == SSA_NAME
660 1347363 : && rhs_code == POINTER_PLUS_EXPR
661 1347363 : && rhs == name
662 2930671 : && TREE_CODE (gimple_assign_rhs2 (use_stmt)) == INTEGER_CST)
663 : {
664 118762 : tree new_def_rhs;
665 : /* As we come here with non-invariant addresses in def_rhs we need
666 : to make sure we can build a valid constant offsetted address
667 : for further propagation. Simply rely on fold building that
668 : and check after the fact. */
669 118762 : new_def_rhs = fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (rhs)),
670 : def_rhs,
671 : fold_convert (ptr_type_node,
672 : gimple_assign_rhs2 (use_stmt)));
673 118762 : if (TREE_CODE (new_def_rhs) == MEM_REF
674 118762 : && !is_gimple_mem_ref_addr (TREE_OPERAND (new_def_rhs, 0)))
675 : return false;
676 114756 : new_def_rhs = build1 (ADDR_EXPR, TREE_TYPE (rhs), new_def_rhs);
677 :
678 : /* Recurse. If we could propagate into all uses of lhs do not
679 : bother to replace into the current use but just pretend we did. */
680 114756 : if (forward_propagate_addr_expr (lhs, new_def_rhs, single_use_p))
681 : return true;
682 :
683 38554 : if (useless_type_conversion_p (TREE_TYPE (lhs),
684 38554 : TREE_TYPE (new_def_rhs)))
685 38554 : gimple_assign_set_rhs_with_ops (use_stmt_gsi, TREE_CODE (new_def_rhs),
686 : new_def_rhs);
687 0 : else if (is_gimple_min_invariant (new_def_rhs))
688 0 : gimple_assign_set_rhs_with_ops (use_stmt_gsi, NOP_EXPR, new_def_rhs);
689 : else
690 : return false;
691 38554 : gcc_assert (gsi_stmt (*use_stmt_gsi) == use_stmt);
692 38554 : update_stmt (use_stmt);
693 38554 : return true;
694 : }
695 :
696 : /* Now strip away any outer COMPONENT_REF/ARRAY_REF nodes from the LHS.
697 : ADDR_EXPR will not appear on the LHS. */
698 2645320 : tree *lhsp = gimple_assign_lhs_ptr (use_stmt);
699 4003997 : while (handled_component_p (*lhsp))
700 1358677 : lhsp = &TREE_OPERAND (*lhsp, 0);
701 2645320 : lhs = *lhsp;
702 :
703 : /* Now see if the LHS node is a MEM_REF using NAME. If so,
704 : propagate the ADDR_EXPR into the use of NAME and fold the result. */
705 2645320 : if (TREE_CODE (lhs) == MEM_REF
706 2645320 : && TREE_OPERAND (lhs, 0) == name)
707 : {
708 895964 : tree def_rhs_base;
709 895964 : poly_int64 def_rhs_offset;
710 : /* If the address is invariant we can always fold it. */
711 895964 : if ((def_rhs_base = get_addr_base_and_unit_offset (TREE_OPERAND (def_rhs, 0),
712 : &def_rhs_offset)))
713 : {
714 848833 : poly_offset_int off = mem_ref_offset (lhs);
715 848833 : tree new_ptr;
716 848833 : off += def_rhs_offset;
717 848833 : if (TREE_CODE (def_rhs_base) == MEM_REF)
718 : {
719 827922 : off += mem_ref_offset (def_rhs_base);
720 827922 : new_ptr = TREE_OPERAND (def_rhs_base, 0);
721 : }
722 : else
723 20911 : new_ptr = build_fold_addr_expr (def_rhs_base);
724 848833 : TREE_OPERAND (lhs, 0) = new_ptr;
725 848833 : TREE_OPERAND (lhs, 1)
726 848833 : = wide_int_to_tree (TREE_TYPE (TREE_OPERAND (lhs, 1)), off);
727 848833 : tidy_after_forward_propagate_addr (use_stmt);
728 : /* Continue propagating into the RHS if this was not the only use. */
729 848833 : if (single_use_p)
730 227700 : return true;
731 : }
732 : /* If the LHS is a plain dereference and the value type is the same as
733 : that of the pointed-to type of the address we can put the
734 : dereferenced address on the LHS preserving the original alias-type. */
735 47131 : else if (integer_zerop (TREE_OPERAND (lhs, 1))
736 18539 : && ((gimple_assign_lhs (use_stmt) == lhs
737 14920 : && useless_type_conversion_p
738 14920 : (TREE_TYPE (TREE_OPERAND (def_rhs, 0)),
739 14920 : TREE_TYPE (gimple_assign_rhs1 (use_stmt))))
740 13617 : || types_compatible_p (TREE_TYPE (lhs),
741 13617 : TREE_TYPE (TREE_OPERAND (def_rhs, 0))))
742 : /* Don't forward anything into clobber stmts if it would result
743 : in the lhs no longer being a MEM_REF. */
744 55309 : && (!gimple_clobber_p (use_stmt)
745 164 : || TREE_CODE (TREE_OPERAND (def_rhs, 0)) == MEM_REF))
746 : {
747 8014 : tree *def_rhs_basep = &TREE_OPERAND (def_rhs, 0);
748 8014 : tree new_offset, new_base, saved, new_lhs;
749 29081 : while (handled_component_p (*def_rhs_basep))
750 13053 : def_rhs_basep = &TREE_OPERAND (*def_rhs_basep, 0);
751 8014 : saved = *def_rhs_basep;
752 8014 : if (TREE_CODE (*def_rhs_basep) == MEM_REF)
753 : {
754 3941 : new_base = TREE_OPERAND (*def_rhs_basep, 0);
755 3941 : new_offset = fold_convert (TREE_TYPE (TREE_OPERAND (lhs, 1)),
756 : TREE_OPERAND (*def_rhs_basep, 1));
757 : }
758 : else
759 : {
760 4073 : new_base = build_fold_addr_expr (*def_rhs_basep);
761 4073 : new_offset = TREE_OPERAND (lhs, 1);
762 : }
763 8014 : tree atype = TREE_TYPE (*def_rhs_basep);
764 8014 : if (TYPE_ALIGN (TREE_TYPE (lhs)) < TYPE_ALIGN (atype))
765 312 : atype = build_aligned_type (atype, TYPE_ALIGN (TREE_TYPE (lhs)));
766 8014 : *def_rhs_basep = build2 (MEM_REF, atype, new_base, new_offset);
767 8014 : TREE_THIS_VOLATILE (*def_rhs_basep) = TREE_THIS_VOLATILE (lhs);
768 8014 : TREE_SIDE_EFFECTS (*def_rhs_basep) = TREE_SIDE_EFFECTS (lhs);
769 8014 : TREE_THIS_NOTRAP (*def_rhs_basep) = TREE_THIS_NOTRAP (lhs);
770 8014 : new_lhs = unshare_expr (TREE_OPERAND (def_rhs, 0));
771 8014 : *lhsp = new_lhs;
772 8014 : TREE_THIS_VOLATILE (new_lhs) = TREE_THIS_VOLATILE (lhs);
773 8014 : TREE_SIDE_EFFECTS (new_lhs) = TREE_SIDE_EFFECTS (lhs);
774 8014 : *def_rhs_basep = saved;
775 8014 : tidy_after_forward_propagate_addr (use_stmt);
776 : /* Continue propagating into the RHS if this was not the
777 : only use. */
778 8014 : if (single_use_p)
779 : return true;
780 : }
781 : else
782 : /* We can have a struct assignment dereferencing our name twice.
783 : Note that we didn't propagate into the lhs to not falsely
784 : claim we did when propagating into the rhs. */
785 : res = false;
786 : }
787 :
788 : /* Strip away any outer COMPONENT_REF, ARRAY_REF or ADDR_EXPR
789 : nodes from the RHS. */
790 2413413 : tree *rhsp = gimple_assign_rhs1_ptr (use_stmt);
791 2413413 : if (TREE_CODE (*rhsp) == ADDR_EXPR)
792 235001 : rhsp = &TREE_OPERAND (*rhsp, 0);
793 3413529 : while (handled_component_p (*rhsp))
794 1000116 : rhsp = &TREE_OPERAND (*rhsp, 0);
795 2413413 : rhs = *rhsp;
796 :
797 : /* Now see if the RHS node is a MEM_REF using NAME. If so,
798 : propagate the ADDR_EXPR into the use of NAME and fold the result. */
799 2413413 : if (TREE_CODE (rhs) == MEM_REF
800 2413413 : && TREE_OPERAND (rhs, 0) == name)
801 : {
802 1146714 : tree def_rhs_base;
803 1146714 : poly_int64 def_rhs_offset;
804 1146714 : if ((def_rhs_base = get_addr_base_and_unit_offset (TREE_OPERAND (def_rhs, 0),
805 : &def_rhs_offset)))
806 : {
807 1109664 : poly_offset_int off = mem_ref_offset (rhs);
808 1109664 : tree new_ptr;
809 1109664 : off += def_rhs_offset;
810 1109664 : if (TREE_CODE (def_rhs_base) == MEM_REF)
811 : {
812 1083489 : off += mem_ref_offset (def_rhs_base);
813 1083489 : new_ptr = TREE_OPERAND (def_rhs_base, 0);
814 : }
815 : else
816 26175 : new_ptr = build_fold_addr_expr (def_rhs_base);
817 1109664 : TREE_OPERAND (rhs, 0) = new_ptr;
818 1109664 : TREE_OPERAND (rhs, 1)
819 1109664 : = wide_int_to_tree (TREE_TYPE (TREE_OPERAND (rhs, 1)), off);
820 1109664 : fold_stmt_inplace (use_stmt_gsi);
821 1109664 : tidy_after_forward_propagate_addr (use_stmt);
822 1109664 : return res;
823 : }
824 : /* If the RHS is a plain dereference and the value type is the same as
825 : that of the pointed-to type of the address we can put the
826 : dereferenced address on the RHS preserving the original alias-type. */
827 37050 : else if (integer_zerop (TREE_OPERAND (rhs, 1))
828 37050 : && ((gimple_assign_rhs1 (use_stmt) == rhs
829 20291 : && useless_type_conversion_p
830 20291 : (TREE_TYPE (gimple_assign_lhs (use_stmt)),
831 20291 : TREE_TYPE (TREE_OPERAND (def_rhs, 0))))
832 22562 : || types_compatible_p (TREE_TYPE (rhs),
833 22562 : TREE_TYPE (TREE_OPERAND (def_rhs, 0)))))
834 : {
835 15270 : tree *def_rhs_basep = &TREE_OPERAND (def_rhs, 0);
836 15270 : tree new_offset, new_base, saved, new_rhs;
837 54336 : while (handled_component_p (*def_rhs_basep))
838 23796 : def_rhs_basep = &TREE_OPERAND (*def_rhs_basep, 0);
839 15270 : saved = *def_rhs_basep;
840 15270 : if (TREE_CODE (*def_rhs_basep) == MEM_REF)
841 : {
842 7359 : new_base = TREE_OPERAND (*def_rhs_basep, 0);
843 7359 : new_offset = fold_convert (TREE_TYPE (TREE_OPERAND (rhs, 1)),
844 : TREE_OPERAND (*def_rhs_basep, 1));
845 : }
846 : else
847 : {
848 7911 : new_base = build_fold_addr_expr (*def_rhs_basep);
849 7911 : new_offset = TREE_OPERAND (rhs, 1);
850 : }
851 15270 : tree atype = TREE_TYPE (*def_rhs_basep);
852 15270 : if (TYPE_ALIGN (TREE_TYPE (rhs)) < TYPE_ALIGN (atype))
853 526 : atype = build_aligned_type (atype, TYPE_ALIGN (TREE_TYPE (rhs)));
854 15270 : *def_rhs_basep = build2 (MEM_REF, atype, new_base, new_offset);
855 15270 : TREE_THIS_VOLATILE (*def_rhs_basep) = TREE_THIS_VOLATILE (rhs);
856 15270 : TREE_SIDE_EFFECTS (*def_rhs_basep) = TREE_SIDE_EFFECTS (rhs);
857 15270 : TREE_THIS_NOTRAP (*def_rhs_basep) = TREE_THIS_NOTRAP (rhs);
858 15270 : new_rhs = unshare_expr (TREE_OPERAND (def_rhs, 0));
859 15270 : *rhsp = new_rhs;
860 15270 : TREE_THIS_VOLATILE (new_rhs) = TREE_THIS_VOLATILE (rhs);
861 15270 : TREE_SIDE_EFFECTS (new_rhs) = TREE_SIDE_EFFECTS (rhs);
862 15270 : *def_rhs_basep = saved;
863 15270 : fold_stmt_inplace (use_stmt_gsi);
864 15270 : tidy_after_forward_propagate_addr (use_stmt);
865 15270 : return res;
866 : }
867 : }
868 :
869 : /* If the use of the ADDR_EXPR is not a POINTER_PLUS_EXPR, there
870 : is nothing to do. */
871 1288479 : if (gimple_assign_rhs_code (use_stmt) != POINTER_PLUS_EXPR
872 1288479 : || gimple_assign_rhs1 (use_stmt) != name)
873 : return false;
874 :
875 : /* The remaining cases are all for turning pointer arithmetic into
876 : array indexing. They only apply when we have the address of
877 : element zero in an array. If that is not the case then there
878 : is nothing to do. */
879 47827 : array_ref = TREE_OPERAND (def_rhs, 0);
880 47827 : if ((TREE_CODE (array_ref) != ARRAY_REF
881 4570 : || TREE_CODE (TREE_TYPE (TREE_OPERAND (array_ref, 0))) != ARRAY_TYPE
882 4570 : || TREE_CODE (TREE_OPERAND (array_ref, 1)) != INTEGER_CST)
883 49326 : && TREE_CODE (TREE_TYPE (array_ref)) != ARRAY_TYPE)
884 : return false;
885 :
886 24567 : rhs2 = gimple_assign_rhs2 (use_stmt);
887 : /* Optimize &x[C1] p+ C2 to &x p+ C3 with C3 = C1 * element_size + C2. */
888 24567 : if (TREE_CODE (rhs2) == INTEGER_CST)
889 : {
890 0 : tree new_rhs = build1_loc (gimple_location (use_stmt),
891 0 : ADDR_EXPR, TREE_TYPE (def_rhs),
892 0 : fold_build2 (MEM_REF,
893 : TREE_TYPE (TREE_TYPE (def_rhs)),
894 : unshare_expr (def_rhs),
895 : fold_convert (ptr_type_node,
896 : rhs2)));
897 0 : gimple_assign_set_rhs_from_tree (use_stmt_gsi, new_rhs);
898 0 : use_stmt = gsi_stmt (*use_stmt_gsi);
899 0 : update_stmt (use_stmt);
900 0 : tidy_after_forward_propagate_addr (use_stmt);
901 0 : return true;
902 : }
903 :
904 : return false;
905 : }
906 :
907 : /* STMT is a statement of the form SSA_NAME = ADDR_EXPR <whatever>.
908 :
909 : Try to forward propagate the ADDR_EXPR into all uses of the SSA_NAME.
910 : Often this will allow for removal of an ADDR_EXPR and INDIRECT_REF
911 : node or for recovery of array indexing from pointer arithmetic.
912 :
913 : PARENT_SINGLE_USE_P tells if, when in a recursive invocation, NAME was
914 : the single use in the previous invocation. Pass true when calling
915 : this as toplevel.
916 :
917 : Returns true, if all uses have been propagated into. */
918 :
919 : static bool
920 3258270 : forward_propagate_addr_expr (tree name, tree rhs, bool parent_single_use_p)
921 : {
922 3258270 : bool all = true;
923 3258270 : bool single_use_p = parent_single_use_p && has_single_use (name);
924 :
925 17205495 : for (gimple *use_stmt : gather_imm_use_stmts (name))
926 : {
927 7430685 : bool result;
928 7430685 : tree use_rhs;
929 :
930 : /* If the use is not in a simple assignment statement, then
931 : there is nothing we can do. */
932 7430685 : if (!is_gimple_assign (use_stmt))
933 : {
934 4613433 : if (!is_gimple_debug (use_stmt))
935 1895955 : all = false;
936 4613433 : continue;
937 : }
938 :
939 2817252 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
940 2817252 : result = forward_propagate_addr_expr_1 (name, rhs, &gsi,
941 : single_use_p);
942 : /* If the use has moved to a different statement adjust
943 : the update machinery for the old statement too. */
944 2817252 : if (use_stmt != gsi_stmt (gsi))
945 : {
946 0 : update_stmt (use_stmt);
947 0 : use_stmt = gsi_stmt (gsi);
948 : }
949 2817252 : update_stmt (use_stmt);
950 2817252 : all &= result;
951 :
952 : /* Remove intermediate now unused copy and conversion chains. */
953 2817252 : use_rhs = gimple_assign_rhs1 (use_stmt);
954 2817252 : if (result
955 1476808 : && TREE_CODE (gimple_assign_lhs (use_stmt)) == SSA_NAME
956 1232281 : && TREE_CODE (use_rhs) == SSA_NAME
957 2898664 : && has_zero_uses (gimple_assign_lhs (use_stmt)))
958 : {
959 81412 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
960 81412 : fwprop_invalidate_lattice (gimple_get_lhs (use_stmt));
961 81412 : release_defs (use_stmt);
962 81412 : gsi_remove (&gsi, true);
963 : }
964 3258270 : }
965 :
966 3258270 : return all && has_zero_uses (name);
967 : }
968 :
969 :
970 : /* Helper function for simplify_gimple_switch. Remove case labels that
971 : have values outside the range of the new type. */
972 :
973 : static void
974 11595 : simplify_gimple_switch_label_vec (gswitch *stmt, tree index_type,
975 : vec<std::pair<int, int> > &edges_to_remove)
976 : {
977 11595 : unsigned int branch_num = gimple_switch_num_labels (stmt);
978 11595 : auto_vec<tree> labels (branch_num);
979 11595 : unsigned int i, len;
980 :
981 : /* Collect the existing case labels in a VEC, and preprocess it as if
982 : we are gimplifying a GENERIC SWITCH_EXPR. */
983 73032 : for (i = 1; i < branch_num; i++)
984 49842 : labels.quick_push (gimple_switch_label (stmt, i));
985 11595 : preprocess_case_label_vec_for_gimple (labels, index_type, NULL);
986 :
987 : /* If any labels were removed, replace the existing case labels
988 : in the GIMPLE_SWITCH statement with the correct ones.
989 : Note that the type updates were done in-place on the case labels,
990 : so we only have to replace the case labels in the GIMPLE_SWITCH
991 : if the number of labels changed. */
992 11595 : len = labels.length ();
993 11595 : if (len < branch_num - 1)
994 : {
995 0 : bitmap target_blocks;
996 0 : edge_iterator ei;
997 0 : edge e;
998 :
999 : /* Corner case: *all* case labels have been removed as being
1000 : out-of-range for INDEX_TYPE. Push one label and let the
1001 : CFG cleanups deal with this further. */
1002 0 : if (len == 0)
1003 : {
1004 0 : tree label, elt;
1005 :
1006 0 : label = CASE_LABEL (gimple_switch_default_label (stmt));
1007 0 : elt = build_case_label (build_int_cst (index_type, 0), NULL, label);
1008 0 : labels.quick_push (elt);
1009 0 : len = 1;
1010 : }
1011 :
1012 0 : for (i = 0; i < labels.length (); i++)
1013 0 : gimple_switch_set_label (stmt, i + 1, labels[i]);
1014 0 : for (i++ ; i < branch_num; i++)
1015 0 : gimple_switch_set_label (stmt, i, NULL_TREE);
1016 0 : gimple_switch_set_num_labels (stmt, len + 1);
1017 :
1018 : /* Cleanup any edges that are now dead. */
1019 0 : target_blocks = BITMAP_ALLOC (NULL);
1020 0 : for (i = 0; i < gimple_switch_num_labels (stmt); i++)
1021 : {
1022 0 : tree elt = gimple_switch_label (stmt, i);
1023 0 : basic_block target = label_to_block (cfun, CASE_LABEL (elt));
1024 0 : bitmap_set_bit (target_blocks, target->index);
1025 : }
1026 0 : for (ei = ei_start (gimple_bb (stmt)->succs); (e = ei_safe_edge (ei)); )
1027 : {
1028 0 : if (! bitmap_bit_p (target_blocks, e->dest->index))
1029 0 : edges_to_remove.safe_push (std::make_pair (e->src->index,
1030 0 : e->dest->index));
1031 : else
1032 0 : ei_next (&ei);
1033 : }
1034 0 : BITMAP_FREE (target_blocks);
1035 : }
1036 11595 : }
1037 :
1038 : /* STMT is a SWITCH_EXPR for which we attempt to find equivalent forms of
1039 : the condition which we may be able to optimize better. */
1040 :
1041 : static bool
1042 106576 : simplify_gimple_switch (gswitch *stmt,
1043 : vec<std::pair<int, int> > &edges_to_remove,
1044 : bitmap simple_dce_worklist)
1045 : {
1046 : /* The optimization that we really care about is removing unnecessary
1047 : casts. That will let us do much better in propagating the inferred
1048 : constant at the switch target. */
1049 106576 : tree cond = gimple_switch_index (stmt);
1050 106576 : if (TREE_CODE (cond) == SSA_NAME)
1051 : {
1052 106575 : gimple *def_stmt = SSA_NAME_DEF_STMT (cond);
1053 106575 : if (gimple_assign_cast_p (def_stmt))
1054 : {
1055 12543 : tree def = gimple_assign_rhs1 (def_stmt);
1056 12543 : if (TREE_CODE (def) != SSA_NAME)
1057 : return false;
1058 :
1059 : /* If we have an extension or sign-change that preserves the
1060 : values we check against then we can copy the source value into
1061 : the switch. */
1062 12543 : tree ti = TREE_TYPE (def);
1063 12543 : if (INTEGRAL_TYPE_P (ti)
1064 12543 : && TYPE_PRECISION (ti) <= TYPE_PRECISION (TREE_TYPE (cond)))
1065 : {
1066 12298 : size_t n = gimple_switch_num_labels (stmt);
1067 12298 : tree min = NULL_TREE, max = NULL_TREE;
1068 12298 : if (n > 1)
1069 : {
1070 12298 : min = CASE_LOW (gimple_switch_label (stmt, 1));
1071 12298 : if (CASE_HIGH (gimple_switch_label (stmt, n - 1)))
1072 159 : max = CASE_HIGH (gimple_switch_label (stmt, n - 1));
1073 : else
1074 12139 : max = CASE_LOW (gimple_switch_label (stmt, n - 1));
1075 : }
1076 12298 : if ((!min || int_fits_type_p (min, ti))
1077 12294 : && (!max || int_fits_type_p (max, ti)))
1078 : {
1079 11595 : bitmap_set_bit (simple_dce_worklist,
1080 11595 : SSA_NAME_VERSION (cond));
1081 11595 : gimple_switch_set_index (stmt, def);
1082 11595 : simplify_gimple_switch_label_vec (stmt, ti,
1083 : edges_to_remove);
1084 11595 : update_stmt (stmt);
1085 11595 : return true;
1086 : }
1087 : }
1088 : }
1089 : }
1090 :
1091 : return false;
1092 : }
1093 :
1094 : /* For pointers p2 and p1 return p2 - p1 if the
1095 : difference is known and constant, otherwise return NULL. */
1096 :
1097 : static tree
1098 5475 : constant_pointer_difference (tree p1, tree p2)
1099 : {
1100 5475 : int i, j;
1101 : #define CPD_ITERATIONS 5
1102 5475 : tree exps[2][CPD_ITERATIONS];
1103 5475 : tree offs[2][CPD_ITERATIONS];
1104 5475 : int cnt[2];
1105 :
1106 16425 : for (i = 0; i < 2; i++)
1107 : {
1108 10950 : tree p = i ? p1 : p2;
1109 10950 : tree off = size_zero_node;
1110 10950 : gimple *stmt;
1111 10950 : enum tree_code code;
1112 :
1113 : /* For each of p1 and p2 we need to iterate at least
1114 : twice, to handle ADDR_EXPR directly in p1/p2,
1115 : SSA_NAME with ADDR_EXPR or POINTER_PLUS_EXPR etc.
1116 : on definition's stmt RHS. Iterate a few extra times. */
1117 10950 : j = 0;
1118 12740 : do
1119 : {
1120 12740 : if (!POINTER_TYPE_P (TREE_TYPE (p)))
1121 : break;
1122 12734 : if (TREE_CODE (p) == ADDR_EXPR)
1123 : {
1124 9587 : tree q = TREE_OPERAND (p, 0);
1125 9587 : poly_int64 offset;
1126 9587 : tree base = get_addr_base_and_unit_offset (q, &offset);
1127 9587 : if (base)
1128 : {
1129 8795 : q = base;
1130 8795 : if (maybe_ne (offset, 0))
1131 3748 : off = size_binop (PLUS_EXPR, off, size_int (offset));
1132 : }
1133 9587 : if (TREE_CODE (q) == MEM_REF
1134 9587 : && TREE_CODE (TREE_OPERAND (q, 0)) == SSA_NAME)
1135 : {
1136 155 : p = TREE_OPERAND (q, 0);
1137 155 : off = size_binop (PLUS_EXPR, off,
1138 : wide_int_to_tree (sizetype,
1139 : mem_ref_offset (q)));
1140 : }
1141 : else
1142 : {
1143 9432 : exps[i][j] = q;
1144 9432 : offs[i][j++] = off;
1145 9432 : break;
1146 : }
1147 : }
1148 3302 : if (TREE_CODE (p) != SSA_NAME)
1149 : break;
1150 3302 : exps[i][j] = p;
1151 3302 : offs[i][j++] = off;
1152 3302 : if (j == CPD_ITERATIONS)
1153 : break;
1154 3302 : stmt = SSA_NAME_DEF_STMT (p);
1155 3302 : if (!is_gimple_assign (stmt) || gimple_assign_lhs (stmt) != p)
1156 : break;
1157 2643 : code = gimple_assign_rhs_code (stmt);
1158 2643 : if (code == POINTER_PLUS_EXPR)
1159 : {
1160 1390 : if (TREE_CODE (gimple_assign_rhs2 (stmt)) != INTEGER_CST)
1161 : break;
1162 863 : off = size_binop (PLUS_EXPR, off, gimple_assign_rhs2 (stmt));
1163 863 : p = gimple_assign_rhs1 (stmt);
1164 : }
1165 1253 : else if (code == ADDR_EXPR || CONVERT_EXPR_CODE_P (code))
1166 927 : p = gimple_assign_rhs1 (stmt);
1167 : else
1168 : break;
1169 : }
1170 : while (1);
1171 10950 : cnt[i] = j;
1172 : }
1173 :
1174 7458 : for (i = 0; i < cnt[0]; i++)
1175 9713 : for (j = 0; j < cnt[1]; j++)
1176 7730 : if (exps[0][i] == exps[1][j])
1177 4591 : return size_binop (MINUS_EXPR, offs[0][i], offs[1][j]);
1178 :
1179 : return NULL_TREE;
1180 : }
1181 :
1182 : /* Helper function for optimize_aggr_zeroprop.
1183 : Props the zeroing (memset, VAL) that was done in DEST+OFFSET:LEN
1184 : (DEFSTMT) into the STMT. Returns true if the STMT was updated. */
1185 : static void
1186 22451460 : optimize_aggr_zeroprop_1 (gimple *defstmt, gimple *stmt,
1187 : tree dest, poly_int64 offset, tree val,
1188 : poly_offset_int len)
1189 : {
1190 22451460 : tree src2;
1191 22451460 : tree len2 = NULL_TREE;
1192 22451460 : poly_int64 offset2;
1193 :
1194 22451460 : if (gimple_call_builtin_p (stmt, BUILT_IN_MEMCPY)
1195 21805 : && TREE_CODE (gimple_call_arg (stmt, 1)) == ADDR_EXPR
1196 22465464 : && poly_int_tree_p (gimple_call_arg (stmt, 2)))
1197 : {
1198 12971 : src2 = TREE_OPERAND (gimple_call_arg (stmt, 1), 0);
1199 12971 : len2 = gimple_call_arg (stmt, 2);
1200 : }
1201 22438489 : else if (gimple_assign_load_p (stmt) && gimple_store_p (stmt))
1202 : {
1203 1907029 : src2 = gimple_assign_rhs1 (stmt);
1204 1907029 : len2 = (TREE_CODE (src2) == COMPONENT_REF
1205 1907029 : ? DECL_SIZE_UNIT (TREE_OPERAND (src2, 1))
1206 1739454 : : TYPE_SIZE_UNIT (TREE_TYPE (src2)));
1207 : /* Can only handle zero memsets. */
1208 1907029 : if (!integer_zerop (val))
1209 22429434 : return;
1210 : }
1211 : else
1212 20531460 : return;
1213 :
1214 1918965 : if (len2 == NULL_TREE
1215 1918965 : || !poly_int_tree_p (len2))
1216 : return;
1217 :
1218 1918965 : src2 = get_addr_base_and_unit_offset (src2, &offset2);
1219 1918965 : if (src2 == NULL_TREE
1220 1918965 : || maybe_lt (offset2, offset))
1221 : return;
1222 :
1223 874538 : if (!operand_equal_p (dest, src2, 0))
1224 : return;
1225 :
1226 : /* [ dest + offset, dest + offset + len - 1 ] is set to val.
1227 : Make sure that
1228 : [ dest + offset2, dest + offset2 + len2 - 1 ] is a subset of that. */
1229 131938 : if (maybe_gt (wi::to_poly_offset (len2) + (offset2 - offset),
1230 : len))
1231 : return;
1232 :
1233 22026 : if (dump_file && (dump_flags & TDF_DETAILS))
1234 : {
1235 32 : fprintf (dump_file, "Simplified\n ");
1236 32 : print_gimple_stmt (dump_file, stmt, 0, dump_flags);
1237 32 : fprintf (dump_file, "after previous\n ");
1238 32 : print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
1239 : }
1240 22026 : gimple *orig_stmt = stmt;
1241 : /* For simplicity, don't change the kind of the stmt,
1242 : turn dest = src; into dest = {}; and memcpy (&dest, &src, len);
1243 : into memset (&dest, val, len);
1244 : In theory we could change dest = src into memset if dest
1245 : is addressable (maybe beneficial if val is not 0), or
1246 : memcpy (&dest, &src, len) into dest = {} if len is the size
1247 : of dest, dest isn't volatile. */
1248 22026 : if (is_gimple_assign (stmt))
1249 : {
1250 22021 : tree ctor_type = TREE_TYPE (gimple_assign_lhs (stmt));
1251 22021 : tree ctor = build_constructor (ctor_type, NULL);
1252 22021 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1253 22021 : gimple_assign_set_rhs_from_tree (&gsi, ctor);
1254 22021 : update_stmt (stmt);
1255 22021 : statistics_counter_event (cfun, "copy zeroing propagation of aggregate", 1);
1256 : }
1257 : else /* If stmt is memcpy, transform it into memset. */
1258 : {
1259 5 : gcall *call = as_a <gcall *> (stmt);
1260 5 : tree fndecl = builtin_decl_implicit (BUILT_IN_MEMSET);
1261 5 : gimple_call_set_fndecl (call, fndecl);
1262 5 : gimple_call_set_fntype (call, TREE_TYPE (fndecl));
1263 5 : gimple_call_set_arg (call, 1, val);
1264 5 : update_stmt (stmt);
1265 5 : statistics_counter_event (cfun, "memcpy to memset changed", 1);
1266 : }
1267 :
1268 22026 : if (dump_file && (dump_flags & TDF_DETAILS))
1269 : {
1270 32 : fprintf (dump_file, "into\n ");
1271 32 : print_gimple_stmt (dump_file, stmt, 0, dump_flags);
1272 : }
1273 :
1274 : /* Mark the bb for eh cleanup if needed. */
1275 22026 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
1276 6 : bitmap_set_bit (to_purge, gimple_bb (stmt)->index);
1277 : }
1278 :
1279 : /* Optimize
1280 : a = {}; // DEST = value ;; LEN(nullptr)
1281 : b = a;
1282 : into
1283 : a = {};
1284 : b = {};
1285 : Similarly for memset (&a, ..., sizeof (a)); instead of a = {};
1286 : and/or memcpy (&b, &a, sizeof (a)); instead of b = a; */
1287 :
1288 : static void
1289 31309763 : optimize_aggr_zeroprop (gimple *stmt, bool full_walk)
1290 : {
1291 31309763 : ao_ref read;
1292 62619526 : if (gimple_has_volatile_ops (stmt))
1293 27281212 : return;
1294 :
1295 30383437 : tree dest = NULL_TREE;
1296 30383437 : tree val = integer_zero_node;
1297 30383437 : tree len = NULL_TREE;
1298 30383437 : bool can_use_tbba = true;
1299 :
1300 30383437 : if (gimple_call_builtin_p (stmt, BUILT_IN_MEMSET)
1301 111262 : && TREE_CODE (gimple_call_arg (stmt, 0)) == ADDR_EXPR
1302 57326 : && TREE_CODE (gimple_call_arg (stmt, 1)) == INTEGER_CST
1303 30438497 : && poly_int_tree_p (gimple_call_arg (stmt, 2)))
1304 : {
1305 52295 : dest = TREE_OPERAND (gimple_call_arg (stmt, 0), 0);
1306 52295 : len = gimple_call_arg (stmt, 2);
1307 52295 : val = gimple_call_arg (stmt, 1);
1308 52295 : ao_ref_init_from_ptr_and_size (&read, gimple_call_arg (stmt, 0), len);
1309 52295 : can_use_tbba = false;
1310 : }
1311 30331142 : else if (gimple_store_p (stmt)
1312 30272015 : && gimple_assign_single_p (stmt)
1313 60603157 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == STRING_CST)
1314 : {
1315 26377 : tree str = gimple_assign_rhs1 (stmt);
1316 26377 : dest = gimple_assign_lhs (stmt);
1317 26377 : ao_ref_init (&read, dest);
1318 : /* The string must contain all null char's for now. */
1319 31719 : for (int i = 0; i < TREE_STRING_LENGTH (str); i++)
1320 : {
1321 29131 : if (TREE_STRING_POINTER (str)[i] != 0)
1322 : {
1323 : dest = NULL_TREE;
1324 : break;
1325 : }
1326 : }
1327 : }
1328 : /* A store of integer (scalar, vector or complex) zeros is
1329 : a zero store. */
1330 30304765 : else if (gimple_store_p (stmt)
1331 30245638 : && gimple_assign_single_p (stmt)
1332 60550403 : && integer_zerop (gimple_assign_rhs1 (stmt)))
1333 : {
1334 3556965 : tree rhs = gimple_assign_rhs1 (stmt);
1335 3556965 : tree type = TREE_TYPE (rhs);
1336 3556965 : dest = gimple_assign_lhs (stmt);
1337 3556965 : ao_ref_init (&read, dest);
1338 : /* For integral types, the type precision needs to be a multiply of BITS_PER_UNIT. */
1339 3556965 : if (INTEGRAL_TYPE_P (type)
1340 3556965 : && (TYPE_PRECISION (type) % BITS_PER_UNIT) != 0)
1341 : dest = NULL_TREE;
1342 : }
1343 26747800 : else if (gimple_store_p (stmt)
1344 26688673 : && gimple_assign_single_p (stmt)
1345 26688673 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == CONSTRUCTOR
1346 27460910 : && !gimple_clobber_p (stmt))
1347 : {
1348 713110 : dest = gimple_assign_lhs (stmt);
1349 713110 : ao_ref_init (&read, dest);
1350 : }
1351 :
1352 4137990 : if (dest == NULL_TREE)
1353 26269236 : return;
1354 :
1355 4114201 : if (len == NULL_TREE)
1356 4061906 : len = (TREE_CODE (dest) == COMPONENT_REF
1357 4061906 : ? DECL_SIZE_UNIT (TREE_OPERAND (dest, 1))
1358 1760704 : : TYPE_SIZE_UNIT (TREE_TYPE (dest)));
1359 4061906 : if (len == NULL_TREE
1360 4114201 : || !poly_int_tree_p (len))
1361 : return;
1362 :
1363 : /* Sometimes memset can have no vdef due to invalid declaration of memset (const, etc.). */
1364 35509590 : if (!gimple_vdef (stmt))
1365 : return;
1366 :
1367 : /* This store needs to be on the byte boundary and pointing to an object. */
1368 4114177 : poly_int64 offset;
1369 4114177 : tree dest_base = get_addr_base_and_unit_offset (dest, &offset);
1370 4114177 : if (dest_base == NULL_TREE)
1371 : return;
1372 :
1373 : /* Setup the worklist. */
1374 4028551 : auto_vec<std::pair<tree, unsigned>> worklist;
1375 4028551 : unsigned limit = full_walk ? param_sccvn_max_alias_queries_per_access : 0;
1376 8057102 : worklist.safe_push (std::make_pair (gimple_vdef (stmt), limit));
1377 :
1378 28005938 : while (!worklist.is_empty ())
1379 : {
1380 19948836 : std::pair<tree, unsigned> top = worklist.pop ();
1381 19948836 : tree vdef = top.first;
1382 19948836 : limit = top.second;
1383 19948836 : gimple *use_stmt;
1384 19948836 : imm_use_iterator iter;
1385 64365490 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
1386 : {
1387 : /* Handling PHI nodes might not be worth it so don't. */
1388 24467818 : if (is_a <gphi*> (use_stmt))
1389 2016358 : continue;
1390 :
1391 : /* If this statement does not clobber add the vdef stmt to the
1392 : worklist.
1393 : After hitting the limit, allow clobbers to able to pass through. */
1394 2028925 : if ((limit != 0 || gimple_clobber_p (use_stmt))
1395 20458069 : && gimple_vdef (use_stmt)
1396 39805386 : && !stmt_may_clobber_ref_p_1 (use_stmt, &read,
1397 : /* tbaa_p = */ can_use_tbba))
1398 : {
1399 15920285 : unsigned new_limit = limit == 0 ? 0 : limit - 1;
1400 31840570 : worklist.safe_push (std::make_pair (gimple_vdef (use_stmt),
1401 : new_limit));
1402 : }
1403 :
1404 22451460 : optimize_aggr_zeroprop_1 (stmt, use_stmt, dest_base, offset,
1405 22451460 : val, wi::to_poly_offset (len));
1406 19948836 : }
1407 : }
1408 :
1409 4028551 : }
1410 :
1411 : /* Returns the pointer to the base of the object of the
1412 : reference EXPR and extracts the information about
1413 : the offset of the access, storing it to PBYTESIZE,
1414 : PBYTEPOS and PREVERSEP.
1415 : If the access is not a byte sized or position is not
1416 : on the byte, return NULL. */
1417 : static tree
1418 5327572 : split_core_and_offset_size (tree expr,
1419 : poly_int64 *pbytesize, poly_int64 *pbytepos,
1420 : tree *poffset, int *preversep)
1421 : {
1422 5327572 : tree core;
1423 5327572 : machine_mode mode;
1424 5327572 : int unsignedp, volatilep;
1425 5327572 : poly_int64 bitsize;
1426 5327572 : poly_int64 bitpos;
1427 5327572 : location_t loc = EXPR_LOCATION (expr);
1428 :
1429 5327572 : core = get_inner_reference (expr, &bitsize, &bitpos,
1430 : poffset, &mode, &unsignedp, preversep,
1431 : &volatilep);
1432 10655144 : if (!multiple_p (bitsize, BITS_PER_UNIT, pbytesize))
1433 : return NULL_TREE;
1434 5327572 : if (!multiple_p (bitpos, BITS_PER_UNIT, pbytepos))
1435 : return NULL_TREE;
1436 : /* If we are left with MEM[a + CST] strip that and add it to the
1437 : pbytepos and return a. */
1438 5327572 : if (TREE_CODE (core) == MEM_REF)
1439 : {
1440 1242172 : poly_offset_int tem;
1441 1242172 : tem = wi::to_poly_offset (TREE_OPERAND (core, 1));
1442 1242172 : tem += *pbytepos;
1443 1242172 : if (tem.to_shwi (pbytepos))
1444 1240286 : return TREE_OPERAND (core, 0);
1445 : }
1446 4087286 : core = build_fold_addr_expr_loc (loc, core);
1447 4087286 : STRIP_NOPS (core);
1448 4087286 : return core;
1449 : }
1450 :
1451 : /* Returns a new src based on the
1452 : copy `DEST = SRC` and for the old SRC2.
1453 : Returns null if SRC2 is not related to DEST. */
1454 :
1455 : static tree
1456 1261492 : new_src_based_on_copy (tree src2, tree dest, tree src)
1457 : {
1458 : /* If the second src is not exactly the same as dest,
1459 : try to handle it separately; see it is address/size equivalent.
1460 : Handles `a` and `a.b` and `MEM<char[N]>(&a)` which all have
1461 : the same size and offsets as address/size equivalent.
1462 : This allows copying over a memcpy and also one for copying
1463 : where one field is the same size as the whole struct. */
1464 1261492 : if (operand_equal_p (dest, src2))
1465 : return src;
1466 : /* if both dest and src2 are decls, then we know these 2
1467 : accesses can't be the same. */
1468 721185 : if (DECL_P (dest) && DECL_P (src2))
1469 : return NULL_TREE;
1470 : /* A VCE can't be used with imag/real or BFR so reject them early. */
1471 380832 : if (TREE_CODE (src) == IMAGPART_EXPR
1472 380832 : || TREE_CODE (src) == REALPART_EXPR
1473 380832 : || TREE_CODE (src) == BIT_FIELD_REF)
1474 : return NULL_TREE;
1475 380832 : tree core1, core2;
1476 380832 : poly_int64 bytepos1, bytepos2;
1477 380832 : poly_int64 bytesize1, bytesize2;
1478 380832 : tree toffset1, toffset2;
1479 380832 : int reversep1 = 0;
1480 380832 : int reversep2 = 0;
1481 380832 : poly_int64 diff = 0;
1482 380832 : core1 = split_core_and_offset_size (dest, &bytesize1, &bytepos1,
1483 : &toffset1, &reversep1);
1484 380832 : core2 = split_core_and_offset_size (src2, &bytesize2, &bytepos2,
1485 : &toffset2, &reversep2);
1486 380832 : if (!core1 || !core2)
1487 : return NULL_TREE;
1488 380832 : if (reversep1 != reversep2)
1489 : return NULL_TREE;
1490 : /* The sizes of the 2 accesses need to be the same. */
1491 380832 : if (!known_eq (bytesize1, bytesize2))
1492 : return NULL_TREE;
1493 171744 : if (!operand_equal_p (core1, core2, 0))
1494 : return NULL_TREE;
1495 :
1496 23263 : if (toffset1 && toffset2)
1497 : {
1498 2 : tree type = TREE_TYPE (toffset1);
1499 2 : if (type != TREE_TYPE (toffset2))
1500 0 : toffset2 = fold_convert (type, toffset2);
1501 :
1502 2 : tree tdiff = fold_build2 (MINUS_EXPR, type, toffset1, toffset2);
1503 2 : if (!cst_and_fits_in_hwi (tdiff))
1504 : return NULL_TREE;
1505 :
1506 0 : diff = int_cst_value (tdiff);
1507 0 : }
1508 23261 : else if (toffset1 || toffset2)
1509 : {
1510 : /* If only one of the offsets is non-constant, the difference cannot
1511 : be a constant. */
1512 : return NULL_TREE;
1513 : }
1514 23229 : diff += bytepos1 - bytepos2;
1515 : /* The offset between the 2 need to be 0. */
1516 23229 : if (!known_eq (diff, 0))
1517 : return NULL_TREE;
1518 22384 : return fold_build1 (VIEW_CONVERT_EXPR,TREE_TYPE (src2), src);
1519 : }
1520 :
1521 : /* Returns true if SRC and DEST are the same address such that
1522 : `SRC == DEST;` is considered a nop. This is more than an
1523 : operand_equal_p check as it needs to be similar to
1524 : new_src_based_on_copy. */
1525 :
1526 : static bool
1527 4493879 : same_for_assignment (tree src, tree dest)
1528 : {
1529 4493879 : if (operand_equal_p (dest, src, 0))
1530 : return true;
1531 : /* if both dest and src2 are decls, then we know these 2
1532 : accesses can't be the same. */
1533 4490972 : if (DECL_P (dest) && DECL_P (src))
1534 : return false;
1535 :
1536 2282954 : tree core1, core2;
1537 2282954 : poly_int64 bytepos1, bytepos2;
1538 2282954 : poly_int64 bytesize1, bytesize2;
1539 2282954 : tree toffset1, toffset2;
1540 2282954 : int reversep1 = 0;
1541 2282954 : int reversep2 = 0;
1542 2282954 : poly_int64 diff = 0;
1543 2282954 : core1 = split_core_and_offset_size (dest, &bytesize1, &bytepos1,
1544 : &toffset1, &reversep1);
1545 2282954 : core2 = split_core_and_offset_size (src, &bytesize2, &bytepos2,
1546 : &toffset2, &reversep2);
1547 2282954 : if (!core1 || !core2)
1548 : return false;
1549 2282954 : if (reversep1 != reversep2)
1550 : return false;
1551 : /* The sizes of the 2 accesses need to be the same. */
1552 2282954 : if (!known_eq (bytesize1, bytesize2))
1553 : return false;
1554 2282036 : if (!operand_equal_p (core1, core2, 0))
1555 : return false;
1556 6106 : if (toffset1 && toffset2)
1557 : {
1558 313 : tree type = TREE_TYPE (toffset1);
1559 313 : if (type != TREE_TYPE (toffset2))
1560 0 : toffset2 = fold_convert (type, toffset2);
1561 :
1562 313 : tree tdiff = fold_build2 (MINUS_EXPR, type, toffset1, toffset2);
1563 313 : if (!cst_and_fits_in_hwi (tdiff))
1564 : return false;
1565 :
1566 0 : diff = int_cst_value (tdiff);
1567 0 : }
1568 5793 : else if (toffset1 || toffset2)
1569 : {
1570 : /* If only one of the offsets is non-constant, the difference cannot
1571 : be a constant. */
1572 : return false;
1573 : }
1574 5793 : diff += bytepos1 - bytepos2;
1575 : /* The offset between the 2 need to be 0. */
1576 5793 : if (!known_eq (diff, 0))
1577 : return false;
1578 : return true;
1579 : }
1580 :
1581 : /* Helper function for optimize_agr_copyprop.
1582 : For aggregate copies in USE_STMT, see if DEST
1583 : is on the lhs of USE_STMT and replace it with SRC. */
1584 : static void
1585 1020614 : optimize_agr_copyprop_1 (gimple *stmt, gimple *use_stmt,
1586 : tree dest, tree src)
1587 : {
1588 1020614 : gcc_assert (gimple_assign_load_p (use_stmt)
1589 : && gimple_store_p (use_stmt));
1590 2041228 : if (gimple_has_volatile_ops (use_stmt))
1591 612992 : return;
1592 1020613 : tree dest2 = gimple_assign_lhs (use_stmt);
1593 1020613 : tree src2 = gimple_assign_rhs1 (use_stmt);
1594 : /* If the new store is `src2 = src2;` skip over it. */
1595 1020613 : if (same_for_assignment (src2, dest2))
1596 : return;
1597 1020050 : src = new_src_based_on_copy (src2, dest, src);
1598 1020050 : if (!src)
1599 : return;
1600 : /* For 2 memory references and using a temporary to do the copy,
1601 : don't remove the temporary as the 2 memory references might overlap.
1602 : Note t does not need to be decl as it could be field.
1603 : See PR 22237 for full details.
1604 : E.g.
1605 : t = *a; #DEST = SRC;
1606 : *b = t; #DEST2 = SRC2;
1607 : Cannot be convert into
1608 : t = *a;
1609 : *b = *a;
1610 : Though the following is allowed to be done:
1611 : t = *a;
1612 : *a = t;
1613 : And convert it into:
1614 : t = *a;
1615 : *a = *a;
1616 : */
1617 436005 : if (!operand_equal_p (dest2, src, 0)
1618 436005 : && !DECL_P (dest2) && !DECL_P (src))
1619 : {
1620 : /* If *a and *b have the same base see if
1621 : the offset between the two is greater than
1622 : or equal to the size of the type. */
1623 31722 : poly_int64 offset1, offset2;
1624 31722 : tree len = TYPE_SIZE_UNIT (TREE_TYPE (src));
1625 31722 : if (len == NULL_TREE
1626 31722 : || !tree_fits_poly_int64_p (len))
1627 28383 : return;
1628 31722 : tree base1 = get_addr_base_and_unit_offset (dest2, &offset1);
1629 31722 : tree base2 = get_addr_base_and_unit_offset (src, &offset2);
1630 31722 : poly_int64 size = tree_to_poly_int64 (len);
1631 : /* If the bases are 2 different decls,
1632 : then there can be no overlapping. */
1633 31722 : if (base1 && base2
1634 30854 : && DECL_P (base1) && DECL_P (base2)
1635 1877 : && base1 != base2)
1636 : ;
1637 : /* If we can't figure out the base or the bases are
1638 : not equal then fall back to an alignment check. */
1639 30074 : else if (!base1
1640 30074 : || !base2
1641 30074 : || !operand_equal_p (base1, base2))
1642 : {
1643 29702 : unsigned int align1 = get_object_alignment (src);
1644 29702 : unsigned int align2 = get_object_alignment (dest2);
1645 29702 : align1 /= BITS_PER_UNIT;
1646 29702 : align2 /= BITS_PER_UNIT;
1647 : /* If the alignment of either object is less
1648 : than the size then there is a possibility
1649 : of overlapping. */
1650 29702 : if (maybe_lt (align1, size)
1651 29702 : || maybe_lt (align2, size))
1652 28383 : return;
1653 : }
1654 : /* Make sure [offset1, offset1 + len - 1] does
1655 : not overlap with [offset2, offset2 + len - 1],
1656 : it is ok if they are at the same location though. */
1657 372 : else if (ranges_maybe_overlap_p (offset1, size, offset2, size)
1658 372 : && !known_eq (offset2, offset1))
1659 : return;
1660 : }
1661 :
1662 407622 : if (dump_file && (dump_flags & TDF_DETAILS))
1663 : {
1664 11 : fprintf (dump_file, "Simplified\n ");
1665 11 : print_gimple_stmt (dump_file, use_stmt, 0, dump_flags);
1666 11 : fprintf (dump_file, "after previous\n ");
1667 11 : print_gimple_stmt (dump_file, stmt, 0, dump_flags);
1668 : }
1669 407622 : gimple *orig_stmt = use_stmt;
1670 407622 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
1671 407622 : gimple_assign_set_rhs_from_tree (&gsi, unshare_expr (src));
1672 407622 : update_stmt (use_stmt);
1673 :
1674 407622 : if (dump_file && (dump_flags & TDF_DETAILS))
1675 : {
1676 11 : fprintf (dump_file, "into\n ");
1677 11 : print_gimple_stmt (dump_file, use_stmt, 0, dump_flags);
1678 : }
1679 407622 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, use_stmt))
1680 0 : bitmap_set_bit (to_purge, gimple_bb (stmt)->index);
1681 407622 : statistics_counter_event (cfun, "copy prop for aggregate", 1);
1682 : }
1683 :
1684 : /* Helper function for optimize_agr_copyprop_1, propagate aggregates
1685 : into the arguments of USE_STMT if the argument matches with DEST;
1686 : replacing it with SRC. */
1687 : static void
1688 710786 : optimize_agr_copyprop_arg (gimple *defstmt, gcall *call,
1689 : tree dest, tree src)
1690 : {
1691 710786 : bool changed = false;
1692 2380222 : for (unsigned arg = 0; arg < gimple_call_num_args (call); arg++)
1693 : {
1694 1669436 : tree *argptr = gimple_call_arg_ptr (call, arg);
1695 3149007 : if (TREE_CODE (*argptr) == SSA_NAME
1696 950025 : || is_gimple_min_invariant (*argptr)
1697 1859301 : || TYPE_VOLATILE (TREE_TYPE (*argptr)))
1698 1479571 : continue;
1699 189865 : tree newsrc = new_src_based_on_copy (*argptr, dest, src);
1700 189865 : if (!newsrc)
1701 114662 : continue;
1702 :
1703 75203 : if (dump_file && (dump_flags & TDF_DETAILS))
1704 : {
1705 9 : fprintf (dump_file, "Simplified\n ");
1706 9 : print_gimple_stmt (dump_file, call, 0, dump_flags);
1707 9 : fprintf (dump_file, "after previous\n ");
1708 9 : print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
1709 : }
1710 75203 : *argptr = unshare_expr (newsrc);
1711 75203 : changed = true;
1712 75203 : if (dump_file && (dump_flags & TDF_DETAILS))
1713 : {
1714 9 : fprintf (dump_file, "into\n ");
1715 9 : print_gimple_stmt (dump_file, call, 0, dump_flags);
1716 : }
1717 : }
1718 710786 : if (changed)
1719 75027 : update_stmt (call);
1720 710786 : }
1721 :
1722 : /* Helper function for optimize_agr_copyprop, propagate aggregates
1723 : into the return stmt USE if the operand of the return matches DEST;
1724 : replacing it with SRC. */
1725 : static void
1726 126436 : optimize_agr_copyprop_return (gimple *defstmt, greturn *use,
1727 : tree dest, tree src)
1728 : {
1729 126436 : tree rvalue = gimple_return_retval (use);
1730 126436 : if (!rvalue
1731 82137 : || TREE_CODE (rvalue) == SSA_NAME
1732 73328 : || is_gimple_min_invariant (rvalue)
1733 199356 : || TYPE_VOLATILE (TREE_TYPE (rvalue)))
1734 53517 : return;
1735 :
1736 : /* `return <retval>;` is already the best it could be.
1737 : Likewise `return *<retval>_N(D)`. */
1738 72919 : if (TREE_CODE (rvalue) == RESULT_DECL
1739 72919 : || (TREE_CODE (rvalue) == MEM_REF
1740 0 : && TREE_CODE (TREE_OPERAND (rvalue, 0)) == SSA_NAME
1741 0 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (rvalue, 0)))
1742 : == RESULT_DECL))
1743 : return;
1744 51577 : tree newsrc = new_src_based_on_copy (rvalue, dest, src);
1745 51577 : if (!newsrc)
1746 : return;
1747 : /* Currently only support non-global vars.
1748 : See PR 124099 on enumtls not supporting expanding for GIMPLE_RETURN.
1749 : FIXME: could support VCEs too? */
1750 51483 : if (!VAR_P (newsrc) || is_global_var (newsrc))
1751 : return;
1752 25893 : if (dump_file && (dump_flags & TDF_DETAILS))
1753 : {
1754 1 : fprintf (dump_file, "Simplified\n ");
1755 1 : print_gimple_stmt (dump_file, use, 0, dump_flags);
1756 1 : fprintf (dump_file, "after previous\n ");
1757 1 : print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
1758 : }
1759 25893 : gimple_return_set_retval (use, newsrc);
1760 25893 : if (dump_file && (dump_flags & TDF_DETAILS))
1761 : {
1762 1 : fprintf (dump_file, "into\n ");
1763 1 : print_gimple_stmt (dump_file, use, 0, dump_flags);
1764 : }
1765 25893 : update_stmt (use);
1766 : }
1767 :
1768 : /* Optimizes
1769 : DEST = SRC;
1770 : DEST2 = DEST; # DEST2 = SRC2;
1771 : into
1772 : DEST = SRC;
1773 : DEST2 = SRC;
1774 : STMT is the first statement and SRC is the common
1775 : between the statements.
1776 :
1777 : Also optimizes:
1778 : DEST = SRC;
1779 : call_func(..., DEST, ...);
1780 : into:
1781 : DEST = SRC;
1782 : call_func(..., SRC, ...);
1783 :
1784 : */
1785 : static void
1786 3882104 : optimize_agr_copyprop (gimple *stmt)
1787 : {
1788 7764208 : if (gimple_has_volatile_ops (stmt))
1789 411428 : return;
1790 :
1791 : /* Can't prop if the statement could throw. */
1792 3880893 : if (stmt_could_throw_p (cfun, stmt))
1793 : return;
1794 :
1795 3473266 : tree dest = gimple_assign_lhs (stmt);
1796 3473266 : tree src = gimple_assign_rhs1 (stmt);
1797 : /* If the statement is `src = src;` then ignore it. */
1798 3473266 : if (same_for_assignment (dest, src))
1799 : return;
1800 :
1801 3470676 : tree vdef = gimple_vdef (stmt);
1802 3470676 : imm_use_iterator iter;
1803 3470676 : gimple *use_stmt;
1804 13487966 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
1805 : {
1806 6546614 : if (gimple_assign_load_p (use_stmt)
1807 6546614 : && gimple_store_p (use_stmt))
1808 1020614 : optimize_agr_copyprop_1 (stmt, use_stmt, dest, src);
1809 5526000 : else if (is_gimple_call (use_stmt))
1810 710786 : optimize_agr_copyprop_arg (stmt, as_a<gcall*>(use_stmt), dest, src);
1811 4815214 : else if (is_a<greturn*> (use_stmt))
1812 126436 : optimize_agr_copyprop_return (stmt, as_a<greturn*>(use_stmt), dest, src);
1813 3470676 : }
1814 : }
1815 :
1816 : /* Simple DSE of the lhs from a clobber STMT.
1817 : This is used mostly to clean up from optimize_agr_copyprop and
1818 : to remove (exactly one) extra copy that might later on confuse SRA.
1819 : An example is:
1820 : ;; write to a and such.
1821 : b = a; // This statement is to be removed
1822 : b = {CLOBBER};
1823 : SRA will totally scalarize b (which means also a) here for the extra copy
1824 : which is not something welcomed. So removing the copy will
1825 : allow SRA to move the scalarization of a further down or not at all.
1826 : */
1827 : static void
1828 7314041 : do_simple_agr_dse (gassign *stmt, bool full_walk)
1829 : {
1830 : /* Don't do this while in -Og as we want to keep around the copy
1831 : for debuggability. */
1832 7314041 : if (optimize_debug)
1833 5052421 : return;
1834 7310648 : ao_ref read;
1835 7310648 : basic_block bb = gimple_bb (stmt);
1836 7310648 : tree lhs = gimple_assign_lhs (stmt);
1837 : /* Only handle clobbers of a full decl. */
1838 7310648 : if (!DECL_P (lhs))
1839 : return;
1840 6602125 : ao_ref_init (&read, lhs);
1841 6602125 : tree vuse = gimple_vuse (stmt);
1842 6602125 : unsigned limit = full_walk ? param_sccvn_max_alias_queries_per_access : 4;
1843 16871840 : while (limit)
1844 : {
1845 16858452 : gimple *ostmt = SSA_NAME_DEF_STMT (vuse);
1846 : /* Don't handle phis, just declare to be done. */
1847 16858452 : if (is_a<gphi*>(ostmt) || gimple_nop_p (ostmt))
1848 : break;
1849 14610220 : basic_block obb = gimple_bb (ostmt);
1850 : /* If the clobber is not fully dominating the statement define,
1851 : then it is not "simple" to detect if the define is fully clobbered. */
1852 14610220 : if (obb != bb && !dominated_by_p (CDI_DOMINATORS, bb, obb))
1853 4340505 : return;
1854 14610220 : gimple *use_stmt;
1855 14610220 : imm_use_iterator iter;
1856 59112279 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, gimple_vdef (ostmt))
1857 : {
1858 17239811 : basic_block ubb = gimple_bb (use_stmt);
1859 17239811 : if (stmt == use_stmt)
1860 5068662 : continue;
1861 : /* If the use is a clobber for lhs,
1862 : then it can be safely skipped; this happens with eh
1863 : and sometimes jump threading. */
1864 12171149 : if (gimple_clobber_p (use_stmt)
1865 12171149 : && lhs == gimple_assign_lhs (use_stmt))
1866 174555 : continue;
1867 : /* If the use is a phi and it is single use then check if that single use
1868 : is a clobber and lhs is the same. */
1869 11996594 : if (gphi *use_phi = dyn_cast<gphi*>(use_stmt))
1870 : {
1871 360050 : use_operand_p ou;
1872 360050 : gimple *ostmt;
1873 360050 : if (single_imm_use (gimple_phi_result (use_phi), &ou, &ostmt)
1874 302113 : && gimple_clobber_p (ostmt)
1875 603497 : && lhs == gimple_assign_lhs (ostmt))
1876 67457 : continue;
1877 : /* A phi node will never be dominating the clobber. */
1878 292593 : return;
1879 : }
1880 : /* The use needs to be dominating the clobber. */
1881 1471148 : if ((ubb != bb && !dominated_by_p (CDI_DOMINATORS, bb, ubb))
1882 12379044 : || ref_maybe_used_by_stmt_p (use_stmt, &read, false))
1883 1209302 : return;
1884 : /* Count the above alias lookup towards the limit. */
1885 10427242 : limit--;
1886 10427242 : if (limit == 0)
1887 : return;
1888 1958192 : }
1889 12652028 : vuse = gimple_vuse (ostmt);
1890 : /* This is a call with an assignment to the clobber decl,
1891 : remove the lhs or the whole stmt if it was pure/const. */
1892 12652028 : if (is_a <gcall*>(ostmt)
1893 12652028 : && lhs == gimple_call_lhs (ostmt))
1894 : {
1895 : /* Don't remove stores/statements that are needed for non-call
1896 : eh to work. */
1897 4050 : if (stmt_unremovable_because_of_non_call_eh_p (cfun, ostmt))
1898 : return;
1899 : /* If we delete a stmt that could throw, mark the block
1900 : in to_purge to cleanup afterwards. */
1901 4044 : if (stmt_could_throw_p (cfun, ostmt))
1902 1006 : bitmap_set_bit (to_purge, obb->index);
1903 4044 : int flags = gimple_call_flags (ostmt);
1904 4044 : if ((flags & (ECF_PURE|ECF_CONST|ECF_NOVOPS))
1905 203 : && !(flags & (ECF_LOOPING_CONST_OR_PURE)))
1906 : {
1907 119 : gimple_stmt_iterator gsi = gsi_for_stmt (ostmt);
1908 119 : if (dump_file && (dump_flags & TDF_DETAILS))
1909 : {
1910 14 : fprintf (dump_file, "Removing dead call store stmt ");
1911 14 : print_gimple_stmt (dump_file, ostmt, 0);
1912 14 : fprintf (dump_file, "\n");
1913 : }
1914 119 : unlink_stmt_vdef (ostmt);
1915 119 : release_defs (ostmt);
1916 119 : gsi_remove (&gsi, true);
1917 119 : statistics_counter_event (cfun, "delete call dead store", 1);
1918 : /* Only remove the first store previous statement. */
1919 119 : return;
1920 : }
1921 : /* Make sure we do not remove a return slot we cannot reconstruct
1922 : later. */
1923 3925 : if (gimple_call_return_slot_opt_p (as_a <gcall *>(ostmt))
1924 3925 : && (TREE_ADDRESSABLE (TREE_TYPE (gimple_call_fntype (ostmt)))
1925 541 : || !poly_int_tree_p
1926 541 : (TYPE_SIZE (TREE_TYPE (gimple_call_fntype (ostmt))))))
1927 : return;
1928 678 : if (dump_file && (dump_flags & TDF_DETAILS))
1929 : {
1930 6 : fprintf (dump_file, "Removing lhs of call stmt ");
1931 6 : print_gimple_stmt (dump_file, ostmt, 0);
1932 6 : fprintf (dump_file, "\n");
1933 : }
1934 678 : gimple_call_set_lhs (ostmt, NULL_TREE);
1935 678 : update_stmt (ostmt);
1936 678 : statistics_counter_event (cfun, "removed lhs call", 1);
1937 678 : return;
1938 : }
1939 : /* This an assignment store to the clobbered decl,
1940 : then maybe remove it. */
1941 12647978 : if (is_a <gassign*>(ostmt)
1942 10693656 : && gimple_store_p (ostmt)
1943 10693656 : && !gimple_clobber_p (ostmt)
1944 15833442 : && lhs == gimple_assign_lhs (ostmt))
1945 : {
1946 : /* Don't remove stores/statements that are needed for non-call
1947 : eh to work. */
1948 165843 : if (stmt_unremovable_because_of_non_call_eh_p (cfun, ostmt))
1949 : return;
1950 : /* If we delete a stmt that could throw, mark the block
1951 : in to_purge to cleanup afterwards. */
1952 160757 : if (stmt_could_throw_p (cfun, ostmt))
1953 0 : bitmap_set_bit (to_purge, obb->index);
1954 160757 : gimple_stmt_iterator gsi = gsi_for_stmt (ostmt);
1955 160757 : if (dump_file && (dump_flags & TDF_DETAILS))
1956 : {
1957 12 : fprintf (dump_file, "Removing dead store stmt ");
1958 12 : print_gimple_stmt (dump_file, ostmt, 0);
1959 12 : fprintf (dump_file, "\n");
1960 : }
1961 160757 : unlink_stmt_vdef (ostmt);
1962 160757 : release_defs (ostmt);
1963 160757 : gsi_remove (&gsi, true);
1964 160757 : statistics_counter_event (cfun, "delete dead store", 1);
1965 : /* Only remove the first store previous statement. */
1966 160757 : return;
1967 : }
1968 : /* If the statement uses or maybe writes to the decl,
1969 : then nothing is to be removed. Don't know if the write
1970 : to the decl is partial write or a full one so the need
1971 : to stop.
1972 : e.g.
1973 : b.c = a;
1974 : Easier to stop here rather than do a full partial
1975 : dse of this statement.
1976 : b = {CLOBBER}; */
1977 12482135 : if (stmt_may_clobber_ref_p_1 (ostmt, &read, false)
1978 12482135 : || ref_maybe_used_by_stmt_p (ostmt, &read, false))
1979 2212420 : return;
1980 10269715 : limit--;
1981 : }
1982 : }
1983 :
1984 : /* Optimizes builtin memcmps for small constant sizes.
1985 : GSI_P is the GSI for the call. STMT is the call itself.
1986 : */
1987 :
1988 : static bool
1989 466027 : simplify_builtin_memcmp (gimple_stmt_iterator *gsi_p, gcall *stmt)
1990 : {
1991 : /* Make sure memcmp arguments are the correct type. */
1992 466027 : if (gimple_call_num_args (stmt) != 3)
1993 : return false;
1994 466027 : tree arg1 = gimple_call_arg (stmt, 0);
1995 466027 : tree arg2 = gimple_call_arg (stmt, 1);
1996 466027 : tree len = gimple_call_arg (stmt, 2);
1997 :
1998 466027 : if (!POINTER_TYPE_P (TREE_TYPE (arg1)))
1999 : return false;
2000 466027 : if (!POINTER_TYPE_P (TREE_TYPE (arg2)))
2001 : return false;
2002 466027 : if (!INTEGRAL_TYPE_P (TREE_TYPE (len)))
2003 : return false;
2004 :
2005 : /* The return value of the memcmp has to be used
2006 : equality comparison to zero. */
2007 466027 : tree res = gimple_call_lhs (stmt);
2008 :
2009 466027 : if (!res || !use_in_zero_equality (res))
2010 14721 : return false;
2011 :
2012 451306 : unsigned HOST_WIDE_INT leni;
2013 :
2014 451306 : if (tree_fits_uhwi_p (len)
2015 629026 : && (leni = tree_to_uhwi (len)) <= GET_MODE_SIZE (word_mode)
2016 531506 : && pow2p_hwi (leni))
2017 : {
2018 19024 : leni *= CHAR_TYPE_SIZE;
2019 19024 : unsigned align1 = get_pointer_alignment (arg1);
2020 19024 : unsigned align2 = get_pointer_alignment (arg2);
2021 19024 : unsigned align = MIN (align1, align2);
2022 19024 : scalar_int_mode mode;
2023 19024 : if (int_mode_for_size (leni, 1).exists (&mode)
2024 19024 : && (align >= leni || !targetm.slow_unaligned_access (mode, align)))
2025 : {
2026 19024 : location_t loc = gimple_location (stmt);
2027 19024 : tree type, off;
2028 19024 : type = build_nonstandard_integer_type (leni, 1);
2029 38048 : gcc_assert (known_eq (GET_MODE_BITSIZE (TYPE_MODE (type)), leni));
2030 19024 : tree ptrtype = build_pointer_type_for_mode (char_type_node,
2031 : ptr_mode, true);
2032 19024 : off = build_int_cst (ptrtype, 0);
2033 :
2034 : /* Create unaligned types if needed. */
2035 19024 : tree type1 = type, type2 = type;
2036 19024 : if (TYPE_ALIGN (type1) > align1)
2037 7702 : type1 = build_aligned_type (type1, align1);
2038 19024 : if (TYPE_ALIGN (type2) > align2)
2039 8197 : type2 = build_aligned_type (type2, align2);
2040 :
2041 19024 : arg1 = build2_loc (loc, MEM_REF, type1, arg1, off);
2042 19024 : arg2 = build2_loc (loc, MEM_REF, type2, arg2, off);
2043 19024 : tree tem1 = fold_const_aggregate_ref (arg1);
2044 19024 : if (tem1)
2045 222 : arg1 = tem1;
2046 19024 : tree tem2 = fold_const_aggregate_ref (arg2);
2047 19024 : if (tem2)
2048 7370 : arg2 = tem2;
2049 19024 : res = fold_convert_loc (loc, TREE_TYPE (res),
2050 : fold_build2_loc (loc, NE_EXPR,
2051 : boolean_type_node,
2052 : arg1, arg2));
2053 19024 : gimplify_and_update_call_from_tree (gsi_p, res);
2054 19024 : return true;
2055 : }
2056 : }
2057 :
2058 : /* Replace memcmp with memcmp_eq if the above fails. */
2059 432282 : if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) == BUILT_IN_MEMCMP_EQ)
2060 : return false;
2061 342422 : if (!fold_before_rtl_expansion_p ())
2062 : return false;
2063 89860 : gimple_call_set_fndecl (stmt, builtin_decl_explicit (BUILT_IN_MEMCMP_EQ));
2064 89860 : update_stmt (stmt);
2065 89860 : return true;
2066 : }
2067 :
2068 : /* Optimizes builtin memchrs for small constant sizes with a const string.
2069 : GSI_P is the GSI for the call. STMT is the call itself.
2070 : */
2071 :
2072 : static bool
2073 12194 : simplify_builtin_memchr (gimple_stmt_iterator *gsi_p, gcall *stmt)
2074 : {
2075 12194 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
2076 : return false;
2077 :
2078 12194 : if (gimple_call_num_args (stmt) != 3)
2079 : return false;
2080 :
2081 12194 : tree res = gimple_call_lhs (stmt);
2082 12194 : if (!res || !use_in_zero_equality (res))
2083 10775 : return false;
2084 :
2085 1419 : tree ptr = gimple_call_arg (stmt, 0);
2086 1419 : if (TREE_CODE (ptr) != ADDR_EXPR
2087 1419 : || TREE_CODE (TREE_OPERAND (ptr, 0)) != STRING_CST)
2088 : return false;
2089 :
2090 412 : unsigned HOST_WIDE_INT slen
2091 412 : = TREE_STRING_LENGTH (TREE_OPERAND (ptr, 0));
2092 : /* It must be a non-empty string constant. */
2093 412 : if (slen < 2)
2094 : return false;
2095 :
2096 : /* For -Os, only simplify strings with a single character. */
2097 408 : if (!optimize_bb_for_speed_p (gimple_bb (stmt))
2098 408 : && slen > 2)
2099 : return false;
2100 :
2101 392 : tree size = gimple_call_arg (stmt, 2);
2102 : /* Size must be a constant which is <= UNITS_PER_WORD and
2103 : <= the string length. */
2104 392 : if (!tree_fits_uhwi_p (size))
2105 : return false;
2106 :
2107 392 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (size);
2108 393 : if (sz == 0 || sz > UNITS_PER_WORD || sz >= slen)
2109 : return false;
2110 :
2111 340 : tree ch = gimple_call_arg (stmt, 1);
2112 340 : location_t loc = gimple_location (stmt);
2113 340 : if (!useless_type_conversion_p (char_type_node,
2114 340 : TREE_TYPE (ch)))
2115 340 : ch = fold_convert_loc (loc, char_type_node, ch);
2116 340 : const char *p = TREE_STRING_POINTER (TREE_OPERAND (ptr, 0));
2117 340 : unsigned int isize = sz;
2118 340 : tree *op = XALLOCAVEC (tree, isize);
2119 1209 : for (unsigned int i = 0; i < isize; i++)
2120 : {
2121 869 : op[i] = build_int_cst (char_type_node, p[i]);
2122 869 : op[i] = fold_build2_loc (loc, EQ_EXPR, boolean_type_node,
2123 : op[i], ch);
2124 : }
2125 869 : for (unsigned int i = isize - 1; i >= 1; i--)
2126 529 : op[i - 1] = fold_convert_loc (loc, boolean_type_node,
2127 : fold_build2_loc (loc,
2128 : BIT_IOR_EXPR,
2129 : boolean_type_node,
2130 529 : op[i - 1],
2131 529 : op[i]));
2132 340 : res = fold_convert_loc (loc, TREE_TYPE (res), op[0]);
2133 340 : gimplify_and_update_call_from_tree (gsi_p, res);
2134 340 : return true;
2135 : }
2136 :
2137 : /* *GSI_P is a GIMPLE_CALL to a builtin function.
2138 : Optimize
2139 : memcpy (p, "abcd", 4); // STMT1
2140 : memset (p + 4, ' ', 3); // STMT2
2141 : into
2142 : memcpy (p, "abcd ", 7);
2143 : call if the latter can be stored by pieces during expansion.
2144 : */
2145 :
2146 : static bool
2147 111422 : simplify_builtin_memcpy_memset (gimple_stmt_iterator *gsi_p, gcall *stmt2)
2148 : {
2149 111422 : if (gimple_call_num_args (stmt2) != 3
2150 111422 : || gimple_call_lhs (stmt2)
2151 : || CHAR_BIT != 8
2152 111422 : || BITS_PER_UNIT != 8)
2153 : return false;
2154 :
2155 214175 : tree vuse = gimple_vuse (stmt2);
2156 105257 : if (vuse == NULL)
2157 : return false;
2158 105241 : gimple *stmt1 = SSA_NAME_DEF_STMT (vuse);
2159 :
2160 105241 : tree callee1;
2161 105241 : tree ptr1, src1, str1, off1, len1, lhs1;
2162 105241 : tree ptr2 = gimple_call_arg (stmt2, 0);
2163 105241 : tree val2 = gimple_call_arg (stmt2, 1);
2164 105241 : tree len2 = gimple_call_arg (stmt2, 2);
2165 105241 : tree diff, vdef, new_str_cst;
2166 105241 : gimple *use_stmt;
2167 105241 : unsigned int ptr1_align;
2168 105241 : unsigned HOST_WIDE_INT src_len;
2169 105241 : char *src_buf;
2170 105241 : use_operand_p use_p;
2171 :
2172 105241 : if (!tree_fits_shwi_p (val2)
2173 100822 : || !tree_fits_uhwi_p (len2)
2174 169605 : || compare_tree_int (len2, 1024) == 1)
2175 46434 : return false;
2176 :
2177 58807 : if (is_gimple_call (stmt1))
2178 : {
2179 : /* If first stmt is a call, it needs to be memcpy
2180 : or mempcpy, with string literal as second argument and
2181 : constant length. */
2182 30562 : callee1 = gimple_call_fndecl (stmt1);
2183 30562 : if (callee1 == NULL_TREE
2184 30446 : || !fndecl_built_in_p (callee1, BUILT_IN_NORMAL)
2185 57379 : || gimple_call_num_args (stmt1) != 3)
2186 : return false;
2187 25495 : if (DECL_FUNCTION_CODE (callee1) != BUILT_IN_MEMCPY
2188 25495 : && DECL_FUNCTION_CODE (callee1) != BUILT_IN_MEMPCPY)
2189 : return false;
2190 11100 : ptr1 = gimple_call_arg (stmt1, 0);
2191 11100 : src1 = gimple_call_arg (stmt1, 1);
2192 11100 : len1 = gimple_call_arg (stmt1, 2);
2193 11100 : lhs1 = gimple_call_lhs (stmt1);
2194 11100 : if (!tree_fits_uhwi_p (len1))
2195 : return false;
2196 11013 : str1 = string_constant (src1, &off1, NULL, NULL);
2197 11013 : if (str1 == NULL_TREE)
2198 : return false;
2199 5126 : if (!tree_fits_uhwi_p (off1)
2200 5126 : || compare_tree_int (off1, TREE_STRING_LENGTH (str1) - 1) > 0
2201 5126 : || compare_tree_int (len1, TREE_STRING_LENGTH (str1)
2202 5126 : - tree_to_uhwi (off1)) > 0
2203 5126 : || TREE_CODE (TREE_TYPE (str1)) != ARRAY_TYPE
2204 15378 : || TYPE_MODE (TREE_TYPE (TREE_TYPE (str1)))
2205 5126 : != TYPE_MODE (char_type_node))
2206 0 : return false;
2207 : }
2208 28245 : else if (gimple_assign_single_p (stmt1))
2209 : {
2210 : /* Otherwise look for length 1 memcpy optimized into
2211 : assignment. */
2212 17297 : ptr1 = gimple_assign_lhs (stmt1);
2213 17297 : src1 = gimple_assign_rhs1 (stmt1);
2214 17297 : if (TREE_CODE (ptr1) != MEM_REF
2215 3432 : || TYPE_MODE (TREE_TYPE (ptr1)) != TYPE_MODE (char_type_node)
2216 18275 : || !tree_fits_shwi_p (src1))
2217 16955 : return false;
2218 342 : ptr1 = build_fold_addr_expr (ptr1);
2219 342 : STRIP_USELESS_TYPE_CONVERSION (ptr1);
2220 342 : callee1 = NULL_TREE;
2221 342 : len1 = size_one_node;
2222 342 : lhs1 = NULL_TREE;
2223 342 : off1 = size_zero_node;
2224 342 : str1 = NULL_TREE;
2225 : }
2226 : else
2227 : return false;
2228 :
2229 5468 : diff = constant_pointer_difference (ptr1, ptr2);
2230 5468 : if (diff == NULL && lhs1 != NULL)
2231 : {
2232 7 : diff = constant_pointer_difference (lhs1, ptr2);
2233 7 : if (DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY
2234 7 : && diff != NULL)
2235 7 : diff = size_binop (PLUS_EXPR, diff,
2236 : fold_convert (sizetype, len1));
2237 : }
2238 : /* If the difference between the second and first destination pointer
2239 : is not constant, or is bigger than memcpy length, bail out. */
2240 5468 : if (diff == NULL
2241 4591 : || !tree_fits_uhwi_p (diff)
2242 4591 : || tree_int_cst_lt (len1, diff)
2243 9803 : || compare_tree_int (diff, 1024) == 1)
2244 1133 : return false;
2245 :
2246 : /* Use maximum of difference plus memset length and memcpy length
2247 : as the new memcpy length, if it is too big, bail out. */
2248 4335 : src_len = tree_to_uhwi (diff);
2249 4335 : src_len += tree_to_uhwi (len2);
2250 4335 : if (src_len < tree_to_uhwi (len1))
2251 : src_len = tree_to_uhwi (len1);
2252 4335 : if (src_len > 1024)
2253 : return false;
2254 :
2255 : /* If mempcpy value is used elsewhere, bail out, as mempcpy
2256 : with bigger length will return different result. */
2257 4335 : if (lhs1 != NULL_TREE
2258 64 : && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY
2259 4342 : && (TREE_CODE (lhs1) != SSA_NAME
2260 7 : || !single_imm_use (lhs1, &use_p, &use_stmt)
2261 7 : || use_stmt != stmt2))
2262 0 : return false;
2263 :
2264 : /* If anything reads memory in between memcpy and memset
2265 : call, the modified memcpy call might change it. */
2266 4335 : vdef = gimple_vdef (stmt1);
2267 4335 : if (vdef != NULL
2268 4335 : && (!single_imm_use (vdef, &use_p, &use_stmt)
2269 3616 : || use_stmt != stmt2))
2270 : return false;
2271 :
2272 3616 : ptr1_align = get_pointer_alignment (ptr1);
2273 : /* Construct the new source string literal. */
2274 3616 : src_buf = XALLOCAVEC (char, src_len + 1);
2275 3616 : if (callee1)
2276 3450 : memcpy (src_buf,
2277 3450 : TREE_STRING_POINTER (str1) + tree_to_uhwi (off1),
2278 : tree_to_uhwi (len1));
2279 : else
2280 166 : src_buf[0] = tree_to_shwi (src1);
2281 3616 : memset (src_buf + tree_to_uhwi (diff),
2282 3616 : tree_to_shwi (val2), tree_to_uhwi (len2));
2283 3616 : src_buf[src_len] = '\0';
2284 : /* Neither builtin_strncpy_read_str nor builtin_memcpy_read_str
2285 : handle embedded '\0's. */
2286 3616 : if (strlen (src_buf) != src_len)
2287 : return false;
2288 3522 : rtl_profile_for_bb (gimple_bb (stmt2));
2289 : /* If the new memcpy wouldn't be emitted by storing the literal
2290 : by pieces, this optimization might enlarge .rodata too much,
2291 : as commonly used string literals couldn't be shared any
2292 : longer. */
2293 3522 : if (!can_store_by_pieces (src_len,
2294 : builtin_strncpy_read_str,
2295 : src_buf, ptr1_align, false))
2296 : return false;
2297 :
2298 2632 : new_str_cst = build_string_literal (src_len, src_buf);
2299 2632 : if (callee1)
2300 : {
2301 : /* If STMT1 is a mem{,p}cpy call, adjust it and remove
2302 : memset call. */
2303 2504 : if (lhs1 && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY)
2304 7 : gimple_call_set_lhs (stmt1, NULL_TREE);
2305 2504 : gimple_call_set_arg (stmt1, 1, new_str_cst);
2306 2504 : gimple_call_set_arg (stmt1, 2,
2307 2504 : build_int_cst (TREE_TYPE (len1), src_len));
2308 2504 : update_stmt (stmt1);
2309 2504 : unlink_stmt_vdef (stmt2);
2310 2504 : gsi_replace (gsi_p, gimple_build_nop (), false);
2311 2504 : fwprop_invalidate_lattice (gimple_get_lhs (stmt2));
2312 2504 : release_defs (stmt2);
2313 2504 : if (lhs1 && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY)
2314 : {
2315 7 : fwprop_invalidate_lattice (lhs1);
2316 7 : release_ssa_name (lhs1);
2317 : }
2318 2504 : return true;
2319 : }
2320 : else
2321 : {
2322 : /* Otherwise, if STMT1 is length 1 memcpy optimized into
2323 : assignment, remove STMT1 and change memset call into
2324 : memcpy call. */
2325 128 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt1);
2326 :
2327 128 : if (!is_gimple_val (ptr1))
2328 12 : ptr1 = force_gimple_operand_gsi (gsi_p, ptr1, true, NULL_TREE,
2329 : true, GSI_SAME_STMT);
2330 128 : tree fndecl = builtin_decl_explicit (BUILT_IN_MEMCPY);
2331 128 : gimple_call_set_fndecl (stmt2, fndecl);
2332 128 : gimple_call_set_fntype (stmt2,
2333 128 : TREE_TYPE (fndecl));
2334 128 : gimple_call_set_arg (stmt2, 0, ptr1);
2335 128 : gimple_call_set_arg (stmt2, 1, new_str_cst);
2336 128 : gimple_call_set_arg (stmt2, 2,
2337 128 : build_int_cst (TREE_TYPE (len2), src_len));
2338 128 : unlink_stmt_vdef (stmt1);
2339 128 : gsi_remove (&gsi, true);
2340 128 : fwprop_invalidate_lattice (gimple_get_lhs (stmt1));
2341 128 : release_defs (stmt1);
2342 128 : update_stmt (stmt2);
2343 128 : return false;
2344 : }
2345 : }
2346 :
2347 :
2348 : /* Try to optimize out __builtin_stack_restore. Optimize it out
2349 : if there is another __builtin_stack_restore in the same basic
2350 : block and no calls or ASM_EXPRs are in between, or if this block's
2351 : only outgoing edge is to EXIT_BLOCK and there are no calls or
2352 : ASM_EXPRs after this __builtin_stack_restore.
2353 : Note restore right before a noreturn function is not needed.
2354 : And skip some cheap calls that will most likely become an instruction.
2355 : Restoring the stack before a call is important to be able to keep
2356 : stack usage down so that call does not run out of stack. */
2357 :
2358 :
2359 : static bool
2360 10369 : optimize_stack_restore (gimple_stmt_iterator *gsi, gimple *call)
2361 : {
2362 10369 : if (!fold_before_rtl_expansion_p ())
2363 : return false;
2364 2525 : tree callee;
2365 2525 : gimple *stmt;
2366 :
2367 2525 : basic_block bb = gsi_bb (*gsi);
2368 :
2369 2525 : if (gimple_call_num_args (call) != 1
2370 2525 : || TREE_CODE (gimple_call_arg (call, 0)) != SSA_NAME
2371 5050 : || !POINTER_TYPE_P (TREE_TYPE (gimple_call_arg (call, 0))))
2372 : return false;
2373 :
2374 2525 : gimple_stmt_iterator i = *gsi;
2375 6288 : for (gsi_next (&i); !gsi_end_p (i); gsi_next (&i))
2376 : {
2377 4218 : stmt = gsi_stmt (i);
2378 4218 : if (is_a<gasm*> (stmt))
2379 : return false;
2380 4217 : gcall *call = dyn_cast<gcall*>(stmt);
2381 4217 : if (!call)
2382 3554 : continue;
2383 :
2384 : /* We can remove the restore in front of noreturn
2385 : calls. Since the restore will happen either
2386 : via an unwind/longjmp or not at all. */
2387 663 : if (gimple_call_noreturn_p (call))
2388 : break;
2389 :
2390 : /* Internal calls are ok, to bypass
2391 : check first since fndecl will be null. */
2392 647 : if (gimple_call_internal_p (call))
2393 1 : continue;
2394 :
2395 646 : callee = gimple_call_fndecl (call);
2396 : /* Non-builtin calls are not ok. */
2397 646 : if (!callee
2398 646 : || !fndecl_built_in_p (callee))
2399 : return false;
2400 :
2401 : /* Do not remove stack updates before strub leave. */
2402 570 : if (fndecl_built_in_p (callee, BUILT_IN___STRUB_LEAVE)
2403 : /* Alloca calls are not ok either. */
2404 570 : || fndecl_builtin_alloc_p (callee))
2405 : return false;
2406 :
2407 355 : if (fndecl_built_in_p (callee, BUILT_IN_STACK_RESTORE))
2408 52 : goto second_stack_restore;
2409 :
2410 : /* If not a simple or inexpensive builtin, then it is not ok either. */
2411 303 : if (!is_simple_builtin (callee)
2412 303 : && !is_inexpensive_builtin (callee))
2413 : return false;
2414 : }
2415 :
2416 : /* Allow one successor of the exit block, or zero successors. */
2417 2086 : switch (EDGE_COUNT (bb->succs))
2418 : {
2419 : case 0:
2420 : break;
2421 1999 : case 1:
2422 1999 : if (single_succ_edge (bb)->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
2423 : return false;
2424 : break;
2425 : default:
2426 : return false;
2427 : }
2428 1729 : second_stack_restore:
2429 :
2430 : /* If there's exactly one use, then zap the call to __builtin_stack_save.
2431 : If there are multiple uses, then the last one should remove the call.
2432 : In any case, whether the call to __builtin_stack_save can be removed
2433 : or not is irrelevant to removing the call to __builtin_stack_restore. */
2434 1729 : if (has_single_use (gimple_call_arg (call, 0)))
2435 : {
2436 1559 : gimple *stack_save = SSA_NAME_DEF_STMT (gimple_call_arg (call, 0));
2437 1559 : if (is_gimple_call (stack_save))
2438 : {
2439 1557 : callee = gimple_call_fndecl (stack_save);
2440 1557 : if (callee && fndecl_built_in_p (callee, BUILT_IN_STACK_SAVE))
2441 : {
2442 1557 : gimple_stmt_iterator stack_save_gsi;
2443 1557 : tree rhs;
2444 :
2445 1557 : stack_save_gsi = gsi_for_stmt (stack_save);
2446 1557 : rhs = build_int_cst (TREE_TYPE (gimple_call_arg (call, 0)), 0);
2447 1557 : replace_call_with_value (&stack_save_gsi, rhs);
2448 : }
2449 : }
2450 : }
2451 :
2452 : /* No effect, so the statement will be deleted. */
2453 1729 : replace_call_with_value (gsi, NULL_TREE);
2454 1729 : return true;
2455 : }
2456 :
2457 : /* Optimizes strlen (s) ==/!= 0 to *s ==/!= 0. */
2458 : static bool
2459 61840 : optimize_strlen_comp (gimple_stmt_iterator *gsi, gimple *call)
2460 : {
2461 61840 : if (!fold_before_rtl_expansion_p ())
2462 : return false;
2463 :
2464 13311 : tree lhs = gimple_call_lhs (call);
2465 13311 : if (lhs == NULL_TREE || use_in_zero_equality (lhs, true) == NULL)
2466 13188 : return false;
2467 :
2468 : /* The string passed to strlen. */
2469 123 : tree ptr = gimple_call_arg (call, 0);
2470 :
2471 : /* Dereference the string. */
2472 123 : tree deref = fold_build2 (MEM_REF, char_type_node, ptr,
2473 : build_zero_cst (ptr_type_node));
2474 :
2475 : /* Perform a type conversion. */
2476 123 : deref = fold_convert_loc (gimple_location (call),
2477 123 : TREE_TYPE (lhs),
2478 : deref);
2479 :
2480 : /* Replace the original call to strlen with the dereference we just built. */
2481 123 : gimplify_and_update_call_from_tree (gsi, deref);
2482 :
2483 123 : return true;
2484 : }
2485 :
2486 : /* *GSI_P is a GIMPLE_CALL to a builtin function.
2487 : Optimize
2488 : memcpy (p, "abcd", 4);
2489 : memset (p + 4, ' ', 3);
2490 : into
2491 : memcpy (p, "abcd ", 7);
2492 : call if the latter can be stored by pieces during expansion.
2493 :
2494 : Optimize
2495 : memchr ("abcd", a, 4) == 0;
2496 : or
2497 : memchr ("abcd", a, 4) != 0;
2498 : to
2499 : (a == 'a' || a == 'b' || a == 'c' || a == 'd') == 0
2500 : or
2501 : (a == 'a' || a == 'b' || a == 'c' || a == 'd') != 0
2502 :
2503 : Also canonicalize __atomic_fetch_op (p, x, y) op x
2504 : to __atomic_op_fetch (p, x, y) or
2505 : __atomic_op_fetch (p, x, y) iop x
2506 : to __atomic_fetch_op (p, x, y) when possible (also __sync). */
2507 :
2508 : static bool
2509 6246747 : simplify_builtin_call (gimple_stmt_iterator *gsi_p, tree callee2, bool full_walk)
2510 : {
2511 6246747 : gimple *stmt2 = gsi_stmt (*gsi_p);
2512 6246747 : enum built_in_function other_atomic = END_BUILTINS;
2513 6246747 : enum tree_code atomic_op = ERROR_MARK;
2514 :
2515 6246747 : switch (DECL_FUNCTION_CODE (callee2))
2516 : {
2517 61840 : case BUILT_IN_STRLEN:
2518 61840 : return optimize_strlen_comp (gsi_p, as_a<gcall*>(stmt2));
2519 10369 : case BUILT_IN_STACK_RESTORE:
2520 10369 : return optimize_stack_restore (gsi_p, as_a<gcall*>(stmt2));
2521 466027 : case BUILT_IN_MEMCMP:
2522 466027 : case BUILT_IN_MEMCMP_EQ:
2523 466027 : return simplify_builtin_memcmp (gsi_p, as_a<gcall*>(stmt2));
2524 12194 : case BUILT_IN_MEMCHR:
2525 12194 : return simplify_builtin_memchr (gsi_p, as_a<gcall*>(stmt2));
2526 :
2527 111422 : case BUILT_IN_MEMSET:
2528 111422 : if (gimple_call_num_args (stmt2) == 3)
2529 : {
2530 : /* Try to prop the zeroing/value of the memset to memcpy
2531 : if the dest is an address and the value is a constant. */
2532 111422 : optimize_aggr_zeroprop (stmt2, full_walk);
2533 : }
2534 111422 : return simplify_builtin_memcpy_memset (gsi_p, as_a<gcall*>(stmt2));
2535 :
2536 : #define CASE_ATOMIC(NAME, OTHER, OP) \
2537 : case BUILT_IN_##NAME##_1: \
2538 : case BUILT_IN_##NAME##_2: \
2539 : case BUILT_IN_##NAME##_4: \
2540 : case BUILT_IN_##NAME##_8: \
2541 : case BUILT_IN_##NAME##_16: \
2542 : atomic_op = OP; \
2543 : other_atomic \
2544 : = (enum built_in_function) (BUILT_IN_##OTHER##_1 \
2545 : + (DECL_FUNCTION_CODE (callee2) \
2546 : - BUILT_IN_##NAME##_1)); \
2547 : goto handle_atomic_fetch_op;
2548 :
2549 48767 : CASE_ATOMIC (ATOMIC_FETCH_ADD, ATOMIC_ADD_FETCH, PLUS_EXPR)
2550 7133 : CASE_ATOMIC (ATOMIC_FETCH_SUB, ATOMIC_SUB_FETCH, MINUS_EXPR)
2551 2876 : CASE_ATOMIC (ATOMIC_FETCH_AND, ATOMIC_AND_FETCH, BIT_AND_EXPR)
2552 2895 : CASE_ATOMIC (ATOMIC_FETCH_XOR, ATOMIC_XOR_FETCH, BIT_XOR_EXPR)
2553 3840 : CASE_ATOMIC (ATOMIC_FETCH_OR, ATOMIC_OR_FETCH, BIT_IOR_EXPR)
2554 :
2555 2373 : CASE_ATOMIC (SYNC_FETCH_AND_ADD, SYNC_ADD_AND_FETCH, PLUS_EXPR)
2556 2012 : CASE_ATOMIC (SYNC_FETCH_AND_SUB, SYNC_SUB_AND_FETCH, MINUS_EXPR)
2557 1876 : CASE_ATOMIC (SYNC_FETCH_AND_AND, SYNC_AND_AND_FETCH, BIT_AND_EXPR)
2558 2144 : CASE_ATOMIC (SYNC_FETCH_AND_XOR, SYNC_XOR_AND_FETCH, BIT_XOR_EXPR)
2559 1987 : CASE_ATOMIC (SYNC_FETCH_AND_OR, SYNC_OR_AND_FETCH, BIT_IOR_EXPR)
2560 :
2561 14409 : CASE_ATOMIC (ATOMIC_ADD_FETCH, ATOMIC_FETCH_ADD, MINUS_EXPR)
2562 8560 : CASE_ATOMIC (ATOMIC_SUB_FETCH, ATOMIC_FETCH_SUB, PLUS_EXPR)
2563 2380 : CASE_ATOMIC (ATOMIC_XOR_FETCH, ATOMIC_FETCH_XOR, BIT_XOR_EXPR)
2564 :
2565 854 : CASE_ATOMIC (SYNC_ADD_AND_FETCH, SYNC_FETCH_AND_ADD, MINUS_EXPR)
2566 740 : CASE_ATOMIC (SYNC_SUB_AND_FETCH, SYNC_FETCH_AND_SUB, PLUS_EXPR)
2567 800 : CASE_ATOMIC (SYNC_XOR_AND_FETCH, SYNC_FETCH_AND_XOR, BIT_XOR_EXPR)
2568 :
2569 : #undef CASE_ATOMIC
2570 :
2571 103646 : handle_atomic_fetch_op:
2572 103646 : if (gimple_call_num_args (stmt2) >= 2 && gimple_call_lhs (stmt2))
2573 : {
2574 60176 : tree lhs2 = gimple_call_lhs (stmt2), lhsc = lhs2;
2575 60176 : tree arg = gimple_call_arg (stmt2, 1);
2576 60176 : gimple *use_stmt, *cast_stmt = NULL;
2577 60176 : use_operand_p use_p;
2578 60176 : tree ndecl = builtin_decl_explicit (other_atomic);
2579 :
2580 60176 : if (ndecl == NULL_TREE || !single_imm_use (lhs2, &use_p, &use_stmt))
2581 : break;
2582 :
2583 59047 : if (gimple_assign_cast_p (use_stmt))
2584 : {
2585 31489 : cast_stmt = use_stmt;
2586 31489 : lhsc = gimple_assign_lhs (cast_stmt);
2587 31489 : if (lhsc == NULL_TREE
2588 31489 : || !INTEGRAL_TYPE_P (TREE_TYPE (lhsc))
2589 30938 : || (TYPE_PRECISION (TREE_TYPE (lhsc))
2590 30938 : != TYPE_PRECISION (TREE_TYPE (lhs2)))
2591 60837 : || !single_imm_use (lhsc, &use_p, &use_stmt))
2592 : {
2593 2669 : use_stmt = cast_stmt;
2594 2669 : cast_stmt = NULL;
2595 2669 : lhsc = lhs2;
2596 : }
2597 : }
2598 :
2599 59047 : bool ok = false;
2600 59047 : tree oarg = NULL_TREE;
2601 59047 : enum tree_code ccode = ERROR_MARK;
2602 59047 : tree crhs1 = NULL_TREE, crhs2 = NULL_TREE;
2603 59047 : if (is_gimple_assign (use_stmt)
2604 59047 : && gimple_assign_rhs_code (use_stmt) == atomic_op)
2605 : {
2606 1416 : if (gimple_assign_rhs1 (use_stmt) == lhsc)
2607 1016 : oarg = gimple_assign_rhs2 (use_stmt);
2608 400 : else if (atomic_op != MINUS_EXPR)
2609 : oarg = gimple_assign_rhs1 (use_stmt);
2610 : }
2611 57631 : else if (atomic_op == MINUS_EXPR
2612 13279 : && is_gimple_assign (use_stmt)
2613 3638 : && gimple_assign_rhs_code (use_stmt) == PLUS_EXPR
2614 199 : && TREE_CODE (arg) == INTEGER_CST
2615 57830 : && (TREE_CODE (gimple_assign_rhs2 (use_stmt))
2616 : == INTEGER_CST))
2617 : {
2618 183 : tree a = fold_convert (TREE_TYPE (lhs2), arg);
2619 183 : tree o = fold_convert (TREE_TYPE (lhs2),
2620 : gimple_assign_rhs2 (use_stmt));
2621 183 : if (wi::to_wide (a) == wi::neg (wi::to_wide (o)))
2622 : ok = true;
2623 : }
2624 57448 : else if (atomic_op == BIT_AND_EXPR || atomic_op == BIT_IOR_EXPR)
2625 : ;
2626 52206 : else if (gimple_code (use_stmt) == GIMPLE_COND)
2627 : {
2628 19582 : ccode = gimple_cond_code (use_stmt);
2629 19582 : crhs1 = gimple_cond_lhs (use_stmt);
2630 19582 : crhs2 = gimple_cond_rhs (use_stmt);
2631 : }
2632 32624 : else if (is_gimple_assign (use_stmt))
2633 : {
2634 9583 : if (gimple_assign_rhs_class (use_stmt) == GIMPLE_BINARY_RHS)
2635 : {
2636 3935 : ccode = gimple_assign_rhs_code (use_stmt);
2637 3935 : crhs1 = gimple_assign_rhs1 (use_stmt);
2638 3935 : crhs2 = gimple_assign_rhs2 (use_stmt);
2639 : }
2640 5648 : else if (gimple_assign_rhs_code (use_stmt) == COND_EXPR)
2641 : {
2642 0 : tree cond = gimple_assign_rhs1 (use_stmt);
2643 0 : if (COMPARISON_CLASS_P (cond))
2644 : {
2645 0 : ccode = TREE_CODE (cond);
2646 0 : crhs1 = TREE_OPERAND (cond, 0);
2647 0 : crhs2 = TREE_OPERAND (cond, 1);
2648 : }
2649 : }
2650 : }
2651 24533 : if (ccode == EQ_EXPR || ccode == NE_EXPR)
2652 : {
2653 : /* Deal with x - y == 0 or x ^ y == 0
2654 : being optimized into x == y and x + cst == 0
2655 : into x == -cst. */
2656 22333 : tree o = NULL_TREE;
2657 22333 : if (crhs1 == lhsc)
2658 : o = crhs2;
2659 133 : else if (crhs2 == lhsc)
2660 133 : o = crhs1;
2661 22333 : if (o && atomic_op != PLUS_EXPR)
2662 : oarg = o;
2663 10117 : else if (o
2664 10117 : && TREE_CODE (o) == INTEGER_CST
2665 10117 : && TREE_CODE (arg) == INTEGER_CST)
2666 : {
2667 9407 : tree a = fold_convert (TREE_TYPE (lhs2), arg);
2668 9407 : o = fold_convert (TREE_TYPE (lhs2), o);
2669 9407 : if (wi::to_wide (a) == wi::neg (wi::to_wide (o)))
2670 59047 : ok = true;
2671 : }
2672 : }
2673 59047 : if (oarg && !ok)
2674 : {
2675 13632 : if (operand_equal_p (arg, oarg, 0))
2676 : ok = true;
2677 12303 : else if (TREE_CODE (arg) == SSA_NAME
2678 2203 : && TREE_CODE (oarg) == SSA_NAME)
2679 : {
2680 745 : tree oarg2 = oarg;
2681 745 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (oarg)))
2682 : {
2683 104 : gimple *g = SSA_NAME_DEF_STMT (oarg);
2684 104 : oarg2 = gimple_assign_rhs1 (g);
2685 104 : if (TREE_CODE (oarg2) != SSA_NAME
2686 104 : || !INTEGRAL_TYPE_P (TREE_TYPE (oarg2))
2687 208 : || (TYPE_PRECISION (TREE_TYPE (oarg2))
2688 104 : != TYPE_PRECISION (TREE_TYPE (oarg))))
2689 : oarg2 = oarg;
2690 : }
2691 745 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (arg)))
2692 : {
2693 544 : gimple *g = SSA_NAME_DEF_STMT (arg);
2694 544 : tree rhs1 = gimple_assign_rhs1 (g);
2695 : /* Handle e.g.
2696 : x.0_1 = (long unsigned int) x_4(D);
2697 : _2 = __atomic_fetch_add_8 (&vlong, x.0_1, 0);
2698 : _3 = (long int) _2;
2699 : _7 = x_4(D) + _3; */
2700 544 : if (rhs1 == oarg || rhs1 == oarg2)
2701 : ok = true;
2702 : /* Handle e.g.
2703 : x.18_1 = (short unsigned int) x_5(D);
2704 : _2 = (int) x.18_1;
2705 : _3 = __atomic_fetch_xor_2 (&vshort, _2, 0);
2706 : _4 = (short int) _3;
2707 : _8 = x_5(D) ^ _4;
2708 : This happens only for char/short. */
2709 160 : else if (TREE_CODE (rhs1) == SSA_NAME
2710 160 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
2711 320 : && (TYPE_PRECISION (TREE_TYPE (rhs1))
2712 160 : == TYPE_PRECISION (TREE_TYPE (lhs2))))
2713 : {
2714 160 : g = SSA_NAME_DEF_STMT (rhs1);
2715 160 : if (gimple_assign_cast_p (g)
2716 160 : && (gimple_assign_rhs1 (g) == oarg
2717 0 : || gimple_assign_rhs1 (g) == oarg2))
2718 : ok = true;
2719 : }
2720 : }
2721 745 : if (!ok && arg == oarg2)
2722 : /* Handle e.g.
2723 : _1 = __sync_fetch_and_add_4 (&v, x_5(D));
2724 : _2 = (int) _1;
2725 : x.0_3 = (int) x_5(D);
2726 : _7 = _2 + x.0_3; */
2727 : ok = true;
2728 : }
2729 : }
2730 :
2731 57718 : if (ok)
2732 : {
2733 2606 : tree new_lhs = make_ssa_name (TREE_TYPE (lhs2));
2734 2606 : gimple_call_set_lhs (stmt2, new_lhs);
2735 2606 : gimple_call_set_fndecl (stmt2, ndecl);
2736 2606 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
2737 2606 : if (ccode == ERROR_MARK)
2738 2000 : gimple_assign_set_rhs_with_ops (&gsi, cast_stmt
2739 : ? NOP_EXPR : SSA_NAME,
2740 : new_lhs);
2741 : else
2742 : {
2743 1383 : crhs1 = new_lhs;
2744 1383 : crhs2 = build_zero_cst (TREE_TYPE (lhs2));
2745 1383 : if (gimple_code (use_stmt) == GIMPLE_COND)
2746 : {
2747 1044 : gcond *cond_stmt = as_a <gcond *> (use_stmt);
2748 1044 : gimple_cond_set_lhs (cond_stmt, crhs1);
2749 1044 : gimple_cond_set_rhs (cond_stmt, crhs2);
2750 : }
2751 339 : else if (gimple_assign_rhs_class (use_stmt)
2752 : == GIMPLE_BINARY_RHS)
2753 : {
2754 339 : gimple_assign_set_rhs1 (use_stmt, crhs1);
2755 339 : gimple_assign_set_rhs2 (use_stmt, crhs2);
2756 : }
2757 : else
2758 : {
2759 0 : gcc_checking_assert (gimple_assign_rhs_code (use_stmt)
2760 : == COND_EXPR);
2761 0 : tree cond = build2 (ccode, boolean_type_node,
2762 : crhs1, crhs2);
2763 0 : gimple_assign_set_rhs1 (use_stmt, cond);
2764 : }
2765 : }
2766 2606 : update_stmt (use_stmt);
2767 2606 : if (atomic_op != BIT_AND_EXPR
2768 2606 : && atomic_op != BIT_IOR_EXPR
2769 2606 : && !stmt_ends_bb_p (stmt2))
2770 : {
2771 : /* For the benefit of debug stmts, emit stmt(s) to set
2772 : lhs2 to the value it had from the new builtin.
2773 : E.g. if it was previously:
2774 : lhs2 = __atomic_fetch_add_8 (ptr, arg, 0);
2775 : emit:
2776 : new_lhs = __atomic_add_fetch_8 (ptr, arg, 0);
2777 : lhs2 = new_lhs - arg;
2778 : We also keep cast_stmt if any in the IL for
2779 : the same reasons.
2780 : These stmts will be DCEd later and proper debug info
2781 : will be emitted.
2782 : This is only possible for reversible operations
2783 : (+/-/^) and without -fnon-call-exceptions. */
2784 2265 : gsi = gsi_for_stmt (stmt2);
2785 2265 : tree type = TREE_TYPE (lhs2);
2786 2265 : if (TREE_CODE (arg) == INTEGER_CST)
2787 1683 : arg = fold_convert (type, arg);
2788 582 : else if (!useless_type_conversion_p (type, TREE_TYPE (arg)))
2789 : {
2790 0 : tree narg = make_ssa_name (type);
2791 0 : gimple *g = gimple_build_assign (narg, NOP_EXPR, arg);
2792 0 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
2793 0 : arg = narg;
2794 : }
2795 2265 : enum tree_code rcode;
2796 2265 : switch (atomic_op)
2797 : {
2798 : case PLUS_EXPR: rcode = MINUS_EXPR; break;
2799 727 : case MINUS_EXPR: rcode = PLUS_EXPR; break;
2800 492 : case BIT_XOR_EXPR: rcode = atomic_op; break;
2801 0 : default: gcc_unreachable ();
2802 : }
2803 2265 : gimple *g = gimple_build_assign (lhs2, rcode, new_lhs, arg);
2804 2265 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
2805 2265 : update_stmt (stmt2);
2806 : }
2807 : else
2808 : {
2809 : /* For e.g.
2810 : lhs2 = __atomic_fetch_or_8 (ptr, arg, 0);
2811 : after we change it to
2812 : new_lhs = __atomic_or_fetch_8 (ptr, arg, 0);
2813 : there is no way to find out the lhs2 value (i.e.
2814 : what the atomic memory contained before the operation),
2815 : values of some bits are lost. We have checked earlier
2816 : that we don't have any non-debug users except for what
2817 : we are already changing, so we need to reset the
2818 : debug stmts and remove the cast_stmt if any. */
2819 341 : imm_use_iterator iter;
2820 676 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs2)
2821 335 : if (use_stmt != cast_stmt)
2822 : {
2823 168 : gcc_assert (is_gimple_debug (use_stmt));
2824 168 : gimple_debug_bind_reset_value (use_stmt);
2825 168 : update_stmt (use_stmt);
2826 341 : }
2827 341 : if (cast_stmt)
2828 : {
2829 167 : gsi = gsi_for_stmt (cast_stmt);
2830 167 : gsi_remove (&gsi, true);
2831 : }
2832 341 : update_stmt (stmt2);
2833 341 : release_ssa_name (lhs2);
2834 : }
2835 : }
2836 : }
2837 : break;
2838 :
2839 : default:
2840 : break;
2841 : }
2842 : return false;
2843 : }
2844 :
2845 : /* Given a ssa_name in NAME see if it was defined by an assignment and
2846 : set CODE to be the code and ARG1 to the first operand on the rhs and ARG2
2847 : to the second operand on the rhs. */
2848 :
2849 : static inline void
2850 17474430 : defcodefor_name (tree name, enum tree_code *code, tree *arg1, tree *arg2)
2851 : {
2852 17474430 : gimple *def;
2853 17474430 : enum tree_code code1;
2854 17474430 : tree arg11;
2855 17474430 : tree arg21;
2856 17474430 : tree arg31;
2857 17474430 : enum gimple_rhs_class grhs_class;
2858 :
2859 17474430 : code1 = TREE_CODE (name);
2860 17474430 : arg11 = name;
2861 17474430 : arg21 = NULL_TREE;
2862 17474430 : arg31 = NULL_TREE;
2863 17474430 : grhs_class = get_gimple_rhs_class (code1);
2864 :
2865 17474430 : if (code1 == SSA_NAME)
2866 : {
2867 11657166 : def = SSA_NAME_DEF_STMT (name);
2868 :
2869 11657166 : if (def && is_gimple_assign (def)
2870 18930003 : && can_propagate_from (def))
2871 : {
2872 5010959 : code1 = gimple_assign_rhs_code (def);
2873 5010959 : arg11 = gimple_assign_rhs1 (def);
2874 5010959 : arg21 = gimple_assign_rhs2 (def);
2875 5010959 : arg31 = gimple_assign_rhs3 (def);
2876 : }
2877 : }
2878 5817264 : else if (grhs_class != GIMPLE_SINGLE_RHS)
2879 0 : code1 = ERROR_MARK;
2880 :
2881 17474430 : *code = code1;
2882 17474430 : *arg1 = arg11;
2883 17474430 : if (arg2)
2884 17457171 : *arg2 = arg21;
2885 17474430 : if (arg31)
2886 2370 : *code = ERROR_MARK;
2887 17474430 : }
2888 :
2889 :
2890 : /* Recognize rotation patterns. Return true if a transformation
2891 : applied, otherwise return false.
2892 :
2893 : We are looking for X with unsigned type T with bitsize B, OP being
2894 : +, | or ^, some type T2 wider than T. For:
2895 : (X << CNT1) OP (X >> CNT2) iff CNT1 + CNT2 == B
2896 : ((T) ((T2) X << CNT1)) OP ((T) ((T2) X >> CNT2)) iff CNT1 + CNT2 == B
2897 :
2898 : transform these into:
2899 : X r<< CNT1
2900 :
2901 : Or for:
2902 : (X << Y) OP (X >> (B - Y))
2903 : (X << (int) Y) OP (X >> (int) (B - Y))
2904 : ((T) ((T2) X << Y)) OP ((T) ((T2) X >> (B - Y)))
2905 : ((T) ((T2) X << (int) Y)) OP ((T) ((T2) X >> (int) (B - Y)))
2906 : (X << Y) | (X >> ((-Y) & (B - 1)))
2907 : (X << (int) Y) | (X >> (int) ((-Y) & (B - 1)))
2908 : ((T) ((T2) X << Y)) | ((T) ((T2) X >> ((-Y) & (B - 1))))
2909 : ((T) ((T2) X << (int) Y)) | ((T) ((T2) X >> (int) ((-Y) & (B - 1))))
2910 :
2911 : transform these into (last 2 only if ranger can prove Y < B
2912 : or Y = N * B):
2913 : X r<< Y
2914 : or
2915 : X r<< (& & (B - 1))
2916 : The latter for the forms with T2 wider than T if ranger can't prove Y < B.
2917 :
2918 : Or for:
2919 : (X << (Y & (B - 1))) | (X >> ((-Y) & (B - 1)))
2920 : (X << (int) (Y & (B - 1))) | (X >> (int) ((-Y) & (B - 1)))
2921 : ((T) ((T2) X << (Y & (B - 1)))) | ((T) ((T2) X >> ((-Y) & (B - 1))))
2922 : ((T) ((T2) X << (int) (Y & (B - 1)))) \
2923 : | ((T) ((T2) X >> (int) ((-Y) & (B - 1))))
2924 :
2925 : transform these into:
2926 : X r<< (Y & (B - 1))
2927 :
2928 : Note, in the patterns with T2 type, the type of OP operands
2929 : might be even a signed type, but should have precision B.
2930 : Expressions with & (B - 1) should be recognized only if B is
2931 : a power of 2. */
2932 :
2933 : static bool
2934 10266882 : simplify_rotate (gimple_stmt_iterator *gsi)
2935 : {
2936 10266882 : gimple *stmt = gsi_stmt (*gsi);
2937 10266882 : tree arg[2], rtype, rotcnt = NULL_TREE;
2938 10266882 : tree def_arg1[2], def_arg2[2];
2939 10266882 : enum tree_code def_code[2];
2940 10266882 : tree lhs;
2941 10266882 : int i;
2942 10266882 : bool swapped_p = false;
2943 10266882 : gimple *g;
2944 10266882 : gimple *def_arg_stmt[2] = { NULL, NULL };
2945 10266882 : int wider_prec = 0;
2946 10266882 : bool add_masking = false;
2947 :
2948 10266882 : arg[0] = gimple_assign_rhs1 (stmt);
2949 10266882 : arg[1] = gimple_assign_rhs2 (stmt);
2950 10266882 : rtype = TREE_TYPE (arg[0]);
2951 :
2952 : /* Only create rotates in complete modes. Other cases are not
2953 : expanded properly. */
2954 10266882 : if (!INTEGRAL_TYPE_P (rtype)
2955 10266882 : || !type_has_mode_precision_p (rtype))
2956 1579316 : return false;
2957 :
2958 26062698 : for (i = 0; i < 2; i++)
2959 : {
2960 17375132 : defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
2961 17375132 : if (TREE_CODE (arg[i]) == SSA_NAME)
2962 11557868 : def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
2963 : }
2964 :
2965 : /* Look through narrowing (or same precision) conversions. */
2966 7724826 : if (CONVERT_EXPR_CODE_P (def_code[0])
2967 962740 : && CONVERT_EXPR_CODE_P (def_code[1])
2968 139673 : && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[0]))
2969 115887 : && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[1]))
2970 108307 : && TYPE_PRECISION (TREE_TYPE (def_arg1[0]))
2971 108307 : == TYPE_PRECISION (TREE_TYPE (def_arg1[1]))
2972 62449 : && TYPE_PRECISION (TREE_TYPE (def_arg1[0])) >= TYPE_PRECISION (rtype)
2973 44123 : && has_single_use (arg[0])
2974 8720359 : && has_single_use (arg[1]))
2975 : {
2976 28342 : wider_prec = TYPE_PRECISION (TREE_TYPE (def_arg1[0]));
2977 85026 : for (i = 0; i < 2; i++)
2978 : {
2979 56684 : arg[i] = def_arg1[i];
2980 56684 : defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
2981 56684 : if (TREE_CODE (arg[i]) == SSA_NAME)
2982 56684 : def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
2983 : }
2984 : }
2985 : else
2986 : {
2987 : /* Handle signed rotate; the RSHIFT_EXPR has to be done
2988 : in unsigned type but LSHIFT_EXPR could be signed. */
2989 8659224 : i = (def_code[0] == LSHIFT_EXPR || def_code[0] == RSHIFT_EXPR);
2990 7707281 : if (CONVERT_EXPR_CODE_P (def_code[i])
2991 951943 : && (def_code[1 - i] == LSHIFT_EXPR || def_code[1 - i] == RSHIFT_EXPR)
2992 30695 : && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[i]))
2993 29432 : && TYPE_PRECISION (rtype) == TYPE_PRECISION (TREE_TYPE (def_arg1[i]))
2994 8664174 : && has_single_use (arg[i]))
2995 : {
2996 2095 : arg[i] = def_arg1[i];
2997 2095 : defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
2998 2095 : if (TREE_CODE (arg[i]) == SSA_NAME)
2999 2095 : def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
3000 : }
3001 : }
3002 :
3003 : /* One operand has to be LSHIFT_EXPR and one RSHIFT_EXPR. */
3004 8886533 : for (i = 0; i < 2; i++)
3005 8861805 : if (def_code[i] != LSHIFT_EXPR && def_code[i] != RSHIFT_EXPR)
3006 : return false;
3007 239527 : else if (!has_single_use (arg[i]))
3008 : return false;
3009 24728 : if (def_code[0] == def_code[1])
3010 : return false;
3011 :
3012 : /* If we've looked through narrowing conversions before, look through
3013 : widening conversions from unsigned type with the same precision
3014 : as rtype here. */
3015 20364 : if (TYPE_PRECISION (TREE_TYPE (def_arg1[0])) != TYPE_PRECISION (rtype))
3016 19348 : for (i = 0; i < 2; i++)
3017 : {
3018 12900 : tree tem;
3019 12900 : enum tree_code code;
3020 12900 : defcodefor_name (def_arg1[i], &code, &tem, NULL);
3021 4 : if (!CONVERT_EXPR_CODE_P (code)
3022 12896 : || !INTEGRAL_TYPE_P (TREE_TYPE (tem))
3023 25796 : || TYPE_PRECISION (TREE_TYPE (tem)) != TYPE_PRECISION (rtype))
3024 4 : return false;
3025 12896 : def_arg1[i] = tem;
3026 : }
3027 : /* Both shifts have to use the same first operand. */
3028 20360 : if (!operand_equal_for_phi_arg_p (def_arg1[0], def_arg1[1])
3029 32296 : || !types_compatible_p (TREE_TYPE (def_arg1[0]),
3030 11936 : TREE_TYPE (def_arg1[1])))
3031 : {
3032 8424 : if ((TYPE_PRECISION (TREE_TYPE (def_arg1[0]))
3033 8424 : != TYPE_PRECISION (TREE_TYPE (def_arg1[1])))
3034 8424 : || (TYPE_UNSIGNED (TREE_TYPE (def_arg1[0]))
3035 8424 : == TYPE_UNSIGNED (TREE_TYPE (def_arg1[1]))))
3036 8400 : return false;
3037 :
3038 : /* Handle signed rotate; the RSHIFT_EXPR has to be done
3039 : in unsigned type but LSHIFT_EXPR could be signed. */
3040 545 : i = def_code[0] != RSHIFT_EXPR;
3041 545 : if (!TYPE_UNSIGNED (TREE_TYPE (def_arg1[i])))
3042 : return false;
3043 :
3044 506 : tree tem;
3045 506 : enum tree_code code;
3046 506 : defcodefor_name (def_arg1[i], &code, &tem, NULL);
3047 303 : if (!CONVERT_EXPR_CODE_P (code)
3048 203 : || !INTEGRAL_TYPE_P (TREE_TYPE (tem))
3049 709 : || TYPE_PRECISION (TREE_TYPE (tem)) != TYPE_PRECISION (rtype))
3050 : return false;
3051 194 : def_arg1[i] = tem;
3052 194 : if (!operand_equal_for_phi_arg_p (def_arg1[0], def_arg1[1])
3053 218 : || !types_compatible_p (TREE_TYPE (def_arg1[0]),
3054 24 : TREE_TYPE (def_arg1[1])))
3055 170 : return false;
3056 : }
3057 11936 : else if (!TYPE_UNSIGNED (TREE_TYPE (def_arg1[0])))
3058 : return false;
3059 :
3060 : /* CNT1 + CNT2 == B case above. */
3061 10705 : if (tree_fits_uhwi_p (def_arg2[0])
3062 1209 : && tree_fits_uhwi_p (def_arg2[1])
3063 10705 : && tree_to_uhwi (def_arg2[0])
3064 1209 : + tree_to_uhwi (def_arg2[1]) == TYPE_PRECISION (rtype))
3065 : rotcnt = def_arg2[0];
3066 9776 : else if (TREE_CODE (def_arg2[0]) != SSA_NAME
3067 9496 : || TREE_CODE (def_arg2[1]) != SSA_NAME)
3068 : return false;
3069 : else
3070 : {
3071 9496 : tree cdef_arg1[2], cdef_arg2[2], def_arg2_alt[2];
3072 9496 : enum tree_code cdef_code[2];
3073 9496 : gimple *def_arg_alt_stmt[2] = { NULL, NULL };
3074 9496 : int check_range = 0;
3075 9496 : gimple *check_range_stmt = NULL;
3076 : /* Look through conversion of the shift count argument.
3077 : The C/C++ FE cast any shift count argument to integer_type_node.
3078 : The only problem might be if the shift count type maximum value
3079 : is equal or smaller than number of bits in rtype. */
3080 28488 : for (i = 0; i < 2; i++)
3081 : {
3082 18992 : def_arg2_alt[i] = def_arg2[i];
3083 18992 : defcodefor_name (def_arg2[i], &cdef_code[i],
3084 : &cdef_arg1[i], &cdef_arg2[i]);
3085 14724 : if (CONVERT_EXPR_CODE_P (cdef_code[i])
3086 4268 : && INTEGRAL_TYPE_P (TREE_TYPE (cdef_arg1[i]))
3087 4268 : && TYPE_PRECISION (TREE_TYPE (cdef_arg1[i]))
3088 8536 : > floor_log2 (TYPE_PRECISION (rtype))
3089 23260 : && type_has_mode_precision_p (TREE_TYPE (cdef_arg1[i])))
3090 : {
3091 4268 : def_arg2_alt[i] = cdef_arg1[i];
3092 4268 : if (TREE_CODE (def_arg2[i]) == SSA_NAME)
3093 4268 : def_arg_alt_stmt[i] = SSA_NAME_DEF_STMT (def_arg2[i]);
3094 4268 : defcodefor_name (def_arg2_alt[i], &cdef_code[i],
3095 : &cdef_arg1[i], &cdef_arg2[i]);
3096 : }
3097 : else
3098 14724 : def_arg_alt_stmt[i] = def_arg_stmt[i];
3099 : }
3100 25812 : for (i = 0; i < 2; i++)
3101 : /* Check for one shift count being Y and the other B - Y,
3102 : with optional casts. */
3103 18641 : if (cdef_code[i] == MINUS_EXPR
3104 862 : && tree_fits_shwi_p (cdef_arg1[i])
3105 862 : && tree_to_shwi (cdef_arg1[i]) == TYPE_PRECISION (rtype)
3106 19463 : && TREE_CODE (cdef_arg2[i]) == SSA_NAME)
3107 : {
3108 822 : tree tem;
3109 822 : enum tree_code code;
3110 :
3111 822 : if (cdef_arg2[i] == def_arg2[1 - i]
3112 472 : || cdef_arg2[i] == def_arg2_alt[1 - i])
3113 : {
3114 350 : rotcnt = cdef_arg2[i];
3115 350 : check_range = -1;
3116 350 : if (cdef_arg2[i] == def_arg2[1 - i])
3117 350 : check_range_stmt = def_arg_stmt[1 - i];
3118 : else
3119 0 : check_range_stmt = def_arg_alt_stmt[1 - i];
3120 806 : break;
3121 : }
3122 472 : defcodefor_name (cdef_arg2[i], &code, &tem, NULL);
3123 16 : if (CONVERT_EXPR_CODE_P (code)
3124 456 : && INTEGRAL_TYPE_P (TREE_TYPE (tem))
3125 456 : && TYPE_PRECISION (TREE_TYPE (tem))
3126 912 : > floor_log2 (TYPE_PRECISION (rtype))
3127 456 : && type_has_mode_precision_p (TREE_TYPE (tem))
3128 928 : && (tem == def_arg2[1 - i]
3129 288 : || tem == def_arg2_alt[1 - i]))
3130 : {
3131 456 : rotcnt = tem;
3132 456 : check_range = -1;
3133 456 : if (tem == def_arg2[1 - i])
3134 168 : check_range_stmt = def_arg_stmt[1 - i];
3135 : else
3136 288 : check_range_stmt = def_arg_alt_stmt[1 - i];
3137 : break;
3138 : }
3139 : }
3140 : /* The above sequence isn't safe for Y being 0,
3141 : because then one of the shifts triggers undefined behavior.
3142 : This alternative is safe even for rotation count of 0.
3143 : One shift count is Y and the other (-Y) & (B - 1).
3144 : Or one shift count is Y & (B - 1) and the other (-Y) & (B - 1). */
3145 17819 : else if (cdef_code[i] == BIT_AND_EXPR
3146 28748 : && pow2p_hwi (TYPE_PRECISION (rtype))
3147 12432 : && tree_fits_shwi_p (cdef_arg2[i])
3148 24864 : && tree_to_shwi (cdef_arg2[i])
3149 12432 : == TYPE_PRECISION (rtype) - 1
3150 12372 : && TREE_CODE (cdef_arg1[i]) == SSA_NAME
3151 30191 : && gimple_assign_rhs_code (stmt) == BIT_IOR_EXPR)
3152 : {
3153 2312 : tree tem;
3154 2312 : enum tree_code code;
3155 :
3156 2312 : defcodefor_name (cdef_arg1[i], &code, &tem, NULL);
3157 2115 : if (CONVERT_EXPR_CODE_P (code)
3158 197 : && INTEGRAL_TYPE_P (TREE_TYPE (tem))
3159 197 : && TYPE_PRECISION (TREE_TYPE (tem))
3160 394 : > floor_log2 (TYPE_PRECISION (rtype))
3161 2509 : && type_has_mode_precision_p (TREE_TYPE (tem)))
3162 197 : defcodefor_name (tem, &code, &tem, NULL);
3163 :
3164 2312 : if (code == NEGATE_EXPR)
3165 : {
3166 1533 : if (tem == def_arg2[1 - i] || tem == def_arg2_alt[1 - i])
3167 : {
3168 854 : rotcnt = tem;
3169 854 : check_range = 1;
3170 854 : if (tem == def_arg2[1 - i])
3171 846 : check_range_stmt = def_arg_stmt[1 - i];
3172 : else
3173 8 : check_range_stmt = def_arg_alt_stmt[1 - i];
3174 1519 : break;
3175 : }
3176 679 : tree tem2;
3177 679 : defcodefor_name (tem, &code, &tem2, NULL);
3178 237 : if (CONVERT_EXPR_CODE_P (code)
3179 442 : && INTEGRAL_TYPE_P (TREE_TYPE (tem2))
3180 442 : && TYPE_PRECISION (TREE_TYPE (tem2))
3181 884 : > floor_log2 (TYPE_PRECISION (rtype))
3182 1121 : && type_has_mode_precision_p (TREE_TYPE (tem2)))
3183 : {
3184 442 : if (tem2 == def_arg2[1 - i]
3185 442 : || tem2 == def_arg2_alt[1 - i])
3186 : {
3187 228 : rotcnt = tem2;
3188 228 : check_range = 1;
3189 228 : if (tem2 == def_arg2[1 - i])
3190 0 : check_range_stmt = def_arg_stmt[1 - i];
3191 : else
3192 228 : check_range_stmt = def_arg_alt_stmt[1 - i];
3193 : break;
3194 : }
3195 : }
3196 : else
3197 237 : tem2 = NULL_TREE;
3198 :
3199 451 : if (cdef_code[1 - i] == BIT_AND_EXPR
3200 438 : && tree_fits_shwi_p (cdef_arg2[1 - i])
3201 876 : && tree_to_shwi (cdef_arg2[1 - i])
3202 438 : == TYPE_PRECISION (rtype) - 1
3203 889 : && TREE_CODE (cdef_arg1[1 - i]) == SSA_NAME)
3204 : {
3205 438 : if (tem == cdef_arg1[1 - i]
3206 213 : || tem2 == cdef_arg1[1 - i])
3207 : {
3208 : rotcnt = def_arg2[1 - i];
3209 437 : break;
3210 : }
3211 193 : tree tem3;
3212 193 : defcodefor_name (cdef_arg1[1 - i], &code, &tem3, NULL);
3213 0 : if (CONVERT_EXPR_CODE_P (code)
3214 193 : && INTEGRAL_TYPE_P (TREE_TYPE (tem3))
3215 193 : && TYPE_PRECISION (TREE_TYPE (tem3))
3216 386 : > floor_log2 (TYPE_PRECISION (rtype))
3217 386 : && type_has_mode_precision_p (TREE_TYPE (tem3)))
3218 : {
3219 193 : if (tem == tem3 || tem2 == tem3)
3220 : {
3221 : rotcnt = def_arg2[1 - i];
3222 : break;
3223 : }
3224 : }
3225 : }
3226 : }
3227 : }
3228 2325 : if (check_range && wider_prec > TYPE_PRECISION (rtype))
3229 : {
3230 1533 : if (TREE_CODE (rotcnt) != SSA_NAME)
3231 573 : return false;
3232 1533 : int_range_max r;
3233 1533 : range_query *q = get_range_query (cfun);
3234 1533 : if (q == get_global_range_query ())
3235 1522 : q = enable_ranger (cfun);
3236 1533 : if (!q->range_of_expr (r, rotcnt, check_range_stmt))
3237 : {
3238 0 : if (check_range > 0)
3239 : return false;
3240 0 : r.set_varying (TREE_TYPE (rotcnt));
3241 : }
3242 1533 : int prec = TYPE_PRECISION (TREE_TYPE (rotcnt));
3243 1533 : signop sign = TYPE_SIGN (TREE_TYPE (rotcnt));
3244 1533 : wide_int min = wide_int::from (TYPE_PRECISION (rtype), prec, sign);
3245 1533 : wide_int max = wide_int::from (wider_prec - 1, prec, sign);
3246 1533 : if (check_range < 0)
3247 616 : max = min;
3248 1533 : int_range<1> r2 (TREE_TYPE (rotcnt), min, max);
3249 1533 : r.intersect (r2);
3250 1533 : if (!r.undefined_p ())
3251 : {
3252 1181 : if (check_range > 0)
3253 : {
3254 589 : int_range_max r3;
3255 1844 : for (int i = TYPE_PRECISION (rtype) + 1; i < wider_prec;
3256 1255 : i += TYPE_PRECISION (rtype))
3257 : {
3258 1255 : int j = i + TYPE_PRECISION (rtype) - 2;
3259 1255 : min = wide_int::from (i, prec, sign);
3260 1255 : max = wide_int::from (MIN (j, wider_prec - 1),
3261 1255 : prec, sign);
3262 1255 : int_range<1> r4 (TREE_TYPE (rotcnt), min, max);
3263 1255 : r3.union_ (r4);
3264 1255 : }
3265 589 : r.intersect (r3);
3266 589 : if (!r.undefined_p ())
3267 573 : return false;
3268 589 : }
3269 : add_masking = true;
3270 : }
3271 1533 : }
3272 8923 : if (rotcnt == NULL_TREE)
3273 : return false;
3274 1752 : swapped_p = i != 1;
3275 : }
3276 :
3277 2681 : if (!useless_type_conversion_p (TREE_TYPE (def_arg2[0]),
3278 2681 : TREE_TYPE (rotcnt)))
3279 : {
3280 496 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (def_arg2[0])),
3281 : NOP_EXPR, rotcnt);
3282 496 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3283 496 : rotcnt = gimple_assign_lhs (g);
3284 : }
3285 2681 : if (add_masking)
3286 : {
3287 608 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rotcnt)),
3288 : BIT_AND_EXPR, rotcnt,
3289 608 : build_int_cst (TREE_TYPE (rotcnt),
3290 608 : TYPE_PRECISION (rtype) - 1));
3291 608 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3292 608 : rotcnt = gimple_assign_lhs (g);
3293 : }
3294 2681 : lhs = gimple_assign_lhs (stmt);
3295 2681 : if (!useless_type_conversion_p (rtype, TREE_TYPE (def_arg1[0])))
3296 1010 : lhs = make_ssa_name (TREE_TYPE (def_arg1[0]));
3297 2681 : g = gimple_build_assign (lhs,
3298 2681 : ((def_code[0] == LSHIFT_EXPR) ^ swapped_p)
3299 : ? LROTATE_EXPR : RROTATE_EXPR, def_arg1[0], rotcnt);
3300 2681 : if (!useless_type_conversion_p (rtype, TREE_TYPE (def_arg1[0])))
3301 : {
3302 1010 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3303 1010 : g = gimple_build_assign (gimple_assign_lhs (stmt), NOP_EXPR, lhs);
3304 : }
3305 2681 : gsi_replace (gsi, g, false);
3306 2681 : return true;
3307 : }
3308 :
3309 :
3310 : /* Check whether an array contains a valid table according to VALIDATE_FN. */
3311 : template<typename ValidateFn>
3312 : static bool
3313 17 : check_table_array (tree ctor, HOST_WIDE_INT &zero_val, unsigned bits,
3314 : ValidateFn validate_fn)
3315 : {
3316 : tree elt, idx;
3317 17 : unsigned HOST_WIDE_INT i, raw_idx = 0;
3318 17 : unsigned matched = 0;
3319 :
3320 17 : zero_val = 0;
3321 :
3322 734 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ctor), i, idx, elt)
3323 : {
3324 734 : if (!tree_fits_shwi_p (idx))
3325 : return false;
3326 734 : if (!tree_fits_shwi_p (elt) && TREE_CODE (elt) != RAW_DATA_CST)
3327 : return false;
3328 :
3329 734 : unsigned HOST_WIDE_INT index = tree_to_shwi (idx);
3330 : HOST_WIDE_INT val;
3331 :
3332 734 : if (TREE_CODE (elt) == INTEGER_CST)
3333 670 : val = tree_to_shwi (elt);
3334 : else
3335 : {
3336 64 : if (raw_idx == (unsigned) RAW_DATA_LENGTH (elt))
3337 : {
3338 0 : raw_idx = 0;
3339 0 : continue;
3340 : }
3341 64 : if (TYPE_UNSIGNED (TREE_TYPE (elt)))
3342 0 : val = RAW_DATA_UCHAR_ELT (elt, raw_idx);
3343 : else
3344 64 : val = RAW_DATA_SCHAR_ELT (elt, raw_idx);
3345 64 : index += raw_idx;
3346 64 : raw_idx++;
3347 64 : i--;
3348 : }
3349 :
3350 734 : if (index > bits * 2)
3351 : return false;
3352 :
3353 734 : if (index == 0)
3354 : {
3355 17 : zero_val = val;
3356 17 : matched++;
3357 : }
3358 :
3359 734 : if (val >= 0 && val < bits && validate_fn (val, index))
3360 672 : matched++;
3361 :
3362 734 : if (matched > bits)
3363 : return true;
3364 : }
3365 :
3366 : return false;
3367 : }
3368 :
3369 : /* Check whether a string contains a valid table according to VALIDATE_FN. */
3370 : template<typename ValidateFn>
3371 : static bool
3372 4 : check_table_string (tree string, HOST_WIDE_INT &zero_val,unsigned bits,
3373 : ValidateFn validate_fn)
3374 : {
3375 4 : unsigned HOST_WIDE_INT len = TREE_STRING_LENGTH (string);
3376 4 : unsigned matched = 0;
3377 4 : const unsigned char *p = (const unsigned char *) TREE_STRING_POINTER (string);
3378 :
3379 4 : if (len < bits || len > bits * 2)
3380 : return false;
3381 :
3382 4 : zero_val = p[0];
3383 :
3384 164 : for (unsigned i = 0; i < len; i++)
3385 160 : if (p[i] < bits && validate_fn (p[i], i))
3386 160 : matched++;
3387 :
3388 4 : return matched == bits;
3389 : }
3390 :
3391 : /* Check whether CTOR contains a valid table according to VALIDATE_FN. */
3392 : template<typename ValidateFn>
3393 : static bool
3394 29 : check_table (tree ctor, tree type, HOST_WIDE_INT &zero_val, unsigned bits,
3395 : ValidateFn validate_fn)
3396 : {
3397 29 : if (TREE_CODE (ctor) == CONSTRUCTOR)
3398 17 : return check_table_array (ctor, zero_val, bits, validate_fn);
3399 : else if (TREE_CODE (ctor) == STRING_CST
3400 12 : && TYPE_PRECISION (type) == CHAR_TYPE_SIZE)
3401 4 : return check_table_string (ctor, zero_val, bits, validate_fn);
3402 : return false;
3403 : }
3404 :
3405 : /* Match.pd function to match the ctz expression. */
3406 : extern bool gimple_ctz_table_index (tree, tree *, tree (*)(tree));
3407 : extern bool gimple_clz_table_index (tree, tree *, tree (*)(tree));
3408 : extern bool gimple_clz_msb_iso_table_index (tree, tree *, tree (*)(tree));
3409 :
3410 : /* Recognize count leading and trailing zeroes idioms.
3411 : The canonical form is array[((x & -x) * C) >> SHIFT] where C is a magic
3412 : constant which when multiplied by a power of 2 creates a unique value
3413 : in the top 5 or 6 bits. This is then indexed into a table which maps it
3414 : to the number of trailing zeroes. Array[0] is returned so the caller can
3415 : emit an appropriate sequence depending on whether ctz (0) is defined on
3416 : the target. */
3417 :
3418 : static bool
3419 1992988 : simplify_count_zeroes (gimple_stmt_iterator *gsi)
3420 : {
3421 1992988 : gimple *stmt = gsi_stmt (*gsi);
3422 1992988 : tree array_ref = gimple_assign_rhs1 (stmt);
3423 1992988 : tree res_ops[3];
3424 :
3425 1992988 : gcc_checking_assert (TREE_CODE (array_ref) == ARRAY_REF);
3426 :
3427 1992988 : internal_fn fn = IFN_LAST;
3428 : /* When true, the matched idiom is a CLZ using DeBruijn CTZ on the
3429 : isolated MSB -- see clz_msb_iso_table_index in match.pd. The
3430 : table stores MSB positions and must satisfy the direct CTZ
3431 : DeBruijn property, so we validate it with the CTZ checkfn even
3432 : though we emit IFN_CLZ code. */
3433 1992988 : bool clz_via_ctz = false;
3434 : /* For CTZ we recognize ((x & -x) * C) >> SHIFT where the array data
3435 : represents the number of trailing zeros. */
3436 1992988 : if (gimple_ctz_table_index (TREE_OPERAND (array_ref, 1), &res_ops[0], NULL))
3437 : fn = IFN_CTZ;
3438 : /* For CLZ we recognize
3439 : x |= x >> 1;
3440 : x |= x >> 2;
3441 : x |= x >> 4;
3442 : x |= x >> 8;
3443 : x |= x >> 16;
3444 : (x * C) >> SHIFT
3445 : where 31 minus the array data represents the number of leading zeros. */
3446 1992965 : else if (gimple_clz_table_index (TREE_OPERAND (array_ref, 1), &res_ops[0],
3447 : NULL))
3448 : fn = IFN_CLZ;
3449 : /* Variant CLZ idiom: after the OR-cascade sets all bits from 0 to
3450 : the original MSB, (value - (value >> 1)) isolates the MSB as a
3451 : power of two (2^k), and the subsequent DeBruijn multiply-and-shift
3452 : is a CTZ-style lookup on 2^k. The table stores MSB positions
3453 : directly. */
3454 1992955 : else if (gimple_clz_msb_iso_table_index (TREE_OPERAND (array_ref, 1),
3455 : &res_ops[0], NULL))
3456 : {
3457 : fn = IFN_CLZ;
3458 : clz_via_ctz = true;
3459 : }
3460 : else
3461 : return false;
3462 :
3463 34 : HOST_WIDE_INT zero_val;
3464 34 : tree type = TREE_TYPE (array_ref);
3465 34 : tree array = TREE_OPERAND (array_ref, 0);
3466 34 : tree input_type = TREE_TYPE (res_ops[0]);
3467 34 : unsigned input_bits = tree_to_shwi (TYPE_SIZE (input_type));
3468 :
3469 : /* Check the array element type is integral and not wider than 64 bits,
3470 : and the input is an unsigned 32-bit or 64-bit type. The table values
3471 : are bit positions in [0, input_bits - 1], so any integer element type
3472 : with at least 6 bits of precision suffices; the cap is just to keep
3473 : the transformation simple. */
3474 34 : if (!INTEGRAL_TYPE_P (type) || TYPE_PRECISION (type) > 64
3475 68 : || !TYPE_UNSIGNED (input_type))
3476 : return false;
3477 30 : if (input_bits != 32 && input_bits != 64)
3478 : return false;
3479 :
3480 30 : if (!direct_internal_fn_supported_p (fn, input_type, OPTIMIZE_FOR_BOTH))
3481 : return false;
3482 :
3483 : /* Check the lower bound of the array is zero. */
3484 30 : tree low = array_ref_low_bound (array_ref);
3485 30 : if (!low || !integer_zerop (low))
3486 0 : return false;
3487 :
3488 : /* Check the shift extracts the top 5..7 bits. */
3489 30 : unsigned shiftval = tree_to_shwi (res_ops[2]);
3490 30 : if (shiftval < input_bits - 7 || shiftval > input_bits - 5)
3491 : return false;
3492 :
3493 29 : tree ctor = ctor_for_folding (array);
3494 29 : if (!ctor)
3495 : return false;
3496 29 : unsigned HOST_WIDE_INT mulval = tree_to_uhwi (res_ops[1]);
3497 : /* CTZ and the MSB-isolation CLZ variant both use the direct CTZ
3498 : DeBruijn check (table[(magic << data) >> shift] == data). */
3499 29 : if (fn == IFN_CTZ || clz_via_ctz)
3500 : {
3501 559 : auto checkfn = [&](unsigned data, unsigned i) -> bool
3502 : {
3503 540 : unsigned HOST_WIDE_INT mask
3504 540 : = ((HOST_WIDE_INT_1U << (input_bits - shiftval)) - 1) << shiftval;
3505 540 : return (((mulval << data) & mask) >> shiftval) == i;
3506 19 : };
3507 19 : if (!check_table (ctor, type, zero_val, input_bits, checkfn))
3508 8 : return false;
3509 : }
3510 10 : else if (fn == IFN_CLZ)
3511 : {
3512 362 : auto checkfn = [&](unsigned data, unsigned i) -> bool
3513 : {
3514 352 : unsigned HOST_WIDE_INT mask
3515 352 : = ((HOST_WIDE_INT_1U << (input_bits - shiftval)) - 1) << shiftval;
3516 : /* The OR-cascade produces a value with all bits from 0 to the
3517 : original MSB set. Compute (1 << (data + 1)) - 1 to simulate
3518 : that value. When data + 1 equals HOST_BITS_PER_WIDE_INT
3519 : (i.e. data is the MSB position of a 64-bit input) the shift
3520 : is undefined behavior, so handle that case explicitly using
3521 : all-ones. Without this, any well-formed 64-bit DeBruijn CLZ
3522 : table is rejected because its entry for the all-ones input
3523 : correctly maps to the MSB (e.g. table[...] == 63).
3524 : PR tree-optimization/122569. */
3525 703 : unsigned HOST_WIDE_INT all_bits_below
3526 : = (data + 1 == HOST_BITS_PER_WIDE_INT)
3527 352 : ? HOST_WIDE_INT_M1U
3528 351 : : ((HOST_WIDE_INT_1U << (data + 1)) - 1);
3529 352 : return (((all_bits_below * mulval) & mask) >> shiftval) == i;
3530 10 : };
3531 10 : if (!check_table (ctor, type, zero_val, input_bits, checkfn))
3532 0 : return false;
3533 : }
3534 :
3535 21 : HOST_WIDE_INT ctz_val = -1;
3536 21 : bool zero_ok;
3537 21 : if (fn == IFN_CTZ)
3538 : {
3539 10 : ctz_val = 0;
3540 20 : zero_ok = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (input_type),
3541 : ctz_val) == 2;
3542 : }
3543 11 : else if (fn == IFN_CLZ)
3544 : {
3545 11 : ctz_val = 32;
3546 11 : zero_ok = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (input_type),
3547 : ctz_val) == 2;
3548 11 : zero_val = input_bits - 1 - zero_val;
3549 : }
3550 21 : int nargs = 2;
3551 :
3552 : /* If the input value can't be zero, don't special case ctz (0). */
3553 21 : range_query *q = get_range_query (cfun);
3554 21 : if (q == get_global_range_query ())
3555 21 : q = enable_ranger (cfun);
3556 21 : int_range_max vr;
3557 21 : if (q->range_of_expr (vr, res_ops[0], stmt)
3558 21 : && !range_includes_zero_p (vr))
3559 : {
3560 4 : zero_ok = true;
3561 4 : zero_val = 0;
3562 4 : ctz_val = 0;
3563 4 : nargs = 1;
3564 : }
3565 :
3566 21 : gimple_seq seq = NULL;
3567 21 : gimple *g;
3568 21 : gcall *call = gimple_build_call_internal (fn, nargs, res_ops[0],
3569 : nargs == 1 ? NULL_TREE
3570 38 : : build_int_cst (integer_type_node,
3571 17 : ctz_val));
3572 21 : gimple_set_location (call, gimple_location (stmt));
3573 21 : gimple_set_lhs (call, make_ssa_name (integer_type_node));
3574 21 : gimple_seq_add_stmt (&seq, call);
3575 :
3576 21 : tree prev_lhs = gimple_call_lhs (call);
3577 :
3578 21 : if (zero_ok && zero_val == ctz_val)
3579 : ;
3580 : /* Emit ctz (x) & 31 if ctz (0) is 32 but we need to return 0. */
3581 6 : else if (zero_ok && zero_val == 0 && ctz_val == input_bits)
3582 : {
3583 5 : g = gimple_build_assign (make_ssa_name (integer_type_node),
3584 : BIT_AND_EXPR, prev_lhs,
3585 : build_int_cst (integer_type_node,
3586 5 : input_bits - 1));
3587 5 : gimple_set_location (g, gimple_location (stmt));
3588 5 : gimple_seq_add_stmt (&seq, g);
3589 5 : prev_lhs = gimple_assign_lhs (g);
3590 : }
3591 : /* As fallback emit a conditional move. */
3592 : else
3593 : {
3594 10 : g = gimple_build_assign (make_ssa_name (boolean_type_node), EQ_EXPR,
3595 : res_ops[0], build_zero_cst (input_type));
3596 10 : gimple_set_location (g, gimple_location (stmt));
3597 10 : gimple_seq_add_stmt (&seq, g);
3598 10 : tree cond = gimple_assign_lhs (g);
3599 10 : g = gimple_build_assign (make_ssa_name (integer_type_node),
3600 : COND_EXPR, cond,
3601 10 : build_int_cst (integer_type_node, zero_val),
3602 : prev_lhs);
3603 10 : gimple_set_location (g, gimple_location (stmt));
3604 10 : gimple_seq_add_stmt (&seq, g);
3605 10 : prev_lhs = gimple_assign_lhs (g);
3606 : }
3607 :
3608 21 : if (fn == IFN_CLZ)
3609 : {
3610 11 : g = gimple_build_assign (make_ssa_name (integer_type_node),
3611 : MINUS_EXPR,
3612 : build_int_cst (integer_type_node,
3613 11 : input_bits - 1),
3614 : prev_lhs);
3615 11 : gimple_set_location (g, gimple_location (stmt));
3616 11 : gimple_seq_add_stmt (&seq, g);
3617 11 : prev_lhs = gimple_assign_lhs (g);
3618 : }
3619 :
3620 21 : g = gimple_build_assign (gimple_assign_lhs (stmt), NOP_EXPR, prev_lhs);
3621 21 : gimple_seq_add_stmt (&seq, g);
3622 21 : gsi_replace_with_seq (gsi, seq, true);
3623 21 : return true;
3624 21 : }
3625 :
3626 :
3627 : /* Determine whether applying the 2 permutations (mask1 then mask2)
3628 : gives back one of the input. */
3629 :
3630 : static int
3631 42 : is_combined_permutation_identity (tree mask1, tree mask2)
3632 : {
3633 42 : tree mask;
3634 42 : unsigned HOST_WIDE_INT nelts, i, j;
3635 42 : bool maybe_identity1 = true;
3636 42 : bool maybe_identity2 = true;
3637 :
3638 42 : gcc_checking_assert (TREE_CODE (mask1) == VECTOR_CST
3639 : && TREE_CODE (mask2) == VECTOR_CST);
3640 :
3641 : /* For VLA masks, check for the following pattern:
3642 : v1 = VEC_PERM_EXPR (v0, ..., mask1)
3643 : v2 = VEC_PERM_EXPR (v1, ..., mask2)
3644 : -->
3645 : v2 = v0
3646 : if mask1 == mask2 == {nelts - 1, nelts - 2, ...}. */
3647 :
3648 42 : if (operand_equal_p (mask1, mask2, 0)
3649 42 : && !VECTOR_CST_NELTS (mask1).is_constant ())
3650 : {
3651 : vec_perm_builder builder;
3652 : if (tree_to_vec_perm_builder (&builder, mask1))
3653 : {
3654 : poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask1));
3655 : vec_perm_indices sel (builder, 1, nelts);
3656 : if (sel.series_p (0, 1, nelts - 1, -1))
3657 : return 1;
3658 : }
3659 : }
3660 :
3661 42 : mask = fold_ternary (VEC_PERM_EXPR, TREE_TYPE (mask1), mask1, mask1, mask2);
3662 42 : if (mask == NULL_TREE || TREE_CODE (mask) != VECTOR_CST)
3663 : return 0;
3664 :
3665 42 : if (!VECTOR_CST_NELTS (mask).is_constant (&nelts))
3666 : return 0;
3667 72 : for (i = 0; i < nelts; i++)
3668 : {
3669 72 : tree val = VECTOR_CST_ELT (mask, i);
3670 72 : gcc_assert (TREE_CODE (val) == INTEGER_CST);
3671 72 : j = TREE_INT_CST_LOW (val) & (2 * nelts - 1);
3672 72 : if (j == i)
3673 : maybe_identity2 = false;
3674 55 : else if (j == i + nelts)
3675 : maybe_identity1 = false;
3676 : else
3677 : return 0;
3678 : }
3679 0 : return maybe_identity1 ? 1 : maybe_identity2 ? 2 : 0;
3680 : }
3681 :
3682 : /* Combine a shuffle with its arguments. Returns true if there were any
3683 : changes made. */
3684 :
3685 : static bool
3686 190834 : simplify_permutation (gimple_stmt_iterator *gsi)
3687 : {
3688 190834 : gimple *stmt = gsi_stmt (*gsi);
3689 190834 : gimple *def_stmt = NULL;
3690 190834 : tree op0, op1, op2, op3, arg0, arg1;
3691 190834 : enum tree_code code, code2 = ERROR_MARK;
3692 190834 : bool single_use_op0 = false;
3693 :
3694 190834 : gcc_checking_assert (gimple_assign_rhs_code (stmt) == VEC_PERM_EXPR);
3695 :
3696 190834 : op0 = gimple_assign_rhs1 (stmt);
3697 190834 : op1 = gimple_assign_rhs2 (stmt);
3698 190834 : op2 = gimple_assign_rhs3 (stmt);
3699 :
3700 190834 : if (TREE_CODE (op2) != VECTOR_CST)
3701 : return false;
3702 :
3703 188065 : if (TREE_CODE (op0) == VECTOR_CST)
3704 : {
3705 : code = VECTOR_CST;
3706 : arg0 = op0;
3707 : }
3708 186199 : else if (TREE_CODE (op0) == SSA_NAME)
3709 : {
3710 186199 : def_stmt = get_prop_source_stmt (op0, false, &single_use_op0);
3711 186199 : if (!def_stmt)
3712 : return false;
3713 177850 : code = gimple_assign_rhs_code (def_stmt);
3714 177850 : if (code == VIEW_CONVERT_EXPR)
3715 : {
3716 1617 : tree rhs = gimple_assign_rhs1 (def_stmt);
3717 1617 : tree name = TREE_OPERAND (rhs, 0);
3718 1617 : if (TREE_CODE (name) != SSA_NAME)
3719 : return false;
3720 1617 : if (!has_single_use (name))
3721 246 : single_use_op0 = false;
3722 : /* Here we update the def_stmt through this VIEW_CONVERT_EXPR,
3723 : but still keep the code to indicate it comes from
3724 : VIEW_CONVERT_EXPR. */
3725 1617 : def_stmt = SSA_NAME_DEF_STMT (name);
3726 1617 : if (!def_stmt || !is_gimple_assign (def_stmt))
3727 : return false;
3728 830 : if (gimple_assign_rhs_code (def_stmt) != CONSTRUCTOR)
3729 : return false;
3730 : }
3731 176499 : if (!can_propagate_from (def_stmt))
3732 : return false;
3733 24472 : arg0 = gimple_assign_rhs1 (def_stmt);
3734 : }
3735 : else
3736 : return false;
3737 :
3738 : /* Two consecutive shuffles. */
3739 24472 : if (code == VEC_PERM_EXPR)
3740 : {
3741 6662 : tree orig;
3742 6662 : int ident;
3743 :
3744 6662 : if (op0 != op1)
3745 : return false;
3746 42 : op3 = gimple_assign_rhs3 (def_stmt);
3747 42 : if (TREE_CODE (op3) != VECTOR_CST)
3748 : return false;
3749 42 : ident = is_combined_permutation_identity (op3, op2);
3750 42 : if (!ident)
3751 : return false;
3752 0 : orig = (ident == 1) ? gimple_assign_rhs1 (def_stmt)
3753 0 : : gimple_assign_rhs2 (def_stmt);
3754 0 : gimple_assign_set_rhs1 (stmt, unshare_expr (orig));
3755 0 : gimple_assign_set_rhs_code (stmt, TREE_CODE (orig));
3756 0 : gimple_set_num_ops (stmt, 2);
3757 0 : update_stmt (stmt);
3758 0 : remove_prop_source_from_use (op0);
3759 0 : return true;
3760 : }
3761 19676 : else if (code == CONSTRUCTOR
3762 19676 : || code == VECTOR_CST
3763 : || code == VIEW_CONVERT_EXPR)
3764 : {
3765 4658 : if (op0 != op1)
3766 : {
3767 4474 : if (TREE_CODE (op0) == SSA_NAME && !single_use_op0)
3768 : return false;
3769 :
3770 3819 : if (TREE_CODE (op1) == VECTOR_CST)
3771 : arg1 = op1;
3772 3186 : else if (TREE_CODE (op1) == SSA_NAME)
3773 : {
3774 3186 : gimple *def_stmt2 = get_prop_source_stmt (op1, true, NULL);
3775 3186 : if (!def_stmt2)
3776 : return false;
3777 1678 : code2 = gimple_assign_rhs_code (def_stmt2);
3778 1678 : if (code2 == VIEW_CONVERT_EXPR)
3779 : {
3780 4 : tree rhs = gimple_assign_rhs1 (def_stmt2);
3781 4 : tree name = TREE_OPERAND (rhs, 0);
3782 4 : if (TREE_CODE (name) != SSA_NAME)
3783 : return false;
3784 4 : if (!has_single_use (name))
3785 : return false;
3786 3 : def_stmt2 = SSA_NAME_DEF_STMT (name);
3787 3 : if (!def_stmt2 || !is_gimple_assign (def_stmt2))
3788 : return false;
3789 0 : if (gimple_assign_rhs_code (def_stmt2) != CONSTRUCTOR)
3790 : return false;
3791 : }
3792 1674 : else if (code2 != CONSTRUCTOR && code2 != VECTOR_CST)
3793 : return false;
3794 1485 : if (!can_propagate_from (def_stmt2))
3795 : return false;
3796 1485 : arg1 = gimple_assign_rhs1 (def_stmt2);
3797 : }
3798 : else
3799 : return false;
3800 : }
3801 : else
3802 : {
3803 : /* Already used twice in this statement. */
3804 184 : if (TREE_CODE (op0) == SSA_NAME && num_imm_uses (op0) > 2)
3805 : return false;
3806 : arg1 = arg0;
3807 : }
3808 :
3809 : /* If there are any VIEW_CONVERT_EXPRs found when finding permutation
3810 : operands source, check whether it's valid to transform and prepare
3811 : the required new operands. */
3812 2234 : if (code == VIEW_CONVERT_EXPR || code2 == VIEW_CONVERT_EXPR)
3813 : {
3814 : /* Figure out the target vector type to which operands should be
3815 : converted. If both are CONSTRUCTOR, the types should be the
3816 : same, otherwise, use the one of CONSTRUCTOR. */
3817 24 : tree tgt_type = NULL_TREE;
3818 24 : if (code == VIEW_CONVERT_EXPR)
3819 : {
3820 24 : gcc_assert (gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR);
3821 24 : code = CONSTRUCTOR;
3822 24 : tgt_type = TREE_TYPE (arg0);
3823 : }
3824 24 : if (code2 == VIEW_CONVERT_EXPR)
3825 : {
3826 0 : tree arg1_type = TREE_TYPE (arg1);
3827 0 : if (tgt_type == NULL_TREE)
3828 : tgt_type = arg1_type;
3829 0 : else if (tgt_type != arg1_type)
3830 23 : return false;
3831 : }
3832 :
3833 24 : if (!VECTOR_TYPE_P (tgt_type))
3834 : return false;
3835 24 : tree op2_type = TREE_TYPE (op2);
3836 :
3837 : /* Figure out the shrunk factor. */
3838 24 : poly_uint64 tgt_units = TYPE_VECTOR_SUBPARTS (tgt_type);
3839 24 : poly_uint64 op2_units = TYPE_VECTOR_SUBPARTS (op2_type);
3840 24 : if (maybe_gt (tgt_units, op2_units))
3841 : return false;
3842 24 : unsigned int factor;
3843 47 : if (!constant_multiple_p (op2_units, tgt_units, &factor))
3844 : return false;
3845 :
3846 : /* Build the new permutation control vector as target vector. */
3847 24 : vec_perm_builder builder;
3848 24 : if (!tree_to_vec_perm_builder (&builder, op2))
3849 : return false;
3850 24 : vec_perm_indices indices (builder, 2, op2_units);
3851 24 : vec_perm_indices new_indices;
3852 24 : if (new_indices.new_shrunk_vector (indices, factor))
3853 : {
3854 1 : tree mask_type = tgt_type;
3855 1 : if (!VECTOR_INTEGER_TYPE_P (mask_type))
3856 : {
3857 0 : tree elem_type = TREE_TYPE (mask_type);
3858 0 : unsigned elem_size = TREE_INT_CST_LOW (TYPE_SIZE (elem_type));
3859 0 : tree int_type = build_nonstandard_integer_type (elem_size, 0);
3860 0 : mask_type = build_vector_type (int_type, tgt_units);
3861 : }
3862 1 : op2 = vec_perm_indices_to_tree (mask_type, new_indices);
3863 : }
3864 : else
3865 23 : return false;
3866 :
3867 : /* Convert the VECTOR_CST to the appropriate vector type. */
3868 1 : if (tgt_type != TREE_TYPE (arg0))
3869 0 : arg0 = fold_build1 (VIEW_CONVERT_EXPR, tgt_type, arg0);
3870 1 : else if (tgt_type != TREE_TYPE (arg1))
3871 0 : arg1 = fold_build1 (VIEW_CONVERT_EXPR, tgt_type, arg1);
3872 47 : }
3873 :
3874 : /* VIEW_CONVERT_EXPR should be updated to CONSTRUCTOR before. */
3875 2211 : gcc_assert (code == CONSTRUCTOR || code == VECTOR_CST);
3876 :
3877 : /* Shuffle of a constructor. */
3878 2211 : tree res_type
3879 2211 : = build_vector_type (TREE_TYPE (TREE_TYPE (arg0)),
3880 2211 : TYPE_VECTOR_SUBPARTS (TREE_TYPE (op2)));
3881 2211 : tree opt = fold_ternary (VEC_PERM_EXPR, res_type, arg0, arg1, op2);
3882 2211 : if (!opt
3883 280 : || (TREE_CODE (opt) != CONSTRUCTOR && TREE_CODE (opt) != VECTOR_CST))
3884 : return false;
3885 : /* Found VIEW_CONVERT_EXPR before, need one explicit conversion. */
3886 280 : if (res_type != TREE_TYPE (op0))
3887 : {
3888 1 : tree name = make_ssa_name (TREE_TYPE (opt));
3889 1 : gimple *ass_stmt = gimple_build_assign (name, opt);
3890 1 : gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
3891 1 : opt = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (op0), name);
3892 : }
3893 280 : gimple_assign_set_rhs_from_tree (gsi, opt);
3894 280 : update_stmt (gsi_stmt (*gsi));
3895 280 : if (TREE_CODE (op0) == SSA_NAME)
3896 1 : remove_prop_source_from_use (op0);
3897 280 : if (op0 != op1 && TREE_CODE (op1) == SSA_NAME)
3898 0 : remove_prop_source_from_use (op1);
3899 280 : return true;
3900 : }
3901 :
3902 : return false;
3903 : }
3904 :
3905 : /* Get the BIT_FIELD_REF definition of VAL, if any, looking through
3906 : conversions with code CONV_CODE or update it if still ERROR_MARK.
3907 : Return NULL_TREE if no such matching def was found. */
3908 :
3909 : static tree
3910 429595 : get_bit_field_ref_def (tree val, enum tree_code &conv_code)
3911 : {
3912 429595 : if (TREE_CODE (val) != SSA_NAME)
3913 : return NULL_TREE ;
3914 402611 : gimple *def_stmt = get_prop_source_stmt (val, false, NULL);
3915 402611 : if (!def_stmt)
3916 : return NULL_TREE;
3917 322890 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
3918 322890 : if (code == FLOAT_EXPR
3919 322890 : || code == FIX_TRUNC_EXPR
3920 : || CONVERT_EXPR_CODE_P (code))
3921 : {
3922 187419 : tree op1 = gimple_assign_rhs1 (def_stmt);
3923 187419 : if (conv_code == ERROR_MARK)
3924 89766 : conv_code = code;
3925 97653 : else if (conv_code != code)
3926 : return NULL_TREE;
3927 187394 : if (TREE_CODE (op1) != SSA_NAME)
3928 : return NULL_TREE;
3929 79093 : def_stmt = SSA_NAME_DEF_STMT (op1);
3930 79093 : if (! is_gimple_assign (def_stmt))
3931 : return NULL_TREE;
3932 63293 : code = gimple_assign_rhs_code (def_stmt);
3933 : }
3934 198764 : if (code != BIT_FIELD_REF)
3935 : return NULL_TREE;
3936 24488 : return gimple_assign_rhs1 (def_stmt);
3937 : }
3938 :
3939 : /* Recognize a VEC_PERM_EXPR. Returns true if there were any changes. */
3940 :
3941 : static bool
3942 165777 : simplify_vector_constructor (gimple_stmt_iterator *gsi)
3943 : {
3944 165777 : gimple *stmt = gsi_stmt (*gsi);
3945 165777 : tree op, orig[2], type;
3946 165777 : unsigned i;
3947 165777 : unsigned HOST_WIDE_INT nelts;
3948 165777 : unsigned HOST_WIDE_INT refnelts;
3949 165777 : enum tree_code conv_code;
3950 165777 : constructor_elt *elt;
3951 :
3952 165777 : op = gimple_assign_rhs1 (stmt);
3953 165777 : type = TREE_TYPE (op);
3954 165777 : gcc_checking_assert (TREE_CODE (op) == CONSTRUCTOR
3955 : && TREE_CODE (type) == VECTOR_TYPE);
3956 :
3957 165777 : if (!TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
3958 : return false;
3959 :
3960 165777 : orig[0] = NULL;
3961 165777 : orig[1] = NULL;
3962 165777 : tree orig_elem_type[2] = {};
3963 165777 : conv_code = ERROR_MARK;
3964 165777 : bool maybe_ident = true;
3965 165777 : bool maybe_blend[2] = { true, true };
3966 165777 : tree one_constant = NULL_TREE;
3967 165777 : tree one_nonconstant = NULL_TREE;
3968 165777 : tree subelt;
3969 165777 : auto_vec<tree> constants;
3970 165777 : constants.safe_grow_cleared (nelts, true);
3971 165777 : auto_vec<std::pair<unsigned, unsigned>, 64> elts;
3972 165777 : unsigned int tsubelts = 0;
3973 463319 : FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (op), i, elt)
3974 : {
3975 429595 : tree ref, op1;
3976 429595 : unsigned int elem, src_elem_size;
3977 429595 : unsigned HOST_WIDE_INT nsubelts = 1;
3978 :
3979 429595 : if (i >= nelts)
3980 165777 : return false;
3981 :
3982 : /* Look for elements extracted and possibly converted from
3983 : another vector. */
3984 429595 : op1 = get_bit_field_ref_def (elt->value, conv_code);
3985 429595 : if (op1
3986 24488 : && TREE_CODE ((ref = TREE_OPERAND (op1, 0))) == SSA_NAME
3987 6103 : && VECTOR_TYPE_P (TREE_TYPE (ref))
3988 6092 : && (tree_nop_conversion_p (TREE_TYPE (op1),
3989 6092 : TREE_TYPE (TREE_TYPE (ref)))
3990 893 : || (VECTOR_TYPE_P (TREE_TYPE (op1))
3991 133 : && tree_nop_conversion_p (TREE_TYPE (TREE_TYPE (op1)),
3992 133 : TREE_TYPE (TREE_TYPE (ref)))
3993 133 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (op1))
3994 133 : .is_constant (&nsubelts)))
3995 5332 : && constant_multiple_p (bit_field_size (op1), nsubelts,
3996 : &src_elem_size)
3997 434927 : && constant_multiple_p (bit_field_offset (op1), src_elem_size, &elem)
3998 434927 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (ref)).is_constant (&refnelts))
3999 : {
4000 : unsigned int j;
4001 5703 : for (j = 0; j < 2; ++j)
4002 : {
4003 5674 : if (!orig[j])
4004 : {
4005 2444 : if (j == 0
4006 2647 : || useless_type_conversion_p (TREE_TYPE (orig[0]),
4007 203 : TREE_TYPE (ref)))
4008 : break;
4009 : }
4010 3230 : else if (ref == orig[j])
4011 : break;
4012 : }
4013 : /* Found a suitable vector element. */
4014 5332 : if (j < 2)
4015 : {
4016 5303 : orig[j] = ref;
4017 : /* Track what element type was actually extracted (which may
4018 : differ in signedness from the vector's element type due to
4019 : tree_nop_conversion_p). */
4020 5303 : if (!orig_elem_type[j])
4021 2440 : orig_elem_type[j] = TREE_TYPE (op1);
4022 5303 : if (elem != i || j != 0)
4023 2209 : maybe_ident = false;
4024 5303 : if (elem != i)
4025 2147 : maybe_blend[j] = false;
4026 10865 : for (unsigned int k = 0; k < nsubelts; ++k)
4027 5562 : elts.safe_push (std::make_pair (j, elem + k));
4028 5303 : tsubelts += nsubelts;
4029 5303 : continue;
4030 5303 : }
4031 : /* Else fallthru. */
4032 : }
4033 : /* Handle elements not extracted from a vector.
4034 : 1. constants by permuting with constant vector
4035 : 2. a unique non-constant element by permuting with a splat vector */
4036 424292 : if (orig[1]
4037 259681 : && orig[1] != error_mark_node)
4038 : return false;
4039 424263 : orig[1] = error_mark_node;
4040 424263 : if (VECTOR_TYPE_P (TREE_TYPE (elt->value))
4041 424263 : && !TYPE_VECTOR_SUBPARTS (TREE_TYPE (elt->value))
4042 5733 : .is_constant (&nsubelts))
4043 : return false;
4044 424263 : if (CONSTANT_CLASS_P (elt->value))
4045 : {
4046 26980 : if (one_nonconstant)
4047 : return false;
4048 18374 : if (!one_constant)
4049 8966 : one_constant = TREE_CODE (elt->value) == VECTOR_CST
4050 8966 : ? VECTOR_CST_ELT (elt->value, 0)
4051 : : elt->value;
4052 18374 : if (TREE_CODE (elt->value) == VECTOR_CST)
4053 : {
4054 687 : for (unsigned int k = 0; k < nsubelts; k++)
4055 507 : constants[tsubelts + k] = VECTOR_CST_ELT (elt->value, k);
4056 : }
4057 : else
4058 18194 : constants[tsubelts] = elt->value;
4059 : }
4060 : else
4061 : {
4062 397283 : if (one_constant)
4063 : return false;
4064 388782 : subelt = VECTOR_TYPE_P (TREE_TYPE (elt->value))
4065 388782 : ? ssa_uniform_vector_p (elt->value)
4066 : : elt->value;
4067 388782 : if (!subelt)
4068 : return false;
4069 383471 : if (!one_nonconstant)
4070 : one_nonconstant = subelt;
4071 233137 : else if (!operand_equal_p (one_nonconstant, subelt, 0))
4072 : return false;
4073 : }
4074 584807 : for (unsigned int k = 0; k < nsubelts; ++k)
4075 292568 : elts.safe_push (std::make_pair (1, tsubelts + k));
4076 292239 : tsubelts += nsubelts;
4077 292239 : maybe_ident = false;
4078 : }
4079 :
4080 67448 : if (elts.length () < nelts)
4081 : return false;
4082 :
4083 32473 : if (! orig[0]
4084 32473 : || ! VECTOR_TYPE_P (TREE_TYPE (orig[0])))
4085 : return false;
4086 1618 : refnelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (orig[0])).to_constant ();
4087 : /* We currently do not handle larger destination vectors. */
4088 1618 : if (refnelts < nelts)
4089 : return false;
4090 :
4091 : /* Determine the element type for the conversion source.
4092 : As orig_elem_type keeps track of the original type, check
4093 : if we need to perform a sign swap after permuting.
4094 : We need to be able to construct a vector type from the element
4095 : type which is not possible for e.g. BitInt or pointers
4096 : so pun with an integer type if needed. */
4097 1377 : tree perm_eltype = TREE_TYPE (TREE_TYPE (orig[0]));
4098 1377 : bool sign_change_p = false;
4099 1377 : if (conv_code != ERROR_MARK
4100 364 : && orig_elem_type[0]
4101 1741 : && TYPE_SIGN (orig_elem_type[0]) != TYPE_SIGN (perm_eltype))
4102 : {
4103 38 : perm_eltype = signed_or_unsigned_type_for
4104 38 : (TYPE_UNSIGNED (orig_elem_type[0]), perm_eltype);
4105 38 : sign_change_p = true;
4106 : }
4107 1377 : tree conv_src_type = build_vector_type (perm_eltype, nelts);
4108 :
4109 1377 : if (maybe_ident)
4110 : {
4111 : /* When there is no conversion, use the target type directly. */
4112 501 : if (conv_code == ERROR_MARK && nelts != refnelts)
4113 501 : conv_src_type = type;
4114 501 : if (conv_code != ERROR_MARK
4115 501 : && !supportable_convert_operation (conv_code, type, conv_src_type,
4116 : &conv_code))
4117 : {
4118 : /* Only few targets implement direct conversion patterns so try
4119 : some simple special cases via VEC_[UN]PACK[_FLOAT]_LO_EXPR. */
4120 99 : optab optab;
4121 99 : insn_code icode;
4122 99 : tree halfvectype, dblvectype;
4123 99 : enum tree_code unpack_op;
4124 :
4125 99 : if (!BYTES_BIG_ENDIAN)
4126 175 : unpack_op = (FLOAT_TYPE_P (TREE_TYPE (type))
4127 99 : ? VEC_UNPACK_FLOAT_LO_EXPR
4128 : : VEC_UNPACK_LO_EXPR);
4129 : else
4130 : unpack_op = (FLOAT_TYPE_P (TREE_TYPE (type))
4131 : ? VEC_UNPACK_FLOAT_HI_EXPR
4132 : : VEC_UNPACK_HI_EXPR);
4133 :
4134 : /* Conversions between DFP and FP have no special tree code
4135 : but we cannot handle those since all relevant vector conversion
4136 : optabs only have a single mode. */
4137 15 : if (CONVERT_EXPR_CODE_P (conv_code)
4138 84 : && FLOAT_TYPE_P (TREE_TYPE (type))
4139 115 : && (DECIMAL_FLOAT_TYPE_P (TREE_TYPE (type))
4140 8 : != DECIMAL_FLOAT_TYPE_P (TREE_TYPE (conv_src_type))))
4141 : return false;
4142 :
4143 15 : if (CONVERT_EXPR_CODE_P (conv_code)
4144 83 : && (2 * TYPE_PRECISION (TREE_TYPE (TREE_TYPE (orig[0])))
4145 83 : == TYPE_PRECISION (TREE_TYPE (type)))
4146 6 : && orig_elem_type[0]
4147 6 : && useless_type_conversion_p (orig_elem_type[0],
4148 6 : TREE_TYPE (TREE_TYPE (orig[0])))
4149 6 : && mode_for_vector (as_a <scalar_mode>
4150 6 : (TYPE_MODE (TREE_TYPE (TREE_TYPE (orig[0])))),
4151 12 : nelts * 2).exists ()
4152 6 : && (dblvectype
4153 6 : = build_vector_type (TREE_TYPE (TREE_TYPE (orig[0])),
4154 6 : nelts * 2))
4155 : /* Only use it for vector modes or for vector booleans
4156 : represented as scalar bitmasks. See PR95528. */
4157 6 : && (VECTOR_MODE_P (TYPE_MODE (dblvectype))
4158 0 : || VECTOR_BOOLEAN_TYPE_P (dblvectype))
4159 6 : && (optab = optab_for_tree_code (unpack_op,
4160 : dblvectype,
4161 : optab_default))
4162 6 : && ((icode = optab_handler (optab, TYPE_MODE (dblvectype)))
4163 : != CODE_FOR_nothing)
4164 98 : && (insn_data[icode].operand[0].mode == TYPE_MODE (type)))
4165 : {
4166 0 : gimple_seq stmts = NULL;
4167 0 : tree dbl;
4168 0 : if (refnelts == nelts)
4169 : {
4170 : /* ??? Paradoxical subregs don't exist, so insert into
4171 : the lower half of a wider zero vector. */
4172 0 : dbl = gimple_build (&stmts, BIT_INSERT_EXPR, dblvectype,
4173 : build_zero_cst (dblvectype), orig[0],
4174 0 : bitsize_zero_node);
4175 : }
4176 0 : else if (refnelts == 2 * nelts)
4177 : dbl = orig[0];
4178 : else
4179 0 : dbl = gimple_build (&stmts, BIT_FIELD_REF, dblvectype,
4180 0 : orig[0], TYPE_SIZE (dblvectype),
4181 0 : bitsize_zero_node);
4182 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
4183 0 : gimple_assign_set_rhs_with_ops (gsi, unpack_op, dbl);
4184 : }
4185 15 : else if (CONVERT_EXPR_CODE_P (conv_code)
4186 83 : && (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (orig[0])))
4187 83 : == 2 * TYPE_PRECISION (TREE_TYPE (type)))
4188 1 : && orig_elem_type[0]
4189 1 : && useless_type_conversion_p (orig_elem_type[0],
4190 1 : TREE_TYPE (TREE_TYPE (orig[0])))
4191 1 : && mode_for_vector (as_a <scalar_mode>
4192 1 : (TYPE_MODE
4193 : (TREE_TYPE (TREE_TYPE (orig[0])))),
4194 2 : nelts / 2).exists ()
4195 1 : && (halfvectype
4196 1 : = build_vector_type (TREE_TYPE (TREE_TYPE (orig[0])),
4197 1 : nelts / 2))
4198 : /* Only use it for vector modes or for vector booleans
4199 : represented as scalar bitmasks. See PR95528. */
4200 1 : && (VECTOR_MODE_P (TYPE_MODE (halfvectype))
4201 0 : || VECTOR_BOOLEAN_TYPE_P (halfvectype))
4202 1 : && (optab = optab_for_tree_code (VEC_PACK_TRUNC_EXPR,
4203 : halfvectype,
4204 : optab_default))
4205 1 : && ((icode = optab_handler (optab, TYPE_MODE (halfvectype)))
4206 : != CODE_FOR_nothing)
4207 99 : && (insn_data[icode].operand[0].mode == TYPE_MODE (type)))
4208 : {
4209 0 : gimple_seq stmts = NULL;
4210 0 : tree low = gimple_build (&stmts, BIT_FIELD_REF, halfvectype,
4211 0 : orig[0], TYPE_SIZE (halfvectype),
4212 0 : bitsize_zero_node);
4213 0 : tree hig = gimple_build (&stmts, BIT_FIELD_REF, halfvectype,
4214 0 : orig[0], TYPE_SIZE (halfvectype),
4215 0 : TYPE_SIZE (halfvectype));
4216 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
4217 0 : gimple_assign_set_rhs_with_ops (gsi, VEC_PACK_TRUNC_EXPR,
4218 : low, hig);
4219 : }
4220 : else
4221 98 : return false;
4222 0 : update_stmt (gsi_stmt (*gsi));
4223 0 : return true;
4224 : }
4225 402 : if (nelts != refnelts)
4226 : {
4227 14 : gassign *lowpart
4228 14 : = gimple_build_assign (make_ssa_name (conv_src_type),
4229 : build3 (BIT_FIELD_REF, conv_src_type,
4230 14 : orig[0], TYPE_SIZE (conv_src_type),
4231 : bitsize_zero_node));
4232 14 : gsi_insert_before (gsi, lowpart, GSI_SAME_STMT);
4233 14 : orig[0] = gimple_assign_lhs (lowpart);
4234 : }
4235 388 : else if (sign_change_p)
4236 : {
4237 0 : gassign *conv
4238 0 : = gimple_build_assign (make_ssa_name (conv_src_type),
4239 : build1 (VIEW_CONVERT_EXPR, conv_src_type,
4240 : orig[0]));
4241 0 : gsi_insert_before (gsi, conv, GSI_SAME_STMT);
4242 0 : orig[0] = gimple_assign_lhs (conv);
4243 : }
4244 402 : if (conv_code == ERROR_MARK)
4245 : {
4246 385 : tree src_type = TREE_TYPE (orig[0]);
4247 385 : if (!useless_type_conversion_p (type, src_type))
4248 : {
4249 0 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (type),
4250 : TYPE_VECTOR_SUBPARTS (src_type))
4251 : && tree_nop_conversion_p (TREE_TYPE (type),
4252 : TREE_TYPE (src_type)));
4253 0 : tree rhs = build1 (VIEW_CONVERT_EXPR, type, orig[0]);
4254 0 : orig[0] = make_ssa_name (type);
4255 0 : gassign *assign = gimple_build_assign (orig[0], rhs);
4256 0 : gsi_insert_before (gsi, assign, GSI_SAME_STMT);
4257 : }
4258 385 : gimple_assign_set_rhs_from_tree (gsi, orig[0]);
4259 : }
4260 : else
4261 17 : gimple_assign_set_rhs_with_ops (gsi, conv_code, orig[0],
4262 : NULL_TREE, NULL_TREE);
4263 : }
4264 : else
4265 : {
4266 : /* If we combine a vector with a non-vector avoid cases where
4267 : we'll obviously end up with more GIMPLE stmts which is when
4268 : we'll later not fold this to a single insert into the vector
4269 : and we had a single extract originally. See PR92819. */
4270 876 : if (nelts == 2
4271 516 : && refnelts > 2
4272 118 : && orig[1] == error_mark_node
4273 31 : && !maybe_blend[0])
4274 269 : return false;
4275 851 : tree mask_type, perm_type;
4276 851 : perm_type = TREE_TYPE (orig[0]);
4277 851 : if (conv_code != ERROR_MARK
4278 851 : && !supportable_convert_operation (conv_code, type, conv_src_type,
4279 : &conv_code))
4280 : return false;
4281 :
4282 : /* Now that we know the number of elements of the source build the
4283 : permute vector.
4284 : ??? When the second vector has constant values we can shuffle
4285 : it and its source indexes to make the permutation supported.
4286 : For now it mimics a blend. */
4287 668 : vec_perm_builder sel (refnelts, refnelts, 1);
4288 668 : bool all_same_p = true;
4289 7452 : for (i = 0; i < elts.length (); ++i)
4290 : {
4291 3058 : sel.quick_push (elts[i].second + elts[i].first * refnelts);
4292 3058 : all_same_p &= known_eq (sel[i], sel[0]);
4293 : }
4294 : /* And fill the tail with "something". It's really don't care,
4295 : and ideally we'd allow VEC_PERM to have a smaller destination
4296 : vector. As a heuristic:
4297 :
4298 : (a) if what we have so far duplicates a single element, make the
4299 : tail do the same
4300 :
4301 : (b) otherwise preserve a uniform orig[0]. This facilitates
4302 : later pattern-matching of VEC_PERM_EXPR to a BIT_INSERT_EXPR. */
4303 1240 : for (; i < refnelts; ++i)
4304 1144 : sel.quick_push (all_same_p
4305 1716 : ? sel[0]
4306 140 : : (elts[0].second == 0 && elts[0].first == 0
4307 828 : ? 0 : refnelts) + i);
4308 863 : vec_perm_indices indices (sel, orig[1] ? 2 : 1, refnelts);
4309 668 : machine_mode vmode = TYPE_MODE (perm_type);
4310 668 : if ((cfun->curr_properties & PROP_gimple_lvec)
4311 668 : && !can_vec_perm_const_p (vmode, vmode, indices))
4312 : return false;
4313 607 : mask_type = build_vector_type (ssizetype, refnelts);
4314 607 : tree op2 = vec_perm_indices_to_tree (mask_type, indices);
4315 607 : bool converted_orig1 = false;
4316 607 : gimple_seq stmts = NULL;
4317 607 : if (!orig[1])
4318 172 : orig[1] = orig[0];
4319 435 : else if (orig[1] == error_mark_node
4320 314 : && one_nonconstant)
4321 : {
4322 : /* ??? We can see if we can safely convert to the original
4323 : element type. */
4324 90 : converted_orig1 = conv_code != ERROR_MARK;
4325 90 : tree target_type = converted_orig1 ? type : perm_type;
4326 90 : tree nonconstant_for_splat = one_nonconstant;
4327 : /* If there's a nop conversion between the target element type and
4328 : the nonconstant's type, convert it. */
4329 90 : if (!useless_type_conversion_p (TREE_TYPE (target_type),
4330 90 : TREE_TYPE (one_nonconstant)))
4331 0 : nonconstant_for_splat
4332 0 : = gimple_build (&stmts, NOP_EXPR, TREE_TYPE (target_type),
4333 : one_nonconstant);
4334 90 : orig[1] = gimple_build_vector_from_val (&stmts, UNKNOWN_LOCATION,
4335 : target_type,
4336 : nonconstant_for_splat);
4337 90 : }
4338 345 : else if (orig[1] == error_mark_node)
4339 : {
4340 : /* ??? See if we can convert the vector to the original type. */
4341 224 : converted_orig1 = conv_code != ERROR_MARK;
4342 224 : unsigned n = converted_orig1 ? nelts : refnelts;
4343 207 : tree target_type = converted_orig1 ? type : perm_type;
4344 224 : tree_vector_builder vec (target_type, n, 1);
4345 1748 : for (unsigned i = 0; i < n; ++i)
4346 2924 : if (i < nelts && constants[i])
4347 : {
4348 778 : tree constant = constants[i];
4349 : /* If there's a nop conversion, convert the constant. */
4350 778 : if (!useless_type_conversion_p (TREE_TYPE (target_type),
4351 778 : TREE_TYPE (constant)))
4352 2 : constant = fold_convert (TREE_TYPE (target_type), constant);
4353 778 : vec.quick_push (constant);
4354 : }
4355 : else
4356 : {
4357 : /* ??? Push a don't-care value. */
4358 746 : tree constant = one_constant;
4359 746 : if (!useless_type_conversion_p (TREE_TYPE (target_type),
4360 746 : TREE_TYPE (constant)))
4361 2 : constant = fold_convert (TREE_TYPE (target_type), constant);
4362 746 : vec.quick_push (constant);
4363 : }
4364 224 : orig[1] = vec.build ();
4365 224 : }
4366 486 : tree blend_op2 = NULL_TREE;
4367 486 : if (converted_orig1)
4368 : {
4369 : /* Make sure we can do a blend in the target type. */
4370 19 : vec_perm_builder sel (nelts, nelts, 1);
4371 87 : for (i = 0; i < elts.length (); ++i)
4372 68 : sel.quick_push (elts[i].first
4373 68 : ? elts[i].second + nelts : i);
4374 19 : vec_perm_indices indices (sel, 2, nelts);
4375 19 : machine_mode vmode = TYPE_MODE (type);
4376 19 : if ((cfun->curr_properties & PROP_gimple_lvec)
4377 19 : && !can_vec_perm_const_p (vmode, vmode, indices))
4378 0 : return false;
4379 19 : mask_type = build_vector_type (ssizetype, nelts);
4380 19 : blend_op2 = vec_perm_indices_to_tree (mask_type, indices);
4381 19 : }
4382 :
4383 : /* For a real orig[1] (no splat, constant etc.) we might need to
4384 : nop-convert it. Do so here. */
4385 607 : if (orig[1] && orig[1] != error_mark_node
4386 607 : && !useless_type_conversion_p (perm_type, TREE_TYPE (orig[1]))
4387 626 : && tree_nop_conversion_p (TREE_TYPE (perm_type),
4388 19 : TREE_TYPE (TREE_TYPE (orig[1]))))
4389 0 : orig[1] = gimple_build (&stmts, VIEW_CONVERT_EXPR, perm_type,
4390 : orig[1]);
4391 :
4392 607 : tree orig1_for_perm
4393 607 : = converted_orig1 ? build_zero_cst (perm_type) : orig[1];
4394 607 : tree res = gimple_build (&stmts, VEC_PERM_EXPR, perm_type,
4395 : orig[0], orig1_for_perm, op2);
4396 : /* If we're building a smaller vector, extract the element
4397 : with the proper type. */
4398 607 : if (nelts != refnelts)
4399 232 : res = gimple_build (&stmts, BIT_FIELD_REF,
4400 : conv_code != ERROR_MARK ? conv_src_type : type,
4401 : res,
4402 116 : TYPE_SIZE (conv_code != ERROR_MARK ? conv_src_type
4403 : : type),
4404 116 : bitsize_zero_node);
4405 : /* Otherwise, we can still have an intermediate sign change.
4406 : ??? In that case we have two subsequent conversions.
4407 : We should be able to merge them. */
4408 491 : else if (sign_change_p)
4409 14 : res = gimple_build (&stmts, VIEW_CONVERT_EXPR, conv_src_type, res);
4410 : /* Finally, apply the conversion. */
4411 607 : if (conv_code != ERROR_MARK)
4412 52 : res = gimple_build (&stmts, conv_code, type, res);
4413 555 : else if (!useless_type_conversion_p (type, TREE_TYPE (res)))
4414 : {
4415 3 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (type),
4416 : TYPE_VECTOR_SUBPARTS (perm_type))
4417 : && tree_nop_conversion_p (TREE_TYPE (type),
4418 : TREE_TYPE (perm_type)));
4419 3 : res = gimple_build (&stmts, VIEW_CONVERT_EXPR, type, res);
4420 : }
4421 : /* Blend in the actual constant. */
4422 607 : if (converted_orig1)
4423 19 : res = gimple_build (&stmts, VEC_PERM_EXPR, type,
4424 19 : res, orig[1], blend_op2);
4425 607 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
4426 607 : gimple_assign_set_rhs_with_ops (gsi, SSA_NAME, res);
4427 668 : }
4428 1009 : update_stmt (gsi_stmt (*gsi));
4429 1009 : return true;
4430 165777 : }
4431 :
4432 : /* Prepare a TARGET_MEM_REF ref so that it can be subsetted as
4433 : lvalue. This splits out an address computation stmt before *GSI
4434 : and returns a MEM_REF wrapping the address. */
4435 :
4436 : static tree
4437 1243 : prepare_target_mem_ref_lvalue (tree ref, gimple_stmt_iterator *gsi)
4438 : {
4439 1243 : if (TREE_CODE (TREE_OPERAND (ref, 0)) == ADDR_EXPR)
4440 250 : mark_addressable (TREE_OPERAND (TREE_OPERAND (ref, 0), 0));
4441 1243 : tree ptrtype = build_pointer_type (TREE_TYPE (ref));
4442 1243 : tree tem = make_ssa_name (ptrtype);
4443 1243 : gimple *new_stmt
4444 1243 : = gimple_build_assign (tem, build1 (ADDR_EXPR, TREE_TYPE (tem),
4445 : unshare_expr (ref)));
4446 1243 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4447 2486 : ref = build2_loc (EXPR_LOCATION (ref),
4448 1243 : MEM_REF, TREE_TYPE (ref), tem,
4449 1243 : build_int_cst (TREE_TYPE (TREE_OPERAND (ref, 1)), 0));
4450 1243 : return ref;
4451 : }
4452 :
4453 : /* Rewrite the vector load at *GSI to component-wise loads if the load
4454 : is only used in BIT_FIELD_REF extractions with eventual intermediate
4455 : widening. */
4456 :
4457 : static void
4458 295325 : optimize_vector_load (gimple_stmt_iterator *gsi)
4459 : {
4460 295325 : gimple *stmt = gsi_stmt (*gsi);
4461 295325 : tree lhs = gimple_assign_lhs (stmt);
4462 295325 : tree rhs = gimple_assign_rhs1 (stmt);
4463 295325 : tree vuse = gimple_vuse (stmt);
4464 :
4465 : /* Gather BIT_FIELD_REFs to rewrite, looking through
4466 : VEC_UNPACK_{LO,HI}_EXPR. */
4467 295325 : use_operand_p use_p;
4468 295325 : imm_use_iterator iter;
4469 295325 : bool rewrite = true;
4470 295325 : bool scalar_use = false;
4471 295325 : bool unpack_use = false;
4472 295325 : auto_vec<gimple *, 8> bf_stmts;
4473 295325 : auto_vec<tree, 8> worklist;
4474 295325 : worklist.quick_push (lhs);
4475 297275 : do
4476 : {
4477 297275 : tree def = worklist.pop ();
4478 297275 : unsigned HOST_WIDE_INT def_eltsize
4479 297275 : = TREE_INT_CST_LOW (TYPE_SIZE (TREE_TYPE (TREE_TYPE (def))));
4480 673462 : FOR_EACH_IMM_USE_FAST (use_p, iter, def)
4481 : {
4482 354965 : gimple *use_stmt = USE_STMT (use_p);
4483 354965 : if (is_gimple_debug (use_stmt))
4484 78912 : continue;
4485 353589 : tree use_lhs;
4486 353589 : if (!is_gimple_assign (use_stmt)
4487 : /* For alias reasons we move the use to the place of the
4488 : load. Avoid this when abnormals are involved. */
4489 353589 : || ((TREE_CODE ((use_lhs = gimple_assign_lhs (use_stmt)))
4490 : == SSA_NAME)
4491 241929 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs)))
4492 : {
4493 : rewrite = false;
4494 276053 : break;
4495 : }
4496 317353 : enum tree_code use_code = gimple_assign_rhs_code (use_stmt);
4497 317353 : tree use_rhs = gimple_assign_rhs1 (use_stmt);
4498 391008 : if (use_code == BIT_FIELD_REF
4499 73656 : && TREE_OPERAND (use_rhs, 0) == def
4500 : /* If its on the VEC_UNPACK_{HI,LO}_EXPR
4501 : def need to verify it is element aligned. */
4502 391009 : && (def == lhs
4503 153 : || (known_eq (bit_field_size (use_rhs), def_eltsize)
4504 153 : && constant_multiple_p (bit_field_offset (use_rhs),
4505 : def_eltsize)
4506 : /* We can simulate the VEC_UNPACK_{HI,LO}_EXPR
4507 : via a NOP_EXPR only for integral types.
4508 : ??? Support VEC_UNPACK_FLOAT_{HI,LO}_EXPR. */
4509 153 : && INTEGRAL_TYPE_P (TREE_TYPE (use_rhs)))))
4510 : {
4511 73655 : if (!VECTOR_TYPE_P (TREE_TYPE (gimple_assign_lhs (use_stmt))))
4512 71338 : scalar_use = true;
4513 73655 : bf_stmts.safe_push (use_stmt);
4514 73655 : continue;
4515 : }
4516 : /* Walk through one level of VEC_UNPACK_{LO,HI}_EXPR. */
4517 243698 : if (def == lhs
4518 241829 : && (use_code == VEC_UNPACK_HI_EXPR
4519 241829 : || use_code == VEC_UNPACK_LO_EXPR)
4520 3881 : && use_rhs == lhs)
4521 : {
4522 3881 : unpack_use = true;
4523 3881 : worklist.safe_push (gimple_assign_lhs (use_stmt));
4524 3881 : continue;
4525 : }
4526 : rewrite = false;
4527 : break;
4528 297275 : }
4529 297275 : if (!rewrite)
4530 : break;
4531 : }
4532 42444 : while (!worklist.is_empty ());
4533 :
4534 295325 : rewrite = rewrite && (scalar_use
4535 19272 : || unpack_use
4536 626 : || !can_implement_p (mov_optab,
4537 626 : TYPE_MODE (TREE_TYPE (lhs))));
4538 295325 : if (!rewrite)
4539 : {
4540 276235 : gsi_next (gsi);
4541 276235 : return;
4542 : }
4543 : /* We now have all ultimate uses of the load to rewrite in bf_stmts. */
4544 :
4545 : /* Prepare the original ref to be wrapped in adjusted BIT_FIELD_REFs.
4546 : For TARGET_MEM_REFs we have to separate the LEA from the reference. */
4547 19090 : tree load_rhs = rhs;
4548 19090 : if (TREE_CODE (load_rhs) == TARGET_MEM_REF)
4549 1242 : load_rhs = prepare_target_mem_ref_lvalue (load_rhs, gsi);
4550 :
4551 : /* Rewrite the BIT_FIELD_REFs to be actual loads, re-emitting them at
4552 : the place of the original load. */
4553 124490 : for (gimple *use_stmt : bf_stmts)
4554 : {
4555 67220 : tree bfr = gimple_assign_rhs1 (use_stmt);
4556 67220 : tree new_rhs = unshare_expr (load_rhs);
4557 67220 : if (TREE_OPERAND (bfr, 0) != lhs)
4558 : {
4559 : /* When the BIT_FIELD_REF is on the promoted vector we have to
4560 : adjust it and emit a conversion afterwards. */
4561 152 : gimple *def_stmt
4562 152 : = SSA_NAME_DEF_STMT (TREE_OPERAND (bfr, 0));
4563 152 : enum tree_code def_code
4564 152 : = gimple_assign_rhs_code (def_stmt);
4565 :
4566 : /* The adjusted BIT_FIELD_REF is of the promotion source
4567 : vector size and at half of the offset... */
4568 152 : new_rhs = fold_build3 (BIT_FIELD_REF,
4569 : TREE_TYPE (TREE_TYPE (lhs)),
4570 : new_rhs,
4571 : TYPE_SIZE (TREE_TYPE (TREE_TYPE (lhs))),
4572 : size_binop (EXACT_DIV_EXPR,
4573 : TREE_OPERAND (bfr, 2),
4574 : bitsize_int (2)));
4575 : /* ... and offsetted by half of the vector if VEC_UNPACK_HI_EXPR. */
4576 152 : if (def_code == (!BYTES_BIG_ENDIAN
4577 : ? VEC_UNPACK_HI_EXPR : VEC_UNPACK_LO_EXPR))
4578 76 : TREE_OPERAND (new_rhs, 2)
4579 152 : = size_binop (PLUS_EXPR, TREE_OPERAND (new_rhs, 2),
4580 : size_binop (EXACT_DIV_EXPR,
4581 : TYPE_SIZE (TREE_TYPE (lhs)),
4582 : bitsize_int (2)));
4583 152 : tree tem = make_ssa_name (TREE_TYPE (TREE_TYPE (lhs)));
4584 152 : gimple *new_stmt = gimple_build_assign (tem, new_rhs);
4585 152 : location_t loc = gimple_location (use_stmt);
4586 152 : gimple_set_location (new_stmt, loc);
4587 152 : gimple_set_vuse (new_stmt, vuse);
4588 152 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4589 : /* Perform scalar promotion. */
4590 152 : new_stmt = gimple_build_assign (gimple_assign_lhs (use_stmt),
4591 : NOP_EXPR, tem);
4592 152 : gimple_set_location (new_stmt, loc);
4593 152 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4594 : }
4595 : else
4596 : {
4597 : /* When the BIT_FIELD_REF is on the original load result
4598 : we can just wrap that. */
4599 67068 : tree new_rhs = fold_build3 (BIT_FIELD_REF, TREE_TYPE (bfr),
4600 : unshare_expr (load_rhs),
4601 : TREE_OPERAND (bfr, 1),
4602 : TREE_OPERAND (bfr, 2));
4603 67068 : gimple *new_stmt = gimple_build_assign (gimple_assign_lhs (use_stmt),
4604 : new_rhs);
4605 67068 : location_t loc = gimple_location (use_stmt);
4606 67068 : gimple_set_location (new_stmt, loc);
4607 67068 : gimple_set_vuse (new_stmt, vuse);
4608 67068 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4609 : }
4610 67220 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
4611 67220 : unlink_stmt_vdef (use_stmt);
4612 67220 : gsi_remove (&gsi2, true);
4613 : }
4614 :
4615 : /* Finally get rid of the intermediate stmts. */
4616 19090 : gimple *use_stmt;
4617 38786 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
4618 : {
4619 606 : if (is_gimple_debug (use_stmt))
4620 : {
4621 544 : if (gimple_debug_bind_p (use_stmt))
4622 : {
4623 544 : gimple_debug_bind_reset_value (use_stmt);
4624 544 : update_stmt (use_stmt);
4625 : }
4626 544 : continue;
4627 : }
4628 62 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
4629 62 : unlink_stmt_vdef (use_stmt);
4630 62 : release_defs (use_stmt);
4631 62 : gsi_remove (&gsi2, true);
4632 19090 : }
4633 : /* And the original load. */
4634 19090 : release_defs (stmt);
4635 19090 : gsi_remove (gsi, true);
4636 295325 : }
4637 :
4638 :
4639 : /* Primitive "lattice" function for gimple_simplify. */
4640 :
4641 : static tree
4642 1679059960 : fwprop_ssa_val (tree name)
4643 : {
4644 : /* First valueize NAME. */
4645 1679059960 : if (TREE_CODE (name) == SSA_NAME
4646 1679059960 : && SSA_NAME_VERSION (name) < lattice.length ())
4647 : {
4648 1678209255 : tree val = lattice[SSA_NAME_VERSION (name)];
4649 1678209255 : if (val)
4650 1679059960 : name = val;
4651 : }
4652 : /* We continue matching along SSA use-def edges for SSA names
4653 : that are not single-use. Currently there are no patterns
4654 : that would cause any issues with that. */
4655 1679059960 : return name;
4656 : }
4657 :
4658 : /* Search for opportunities to free half of the lanes in the following pattern:
4659 :
4660 : v_in = {e0, e1, e2, e3}
4661 : v_1 = VEC_PERM <v_in, v_in, {0, 2, 0, 2}>
4662 : // v_1 = {e0, e2, e0, e2}
4663 : v_2 = VEC_PERM <v_in, v_in, {1, 3, 1, 3}>
4664 : // v_2 = {e1, e3, e1, e3}
4665 :
4666 : v_x = v_1 + v_2
4667 : // v_x = {e0+e1, e2+e3, e0+e1, e2+e3}
4668 : v_y = v_1 - v_2
4669 : // v_y = {e0-e1, e2-e3, e0-e1, e2-e3}
4670 :
4671 : v_out = VEC_PERM <v_x, v_y, {0, 1, 6, 7}>
4672 : // v_out = {e0+e1, e2+e3, e0-e1, e2-e3}
4673 :
4674 : The last statement could be simplified to:
4675 : v_out' = VEC_PERM <v_x, v_y, {0, 1, 4, 5}>
4676 : // v_out' = {e0+e1, e2+e3, e0-e1, e2-e3}
4677 :
4678 : Characteristic properties:
4679 : - v_1 and v_2 are created from the same input vector v_in and introduce the
4680 : lane duplication (in the selection operand) that we can eliminate.
4681 : - v_x and v_y are results from lane-preserving operations that use v_1 and
4682 : v_2 as inputs.
4683 : - v_out is created by selecting from duplicated lanes. */
4684 :
4685 : static bool
4686 188618 : recognise_vec_perm_simplify_seq (gassign *stmt, vec_perm_simplify_seq *seq)
4687 : {
4688 188618 : unsigned HOST_WIDE_INT nelts;
4689 :
4690 188618 : gcc_checking_assert (stmt);
4691 188618 : gcc_checking_assert (gimple_assign_rhs_code (stmt) == VEC_PERM_EXPR);
4692 188618 : basic_block bb = gimple_bb (stmt);
4693 :
4694 : /* Decompose the final vec permute statement. */
4695 188618 : tree v_x = gimple_assign_rhs1 (stmt);
4696 188618 : tree v_y = gimple_assign_rhs2 (stmt);
4697 188618 : tree sel = gimple_assign_rhs3 (stmt);
4698 :
4699 188618 : if (TREE_CODE (sel) != VECTOR_CST
4700 185849 : || !VECTOR_CST_NELTS (sel).is_constant (&nelts)
4701 185849 : || TREE_CODE (v_x) != SSA_NAME
4702 183993 : || TREE_CODE (v_y) != SSA_NAME
4703 178724 : || !has_single_use (v_x)
4704 299845 : || !has_single_use (v_y))
4705 79186 : return false;
4706 :
4707 : /* Don't analyse sequences with many lanes. */
4708 109432 : if (nelts > 4)
4709 : return false;
4710 :
4711 : /* Lookup the definition of v_x and v_y. */
4712 105820 : gassign *v_x_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x));
4713 105820 : gassign *v_y_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_y));
4714 105443 : if (!v_x_stmt || gimple_bb (v_x_stmt) != bb
4715 211263 : || !v_y_stmt || gimple_bb (v_y_stmt) != bb)
4716 : return false;
4717 :
4718 : /* Check the operations that define v_x and v_y. */
4719 105436 : if (TREE_CODE_CLASS (gimple_assign_rhs_code (v_x_stmt)) != tcc_binary
4720 107522 : || TREE_CODE_CLASS (gimple_assign_rhs_code (v_y_stmt)) != tcc_binary)
4721 : return false;
4722 :
4723 2086 : tree v_x_1 = gimple_assign_rhs1 (v_x_stmt);
4724 2086 : tree v_x_2 = gimple_assign_rhs2 (v_x_stmt);
4725 2086 : tree v_y_1 = gimple_assign_rhs1 (v_y_stmt);
4726 2086 : tree v_y_2 = gimple_assign_rhs2 (v_y_stmt);
4727 :
4728 2086 : if (v_x_stmt == v_y_stmt
4729 2086 : || TREE_CODE (v_x_1) != SSA_NAME
4730 2083 : || TREE_CODE (v_x_2) != SSA_NAME
4731 2059 : || num_imm_uses (v_x_1) != 2
4732 3989 : || num_imm_uses (v_x_2) != 2)
4733 : return false;
4734 :
4735 1865 : if (v_x_1 != v_y_1 || v_x_2 != v_y_2)
4736 : {
4737 : /* Allow operands of commutative operators to swap. */
4738 653 : if (commutative_tree_code (gimple_assign_rhs_code (v_x_stmt)))
4739 : {
4740 : /* Keep v_x_1 the first operand for non-commutative operators. */
4741 245 : std::swap (v_x_1, v_x_2);
4742 245 : if (v_x_1 != v_y_1 || v_x_2 != v_y_2)
4743 : return false;
4744 : }
4745 408 : else if (commutative_tree_code (gimple_assign_rhs_code (v_y_stmt)))
4746 : {
4747 408 : if (v_x_1 != v_y_2 || v_x_2 != v_y_1)
4748 : return false;
4749 : }
4750 : else
4751 : return false;
4752 : }
4753 1865 : gassign *v_1_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x_1));
4754 1865 : gassign *v_2_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x_2));
4755 1801 : if (!v_1_stmt || gimple_bb (v_1_stmt) != bb
4756 3666 : || !v_2_stmt || gimple_bb (v_2_stmt) != bb)
4757 : return false;
4758 :
4759 1797 : if (gimple_assign_rhs_code (v_1_stmt) != VEC_PERM_EXPR
4760 1919 : || gimple_assign_rhs_code (v_2_stmt) != VEC_PERM_EXPR)
4761 : return false;
4762 :
4763 : /* Decompose initial VEC_PERM_EXPRs. */
4764 108 : tree v_in = gimple_assign_rhs1 (v_1_stmt);
4765 108 : tree v_1_sel = gimple_assign_rhs3 (v_1_stmt);
4766 108 : tree v_2_sel = gimple_assign_rhs3 (v_2_stmt);
4767 108 : if (v_in != gimple_assign_rhs2 (v_1_stmt)
4768 103 : || v_in != gimple_assign_rhs1 (v_2_stmt)
4769 209 : || v_in != gimple_assign_rhs2 (v_2_stmt))
4770 : return false;
4771 :
4772 101 : unsigned HOST_WIDE_INT v_1_nelts, v_2_nelts;
4773 101 : if (TREE_CODE (v_1_sel) != VECTOR_CST
4774 101 : || !VECTOR_CST_NELTS (v_1_sel).is_constant (&v_1_nelts)
4775 101 : || TREE_CODE (v_2_sel) != VECTOR_CST
4776 202 : || !VECTOR_CST_NELTS (v_2_sel).is_constant (&v_2_nelts))
4777 0 : return false;
4778 :
4779 101 : if (nelts != v_1_nelts || nelts != v_2_nelts)
4780 : return false;
4781 :
4782 : /* Create the new selector. */
4783 101 : vec_perm_builder new_sel_perm (nelts, nelts, 1);
4784 101 : auto_vec<bool> lanes (nelts);
4785 101 : lanes.quick_grow_cleared (nelts);
4786 505 : for (unsigned int i = 0; i < nelts; i++)
4787 : {
4788 : /* Extract the i-th value from the selector. */
4789 404 : unsigned int sel_cst = TREE_INT_CST_LOW (VECTOR_CST_ELT (sel, i));
4790 404 : unsigned int lane = sel_cst % nelts;
4791 404 : unsigned int offs = sel_cst / nelts;
4792 :
4793 : /* Check what's in the lane. */
4794 404 : unsigned int e_1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (v_1_sel, lane));
4795 404 : unsigned int e_2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (v_2_sel, lane));
4796 :
4797 : /* Reuse previous lane (if any). */
4798 404 : unsigned int l = 0;
4799 687 : for (; l < lane; l++)
4800 : {
4801 481 : if ((TREE_INT_CST_LOW (VECTOR_CST_ELT (v_1_sel, l)) == e_1)
4802 481 : && (TREE_INT_CST_LOW (VECTOR_CST_ELT (v_2_sel, l)) == e_2))
4803 : break;
4804 : }
4805 :
4806 : /* Add to narrowed selector. */
4807 404 : new_sel_perm.quick_push (l + offs * nelts);
4808 :
4809 : /* Mark lane as used. */
4810 404 : lanes[l] = true;
4811 : }
4812 :
4813 : /* Count how many lanes are need. */
4814 : unsigned int cnt = 0;
4815 505 : for (unsigned int i = 0; i < nelts; i++)
4816 404 : cnt += lanes[i];
4817 :
4818 : /* If more than (nelts/2) lanes are needed, skip the sequence. */
4819 101 : if (cnt > nelts / 2)
4820 : return false;
4821 :
4822 : /* Check if the resulting permutation is cheap. */
4823 101 : vec_perm_indices new_indices (new_sel_perm, 2, nelts);
4824 101 : tree vectype = TREE_TYPE (gimple_assign_lhs (stmt));
4825 101 : machine_mode vmode = TYPE_MODE (vectype);
4826 101 : if (!can_vec_perm_const_p (vmode, vmode, new_indices, false))
4827 : return false;
4828 :
4829 101 : *seq = XNEW (struct _vec_perm_simplify_seq);
4830 101 : (*seq)->stmt = stmt;
4831 101 : (*seq)->v_1_stmt = v_1_stmt;
4832 101 : (*seq)->v_2_stmt = v_2_stmt;
4833 101 : (*seq)->v_x_stmt = v_x_stmt;
4834 101 : (*seq)->v_y_stmt = v_y_stmt;
4835 101 : (*seq)->nelts = nelts;
4836 101 : (*seq)->new_sel = vect_gen_perm_mask_checked (vectype, new_indices);
4837 :
4838 101 : if (dump_file)
4839 : {
4840 28 : fprintf (dump_file, "Found vec perm simplify sequence ending with:\n\t");
4841 28 : print_gimple_stmt (dump_file, stmt, 0);
4842 :
4843 28 : if (dump_flags & TDF_DETAILS)
4844 : {
4845 28 : fprintf (dump_file, "\tNarrowed vec_perm selector: ");
4846 28 : print_generic_expr (dump_file, (*seq)->new_sel);
4847 28 : fprintf (dump_file, "\n");
4848 : }
4849 : }
4850 :
4851 : return true;
4852 202 : }
4853 :
4854 : /* Reduce the lane consumption of a simplifiable vec perm sequence. */
4855 :
4856 : static void
4857 74 : narrow_vec_perm_simplify_seq (const vec_perm_simplify_seq &seq)
4858 : {
4859 74 : gassign *stmt = seq->stmt;
4860 74 : if (dump_file && (dump_flags & TDF_DETAILS))
4861 : {
4862 22 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
4863 22 : fprintf (dump_file, "Old stmt: ");
4864 22 : print_gimple_stmt (dump_file, stmt, 0);
4865 : }
4866 :
4867 : /* Update the last VEC_PERM statement. */
4868 74 : gimple_assign_set_rhs3 (stmt, seq->new_sel);
4869 74 : update_stmt (stmt);
4870 :
4871 74 : if (dump_file && (dump_flags & TDF_DETAILS))
4872 : {
4873 22 : fprintf (dump_file, "New stmt: ");
4874 22 : print_gimple_stmt (dump_file, stmt, 0);
4875 : }
4876 74 : }
4877 :
4878 : /* Test if we can blend two simplifiable vec permute sequences.
4879 : NEED_SWAP will be set, if sequences must be swapped for blending. */
4880 :
4881 : static bool
4882 47 : can_blend_vec_perm_simplify_seqs_p (vec_perm_simplify_seq seq1,
4883 : vec_perm_simplify_seq seq2,
4884 : bool *need_swap)
4885 : {
4886 47 : unsigned int nelts = seq1->nelts;
4887 47 : basic_block bb = gimple_bb (seq1->stmt);
4888 :
4889 47 : gcc_assert (gimple_bb (seq2->stmt) == bb);
4890 :
4891 : /* BBs and number of elements must be equal. */
4892 47 : if (gimple_bb (seq2->stmt) != bb || seq2->nelts != nelts)
4893 : return false;
4894 :
4895 : /* We need vectors of the same type. */
4896 47 : if (TREE_TYPE (gimple_assign_lhs (seq1->stmt))
4897 47 : != TREE_TYPE (gimple_assign_lhs (seq2->stmt)))
4898 : return false;
4899 :
4900 : /* We require isomorphic operators. */
4901 41 : if (((gimple_assign_rhs_code (seq1->v_x_stmt)
4902 41 : != gimple_assign_rhs_code (seq2->v_x_stmt))
4903 41 : || (gimple_assign_rhs_code (seq1->v_y_stmt)
4904 41 : != gimple_assign_rhs_code (seq2->v_y_stmt))))
4905 : return false;
4906 :
4907 : /* We cannot have any dependencies between the sequences.
4908 :
4909 : For merging, we will reuse seq1->v_1_stmt and seq1->v_2_stmt.
4910 : seq1's v_in is defined before these statements, but we need
4911 : to check if seq2's v_in is defined before them as well.
4912 :
4913 : Further, we will reuse seq2->stmt. We need to ensure that
4914 : seq1->v_x_stmt and seq1->v_y_stmt are before it.
4915 :
4916 : Note, that we don't need to check the BBs here, because all
4917 : statements of both sequences have to be in the same BB. */
4918 :
4919 41 : tree seq2_v_in = gimple_assign_rhs1 (seq2->v_1_stmt);
4920 41 : if (TREE_CODE (seq2_v_in) != SSA_NAME)
4921 : return false;
4922 :
4923 41 : gassign *seq2_v_in_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (seq2_v_in));
4924 41 : if (!seq2_v_in_stmt || gimple_bb (seq2_v_in_stmt) != bb
4925 41 : || (gimple_uid (seq2_v_in_stmt) > gimple_uid (seq1->v_1_stmt))
4926 37 : || (gimple_uid (seq1->v_x_stmt) > gimple_uid (seq2->stmt))
4927 37 : || (gimple_uid (seq1->v_y_stmt) > gimple_uid (seq2->stmt)))
4928 : {
4929 4 : tree seq1_v_in = gimple_assign_rhs1 (seq1->v_1_stmt);
4930 4 : if (TREE_CODE (seq1_v_in) != SSA_NAME)
4931 : return false;
4932 :
4933 4 : gassign *seq1_v_in_stmt
4934 4 : = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (seq1_v_in));
4935 : /* Let's try to see if we succeed when swapping the sequences. */
4936 4 : if (!seq1_v_in_stmt || gimple_bb (seq1_v_in_stmt)
4937 0 : || (gimple_uid (seq1_v_in_stmt) > gimple_uid (seq2->v_1_stmt))
4938 0 : || (gimple_uid (seq2->v_x_stmt) > gimple_uid (seq1->stmt))
4939 0 : || (gimple_uid (seq2->v_y_stmt) > gimple_uid (seq1->stmt)))
4940 : return false;
4941 0 : *need_swap = true;
4942 : }
4943 : else
4944 37 : *need_swap = false;
4945 :
4946 37 : if (dump_file && (dump_flags & TDF_DETAILS))
4947 11 : fprintf (dump_file, "Found vec perm simplify sequence pair.\n");
4948 :
4949 : return true;
4950 : }
4951 :
4952 : /* Calculate the permutations for blending the two given vec permute
4953 : sequences. This may fail if the resulting permutation is not
4954 : supported. */
4955 :
4956 : static bool
4957 37 : calc_perm_vec_perm_simplify_seqs (vec_perm_simplify_seq seq1,
4958 : vec_perm_simplify_seq seq2,
4959 : vec_perm_indices *seq2_stmt_indices,
4960 : vec_perm_indices *seq1_v_1_stmt_indices,
4961 : vec_perm_indices *seq1_v_2_stmt_indices)
4962 : {
4963 37 : unsigned int i;
4964 37 : unsigned int nelts = seq1->nelts;
4965 37 : auto_vec<unsigned int> lane_assignment;
4966 37 : lane_assignment.create (nelts);
4967 :
4968 : /* Mark all lanes as free. */
4969 37 : lane_assignment.quick_grow_cleared (nelts);
4970 :
4971 : /* Allocate lanes for seq1. */
4972 185 : for (i = 0; i < nelts; i++)
4973 : {
4974 148 : unsigned int l = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq1->new_sel, i));
4975 148 : l %= nelts;
4976 148 : lane_assignment[l] = 1;
4977 : }
4978 :
4979 : /* Allocate lanes for seq2 and calculate selector for seq2->stmt. */
4980 37 : vec_perm_builder seq2_stmt_sel_perm (nelts, nelts, 1);
4981 185 : for (i = 0; i < nelts; i++)
4982 : {
4983 148 : unsigned int sel = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq2->new_sel, i));
4984 148 : unsigned int lane = sel % nelts;
4985 148 : unsigned int offs = sel / nelts;
4986 148 : unsigned int new_sel;
4987 :
4988 : /* Check if we already allocated the lane for seq2. */
4989 148 : unsigned int j = 0;
4990 263 : for (; j < i; j++)
4991 : {
4992 189 : unsigned int sel_old;
4993 189 : sel_old = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq2->new_sel, j));
4994 189 : unsigned int lane_old = sel_old % nelts;
4995 189 : if (lane == lane_old)
4996 : {
4997 74 : new_sel = seq2_stmt_sel_perm[j].to_constant ();
4998 74 : new_sel = (new_sel % nelts) + offs * nelts;
4999 74 : break;
5000 : }
5001 : }
5002 :
5003 : /* If the lane is not allocated, we need to do that now. */
5004 148 : if (j == i)
5005 : {
5006 : unsigned int l_orig = lane;
5007 182 : while (lane_assignment[lane] != 0)
5008 : {
5009 108 : lane = (lane + 1) % nelts;
5010 :
5011 : /* This should not happen if both sequences utilize no more than
5012 : half of the lanes. Test anyway to guarantee termination. */
5013 108 : if (lane == l_orig)
5014 0 : return false;
5015 : }
5016 :
5017 : /* Allocate lane. */
5018 74 : lane_assignment[lane] = 2 + l_orig;
5019 74 : new_sel = lane + offs * nelts;
5020 : }
5021 :
5022 148 : seq2_stmt_sel_perm.quick_push (new_sel);
5023 : }
5024 :
5025 : /* Check if the resulting permutation is cheap. */
5026 37 : seq2_stmt_indices->new_vector (seq2_stmt_sel_perm, 2, nelts);
5027 37 : tree vectype = TREE_TYPE (gimple_assign_lhs (seq2->stmt));
5028 37 : machine_mode vmode = TYPE_MODE (vectype);
5029 37 : if (!can_vec_perm_const_p (vmode, vmode, *seq2_stmt_indices, false))
5030 : return false;
5031 :
5032 : /* Calculate selectors for seq1->v_1_stmt and seq1->v_2_stmt. */
5033 37 : vec_perm_builder seq1_v_1_stmt_sel_perm (nelts, nelts, 1);
5034 37 : vec_perm_builder seq1_v_2_stmt_sel_perm (nelts, nelts, 1);
5035 185 : for (i = 0; i < nelts; i++)
5036 : {
5037 148 : bool use_seq1 = lane_assignment[i] < 2;
5038 148 : unsigned int l1, l2;
5039 :
5040 148 : if (use_seq1)
5041 : {
5042 : /* Just reuse the selector indices. */
5043 74 : tree s1 = gimple_assign_rhs3 (seq1->v_1_stmt);
5044 74 : tree s2 = gimple_assign_rhs3 (seq1->v_2_stmt);
5045 74 : l1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s1, i));
5046 74 : l2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s2, i));
5047 : }
5048 : else
5049 : {
5050 : /* We moved the lanes for seq2, so we need to adjust for that. */
5051 74 : tree s1 = gimple_assign_rhs3 (seq2->v_1_stmt);
5052 74 : tree s2 = gimple_assign_rhs3 (seq2->v_2_stmt);
5053 74 : l1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s1, lane_assignment[i] - 2));
5054 74 : l2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s2, lane_assignment[i] - 2));
5055 : }
5056 :
5057 148 : l1 %= nelts;
5058 148 : l2 %= nelts;
5059 222 : seq1_v_1_stmt_sel_perm.quick_push (l1 + (use_seq1 ? 0 : nelts));
5060 148 : seq1_v_2_stmt_sel_perm.quick_push (l2 + (use_seq1 ? 0 : nelts));
5061 : }
5062 :
5063 37 : seq1_v_1_stmt_indices->new_vector (seq1_v_1_stmt_sel_perm, 2, nelts);
5064 37 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_1_stmt));
5065 37 : vmode = TYPE_MODE (vectype);
5066 37 : if (!can_vec_perm_const_p (vmode, vmode, *seq1_v_1_stmt_indices, false))
5067 : return false;
5068 :
5069 37 : seq1_v_2_stmt_indices->new_vector (seq1_v_2_stmt_sel_perm, 2, nelts);
5070 37 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_2_stmt));
5071 37 : vmode = TYPE_MODE (vectype);
5072 37 : if (!can_vec_perm_const_p (vmode, vmode, *seq1_v_2_stmt_indices, false))
5073 : return false;
5074 :
5075 : return true;
5076 74 : }
5077 :
5078 : /* Blend the two given simplifiable vec permute sequences using the
5079 : given permutations. */
5080 :
5081 : static void
5082 37 : blend_vec_perm_simplify_seqs (vec_perm_simplify_seq seq1,
5083 : vec_perm_simplify_seq seq2,
5084 : const vec_perm_indices &seq2_stmt_indices,
5085 : const vec_perm_indices &seq1_v_1_stmt_indices,
5086 : const vec_perm_indices &seq1_v_2_stmt_indices)
5087 : {
5088 : /* We don't need to adjust seq1->stmt because its lanes consumption
5089 : was already narrowed before entering this function. */
5090 :
5091 : /* Adjust seq2->stmt: copy RHS1/RHS2 from seq1->stmt and set new sel. */
5092 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5093 : {
5094 11 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
5095 11 : fprintf (dump_file, "Old stmt: ");
5096 11 : print_gimple_stmt (dump_file, seq2->stmt, 0);
5097 : }
5098 :
5099 37 : gimple_assign_set_rhs1 (seq2->stmt, gimple_assign_rhs1 (seq1->stmt));
5100 74 : gimple_assign_set_rhs2 (seq2->stmt, gimple_assign_rhs2 (seq1->stmt));
5101 37 : tree vectype = TREE_TYPE (gimple_assign_lhs (seq2->stmt));
5102 37 : tree sel = vect_gen_perm_mask_checked (vectype, seq2_stmt_indices);
5103 37 : gimple_assign_set_rhs3 (seq2->stmt, sel);
5104 37 : update_stmt (seq2->stmt);
5105 :
5106 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5107 : {
5108 11 : fprintf (dump_file, "New stmt: ");
5109 11 : print_gimple_stmt (dump_file, seq2->stmt, 0);
5110 : }
5111 :
5112 : /* Adjust seq1->v_1_stmt: copy RHS2 from seq2->v_1_stmt and set new sel. */
5113 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5114 : {
5115 11 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
5116 11 : fprintf (dump_file, "Old stmt: ");
5117 11 : print_gimple_stmt (dump_file, seq1->v_1_stmt, 0);
5118 : }
5119 :
5120 37 : gimple_assign_set_rhs2 (seq1->v_1_stmt, gimple_assign_rhs1 (seq2->v_1_stmt));
5121 37 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_1_stmt));
5122 37 : sel = vect_gen_perm_mask_checked (vectype, seq1_v_1_stmt_indices);
5123 37 : gimple_assign_set_rhs3 (seq1->v_1_stmt, sel);
5124 37 : update_stmt (seq1->v_1_stmt);
5125 :
5126 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5127 : {
5128 11 : fprintf (dump_file, "New stmt: ");
5129 11 : print_gimple_stmt (dump_file, seq1->v_1_stmt, 0);
5130 : }
5131 :
5132 : /* Adjust seq1->v_2_stmt: copy RHS2 from seq2->v_2_stmt and set new sel. */
5133 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5134 : {
5135 11 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
5136 11 : fprintf (dump_file, "Old stmt: ");
5137 11 : print_gimple_stmt (dump_file, seq1->v_2_stmt, 0);
5138 : }
5139 :
5140 37 : gimple_assign_set_rhs2 (seq1->v_2_stmt, gimple_assign_rhs1 (seq2->v_2_stmt));
5141 37 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_2_stmt));
5142 37 : sel = vect_gen_perm_mask_checked (vectype, seq1_v_2_stmt_indices);
5143 37 : gimple_assign_set_rhs3 (seq1->v_2_stmt, sel);
5144 37 : update_stmt (seq1->v_2_stmt);
5145 :
5146 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5147 : {
5148 11 : fprintf (dump_file, "New stmt: ");
5149 11 : print_gimple_stmt (dump_file, seq1->v_2_stmt, 0);
5150 : }
5151 :
5152 : /* At this point, we have four unmodified seq2 stmts, which will be
5153 : eliminated by DCE. */
5154 :
5155 37 : if (dump_file)
5156 11 : fprintf (dump_file, "Vec perm simplify sequences have been blended.\n\n");
5157 37 : }
5158 :
5159 : /* Try to blend narrowed vec_perm_simplify_seqs pairwise.
5160 : The provided list will be empty after this call. */
5161 :
5162 : static void
5163 325951771 : process_vec_perm_simplify_seq_list (vec<vec_perm_simplify_seq> *l)
5164 : {
5165 325951771 : unsigned int i, j;
5166 325951771 : vec_perm_simplify_seq seq1, seq2;
5167 :
5168 325951771 : if (l->is_empty ())
5169 325951726 : return;
5170 :
5171 45 : if (dump_file && (dump_flags & TDF_DETAILS))
5172 13 : fprintf (dump_file, "\nProcessing %u vec perm simplify sequences.\n",
5173 : l->length ());
5174 :
5175 109 : FOR_EACH_VEC_ELT (*l, i, seq1)
5176 : {
5177 64 : if (i + 1 < l->length ())
5178 : {
5179 51 : FOR_EACH_VEC_ELT_FROM (*l, j, seq2, i + 1)
5180 : {
5181 47 : bool swap = false;
5182 47 : if (can_blend_vec_perm_simplify_seqs_p (seq1, seq2, &swap))
5183 : {
5184 37 : vec_perm_indices seq2_stmt_indices;
5185 37 : vec_perm_indices seq1_v_1_stmt_indices;
5186 37 : vec_perm_indices seq1_v_2_stmt_indices;
5187 111 : if (calc_perm_vec_perm_simplify_seqs (swap ? seq2 : seq1,
5188 : swap ? seq1 : seq2,
5189 : &seq2_stmt_indices,
5190 : &seq1_v_1_stmt_indices,
5191 : &seq1_v_2_stmt_indices))
5192 : {
5193 : /* Narrow lane usage. */
5194 37 : narrow_vec_perm_simplify_seq (seq1);
5195 37 : narrow_vec_perm_simplify_seq (seq2);
5196 :
5197 : /* Blend sequences. */
5198 37 : blend_vec_perm_simplify_seqs (swap ? seq2 : seq1,
5199 : swap ? seq1 : seq2,
5200 : seq2_stmt_indices,
5201 : seq1_v_1_stmt_indices,
5202 : seq1_v_2_stmt_indices);
5203 :
5204 : /* We can use unordered_remove as we break the loop. */
5205 37 : l->unordered_remove (j);
5206 37 : XDELETE (seq2);
5207 37 : break;
5208 : }
5209 37 : }
5210 : }
5211 : }
5212 :
5213 : /* We don't need to call l->remove for seq1. */
5214 64 : XDELETE (seq1);
5215 : }
5216 :
5217 45 : l->truncate (0);
5218 : }
5219 :
5220 : static void
5221 101 : append_vec_perm_simplify_seq_list (vec<vec_perm_simplify_seq> *l,
5222 : const vec_perm_simplify_seq &seq)
5223 : {
5224 : /* If no space on list left, then process the list. */
5225 101 : if (!l->space (1))
5226 0 : process_vec_perm_simplify_seq_list (l);
5227 :
5228 101 : l->quick_push (seq);
5229 101 : }
5230 :
5231 : /* Main entry point for the forward propagation and statement combine
5232 : optimizer. */
5233 :
5234 : namespace {
5235 :
5236 : const pass_data pass_data_forwprop =
5237 : {
5238 : GIMPLE_PASS, /* type */
5239 : "forwprop", /* name */
5240 : OPTGROUP_NONE, /* optinfo_flags */
5241 : TV_TREE_FORWPROP, /* tv_id */
5242 : ( PROP_cfg | PROP_ssa ), /* properties_required */
5243 : 0, /* properties_provided */
5244 : 0, /* properties_destroyed */
5245 : 0, /* todo_flags_start */
5246 : 0, /* todo_flags_finish */
5247 : };
5248 :
5249 : class pass_forwprop : public gimple_opt_pass
5250 : {
5251 : public:
5252 1469140 : pass_forwprop (gcc::context *ctxt)
5253 2938280 : : gimple_opt_pass (pass_data_forwprop, ctxt), last_p (false)
5254 : {}
5255 :
5256 : /* opt_pass methods: */
5257 1175312 : opt_pass * clone () final override { return new pass_forwprop (m_ctxt); }
5258 1762968 : void set_pass_param (unsigned int n, bool param) final override
5259 : {
5260 1762968 : switch (n)
5261 : {
5262 1175312 : case 0:
5263 1175312 : m_full_walk = param;
5264 1175312 : break;
5265 587656 : case 1:
5266 587656 : last_p = param;
5267 587656 : break;
5268 0 : default:
5269 0 : gcc_unreachable();
5270 : }
5271 1762968 : }
5272 5646528 : bool gate (function *) final override { return flag_tree_forwprop; }
5273 : unsigned int execute (function *) final override;
5274 :
5275 : private:
5276 : /* Determines whether the pass instance should set PROP_last_full_fold. */
5277 : bool last_p;
5278 :
5279 : /* True if the aggregate props are doing a full walk or not. */
5280 : bool m_full_walk = false;
5281 : }; // class pass_forwprop
5282 :
5283 : /* Attempt to make the BB block of __builtin_unreachable unreachable by changing
5284 : the incoming jumps. Return true if at least one jump was changed. */
5285 :
5286 : static bool
5287 1131 : optimize_unreachable (basic_block bb)
5288 : {
5289 1131 : gimple_stmt_iterator gsi;
5290 1131 : gimple *stmt;
5291 1131 : edge_iterator ei;
5292 1131 : edge e;
5293 1131 : bool ret;
5294 :
5295 1131 : ret = false;
5296 2331 : FOR_EACH_EDGE (e, ei, bb->preds)
5297 : {
5298 1200 : gsi = gsi_last_bb (e->src);
5299 1200 : if (gsi_end_p (gsi))
5300 328 : continue;
5301 :
5302 872 : stmt = gsi_stmt (gsi);
5303 872 : if (gcond *cond_stmt = dyn_cast <gcond *> (stmt))
5304 : {
5305 : /* If the condition is already true/false
5306 : ignore it. This can happen during copy prop of forwprop. */
5307 686 : if (gimple_cond_true_p (cond_stmt)
5308 678 : || gimple_cond_false_p (cond_stmt))
5309 8 : continue;
5310 670 : else if (e->flags & EDGE_TRUE_VALUE)
5311 582 : gimple_cond_make_false (cond_stmt);
5312 88 : else if (e->flags & EDGE_FALSE_VALUE)
5313 88 : gimple_cond_make_true (cond_stmt);
5314 : else
5315 0 : gcc_unreachable ();
5316 670 : update_stmt (cond_stmt);
5317 : }
5318 : else
5319 : {
5320 : /* Todo: handle other cases. e.g. switch. */
5321 194 : continue;
5322 : }
5323 :
5324 670 : ret = true;
5325 : }
5326 :
5327 1131 : return ret;
5328 : }
5329 :
5330 : unsigned int
5331 5643906 : pass_forwprop::execute (function *fun)
5332 : {
5333 5643906 : unsigned int todoflags = 0;
5334 : /* Handle a full walk only when expensive optimizations are on. */
5335 5643906 : bool full_walk = m_full_walk && flag_expensive_optimizations;
5336 :
5337 5643906 : cfg_changed = false;
5338 5643906 : if (last_p)
5339 1050776 : fun->curr_properties |= PROP_last_full_fold;
5340 :
5341 5643906 : calculate_dominance_info (CDI_DOMINATORS);
5342 :
5343 : /* Combine stmts with the stmts defining their operands. Do that
5344 : in an order that guarantees visiting SSA defs before SSA uses. */
5345 11287812 : lattice.create (num_ssa_names);
5346 11287812 : lattice.quick_grow_cleared (num_ssa_names);
5347 5643906 : int *postorder = XNEWVEC (int, n_basic_blocks_for_fn (fun));
5348 5643906 : int postorder_num = pre_and_rev_post_order_compute_fn (fun, NULL,
5349 : postorder, false);
5350 5643906 : int *bb_to_rpo = XNEWVEC (int, last_basic_block_for_fn (fun));
5351 50858599 : for (int i = 0; i < postorder_num; ++i)
5352 : {
5353 45214693 : bb_to_rpo[postorder[i]] = i;
5354 45214693 : edge_iterator ei;
5355 45214693 : edge e;
5356 108805057 : FOR_EACH_EDGE (e, ei, BASIC_BLOCK_FOR_FN (fun, postorder[i])->succs)
5357 63590364 : e->flags &= ~EDGE_EXECUTABLE;
5358 : }
5359 5643906 : single_succ_edge (BASIC_BLOCK_FOR_FN (fun, ENTRY_BLOCK))->flags
5360 5643906 : |= EDGE_EXECUTABLE;
5361 5643906 : auto_vec<gimple *, 4> to_fixup;
5362 5643906 : auto_vec<gimple *, 32> to_remove;
5363 5643906 : auto_vec<unsigned, 32> to_remove_defs;
5364 5643906 : auto_vec<std::pair<int, int>, 10> edges_to_remove;
5365 5643906 : auto_bitmap simple_dce_worklist;
5366 5643906 : auto_bitmap need_ab_cleanup;
5367 5643906 : to_purge = BITMAP_ALLOC (NULL);
5368 5643906 : auto_vec<vec_perm_simplify_seq, 8> vec_perm_simplify_seq_list;
5369 50858599 : for (int i = 0; i < postorder_num; ++i)
5370 : {
5371 45214693 : gimple_stmt_iterator gsi;
5372 45214693 : basic_block bb = BASIC_BLOCK_FOR_FN (fun, postorder[i]);
5373 45214693 : edge_iterator ei;
5374 45214693 : edge e;
5375 :
5376 : /* Skip processing not executable blocks. We could improve
5377 : single_use tracking by at least unlinking uses from unreachable
5378 : blocks but since blocks with uses are not processed in a
5379 : meaningful order this is probably not worth it. */
5380 45214693 : bool any = false;
5381 46358931 : FOR_EACH_EDGE (e, ei, bb->preds)
5382 : {
5383 46345311 : if ((e->flags & EDGE_EXECUTABLE)
5384 : /* We can handle backedges in natural loops correctly but
5385 : for irreducible regions we have to take all backedges
5386 : conservatively when we did not visit the source yet. */
5387 46345311 : || (bb_to_rpo[e->src->index] > i
5388 669796 : && !dominated_by_p (CDI_DOMINATORS, e->src, e->dest)))
5389 : {
5390 : any = true;
5391 : break;
5392 : }
5393 : }
5394 45214693 : if (!any)
5395 14257 : continue;
5396 :
5397 : /* Remove conditions that go directly to unreachable when this is the last forwprop. */
5398 45201073 : if (last_p
5399 9844883 : && !(flag_sanitize & SANITIZE_UNREACHABLE))
5400 : {
5401 9839908 : gimple_stmt_iterator gsi;
5402 9839908 : gsi = gsi_start_nondebug_after_labels_bb (bb);
5403 9840545 : if (!gsi_end_p (gsi)
5404 8999453 : && gimple_call_builtin_p (*gsi, BUILT_IN_UNREACHABLE)
5405 9841039 : && optimize_unreachable (bb))
5406 : {
5407 637 : cfg_changed = true;
5408 637 : continue;
5409 : }
5410 : }
5411 :
5412 : /* Record degenerate PHIs in the lattice. */
5413 61108919 : for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
5414 15908483 : gsi_next (&si))
5415 : {
5416 15908483 : gphi *phi = si.phi ();
5417 15908483 : tree res = gimple_phi_result (phi);
5418 31816966 : if (virtual_operand_p (res))
5419 7290070 : continue;
5420 :
5421 8618413 : tree first = NULL_TREE;
5422 8618413 : bool all_same = true;
5423 8618413 : edge_iterator ei;
5424 8618413 : edge e;
5425 17730223 : FOR_EACH_EDGE (e, ei, bb->preds)
5426 : {
5427 : /* Ignore not executable forward edges. */
5428 17518830 : if (!(e->flags & EDGE_EXECUTABLE))
5429 : {
5430 4052493 : if (bb_to_rpo[e->src->index] < i)
5431 6947 : continue;
5432 : /* Avoid equivalences from backedges - while we might
5433 : be able to make irreducible regions reducible and
5434 : thus turning a back into a forward edge we do not
5435 : want to deal with the intermediate SSA issues that
5436 : exposes. */
5437 : all_same = false;
5438 : }
5439 17511883 : tree use = PHI_ARG_DEF_FROM_EDGE (phi, e);
5440 17511883 : if (use == res)
5441 : /* The PHI result can also appear on a backedge, if so
5442 : we can ignore this case for the purpose of determining
5443 : the singular value. */
5444 : ;
5445 17499081 : else if (! first)
5446 : first = use;
5447 8880668 : else if (! operand_equal_p (first, use, 0))
5448 : {
5449 : all_same = false;
5450 : break;
5451 : }
5452 : }
5453 8618413 : if (all_same)
5454 : {
5455 206815 : if (may_propagate_copy (res, first))
5456 206260 : to_remove_defs.safe_push (SSA_NAME_VERSION (res));
5457 206815 : fwprop_set_lattice_val (res, first);
5458 : }
5459 : }
5460 :
5461 : /* Apply forward propagation to all stmts in the basic-block.
5462 : Note we update GSI within the loop as necessary. */
5463 45200436 : unsigned int uid = 1;
5464 438646133 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); )
5465 : {
5466 348245261 : gimple *stmt = gsi_stmt (gsi);
5467 348245261 : tree lhs, rhs;
5468 348245261 : enum tree_code code;
5469 :
5470 348245261 : gimple_set_uid (stmt, uid++);
5471 :
5472 348245261 : if (!is_gimple_assign (stmt))
5473 : {
5474 241616616 : process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
5475 241616616 : gsi_next (&gsi);
5476 241616616 : continue;
5477 : }
5478 :
5479 106628645 : lhs = gimple_assign_lhs (stmt);
5480 106628645 : rhs = gimple_assign_rhs1 (stmt);
5481 106628645 : code = gimple_assign_rhs_code (stmt);
5482 :
5483 145763364 : if (TREE_CODE (lhs) != SSA_NAME
5484 106628645 : || has_zero_uses (lhs))
5485 : {
5486 39134719 : process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
5487 39134719 : gsi_next (&gsi);
5488 39134719 : continue;
5489 : }
5490 :
5491 : /* If this statement sets an SSA_NAME to an address,
5492 : try to propagate the address into the uses of the SSA_NAME. */
5493 67493926 : if ((code == ADDR_EXPR
5494 : /* Handle pointer conversions on invariant addresses
5495 : as well, as this is valid gimple. */
5496 65208123 : || (CONVERT_EXPR_CODE_P (code)
5497 9000615 : && TREE_CODE (rhs) == ADDR_EXPR
5498 357137 : && POINTER_TYPE_P (TREE_TYPE (lhs))))
5499 67494150 : && TREE_CODE (TREE_OPERAND (rhs, 0)) != TARGET_MEM_REF)
5500 : {
5501 2285438 : tree base = get_base_address (TREE_OPERAND (rhs, 0));
5502 2285438 : if ((!base
5503 2285438 : || !DECL_P (base)
5504 131995 : || decl_address_invariant_p (base))
5505 2285438 : && !stmt_references_abnormal_ssa_name (stmt)
5506 4570860 : && forward_propagate_addr_expr (lhs, rhs, true))
5507 : {
5508 470006 : fwprop_invalidate_lattice (gimple_get_lhs (stmt));
5509 470006 : release_defs (stmt);
5510 470006 : gsi_remove (&gsi, true);
5511 : }
5512 : else
5513 1815432 : gsi_next (&gsi);
5514 : }
5515 65208488 : else if (code == POINTER_PLUS_EXPR)
5516 : {
5517 3634572 : tree off = gimple_assign_rhs2 (stmt);
5518 3634572 : if (TREE_CODE (off) == INTEGER_CST
5519 1117432 : && can_propagate_from (stmt)
5520 1117079 : && !simple_iv_increment_p (stmt)
5521 : /* ??? Better adjust the interface to that function
5522 : instead of building new trees here. */
5523 4460168 : && forward_propagate_addr_expr
5524 2476788 : (lhs,
5525 : build1_loc (gimple_location (stmt),
5526 825596 : ADDR_EXPR, TREE_TYPE (rhs),
5527 825596 : fold_build2 (MEM_REF,
5528 : TREE_TYPE (TREE_TYPE (rhs)),
5529 : rhs,
5530 : fold_convert (ptr_type_node,
5531 : off))), true))
5532 : {
5533 313824 : fwprop_invalidate_lattice (gimple_get_lhs (stmt));
5534 313824 : release_defs (stmt);
5535 313824 : gsi_remove (&gsi, true);
5536 : }
5537 3320748 : else if (is_gimple_min_invariant (rhs))
5538 : {
5539 : /* Make sure to fold &a[0] + off_1 here. */
5540 412222 : fold_stmt_inplace (&gsi);
5541 412222 : update_stmt (stmt);
5542 412222 : if (gimple_assign_rhs_code (stmt) == POINTER_PLUS_EXPR)
5543 412204 : gsi_next (&gsi);
5544 : }
5545 : else
5546 2908526 : gsi_next (&gsi);
5547 : }
5548 61573916 : else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE
5549 214046 : && gimple_assign_load_p (stmt)
5550 134974 : && !gimple_has_volatile_ops (stmt)
5551 40899 : && TREE_CODE (rhs) != TARGET_MEM_REF
5552 40870 : && TREE_CODE (rhs) != BIT_FIELD_REF
5553 61614782 : && !stmt_can_throw_internal (fun, stmt))
5554 : {
5555 : /* Rewrite loads used only in real/imagpart extractions to
5556 : component-wise loads. */
5557 40741 : use_operand_p use_p;
5558 40741 : imm_use_iterator iter;
5559 40741 : tree vuse = gimple_vuse (stmt);
5560 40741 : bool rewrite = true;
5561 86665 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
5562 : {
5563 43798 : gimple *use_stmt = USE_STMT (use_p);
5564 43798 : if (is_gimple_debug (use_stmt))
5565 1011 : continue;
5566 42787 : if (!is_gimple_assign (use_stmt)
5567 28092 : || (gimple_assign_rhs_code (use_stmt) != REALPART_EXPR
5568 25991 : && gimple_assign_rhs_code (use_stmt) != IMAGPART_EXPR)
5569 46959 : || TREE_OPERAND (gimple_assign_rhs1 (use_stmt), 0) != lhs)
5570 : {
5571 : rewrite = false;
5572 : break;
5573 : }
5574 40741 : }
5575 40741 : if (rewrite)
5576 : {
5577 2126 : gimple *use_stmt;
5578 8889 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
5579 : {
5580 4637 : if (is_gimple_debug (use_stmt))
5581 : {
5582 498 : if (gimple_debug_bind_p (use_stmt))
5583 : {
5584 498 : gimple_debug_bind_reset_value (use_stmt);
5585 498 : update_stmt (use_stmt);
5586 : }
5587 498 : continue;
5588 : }
5589 :
5590 8278 : tree new_rhs = build1 (gimple_assign_rhs_code (use_stmt),
5591 4139 : TREE_TYPE (TREE_TYPE (rhs)),
5592 : unshare_expr (rhs));
5593 4139 : gimple *new_stmt
5594 4139 : = gimple_build_assign (gimple_assign_lhs (use_stmt),
5595 : new_rhs);
5596 :
5597 4139 : location_t loc = gimple_location (use_stmt);
5598 4139 : gimple_set_location (new_stmt, loc);
5599 4139 : gimple_set_vuse (new_stmt, vuse);
5600 4139 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5601 4139 : unlink_stmt_vdef (use_stmt);
5602 4139 : gsi_remove (&gsi2, true);
5603 :
5604 4139 : gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
5605 2126 : }
5606 :
5607 2126 : release_defs (stmt);
5608 2126 : gsi_remove (&gsi, true);
5609 : }
5610 : else
5611 38615 : gsi_next (&gsi);
5612 : }
5613 61533175 : else if (TREE_CODE (TREE_TYPE (lhs)) == VECTOR_TYPE
5614 1812548 : && (TYPE_MODE (TREE_TYPE (lhs)) == BLKmode
5615 : /* After vector lowering rewrite all loads, but
5616 : initially do not since this conflicts with
5617 : vector CONSTRUCTOR to shuffle optimization. */
5618 1787880 : || (fun->curr_properties & PROP_gimple_lvec))
5619 932852 : && gimple_assign_load_p (stmt)
5620 309704 : && !gimple_has_volatile_ops (stmt)
5621 295827 : && !stmt_can_throw_internal (fun, stmt)
5622 61829002 : && (!VAR_P (rhs) || !DECL_HARD_REGISTER (rhs)))
5623 295325 : optimize_vector_load (&gsi);
5624 :
5625 61237850 : else if (code == COMPLEX_EXPR)
5626 : {
5627 : /* Rewrite stores of a single-use complex build expression
5628 : to component-wise stores. */
5629 37863 : use_operand_p use_p;
5630 37863 : gimple *use_stmt, *def1, *def2;
5631 37863 : tree rhs2;
5632 37863 : if (single_imm_use (lhs, &use_p, &use_stmt)
5633 35678 : && gimple_store_p (use_stmt)
5634 42088 : && !gimple_has_volatile_ops (use_stmt)
5635 3128 : && is_gimple_assign (use_stmt)
5636 3124 : && (TREE_CODE (TREE_TYPE (gimple_assign_lhs (use_stmt)))
5637 : == COMPLEX_TYPE)
5638 40982 : && (TREE_CODE (gimple_assign_lhs (use_stmt))
5639 : != TARGET_MEM_REF))
5640 : {
5641 3115 : tree use_lhs = gimple_assign_lhs (use_stmt);
5642 3115 : if (auto_var_p (use_lhs))
5643 601 : DECL_NOT_GIMPLE_REG_P (use_lhs) = 1;
5644 6230 : tree new_lhs = build1 (REALPART_EXPR,
5645 3115 : TREE_TYPE (TREE_TYPE (use_lhs)),
5646 : unshare_expr (use_lhs));
5647 3115 : gimple *new_stmt = gimple_build_assign (new_lhs, rhs);
5648 3115 : location_t loc = gimple_location (use_stmt);
5649 3115 : gimple_set_location (new_stmt, loc);
5650 6230 : gimple_set_vuse (new_stmt, gimple_vuse (use_stmt));
5651 3115 : gimple_set_vdef (new_stmt, make_ssa_name (gimple_vop (fun)));
5652 6230 : SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
5653 6230 : gimple_set_vuse (use_stmt, gimple_vdef (new_stmt));
5654 3115 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5655 3115 : gsi_insert_before (&gsi2, new_stmt, GSI_SAME_STMT);
5656 :
5657 6230 : new_lhs = build1 (IMAGPART_EXPR,
5658 3115 : TREE_TYPE (TREE_TYPE (use_lhs)),
5659 : unshare_expr (use_lhs));
5660 3115 : gimple_assign_set_lhs (use_stmt, new_lhs);
5661 3115 : gimple_assign_set_rhs1 (use_stmt, gimple_assign_rhs2 (stmt));
5662 3115 : update_stmt (use_stmt);
5663 :
5664 3115 : release_defs (stmt);
5665 3115 : gsi_remove (&gsi, true);
5666 : }
5667 : /* Rewrite a component-wise load of a complex to a complex
5668 : load if the components are not used separately. */
5669 34748 : else if (TREE_CODE (rhs) == SSA_NAME
5670 34307 : && has_single_use (rhs)
5671 30771 : && ((rhs2 = gimple_assign_rhs2 (stmt)), true)
5672 30771 : && TREE_CODE (rhs2) == SSA_NAME
5673 28960 : && has_single_use (rhs2)
5674 28540 : && (def1 = SSA_NAME_DEF_STMT (rhs),
5675 28540 : gimple_assign_load_p (def1))
5676 1088 : && (def2 = SSA_NAME_DEF_STMT (rhs2),
5677 1088 : gimple_assign_load_p (def2))
5678 1588 : && (gimple_vuse (def1) == gimple_vuse (def2))
5679 791 : && !gimple_has_volatile_ops (def1)
5680 791 : && !gimple_has_volatile_ops (def2)
5681 791 : && !stmt_can_throw_internal (fun, def1)
5682 791 : && !stmt_can_throw_internal (fun, def2)
5683 791 : && gimple_assign_rhs_code (def1) == REALPART_EXPR
5684 545 : && gimple_assign_rhs_code (def2) == IMAGPART_EXPR
5685 35293 : && operand_equal_p (TREE_OPERAND (gimple_assign_rhs1
5686 : (def1), 0),
5687 545 : TREE_OPERAND (gimple_assign_rhs1
5688 : (def2), 0)))
5689 : {
5690 545 : tree cl = TREE_OPERAND (gimple_assign_rhs1 (def1), 0);
5691 545 : gimple_assign_set_rhs_from_tree (&gsi, unshare_expr (cl));
5692 545 : gcc_assert (gsi_stmt (gsi) == stmt);
5693 1090 : gimple_set_vuse (stmt, gimple_vuse (def1));
5694 545 : gimple_set_modified (stmt, true);
5695 545 : gimple_stmt_iterator gsi2 = gsi_for_stmt (def1);
5696 545 : gsi_remove (&gsi, false);
5697 545 : gsi_insert_after (&gsi2, stmt, GSI_SAME_STMT);
5698 : }
5699 : else
5700 34203 : gsi_next (&gsi);
5701 : }
5702 61199987 : else if (code == CONSTRUCTOR
5703 168004 : && VECTOR_TYPE_P (TREE_TYPE (rhs))
5704 168004 : && TYPE_MODE (TREE_TYPE (rhs)) == BLKmode
5705 4541 : && CONSTRUCTOR_NELTS (rhs) > 0
5706 61204528 : && (!VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
5707 2095 : || (TYPE_MODE (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
5708 : != BLKmode)))
5709 : {
5710 : /* Rewrite stores of a single-use vector constructors
5711 : to component-wise stores if the mode isn't supported. */
5712 4184 : use_operand_p use_p;
5713 4184 : gimple *use_stmt;
5714 4184 : if (single_imm_use (lhs, &use_p, &use_stmt)
5715 3718 : && gimple_store_p (use_stmt)
5716 3076 : && !gimple_has_volatile_ops (use_stmt)
5717 1532 : && !stmt_can_throw_internal (fun, use_stmt)
5718 5709 : && is_gimple_assign (use_stmt))
5719 : {
5720 1525 : tree elt_t = TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value);
5721 1525 : unsigned HOST_WIDE_INT elt_w
5722 1525 : = tree_to_uhwi (TYPE_SIZE (elt_t));
5723 1525 : unsigned HOST_WIDE_INT n
5724 1525 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs)));
5725 1525 : tree use_lhs = gimple_assign_lhs (use_stmt);
5726 1525 : if (auto_var_p (use_lhs))
5727 575 : DECL_NOT_GIMPLE_REG_P (use_lhs) = 1;
5728 950 : else if (TREE_CODE (use_lhs) == TARGET_MEM_REF)
5729 : {
5730 1 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5731 1 : use_lhs = prepare_target_mem_ref_lvalue (use_lhs, &gsi2);
5732 : }
5733 33572 : for (unsigned HOST_WIDE_INT bi = 0; bi < n; bi += elt_w)
5734 : {
5735 32047 : unsigned HOST_WIDE_INT ci = bi / elt_w;
5736 32047 : tree new_rhs;
5737 32047 : if (ci < CONSTRUCTOR_NELTS (rhs))
5738 31429 : new_rhs = CONSTRUCTOR_ELT (rhs, ci)->value;
5739 : else
5740 618 : new_rhs = build_zero_cst (elt_t);
5741 32047 : tree new_lhs = build3 (BIT_FIELD_REF,
5742 : elt_t,
5743 : unshare_expr (use_lhs),
5744 32047 : bitsize_int (elt_w),
5745 32047 : bitsize_int (bi));
5746 32047 : gimple *new_stmt = gimple_build_assign (new_lhs, new_rhs);
5747 32047 : location_t loc = gimple_location (use_stmt);
5748 32047 : gimple_set_location (new_stmt, loc);
5749 64094 : gimple_set_vuse (new_stmt, gimple_vuse (use_stmt));
5750 32047 : gimple_set_vdef (new_stmt,
5751 : make_ssa_name (gimple_vop (fun)));
5752 64094 : SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
5753 64094 : gimple_set_vuse (use_stmt, gimple_vdef (new_stmt));
5754 32047 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5755 32047 : gsi_insert_before (&gsi2, new_stmt, GSI_SAME_STMT);
5756 : }
5757 1525 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5758 1525 : unlink_stmt_vdef (use_stmt);
5759 1525 : release_defs (use_stmt);
5760 1525 : gsi_remove (&gsi2, true);
5761 1525 : release_defs (stmt);
5762 1525 : gsi_remove (&gsi, true);
5763 : }
5764 : else
5765 2659 : gsi_next (&gsi);
5766 : }
5767 61195803 : else if (code == VEC_PERM_EXPR)
5768 : {
5769 : /* Find vectorized sequences, where we can reduce the lane
5770 : utilization. The narrowing will be donw later and only
5771 : if we find a pair of sequences that can be blended. */
5772 188618 : gassign *assign = dyn_cast <gassign *> (stmt);
5773 188618 : vec_perm_simplify_seq seq;
5774 188618 : if (recognise_vec_perm_simplify_seq (assign, &seq))
5775 101 : append_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list,
5776 : seq);
5777 :
5778 188618 : gsi_next (&gsi);
5779 : }
5780 : else
5781 61007185 : gsi_next (&gsi);
5782 : }
5783 :
5784 45200436 : process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
5785 :
5786 : /* Combine stmts with the stmts defining their operands.
5787 : Note we update GSI within the loop as necessary. */
5788 438292415 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
5789 : {
5790 347891543 : gimple *stmt = gsi_stmt (gsi);
5791 :
5792 : /* Mark stmt as potentially needing revisiting. */
5793 347891543 : gimple_set_plf (stmt, GF_PLF_1, false);
5794 :
5795 347891543 : bool can_make_abnormal_goto = (is_gimple_call (stmt)
5796 347891543 : && stmt_can_make_abnormal_goto (stmt));
5797 :
5798 : /* Substitute from our lattice. We need to do so only once. */
5799 347891543 : bool substituted_p = false;
5800 347891543 : use_operand_p usep;
5801 347891543 : ssa_op_iter iter;
5802 511764121 : FOR_EACH_SSA_USE_OPERAND (usep, stmt, iter, SSA_OP_USE)
5803 : {
5804 163872578 : tree use = USE_FROM_PTR (usep);
5805 163872578 : tree val = fwprop_ssa_val (use);
5806 163872578 : if (val && val != use)
5807 : {
5808 1840977 : if (!is_gimple_debug (stmt))
5809 1535857 : bitmap_set_bit (simple_dce_worklist, SSA_NAME_VERSION (use));
5810 1840977 : if (may_propagate_copy (use, val))
5811 : {
5812 1837749 : propagate_value (usep, val);
5813 1837749 : substituted_p = true;
5814 : }
5815 : }
5816 : }
5817 347891543 : if (substituted_p)
5818 1783458 : update_stmt (stmt);
5819 1783458 : if (substituted_p
5820 1783458 : && is_gimple_assign (stmt)
5821 1074823 : && gimple_assign_rhs_code (stmt) == ADDR_EXPR)
5822 20282 : recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
5823 347891543 : if (substituted_p
5824 347891543 : && can_make_abnormal_goto
5825 347891543 : && !stmt_can_make_abnormal_goto (stmt))
5826 3 : bitmap_set_bit (need_ab_cleanup, bb->index);
5827 :
5828 350715627 : bool changed;
5829 701431254 : do
5830 : {
5831 350715627 : gimple *orig_stmt = stmt = gsi_stmt (gsi);
5832 350715627 : bool was_call = is_gimple_call (stmt);
5833 350715627 : bool was_noreturn = (was_call
5834 350715627 : && gimple_call_noreturn_p (stmt));
5835 350715627 : changed = false;
5836 :
5837 350715627 : auto_vec<tree, 8> uses;
5838 517594773 : FOR_EACH_SSA_USE_OPERAND (usep, stmt, iter, SSA_OP_USE)
5839 166879146 : if (uses.space (1))
5840 166488273 : uses.quick_push (USE_FROM_PTR (usep));
5841 :
5842 350715627 : if (fold_stmt (&gsi, fwprop_ssa_val, simple_dce_worklist))
5843 : {
5844 2503859 : changed = true;
5845 : /* There is no updating of the address
5846 : taken after the last forwprop so update
5847 : the addresses when a folding happened to a call.
5848 : The va_* builtins can remove taking of the address so
5849 : can the sincos->cexpi transformation. See PR 39643 and PR 20983. */
5850 2503859 : if (was_call && last_p)
5851 2503859 : todoflags |= TODO_update_address_taken;
5852 2503859 : stmt = gsi_stmt (gsi);
5853 : /* Cleanup the CFG if we simplified a condition to
5854 : true or false. */
5855 2503859 : if (gcond *cond = dyn_cast <gcond *> (stmt))
5856 997050 : if (gimple_cond_true_p (cond)
5857 997050 : || gimple_cond_false_p (cond))
5858 14234 : cfg_changed = true;
5859 : /* Queue old uses for simple DCE if not debug statement. */
5860 2503859 : if (!is_gimple_debug (stmt))
5861 10587547 : for (tree use : uses)
5862 3097114 : if (TREE_CODE (use) == SSA_NAME
5863 3097114 : && !SSA_NAME_IS_DEFAULT_DEF (use))
5864 2900402 : bitmap_set_bit (simple_dce_worklist,
5865 2900402 : SSA_NAME_VERSION (use));
5866 2503859 : update_stmt (stmt);
5867 : }
5868 :
5869 350715627 : switch (gimple_code (stmt))
5870 : {
5871 107620108 : case GIMPLE_ASSIGN:
5872 107620108 : {
5873 107620108 : tree rhs1 = gimple_assign_rhs1 (stmt);
5874 107620108 : enum tree_code code = gimple_assign_rhs_code (stmt);
5875 107620108 : if (gimple_clobber_p (stmt))
5876 7314041 : do_simple_agr_dse (as_a<gassign*>(stmt), full_walk);
5877 100306067 : else if (gimple_store_p (stmt))
5878 : {
5879 31198341 : optimize_aggr_zeroprop (stmt, full_walk);
5880 31198341 : if (gimple_assign_load_p (stmt))
5881 3882104 : optimize_agr_copyprop (stmt);
5882 : }
5883 69107726 : else if (TREE_CODE_CLASS (code) == tcc_comparison)
5884 2654692 : changed |= forward_propagate_into_comparison (&gsi);
5885 66453034 : else if ((code == PLUS_EXPR
5886 66453034 : || code == BIT_IOR_EXPR
5887 56325111 : || code == BIT_XOR_EXPR)
5888 66591993 : && simplify_rotate (&gsi))
5889 : changed = true;
5890 66450353 : else if (code == VEC_PERM_EXPR)
5891 190834 : changed |= simplify_permutation (&gsi);
5892 66259519 : else if (code == CONSTRUCTOR
5893 66259519 : && TREE_CODE (TREE_TYPE (rhs1)) == VECTOR_TYPE)
5894 165777 : changed |= simplify_vector_constructor (&gsi);
5895 66093742 : else if (code == ARRAY_REF)
5896 1992988 : changed |= simplify_count_zeroes (&gsi);
5897 : break;
5898 : }
5899 :
5900 106576 : case GIMPLE_SWITCH:
5901 106576 : changed |= simplify_gimple_switch (as_a <gswitch *> (stmt),
5902 : edges_to_remove,
5903 : simple_dce_worklist);
5904 106576 : break;
5905 :
5906 19524402 : case GIMPLE_COND:
5907 19524402 : {
5908 19524402 : int did_something = forward_propagate_into_gimple_cond
5909 19524402 : (as_a <gcond *> (stmt));
5910 19524402 : if (did_something == 2)
5911 1655 : cfg_changed = true;
5912 19524402 : changed |= did_something != 0;
5913 19524402 : break;
5914 : }
5915 :
5916 23541840 : case GIMPLE_CALL:
5917 23541840 : {
5918 23541840 : tree callee = gimple_call_fndecl (stmt);
5919 23541840 : if (callee != NULL_TREE
5920 23541840 : && fndecl_built_in_p (callee, BUILT_IN_NORMAL))
5921 6246747 : changed |= simplify_builtin_call (&gsi, callee, full_walk);
5922 : break;
5923 : }
5924 :
5925 350712946 : default:;
5926 : }
5927 :
5928 350712946 : if (changed || substituted_p)
5929 : {
5930 4085167 : substituted_p = false;
5931 4085167 : stmt = gsi_stmt (gsi);
5932 4085167 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
5933 70 : bitmap_set_bit (to_purge, bb->index);
5934 4085167 : if (!was_noreturn
5935 4085167 : && is_gimple_call (stmt) && gimple_call_noreturn_p (stmt))
5936 12 : to_fixup.safe_push (stmt);
5937 : }
5938 4085167 : if (changed)
5939 : {
5940 : /* If the stmt changed then re-visit it and the statements
5941 : inserted before it. */
5942 8837584 : for (; !gsi_end_p (gsi); gsi_prev (&gsi))
5943 5609470 : if (gimple_plf (gsi_stmt (gsi), GF_PLF_1))
5944 : break;
5945 2824084 : if (gsi_end_p (gsi))
5946 442728 : gsi = gsi_start_bb (bb);
5947 : else
5948 2602720 : gsi_next (&gsi);
5949 : }
5950 350715627 : }
5951 : while (changed);
5952 :
5953 : /* Stmt no longer needs to be revisited. */
5954 347891543 : stmt = gsi_stmt (gsi);
5955 347891543 : gcc_checking_assert (!gimple_plf (stmt, GF_PLF_1));
5956 347891543 : gimple_set_plf (stmt, GF_PLF_1, true);
5957 :
5958 : /* Fill up the lattice. */
5959 347891543 : if (gimple_assign_single_p (stmt))
5960 : {
5961 71200928 : tree lhs = gimple_assign_lhs (stmt);
5962 71200928 : tree rhs = gimple_assign_rhs1 (stmt);
5963 71200928 : if (TREE_CODE (lhs) == SSA_NAME)
5964 : {
5965 32700899 : tree val = lhs;
5966 32700899 : if (TREE_CODE (rhs) == SSA_NAME)
5967 783610 : val = fwprop_ssa_val (rhs);
5968 31917289 : else if (is_gimple_min_invariant (rhs))
5969 426666 : val = rhs;
5970 : /* If we can propagate the lattice-value mark the
5971 : stmt for removal. */
5972 32700899 : if (val != lhs
5973 32700899 : && may_propagate_copy (lhs, val))
5974 1206846 : to_remove_defs.safe_push (SSA_NAME_VERSION (lhs));
5975 32700899 : fwprop_set_lattice_val (lhs, val);
5976 : }
5977 : }
5978 276690615 : else if (gimple_nop_p (stmt))
5979 89728 : to_remove.safe_push (stmt);
5980 : }
5981 :
5982 : /* Substitute in destination PHI arguments. */
5983 108778932 : FOR_EACH_EDGE (e, ei, bb->succs)
5984 63578496 : for (gphi_iterator gsi = gsi_start_phis (e->dest);
5985 105456762 : !gsi_end_p (gsi); gsi_next (&gsi))
5986 : {
5987 41878266 : gphi *phi = gsi.phi ();
5988 41878266 : use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
5989 41878266 : tree arg = USE_FROM_PTR (use_p);
5990 69115194 : if (TREE_CODE (arg) != SSA_NAME
5991 41878266 : || virtual_operand_p (arg))
5992 27236928 : continue;
5993 14641338 : tree val = fwprop_ssa_val (arg);
5994 14641338 : if (val != arg
5995 14641338 : && may_propagate_copy (arg, val, !(e->flags & EDGE_ABNORMAL)))
5996 242655 : propagate_value (use_p, val);
5997 : }
5998 :
5999 : /* Mark outgoing executable edges. */
6000 45200436 : if (edge e = find_taken_edge (bb, NULL))
6001 : {
6002 19200181 : e->flags |= EDGE_EXECUTABLE;
6003 45220652 : if (EDGE_COUNT (bb->succs) > 1)
6004 20216 : cfg_changed = true;
6005 : }
6006 : else
6007 : {
6008 70358353 : FOR_EACH_EDGE (e, ei, bb->succs)
6009 44358098 : e->flags |= EDGE_EXECUTABLE;
6010 : }
6011 : }
6012 5643906 : free (postorder);
6013 5643906 : free (bb_to_rpo);
6014 5643906 : lattice.release ();
6015 :
6016 : /* First remove chains of stmts where we check no uses remain. */
6017 5643906 : simple_dce_from_worklist (simple_dce_worklist, to_purge);
6018 :
6019 5985714 : auto remove = [](gimple *stmt)
6020 : {
6021 341808 : if (dump_file && (dump_flags & TDF_DETAILS))
6022 : {
6023 1 : fprintf (dump_file, "Removing dead stmt ");
6024 1 : print_gimple_stmt (dump_file, stmt, 0);
6025 1 : fprintf (dump_file, "\n");
6026 : }
6027 341808 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6028 341808 : if (gimple_code (stmt) == GIMPLE_PHI)
6029 91438 : remove_phi_node (&gsi, true);
6030 : else
6031 : {
6032 250370 : unlink_stmt_vdef (stmt);
6033 250370 : gsi_remove (&gsi, true);
6034 250370 : release_defs (stmt);
6035 : }
6036 341808 : };
6037 :
6038 : /* Then remove stmts we know we can remove even though we did not
6039 : substitute in dead code regions, so uses can remain. Do so in reverse
6040 : order to make debug stmt creation possible. */
6041 12700918 : while (!to_remove_defs.is_empty())
6042 : {
6043 1413106 : tree def = ssa_name (to_remove_defs.pop ());
6044 : /* For example remove_prop_source_from_use can remove stmts queued
6045 : for removal. Deal with this gracefully. */
6046 1413106 : if (!def)
6047 1161026 : continue;
6048 252080 : gimple *stmt = SSA_NAME_DEF_STMT (def);
6049 252080 : remove (stmt);
6050 : }
6051 :
6052 : /* Wipe other queued stmts that do not have SSA defs. */
6053 5733634 : while (!to_remove.is_empty())
6054 : {
6055 89728 : gimple *stmt = to_remove.pop ();
6056 89728 : remove (stmt);
6057 : }
6058 :
6059 : /* Fixup stmts that became noreturn calls. This may require splitting
6060 : blocks and thus isn't possible during the walk. Do this
6061 : in reverse order so we don't inadvertently remove a stmt we want to
6062 : fixup by visiting a dominating now noreturn call first. */
6063 5643918 : while (!to_fixup.is_empty ())
6064 : {
6065 12 : gimple *stmt = to_fixup.pop ();
6066 12 : if (dump_file && dump_flags & TDF_DETAILS)
6067 : {
6068 0 : fprintf (dump_file, "Fixing up noreturn call ");
6069 0 : print_gimple_stmt (dump_file, stmt, 0);
6070 0 : fprintf (dump_file, "\n");
6071 : }
6072 12 : cfg_changed |= fixup_noreturn_call (stmt);
6073 : }
6074 :
6075 5643906 : cfg_changed |= gimple_purge_all_dead_eh_edges (to_purge);
6076 5643906 : cfg_changed |= gimple_purge_all_dead_abnormal_call_edges (need_ab_cleanup);
6077 5643906 : BITMAP_FREE (to_purge);
6078 :
6079 : /* Remove edges queued from switch stmt simplification. */
6080 16931718 : for (auto ep : edges_to_remove)
6081 : {
6082 0 : basic_block src = BASIC_BLOCK_FOR_FN (fun, ep.first);
6083 0 : basic_block dest = BASIC_BLOCK_FOR_FN (fun, ep.second);
6084 0 : edge e;
6085 0 : if (src && dest && (e = find_edge (src, dest)))
6086 : {
6087 0 : free_dominance_info (CDI_DOMINATORS);
6088 0 : remove_edge (e);
6089 0 : cfg_changed = true;
6090 : }
6091 : }
6092 :
6093 11286269 : if (get_range_query (fun) != get_global_range_query ())
6094 1543 : disable_ranger (fun);
6095 :
6096 5643906 : if (cfg_changed)
6097 9488 : todoflags |= TODO_cleanup_cfg;
6098 :
6099 5643906 : return todoflags;
6100 5643906 : }
6101 :
6102 : } // anon namespace
6103 :
6104 : gimple_opt_pass *
6105 293828 : make_pass_forwprop (gcc::context *ctxt)
6106 : {
6107 293828 : return new pass_forwprop (ctxt);
6108 : }
|
