Line data Source code
1 : /* Forward propagation of expressions for single use variables.
2 : Copyright (C) 2004-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify
7 : it under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful,
12 : but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : GNU General Public License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "rtl.h"
25 : #include "tree.h"
26 : #include "gimple.h"
27 : #include "cfghooks.h"
28 : #include "tree-pass.h"
29 : #include "ssa.h"
30 : #include "expmed.h"
31 : #include "optabs-query.h"
32 : #include "gimple-pretty-print.h"
33 : #include "fold-const.h"
34 : #include "stor-layout.h"
35 : #include "gimple-iterator.h"
36 : #include "gimple-fold.h"
37 : #include "tree-eh.h"
38 : #include "gimplify.h"
39 : #include "gimplify-me.h"
40 : #include "tree-cfg.h"
41 : #include "expr.h"
42 : #include "tree-dfa.h"
43 : #include "tree-ssa-propagate.h"
44 : #include "tree-ssa-dom.h"
45 : #include "tree-ssa-strlen.h"
46 : #include "builtins.h"
47 : #include "tree-cfgcleanup.h"
48 : #include "cfganal.h"
49 : #include "optabs-tree.h"
50 : #include "insn-config.h"
51 : #include "recog.h"
52 : #include "cfgloop.h"
53 : #include "tree-vectorizer.h"
54 : #include "tree-vector-builder.h"
55 : #include "vec-perm-indices.h"
56 : #include "internal-fn.h"
57 : #include "cgraph.h"
58 : #include "tree-ssa.h"
59 : #include "gimple-range.h"
60 : #include "tree-ssa-dce.h"
61 :
62 : /* This pass propagates the RHS of assignment statements into use
63 : sites of the LHS of the assignment. It's basically a specialized
64 : form of tree combination. It is hoped all of this can disappear
65 : when we have a generalized tree combiner.
66 :
67 : One class of common cases we handle is forward propagating a single use
68 : variable into a COND_EXPR.
69 :
70 : bb0:
71 : x = a COND b;
72 : if (x) goto ... else goto ...
73 :
74 : Will be transformed into:
75 :
76 : bb0:
77 : if (a COND b) goto ... else goto ...
78 :
79 : Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
80 :
81 : Or (assuming c1 and c2 are constants):
82 :
83 : bb0:
84 : x = a + c1;
85 : if (x EQ/NEQ c2) goto ... else goto ...
86 :
87 : Will be transformed into:
88 :
89 : bb0:
90 : if (a EQ/NEQ (c2 - c1)) goto ... else goto ...
91 :
92 : Similarly for x = a - c1.
93 :
94 : Or
95 :
96 : bb0:
97 : x = !a
98 : if (x) goto ... else goto ...
99 :
100 : Will be transformed into:
101 :
102 : bb0:
103 : if (a == 0) goto ... else goto ...
104 :
105 : Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
106 : For these cases, we propagate A into all, possibly more than one,
107 : COND_EXPRs that use X.
108 :
109 : Or
110 :
111 : bb0:
112 : x = (typecast) a
113 : if (x) goto ... else goto ...
114 :
115 : Will be transformed into:
116 :
117 : bb0:
118 : if (a != 0) goto ... else goto ...
119 :
120 : (Assuming a is an integral type and x is a boolean or x is an
121 : integral and a is a boolean.)
122 :
123 : Similarly for the tests (x == 0), (x != 0), (x == 1) and (x != 1).
124 : For these cases, we propagate A into all, possibly more than one,
125 : COND_EXPRs that use X.
126 :
127 : In addition to eliminating the variable and the statement which assigns
128 : a value to the variable, we may be able to later thread the jump without
129 : adding insane complexity in the dominator optimizer.
130 :
131 : Also note these transformations can cascade. We handle this by having
132 : a worklist of COND_EXPR statements to examine. As we make a change to
133 : a statement, we put it back on the worklist to examine on the next
134 : iteration of the main loop.
135 :
136 : A second class of propagation opportunities arises for ADDR_EXPR
137 : nodes.
138 :
139 : ptr = &x->y->z;
140 : res = *ptr;
141 :
142 : Will get turned into
143 :
144 : res = x->y->z;
145 :
146 : Or
147 : ptr = (type1*)&type2var;
148 : res = *ptr
149 :
150 : Will get turned into (if type1 and type2 are the same size
151 : and neither have volatile on them):
152 : res = VIEW_CONVERT_EXPR<type1>(type2var)
153 :
154 : Or
155 :
156 : ptr = &x[0];
157 : ptr2 = ptr + <constant>;
158 :
159 : Will get turned into
160 :
161 : ptr2 = &x[constant/elementsize];
162 :
163 : Or
164 :
165 : ptr = &x[0];
166 : offset = index * element_size;
167 : offset_p = (pointer) offset;
168 : ptr2 = ptr + offset_p
169 :
170 : Will get turned into:
171 :
172 : ptr2 = &x[index];
173 :
174 : Or
175 : ssa = (int) decl
176 : res = ssa & 1
177 :
178 : Provided that decl has known alignment >= 2, will get turned into
179 :
180 : res = 0
181 :
182 : We also propagate casts into SWITCH_EXPR and COND_EXPR conditions to
183 : allow us to remove the cast and {NOT_EXPR,NEG_EXPR} into a subsequent
184 : {NOT_EXPR,NEG_EXPR}.
185 :
186 : This will (of course) be extended as other needs arise. */
187 :
188 : /* Data structure that contains simplifiable vectorized permute sequences.
189 : See recognise_vec_perm_simplify_seq () for a description of the sequence. */
190 :
191 : struct _vec_perm_simplify_seq
192 : {
193 : /* Defining stmts of vectors in the sequence. */
194 : gassign *v_1_stmt;
195 : gassign *v_2_stmt;
196 : gassign *v_x_stmt;
197 : gassign *v_y_stmt;
198 : /* Final permute statement. */
199 : gassign *stmt;
200 : /* New selector indices for stmt. */
201 : tree new_sel;
202 : /* Elements of each vector and selector. */
203 : unsigned int nelts;
204 : };
205 : typedef struct _vec_perm_simplify_seq *vec_perm_simplify_seq;
206 :
207 : static bool forward_propagate_addr_expr (tree, tree, bool);
208 :
209 : /* Set to true if we delete dead edges during the optimization. */
210 : static bool cfg_changed;
211 :
212 : static tree rhs_to_tree (tree type, gimple *stmt);
213 :
214 : static bitmap to_purge;
215 :
216 : /* Const-and-copy lattice. */
217 : static vec<tree> lattice;
218 :
219 : /* Set the lattice entry for NAME to VAL. */
220 : static void
221 32202863 : fwprop_set_lattice_val (tree name, tree val)
222 : {
223 32202863 : if (TREE_CODE (name) == SSA_NAME)
224 : {
225 32202863 : if (SSA_NAME_VERSION (name) >= lattice.length ())
226 : {
227 32169 : lattice.reserve (num_ssa_names - lattice.length ());
228 21446 : lattice.quick_grow_cleared (num_ssa_names);
229 : }
230 32202863 : lattice[SSA_NAME_VERSION (name)] = val;
231 : /* As this now constitutes a copy duplicate points-to
232 : and range info appropriately. */
233 32202863 : if (TREE_CODE (val) == SSA_NAME)
234 31760542 : maybe_duplicate_ssa_info_at_copy (name, val);
235 : }
236 32202863 : }
237 :
238 : /* Invalidate the lattice entry for NAME, done when releasing SSA names. */
239 : static void
240 907688 : fwprop_invalidate_lattice (tree name)
241 : {
242 907688 : if (name
243 905346 : && TREE_CODE (name) == SSA_NAME
244 1812907 : && SSA_NAME_VERSION (name) < lattice.length ())
245 905186 : lattice[SSA_NAME_VERSION (name)] = NULL_TREE;
246 907688 : }
247 :
248 : /* Get the statement we can propagate from into NAME skipping
249 : trivial copies. Returns the statement which defines the
250 : propagation source or NULL_TREE if there is no such one.
251 : If SINGLE_USE_ONLY is set considers only sources which have
252 : a single use chain up to NAME. If SINGLE_USE_P is non-null,
253 : it is set to whether the chain to NAME is a single use chain
254 : or not. SINGLE_USE_P is not written to if SINGLE_USE_ONLY is set. */
255 :
256 : static gimple *
257 27725714 : get_prop_source_stmt (tree name, bool single_use_only, bool *single_use_p)
258 : {
259 27725714 : bool single_use = true;
260 :
261 27726702 : do {
262 27726208 : gimple *def_stmt = SSA_NAME_DEF_STMT (name);
263 :
264 27726208 : if (!has_single_use (name))
265 : {
266 15102627 : single_use = false;
267 15102627 : if (single_use_only)
268 : return NULL;
269 : }
270 :
271 : /* If name is defined by a PHI node or is the default def, bail out. */
272 27724772 : if (!is_gimple_assign (def_stmt))
273 : return NULL;
274 :
275 : /* If def_stmt is a simple copy, continue looking. */
276 19512755 : if (gimple_assign_rhs_code (def_stmt) == SSA_NAME)
277 494 : name = gimple_assign_rhs1 (def_stmt);
278 : else
279 : {
280 19512261 : if (!single_use_only && single_use_p)
281 19208306 : *single_use_p = single_use;
282 :
283 19512261 : return def_stmt;
284 : }
285 494 : } while (1);
286 : }
287 :
288 : /* Checks if the destination ssa name in DEF_STMT can be used as
289 : propagation source. Returns true if so, otherwise false. */
290 :
291 : static bool
292 27476829 : can_propagate_from (gimple *def_stmt)
293 : {
294 27476829 : gcc_assert (is_gimple_assign (def_stmt));
295 :
296 : /* If the rhs has side-effects we cannot propagate from it. */
297 27476829 : if (gimple_has_volatile_ops (def_stmt))
298 : return false;
299 :
300 : /* If the rhs is a load we cannot propagate from it. */
301 26884734 : if (TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_reference
302 26884734 : || TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_declaration)
303 : return false;
304 :
305 : /* Constants can be always propagated. */
306 13312926 : if (gimple_assign_single_p (def_stmt)
307 13312926 : && is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
308 : return true;
309 :
310 : /* We cannot propagate ssa names that occur in abnormal phi nodes. */
311 13312926 : if (stmt_references_abnormal_ssa_name (def_stmt))
312 : return false;
313 :
314 : /* If the definition is a conversion of a pointer to a function type,
315 : then we cannot apply optimizations as some targets require
316 : function pointers to be canonicalized and in this case this
317 : optimization could eliminate a necessary canonicalization. */
318 13312235 : if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def_stmt)))
319 : {
320 3215565 : tree rhs = gimple_assign_rhs1 (def_stmt);
321 3215565 : if (FUNCTION_POINTER_TYPE_P (TREE_TYPE (rhs)))
322 : return false;
323 : }
324 :
325 : return true;
326 : }
327 :
328 : /* Remove a chain of dead statements starting at the definition of
329 : NAME. The chain is linked via the first operand of the defining statements.
330 : If NAME was replaced in its only use then this function can be used
331 : to clean up dead stmts. The function handles already released SSA
332 : names gracefully. */
333 :
334 : static void
335 236201 : remove_prop_source_from_use (tree name)
336 : {
337 296042 : gimple_stmt_iterator gsi;
338 296042 : gimple *stmt;
339 :
340 296042 : do {
341 296042 : basic_block bb;
342 :
343 296042 : if (SSA_NAME_IN_FREE_LIST (name)
344 295999 : || SSA_NAME_IS_DEFAULT_DEF (name)
345 590109 : || !has_zero_uses (name))
346 : break;
347 :
348 60298 : stmt = SSA_NAME_DEF_STMT (name);
349 60298 : if (gimple_code (stmt) == GIMPLE_PHI
350 60298 : || gimple_has_side_effects (stmt))
351 : break;
352 :
353 60298 : bb = gimple_bb (stmt);
354 60298 : gsi = gsi_for_stmt (stmt);
355 60298 : unlink_stmt_vdef (stmt);
356 60298 : if (gsi_remove (&gsi, true))
357 6 : bitmap_set_bit (to_purge, bb->index);
358 60298 : fwprop_invalidate_lattice (gimple_get_lhs (stmt));
359 60298 : release_defs (stmt);
360 :
361 60298 : name = is_gimple_assign (stmt) ? gimple_assign_rhs1 (stmt) : NULL_TREE;
362 60298 : } while (name && TREE_CODE (name) == SSA_NAME);
363 :
364 236201 : }
365 :
366 : /* Return the rhs of a gassign *STMT in a form of a single tree,
367 : converted to type TYPE.
368 :
369 : This should disappear, but is needed so we can combine expressions and use
370 : the fold() interfaces. Long term, we need to develop folding and combine
371 : routines that deal with gimple exclusively . */
372 :
373 : static tree
374 7249070 : rhs_to_tree (tree type, gimple *stmt)
375 : {
376 7249070 : location_t loc = gimple_location (stmt);
377 7249070 : enum tree_code code = gimple_assign_rhs_code (stmt);
378 7249070 : switch (get_gimple_rhs_class (code))
379 : {
380 12510 : case GIMPLE_TERNARY_RHS:
381 12510 : return fold_build3_loc (loc, code, type, gimple_assign_rhs1 (stmt),
382 : gimple_assign_rhs2 (stmt),
383 12510 : gimple_assign_rhs3 (stmt));
384 4955518 : case GIMPLE_BINARY_RHS:
385 4955518 : return fold_build2_loc (loc, code, type, gimple_assign_rhs1 (stmt),
386 4955518 : gimple_assign_rhs2 (stmt));
387 2012228 : case GIMPLE_UNARY_RHS:
388 2012228 : return build1 (code, type, gimple_assign_rhs1 (stmt));
389 268814 : case GIMPLE_SINGLE_RHS:
390 268814 : return gimple_assign_rhs1 (stmt);
391 0 : default:
392 0 : gcc_unreachable ();
393 : }
394 : }
395 :
396 : /* Combine OP0 CODE OP1 in the context of a COND_EXPR. Returns
397 : the folded result in a form suitable for COND_EXPR_COND or
398 : NULL_TREE, if there is no suitable simplified form. If
399 : INVARIANT_ONLY is true only gimple_min_invariant results are
400 : considered simplified. */
401 :
402 : static tree
403 8164490 : combine_cond_expr_cond (gimple *stmt, enum tree_code code, tree type,
404 : tree op0, tree op1, bool invariant_only)
405 : {
406 8164490 : tree t;
407 :
408 8164490 : gcc_assert (TREE_CODE_CLASS (code) == tcc_comparison);
409 :
410 8164490 : fold_defer_overflow_warnings ();
411 8164490 : t = fold_binary_loc (gimple_location (stmt), code, type, op0, op1);
412 8164490 : if (!t)
413 : {
414 4629249 : fold_undefer_overflow_warnings (false, NULL, 0);
415 4629249 : return NULL_TREE;
416 : }
417 :
418 : /* Require that we got a boolean type out if we put one in. */
419 3535241 : gcc_assert (TREE_CODE (TREE_TYPE (t)) == TREE_CODE (type));
420 :
421 : /* Canonicalize the combined condition for use in a COND_EXPR. */
422 3535241 : t = canonicalize_cond_expr_cond (t);
423 :
424 : /* Bail out if we required an invariant but didn't get one. */
425 3535241 : if (!t || (invariant_only && !is_gimple_min_invariant (t)))
426 : {
427 3301235 : fold_undefer_overflow_warnings (false, NULL, 0);
428 3301235 : return NULL_TREE;
429 : }
430 :
431 234006 : bool nowarn = warning_suppressed_p (stmt, OPT_Wstrict_overflow);
432 234006 : fold_undefer_overflow_warnings (!nowarn, stmt, 0);
433 :
434 234006 : return t;
435 : }
436 :
437 : /* Combine the comparison OP0 CODE OP1 at LOC with the defining statements
438 : of its operand. Return a new comparison tree or NULL_TREE if there
439 : were no simplifying combines. */
440 :
441 : static tree
442 21786927 : forward_propagate_into_comparison_1 (gimple *stmt,
443 : enum tree_code code, tree type,
444 : tree op0, tree op1)
445 : {
446 21786927 : tree tmp = NULL_TREE;
447 21786927 : tree rhs0 = NULL_TREE, rhs1 = NULL_TREE;
448 21786927 : bool single_use0_p = false, single_use1_p = false;
449 :
450 : /* For comparisons use the first operand, that is likely to
451 : simplify comparisons against constants. */
452 21786927 : if (TREE_CODE (op0) == SSA_NAME)
453 : {
454 21750774 : gimple *def_stmt = get_prop_source_stmt (op0, false, &single_use0_p);
455 21750774 : if (def_stmt && can_propagate_from (def_stmt))
456 : {
457 5489210 : enum tree_code def_code = gimple_assign_rhs_code (def_stmt);
458 5489210 : bool invariant_only_p = !single_use0_p;
459 :
460 5489210 : rhs0 = rhs_to_tree (TREE_TYPE (op1), def_stmt);
461 :
462 : /* Always combine comparisons or conversions from booleans. */
463 5489210 : if (TREE_CODE (op1) == INTEGER_CST
464 5489210 : && ((CONVERT_EXPR_CODE_P (def_code)
465 869621 : && TREE_CODE (TREE_TYPE (TREE_OPERAND (rhs0, 0)))
466 : == BOOLEAN_TYPE)
467 3515231 : || TREE_CODE_CLASS (def_code) == tcc_comparison))
468 : invariant_only_p = false;
469 :
470 5489210 : tmp = combine_cond_expr_cond (stmt, code, type,
471 : rhs0, op1, invariant_only_p);
472 5489210 : if (tmp)
473 : return tmp;
474 : }
475 : }
476 :
477 : /* If that wasn't successful, try the second operand. */
478 21561154 : if (TREE_CODE (op1) == SSA_NAME)
479 : {
480 5415318 : gimple *def_stmt = get_prop_source_stmt (op1, false, &single_use1_p);
481 5415318 : if (def_stmt && can_propagate_from (def_stmt))
482 : {
483 1759860 : rhs1 = rhs_to_tree (TREE_TYPE (op0), def_stmt);
484 3519720 : tmp = combine_cond_expr_cond (stmt, code, type,
485 1759860 : op0, rhs1, !single_use1_p);
486 1759860 : if (tmp)
487 : return tmp;
488 : }
489 : }
490 :
491 : /* If that wasn't successful either, try both operands. */
492 21554829 : if (rhs0 != NULL_TREE
493 21554829 : && rhs1 != NULL_TREE)
494 915420 : tmp = combine_cond_expr_cond (stmt, code, type,
495 : rhs0, rhs1,
496 915420 : !(single_use0_p && single_use1_p));
497 :
498 : return tmp;
499 : }
500 :
501 : /* Propagate from the ssa name definition statements of the assignment
502 : from a comparison at *GSI into the conditional if that simplifies it.
503 : Returns true if the stmt was modified. */
504 :
505 : static bool
506 2560141 : forward_propagate_into_comparison (gimple_stmt_iterator *gsi)
507 : {
508 2560141 : gimple *stmt = gsi_stmt (*gsi);
509 2560141 : tree tmp;
510 2560141 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
511 2560141 : tree rhs1 = gimple_assign_rhs1 (stmt);
512 2560141 : tree rhs2 = gimple_assign_rhs2 (stmt);
513 :
514 : /* Combine the comparison with defining statements. */
515 2560141 : tmp = forward_propagate_into_comparison_1 (stmt,
516 : gimple_assign_rhs_code (stmt),
517 : type, rhs1, rhs2);
518 2560141 : if (tmp && useless_type_conversion_p (type, TREE_TYPE (tmp)))
519 : {
520 7136 : if (dump_file)
521 : {
522 0 : fprintf (dump_file, " Replaced '");
523 0 : print_gimple_expr (dump_file, stmt, 0);
524 0 : fprintf (dump_file, "' with '");
525 0 : print_generic_expr (dump_file, tmp);
526 0 : fprintf (dump_file, "'\n");
527 : }
528 7136 : gimple_assign_set_rhs_from_tree (gsi, tmp);
529 7136 : fold_stmt (gsi);
530 7136 : update_stmt (gsi_stmt (*gsi));
531 :
532 7136 : if (TREE_CODE (rhs1) == SSA_NAME)
533 7136 : remove_prop_source_from_use (rhs1);
534 7136 : if (TREE_CODE (rhs2) == SSA_NAME)
535 2970 : remove_prop_source_from_use (rhs2);
536 7136 : return true;
537 : }
538 :
539 : return false;
540 : }
541 :
542 : /* Propagate from the ssa name definition statements of COND_EXPR
543 : in GIMPLE_COND statement STMT into the conditional if that simplifies it.
544 : Returns zero if no statement was changed, one if there were
545 : changes and two if cfg_cleanup needs to run. */
546 :
547 : static int
548 19226786 : forward_propagate_into_gimple_cond (gcond *stmt)
549 : {
550 19226786 : tree tmp;
551 19226786 : enum tree_code code = gimple_cond_code (stmt);
552 19226786 : tree rhs1 = gimple_cond_lhs (stmt);
553 19226786 : tree rhs2 = gimple_cond_rhs (stmt);
554 :
555 : /* GIMPLE_COND will always be a comparison. */
556 19226786 : gcc_assert (TREE_CODE_CLASS (gimple_cond_code (stmt)) == tcc_comparison);
557 :
558 19226786 : tmp = forward_propagate_into_comparison_1 (stmt, code,
559 : boolean_type_node,
560 : rhs1, rhs2);
561 19226786 : if (tmp
562 19226786 : && is_gimple_condexpr_for_cond (tmp))
563 : {
564 220538 : if (dump_file)
565 : {
566 9 : fprintf (dump_file, " Replaced '");
567 9 : print_gimple_expr (dump_file, stmt, 0);
568 9 : fprintf (dump_file, "' with '");
569 9 : print_generic_expr (dump_file, tmp);
570 9 : fprintf (dump_file, "'\n");
571 : }
572 :
573 220538 : gimple_cond_set_condition_from_tree (stmt, unshare_expr (tmp));
574 220538 : update_stmt (stmt);
575 :
576 220538 : if (TREE_CODE (rhs1) == SSA_NAME)
577 220538 : remove_prop_source_from_use (rhs1);
578 220538 : if (TREE_CODE (rhs2) == SSA_NAME)
579 5556 : remove_prop_source_from_use (rhs2);
580 220538 : return is_gimple_min_invariant (tmp) ? 2 : 1;
581 : }
582 :
583 19006248 : if (canonicalize_bool_cond (stmt, gimple_bb (stmt)))
584 : return 1;
585 :
586 : return 0;
587 : }
588 :
589 : /* We've just substituted an ADDR_EXPR into stmt. Update all the
590 : relevant data structures to match. */
591 :
592 : static void
593 1942134 : tidy_after_forward_propagate_addr (gimple *stmt)
594 : {
595 : /* We may have turned a trapping insn into a non-trapping insn. */
596 1942134 : if (maybe_clean_or_replace_eh_stmt (stmt, stmt))
597 131 : bitmap_set_bit (to_purge, gimple_bb (stmt)->index);
598 :
599 1942134 : if (TREE_CODE (gimple_assign_rhs1 (stmt)) == ADDR_EXPR)
600 242259 : recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
601 1942134 : }
602 :
603 : /* NAME is a SSA_NAME representing DEF_RHS which is of the form
604 : ADDR_EXPR <whatever>.
605 :
606 : Try to forward propagate the ADDR_EXPR into the use USE_STMT.
607 : Often this will allow for removal of an ADDR_EXPR and INDIRECT_REF
608 : node or for recovery of array indexing from pointer arithmetic.
609 :
610 : Return true if the propagation was successful (the propagation can
611 : be not totally successful, yet things may have been changed). */
612 :
613 : static bool
614 2766341 : forward_propagate_addr_expr_1 (tree name, tree def_rhs,
615 : gimple_stmt_iterator *use_stmt_gsi,
616 : bool single_use_p)
617 : {
618 2766341 : tree lhs, rhs, rhs2, array_ref;
619 2766341 : gimple *use_stmt = gsi_stmt (*use_stmt_gsi);
620 2766341 : enum tree_code rhs_code;
621 2766341 : bool res = true;
622 :
623 2766341 : gcc_assert (TREE_CODE (def_rhs) == ADDR_EXPR);
624 :
625 2766341 : lhs = gimple_assign_lhs (use_stmt);
626 2766341 : rhs_code = gimple_assign_rhs_code (use_stmt);
627 2766341 : rhs = gimple_assign_rhs1 (use_stmt);
628 :
629 : /* Do not perform copy-propagation but recurse through copy chains. */
630 2766341 : if (TREE_CODE (lhs) == SSA_NAME
631 1369592 : && rhs_code == SSA_NAME)
632 6845 : return forward_propagate_addr_expr (lhs, def_rhs, single_use_p);
633 :
634 : /* The use statement could be a conversion. Recurse to the uses of the
635 : lhs as copyprop does not copy through pointer to integer to pointer
636 : conversions and FRE does not catch all cases either.
637 : Treat the case of a single-use name and
638 : a conversion to def_rhs type separate, though. */
639 2759496 : if (TREE_CODE (lhs) == SSA_NAME
640 1362747 : && CONVERT_EXPR_CODE_P (rhs_code))
641 : {
642 : /* If there is a point in a conversion chain where the types match
643 : so we can remove a conversion re-materialize the address here
644 : and stop. */
645 24032 : if (single_use_p
646 24032 : && useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (def_rhs)))
647 : {
648 1 : gimple_assign_set_rhs1 (use_stmt, unshare_expr (def_rhs));
649 1 : gimple_assign_set_rhs_code (use_stmt, TREE_CODE (def_rhs));
650 1 : return true;
651 : }
652 :
653 : /* Else recurse if the conversion preserves the address value. */
654 48062 : if ((INTEGRAL_TYPE_P (TREE_TYPE (lhs))
655 2 : || POINTER_TYPE_P (TREE_TYPE (lhs)))
656 48062 : && (TYPE_PRECISION (TREE_TYPE (lhs))
657 24031 : >= TYPE_PRECISION (TREE_TYPE (def_rhs))))
658 23964 : return forward_propagate_addr_expr (lhs, def_rhs, single_use_p);
659 :
660 : return false;
661 : }
662 :
663 : /* If this isn't a conversion chain from this on we only can propagate
664 : into compatible pointer contexts. */
665 2735464 : if (!types_compatible_p (TREE_TYPE (name), TREE_TYPE (def_rhs)))
666 : return false;
667 :
668 : /* Propagate through constant pointer adjustments. */
669 2714804 : if (TREE_CODE (lhs) == SSA_NAME
670 1319228 : && rhs_code == POINTER_PLUS_EXPR
671 1319228 : && rhs == name
672 2877910 : && TREE_CODE (gimple_assign_rhs2 (use_stmt)) == INTEGER_CST)
673 : {
674 117835 : tree new_def_rhs;
675 : /* As we come here with non-invariant addresses in def_rhs we need
676 : to make sure we can build a valid constant offsetted address
677 : for further propagation. Simply rely on fold building that
678 : and check after the fact. */
679 117835 : new_def_rhs = fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (rhs)),
680 : def_rhs,
681 : fold_convert (ptr_type_node,
682 : gimple_assign_rhs2 (use_stmt)));
683 117835 : if (TREE_CODE (new_def_rhs) == MEM_REF
684 117835 : && !is_gimple_mem_ref_addr (TREE_OPERAND (new_def_rhs, 0)))
685 : return false;
686 113862 : new_def_rhs = build1 (ADDR_EXPR, TREE_TYPE (rhs), new_def_rhs);
687 :
688 : /* Recurse. If we could propagate into all uses of lhs do not
689 : bother to replace into the current use but just pretend we did. */
690 113862 : if (forward_propagate_addr_expr (lhs, new_def_rhs, single_use_p))
691 : return true;
692 :
693 37738 : if (useless_type_conversion_p (TREE_TYPE (lhs),
694 37738 : TREE_TYPE (new_def_rhs)))
695 37738 : gimple_assign_set_rhs_with_ops (use_stmt_gsi, TREE_CODE (new_def_rhs),
696 : new_def_rhs);
697 0 : else if (is_gimple_min_invariant (new_def_rhs))
698 0 : gimple_assign_set_rhs_with_ops (use_stmt_gsi, NOP_EXPR, new_def_rhs);
699 : else
700 : return false;
701 37738 : gcc_assert (gsi_stmt (*use_stmt_gsi) == use_stmt);
702 37738 : update_stmt (use_stmt);
703 37738 : return true;
704 : }
705 :
706 : /* Now strip away any outer COMPONENT_REF/ARRAY_REF nodes from the LHS.
707 : ADDR_EXPR will not appear on the LHS. */
708 2596969 : tree *lhsp = gimple_assign_lhs_ptr (use_stmt);
709 3928633 : while (handled_component_p (*lhsp))
710 1331664 : lhsp = &TREE_OPERAND (*lhsp, 0);
711 2596969 : lhs = *lhsp;
712 :
713 : /* Now see if the LHS node is a MEM_REF using NAME. If so,
714 : propagate the ADDR_EXPR into the use of NAME and fold the result. */
715 2596969 : if (TREE_CODE (lhs) == MEM_REF
716 2596969 : && TREE_OPERAND (lhs, 0) == name)
717 : {
718 880195 : tree def_rhs_base;
719 880195 : poly_int64 def_rhs_offset;
720 : /* If the address is invariant we can always fold it. */
721 880195 : if ((def_rhs_base = get_addr_base_and_unit_offset (TREE_OPERAND (def_rhs, 0),
722 : &def_rhs_offset)))
723 : {
724 834646 : poly_offset_int off = mem_ref_offset (lhs);
725 834646 : tree new_ptr;
726 834646 : off += def_rhs_offset;
727 834646 : if (TREE_CODE (def_rhs_base) == MEM_REF)
728 : {
729 812905 : off += mem_ref_offset (def_rhs_base);
730 812905 : new_ptr = TREE_OPERAND (def_rhs_base, 0);
731 : }
732 : else
733 21741 : new_ptr = build_fold_addr_expr (def_rhs_base);
734 834646 : TREE_OPERAND (lhs, 0) = new_ptr;
735 834646 : TREE_OPERAND (lhs, 1)
736 834646 : = wide_int_to_tree (TREE_TYPE (TREE_OPERAND (lhs, 1)), off);
737 834646 : tidy_after_forward_propagate_addr (use_stmt);
738 : /* Continue propagating into the RHS if this was not the only use. */
739 834646 : if (single_use_p)
740 225842 : return true;
741 : }
742 : /* If the LHS is a plain dereference and the value type is the same as
743 : that of the pointed-to type of the address we can put the
744 : dereferenced address on the LHS preserving the original alias-type. */
745 45549 : else if (integer_zerop (TREE_OPERAND (lhs, 1))
746 17659 : && ((gimple_assign_lhs (use_stmt) == lhs
747 14101 : && useless_type_conversion_p
748 14101 : (TREE_TYPE (TREE_OPERAND (def_rhs, 0)),
749 14101 : TREE_TYPE (gimple_assign_rhs1 (use_stmt))))
750 13416 : || types_compatible_p (TREE_TYPE (lhs),
751 13416 : TREE_TYPE (TREE_OPERAND (def_rhs, 0))))
752 : /* Don't forward anything into clobber stmts if it would result
753 : in the lhs no longer being a MEM_REF. */
754 53003 : && (!gimple_clobber_p (use_stmt)
755 161 : || TREE_CODE (TREE_OPERAND (def_rhs, 0)) == MEM_REF))
756 : {
757 7293 : tree *def_rhs_basep = &TREE_OPERAND (def_rhs, 0);
758 7293 : tree new_offset, new_base, saved, new_lhs;
759 26183 : while (handled_component_p (*def_rhs_basep))
760 11597 : def_rhs_basep = &TREE_OPERAND (*def_rhs_basep, 0);
761 7293 : saved = *def_rhs_basep;
762 7293 : if (TREE_CODE (*def_rhs_basep) == MEM_REF)
763 : {
764 3757 : new_base = TREE_OPERAND (*def_rhs_basep, 0);
765 3757 : new_offset = fold_convert (TREE_TYPE (TREE_OPERAND (lhs, 1)),
766 : TREE_OPERAND (*def_rhs_basep, 1));
767 : }
768 : else
769 : {
770 3536 : new_base = build_fold_addr_expr (*def_rhs_basep);
771 3536 : new_offset = TREE_OPERAND (lhs, 1);
772 : }
773 7293 : *def_rhs_basep = build2 (MEM_REF, TREE_TYPE (*def_rhs_basep),
774 : new_base, new_offset);
775 7293 : TREE_THIS_VOLATILE (*def_rhs_basep) = TREE_THIS_VOLATILE (lhs);
776 7293 : TREE_SIDE_EFFECTS (*def_rhs_basep) = TREE_SIDE_EFFECTS (lhs);
777 7293 : TREE_THIS_NOTRAP (*def_rhs_basep) = TREE_THIS_NOTRAP (lhs);
778 7293 : new_lhs = unshare_expr (TREE_OPERAND (def_rhs, 0));
779 7293 : *lhsp = new_lhs;
780 7293 : TREE_THIS_VOLATILE (new_lhs) = TREE_THIS_VOLATILE (lhs);
781 7293 : TREE_SIDE_EFFECTS (new_lhs) = TREE_SIDE_EFFECTS (lhs);
782 7293 : *def_rhs_basep = saved;
783 7293 : tidy_after_forward_propagate_addr (use_stmt);
784 : /* Continue propagating into the RHS if this was not the
785 : only use. */
786 7293 : if (single_use_p)
787 : return true;
788 : }
789 : else
790 : /* We can have a struct assignment dereferencing our name twice.
791 : Note that we didn't propagate into the lhs to not falsely
792 : claim we did when propagating into the rhs. */
793 : res = false;
794 : }
795 :
796 : /* Strip away any outer COMPONENT_REF, ARRAY_REF or ADDR_EXPR
797 : nodes from the RHS. */
798 2367599 : tree *rhsp = gimple_assign_rhs1_ptr (use_stmt);
799 2367599 : if (TREE_CODE (*rhsp) == ADDR_EXPR)
800 230619 : rhsp = &TREE_OPERAND (*rhsp, 0);
801 3344005 : while (handled_component_p (*rhsp))
802 976406 : rhsp = &TREE_OPERAND (*rhsp, 0);
803 2367599 : rhs = *rhsp;
804 :
805 : /* Now see if the RHS node is a MEM_REF using NAME. If so,
806 : propagate the ADDR_EXPR into the use of NAME and fold the result. */
807 2367599 : if (TREE_CODE (rhs) == MEM_REF
808 2367599 : && TREE_OPERAND (rhs, 0) == name)
809 : {
810 1121584 : tree def_rhs_base;
811 1121584 : poly_int64 def_rhs_offset;
812 1121584 : if ((def_rhs_base = get_addr_base_and_unit_offset (TREE_OPERAND (def_rhs, 0),
813 : &def_rhs_offset)))
814 : {
815 1085536 : poly_offset_int off = mem_ref_offset (rhs);
816 1085536 : tree new_ptr;
817 1085536 : off += def_rhs_offset;
818 1085536 : if (TREE_CODE (def_rhs_base) == MEM_REF)
819 : {
820 1059974 : off += mem_ref_offset (def_rhs_base);
821 1059974 : new_ptr = TREE_OPERAND (def_rhs_base, 0);
822 : }
823 : else
824 25562 : new_ptr = build_fold_addr_expr (def_rhs_base);
825 1085536 : TREE_OPERAND (rhs, 0) = new_ptr;
826 1085536 : TREE_OPERAND (rhs, 1)
827 1085536 : = wide_int_to_tree (TREE_TYPE (TREE_OPERAND (rhs, 1)), off);
828 1085536 : fold_stmt_inplace (use_stmt_gsi);
829 1085536 : tidy_after_forward_propagate_addr (use_stmt);
830 1085536 : return res;
831 : }
832 : /* If the RHS is a plain dereference and the value type is the same as
833 : that of the pointed-to type of the address we can put the
834 : dereferenced address on the RHS preserving the original alias-type. */
835 36048 : else if (integer_zerop (TREE_OPERAND (rhs, 1))
836 36048 : && ((gimple_assign_rhs1 (use_stmt) == rhs
837 19685 : && useless_type_conversion_p
838 19685 : (TREE_TYPE (gimple_assign_lhs (use_stmt)),
839 19685 : TREE_TYPE (TREE_OPERAND (def_rhs, 0))))
840 22451 : || types_compatible_p (TREE_TYPE (rhs),
841 22451 : TREE_TYPE (TREE_OPERAND (def_rhs, 0)))))
842 : {
843 14659 : tree *def_rhs_basep = &TREE_OPERAND (def_rhs, 0);
844 14659 : tree new_offset, new_base, saved, new_rhs;
845 51859 : while (handled_component_p (*def_rhs_basep))
846 22541 : def_rhs_basep = &TREE_OPERAND (*def_rhs_basep, 0);
847 14659 : saved = *def_rhs_basep;
848 14659 : if (TREE_CODE (*def_rhs_basep) == MEM_REF)
849 : {
850 7042 : new_base = TREE_OPERAND (*def_rhs_basep, 0);
851 7042 : new_offset = fold_convert (TREE_TYPE (TREE_OPERAND (rhs, 1)),
852 : TREE_OPERAND (*def_rhs_basep, 1));
853 : }
854 : else
855 : {
856 7617 : new_base = build_fold_addr_expr (*def_rhs_basep);
857 7617 : new_offset = TREE_OPERAND (rhs, 1);
858 : }
859 14659 : tree atype = TREE_TYPE (*def_rhs_basep);
860 14659 : if (TYPE_ALIGN (TREE_TYPE (rhs)) < TYPE_ALIGN (atype))
861 525 : atype = build_aligned_type (atype, TYPE_ALIGN (TREE_TYPE (rhs)));
862 14659 : *def_rhs_basep = build2 (MEM_REF, atype, new_base, new_offset);
863 14659 : TREE_THIS_VOLATILE (*def_rhs_basep) = TREE_THIS_VOLATILE (rhs);
864 14659 : TREE_SIDE_EFFECTS (*def_rhs_basep) = TREE_SIDE_EFFECTS (rhs);
865 14659 : TREE_THIS_NOTRAP (*def_rhs_basep) = TREE_THIS_NOTRAP (rhs);
866 14659 : new_rhs = unshare_expr (TREE_OPERAND (def_rhs, 0));
867 14659 : *rhsp = new_rhs;
868 14659 : TREE_THIS_VOLATILE (new_rhs) = TREE_THIS_VOLATILE (rhs);
869 14659 : TREE_SIDE_EFFECTS (new_rhs) = TREE_SIDE_EFFECTS (rhs);
870 14659 : *def_rhs_basep = saved;
871 14659 : fold_stmt_inplace (use_stmt_gsi);
872 14659 : tidy_after_forward_propagate_addr (use_stmt);
873 14659 : return res;
874 : }
875 : }
876 :
877 : /* If the use of the ADDR_EXPR is not a POINTER_PLUS_EXPR, there
878 : is nothing to do. */
879 1267404 : if (gimple_assign_rhs_code (use_stmt) != POINTER_PLUS_EXPR
880 1267404 : || gimple_assign_rhs1 (use_stmt) != name)
881 : return false;
882 :
883 : /* The remaining cases are all for turning pointer arithmetic into
884 : array indexing. They only apply when we have the address of
885 : element zero in an array. If that is not the case then there
886 : is nothing to do. */
887 45271 : array_ref = TREE_OPERAND (def_rhs, 0);
888 45271 : if ((TREE_CODE (array_ref) != ARRAY_REF
889 4525 : || TREE_CODE (TREE_TYPE (TREE_OPERAND (array_ref, 0))) != ARRAY_TYPE
890 4525 : || TREE_CODE (TREE_OPERAND (array_ref, 1)) != INTEGER_CST)
891 46749 : && TREE_CODE (TREE_TYPE (array_ref)) != ARRAY_TYPE)
892 : return false;
893 :
894 22575 : rhs2 = gimple_assign_rhs2 (use_stmt);
895 : /* Optimize &x[C1] p+ C2 to &x p+ C3 with C3 = C1 * element_size + C2. */
896 22575 : if (TREE_CODE (rhs2) == INTEGER_CST)
897 : {
898 0 : tree new_rhs = build1_loc (gimple_location (use_stmt),
899 0 : ADDR_EXPR, TREE_TYPE (def_rhs),
900 0 : fold_build2 (MEM_REF,
901 : TREE_TYPE (TREE_TYPE (def_rhs)),
902 : unshare_expr (def_rhs),
903 : fold_convert (ptr_type_node,
904 : rhs2)));
905 0 : gimple_assign_set_rhs_from_tree (use_stmt_gsi, new_rhs);
906 0 : use_stmt = gsi_stmt (*use_stmt_gsi);
907 0 : update_stmt (use_stmt);
908 0 : tidy_after_forward_propagate_addr (use_stmt);
909 0 : return true;
910 : }
911 :
912 : return false;
913 : }
914 :
915 : /* STMT is a statement of the form SSA_NAME = ADDR_EXPR <whatever>.
916 :
917 : Try to forward propagate the ADDR_EXPR into all uses of the SSA_NAME.
918 : Often this will allow for removal of an ADDR_EXPR and INDIRECT_REF
919 : node or for recovery of array indexing from pointer arithmetic.
920 :
921 : PARENT_SINGLE_USE_P tells if, when in a recursive invocation, NAME was
922 : the single use in the previous invocation. Pass true when calling
923 : this as toplevel.
924 :
925 : Returns true, if all uses have been propagated into. */
926 :
927 : static bool
928 3211574 : forward_propagate_addr_expr (tree name, tree rhs, bool parent_single_use_p)
929 : {
930 3211574 : bool all = true;
931 3211574 : bool single_use_p = parent_single_use_p && has_single_use (name);
932 :
933 16923303 : for (gimple *use_stmt : gather_imm_use_stmts (name))
934 : {
935 7288581 : bool result;
936 7288581 : tree use_rhs;
937 :
938 : /* If the use is not in a simple assignment statement, then
939 : there is nothing we can do. */
940 7288581 : if (!is_gimple_assign (use_stmt))
941 : {
942 4522240 : if (!is_gimple_debug (use_stmt))
943 1874258 : all = false;
944 4522240 : continue;
945 : }
946 :
947 2766341 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
948 2766341 : result = forward_propagate_addr_expr_1 (name, rhs, &gsi,
949 : single_use_p);
950 : /* If the use has moved to a different statement adjust
951 : the update machinery for the old statement too. */
952 2766341 : if (use_stmt != gsi_stmt (gsi))
953 : {
954 0 : update_stmt (use_stmt);
955 0 : use_stmt = gsi_stmt (gsi);
956 : }
957 2766341 : update_stmt (use_stmt);
958 2766341 : all &= result;
959 :
960 : /* Remove intermediate now unused copy and conversion chains. */
961 2766341 : use_rhs = gimple_assign_rhs1 (use_stmt);
962 2766341 : if (result
963 1447126 : && TREE_CODE (gimple_assign_lhs (use_stmt)) == SSA_NAME
964 1205640 : && TREE_CODE (use_rhs) == SSA_NAME
965 2846163 : && has_zero_uses (gimple_assign_lhs (use_stmt)))
966 : {
967 79822 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
968 79822 : fwprop_invalidate_lattice (gimple_get_lhs (use_stmt));
969 79822 : release_defs (use_stmt);
970 79822 : gsi_remove (&gsi, true);
971 : }
972 3211574 : }
973 :
974 3211574 : return all && has_zero_uses (name);
975 : }
976 :
977 :
978 : /* Helper function for simplify_gimple_switch. Remove case labels that
979 : have values outside the range of the new type. */
980 :
981 : static void
982 11554 : simplify_gimple_switch_label_vec (gswitch *stmt, tree index_type,
983 : vec<std::pair<int, int> > &edges_to_remove)
984 : {
985 11554 : unsigned int branch_num = gimple_switch_num_labels (stmt);
986 11554 : auto_vec<tree> labels (branch_num);
987 11554 : unsigned int i, len;
988 :
989 : /* Collect the existing case labels in a VEC, and preprocess it as if
990 : we are gimplifying a GENERIC SWITCH_EXPR. */
991 72354 : for (i = 1; i < branch_num; i++)
992 49246 : labels.quick_push (gimple_switch_label (stmt, i));
993 11554 : preprocess_case_label_vec_for_gimple (labels, index_type, NULL);
994 :
995 : /* If any labels were removed, replace the existing case labels
996 : in the GIMPLE_SWITCH statement with the correct ones.
997 : Note that the type updates were done in-place on the case labels,
998 : so we only have to replace the case labels in the GIMPLE_SWITCH
999 : if the number of labels changed. */
1000 11554 : len = labels.length ();
1001 11554 : if (len < branch_num - 1)
1002 : {
1003 0 : bitmap target_blocks;
1004 0 : edge_iterator ei;
1005 0 : edge e;
1006 :
1007 : /* Corner case: *all* case labels have been removed as being
1008 : out-of-range for INDEX_TYPE. Push one label and let the
1009 : CFG cleanups deal with this further. */
1010 0 : if (len == 0)
1011 : {
1012 0 : tree label, elt;
1013 :
1014 0 : label = CASE_LABEL (gimple_switch_default_label (stmt));
1015 0 : elt = build_case_label (build_int_cst (index_type, 0), NULL, label);
1016 0 : labels.quick_push (elt);
1017 0 : len = 1;
1018 : }
1019 :
1020 0 : for (i = 0; i < labels.length (); i++)
1021 0 : gimple_switch_set_label (stmt, i + 1, labels[i]);
1022 0 : for (i++ ; i < branch_num; i++)
1023 0 : gimple_switch_set_label (stmt, i, NULL_TREE);
1024 0 : gimple_switch_set_num_labels (stmt, len + 1);
1025 :
1026 : /* Cleanup any edges that are now dead. */
1027 0 : target_blocks = BITMAP_ALLOC (NULL);
1028 0 : for (i = 0; i < gimple_switch_num_labels (stmt); i++)
1029 : {
1030 0 : tree elt = gimple_switch_label (stmt, i);
1031 0 : basic_block target = label_to_block (cfun, CASE_LABEL (elt));
1032 0 : bitmap_set_bit (target_blocks, target->index);
1033 : }
1034 0 : for (ei = ei_start (gimple_bb (stmt)->succs); (e = ei_safe_edge (ei)); )
1035 : {
1036 0 : if (! bitmap_bit_p (target_blocks, e->dest->index))
1037 0 : edges_to_remove.safe_push (std::make_pair (e->src->index,
1038 0 : e->dest->index));
1039 : else
1040 0 : ei_next (&ei);
1041 : }
1042 0 : BITMAP_FREE (target_blocks);
1043 : }
1044 11554 : }
1045 :
1046 : /* STMT is a SWITCH_EXPR for which we attempt to find equivalent forms of
1047 : the condition which we may be able to optimize better. */
1048 :
1049 : static bool
1050 103322 : simplify_gimple_switch (gswitch *stmt,
1051 : vec<std::pair<int, int> > &edges_to_remove,
1052 : bitmap simple_dce_worklist)
1053 : {
1054 : /* The optimization that we really care about is removing unnecessary
1055 : casts. That will let us do much better in propagating the inferred
1056 : constant at the switch target. */
1057 103322 : tree cond = gimple_switch_index (stmt);
1058 103322 : if (TREE_CODE (cond) == SSA_NAME)
1059 : {
1060 103321 : gimple *def_stmt = SSA_NAME_DEF_STMT (cond);
1061 103321 : if (gimple_assign_cast_p (def_stmt))
1062 : {
1063 12440 : tree def = gimple_assign_rhs1 (def_stmt);
1064 12440 : if (TREE_CODE (def) != SSA_NAME)
1065 : return false;
1066 :
1067 : /* If we have an extension or sign-change that preserves the
1068 : values we check against then we can copy the source value into
1069 : the switch. */
1070 12440 : tree ti = TREE_TYPE (def);
1071 12440 : if (INTEGRAL_TYPE_P (ti)
1072 12440 : && TYPE_PRECISION (ti) <= TYPE_PRECISION (TREE_TYPE (cond)))
1073 : {
1074 12195 : size_t n = gimple_switch_num_labels (stmt);
1075 12195 : tree min = NULL_TREE, max = NULL_TREE;
1076 12195 : if (n > 1)
1077 : {
1078 12195 : min = CASE_LOW (gimple_switch_label (stmt, 1));
1079 12195 : if (CASE_HIGH (gimple_switch_label (stmt, n - 1)))
1080 154 : max = CASE_HIGH (gimple_switch_label (stmt, n - 1));
1081 : else
1082 12041 : max = CASE_LOW (gimple_switch_label (stmt, n - 1));
1083 : }
1084 12195 : if ((!min || int_fits_type_p (min, ti))
1085 12191 : && (!max || int_fits_type_p (max, ti)))
1086 : {
1087 11554 : bitmap_set_bit (simple_dce_worklist,
1088 11554 : SSA_NAME_VERSION (cond));
1089 11554 : gimple_switch_set_index (stmt, def);
1090 11554 : simplify_gimple_switch_label_vec (stmt, ti,
1091 : edges_to_remove);
1092 11554 : update_stmt (stmt);
1093 11554 : return true;
1094 : }
1095 : }
1096 : }
1097 : }
1098 :
1099 : return false;
1100 : }
1101 :
1102 : /* For pointers p2 and p1 return p2 - p1 if the
1103 : difference is known and constant, otherwise return NULL. */
1104 :
1105 : static tree
1106 5232 : constant_pointer_difference (tree p1, tree p2)
1107 : {
1108 5232 : int i, j;
1109 : #define CPD_ITERATIONS 5
1110 5232 : tree exps[2][CPD_ITERATIONS];
1111 5232 : tree offs[2][CPD_ITERATIONS];
1112 5232 : int cnt[2];
1113 :
1114 15696 : for (i = 0; i < 2; i++)
1115 : {
1116 10464 : tree p = i ? p1 : p2;
1117 10464 : tree off = size_zero_node;
1118 10464 : gimple *stmt;
1119 10464 : enum tree_code code;
1120 :
1121 : /* For each of p1 and p2 we need to iterate at least
1122 : twice, to handle ADDR_EXPR directly in p1/p2,
1123 : SSA_NAME with ADDR_EXPR or POINTER_PLUS_EXPR etc.
1124 : on definition's stmt RHS. Iterate a few extra times. */
1125 10464 : j = 0;
1126 12400 : do
1127 : {
1128 12400 : if (!POINTER_TYPE_P (TREE_TYPE (p)))
1129 : break;
1130 12394 : if (TREE_CODE (p) == ADDR_EXPR)
1131 : {
1132 8988 : tree q = TREE_OPERAND (p, 0);
1133 8988 : poly_int64 offset;
1134 8988 : tree base = get_addr_base_and_unit_offset (q, &offset);
1135 8988 : if (base)
1136 : {
1137 8172 : q = base;
1138 8172 : if (maybe_ne (offset, 0))
1139 3409 : off = size_binop (PLUS_EXPR, off, size_int (offset));
1140 : }
1141 8988 : if (TREE_CODE (q) == MEM_REF
1142 8988 : && TREE_CODE (TREE_OPERAND (q, 0)) == SSA_NAME)
1143 : {
1144 213 : p = TREE_OPERAND (q, 0);
1145 213 : off = size_binop (PLUS_EXPR, off,
1146 : wide_int_to_tree (sizetype,
1147 : mem_ref_offset (q)));
1148 : }
1149 : else
1150 : {
1151 8775 : exps[i][j] = q;
1152 8775 : offs[i][j++] = off;
1153 8775 : break;
1154 : }
1155 : }
1156 3619 : if (TREE_CODE (p) != SSA_NAME)
1157 : break;
1158 3619 : exps[i][j] = p;
1159 3619 : offs[i][j++] = off;
1160 3619 : if (j == CPD_ITERATIONS)
1161 : break;
1162 3619 : stmt = SSA_NAME_DEF_STMT (p);
1163 3619 : if (!is_gimple_assign (stmt) || gimple_assign_lhs (stmt) != p)
1164 : break;
1165 2840 : code = gimple_assign_rhs_code (stmt);
1166 2840 : if (code == POINTER_PLUS_EXPR)
1167 : {
1168 1482 : if (TREE_CODE (gimple_assign_rhs2 (stmt)) != INTEGER_CST)
1169 : break;
1170 951 : off = size_binop (PLUS_EXPR, off, gimple_assign_rhs2 (stmt));
1171 951 : p = gimple_assign_rhs1 (stmt);
1172 : }
1173 1358 : else if (code == ADDR_EXPR || CONVERT_EXPR_CODE_P (code))
1174 985 : p = gimple_assign_rhs1 (stmt);
1175 : else
1176 : break;
1177 : }
1178 : while (1);
1179 10464 : cnt[i] = j;
1180 : }
1181 :
1182 7310 : for (i = 0; i < cnt[0]; i++)
1183 9743 : for (j = 0; j < cnt[1]; j++)
1184 7665 : if (exps[0][i] == exps[1][j])
1185 4342 : return size_binop (MINUS_EXPR, offs[0][i], offs[1][j]);
1186 :
1187 : return NULL_TREE;
1188 : }
1189 :
1190 : /* Helper function for optimize_aggr_zeroprop.
1191 : Props the zeroing (memset, VAL) that was done in DEST+OFFSET:LEN
1192 : (DEFSTMT) into the STMT. Returns true if the STMT was updated. */
1193 : static void
1194 22158124 : optimize_aggr_zeroprop_1 (gimple *defstmt, gimple *stmt,
1195 : tree dest, poly_int64 offset, tree val,
1196 : poly_offset_int len)
1197 : {
1198 22158124 : tree src2;
1199 22158124 : tree len2 = NULL_TREE;
1200 22158124 : poly_int64 offset2;
1201 :
1202 22158124 : if (gimple_call_builtin_p (stmt, BUILT_IN_MEMCPY)
1203 21606 : && TREE_CODE (gimple_call_arg (stmt, 1)) == ADDR_EXPR
1204 22171853 : && poly_int_tree_p (gimple_call_arg (stmt, 2)))
1205 : {
1206 12712 : src2 = TREE_OPERAND (gimple_call_arg (stmt, 1), 0);
1207 12712 : len2 = gimple_call_arg (stmt, 2);
1208 : }
1209 22145412 : else if (gimple_assign_load_p (stmt) && gimple_store_p (stmt))
1210 : {
1211 1888127 : src2 = gimple_assign_rhs1 (stmt);
1212 1888127 : len2 = (TREE_CODE (src2) == COMPONENT_REF
1213 1888127 : ? DECL_SIZE_UNIT (TREE_OPERAND (src2, 1))
1214 1720721 : : TYPE_SIZE_UNIT (TREE_TYPE (src2)));
1215 : /* Can only handle zero memsets. */
1216 1888127 : if (!integer_zerop (val))
1217 22136409 : return;
1218 : }
1219 : else
1220 20257285 : return;
1221 :
1222 1899877 : if (len2 == NULL_TREE
1223 1899877 : || !poly_int_tree_p (len2))
1224 : return;
1225 :
1226 1899877 : src2 = get_addr_base_and_unit_offset (src2, &offset2);
1227 1899877 : if (src2 == NULL_TREE
1228 1899877 : || maybe_lt (offset2, offset))
1229 : return;
1230 :
1231 861864 : if (!operand_equal_p (dest, src2, 0))
1232 : return;
1233 :
1234 : /* [ dest + offset, dest + offset + len - 1 ] is set to val.
1235 : Make sure that
1236 : [ dest + offset2, dest + offset2 + len2 - 1 ] is a subset of that. */
1237 130909 : if (maybe_gt (wi::to_poly_offset (len2) + (offset2 - offset),
1238 : len))
1239 : return;
1240 :
1241 21715 : if (dump_file && (dump_flags & TDF_DETAILS))
1242 : {
1243 32 : fprintf (dump_file, "Simplified\n ");
1244 32 : print_gimple_stmt (dump_file, stmt, 0, dump_flags);
1245 32 : fprintf (dump_file, "after previous\n ");
1246 32 : print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
1247 : }
1248 21715 : gimple *orig_stmt = stmt;
1249 : /* For simplicity, don't change the kind of the stmt,
1250 : turn dest = src; into dest = {}; and memcpy (&dest, &src, len);
1251 : into memset (&dest, val, len);
1252 : In theory we could change dest = src into memset if dest
1253 : is addressable (maybe beneficial if val is not 0), or
1254 : memcpy (&dest, &src, len) into dest = {} if len is the size
1255 : of dest, dest isn't volatile. */
1256 21715 : if (is_gimple_assign (stmt))
1257 : {
1258 21710 : tree ctor_type = TREE_TYPE (gimple_assign_lhs (stmt));
1259 21710 : tree ctor = build_constructor (ctor_type, NULL);
1260 21710 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1261 21710 : gimple_assign_set_rhs_from_tree (&gsi, ctor);
1262 21710 : update_stmt (stmt);
1263 21710 : statistics_counter_event (cfun, "copy zeroing propagation of aggregate", 1);
1264 : }
1265 : else /* If stmt is memcpy, transform it into memset. */
1266 : {
1267 5 : gcall *call = as_a <gcall *> (stmt);
1268 5 : tree fndecl = builtin_decl_implicit (BUILT_IN_MEMSET);
1269 5 : gimple_call_set_fndecl (call, fndecl);
1270 5 : gimple_call_set_fntype (call, TREE_TYPE (fndecl));
1271 5 : gimple_call_set_arg (call, 1, val);
1272 5 : update_stmt (stmt);
1273 5 : statistics_counter_event (cfun, "memcpy to memset changed", 1);
1274 : }
1275 :
1276 21715 : if (dump_file && (dump_flags & TDF_DETAILS))
1277 : {
1278 32 : fprintf (dump_file, "into\n ");
1279 32 : print_gimple_stmt (dump_file, stmt, 0, dump_flags);
1280 : }
1281 :
1282 : /* Mark the bb for eh cleanup if needed. */
1283 21715 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
1284 6 : bitmap_set_bit (to_purge, gimple_bb (stmt)->index);
1285 : }
1286 :
1287 : /* Optimize
1288 : a = {}; // DEST = value ;; LEN(nullptr)
1289 : b = a;
1290 : into
1291 : a = {};
1292 : b = {};
1293 : Similarly for memset (&a, ..., sizeof (a)); instead of a = {};
1294 : and/or memcpy (&b, &a, sizeof (a)); instead of b = a; */
1295 :
1296 : static void
1297 30661181 : optimize_aggr_zeroprop (gimple *stmt, bool full_walk)
1298 : {
1299 30661181 : ao_ref read;
1300 61322362 : if (gimple_has_volatile_ops (stmt))
1301 26694315 : return;
1302 :
1303 29736058 : tree dest = NULL_TREE;
1304 29736058 : tree val = integer_zero_node;
1305 29736058 : tree len = NULL_TREE;
1306 29736058 : bool can_use_tbba = true;
1307 :
1308 29736058 : if (gimple_call_builtin_p (stmt, BUILT_IN_MEMSET)
1309 109412 : && TREE_CODE (gimple_call_arg (stmt, 0)) == ADDR_EXPR
1310 55481 : && TREE_CODE (gimple_call_arg (stmt, 1)) == INTEGER_CST
1311 29789247 : && poly_int_tree_p (gimple_call_arg (stmt, 2)))
1312 : {
1313 50422 : dest = TREE_OPERAND (gimple_call_arg (stmt, 0), 0);
1314 50422 : len = gimple_call_arg (stmt, 2);
1315 50422 : val = gimple_call_arg (stmt, 1);
1316 50422 : ao_ref_init_from_ptr_and_size (&read, gimple_call_arg (stmt, 0), len);
1317 50422 : can_use_tbba = false;
1318 : }
1319 29685636 : else if (gimple_store_p (stmt)
1320 29626486 : && gimple_assign_single_p (stmt)
1321 59312122 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == STRING_CST)
1322 : {
1323 33498 : tree str = gimple_assign_rhs1 (stmt);
1324 33498 : dest = gimple_assign_lhs (stmt);
1325 33498 : ao_ref_init (&read, dest);
1326 : /* The string must contain all null char's for now. */
1327 38765 : for (int i = 0; i < TREE_STRING_LENGTH (str); i++)
1328 : {
1329 36249 : if (TREE_STRING_POINTER (str)[i] != 0)
1330 : {
1331 : dest = NULL_TREE;
1332 : break;
1333 : }
1334 : }
1335 : }
1336 : /* A store of integer (scalar, vector or complex) zeros is
1337 : a zero store. */
1338 29652138 : else if (gimple_store_p (stmt)
1339 29592988 : && gimple_assign_single_p (stmt)
1340 59245126 : && integer_zerop (gimple_assign_rhs1 (stmt)))
1341 : {
1342 3506435 : tree rhs = gimple_assign_rhs1 (stmt);
1343 3506435 : tree type = TREE_TYPE (rhs);
1344 3506435 : dest = gimple_assign_lhs (stmt);
1345 3506435 : ao_ref_init (&read, dest);
1346 : /* For integral types, the type precision needs to be a multiply of BITS_PER_UNIT. */
1347 3506435 : if (INTEGRAL_TYPE_P (type)
1348 3506435 : && (TYPE_PRECISION (type) % BITS_PER_UNIT) != 0)
1349 : dest = NULL_TREE;
1350 : }
1351 26145703 : else if (gimple_store_p (stmt)
1352 26086553 : && gimple_assign_single_p (stmt)
1353 26086553 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == CONSTRUCTOR
1354 26842762 : && !gimple_clobber_p (stmt))
1355 : {
1356 697059 : dest = gimple_assign_lhs (stmt);
1357 697059 : ao_ref_init (&read, dest);
1358 : }
1359 :
1360 4083533 : if (dest == NULL_TREE)
1361 25683507 : return;
1362 :
1363 4052551 : if (len == NULL_TREE)
1364 4002129 : len = (TREE_CODE (dest) == COMPONENT_REF
1365 4002129 : ? DECL_SIZE_UNIT (TREE_OPERAND (dest, 1))
1366 1742527 : : TYPE_SIZE_UNIT (TREE_TYPE (dest)));
1367 4002129 : if (len == NULL_TREE
1368 4052551 : || !poly_int_tree_p (len))
1369 : return;
1370 :
1371 : /* Sometimes memset can have no vdef due to invalid declaration of memset (const, etc.). */
1372 34799393 : if (!gimple_vdef (stmt))
1373 : return;
1374 :
1375 : /* This store needs to be on the byte boundary and pointing to an object. */
1376 4052527 : poly_int64 offset;
1377 4052527 : tree dest_base = get_addr_base_and_unit_offset (dest, &offset);
1378 4052527 : if (dest_base == NULL_TREE)
1379 : return;
1380 :
1381 : /* Setup the worklist. */
1382 3966866 : auto_vec<std::pair<tree, unsigned>> worklist;
1383 3966866 : unsigned limit = full_walk ? param_sccvn_max_alias_queries_per_access : 0;
1384 7933732 : worklist.safe_push (std::make_pair (gimple_vdef (stmt), limit));
1385 :
1386 27642416 : while (!worklist.is_empty ())
1387 : {
1388 19708684 : std::pair<tree, unsigned> top = worklist.pop ();
1389 19708684 : tree vdef = top.first;
1390 19708684 : limit = top.second;
1391 19708684 : gimple *use_stmt;
1392 19708684 : imm_use_iterator iter;
1393 63560226 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
1394 : {
1395 : /* Handling PHI nodes might not be worth it so don't. */
1396 24142858 : if (is_a <gphi*> (use_stmt))
1397 1984734 : continue;
1398 :
1399 : /* If this statement does not clobber add the vdef stmt to the
1400 : worklist.
1401 : After hitting the limit, allow clobbers to able to pass through. */
1402 2003861 : if ((limit != 0 || gimple_clobber_p (use_stmt))
1403 20189676 : && gimple_vdef (use_stmt)
1404 39307631 : && !stmt_may_clobber_ref_p_1 (use_stmt, &read,
1405 : /* tbaa_p = */ can_use_tbba))
1406 : {
1407 15741818 : unsigned new_limit = limit == 0 ? 0 : limit - 1;
1408 31483636 : worklist.safe_push (std::make_pair (gimple_vdef (use_stmt),
1409 : new_limit));
1410 : }
1411 :
1412 22158124 : optimize_aggr_zeroprop_1 (stmt, use_stmt, dest_base, offset,
1413 22158124 : val, wi::to_poly_offset (len));
1414 19708684 : }
1415 : }
1416 :
1417 3966866 : }
1418 :
1419 : /* Returns the pointer to the base of the object of the
1420 : reference EXPR and extracts the information about
1421 : the offset of the access, storing it to PBYTESIZE,
1422 : PBYTEPOS and PREVERSEP.
1423 : If the access is not a byte sized or position is not
1424 : on the byte, return NULL. */
1425 : static tree
1426 5241030 : split_core_and_offset_size (tree expr,
1427 : poly_int64 *pbytesize, poly_int64 *pbytepos,
1428 : tree *poffset, int *preversep)
1429 : {
1430 5241030 : tree core;
1431 5241030 : machine_mode mode;
1432 5241030 : int unsignedp, volatilep;
1433 5241030 : poly_int64 bitsize;
1434 5241030 : poly_int64 bitpos;
1435 5241030 : location_t loc = EXPR_LOCATION (expr);
1436 :
1437 5241030 : core = get_inner_reference (expr, &bitsize, &bitpos,
1438 : poffset, &mode, &unsignedp, preversep,
1439 : &volatilep);
1440 10482060 : if (!multiple_p (bitsize, BITS_PER_UNIT, pbytesize))
1441 : return NULL_TREE;
1442 5241030 : if (!multiple_p (bitpos, BITS_PER_UNIT, pbytepos))
1443 : return NULL_TREE;
1444 : /* If we are left with MEM[a + CST] strip that and add it to the
1445 : pbytepos and return a. */
1446 5241030 : if (TREE_CODE (core) == MEM_REF)
1447 : {
1448 1217618 : poly_offset_int tem;
1449 1217618 : tem = wi::to_poly_offset (TREE_OPERAND (core, 1));
1450 1217618 : tem += *pbytepos;
1451 1217618 : if (tem.to_shwi (pbytepos))
1452 1215692 : return TREE_OPERAND (core, 0);
1453 : }
1454 4025338 : core = build_fold_addr_expr_loc (loc, core);
1455 4025338 : STRIP_NOPS (core);
1456 4025338 : return core;
1457 : }
1458 :
1459 : /* Returns a new src based on the
1460 : copy `DEST = SRC` and for the old SRC2.
1461 : Returns null if SRC2 is not related to DEST. */
1462 :
1463 : static tree
1464 1226498 : new_src_based_on_copy (tree src2, tree dest, tree src)
1465 : {
1466 : /* If the second src is not exactly the same as dest,
1467 : try to handle it seperately; see it is address/size equivalent.
1468 : Handles `a` and `a.b` and `MEM<char[N]>(&a)` which all have
1469 : the same size and offsets as address/size equivalent.
1470 : This allows copying over a memcpy and also one for copying
1471 : where one field is the same size as the whole struct. */
1472 1226498 : if (operand_equal_p (dest, src2))
1473 : return src;
1474 : /* if both dest and src2 are decls, then we know these 2
1475 : accesses can't be the same. */
1476 709092 : if (DECL_P (dest) && DECL_P (src2))
1477 : return NULL_TREE;
1478 : /* A VCE can't be used with imag/real or BFR so reject them early. */
1479 375948 : if (TREE_CODE (src) == IMAGPART_EXPR
1480 375948 : || TREE_CODE (src) == REALPART_EXPR
1481 375948 : || TREE_CODE (src) == BIT_FIELD_REF)
1482 : return NULL_TREE;
1483 375948 : tree core1, core2;
1484 375948 : poly_int64 bytepos1, bytepos2;
1485 375948 : poly_int64 bytesize1, bytesize2;
1486 375948 : tree toffset1, toffset2;
1487 375948 : int reversep1 = 0;
1488 375948 : int reversep2 = 0;
1489 375948 : poly_int64 diff = 0;
1490 375948 : core1 = split_core_and_offset_size (dest, &bytesize1, &bytepos1,
1491 : &toffset1, &reversep1);
1492 375948 : core2 = split_core_and_offset_size (src2, &bytesize2, &bytepos2,
1493 : &toffset2, &reversep2);
1494 375948 : if (!core1 || !core2)
1495 : return NULL_TREE;
1496 375948 : if (reversep1 != reversep2)
1497 : return NULL_TREE;
1498 : /* The sizes of the 2 accesses need to be the same. */
1499 375948 : if (!known_eq (bytesize1, bytesize2))
1500 : return NULL_TREE;
1501 167429 : if (!operand_equal_p (core1, core2, 0))
1502 : return NULL_TREE;
1503 :
1504 22485 : if (toffset1 && toffset2)
1505 : {
1506 2 : tree type = TREE_TYPE (toffset1);
1507 2 : if (type != TREE_TYPE (toffset2))
1508 0 : toffset2 = fold_convert (type, toffset2);
1509 :
1510 2 : tree tdiff = fold_build2 (MINUS_EXPR, type, toffset1, toffset2);
1511 2 : if (!cst_and_fits_in_hwi (tdiff))
1512 : return NULL_TREE;
1513 :
1514 0 : diff = int_cst_value (tdiff);
1515 0 : }
1516 22483 : else if (toffset1 || toffset2)
1517 : {
1518 : /* If only one of the offsets is non-constant, the difference cannot
1519 : be a constant. */
1520 : return NULL_TREE;
1521 : }
1522 22451 : diff += bytepos1 - bytepos2;
1523 : /* The offset between the 2 need to be 0. */
1524 22451 : if (!known_eq (diff, 0))
1525 : return NULL_TREE;
1526 21638 : return fold_build1 (VIEW_CONVERT_EXPR,TREE_TYPE (src2), src);
1527 : }
1528 :
1529 : /* Returns true if SRC and DEST are the same address such that
1530 : `SRC == DEST;` is considered a nop. This is more than an
1531 : operand_equal_p check as it needs to be similar to
1532 : new_src_based_on_copy. */
1533 :
1534 : static bool
1535 4367156 : same_for_assignment (tree src, tree dest)
1536 : {
1537 4367156 : if (operand_equal_p (dest, src, 0))
1538 : return true;
1539 : /* if both dest and src2 are decls, then we know these 2
1540 : accesses can't be the same. */
1541 4364262 : if (DECL_P (dest) && DECL_P (src))
1542 : return false;
1543 :
1544 2244567 : tree core1, core2;
1545 2244567 : poly_int64 bytepos1, bytepos2;
1546 2244567 : poly_int64 bytesize1, bytesize2;
1547 2244567 : tree toffset1, toffset2;
1548 2244567 : int reversep1 = 0;
1549 2244567 : int reversep2 = 0;
1550 2244567 : poly_int64 diff = 0;
1551 2244567 : core1 = split_core_and_offset_size (dest, &bytesize1, &bytepos1,
1552 : &toffset1, &reversep1);
1553 2244567 : core2 = split_core_and_offset_size (src, &bytesize2, &bytepos2,
1554 : &toffset2, &reversep2);
1555 2244567 : if (!core1 || !core2)
1556 : return false;
1557 2244567 : if (reversep1 != reversep2)
1558 : return false;
1559 : /* The sizes of the 2 accesses need to be the same. */
1560 2244567 : if (!known_eq (bytesize1, bytesize2))
1561 : return false;
1562 2243646 : if (!operand_equal_p (core1, core2, 0))
1563 : return false;
1564 6029 : if (toffset1 && toffset2)
1565 : {
1566 313 : tree type = TREE_TYPE (toffset1);
1567 313 : if (type != TREE_TYPE (toffset2))
1568 0 : toffset2 = fold_convert (type, toffset2);
1569 :
1570 313 : tree tdiff = fold_build2 (MINUS_EXPR, type, toffset1, toffset2);
1571 313 : if (!cst_and_fits_in_hwi (tdiff))
1572 : return false;
1573 :
1574 0 : diff = int_cst_value (tdiff);
1575 0 : }
1576 5716 : else if (toffset1 || toffset2)
1577 : {
1578 : /* If only one of the offsets is non-constant, the difference cannot
1579 : be a constant. */
1580 : return false;
1581 : }
1582 5716 : diff += bytepos1 - bytepos2;
1583 : /* The offset between the 2 need to be 0. */
1584 5716 : if (!known_eq (diff, 0))
1585 : return false;
1586 : return true;
1587 : }
1588 :
1589 : /* Helper function for optimize_agr_copyprop.
1590 : For aggregate copies in USE_STMT, see if DEST
1591 : is on the lhs of USE_STMT and replace it with SRC. */
1592 : static void
1593 995223 : optimize_agr_copyprop_1 (gimple *stmt, gimple *use_stmt,
1594 : tree dest, tree src)
1595 : {
1596 995223 : gcc_assert (gimple_assign_load_p (use_stmt)
1597 : && gimple_store_p (use_stmt));
1598 1990446 : if (gimple_has_volatile_ops (use_stmt))
1599 604166 : return;
1600 995222 : tree dest2 = gimple_assign_lhs (use_stmt);
1601 995222 : tree src2 = gimple_assign_rhs1 (use_stmt);
1602 : /* If the new store is `src2 = src2;` skip over it. */
1603 995222 : if (same_for_assignment (src2, dest2))
1604 : return;
1605 994659 : src = new_src_based_on_copy (src2, dest, src);
1606 994659 : if (!src)
1607 : return;
1608 : /* For 2 memory refences and using a temporary to do the copy,
1609 : don't remove the temporary as the 2 memory references might overlap.
1610 : Note t does not need to be decl as it could be field.
1611 : See PR 22237 for full details.
1612 : E.g.
1613 : t = *a; #DEST = SRC;
1614 : *b = t; #DEST2 = SRC2;
1615 : Cannot be convert into
1616 : t = *a;
1617 : *b = *a;
1618 : Though the following is allowed to be done:
1619 : t = *a;
1620 : *a = t;
1621 : And convert it into:
1622 : t = *a;
1623 : *a = *a;
1624 : */
1625 419196 : if (!operand_equal_p (dest2, src, 0)
1626 419196 : && !DECL_P (dest2) && !DECL_P (src))
1627 : {
1628 : /* If *a and *b have the same base see if
1629 : the offset between the two is greater than
1630 : or equal to the size of the type. */
1631 31401 : poly_int64 offset1, offset2;
1632 31401 : tree len = TYPE_SIZE_UNIT (TREE_TYPE (src));
1633 31401 : if (len == NULL_TREE
1634 31401 : || !tree_fits_poly_int64_p (len))
1635 28139 : return;
1636 31401 : tree base1 = get_addr_base_and_unit_offset (dest2, &offset1);
1637 31401 : tree base2 = get_addr_base_and_unit_offset (src, &offset2);
1638 31401 : poly_int64 size = tree_to_poly_int64 (len);
1639 : /* If the bases are 2 different decls,
1640 : then there can be no overlapping. */
1641 31401 : if (base1 && base2
1642 30596 : && DECL_P (base1) && DECL_P (base2)
1643 1858 : && base1 != base2)
1644 : ;
1645 : /* If we can't figure out the base or the bases are
1646 : not equal then fall back to an alignment check. */
1647 29767 : else if (!base1
1648 29767 : || !base2
1649 29767 : || !operand_equal_p (base1, base2))
1650 : {
1651 29400 : unsigned int align1 = get_object_alignment (src);
1652 29400 : unsigned int align2 = get_object_alignment (dest2);
1653 29400 : align1 /= BITS_PER_UNIT;
1654 29400 : align2 /= BITS_PER_UNIT;
1655 : /* If the alignment of either object is less
1656 : than the size then there is a possibility
1657 : of overlapping. */
1658 29400 : if (maybe_lt (align1, size)
1659 29400 : || maybe_lt (align2, size))
1660 28139 : return;
1661 : }
1662 : /* Make sure [offset1, offset1 + len - 1] does
1663 : not overlap with [offset2, offset2 + len - 1],
1664 : it is ok if they are at the same location though. */
1665 367 : else if (ranges_maybe_overlap_p (offset1, size, offset2, size)
1666 367 : && !known_eq (offset2, offset1))
1667 : return;
1668 : }
1669 :
1670 391057 : if (dump_file && (dump_flags & TDF_DETAILS))
1671 : {
1672 11 : fprintf (dump_file, "Simplified\n ");
1673 11 : print_gimple_stmt (dump_file, use_stmt, 0, dump_flags);
1674 11 : fprintf (dump_file, "after previous\n ");
1675 11 : print_gimple_stmt (dump_file, stmt, 0, dump_flags);
1676 : }
1677 391057 : gimple *orig_stmt = use_stmt;
1678 391057 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
1679 391057 : gimple_assign_set_rhs_from_tree (&gsi, unshare_expr (src));
1680 391057 : update_stmt (use_stmt);
1681 :
1682 391057 : if (dump_file && (dump_flags & TDF_DETAILS))
1683 : {
1684 11 : fprintf (dump_file, "into\n ");
1685 11 : print_gimple_stmt (dump_file, use_stmt, 0, dump_flags);
1686 : }
1687 391057 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, use_stmt))
1688 0 : bitmap_set_bit (to_purge, gimple_bb (stmt)->index);
1689 391057 : statistics_counter_event (cfun, "copy prop for aggregate", 1);
1690 : }
1691 :
1692 : /* Helper function for optimize_agr_copyprop_1, propagate aggregates
1693 : into the arguments of USE_STMT if the argument matches with DEST;
1694 : replacing it with SRC. */
1695 : static void
1696 692032 : optimize_agr_copyprop_arg (gimple *defstmt, gcall *call,
1697 : tree dest, tree src)
1698 : {
1699 692032 : bool changed = false;
1700 2313064 : for (unsigned arg = 0; arg < gimple_call_num_args (call); arg++)
1701 : {
1702 1621032 : tree *argptr = gimple_call_arg_ptr (call, arg);
1703 3058090 : if (TREE_CODE (*argptr) == SSA_NAME
1704 915258 : || is_gimple_min_invariant (*argptr)
1705 1805006 : || TYPE_VOLATILE (TREE_TYPE (*argptr)))
1706 1437058 : continue;
1707 183974 : tree newsrc = new_src_based_on_copy (*argptr, dest, src);
1708 183974 : if (!newsrc)
1709 111909 : continue;
1710 :
1711 72065 : if (dump_file && (dump_flags & TDF_DETAILS))
1712 : {
1713 9 : fprintf (dump_file, "Simplified\n ");
1714 9 : print_gimple_stmt (dump_file, call, 0, dump_flags);
1715 9 : fprintf (dump_file, "after previous\n ");
1716 9 : print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
1717 : }
1718 72065 : *argptr = unshare_expr (newsrc);
1719 72065 : changed = true;
1720 72065 : if (dump_file && (dump_flags & TDF_DETAILS))
1721 : {
1722 9 : fprintf (dump_file, "into\n ");
1723 9 : print_gimple_stmt (dump_file, call, 0, dump_flags);
1724 : }
1725 : }
1726 692032 : if (changed)
1727 71895 : update_stmt (call);
1728 692032 : }
1729 :
1730 : /* Helper function for optimize_agr_copyprop, propagate aggregates
1731 : into the return stmt USE if the operand of the return matches DEST;
1732 : replacing it with SRC. */
1733 : static void
1734 120698 : optimize_agr_copyprop_return (gimple *defstmt, greturn *use,
1735 : tree dest, tree src)
1736 : {
1737 120698 : tree rvalue = gimple_return_retval (use);
1738 120698 : if (!rvalue
1739 77996 : || TREE_CODE (rvalue) == SSA_NAME
1740 69387 : || is_gimple_min_invariant (rvalue)
1741 189690 : || TYPE_VOLATILE (TREE_TYPE (rvalue)))
1742 51707 : return;
1743 :
1744 : /* `return <retval>;` is already the best it could be.
1745 : Likewise `return *<retval>_N(D)`. */
1746 68991 : if (TREE_CODE (rvalue) == RESULT_DECL
1747 68991 : || (TREE_CODE (rvalue) == MEM_REF
1748 0 : && TREE_CODE (TREE_OPERAND (rvalue, 0)) == SSA_NAME
1749 0 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (rvalue, 0)))
1750 : == RESULT_DECL))
1751 : return;
1752 47865 : tree newsrc = new_src_based_on_copy (rvalue, dest, src);
1753 47865 : if (!newsrc)
1754 : return;
1755 : /* Currently only support non-global vars.
1756 : See PR 124099 on enumtls not supporting expanding for GIMPLE_RETURN.
1757 : FIXME: could support VCEs too? */
1758 47783 : if (!VAR_P (newsrc) || is_global_var (newsrc))
1759 : return;
1760 22781 : if (dump_file && (dump_flags & TDF_DETAILS))
1761 : {
1762 1 : fprintf (dump_file, "Simplified\n ");
1763 1 : print_gimple_stmt (dump_file, use, 0, dump_flags);
1764 1 : fprintf (dump_file, "after previous\n ");
1765 1 : print_gimple_stmt (dump_file, defstmt, 0, dump_flags);
1766 : }
1767 22781 : gimple_return_set_retval (use, newsrc);
1768 22781 : if (dump_file && (dump_flags & TDF_DETAILS))
1769 : {
1770 1 : fprintf (dump_file, "into\n ");
1771 1 : print_gimple_stmt (dump_file, use, 0, dump_flags);
1772 : }
1773 22781 : update_stmt (use);
1774 : }
1775 :
1776 : /* Optimizes
1777 : DEST = SRC;
1778 : DEST2 = DEST; # DEST2 = SRC2;
1779 : into
1780 : DEST = SRC;
1781 : DEST2 = SRC;
1782 : STMT is the first statement and SRC is the common
1783 : between the statements.
1784 :
1785 : Also optimizes:
1786 : DEST = SRC;
1787 : call_func(..., DEST, ...);
1788 : into:
1789 : DEST = SRC;
1790 : call_func(..., SRC, ...);
1791 :
1792 : */
1793 : static void
1794 3780716 : optimize_agr_copyprop (gimple *stmt)
1795 : {
1796 7561432 : if (gimple_has_volatile_ops (stmt))
1797 411359 : return;
1798 :
1799 : /* Can't prop if the statement could throw. */
1800 3779577 : if (stmt_could_throw_p (cfun, stmt))
1801 : return;
1802 :
1803 3371934 : tree dest = gimple_assign_lhs (stmt);
1804 3371934 : tree src = gimple_assign_rhs1 (stmt);
1805 : /* If the statement is `src = src;` then ignore it. */
1806 3371934 : if (same_for_assignment (dest, src))
1807 : return;
1808 :
1809 3369357 : tree vdef = gimple_vdef (stmt);
1810 3369357 : imm_use_iterator iter;
1811 3369357 : gimple *use_stmt;
1812 13131162 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, vdef)
1813 : {
1814 6392448 : if (gimple_assign_load_p (use_stmt)
1815 6392448 : && gimple_store_p (use_stmt))
1816 995223 : optimize_agr_copyprop_1 (stmt, use_stmt, dest, src);
1817 5397225 : else if (is_gimple_call (use_stmt))
1818 692032 : optimize_agr_copyprop_arg (stmt, as_a<gcall*>(use_stmt), dest, src);
1819 4705193 : else if (is_a<greturn*> (use_stmt))
1820 120698 : optimize_agr_copyprop_return (stmt, as_a<greturn*>(use_stmt), dest, src);
1821 3369357 : }
1822 : }
1823 :
1824 : /* Simple DSE of the lhs from a clobber STMT.
1825 : This is used mostly to clean up from optimize_agr_copyprop and
1826 : to remove (exactly one) extra copy that might later on confuse SRA.
1827 : An example is:
1828 : ;; write to a and such.
1829 : b = a; // This statement is to be removed
1830 : b = {CLOBBER};
1831 : SRA will totally scalarize b (which means also a) here for the extra copy
1832 : which is not something welcomed. So removing the copy will
1833 : allow SRA to move the scalarization of a further down or not at all.
1834 : */
1835 : static void
1836 6987907 : do_simple_agr_dse (gassign *stmt, bool full_walk)
1837 : {
1838 : /* Don't do this while in -Og as we want to keep around the copy
1839 : for debuggability. */
1840 6987907 : if (optimize_debug)
1841 4840466 : return;
1842 6984514 : ao_ref read;
1843 6984514 : basic_block bb = gimple_bb (stmt);
1844 6984514 : tree lhs = gimple_assign_lhs (stmt);
1845 : /* Only handle clobbers of a full decl. */
1846 6984514 : if (!DECL_P (lhs))
1847 : return;
1848 6285026 : ao_ref_init (&read, lhs);
1849 6285026 : tree vuse = gimple_vuse (stmt);
1850 6285026 : unsigned limit = full_walk ? param_sccvn_max_alias_queries_per_access : 4;
1851 16051212 : while (limit)
1852 : {
1853 16038512 : gimple *ostmt = SSA_NAME_DEF_STMT (vuse);
1854 : /* Don't handle phis, just declare to be done. */
1855 16038512 : if (is_a<gphi*>(ostmt) || gimple_nop_p (ostmt))
1856 : break;
1857 13903771 : basic_block obb = gimple_bb (ostmt);
1858 : /* If the clobber is not fully dominating the statement define,
1859 : then it is not "simple" to detect if the define is fully clobbered. */
1860 13903771 : if (obb != bb && !dominated_by_p (CDI_DOMINATORS, bb, obb))
1861 4137585 : return;
1862 13903771 : gimple *use_stmt;
1863 13903771 : imm_use_iterator iter;
1864 56245110 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, gimple_vdef (ostmt))
1865 : {
1866 16399597 : basic_block ubb = gimple_bb (use_stmt);
1867 16399597 : if (stmt == use_stmt)
1868 4787684 : continue;
1869 : /* If the use is a clobber for lhs,
1870 : then it can be safely skipped; this happens with eh
1871 : and sometimes jump threading. */
1872 11611913 : if (gimple_clobber_p (use_stmt)
1873 11611913 : && lhs == gimple_assign_lhs (use_stmt))
1874 159682 : continue;
1875 : /* If the use is a phi and it is single use then check if that single use
1876 : is a clobber and lhs is the same. */
1877 11452231 : if (gphi *use_phi = dyn_cast<gphi*>(use_stmt))
1878 : {
1879 340949 : use_operand_p ou;
1880 340949 : gimple *ostmt;
1881 340949 : if (single_imm_use (gimple_phi_result (use_phi), &ou, &ostmt)
1882 283171 : && gimple_clobber_p (ostmt)
1883 565939 : && lhs == gimple_assign_lhs (ostmt))
1884 66360 : continue;
1885 : /* A phi node will never be dominating the clobber. */
1886 274589 : return;
1887 : }
1888 : /* The use needs to be dominating the clobber. */
1889 1394575 : if ((ubb != bb && !dominated_by_p (CDI_DOMINATORS, bb, ubb))
1890 11819594 : || ref_maybe_used_by_stmt_p (use_stmt, &read, false))
1891 1149333 : return;
1892 : /* Count the above alias lookup towards the limit. */
1893 9961949 : limit--;
1894 9961949 : if (limit == 0)
1895 : return;
1896 1865800 : }
1897 12037971 : vuse = gimple_vuse (ostmt);
1898 : /* This is a call with an assignment to the clobber decl,
1899 : remove the lhs or the whole stmt if it was pure/const. */
1900 12037971 : if (is_a <gcall*>(ostmt)
1901 12037971 : && lhs == gimple_call_lhs (ostmt))
1902 : {
1903 : /* Don't remove stores/statements that are needed for non-call
1904 : eh to work. */
1905 3926 : if (stmt_unremovable_because_of_non_call_eh_p (cfun, ostmt))
1906 : return;
1907 : /* If we delete a stmt that could throw, mark the block
1908 : in to_purge to cleanup afterwards. */
1909 3920 : if (stmt_could_throw_p (cfun, ostmt))
1910 953 : bitmap_set_bit (to_purge, obb->index);
1911 3920 : int flags = gimple_call_flags (ostmt);
1912 3920 : if ((flags & (ECF_PURE|ECF_CONST|ECF_NOVOPS))
1913 203 : && !(flags & (ECF_LOOPING_CONST_OR_PURE)))
1914 : {
1915 119 : gimple_stmt_iterator gsi = gsi_for_stmt (ostmt);
1916 119 : if (dump_file && (dump_flags & TDF_DETAILS))
1917 : {
1918 14 : fprintf (dump_file, "Removing dead call store stmt ");
1919 14 : print_gimple_stmt (dump_file, ostmt, 0);
1920 14 : fprintf (dump_file, "\n");
1921 : }
1922 119 : unlink_stmt_vdef (ostmt);
1923 119 : release_defs (ostmt);
1924 119 : gsi_remove (&gsi, true);
1925 119 : statistics_counter_event (cfun, "delete call dead store", 1);
1926 : /* Only remove the first store previous statement. */
1927 119 : return;
1928 : }
1929 : /* Make sure we do not remove a return slot we cannot reconstruct
1930 : later. */
1931 3801 : if (gimple_call_return_slot_opt_p (as_a <gcall *>(ostmt))
1932 3801 : && (TREE_ADDRESSABLE (TREE_TYPE (gimple_call_fntype (ostmt)))
1933 451 : || !poly_int_tree_p
1934 451 : (TYPE_SIZE (TREE_TYPE (gimple_call_fntype (ostmt))))))
1935 : return;
1936 567 : if (dump_file && (dump_flags & TDF_DETAILS))
1937 : {
1938 6 : fprintf (dump_file, "Removing lhs of call stmt ");
1939 6 : print_gimple_stmt (dump_file, ostmt, 0);
1940 6 : fprintf (dump_file, "\n");
1941 : }
1942 567 : gimple_call_set_lhs (ostmt, NULL_TREE);
1943 567 : update_stmt (ostmt);
1944 567 : statistics_counter_event (cfun, "removed lhs call", 1);
1945 567 : return;
1946 : }
1947 : /* This an assignment store to the clobbered decl,
1948 : then maybe remove it. */
1949 12034045 : if (is_a <gassign*>(ostmt)
1950 10172055 : && gimple_store_p (ostmt)
1951 10172055 : && !gimple_clobber_p (ostmt)
1952 15063604 : && lhs == gimple_assign_lhs (ostmt))
1953 : {
1954 : /* Don't remove stores/statements that are needed for non-call
1955 : eh to work. */
1956 152369 : if (stmt_unremovable_because_of_non_call_eh_p (cfun, ostmt))
1957 : return;
1958 : /* If we delete a stmt that could throw, mark the block
1959 : in to_purge to cleanup afterwards. */
1960 147285 : if (stmt_could_throw_p (cfun, ostmt))
1961 0 : bitmap_set_bit (to_purge, obb->index);
1962 147285 : gimple_stmt_iterator gsi = gsi_for_stmt (ostmt);
1963 147285 : if (dump_file && (dump_flags & TDF_DETAILS))
1964 : {
1965 12 : fprintf (dump_file, "Removing dead store stmt ");
1966 12 : print_gimple_stmt (dump_file, ostmt, 0);
1967 12 : fprintf (dump_file, "\n");
1968 : }
1969 147285 : unlink_stmt_vdef (ostmt);
1970 147285 : release_defs (ostmt);
1971 147285 : gsi_remove (&gsi, true);
1972 147285 : statistics_counter_event (cfun, "delete dead store", 1);
1973 : /* Only remove the first store previous statement. */
1974 147285 : return;
1975 : }
1976 : /* If the statement uses or maybe writes to the decl,
1977 : then nothing is to be removed. Don't know if the write
1978 : to the decl is partial write or a full one so the need
1979 : to stop.
1980 : e.g.
1981 : b.c = a;
1982 : Easier to stop here rather than do a full partial
1983 : dse of this statement.
1984 : b = {CLOBBER}; */
1985 11881676 : if (stmt_may_clobber_ref_p_1 (ostmt, &read, false)
1986 11881676 : || ref_maybe_used_by_stmt_p (ostmt, &read, false))
1987 2115490 : return;
1988 9766186 : limit--;
1989 : }
1990 : }
1991 :
1992 : /* Optimizes builtin memcmps for small constant sizes.
1993 : GSI_P is the GSI for the call. STMT is the call itself.
1994 : */
1995 :
1996 : static bool
1997 466872 : simplify_builtin_memcmp (gimple_stmt_iterator *gsi_p, gcall *stmt)
1998 : {
1999 : /* Make sure memcmp arguments are the correct type. */
2000 466872 : if (gimple_call_num_args (stmt) != 3)
2001 : return false;
2002 466872 : tree arg1 = gimple_call_arg (stmt, 0);
2003 466872 : tree arg2 = gimple_call_arg (stmt, 1);
2004 466872 : tree len = gimple_call_arg (stmt, 2);
2005 :
2006 466872 : if (!POINTER_TYPE_P (TREE_TYPE (arg1)))
2007 : return false;
2008 466872 : if (!POINTER_TYPE_P (TREE_TYPE (arg2)))
2009 : return false;
2010 466872 : if (!INTEGRAL_TYPE_P (TREE_TYPE (len)))
2011 : return false;
2012 :
2013 : /* The return value of the memcmp has to be used
2014 : equality comparison to zero. */
2015 466872 : tree res = gimple_call_lhs (stmt);
2016 :
2017 466872 : if (!res || !use_in_zero_equality (res))
2018 14226 : return false;
2019 :
2020 452646 : unsigned HOST_WIDE_INT leni;
2021 :
2022 452646 : if (tree_fits_uhwi_p (len)
2023 631560 : && (leni = tree_to_uhwi (len)) <= GET_MODE_SIZE (word_mode)
2024 533349 : && pow2p_hwi (leni))
2025 : {
2026 19111 : leni *= CHAR_TYPE_SIZE;
2027 19111 : unsigned align1 = get_pointer_alignment (arg1);
2028 19111 : unsigned align2 = get_pointer_alignment (arg2);
2029 19111 : unsigned align = MIN (align1, align2);
2030 19111 : scalar_int_mode mode;
2031 19111 : if (int_mode_for_size (leni, 1).exists (&mode)
2032 19111 : && (align >= leni || !targetm.slow_unaligned_access (mode, align)))
2033 : {
2034 19111 : location_t loc = gimple_location (stmt);
2035 19111 : tree type, off;
2036 19111 : type = build_nonstandard_integer_type (leni, 1);
2037 38222 : gcc_assert (known_eq (GET_MODE_BITSIZE (TYPE_MODE (type)), leni));
2038 19111 : tree ptrtype = build_pointer_type_for_mode (char_type_node,
2039 : ptr_mode, true);
2040 19111 : off = build_int_cst (ptrtype, 0);
2041 :
2042 : /* Create unaligned types if needed. */
2043 19111 : tree type1 = type, type2 = type;
2044 19111 : if (TYPE_ALIGN (type1) > align1)
2045 7794 : type1 = build_aligned_type (type1, align1);
2046 19111 : if (TYPE_ALIGN (type2) > align2)
2047 8292 : type2 = build_aligned_type (type2, align2);
2048 :
2049 19111 : arg1 = build2_loc (loc, MEM_REF, type1, arg1, off);
2050 19111 : arg2 = build2_loc (loc, MEM_REF, type2, arg2, off);
2051 19111 : tree tem1 = fold_const_aggregate_ref (arg1);
2052 19111 : if (tem1)
2053 219 : arg1 = tem1;
2054 19111 : tree tem2 = fold_const_aggregate_ref (arg2);
2055 19111 : if (tem2)
2056 7464 : arg2 = tem2;
2057 19111 : res = fold_convert_loc (loc, TREE_TYPE (res),
2058 : fold_build2_loc (loc, NE_EXPR,
2059 : boolean_type_node,
2060 : arg1, arg2));
2061 19111 : gimplify_and_update_call_from_tree (gsi_p, res);
2062 19111 : return true;
2063 : }
2064 : }
2065 :
2066 : /* Replace memcmp with memcmp_eq if the above fails. */
2067 433535 : if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) == BUILT_IN_MEMCMP_EQ)
2068 : return false;
2069 343362 : if (!fold_before_rtl_expansion_p ())
2070 : return false;
2071 90173 : gimple_call_set_fndecl (stmt, builtin_decl_explicit (BUILT_IN_MEMCMP_EQ));
2072 90173 : update_stmt (stmt);
2073 90173 : return true;
2074 : }
2075 :
2076 : /* Optimizes builtin memchrs for small constant sizes with a const string.
2077 : GSI_P is the GSI for the call. STMT is the call itself.
2078 : */
2079 :
2080 : static bool
2081 13801 : simplify_builtin_memchr (gimple_stmt_iterator *gsi_p, gcall *stmt)
2082 : {
2083 13801 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
2084 : return false;
2085 :
2086 13801 : if (gimple_call_num_args (stmt) != 3)
2087 : return false;
2088 :
2089 13801 : tree res = gimple_call_lhs (stmt);
2090 13801 : if (!res || !use_in_zero_equality (res))
2091 12360 : return false;
2092 :
2093 1441 : tree ptr = gimple_call_arg (stmt, 0);
2094 1441 : if (TREE_CODE (ptr) != ADDR_EXPR
2095 1441 : || TREE_CODE (TREE_OPERAND (ptr, 0)) != STRING_CST)
2096 : return false;
2097 :
2098 434 : unsigned HOST_WIDE_INT slen
2099 434 : = TREE_STRING_LENGTH (TREE_OPERAND (ptr, 0));
2100 : /* It must be a non-empty string constant. */
2101 434 : if (slen < 2)
2102 : return false;
2103 :
2104 : /* For -Os, only simplify strings with a single character. */
2105 430 : if (!optimize_bb_for_speed_p (gimple_bb (stmt))
2106 430 : && slen > 2)
2107 : return false;
2108 :
2109 414 : tree size = gimple_call_arg (stmt, 2);
2110 : /* Size must be a constant which is <= UNITS_PER_WORD and
2111 : <= the string length. */
2112 414 : if (!tree_fits_uhwi_p (size))
2113 : return false;
2114 :
2115 414 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (size);
2116 415 : if (sz == 0 || sz > UNITS_PER_WORD || sz >= slen)
2117 : return false;
2118 :
2119 362 : tree ch = gimple_call_arg (stmt, 1);
2120 362 : location_t loc = gimple_location (stmt);
2121 362 : if (!useless_type_conversion_p (char_type_node,
2122 362 : TREE_TYPE (ch)))
2123 362 : ch = fold_convert_loc (loc, char_type_node, ch);
2124 362 : const char *p = TREE_STRING_POINTER (TREE_OPERAND (ptr, 0));
2125 362 : unsigned int isize = sz;
2126 362 : tree *op = XALLOCAVEC (tree, isize);
2127 1305 : for (unsigned int i = 0; i < isize; i++)
2128 : {
2129 943 : op[i] = build_int_cst (char_type_node, p[i]);
2130 943 : op[i] = fold_build2_loc (loc, EQ_EXPR, boolean_type_node,
2131 : op[i], ch);
2132 : }
2133 943 : for (unsigned int i = isize - 1; i >= 1; i--)
2134 581 : op[i - 1] = fold_convert_loc (loc, boolean_type_node,
2135 : fold_build2_loc (loc,
2136 : BIT_IOR_EXPR,
2137 : boolean_type_node,
2138 581 : op[i - 1],
2139 581 : op[i]));
2140 362 : res = fold_convert_loc (loc, TREE_TYPE (res), op[0]);
2141 362 : gimplify_and_update_call_from_tree (gsi_p, res);
2142 362 : return true;
2143 : }
2144 :
2145 : /* *GSI_P is a GIMPLE_CALL to a builtin function.
2146 : Optimize
2147 : memcpy (p, "abcd", 4); // STMT1
2148 : memset (p + 4, ' ', 3); // STMT2
2149 : into
2150 : memcpy (p, "abcd ", 7);
2151 : call if the latter can be stored by pieces during expansion.
2152 : */
2153 :
2154 : static bool
2155 109572 : simplify_builtin_memcpy_memset (gimple_stmt_iterator *gsi_p, gcall *stmt2)
2156 : {
2157 109572 : if (gimple_call_num_args (stmt2) != 3
2158 109572 : || gimple_call_lhs (stmt2)
2159 : || CHAR_BIT != 8
2160 109572 : || BITS_PER_UNIT != 8)
2161 : return false;
2162 :
2163 209241 : tree vuse = gimple_vuse (stmt2);
2164 102011 : if (vuse == NULL)
2165 : return false;
2166 101997 : gimple *stmt1 = SSA_NAME_DEF_STMT (vuse);
2167 :
2168 101997 : tree callee1;
2169 101997 : tree ptr1, src1, str1, off1, len1, lhs1;
2170 101997 : tree ptr2 = gimple_call_arg (stmt2, 0);
2171 101997 : tree val2 = gimple_call_arg (stmt2, 1);
2172 101997 : tree len2 = gimple_call_arg (stmt2, 2);
2173 101997 : tree diff, vdef, new_str_cst;
2174 101997 : gimple *use_stmt;
2175 101997 : unsigned int ptr1_align;
2176 101997 : unsigned HOST_WIDE_INT src_len;
2177 101997 : char *src_buf;
2178 101997 : use_operand_p use_p;
2179 :
2180 101997 : if (!tree_fits_shwi_p (val2)
2181 97982 : || !tree_fits_uhwi_p (len2)
2182 164681 : || compare_tree_int (len2, 1024) == 1)
2183 44396 : return false;
2184 :
2185 57601 : if (is_gimple_call (stmt1))
2186 : {
2187 : /* If first stmt is a call, it needs to be memcpy
2188 : or mempcpy, with string literal as second argument and
2189 : constant length. */
2190 29952 : callee1 = gimple_call_fndecl (stmt1);
2191 29952 : if (callee1 == NULL_TREE
2192 29836 : || !fndecl_built_in_p (callee1, BUILT_IN_NORMAL)
2193 56290 : || gimple_call_num_args (stmt1) != 3)
2194 : return false;
2195 25040 : if (DECL_FUNCTION_CODE (callee1) != BUILT_IN_MEMCPY
2196 25040 : && DECL_FUNCTION_CODE (callee1) != BUILT_IN_MEMPCPY)
2197 : return false;
2198 10865 : ptr1 = gimple_call_arg (stmt1, 0);
2199 10865 : src1 = gimple_call_arg (stmt1, 1);
2200 10865 : len1 = gimple_call_arg (stmt1, 2);
2201 10865 : lhs1 = gimple_call_lhs (stmt1);
2202 10865 : if (!tree_fits_uhwi_p (len1))
2203 : return false;
2204 10778 : str1 = string_constant (src1, &off1, NULL, NULL);
2205 10778 : if (str1 == NULL_TREE)
2206 : return false;
2207 4873 : if (!tree_fits_uhwi_p (off1)
2208 4873 : || compare_tree_int (off1, TREE_STRING_LENGTH (str1) - 1) > 0
2209 4873 : || compare_tree_int (len1, TREE_STRING_LENGTH (str1)
2210 4873 : - tree_to_uhwi (off1)) > 0
2211 4873 : || TREE_CODE (TREE_TYPE (str1)) != ARRAY_TYPE
2212 14619 : || TYPE_MODE (TREE_TYPE (TREE_TYPE (str1)))
2213 4873 : != TYPE_MODE (char_type_node))
2214 0 : return false;
2215 : }
2216 27649 : else if (gimple_assign_single_p (stmt1))
2217 : {
2218 : /* Otherwise look for length 1 memcpy optimized into
2219 : assignment. */
2220 16902 : ptr1 = gimple_assign_lhs (stmt1);
2221 16902 : src1 = gimple_assign_rhs1 (stmt1);
2222 16902 : if (TREE_CODE (ptr1) != MEM_REF
2223 3509 : || TYPE_MODE (TREE_TYPE (ptr1)) != TYPE_MODE (char_type_node)
2224 17936 : || !tree_fits_shwi_p (src1))
2225 16550 : return false;
2226 352 : ptr1 = build_fold_addr_expr (ptr1);
2227 352 : STRIP_USELESS_TYPE_CONVERSION (ptr1);
2228 352 : callee1 = NULL_TREE;
2229 352 : len1 = size_one_node;
2230 352 : lhs1 = NULL_TREE;
2231 352 : off1 = size_zero_node;
2232 352 : str1 = NULL_TREE;
2233 : }
2234 : else
2235 : return false;
2236 :
2237 5225 : diff = constant_pointer_difference (ptr1, ptr2);
2238 5225 : if (diff == NULL && lhs1 != NULL)
2239 : {
2240 7 : diff = constant_pointer_difference (lhs1, ptr2);
2241 7 : if (DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY
2242 7 : && diff != NULL)
2243 7 : diff = size_binop (PLUS_EXPR, diff,
2244 : fold_convert (sizetype, len1));
2245 : }
2246 : /* If the difference between the second and first destination pointer
2247 : is not constant, or is bigger than memcpy length, bail out. */
2248 5225 : if (diff == NULL
2249 4342 : || !tree_fits_uhwi_p (diff)
2250 4342 : || tree_int_cst_lt (len1, diff)
2251 9323 : || compare_tree_int (diff, 1024) == 1)
2252 1127 : return false;
2253 :
2254 : /* Use maximum of difference plus memset length and memcpy length
2255 : as the new memcpy length, if it is too big, bail out. */
2256 4098 : src_len = tree_to_uhwi (diff);
2257 4098 : src_len += tree_to_uhwi (len2);
2258 4098 : if (src_len < tree_to_uhwi (len1))
2259 : src_len = tree_to_uhwi (len1);
2260 4098 : if (src_len > 1024)
2261 : return false;
2262 :
2263 : /* If mempcpy value is used elsewhere, bail out, as mempcpy
2264 : with bigger length will return different result. */
2265 4098 : if (lhs1 != NULL_TREE
2266 193 : && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY
2267 4105 : && (TREE_CODE (lhs1) != SSA_NAME
2268 7 : || !single_imm_use (lhs1, &use_p, &use_stmt)
2269 7 : || use_stmt != stmt2))
2270 0 : return false;
2271 :
2272 : /* If anything reads memory in between memcpy and memset
2273 : call, the modified memcpy call might change it. */
2274 4098 : vdef = gimple_vdef (stmt1);
2275 4098 : if (vdef != NULL
2276 4098 : && (!single_imm_use (vdef, &use_p, &use_stmt)
2277 3343 : || use_stmt != stmt2))
2278 : return false;
2279 :
2280 3343 : ptr1_align = get_pointer_alignment (ptr1);
2281 : /* Construct the new source string literal. */
2282 3343 : src_buf = XALLOCAVEC (char, src_len + 1);
2283 3343 : if (callee1)
2284 3182 : memcpy (src_buf,
2285 3182 : TREE_STRING_POINTER (str1) + tree_to_uhwi (off1),
2286 : tree_to_uhwi (len1));
2287 : else
2288 161 : src_buf[0] = tree_to_shwi (src1);
2289 3343 : memset (src_buf + tree_to_uhwi (diff),
2290 3343 : tree_to_shwi (val2), tree_to_uhwi (len2));
2291 3343 : src_buf[src_len] = '\0';
2292 : /* Neither builtin_strncpy_read_str nor builtin_memcpy_read_str
2293 : handle embedded '\0's. */
2294 3343 : if (strlen (src_buf) != src_len)
2295 : return false;
2296 3253 : rtl_profile_for_bb (gimple_bb (stmt2));
2297 : /* If the new memcpy wouldn't be emitted by storing the literal
2298 : by pieces, this optimization might enlarge .rodata too much,
2299 : as commonly used string literals couldn't be shared any
2300 : longer. */
2301 3253 : if (!can_store_by_pieces (src_len,
2302 : builtin_strncpy_read_str,
2303 : src_buf, ptr1_align, false))
2304 : return false;
2305 :
2306 2469 : new_str_cst = build_string_literal (src_len, src_buf);
2307 2469 : if (callee1)
2308 : {
2309 : /* If STMT1 is a mem{,p}cpy call, adjust it and remove
2310 : memset call. */
2311 2342 : if (lhs1 && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY)
2312 7 : gimple_call_set_lhs (stmt1, NULL_TREE);
2313 2342 : gimple_call_set_arg (stmt1, 1, new_str_cst);
2314 2342 : gimple_call_set_arg (stmt1, 2,
2315 2342 : build_int_cst (TREE_TYPE (len1), src_len));
2316 2342 : update_stmt (stmt1);
2317 2342 : unlink_stmt_vdef (stmt2);
2318 2342 : gsi_replace (gsi_p, gimple_build_nop (), false);
2319 2342 : fwprop_invalidate_lattice (gimple_get_lhs (stmt2));
2320 2342 : release_defs (stmt2);
2321 2342 : if (lhs1 && DECL_FUNCTION_CODE (callee1) == BUILT_IN_MEMPCPY)
2322 : {
2323 7 : fwprop_invalidate_lattice (lhs1);
2324 7 : release_ssa_name (lhs1);
2325 : }
2326 2342 : return true;
2327 : }
2328 : else
2329 : {
2330 : /* Otherwise, if STMT1 is length 1 memcpy optimized into
2331 : assignment, remove STMT1 and change memset call into
2332 : memcpy call. */
2333 127 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt1);
2334 :
2335 127 : if (!is_gimple_val (ptr1))
2336 12 : ptr1 = force_gimple_operand_gsi (gsi_p, ptr1, true, NULL_TREE,
2337 : true, GSI_SAME_STMT);
2338 127 : tree fndecl = builtin_decl_explicit (BUILT_IN_MEMCPY);
2339 127 : gimple_call_set_fndecl (stmt2, fndecl);
2340 127 : gimple_call_set_fntype (stmt2,
2341 127 : TREE_TYPE (fndecl));
2342 127 : gimple_call_set_arg (stmt2, 0, ptr1);
2343 127 : gimple_call_set_arg (stmt2, 1, new_str_cst);
2344 127 : gimple_call_set_arg (stmt2, 2,
2345 127 : build_int_cst (TREE_TYPE (len2), src_len));
2346 127 : unlink_stmt_vdef (stmt1);
2347 127 : gsi_remove (&gsi, true);
2348 127 : fwprop_invalidate_lattice (gimple_get_lhs (stmt1));
2349 127 : release_defs (stmt1);
2350 127 : update_stmt (stmt2);
2351 127 : return false;
2352 : }
2353 : }
2354 :
2355 :
2356 : /* Try to optimize out __builtin_stack_restore. Optimize it out
2357 : if there is another __builtin_stack_restore in the same basic
2358 : block and no calls or ASM_EXPRs are in between, or if this block's
2359 : only outgoing edge is to EXIT_BLOCK and there are no calls or
2360 : ASM_EXPRs after this __builtin_stack_restore.
2361 : Note restore right before a noreturn function is not needed.
2362 : And skip some cheap calls that will most likely become an instruction.
2363 : Restoring the stack before a call is important to be able to keep
2364 : stack usage down so that call does not run out of stack. */
2365 :
2366 :
2367 : static bool
2368 10376 : optimize_stack_restore (gimple_stmt_iterator *gsi, gimple *call)
2369 : {
2370 10376 : if (!fold_before_rtl_expansion_p ())
2371 : return false;
2372 2527 : tree callee;
2373 2527 : gimple *stmt;
2374 :
2375 2527 : basic_block bb = gsi_bb (*gsi);
2376 :
2377 2527 : if (gimple_call_num_args (call) != 1
2378 2527 : || TREE_CODE (gimple_call_arg (call, 0)) != SSA_NAME
2379 5054 : || !POINTER_TYPE_P (TREE_TYPE (gimple_call_arg (call, 0))))
2380 : return false;
2381 :
2382 2527 : gimple_stmt_iterator i = *gsi;
2383 6312 : for (gsi_next (&i); !gsi_end_p (i); gsi_next (&i))
2384 : {
2385 4240 : stmt = gsi_stmt (i);
2386 4240 : if (is_a<gasm*> (stmt))
2387 : return false;
2388 4239 : gcall *call = dyn_cast<gcall*>(stmt);
2389 4239 : if (!call)
2390 3576 : continue;
2391 :
2392 : /* We can remove the restore in front of noreturn
2393 : calls. Since the restore will happen either
2394 : via an unwind/longjmp or not at all. */
2395 663 : if (gimple_call_noreturn_p (call))
2396 : break;
2397 :
2398 : /* Internal calls are ok, to bypass
2399 : check first since fndecl will be null. */
2400 647 : if (gimple_call_internal_p (call))
2401 1 : continue;
2402 :
2403 646 : callee = gimple_call_fndecl (call);
2404 : /* Non-builtin calls are not ok. */
2405 646 : if (!callee
2406 646 : || !fndecl_built_in_p (callee))
2407 : return false;
2408 :
2409 : /* Do not remove stack updates before strub leave. */
2410 570 : if (fndecl_built_in_p (callee, BUILT_IN___STRUB_LEAVE)
2411 : /* Alloca calls are not ok either. */
2412 570 : || fndecl_builtin_alloc_p (callee))
2413 : return false;
2414 :
2415 358 : if (fndecl_built_in_p (callee, BUILT_IN_STACK_RESTORE))
2416 52 : goto second_stack_restore;
2417 :
2418 : /* If not a simple or inexpensive builtin, then it is not ok either. */
2419 306 : if (!is_simple_builtin (callee)
2420 306 : && !is_inexpensive_builtin (callee))
2421 : return false;
2422 : }
2423 :
2424 : /* Allow one successor of the exit block, or zero successors. */
2425 2088 : switch (EDGE_COUNT (bb->succs))
2426 : {
2427 : case 0:
2428 : break;
2429 2001 : case 1:
2430 2001 : if (single_succ_edge (bb)->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
2431 : return false;
2432 : break;
2433 : default:
2434 : return false;
2435 : }
2436 1731 : second_stack_restore:
2437 :
2438 : /* If there's exactly one use, then zap the call to __builtin_stack_save.
2439 : If there are multiple uses, then the last one should remove the call.
2440 : In any case, whether the call to __builtin_stack_save can be removed
2441 : or not is irrelevant to removing the call to __builtin_stack_restore. */
2442 1731 : if (has_single_use (gimple_call_arg (call, 0)))
2443 : {
2444 1561 : gimple *stack_save = SSA_NAME_DEF_STMT (gimple_call_arg (call, 0));
2445 1561 : if (is_gimple_call (stack_save))
2446 : {
2447 1559 : callee = gimple_call_fndecl (stack_save);
2448 1559 : if (callee && fndecl_built_in_p (callee, BUILT_IN_STACK_SAVE))
2449 : {
2450 1559 : gimple_stmt_iterator stack_save_gsi;
2451 1559 : tree rhs;
2452 :
2453 1559 : stack_save_gsi = gsi_for_stmt (stack_save);
2454 1559 : rhs = build_int_cst (TREE_TYPE (gimple_call_arg (call, 0)), 0);
2455 1559 : replace_call_with_value (&stack_save_gsi, rhs);
2456 : }
2457 : }
2458 : }
2459 :
2460 : /* No effect, so the statement will be deleted. */
2461 1731 : replace_call_with_value (gsi, NULL_TREE);
2462 1731 : return true;
2463 : }
2464 :
2465 : /* *GSI_P is a GIMPLE_CALL to a builtin function.
2466 : Optimize
2467 : memcpy (p, "abcd", 4);
2468 : memset (p + 4, ' ', 3);
2469 : into
2470 : memcpy (p, "abcd ", 7);
2471 : call if the latter can be stored by pieces during expansion.
2472 :
2473 : Optimize
2474 : memchr ("abcd", a, 4) == 0;
2475 : or
2476 : memchr ("abcd", a, 4) != 0;
2477 : to
2478 : (a == 'a' || a == 'b' || a == 'c' || a == 'd') == 0
2479 : or
2480 : (a == 'a' || a == 'b' || a == 'c' || a == 'd') != 0
2481 :
2482 : Also canonicalize __atomic_fetch_op (p, x, y) op x
2483 : to __atomic_op_fetch (p, x, y) or
2484 : __atomic_op_fetch (p, x, y) iop x
2485 : to __atomic_fetch_op (p, x, y) when possible (also __sync). */
2486 :
2487 : static bool
2488 6203131 : simplify_builtin_call (gimple_stmt_iterator *gsi_p, tree callee2, bool full_walk)
2489 : {
2490 6203131 : gimple *stmt2 = gsi_stmt (*gsi_p);
2491 6203131 : enum built_in_function other_atomic = END_BUILTINS;
2492 6203131 : enum tree_code atomic_op = ERROR_MARK;
2493 :
2494 6203131 : switch (DECL_FUNCTION_CODE (callee2))
2495 : {
2496 10376 : case BUILT_IN_STACK_RESTORE:
2497 10376 : return optimize_stack_restore (gsi_p, as_a<gcall*>(stmt2));
2498 466872 : case BUILT_IN_MEMCMP:
2499 466872 : case BUILT_IN_MEMCMP_EQ:
2500 466872 : return simplify_builtin_memcmp (gsi_p, as_a<gcall*>(stmt2));
2501 13801 : case BUILT_IN_MEMCHR:
2502 13801 : return simplify_builtin_memchr (gsi_p, as_a<gcall*>(stmt2));
2503 :
2504 109572 : case BUILT_IN_MEMSET:
2505 109572 : if (gimple_call_num_args (stmt2) == 3)
2506 : {
2507 : /* Try to prop the zeroing/value of the memset to memcpy
2508 : if the dest is an address and the value is a constant. */
2509 109572 : optimize_aggr_zeroprop (stmt2, full_walk);
2510 : }
2511 109572 : return simplify_builtin_memcpy_memset (gsi_p, as_a<gcall*>(stmt2));
2512 :
2513 : #define CASE_ATOMIC(NAME, OTHER, OP) \
2514 : case BUILT_IN_##NAME##_1: \
2515 : case BUILT_IN_##NAME##_2: \
2516 : case BUILT_IN_##NAME##_4: \
2517 : case BUILT_IN_##NAME##_8: \
2518 : case BUILT_IN_##NAME##_16: \
2519 : atomic_op = OP; \
2520 : other_atomic \
2521 : = (enum built_in_function) (BUILT_IN_##OTHER##_1 \
2522 : + (DECL_FUNCTION_CODE (callee2) \
2523 : - BUILT_IN_##NAME##_1)); \
2524 : goto handle_atomic_fetch_op;
2525 :
2526 48326 : CASE_ATOMIC (ATOMIC_FETCH_ADD, ATOMIC_ADD_FETCH, PLUS_EXPR)
2527 7125 : CASE_ATOMIC (ATOMIC_FETCH_SUB, ATOMIC_SUB_FETCH, MINUS_EXPR)
2528 2876 : CASE_ATOMIC (ATOMIC_FETCH_AND, ATOMIC_AND_FETCH, BIT_AND_EXPR)
2529 2895 : CASE_ATOMIC (ATOMIC_FETCH_XOR, ATOMIC_XOR_FETCH, BIT_XOR_EXPR)
2530 3823 : CASE_ATOMIC (ATOMIC_FETCH_OR, ATOMIC_OR_FETCH, BIT_IOR_EXPR)
2531 :
2532 2365 : CASE_ATOMIC (SYNC_FETCH_AND_ADD, SYNC_ADD_AND_FETCH, PLUS_EXPR)
2533 2004 : CASE_ATOMIC (SYNC_FETCH_AND_SUB, SYNC_SUB_AND_FETCH, MINUS_EXPR)
2534 1876 : CASE_ATOMIC (SYNC_FETCH_AND_AND, SYNC_AND_AND_FETCH, BIT_AND_EXPR)
2535 2144 : CASE_ATOMIC (SYNC_FETCH_AND_XOR, SYNC_XOR_AND_FETCH, BIT_XOR_EXPR)
2536 1987 : CASE_ATOMIC (SYNC_FETCH_AND_OR, SYNC_OR_AND_FETCH, BIT_IOR_EXPR)
2537 :
2538 14351 : CASE_ATOMIC (ATOMIC_ADD_FETCH, ATOMIC_FETCH_ADD, MINUS_EXPR)
2539 8552 : CASE_ATOMIC (ATOMIC_SUB_FETCH, ATOMIC_FETCH_SUB, PLUS_EXPR)
2540 2380 : CASE_ATOMIC (ATOMIC_XOR_FETCH, ATOMIC_FETCH_XOR, BIT_XOR_EXPR)
2541 :
2542 846 : CASE_ATOMIC (SYNC_ADD_AND_FETCH, SYNC_FETCH_AND_ADD, MINUS_EXPR)
2543 732 : CASE_ATOMIC (SYNC_SUB_AND_FETCH, SYNC_FETCH_AND_SUB, PLUS_EXPR)
2544 800 : CASE_ATOMIC (SYNC_XOR_AND_FETCH, SYNC_FETCH_AND_XOR, BIT_XOR_EXPR)
2545 :
2546 : #undef CASE_ATOMIC
2547 :
2548 103082 : handle_atomic_fetch_op:
2549 103082 : if (gimple_call_num_args (stmt2) >= 2 && gimple_call_lhs (stmt2))
2550 : {
2551 59613 : tree lhs2 = gimple_call_lhs (stmt2), lhsc = lhs2;
2552 59613 : tree arg = gimple_call_arg (stmt2, 1);
2553 59613 : gimple *use_stmt, *cast_stmt = NULL;
2554 59613 : use_operand_p use_p;
2555 59613 : tree ndecl = builtin_decl_explicit (other_atomic);
2556 :
2557 59613 : if (ndecl == NULL_TREE || !single_imm_use (lhs2, &use_p, &use_stmt))
2558 : break;
2559 :
2560 58484 : if (gimple_assign_cast_p (use_stmt))
2561 : {
2562 30999 : cast_stmt = use_stmt;
2563 30999 : lhsc = gimple_assign_lhs (cast_stmt);
2564 30999 : if (lhsc == NULL_TREE
2565 30999 : || !INTEGRAL_TYPE_P (TREE_TYPE (lhsc))
2566 30448 : || (TYPE_PRECISION (TREE_TYPE (lhsc))
2567 30448 : != TYPE_PRECISION (TREE_TYPE (lhs2)))
2568 59921 : || !single_imm_use (lhsc, &use_p, &use_stmt))
2569 : {
2570 2605 : use_stmt = cast_stmt;
2571 2605 : cast_stmt = NULL;
2572 2605 : lhsc = lhs2;
2573 : }
2574 : }
2575 :
2576 58484 : bool ok = false;
2577 58484 : tree oarg = NULL_TREE;
2578 58484 : enum tree_code ccode = ERROR_MARK;
2579 58484 : tree crhs1 = NULL_TREE, crhs2 = NULL_TREE;
2580 58484 : if (is_gimple_assign (use_stmt)
2581 58484 : && gimple_assign_rhs_code (use_stmt) == atomic_op)
2582 : {
2583 1416 : if (gimple_assign_rhs1 (use_stmt) == lhsc)
2584 1016 : oarg = gimple_assign_rhs2 (use_stmt);
2585 400 : else if (atomic_op != MINUS_EXPR)
2586 : oarg = gimple_assign_rhs1 (use_stmt);
2587 : }
2588 57068 : else if (atomic_op == MINUS_EXPR
2589 13188 : && is_gimple_assign (use_stmt)
2590 3612 : && gimple_assign_rhs_code (use_stmt) == PLUS_EXPR
2591 199 : && TREE_CODE (arg) == INTEGER_CST
2592 57267 : && (TREE_CODE (gimple_assign_rhs2 (use_stmt))
2593 : == INTEGER_CST))
2594 : {
2595 183 : tree a = fold_convert (TREE_TYPE (lhs2), arg);
2596 183 : tree o = fold_convert (TREE_TYPE (lhs2),
2597 : gimple_assign_rhs2 (use_stmt));
2598 183 : if (wi::to_wide (a) == wi::neg (wi::to_wide (o)))
2599 : ok = true;
2600 : }
2601 56885 : else if (atomic_op == BIT_AND_EXPR || atomic_op == BIT_IOR_EXPR)
2602 : ;
2603 51643 : else if (gimple_code (use_stmt) == GIMPLE_COND)
2604 : {
2605 19457 : ccode = gimple_cond_code (use_stmt);
2606 19457 : crhs1 = gimple_cond_lhs (use_stmt);
2607 19457 : crhs2 = gimple_cond_rhs (use_stmt);
2608 : }
2609 32186 : else if (is_gimple_assign (use_stmt))
2610 : {
2611 9525 : if (gimple_assign_rhs_class (use_stmt) == GIMPLE_BINARY_RHS)
2612 : {
2613 3941 : ccode = gimple_assign_rhs_code (use_stmt);
2614 3941 : crhs1 = gimple_assign_rhs1 (use_stmt);
2615 3941 : crhs2 = gimple_assign_rhs2 (use_stmt);
2616 : }
2617 5584 : else if (gimple_assign_rhs_code (use_stmt) == COND_EXPR)
2618 : {
2619 0 : tree cond = gimple_assign_rhs1 (use_stmt);
2620 0 : if (COMPARISON_CLASS_P (cond))
2621 : {
2622 0 : ccode = TREE_CODE (cond);
2623 0 : crhs1 = TREE_OPERAND (cond, 0);
2624 0 : crhs2 = TREE_OPERAND (cond, 1);
2625 : }
2626 : }
2627 : }
2628 24414 : if (ccode == EQ_EXPR || ccode == NE_EXPR)
2629 : {
2630 : /* Deal with x - y == 0 or x ^ y == 0
2631 : being optimized into x == y and x + cst == 0
2632 : into x == -cst. */
2633 22214 : tree o = NULL_TREE;
2634 22214 : if (crhs1 == lhsc)
2635 : o = crhs2;
2636 133 : else if (crhs2 == lhsc)
2637 133 : o = crhs1;
2638 22214 : if (o && atomic_op != PLUS_EXPR)
2639 : oarg = o;
2640 10057 : else if (o
2641 10057 : && TREE_CODE (o) == INTEGER_CST
2642 10057 : && TREE_CODE (arg) == INTEGER_CST)
2643 : {
2644 9347 : tree a = fold_convert (TREE_TYPE (lhs2), arg);
2645 9347 : o = fold_convert (TREE_TYPE (lhs2), o);
2646 9347 : if (wi::to_wide (a) == wi::neg (wi::to_wide (o)))
2647 58484 : ok = true;
2648 : }
2649 : }
2650 58484 : if (oarg && !ok)
2651 : {
2652 13573 : if (operand_equal_p (arg, oarg, 0))
2653 : ok = true;
2654 12244 : else if (TREE_CODE (arg) == SSA_NAME
2655 2203 : && TREE_CODE (oarg) == SSA_NAME)
2656 : {
2657 745 : tree oarg2 = oarg;
2658 745 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (oarg)))
2659 : {
2660 104 : gimple *g = SSA_NAME_DEF_STMT (oarg);
2661 104 : oarg2 = gimple_assign_rhs1 (g);
2662 104 : if (TREE_CODE (oarg2) != SSA_NAME
2663 104 : || !INTEGRAL_TYPE_P (TREE_TYPE (oarg2))
2664 208 : || (TYPE_PRECISION (TREE_TYPE (oarg2))
2665 104 : != TYPE_PRECISION (TREE_TYPE (oarg))))
2666 : oarg2 = oarg;
2667 : }
2668 745 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (arg)))
2669 : {
2670 544 : gimple *g = SSA_NAME_DEF_STMT (arg);
2671 544 : tree rhs1 = gimple_assign_rhs1 (g);
2672 : /* Handle e.g.
2673 : x.0_1 = (long unsigned int) x_4(D);
2674 : _2 = __atomic_fetch_add_8 (&vlong, x.0_1, 0);
2675 : _3 = (long int) _2;
2676 : _7 = x_4(D) + _3; */
2677 544 : if (rhs1 == oarg || rhs1 == oarg2)
2678 : ok = true;
2679 : /* Handle e.g.
2680 : x.18_1 = (short unsigned int) x_5(D);
2681 : _2 = (int) x.18_1;
2682 : _3 = __atomic_fetch_xor_2 (&vshort, _2, 0);
2683 : _4 = (short int) _3;
2684 : _8 = x_5(D) ^ _4;
2685 : This happens only for char/short. */
2686 160 : else if (TREE_CODE (rhs1) == SSA_NAME
2687 160 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
2688 320 : && (TYPE_PRECISION (TREE_TYPE (rhs1))
2689 160 : == TYPE_PRECISION (TREE_TYPE (lhs2))))
2690 : {
2691 160 : g = SSA_NAME_DEF_STMT (rhs1);
2692 160 : if (gimple_assign_cast_p (g)
2693 160 : && (gimple_assign_rhs1 (g) == oarg
2694 0 : || gimple_assign_rhs1 (g) == oarg2))
2695 : ok = true;
2696 : }
2697 : }
2698 745 : if (!ok && arg == oarg2)
2699 : /* Handle e.g.
2700 : _1 = __sync_fetch_and_add_4 (&v, x_5(D));
2701 : _2 = (int) _1;
2702 : x.0_3 = (int) x_5(D);
2703 : _7 = _2 + x.0_3; */
2704 : ok = true;
2705 : }
2706 : }
2707 :
2708 57155 : if (ok)
2709 : {
2710 2546 : tree new_lhs = make_ssa_name (TREE_TYPE (lhs2));
2711 2546 : gimple_call_set_lhs (stmt2, new_lhs);
2712 2546 : gimple_call_set_fndecl (stmt2, ndecl);
2713 2546 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
2714 2546 : if (ccode == ERROR_MARK)
2715 2000 : gimple_assign_set_rhs_with_ops (&gsi, cast_stmt
2716 : ? NOP_EXPR : SSA_NAME,
2717 : new_lhs);
2718 : else
2719 : {
2720 1323 : crhs1 = new_lhs;
2721 1323 : crhs2 = build_zero_cst (TREE_TYPE (lhs2));
2722 1323 : if (gimple_code (use_stmt) == GIMPLE_COND)
2723 : {
2724 984 : gcond *cond_stmt = as_a <gcond *> (use_stmt);
2725 984 : gimple_cond_set_lhs (cond_stmt, crhs1);
2726 984 : gimple_cond_set_rhs (cond_stmt, crhs2);
2727 : }
2728 339 : else if (gimple_assign_rhs_class (use_stmt)
2729 : == GIMPLE_BINARY_RHS)
2730 : {
2731 339 : gimple_assign_set_rhs1 (use_stmt, crhs1);
2732 339 : gimple_assign_set_rhs2 (use_stmt, crhs2);
2733 : }
2734 : else
2735 : {
2736 0 : gcc_checking_assert (gimple_assign_rhs_code (use_stmt)
2737 : == COND_EXPR);
2738 0 : tree cond = build2 (ccode, boolean_type_node,
2739 : crhs1, crhs2);
2740 0 : gimple_assign_set_rhs1 (use_stmt, cond);
2741 : }
2742 : }
2743 2546 : update_stmt (use_stmt);
2744 2546 : if (atomic_op != BIT_AND_EXPR
2745 2546 : && atomic_op != BIT_IOR_EXPR
2746 2546 : && !stmt_ends_bb_p (stmt2))
2747 : {
2748 : /* For the benefit of debug stmts, emit stmt(s) to set
2749 : lhs2 to the value it had from the new builtin.
2750 : E.g. if it was previously:
2751 : lhs2 = __atomic_fetch_add_8 (ptr, arg, 0);
2752 : emit:
2753 : new_lhs = __atomic_add_fetch_8 (ptr, arg, 0);
2754 : lhs2 = new_lhs - arg;
2755 : We also keep cast_stmt if any in the IL for
2756 : the same reasons.
2757 : These stmts will be DCEd later and proper debug info
2758 : will be emitted.
2759 : This is only possible for reversible operations
2760 : (+/-/^) and without -fnon-call-exceptions. */
2761 2205 : gsi = gsi_for_stmt (stmt2);
2762 2205 : tree type = TREE_TYPE (lhs2);
2763 2205 : if (TREE_CODE (arg) == INTEGER_CST)
2764 1623 : arg = fold_convert (type, arg);
2765 582 : else if (!useless_type_conversion_p (type, TREE_TYPE (arg)))
2766 : {
2767 0 : tree narg = make_ssa_name (type);
2768 0 : gimple *g = gimple_build_assign (narg, NOP_EXPR, arg);
2769 0 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
2770 0 : arg = narg;
2771 : }
2772 2205 : enum tree_code rcode;
2773 2205 : switch (atomic_op)
2774 : {
2775 : case PLUS_EXPR: rcode = MINUS_EXPR; break;
2776 727 : case MINUS_EXPR: rcode = PLUS_EXPR; break;
2777 492 : case BIT_XOR_EXPR: rcode = atomic_op; break;
2778 0 : default: gcc_unreachable ();
2779 : }
2780 2205 : gimple *g = gimple_build_assign (lhs2, rcode, new_lhs, arg);
2781 2205 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
2782 2205 : update_stmt (stmt2);
2783 : }
2784 : else
2785 : {
2786 : /* For e.g.
2787 : lhs2 = __atomic_fetch_or_8 (ptr, arg, 0);
2788 : after we change it to
2789 : new_lhs = __atomic_or_fetch_8 (ptr, arg, 0);
2790 : there is no way to find out the lhs2 value (i.e.
2791 : what the atomic memory contained before the operation),
2792 : values of some bits are lost. We have checked earlier
2793 : that we don't have any non-debug users except for what
2794 : we are already changing, so we need to reset the
2795 : debug stmts and remove the cast_stmt if any. */
2796 341 : imm_use_iterator iter;
2797 676 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs2)
2798 335 : if (use_stmt != cast_stmt)
2799 : {
2800 168 : gcc_assert (is_gimple_debug (use_stmt));
2801 168 : gimple_debug_bind_reset_value (use_stmt);
2802 168 : update_stmt (use_stmt);
2803 341 : }
2804 341 : if (cast_stmt)
2805 : {
2806 167 : gsi = gsi_for_stmt (cast_stmt);
2807 167 : gsi_remove (&gsi, true);
2808 : }
2809 341 : update_stmt (stmt2);
2810 341 : release_ssa_name (lhs2);
2811 : }
2812 : }
2813 : }
2814 : break;
2815 :
2816 : default:
2817 : break;
2818 : }
2819 : return false;
2820 : }
2821 :
2822 : /* Given a ssa_name in NAME see if it was defined by an assignment and
2823 : set CODE to be the code and ARG1 to the first operand on the rhs and ARG2
2824 : to the second operand on the rhs. */
2825 :
2826 : static inline void
2827 17283556 : defcodefor_name (tree name, enum tree_code *code, tree *arg1, tree *arg2)
2828 : {
2829 17283556 : gimple *def;
2830 17283556 : enum tree_code code1;
2831 17283556 : tree arg11;
2832 17283556 : tree arg21;
2833 17283556 : tree arg31;
2834 17283556 : enum gimple_rhs_class grhs_class;
2835 :
2836 17283556 : code1 = TREE_CODE (name);
2837 17283556 : arg11 = name;
2838 17283556 : arg21 = NULL_TREE;
2839 17283556 : arg31 = NULL_TREE;
2840 17283556 : grhs_class = get_gimple_rhs_class (code1);
2841 :
2842 17283556 : if (code1 == SSA_NAME)
2843 : {
2844 11524629 : def = SSA_NAME_DEF_STMT (name);
2845 :
2846 11524629 : if (def && is_gimple_assign (def)
2847 18688968 : && can_propagate_from (def))
2848 : {
2849 4935592 : code1 = gimple_assign_rhs_code (def);
2850 4935592 : arg11 = gimple_assign_rhs1 (def);
2851 4935592 : arg21 = gimple_assign_rhs2 (def);
2852 4935592 : arg31 = gimple_assign_rhs3 (def);
2853 : }
2854 : }
2855 5758927 : else if (grhs_class != GIMPLE_SINGLE_RHS)
2856 0 : code1 = ERROR_MARK;
2857 :
2858 17283556 : *code = code1;
2859 17283556 : *arg1 = arg11;
2860 17283556 : if (arg2)
2861 17266320 : *arg2 = arg21;
2862 17283556 : if (arg31)
2863 2335 : *code = ERROR_MARK;
2864 17283556 : }
2865 :
2866 :
2867 : /* Recognize rotation patterns. Return true if a transformation
2868 : applied, otherwise return false.
2869 :
2870 : We are looking for X with unsigned type T with bitsize B, OP being
2871 : +, | or ^, some type T2 wider than T. For:
2872 : (X << CNT1) OP (X >> CNT2) iff CNT1 + CNT2 == B
2873 : ((T) ((T2) X << CNT1)) OP ((T) ((T2) X >> CNT2)) iff CNT1 + CNT2 == B
2874 :
2875 : transform these into:
2876 : X r<< CNT1
2877 :
2878 : Or for:
2879 : (X << Y) OP (X >> (B - Y))
2880 : (X << (int) Y) OP (X >> (int) (B - Y))
2881 : ((T) ((T2) X << Y)) OP ((T) ((T2) X >> (B - Y)))
2882 : ((T) ((T2) X << (int) Y)) OP ((T) ((T2) X >> (int) (B - Y)))
2883 : (X << Y) | (X >> ((-Y) & (B - 1)))
2884 : (X << (int) Y) | (X >> (int) ((-Y) & (B - 1)))
2885 : ((T) ((T2) X << Y)) | ((T) ((T2) X >> ((-Y) & (B - 1))))
2886 : ((T) ((T2) X << (int) Y)) | ((T) ((T2) X >> (int) ((-Y) & (B - 1))))
2887 :
2888 : transform these into (last 2 only if ranger can prove Y < B
2889 : or Y = N * B):
2890 : X r<< Y
2891 : or
2892 : X r<< (& & (B - 1))
2893 : The latter for the forms with T2 wider than T if ranger can't prove Y < B.
2894 :
2895 : Or for:
2896 : (X << (Y & (B - 1))) | (X >> ((-Y) & (B - 1)))
2897 : (X << (int) (Y & (B - 1))) | (X >> (int) ((-Y) & (B - 1)))
2898 : ((T) ((T2) X << (Y & (B - 1)))) | ((T) ((T2) X >> ((-Y) & (B - 1))))
2899 : ((T) ((T2) X << (int) (Y & (B - 1)))) \
2900 : | ((T) ((T2) X >> (int) ((-Y) & (B - 1))))
2901 :
2902 : transform these into:
2903 : X r<< (Y & (B - 1))
2904 :
2905 : Note, in the patterns with T2 type, the type of OP operands
2906 : might be even a signed type, but should have precision B.
2907 : Expressions with & (B - 1) should be recognized only if B is
2908 : a power of 2. */
2909 :
2910 : static bool
2911 10151336 : simplify_rotate (gimple_stmt_iterator *gsi)
2912 : {
2913 10151336 : gimple *stmt = gsi_stmt (*gsi);
2914 10151336 : tree arg[2], rtype, rotcnt = NULL_TREE;
2915 10151336 : tree def_arg1[2], def_arg2[2];
2916 10151336 : enum tree_code def_code[2];
2917 10151336 : tree lhs;
2918 10151336 : int i;
2919 10151336 : bool swapped_p = false;
2920 10151336 : gimple *g;
2921 10151336 : gimple *def_arg_stmt[2] = { NULL, NULL };
2922 10151336 : int wider_prec = 0;
2923 10151336 : bool add_masking = false;
2924 :
2925 10151336 : arg[0] = gimple_assign_rhs1 (stmt);
2926 10151336 : arg[1] = gimple_assign_rhs2 (stmt);
2927 10151336 : rtype = TREE_TYPE (arg[0]);
2928 :
2929 : /* Only create rotates in complete modes. Other cases are not
2930 : expanded properly. */
2931 10151336 : if (!INTEGRAL_TYPE_P (rtype)
2932 10151336 : || !type_has_mode_precision_p (rtype))
2933 1559464 : return false;
2934 :
2935 25775616 : for (i = 0; i < 2; i++)
2936 : {
2937 17183744 : defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
2938 17183744 : if (TREE_CODE (arg[i]) == SSA_NAME)
2939 11424817 : def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
2940 : }
2941 :
2942 : /* Look through narrowing (or same precision) conversions. */
2943 7630866 : if (CONVERT_EXPR_CODE_P (def_code[0])
2944 961006 : && CONVERT_EXPR_CODE_P (def_code[1])
2945 140277 : && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[0]))
2946 116856 : && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[1]))
2947 109783 : && TYPE_PRECISION (TREE_TYPE (def_arg1[0]))
2948 109783 : == TYPE_PRECISION (TREE_TYPE (def_arg1[1]))
2949 64031 : && TYPE_PRECISION (TREE_TYPE (def_arg1[0])) >= TYPE_PRECISION (rtype)
2950 44405 : && has_single_use (arg[0])
2951 8624876 : && has_single_use (arg[1]))
2952 : {
2953 28640 : wider_prec = TYPE_PRECISION (TREE_TYPE (def_arg1[0]));
2954 85920 : for (i = 0; i < 2; i++)
2955 : {
2956 57280 : arg[i] = def_arg1[i];
2957 57280 : defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
2958 57280 : if (TREE_CODE (arg[i]) == SSA_NAME)
2959 57280 : def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
2960 : }
2961 : }
2962 : else
2963 : {
2964 : /* Handle signed rotate; the RSHIFT_EXPR has to be done
2965 : in unsigned type but LSHIFT_EXPR could be signed. */
2966 8563232 : i = (def_code[0] == LSHIFT_EXPR || def_code[0] == RSHIFT_EXPR);
2967 7614024 : if (CONVERT_EXPR_CODE_P (def_code[i])
2968 949208 : && (def_code[1 - i] == LSHIFT_EXPR || def_code[1 - i] == RSHIFT_EXPR)
2969 29525 : && INTEGRAL_TYPE_P (TREE_TYPE (def_arg1[i]))
2970 28445 : && TYPE_PRECISION (rtype) == TYPE_PRECISION (TREE_TYPE (def_arg1[i]))
2971 8567888 : && has_single_use (arg[i]))
2972 : {
2973 2060 : arg[i] = def_arg1[i];
2974 2060 : defcodefor_name (arg[i], &def_code[i], &def_arg1[i], &def_arg2[i]);
2975 2060 : if (TREE_CODE (arg[i]) == SSA_NAME)
2976 2060 : def_arg_stmt[i] = SSA_NAME_DEF_STMT (arg[i]);
2977 : }
2978 : }
2979 :
2980 : /* One operand has to be LSHIFT_EXPR and one RSHIFT_EXPR. */
2981 8789532 : for (i = 0; i < 2; i++)
2982 8764858 : if (def_code[i] != LSHIFT_EXPR && def_code[i] != RSHIFT_EXPR)
2983 : return false;
2984 237968 : else if (!has_single_use (arg[i]))
2985 : return false;
2986 24674 : if (def_code[0] == def_code[1])
2987 : return false;
2988 :
2989 : /* If we've looked through narrowing conversions before, look through
2990 : widening conversions from unsigned type with the same precision
2991 : as rtype here. */
2992 20326 : if (TYPE_PRECISION (TREE_TYPE (def_arg1[0])) != TYPE_PRECISION (rtype))
2993 19348 : for (i = 0; i < 2; i++)
2994 : {
2995 12900 : tree tem;
2996 12900 : enum tree_code code;
2997 12900 : defcodefor_name (def_arg1[i], &code, &tem, NULL);
2998 4 : if (!CONVERT_EXPR_CODE_P (code)
2999 12896 : || !INTEGRAL_TYPE_P (TREE_TYPE (tem))
3000 25796 : || TYPE_PRECISION (TREE_TYPE (tem)) != TYPE_PRECISION (rtype))
3001 4 : return false;
3002 12896 : def_arg1[i] = tem;
3003 : }
3004 : /* Both shifts have to use the same first operand. */
3005 20322 : if (!operand_equal_for_phi_arg_p (def_arg1[0], def_arg1[1])
3006 32251 : || !types_compatible_p (TREE_TYPE (def_arg1[0]),
3007 11929 : TREE_TYPE (def_arg1[1])))
3008 : {
3009 8393 : if ((TYPE_PRECISION (TREE_TYPE (def_arg1[0]))
3010 8393 : != TYPE_PRECISION (TREE_TYPE (def_arg1[1])))
3011 8393 : || (TYPE_UNSIGNED (TREE_TYPE (def_arg1[0]))
3012 8393 : == TYPE_UNSIGNED (TREE_TYPE (def_arg1[1]))))
3013 8369 : return false;
3014 :
3015 : /* Handle signed rotate; the RSHIFT_EXPR has to be done
3016 : in unsigned type but LSHIFT_EXPR could be signed. */
3017 540 : i = def_code[0] != RSHIFT_EXPR;
3018 540 : if (!TYPE_UNSIGNED (TREE_TYPE (def_arg1[i])))
3019 : return false;
3020 :
3021 507 : tree tem;
3022 507 : enum tree_code code;
3023 507 : defcodefor_name (def_arg1[i], &code, &tem, NULL);
3024 304 : if (!CONVERT_EXPR_CODE_P (code)
3025 203 : || !INTEGRAL_TYPE_P (TREE_TYPE (tem))
3026 710 : || TYPE_PRECISION (TREE_TYPE (tem)) != TYPE_PRECISION (rtype))
3027 : return false;
3028 194 : def_arg1[i] = tem;
3029 194 : if (!operand_equal_for_phi_arg_p (def_arg1[0], def_arg1[1])
3030 218 : || !types_compatible_p (TREE_TYPE (def_arg1[0]),
3031 24 : TREE_TYPE (def_arg1[1])))
3032 170 : return false;
3033 : }
3034 11929 : else if (!TYPE_UNSIGNED (TREE_TYPE (def_arg1[0])))
3035 : return false;
3036 :
3037 : /* CNT1 + CNT2 == B case above. */
3038 10698 : if (tree_fits_uhwi_p (def_arg2[0])
3039 1210 : && tree_fits_uhwi_p (def_arg2[1])
3040 10698 : && tree_to_uhwi (def_arg2[0])
3041 1210 : + tree_to_uhwi (def_arg2[1]) == TYPE_PRECISION (rtype))
3042 : rotcnt = def_arg2[0];
3043 9768 : else if (TREE_CODE (def_arg2[0]) != SSA_NAME
3044 9488 : || TREE_CODE (def_arg2[1]) != SSA_NAME)
3045 : return false;
3046 : else
3047 : {
3048 9488 : tree cdef_arg1[2], cdef_arg2[2], def_arg2_alt[2];
3049 9488 : enum tree_code cdef_code[2];
3050 9488 : gimple *def_arg_alt_stmt[2] = { NULL, NULL };
3051 9488 : int check_range = 0;
3052 9488 : gimple *check_range_stmt = NULL;
3053 : /* Look through conversion of the shift count argument.
3054 : The C/C++ FE cast any shift count argument to integer_type_node.
3055 : The only problem might be if the shift count type maximum value
3056 : is equal or smaller than number of bits in rtype. */
3057 28464 : for (i = 0; i < 2; i++)
3058 : {
3059 18976 : def_arg2_alt[i] = def_arg2[i];
3060 18976 : defcodefor_name (def_arg2[i], &cdef_code[i],
3061 : &cdef_arg1[i], &cdef_arg2[i]);
3062 14716 : if (CONVERT_EXPR_CODE_P (cdef_code[i])
3063 4260 : && INTEGRAL_TYPE_P (TREE_TYPE (cdef_arg1[i]))
3064 4260 : && TYPE_PRECISION (TREE_TYPE (cdef_arg1[i]))
3065 8520 : > floor_log2 (TYPE_PRECISION (rtype))
3066 23236 : && type_has_mode_precision_p (TREE_TYPE (cdef_arg1[i])))
3067 : {
3068 4260 : def_arg2_alt[i] = cdef_arg1[i];
3069 4260 : if (TREE_CODE (def_arg2[i]) == SSA_NAME)
3070 4260 : def_arg_alt_stmt[i] = SSA_NAME_DEF_STMT (def_arg2[i]);
3071 4260 : defcodefor_name (def_arg2_alt[i], &cdef_code[i],
3072 : &cdef_arg1[i], &cdef_arg2[i]);
3073 : }
3074 : else
3075 14716 : def_arg_alt_stmt[i] = def_arg_stmt[i];
3076 : }
3077 25796 : for (i = 0; i < 2; i++)
3078 : /* Check for one shift count being Y and the other B - Y,
3079 : with optional casts. */
3080 18625 : if (cdef_code[i] == MINUS_EXPR
3081 862 : && tree_fits_shwi_p (cdef_arg1[i])
3082 862 : && tree_to_shwi (cdef_arg1[i]) == TYPE_PRECISION (rtype)
3083 19447 : && TREE_CODE (cdef_arg2[i]) == SSA_NAME)
3084 : {
3085 822 : tree tem;
3086 822 : enum tree_code code;
3087 :
3088 822 : if (cdef_arg2[i] == def_arg2[1 - i]
3089 472 : || cdef_arg2[i] == def_arg2_alt[1 - i])
3090 : {
3091 350 : rotcnt = cdef_arg2[i];
3092 350 : check_range = -1;
3093 350 : if (cdef_arg2[i] == def_arg2[1 - i])
3094 350 : check_range_stmt = def_arg_stmt[1 - i];
3095 : else
3096 0 : check_range_stmt = def_arg_alt_stmt[1 - i];
3097 806 : break;
3098 : }
3099 472 : defcodefor_name (cdef_arg2[i], &code, &tem, NULL);
3100 16 : if (CONVERT_EXPR_CODE_P (code)
3101 456 : && INTEGRAL_TYPE_P (TREE_TYPE (tem))
3102 456 : && TYPE_PRECISION (TREE_TYPE (tem))
3103 912 : > floor_log2 (TYPE_PRECISION (rtype))
3104 456 : && type_has_mode_precision_p (TREE_TYPE (tem))
3105 928 : && (tem == def_arg2[1 - i]
3106 288 : || tem == def_arg2_alt[1 - i]))
3107 : {
3108 456 : rotcnt = tem;
3109 456 : check_range = -1;
3110 456 : if (tem == def_arg2[1 - i])
3111 168 : check_range_stmt = def_arg_stmt[1 - i];
3112 : else
3113 288 : check_range_stmt = def_arg_alt_stmt[1 - i];
3114 : break;
3115 : }
3116 : }
3117 : /* The above sequence isn't safe for Y being 0,
3118 : because then one of the shifts triggers undefined behavior.
3119 : This alternative is safe even for rotation count of 0.
3120 : One shift count is Y and the other (-Y) & (B - 1).
3121 : Or one shift count is Y & (B - 1) and the other (-Y) & (B - 1). */
3122 17803 : else if (cdef_code[i] == BIT_AND_EXPR
3123 28724 : && pow2p_hwi (TYPE_PRECISION (rtype))
3124 12416 : && tree_fits_shwi_p (cdef_arg2[i])
3125 24832 : && tree_to_shwi (cdef_arg2[i])
3126 12416 : == TYPE_PRECISION (rtype) - 1
3127 12356 : && TREE_CODE (cdef_arg1[i]) == SSA_NAME
3128 30159 : && gimple_assign_rhs_code (stmt) == BIT_IOR_EXPR)
3129 : {
3130 2296 : tree tem;
3131 2296 : enum tree_code code;
3132 :
3133 2296 : defcodefor_name (cdef_arg1[i], &code, &tem, NULL);
3134 2099 : if (CONVERT_EXPR_CODE_P (code)
3135 197 : && INTEGRAL_TYPE_P (TREE_TYPE (tem))
3136 197 : && TYPE_PRECISION (TREE_TYPE (tem))
3137 394 : > floor_log2 (TYPE_PRECISION (rtype))
3138 2493 : && type_has_mode_precision_p (TREE_TYPE (tem)))
3139 197 : defcodefor_name (tem, &code, &tem, NULL);
3140 :
3141 2296 : if (code == NEGATE_EXPR)
3142 : {
3143 1525 : if (tem == def_arg2[1 - i] || tem == def_arg2_alt[1 - i])
3144 : {
3145 854 : rotcnt = tem;
3146 854 : check_range = 1;
3147 854 : if (tem == def_arg2[1 - i])
3148 846 : check_range_stmt = def_arg_stmt[1 - i];
3149 : else
3150 8 : check_range_stmt = def_arg_alt_stmt[1 - i];
3151 1511 : break;
3152 : }
3153 671 : tree tem2;
3154 671 : defcodefor_name (tem, &code, &tem2, NULL);
3155 237 : if (CONVERT_EXPR_CODE_P (code)
3156 434 : && INTEGRAL_TYPE_P (TREE_TYPE (tem2))
3157 434 : && TYPE_PRECISION (TREE_TYPE (tem2))
3158 868 : > floor_log2 (TYPE_PRECISION (rtype))
3159 1105 : && type_has_mode_precision_p (TREE_TYPE (tem2)))
3160 : {
3161 434 : if (tem2 == def_arg2[1 - i]
3162 434 : || tem2 == def_arg2_alt[1 - i])
3163 : {
3164 228 : rotcnt = tem2;
3165 228 : check_range = 1;
3166 228 : if (tem2 == def_arg2[1 - i])
3167 0 : check_range_stmt = def_arg_stmt[1 - i];
3168 : else
3169 228 : check_range_stmt = def_arg_alt_stmt[1 - i];
3170 : break;
3171 : }
3172 : }
3173 : else
3174 237 : tem2 = NULL_TREE;
3175 :
3176 443 : if (cdef_code[1 - i] == BIT_AND_EXPR
3177 430 : && tree_fits_shwi_p (cdef_arg2[1 - i])
3178 860 : && tree_to_shwi (cdef_arg2[1 - i])
3179 430 : == TYPE_PRECISION (rtype) - 1
3180 873 : && TREE_CODE (cdef_arg1[1 - i]) == SSA_NAME)
3181 : {
3182 430 : if (tem == cdef_arg1[1 - i]
3183 205 : || tem2 == cdef_arg1[1 - i])
3184 : {
3185 : rotcnt = def_arg2[1 - i];
3186 429 : break;
3187 : }
3188 193 : tree tem3;
3189 193 : defcodefor_name (cdef_arg1[1 - i], &code, &tem3, NULL);
3190 0 : if (CONVERT_EXPR_CODE_P (code)
3191 193 : && INTEGRAL_TYPE_P (TREE_TYPE (tem3))
3192 193 : && TYPE_PRECISION (TREE_TYPE (tem3))
3193 386 : > floor_log2 (TYPE_PRECISION (rtype))
3194 386 : && type_has_mode_precision_p (TREE_TYPE (tem3)))
3195 : {
3196 193 : if (tem == tem3 || tem2 == tem3)
3197 : {
3198 : rotcnt = def_arg2[1 - i];
3199 : break;
3200 : }
3201 : }
3202 : }
3203 : }
3204 : }
3205 2317 : if (check_range && wider_prec > TYPE_PRECISION (rtype))
3206 : {
3207 1533 : if (TREE_CODE (rotcnt) != SSA_NAME)
3208 573 : return false;
3209 1533 : int_range_max r;
3210 1533 : range_query *q = get_range_query (cfun);
3211 1533 : if (q == get_global_range_query ())
3212 1522 : q = enable_ranger (cfun);
3213 1533 : if (!q->range_of_expr (r, rotcnt, check_range_stmt))
3214 : {
3215 0 : if (check_range > 0)
3216 : return false;
3217 0 : r.set_varying (TREE_TYPE (rotcnt));
3218 : }
3219 1533 : int prec = TYPE_PRECISION (TREE_TYPE (rotcnt));
3220 1533 : signop sign = TYPE_SIGN (TREE_TYPE (rotcnt));
3221 1533 : wide_int min = wide_int::from (TYPE_PRECISION (rtype), prec, sign);
3222 1533 : wide_int max = wide_int::from (wider_prec - 1, prec, sign);
3223 1533 : if (check_range < 0)
3224 616 : max = min;
3225 1533 : int_range<1> r2 (TREE_TYPE (rotcnt), min, max);
3226 1533 : r.intersect (r2);
3227 1533 : if (!r.undefined_p ())
3228 : {
3229 1181 : if (check_range > 0)
3230 : {
3231 589 : int_range_max r3;
3232 1844 : for (int i = TYPE_PRECISION (rtype) + 1; i < wider_prec;
3233 1255 : i += TYPE_PRECISION (rtype))
3234 : {
3235 1255 : int j = i + TYPE_PRECISION (rtype) - 2;
3236 1255 : min = wide_int::from (i, prec, sign);
3237 1255 : max = wide_int::from (MIN (j, wider_prec - 1),
3238 1255 : prec, sign);
3239 1255 : int_range<1> r4 (TREE_TYPE (rotcnt), min, max);
3240 1255 : r3.union_ (r4);
3241 1255 : }
3242 589 : r.intersect (r3);
3243 589 : if (!r.undefined_p ())
3244 573 : return false;
3245 589 : }
3246 : add_masking = true;
3247 : }
3248 1533 : }
3249 8915 : if (rotcnt == NULL_TREE)
3250 : return false;
3251 1744 : swapped_p = i != 1;
3252 : }
3253 :
3254 2674 : if (!useless_type_conversion_p (TREE_TYPE (def_arg2[0]),
3255 2674 : TREE_TYPE (rotcnt)))
3256 : {
3257 496 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (def_arg2[0])),
3258 : NOP_EXPR, rotcnt);
3259 496 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3260 496 : rotcnt = gimple_assign_lhs (g);
3261 : }
3262 2674 : if (add_masking)
3263 : {
3264 608 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rotcnt)),
3265 : BIT_AND_EXPR, rotcnt,
3266 608 : build_int_cst (TREE_TYPE (rotcnt),
3267 608 : TYPE_PRECISION (rtype) - 1));
3268 608 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3269 608 : rotcnt = gimple_assign_lhs (g);
3270 : }
3271 2674 : lhs = gimple_assign_lhs (stmt);
3272 2674 : if (!useless_type_conversion_p (rtype, TREE_TYPE (def_arg1[0])))
3273 1010 : lhs = make_ssa_name (TREE_TYPE (def_arg1[0]));
3274 2674 : g = gimple_build_assign (lhs,
3275 2674 : ((def_code[0] == LSHIFT_EXPR) ^ swapped_p)
3276 : ? LROTATE_EXPR : RROTATE_EXPR, def_arg1[0], rotcnt);
3277 2674 : if (!useless_type_conversion_p (rtype, TREE_TYPE (def_arg1[0])))
3278 : {
3279 1010 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3280 1010 : g = gimple_build_assign (gimple_assign_lhs (stmt), NOP_EXPR, lhs);
3281 : }
3282 2674 : gsi_replace (gsi, g, false);
3283 2674 : return true;
3284 : }
3285 :
3286 :
3287 : /* Check whether an array contains a valid table according to VALIDATE_FN. */
3288 : template<typename ValidateFn>
3289 : static bool
3290 16 : check_table_array (tree ctor, HOST_WIDE_INT &zero_val, unsigned bits,
3291 : ValidateFn validate_fn)
3292 : {
3293 : tree elt, idx;
3294 16 : unsigned HOST_WIDE_INT i, raw_idx = 0;
3295 16 : unsigned matched = 0;
3296 :
3297 16 : zero_val = 0;
3298 :
3299 670 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ctor), i, idx, elt)
3300 : {
3301 670 : if (!tree_fits_shwi_p (idx))
3302 : return false;
3303 670 : if (!tree_fits_shwi_p (elt) && TREE_CODE (elt) != RAW_DATA_CST)
3304 : return false;
3305 :
3306 670 : unsigned HOST_WIDE_INT index = tree_to_shwi (idx);
3307 : HOST_WIDE_INT val;
3308 :
3309 670 : if (TREE_CODE (elt) == INTEGER_CST)
3310 606 : val = tree_to_shwi (elt);
3311 : else
3312 : {
3313 64 : if (raw_idx == (unsigned) RAW_DATA_LENGTH (elt))
3314 : {
3315 0 : raw_idx = 0;
3316 0 : continue;
3317 : }
3318 64 : if (TYPE_UNSIGNED (TREE_TYPE (elt)))
3319 0 : val = RAW_DATA_UCHAR_ELT (elt, raw_idx);
3320 : else
3321 64 : val = RAW_DATA_SCHAR_ELT (elt, raw_idx);
3322 64 : index += raw_idx;
3323 64 : raw_idx++;
3324 64 : i--;
3325 : }
3326 :
3327 670 : if (index > bits * 2)
3328 : return false;
3329 :
3330 670 : if (index == 0)
3331 : {
3332 16 : zero_val = val;
3333 16 : matched++;
3334 : }
3335 :
3336 670 : if (val >= 0 && val < bits && validate_fn (val, index))
3337 608 : matched++;
3338 :
3339 670 : if (matched > bits)
3340 : return true;
3341 : }
3342 :
3343 : return false;
3344 : }
3345 :
3346 : /* Check whether a string contains a valid table according to VALIDATE_FN. */
3347 : template<typename ValidateFn>
3348 : static bool
3349 4 : check_table_string (tree string, HOST_WIDE_INT &zero_val,unsigned bits,
3350 : ValidateFn validate_fn)
3351 : {
3352 4 : unsigned HOST_WIDE_INT len = TREE_STRING_LENGTH (string);
3353 4 : unsigned matched = 0;
3354 4 : const unsigned char *p = (const unsigned char *) TREE_STRING_POINTER (string);
3355 :
3356 4 : if (len < bits || len > bits * 2)
3357 : return false;
3358 :
3359 4 : zero_val = p[0];
3360 :
3361 164 : for (unsigned i = 0; i < len; i++)
3362 160 : if (p[i] < bits && validate_fn (p[i], i))
3363 160 : matched++;
3364 :
3365 4 : return matched == bits;
3366 : }
3367 :
3368 : /* Check whether CTOR contains a valid table according to VALIDATE_FN. */
3369 : template<typename ValidateFn>
3370 : static bool
3371 28 : check_table (tree ctor, tree type, HOST_WIDE_INT &zero_val, unsigned bits,
3372 : ValidateFn validate_fn)
3373 : {
3374 28 : if (TREE_CODE (ctor) == CONSTRUCTOR)
3375 16 : return check_table_array (ctor, zero_val, bits, validate_fn);
3376 : else if (TREE_CODE (ctor) == STRING_CST
3377 12 : && TYPE_PRECISION (type) == CHAR_TYPE_SIZE)
3378 4 : return check_table_string (ctor, zero_val, bits, validate_fn);
3379 : return false;
3380 : }
3381 :
3382 : /* Match.pd function to match the ctz expression. */
3383 : extern bool gimple_ctz_table_index (tree, tree *, tree (*)(tree));
3384 : extern bool gimple_clz_table_index (tree, tree *, tree (*)(tree));
3385 :
3386 : /* Recognize count leading and trailing zeroes idioms.
3387 : The canonical form is array[((x & -x) * C) >> SHIFT] where C is a magic
3388 : constant which when multiplied by a power of 2 creates a unique value
3389 : in the top 5 or 6 bits. This is then indexed into a table which maps it
3390 : to the number of trailing zeroes. Array[0] is returned so the caller can
3391 : emit an appropriate sequence depending on whether ctz (0) is defined on
3392 : the target. */
3393 :
3394 : static bool
3395 1956766 : simplify_count_zeroes (gimple_stmt_iterator *gsi)
3396 : {
3397 1956766 : gimple *stmt = gsi_stmt (*gsi);
3398 1956766 : tree array_ref = gimple_assign_rhs1 (stmt);
3399 1956766 : tree res_ops[3];
3400 :
3401 1956766 : gcc_checking_assert (TREE_CODE (array_ref) == ARRAY_REF);
3402 :
3403 1956766 : internal_fn fn = IFN_LAST;
3404 : /* For CTZ we recognize ((x & -x) * C) >> SHIFT where the array data
3405 : represents the number of trailing zeros. */
3406 1956766 : if (gimple_ctz_table_index (TREE_OPERAND (array_ref, 1), &res_ops[0], NULL))
3407 : fn = IFN_CTZ;
3408 : /* For CLZ we recognize
3409 : x |= x >> 1;
3410 : x |= x >> 2;
3411 : x |= x >> 4;
3412 : x |= x >> 8;
3413 : x |= x >> 16;
3414 : (x * C) >> SHIFT
3415 : where 31 minus the array data represents the number of leading zeros. */
3416 1956743 : else if (gimple_clz_table_index (TREE_OPERAND (array_ref, 1), &res_ops[0],
3417 : NULL))
3418 : fn = IFN_CLZ;
3419 : else
3420 : return false;
3421 :
3422 33 : HOST_WIDE_INT zero_val;
3423 33 : tree type = TREE_TYPE (array_ref);
3424 33 : tree array = TREE_OPERAND (array_ref, 0);
3425 33 : tree input_type = TREE_TYPE (res_ops[0]);
3426 33 : unsigned input_bits = tree_to_shwi (TYPE_SIZE (input_type));
3427 :
3428 : /* Check the array element type is not wider than 32 bits and the input is
3429 : an unsigned 32-bit or 64-bit type. */
3430 33 : if (TYPE_PRECISION (type) > 32 || !TYPE_UNSIGNED (input_type))
3431 : return false;
3432 29 : if (input_bits != 32 && input_bits != 64)
3433 : return false;
3434 :
3435 29 : if (!direct_internal_fn_supported_p (fn, input_type, OPTIMIZE_FOR_BOTH))
3436 : return false;
3437 :
3438 : /* Check the lower bound of the array is zero. */
3439 29 : tree low = array_ref_low_bound (array_ref);
3440 29 : if (!low || !integer_zerop (low))
3441 0 : return false;
3442 :
3443 : /* Check the shift extracts the top 5..7 bits. */
3444 29 : unsigned shiftval = tree_to_shwi (res_ops[2]);
3445 29 : if (shiftval < input_bits - 7 || shiftval > input_bits - 5)
3446 : return false;
3447 :
3448 28 : tree ctor = ctor_for_folding (array);
3449 28 : if (!ctor)
3450 : return false;
3451 28 : unsigned HOST_WIDE_INT mulval = tree_to_uhwi (res_ops[1]);
3452 28 : if (fn == IFN_CTZ)
3453 : {
3454 494 : auto checkfn = [&](unsigned data, unsigned i) -> bool
3455 : {
3456 476 : unsigned HOST_WIDE_INT mask
3457 476 : = ((HOST_WIDE_INT_1U << (input_bits - shiftval)) - 1) << shiftval;
3458 476 : return (((mulval << data) & mask) >> shiftval) == i;
3459 18 : };
3460 18 : if (!check_table (ctor, type, zero_val, input_bits, checkfn))
3461 8 : return false;
3462 : }
3463 10 : else if (fn == IFN_CLZ)
3464 : {
3465 362 : auto checkfn = [&](unsigned data, unsigned i) -> bool
3466 : {
3467 352 : unsigned HOST_WIDE_INT mask
3468 352 : = ((HOST_WIDE_INT_1U << (input_bits - shiftval)) - 1) << shiftval;
3469 : /* The OR-cascade produces a value with all bits from 0 to the
3470 : original MSB set. Compute (1 << (data + 1)) - 1 to simulate
3471 : that value. When data + 1 equals HOST_BITS_PER_WIDE_INT
3472 : (i.e. data is the MSB position of a 64-bit input) the shift
3473 : is undefined behavior, so handle that case explicitly using
3474 : all-ones. Without this, any well-formed 64-bit DeBruijn CLZ
3475 : table is rejected because its entry for the all-ones input
3476 : correctly maps to the MSB (e.g. table[...] == 63).
3477 : PR tree-optimization/122569. */
3478 703 : unsigned HOST_WIDE_INT all_bits_below
3479 : = (data + 1 == HOST_BITS_PER_WIDE_INT)
3480 352 : ? HOST_WIDE_INT_M1U
3481 351 : : ((HOST_WIDE_INT_1U << (data + 1)) - 1);
3482 352 : return (((all_bits_below * mulval) & mask) >> shiftval) == i;
3483 10 : };
3484 10 : if (!check_table (ctor, type, zero_val, input_bits, checkfn))
3485 0 : return false;
3486 : }
3487 :
3488 20 : HOST_WIDE_INT ctz_val = -1;
3489 20 : bool zero_ok;
3490 20 : if (fn == IFN_CTZ)
3491 : {
3492 10 : ctz_val = 0;
3493 20 : zero_ok = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (input_type),
3494 : ctz_val) == 2;
3495 : }
3496 10 : else if (fn == IFN_CLZ)
3497 : {
3498 10 : ctz_val = 32;
3499 10 : zero_ok = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (input_type),
3500 : ctz_val) == 2;
3501 10 : zero_val = input_bits - 1 - zero_val;
3502 : }
3503 20 : int nargs = 2;
3504 :
3505 : /* If the input value can't be zero, don't special case ctz (0). */
3506 20 : range_query *q = get_range_query (cfun);
3507 20 : if (q == get_global_range_query ())
3508 20 : q = enable_ranger (cfun);
3509 20 : int_range_max vr;
3510 20 : if (q->range_of_expr (vr, res_ops[0], stmt)
3511 20 : && !range_includes_zero_p (vr))
3512 : {
3513 4 : zero_ok = true;
3514 4 : zero_val = 0;
3515 4 : ctz_val = 0;
3516 4 : nargs = 1;
3517 : }
3518 :
3519 20 : gimple_seq seq = NULL;
3520 20 : gimple *g;
3521 20 : gcall *call = gimple_build_call_internal (fn, nargs, res_ops[0],
3522 : nargs == 1 ? NULL_TREE
3523 36 : : build_int_cst (integer_type_node,
3524 16 : ctz_val));
3525 20 : gimple_set_location (call, gimple_location (stmt));
3526 20 : gimple_set_lhs (call, make_ssa_name (integer_type_node));
3527 20 : gimple_seq_add_stmt (&seq, call);
3528 :
3529 20 : tree prev_lhs = gimple_call_lhs (call);
3530 :
3531 20 : if (zero_ok && zero_val == ctz_val)
3532 : ;
3533 : /* Emit ctz (x) & 31 if ctz (0) is 32 but we need to return 0. */
3534 6 : else if (zero_ok && zero_val == 0 && ctz_val == input_bits)
3535 : {
3536 5 : g = gimple_build_assign (make_ssa_name (integer_type_node),
3537 : BIT_AND_EXPR, prev_lhs,
3538 : build_int_cst (integer_type_node,
3539 5 : input_bits - 1));
3540 5 : gimple_set_location (g, gimple_location (stmt));
3541 5 : gimple_seq_add_stmt (&seq, g);
3542 5 : prev_lhs = gimple_assign_lhs (g);
3543 : }
3544 : /* As fallback emit a conditional move. */
3545 : else
3546 : {
3547 9 : g = gimple_build_assign (make_ssa_name (boolean_type_node), EQ_EXPR,
3548 : res_ops[0], build_zero_cst (input_type));
3549 9 : gimple_set_location (g, gimple_location (stmt));
3550 9 : gimple_seq_add_stmt (&seq, g);
3551 9 : tree cond = gimple_assign_lhs (g);
3552 9 : g = gimple_build_assign (make_ssa_name (integer_type_node),
3553 : COND_EXPR, cond,
3554 9 : build_int_cst (integer_type_node, zero_val),
3555 : prev_lhs);
3556 9 : gimple_set_location (g, gimple_location (stmt));
3557 9 : gimple_seq_add_stmt (&seq, g);
3558 9 : prev_lhs = gimple_assign_lhs (g);
3559 : }
3560 :
3561 20 : if (fn == IFN_CLZ)
3562 : {
3563 10 : g = gimple_build_assign (make_ssa_name (integer_type_node),
3564 : MINUS_EXPR,
3565 : build_int_cst (integer_type_node,
3566 10 : input_bits - 1),
3567 : prev_lhs);
3568 10 : gimple_set_location (g, gimple_location (stmt));
3569 10 : gimple_seq_add_stmt (&seq, g);
3570 10 : prev_lhs = gimple_assign_lhs (g);
3571 : }
3572 :
3573 20 : g = gimple_build_assign (gimple_assign_lhs (stmt), NOP_EXPR, prev_lhs);
3574 20 : gimple_seq_add_stmt (&seq, g);
3575 20 : gsi_replace_with_seq (gsi, seq, true);
3576 20 : return true;
3577 20 : }
3578 :
3579 :
3580 : /* Determine whether applying the 2 permutations (mask1 then mask2)
3581 : gives back one of the input. */
3582 :
3583 : static int
3584 34 : is_combined_permutation_identity (tree mask1, tree mask2)
3585 : {
3586 34 : tree mask;
3587 34 : unsigned HOST_WIDE_INT nelts, i, j;
3588 34 : bool maybe_identity1 = true;
3589 34 : bool maybe_identity2 = true;
3590 :
3591 34 : gcc_checking_assert (TREE_CODE (mask1) == VECTOR_CST
3592 : && TREE_CODE (mask2) == VECTOR_CST);
3593 :
3594 : /* For VLA masks, check for the following pattern:
3595 : v1 = VEC_PERM_EXPR (v0, ..., mask1)
3596 : v2 = VEC_PERM_EXPR (v1, ..., mask2)
3597 : -->
3598 : v2 = v0
3599 : if mask1 == mask2 == {nelts - 1, nelts - 2, ...}. */
3600 :
3601 34 : if (operand_equal_p (mask1, mask2, 0)
3602 34 : && !VECTOR_CST_NELTS (mask1).is_constant ())
3603 : {
3604 : vec_perm_builder builder;
3605 : if (tree_to_vec_perm_builder (&builder, mask1))
3606 : {
3607 : poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask1));
3608 : vec_perm_indices sel (builder, 1, nelts);
3609 : if (sel.series_p (0, 1, nelts - 1, -1))
3610 : return 1;
3611 : }
3612 : }
3613 :
3614 34 : mask = fold_ternary (VEC_PERM_EXPR, TREE_TYPE (mask1), mask1, mask1, mask2);
3615 34 : if (mask == NULL_TREE || TREE_CODE (mask) != VECTOR_CST)
3616 : return 0;
3617 :
3618 34 : if (!VECTOR_CST_NELTS (mask).is_constant (&nelts))
3619 : return 0;
3620 60 : for (i = 0; i < nelts; i++)
3621 : {
3622 60 : tree val = VECTOR_CST_ELT (mask, i);
3623 60 : gcc_assert (TREE_CODE (val) == INTEGER_CST);
3624 60 : j = TREE_INT_CST_LOW (val) & (2 * nelts - 1);
3625 60 : if (j == i)
3626 : maybe_identity2 = false;
3627 47 : else if (j == i + nelts)
3628 : maybe_identity1 = false;
3629 : else
3630 : return 0;
3631 : }
3632 0 : return maybe_identity1 ? 1 : maybe_identity2 ? 2 : 0;
3633 : }
3634 :
3635 : /* Combine a shuffle with its arguments. Returns true if there were any
3636 : changes made. */
3637 :
3638 : static bool
3639 184588 : simplify_permutation (gimple_stmt_iterator *gsi)
3640 : {
3641 184588 : gimple *stmt = gsi_stmt (*gsi);
3642 184588 : gimple *def_stmt = NULL;
3643 184588 : tree op0, op1, op2, op3, arg0, arg1;
3644 184588 : enum tree_code code, code2 = ERROR_MARK;
3645 184588 : bool single_use_op0 = false;
3646 :
3647 184588 : gcc_checking_assert (gimple_assign_rhs_code (stmt) == VEC_PERM_EXPR);
3648 :
3649 184588 : op0 = gimple_assign_rhs1 (stmt);
3650 184588 : op1 = gimple_assign_rhs2 (stmt);
3651 184588 : op2 = gimple_assign_rhs3 (stmt);
3652 :
3653 184588 : if (TREE_CODE (op2) != VECTOR_CST)
3654 : return false;
3655 :
3656 181819 : if (TREE_CODE (op0) == VECTOR_CST)
3657 : {
3658 : code = VECTOR_CST;
3659 : arg0 = op0;
3660 : }
3661 179957 : else if (TREE_CODE (op0) == SSA_NAME)
3662 : {
3663 179957 : def_stmt = get_prop_source_stmt (op0, false, &single_use_op0);
3664 179957 : if (!def_stmt)
3665 : return false;
3666 171895 : code = gimple_assign_rhs_code (def_stmt);
3667 171895 : if (code == VIEW_CONVERT_EXPR)
3668 : {
3669 1433 : tree rhs = gimple_assign_rhs1 (def_stmt);
3670 1433 : tree name = TREE_OPERAND (rhs, 0);
3671 1433 : if (TREE_CODE (name) != SSA_NAME)
3672 : return false;
3673 1433 : if (!has_single_use (name))
3674 216 : single_use_op0 = false;
3675 : /* Here we update the def_stmt through this VIEW_CONVERT_EXPR,
3676 : but still keep the code to indicate it comes from
3677 : VIEW_CONVERT_EXPR. */
3678 1433 : def_stmt = SSA_NAME_DEF_STMT (name);
3679 1433 : if (!def_stmt || !is_gimple_assign (def_stmt))
3680 : return false;
3681 646 : if (gimple_assign_rhs_code (def_stmt) != CONSTRUCTOR)
3682 : return false;
3683 : }
3684 170600 : if (!can_propagate_from (def_stmt))
3685 : return false;
3686 21628 : arg0 = gimple_assign_rhs1 (def_stmt);
3687 : }
3688 : else
3689 : return false;
3690 :
3691 : /* Two consecutive shuffles. */
3692 21628 : if (code == VEC_PERM_EXPR)
3693 : {
3694 6451 : tree orig;
3695 6451 : int ident;
3696 :
3697 6451 : if (op0 != op1)
3698 : return false;
3699 34 : op3 = gimple_assign_rhs3 (def_stmt);
3700 34 : if (TREE_CODE (op3) != VECTOR_CST)
3701 : return false;
3702 34 : ident = is_combined_permutation_identity (op3, op2);
3703 34 : if (!ident)
3704 : return false;
3705 0 : orig = (ident == 1) ? gimple_assign_rhs1 (def_stmt)
3706 0 : : gimple_assign_rhs2 (def_stmt);
3707 0 : gimple_assign_set_rhs1 (stmt, unshare_expr (orig));
3708 0 : gimple_assign_set_rhs_code (stmt, TREE_CODE (orig));
3709 0 : gimple_set_num_ops (stmt, 2);
3710 0 : update_stmt (stmt);
3711 0 : remove_prop_source_from_use (op0);
3712 0 : return true;
3713 : }
3714 17039 : else if (code == CONSTRUCTOR
3715 17039 : || code == VECTOR_CST
3716 : || code == VIEW_CONVERT_EXPR)
3717 : {
3718 3121 : if (op0 != op1)
3719 : {
3720 2949 : if (TREE_CODE (op0) == SSA_NAME && !single_use_op0)
3721 : return false;
3722 :
3723 2746 : if (TREE_CODE (op1) == VECTOR_CST)
3724 : arg1 = op1;
3725 2246 : else if (TREE_CODE (op1) == SSA_NAME)
3726 : {
3727 2246 : gimple *def_stmt2 = get_prop_source_stmt (op1, true, NULL);
3728 2246 : if (!def_stmt2)
3729 : return false;
3730 785 : code2 = gimple_assign_rhs_code (def_stmt2);
3731 785 : if (code2 == VIEW_CONVERT_EXPR)
3732 : {
3733 4 : tree rhs = gimple_assign_rhs1 (def_stmt2);
3734 4 : tree name = TREE_OPERAND (rhs, 0);
3735 4 : if (TREE_CODE (name) != SSA_NAME)
3736 : return false;
3737 4 : if (!has_single_use (name))
3738 : return false;
3739 3 : def_stmt2 = SSA_NAME_DEF_STMT (name);
3740 3 : if (!def_stmt2 || !is_gimple_assign (def_stmt2))
3741 : return false;
3742 0 : if (gimple_assign_rhs_code (def_stmt2) != CONSTRUCTOR)
3743 : return false;
3744 : }
3745 781 : else if (code2 != CONSTRUCTOR && code2 != VECTOR_CST)
3746 : return false;
3747 651 : if (!can_propagate_from (def_stmt2))
3748 : return false;
3749 651 : arg1 = gimple_assign_rhs1 (def_stmt2);
3750 : }
3751 : else
3752 : return false;
3753 : }
3754 : else
3755 : {
3756 : /* Already used twice in this statement. */
3757 172 : if (TREE_CODE (op0) == SSA_NAME && num_imm_uses (op0) > 2)
3758 : return false;
3759 : arg1 = arg0;
3760 : }
3761 :
3762 : /* If there are any VIEW_CONVERT_EXPRs found when finding permutation
3763 : operands source, check whether it's valid to transform and prepare
3764 : the required new operands. */
3765 1249 : if (code == VIEW_CONVERT_EXPR || code2 == VIEW_CONVERT_EXPR)
3766 : {
3767 : /* Figure out the target vector type to which operands should be
3768 : converted. If both are CONSTRUCTOR, the types should be the
3769 : same, otherwise, use the one of CONSTRUCTOR. */
3770 24 : tree tgt_type = NULL_TREE;
3771 24 : if (code == VIEW_CONVERT_EXPR)
3772 : {
3773 24 : gcc_assert (gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR);
3774 24 : code = CONSTRUCTOR;
3775 24 : tgt_type = TREE_TYPE (arg0);
3776 : }
3777 24 : if (code2 == VIEW_CONVERT_EXPR)
3778 : {
3779 0 : tree arg1_type = TREE_TYPE (arg1);
3780 0 : if (tgt_type == NULL_TREE)
3781 : tgt_type = arg1_type;
3782 0 : else if (tgt_type != arg1_type)
3783 23 : return false;
3784 : }
3785 :
3786 24 : if (!VECTOR_TYPE_P (tgt_type))
3787 : return false;
3788 24 : tree op2_type = TREE_TYPE (op2);
3789 :
3790 : /* Figure out the shrunk factor. */
3791 24 : poly_uint64 tgt_units = TYPE_VECTOR_SUBPARTS (tgt_type);
3792 24 : poly_uint64 op2_units = TYPE_VECTOR_SUBPARTS (op2_type);
3793 24 : if (maybe_gt (tgt_units, op2_units))
3794 : return false;
3795 24 : unsigned int factor;
3796 47 : if (!constant_multiple_p (op2_units, tgt_units, &factor))
3797 : return false;
3798 :
3799 : /* Build the new permutation control vector as target vector. */
3800 24 : vec_perm_builder builder;
3801 24 : if (!tree_to_vec_perm_builder (&builder, op2))
3802 : return false;
3803 24 : vec_perm_indices indices (builder, 2, op2_units);
3804 24 : vec_perm_indices new_indices;
3805 24 : if (new_indices.new_shrunk_vector (indices, factor))
3806 : {
3807 1 : tree mask_type = tgt_type;
3808 1 : if (!VECTOR_INTEGER_TYPE_P (mask_type))
3809 : {
3810 0 : tree elem_type = TREE_TYPE (mask_type);
3811 0 : unsigned elem_size = TREE_INT_CST_LOW (TYPE_SIZE (elem_type));
3812 0 : tree int_type = build_nonstandard_integer_type (elem_size, 0);
3813 0 : mask_type = build_vector_type (int_type, tgt_units);
3814 : }
3815 1 : op2 = vec_perm_indices_to_tree (mask_type, new_indices);
3816 : }
3817 : else
3818 23 : return false;
3819 :
3820 : /* Convert the VECTOR_CST to the appropriate vector type. */
3821 1 : if (tgt_type != TREE_TYPE (arg0))
3822 0 : arg0 = fold_build1 (VIEW_CONVERT_EXPR, tgt_type, arg0);
3823 1 : else if (tgt_type != TREE_TYPE (arg1))
3824 0 : arg1 = fold_build1 (VIEW_CONVERT_EXPR, tgt_type, arg1);
3825 47 : }
3826 :
3827 : /* VIEW_CONVERT_EXPR should be updated to CONSTRUCTOR before. */
3828 1226 : gcc_assert (code == CONSTRUCTOR || code == VECTOR_CST);
3829 :
3830 : /* Shuffle of a constructor. */
3831 1226 : tree res_type
3832 1226 : = build_vector_type (TREE_TYPE (TREE_TYPE (arg0)),
3833 1226 : TYPE_VECTOR_SUBPARTS (TREE_TYPE (op2)));
3834 1226 : tree opt = fold_ternary (VEC_PERM_EXPR, res_type, arg0, arg1, op2);
3835 1226 : if (!opt
3836 280 : || (TREE_CODE (opt) != CONSTRUCTOR && TREE_CODE (opt) != VECTOR_CST))
3837 : return false;
3838 : /* Found VIEW_CONVERT_EXPR before, need one explicit conversion. */
3839 280 : if (res_type != TREE_TYPE (op0))
3840 : {
3841 1 : tree name = make_ssa_name (TREE_TYPE (opt));
3842 1 : gimple *ass_stmt = gimple_build_assign (name, opt);
3843 1 : gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
3844 1 : opt = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (op0), name);
3845 : }
3846 280 : gimple_assign_set_rhs_from_tree (gsi, opt);
3847 280 : update_stmt (gsi_stmt (*gsi));
3848 280 : if (TREE_CODE (op0) == SSA_NAME)
3849 1 : remove_prop_source_from_use (op0);
3850 280 : if (op0 != op1 && TREE_CODE (op1) == SSA_NAME)
3851 0 : remove_prop_source_from_use (op1);
3852 280 : return true;
3853 : }
3854 :
3855 : return false;
3856 : }
3857 :
3858 : /* Get the BIT_FIELD_REF definition of VAL, if any, looking through
3859 : conversions with code CONV_CODE or update it if still ERROR_MARK.
3860 : Return NULL_TREE if no such matching def was found. */
3861 :
3862 : static tree
3863 403154 : get_bit_field_ref_def (tree val, enum tree_code &conv_code)
3864 : {
3865 403154 : if (TREE_CODE (val) != SSA_NAME)
3866 : return NULL_TREE ;
3867 377419 : gimple *def_stmt = get_prop_source_stmt (val, false, NULL);
3868 377419 : if (!def_stmt)
3869 : return NULL_TREE;
3870 303170 : enum tree_code code = gimple_assign_rhs_code (def_stmt);
3871 303170 : if (code == FLOAT_EXPR
3872 303170 : || code == FIX_TRUNC_EXPR
3873 : || CONVERT_EXPR_CODE_P (code))
3874 : {
3875 181769 : tree op1 = gimple_assign_rhs1 (def_stmt);
3876 181769 : if (conv_code == ERROR_MARK)
3877 87075 : conv_code = code;
3878 94694 : else if (conv_code != code)
3879 : return NULL_TREE;
3880 181744 : if (TREE_CODE (op1) != SSA_NAME)
3881 : return NULL_TREE;
3882 74922 : def_stmt = SSA_NAME_DEF_STMT (op1);
3883 74922 : if (! is_gimple_assign (def_stmt))
3884 : return NULL_TREE;
3885 59977 : code = gimple_assign_rhs_code (def_stmt);
3886 : }
3887 181378 : if (code != BIT_FIELD_REF)
3888 : return NULL_TREE;
3889 23215 : return gimple_assign_rhs1 (def_stmt);
3890 : }
3891 :
3892 : /* Recognize a VEC_PERM_EXPR. Returns true if there were any changes. */
3893 :
3894 : static bool
3895 153302 : simplify_vector_constructor (gimple_stmt_iterator *gsi)
3896 : {
3897 153302 : gimple *stmt = gsi_stmt (*gsi);
3898 153302 : tree op, orig[2], type;
3899 153302 : unsigned i;
3900 153302 : unsigned HOST_WIDE_INT nelts;
3901 153302 : unsigned HOST_WIDE_INT refnelts;
3902 153302 : enum tree_code conv_code;
3903 153302 : constructor_elt *elt;
3904 :
3905 153302 : op = gimple_assign_rhs1 (stmt);
3906 153302 : type = TREE_TYPE (op);
3907 153302 : gcc_checking_assert (TREE_CODE (op) == CONSTRUCTOR
3908 : && TREE_CODE (type) == VECTOR_TYPE);
3909 :
3910 153302 : if (!TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
3911 : return false;
3912 :
3913 153302 : orig[0] = NULL;
3914 153302 : orig[1] = NULL;
3915 153302 : tree orig_elem_type[2] = {};
3916 153302 : conv_code = ERROR_MARK;
3917 153302 : bool maybe_ident = true;
3918 153302 : bool maybe_blend[2] = { true, true };
3919 153302 : tree one_constant = NULL_TREE;
3920 153302 : tree one_nonconstant = NULL_TREE;
3921 153302 : tree subelt;
3922 153302 : auto_vec<tree> constants;
3923 153302 : constants.safe_grow_cleared (nelts, true);
3924 153302 : auto_vec<std::pair<unsigned, unsigned>, 64> elts;
3925 153302 : unsigned int tsubelts = 0;
3926 435120 : FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (op), i, elt)
3927 : {
3928 403154 : tree ref, op1;
3929 403154 : unsigned int elem, src_elem_size;
3930 403154 : unsigned HOST_WIDE_INT nsubelts = 1;
3931 :
3932 403154 : if (i >= nelts)
3933 153302 : return false;
3934 :
3935 : /* Look for elements extracted and possibly converted from
3936 : another vector. */
3937 403154 : op1 = get_bit_field_ref_def (elt->value, conv_code);
3938 403154 : if (op1
3939 23215 : && TREE_CODE ((ref = TREE_OPERAND (op1, 0))) == SSA_NAME
3940 5189 : && VECTOR_TYPE_P (TREE_TYPE (ref))
3941 5182 : && (tree_nop_conversion_p (TREE_TYPE (op1),
3942 5182 : TREE_TYPE (TREE_TYPE (ref)))
3943 840 : || (VECTOR_TYPE_P (TREE_TYPE (op1))
3944 76 : && tree_nop_conversion_p (TREE_TYPE (TREE_TYPE (op1)),
3945 76 : TREE_TYPE (TREE_TYPE (ref)))
3946 76 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (op1))
3947 76 : .is_constant (&nsubelts)))
3948 4418 : && constant_multiple_p (bit_field_size (op1), nsubelts,
3949 : &src_elem_size)
3950 407572 : && constant_multiple_p (bit_field_offset (op1), src_elem_size, &elem)
3951 407572 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (ref)).is_constant (&refnelts))
3952 : {
3953 : unsigned int j;
3954 4715 : for (j = 0; j < 2; ++j)
3955 : {
3956 4696 : if (!orig[j])
3957 : {
3958 2087 : if (j == 0
3959 2289 : || useless_type_conversion_p (TREE_TYPE (orig[0]),
3960 202 : TREE_TYPE (ref)))
3961 : break;
3962 : }
3963 2609 : else if (ref == orig[j])
3964 : break;
3965 : }
3966 : /* Found a suitable vector element. */
3967 4418 : if (j < 2)
3968 : {
3969 4399 : orig[j] = ref;
3970 : /* Track what element type was actually extracted (which may
3971 : differ in signedness from the vector's element type due to
3972 : tree_nop_conversion_p). */
3973 4399 : if (!orig_elem_type[j])
3974 2081 : orig_elem_type[j] = TREE_TYPE (op1);
3975 4399 : if (elem != i || j != 0)
3976 1965 : maybe_ident = false;
3977 4399 : if (elem != i)
3978 1908 : maybe_blend[j] = false;
3979 8933 : for (unsigned int k = 0; k < nsubelts; ++k)
3980 4534 : elts.safe_push (std::make_pair (j, elem + k));
3981 4399 : tsubelts += nsubelts;
3982 4399 : continue;
3983 4399 : }
3984 : /* Else fallthru. */
3985 : }
3986 : /* Handle elements not extracted from a vector.
3987 : 1. constants by permuting with constant vector
3988 : 2. a unique non-constant element by permuting with a splat vector */
3989 398755 : if (orig[1]
3990 246549 : && orig[1] != error_mark_node)
3991 : return false;
3992 398738 : orig[1] = error_mark_node;
3993 398738 : if (VECTOR_TYPE_P (TREE_TYPE (elt->value))
3994 398738 : && !TYPE_VECTOR_SUBPARTS (TREE_TYPE (elt->value))
3995 3122 : .is_constant (&nsubelts))
3996 : return false;
3997 398738 : if (CONSTANT_CLASS_P (elt->value))
3998 : {
3999 25731 : if (one_nonconstant)
4000 : return false;
4001 17433 : if (!one_constant)
4002 8175 : one_constant = TREE_CODE (elt->value) == VECTOR_CST
4003 8175 : ? VECTOR_CST_ELT (elt->value, 0)
4004 : : elt->value;
4005 17433 : if (TREE_CODE (elt->value) == VECTOR_CST)
4006 : {
4007 347 : for (unsigned int k = 0; k < nsubelts; k++)
4008 255 : constants[tsubelts + k] = VECTOR_CST_ELT (elt->value, k);
4009 : }
4010 : else
4011 17341 : constants[tsubelts] = elt->value;
4012 : }
4013 : else
4014 : {
4015 373007 : if (one_constant)
4016 : return false;
4017 365122 : subelt = VECTOR_TYPE_P (TREE_TYPE (elt->value))
4018 365122 : ? ssa_uniform_vector_p (elt->value)
4019 : : elt->value;
4020 365122 : if (!subelt)
4021 : return false;
4022 362194 : if (!one_nonconstant)
4023 : one_nonconstant = subelt;
4024 221091 : else if (!operand_equal_p (one_nonconstant, subelt, 0))
4025 : return false;
4026 : }
4027 555003 : for (unsigned int k = 0; k < nsubelts; ++k)
4028 277584 : elts.safe_push (std::make_pair (1, tsubelts + k));
4029 277419 : tsubelts += nsubelts;
4030 277419 : maybe_ident = false;
4031 : }
4032 :
4033 63932 : if (elts.length () < nelts)
4034 : return false;
4035 :
4036 30718 : if (! orig[0]
4037 30718 : || ! VECTOR_TYPE_P (TREE_TYPE (orig[0])))
4038 : return false;
4039 1383 : refnelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (orig[0])).to_constant ();
4040 : /* We currently do not handle larger destination vectors. */
4041 1383 : if (refnelts < nelts)
4042 : return false;
4043 :
4044 : /* Determine the element type for the conversion source.
4045 : As orig_elem_type keeps track of the original type, check
4046 : if we need to perform a sign swap after permuting.
4047 : We need to be able to construct a vector type from the element
4048 : type which is not possible for e.g. BitInt or pointers
4049 : so pun with an integer type if needed. */
4050 1222 : tree perm_eltype = TREE_TYPE (TREE_TYPE (orig[0]));
4051 1222 : bool sign_change_p = false;
4052 1222 : if (conv_code != ERROR_MARK
4053 300 : && orig_elem_type[0]
4054 1522 : && TYPE_SIGN (orig_elem_type[0]) != TYPE_SIGN (perm_eltype))
4055 : {
4056 38 : perm_eltype = signed_or_unsigned_type_for
4057 38 : (TYPE_UNSIGNED (orig_elem_type[0]), perm_eltype);
4058 38 : sign_change_p = true;
4059 : }
4060 1222 : tree conv_src_type = build_vector_type (perm_eltype, nelts);
4061 :
4062 1222 : if (maybe_ident)
4063 : {
4064 : /* When there is no conversion, use the target type directly. */
4065 439 : if (conv_code == ERROR_MARK && nelts != refnelts)
4066 439 : conv_src_type = type;
4067 439 : if (conv_code != ERROR_MARK
4068 439 : && !supportable_convert_operation (conv_code, type, conv_src_type,
4069 : &conv_code))
4070 : {
4071 : /* Only few targets implement direct conversion patterns so try
4072 : some simple special cases via VEC_[UN]PACK[_FLOAT]_LO_EXPR. */
4073 46 : optab optab;
4074 46 : insn_code icode;
4075 46 : tree halfvectype, dblvectype;
4076 46 : enum tree_code unpack_op;
4077 :
4078 46 : if (!BYTES_BIG_ENDIAN)
4079 84 : unpack_op = (FLOAT_TYPE_P (TREE_TYPE (type))
4080 46 : ? VEC_UNPACK_FLOAT_LO_EXPR
4081 : : VEC_UNPACK_LO_EXPR);
4082 : else
4083 : unpack_op = (FLOAT_TYPE_P (TREE_TYPE (type))
4084 : ? VEC_UNPACK_FLOAT_HI_EXPR
4085 : : VEC_UNPACK_HI_EXPR);
4086 :
4087 : /* Conversions between DFP and FP have no special tree code
4088 : but we cannot handle those since all relevant vector conversion
4089 : optabs only have a single mode. */
4090 6 : if (CONVERT_EXPR_CODE_P (conv_code)
4091 40 : && FLOAT_TYPE_P (TREE_TYPE (type))
4092 50 : && (DECIMAL_FLOAT_TYPE_P (TREE_TYPE (type))
4093 2 : != DECIMAL_FLOAT_TYPE_P (TREE_TYPE (conv_src_type))))
4094 : return false;
4095 :
4096 6 : if (CONVERT_EXPR_CODE_P (conv_code)
4097 39 : && (2 * TYPE_PRECISION (TREE_TYPE (TREE_TYPE (orig[0])))
4098 39 : == TYPE_PRECISION (TREE_TYPE (type)))
4099 0 : && orig_elem_type[0]
4100 0 : && useless_type_conversion_p (orig_elem_type[0],
4101 0 : TREE_TYPE (TREE_TYPE (orig[0])))
4102 0 : && mode_for_vector (as_a <scalar_mode>
4103 0 : (TYPE_MODE (TREE_TYPE (TREE_TYPE (orig[0])))),
4104 0 : nelts * 2).exists ()
4105 0 : && (dblvectype
4106 0 : = build_vector_type (TREE_TYPE (TREE_TYPE (orig[0])),
4107 0 : nelts * 2))
4108 : /* Only use it for vector modes or for vector booleans
4109 : represented as scalar bitmasks. See PR95528. */
4110 0 : && (VECTOR_MODE_P (TYPE_MODE (dblvectype))
4111 0 : || VECTOR_BOOLEAN_TYPE_P (dblvectype))
4112 0 : && (optab = optab_for_tree_code (unpack_op,
4113 : dblvectype,
4114 : optab_default))
4115 0 : && ((icode = optab_handler (optab, TYPE_MODE (dblvectype)))
4116 : != CODE_FOR_nothing)
4117 45 : && (insn_data[icode].operand[0].mode == TYPE_MODE (type)))
4118 : {
4119 0 : gimple_seq stmts = NULL;
4120 0 : tree dbl;
4121 0 : if (refnelts == nelts)
4122 : {
4123 : /* ??? Paradoxical subregs don't exist, so insert into
4124 : the lower half of a wider zero vector. */
4125 0 : dbl = gimple_build (&stmts, BIT_INSERT_EXPR, dblvectype,
4126 : build_zero_cst (dblvectype), orig[0],
4127 0 : bitsize_zero_node);
4128 : }
4129 0 : else if (refnelts == 2 * nelts)
4130 : dbl = orig[0];
4131 : else
4132 0 : dbl = gimple_build (&stmts, BIT_FIELD_REF, dblvectype,
4133 0 : orig[0], TYPE_SIZE (dblvectype),
4134 0 : bitsize_zero_node);
4135 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
4136 0 : gimple_assign_set_rhs_with_ops (gsi, unpack_op, dbl);
4137 : }
4138 6 : else if (CONVERT_EXPR_CODE_P (conv_code)
4139 39 : && (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (orig[0])))
4140 39 : == 2 * TYPE_PRECISION (TREE_TYPE (type)))
4141 1 : && orig_elem_type[0]
4142 1 : && useless_type_conversion_p (orig_elem_type[0],
4143 1 : TREE_TYPE (TREE_TYPE (orig[0])))
4144 1 : && mode_for_vector (as_a <scalar_mode>
4145 1 : (TYPE_MODE
4146 : (TREE_TYPE (TREE_TYPE (orig[0])))),
4147 2 : nelts / 2).exists ()
4148 1 : && (halfvectype
4149 1 : = build_vector_type (TREE_TYPE (TREE_TYPE (orig[0])),
4150 1 : nelts / 2))
4151 : /* Only use it for vector modes or for vector booleans
4152 : represented as scalar bitmasks. See PR95528. */
4153 1 : && (VECTOR_MODE_P (TYPE_MODE (halfvectype))
4154 0 : || VECTOR_BOOLEAN_TYPE_P (halfvectype))
4155 1 : && (optab = optab_for_tree_code (VEC_PACK_TRUNC_EXPR,
4156 : halfvectype,
4157 : optab_default))
4158 1 : && ((icode = optab_handler (optab, TYPE_MODE (halfvectype)))
4159 : != CODE_FOR_nothing)
4160 46 : && (insn_data[icode].operand[0].mode == TYPE_MODE (type)))
4161 : {
4162 0 : gimple_seq stmts = NULL;
4163 0 : tree low = gimple_build (&stmts, BIT_FIELD_REF, halfvectype,
4164 0 : orig[0], TYPE_SIZE (halfvectype),
4165 0 : bitsize_zero_node);
4166 0 : tree hig = gimple_build (&stmts, BIT_FIELD_REF, halfvectype,
4167 0 : orig[0], TYPE_SIZE (halfvectype),
4168 0 : TYPE_SIZE (halfvectype));
4169 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
4170 0 : gimple_assign_set_rhs_with_ops (gsi, VEC_PACK_TRUNC_EXPR,
4171 : low, hig);
4172 : }
4173 : else
4174 45 : return false;
4175 0 : update_stmt (gsi_stmt (*gsi));
4176 0 : return true;
4177 : }
4178 393 : if (nelts != refnelts)
4179 : {
4180 20 : gassign *lowpart
4181 20 : = gimple_build_assign (make_ssa_name (conv_src_type),
4182 : build3 (BIT_FIELD_REF, conv_src_type,
4183 20 : orig[0], TYPE_SIZE (conv_src_type),
4184 : bitsize_zero_node));
4185 20 : gsi_insert_before (gsi, lowpart, GSI_SAME_STMT);
4186 20 : orig[0] = gimple_assign_lhs (lowpart);
4187 : }
4188 373 : else if (sign_change_p)
4189 : {
4190 0 : gassign *conv
4191 0 : = gimple_build_assign (make_ssa_name (conv_src_type),
4192 : build1 (VIEW_CONVERT_EXPR, conv_src_type,
4193 : orig[0]));
4194 0 : gsi_insert_before (gsi, conv, GSI_SAME_STMT);
4195 0 : orig[0] = gimple_assign_lhs (conv);
4196 : }
4197 393 : if (conv_code == ERROR_MARK)
4198 : {
4199 376 : tree src_type = TREE_TYPE (orig[0]);
4200 376 : if (!useless_type_conversion_p (type, src_type))
4201 : {
4202 0 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (type),
4203 : TYPE_VECTOR_SUBPARTS (src_type))
4204 : && tree_nop_conversion_p (TREE_TYPE (type),
4205 : TREE_TYPE (src_type)));
4206 0 : tree rhs = build1 (VIEW_CONVERT_EXPR, type, orig[0]);
4207 0 : orig[0] = make_ssa_name (type);
4208 0 : gassign *assign = gimple_build_assign (orig[0], rhs);
4209 0 : gsi_insert_before (gsi, assign, GSI_SAME_STMT);
4210 : }
4211 376 : gimple_assign_set_rhs_from_tree (gsi, orig[0]);
4212 : }
4213 : else
4214 17 : gimple_assign_set_rhs_with_ops (gsi, conv_code, orig[0],
4215 : NULL_TREE, NULL_TREE);
4216 : }
4217 : else
4218 : {
4219 : /* If we combine a vector with a non-vector avoid cases where
4220 : we'll obviously end up with more GIMPLE stmts which is when
4221 : we'll later not fold this to a single insert into the vector
4222 : and we had a single extract originally. See PR92819. */
4223 783 : if (nelts == 2
4224 536 : && refnelts > 2
4225 159 : && orig[1] == error_mark_node
4226 34 : && !maybe_blend[0])
4227 240 : return false;
4228 755 : tree mask_type, perm_type;
4229 755 : perm_type = TREE_TYPE (orig[0]);
4230 755 : if (conv_code != ERROR_MARK
4231 755 : && !supportable_convert_operation (conv_code, type, conv_src_type,
4232 : &conv_code))
4233 : return false;
4234 :
4235 : /* Now that we know the number of elements of the source build the
4236 : permute vector.
4237 : ??? When the second vector has constant values we can shuffle
4238 : it and its source indexes to make the permutation supported.
4239 : For now it mimics a blend. */
4240 584 : vec_perm_builder sel (refnelts, refnelts, 1);
4241 584 : bool all_same_p = true;
4242 5838 : for (i = 0; i < elts.length (); ++i)
4243 : {
4244 2335 : sel.quick_push (elts[i].second + elts[i].first * refnelts);
4245 2335 : all_same_p &= known_eq (sel[i], sel[0]);
4246 : }
4247 : /* And fill the tail with "something". It's really don't care,
4248 : and ideally we'd allow VEC_PERM to have a smaller destination
4249 : vector. As a heuristic:
4250 :
4251 : (a) if what we have so far duplicates a single element, make the
4252 : tail do the same
4253 :
4254 : (b) otherwise preserve a uniform orig[0]. This facilitates
4255 : later pattern-matching of VEC_PERM_EXPR to a BIT_INSERT_EXPR. */
4256 1263 : for (; i < refnelts; ++i)
4257 1358 : sel.quick_push (all_same_p
4258 2037 : ? sel[0]
4259 92 : : (elts[0].second == 0 && elts[0].first == 0
4260 1076 : ? 0 : refnelts) + i);
4261 787 : vec_perm_indices indices (sel, orig[1] ? 2 : 1, refnelts);
4262 584 : machine_mode vmode = TYPE_MODE (perm_type);
4263 584 : if ((cfun->curr_properties & PROP_gimple_lvec)
4264 584 : && !can_vec_perm_const_p (vmode, vmode, indices))
4265 : return false;
4266 543 : mask_type = build_vector_type (ssizetype, refnelts);
4267 543 : tree op2 = vec_perm_indices_to_tree (mask_type, indices);
4268 543 : bool converted_orig1 = false;
4269 543 : gimple_seq stmts = NULL;
4270 543 : if (!orig[1])
4271 190 : orig[1] = orig[0];
4272 353 : else if (orig[1] == error_mark_node
4273 214 : && one_nonconstant)
4274 : {
4275 : /* ??? We can see if we can safely convert to the original
4276 : element type. */
4277 87 : converted_orig1 = conv_code != ERROR_MARK;
4278 87 : tree target_type = converted_orig1 ? type : perm_type;
4279 87 : tree nonconstant_for_splat = one_nonconstant;
4280 : /* If there's a nop conversion between the target element type and
4281 : the nonconstant's type, convert it. */
4282 87 : if (!useless_type_conversion_p (TREE_TYPE (target_type),
4283 87 : TREE_TYPE (one_nonconstant)))
4284 0 : nonconstant_for_splat
4285 0 : = gimple_build (&stmts, NOP_EXPR, TREE_TYPE (target_type),
4286 : one_nonconstant);
4287 87 : orig[1] = gimple_build_vector_from_val (&stmts, UNKNOWN_LOCATION,
4288 : target_type,
4289 : nonconstant_for_splat);
4290 87 : }
4291 266 : else if (orig[1] == error_mark_node)
4292 : {
4293 : /* ??? See if we can convert the vector to the original type. */
4294 127 : converted_orig1 = conv_code != ERROR_MARK;
4295 127 : unsigned n = converted_orig1 ? nelts : refnelts;
4296 110 : tree target_type = converted_orig1 ? type : perm_type;
4297 127 : tree_vector_builder vec (target_type, n, 1);
4298 1011 : for (unsigned i = 0; i < n; ++i)
4299 1692 : if (i < nelts && constants[i])
4300 : {
4301 451 : tree constant = constants[i];
4302 : /* If there's a nop conversion, convert the constant. */
4303 451 : if (!useless_type_conversion_p (TREE_TYPE (target_type),
4304 451 : TREE_TYPE (constant)))
4305 2 : constant = fold_convert (TREE_TYPE (target_type), constant);
4306 451 : vec.quick_push (constant);
4307 : }
4308 : else
4309 : {
4310 : /* ??? Push a don't-care value. */
4311 433 : tree constant = one_constant;
4312 433 : if (!useless_type_conversion_p (TREE_TYPE (target_type),
4313 433 : TREE_TYPE (constant)))
4314 2 : constant = fold_convert (TREE_TYPE (target_type), constant);
4315 433 : vec.quick_push (constant);
4316 : }
4317 127 : orig[1] = vec.build ();
4318 127 : }
4319 404 : tree blend_op2 = NULL_TREE;
4320 404 : if (converted_orig1)
4321 : {
4322 : /* Make sure we can do a blend in the target type. */
4323 19 : vec_perm_builder sel (nelts, nelts, 1);
4324 87 : for (i = 0; i < elts.length (); ++i)
4325 68 : sel.quick_push (elts[i].first
4326 68 : ? elts[i].second + nelts : i);
4327 19 : vec_perm_indices indices (sel, 2, nelts);
4328 19 : machine_mode vmode = TYPE_MODE (type);
4329 19 : if ((cfun->curr_properties & PROP_gimple_lvec)
4330 19 : && !can_vec_perm_const_p (vmode, vmode, indices))
4331 0 : return false;
4332 19 : mask_type = build_vector_type (ssizetype, nelts);
4333 19 : blend_op2 = vec_perm_indices_to_tree (mask_type, indices);
4334 19 : }
4335 :
4336 : /* For a real orig[1] (no splat, constant etc.) we might need to
4337 : nop-convert it. Do so here. */
4338 543 : if (orig[1] && orig[1] != error_mark_node
4339 543 : && !useless_type_conversion_p (perm_type, TREE_TYPE (orig[1]))
4340 562 : && tree_nop_conversion_p (TREE_TYPE (perm_type),
4341 19 : TREE_TYPE (TREE_TYPE (orig[1]))))
4342 0 : orig[1] = gimple_build (&stmts, VIEW_CONVERT_EXPR, perm_type,
4343 : orig[1]);
4344 :
4345 543 : tree orig1_for_perm
4346 543 : = converted_orig1 ? build_zero_cst (perm_type) : orig[1];
4347 543 : tree res = gimple_build (&stmts, VEC_PERM_EXPR, perm_type,
4348 : orig[0], orig1_for_perm, op2);
4349 : /* If we're building a smaller vector, extract the element
4350 : with the proper type. */
4351 543 : if (nelts != refnelts)
4352 298 : res = gimple_build (&stmts, BIT_FIELD_REF,
4353 : conv_code != ERROR_MARK ? conv_src_type : type,
4354 : res,
4355 149 : TYPE_SIZE (conv_code != ERROR_MARK ? conv_src_type
4356 : : type),
4357 149 : bitsize_zero_node);
4358 : /* Otherwise, we can still have an intermediate sign change.
4359 : ??? In that case we have two subsequent conversions.
4360 : We should be able to merge them. */
4361 394 : else if (sign_change_p)
4362 14 : res = gimple_build (&stmts, VIEW_CONVERT_EXPR, conv_src_type, res);
4363 : /* Finally, apply the conversion. */
4364 543 : if (conv_code != ERROR_MARK)
4365 52 : res = gimple_build (&stmts, conv_code, type, res);
4366 491 : else if (!useless_type_conversion_p (type, TREE_TYPE (res)))
4367 : {
4368 3 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (type),
4369 : TYPE_VECTOR_SUBPARTS (perm_type))
4370 : && tree_nop_conversion_p (TREE_TYPE (type),
4371 : TREE_TYPE (perm_type)));
4372 3 : res = gimple_build (&stmts, VIEW_CONVERT_EXPR, type, res);
4373 : }
4374 : /* Blend in the actual constant. */
4375 543 : if (converted_orig1)
4376 19 : res = gimple_build (&stmts, VEC_PERM_EXPR, type,
4377 19 : res, orig[1], blend_op2);
4378 543 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
4379 543 : gimple_assign_set_rhs_with_ops (gsi, SSA_NAME, res);
4380 584 : }
4381 936 : update_stmt (gsi_stmt (*gsi));
4382 936 : return true;
4383 153302 : }
4384 :
4385 : /* Prepare a TARGET_MEM_REF ref so that it can be subsetted as
4386 : lvalue. This splits out an address computation stmt before *GSI
4387 : and returns a MEM_REF wrapping the address. */
4388 :
4389 : static tree
4390 1096 : prepare_target_mem_ref_lvalue (tree ref, gimple_stmt_iterator *gsi)
4391 : {
4392 1096 : if (TREE_CODE (TREE_OPERAND (ref, 0)) == ADDR_EXPR)
4393 215 : mark_addressable (TREE_OPERAND (TREE_OPERAND (ref, 0), 0));
4394 1096 : tree ptrtype = build_pointer_type (TREE_TYPE (ref));
4395 1096 : tree tem = make_ssa_name (ptrtype);
4396 1096 : gimple *new_stmt
4397 1096 : = gimple_build_assign (tem, build1 (ADDR_EXPR, TREE_TYPE (tem),
4398 : unshare_expr (ref)));
4399 1096 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4400 2192 : ref = build2_loc (EXPR_LOCATION (ref),
4401 1096 : MEM_REF, TREE_TYPE (ref), tem,
4402 1096 : build_int_cst (TREE_TYPE (TREE_OPERAND (ref, 1)), 0));
4403 1096 : return ref;
4404 : }
4405 :
4406 : /* Rewrite the vector load at *GSI to component-wise loads if the load
4407 : is only used in BIT_FIELD_REF extractions with eventual intermediate
4408 : widening. */
4409 :
4410 : static void
4411 289721 : optimize_vector_load (gimple_stmt_iterator *gsi)
4412 : {
4413 289721 : gimple *stmt = gsi_stmt (*gsi);
4414 289721 : tree lhs = gimple_assign_lhs (stmt);
4415 289721 : tree rhs = gimple_assign_rhs1 (stmt);
4416 289721 : tree vuse = gimple_vuse (stmt);
4417 :
4418 : /* Gather BIT_FIELD_REFs to rewrite, looking through
4419 : VEC_UNPACK_{LO,HI}_EXPR. */
4420 289721 : use_operand_p use_p;
4421 289721 : imm_use_iterator iter;
4422 289721 : bool rewrite = true;
4423 289721 : bool scalar_use = false;
4424 289721 : bool unpack_use = false;
4425 289721 : auto_vec<gimple *, 8> bf_stmts;
4426 289721 : auto_vec<tree, 8> worklist;
4427 289721 : worklist.quick_push (lhs);
4428 291567 : do
4429 : {
4430 291567 : tree def = worklist.pop ();
4431 291567 : unsigned HOST_WIDE_INT def_eltsize
4432 291567 : = TREE_INT_CST_LOW (TYPE_SIZE (TREE_TYPE (TREE_TYPE (def))));
4433 659213 : FOR_EACH_IMM_USE_FAST (use_p, iter, def)
4434 : {
4435 347288 : gimple *use_stmt = USE_STMT (use_p);
4436 347288 : if (is_gimple_debug (use_stmt))
4437 76079 : continue;
4438 346701 : if (!is_gimple_assign (use_stmt))
4439 : {
4440 : rewrite = false;
4441 271209 : break;
4442 : }
4443 310937 : enum tree_code use_code = gimple_assign_rhs_code (use_stmt);
4444 310937 : tree use_rhs = gimple_assign_rhs1 (use_stmt);
4445 382726 : if (use_code == BIT_FIELD_REF
4446 71790 : && TREE_OPERAND (use_rhs, 0) == def
4447 : /* If its on the VEC_UNPACK_{HI,LO}_EXPR
4448 : def need to verify it is element aligned. */
4449 382727 : && (def == lhs
4450 85 : || (known_eq (bit_field_size (use_rhs), def_eltsize)
4451 85 : && constant_multiple_p (bit_field_offset (use_rhs),
4452 : def_eltsize)
4453 : /* We can simulate the VEC_UNPACK_{HI,LO}_EXPR
4454 : via a NOP_EXPR only for integral types.
4455 : ??? Support VEC_UNPACK_FLOAT_{HI,LO}_EXPR. */
4456 85 : && INTEGRAL_TYPE_P (TREE_TYPE (use_rhs)))))
4457 : {
4458 71789 : if (!VECTOR_TYPE_P (TREE_TYPE (gimple_assign_lhs (use_stmt))))
4459 69675 : scalar_use = true;
4460 71789 : bf_stmts.safe_push (use_stmt);
4461 71789 : continue;
4462 : }
4463 : /* Walk through one level of VEC_UNPACK_{LO,HI}_EXPR. */
4464 239148 : if (def == lhs
4465 237349 : && (use_code == VEC_UNPACK_HI_EXPR
4466 237349 : || use_code == VEC_UNPACK_LO_EXPR)
4467 3703 : && use_rhs == lhs)
4468 : {
4469 3703 : unpack_use = true;
4470 3703 : worklist.safe_push (gimple_assign_lhs (use_stmt));
4471 3703 : continue;
4472 : }
4473 : rewrite = false;
4474 : break;
4475 291567 : }
4476 291567 : if (!rewrite)
4477 : break;
4478 : }
4479 40716 : while (!worklist.is_empty ());
4480 :
4481 289721 : rewrite = rewrite && (scalar_use
4482 18512 : || unpack_use
4483 588 : || !can_implement_p (mov_optab,
4484 588 : TYPE_MODE (TREE_TYPE (lhs))));
4485 289721 : if (!rewrite)
4486 : {
4487 271379 : gsi_next (gsi);
4488 271379 : return;
4489 : }
4490 : /* We now have all ultimate uses of the load to rewrite in bf_stmts. */
4491 :
4492 : /* Prepare the original ref to be wrapped in adjusted BIT_FIELD_REFs.
4493 : For TARGET_MEM_REFs we have to separate the LEA from the reference. */
4494 18342 : tree load_rhs = rhs;
4495 18342 : if (TREE_CODE (load_rhs) == TARGET_MEM_REF)
4496 1095 : load_rhs = prepare_target_mem_ref_lvalue (load_rhs, gsi);
4497 :
4498 : /* Rewrite the BIT_FIELD_REFs to be actual loads, re-emitting them at
4499 : the place of the original load. */
4500 120956 : for (gimple *use_stmt : bf_stmts)
4501 : {
4502 65930 : tree bfr = gimple_assign_rhs1 (use_stmt);
4503 65930 : tree new_rhs = unshare_expr (load_rhs);
4504 65930 : if (TREE_OPERAND (bfr, 0) != lhs)
4505 : {
4506 : /* When the BIT_FIELD_REF is on the promoted vector we have to
4507 : adjust it and emit a conversion afterwards. */
4508 84 : gimple *def_stmt
4509 84 : = SSA_NAME_DEF_STMT (TREE_OPERAND (bfr, 0));
4510 84 : enum tree_code def_code
4511 84 : = gimple_assign_rhs_code (def_stmt);
4512 :
4513 : /* The adjusted BIT_FIELD_REF is of the promotion source
4514 : vector size and at half of the offset... */
4515 84 : new_rhs = fold_build3 (BIT_FIELD_REF,
4516 : TREE_TYPE (TREE_TYPE (lhs)),
4517 : new_rhs,
4518 : TYPE_SIZE (TREE_TYPE (TREE_TYPE (lhs))),
4519 : size_binop (EXACT_DIV_EXPR,
4520 : TREE_OPERAND (bfr, 2),
4521 : bitsize_int (2)));
4522 : /* ... and offsetted by half of the vector if VEC_UNPACK_HI_EXPR. */
4523 84 : if (def_code == (!BYTES_BIG_ENDIAN
4524 : ? VEC_UNPACK_HI_EXPR : VEC_UNPACK_LO_EXPR))
4525 42 : TREE_OPERAND (new_rhs, 2)
4526 84 : = size_binop (PLUS_EXPR, TREE_OPERAND (new_rhs, 2),
4527 : size_binop (EXACT_DIV_EXPR,
4528 : TYPE_SIZE (TREE_TYPE (lhs)),
4529 : bitsize_int (2)));
4530 84 : tree tem = make_ssa_name (TREE_TYPE (TREE_TYPE (lhs)));
4531 84 : gimple *new_stmt = gimple_build_assign (tem, new_rhs);
4532 84 : location_t loc = gimple_location (use_stmt);
4533 84 : gimple_set_location (new_stmt, loc);
4534 84 : gimple_set_vuse (new_stmt, vuse);
4535 84 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4536 : /* Perform scalar promotion. */
4537 84 : new_stmt = gimple_build_assign (gimple_assign_lhs (use_stmt),
4538 : NOP_EXPR, tem);
4539 84 : gimple_set_location (new_stmt, loc);
4540 84 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4541 : }
4542 : else
4543 : {
4544 : /* When the BIT_FIELD_REF is on the original load result
4545 : we can just wrap that. */
4546 65846 : tree new_rhs = fold_build3 (BIT_FIELD_REF, TREE_TYPE (bfr),
4547 : unshare_expr (load_rhs),
4548 : TREE_OPERAND (bfr, 1),
4549 : TREE_OPERAND (bfr, 2));
4550 65846 : gimple *new_stmt = gimple_build_assign (gimple_assign_lhs (use_stmt),
4551 : new_rhs);
4552 65846 : location_t loc = gimple_location (use_stmt);
4553 65846 : gimple_set_location (new_stmt, loc);
4554 65846 : gimple_set_vuse (new_stmt, vuse);
4555 65846 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
4556 : }
4557 65930 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
4558 65930 : unlink_stmt_vdef (use_stmt);
4559 65930 : gsi_remove (&gsi2, true);
4560 : }
4561 :
4562 : /* Finally get rid of the intermediate stmts. */
4563 18342 : gimple *use_stmt;
4564 36984 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
4565 : {
4566 300 : if (is_gimple_debug (use_stmt))
4567 : {
4568 272 : if (gimple_debug_bind_p (use_stmt))
4569 : {
4570 272 : gimple_debug_bind_reset_value (use_stmt);
4571 272 : update_stmt (use_stmt);
4572 : }
4573 272 : continue;
4574 : }
4575 28 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
4576 28 : unlink_stmt_vdef (use_stmt);
4577 28 : release_defs (use_stmt);
4578 28 : gsi_remove (&gsi2, true);
4579 18342 : }
4580 : /* And the original load. */
4581 18342 : release_defs (stmt);
4582 18342 : gsi_remove (gsi, true);
4583 289721 : }
4584 :
4585 :
4586 : /* Primitive "lattice" function for gimple_simplify. */
4587 :
4588 : static tree
4589 1613734822 : fwprop_ssa_val (tree name)
4590 : {
4591 : /* First valueize NAME. */
4592 1613734822 : if (TREE_CODE (name) == SSA_NAME
4593 1613734822 : && SSA_NAME_VERSION (name) < lattice.length ())
4594 : {
4595 1612871404 : tree val = lattice[SSA_NAME_VERSION (name)];
4596 1612871404 : if (val)
4597 1613734822 : name = val;
4598 : }
4599 : /* We continue matching along SSA use-def edges for SSA names
4600 : that are not single-use. Currently there are no patterns
4601 : that would cause any issues with that. */
4602 1613734822 : return name;
4603 : }
4604 :
4605 : /* Search for opportunities to free half of the lanes in the following pattern:
4606 :
4607 : v_in = {e0, e1, e2, e3}
4608 : v_1 = VEC_PERM <v_in, v_in, {0, 2, 0, 2}>
4609 : // v_1 = {e0, e2, e0, e2}
4610 : v_2 = VEC_PERM <v_in, v_in, {1, 3, 1, 3}>
4611 : // v_2 = {e1, e3, e1, e3}
4612 :
4613 : v_x = v_1 + v_2
4614 : // v_x = {e0+e1, e2+e3, e0+e1, e2+e3}
4615 : v_y = v_1 - v_2
4616 : // v_y = {e0-e1, e2-e3, e0-e1, e2-e3}
4617 :
4618 : v_out = VEC_PERM <v_x, v_y, {0, 1, 6, 7}>
4619 : // v_out = {e0+e1, e2+e3, e0-e1, e2-e3}
4620 :
4621 : The last statement could be simplified to:
4622 : v_out' = VEC_PERM <v_x, v_y, {0, 1, 4, 5}>
4623 : // v_out' = {e0+e1, e2+e3, e0-e1, e2-e3}
4624 :
4625 : Characteristic properties:
4626 : - v_1 and v_2 are created from the same input vector v_in and introduce the
4627 : lane duplication (in the selection operand) that we can eliminate.
4628 : - v_x and v_y are results from lane-preserving operations that use v_1 and
4629 : v_2 as inputs.
4630 : - v_out is created by selecting from duplicated lanes. */
4631 :
4632 : static bool
4633 182530 : recognise_vec_perm_simplify_seq (gassign *stmt, vec_perm_simplify_seq *seq)
4634 : {
4635 182530 : unsigned HOST_WIDE_INT nelts;
4636 :
4637 182530 : gcc_checking_assert (stmt);
4638 182530 : gcc_checking_assert (gimple_assign_rhs_code (stmt) == VEC_PERM_EXPR);
4639 182530 : basic_block bb = gimple_bb (stmt);
4640 :
4641 : /* Decompose the final vec permute statement. */
4642 182530 : tree v_x = gimple_assign_rhs1 (stmt);
4643 182530 : tree v_y = gimple_assign_rhs2 (stmt);
4644 182530 : tree sel = gimple_assign_rhs3 (stmt);
4645 :
4646 182530 : if (TREE_CODE (sel) != VECTOR_CST
4647 179761 : || !VECTOR_CST_NELTS (sel).is_constant (&nelts)
4648 179761 : || TREE_CODE (v_x) != SSA_NAME
4649 177909 : || TREE_CODE (v_y) != SSA_NAME
4650 173869 : || !has_single_use (v_x)
4651 292196 : || !has_single_use (v_y))
4652 74589 : return false;
4653 :
4654 : /* Don't analyse sequences with many lanes. */
4655 107941 : if (nelts > 4)
4656 : return false;
4657 :
4658 : /* Lookup the definition of v_x and v_y. */
4659 105227 : gassign *v_x_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x));
4660 105227 : gassign *v_y_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_y));
4661 104850 : if (!v_x_stmt || gimple_bb (v_x_stmt) != bb
4662 210077 : || !v_y_stmt || gimple_bb (v_y_stmt) != bb)
4663 : return false;
4664 :
4665 : /* Check the operations that define v_x and v_y. */
4666 104843 : if (TREE_CODE_CLASS (gimple_assign_rhs_code (v_x_stmt)) != tcc_binary
4667 106891 : || TREE_CODE_CLASS (gimple_assign_rhs_code (v_y_stmt)) != tcc_binary)
4668 : return false;
4669 :
4670 2048 : tree v_x_1 = gimple_assign_rhs1 (v_x_stmt);
4671 2048 : tree v_x_2 = gimple_assign_rhs2 (v_x_stmt);
4672 2048 : tree v_y_1 = gimple_assign_rhs1 (v_y_stmt);
4673 2048 : tree v_y_2 = gimple_assign_rhs2 (v_y_stmt);
4674 :
4675 2048 : if (v_x_stmt == v_y_stmt
4676 2048 : || TREE_CODE (v_x_1) != SSA_NAME
4677 2045 : || TREE_CODE (v_x_2) != SSA_NAME
4678 2033 : || num_imm_uses (v_x_1) != 2
4679 3924 : || num_imm_uses (v_x_2) != 2)
4680 : return false;
4681 :
4682 1841 : if (v_x_1 != v_y_1 || v_x_2 != v_y_2)
4683 : {
4684 : /* Allow operands of commutative operators to swap. */
4685 635 : if (commutative_tree_code (gimple_assign_rhs_code (v_x_stmt)))
4686 : {
4687 : /* Keep v_x_1 the first operand for non-commutative operators. */
4688 252 : std::swap (v_x_1, v_x_2);
4689 252 : if (v_x_1 != v_y_1 || v_x_2 != v_y_2)
4690 : return false;
4691 : }
4692 383 : else if (commutative_tree_code (gimple_assign_rhs_code (v_y_stmt)))
4693 : {
4694 383 : if (v_x_1 != v_y_2 || v_x_2 != v_y_1)
4695 : return false;
4696 : }
4697 : else
4698 : return false;
4699 : }
4700 1841 : gassign *v_1_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x_1));
4701 1841 : gassign *v_2_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (v_x_2));
4702 1777 : if (!v_1_stmt || gimple_bb (v_1_stmt) != bb
4703 3618 : || !v_2_stmt || gimple_bb (v_2_stmt) != bb)
4704 : return false;
4705 :
4706 1773 : if (gimple_assign_rhs_code (v_1_stmt) != VEC_PERM_EXPR
4707 1895 : || gimple_assign_rhs_code (v_2_stmt) != VEC_PERM_EXPR)
4708 : return false;
4709 :
4710 : /* Decompose initial VEC_PERM_EXPRs. */
4711 108 : tree v_in = gimple_assign_rhs1 (v_1_stmt);
4712 108 : tree v_1_sel = gimple_assign_rhs3 (v_1_stmt);
4713 108 : tree v_2_sel = gimple_assign_rhs3 (v_2_stmt);
4714 108 : if (v_in != gimple_assign_rhs2 (v_1_stmt)
4715 103 : || v_in != gimple_assign_rhs1 (v_2_stmt)
4716 209 : || v_in != gimple_assign_rhs2 (v_2_stmt))
4717 : return false;
4718 :
4719 101 : unsigned HOST_WIDE_INT v_1_nelts, v_2_nelts;
4720 101 : if (TREE_CODE (v_1_sel) != VECTOR_CST
4721 101 : || !VECTOR_CST_NELTS (v_1_sel).is_constant (&v_1_nelts)
4722 101 : || TREE_CODE (v_2_sel) != VECTOR_CST
4723 202 : || !VECTOR_CST_NELTS (v_2_sel).is_constant (&v_2_nelts))
4724 0 : return false;
4725 :
4726 101 : if (nelts != v_1_nelts || nelts != v_2_nelts)
4727 : return false;
4728 :
4729 : /* Create the new selector. */
4730 101 : vec_perm_builder new_sel_perm (nelts, nelts, 1);
4731 101 : auto_vec<bool> lanes (nelts);
4732 101 : lanes.quick_grow_cleared (nelts);
4733 505 : for (unsigned int i = 0; i < nelts; i++)
4734 : {
4735 : /* Extract the i-th value from the selector. */
4736 404 : unsigned int sel_cst = TREE_INT_CST_LOW (VECTOR_CST_ELT (sel, i));
4737 404 : unsigned int lane = sel_cst % nelts;
4738 404 : unsigned int offs = sel_cst / nelts;
4739 :
4740 : /* Check what's in the lane. */
4741 404 : unsigned int e_1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (v_1_sel, lane));
4742 404 : unsigned int e_2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (v_2_sel, lane));
4743 :
4744 : /* Reuse previous lane (if any). */
4745 404 : unsigned int l = 0;
4746 687 : for (; l < lane; l++)
4747 : {
4748 481 : if ((TREE_INT_CST_LOW (VECTOR_CST_ELT (v_1_sel, l)) == e_1)
4749 481 : && (TREE_INT_CST_LOW (VECTOR_CST_ELT (v_2_sel, l)) == e_2))
4750 : break;
4751 : }
4752 :
4753 : /* Add to narrowed selector. */
4754 404 : new_sel_perm.quick_push (l + offs * nelts);
4755 :
4756 : /* Mark lane as used. */
4757 404 : lanes[l] = true;
4758 : }
4759 :
4760 : /* Count how many lanes are need. */
4761 : unsigned int cnt = 0;
4762 505 : for (unsigned int i = 0; i < nelts; i++)
4763 404 : cnt += lanes[i];
4764 :
4765 : /* If more than (nelts/2) lanes are needed, skip the sequence. */
4766 101 : if (cnt > nelts / 2)
4767 : return false;
4768 :
4769 : /* Check if the resulting permutation is cheap. */
4770 101 : vec_perm_indices new_indices (new_sel_perm, 2, nelts);
4771 101 : tree vectype = TREE_TYPE (gimple_assign_lhs (stmt));
4772 101 : machine_mode vmode = TYPE_MODE (vectype);
4773 101 : if (!can_vec_perm_const_p (vmode, vmode, new_indices, false))
4774 : return false;
4775 :
4776 101 : *seq = XNEW (struct _vec_perm_simplify_seq);
4777 101 : (*seq)->stmt = stmt;
4778 101 : (*seq)->v_1_stmt = v_1_stmt;
4779 101 : (*seq)->v_2_stmt = v_2_stmt;
4780 101 : (*seq)->v_x_stmt = v_x_stmt;
4781 101 : (*seq)->v_y_stmt = v_y_stmt;
4782 101 : (*seq)->nelts = nelts;
4783 101 : (*seq)->new_sel = vect_gen_perm_mask_checked (vectype, new_indices);
4784 :
4785 101 : if (dump_file)
4786 : {
4787 28 : fprintf (dump_file, "Found vec perm simplify sequence ending with:\n\t");
4788 28 : print_gimple_stmt (dump_file, stmt, 0);
4789 :
4790 28 : if (dump_flags & TDF_DETAILS)
4791 : {
4792 28 : fprintf (dump_file, "\tNarrowed vec_perm selector: ");
4793 28 : print_generic_expr (dump_file, (*seq)->new_sel);
4794 28 : fprintf (dump_file, "\n");
4795 : }
4796 : }
4797 :
4798 : return true;
4799 202 : }
4800 :
4801 : /* Reduce the lane consumption of a simplifiable vec perm sequence. */
4802 :
4803 : static void
4804 74 : narrow_vec_perm_simplify_seq (const vec_perm_simplify_seq &seq)
4805 : {
4806 74 : gassign *stmt = seq->stmt;
4807 74 : if (dump_file && (dump_flags & TDF_DETAILS))
4808 : {
4809 22 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
4810 22 : fprintf (dump_file, "Old stmt: ");
4811 22 : print_gimple_stmt (dump_file, stmt, 0);
4812 : }
4813 :
4814 : /* Update the last VEC_PERM statement. */
4815 74 : gimple_assign_set_rhs3 (stmt, seq->new_sel);
4816 74 : update_stmt (stmt);
4817 :
4818 74 : if (dump_file && (dump_flags & TDF_DETAILS))
4819 : {
4820 22 : fprintf (dump_file, "New stmt: ");
4821 22 : print_gimple_stmt (dump_file, stmt, 0);
4822 : }
4823 74 : }
4824 :
4825 : /* Test if we can blend two simplifiable vec permute sequences.
4826 : NEED_SWAP will be set, if sequences must be swapped for blending. */
4827 :
4828 : static bool
4829 47 : can_blend_vec_perm_simplify_seqs_p (vec_perm_simplify_seq seq1,
4830 : vec_perm_simplify_seq seq2,
4831 : bool *need_swap)
4832 : {
4833 47 : unsigned int nelts = seq1->nelts;
4834 47 : basic_block bb = gimple_bb (seq1->stmt);
4835 :
4836 47 : gcc_assert (gimple_bb (seq2->stmt) == bb);
4837 :
4838 : /* BBs and number of elements must be equal. */
4839 47 : if (gimple_bb (seq2->stmt) != bb || seq2->nelts != nelts)
4840 : return false;
4841 :
4842 : /* We need vectors of the same type. */
4843 47 : if (TREE_TYPE (gimple_assign_lhs (seq1->stmt))
4844 47 : != TREE_TYPE (gimple_assign_lhs (seq2->stmt)))
4845 : return false;
4846 :
4847 : /* We require isomorphic operators. */
4848 41 : if (((gimple_assign_rhs_code (seq1->v_x_stmt)
4849 41 : != gimple_assign_rhs_code (seq2->v_x_stmt))
4850 41 : || (gimple_assign_rhs_code (seq1->v_y_stmt)
4851 41 : != gimple_assign_rhs_code (seq2->v_y_stmt))))
4852 : return false;
4853 :
4854 : /* We cannot have any dependencies between the sequences.
4855 :
4856 : For merging, we will reuse seq1->v_1_stmt and seq1->v_2_stmt.
4857 : seq1's v_in is defined before these statements, but we need
4858 : to check if seq2's v_in is defined before them as well.
4859 :
4860 : Further, we will reuse seq2->stmt. We need to ensure that
4861 : seq1->v_x_stmt and seq1->v_y_stmt are before it.
4862 :
4863 : Note, that we don't need to check the BBs here, because all
4864 : statements of both sequences have to be in the same BB. */
4865 :
4866 41 : tree seq2_v_in = gimple_assign_rhs1 (seq2->v_1_stmt);
4867 41 : if (TREE_CODE (seq2_v_in) != SSA_NAME)
4868 : return false;
4869 :
4870 41 : gassign *seq2_v_in_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (seq2_v_in));
4871 41 : if (!seq2_v_in_stmt || gimple_bb (seq2_v_in_stmt) != bb
4872 41 : || (gimple_uid (seq2_v_in_stmt) > gimple_uid (seq1->v_1_stmt))
4873 37 : || (gimple_uid (seq1->v_x_stmt) > gimple_uid (seq2->stmt))
4874 37 : || (gimple_uid (seq1->v_y_stmt) > gimple_uid (seq2->stmt)))
4875 : {
4876 4 : tree seq1_v_in = gimple_assign_rhs1 (seq1->v_1_stmt);
4877 4 : if (TREE_CODE (seq1_v_in) != SSA_NAME)
4878 : return false;
4879 :
4880 4 : gassign *seq1_v_in_stmt
4881 4 : = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (seq1_v_in));
4882 : /* Let's try to see if we succeed when swapping the sequences. */
4883 4 : if (!seq1_v_in_stmt || gimple_bb (seq1_v_in_stmt)
4884 0 : || (gimple_uid (seq1_v_in_stmt) > gimple_uid (seq2->v_1_stmt))
4885 0 : || (gimple_uid (seq2->v_x_stmt) > gimple_uid (seq1->stmt))
4886 0 : || (gimple_uid (seq2->v_y_stmt) > gimple_uid (seq1->stmt)))
4887 : return false;
4888 0 : *need_swap = true;
4889 : }
4890 : else
4891 37 : *need_swap = false;
4892 :
4893 37 : if (dump_file && (dump_flags & TDF_DETAILS))
4894 11 : fprintf (dump_file, "Found vec perm simplify sequence pair.\n");
4895 :
4896 : return true;
4897 : }
4898 :
4899 : /* Calculate the permutations for blending the two given vec permute
4900 : sequences. This may fail if the resulting permutation is not
4901 : supported. */
4902 :
4903 : static bool
4904 37 : calc_perm_vec_perm_simplify_seqs (vec_perm_simplify_seq seq1,
4905 : vec_perm_simplify_seq seq2,
4906 : vec_perm_indices *seq2_stmt_indices,
4907 : vec_perm_indices *seq1_v_1_stmt_indices,
4908 : vec_perm_indices *seq1_v_2_stmt_indices)
4909 : {
4910 37 : unsigned int i;
4911 37 : unsigned int nelts = seq1->nelts;
4912 37 : auto_vec<unsigned int> lane_assignment;
4913 37 : lane_assignment.create (nelts);
4914 :
4915 : /* Mark all lanes as free. */
4916 37 : lane_assignment.quick_grow_cleared (nelts);
4917 :
4918 : /* Allocate lanes for seq1. */
4919 185 : for (i = 0; i < nelts; i++)
4920 : {
4921 148 : unsigned int l = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq1->new_sel, i));
4922 148 : l %= nelts;
4923 148 : lane_assignment[l] = 1;
4924 : }
4925 :
4926 : /* Allocate lanes for seq2 and calculate selector for seq2->stmt. */
4927 37 : vec_perm_builder seq2_stmt_sel_perm (nelts, nelts, 1);
4928 185 : for (i = 0; i < nelts; i++)
4929 : {
4930 148 : unsigned int sel = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq2->new_sel, i));
4931 148 : unsigned int lane = sel % nelts;
4932 148 : unsigned int offs = sel / nelts;
4933 148 : unsigned int new_sel;
4934 :
4935 : /* Check if we already allocated the lane for seq2. */
4936 148 : unsigned int j = 0;
4937 263 : for (; j < i; j++)
4938 : {
4939 189 : unsigned int sel_old;
4940 189 : sel_old = TREE_INT_CST_LOW (VECTOR_CST_ELT (seq2->new_sel, j));
4941 189 : unsigned int lane_old = sel_old % nelts;
4942 189 : if (lane == lane_old)
4943 : {
4944 74 : new_sel = seq2_stmt_sel_perm[j].to_constant ();
4945 74 : new_sel = (new_sel % nelts) + offs * nelts;
4946 74 : break;
4947 : }
4948 : }
4949 :
4950 : /* If the lane is not allocated, we need to do that now. */
4951 148 : if (j == i)
4952 : {
4953 : unsigned int l_orig = lane;
4954 182 : while (lane_assignment[lane] != 0)
4955 : {
4956 108 : lane = (lane + 1) % nelts;
4957 :
4958 : /* This should not happen if both sequences utilize no more than
4959 : half of the lanes. Test anyway to guarantee termination. */
4960 108 : if (lane == l_orig)
4961 0 : return false;
4962 : }
4963 :
4964 : /* Allocate lane. */
4965 74 : lane_assignment[lane] = 2 + l_orig;
4966 74 : new_sel = lane + offs * nelts;
4967 : }
4968 :
4969 148 : seq2_stmt_sel_perm.quick_push (new_sel);
4970 : }
4971 :
4972 : /* Check if the resulting permutation is cheap. */
4973 37 : seq2_stmt_indices->new_vector (seq2_stmt_sel_perm, 2, nelts);
4974 37 : tree vectype = TREE_TYPE (gimple_assign_lhs (seq2->stmt));
4975 37 : machine_mode vmode = TYPE_MODE (vectype);
4976 37 : if (!can_vec_perm_const_p (vmode, vmode, *seq2_stmt_indices, false))
4977 : return false;
4978 :
4979 : /* Calculate selectors for seq1->v_1_stmt and seq1->v_2_stmt. */
4980 37 : vec_perm_builder seq1_v_1_stmt_sel_perm (nelts, nelts, 1);
4981 37 : vec_perm_builder seq1_v_2_stmt_sel_perm (nelts, nelts, 1);
4982 185 : for (i = 0; i < nelts; i++)
4983 : {
4984 148 : bool use_seq1 = lane_assignment[i] < 2;
4985 148 : unsigned int l1, l2;
4986 :
4987 148 : if (use_seq1)
4988 : {
4989 : /* Just reuse the selector indices. */
4990 74 : tree s1 = gimple_assign_rhs3 (seq1->v_1_stmt);
4991 74 : tree s2 = gimple_assign_rhs3 (seq1->v_2_stmt);
4992 74 : l1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s1, i));
4993 74 : l2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s2, i));
4994 : }
4995 : else
4996 : {
4997 : /* We moved the lanes for seq2, so we need to adjust for that. */
4998 74 : tree s1 = gimple_assign_rhs3 (seq2->v_1_stmt);
4999 74 : tree s2 = gimple_assign_rhs3 (seq2->v_2_stmt);
5000 74 : l1 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s1, lane_assignment[i] - 2));
5001 74 : l2 = TREE_INT_CST_LOW (VECTOR_CST_ELT (s2, lane_assignment[i] - 2));
5002 : }
5003 :
5004 148 : l1 %= nelts;
5005 148 : l2 %= nelts;
5006 222 : seq1_v_1_stmt_sel_perm.quick_push (l1 + (use_seq1 ? 0 : nelts));
5007 148 : seq1_v_2_stmt_sel_perm.quick_push (l2 + (use_seq1 ? 0 : nelts));
5008 : }
5009 :
5010 37 : seq1_v_1_stmt_indices->new_vector (seq1_v_1_stmt_sel_perm, 2, nelts);
5011 37 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_1_stmt));
5012 37 : vmode = TYPE_MODE (vectype);
5013 37 : if (!can_vec_perm_const_p (vmode, vmode, *seq1_v_1_stmt_indices, false))
5014 : return false;
5015 :
5016 37 : seq1_v_2_stmt_indices->new_vector (seq1_v_2_stmt_sel_perm, 2, nelts);
5017 37 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_2_stmt));
5018 37 : vmode = TYPE_MODE (vectype);
5019 37 : if (!can_vec_perm_const_p (vmode, vmode, *seq1_v_2_stmt_indices, false))
5020 : return false;
5021 :
5022 : return true;
5023 74 : }
5024 :
5025 : /* Blend the two given simplifiable vec permute sequences using the
5026 : given permutations. */
5027 :
5028 : static void
5029 37 : blend_vec_perm_simplify_seqs (vec_perm_simplify_seq seq1,
5030 : vec_perm_simplify_seq seq2,
5031 : const vec_perm_indices &seq2_stmt_indices,
5032 : const vec_perm_indices &seq1_v_1_stmt_indices,
5033 : const vec_perm_indices &seq1_v_2_stmt_indices)
5034 : {
5035 : /* We don't need to adjust seq1->stmt because its lanes consumption
5036 : was already narrowed before entering this function. */
5037 :
5038 : /* Adjust seq2->stmt: copy RHS1/RHS2 from seq1->stmt and set new sel. */
5039 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5040 : {
5041 11 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
5042 11 : fprintf (dump_file, "Old stmt: ");
5043 11 : print_gimple_stmt (dump_file, seq2->stmt, 0);
5044 : }
5045 :
5046 37 : gimple_assign_set_rhs1 (seq2->stmt, gimple_assign_rhs1 (seq1->stmt));
5047 74 : gimple_assign_set_rhs2 (seq2->stmt, gimple_assign_rhs2 (seq1->stmt));
5048 37 : tree vectype = TREE_TYPE (gimple_assign_lhs (seq2->stmt));
5049 37 : tree sel = vect_gen_perm_mask_checked (vectype, seq2_stmt_indices);
5050 37 : gimple_assign_set_rhs3 (seq2->stmt, sel);
5051 37 : update_stmt (seq2->stmt);
5052 :
5053 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5054 : {
5055 11 : fprintf (dump_file, "New stmt: ");
5056 11 : print_gimple_stmt (dump_file, seq2->stmt, 0);
5057 : }
5058 :
5059 : /* Adjust seq1->v_1_stmt: copy RHS2 from seq2->v_1_stmt and set new sel. */
5060 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5061 : {
5062 11 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
5063 11 : fprintf (dump_file, "Old stmt: ");
5064 11 : print_gimple_stmt (dump_file, seq1->v_1_stmt, 0);
5065 : }
5066 :
5067 37 : gimple_assign_set_rhs2 (seq1->v_1_stmt, gimple_assign_rhs1 (seq2->v_1_stmt));
5068 37 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_1_stmt));
5069 37 : sel = vect_gen_perm_mask_checked (vectype, seq1_v_1_stmt_indices);
5070 37 : gimple_assign_set_rhs3 (seq1->v_1_stmt, sel);
5071 37 : update_stmt (seq1->v_1_stmt);
5072 :
5073 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5074 : {
5075 11 : fprintf (dump_file, "New stmt: ");
5076 11 : print_gimple_stmt (dump_file, seq1->v_1_stmt, 0);
5077 : }
5078 :
5079 : /* Adjust seq1->v_2_stmt: copy RHS2 from seq2->v_2_stmt and set new sel. */
5080 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5081 : {
5082 11 : fprintf (dump_file, "Updating VEC_PERM statement:\n");
5083 11 : fprintf (dump_file, "Old stmt: ");
5084 11 : print_gimple_stmt (dump_file, seq1->v_2_stmt, 0);
5085 : }
5086 :
5087 37 : gimple_assign_set_rhs2 (seq1->v_2_stmt, gimple_assign_rhs1 (seq2->v_2_stmt));
5088 37 : vectype = TREE_TYPE (gimple_assign_lhs (seq1->v_2_stmt));
5089 37 : sel = vect_gen_perm_mask_checked (vectype, seq1_v_2_stmt_indices);
5090 37 : gimple_assign_set_rhs3 (seq1->v_2_stmt, sel);
5091 37 : update_stmt (seq1->v_2_stmt);
5092 :
5093 37 : if (dump_file && (dump_flags & TDF_DETAILS))
5094 : {
5095 11 : fprintf (dump_file, "New stmt: ");
5096 11 : print_gimple_stmt (dump_file, seq1->v_2_stmt, 0);
5097 : }
5098 :
5099 : /* At this point, we have four unmodified seq2 stmts, which will be
5100 : eliminated by DCE. */
5101 :
5102 37 : if (dump_file)
5103 11 : fprintf (dump_file, "Vec perm simplify sequences have been blended.\n\n");
5104 37 : }
5105 :
5106 : /* Try to blend narrowed vec_perm_simplify_seqs pairwise.
5107 : The provided list will be empty after this call. */
5108 :
5109 : static void
5110 315706922 : process_vec_perm_simplify_seq_list (vec<vec_perm_simplify_seq> *l)
5111 : {
5112 315706922 : unsigned int i, j;
5113 315706922 : vec_perm_simplify_seq seq1, seq2;
5114 :
5115 315706922 : if (l->is_empty ())
5116 315706877 : return;
5117 :
5118 45 : if (dump_file && (dump_flags & TDF_DETAILS))
5119 13 : fprintf (dump_file, "\nProcessing %u vec perm simplify sequences.\n",
5120 : l->length ());
5121 :
5122 109 : FOR_EACH_VEC_ELT (*l, i, seq1)
5123 : {
5124 64 : if (i + 1 < l->length ())
5125 : {
5126 51 : FOR_EACH_VEC_ELT_FROM (*l, j, seq2, i + 1)
5127 : {
5128 47 : bool swap = false;
5129 47 : if (can_blend_vec_perm_simplify_seqs_p (seq1, seq2, &swap))
5130 : {
5131 37 : vec_perm_indices seq2_stmt_indices;
5132 37 : vec_perm_indices seq1_v_1_stmt_indices;
5133 37 : vec_perm_indices seq1_v_2_stmt_indices;
5134 111 : if (calc_perm_vec_perm_simplify_seqs (swap ? seq2 : seq1,
5135 : swap ? seq1 : seq2,
5136 : &seq2_stmt_indices,
5137 : &seq1_v_1_stmt_indices,
5138 : &seq1_v_2_stmt_indices))
5139 : {
5140 : /* Narrow lane usage. */
5141 37 : narrow_vec_perm_simplify_seq (seq1);
5142 37 : narrow_vec_perm_simplify_seq (seq2);
5143 :
5144 : /* Blend sequences. */
5145 37 : blend_vec_perm_simplify_seqs (swap ? seq2 : seq1,
5146 : swap ? seq1 : seq2,
5147 : seq2_stmt_indices,
5148 : seq1_v_1_stmt_indices,
5149 : seq1_v_2_stmt_indices);
5150 :
5151 : /* We can use unordered_remove as we break the loop. */
5152 37 : l->unordered_remove (j);
5153 37 : XDELETE (seq2);
5154 37 : break;
5155 : }
5156 37 : }
5157 : }
5158 : }
5159 :
5160 : /* We don't need to call l->remove for seq1. */
5161 64 : XDELETE (seq1);
5162 : }
5163 :
5164 45 : l->truncate (0);
5165 : }
5166 :
5167 : static void
5168 101 : append_vec_perm_simplify_seq_list (vec<vec_perm_simplify_seq> *l,
5169 : const vec_perm_simplify_seq &seq)
5170 : {
5171 : /* If no space on list left, then process the list. */
5172 101 : if (!l->space (1))
5173 0 : process_vec_perm_simplify_seq_list (l);
5174 :
5175 101 : l->quick_push (seq);
5176 101 : }
5177 :
5178 : /* Main entry point for the forward propagation and statement combine
5179 : optimizer. */
5180 :
5181 : namespace {
5182 :
5183 : const pass_data pass_data_forwprop =
5184 : {
5185 : GIMPLE_PASS, /* type */
5186 : "forwprop", /* name */
5187 : OPTGROUP_NONE, /* optinfo_flags */
5188 : TV_TREE_FORWPROP, /* tv_id */
5189 : ( PROP_cfg | PROP_ssa ), /* properties_required */
5190 : 0, /* properties_provided */
5191 : 0, /* properties_destroyed */
5192 : 0, /* todo_flags_start */
5193 : 0, /* todo_flags_finish */
5194 : };
5195 :
5196 : class pass_forwprop : public gimple_opt_pass
5197 : {
5198 : public:
5199 1443835 : pass_forwprop (gcc::context *ctxt)
5200 2887670 : : gimple_opt_pass (pass_data_forwprop, ctxt), last_p (false)
5201 : {}
5202 :
5203 : /* opt_pass methods: */
5204 1155068 : opt_pass * clone () final override { return new pass_forwprop (m_ctxt); }
5205 1732602 : void set_pass_param (unsigned int n, bool param) final override
5206 : {
5207 1732602 : switch (n)
5208 : {
5209 1155068 : case 0:
5210 1155068 : m_full_walk = param;
5211 1155068 : break;
5212 577534 : case 1:
5213 577534 : last_p = param;
5214 577534 : break;
5215 0 : default:
5216 0 : gcc_unreachable();
5217 : }
5218 1732602 : }
5219 5570686 : bool gate (function *) final override { return flag_tree_forwprop; }
5220 : unsigned int execute (function *) final override;
5221 :
5222 : private:
5223 : /* Determines whether the pass instance should set PROP_last_full_fold. */
5224 : bool last_p;
5225 :
5226 : /* True if the aggregate props are doing a full walk or not. */
5227 : bool m_full_walk = false;
5228 : }; // class pass_forwprop
5229 :
5230 : /* Attemp to make the BB block of __builtin_unreachable unreachable by changing
5231 : the incoming jumps. Return true if at least one jump was changed. */
5232 :
5233 : static bool
5234 2279 : optimize_unreachable (basic_block bb)
5235 : {
5236 2279 : gimple_stmt_iterator gsi;
5237 2279 : gimple *stmt;
5238 2279 : edge_iterator ei;
5239 2279 : edge e;
5240 2279 : bool ret;
5241 :
5242 2279 : ret = false;
5243 5240 : FOR_EACH_EDGE (e, ei, bb->preds)
5244 : {
5245 2961 : gsi = gsi_last_bb (e->src);
5246 2961 : if (gsi_end_p (gsi))
5247 330 : continue;
5248 :
5249 2631 : stmt = gsi_stmt (gsi);
5250 2631 : if (gcond *cond_stmt = dyn_cast <gcond *> (stmt))
5251 : {
5252 : /* If the condition is already true/false
5253 : ignore it. This can happen during copy prop of forwprop. */
5254 677 : if (gimple_cond_true_p (cond_stmt)
5255 669 : || gimple_cond_false_p (cond_stmt))
5256 8 : continue;
5257 661 : else if (e->flags & EDGE_TRUE_VALUE)
5258 578 : gimple_cond_make_false (cond_stmt);
5259 83 : else if (e->flags & EDGE_FALSE_VALUE)
5260 83 : gimple_cond_make_true (cond_stmt);
5261 : else
5262 0 : gcc_unreachable ();
5263 661 : update_stmt (cond_stmt);
5264 : }
5265 : else
5266 : {
5267 : /* Todo: handle other cases. e.g. switch. */
5268 1962 : continue;
5269 : }
5270 :
5271 661 : ret = true;
5272 : }
5273 :
5274 2279 : return ret;
5275 : }
5276 :
5277 : unsigned int
5278 5568120 : pass_forwprop::execute (function *fun)
5279 : {
5280 5568120 : unsigned int todoflags = 0;
5281 : /* Handle a full walk only when expensive optimizations are on. */
5282 5568120 : bool full_walk = m_full_walk && flag_expensive_optimizations;
5283 :
5284 5568120 : cfg_changed = false;
5285 5568120 : if (last_p)
5286 1046221 : fun->curr_properties |= PROP_last_full_fold;
5287 :
5288 5568120 : calculate_dominance_info (CDI_DOMINATORS);
5289 :
5290 : /* Combine stmts with the stmts defining their operands. Do that
5291 : in an order that guarantees visiting SSA defs before SSA uses. */
5292 11136240 : lattice.create (num_ssa_names);
5293 11136240 : lattice.quick_grow_cleared (num_ssa_names);
5294 5568120 : int *postorder = XNEWVEC (int, n_basic_blocks_for_fn (fun));
5295 5568120 : int postorder_num = pre_and_rev_post_order_compute_fn (fun, NULL,
5296 : postorder, false);
5297 5568120 : int *bb_to_rpo = XNEWVEC (int, last_basic_block_for_fn (fun));
5298 50095994 : for (int i = 0; i < postorder_num; ++i)
5299 : {
5300 44527874 : bb_to_rpo[postorder[i]] = i;
5301 44527874 : edge_iterator ei;
5302 44527874 : edge e;
5303 107123319 : FOR_EACH_EDGE (e, ei, BASIC_BLOCK_FOR_FN (fun, postorder[i])->succs)
5304 62595445 : e->flags &= ~EDGE_EXECUTABLE;
5305 : }
5306 5568120 : single_succ_edge (BASIC_BLOCK_FOR_FN (fun, ENTRY_BLOCK))->flags
5307 5568120 : |= EDGE_EXECUTABLE;
5308 5568120 : auto_vec<gimple *, 4> to_fixup;
5309 5568120 : auto_vec<gimple *, 32> to_remove;
5310 5568120 : auto_vec<unsigned, 32> to_remove_defs;
5311 5568120 : auto_vec<std::pair<int, int>, 10> edges_to_remove;
5312 5568120 : auto_bitmap simple_dce_worklist;
5313 5568120 : auto_bitmap need_ab_cleanup;
5314 5568120 : to_purge = BITMAP_ALLOC (NULL);
5315 5568120 : auto_vec<vec_perm_simplify_seq, 8> vec_perm_simplify_seq_list;
5316 50095994 : for (int i = 0; i < postorder_num; ++i)
5317 : {
5318 44527874 : gimple_stmt_iterator gsi;
5319 44527874 : basic_block bb = BASIC_BLOCK_FOR_FN (fun, postorder[i]);
5320 44527874 : edge_iterator ei;
5321 44527874 : edge e;
5322 :
5323 : /* Skip processing not executable blocks. We could improve
5324 : single_use tracking by at least unlinking uses from unreachable
5325 : blocks but since blocks with uses are not processed in a
5326 : meaningful order this is probably not worth it. */
5327 44527874 : bool any = false;
5328 45661774 : FOR_EACH_EDGE (e, ei, bb->preds)
5329 : {
5330 45648913 : if ((e->flags & EDGE_EXECUTABLE)
5331 : /* We can handle backedges in natural loops correctly but
5332 : for irreducible regions we have to take all backedges
5333 : conservatively when we did not visit the source yet. */
5334 45648913 : || (bb_to_rpo[e->src->index] > i
5335 662991 : && !dominated_by_p (CDI_DOMINATORS, e->src, e->dest)))
5336 : {
5337 : any = true;
5338 : break;
5339 : }
5340 : }
5341 44527874 : if (!any)
5342 13490 : continue;
5343 :
5344 : /* Remove conditions that go directly to unreachable when this is the last forwprop. */
5345 44515013 : if (last_p
5346 9723606 : && !(flag_sanitize & SANITIZE_UNREACHABLE))
5347 : {
5348 9718626 : gimple_stmt_iterator gsi;
5349 9718626 : gsi = gsi_start_nondebug_after_labels_bb (bb);
5350 9719255 : if (!gsi_end_p (gsi)
5351 8897656 : && gimple_call_builtin_p (*gsi, BUILT_IN_UNREACHABLE)
5352 9720905 : && optimize_unreachable (bb))
5353 : {
5354 629 : cfg_changed = true;
5355 629 : continue;
5356 : }
5357 : }
5358 :
5359 : /* Record degenerate PHIs in the lattice. */
5360 60213810 : for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
5361 15699426 : gsi_next (&si))
5362 : {
5363 15699426 : gphi *phi = si.phi ();
5364 15699426 : tree res = gimple_phi_result (phi);
5365 31398852 : if (virtual_operand_p (res))
5366 7199475 : continue;
5367 :
5368 8499951 : tree first = NULL_TREE;
5369 8499951 : bool all_same = true;
5370 8499951 : edge_iterator ei;
5371 8499951 : edge e;
5372 17479848 : FOR_EACH_EDGE (e, ei, bb->preds)
5373 : {
5374 : /* Ignore not executable forward edges. */
5375 17274599 : if (!(e->flags & EDGE_EXECUTABLE))
5376 : {
5377 4013149 : if (bb_to_rpo[e->src->index] < i)
5378 5528 : continue;
5379 : /* Avoid equivalences from backedges - while we might
5380 : be able to make irreducible regions reducible and
5381 : thus turning a back into a forward edge we do not
5382 : want to deal with the intermediate SSA issues that
5383 : exposes. */
5384 : all_same = false;
5385 : }
5386 17269071 : tree use = PHI_ARG_DEF_FROM_EDGE (phi, e);
5387 17269071 : if (use == res)
5388 : /* The PHI result can also appear on a backedge, if so
5389 : we can ignore this case for the purpose of determining
5390 : the singular value. */
5391 : ;
5392 17256540 : else if (! first)
5393 : first = use;
5394 8756589 : else if (! operand_equal_p (first, use, 0))
5395 : {
5396 : all_same = false;
5397 : break;
5398 : }
5399 : }
5400 8499951 : if (all_same)
5401 : {
5402 200695 : if (may_propagate_copy (res, first))
5403 200159 : to_remove_defs.safe_push (SSA_NAME_VERSION (res));
5404 200695 : fwprop_set_lattice_val (res, first);
5405 : }
5406 : }
5407 :
5408 : /* Apply forward propagation to all stmts in the basic-block.
5409 : Note we update GSI within the loop as necessary. */
5410 44514384 : unsigned int uid = 1;
5411 426266066 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); )
5412 : {
5413 337237298 : gimple *stmt = gsi_stmt (gsi);
5414 337237298 : tree lhs, rhs;
5415 337237298 : enum tree_code code;
5416 :
5417 337237298 : gimple_set_uid (stmt, uid++);
5418 :
5419 337237298 : if (!is_gimple_assign (stmt))
5420 : {
5421 233040125 : process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
5422 233040125 : gsi_next (&gsi);
5423 233040125 : continue;
5424 : }
5425 :
5426 104197173 : lhs = gimple_assign_lhs (stmt);
5427 104197173 : rhs = gimple_assign_rhs1 (stmt);
5428 104197173 : code = gimple_assign_rhs_code (stmt);
5429 :
5430 142349586 : if (TREE_CODE (lhs) != SSA_NAME
5431 104197173 : || has_zero_uses (lhs))
5432 : {
5433 38152413 : process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
5434 38152413 : gsi_next (&gsi);
5435 38152413 : continue;
5436 : }
5437 :
5438 : /* If this statement sets an SSA_NAME to an address,
5439 : try to propagate the address into the uses of the SSA_NAME. */
5440 66044760 : if ((code == ADDR_EXPR
5441 : /* Handle pointer conversions on invariant addresses
5442 : as well, as this is valid gimple. */
5443 63797726 : || (CONVERT_EXPR_CODE_P (code)
5444 8794591 : && TREE_CODE (rhs) == ADDR_EXPR
5445 352016 : && POINTER_TYPE_P (TREE_TYPE (lhs))))
5446 66044984 : && TREE_CODE (TREE_OPERAND (rhs, 0)) != TARGET_MEM_REF)
5447 : {
5448 2246603 : tree base = get_base_address (TREE_OPERAND (rhs, 0));
5449 2246603 : if ((!base
5450 2246603 : || !DECL_P (base)
5451 131665 : || decl_address_invariant_p (base))
5452 2246603 : && !stmt_references_abnormal_ssa_name (stmt)
5453 4493190 : && forward_propagate_addr_expr (lhs, rhs, true))
5454 : {
5455 452077 : fwprop_invalidate_lattice (gimple_get_lhs (stmt));
5456 452077 : release_defs (stmt);
5457 452077 : gsi_remove (&gsi, true);
5458 : }
5459 : else
5460 1794526 : gsi_next (&gsi);
5461 : }
5462 63798157 : else if (code == POINTER_PLUS_EXPR)
5463 : {
5464 3585542 : tree off = gimple_assign_rhs2 (stmt);
5465 3585542 : if (TREE_CODE (off) == INTEGER_CST
5466 1104828 : && can_propagate_from (stmt)
5467 1104475 : && !simple_iv_increment_p (stmt)
5468 : /* ??? Better adjust the interface to that function
5469 : instead of building new trees here. */
5470 4405858 : && forward_propagate_addr_expr
5471 2460948 : (lhs,
5472 : build1_loc (gimple_location (stmt),
5473 820316 : ADDR_EXPR, TREE_TYPE (rhs),
5474 820316 : fold_build2 (MEM_REF,
5475 : TREE_TYPE (TREE_TYPE (rhs)),
5476 : rhs,
5477 : fold_convert (ptr_type_node,
5478 : off))), true))
5479 : {
5480 313015 : fwprop_invalidate_lattice (gimple_get_lhs (stmt));
5481 313015 : release_defs (stmt);
5482 313015 : gsi_remove (&gsi, true);
5483 : }
5484 3272527 : else if (is_gimple_min_invariant (rhs))
5485 : {
5486 : /* Make sure to fold &a[0] + off_1 here. */
5487 404447 : fold_stmt_inplace (&gsi);
5488 404447 : update_stmt (stmt);
5489 404447 : if (gimple_assign_rhs_code (stmt) == POINTER_PLUS_EXPR)
5490 404429 : gsi_next (&gsi);
5491 : }
5492 : else
5493 2868080 : gsi_next (&gsi);
5494 : }
5495 60212615 : else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE
5496 212412 : && gimple_assign_load_p (stmt)
5497 134595 : && !gimple_has_volatile_ops (stmt)
5498 40520 : && TREE_CODE (rhs) != TARGET_MEM_REF
5499 40491 : && TREE_CODE (rhs) != BIT_FIELD_REF
5500 60253102 : && !stmt_can_throw_internal (fun, stmt))
5501 : {
5502 : /* Rewrite loads used only in real/imagpart extractions to
5503 : component-wise loads. */
5504 40362 : use_operand_p use_p;
5505 40362 : imm_use_iterator iter;
5506 40362 : tree vuse = gimple_vuse (stmt);
5507 40362 : bool rewrite = true;
5508 85483 : FOR_EACH_IMM_USE_FAST (use_p, iter, lhs)
5509 : {
5510 43035 : gimple *use_stmt = USE_STMT (use_p);
5511 43035 : if (is_gimple_debug (use_stmt))
5512 691 : continue;
5513 42344 : if (!is_gimple_assign (use_stmt)
5514 27674 : || (gimple_assign_rhs_code (use_stmt) != REALPART_EXPR
5515 25625 : && gimple_assign_rhs_code (use_stmt) != IMAGPART_EXPR)
5516 46412 : || TREE_OPERAND (gimple_assign_rhs1 (use_stmt), 0) != lhs)
5517 : {
5518 : rewrite = false;
5519 : break;
5520 : }
5521 40362 : }
5522 40362 : if (rewrite)
5523 : {
5524 2086 : gimple *use_stmt;
5525 8665 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
5526 : {
5527 4493 : if (is_gimple_debug (use_stmt))
5528 : {
5529 454 : if (gimple_debug_bind_p (use_stmt))
5530 : {
5531 454 : gimple_debug_bind_reset_value (use_stmt);
5532 454 : update_stmt (use_stmt);
5533 : }
5534 454 : continue;
5535 : }
5536 :
5537 8078 : tree new_rhs = build1 (gimple_assign_rhs_code (use_stmt),
5538 4039 : TREE_TYPE (TREE_TYPE (rhs)),
5539 : unshare_expr (rhs));
5540 4039 : gimple *new_stmt
5541 4039 : = gimple_build_assign (gimple_assign_lhs (use_stmt),
5542 : new_rhs);
5543 :
5544 4039 : location_t loc = gimple_location (use_stmt);
5545 4039 : gimple_set_location (new_stmt, loc);
5546 4039 : gimple_set_vuse (new_stmt, vuse);
5547 4039 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5548 4039 : unlink_stmt_vdef (use_stmt);
5549 4039 : gsi_remove (&gsi2, true);
5550 :
5551 4039 : gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
5552 2086 : }
5553 :
5554 2086 : release_defs (stmt);
5555 2086 : gsi_remove (&gsi, true);
5556 : }
5557 : else
5558 38276 : gsi_next (&gsi);
5559 : }
5560 60172253 : else if (TREE_CODE (TREE_TYPE (lhs)) == VECTOR_TYPE
5561 1694681 : && (TYPE_MODE (TREE_TYPE (lhs)) == BLKmode
5562 : /* After vector lowering rewrite all loads, but
5563 : initially do not since this conflicts with
5564 : vector CONSTRUCTOR to shuffle optimization. */
5565 1672691 : || (fun->curr_properties & PROP_gimple_lvec))
5566 903261 : && gimple_assign_load_p (stmt)
5567 304092 : && !gimple_has_volatile_ops (stmt)
5568 290223 : && !stmt_can_throw_internal (fun, stmt)
5569 60462476 : && (!VAR_P (rhs) || !DECL_HARD_REGISTER (rhs)))
5570 289721 : optimize_vector_load (&gsi);
5571 :
5572 59882532 : else if (code == COMPLEX_EXPR)
5573 : {
5574 : /* Rewrite stores of a single-use complex build expression
5575 : to component-wise stores. */
5576 36972 : use_operand_p use_p;
5577 36972 : gimple *use_stmt, *def1, *def2;
5578 36972 : tree rhs2;
5579 36972 : if (single_imm_use (lhs, &use_p, &use_stmt)
5580 34807 : && gimple_store_p (use_stmt)
5581 41040 : && !gimple_has_volatile_ops (use_stmt)
5582 2604 : && is_gimple_assign (use_stmt)
5583 2600 : && (TREE_CODE (TREE_TYPE (gimple_assign_lhs (use_stmt)))
5584 : == COMPLEX_TYPE)
5585 39567 : && (TREE_CODE (gimple_assign_lhs (use_stmt))
5586 : != TARGET_MEM_REF))
5587 : {
5588 2591 : tree use_lhs = gimple_assign_lhs (use_stmt);
5589 2591 : if (auto_var_p (use_lhs))
5590 601 : DECL_NOT_GIMPLE_REG_P (use_lhs) = 1;
5591 5182 : tree new_lhs = build1 (REALPART_EXPR,
5592 2591 : TREE_TYPE (TREE_TYPE (use_lhs)),
5593 : unshare_expr (use_lhs));
5594 2591 : gimple *new_stmt = gimple_build_assign (new_lhs, rhs);
5595 2591 : location_t loc = gimple_location (use_stmt);
5596 2591 : gimple_set_location (new_stmt, loc);
5597 5182 : gimple_set_vuse (new_stmt, gimple_vuse (use_stmt));
5598 2591 : gimple_set_vdef (new_stmt, make_ssa_name (gimple_vop (fun)));
5599 5182 : SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
5600 5182 : gimple_set_vuse (use_stmt, gimple_vdef (new_stmt));
5601 2591 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5602 2591 : gsi_insert_before (&gsi2, new_stmt, GSI_SAME_STMT);
5603 :
5604 5182 : new_lhs = build1 (IMAGPART_EXPR,
5605 2591 : TREE_TYPE (TREE_TYPE (use_lhs)),
5606 : unshare_expr (use_lhs));
5607 2591 : gimple_assign_set_lhs (use_stmt, new_lhs);
5608 2591 : gimple_assign_set_rhs1 (use_stmt, gimple_assign_rhs2 (stmt));
5609 2591 : update_stmt (use_stmt);
5610 :
5611 2591 : release_defs (stmt);
5612 2591 : gsi_remove (&gsi, true);
5613 : }
5614 : /* Rewrite a component-wise load of a complex to a complex
5615 : load if the components are not used separately. */
5616 34381 : else if (TREE_CODE (rhs) == SSA_NAME
5617 33940 : && has_single_use (rhs)
5618 30450 : && ((rhs2 = gimple_assign_rhs2 (stmt)), true)
5619 30450 : && TREE_CODE (rhs2) == SSA_NAME
5620 28703 : && has_single_use (rhs2)
5621 28283 : && (def1 = SSA_NAME_DEF_STMT (rhs),
5622 28283 : gimple_assign_load_p (def1))
5623 1085 : && (def2 = SSA_NAME_DEF_STMT (rhs2),
5624 1085 : gimple_assign_load_p (def2))
5625 1582 : && (gimple_vuse (def1) == gimple_vuse (def2))
5626 788 : && !gimple_has_volatile_ops (def1)
5627 788 : && !gimple_has_volatile_ops (def2)
5628 788 : && !stmt_can_throw_internal (fun, def1)
5629 788 : && !stmt_can_throw_internal (fun, def2)
5630 788 : && gimple_assign_rhs_code (def1) == REALPART_EXPR
5631 542 : && gimple_assign_rhs_code (def2) == IMAGPART_EXPR
5632 34923 : && operand_equal_p (TREE_OPERAND (gimple_assign_rhs1
5633 : (def1), 0),
5634 542 : TREE_OPERAND (gimple_assign_rhs1
5635 : (def2), 0)))
5636 : {
5637 542 : tree cl = TREE_OPERAND (gimple_assign_rhs1 (def1), 0);
5638 542 : gimple_assign_set_rhs_from_tree (&gsi, unshare_expr (cl));
5639 542 : gcc_assert (gsi_stmt (gsi) == stmt);
5640 1084 : gimple_set_vuse (stmt, gimple_vuse (def1));
5641 542 : gimple_set_modified (stmt, true);
5642 542 : gimple_stmt_iterator gsi2 = gsi_for_stmt (def1);
5643 542 : gsi_remove (&gsi, false);
5644 542 : gsi_insert_after (&gsi2, stmt, GSI_SAME_STMT);
5645 : }
5646 : else
5647 33839 : gsi_next (&gsi);
5648 : }
5649 59845560 : else if (code == CONSTRUCTOR
5650 155471 : && VECTOR_TYPE_P (TREE_TYPE (rhs))
5651 155471 : && TYPE_MODE (TREE_TYPE (rhs)) == BLKmode
5652 3436 : && CONSTRUCTOR_NELTS (rhs) > 0
5653 59848996 : && (!VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
5654 1050 : || (TYPE_MODE (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
5655 : != BLKmode)))
5656 : {
5657 : /* Rewrite stores of a single-use vector constructors
5658 : to component-wise stores if the mode isn't supported. */
5659 3341 : use_operand_p use_p;
5660 3341 : gimple *use_stmt;
5661 3341 : if (single_imm_use (lhs, &use_p, &use_stmt)
5662 2903 : && gimple_store_p (use_stmt)
5663 2948 : && !gimple_has_volatile_ops (use_stmt)
5664 1468 : && !stmt_can_throw_internal (fun, use_stmt)
5665 4802 : && is_gimple_assign (use_stmt))
5666 : {
5667 1461 : tree elt_t = TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value);
5668 1461 : unsigned HOST_WIDE_INT elt_w
5669 1461 : = tree_to_uhwi (TYPE_SIZE (elt_t));
5670 1461 : unsigned HOST_WIDE_INT n
5671 1461 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs)));
5672 1461 : tree use_lhs = gimple_assign_lhs (use_stmt);
5673 1461 : if (auto_var_p (use_lhs))
5674 554 : DECL_NOT_GIMPLE_REG_P (use_lhs) = 1;
5675 907 : else if (TREE_CODE (use_lhs) == TARGET_MEM_REF)
5676 : {
5677 1 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5678 1 : use_lhs = prepare_target_mem_ref_lvalue (use_lhs, &gsi2);
5679 : }
5680 32835 : for (unsigned HOST_WIDE_INT bi = 0; bi < n; bi += elt_w)
5681 : {
5682 31374 : unsigned HOST_WIDE_INT ci = bi / elt_w;
5683 31374 : tree new_rhs;
5684 31374 : if (ci < CONSTRUCTOR_NELTS (rhs))
5685 30756 : new_rhs = CONSTRUCTOR_ELT (rhs, ci)->value;
5686 : else
5687 618 : new_rhs = build_zero_cst (elt_t);
5688 31374 : tree new_lhs = build3 (BIT_FIELD_REF,
5689 : elt_t,
5690 : unshare_expr (use_lhs),
5691 31374 : bitsize_int (elt_w),
5692 31374 : bitsize_int (bi));
5693 31374 : gimple *new_stmt = gimple_build_assign (new_lhs, new_rhs);
5694 31374 : location_t loc = gimple_location (use_stmt);
5695 31374 : gimple_set_location (new_stmt, loc);
5696 62748 : gimple_set_vuse (new_stmt, gimple_vuse (use_stmt));
5697 31374 : gimple_set_vdef (new_stmt,
5698 : make_ssa_name (gimple_vop (fun)));
5699 62748 : SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
5700 62748 : gimple_set_vuse (use_stmt, gimple_vdef (new_stmt));
5701 31374 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5702 31374 : gsi_insert_before (&gsi2, new_stmt, GSI_SAME_STMT);
5703 : }
5704 1461 : gimple_stmt_iterator gsi2 = gsi_for_stmt (use_stmt);
5705 1461 : unlink_stmt_vdef (use_stmt);
5706 1461 : release_defs (use_stmt);
5707 1461 : gsi_remove (&gsi2, true);
5708 1461 : release_defs (stmt);
5709 1461 : gsi_remove (&gsi, true);
5710 : }
5711 : else
5712 1880 : gsi_next (&gsi);
5713 : }
5714 59842219 : else if (code == VEC_PERM_EXPR)
5715 : {
5716 : /* Find vectorized sequences, where we can reduce the lane
5717 : utilization. The narrowing will be donw later and only
5718 : if we find a pair of sequences that can be blended. */
5719 182530 : gassign *assign = dyn_cast <gassign *> (stmt);
5720 182530 : vec_perm_simplify_seq seq;
5721 182530 : if (recognise_vec_perm_simplify_seq (assign, &seq))
5722 101 : append_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list,
5723 : seq);
5724 :
5725 182530 : gsi_next (&gsi);
5726 : }
5727 : else
5728 59659689 : gsi_next (&gsi);
5729 : }
5730 :
5731 44514384 : process_vec_perm_simplify_seq_list (&vec_perm_simplify_seq_list);
5732 :
5733 : /* Combine stmts with the stmts defining their operands.
5734 : Note we update GSI within the loop as necessary. */
5735 425924890 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
5736 : {
5737 336896122 : gimple *stmt = gsi_stmt (gsi);
5738 :
5739 : /* Mark stmt as potentially needing revisiting. */
5740 336896122 : gimple_set_plf (stmt, GF_PLF_1, false);
5741 :
5742 336896122 : bool can_make_abnormal_goto = (is_gimple_call (stmt)
5743 336896122 : && stmt_can_make_abnormal_goto (stmt));
5744 :
5745 : /* Substitute from our lattice. We need to do so only once. */
5746 336896122 : bool substituted_p = false;
5747 336896122 : use_operand_p usep;
5748 336896122 : ssa_op_iter iter;
5749 497330038 : FOR_EACH_SSA_USE_OPERAND (usep, stmt, iter, SSA_OP_USE)
5750 : {
5751 160433916 : tree use = USE_FROM_PTR (usep);
5752 160433916 : tree val = fwprop_ssa_val (use);
5753 160433916 : if (val && val != use)
5754 : {
5755 1804097 : if (!is_gimple_debug (stmt))
5756 1508934 : bitmap_set_bit (simple_dce_worklist, SSA_NAME_VERSION (use));
5757 1804097 : if (may_propagate_copy (use, val))
5758 : {
5759 1800888 : propagate_value (usep, val);
5760 1800888 : substituted_p = true;
5761 : }
5762 : }
5763 : }
5764 336896122 : if (substituted_p)
5765 1747488 : update_stmt (stmt);
5766 1747488 : if (substituted_p
5767 1747488 : && is_gimple_assign (stmt)
5768 1054856 : && gimple_assign_rhs_code (stmt) == ADDR_EXPR)
5769 20060 : recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
5770 336896122 : if (substituted_p
5771 336896122 : && can_make_abnormal_goto
5772 336896122 : && !stmt_can_make_abnormal_goto (stmt))
5773 3 : bitmap_set_bit (need_ab_cleanup, bb->index);
5774 :
5775 339681324 : bool changed;
5776 679362648 : do
5777 : {
5778 339681324 : gimple *orig_stmt = stmt = gsi_stmt (gsi);
5779 339681324 : bool was_call = is_gimple_call (stmt);
5780 339681324 : bool was_noreturn = (was_call
5781 339681324 : && gimple_call_noreturn_p (stmt));
5782 339681324 : changed = false;
5783 :
5784 339681324 : auto_vec<tree, 8> uses;
5785 503093077 : FOR_EACH_SSA_USE_OPERAND (usep, stmt, iter, SSA_OP_USE)
5786 163411753 : if (uses.space (1))
5787 163024130 : uses.quick_push (USE_FROM_PTR (usep));
5788 :
5789 339681324 : if (fold_stmt (&gsi, fwprop_ssa_val, simple_dce_worklist))
5790 : {
5791 2469812 : changed = true;
5792 : /* There is no updating of the address
5793 : taken after the last forwprop so update
5794 : the addresses when a folding happened to a call.
5795 : The va_* builtins can remove taking of the address so
5796 : can the sincos->cexpi transformation. See PR 39643 and PR 20983. */
5797 2469812 : if (was_call && last_p)
5798 2469812 : todoflags |= TODO_update_address_taken;
5799 2469812 : stmt = gsi_stmt (gsi);
5800 : /* Cleanup the CFG if we simplified a condition to
5801 : true or false. */
5802 2469812 : if (gcond *cond = dyn_cast <gcond *> (stmt))
5803 981876 : if (gimple_cond_true_p (cond)
5804 981876 : || gimple_cond_false_p (cond))
5805 13899 : cfg_changed = true;
5806 : /* Queue old uses for simple DCE if not debug statement. */
5807 2469812 : if (!is_gimple_debug (stmt))
5808 10453230 : for (tree use : uses)
5809 3064614 : if (TREE_CODE (use) == SSA_NAME
5810 3064614 : && !SSA_NAME_IS_DEFAULT_DEF (use))
5811 2868872 : bitmap_set_bit (simple_dce_worklist,
5812 2868872 : SSA_NAME_VERSION (use));
5813 2469812 : update_stmt (stmt);
5814 : }
5815 :
5816 339681324 : switch (gimple_code (stmt))
5817 : {
5818 105188861 : case GIMPLE_ASSIGN:
5819 105188861 : {
5820 105188861 : tree rhs1 = gimple_assign_rhs1 (stmt);
5821 105188861 : enum tree_code code = gimple_assign_rhs_code (stmt);
5822 105188861 : if (gimple_clobber_p (stmt))
5823 6987907 : do_simple_agr_dse (as_a<gassign*>(stmt), full_walk);
5824 98200954 : else if (gimple_store_p (stmt))
5825 : {
5826 30551609 : optimize_aggr_zeroprop (stmt, full_walk);
5827 30551609 : if (gimple_assign_load_p (stmt))
5828 3780716 : optimize_agr_copyprop (stmt);
5829 : }
5830 67649345 : else if (TREE_CODE_CLASS (code) == tcc_comparison)
5831 2560141 : changed |= forward_propagate_into_comparison (&gsi);
5832 65089204 : else if ((code == PLUS_EXPR
5833 65089204 : || code == BIT_IOR_EXPR
5834 55071508 : || code == BIT_XOR_EXPR)
5835 65222844 : && simplify_rotate (&gsi))
5836 : changed = true;
5837 65086530 : else if (code == VEC_PERM_EXPR)
5838 184588 : changed |= simplify_permutation (&gsi);
5839 64901942 : else if (code == CONSTRUCTOR
5840 64901942 : && TREE_CODE (TREE_TYPE (rhs1)) == VECTOR_TYPE)
5841 153302 : changed |= simplify_vector_constructor (&gsi);
5842 64748640 : else if (code == ARRAY_REF)
5843 1956766 : changed |= simplify_count_zeroes (&gsi);
5844 : break;
5845 : }
5846 :
5847 103322 : case GIMPLE_SWITCH:
5848 103322 : changed |= simplify_gimple_switch (as_a <gswitch *> (stmt),
5849 : edges_to_remove,
5850 : simple_dce_worklist);
5851 103322 : break;
5852 :
5853 19226786 : case GIMPLE_COND:
5854 19226786 : {
5855 19226786 : int did_something = forward_propagate_into_gimple_cond
5856 19226786 : (as_a <gcond *> (stmt));
5857 19226786 : if (did_something == 2)
5858 1657 : cfg_changed = true;
5859 19226786 : changed |= did_something != 0;
5860 19226786 : break;
5861 : }
5862 :
5863 23244796 : case GIMPLE_CALL:
5864 23244796 : {
5865 23244796 : tree callee = gimple_call_fndecl (stmt);
5866 23244796 : if (callee != NULL_TREE
5867 23244796 : && fndecl_built_in_p (callee, BUILT_IN_NORMAL))
5868 6203131 : changed |= simplify_builtin_call (&gsi, callee, full_walk);
5869 : break;
5870 : }
5871 :
5872 339678650 : default:;
5873 : }
5874 :
5875 339678650 : if (changed || substituted_p)
5876 : {
5877 4021348 : substituted_p = false;
5878 4021348 : stmt = gsi_stmt (gsi);
5879 4021348 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
5880 70 : bitmap_set_bit (to_purge, bb->index);
5881 4021348 : if (!was_noreturn
5882 4021348 : && is_gimple_call (stmt) && gimple_call_noreturn_p (stmt))
5883 12 : to_fixup.safe_push (stmt);
5884 : }
5885 4021348 : if (changed)
5886 : {
5887 : /* If the stmt changed then re-visit it and the statements
5888 : inserted before it. */
5889 8723292 : for (; !gsi_end_p (gsi); gsi_prev (&gsi))
5890 5535552 : if (gimple_plf (gsi_stmt (gsi), GF_PLF_1))
5891 : break;
5892 2785202 : if (gsi_end_p (gsi))
5893 437390 : gsi = gsi_start_bb (bb);
5894 : else
5895 2566507 : gsi_next (&gsi);
5896 : }
5897 339681324 : }
5898 : while (changed);
5899 :
5900 : /* Stmt no longer needs to be revisited. */
5901 336896122 : stmt = gsi_stmt (gsi);
5902 336896122 : gcc_checking_assert (!gimple_plf (stmt, GF_PLF_1));
5903 336896122 : gimple_set_plf (stmt, GF_PLF_1, true);
5904 :
5905 : /* Fill up the lattice. */
5906 336896122 : if (gimple_assign_single_p (stmt))
5907 : {
5908 69528659 : tree lhs = gimple_assign_lhs (stmt);
5909 69528659 : tree rhs = gimple_assign_rhs1 (stmt);
5910 69528659 : if (TREE_CODE (lhs) == SSA_NAME)
5911 : {
5912 32002168 : tree val = lhs;
5913 32002168 : if (TREE_CODE (rhs) == SSA_NAME)
5914 768358 : val = fwprop_ssa_val (rhs);
5915 31233810 : else if (is_gimple_min_invariant (rhs))
5916 419721 : val = rhs;
5917 : /* If we can propagate the lattice-value mark the
5918 : stmt for removal. */
5919 32002168 : if (val != lhs
5920 32002168 : && may_propagate_copy (lhs, val))
5921 1184649 : to_remove_defs.safe_push (SSA_NAME_VERSION (lhs));
5922 32002168 : fwprop_set_lattice_val (lhs, val);
5923 : }
5924 : }
5925 267367463 : else if (gimple_nop_p (stmt))
5926 88712 : to_remove.safe_push (stmt);
5927 : }
5928 :
5929 : /* Substitute in destination PHI arguments. */
5930 107098882 : FOR_EACH_EDGE (e, ei, bb->succs)
5931 62584498 : for (gphi_iterator gsi = gsi_start_phis (e->dest);
5932 104003001 : !gsi_end_p (gsi); gsi_next (&gsi))
5933 : {
5934 41418503 : gphi *phi = gsi.phi ();
5935 41418503 : use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
5936 41418503 : tree arg = USE_FROM_PTR (use_p);
5937 68275908 : if (TREE_CODE (arg) != SSA_NAME
5938 41418503 : || virtual_operand_p (arg))
5939 26857405 : continue;
5940 14561098 : tree val = fwprop_ssa_val (arg);
5941 14561098 : if (val != arg
5942 14561098 : && may_propagate_copy (arg, val, !(e->flags & EDGE_ABNORMAL)))
5943 236615 : propagate_value (use_p, val);
5944 : }
5945 :
5946 : /* Mark outgoing exectuable edges. */
5947 44514384 : if (edge e = find_taken_edge (bb, NULL))
5948 : {
5949 18872637 : e->flags |= EDGE_EXECUTABLE;
5950 44534153 : if (EDGE_COUNT (bb->succs) > 1)
5951 19769 : cfg_changed = true;
5952 : }
5953 : else
5954 : {
5955 69333838 : FOR_EACH_EDGE (e, ei, bb->succs)
5956 43692091 : e->flags |= EDGE_EXECUTABLE;
5957 : }
5958 : }
5959 5568120 : free (postorder);
5960 5568120 : free (bb_to_rpo);
5961 5568120 : lattice.release ();
5962 :
5963 : /* First remove chains of stmts where we check no uses remain. */
5964 5568120 : simple_dce_from_worklist (simple_dce_worklist, to_purge);
5965 :
5966 5902765 : auto remove = [](gimple *stmt)
5967 : {
5968 334645 : if (dump_file && (dump_flags & TDF_DETAILS))
5969 : {
5970 1 : fprintf (dump_file, "Removing dead stmt ");
5971 1 : print_gimple_stmt (dump_file, stmt, 0);
5972 1 : fprintf (dump_file, "\n");
5973 : }
5974 334645 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5975 334645 : if (gimple_code (stmt) == GIMPLE_PHI)
5976 87264 : remove_phi_node (&gsi, true);
5977 : else
5978 : {
5979 247381 : unlink_stmt_vdef (stmt);
5980 247381 : gsi_remove (&gsi, true);
5981 247381 : release_defs (stmt);
5982 : }
5983 334645 : };
5984 :
5985 : /* Then remove stmts we know we can remove even though we did not
5986 : substitute in dead code regions, so uses can remain. Do so in reverse
5987 : order to make debug stmt creation possible. */
5988 12521048 : while (!to_remove_defs.is_empty())
5989 : {
5990 1384808 : tree def = ssa_name (to_remove_defs.pop ());
5991 : /* For example remove_prop_source_from_use can remove stmts queued
5992 : for removal. Deal with this gracefully. */
5993 1384808 : if (!def)
5994 1138875 : continue;
5995 245933 : gimple *stmt = SSA_NAME_DEF_STMT (def);
5996 245933 : remove (stmt);
5997 : }
5998 :
5999 : /* Wipe other queued stmts that do not have SSA defs. */
6000 5656832 : while (!to_remove.is_empty())
6001 : {
6002 88712 : gimple *stmt = to_remove.pop ();
6003 88712 : remove (stmt);
6004 : }
6005 :
6006 : /* Fixup stmts that became noreturn calls. This may require splitting
6007 : blocks and thus isn't possible during the walk. Do this
6008 : in reverse order so we don't inadvertedly remove a stmt we want to
6009 : fixup by visiting a dominating now noreturn call first. */
6010 5568132 : while (!to_fixup.is_empty ())
6011 : {
6012 12 : gimple *stmt = to_fixup.pop ();
6013 12 : if (dump_file && dump_flags & TDF_DETAILS)
6014 : {
6015 0 : fprintf (dump_file, "Fixing up noreturn call ");
6016 0 : print_gimple_stmt (dump_file, stmt, 0);
6017 0 : fprintf (dump_file, "\n");
6018 : }
6019 12 : cfg_changed |= fixup_noreturn_call (stmt);
6020 : }
6021 :
6022 5568120 : cfg_changed |= gimple_purge_all_dead_eh_edges (to_purge);
6023 5568120 : cfg_changed |= gimple_purge_all_dead_abnormal_call_edges (need_ab_cleanup);
6024 5568120 : BITMAP_FREE (to_purge);
6025 :
6026 : /* Remove edges queued from switch stmt simplification. */
6027 16704360 : for (auto ep : edges_to_remove)
6028 : {
6029 0 : basic_block src = BASIC_BLOCK_FOR_FN (fun, ep.first);
6030 0 : basic_block dest = BASIC_BLOCK_FOR_FN (fun, ep.second);
6031 0 : edge e;
6032 0 : if (src && dest && (e = find_edge (src, dest)))
6033 : {
6034 0 : free_dominance_info (CDI_DOMINATORS);
6035 0 : remove_edge (e);
6036 0 : cfg_changed = true;
6037 : }
6038 : }
6039 :
6040 11134698 : if (get_range_query (fun) != get_global_range_query ())
6041 1542 : disable_ranger (fun);
6042 :
6043 5568120 : if (cfg_changed)
6044 9343 : todoflags |= TODO_cleanup_cfg;
6045 :
6046 5568120 : return todoflags;
6047 5568120 : }
6048 :
6049 : } // anon namespace
6050 :
6051 : gimple_opt_pass *
6052 288767 : make_pass_forwprop (gcc::context *ctxt)
6053 : {
6054 288767 : return new pass_forwprop (ctxt);
6055 : }
|
