Line data Source code
1 : /* SCC value numbering for trees
2 : Copyright (C) 2006-2026 Free Software Foundation, Inc.
3 : Contributed by Daniel Berlin <dan@dberlin.org>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify
8 : it under the terms of the GNU General Public License as published by
9 : the Free Software Foundation; either version 3, or (at your option)
10 : any later version.
11 :
12 : GCC is distributed in the hope that it will be useful,
13 : but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 : GNU General Public License for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "ssa.h"
29 : #include "expmed.h"
30 : #include "insn-config.h"
31 : #include "memmodel.h"
32 : #include "emit-rtl.h"
33 : #include "cgraph.h"
34 : #include "gimple-pretty-print.h"
35 : #include "splay-tree-utils.h"
36 : #include "alias.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "cfganal.h"
40 : #include "tree-inline.h"
41 : #include "internal-fn.h"
42 : #include "gimple-iterator.h"
43 : #include "gimple-fold.h"
44 : #include "tree-eh.h"
45 : #include "flags.h"
46 : #include "dojump.h"
47 : #include "explow.h"
48 : #include "calls.h"
49 : #include "varasm.h"
50 : #include "stmt.h"
51 : #include "expr.h"
52 : #include "tree-dfa.h"
53 : #include "tree-ssa.h"
54 : #include "dumpfile.h"
55 : #include "cfgloop.h"
56 : #include "tree-ssa-propagate.h"
57 : #include "tree-cfg.h"
58 : #include "domwalk.h"
59 : #include "gimple-match.h"
60 : #include "stringpool.h"
61 : #include "attribs.h"
62 : #include "tree-pass.h"
63 : #include "statistics.h"
64 : #include "langhooks.h"
65 : #include "ipa-utils.h"
66 : #include "dbgcnt.h"
67 : #include "tree-cfgcleanup.h"
68 : #include "tree-ssa-loop.h"
69 : #include "tree-scalar-evolution.h"
70 : #include "tree-ssa-loop-niter.h"
71 : #include "builtins.h"
72 : #include "fold-const-call.h"
73 : #include "ipa-modref-tree.h"
74 : #include "ipa-modref.h"
75 : #include "tree-ssa-sccvn.h"
76 : #include "alloc-pool.h"
77 : #include "symbol-summary.h"
78 : #include "sreal.h"
79 : #include "ipa-cp.h"
80 : #include "ipa-prop.h"
81 : #include "target.h"
82 :
83 : /* This algorithm is based on the SCC algorithm presented by Keith
84 : Cooper and L. Taylor Simpson in "SCC-Based Value numbering"
85 : (http://citeseer.ist.psu.edu/41805.html). In
86 : straight line code, it is equivalent to a regular hash based value
87 : numbering that is performed in reverse postorder.
88 :
89 : For code with cycles, there are two alternatives, both of which
90 : require keeping the hashtables separate from the actual list of
91 : value numbers for SSA names.
92 :
93 : 1. Iterate value numbering in an RPO walk of the blocks, removing
94 : all the entries from the hashtable after each iteration (but
95 : keeping the SSA name->value number mapping between iterations).
96 : Iterate until it does not change.
97 :
98 : 2. Perform value numbering as part of an SCC walk on the SSA graph,
99 : iterating only the cycles in the SSA graph until they do not change
100 : (using a separate, optimistic hashtable for value numbering the SCC
101 : operands).
102 :
103 : The second is not just faster in practice (because most SSA graph
104 : cycles do not involve all the variables in the graph), it also has
105 : some nice properties.
106 :
107 : One of these nice properties is that when we pop an SCC off the
108 : stack, we are guaranteed to have processed all the operands coming from
109 : *outside of that SCC*, so we do not need to do anything special to
110 : ensure they have value numbers.
111 :
112 : Another nice property is that the SCC walk is done as part of a DFS
113 : of the SSA graph, which makes it easy to perform combining and
114 : simplifying operations at the same time.
115 :
116 : The code below is deliberately written in a way that makes it easy
117 : to separate the SCC walk from the other work it does.
118 :
119 : In order to propagate constants through the code, we track which
120 : expressions contain constants, and use those while folding. In
121 : theory, we could also track expressions whose value numbers are
122 : replaced, in case we end up folding based on expression
123 : identities.
124 :
125 : In order to value number memory, we assign value numbers to vuses.
126 : This enables us to note that, for example, stores to the same
127 : address of the same value from the same starting memory states are
128 : equivalent.
129 : TODO:
130 :
131 : 1. We can iterate only the changing portions of the SCC's, but
132 : I have not seen an SCC big enough for this to be a win.
133 : 2. If you differentiate between phi nodes for loops and phi nodes
134 : for if-then-else, you can properly consider phi nodes in different
135 : blocks for equivalence.
136 : 3. We could value number vuses in more cases, particularly, whole
137 : structure copies.
138 : */
139 :
140 : /* There's no BB_EXECUTABLE but we can use BB_VISITED. */
141 : #define BB_EXECUTABLE BB_VISITED
142 :
143 : static vn_lookup_kind default_vn_walk_kind;
144 :
145 : /* vn_nary_op hashtable helpers. */
146 :
147 : struct vn_nary_op_hasher : nofree_ptr_hash <vn_nary_op_s>
148 : {
149 : typedef vn_nary_op_s *compare_type;
150 : static inline hashval_t hash (const vn_nary_op_s *);
151 : static inline bool equal (const vn_nary_op_s *, const vn_nary_op_s *);
152 : };
153 :
154 : /* Return the computed hashcode for nary operation P1. */
155 :
156 : inline hashval_t
157 777503687 : vn_nary_op_hasher::hash (const vn_nary_op_s *vno1)
158 : {
159 777503687 : return vno1->hashcode;
160 : }
161 :
162 : /* Compare nary operations P1 and P2 and return true if they are
163 : equivalent. */
164 :
165 : inline bool
166 986201343 : vn_nary_op_hasher::equal (const vn_nary_op_s *vno1, const vn_nary_op_s *vno2)
167 : {
168 986201343 : return vno1 == vno2 || vn_nary_op_eq (vno1, vno2);
169 : }
170 :
171 : typedef hash_table<vn_nary_op_hasher> vn_nary_op_table_type;
172 : typedef vn_nary_op_table_type::iterator vn_nary_op_iterator_type;
173 :
174 :
175 : /* vn_phi hashtable helpers. */
176 :
177 : static int
178 : vn_phi_eq (const_vn_phi_t const vp1, const_vn_phi_t const vp2);
179 :
180 : struct vn_phi_hasher : nofree_ptr_hash <vn_phi_s>
181 : {
182 : static inline hashval_t hash (const vn_phi_s *);
183 : static inline bool equal (const vn_phi_s *, const vn_phi_s *);
184 : };
185 :
186 : /* Return the computed hashcode for phi operation P1. */
187 :
188 : inline hashval_t
189 25553804 : vn_phi_hasher::hash (const vn_phi_s *vp1)
190 : {
191 25553804 : return vp1->hashcode;
192 : }
193 :
194 : /* Compare two phi entries for equality, ignoring VN_TOP arguments. */
195 :
196 : inline bool
197 46557014 : vn_phi_hasher::equal (const vn_phi_s *vp1, const vn_phi_s *vp2)
198 : {
199 46557014 : return vp1 == vp2 || vn_phi_eq (vp1, vp2);
200 : }
201 :
202 : typedef hash_table<vn_phi_hasher> vn_phi_table_type;
203 : typedef vn_phi_table_type::iterator vn_phi_iterator_type;
204 :
205 :
206 : /* Compare two reference operands P1 and P2 for equality. Return true if
207 : they are equal, and false otherwise. */
208 :
209 : static int
210 26047914 : vn_reference_op_eq (const void *p1, const void *p2)
211 : {
212 26047914 : const_vn_reference_op_t const vro1 = (const_vn_reference_op_t) p1;
213 26047914 : const_vn_reference_op_t const vro2 = (const_vn_reference_op_t) p2;
214 :
215 26047914 : return (vro1->opcode == vro2->opcode
216 : /* We do not care for differences in type qualification. */
217 26046072 : && (vro1->type == vro2->type
218 1190756 : || (vro1->type && vro2->type
219 1190756 : && types_compatible_p (TYPE_MAIN_VARIANT (vro1->type),
220 1190756 : TYPE_MAIN_VARIANT (vro2->type))))
221 25043575 : && expressions_equal_p (vro1->op0, vro2->op0)
222 25001993 : && expressions_equal_p (vro1->op1, vro2->op1)
223 25001993 : && expressions_equal_p (vro1->op2, vro2->op2)
224 51049907 : && (vro1->opcode != CALL_EXPR || vro1->clique == vro2->clique));
225 : }
226 :
227 : /* Free a reference operation structure VP. */
228 :
229 : static inline void
230 0 : free_reference (vn_reference_s *vr)
231 : {
232 0 : vr->operands.release ();
233 : }
234 :
235 :
236 : /* vn_reference hashtable helpers. */
237 :
238 : struct vn_reference_hasher : nofree_ptr_hash <vn_reference_s>
239 : {
240 : static inline hashval_t hash (const vn_reference_s *);
241 : static inline bool equal (const vn_reference_s *, const vn_reference_s *);
242 : };
243 :
244 : /* Return the hashcode for a given reference operation P1. */
245 :
246 : inline hashval_t
247 3775215151 : vn_reference_hasher::hash (const vn_reference_s *vr1)
248 : {
249 3775215151 : return vr1->hashcode;
250 : }
251 :
252 : inline bool
253 4495998759 : vn_reference_hasher::equal (const vn_reference_s *v, const vn_reference_s *c)
254 : {
255 4495998759 : return v == c || vn_reference_eq (v, c);
256 : }
257 :
258 : typedef hash_table<vn_reference_hasher> vn_reference_table_type;
259 : typedef vn_reference_table_type::iterator vn_reference_iterator_type;
260 :
261 : /* Pretty-print OPS to OUTFILE. */
262 :
263 : void
264 287 : print_vn_reference_ops (FILE *outfile, const vec<vn_reference_op_s> ops)
265 : {
266 287 : vn_reference_op_t vro;
267 287 : unsigned int i;
268 287 : fprintf (outfile, "{");
269 1304 : for (i = 0; ops.iterate (i, &vro); i++)
270 : {
271 1017 : bool closebrace = false;
272 1017 : if (vro->opcode != SSA_NAME
273 803 : && TREE_CODE_CLASS (vro->opcode) != tcc_declaration)
274 : {
275 803 : fprintf (outfile, "%s", get_tree_code_name (vro->opcode));
276 803 : if (vro->op0 || vro->opcode == CALL_EXPR)
277 : {
278 803 : fprintf (outfile, "<");
279 803 : closebrace = true;
280 : }
281 : }
282 1017 : if (vro->opcode == MEM_REF || vro->opcode == TARGET_MEM_REF)
283 275 : fprintf (outfile, "(A%d)", TYPE_ALIGN (vro->type));
284 1017 : if (vro->op0 || vro->opcode == CALL_EXPR)
285 : {
286 1017 : if (!vro->op0)
287 0 : fprintf (outfile, internal_fn_name ((internal_fn)vro->clique));
288 : else
289 : {
290 1017 : if (vro->opcode == MEM_REF || vro->opcode == TARGET_MEM_REF)
291 : {
292 275 : fprintf (outfile, "(");
293 275 : print_generic_expr (outfile, TREE_TYPE (vro->op0));
294 275 : fprintf (outfile, ")");
295 : }
296 1017 : print_generic_expr (outfile, vro->op0);
297 : }
298 1017 : if (vro->op1)
299 : {
300 185 : fprintf (outfile, ",");
301 185 : print_generic_expr (outfile, vro->op1);
302 : }
303 1017 : if (vro->op2)
304 : {
305 185 : fprintf (outfile, ",");
306 185 : print_generic_expr (outfile, vro->op2);
307 : }
308 : }
309 1017 : if (closebrace)
310 803 : fprintf (outfile, ">");
311 1017 : if (i != ops.length () - 1)
312 730 : fprintf (outfile, ",");
313 : }
314 287 : fprintf (outfile, "}");
315 287 : }
316 :
317 : DEBUG_FUNCTION void
318 0 : debug_vn_reference_ops (const vec<vn_reference_op_s> ops)
319 : {
320 0 : print_vn_reference_ops (stderr, ops);
321 0 : fputc ('\n', stderr);
322 0 : }
323 :
324 : /* The set of VN hashtables. */
325 :
326 : typedef struct vn_tables_s
327 : {
328 : vn_nary_op_table_type *nary;
329 : vn_phi_table_type *phis;
330 : vn_reference_table_type *references;
331 : } *vn_tables_t;
332 :
333 :
334 : /* vn_constant hashtable helpers. */
335 :
336 : struct vn_constant_hasher : free_ptr_hash <vn_constant_s>
337 : {
338 : static inline hashval_t hash (const vn_constant_s *);
339 : static inline bool equal (const vn_constant_s *, const vn_constant_s *);
340 : };
341 :
342 : /* Hash table hash function for vn_constant_t. */
343 :
344 : inline hashval_t
345 12380298 : vn_constant_hasher::hash (const vn_constant_s *vc1)
346 : {
347 12380298 : return vc1->hashcode;
348 : }
349 :
350 : /* Hash table equality function for vn_constant_t. */
351 :
352 : inline bool
353 14936449 : vn_constant_hasher::equal (const vn_constant_s *vc1, const vn_constant_s *vc2)
354 : {
355 14936449 : if (vc1->hashcode != vc2->hashcode)
356 : return false;
357 :
358 2244893 : return vn_constant_eq_with_type (vc1->constant, vc2->constant);
359 : }
360 :
361 : static hash_table<vn_constant_hasher> *constant_to_value_id;
362 :
363 :
364 : /* Obstack we allocate the vn-tables elements from. */
365 : static obstack vn_tables_obstack;
366 : /* Special obstack we never unwind. */
367 : static obstack vn_tables_insert_obstack;
368 :
369 : static vn_reference_t last_inserted_ref;
370 : static vn_phi_t last_inserted_phi;
371 : static vn_nary_op_t last_inserted_nary;
372 : static vn_ssa_aux_t last_pushed_avail;
373 :
374 : /* Valid hashtables storing information we have proven to be
375 : correct. */
376 : static vn_tables_t valid_info;
377 :
378 : /* Global RPO state for access from hooks. */
379 : static class eliminate_dom_walker *rpo_avail;
380 : basic_block vn_context_bb;
381 : int *vn_bb_to_rpo;
382 :
383 :
384 : /* Valueization hook for simplify_replace_tree. Valueize NAME if it is
385 : an SSA name, otherwise just return it. */
386 : tree (*vn_valueize) (tree);
387 : static tree
388 82588 : vn_valueize_for_srt (tree t, void* context ATTRIBUTE_UNUSED)
389 : {
390 82588 : basic_block saved_vn_context_bb = vn_context_bb;
391 : /* Look for sth available at the definition block of the argument.
392 : This avoids inconsistencies between availability there which
393 : decides if the stmt can be removed and availability at the
394 : use site. The SSA property ensures that things available
395 : at the definition are also available at uses. */
396 82588 : if (!SSA_NAME_IS_DEFAULT_DEF (t))
397 78757 : vn_context_bb = gimple_bb (SSA_NAME_DEF_STMT (t));
398 82588 : tree res = vn_valueize (t);
399 82588 : vn_context_bb = saved_vn_context_bb;
400 82588 : return res;
401 : }
402 :
403 :
404 : /* This represents the top of the VN lattice, which is the universal
405 : value. */
406 :
407 : tree VN_TOP;
408 :
409 : /* Unique counter for our value ids. */
410 :
411 : static unsigned int next_value_id;
412 : static int next_constant_value_id;
413 :
414 :
415 : /* Table of vn_ssa_aux_t's, one per ssa_name. The vn_ssa_aux_t objects
416 : are allocated on an obstack for locality reasons, and to free them
417 : without looping over the vec. */
418 :
419 : struct vn_ssa_aux_hasher : typed_noop_remove <vn_ssa_aux_t>
420 : {
421 : typedef vn_ssa_aux_t value_type;
422 : typedef tree compare_type;
423 : static inline hashval_t hash (const value_type &);
424 : static inline bool equal (const value_type &, const compare_type &);
425 : static inline void mark_deleted (value_type &) {}
426 : static const bool empty_zero_p = true;
427 0 : static inline void mark_empty (value_type &e) { e = NULL; }
428 : static inline bool is_deleted (value_type &) { return false; }
429 >13715*10^7 : static inline bool is_empty (value_type &e) { return e == NULL; }
430 : };
431 :
432 : hashval_t
433 45100841063 : vn_ssa_aux_hasher::hash (const value_type &entry)
434 : {
435 45100841063 : return SSA_NAME_VERSION (entry->name);
436 : }
437 :
438 : bool
439 51613614629 : vn_ssa_aux_hasher::equal (const value_type &entry, const compare_type &name)
440 : {
441 51613614629 : return name == entry->name;
442 : }
443 :
444 : static hash_table<vn_ssa_aux_hasher> *vn_ssa_aux_hash;
445 : typedef hash_table<vn_ssa_aux_hasher>::iterator vn_ssa_aux_iterator_type;
446 : static struct obstack vn_ssa_aux_obstack;
447 :
448 : static vn_nary_op_t vn_nary_op_insert_stmt (gimple *, tree);
449 : static vn_nary_op_t vn_nary_op_insert_into (vn_nary_op_t,
450 : vn_nary_op_table_type *);
451 : static void init_vn_nary_op_from_pieces (vn_nary_op_t, unsigned int,
452 : enum tree_code, tree, tree *);
453 : static tree vn_lookup_simplify_result (gimple_match_op *);
454 : static vn_reference_t vn_reference_lookup_or_insert_for_pieces
455 : (tree, alias_set_type, alias_set_type, poly_int64, poly_int64, tree,
456 : vec<vn_reference_op_s, va_heap>, tree);
457 :
458 : /* Return whether there is value numbering information for a given SSA name. */
459 :
460 : bool
461 5230816 : has_VN_INFO (tree name)
462 : {
463 5230816 : return vn_ssa_aux_hash->find_with_hash (name, SSA_NAME_VERSION (name));
464 : }
465 :
466 : vn_ssa_aux_t
467 3946892882 : VN_INFO (tree name)
468 : {
469 3946892882 : vn_ssa_aux_t *res
470 3946892882 : = vn_ssa_aux_hash->find_slot_with_hash (name, SSA_NAME_VERSION (name),
471 : INSERT);
472 3946892882 : if (*res != NULL)
473 : return *res;
474 :
475 175915463 : vn_ssa_aux_t newinfo = *res = XOBNEW (&vn_ssa_aux_obstack, struct vn_ssa_aux);
476 175915463 : memset (newinfo, 0, sizeof (struct vn_ssa_aux));
477 175915463 : newinfo->name = name;
478 175915463 : newinfo->valnum = VN_TOP;
479 : /* We are using the visited flag to handle uses with defs not within the
480 : region being value-numbered. */
481 175915463 : newinfo->visited = false;
482 :
483 : /* Given we create the VN_INFOs on-demand now we have to do initialization
484 : different than VN_TOP here. */
485 175915463 : if (SSA_NAME_IS_DEFAULT_DEF (name))
486 9474613 : switch (TREE_CODE (SSA_NAME_VAR (name)))
487 : {
488 1696532 : case VAR_DECL:
489 : /* All undefined vars are VARYING. */
490 1696532 : newinfo->valnum = name;
491 1696532 : newinfo->visited = true;
492 1696532 : break;
493 :
494 7717301 : case PARM_DECL:
495 : /* Parameters are VARYING but we can record a condition
496 : if we know it is a non-NULL pointer. */
497 7717301 : newinfo->visited = true;
498 7717301 : newinfo->valnum = name;
499 11862545 : if (POINTER_TYPE_P (TREE_TYPE (name))
500 8889064 : && nonnull_arg_p (SSA_NAME_VAR (name)))
501 : {
502 2403178 : tree ops[2];
503 2403178 : ops[0] = name;
504 2403178 : ops[1] = build_int_cst (TREE_TYPE (name), 0);
505 2403178 : vn_nary_op_t nary;
506 : /* Allocate from non-unwinding stack. */
507 2403178 : nary = alloc_vn_nary_op_noinit (2, &vn_tables_insert_obstack);
508 2403178 : init_vn_nary_op_from_pieces (nary, 2, NE_EXPR,
509 : boolean_type_node, ops);
510 2403178 : nary->predicated_values = 0;
511 2403178 : nary->u.result = boolean_true_node;
512 2403178 : vn_nary_op_insert_into (nary, valid_info->nary);
513 2403178 : gcc_assert (nary->unwind_to == NULL);
514 : /* Also do not link it into the undo chain. */
515 2403178 : last_inserted_nary = nary->next;
516 2403178 : nary->next = (vn_nary_op_t)(void *)-1;
517 2403178 : nary = alloc_vn_nary_op_noinit (2, &vn_tables_insert_obstack);
518 2403178 : init_vn_nary_op_from_pieces (nary, 2, EQ_EXPR,
519 : boolean_type_node, ops);
520 2403178 : nary->predicated_values = 0;
521 2403178 : nary->u.result = boolean_false_node;
522 2403178 : vn_nary_op_insert_into (nary, valid_info->nary);
523 2403178 : gcc_assert (nary->unwind_to == NULL);
524 2403178 : last_inserted_nary = nary->next;
525 2403178 : nary->next = (vn_nary_op_t)(void *)-1;
526 2403178 : if (dump_file && (dump_flags & TDF_DETAILS))
527 : {
528 38 : fprintf (dump_file, "Recording ");
529 38 : print_generic_expr (dump_file, name, TDF_SLIM);
530 38 : fprintf (dump_file, " != 0\n");
531 : }
532 : }
533 : break;
534 :
535 60780 : case RESULT_DECL:
536 : /* If the result is passed by invisible reference the default
537 : def is initialized, otherwise it's uninitialized. Still
538 : undefined is varying. */
539 60780 : newinfo->visited = true;
540 60780 : newinfo->valnum = name;
541 60780 : break;
542 :
543 0 : default:
544 0 : gcc_unreachable ();
545 : }
546 : return newinfo;
547 : }
548 :
549 : /* Return the SSA value of X. */
550 :
551 : inline tree
552 3488606404 : SSA_VAL (tree x, bool *visited = NULL)
553 : {
554 3488606404 : vn_ssa_aux_t tem = vn_ssa_aux_hash->find_with_hash (x, SSA_NAME_VERSION (x));
555 3488606404 : if (visited)
556 1416844826 : *visited = tem && tem->visited;
557 3488606404 : return tem && tem->visited ? tem->valnum : x;
558 : }
559 :
560 : /* Return the SSA value of the VUSE x, supporting released VDEFs
561 : during elimination which will value-number the VDEF to the
562 : associated VUSE (but not substitute in the whole lattice). */
563 :
564 : static inline tree
565 1287799807 : vuse_ssa_val (tree x)
566 : {
567 1287799807 : if (!x)
568 : return NULL_TREE;
569 :
570 1284352947 : do
571 : {
572 1284352947 : x = SSA_VAL (x);
573 1284352947 : gcc_assert (x != VN_TOP);
574 : }
575 1284352947 : while (SSA_NAME_IN_FREE_LIST (x));
576 :
577 : return x;
578 : }
579 :
580 : /* Similar to the above but used as callback for walk_non_aliased_vuses
581 : and thus should stop at unvisited VUSE to not walk across region
582 : boundaries. */
583 :
584 : static tree
585 1088180065 : vuse_valueize (tree vuse)
586 : {
587 1088180065 : do
588 : {
589 1088180065 : bool visited;
590 1088180065 : vuse = SSA_VAL (vuse, &visited);
591 1088180065 : if (!visited)
592 16376746 : return NULL_TREE;
593 1071803319 : gcc_assert (vuse != VN_TOP);
594 : }
595 1071803319 : while (SSA_NAME_IN_FREE_LIST (vuse));
596 : return vuse;
597 : }
598 :
599 :
600 : /* Return the vn_kind the expression computed by the stmt should be
601 : associated with. */
602 :
603 : enum vn_kind
604 104384895 : vn_get_stmt_kind (gimple *stmt)
605 : {
606 104384895 : switch (gimple_code (stmt))
607 : {
608 : case GIMPLE_CALL:
609 : return VN_REFERENCE;
610 : case GIMPLE_PHI:
611 : return VN_PHI;
612 104384895 : case GIMPLE_ASSIGN:
613 104384895 : {
614 104384895 : enum tree_code code = gimple_assign_rhs_code (stmt);
615 104384895 : tree rhs1 = gimple_assign_rhs1 (stmt);
616 104384895 : switch (get_gimple_rhs_class (code))
617 : {
618 : case GIMPLE_UNARY_RHS:
619 : case GIMPLE_BINARY_RHS:
620 : case GIMPLE_TERNARY_RHS:
621 : return VN_NARY;
622 48999875 : case GIMPLE_SINGLE_RHS:
623 48999875 : switch (TREE_CODE_CLASS (code))
624 : {
625 36958761 : case tcc_reference:
626 : /* VOP-less references can go through unary case. */
627 36958761 : if ((code == REALPART_EXPR
628 : || code == IMAGPART_EXPR
629 36958761 : || code == VIEW_CONVERT_EXPR
630 36958761 : || code == BIT_FIELD_REF)
631 36958761 : && (TREE_CODE (TREE_OPERAND (rhs1, 0)) == SSA_NAME
632 661145 : || is_gimple_min_invariant (TREE_OPERAND (rhs1, 0))))
633 2249049 : return VN_NARY;
634 :
635 : /* Fallthrough. */
636 : case tcc_declaration:
637 : return VN_REFERENCE;
638 :
639 : case tcc_constant:
640 : return VN_CONSTANT;
641 :
642 6041554 : default:
643 6041554 : if (code == ADDR_EXPR)
644 3283749 : return (is_gimple_min_invariant (rhs1)
645 3283749 : ? VN_CONSTANT : VN_REFERENCE);
646 2757805 : else if (code == CONSTRUCTOR)
647 : return VN_NARY;
648 : return VN_NONE;
649 : }
650 : default:
651 : return VN_NONE;
652 : }
653 : }
654 : default:
655 : return VN_NONE;
656 : }
657 : }
658 :
659 : /* Lookup a value id for CONSTANT and return it. If it does not
660 : exist returns 0. */
661 :
662 : unsigned int
663 0 : get_constant_value_id (tree constant)
664 : {
665 0 : vn_constant_s **slot;
666 0 : struct vn_constant_s vc;
667 :
668 0 : vc.hashcode = vn_hash_constant_with_type (constant);
669 0 : vc.constant = constant;
670 0 : slot = constant_to_value_id->find_slot (&vc, NO_INSERT);
671 0 : if (slot)
672 0 : return (*slot)->value_id;
673 : return 0;
674 : }
675 :
676 : /* Lookup a value id for CONSTANT, and if it does not exist, create a
677 : new one and return it. If it does exist, return it. */
678 :
679 : unsigned int
680 29128363 : get_or_alloc_constant_value_id (tree constant)
681 : {
682 29128363 : vn_constant_s **slot;
683 29128363 : struct vn_constant_s vc;
684 29128363 : vn_constant_t vcp;
685 :
686 : /* If the hashtable isn't initialized we're not running from PRE and thus
687 : do not need value-ids. */
688 29128363 : if (!constant_to_value_id)
689 : return 0;
690 :
691 4790644 : vc.hashcode = vn_hash_constant_with_type (constant);
692 4790644 : vc.constant = constant;
693 4790644 : slot = constant_to_value_id->find_slot (&vc, INSERT);
694 4790644 : if (*slot)
695 2227588 : return (*slot)->value_id;
696 :
697 2563056 : vcp = XNEW (struct vn_constant_s);
698 2563056 : vcp->hashcode = vc.hashcode;
699 2563056 : vcp->constant = constant;
700 2563056 : vcp->value_id = get_next_constant_value_id ();
701 2563056 : *slot = vcp;
702 2563056 : return vcp->value_id;
703 : }
704 :
705 : /* Compute the hash for a reference operand VRO1. */
706 :
707 : static void
708 138390020 : vn_reference_op_compute_hash (const vn_reference_op_t vro1, inchash::hash &hstate)
709 : {
710 138390020 : hstate.add_int (vro1->opcode);
711 138390020 : if (vro1->opcode == CALL_EXPR && !vro1->op0)
712 553784 : hstate.add_int (vro1->clique);
713 138390020 : if (vro1->op0)
714 131939093 : inchash::add_expr (vro1->op0, hstate);
715 138390020 : if (vro1->op1)
716 12138080 : inchash::add_expr (vro1->op1, hstate);
717 138390020 : if (vro1->op2)
718 13885173 : inchash::add_expr (vro1->op2, hstate);
719 138390020 : }
720 :
721 : /* Compute a hash for the reference operation VR1 and return it. */
722 :
723 : hashval_t
724 206314472 : vn_reference_compute_hash (const vn_reference_t vr1)
725 : {
726 206314472 : inchash::hash hstate;
727 206314472 : hashval_t result;
728 206314472 : int i;
729 206314472 : vn_reference_op_t vro;
730 206314472 : poly_offset_int off = -1;
731 206314472 : bool deref = false;
732 :
733 839484457 : FOR_EACH_VEC_ELT (vr1->operands, i, vro)
734 : {
735 633169985 : if (vro->opcode == MEM_REF)
736 : deref = true;
737 437633738 : else if (vro->opcode != ADDR_EXPR)
738 306820948 : deref = false;
739 633169985 : if (maybe_ne (vro->off, -1))
740 : {
741 372279613 : if (known_eq (off, -1))
742 197937705 : off = 0;
743 633169985 : off += vro->off;
744 : }
745 : else
746 : {
747 260890372 : if (maybe_ne (off, -1)
748 260890372 : && maybe_ne (off, 0))
749 104946423 : hstate.add_poly_hwi (off.force_shwi ());
750 260890372 : off = -1;
751 260890372 : if (deref
752 122719121 : && vro->opcode == ADDR_EXPR)
753 : {
754 122500352 : if (vro->op0)
755 : {
756 122500352 : tree op = TREE_OPERAND (vro->op0, 0);
757 122500352 : hstate.add_int (TREE_CODE (op));
758 122500352 : inchash::add_expr (op, hstate);
759 : }
760 : }
761 : else
762 138390020 : vn_reference_op_compute_hash (vro, hstate);
763 : }
764 : }
765 : /* Do not hash vr1->offset or vr1->max_size, we want to get collisions
766 : to be able to identify compatible results. */
767 206314472 : result = hstate.end ();
768 : /* ??? We would ICE later if we hash instead of adding that in. */
769 206314472 : if (vr1->vuse)
770 201297800 : result += SSA_NAME_VERSION (vr1->vuse);
771 :
772 206314472 : return result;
773 : }
774 :
775 : /* Return true if reference operations VR1 and VR2 are equivalent. This
776 : means they have the same set of operands and vuses. If LEXICAL
777 : is true then the full access path has to be the same. */
778 :
779 : bool
780 4491157676 : vn_reference_eq (const_vn_reference_t const vr1, const_vn_reference_t const vr2,
781 : bool lexical)
782 : {
783 4491157676 : unsigned i, j;
784 :
785 : /* Early out if this is not a hash collision. */
786 4491157676 : if (vr1->hashcode != vr2->hashcode)
787 : return false;
788 :
789 : /* The VOP needs to be the same. */
790 18050438 : if (vr1->vuse != vr2->vuse)
791 : return false;
792 :
793 : /* The offset/max_size used for the ao_ref during lookup has to be
794 : the same. */
795 18049972 : if (maybe_ne (vr1->offset, vr2->offset)
796 18049972 : || maybe_ne (vr1->max_size, vr2->max_size))
797 : {
798 : /* But nothing known in the prevailing entry is OK to be used. */
799 7004839 : if (maybe_ne (vr1->offset, 0) || known_size_p (vr1->max_size))
800 : return false;
801 : }
802 :
803 : /* If the operands are the same we are done. */
804 36008252 : if (vr1->operands == vr2->operands)
805 : return true;
806 :
807 18004126 : if (!vr1->type || !vr2->type)
808 : {
809 575942 : if (vr1->type != vr2->type)
810 : return false;
811 : }
812 17428184 : else if (vr1->type == vr2->type)
813 : ;
814 2225896 : else if (COMPLETE_TYPE_P (vr1->type) != COMPLETE_TYPE_P (vr2->type)
815 2225896 : || (COMPLETE_TYPE_P (vr1->type)
816 2225896 : && !expressions_equal_p (TYPE_SIZE (vr1->type),
817 2225896 : TYPE_SIZE (vr2->type))))
818 788021 : return false;
819 1437875 : else if (vr1->operands[0].opcode == CALL_EXPR
820 1437875 : && !types_compatible_p (vr1->type, vr2->type))
821 : return false;
822 1437875 : else if (INTEGRAL_TYPE_P (vr1->type)
823 579647 : && INTEGRAL_TYPE_P (vr2->type))
824 : {
825 539541 : if (TYPE_PRECISION (vr1->type) != TYPE_PRECISION (vr2->type))
826 : return false;
827 : }
828 898334 : else if (INTEGRAL_TYPE_P (vr1->type)
829 898334 : && (TYPE_PRECISION (vr1->type)
830 40106 : != TREE_INT_CST_LOW (TYPE_SIZE (vr1->type))))
831 : return false;
832 898288 : else if (INTEGRAL_TYPE_P (vr2->type)
833 898288 : && (TYPE_PRECISION (vr2->type)
834 9304 : != TREE_INT_CST_LOW (TYPE_SIZE (vr2->type))))
835 : return false;
836 19599 : else if (VECTOR_BOOLEAN_TYPE_P (vr1->type)
837 897693 : && VECTOR_BOOLEAN_TYPE_P (vr2->type))
838 : {
839 : /* Vector boolean types can have padding, verify we are dealing with
840 : the same number of elements, aka the precision of the types.
841 : For example, In most architecture the precision_size of vbool*_t
842 : types are calculated like below:
843 : precision_size = type_size * 8
844 :
845 : Unfortunately, the RISC-V will adjust the precision_size for the
846 : vbool*_t in order to align the ISA as below:
847 : type_size = [1, 1, 1, 1, 2, 4, 8]
848 : precision_size = [1, 2, 4, 8, 16, 32, 64]
849 :
850 : Then the precision_size of RISC-V vbool*_t will not be the multiple
851 : of the type_size. We take care of this case consolidated here. */
852 0 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vr1->type),
853 0 : TYPE_VECTOR_SUBPARTS (vr2->type)))
854 : return false;
855 : }
856 897693 : else if (TYPE_MODE (vr1->type) != TYPE_MODE (vr2->type)
857 897693 : && (!mode_can_transfer_bits (TYPE_MODE (vr1->type))
858 45192 : || !mode_can_transfer_bits (TYPE_MODE (vr2->type))))
859 1037 : return false;
860 :
861 : i = 0;
862 : j = 0;
863 22146044 : do
864 : {
865 22146044 : poly_offset_int off1 = 0, off2 = 0;
866 22146044 : vn_reference_op_t vro1, vro2;
867 22146044 : vn_reference_op_s tem1, tem2;
868 22146044 : bool deref1 = false, deref2 = false;
869 22146044 : bool reverse1 = false, reverse2 = false;
870 72003164 : for (; vr1->operands.iterate (i, &vro1); i++)
871 : {
872 49857120 : if (vro1->opcode == MEM_REF)
873 : deref1 = true;
874 : /* Do not look through a storage order barrier. */
875 34120908 : else if (vro1->opcode == VIEW_CONVERT_EXPR && vro1->reverse)
876 73760 : return false;
877 49857120 : reverse1 |= vro1->reverse;
878 49857120 : if (lexical || known_eq (vro1->off, -1))
879 : break;
880 27711076 : off1 += vro1->off;
881 : }
882 50001689 : for (; vr2->operands.iterate (j, &vro2); j++)
883 : {
884 50001689 : if (vro2->opcode == MEM_REF)
885 : deref2 = true;
886 : /* Do not look through a storage order barrier. */
887 34241575 : else if (vro2->opcode == VIEW_CONVERT_EXPR && vro2->reverse)
888 : return false;
889 50001689 : reverse2 |= vro2->reverse;
890 50001689 : if (lexical || known_eq (vro2->off, -1))
891 : break;
892 27855645 : off2 += vro2->off;
893 : }
894 22146044 : if (maybe_ne (off1, off2) || reverse1 != reverse2)
895 : return false;
896 22145906 : if (deref1 && vro1->opcode == ADDR_EXPR)
897 : {
898 8360446 : memset (&tem1, 0, sizeof (tem1));
899 8360446 : tem1.op0 = TREE_OPERAND (vro1->op0, 0);
900 8360446 : tem1.type = TREE_TYPE (tem1.op0);
901 8360446 : tem1.opcode = TREE_CODE (tem1.op0);
902 8360446 : vro1 = &tem1;
903 8360446 : deref1 = false;
904 : }
905 22145906 : if (deref2 && vro2->opcode == ADDR_EXPR)
906 : {
907 8360456 : memset (&tem2, 0, sizeof (tem2));
908 8360456 : tem2.op0 = TREE_OPERAND (vro2->op0, 0);
909 8360456 : tem2.type = TREE_TYPE (tem2.op0);
910 8360456 : tem2.opcode = TREE_CODE (tem2.op0);
911 8360456 : vro2 = &tem2;
912 8360456 : deref2 = false;
913 : }
914 22145906 : if (deref1 != deref2)
915 : return false;
916 22088214 : if (!vn_reference_op_eq (vro1, vro2))
917 : return false;
918 : /* Both alignment and alias set are not relevant for the produced
919 : value but need to be included when doing lexical comparison.
920 : We also need to make sure that the access path ends in an
921 : access of the same size as otherwise we might assume an access
922 : may not trap while in fact it might. */
923 22076063 : if (lexical
924 2245997 : && (vro1->opcode == MEM_REF
925 2245997 : || vro1->opcode == TARGET_MEM_REF)
926 22812422 : && (TYPE_ALIGN (vro1->type) != TYPE_ALIGN (vro2->type)
927 736152 : || (TYPE_SIZE (vro1->type) != TYPE_SIZE (vro2->type)
928 6 : && (! TYPE_SIZE (vro1->type)
929 6 : || ! TYPE_SIZE (vro2->type)
930 6 : || ! operand_equal_p (TYPE_SIZE (vro1->type),
931 6 : TYPE_SIZE (vro2->type))))
932 2208438 : || (get_deref_alias_set (vro1->opcode == MEM_REF
933 736146 : ? TREE_TYPE (vro1->op0)
934 0 : : TREE_TYPE (vro1->op2))
935 1472292 : != get_deref_alias_set (vro2->opcode == MEM_REF
936 736146 : ? TREE_TYPE (vro2->op0)
937 0 : : TREE_TYPE (vro2->op2)))))
938 3779 : return false;
939 22072284 : ++j;
940 22072284 : ++i;
941 : }
942 44144568 : while (vr1->operands.length () != i
943 66216852 : || vr2->operands.length () != j);
944 :
945 : return true;
946 : }
947 :
948 : /* Copy the operations present in load/store REF into RESULT, a vector of
949 : vn_reference_op_s's. */
950 :
951 : void
952 225697654 : copy_reference_ops_from_ref (tree ref, vec<vn_reference_op_s> *result)
953 : {
954 : /* For non-calls, store the information that makes up the address. */
955 225697654 : tree orig = ref;
956 783726988 : while (ref)
957 : {
958 558029334 : vn_reference_op_s temp;
959 :
960 558029334 : memset (&temp, 0, sizeof (temp));
961 558029334 : temp.type = TREE_TYPE (ref);
962 558029334 : temp.opcode = TREE_CODE (ref);
963 558029334 : temp.off = -1;
964 :
965 558029334 : switch (temp.opcode)
966 : {
967 15151090 : case MODIFY_EXPR:
968 15151090 : temp.op0 = TREE_OPERAND (ref, 1);
969 15151090 : break;
970 137 : case WITH_SIZE_EXPR:
971 137 : temp.op0 = TREE_OPERAND (ref, 1);
972 137 : temp.off = 0;
973 137 : break;
974 119270962 : case MEM_REF:
975 : /* The base address gets its own vn_reference_op_s structure. */
976 119270962 : temp.op0 = TREE_OPERAND (ref, 1);
977 119270962 : if (!mem_ref_offset (ref).to_shwi (&temp.off))
978 0 : temp.off = -1;
979 119270962 : temp.clique = MR_DEPENDENCE_CLIQUE (ref);
980 119270962 : temp.base = MR_DEPENDENCE_BASE (ref);
981 119270962 : temp.reverse = REF_REVERSE_STORAGE_ORDER (ref);
982 119270962 : break;
983 2538761 : case TARGET_MEM_REF:
984 : /* The base address gets its own vn_reference_op_s structure. */
985 2538761 : temp.op0 = TMR_INDEX (ref);
986 2538761 : temp.op1 = TMR_STEP (ref);
987 2538761 : temp.op2 = TMR_OFFSET (ref);
988 2538761 : temp.clique = MR_DEPENDENCE_CLIQUE (ref);
989 2538761 : temp.base = MR_DEPENDENCE_BASE (ref);
990 2538761 : result->safe_push (temp);
991 2538761 : memset (&temp, 0, sizeof (temp));
992 2538761 : temp.type = NULL_TREE;
993 2538761 : temp.opcode = ERROR_MARK;
994 2538761 : temp.op0 = TMR_INDEX2 (ref);
995 2538761 : temp.off = -1;
996 2538761 : break;
997 786107 : case BIT_FIELD_REF:
998 : /* Record bits, position and storage order. */
999 786107 : temp.op0 = TREE_OPERAND (ref, 1);
1000 786107 : temp.op1 = TREE_OPERAND (ref, 2);
1001 1571516 : if (!multiple_p (bit_field_offset (ref), BITS_PER_UNIT, &temp.off))
1002 698 : temp.off = -1;
1003 786107 : temp.reverse = REF_REVERSE_STORAGE_ORDER (ref);
1004 786107 : break;
1005 148906698 : case COMPONENT_REF:
1006 : /* The field decl is enough to unambiguously specify the field,
1007 : so use its type here. */
1008 148906698 : temp.type = TREE_TYPE (TREE_OPERAND (ref, 1));
1009 148906698 : temp.op0 = TREE_OPERAND (ref, 1);
1010 148906698 : temp.op1 = TREE_OPERAND (ref, 2);
1011 297810964 : temp.reverse = (AGGREGATE_TYPE_P (TREE_TYPE (TREE_OPERAND (ref, 0)))
1012 297810699 : && TYPE_REVERSE_STORAGE_ORDER
1013 : (TREE_TYPE (TREE_OPERAND (ref, 0))));
1014 148906698 : {
1015 148906698 : tree this_offset = component_ref_field_offset (ref);
1016 148906698 : if (this_offset
1017 148906698 : && poly_int_tree_p (this_offset))
1018 : {
1019 148904562 : tree bit_offset = DECL_FIELD_BIT_OFFSET (TREE_OPERAND (ref, 1));
1020 148904562 : if (TREE_INT_CST_LOW (bit_offset) % BITS_PER_UNIT == 0)
1021 : {
1022 148429484 : poly_offset_int off
1023 148429484 : = (wi::to_poly_offset (this_offset)
1024 148429484 : + (wi::to_offset (bit_offset) >> LOG2_BITS_PER_UNIT));
1025 : /* Prohibit value-numbering zero offset components
1026 : of addresses the same before the pass folding
1027 : __builtin_object_size had a chance to run. Likewise
1028 : for components of zero size at arbitrary offset. */
1029 148429484 : if (TREE_CODE (orig) != ADDR_EXPR
1030 4980080 : || (TYPE_SIZE (temp.type)
1031 4967078 : && integer_nonzerop (TYPE_SIZE (temp.type))
1032 6418260 : && maybe_ne (off, 0))
1033 151496646 : || (cfun->curr_properties & PROP_objsz))
1034 146976132 : off.to_shwi (&temp.off);
1035 : }
1036 : }
1037 : }
1038 : break;
1039 38982371 : case ARRAY_RANGE_REF:
1040 38982371 : case ARRAY_REF:
1041 38982371 : {
1042 38982371 : tree eltype = TREE_TYPE (TREE_TYPE (TREE_OPERAND (ref, 0)));
1043 : /* Record index as operand. */
1044 38982371 : temp.op0 = TREE_OPERAND (ref, 1);
1045 : /* Always record lower bounds and element size. */
1046 38982371 : temp.op1 = array_ref_low_bound (ref);
1047 : /* But record element size in units of the type alignment. */
1048 38982371 : temp.op2 = TREE_OPERAND (ref, 3);
1049 38982371 : temp.align = eltype->type_common.align;
1050 38982371 : if (! temp.op2)
1051 38772075 : temp.op2 = size_binop (EXACT_DIV_EXPR, TYPE_SIZE_UNIT (eltype),
1052 : size_int (TYPE_ALIGN_UNIT (eltype)));
1053 : /* Prohibit value-numbering addresses of one-after-the-last
1054 : element ARRAY_REFs the same as addresses of other components
1055 : before the pass folding __builtin_object_size had a chance
1056 : to run. */
1057 38982371 : bool avoid_oob = true;
1058 38982371 : if (TREE_CODE (orig) != ADDR_EXPR
1059 477353 : || cfun->curr_properties & PROP_objsz)
1060 : avoid_oob = false;
1061 224478 : else if (poly_int_tree_p (temp.op0))
1062 : {
1063 75221 : tree ub = array_ref_up_bound (ref);
1064 75221 : if (ub
1065 73583 : && poly_int_tree_p (ub)
1066 : /* ??? The C frontend for T[0] uses [0:] and the
1067 : C++ frontend [0:-1U]. See layout_type for how
1068 : awkward this is. */
1069 65282 : && !integer_minus_onep (ub)
1070 148804 : && known_le (wi::to_poly_offset (temp.op0),
1071 : wi::to_poly_offset (ub)))
1072 64435 : avoid_oob = false;
1073 : }
1074 38982371 : if (poly_int_tree_p (temp.op0)
1075 22335140 : && poly_int_tree_p (temp.op1)
1076 22335112 : && TREE_CODE (temp.op2) == INTEGER_CST
1077 61256875 : && !avoid_oob)
1078 : {
1079 44529146 : poly_offset_int off = ((wi::to_poly_offset (temp.op0)
1080 66793719 : - wi::to_poly_offset (temp.op1))
1081 44529146 : * wi::to_offset (temp.op2)
1082 22264573 : * vn_ref_op_align_unit (&temp));
1083 22264573 : off.to_shwi (&temp.off);
1084 : }
1085 38982371 : temp.reverse = (AGGREGATE_TYPE_P (TREE_TYPE (TREE_OPERAND (ref, 0)))
1086 38982371 : && TYPE_REVERSE_STORAGE_ORDER
1087 : (TREE_TYPE (TREE_OPERAND (ref, 0))));
1088 : }
1089 38982371 : break;
1090 82983466 : case VAR_DECL:
1091 82983466 : if (DECL_HARD_REGISTER (ref))
1092 : {
1093 20325 : temp.op0 = ref;
1094 20325 : break;
1095 : }
1096 : /* Fallthru. */
1097 86375837 : case PARM_DECL:
1098 86375837 : case CONST_DECL:
1099 86375837 : case RESULT_DECL:
1100 : /* Canonicalize decls to MEM[&decl] which is what we end up with
1101 : when valueizing MEM[ptr] with ptr = &decl. */
1102 86375837 : temp.opcode = MEM_REF;
1103 86375837 : temp.op0 = build_int_cst (build_pointer_type (TREE_TYPE (ref)), 0);
1104 86375837 : temp.off = 0;
1105 86375837 : result->safe_push (temp);
1106 86375837 : temp.opcode = ADDR_EXPR;
1107 86375837 : temp.op0 = build1 (ADDR_EXPR, TREE_TYPE (temp.op0), ref);
1108 86375837 : temp.type = TREE_TYPE (temp.op0);
1109 86375837 : temp.off = -1;
1110 86375837 : break;
1111 97582350 : case STRING_CST:
1112 97582350 : case INTEGER_CST:
1113 97582350 : case POLY_INT_CST:
1114 97582350 : case COMPLEX_CST:
1115 97582350 : case VECTOR_CST:
1116 97582350 : case REAL_CST:
1117 97582350 : case FIXED_CST:
1118 97582350 : case CONSTRUCTOR:
1119 97582350 : case SSA_NAME:
1120 97582350 : temp.op0 = ref;
1121 97582350 : break;
1122 45973719 : case ADDR_EXPR:
1123 45973719 : if (is_gimple_min_invariant (ref))
1124 : {
1125 41719142 : temp.op0 = ref;
1126 41719142 : break;
1127 : }
1128 : break;
1129 : /* These are only interesting for their operands, their
1130 : existence, and their type. They will never be the last
1131 : ref in the chain of references (IE they require an
1132 : operand), so we don't have to put anything
1133 : for op* as it will be handled by the iteration */
1134 493788 : case REALPART_EXPR:
1135 493788 : temp.off = 0;
1136 493788 : break;
1137 1448814 : case VIEW_CONVERT_EXPR:
1138 1448814 : temp.off = 0;
1139 1448814 : temp.reverse = storage_order_barrier_p (ref);
1140 1448814 : break;
1141 498375 : case IMAGPART_EXPR:
1142 : /* This is only interesting for its constant offset. */
1143 498375 : temp.off = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (ref)));
1144 498375 : break;
1145 0 : default:
1146 0 : gcc_unreachable ();
1147 : }
1148 558029334 : result->safe_push (temp);
1149 :
1150 558029334 : if (REFERENCE_CLASS_P (ref)
1151 245103458 : || TREE_CODE (ref) == MODIFY_EXPR
1152 229952368 : || TREE_CODE (ref) == WITH_SIZE_EXPR
1153 787981565 : || (TREE_CODE (ref) == ADDR_EXPR
1154 45973719 : && !is_gimple_min_invariant (ref)))
1155 332331680 : ref = TREE_OPERAND (ref, 0);
1156 : else
1157 : ref = NULL_TREE;
1158 : }
1159 225697654 : }
1160 :
1161 : /* Build a alias-oracle reference abstraction in *REF from the vn_reference
1162 : operands in *OPS, the reference alias set SET and the reference type TYPE.
1163 : Return true if something useful was produced. */
1164 :
1165 : bool
1166 14749435 : ao_ref_init_from_vn_reference (ao_ref *ref,
1167 : alias_set_type set, alias_set_type base_set,
1168 : tree type, const vec<vn_reference_op_s> &ops)
1169 : {
1170 14749435 : unsigned i;
1171 14749435 : tree base = NULL_TREE;
1172 14749435 : tree *op0_p = &base;
1173 14749435 : poly_offset_int offset = 0;
1174 14749435 : poly_offset_int max_size;
1175 14749435 : poly_offset_int size = -1;
1176 14749435 : tree size_tree = NULL_TREE;
1177 :
1178 : /* We don't handle calls. */
1179 14749435 : if (!type)
1180 : return false;
1181 :
1182 14749435 : machine_mode mode = TYPE_MODE (type);
1183 14749435 : if (mode == BLKmode)
1184 66003 : size_tree = TYPE_SIZE (type);
1185 : else
1186 29366864 : size = GET_MODE_BITSIZE (mode);
1187 14683432 : if (size_tree != NULL_TREE
1188 66003 : && poly_int_tree_p (size_tree))
1189 66003 : size = wi::to_poly_offset (size_tree);
1190 :
1191 : /* Lower the final access size from the outermost expression. */
1192 14749435 : const_vn_reference_op_t cst_op = &ops[0];
1193 : /* Cast away constness for the sake of the const-unsafe
1194 : FOR_EACH_VEC_ELT(). */
1195 14749435 : vn_reference_op_t op = const_cast<vn_reference_op_t>(cst_op);
1196 14749435 : size_tree = NULL_TREE;
1197 14749435 : if (op->opcode == COMPONENT_REF)
1198 5141767 : size_tree = DECL_SIZE (op->op0);
1199 9607668 : else if (op->opcode == BIT_FIELD_REF)
1200 69192 : size_tree = op->op0;
1201 5210959 : if (size_tree != NULL_TREE
1202 5210959 : && poly_int_tree_p (size_tree)
1203 10421918 : && (!known_size_p (size)
1204 14749435 : || known_lt (wi::to_poly_offset (size_tree), size)))
1205 39075 : size = wi::to_poly_offset (size_tree);
1206 :
1207 : /* Initially, maxsize is the same as the accessed element size.
1208 : In the following it will only grow (or become -1). */
1209 14749435 : max_size = size;
1210 :
1211 : /* Compute cumulative bit-offset for nested component-refs and array-refs,
1212 : and find the ultimate containing object. */
1213 56811950 : FOR_EACH_VEC_ELT (ops, i, op)
1214 : {
1215 42211137 : switch (op->opcode)
1216 : {
1217 : case CALL_EXPR:
1218 : return false;
1219 :
1220 : /* Record the base objects. */
1221 14292019 : case MEM_REF:
1222 14292019 : *op0_p = build2 (MEM_REF, op->type,
1223 : NULL_TREE, op->op0);
1224 14292019 : MR_DEPENDENCE_CLIQUE (*op0_p) = op->clique;
1225 14292019 : MR_DEPENDENCE_BASE (*op0_p) = op->base;
1226 14292019 : op0_p = &TREE_OPERAND (*op0_p, 0);
1227 14292019 : break;
1228 :
1229 308262 : case TARGET_MEM_REF:
1230 924786 : *op0_p = build5 (TARGET_MEM_REF, op->type,
1231 : NULL_TREE, op->op2, op->op0,
1232 308262 : op->op1, ops[i+1].op0);
1233 308262 : MR_DEPENDENCE_CLIQUE (*op0_p) = op->clique;
1234 308262 : MR_DEPENDENCE_BASE (*op0_p) = op->base;
1235 308262 : op0_p = &TREE_OPERAND (*op0_p, 0);
1236 308262 : ++i;
1237 308262 : break;
1238 :
1239 : /* Unwrap some of the wrapped decls. */
1240 6702997 : case ADDR_EXPR:
1241 : /* Apart from ADDR_EXPR arguments to MEM_REF. */
1242 6702997 : if (base != NULL_TREE
1243 6702996 : && TREE_CODE (base) == MEM_REF
1244 6667954 : && op->op0
1245 13370951 : && DECL_P (TREE_OPERAND (op->op0, 0)))
1246 : {
1247 6660661 : const_vn_reference_op_t pop = &ops[i-1];
1248 6660661 : base = TREE_OPERAND (op->op0, 0);
1249 6660661 : if (known_eq (pop->off, -1))
1250 : {
1251 25 : max_size = -1;
1252 25 : offset = 0;
1253 : }
1254 : else
1255 19981908 : offset += poly_offset_int (pop->off) * BITS_PER_UNIT;
1256 : op0_p = NULL;
1257 : break;
1258 : }
1259 : /* Fallthru. */
1260 7940152 : case PARM_DECL:
1261 7940152 : case CONST_DECL:
1262 7940152 : case RESULT_DECL:
1263 : /* ??? We shouldn't see these, but un-canonicalize what
1264 : copy_reference_ops_from_ref does when visiting MEM_REF. */
1265 7940152 : case VAR_DECL:
1266 : /* ??? And for this only have DECL_HARD_REGISTER. */
1267 7940152 : case STRING_CST:
1268 : /* This can show up in ARRAY_REF bases. */
1269 7940152 : case INTEGER_CST:
1270 7940152 : case SSA_NAME:
1271 7940152 : *op0_p = op->op0;
1272 7940152 : op0_p = NULL;
1273 7940152 : break;
1274 :
1275 : /* And now the usual component-reference style ops. */
1276 69192 : case BIT_FIELD_REF:
1277 69192 : offset += wi::to_poly_offset (op->op1);
1278 69192 : break;
1279 :
1280 8474605 : case COMPONENT_REF:
1281 8474605 : {
1282 8474605 : tree field = op->op0;
1283 : /* We do not have a complete COMPONENT_REF tree here so we
1284 : cannot use component_ref_field_offset. Do the interesting
1285 : parts manually. */
1286 8474605 : tree this_offset = DECL_FIELD_OFFSET (field);
1287 :
1288 8474605 : if (op->op1 || !poly_int_tree_p (this_offset))
1289 234 : max_size = -1;
1290 : else
1291 : {
1292 8474371 : poly_offset_int woffset = (wi::to_poly_offset (this_offset)
1293 8474371 : << LOG2_BITS_PER_UNIT);
1294 8474371 : woffset += wi::to_offset (DECL_FIELD_BIT_OFFSET (field));
1295 8474371 : offset += woffset;
1296 : }
1297 : break;
1298 : }
1299 :
1300 3110806 : case ARRAY_RANGE_REF:
1301 3110806 : case ARRAY_REF:
1302 : /* Use the recorded constant offset. */
1303 3110806 : if (maybe_eq (op->off, -1))
1304 1202640 : max_size = -1;
1305 : else
1306 5724498 : offset += poly_offset_int (op->off) * BITS_PER_UNIT;
1307 : break;
1308 :
1309 : case REALPART_EXPR:
1310 : break;
1311 :
1312 : case IMAGPART_EXPR:
1313 42062515 : offset += size;
1314 : break;
1315 :
1316 : case VIEW_CONVERT_EXPR:
1317 : break;
1318 :
1319 : case POLY_INT_CST:
1320 : case COMPLEX_CST:
1321 : case VECTOR_CST:
1322 : case REAL_CST:
1323 : case FIXED_CST:
1324 : case CONSTRUCTOR:
1325 : return false;
1326 :
1327 : default:
1328 : return false;
1329 : }
1330 : }
1331 :
1332 14600813 : if (base == NULL_TREE)
1333 : return false;
1334 :
1335 14600813 : ref->ref = NULL_TREE;
1336 14600813 : ref->base = base;
1337 14600813 : ref->ref_alias_set = set;
1338 14600813 : ref->base_alias_set = base_set;
1339 : /* We discount volatiles from value-numbering elsewhere. */
1340 14600813 : ref->volatile_p = false;
1341 :
1342 14600813 : if (!size.to_shwi (&ref->size) || maybe_lt (ref->size, 0))
1343 : {
1344 0 : ref->offset = 0;
1345 0 : ref->size = -1;
1346 0 : ref->max_size = -1;
1347 0 : return true;
1348 : }
1349 :
1350 14600813 : if (!offset.to_shwi (&ref->offset))
1351 : {
1352 26 : ref->offset = 0;
1353 26 : ref->max_size = -1;
1354 26 : return true;
1355 : }
1356 :
1357 14600787 : if (!max_size.to_shwi (&ref->max_size) || maybe_lt (ref->max_size, 0))
1358 1052005 : ref->max_size = -1;
1359 :
1360 : return true;
1361 : }
1362 :
1363 : /* Copy the operations present in load/store/call REF into RESULT, a vector of
1364 : vn_reference_op_s's. */
1365 :
1366 : static void
1367 9346632 : copy_reference_ops_from_call (gcall *call,
1368 : vec<vn_reference_op_s> *result)
1369 : {
1370 9346632 : vn_reference_op_s temp;
1371 9346632 : unsigned i;
1372 9346632 : tree lhs = gimple_call_lhs (call);
1373 9346632 : int lr;
1374 :
1375 : /* If 2 calls have a different non-ssa lhs, vdef value numbers should be
1376 : different. By adding the lhs here in the vector, we ensure that the
1377 : hashcode is different, guaranteeing a different value number. */
1378 9346632 : if (lhs && TREE_CODE (lhs) != SSA_NAME)
1379 : {
1380 454979 : memset (&temp, 0, sizeof (temp));
1381 454979 : temp.opcode = MODIFY_EXPR;
1382 454979 : temp.type = TREE_TYPE (lhs);
1383 454979 : temp.op0 = lhs;
1384 454979 : temp.off = -1;
1385 454979 : result->safe_push (temp);
1386 : }
1387 :
1388 : /* Copy the type, opcode, function, static chain and EH region, if any. */
1389 9346632 : memset (&temp, 0, sizeof (temp));
1390 9346632 : temp.type = gimple_call_fntype (call);
1391 9346632 : temp.opcode = CALL_EXPR;
1392 9346632 : temp.op0 = gimple_call_fn (call);
1393 9346632 : if (gimple_call_internal_p (call))
1394 538998 : temp.clique = gimple_call_internal_fn (call);
1395 9346632 : temp.op1 = gimple_call_chain (call);
1396 9346632 : if (stmt_could_throw_p (cfun, call) && (lr = lookup_stmt_eh_lp (call)) > 0)
1397 623815 : temp.op2 = size_int (lr);
1398 9346632 : temp.off = -1;
1399 9346632 : result->safe_push (temp);
1400 :
1401 : /* Copy the call arguments. As they can be references as well,
1402 : just chain them together. */
1403 27646869 : for (i = 0; i < gimple_call_num_args (call); ++i)
1404 : {
1405 18300237 : tree callarg = gimple_call_arg (call, i);
1406 18300237 : copy_reference_ops_from_ref (callarg, result);
1407 : }
1408 9346632 : }
1409 :
1410 : /* Fold *& at position *I_P in a vn_reference_op_s vector *OPS. Updates
1411 : *I_P to point to the last element of the replacement. */
1412 : static bool
1413 128661954 : vn_reference_fold_indirect (vec<vn_reference_op_s> *ops,
1414 : unsigned int *i_p)
1415 : {
1416 128661954 : unsigned int i = *i_p;
1417 128661954 : vn_reference_op_t op = &(*ops)[i];
1418 128661954 : vn_reference_op_t mem_op = &(*ops)[i - 1];
1419 128661954 : tree addr_base;
1420 128661954 : poly_int64 addr_offset = 0;
1421 :
1422 : /* The only thing we have to do is from &OBJ.foo.bar add the offset
1423 : from .foo.bar to the preceding MEM_REF offset and replace the
1424 : address with &OBJ. */
1425 128661954 : addr_base = get_addr_base_and_unit_offset_1 (TREE_OPERAND (op->op0, 0),
1426 : &addr_offset, vn_valueize);
1427 128661954 : gcc_checking_assert (addr_base && TREE_CODE (addr_base) != MEM_REF);
1428 128661954 : if (addr_base != TREE_OPERAND (op->op0, 0))
1429 : {
1430 683865 : poly_offset_int off
1431 683865 : = (poly_offset_int::from (wi::to_poly_wide (mem_op->op0),
1432 : SIGNED)
1433 683865 : + addr_offset);
1434 683865 : mem_op->op0 = wide_int_to_tree (TREE_TYPE (mem_op->op0), off);
1435 683865 : op->op0 = build_fold_addr_expr (addr_base);
1436 683865 : if (tree_fits_shwi_p (mem_op->op0))
1437 683798 : mem_op->off = tree_to_shwi (mem_op->op0);
1438 : else
1439 67 : mem_op->off = -1;
1440 683865 : return true;
1441 : }
1442 : return false;
1443 : }
1444 :
1445 : /* Fold *& at position *I_P in a vn_reference_op_s vector *OPS. Updates
1446 : *I_P to point to the last element of the replacement. */
1447 : static bool
1448 86764400 : vn_reference_maybe_forwprop_address (vec<vn_reference_op_s> *ops,
1449 : unsigned int *i_p)
1450 : {
1451 86764400 : bool changed = false;
1452 94215886 : vn_reference_op_t op;
1453 :
1454 94215886 : do
1455 : {
1456 94215886 : unsigned int i = *i_p;
1457 94215886 : op = &(*ops)[i];
1458 94215886 : vn_reference_op_t mem_op = &(*ops)[i - 1];
1459 94215886 : gimple *def_stmt;
1460 94215886 : enum tree_code code;
1461 94215886 : poly_offset_int off;
1462 :
1463 94215886 : def_stmt = SSA_NAME_DEF_STMT (op->op0);
1464 94215886 : if (!is_gimple_assign (def_stmt))
1465 86762449 : return changed;
1466 :
1467 38066951 : code = gimple_assign_rhs_code (def_stmt);
1468 38066951 : if (code != ADDR_EXPR
1469 38066951 : && code != POINTER_PLUS_EXPR)
1470 : return changed;
1471 :
1472 20211312 : off = poly_offset_int::from (wi::to_poly_wide (mem_op->op0), SIGNED);
1473 :
1474 : /* The only thing we have to do is from &OBJ.foo.bar add the offset
1475 : from .foo.bar to the preceding MEM_REF offset and replace the
1476 : address with &OBJ. */
1477 20211312 : if (code == ADDR_EXPR)
1478 : {
1479 963165 : tree addr, addr_base;
1480 963165 : poly_int64 addr_offset;
1481 :
1482 963165 : addr = gimple_assign_rhs1 (def_stmt);
1483 963165 : addr_base = get_addr_base_and_unit_offset_1 (TREE_OPERAND (addr, 0),
1484 : &addr_offset,
1485 : vn_valueize);
1486 : /* If that didn't work because the address isn't invariant propagate
1487 : the reference tree from the address operation in case the current
1488 : dereference isn't offsetted. */
1489 963165 : if (!addr_base
1490 284892 : && *i_p == ops->length () - 1
1491 142446 : && known_eq (off, 0)
1492 : /* This makes us disable this transform for PRE where the
1493 : reference ops might be also used for code insertion which
1494 : is invalid. */
1495 1049121 : && default_vn_walk_kind == VN_WALKREWRITE)
1496 : {
1497 85866 : auto_vec<vn_reference_op_s, 32> tem;
1498 85866 : copy_reference_ops_from_ref (TREE_OPERAND (addr, 0), &tem);
1499 : /* Make sure to preserve TBAA info. The only objects not
1500 : wrapped in MEM_REFs that can have their address taken are
1501 : STRING_CSTs. */
1502 85866 : if (tem.length () >= 2
1503 85866 : && tem[tem.length () - 2].opcode == MEM_REF)
1504 : {
1505 85851 : vn_reference_op_t new_mem_op = &tem[tem.length () - 2];
1506 85851 : new_mem_op->op0
1507 85851 : = wide_int_to_tree (TREE_TYPE (mem_op->op0),
1508 171702 : wi::to_poly_wide (new_mem_op->op0));
1509 : }
1510 : /* Do not forward addresses of TARGET_MEM_REF. */
1511 15 : else if (tem[0].opcode == TARGET_MEM_REF)
1512 : return changed;
1513 : else
1514 15 : gcc_assert (tem.last ().opcode == STRING_CST);
1515 85866 : ops->pop ();
1516 85866 : ops->pop ();
1517 85866 : ops->safe_splice (tem);
1518 85866 : --*i_p;
1519 85866 : return true;
1520 85866 : }
1521 877299 : if (!addr_base
1522 820719 : || TREE_CODE (addr_base) != MEM_REF
1523 1696215 : || (TREE_CODE (TREE_OPERAND (addr_base, 0)) == SSA_NAME
1524 817055 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (TREE_OPERAND (addr_base,
1525 : 0))))
1526 : return changed;
1527 :
1528 818916 : off += addr_offset;
1529 818916 : off += mem_ref_offset (addr_base);
1530 818916 : op->op0 = TREE_OPERAND (addr_base, 0);
1531 : }
1532 : else
1533 : {
1534 19248147 : tree ptr, ptroff;
1535 19248147 : ptr = gimple_assign_rhs1 (def_stmt);
1536 19248147 : ptroff = gimple_assign_rhs2 (def_stmt);
1537 19248147 : if (TREE_CODE (ptr) != SSA_NAME
1538 17523781 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ptr)
1539 : /* Make sure to not endlessly recurse.
1540 : See gcc.dg/tree-ssa/20040408-1.c for an example. Can easily
1541 : happen when we value-number a PHI to its backedge value. */
1542 17522384 : || SSA_VAL (ptr) == op->op0
1543 36770531 : || !poly_int_tree_p (ptroff))
1544 12613626 : return changed;
1545 :
1546 6634521 : off += wi::to_poly_offset (ptroff);
1547 6634521 : op->op0 = ptr;
1548 : }
1549 :
1550 7453437 : mem_op->op0 = wide_int_to_tree (TREE_TYPE (mem_op->op0), off);
1551 7453437 : if (tree_fits_shwi_p (mem_op->op0))
1552 7143764 : mem_op->off = tree_to_shwi (mem_op->op0);
1553 : else
1554 309673 : mem_op->off = -1;
1555 : /* ??? Can end up with endless recursion here!?
1556 : gcc.c-torture/execute/strcmp-1.c */
1557 7453437 : if (TREE_CODE (op->op0) == SSA_NAME)
1558 7451576 : op->op0 = SSA_VAL (op->op0);
1559 7453437 : if (TREE_CODE (op->op0) != SSA_NAME)
1560 1951 : op->opcode = TREE_CODE (op->op0);
1561 :
1562 7453437 : changed = true;
1563 : }
1564 : /* Tail-recurse. */
1565 7453437 : while (TREE_CODE (op->op0) == SSA_NAME);
1566 :
1567 : /* Fold a remaining *&. */
1568 1951 : if (TREE_CODE (op->op0) == ADDR_EXPR)
1569 261 : vn_reference_fold_indirect (ops, i_p);
1570 :
1571 : return changed;
1572 : }
1573 :
1574 : /* Optimize the reference REF to a constant if possible or return
1575 : NULL_TREE if not. */
1576 :
1577 : tree
1578 111352126 : fully_constant_vn_reference_p (vn_reference_t ref)
1579 : {
1580 111352126 : vec<vn_reference_op_s> operands = ref->operands;
1581 111352126 : vn_reference_op_t op;
1582 :
1583 : /* Try to simplify the translated expression if it is
1584 : a call to a builtin function with at most two arguments. */
1585 111352126 : op = &operands[0];
1586 111352126 : if (op->opcode == CALL_EXPR
1587 89860 : && (!op->op0
1588 82441 : || (TREE_CODE (op->op0) == ADDR_EXPR
1589 82441 : && TREE_CODE (TREE_OPERAND (op->op0, 0)) == FUNCTION_DECL
1590 82441 : && fndecl_built_in_p (TREE_OPERAND (op->op0, 0),
1591 : BUILT_IN_NORMAL)))
1592 72399 : && operands.length () >= 2
1593 111424493 : && operands.length () <= 3)
1594 : {
1595 34037 : vn_reference_op_t arg0, arg1 = NULL;
1596 34037 : bool anyconst = false;
1597 34037 : arg0 = &operands[1];
1598 34037 : if (operands.length () > 2)
1599 5592 : arg1 = &operands[2];
1600 34037 : if (TREE_CODE_CLASS (arg0->opcode) == tcc_constant
1601 34037 : || (arg0->opcode == ADDR_EXPR
1602 13869 : && is_gimple_min_invariant (arg0->op0)))
1603 : anyconst = true;
1604 34037 : if (arg1
1605 34037 : && (TREE_CODE_CLASS (arg1->opcode) == tcc_constant
1606 4072 : || (arg1->opcode == ADDR_EXPR
1607 587 : && is_gimple_min_invariant (arg1->op0))))
1608 : anyconst = true;
1609 31930 : if (anyconst)
1610 : {
1611 22411 : combined_fn fn;
1612 22411 : if (op->op0)
1613 21459 : fn = as_combined_fn (DECL_FUNCTION_CODE
1614 21459 : (TREE_OPERAND (op->op0, 0)));
1615 : else
1616 952 : fn = as_combined_fn ((internal_fn) op->clique);
1617 22411 : tree folded;
1618 22411 : if (arg1)
1619 2717 : folded = fold_const_call (fn, ref->type, arg0->op0, arg1->op0);
1620 : else
1621 19694 : folded = fold_const_call (fn, ref->type, arg0->op0);
1622 22411 : if (folded
1623 22411 : && is_gimple_min_invariant (folded))
1624 : return folded;
1625 : }
1626 : }
1627 :
1628 : /* Simplify reads from constants or constant initializers. */
1629 111318089 : else if (BITS_PER_UNIT == 8
1630 111318089 : && ref->type
1631 111318089 : && COMPLETE_TYPE_P (ref->type)
1632 222636136 : && is_gimple_reg_type (ref->type))
1633 : {
1634 106950630 : poly_int64 off = 0;
1635 106950630 : HOST_WIDE_INT size;
1636 106950630 : if (INTEGRAL_TYPE_P (ref->type))
1637 54414415 : size = TYPE_PRECISION (ref->type);
1638 52536215 : else if (tree_fits_shwi_p (TYPE_SIZE (ref->type)))
1639 52536215 : size = tree_to_shwi (TYPE_SIZE (ref->type));
1640 : else
1641 111352126 : return NULL_TREE;
1642 106950630 : if (size % BITS_PER_UNIT != 0
1643 105153870 : || size > MAX_BITSIZE_MODE_ANY_MODE)
1644 : return NULL_TREE;
1645 105152543 : size /= BITS_PER_UNIT;
1646 105152543 : unsigned i;
1647 194620425 : for (i = 0; i < operands.length (); ++i)
1648 : {
1649 194620425 : if (TREE_CODE_CLASS (operands[i].opcode) == tcc_constant)
1650 : {
1651 309 : ++i;
1652 309 : break;
1653 : }
1654 194620116 : if (operands[i].reverse)
1655 : return NULL_TREE;
1656 194611758 : if (known_eq (operands[i].off, -1))
1657 : return NULL_TREE;
1658 180689874 : off += operands[i].off;
1659 180689874 : if (operands[i].opcode == MEM_REF)
1660 : {
1661 91221992 : ++i;
1662 91221992 : break;
1663 : }
1664 : }
1665 91222301 : vn_reference_op_t base = &operands[--i];
1666 91222301 : tree ctor = error_mark_node;
1667 91222301 : tree decl = NULL_TREE;
1668 91222301 : if (TREE_CODE_CLASS (base->opcode) == tcc_constant)
1669 309 : ctor = base->op0;
1670 91221992 : else if (base->opcode == MEM_REF
1671 91221992 : && base[1].opcode == ADDR_EXPR
1672 149870958 : && (VAR_P (TREE_OPERAND (base[1].op0, 0))
1673 3585888 : || TREE_CODE (TREE_OPERAND (base[1].op0, 0)) == CONST_DECL
1674 3585828 : || TREE_CODE (TREE_OPERAND (base[1].op0, 0)) == STRING_CST))
1675 : {
1676 55069204 : decl = TREE_OPERAND (base[1].op0, 0);
1677 55069204 : if (TREE_CODE (decl) == STRING_CST)
1678 : ctor = decl;
1679 : else
1680 55063138 : ctor = ctor_for_folding (decl);
1681 : }
1682 91216235 : if (ctor == NULL_TREE)
1683 386 : return build_zero_cst (ref->type);
1684 91221915 : else if (ctor != error_mark_node)
1685 : {
1686 104011 : HOST_WIDE_INT const_off;
1687 104011 : if (decl)
1688 : {
1689 207404 : tree res = fold_ctor_reference (ref->type, ctor,
1690 103702 : off * BITS_PER_UNIT,
1691 103702 : size * BITS_PER_UNIT, decl);
1692 103702 : if (res)
1693 : {
1694 58891 : STRIP_USELESS_TYPE_CONVERSION (res);
1695 58891 : if (is_gimple_min_invariant (res))
1696 111352126 : return res;
1697 : }
1698 : }
1699 309 : else if (off.is_constant (&const_off))
1700 : {
1701 309 : unsigned char buf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
1702 309 : int len = native_encode_expr (ctor, buf, size, const_off);
1703 309 : if (len > 0)
1704 139 : return native_interpret_expr (ref->type, buf, len);
1705 : }
1706 : }
1707 : }
1708 :
1709 : return NULL_TREE;
1710 : }
1711 :
1712 : /* Return true if OPS contain a storage order barrier. */
1713 :
1714 : static bool
1715 60195914 : contains_storage_order_barrier_p (vec<vn_reference_op_s> ops)
1716 : {
1717 60195914 : vn_reference_op_t op;
1718 60195914 : unsigned i;
1719 :
1720 235817581 : FOR_EACH_VEC_ELT (ops, i, op)
1721 175621667 : if (op->opcode == VIEW_CONVERT_EXPR && op->reverse)
1722 : return true;
1723 :
1724 : return false;
1725 : }
1726 :
1727 : /* Return true if OPS represent an access with reverse storage order. */
1728 :
1729 : static bool
1730 60204182 : reverse_storage_order_for_component_p (vec<vn_reference_op_s> ops)
1731 : {
1732 60204182 : unsigned i = 0;
1733 60204182 : if (ops[i].opcode == REALPART_EXPR || ops[i].opcode == IMAGPART_EXPR)
1734 : ++i;
1735 60204182 : switch (ops[i].opcode)
1736 : {
1737 58123992 : case ARRAY_REF:
1738 58123992 : case COMPONENT_REF:
1739 58123992 : case BIT_FIELD_REF:
1740 58123992 : case MEM_REF:
1741 58123992 : return ops[i].reverse;
1742 : default:
1743 : return false;
1744 : }
1745 : }
1746 :
1747 : /* Transform any SSA_NAME's in a vector of vn_reference_op_s
1748 : structures into their value numbers. This is done in-place, and
1749 : the vector passed in is returned. *VALUEIZED_ANYTHING will specify
1750 : whether any operands were valueized. */
1751 :
1752 : static void
1753 222694605 : valueize_refs_1 (vec<vn_reference_op_s> *orig, bool *valueized_anything,
1754 : bool with_avail = false)
1755 : {
1756 222694605 : *valueized_anything = false;
1757 :
1758 898709376 : for (unsigned i = 0; i < orig->length (); ++i)
1759 : {
1760 676014771 : re_valueize:
1761 679967015 : vn_reference_op_t vro = &(*orig)[i];
1762 679967015 : if (vro->opcode == SSA_NAME
1763 581125461 : || (vro->op0 && TREE_CODE (vro->op0) == SSA_NAME))
1764 : {
1765 123557221 : tree tem = with_avail ? vn_valueize (vro->op0) : SSA_VAL (vro->op0);
1766 123557221 : if (tem != vro->op0)
1767 : {
1768 18420487 : *valueized_anything = true;
1769 18420487 : vro->op0 = tem;
1770 : }
1771 : /* If it transforms from an SSA_NAME to a constant, update
1772 : the opcode. */
1773 123557221 : if (TREE_CODE (vro->op0) != SSA_NAME && vro->opcode == SSA_NAME)
1774 2149869 : vro->opcode = TREE_CODE (vro->op0);
1775 : }
1776 679967015 : if (vro->op1 && TREE_CODE (vro->op1) == SSA_NAME)
1777 : {
1778 26286 : tree tem = with_avail ? vn_valueize (vro->op1) : SSA_VAL (vro->op1);
1779 26286 : if (tem != vro->op1)
1780 : {
1781 609 : *valueized_anything = true;
1782 609 : vro->op1 = tem;
1783 : }
1784 : }
1785 679967015 : if (vro->op2 && TREE_CODE (vro->op2) == SSA_NAME)
1786 : {
1787 205492 : tree tem = with_avail ? vn_valueize (vro->op2) : SSA_VAL (vro->op2);
1788 205492 : if (tem != vro->op2)
1789 : {
1790 119592 : *valueized_anything = true;
1791 119592 : vro->op2 = tem;
1792 : }
1793 : }
1794 : /* If it transforms from an SSA_NAME to an address, fold with
1795 : a preceding indirect reference. */
1796 679967015 : if (i > 0
1797 457193086 : && vro->op0
1798 453665194 : && TREE_CODE (vro->op0) == ADDR_EXPR
1799 814592863 : && (*orig)[i - 1].opcode == MEM_REF)
1800 : {
1801 128661693 : if (vn_reference_fold_indirect (orig, &i))
1802 683865 : *valueized_anything = true;
1803 : }
1804 551305322 : else if (i > 0
1805 328531393 : && vro->opcode == SSA_NAME
1806 647997007 : && (*orig)[i - 1].opcode == MEM_REF)
1807 : {
1808 86764400 : if (vn_reference_maybe_forwprop_address (orig, &i))
1809 : {
1810 3952244 : *valueized_anything = true;
1811 : /* Re-valueize the current operand. */
1812 3952244 : goto re_valueize;
1813 : }
1814 : }
1815 : /* If it transforms a non-constant ARRAY_REF into a constant
1816 : one, adjust the constant offset. */
1817 464540922 : else if ((vro->opcode == ARRAY_REF
1818 464540922 : || vro->opcode == ARRAY_RANGE_REF)
1819 40082175 : && known_eq (vro->off, -1)
1820 17441363 : && poly_int_tree_p (vro->op0)
1821 5009067 : && poly_int_tree_p (vro->op1)
1822 469549989 : && TREE_CODE (vro->op2) == INTEGER_CST)
1823 : {
1824 : /* Prohibit value-numbering addresses of one-after-the-last
1825 : element ARRAY_REFs the same as addresses of other components
1826 : before the pass folding __builtin_object_size had a chance
1827 : to run. */
1828 4875356 : if (!(cfun->curr_properties & PROP_objsz)
1829 6128181 : && (*orig)[0].opcode == ADDR_EXPR)
1830 : {
1831 35707 : tree dom = TYPE_DOMAIN ((*orig)[i + 1].type);
1832 54277 : if (!dom
1833 35557 : || !TYPE_MAX_VALUE (dom)
1834 25577 : || !poly_int_tree_p (TYPE_MAX_VALUE (dom))
1835 52930 : || integer_minus_onep (TYPE_MAX_VALUE (dom)))
1836 19377 : continue;
1837 17137 : if (!known_le (wi::to_poly_offset (vro->op0),
1838 : wi::to_poly_offset (TYPE_MAX_VALUE (dom))))
1839 807 : continue;
1840 : }
1841 :
1842 9711958 : poly_offset_int off = ((wi::to_poly_offset (vro->op0)
1843 14567937 : - wi::to_poly_offset (vro->op1))
1844 9711958 : * wi::to_offset (vro->op2)
1845 4855979 : * vn_ref_op_align_unit (vro));
1846 4855979 : off.to_shwi (&vro->off);
1847 : }
1848 : }
1849 222694605 : }
1850 :
1851 : static void
1852 12838624 : valueize_refs (vec<vn_reference_op_s> *orig)
1853 : {
1854 12838624 : bool tem;
1855 0 : valueize_refs_1 (orig, &tem);
1856 0 : }
1857 :
1858 : static vec<vn_reference_op_s> shared_lookup_references;
1859 :
1860 : /* Create a vector of vn_reference_op_s structures from REF, a
1861 : REFERENCE_CLASS_P tree. The vector is shared among all callers of
1862 : this function. *VALUEIZED_ANYTHING will specify whether any
1863 : operands were valueized. */
1864 :
1865 : static vec<vn_reference_op_s>
1866 183378293 : valueize_shared_reference_ops_from_ref (tree ref, bool *valueized_anything)
1867 : {
1868 183378293 : if (!ref)
1869 0 : return vNULL;
1870 183378293 : shared_lookup_references.truncate (0);
1871 183378293 : copy_reference_ops_from_ref (ref, &shared_lookup_references);
1872 183378293 : valueize_refs_1 (&shared_lookup_references, valueized_anything);
1873 183378293 : return shared_lookup_references;
1874 : }
1875 :
1876 : /* Create a vector of vn_reference_op_s structures from CALL, a
1877 : call statement. The vector is shared among all callers of
1878 : this function. */
1879 :
1880 : static vec<vn_reference_op_s>
1881 9346632 : valueize_shared_reference_ops_from_call (gcall *call)
1882 : {
1883 9346632 : if (!call)
1884 0 : return vNULL;
1885 9346632 : shared_lookup_references.truncate (0);
1886 9346632 : copy_reference_ops_from_call (call, &shared_lookup_references);
1887 9346632 : valueize_refs (&shared_lookup_references);
1888 9346632 : return shared_lookup_references;
1889 : }
1890 :
1891 : /* Lookup a SCCVN reference operation VR in the current hash table.
1892 : Returns the resulting value number if it exists in the hash table,
1893 : NULL_TREE otherwise. VNRESULT will be filled in with the actual
1894 : vn_reference_t stored in the hashtable if something is found. */
1895 :
1896 : static tree
1897 66379306 : vn_reference_lookup_1 (vn_reference_t vr, vn_reference_t *vnresult)
1898 : {
1899 66379306 : vn_reference_s **slot;
1900 66379306 : hashval_t hash;
1901 :
1902 66379306 : hash = vr->hashcode;
1903 66379306 : slot = valid_info->references->find_slot_with_hash (vr, hash, NO_INSERT);
1904 66379306 : if (slot)
1905 : {
1906 8301472 : if (vnresult)
1907 8301472 : *vnresult = (vn_reference_t)*slot;
1908 8301472 : return ((vn_reference_t)*slot)->result;
1909 : }
1910 :
1911 : return NULL_TREE;
1912 : }
1913 :
1914 :
1915 : /* Partial definition tracking support. */
1916 :
1917 : struct pd_range
1918 : {
1919 : HOST_WIDE_INT offset;
1920 : HOST_WIDE_INT size;
1921 : pd_range *m_children[2];
1922 : };
1923 :
1924 : struct pd_data
1925 : {
1926 : tree rhs;
1927 : HOST_WIDE_INT rhs_off;
1928 : HOST_WIDE_INT offset;
1929 : HOST_WIDE_INT size;
1930 : };
1931 :
1932 : /* Context for alias walking. */
1933 :
1934 : struct vn_walk_cb_data
1935 : {
1936 62194097 : vn_walk_cb_data (vn_reference_t vr_, tree orig_ref_, tree *last_vuse_ptr_,
1937 : vn_lookup_kind vn_walk_kind_, bool tbaa_p_, tree mask_,
1938 : bool redundant_store_removal_p_)
1939 62194097 : : vr (vr_), last_vuse_ptr (last_vuse_ptr_), last_vuse (NULL_TREE),
1940 62194097 : mask (mask_), masked_result (NULL_TREE), same_val (NULL_TREE),
1941 62194097 : vn_walk_kind (vn_walk_kind_),
1942 62194097 : tbaa_p (tbaa_p_), redundant_store_removal_p (redundant_store_removal_p_),
1943 124388194 : saved_operands (vNULL), first_range (), first_set (-2),
1944 124388194 : first_base_set (-2)
1945 : {
1946 62194097 : if (!last_vuse_ptr)
1947 28811867 : last_vuse_ptr = &last_vuse;
1948 62194097 : ao_ref_init (&orig_ref, orig_ref_);
1949 62194097 : if (mask)
1950 : {
1951 303292 : wide_int w = wi::to_wide (mask);
1952 303292 : unsigned int pos = 0, prec = w.get_precision ();
1953 303292 : pd_data pd;
1954 303292 : pd.rhs = build_constructor (NULL_TREE, NULL);
1955 303292 : pd.rhs_off = 0;
1956 : /* When bitwise and with a constant is done on a memory load,
1957 : we don't really need all the bits to be defined or defined
1958 : to constants, we don't really care what is in the position
1959 : corresponding to 0 bits in the mask.
1960 : So, push the ranges of those 0 bits in the mask as artificial
1961 : zero stores and let the partial def handling code do the
1962 : rest. */
1963 650571 : while (pos < prec)
1964 : {
1965 630238 : int tz = wi::ctz (w);
1966 630238 : if (pos + tz > prec)
1967 282959 : tz = prec - pos;
1968 630238 : if (tz)
1969 : {
1970 477762 : if (BYTES_BIG_ENDIAN)
1971 : pd.offset = prec - pos - tz;
1972 : else
1973 477762 : pd.offset = pos;
1974 477762 : pd.size = tz;
1975 477762 : void *r = push_partial_def (pd, 0, 0, 0, prec);
1976 477762 : gcc_assert (r == NULL_TREE);
1977 : }
1978 630238 : pos += tz;
1979 630238 : if (pos == prec)
1980 : break;
1981 347279 : w = wi::lrshift (w, tz);
1982 347279 : tz = wi::ctz (wi::bit_not (w));
1983 347279 : if (pos + tz > prec)
1984 0 : tz = prec - pos;
1985 347279 : pos += tz;
1986 347279 : w = wi::lrshift (w, tz);
1987 : }
1988 303292 : }
1989 62194097 : }
1990 : ~vn_walk_cb_data ();
1991 : void *finish (alias_set_type, alias_set_type, tree);
1992 : void *push_partial_def (pd_data pd,
1993 : alias_set_type, alias_set_type, HOST_WIDE_INT,
1994 : HOST_WIDE_INT);
1995 :
1996 : vn_reference_t vr;
1997 : ao_ref orig_ref;
1998 : tree *last_vuse_ptr;
1999 : tree last_vuse;
2000 : tree mask;
2001 : tree masked_result;
2002 : tree same_val;
2003 : vn_lookup_kind vn_walk_kind;
2004 : bool tbaa_p;
2005 : bool redundant_store_removal_p;
2006 : vec<vn_reference_op_s> saved_operands;
2007 :
2008 : /* The VDEFs of partial defs we come along. */
2009 : auto_vec<pd_data, 2> partial_defs;
2010 : /* The first defs range to avoid splay tree setup in most cases. */
2011 : pd_range first_range;
2012 : alias_set_type first_set;
2013 : alias_set_type first_base_set;
2014 : default_splay_tree<pd_range *> known_ranges;
2015 : obstack ranges_obstack;
2016 : static constexpr HOST_WIDE_INT bufsize = 64;
2017 : };
2018 :
2019 62194097 : vn_walk_cb_data::~vn_walk_cb_data ()
2020 : {
2021 62194097 : if (known_ranges)
2022 170058 : obstack_free (&ranges_obstack, NULL);
2023 62194097 : saved_operands.release ();
2024 62194097 : }
2025 :
2026 : void *
2027 1569024 : vn_walk_cb_data::finish (alias_set_type set, alias_set_type base_set, tree val)
2028 : {
2029 1569024 : if (first_set != -2)
2030 : {
2031 447426 : set = first_set;
2032 447426 : base_set = first_base_set;
2033 : }
2034 1569024 : if (mask)
2035 : {
2036 459 : masked_result = val;
2037 459 : return (void *) -1;
2038 : }
2039 1568565 : if (same_val && !operand_equal_p (val, same_val))
2040 : return (void *) -1;
2041 1564785 : vec<vn_reference_op_s> &operands
2042 1564785 : = saved_operands.exists () ? saved_operands : vr->operands;
2043 1564785 : return vn_reference_lookup_or_insert_for_pieces (last_vuse, set, base_set,
2044 : vr->offset, vr->max_size,
2045 1564785 : vr->type, operands, val);
2046 : }
2047 :
2048 : /* Push PD to the vector of partial definitions returning a
2049 : value when we are ready to combine things with VUSE, SET and MAXSIZEI,
2050 : NULL when we want to continue looking for partial defs or -1
2051 : on failure. */
2052 :
2053 : void *
2054 562479 : vn_walk_cb_data::push_partial_def (pd_data pd,
2055 : alias_set_type set, alias_set_type base_set,
2056 : HOST_WIDE_INT offseti,
2057 : HOST_WIDE_INT maxsizei)
2058 : {
2059 : /* We're using a fixed buffer for encoding so fail early if the object
2060 : we want to interpret is bigger. */
2061 562479 : if (maxsizei > bufsize * BITS_PER_UNIT
2062 : || CHAR_BIT != 8
2063 : || BITS_PER_UNIT != 8
2064 : /* Not prepared to handle PDP endian. */
2065 : || BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
2066 : return (void *)-1;
2067 :
2068 : /* Turn too large constant stores into non-constant stores. */
2069 562408 : if (CONSTANT_CLASS_P (pd.rhs) && pd.size > bufsize * BITS_PER_UNIT)
2070 0 : pd.rhs = error_mark_node;
2071 :
2072 : /* And for non-constant or CONSTRUCTOR stores shrink them to only keep at
2073 : most a partial byte before and/or after the region. */
2074 562408 : if (!CONSTANT_CLASS_P (pd.rhs))
2075 : {
2076 521479 : if (pd.offset < offseti)
2077 : {
2078 8613 : HOST_WIDE_INT o = ROUND_DOWN (offseti - pd.offset, BITS_PER_UNIT);
2079 8613 : gcc_assert (pd.size > o);
2080 8613 : pd.size -= o;
2081 8613 : pd.offset += o;
2082 : }
2083 521479 : if (pd.size > maxsizei)
2084 7718 : pd.size = maxsizei + ((pd.size - maxsizei) % BITS_PER_UNIT);
2085 : }
2086 :
2087 562408 : pd.offset -= offseti;
2088 :
2089 1124816 : bool pd_constant_p = (TREE_CODE (pd.rhs) == CONSTRUCTOR
2090 562408 : || CONSTANT_CLASS_P (pd.rhs));
2091 562408 : pd_range *r;
2092 562408 : if (partial_defs.is_empty ())
2093 : {
2094 : /* If we get a clobber upfront, fail. */
2095 360954 : if (TREE_CLOBBER_P (pd.rhs))
2096 : return (void *)-1;
2097 360599 : if (!pd_constant_p)
2098 : return (void *)-1;
2099 327945 : partial_defs.safe_push (pd);
2100 327945 : first_range.offset = pd.offset;
2101 327945 : first_range.size = pd.size;
2102 327945 : first_set = set;
2103 327945 : first_base_set = base_set;
2104 327945 : last_vuse_ptr = NULL;
2105 327945 : r = &first_range;
2106 : /* Go check if the first partial definition was a full one in case
2107 : the caller didn't optimize for this. */
2108 : }
2109 : else
2110 : {
2111 201454 : if (!known_ranges)
2112 : {
2113 : /* ??? Optimize the case where the 2nd partial def completes
2114 : things. */
2115 170058 : gcc_obstack_init (&ranges_obstack);
2116 170058 : known_ranges.insert_max_node (&first_range);
2117 : }
2118 : /* Lookup the offset and see if we need to merge. */
2119 201454 : int comparison = known_ranges.lookup_le
2120 407204 : ([&] (pd_range *r) { return pd.offset < r->offset; },
2121 180743 : [&] (pd_range *r) { return pd.offset > r->offset; });
2122 201454 : r = known_ranges.root ();
2123 201454 : if (comparison >= 0
2124 201454 : && ranges_known_overlap_p (r->offset, r->size + 1,
2125 : pd.offset, pd.size))
2126 : {
2127 : /* Ignore partial defs already covered. Here we also drop shadowed
2128 : clobbers arriving here at the floor. */
2129 5843 : if (known_subrange_p (pd.offset, pd.size, r->offset, r->size))
2130 : return NULL;
2131 5002 : r->size = MAX (r->offset + r->size, pd.offset + pd.size) - r->offset;
2132 : }
2133 : else
2134 : {
2135 : /* pd.offset wasn't covered yet, insert the range. */
2136 195611 : void *addr = XOBNEW (&ranges_obstack, pd_range);
2137 195611 : r = new (addr) pd_range { pd.offset, pd.size, {} };
2138 195611 : known_ranges.insert_relative (comparison, r);
2139 : }
2140 : /* Merge r which now contains pd's range and is a member of the splay
2141 : tree with adjacent overlapping ranges. */
2142 200613 : if (known_ranges.splay_next_node ())
2143 22816 : do
2144 : {
2145 22816 : pd_range *rafter = known_ranges.root ();
2146 22816 : if (!ranges_known_overlap_p (r->offset, r->size + 1,
2147 22816 : rafter->offset, rafter->size))
2148 : break;
2149 22546 : r->size = MAX (r->offset + r->size,
2150 22546 : rafter->offset + rafter->size) - r->offset;
2151 : }
2152 22546 : while (known_ranges.remove_root_and_splay_next ());
2153 : /* If we get a clobber, fail. */
2154 200613 : if (TREE_CLOBBER_P (pd.rhs))
2155 : return (void *)-1;
2156 : /* Non-constants are OK as long as they are shadowed by a constant. */
2157 198417 : if (!pd_constant_p)
2158 : return (void *)-1;
2159 191979 : partial_defs.safe_push (pd);
2160 : }
2161 :
2162 : /* Now we have merged pd's range into the range tree. When we have covered
2163 : [offseti, sizei] then the tree will contain exactly one node which has
2164 : the desired properties and it will be 'r'. */
2165 519924 : if (!known_subrange_p (0, maxsizei, r->offset, r->size))
2166 : /* Continue looking for partial defs. */
2167 : return NULL;
2168 :
2169 : /* Now simply native encode all partial defs in reverse order. */
2170 8822 : unsigned ndefs = partial_defs.length ();
2171 : /* We support up to 512-bit values (for V8DFmode). */
2172 8822 : unsigned char buffer[bufsize + 1];
2173 8822 : unsigned char this_buffer[bufsize + 1];
2174 8822 : int len;
2175 :
2176 8822 : memset (buffer, 0, bufsize + 1);
2177 8822 : unsigned needed_len = ROUND_UP (maxsizei, BITS_PER_UNIT) / BITS_PER_UNIT;
2178 43489 : while (!partial_defs.is_empty ())
2179 : {
2180 25845 : pd_data pd = partial_defs.pop ();
2181 25845 : unsigned int amnt;
2182 25845 : if (TREE_CODE (pd.rhs) == CONSTRUCTOR)
2183 : {
2184 : /* Empty CONSTRUCTOR. */
2185 2120 : if (pd.size >= needed_len * BITS_PER_UNIT)
2186 2120 : len = needed_len;
2187 : else
2188 1801 : len = ROUND_UP (pd.size, BITS_PER_UNIT) / BITS_PER_UNIT;
2189 2120 : memset (this_buffer, 0, len);
2190 : }
2191 23725 : else if (pd.rhs_off >= 0)
2192 : {
2193 47450 : len = native_encode_expr (pd.rhs, this_buffer, bufsize,
2194 23725 : (MAX (0, -pd.offset)
2195 23725 : + pd.rhs_off) / BITS_PER_UNIT);
2196 23725 : if (len <= 0
2197 23725 : || len < (ROUND_UP (pd.size, BITS_PER_UNIT) / BITS_PER_UNIT
2198 23725 : - MAX (0, -pd.offset) / BITS_PER_UNIT))
2199 : {
2200 0 : if (dump_file && (dump_flags & TDF_DETAILS))
2201 0 : fprintf (dump_file, "Failed to encode %u "
2202 : "partial definitions\n", ndefs);
2203 0 : return (void *)-1;
2204 : }
2205 : }
2206 : else /* negative pd.rhs_off indicates we want to chop off first bits */
2207 : {
2208 0 : if (-pd.rhs_off >= bufsize)
2209 : return (void *)-1;
2210 0 : len = native_encode_expr (pd.rhs,
2211 0 : this_buffer + -pd.rhs_off / BITS_PER_UNIT,
2212 0 : bufsize - -pd.rhs_off / BITS_PER_UNIT,
2213 0 : MAX (0, -pd.offset) / BITS_PER_UNIT);
2214 0 : if (len <= 0
2215 0 : || len < (ROUND_UP (pd.size, BITS_PER_UNIT) / BITS_PER_UNIT
2216 0 : - MAX (0, -pd.offset) / BITS_PER_UNIT))
2217 : {
2218 0 : if (dump_file && (dump_flags & TDF_DETAILS))
2219 0 : fprintf (dump_file, "Failed to encode %u "
2220 : "partial definitions\n", ndefs);
2221 0 : return (void *)-1;
2222 : }
2223 : }
2224 :
2225 25845 : unsigned char *p = buffer;
2226 25845 : HOST_WIDE_INT size = pd.size;
2227 25845 : if (pd.offset < 0)
2228 312 : size -= ROUND_DOWN (-pd.offset, BITS_PER_UNIT);
2229 25845 : this_buffer[len] = 0;
2230 25845 : if (BYTES_BIG_ENDIAN)
2231 : {
2232 : /* LSB of this_buffer[len - 1] byte should be at
2233 : pd.offset + pd.size - 1 bits in buffer. */
2234 : amnt = ((unsigned HOST_WIDE_INT) pd.offset
2235 : + pd.size) % BITS_PER_UNIT;
2236 : if (amnt)
2237 : shift_bytes_in_array_right (this_buffer, len + 1, amnt);
2238 : unsigned char *q = this_buffer;
2239 : unsigned int off = 0;
2240 : if (pd.offset >= 0)
2241 : {
2242 : unsigned int msk;
2243 : off = pd.offset / BITS_PER_UNIT;
2244 : gcc_assert (off < needed_len);
2245 : p = buffer + off;
2246 : if (size <= amnt)
2247 : {
2248 : msk = ((1 << size) - 1) << (BITS_PER_UNIT - amnt);
2249 : *p = (*p & ~msk) | (this_buffer[len] & msk);
2250 : size = 0;
2251 : }
2252 : else
2253 : {
2254 : if (TREE_CODE (pd.rhs) != CONSTRUCTOR)
2255 : q = (this_buffer + len
2256 : - (ROUND_UP (size - amnt, BITS_PER_UNIT)
2257 : / BITS_PER_UNIT));
2258 : if (pd.offset % BITS_PER_UNIT)
2259 : {
2260 : msk = -1U << (BITS_PER_UNIT
2261 : - (pd.offset % BITS_PER_UNIT));
2262 : *p = (*p & msk) | (*q & ~msk);
2263 : p++;
2264 : q++;
2265 : off++;
2266 : size -= BITS_PER_UNIT - (pd.offset % BITS_PER_UNIT);
2267 : gcc_assert (size >= 0);
2268 : }
2269 : }
2270 : }
2271 : else if (TREE_CODE (pd.rhs) != CONSTRUCTOR)
2272 : {
2273 : q = (this_buffer + len
2274 : - (ROUND_UP (size - amnt, BITS_PER_UNIT)
2275 : / BITS_PER_UNIT));
2276 : if (pd.offset % BITS_PER_UNIT)
2277 : {
2278 : q++;
2279 : size -= BITS_PER_UNIT - ((unsigned HOST_WIDE_INT) pd.offset
2280 : % BITS_PER_UNIT);
2281 : gcc_assert (size >= 0);
2282 : }
2283 : }
2284 : if ((unsigned HOST_WIDE_INT) size / BITS_PER_UNIT + off
2285 : > needed_len)
2286 : size = (needed_len - off) * BITS_PER_UNIT;
2287 : memcpy (p, q, size / BITS_PER_UNIT);
2288 : if (size % BITS_PER_UNIT)
2289 : {
2290 : unsigned int msk
2291 : = -1U << (BITS_PER_UNIT - (size % BITS_PER_UNIT));
2292 : p += size / BITS_PER_UNIT;
2293 : q += size / BITS_PER_UNIT;
2294 : *p = (*q & msk) | (*p & ~msk);
2295 : }
2296 : }
2297 : else
2298 : {
2299 25845 : if (pd.offset >= 0)
2300 : {
2301 : /* LSB of this_buffer[0] byte should be at pd.offset bits
2302 : in buffer. */
2303 25533 : unsigned int msk;
2304 25533 : size = MIN (size, (HOST_WIDE_INT) needed_len * BITS_PER_UNIT);
2305 25533 : amnt = pd.offset % BITS_PER_UNIT;
2306 25533 : if (amnt)
2307 1516 : shift_bytes_in_array_left (this_buffer, len + 1, amnt);
2308 25533 : unsigned int off = pd.offset / BITS_PER_UNIT;
2309 25533 : gcc_assert (off < needed_len);
2310 25533 : size = MIN (size,
2311 : (HOST_WIDE_INT) (needed_len - off) * BITS_PER_UNIT);
2312 25533 : p = buffer + off;
2313 25533 : if (amnt + size < BITS_PER_UNIT)
2314 : {
2315 : /* Low amnt bits come from *p, then size bits
2316 : from this_buffer[0] and the remaining again from
2317 : *p. */
2318 1088 : msk = ((1 << size) - 1) << amnt;
2319 1088 : *p = (*p & ~msk) | (this_buffer[0] & msk);
2320 1088 : size = 0;
2321 : }
2322 24445 : else if (amnt)
2323 : {
2324 1140 : msk = -1U << amnt;
2325 1140 : *p = (*p & ~msk) | (this_buffer[0] & msk);
2326 1140 : p++;
2327 1140 : size -= (BITS_PER_UNIT - amnt);
2328 : }
2329 : }
2330 : else
2331 : {
2332 312 : amnt = (unsigned HOST_WIDE_INT) pd.offset % BITS_PER_UNIT;
2333 312 : if (amnt)
2334 16 : size -= BITS_PER_UNIT - amnt;
2335 312 : size = MIN (size, (HOST_WIDE_INT) needed_len * BITS_PER_UNIT);
2336 312 : if (amnt)
2337 16 : shift_bytes_in_array_left (this_buffer, len + 1, amnt);
2338 : }
2339 25845 : memcpy (p, this_buffer + (amnt != 0), size / BITS_PER_UNIT);
2340 25845 : p += size / BITS_PER_UNIT;
2341 25845 : if (size % BITS_PER_UNIT)
2342 : {
2343 626 : unsigned int msk = -1U << (size % BITS_PER_UNIT);
2344 626 : *p = (this_buffer[(amnt != 0) + size / BITS_PER_UNIT]
2345 626 : & ~msk) | (*p & msk);
2346 : }
2347 : }
2348 : }
2349 :
2350 8822 : tree type = vr->type;
2351 : /* Make sure to interpret in a type that has a range covering the whole
2352 : access size. */
2353 8822 : if (INTEGRAL_TYPE_P (vr->type) && maxsizei != TYPE_PRECISION (vr->type))
2354 : {
2355 0 : if (BITINT_TYPE_P (vr->type)
2356 26 : && maxsizei > MAX_FIXED_MODE_SIZE)
2357 13 : type = build_bitint_type (maxsizei, TYPE_UNSIGNED (type));
2358 : else
2359 0 : type = build_nonstandard_integer_type (maxsizei, TYPE_UNSIGNED (type));
2360 : }
2361 8822 : tree val;
2362 8822 : if (BYTES_BIG_ENDIAN)
2363 : {
2364 : unsigned sz = needed_len;
2365 : if (maxsizei % BITS_PER_UNIT)
2366 : shift_bytes_in_array_right (buffer, needed_len,
2367 : BITS_PER_UNIT
2368 : - (maxsizei % BITS_PER_UNIT));
2369 : if (INTEGRAL_TYPE_P (type))
2370 : {
2371 : if (TYPE_MODE (type) != BLKmode)
2372 : sz = GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (type));
2373 : else
2374 : sz = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (type));
2375 : }
2376 : if (sz > needed_len)
2377 : {
2378 : memcpy (this_buffer + (sz - needed_len), buffer, needed_len);
2379 : val = native_interpret_expr (type, this_buffer, sz);
2380 : }
2381 : else
2382 : val = native_interpret_expr (type, buffer, needed_len);
2383 : }
2384 : else
2385 8822 : val = native_interpret_expr (type, buffer, bufsize);
2386 : /* If we chop off bits because the types precision doesn't match the memory
2387 : access size this is ok when optimizing reads but not when called from
2388 : the DSE code during elimination. */
2389 8822 : if (val && type != vr->type)
2390 : {
2391 13 : if (! int_fits_type_p (val, vr->type))
2392 : val = NULL_TREE;
2393 : else
2394 13 : val = fold_convert (vr->type, val);
2395 : }
2396 :
2397 8818 : if (val)
2398 : {
2399 8818 : if (dump_file && (dump_flags & TDF_DETAILS))
2400 0 : fprintf (dump_file,
2401 : "Successfully combined %u partial definitions\n", ndefs);
2402 : /* We are using the alias-set of the first store we encounter which
2403 : should be appropriate here. */
2404 8818 : return finish (first_set, first_base_set, val);
2405 : }
2406 : else
2407 : {
2408 4 : if (dump_file && (dump_flags & TDF_DETAILS))
2409 0 : fprintf (dump_file,
2410 : "Failed to interpret %u encoded partial definitions\n", ndefs);
2411 4 : return (void *)-1;
2412 : }
2413 : }
2414 :
2415 : /* Callback for walk_non_aliased_vuses. Adjusts the vn_reference_t VR_
2416 : with the current VUSE and performs the expression lookup. */
2417 :
2418 : static void *
2419 1096163876 : vn_reference_lookup_2 (ao_ref *op, tree vuse, void *data_)
2420 : {
2421 1096163876 : vn_walk_cb_data *data = (vn_walk_cb_data *)data_;
2422 1096163876 : vn_reference_t vr = data->vr;
2423 1096163876 : vn_reference_s **slot;
2424 1096163876 : hashval_t hash;
2425 :
2426 : /* If we have partial definitions recorded we have to go through
2427 : vn_reference_lookup_3. */
2428 2184343941 : if (!data->partial_defs.is_empty ())
2429 : return NULL;
2430 :
2431 1095376184 : if (data->last_vuse_ptr)
2432 : {
2433 1074076533 : *data->last_vuse_ptr = vuse;
2434 1074076533 : data->last_vuse = vuse;
2435 : }
2436 :
2437 : /* Fixup vuse and hash. */
2438 1095376184 : if (vr->vuse)
2439 1095376184 : vr->hashcode = vr->hashcode - SSA_NAME_VERSION (vr->vuse);
2440 1095376184 : vr->vuse = vuse_ssa_val (vuse);
2441 1095376184 : if (vr->vuse)
2442 1095376184 : vr->hashcode = vr->hashcode + SSA_NAME_VERSION (vr->vuse);
2443 :
2444 1095376184 : hash = vr->hashcode;
2445 1095376184 : slot = valid_info->references->find_slot_with_hash (vr, hash, NO_INSERT);
2446 1095376184 : if (slot)
2447 : {
2448 7982635 : if ((*slot)->result && data->saved_operands.exists ())
2449 432774 : return data->finish (vr->set, vr->base_set, (*slot)->result);
2450 : return *slot;
2451 : }
2452 :
2453 1087393549 : if (SSA_NAME_IS_DEFAULT_DEF (vuse))
2454 : {
2455 18405989 : HOST_WIDE_INT op_offset, op_size;
2456 18405989 : tree v = NULL_TREE;
2457 18405989 : tree base = ao_ref_base (op);
2458 :
2459 18405989 : if (base
2460 18405989 : && op->offset.is_constant (&op_offset)
2461 18405989 : && op->size.is_constant (&op_size)
2462 18405989 : && op->max_size_known_p ()
2463 36357647 : && known_eq (op->size, op->max_size))
2464 : {
2465 17649698 : if (TREE_CODE (base) == PARM_DECL)
2466 678100 : v = ipcp_get_aggregate_const (cfun, base, false, op_offset,
2467 : op_size);
2468 16971598 : else if (TREE_CODE (base) == MEM_REF
2469 7060242 : && integer_zerop (TREE_OPERAND (base, 1))
2470 5679085 : && TREE_CODE (TREE_OPERAND (base, 0)) == SSA_NAME
2471 5673880 : && SSA_NAME_IS_DEFAULT_DEF (TREE_OPERAND (base, 0))
2472 20753319 : && (TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (base, 0)))
2473 : == PARM_DECL))
2474 3727831 : v = ipcp_get_aggregate_const (cfun,
2475 3727831 : SSA_NAME_VAR (TREE_OPERAND (base, 0)),
2476 : true, op_offset, op_size);
2477 : }
2478 4405931 : if (v)
2479 1176 : return data->finish (vr->set, vr->base_set, v);
2480 : }
2481 :
2482 : return NULL;
2483 : }
2484 :
2485 : /* Lookup an existing or insert a new vn_reference entry into the
2486 : value table for the VUSE, SET, TYPE, OPERANDS reference which
2487 : has the value VALUE which is either a constant or an SSA name. */
2488 :
2489 : static vn_reference_t
2490 1564785 : vn_reference_lookup_or_insert_for_pieces (tree vuse,
2491 : alias_set_type set,
2492 : alias_set_type base_set,
2493 : poly_int64 offset,
2494 : poly_int64 max_size,
2495 : tree type,
2496 : vec<vn_reference_op_s,
2497 : va_heap> operands,
2498 : tree value)
2499 : {
2500 1564785 : vn_reference_s vr1;
2501 1564785 : vn_reference_t result;
2502 1564785 : unsigned value_id;
2503 1564785 : vr1.vuse = vuse ? SSA_VAL (vuse) : NULL_TREE;
2504 1564785 : vr1.operands = operands;
2505 1564785 : vr1.type = type;
2506 1564785 : vr1.set = set;
2507 1564785 : vr1.base_set = base_set;
2508 1564785 : vr1.offset = offset;
2509 1564785 : vr1.max_size = max_size;
2510 1564785 : vr1.hashcode = vn_reference_compute_hash (&vr1);
2511 1564785 : if (vn_reference_lookup_1 (&vr1, &result))
2512 8287 : return result;
2513 :
2514 1556498 : if (TREE_CODE (value) == SSA_NAME)
2515 361391 : value_id = VN_INFO (value)->value_id;
2516 : else
2517 1195107 : value_id = get_or_alloc_constant_value_id (value);
2518 1556498 : return vn_reference_insert_pieces (vuse, set, base_set, offset, max_size,
2519 1556498 : type, operands.copy (), value, value_id);
2520 : }
2521 :
2522 : /* Return a value-number for RCODE OPS... either by looking up an existing
2523 : value-number for the possibly simplified result or by inserting the
2524 : operation if INSERT is true. If SIMPLIFY is false, return a value
2525 : number for the unsimplified expression. */
2526 :
2527 : static tree
2528 18878168 : vn_nary_build_or_lookup_1 (gimple_match_op *res_op, bool insert,
2529 : bool simplify)
2530 : {
2531 18878168 : tree result = NULL_TREE;
2532 : /* We will be creating a value number for
2533 : RCODE (OPS...).
2534 : So first simplify and lookup this expression to see if it
2535 : is already available. */
2536 : /* For simplification valueize. */
2537 18878168 : unsigned i = 0;
2538 18878168 : if (simplify)
2539 43809323 : for (i = 0; i < res_op->num_ops; ++i)
2540 24936829 : if (TREE_CODE (res_op->ops[i]) == SSA_NAME)
2541 : {
2542 16001284 : tree tem = vn_valueize (res_op->ops[i]);
2543 16001284 : if (!tem)
2544 : break;
2545 16001284 : res_op->ops[i] = tem;
2546 : }
2547 : /* If valueization of an operand fails (it is not available), skip
2548 : simplification. */
2549 18878168 : bool res = false;
2550 18878168 : if (i == res_op->num_ops)
2551 : {
2552 : /* Do not leak not available operands into the simplified expression
2553 : when called from PRE context. */
2554 18872494 : if (rpo_avail)
2555 11326434 : mprts_hook = vn_lookup_simplify_result;
2556 18872494 : res = res_op->resimplify (NULL, vn_valueize);
2557 18872494 : mprts_hook = NULL;
2558 : }
2559 32512334 : gimple *new_stmt = NULL;
2560 18872494 : if (res
2561 18872494 : && gimple_simplified_result_is_gimple_val (res_op))
2562 : {
2563 : /* The expression is already available. */
2564 5238328 : result = res_op->ops[0];
2565 : /* Valueize it, simplification returns sth in AVAIL only. */
2566 5238328 : if (TREE_CODE (result) == SSA_NAME)
2567 293424 : result = SSA_VAL (result);
2568 : }
2569 : else
2570 : {
2571 13639840 : tree val = vn_lookup_simplify_result (res_op);
2572 : /* ??? In weird cases we can end up with internal-fn calls,
2573 : but this isn't expected so throw the result away. See
2574 : PR123040 for an example. */
2575 13639840 : if (!val && insert && res_op->code.is_tree_code ())
2576 : {
2577 136844 : gimple_seq stmts = NULL;
2578 136844 : result = maybe_push_res_to_seq (res_op, &stmts);
2579 136844 : if (result)
2580 : {
2581 136838 : gcc_assert (gimple_seq_singleton_p (stmts));
2582 136838 : new_stmt = gimple_seq_first_stmt (stmts);
2583 : }
2584 : }
2585 : else
2586 : /* The expression is already available. */
2587 : result = val;
2588 : }
2589 293430 : if (new_stmt)
2590 : {
2591 : /* The expression is not yet available, value-number lhs to
2592 : the new SSA_NAME we created. */
2593 : /* Initialize value-number information properly. */
2594 136838 : vn_ssa_aux_t result_info = VN_INFO (result);
2595 136838 : result_info->valnum = result;
2596 136838 : result_info->value_id = get_next_value_id ();
2597 136838 : result_info->visited = 1;
2598 136838 : gimple_seq_add_stmt_without_update (&VN_INFO (result)->expr,
2599 : new_stmt);
2600 136838 : result_info->needs_insertion = true;
2601 : /* ??? PRE phi-translation inserts NARYs without corresponding
2602 : SSA name result. Re-use those but set their result according
2603 : to the stmt we just built. */
2604 136838 : vn_nary_op_t nary = NULL;
2605 136838 : vn_nary_op_lookup_stmt (new_stmt, &nary);
2606 136838 : if (nary)
2607 : {
2608 0 : gcc_assert (! nary->predicated_values && nary->u.result == NULL_TREE);
2609 0 : nary->u.result = gimple_assign_lhs (new_stmt);
2610 : }
2611 : /* As all "inserted" statements are singleton SCCs, insert
2612 : to the valid table. This is strictly needed to
2613 : avoid re-generating new value SSA_NAMEs for the same
2614 : expression during SCC iteration over and over (the
2615 : optimistic table gets cleared after each iteration).
2616 : We do not need to insert into the optimistic table, as
2617 : lookups there will fall back to the valid table. */
2618 : else
2619 : {
2620 136838 : unsigned int length = vn_nary_length_from_stmt (new_stmt);
2621 136838 : vn_nary_op_t vno1
2622 136838 : = alloc_vn_nary_op_noinit (length, &vn_tables_insert_obstack);
2623 136838 : vno1->value_id = result_info->value_id;
2624 136838 : vno1->length = length;
2625 136838 : vno1->predicated_values = 0;
2626 136838 : vno1->u.result = result;
2627 136838 : init_vn_nary_op_from_stmt (vno1, as_a <gassign *> (new_stmt));
2628 136838 : vn_nary_op_insert_into (vno1, valid_info->nary);
2629 : /* Also do not link it into the undo chain. */
2630 136838 : last_inserted_nary = vno1->next;
2631 136838 : vno1->next = (vn_nary_op_t)(void *)-1;
2632 : }
2633 136838 : if (dump_file && (dump_flags & TDF_DETAILS))
2634 : {
2635 595 : fprintf (dump_file, "Inserting name ");
2636 595 : print_generic_expr (dump_file, result);
2637 595 : fprintf (dump_file, " for expression ");
2638 595 : print_gimple_expr (dump_file, new_stmt, 0, TDF_SLIM);
2639 595 : fprintf (dump_file, "\n");
2640 : }
2641 : }
2642 18878168 : return result;
2643 : }
2644 :
2645 : /* Return a value-number for RCODE OPS... either by looking up an existing
2646 : value-number for the simplified result or by inserting the operation. */
2647 :
2648 : static tree
2649 183047 : vn_nary_build_or_lookup (gimple_match_op *res_op)
2650 : {
2651 0 : return vn_nary_build_or_lookup_1 (res_op, true, true);
2652 : }
2653 :
2654 : /* Try to simplify the expression RCODE OPS... of type TYPE and return
2655 : its value if present. Update NARY with a simplified expression if
2656 : it fits. */
2657 :
2658 : tree
2659 7543053 : vn_nary_simplify (vn_nary_op_t nary)
2660 : {
2661 7543053 : if (nary->length > gimple_match_op::MAX_NUM_OPS
2662 : /* For CONSTRUCTOR the vn_nary_op_t and gimple_match_op representation
2663 : does not match. */
2664 7542507 : || nary->opcode == CONSTRUCTOR)
2665 : return NULL_TREE;
2666 7539806 : gimple_match_op op (gimple_match_cond::UNCOND, nary->opcode,
2667 7539806 : nary->type, nary->length);
2668 7539806 : memcpy (op.ops, nary->op, sizeof (tree) * nary->length);
2669 7539806 : tree res = vn_nary_build_or_lookup_1 (&op, false, true);
2670 : /* Do not update *NARY with a simplified result that contains abnormals.
2671 : This matches what maybe_push_res_to_seq does when requesting insertion. */
2672 19791638 : for (unsigned i = 0; i < op.num_ops; ++i)
2673 12251913 : if (TREE_CODE (op.ops[i]) == SSA_NAME
2674 12251913 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op.ops[i]))
2675 : return res;
2676 7539725 : if (op.code.is_tree_code ()
2677 7539725 : && op.num_ops <= nary->length
2678 15078637 : && (tree_code) op.code != CONSTRUCTOR)
2679 : {
2680 7538911 : nary->opcode = (tree_code) op.code;
2681 7538911 : nary->length = op.num_ops;
2682 19789080 : for (unsigned i = 0; i < op.num_ops; ++i)
2683 12250169 : nary->op[i] = op.ops[i];
2684 : }
2685 : return res;
2686 : }
2687 :
2688 : /* Elimination engine. */
2689 :
2690 : class eliminate_dom_walker : public dom_walker
2691 : {
2692 : public:
2693 : eliminate_dom_walker (cdi_direction, bitmap);
2694 : ~eliminate_dom_walker ();
2695 :
2696 : edge before_dom_children (basic_block) final override;
2697 : void after_dom_children (basic_block) final override;
2698 :
2699 : virtual tree eliminate_avail (basic_block, tree op);
2700 : virtual void eliminate_push_avail (basic_block, tree op);
2701 : tree eliminate_insert (basic_block, gimple_stmt_iterator *gsi, tree val);
2702 :
2703 : void eliminate_stmt (basic_block, gimple_stmt_iterator *);
2704 :
2705 : unsigned eliminate_cleanup (bool region_p = false);
2706 :
2707 : bool do_pre;
2708 : unsigned int el_todo;
2709 : unsigned int eliminations;
2710 : unsigned int insertions;
2711 :
2712 : /* SSA names that had their defs inserted by PRE if do_pre. */
2713 : bitmap inserted_exprs;
2714 :
2715 : /* Blocks with statements that have had their EH properties changed. */
2716 : bitmap need_eh_cleanup;
2717 :
2718 : /* Blocks with statements that have had their AB properties changed. */
2719 : bitmap need_ab_cleanup;
2720 :
2721 : /* Local state for the eliminate domwalk. */
2722 : auto_vec<gimple *> to_remove;
2723 : auto_vec<gimple *> to_fixup;
2724 : auto_vec<tree> avail;
2725 : auto_vec<tree> avail_stack;
2726 : };
2727 :
2728 : /* Adaptor to the elimination engine using RPO availability. */
2729 :
2730 12564182 : class rpo_elim : public eliminate_dom_walker
2731 : {
2732 : public:
2733 6282091 : rpo_elim(basic_block entry_)
2734 12564182 : : eliminate_dom_walker (CDI_DOMINATORS, NULL), entry (entry_),
2735 12564182 : m_avail_freelist (NULL) {}
2736 :
2737 : tree eliminate_avail (basic_block, tree op) final override;
2738 :
2739 : void eliminate_push_avail (basic_block, tree) final override;
2740 :
2741 : basic_block entry;
2742 : /* Freelist of avail entries which are allocated from the vn_ssa_aux
2743 : obstack. */
2744 : vn_avail *m_avail_freelist;
2745 : };
2746 :
2747 : /* Return true if BASE1 and BASE2 can be adjusted so they have the
2748 : same address and adjust *OFFSET1 and *OFFSET2 accordingly.
2749 : Otherwise return false. */
2750 :
2751 : static bool
2752 6906664 : adjust_offsets_for_equal_base_address (tree base1, poly_int64 *offset1,
2753 : tree base2, poly_int64 *offset2)
2754 : {
2755 6906664 : poly_int64 soff;
2756 6906664 : if (TREE_CODE (base1) == MEM_REF
2757 3150182 : && TREE_CODE (base2) == MEM_REF)
2758 : {
2759 2527464 : if (mem_ref_offset (base1).to_shwi (&soff))
2760 : {
2761 2527464 : base1 = TREE_OPERAND (base1, 0);
2762 2527464 : *offset1 += soff * BITS_PER_UNIT;
2763 : }
2764 2527464 : if (mem_ref_offset (base2).to_shwi (&soff))
2765 : {
2766 2527464 : base2 = TREE_OPERAND (base2, 0);
2767 2527464 : *offset2 += soff * BITS_PER_UNIT;
2768 : }
2769 2527464 : return operand_equal_p (base1, base2, 0);
2770 : }
2771 4379200 : return operand_equal_p (base1, base2, OEP_ADDRESS_OF);
2772 : }
2773 :
2774 : /* Callback for walk_non_aliased_vuses. Tries to perform a lookup
2775 : from the statement defining VUSE and if not successful tries to
2776 : translate *REFP and VR_ through an aggregate copy at the definition
2777 : of VUSE. If *DISAMBIGUATE_ONLY is true then do not perform translation
2778 : of *REF and *VR. If only disambiguation was performed then
2779 : *DISAMBIGUATE_ONLY is set to true. */
2780 :
2781 : static void *
2782 43124492 : vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
2783 : translate_flags *disambiguate_only)
2784 : {
2785 43124492 : vn_walk_cb_data *data = (vn_walk_cb_data *)data_;
2786 43124492 : vn_reference_t vr = data->vr;
2787 43124492 : gimple *def_stmt = SSA_NAME_DEF_STMT (vuse);
2788 43124492 : tree base = ao_ref_base (ref);
2789 43124492 : HOST_WIDE_INT offseti = 0, maxsizei, sizei = 0;
2790 43124492 : static vec<vn_reference_op_s> lhs_ops;
2791 43124492 : ao_ref lhs_ref;
2792 43124492 : bool lhs_ref_ok = false;
2793 43124492 : poly_int64 copy_size;
2794 :
2795 : /* First try to disambiguate after value-replacing in the definitions LHS. */
2796 43124492 : if (is_gimple_assign (def_stmt))
2797 : {
2798 21142375 : tree lhs = gimple_assign_lhs (def_stmt);
2799 21142375 : bool valueized_anything = false;
2800 : /* Avoid re-allocation overhead. */
2801 21142375 : lhs_ops.truncate (0);
2802 21142375 : basic_block saved_rpo_bb = vn_context_bb;
2803 21142375 : vn_context_bb = gimple_bb (def_stmt);
2804 21142375 : if (*disambiguate_only <= TR_VALUEIZE_AND_DISAMBIGUATE)
2805 : {
2806 13778895 : copy_reference_ops_from_ref (lhs, &lhs_ops);
2807 13778895 : valueize_refs_1 (&lhs_ops, &valueized_anything, true);
2808 : }
2809 21142375 : vn_context_bb = saved_rpo_bb;
2810 21142375 : ao_ref_init (&lhs_ref, lhs);
2811 21142375 : lhs_ref_ok = true;
2812 21142375 : if (valueized_anything
2813 2022436 : && ao_ref_init_from_vn_reference
2814 2022436 : (&lhs_ref, ao_ref_alias_set (&lhs_ref),
2815 2022436 : ao_ref_base_alias_set (&lhs_ref), TREE_TYPE (lhs), lhs_ops)
2816 23164811 : && !refs_may_alias_p_1 (ref, &lhs_ref, data->tbaa_p))
2817 : {
2818 1728920 : *disambiguate_only = TR_VALUEIZE_AND_DISAMBIGUATE;
2819 8391410 : return NULL;
2820 : }
2821 :
2822 : /* When the def is a CLOBBER we can optimistically disambiguate
2823 : against it since any overlap it would be undefined behavior.
2824 : Avoid this for obvious must aliases to save compile-time though.
2825 : We also may not do this when the query is used for redundant
2826 : store removal. */
2827 19413455 : if (!data->redundant_store_removal_p
2828 10699412 : && gimple_clobber_p (def_stmt)
2829 19930263 : && !operand_equal_p (ao_ref_base (&lhs_ref), base, OEP_ADDRESS_OF))
2830 : {
2831 490684 : *disambiguate_only = TR_DISAMBIGUATE;
2832 490684 : return NULL;
2833 : }
2834 :
2835 : /* Besides valueizing the LHS we can also use access-path based
2836 : disambiguation on the original non-valueized ref. */
2837 18922771 : if (!ref->ref
2838 : && lhs_ref_ok
2839 2717609 : && data->orig_ref.ref)
2840 : {
2841 : /* We want to use the non-valueized LHS for this, but avoid redundant
2842 : work. */
2843 1895667 : ao_ref *lref = &lhs_ref;
2844 1895667 : ao_ref lref_alt;
2845 1895667 : if (valueized_anything)
2846 : {
2847 114648 : ao_ref_init (&lref_alt, lhs);
2848 114648 : lref = &lref_alt;
2849 : }
2850 1895667 : if (!refs_may_alias_p_1 (&data->orig_ref, lref, data->tbaa_p))
2851 : {
2852 313418 : *disambiguate_only = (valueized_anything
2853 156709 : ? TR_VALUEIZE_AND_DISAMBIGUATE
2854 : : TR_DISAMBIGUATE);
2855 156709 : return NULL;
2856 : }
2857 : }
2858 :
2859 : /* If we reach a clobbering statement try to skip it and see if
2860 : we find a VN result with exactly the same value as the
2861 : possible clobber. In this case we can ignore the clobber
2862 : and return the found value. */
2863 18766062 : if (!gimple_has_volatile_ops (def_stmt)
2864 17370097 : && ((is_gimple_reg_type (TREE_TYPE (lhs))
2865 12772329 : && types_compatible_p (TREE_TYPE (lhs), vr->type)
2866 9933372 : && !storage_order_barrier_p (lhs)
2867 9933372 : && !reverse_storage_order_for_component_p (lhs))
2868 7436729 : || TREE_CODE (gimple_assign_rhs1 (def_stmt)) == CONSTRUCTOR)
2869 10999477 : && (ref->ref || data->orig_ref.ref)
2870 10527491 : && !data->mask
2871 10505419 : && data->partial_defs.is_empty ()
2872 10503137 : && multiple_p (get_object_alignment
2873 : (ref->ref ? ref->ref : data->orig_ref.ref),
2874 : ref->size)
2875 41960090 : && multiple_p (get_object_alignment (lhs), ref->size))
2876 : {
2877 10110108 : HOST_WIDE_INT offset2i, size2i;
2878 10110108 : poly_int64 offset = ref->offset;
2879 10110108 : poly_int64 maxsize = ref->max_size;
2880 :
2881 10110108 : gcc_assert (lhs_ref_ok);
2882 10110108 : tree base2 = ao_ref_base (&lhs_ref);
2883 10110108 : poly_int64 offset2 = lhs_ref.offset;
2884 10110108 : poly_int64 size2 = lhs_ref.size;
2885 10110108 : poly_int64 maxsize2 = lhs_ref.max_size;
2886 :
2887 10110108 : tree rhs = gimple_assign_rhs1 (def_stmt);
2888 10110108 : if (TREE_CODE (rhs) == CONSTRUCTOR)
2889 1036651 : rhs = integer_zero_node;
2890 : /* ??? We may not compare to ahead values which might be from
2891 : a different loop iteration but only to loop invariants. Use
2892 : CONSTANT_CLASS_P (unvalueized!) as conservative approximation.
2893 : The one-hop lookup below doesn't have this issue since there's
2894 : a virtual PHI before we ever reach a backedge to cross.
2895 : We can skip multiple defs as long as they are from the same
2896 : value though. */
2897 10110108 : if (data->same_val
2898 10110108 : && !operand_equal_p (data->same_val, rhs))
2899 : ;
2900 : /* When this is a (partial) must-def, leave it to handling
2901 : below in case we are interested in the value. */
2902 9816475 : else if (!(*disambiguate_only > TR_TRANSLATE)
2903 3356670 : && base2
2904 3356670 : && known_eq (maxsize2, size2)
2905 2361623 : && adjust_offsets_for_equal_base_address (base, &offset,
2906 : base2, &offset2)
2907 1151915 : && offset2.is_constant (&offset2i)
2908 1151915 : && size2.is_constant (&size2i)
2909 1151915 : && maxsize.is_constant (&maxsizei)
2910 1151915 : && offset.is_constant (&offseti)
2911 10968390 : && ranges_known_overlap_p (offseti, maxsizei, offset2i,
2912 : size2i))
2913 : ;
2914 8760108 : else if (CONSTANT_CLASS_P (rhs))
2915 : {
2916 4220826 : if (dump_file && (dump_flags & TDF_DETAILS))
2917 : {
2918 2194 : fprintf (dump_file,
2919 : "Skipping possible redundant definition ");
2920 2194 : print_gimple_stmt (dump_file, def_stmt, 0);
2921 : }
2922 : /* Delay the actual compare of the values to the end of the walk
2923 : but do not update last_vuse from here. */
2924 4220826 : data->last_vuse_ptr = NULL;
2925 4220826 : data->same_val = rhs;
2926 4286177 : return NULL;
2927 : }
2928 : else
2929 : {
2930 4539282 : tree saved_vuse = vr->vuse;
2931 4539282 : hashval_t saved_hashcode = vr->hashcode;
2932 4539282 : if (vr->vuse)
2933 4539282 : vr->hashcode = vr->hashcode - SSA_NAME_VERSION (vr->vuse);
2934 9078564 : vr->vuse = vuse_ssa_val (gimple_vuse (def_stmt));
2935 4539282 : if (vr->vuse)
2936 4539282 : vr->hashcode = vr->hashcode + SSA_NAME_VERSION (vr->vuse);
2937 4539282 : vn_reference_t vnresult = NULL;
2938 : /* Do not use vn_reference_lookup_2 since that might perform
2939 : expression hashtable insertion but this lookup crosses
2940 : a possible may-alias making such insertion conditionally
2941 : invalid. */
2942 4539282 : vn_reference_lookup_1 (vr, &vnresult);
2943 : /* Need to restore vr->vuse and vr->hashcode. */
2944 4539282 : vr->vuse = saved_vuse;
2945 4539282 : vr->hashcode = saved_hashcode;
2946 4539282 : if (vnresult)
2947 : {
2948 250386 : if (TREE_CODE (rhs) == SSA_NAME)
2949 248865 : rhs = SSA_VAL (rhs);
2950 250386 : if (vnresult->result
2951 250386 : && operand_equal_p (vnresult->result, rhs, 0))
2952 65351 : return vnresult;
2953 : }
2954 : }
2955 : }
2956 : }
2957 21982117 : else if (*disambiguate_only <= TR_VALUEIZE_AND_DISAMBIGUATE
2958 19752697 : && gimple_call_builtin_p (def_stmt, BUILT_IN_NORMAL)
2959 24081979 : && gimple_call_num_args (def_stmt) <= 4)
2960 : {
2961 : /* For builtin calls valueize its arguments and call the
2962 : alias oracle again. Valueization may improve points-to
2963 : info of pointers and constify size and position arguments.
2964 : Originally this was motivated by PR61034 which has
2965 : conditional calls to free falsely clobbering ref because
2966 : of imprecise points-to info of the argument. */
2967 : tree oldargs[4];
2968 : bool valueized_anything = false;
2969 4962779 : for (unsigned i = 0; i < gimple_call_num_args (def_stmt); ++i)
2970 : {
2971 3421982 : oldargs[i] = gimple_call_arg (def_stmt, i);
2972 3421982 : tree val = vn_valueize (oldargs[i]);
2973 3421982 : if (val != oldargs[i])
2974 : {
2975 127233 : gimple_call_set_arg (def_stmt, i, val);
2976 127233 : valueized_anything = true;
2977 : }
2978 : }
2979 1540797 : if (valueized_anything)
2980 : {
2981 197940 : bool res = call_may_clobber_ref_p_1 (as_a <gcall *> (def_stmt),
2982 98970 : ref, data->tbaa_p);
2983 361224 : for (unsigned i = 0; i < gimple_call_num_args (def_stmt); ++i)
2984 262254 : gimple_call_set_arg (def_stmt, i, oldargs[i]);
2985 98970 : if (!res)
2986 : {
2987 31581 : *disambiguate_only = TR_VALUEIZE_AND_DISAMBIGUATE;
2988 31581 : return NULL;
2989 : }
2990 : }
2991 : }
2992 :
2993 36430421 : if (*disambiguate_only > TR_TRANSLATE)
2994 : return (void *)-1;
2995 :
2996 : /* If we cannot constrain the size of the reference we cannot
2997 : test if anything kills it. */
2998 24256382 : if (!ref->max_size_known_p ())
2999 : return (void *)-1;
3000 :
3001 23831329 : poly_int64 offset = ref->offset;
3002 23831329 : poly_int64 maxsize = ref->max_size;
3003 :
3004 : /* def_stmt may-defs *ref. See if we can derive a value for *ref
3005 : from that definition.
3006 : 1) Memset. */
3007 23831329 : if (is_gimple_reg_type (vr->type)
3008 23825516 : && (gimple_call_builtin_p (def_stmt, BUILT_IN_MEMSET)
3009 23735134 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMSET_CHK))
3010 90924 : && (integer_zerop (gimple_call_arg (def_stmt, 1))
3011 32928 : || ((TREE_CODE (gimple_call_arg (def_stmt, 1)) == INTEGER_CST
3012 9097 : || (INTEGRAL_TYPE_P (vr->type) && known_eq (ref->size, 8)))
3013 : && CHAR_BIT == 8
3014 : && BITS_PER_UNIT == 8
3015 : && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
3016 31645 : && offset.is_constant (&offseti)
3017 31645 : && ref->size.is_constant (&sizei)
3018 31645 : && (offseti % BITS_PER_UNIT == 0
3019 39 : || TREE_CODE (gimple_call_arg (def_stmt, 1)) == INTEGER_CST)))
3020 89641 : && (poly_int_tree_p (gimple_call_arg (def_stmt, 2))
3021 36341 : || (TREE_CODE (gimple_call_arg (def_stmt, 2)) == SSA_NAME
3022 36341 : && poly_int_tree_p (SSA_VAL (gimple_call_arg (def_stmt, 2)))))
3023 23885192 : && (TREE_CODE (gimple_call_arg (def_stmt, 0)) == ADDR_EXPR
3024 30925 : || TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME))
3025 : {
3026 53822 : tree base2;
3027 53822 : poly_int64 offset2, size2, maxsize2;
3028 53822 : bool reverse;
3029 53822 : tree ref2 = gimple_call_arg (def_stmt, 0);
3030 53822 : if (TREE_CODE (ref2) == SSA_NAME)
3031 : {
3032 30884 : ref2 = SSA_VAL (ref2);
3033 30884 : if (TREE_CODE (ref2) == SSA_NAME
3034 30884 : && (TREE_CODE (base) != MEM_REF
3035 19618 : || TREE_OPERAND (base, 0) != ref2))
3036 : {
3037 24527 : gimple *def_stmt = SSA_NAME_DEF_STMT (ref2);
3038 24527 : if (gimple_assign_single_p (def_stmt)
3039 24527 : && gimple_assign_rhs_code (def_stmt) == ADDR_EXPR)
3040 834 : ref2 = gimple_assign_rhs1 (def_stmt);
3041 : }
3042 : }
3043 53822 : if (TREE_CODE (ref2) == ADDR_EXPR)
3044 : {
3045 26757 : ref2 = TREE_OPERAND (ref2, 0);
3046 26757 : base2 = get_ref_base_and_extent (ref2, &offset2, &size2, &maxsize2,
3047 : &reverse);
3048 26757 : if (!known_size_p (maxsize2)
3049 26717 : || !known_eq (maxsize2, size2)
3050 53400 : || !operand_equal_p (base, base2, OEP_ADDRESS_OF))
3051 57337 : return (void *)-1;
3052 : }
3053 27065 : else if (TREE_CODE (ref2) == SSA_NAME)
3054 : {
3055 27065 : poly_int64 soff;
3056 27065 : if (TREE_CODE (base) != MEM_REF
3057 46072 : || !(mem_ref_offset (base)
3058 38014 : << LOG2_BITS_PER_UNIT).to_shwi (&soff))
3059 22997 : return (void *)-1;
3060 19007 : offset += soff;
3061 19007 : offset2 = 0;
3062 19007 : if (TREE_OPERAND (base, 0) != ref2)
3063 : {
3064 15635 : gimple *def = SSA_NAME_DEF_STMT (ref2);
3065 15635 : if (is_gimple_assign (def)
3066 14246 : && gimple_assign_rhs_code (def) == POINTER_PLUS_EXPR
3067 12246 : && gimple_assign_rhs1 (def) == TREE_OPERAND (base, 0)
3068 16361 : && poly_int_tree_p (gimple_assign_rhs2 (def)))
3069 : {
3070 696 : tree rhs2 = gimple_assign_rhs2 (def);
3071 696 : if (!(poly_offset_int::from (wi::to_poly_wide (rhs2),
3072 : SIGNED)
3073 696 : << LOG2_BITS_PER_UNIT).to_shwi (&offset2))
3074 : return (void *)-1;
3075 696 : ref2 = gimple_assign_rhs1 (def);
3076 696 : if (TREE_CODE (ref2) == SSA_NAME)
3077 696 : ref2 = SSA_VAL (ref2);
3078 : }
3079 : else
3080 : return (void *)-1;
3081 : }
3082 : }
3083 : else
3084 : return (void *)-1;
3085 26919 : tree len = gimple_call_arg (def_stmt, 2);
3086 26919 : HOST_WIDE_INT leni, offset2i;
3087 26919 : if (TREE_CODE (len) == SSA_NAME)
3088 255 : len = SSA_VAL (len);
3089 : /* Sometimes the above trickery is smarter than alias analysis. Take
3090 : advantage of that. */
3091 26919 : if (!ranges_maybe_overlap_p (offset, maxsize, offset2,
3092 53838 : (wi::to_poly_offset (len)
3093 26919 : << LOG2_BITS_PER_UNIT)))
3094 : return NULL;
3095 53781 : if (data->partial_defs.is_empty ()
3096 26862 : && known_subrange_p (offset, maxsize, offset2,
3097 26862 : wi::to_poly_offset (len) << LOG2_BITS_PER_UNIT))
3098 : {
3099 26327 : tree val;
3100 26327 : if (integer_zerop (gimple_call_arg (def_stmt, 1)))
3101 21482 : val = build_zero_cst (vr->type);
3102 4845 : else if (INTEGRAL_TYPE_P (vr->type)
3103 3705 : && known_eq (ref->size, 8)
3104 7787 : && offseti % BITS_PER_UNIT == 0)
3105 : {
3106 2942 : gimple_match_op res_op (gimple_match_cond::UNCOND, NOP_EXPR,
3107 2942 : vr->type, gimple_call_arg (def_stmt, 1));
3108 2942 : val = vn_nary_build_or_lookup (&res_op);
3109 2942 : if (!val
3110 2942 : || (TREE_CODE (val) == SSA_NAME
3111 626 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (val)))
3112 0 : return (void *)-1;
3113 : }
3114 : else
3115 : {
3116 1903 : unsigned buflen
3117 1903 : = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (vr->type)) + 1;
3118 1903 : if (INTEGRAL_TYPE_P (vr->type)
3119 1903 : && TYPE_MODE (vr->type) != BLKmode)
3120 1524 : buflen = GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (vr->type)) + 1;
3121 1903 : unsigned char *buf = XALLOCAVEC (unsigned char, buflen);
3122 1903 : memset (buf, TREE_INT_CST_LOW (gimple_call_arg (def_stmt, 1)),
3123 : buflen);
3124 1903 : if (BYTES_BIG_ENDIAN)
3125 : {
3126 : unsigned int amnt
3127 : = (((unsigned HOST_WIDE_INT) offseti + sizei)
3128 : % BITS_PER_UNIT);
3129 : if (amnt)
3130 : {
3131 : shift_bytes_in_array_right (buf, buflen,
3132 : BITS_PER_UNIT - amnt);
3133 : buf++;
3134 : buflen--;
3135 : }
3136 : }
3137 1903 : else if (offseti % BITS_PER_UNIT != 0)
3138 : {
3139 7 : unsigned int amnt
3140 : = BITS_PER_UNIT - ((unsigned HOST_WIDE_INT) offseti
3141 7 : % BITS_PER_UNIT);
3142 7 : shift_bytes_in_array_left (buf, buflen, amnt);
3143 7 : buf++;
3144 7 : buflen--;
3145 : }
3146 1903 : val = native_interpret_expr (vr->type, buf, buflen);
3147 1903 : if (!val)
3148 : return (void *)-1;
3149 : }
3150 26327 : return data->finish (0, 0, val);
3151 : }
3152 : /* For now handle clearing memory with partial defs. */
3153 592 : else if (known_eq (ref->size, maxsize)
3154 518 : && integer_zerop (gimple_call_arg (def_stmt, 1))
3155 205 : && tree_fits_poly_int64_p (len)
3156 201 : && tree_to_poly_int64 (len).is_constant (&leni)
3157 201 : && leni <= INTTYPE_MAXIMUM (HOST_WIDE_INT) / BITS_PER_UNIT
3158 201 : && offset.is_constant (&offseti)
3159 201 : && offset2.is_constant (&offset2i)
3160 201 : && maxsize.is_constant (&maxsizei)
3161 592 : && ranges_known_overlap_p (offseti, maxsizei, offset2i,
3162 592 : leni << LOG2_BITS_PER_UNIT))
3163 : {
3164 201 : pd_data pd;
3165 201 : pd.rhs = build_constructor (NULL_TREE, NULL);
3166 201 : pd.rhs_off = 0;
3167 201 : pd.offset = offset2i;
3168 201 : pd.size = leni << LOG2_BITS_PER_UNIT;
3169 201 : return data->push_partial_def (pd, 0, 0, offseti, maxsizei);
3170 : }
3171 : }
3172 :
3173 : /* 2) Assignment from an empty CONSTRUCTOR. */
3174 23777507 : else if (is_gimple_reg_type (vr->type)
3175 23771694 : && gimple_assign_single_p (def_stmt)
3176 7832030 : && gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR
3177 1969241 : && CONSTRUCTOR_NELTS (gimple_assign_rhs1 (def_stmt)) == 0
3178 25746748 : && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt)))
3179 : {
3180 1969209 : tree base2;
3181 1969209 : poly_int64 offset2, size2, maxsize2;
3182 1969209 : HOST_WIDE_INT offset2i, size2i;
3183 1969209 : gcc_assert (lhs_ref_ok);
3184 1969209 : base2 = ao_ref_base (&lhs_ref);
3185 1969209 : offset2 = lhs_ref.offset;
3186 1969209 : size2 = lhs_ref.size;
3187 1969209 : maxsize2 = lhs_ref.max_size;
3188 1969209 : if (known_size_p (maxsize2)
3189 1969171 : && known_eq (maxsize2, size2)
3190 3938334 : && adjust_offsets_for_equal_base_address (base, &offset,
3191 : base2, &offset2))
3192 : {
3193 1941533 : if (data->partial_defs.is_empty ()
3194 1938002 : && known_subrange_p (offset, maxsize, offset2, size2))
3195 : {
3196 : /* While technically undefined behavior do not optimize
3197 : a full read from a clobber. */
3198 1937131 : if (gimple_clobber_p (def_stmt))
3199 1941483 : return (void *)-1;
3200 987391 : tree val = build_zero_cst (vr->type);
3201 987391 : return data->finish (ao_ref_alias_set (&lhs_ref),
3202 987391 : ao_ref_base_alias_set (&lhs_ref), val);
3203 : }
3204 4402 : else if (known_eq (ref->size, maxsize)
3205 4352 : && maxsize.is_constant (&maxsizei)
3206 4352 : && offset.is_constant (&offseti)
3207 4352 : && offset2.is_constant (&offset2i)
3208 4352 : && size2.is_constant (&size2i)
3209 4402 : && ranges_known_overlap_p (offseti, maxsizei,
3210 : offset2i, size2i))
3211 : {
3212 : /* Let clobbers be consumed by the partial-def tracker
3213 : which can choose to ignore them if they are shadowed
3214 : by a later def. */
3215 4352 : pd_data pd;
3216 4352 : pd.rhs = gimple_assign_rhs1 (def_stmt);
3217 4352 : pd.rhs_off = 0;
3218 4352 : pd.offset = offset2i;
3219 4352 : pd.size = size2i;
3220 4352 : return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
3221 : ao_ref_base_alias_set (&lhs_ref),
3222 : offseti, maxsizei);
3223 : }
3224 : }
3225 : }
3226 :
3227 : /* 3) Assignment from a constant. We can use folds native encode/interpret
3228 : routines to extract the assigned bits. */
3229 21808298 : else if (known_eq (ref->size, maxsize)
3230 21284296 : && is_gimple_reg_type (vr->type)
3231 21278483 : && !reverse_storage_order_for_component_p (vr->operands)
3232 21275727 : && !contains_storage_order_barrier_p (vr->operands)
3233 21275727 : && gimple_assign_single_p (def_stmt)
3234 5540668 : && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt))
3235 : && CHAR_BIT == 8
3236 : && BITS_PER_UNIT == 8
3237 : && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
3238 : /* native_encode and native_decode operate on arrays of bytes
3239 : and so fundamentally need a compile-time size and offset. */
3240 5537707 : && maxsize.is_constant (&maxsizei)
3241 5537707 : && offset.is_constant (&offseti)
3242 27346005 : && (is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt))
3243 4677968 : || (TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME
3244 1895073 : && is_gimple_min_invariant (SSA_VAL (gimple_assign_rhs1 (def_stmt))))))
3245 : {
3246 876455 : tree lhs = gimple_assign_lhs (def_stmt);
3247 876455 : tree base2;
3248 876455 : poly_int64 offset2, size2, maxsize2;
3249 876455 : HOST_WIDE_INT offset2i, size2i;
3250 876455 : bool reverse;
3251 876455 : gcc_assert (lhs_ref_ok);
3252 876455 : base2 = ao_ref_base (&lhs_ref);
3253 876455 : offset2 = lhs_ref.offset;
3254 876455 : size2 = lhs_ref.size;
3255 876455 : maxsize2 = lhs_ref.max_size;
3256 876455 : reverse = reverse_storage_order_for_component_p (lhs);
3257 876455 : if (base2
3258 876455 : && !reverse
3259 875627 : && !storage_order_barrier_p (lhs)
3260 875627 : && known_eq (maxsize2, size2)
3261 843864 : && adjust_offsets_for_equal_base_address (base, &offset,
3262 : base2, &offset2)
3263 84903 : && offset.is_constant (&offseti)
3264 84903 : && offset2.is_constant (&offset2i)
3265 876455 : && size2.is_constant (&size2i))
3266 : {
3267 84903 : if (data->partial_defs.is_empty ()
3268 67926 : && known_subrange_p (offseti, maxsizei, offset2, size2))
3269 : {
3270 : /* We support up to 512-bit values (for V8DFmode). */
3271 43863 : unsigned char buffer[65];
3272 43863 : int len;
3273 :
3274 43863 : tree rhs = gimple_assign_rhs1 (def_stmt);
3275 43863 : if (TREE_CODE (rhs) == SSA_NAME)
3276 1765 : rhs = SSA_VAL (rhs);
3277 87726 : len = native_encode_expr (rhs,
3278 : buffer, sizeof (buffer) - 1,
3279 43863 : (offseti - offset2i) / BITS_PER_UNIT);
3280 43863 : if (len > 0 && len * BITS_PER_UNIT >= maxsizei)
3281 : {
3282 40841 : tree type = vr->type;
3283 40841 : unsigned char *buf = buffer;
3284 40841 : unsigned int amnt = 0;
3285 : /* Make sure to interpret in a type that has a range
3286 : covering the whole access size. */
3287 40841 : if (INTEGRAL_TYPE_P (vr->type)
3288 40841 : && maxsizei != TYPE_PRECISION (vr->type))
3289 : {
3290 1012 : bool uns = TYPE_UNSIGNED (type);
3291 1011 : if (BITINT_TYPE_P (vr->type)
3292 1013 : && maxsizei > MAX_FIXED_MODE_SIZE)
3293 1 : type = build_bitint_type (maxsizei, uns);
3294 : else
3295 1011 : type = build_nonstandard_integer_type (maxsizei, uns);
3296 : }
3297 40841 : if (BYTES_BIG_ENDIAN)
3298 : {
3299 : /* For big-endian native_encode_expr stored the rhs
3300 : such that the LSB of it is the LSB of buffer[len - 1].
3301 : That bit is stored into memory at position
3302 : offset2 + size2 - 1, i.e. in byte
3303 : base + (offset2 + size2 - 1) / BITS_PER_UNIT.
3304 : E.g. for offset2 1 and size2 14, rhs -1 and memory
3305 : previously cleared that is:
3306 : 0 1
3307 : 01111111|11111110
3308 : Now, if we want to extract offset 2 and size 12 from
3309 : it using native_interpret_expr (which actually works
3310 : for integral bitfield types in terms of byte size of
3311 : the mode), the native_encode_expr stored the value
3312 : into buffer as
3313 : XX111111|11111111
3314 : and returned len 2 (the X bits are outside of
3315 : precision).
3316 : Let sz be maxsize / BITS_PER_UNIT if not extracting
3317 : a bitfield, and GET_MODE_SIZE otherwise.
3318 : We need to align the LSB of the value we want to
3319 : extract as the LSB of buf[sz - 1].
3320 : The LSB from memory we need to read is at position
3321 : offset + maxsize - 1. */
3322 : HOST_WIDE_INT sz = maxsizei / BITS_PER_UNIT;
3323 : if (INTEGRAL_TYPE_P (type))
3324 : {
3325 : if (TYPE_MODE (type) != BLKmode)
3326 : sz = GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (type));
3327 : else
3328 : sz = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (type));
3329 : }
3330 : amnt = ((unsigned HOST_WIDE_INT) offset2i + size2i
3331 : - offseti - maxsizei) % BITS_PER_UNIT;
3332 : if (amnt)
3333 : shift_bytes_in_array_right (buffer, len, amnt);
3334 : amnt = ((unsigned HOST_WIDE_INT) offset2i + size2i
3335 : - offseti - maxsizei - amnt) / BITS_PER_UNIT;
3336 : if ((unsigned HOST_WIDE_INT) sz + amnt > (unsigned) len)
3337 : len = 0;
3338 : else
3339 : {
3340 : buf = buffer + len - sz - amnt;
3341 : len -= (buf - buffer);
3342 : }
3343 : }
3344 : else
3345 : {
3346 40841 : amnt = ((unsigned HOST_WIDE_INT) offset2i
3347 40841 : - offseti) % BITS_PER_UNIT;
3348 40841 : if (amnt)
3349 : {
3350 315 : buffer[len] = 0;
3351 315 : shift_bytes_in_array_left (buffer, len + 1, amnt);
3352 315 : buf = buffer + 1;
3353 : }
3354 : }
3355 40841 : tree val = native_interpret_expr (type, buf, len);
3356 : /* If we chop off bits because the types precision doesn't
3357 : match the memory access size this is ok when optimizing
3358 : reads but not when called from the DSE code during
3359 : elimination. */
3360 40841 : if (val
3361 40839 : && type != vr->type)
3362 : {
3363 1012 : if (! int_fits_type_p (val, vr->type))
3364 : val = NULL_TREE;
3365 : else
3366 1012 : val = fold_convert (vr->type, val);
3367 : }
3368 :
3369 40839 : if (val)
3370 40839 : return data->finish (ao_ref_alias_set (&lhs_ref),
3371 40839 : ao_ref_base_alias_set (&lhs_ref), val);
3372 : }
3373 : }
3374 41040 : else if (ranges_known_overlap_p (offseti, maxsizei, offset2i,
3375 : size2i))
3376 : {
3377 41040 : pd_data pd;
3378 41040 : tree rhs = gimple_assign_rhs1 (def_stmt);
3379 41040 : if (TREE_CODE (rhs) == SSA_NAME)
3380 2226 : rhs = SSA_VAL (rhs);
3381 41040 : pd.rhs = rhs;
3382 41040 : pd.rhs_off = 0;
3383 41040 : pd.offset = offset2i;
3384 41040 : pd.size = size2i;
3385 41040 : return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
3386 : ao_ref_base_alias_set (&lhs_ref),
3387 : offseti, maxsizei);
3388 : }
3389 : }
3390 : }
3391 :
3392 : /* 4) Assignment from an SSA name which definition we may be able
3393 : to access pieces from or we can combine to a larger entity. */
3394 20931843 : else if (known_eq (ref->size, maxsize)
3395 20407841 : && is_gimple_reg_type (vr->type)
3396 20402028 : && !reverse_storage_order_for_component_p (vr->operands)
3397 20399272 : && !contains_storage_order_barrier_p (vr->operands)
3398 20399272 : && gimple_assign_single_p (def_stmt)
3399 4664213 : && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt))
3400 25593095 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
3401 : {
3402 1878357 : tree lhs = gimple_assign_lhs (def_stmt);
3403 1878357 : tree base2;
3404 1878357 : poly_int64 offset2, size2, maxsize2;
3405 1878357 : HOST_WIDE_INT offset2i, size2i, offseti;
3406 1878357 : bool reverse;
3407 1878357 : gcc_assert (lhs_ref_ok);
3408 1878357 : base2 = ao_ref_base (&lhs_ref);
3409 1878357 : offset2 = lhs_ref.offset;
3410 1878357 : size2 = lhs_ref.size;
3411 1878357 : maxsize2 = lhs_ref.max_size;
3412 1878357 : reverse = reverse_storage_order_for_component_p (lhs);
3413 1878357 : tree def_rhs = gimple_assign_rhs1 (def_stmt);
3414 1878357 : if (!reverse
3415 1878145 : && !storage_order_barrier_p (lhs)
3416 1878145 : && known_size_p (maxsize2)
3417 1853155 : && known_eq (maxsize2, size2)
3418 3610395 : && adjust_offsets_for_equal_base_address (base, &offset,
3419 : base2, &offset2))
3420 : {
3421 85621 : if (data->partial_defs.is_empty ()
3422 79193 : && known_subrange_p (offset, maxsize, offset2, size2)
3423 : /* ??? We can't handle bitfield precision extracts without
3424 : either using an alternate type for the BIT_FIELD_REF and
3425 : then doing a conversion or possibly adjusting the offset
3426 : according to endianness. */
3427 55058 : && (! INTEGRAL_TYPE_P (vr->type)
3428 40881 : || known_eq (ref->size, TYPE_PRECISION (vr->type)))
3429 96437 : && multiple_p (ref->size, BITS_PER_UNIT))
3430 : {
3431 46508 : tree val = NULL_TREE;
3432 93010 : if (! INTEGRAL_TYPE_P (TREE_TYPE (def_rhs))
3433 51177 : || type_has_mode_precision_p (TREE_TYPE (def_rhs)))
3434 : {
3435 45417 : gimple_match_op op (gimple_match_cond::UNCOND,
3436 45417 : BIT_FIELD_REF, vr->type,
3437 : SSA_VAL (def_rhs),
3438 : bitsize_int (ref->size),
3439 45417 : bitsize_int (offset - offset2));
3440 45417 : val = vn_nary_build_or_lookup (&op);
3441 : }
3442 1091 : else if (known_eq (ref->size, size2))
3443 : {
3444 1017 : gimple_match_op op (gimple_match_cond::UNCOND,
3445 1017 : VIEW_CONVERT_EXPR, vr->type,
3446 1017 : SSA_VAL (def_rhs));
3447 1017 : val = vn_nary_build_or_lookup (&op);
3448 : }
3449 46434 : if (val
3450 46434 : && (TREE_CODE (val) != SSA_NAME
3451 45634 : || ! SSA_NAME_OCCURS_IN_ABNORMAL_PHI (val)))
3452 46415 : return data->finish (ao_ref_alias_set (&lhs_ref),
3453 85528 : ao_ref_base_alias_set (&lhs_ref), val);
3454 : }
3455 39113 : else if (maxsize.is_constant (&maxsizei)
3456 39113 : && offset.is_constant (&offseti)
3457 39113 : && offset2.is_constant (&offset2i)
3458 39113 : && size2.is_constant (&size2i)
3459 39113 : && ranges_known_overlap_p (offset, maxsize, offset2, size2))
3460 : {
3461 39113 : pd_data pd;
3462 39113 : pd.rhs = SSA_VAL (def_rhs);
3463 39113 : pd.rhs_off = 0;
3464 39113 : pd.offset = offset2i;
3465 39113 : pd.size = size2i;
3466 39113 : return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
3467 : ao_ref_base_alias_set (&lhs_ref),
3468 : offseti, maxsizei);
3469 : }
3470 : }
3471 : }
3472 :
3473 : /* 4b) Assignment done via one of the vectorizer internal store
3474 : functions where we may be able to access pieces from or we can
3475 : combine to a larger entity. */
3476 19053486 : else if (known_eq (ref->size, maxsize)
3477 18529484 : && is_gimple_reg_type (vr->type)
3478 18523671 : && !reverse_storage_order_for_component_p (vr->operands)
3479 18520915 : && !contains_storage_order_barrier_p (vr->operands)
3480 18520915 : && is_gimple_call (def_stmt)
3481 14894599 : && gimple_call_internal_p (def_stmt)
3482 19364136 : && internal_store_fn_p (gimple_call_internal_fn (def_stmt)))
3483 : {
3484 36 : gcall *call = as_a <gcall *> (def_stmt);
3485 36 : internal_fn fn = gimple_call_internal_fn (call);
3486 :
3487 36 : tree mask = NULL_TREE, len = NULL_TREE, bias = NULL_TREE;
3488 36 : switch (fn)
3489 : {
3490 36 : case IFN_MASK_STORE:
3491 36 : mask = gimple_call_arg (call, internal_fn_mask_index (fn));
3492 36 : mask = vn_valueize (mask);
3493 36 : if (TREE_CODE (mask) != VECTOR_CST)
3494 28 : return (void *)-1;
3495 : break;
3496 0 : case IFN_LEN_STORE:
3497 0 : {
3498 0 : int len_index = internal_fn_len_index (fn);
3499 0 : len = gimple_call_arg (call, len_index);
3500 0 : bias = gimple_call_arg (call, len_index + 1);
3501 0 : if (!tree_fits_uhwi_p (len) || !tree_fits_shwi_p (bias))
3502 : return (void *) -1;
3503 : break;
3504 : }
3505 : default:
3506 : return (void *)-1;
3507 : }
3508 14 : tree def_rhs = gimple_call_arg (call,
3509 14 : internal_fn_stored_value_index (fn));
3510 14 : def_rhs = vn_valueize (def_rhs);
3511 14 : if (TREE_CODE (def_rhs) != VECTOR_CST)
3512 : return (void *)-1;
3513 :
3514 14 : ao_ref_init_from_ptr_and_size (&lhs_ref,
3515 : vn_valueize (gimple_call_arg (call, 0)),
3516 14 : TYPE_SIZE_UNIT (TREE_TYPE (def_rhs)));
3517 14 : tree base2;
3518 14 : poly_int64 offset2, size2, maxsize2;
3519 14 : HOST_WIDE_INT offset2i, size2i, offseti;
3520 14 : base2 = ao_ref_base (&lhs_ref);
3521 14 : offset2 = lhs_ref.offset;
3522 14 : size2 = lhs_ref.size;
3523 14 : maxsize2 = lhs_ref.max_size;
3524 14 : if (known_size_p (maxsize2)
3525 14 : && known_eq (maxsize2, size2)
3526 14 : && adjust_offsets_for_equal_base_address (base, &offset,
3527 : base2, &offset2)
3528 6 : && maxsize.is_constant (&maxsizei)
3529 6 : && offset.is_constant (&offseti)
3530 6 : && offset2.is_constant (&offset2i)
3531 14 : && size2.is_constant (&size2i))
3532 : {
3533 6 : if (!ranges_maybe_overlap_p (offset, maxsize, offset2, size2))
3534 : /* Poor-mans disambiguation. */
3535 : return NULL;
3536 6 : else if (ranges_known_overlap_p (offset, maxsize, offset2, size2))
3537 : {
3538 6 : pd_data pd;
3539 6 : pd.rhs = def_rhs;
3540 6 : tree aa = gimple_call_arg (call, 1);
3541 6 : alias_set_type set = get_deref_alias_set (TREE_TYPE (aa));
3542 6 : tree vectype = TREE_TYPE (def_rhs);
3543 6 : unsigned HOST_WIDE_INT elsz
3544 6 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (vectype)));
3545 6 : if (mask)
3546 : {
3547 : HOST_WIDE_INT start = 0, length = 0;
3548 : unsigned mask_idx = 0;
3549 48 : do
3550 : {
3551 48 : if (integer_zerop (VECTOR_CST_ELT (mask, mask_idx)))
3552 : {
3553 24 : if (length != 0)
3554 : {
3555 18 : pd.rhs_off = start;
3556 18 : pd.offset = offset2i + start;
3557 18 : pd.size = length;
3558 18 : if (ranges_known_overlap_p
3559 18 : (offset, maxsize, pd.offset, pd.size))
3560 : {
3561 0 : void *res = data->push_partial_def
3562 0 : (pd, set, set, offseti, maxsizei);
3563 0 : if (res != NULL)
3564 6 : return res;
3565 : }
3566 : }
3567 24 : start = (mask_idx + 1) * elsz;
3568 24 : length = 0;
3569 : }
3570 : else
3571 24 : length += elsz;
3572 48 : mask_idx++;
3573 : }
3574 48 : while (known_lt (mask_idx, TYPE_VECTOR_SUBPARTS (vectype)));
3575 6 : if (length != 0)
3576 : {
3577 6 : pd.rhs_off = start;
3578 6 : pd.offset = offset2i + start;
3579 6 : pd.size = length;
3580 6 : if (ranges_known_overlap_p (offset, maxsize,
3581 : pd.offset, pd.size))
3582 2 : return data->push_partial_def (pd, set, set,
3583 2 : offseti, maxsizei);
3584 : }
3585 : }
3586 0 : else if (fn == IFN_LEN_STORE)
3587 : {
3588 0 : pd.offset = offset2i;
3589 0 : pd.size = (tree_to_uhwi (len)
3590 0 : + -tree_to_shwi (bias)) * BITS_PER_UNIT;
3591 0 : if (BYTES_BIG_ENDIAN)
3592 : pd.rhs_off = pd.size - tree_to_uhwi (TYPE_SIZE (vectype));
3593 : else
3594 0 : pd.rhs_off = 0;
3595 0 : if (ranges_known_overlap_p (offset, maxsize,
3596 : pd.offset, pd.size))
3597 0 : return data->push_partial_def (pd, set, set,
3598 0 : offseti, maxsizei);
3599 : }
3600 : else
3601 0 : gcc_unreachable ();
3602 4 : return NULL;
3603 : }
3604 : }
3605 : }
3606 :
3607 : /* 5) For aggregate copies translate the reference through them if
3608 : the copy kills ref. */
3609 19053450 : else if (data->vn_walk_kind == VN_WALKREWRITE
3610 15390874 : && gimple_assign_single_p (def_stmt)
3611 2572650 : && !gimple_has_volatile_ops (def_stmt)
3612 21623802 : && (DECL_P (gimple_assign_rhs1 (def_stmt))
3613 1989175 : || TREE_CODE (gimple_assign_rhs1 (def_stmt)) == MEM_REF
3614 1577240 : || handled_component_p (gimple_assign_rhs1 (def_stmt))))
3615 : {
3616 2361967 : tree base2;
3617 2361967 : int i, j, k;
3618 2361967 : auto_vec<vn_reference_op_s> rhs;
3619 2361967 : vn_reference_op_t vro;
3620 2361967 : ao_ref r;
3621 :
3622 2361967 : gcc_assert (lhs_ref_ok);
3623 :
3624 : /* See if the assignment kills REF. */
3625 2361967 : base2 = ao_ref_base (&lhs_ref);
3626 2361967 : if (!lhs_ref.max_size_known_p ()
3627 2361392 : || (base != base2
3628 87427 : && (TREE_CODE (base) != MEM_REF
3629 71714 : || TREE_CODE (base2) != MEM_REF
3630 54637 : || TREE_OPERAND (base, 0) != TREE_OPERAND (base2, 0)
3631 18696 : || !tree_int_cst_equal (TREE_OPERAND (base, 1),
3632 18696 : TREE_OPERAND (base2, 1))))
3633 4653048 : || !stmt_kills_ref_p (def_stmt, ref))
3634 395454 : return (void *)-1;
3635 :
3636 : /* Find the common base of ref and the lhs. lhs_ops already
3637 : contains valueized operands for the lhs. */
3638 1966513 : poly_int64 extra_off = 0;
3639 1966513 : i = vr->operands.length () - 1;
3640 1966513 : j = lhs_ops.length () - 1;
3641 :
3642 : /* The base should be always equal due to the above check. */
3643 1966513 : if (! vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
3644 : return (void *)-1;
3645 1966253 : i--, j--;
3646 :
3647 : /* The 2nd component should always exist and be a MEM_REF. */
3648 1966253 : if (!(i >= 0 && j >= 0))
3649 : ;
3650 1966253 : else if (vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
3651 934643 : i--, j--;
3652 1031610 : else if (vr->operands[i].opcode == MEM_REF
3653 1030132 : && lhs_ops[j].opcode == MEM_REF
3654 1030132 : && known_ne (lhs_ops[j].off, -1)
3655 2061742 : && known_ne (vr->operands[i].off, -1))
3656 : {
3657 1030132 : bool found = false;
3658 : /* When we ge a mismatch at a MEM_REF that is not the sole component
3659 : try finding a match in one of the outer components and continue
3660 : stripping there. This happens when addresses of components get
3661 : forwarded into dereferences. */
3662 1030132 : if (i > 0)
3663 : {
3664 112945 : int temi = i - 1;
3665 112945 : poly_int64 tem_extra_off = extra_off + vr->operands[i].off;
3666 112945 : while (temi >= 0
3667 245961 : && known_ne (vr->operands[temi].off, -1))
3668 : {
3669 134499 : if (vr->operands[temi].type
3670 134499 : && lhs_ops[j].type
3671 268998 : && (TYPE_MAIN_VARIANT (vr->operands[temi].type)
3672 134499 : == TYPE_MAIN_VARIANT (lhs_ops[j].type)))
3673 : {
3674 1483 : i = temi;
3675 : /* Strip the component that was type matched to
3676 : the MEM_REF. */
3677 1483 : extra_off = (tem_extra_off
3678 1483 : + vr->operands[i].off - lhs_ops[j].off);
3679 1483 : i--, j--;
3680 : /* Strip further equal components. */
3681 1483 : found = true;
3682 1483 : break;
3683 : }
3684 133016 : tem_extra_off += vr->operands[temi].off;
3685 133016 : temi--;
3686 : }
3687 : }
3688 1030132 : if (!found && j > 0)
3689 : {
3690 33030 : int temj = j - 1;
3691 33030 : poly_int64 tem_extra_off = extra_off - lhs_ops[j].off;
3692 33030 : while (temj >= 0
3693 63254 : && known_ne (lhs_ops[temj].off, -1))
3694 : {
3695 35261 : if (vr->operands[i].type
3696 35261 : && lhs_ops[temj].type
3697 70522 : && (TYPE_MAIN_VARIANT (vr->operands[i].type)
3698 35261 : == TYPE_MAIN_VARIANT (lhs_ops[temj].type)))
3699 : {
3700 5037 : j = temj;
3701 : /* Strip the component that was type matched to
3702 : the MEM_REF. */
3703 5037 : extra_off = (tem_extra_off
3704 5037 : + vr->operands[i].off - lhs_ops[j].off);
3705 5037 : i--, j--;
3706 : /* Strip further equal components. */
3707 5037 : found = true;
3708 5037 : break;
3709 : }
3710 30224 : tem_extra_off += -lhs_ops[temj].off;
3711 30224 : temj--;
3712 : }
3713 : }
3714 : /* When we cannot find a common base to reconstruct the full
3715 : reference instead try to reduce the lookup to the new
3716 : base plus a constant offset. */
3717 1030132 : if (!found)
3718 : {
3719 : while (j >= 0
3720 2077074 : && known_ne (lhs_ops[j].off, -1))
3721 : {
3722 1053462 : extra_off += -lhs_ops[j].off;
3723 1053462 : j--;
3724 : }
3725 1023612 : if (j != -1)
3726 : return (void *)-1;
3727 : while (i >= 0
3728 2172976 : && known_ne (vr->operands[i].off, -1))
3729 : {
3730 : /* Punt if the additional ops contain a storage order
3731 : barrier. */
3732 1149364 : if (vr->operands[i].opcode == VIEW_CONVERT_EXPR
3733 1149364 : && vr->operands[i].reverse)
3734 : break;
3735 1149364 : extra_off += vr->operands[i].off;
3736 1149364 : i--;
3737 : }
3738 1023612 : if (i != -1)
3739 : return (void *)-1;
3740 : found = true;
3741 : }
3742 : /* If we did find a match we'd eventually append a MEM_REF
3743 : as component. Don't. */
3744 : if (!found)
3745 : return (void *)-1;
3746 : }
3747 : else
3748 : return (void *)-1;
3749 :
3750 : /* Strip further common components, attempting to consume lhs_ops
3751 : in full. */
3752 1963459 : while (j >= 0 && i >= 0
3753 1963459 : && vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
3754 : {
3755 25034 : i--;
3756 25034 : j--;
3757 : }
3758 :
3759 : /* i now points to the first additional op.
3760 : ??? LHS may not be completely contained in VR, one or more
3761 : VIEW_CONVERT_EXPRs could be in its way. We could at least
3762 : try handling outermost VIEW_CONVERT_EXPRs. */
3763 1938425 : if (j != -1)
3764 : return (void *)-1;
3765 :
3766 : /* Punt if the additional ops contain a storage order barrier. */
3767 3031577 : for (k = i; k >= 0; k--)
3768 : {
3769 1096065 : vro = &vr->operands[k];
3770 1096065 : if (vro->opcode == VIEW_CONVERT_EXPR && vro->reverse)
3771 : return (void *)-1;
3772 : }
3773 :
3774 : /* Now re-write REF to be based on the rhs of the assignment. */
3775 1935512 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
3776 1935512 : copy_reference_ops_from_ref (rhs1, &rhs);
3777 :
3778 : /* Apply an extra offset to the inner MEM_REF of the RHS. */
3779 1935512 : bool force_no_tbaa = false;
3780 1935512 : if (maybe_ne (extra_off, 0))
3781 : {
3782 737731 : if (rhs.length () < 2)
3783 : return (void *)-1;
3784 737731 : int ix = rhs.length () - 2;
3785 737731 : if (rhs[ix].opcode != MEM_REF
3786 737731 : || known_eq (rhs[ix].off, -1))
3787 : return (void *)-1;
3788 737713 : rhs[ix].off += extra_off;
3789 737713 : rhs[ix].op0 = int_const_binop (PLUS_EXPR, rhs[ix].op0,
3790 737713 : build_int_cst (TREE_TYPE (rhs[ix].op0),
3791 : extra_off));
3792 : /* When we have offsetted the RHS, reading only parts of it,
3793 : we can no longer use the original TBAA type, force alias-set
3794 : zero. */
3795 737713 : force_no_tbaa = true;
3796 : }
3797 :
3798 : /* Save the operands since we need to use the original ones for
3799 : the hash entry we use. */
3800 1935494 : if (!data->saved_operands.exists ())
3801 1824116 : data->saved_operands = vr->operands.copy ();
3802 :
3803 : /* We need to pre-pend vr->operands[0..i] to rhs. */
3804 1935494 : vec<vn_reference_op_s> old = vr->operands;
3805 5806482 : if (i + 1 + rhs.length () > vr->operands.length ())
3806 1154213 : vr->operands.safe_grow (i + 1 + rhs.length (), true);
3807 : else
3808 781281 : vr->operands.truncate (i + 1 + rhs.length ());
3809 7009267 : FOR_EACH_VEC_ELT (rhs, j, vro)
3810 5073773 : vr->operands[i + 1 + j] = *vro;
3811 1935494 : valueize_refs (&vr->operands);
3812 3870988 : if (old == shared_lookup_references)
3813 1935494 : shared_lookup_references = vr->operands;
3814 1935494 : vr->hashcode = vn_reference_compute_hash (vr);
3815 :
3816 : /* Try folding the new reference to a constant. */
3817 1935494 : tree val = fully_constant_vn_reference_p (vr);
3818 1935494 : if (val)
3819 : {
3820 22090 : if (data->partial_defs.is_empty ())
3821 22081 : return data->finish (ao_ref_alias_set (&lhs_ref),
3822 22081 : ao_ref_base_alias_set (&lhs_ref), val);
3823 : /* This is the only interesting case for partial-def handling
3824 : coming from targets that like to gimplify init-ctors as
3825 : aggregate copies from constant data like aarch64 for
3826 : PR83518. */
3827 9 : if (maxsize.is_constant (&maxsizei) && known_eq (ref->size, maxsize))
3828 : {
3829 9 : pd_data pd;
3830 9 : pd.rhs = val;
3831 9 : pd.rhs_off = 0;
3832 9 : pd.offset = 0;
3833 9 : pd.size = maxsizei;
3834 9 : return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
3835 : ao_ref_base_alias_set (&lhs_ref),
3836 : 0, maxsizei);
3837 : }
3838 : }
3839 :
3840 : /* Continuing with partial defs isn't easily possible here, we
3841 : have to find a full def from further lookups from here. Probably
3842 : not worth the special-casing everywhere. */
3843 2345879 : if (!data->partial_defs.is_empty ())
3844 : return (void *)-1;
3845 :
3846 : /* Adjust *ref from the new operands. */
3847 1907402 : ao_ref rhs1_ref;
3848 1907402 : ao_ref_init (&rhs1_ref, rhs1);
3849 3091713 : if (!ao_ref_init_from_vn_reference (&r,
3850 : force_no_tbaa ? 0
3851 1184311 : : ao_ref_alias_set (&rhs1_ref),
3852 : force_no_tbaa ? 0
3853 1184311 : : ao_ref_base_alias_set (&rhs1_ref),
3854 : vr->type, vr->operands))
3855 : return (void *)-1;
3856 : /* This can happen with bitfields. */
3857 1907402 : if (maybe_ne (ref->size, r.size))
3858 : {
3859 : /* If the access lacks some subsetting simply apply that by
3860 : shortening it. That in the end can only be successful
3861 : if we can pun the lookup result which in turn requires
3862 : exact offsets. */
3863 1583 : if (known_eq (r.size, r.max_size)
3864 1583 : && known_lt (ref->size, r.size))
3865 1583 : r.size = r.max_size = ref->size;
3866 : else
3867 : return (void *)-1;
3868 : }
3869 1907402 : *ref = r;
3870 1907402 : vr->offset = r.offset;
3871 1907402 : vr->max_size = r.max_size;
3872 :
3873 : /* Do not update last seen VUSE after translating. */
3874 1907402 : data->last_vuse_ptr = NULL;
3875 : /* Invalidate the original access path since it now contains
3876 : the wrong base. */
3877 1907402 : data->orig_ref.ref = NULL_TREE;
3878 : /* Use the alias-set of this LHS for recording an eventual result. */
3879 1907402 : if (data->first_set == -2)
3880 : {
3881 1797552 : data->first_set = ao_ref_alias_set (&lhs_ref);
3882 1797552 : data->first_base_set = ao_ref_base_alias_set (&lhs_ref);
3883 : }
3884 :
3885 : /* Keep looking for the adjusted *REF / VR pair. */
3886 1907402 : return NULL;
3887 2361967 : }
3888 :
3889 : /* 6) For memcpy copies translate the reference through them if the copy
3890 : kills ref. But we cannot (easily) do this translation if the memcpy is
3891 : a storage order barrier, i.e. is equivalent to a VIEW_CONVERT_EXPR that
3892 : can modify the storage order of objects (see storage_order_barrier_p). */
3893 16691483 : else if (data->vn_walk_kind == VN_WALKREWRITE
3894 13028907 : && is_gimple_reg_type (vr->type)
3895 : /* ??? Handle BCOPY as well. */
3896 13023094 : && (gimple_call_builtin_p (def_stmt, BUILT_IN_MEMCPY)
3897 12953930 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMCPY_CHK)
3898 12953507 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMPCPY)
3899 12952331 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMPCPY_CHK)
3900 12952089 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMMOVE)
3901 12926738 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMMOVE_CHK))
3902 96684 : && (TREE_CODE (gimple_call_arg (def_stmt, 0)) == ADDR_EXPR
3903 84658 : || TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME)
3904 96648 : && (TREE_CODE (gimple_call_arg (def_stmt, 1)) == ADDR_EXPR
3905 68385 : || TREE_CODE (gimple_call_arg (def_stmt, 1)) == SSA_NAME)
3906 96633 : && (poly_int_tree_p (gimple_call_arg (def_stmt, 2), ©_size)
3907 55162 : || (TREE_CODE (gimple_call_arg (def_stmt, 2)) == SSA_NAME
3908 55162 : && poly_int_tree_p (SSA_VAL (gimple_call_arg (def_stmt, 2)),
3909 : ©_size)))
3910 : /* Handling this is more complicated, give up for now. */
3911 16735563 : && data->partial_defs.is_empty ())
3912 : {
3913 43382 : tree lhs, rhs;
3914 43382 : ao_ref r;
3915 43382 : poly_int64 rhs_offset, lhs_offset;
3916 43382 : vn_reference_op_s op;
3917 43382 : poly_uint64 mem_offset;
3918 43382 : poly_int64 at, byte_maxsize;
3919 :
3920 : /* Only handle non-variable, addressable refs. */
3921 43382 : if (maybe_ne (ref->size, maxsize)
3922 42903 : || !multiple_p (offset, BITS_PER_UNIT, &at)
3923 43382 : || !multiple_p (maxsize, BITS_PER_UNIT, &byte_maxsize))
3924 479 : return (void *)-1;
3925 :
3926 : /* Extract a pointer base and an offset for the destination. */
3927 42903 : lhs = gimple_call_arg (def_stmt, 0);
3928 42903 : lhs_offset = 0;
3929 42903 : if (TREE_CODE (lhs) == SSA_NAME)
3930 : {
3931 32613 : lhs = vn_valueize (lhs);
3932 32613 : if (TREE_CODE (lhs) == SSA_NAME)
3933 : {
3934 32276 : gimple *def_stmt = SSA_NAME_DEF_STMT (lhs);
3935 32276 : if (gimple_assign_single_p (def_stmt)
3936 32276 : && gimple_assign_rhs_code (def_stmt) == ADDR_EXPR)
3937 2381 : lhs = gimple_assign_rhs1 (def_stmt);
3938 : }
3939 : }
3940 42903 : if (TREE_CODE (lhs) == ADDR_EXPR)
3941 : {
3942 18273 : if (AGGREGATE_TYPE_P (TREE_TYPE (TREE_TYPE (lhs)))
3943 17976 : && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (TREE_TYPE (lhs))))
3944 : return (void *)-1;
3945 12868 : tree tem = get_addr_base_and_unit_offset (TREE_OPERAND (lhs, 0),
3946 : &lhs_offset);
3947 12868 : if (!tem)
3948 : return (void *)-1;
3949 12168 : if (TREE_CODE (tem) == MEM_REF
3950 12168 : && poly_int_tree_p (TREE_OPERAND (tem, 1), &mem_offset))
3951 : {
3952 1681 : lhs = TREE_OPERAND (tem, 0);
3953 1681 : if (TREE_CODE (lhs) == SSA_NAME)
3954 1681 : lhs = vn_valueize (lhs);
3955 1681 : lhs_offset += mem_offset;
3956 : }
3957 10487 : else if (DECL_P (tem))
3958 10487 : lhs = build_fold_addr_expr (tem);
3959 : else
3960 : return (void *)-1;
3961 : }
3962 42063 : if (TREE_CODE (lhs) != SSA_NAME
3963 10488 : && TREE_CODE (lhs) != ADDR_EXPR)
3964 : return (void *)-1;
3965 :
3966 : /* Extract a pointer base and an offset for the source. */
3967 42063 : rhs = gimple_call_arg (def_stmt, 1);
3968 42063 : rhs_offset = 0;
3969 42063 : if (TREE_CODE (rhs) == SSA_NAME)
3970 19651 : rhs = vn_valueize (rhs);
3971 42063 : if (TREE_CODE (rhs) == ADDR_EXPR)
3972 : {
3973 35188 : if (AGGREGATE_TYPE_P (TREE_TYPE (TREE_TYPE (rhs)))
3974 24655 : && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (TREE_TYPE (rhs))))
3975 : return (void *)-1;
3976 24053 : tree tem = get_addr_base_and_unit_offset (TREE_OPERAND (rhs, 0),
3977 : &rhs_offset);
3978 24053 : if (!tem)
3979 : return (void *)-1;
3980 24053 : if (TREE_CODE (tem) == MEM_REF
3981 24053 : && poly_int_tree_p (TREE_OPERAND (tem, 1), &mem_offset))
3982 : {
3983 0 : rhs = TREE_OPERAND (tem, 0);
3984 0 : rhs_offset += mem_offset;
3985 : }
3986 24053 : else if (DECL_P (tem)
3987 17620 : || TREE_CODE (tem) == STRING_CST)
3988 24053 : rhs = build_fold_addr_expr (tem);
3989 : else
3990 : return (void *)-1;
3991 : }
3992 42063 : if (TREE_CODE (rhs) == SSA_NAME)
3993 18010 : rhs = SSA_VAL (rhs);
3994 24053 : else if (TREE_CODE (rhs) != ADDR_EXPR)
3995 : return (void *)-1;
3996 :
3997 : /* The bases of the destination and the references have to agree. */
3998 42063 : if (TREE_CODE (base) == MEM_REF)
3999 : {
4000 15681 : if (TREE_OPERAND (base, 0) != lhs
4001 15681 : || !poly_int_tree_p (TREE_OPERAND (base, 1), &mem_offset))
4002 12016 : return (void *) -1;
4003 12963 : at += mem_offset;
4004 : }
4005 26382 : else if (!DECL_P (base)
4006 25441 : || TREE_CODE (lhs) != ADDR_EXPR
4007 35681 : || TREE_OPERAND (lhs, 0) != base)
4008 : return (void *)-1;
4009 :
4010 : /* If the access is completely outside of the memcpy destination
4011 : area there is no aliasing. */
4012 12963 : if (!ranges_maybe_overlap_p (lhs_offset, copy_size, at, byte_maxsize))
4013 : return NULL;
4014 : /* And the access has to be contained within the memcpy destination. */
4015 12930 : if (!known_subrange_p (at, byte_maxsize, lhs_offset, copy_size))
4016 : return (void *)-1;
4017 :
4018 : /* Save the operands since we need to use the original ones for
4019 : the hash entry we use. */
4020 12311 : if (!data->saved_operands.exists ())
4021 11878 : data->saved_operands = vr->operands.copy ();
4022 :
4023 : /* Make room for 2 operands in the new reference. */
4024 12311 : if (vr->operands.length () < 2)
4025 : {
4026 0 : vec<vn_reference_op_s> old = vr->operands;
4027 0 : vr->operands.safe_grow_cleared (2, true);
4028 0 : if (old == shared_lookup_references)
4029 0 : shared_lookup_references = vr->operands;
4030 : }
4031 : else
4032 12311 : vr->operands.truncate (2);
4033 :
4034 : /* The looked-through reference is a simple MEM_REF. */
4035 12311 : memset (&op, 0, sizeof (op));
4036 12311 : op.type = vr->type;
4037 12311 : op.opcode = MEM_REF;
4038 12311 : op.op0 = build_int_cst (ptr_type_node, at - lhs_offset + rhs_offset);
4039 12311 : op.off = at - lhs_offset + rhs_offset;
4040 12311 : vr->operands[0] = op;
4041 12311 : op.type = TREE_TYPE (rhs);
4042 12311 : op.opcode = TREE_CODE (rhs);
4043 12311 : op.op0 = rhs;
4044 12311 : op.off = -1;
4045 12311 : vr->operands[1] = op;
4046 12311 : vr->hashcode = vn_reference_compute_hash (vr);
4047 :
4048 : /* Try folding the new reference to a constant. */
4049 12311 : tree val = fully_constant_vn_reference_p (vr);
4050 12311 : if (val)
4051 3203 : return data->finish (0, 0, val);
4052 :
4053 : /* Adjust *ref from the new operands. */
4054 9108 : if (!ao_ref_init_from_vn_reference (&r, 0, 0, vr->type, vr->operands))
4055 : return (void *)-1;
4056 : /* This can happen with bitfields. */
4057 9108 : if (maybe_ne (ref->size, r.size))
4058 : return (void *)-1;
4059 9108 : *ref = r;
4060 9108 : vr->offset = r.offset;
4061 9108 : vr->max_size = r.max_size;
4062 :
4063 : /* Do not update last seen VUSE after translating. */
4064 9108 : data->last_vuse_ptr = NULL;
4065 : /* Invalidate the original access path since it now contains
4066 : the wrong base. */
4067 9108 : data->orig_ref.ref = NULL_TREE;
4068 : /* Use the alias-set of this stmt for recording an eventual result. */
4069 9108 : if (data->first_set == -2)
4070 : {
4071 8726 : data->first_set = 0;
4072 8726 : data->first_base_set = 0;
4073 : }
4074 :
4075 : /* Keep looking for the adjusted *REF / VR pair. */
4076 9108 : return NULL;
4077 : }
4078 :
4079 : /* Bail out and stop walking. */
4080 : return (void *)-1;
4081 : }
4082 :
4083 : /* Return true if E is a backedge with respect to our CFG walk order. */
4084 :
4085 : static bool
4086 124406619 : vn_is_backedge (edge e, void *)
4087 : {
4088 : /* During PRE elimination we no longer have access to this info. */
4089 124406619 : return (!vn_bb_to_rpo
4090 124406619 : || vn_bb_to_rpo[e->dest->index] <= vn_bb_to_rpo[e->src->index]);
4091 : }
4092 :
4093 : /* Return a reference op vector from OP that can be used for
4094 : vn_reference_lookup_pieces. The caller is responsible for releasing
4095 : the vector. */
4096 :
4097 : vec<vn_reference_op_s>
4098 4805962 : vn_reference_operands_for_lookup (tree op)
4099 : {
4100 4805962 : bool valueized;
4101 4805962 : return valueize_shared_reference_ops_from_ref (op, &valueized).copy ();
4102 : }
4103 :
4104 : /* Lookup a reference operation by it's parts, in the current hash table.
4105 : Returns the resulting value number if it exists in the hash table,
4106 : NULL_TREE otherwise. VNRESULT will be filled in with the actual
4107 : vn_reference_t stored in the hashtable if something is found. */
4108 :
4109 : tree
4110 7902899 : vn_reference_lookup_pieces (tree vuse, alias_set_type set,
4111 : alias_set_type base_set, tree type,
4112 : vec<vn_reference_op_s> operands,
4113 : vn_reference_t *vnresult, vn_lookup_kind kind)
4114 : {
4115 7902899 : struct vn_reference_s vr1;
4116 7902899 : vn_reference_t tmp;
4117 7902899 : tree cst;
4118 :
4119 7902899 : if (!vnresult)
4120 0 : vnresult = &tmp;
4121 7902899 : *vnresult = NULL;
4122 :
4123 7902899 : vr1.vuse = vuse_ssa_val (vuse);
4124 7902899 : shared_lookup_references.truncate (0);
4125 15805798 : shared_lookup_references.safe_grow (operands.length (), true);
4126 7902899 : memcpy (shared_lookup_references.address (),
4127 7902899 : operands.address (),
4128 : sizeof (vn_reference_op_s)
4129 7902899 : * operands.length ());
4130 7902899 : bool valueized_p;
4131 7902899 : valueize_refs_1 (&shared_lookup_references, &valueized_p);
4132 7902899 : vr1.operands = shared_lookup_references;
4133 7902899 : vr1.type = type;
4134 7902899 : vr1.set = set;
4135 7902899 : vr1.base_set = base_set;
4136 : /* We can pretend there's no extra info fed in since the ao_refs offset
4137 : and max_size are computed only from the VN reference ops. */
4138 7902899 : vr1.offset = 0;
4139 7902899 : vr1.max_size = -1;
4140 7902899 : vr1.hashcode = vn_reference_compute_hash (&vr1);
4141 7902899 : if ((cst = fully_constant_vn_reference_p (&vr1)))
4142 : return cst;
4143 :
4144 7883357 : vn_reference_lookup_1 (&vr1, vnresult);
4145 7883357 : if (!*vnresult
4146 3058641 : && kind != VN_NOWALK
4147 3058641 : && vr1.vuse)
4148 : {
4149 3028939 : ao_ref r;
4150 3028939 : unsigned limit = param_sccvn_max_alias_queries_per_access;
4151 3028939 : vn_walk_cb_data data (&vr1, NULL_TREE, NULL, kind, true, NULL_TREE,
4152 3028939 : false);
4153 3028939 : vec<vn_reference_op_s> ops_for_ref;
4154 3028939 : if (!valueized_p)
4155 2934103 : ops_for_ref = vr1.operands;
4156 : else
4157 : {
4158 : /* For ao_ref_from_mem we have to ensure only available SSA names
4159 : end up in base and the only convenient way to make this work
4160 : for PRE is to re-valueize with that in mind. */
4161 189672 : ops_for_ref.create (operands.length ());
4162 189672 : ops_for_ref.quick_grow (operands.length ());
4163 94836 : memcpy (ops_for_ref.address (),
4164 94836 : operands.address (),
4165 : sizeof (vn_reference_op_s)
4166 94836 : * operands.length ());
4167 94836 : valueize_refs_1 (&ops_for_ref, &valueized_p, true);
4168 : }
4169 3028939 : if (ao_ref_init_from_vn_reference (&r, set, base_set, type,
4170 : ops_for_ref))
4171 2958964 : *vnresult
4172 2958964 : = ((vn_reference_t)
4173 2958964 : walk_non_aliased_vuses (&r, vr1.vuse, true, vn_reference_lookup_2,
4174 : vn_reference_lookup_3, vn_is_backedge,
4175 : vuse_valueize, limit, &data));
4176 6057878 : if (ops_for_ref != shared_lookup_references)
4177 94836 : ops_for_ref.release ();
4178 6057878 : gcc_checking_assert (vr1.operands == shared_lookup_references);
4179 3028939 : if (*vnresult
4180 430338 : && data.same_val
4181 3028939 : && (!(*vnresult)->result
4182 0 : || !operand_equal_p ((*vnresult)->result, data.same_val)))
4183 : {
4184 0 : *vnresult = NULL;
4185 0 : return NULL_TREE;
4186 : }
4187 3028939 : }
4188 :
4189 7883357 : if (*vnresult)
4190 5255054 : return (*vnresult)->result;
4191 :
4192 : return NULL_TREE;
4193 : }
4194 :
4195 : /* When OPERANDS is an ADDR_EXPR that can be possibly expressed as a
4196 : POINTER_PLUS_EXPR return true and fill in its operands in OPS. */
4197 :
4198 : bool
4199 2261205 : vn_pp_nary_for_addr (const vec<vn_reference_op_s>& operands, tree ops[2])
4200 : {
4201 4522410 : gcc_assert (operands[0].opcode == ADDR_EXPR
4202 : && operands.last ().opcode == SSA_NAME);
4203 : poly_int64 off = 0;
4204 : vn_reference_op_t vro;
4205 : unsigned i;
4206 7331859 : for (i = 1; operands.iterate (i, &vro); ++i)
4207 : {
4208 7331859 : if (vro->opcode == SSA_NAME)
4209 : break;
4210 5121029 : else if (known_eq (vro->off, -1))
4211 : break;
4212 5070654 : off += vro->off;
4213 : }
4214 2261205 : if (i == operands.length () - 1
4215 2210830 : && maybe_ne (off, 0)
4216 : /* Make sure we the offset we accumulated in a 64bit int
4217 : fits the address computation carried out in target
4218 : offset precision. */
4219 3726371 : && (off.coeffs[0]
4220 1465166 : == sext_hwi (off.coeffs[0], TYPE_PRECISION (sizetype))))
4221 : {
4222 1464622 : gcc_assert (operands[i-1].opcode == MEM_REF);
4223 1464622 : ops[0] = operands[i].op0;
4224 1464622 : ops[1] = wide_int_to_tree (sizetype, off);
4225 1464622 : return true;
4226 : }
4227 : return false;
4228 : }
4229 :
4230 : /* Lookup OP in the current hash table, and return the resulting value
4231 : number if it exists in the hash table. Return NULL_TREE if it does
4232 : not exist in the hash table or if the result field of the structure
4233 : was NULL.. VNRESULT will be filled in with the vn_reference_t
4234 : stored in the hashtable if one exists. When TBAA_P is false assume
4235 : we are looking up a store and treat it as having alias-set zero.
4236 : *LAST_VUSE_PTR will be updated with the VUSE the value lookup succeeded.
4237 : MASK is either NULL_TREE, or can be an INTEGER_CST if the result of the
4238 : load is bitwise anded with MASK and so we are only interested in a subset
4239 : of the bits and can ignore if the other bits are uninitialized or
4240 : not initialized with constants. When doing redundant store removal
4241 : the caller has to set REDUNDANT_STORE_REMOVAL_P. */
4242 :
4243 : tree
4244 102518281 : vn_reference_lookup (tree op, tree vuse, vn_lookup_kind kind,
4245 : vn_reference_t *vnresult, bool tbaa_p,
4246 : tree *last_vuse_ptr, tree mask,
4247 : bool redundant_store_removal_p)
4248 : {
4249 102518281 : vec<vn_reference_op_s> operands;
4250 102518281 : struct vn_reference_s vr1;
4251 102518281 : bool valueized_anything;
4252 :
4253 102518281 : if (vnresult)
4254 102124878 : *vnresult = NULL;
4255 :
4256 102518281 : vr1.vuse = vuse_ssa_val (vuse);
4257 205036562 : vr1.operands = operands
4258 102518281 : = valueize_shared_reference_ops_from_ref (op, &valueized_anything);
4259 :
4260 : /* Handle &MEM[ptr + 5].b[1].c as POINTER_PLUS_EXPR. Avoid doing
4261 : this before the pass folding __builtin_object_size had a chance to run. */
4262 102518281 : if ((cfun->curr_properties & PROP_objsz)
4263 74368435 : && operands[0].opcode == ADDR_EXPR
4264 103653363 : && operands.last ().opcode == SSA_NAME)
4265 : {
4266 1100566 : tree ops[2];
4267 1100566 : if (vn_pp_nary_for_addr (operands, ops))
4268 : {
4269 713567 : tree res = vn_nary_op_lookup_pieces (2, POINTER_PLUS_EXPR,
4270 713567 : TREE_TYPE (op), ops, NULL);
4271 713567 : if (res)
4272 713567 : return res;
4273 713567 : return NULL_TREE;
4274 : }
4275 : }
4276 :
4277 101804714 : vr1.type = TREE_TYPE (op);
4278 101804714 : ao_ref op_ref;
4279 101804714 : ao_ref_init (&op_ref, op);
4280 101804714 : vr1.set = ao_ref_alias_set (&op_ref);
4281 101804714 : vr1.base_set = ao_ref_base_alias_set (&op_ref);
4282 101804714 : vr1.offset = 0;
4283 101804714 : vr1.max_size = -1;
4284 101804714 : vr1.hashcode = vn_reference_compute_hash (&vr1);
4285 101804714 : if (mask == NULL_TREE)
4286 101501422 : if (tree cst = fully_constant_vn_reference_p (&vr1))
4287 : return cst;
4288 :
4289 101789303 : if (kind != VN_NOWALK && vr1.vuse)
4290 : {
4291 59165158 : vn_reference_t wvnresult;
4292 59165158 : ao_ref r;
4293 59165158 : unsigned limit = param_sccvn_max_alias_queries_per_access;
4294 59165158 : auto_vec<vn_reference_op_s> ops_for_ref;
4295 59165158 : if (valueized_anything)
4296 : {
4297 4701058 : copy_reference_ops_from_ref (op, &ops_for_ref);
4298 4701058 : bool tem;
4299 4701058 : valueize_refs_1 (&ops_for_ref, &tem, true);
4300 : }
4301 : /* Make sure to use a valueized reference if we valueized anything.
4302 : Otherwise preserve the full reference for advanced TBAA. */
4303 59165158 : if (!valueized_anything
4304 59165158 : || !ao_ref_init_from_vn_reference (&r, vr1.set, vr1.base_set,
4305 : vr1.type, ops_for_ref))
4306 : {
4307 54464100 : ao_ref_init (&r, op);
4308 : /* Record the extra info we're getting from the full ref. */
4309 54464100 : ao_ref_base (&r);
4310 54464100 : vr1.offset = r.offset;
4311 54464100 : vr1.max_size = r.max_size;
4312 : }
4313 59165158 : vn_walk_cb_data data (&vr1, r.ref ? NULL_TREE : op,
4314 : last_vuse_ptr, kind, tbaa_p, mask,
4315 113629258 : redundant_store_removal_p);
4316 :
4317 59165158 : wvnresult
4318 : = ((vn_reference_t)
4319 59165158 : walk_non_aliased_vuses (&r, vr1.vuse, tbaa_p, vn_reference_lookup_2,
4320 : vn_reference_lookup_3, vn_is_backedge,
4321 : vuse_valueize, limit, &data));
4322 118330316 : gcc_checking_assert (vr1.operands == shared_lookup_references);
4323 59165158 : if (wvnresult)
4324 : {
4325 8745333 : gcc_assert (mask == NULL_TREE);
4326 8745333 : if (data.same_val
4327 8745333 : && (!wvnresult->result
4328 66673 : || !operand_equal_p (wvnresult->result, data.same_val)))
4329 46630 : return NULL_TREE;
4330 8698703 : if (vnresult)
4331 8697182 : *vnresult = wvnresult;
4332 8698703 : return wvnresult->result;
4333 : }
4334 50419825 : else if (mask)
4335 303292 : return data.masked_result;
4336 :
4337 : return NULL_TREE;
4338 59165158 : }
4339 :
4340 42624145 : if (last_vuse_ptr)
4341 1470188 : *last_vuse_ptr = vr1.vuse;
4342 42624145 : if (mask)
4343 : return NULL_TREE;
4344 42624145 : return vn_reference_lookup_1 (&vr1, vnresult);
4345 : }
4346 :
4347 : /* Lookup CALL in the current hash table and return the entry in
4348 : *VNRESULT if found. Populates *VR for the hashtable lookup. */
4349 :
4350 : void
4351 9346632 : vn_reference_lookup_call (gcall *call, vn_reference_t *vnresult,
4352 : vn_reference_t vr)
4353 : {
4354 9346632 : if (vnresult)
4355 9346632 : *vnresult = NULL;
4356 :
4357 9346632 : tree vuse = gimple_vuse (call);
4358 :
4359 9346632 : vr->vuse = vuse ? SSA_VAL (vuse) : NULL_TREE;
4360 9346632 : vr->operands = valueize_shared_reference_ops_from_call (call);
4361 9346632 : tree lhs = gimple_call_lhs (call);
4362 : /* For non-SSA return values the reference ops contain the LHS. */
4363 5078519 : vr->type = ((lhs && TREE_CODE (lhs) == SSA_NAME)
4364 13970172 : ? TREE_TYPE (lhs) : NULL_TREE);
4365 9346632 : vr->punned = false;
4366 9346632 : vr->set = 0;
4367 9346632 : vr->base_set = 0;
4368 9346632 : vr->offset = 0;
4369 9346632 : vr->max_size = -1;
4370 9346632 : vr->hashcode = vn_reference_compute_hash (vr);
4371 9346632 : vn_reference_lookup_1 (vr, vnresult);
4372 9346632 : }
4373 :
4374 : /* Insert OP into the current hash table with a value number of RESULT. */
4375 :
4376 : static void
4377 76054050 : vn_reference_insert (tree op, tree result, tree vuse, tree vdef)
4378 : {
4379 76054050 : vn_reference_s **slot;
4380 76054050 : vn_reference_t vr1;
4381 76054050 : bool tem;
4382 :
4383 76054050 : vec<vn_reference_op_s> operands
4384 76054050 : = valueize_shared_reference_ops_from_ref (op, &tem);
4385 : /* Handle &MEM[ptr + 5].b[1].c as POINTER_PLUS_EXPR. Avoid doing this
4386 : before the pass folding __builtin_object_size had a chance to run. */
4387 76054050 : if ((cfun->curr_properties & PROP_objsz)
4388 57075126 : && operands[0].opcode == ADDR_EXPR
4389 76983126 : && operands.last ().opcode == SSA_NAME)
4390 : {
4391 897045 : tree ops[2];
4392 897045 : if (vn_pp_nary_for_addr (operands, ops))
4393 : {
4394 575025 : vn_nary_op_insert_pieces (2, POINTER_PLUS_EXPR,
4395 575025 : TREE_TYPE (op), ops, result,
4396 575025 : VN_INFO (result)->value_id);
4397 575025 : return;
4398 : }
4399 : }
4400 :
4401 75479025 : vr1 = XOBNEW (&vn_tables_obstack, vn_reference_s);
4402 75479025 : if (TREE_CODE (result) == SSA_NAME)
4403 52176602 : vr1->value_id = VN_INFO (result)->value_id;
4404 : else
4405 23302423 : vr1->value_id = get_or_alloc_constant_value_id (result);
4406 75479025 : vr1->vuse = vuse_ssa_val (vuse);
4407 75479025 : vr1->operands = operands.copy ();
4408 75479025 : vr1->type = TREE_TYPE (op);
4409 75479025 : vr1->punned = false;
4410 75479025 : ao_ref op_ref;
4411 75479025 : ao_ref_init (&op_ref, op);
4412 75479025 : vr1->set = ao_ref_alias_set (&op_ref);
4413 75479025 : vr1->base_set = ao_ref_base_alias_set (&op_ref);
4414 : /* Specifically use an unknown extent here, we're not doing any lookup
4415 : and assume the caller didn't either (or it went VARYING). */
4416 75479025 : vr1->offset = 0;
4417 75479025 : vr1->max_size = -1;
4418 75479025 : vr1->hashcode = vn_reference_compute_hash (vr1);
4419 75479025 : vr1->result = TREE_CODE (result) == SSA_NAME ? SSA_VAL (result) : result;
4420 75479025 : vr1->result_vdef = vdef;
4421 :
4422 75479025 : slot = valid_info->references->find_slot_with_hash (vr1, vr1->hashcode,
4423 : INSERT);
4424 :
4425 : /* Because IL walking on reference lookup can end up visiting
4426 : a def that is only to be visited later in iteration order
4427 : when we are about to make an irreducible region reducible
4428 : the def can be effectively processed and its ref being inserted
4429 : by vn_reference_lookup_3 already. So we cannot assert (!*slot)
4430 : but save a lookup if we deal with already inserted refs here. */
4431 75479025 : if (*slot)
4432 : {
4433 : /* We cannot assert that we have the same value either because
4434 : when disentangling an irreducible region we may end up visiting
4435 : a use before the corresponding def. That's a missed optimization
4436 : only though. See gcc.dg/tree-ssa/pr87126.c for example. */
4437 0 : if (dump_file && (dump_flags & TDF_DETAILS)
4438 0 : && !operand_equal_p ((*slot)->result, vr1->result, 0))
4439 : {
4440 0 : fprintf (dump_file, "Keeping old value ");
4441 0 : print_generic_expr (dump_file, (*slot)->result);
4442 0 : fprintf (dump_file, " because of collision\n");
4443 : }
4444 0 : free_reference (vr1);
4445 0 : obstack_free (&vn_tables_obstack, vr1);
4446 0 : return;
4447 : }
4448 :
4449 75479025 : *slot = vr1;
4450 75479025 : vr1->next = last_inserted_ref;
4451 75479025 : last_inserted_ref = vr1;
4452 : }
4453 :
4454 : /* Insert a reference by it's pieces into the current hash table with
4455 : a value number of RESULT. Return the resulting reference
4456 : structure we created. */
4457 :
4458 : vn_reference_t
4459 1556498 : vn_reference_insert_pieces (tree vuse, alias_set_type set,
4460 : alias_set_type base_set,
4461 : poly_int64 offset, poly_int64 max_size, tree type,
4462 : vec<vn_reference_op_s> operands,
4463 : tree result, unsigned int value_id)
4464 :
4465 : {
4466 1556498 : vn_reference_s **slot;
4467 1556498 : vn_reference_t vr1;
4468 :
4469 1556498 : vr1 = XOBNEW (&vn_tables_obstack, vn_reference_s);
4470 1556498 : vr1->value_id = value_id;
4471 1556498 : vr1->vuse = vuse_ssa_val (vuse);
4472 1556498 : vr1->operands = operands;
4473 1556498 : valueize_refs (&vr1->operands);
4474 1556498 : vr1->type = type;
4475 1556498 : vr1->punned = false;
4476 1556498 : vr1->set = set;
4477 1556498 : vr1->base_set = base_set;
4478 1556498 : vr1->offset = offset;
4479 1556498 : vr1->max_size = max_size;
4480 1556498 : vr1->hashcode = vn_reference_compute_hash (vr1);
4481 1556498 : if (result && TREE_CODE (result) == SSA_NAME)
4482 361391 : result = SSA_VAL (result);
4483 1556498 : vr1->result = result;
4484 1556498 : vr1->result_vdef = NULL_TREE;
4485 :
4486 1556498 : slot = valid_info->references->find_slot_with_hash (vr1, vr1->hashcode,
4487 : INSERT);
4488 :
4489 : /* At this point we should have all the things inserted that we have
4490 : seen before, and we should never try inserting something that
4491 : already exists. */
4492 1556498 : gcc_assert (!*slot);
4493 :
4494 1556498 : *slot = vr1;
4495 1556498 : vr1->next = last_inserted_ref;
4496 1556498 : last_inserted_ref = vr1;
4497 1556498 : return vr1;
4498 : }
4499 :
4500 : /* Compute and return the hash value for nary operation VBO1. */
4501 :
4502 : hashval_t
4503 308438705 : vn_nary_op_compute_hash (const vn_nary_op_t vno1)
4504 : {
4505 308438705 : inchash::hash hstate;
4506 308438705 : unsigned i;
4507 :
4508 308438705 : if (((vno1->length == 2
4509 259519676 : && commutative_tree_code (vno1->opcode))
4510 141405459 : || (vno1->length == 3
4511 1695484 : && commutative_ternary_tree_code (vno1->opcode)))
4512 475474142 : && tree_swap_operands_p (vno1->op[0], vno1->op[1]))
4513 2466728 : std::swap (vno1->op[0], vno1->op[1]);
4514 305971977 : else if (TREE_CODE_CLASS (vno1->opcode) == tcc_comparison
4515 305971977 : && tree_swap_operands_p (vno1->op[0], vno1->op[1]))
4516 : {
4517 469625 : std::swap (vno1->op[0], vno1->op[1]);
4518 469625 : vno1->opcode = swap_tree_comparison (vno1->opcode);
4519 : }
4520 :
4521 308438705 : hstate.add_int (vno1->opcode);
4522 880571855 : for (i = 0; i < vno1->length; ++i)
4523 572133150 : inchash::add_expr (vno1->op[i], hstate);
4524 :
4525 308438705 : return hstate.end ();
4526 : }
4527 :
4528 : /* Compare nary operations VNO1 and VNO2 and return true if they are
4529 : equivalent. */
4530 :
4531 : bool
4532 975211577 : vn_nary_op_eq (const_vn_nary_op_t const vno1, const_vn_nary_op_t const vno2)
4533 : {
4534 975211577 : unsigned i;
4535 :
4536 975211577 : if (vno1->hashcode != vno2->hashcode)
4537 : return false;
4538 :
4539 51185428 : if (vno1->length != vno2->length)
4540 : return false;
4541 :
4542 51185428 : if (vno1->opcode != vno2->opcode
4543 51185428 : || !types_compatible_p (vno1->type, vno2->type))
4544 1158812 : return false;
4545 :
4546 144632766 : for (i = 0; i < vno1->length; ++i)
4547 94705042 : if (!expressions_equal_p (vno1->op[i], vno2->op[i]))
4548 : return false;
4549 :
4550 : /* BIT_INSERT_EXPR has an implicit operand as the type precision
4551 : of op1. Need to check to make sure they are the same. */
4552 49927724 : if (vno1->opcode == BIT_INSERT_EXPR
4553 541 : && TREE_CODE (vno1->op[1]) == INTEGER_CST
4554 49927827 : && TYPE_PRECISION (TREE_TYPE (vno1->op[1]))
4555 103 : != TYPE_PRECISION (TREE_TYPE (vno2->op[1])))
4556 : return false;
4557 :
4558 : return true;
4559 : }
4560 :
4561 : /* Initialize VNO from the pieces provided. */
4562 :
4563 : static void
4564 191013738 : init_vn_nary_op_from_pieces (vn_nary_op_t vno, unsigned int length,
4565 : enum tree_code code, tree type, tree *ops)
4566 : {
4567 191013738 : vno->opcode = code;
4568 191013738 : vno->length = length;
4569 191013738 : vno->type = type;
4570 4806356 : memcpy (&vno->op[0], ops, sizeof (tree) * length);
4571 0 : }
4572 :
4573 : /* Return the number of operands for a vn_nary ops structure from STMT. */
4574 :
4575 : unsigned int
4576 111314580 : vn_nary_length_from_stmt (gimple *stmt)
4577 : {
4578 111314580 : switch (gimple_assign_rhs_code (stmt))
4579 : {
4580 : case REALPART_EXPR:
4581 : case IMAGPART_EXPR:
4582 : case VIEW_CONVERT_EXPR:
4583 : return 1;
4584 :
4585 682444 : case BIT_FIELD_REF:
4586 682444 : return 3;
4587 :
4588 538617 : case CONSTRUCTOR:
4589 538617 : return CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt));
4590 :
4591 106406553 : default:
4592 106406553 : return gimple_num_ops (stmt) - 1;
4593 : }
4594 : }
4595 :
4596 : /* Initialize VNO from STMT. */
4597 :
4598 : void
4599 111314580 : init_vn_nary_op_from_stmt (vn_nary_op_t vno, gassign *stmt)
4600 : {
4601 111314580 : unsigned i;
4602 :
4603 111314580 : vno->opcode = gimple_assign_rhs_code (stmt);
4604 111314580 : vno->type = TREE_TYPE (gimple_assign_lhs (stmt));
4605 111314580 : switch (vno->opcode)
4606 : {
4607 3686966 : case REALPART_EXPR:
4608 3686966 : case IMAGPART_EXPR:
4609 3686966 : case VIEW_CONVERT_EXPR:
4610 3686966 : vno->length = 1;
4611 3686966 : vno->op[0] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
4612 3686966 : break;
4613 :
4614 682444 : case BIT_FIELD_REF:
4615 682444 : vno->length = 3;
4616 682444 : vno->op[0] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
4617 682444 : vno->op[1] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 1);
4618 682444 : vno->op[2] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 2);
4619 682444 : break;
4620 :
4621 538617 : case CONSTRUCTOR:
4622 538617 : vno->length = CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt));
4623 2134882 : for (i = 0; i < vno->length; ++i)
4624 1596265 : vno->op[i] = CONSTRUCTOR_ELT (gimple_assign_rhs1 (stmt), i)->value;
4625 : break;
4626 :
4627 106406553 : default:
4628 106406553 : gcc_checking_assert (!gimple_assign_single_p (stmt));
4629 106406553 : vno->length = gimple_num_ops (stmt) - 1;
4630 291642993 : for (i = 0; i < vno->length; ++i)
4631 185236440 : vno->op[i] = gimple_op (stmt, i + 1);
4632 : }
4633 111314580 : }
4634 :
4635 : /* Compute the hashcode for VNO and look for it in the hash table;
4636 : return the resulting value number if it exists in the hash table.
4637 : Return NULL_TREE if it does not exist in the hash table or if the
4638 : result field of the operation is NULL. VNRESULT will contain the
4639 : vn_nary_op_t from the hashtable if it exists. */
4640 :
4641 : static tree
4642 133674256 : vn_nary_op_lookup_1 (vn_nary_op_t vno, vn_nary_op_t *vnresult)
4643 : {
4644 133674256 : vn_nary_op_s **slot;
4645 :
4646 133674256 : if (vnresult)
4647 126384643 : *vnresult = NULL;
4648 :
4649 371697233 : for (unsigned i = 0; i < vno->length; ++i)
4650 238022977 : if (TREE_CODE (vno->op[i]) == SSA_NAME)
4651 168524949 : vno->op[i] = SSA_VAL (vno->op[i]);
4652 :
4653 133674256 : vno->hashcode = vn_nary_op_compute_hash (vno);
4654 133674256 : slot = valid_info->nary->find_slot_with_hash (vno, vno->hashcode, NO_INSERT);
4655 133674256 : if (!slot)
4656 : return NULL_TREE;
4657 18059144 : if (vnresult)
4658 17614192 : *vnresult = *slot;
4659 18059144 : return (*slot)->predicated_values ? NULL_TREE : (*slot)->u.result;
4660 : }
4661 :
4662 : /* Lookup a n-ary operation by its pieces and return the resulting value
4663 : number if it exists in the hash table. Return NULL_TREE if it does
4664 : not exist in the hash table or if the result field of the operation
4665 : is NULL. VNRESULT will contain the vn_nary_op_t from the hashtable
4666 : if it exists. */
4667 :
4668 : tree
4669 76057348 : vn_nary_op_lookup_pieces (unsigned int length, enum tree_code code,
4670 : tree type, tree *ops, vn_nary_op_t *vnresult)
4671 : {
4672 76057348 : vn_nary_op_t vno1 = XALLOCAVAR (struct vn_nary_op_s,
4673 : sizeof_vn_nary_op (length));
4674 76057348 : init_vn_nary_op_from_pieces (vno1, length, code, type, ops);
4675 76057348 : return vn_nary_op_lookup_1 (vno1, vnresult);
4676 : }
4677 :
4678 : /* Lookup the rhs of STMT in the current hash table, and return the resulting
4679 : value number if it exists in the hash table. Return NULL_TREE if
4680 : it does not exist in the hash table. VNRESULT will contain the
4681 : vn_nary_op_t from the hashtable if it exists. */
4682 :
4683 : tree
4684 57616908 : vn_nary_op_lookup_stmt (gimple *stmt, vn_nary_op_t *vnresult)
4685 : {
4686 57616908 : vn_nary_op_t vno1
4687 57616908 : = XALLOCAVAR (struct vn_nary_op_s,
4688 : sizeof_vn_nary_op (vn_nary_length_from_stmt (stmt)));
4689 57616908 : init_vn_nary_op_from_stmt (vno1, as_a <gassign *> (stmt));
4690 57616908 : return vn_nary_op_lookup_1 (vno1, vnresult);
4691 : }
4692 :
4693 : /* Allocate a vn_nary_op_t with LENGTH operands on STACK. */
4694 :
4695 : vn_nary_op_t
4696 173772233 : alloc_vn_nary_op_noinit (unsigned int length, struct obstack *stack)
4697 : {
4698 173772233 : return (vn_nary_op_t) obstack_alloc (stack, sizeof_vn_nary_op (length));
4699 : }
4700 :
4701 : /* Allocate and initialize a vn_nary_op_t on CURRENT_INFO's
4702 : obstack. */
4703 :
4704 : static vn_nary_op_t
4705 156179879 : alloc_vn_nary_op (unsigned int length, tree result, unsigned int value_id)
4706 : {
4707 0 : vn_nary_op_t vno1 = alloc_vn_nary_op_noinit (length, &vn_tables_obstack);
4708 :
4709 156179879 : vno1->value_id = value_id;
4710 156179879 : vno1->length = length;
4711 156179879 : vno1->predicated_values = 0;
4712 156179879 : vno1->u.result = result;
4713 :
4714 156179879 : return vno1;
4715 : }
4716 :
4717 : /* Insert VNO into TABLE. */
4718 :
4719 : static vn_nary_op_t
4720 161123073 : vn_nary_op_insert_into (vn_nary_op_t vno, vn_nary_op_table_type *table)
4721 : {
4722 161123073 : vn_nary_op_s **slot;
4723 :
4724 161123073 : gcc_assert (! vno->predicated_values
4725 : || (! vno->u.values->next
4726 : && vno->u.values->n == 1));
4727 :
4728 471484086 : for (unsigned i = 0; i < vno->length; ++i)
4729 310361013 : if (TREE_CODE (vno->op[i]) == SSA_NAME)
4730 202139678 : vno->op[i] = SSA_VAL (vno->op[i]);
4731 :
4732 161123073 : vno->hashcode = vn_nary_op_compute_hash (vno);
4733 161123073 : slot = table->find_slot_with_hash (vno, vno->hashcode, INSERT);
4734 161123073 : vno->unwind_to = *slot;
4735 161123073 : if (*slot)
4736 : {
4737 : /* Prefer non-predicated values.
4738 : ??? Only if those are constant, otherwise, with constant predicated
4739 : value, turn them into predicated values with entry-block validity
4740 : (??? but we always find the first valid result currently). */
4741 30876364 : if ((*slot)->predicated_values
4742 30106418 : && ! vno->predicated_values)
4743 : {
4744 : /* ??? We cannot remove *slot from the unwind stack list.
4745 : For the moment we deal with this by skipping not found
4746 : entries but this isn't ideal ... */
4747 86794 : *slot = vno;
4748 : /* ??? Maintain a stack of states we can unwind in
4749 : vn_nary_op_s? But how far do we unwind? In reality
4750 : we need to push change records somewhere... Or not
4751 : unwind vn_nary_op_s and linking them but instead
4752 : unwind the results "list", linking that, which also
4753 : doesn't move on hashtable resize. */
4754 : /* We can also have a ->unwind_to recording *slot there.
4755 : That way we can make u.values a fixed size array with
4756 : recording the number of entries but of course we then
4757 : have always N copies for each unwind_to-state. Or we
4758 : make sure to only ever append and each unwinding will
4759 : pop off one entry (but how to deal with predicated
4760 : replaced with non-predicated here?) */
4761 86794 : vno->next = last_inserted_nary;
4762 86794 : last_inserted_nary = vno;
4763 86794 : return vno;
4764 : }
4765 30789570 : else if (vno->predicated_values
4766 30789218 : && ! (*slot)->predicated_values)
4767 : return *slot;
4768 30019976 : else if (vno->predicated_values
4769 30019624 : && (*slot)->predicated_values)
4770 : {
4771 : /* ??? Factor this all into a insert_single_predicated_value
4772 : routine. */
4773 30019624 : gcc_assert (!vno->u.values->next && vno->u.values->n == 1);
4774 30019624 : basic_block vno_bb
4775 30019624 : = BASIC_BLOCK_FOR_FN (cfun, vno->u.values->valid_dominated_by_p[0]);
4776 30019624 : vn_pval *nval = vno->u.values;
4777 30019624 : vn_pval **next = &vno->u.values;
4778 30019624 : vn_pval *ins = NULL;
4779 30019624 : vn_pval *ins_at = NULL;
4780 : /* Find an existing value to append to. */
4781 56384144 : for (vn_pval *val = (*slot)->u.values; val; val = val->next)
4782 : {
4783 31048309 : if (expressions_equal_p (val->result, nval->result))
4784 : {
4785 : /* Limit the number of places we register a predicate
4786 : as valid. */
4787 4683789 : if (val->n > 8)
4788 138955 : return *slot;
4789 11681911 : for (unsigned i = 0; i < val->n; ++i)
4790 : {
4791 7376704 : basic_block val_bb
4792 7376704 : = BASIC_BLOCK_FOR_FN (cfun,
4793 : val->valid_dominated_by_p[i]);
4794 7376704 : if (dominated_by_p (CDI_DOMINATORS, vno_bb, val_bb))
4795 : /* Value registered with more generic predicate. */
4796 239627 : return *slot;
4797 7137077 : else if (flag_checking)
4798 : /* Shouldn't happen, we insert in RPO order. */
4799 7137077 : gcc_assert (!dominated_by_p (CDI_DOMINATORS,
4800 : val_bb, vno_bb));
4801 : }
4802 : /* Append the location. */
4803 4305207 : ins_at = val;
4804 4305207 : ins = (vn_pval *) obstack_alloc (&vn_tables_obstack,
4805 : sizeof (vn_pval)
4806 : + val->n * sizeof (int));
4807 4305207 : ins->next = NULL;
4808 4305207 : ins->result = val->result;
4809 4305207 : ins->n = val->n + 1;
4810 4305207 : memcpy (ins->valid_dominated_by_p,
4811 4305207 : val->valid_dominated_by_p,
4812 4305207 : val->n * sizeof (int));
4813 4305207 : ins->valid_dominated_by_p[val->n] = vno_bb->index;
4814 4305207 : if (dump_file && (dump_flags & TDF_DETAILS))
4815 4 : fprintf (dump_file, "Appending predicate to value.\n");
4816 : break;
4817 : }
4818 : }
4819 : /* Copy the rest of the value chain. */
4820 61205371 : for (vn_pval *val = (*slot)->u.values; val; val = val->next)
4821 : {
4822 31564329 : if (val == ins_at)
4823 : /* Replace the node we appended to. */
4824 4305207 : *next = ins;
4825 : else
4826 : {
4827 : /* Copy other predicated values. */
4828 27259122 : *next = (vn_pval *) obstack_alloc (&vn_tables_obstack,
4829 : sizeof (vn_pval)
4830 : + ((val->n-1)
4831 : * sizeof (int)));
4832 27259122 : memcpy (*next, val,
4833 27259122 : sizeof (vn_pval) + (val->n-1) * sizeof (int));
4834 27259122 : (*next)->next = NULL;
4835 : }
4836 31564329 : next = &(*next)->next;
4837 : }
4838 : /* Append the value if we didn't find it. */
4839 29641042 : if (!ins_at)
4840 25335835 : *next = nval;
4841 29641042 : *slot = vno;
4842 29641042 : vno->next = last_inserted_nary;
4843 29641042 : last_inserted_nary = vno;
4844 29641042 : return vno;
4845 : }
4846 :
4847 : /* While we do not want to insert things twice it's awkward to
4848 : avoid it in the case where visit_nary_op pattern-matches stuff
4849 : and ends up simplifying the replacement to itself. We then
4850 : get two inserts, one from visit_nary_op and one from
4851 : vn_nary_build_or_lookup.
4852 : So allow inserts with the same value number. */
4853 352 : if ((*slot)->u.result == vno->u.result)
4854 : return *slot;
4855 : }
4856 :
4857 : /* ??? There's also optimistic vs. previous committed state merging
4858 : that is problematic for the case of unwinding. */
4859 :
4860 : /* ??? We should return NULL if we do not use 'vno' and have the
4861 : caller release it. */
4862 130246709 : gcc_assert (!*slot);
4863 :
4864 130246709 : *slot = vno;
4865 130246709 : vno->next = last_inserted_nary;
4866 130246709 : last_inserted_nary = vno;
4867 130246709 : return vno;
4868 : }
4869 :
4870 : /* Insert a n-ary operation into the current hash table using it's
4871 : pieces. Return the vn_nary_op_t structure we created and put in
4872 : the hashtable. */
4873 :
4874 : vn_nary_op_t
4875 575025 : vn_nary_op_insert_pieces (unsigned int length, enum tree_code code,
4876 : tree type, tree *ops,
4877 : tree result, unsigned int value_id)
4878 : {
4879 575025 : vn_nary_op_t vno1 = alloc_vn_nary_op (length, result, value_id);
4880 575025 : init_vn_nary_op_from_pieces (vno1, length, code, type, ops);
4881 575025 : return vn_nary_op_insert_into (vno1, valid_info->nary);
4882 : }
4883 :
4884 : /* Return whether we can track a predicate valid when PRED_E is executed. */
4885 :
4886 : static bool
4887 154645035 : can_track_predicate_on_edge (edge pred_e)
4888 : {
4889 : /* ??? As we are currently recording the destination basic-block index in
4890 : vn_pval.valid_dominated_by_p and using dominance for the
4891 : validity check we cannot track predicates on all edges. */
4892 154645035 : if (single_pred_p (pred_e->dest))
4893 : return true;
4894 : /* Never record for backedges. */
4895 12277280 : if (pred_e->flags & EDGE_DFS_BACK)
4896 : return false;
4897 : /* When there's more than one predecessor we cannot track
4898 : predicate validity based on the destination block. The
4899 : exception is when all other incoming edges sources are
4900 : dominated by the destination block. */
4901 11577722 : edge_iterator ei;
4902 11577722 : edge e;
4903 19845064 : FOR_EACH_EDGE (e, ei, pred_e->dest->preds)
4904 17953706 : if (e != pred_e && ! dominated_by_p (CDI_DOMINATORS, e->src, e->dest))
4905 : return false;
4906 : return true;
4907 : }
4908 :
4909 : static vn_nary_op_t
4910 109575009 : vn_nary_op_insert_pieces_predicated (unsigned int length, enum tree_code code,
4911 : tree type, tree *ops,
4912 : tree result, unsigned int value_id,
4913 : edge pred_e)
4914 : {
4915 109575009 : if (flag_checking)
4916 109574189 : gcc_assert (can_track_predicate_on_edge (pred_e));
4917 :
4918 75350 : if (dump_file && (dump_flags & TDF_DETAILS)
4919 : /* ??? Fix dumping, but currently we only get comparisons. */
4920 109646263 : && TREE_CODE_CLASS (code) == tcc_comparison)
4921 : {
4922 71254 : fprintf (dump_file, "Recording on edge %d->%d ", pred_e->src->index,
4923 71254 : pred_e->dest->index);
4924 71254 : print_generic_expr (dump_file, ops[0], TDF_SLIM);
4925 71254 : fprintf (dump_file, " %s ", get_tree_code_name (code));
4926 71254 : print_generic_expr (dump_file, ops[1], TDF_SLIM);
4927 106510 : fprintf (dump_file, " == %s\n",
4928 71254 : integer_zerop (result) ? "false" : "true");
4929 : }
4930 109575009 : vn_nary_op_t vno1 = alloc_vn_nary_op (length, NULL_TREE, value_id);
4931 109575009 : init_vn_nary_op_from_pieces (vno1, length, code, type, ops);
4932 109575009 : vno1->predicated_values = 1;
4933 109575009 : vno1->u.values = (vn_pval *) obstack_alloc (&vn_tables_obstack,
4934 : sizeof (vn_pval));
4935 109575009 : vno1->u.values->next = NULL;
4936 109575009 : vno1->u.values->result = result;
4937 109575009 : vno1->u.values->n = 1;
4938 109575009 : vno1->u.values->valid_dominated_by_p[0] = pred_e->dest->index;
4939 109575009 : return vn_nary_op_insert_into (vno1, valid_info->nary);
4940 : }
4941 :
4942 : static bool
4943 : dominated_by_p_w_unex (basic_block bb1, basic_block bb2, bool);
4944 :
4945 : static tree
4946 1792458 : vn_nary_op_get_predicated_value (vn_nary_op_t vno, basic_block bb,
4947 : edge e = NULL)
4948 : {
4949 1792458 : if (! vno->predicated_values)
4950 0 : return vno->u.result;
4951 3732800 : for (vn_pval *val = vno->u.values; val; val = val->next)
4952 5750022 : for (unsigned i = 0; i < val->n; ++i)
4953 : {
4954 3809680 : basic_block cand
4955 3809680 : = BASIC_BLOCK_FOR_FN (cfun, val->valid_dominated_by_p[i]);
4956 : /* Do not handle backedge executability optimistically since
4957 : when figuring out whether to iterate we do not consider
4958 : changed predication.
4959 : When asking for predicated values on an edge avoid looking
4960 : at edge executability for edges forward in our iteration
4961 : as well. */
4962 3809680 : if (e && (e->flags & EDGE_DFS_BACK))
4963 : {
4964 23373 : if (dominated_by_p (CDI_DOMINATORS, bb, cand))
4965 7770 : return val->result;
4966 : }
4967 3786307 : else if (dominated_by_p_w_unex (bb, cand, false))
4968 550742 : return val->result;
4969 : }
4970 : return NULL_TREE;
4971 : }
4972 :
4973 : static tree
4974 214908 : vn_nary_op_get_predicated_value (vn_nary_op_t vno, edge e)
4975 : {
4976 0 : return vn_nary_op_get_predicated_value (vno, e->src, e);
4977 : }
4978 :
4979 : /* Insert the rhs of STMT into the current hash table with a value number of
4980 : RESULT. */
4981 :
4982 : static vn_nary_op_t
4983 46029845 : vn_nary_op_insert_stmt (gimple *stmt, tree result)
4984 : {
4985 46029845 : vn_nary_op_t vno1
4986 46029845 : = alloc_vn_nary_op (vn_nary_length_from_stmt (stmt),
4987 46029845 : result, VN_INFO (result)->value_id);
4988 46029845 : init_vn_nary_op_from_stmt (vno1, as_a <gassign *> (stmt));
4989 46029845 : return vn_nary_op_insert_into (vno1, valid_info->nary);
4990 : }
4991 :
4992 : /* Compute a hashcode for PHI operation VP1 and return it. */
4993 :
4994 : static inline hashval_t
4995 50678789 : vn_phi_compute_hash (vn_phi_t vp1)
4996 : {
4997 50678789 : inchash::hash hstate;
4998 50678789 : tree phi1op;
4999 50678789 : tree type;
5000 50678789 : edge e;
5001 50678789 : edge_iterator ei;
5002 :
5003 101357578 : hstate.add_int (EDGE_COUNT (vp1->block->preds));
5004 50678789 : switch (EDGE_COUNT (vp1->block->preds))
5005 : {
5006 : case 1:
5007 : break;
5008 43601241 : case 2:
5009 : /* When this is a PHI node subject to CSE for different blocks
5010 : avoid hashing the block index. */
5011 43601241 : if (vp1->cclhs)
5012 : break;
5013 : /* Fallthru. */
5014 34118892 : default:
5015 34118892 : hstate.add_int (vp1->block->index);
5016 : }
5017 :
5018 : /* If all PHI arguments are constants we need to distinguish
5019 : the PHI node via its type. */
5020 50678789 : type = vp1->type;
5021 50678789 : hstate.merge_hash (vn_hash_type (type));
5022 :
5023 176403520 : FOR_EACH_EDGE (e, ei, vp1->block->preds)
5024 : {
5025 : /* Don't hash backedge values they need to be handled as VN_TOP
5026 : for optimistic value-numbering. */
5027 125724731 : if (e->flags & EDGE_DFS_BACK)
5028 28036155 : continue;
5029 :
5030 97688576 : phi1op = vp1->phiargs[e->dest_idx];
5031 97688576 : if (phi1op == VN_TOP)
5032 247550 : continue;
5033 97441026 : inchash::add_expr (phi1op, hstate);
5034 : }
5035 :
5036 50678789 : return hstate.end ();
5037 : }
5038 :
5039 :
5040 : /* Return true if COND1 and COND2 represent the same condition, set
5041 : *INVERTED_P if one needs to be inverted to make it the same as
5042 : the other. */
5043 :
5044 : static bool
5045 3813923 : cond_stmts_equal_p (gcond *cond1, tree lhs1, tree rhs1,
5046 : gcond *cond2, tree lhs2, tree rhs2, bool *inverted_p)
5047 : {
5048 3813923 : enum tree_code code1 = gimple_cond_code (cond1);
5049 3813923 : enum tree_code code2 = gimple_cond_code (cond2);
5050 :
5051 3813923 : *inverted_p = false;
5052 3813923 : if (code1 == code2)
5053 : ;
5054 300523 : else if (code1 == swap_tree_comparison (code2))
5055 : std::swap (lhs2, rhs2);
5056 264554 : else if (code1 == invert_tree_comparison (code2, HONOR_NANS (lhs2)))
5057 131589 : *inverted_p = true;
5058 132965 : else if (code1 == invert_tree_comparison
5059 132965 : (swap_tree_comparison (code2), HONOR_NANS (lhs2)))
5060 : {
5061 10372 : std::swap (lhs2, rhs2);
5062 10372 : *inverted_p = true;
5063 : }
5064 : else
5065 : return false;
5066 :
5067 3691330 : return ((expressions_equal_p (lhs1, lhs2)
5068 108741 : && expressions_equal_p (rhs1, rhs2))
5069 3716372 : || (commutative_tree_code (code1)
5070 1822642 : && expressions_equal_p (lhs1, rhs2)
5071 2427 : && expressions_equal_p (rhs1, lhs2)));
5072 : }
5073 :
5074 : /* Compare two phi entries for equality, ignoring VN_TOP arguments. */
5075 :
5076 : static int
5077 40911816 : vn_phi_eq (const_vn_phi_t const vp1, const_vn_phi_t const vp2)
5078 : {
5079 40911816 : if (vp1->hashcode != vp2->hashcode)
5080 : return false;
5081 :
5082 12830979 : if (vp1->block != vp2->block)
5083 : {
5084 11464770 : if (EDGE_COUNT (vp1->block->preds) != EDGE_COUNT (vp2->block->preds))
5085 : return false;
5086 :
5087 36732731 : switch (EDGE_COUNT (vp1->block->preds))
5088 : {
5089 : case 1:
5090 : /* Single-arg PHIs are just copies. */
5091 : break;
5092 :
5093 3821590 : case 2:
5094 3821590 : {
5095 : /* Make sure both PHIs are classified as CSEable. */
5096 3821590 : if (! vp1->cclhs || ! vp2->cclhs)
5097 : return false;
5098 :
5099 : /* Rule out backedges into the PHI. */
5100 3821590 : gcc_checking_assert
5101 : (vp1->block->loop_father->header != vp1->block
5102 : && vp2->block->loop_father->header != vp2->block);
5103 :
5104 : /* If the PHI nodes do not have compatible types
5105 : they are not the same. */
5106 3821590 : if (!types_compatible_p (vp1->type, vp2->type))
5107 : return false;
5108 :
5109 : /* If the immediate dominator end in switch stmts multiple
5110 : values may end up in the same PHI arg via intermediate
5111 : CFG merges. */
5112 3813923 : basic_block idom1
5113 3813923 : = get_immediate_dominator (CDI_DOMINATORS, vp1->block);
5114 3813923 : basic_block idom2
5115 3813923 : = get_immediate_dominator (CDI_DOMINATORS, vp2->block);
5116 3813923 : gcc_checking_assert (EDGE_COUNT (idom1->succs) == 2
5117 : && EDGE_COUNT (idom2->succs) == 2);
5118 :
5119 : /* Verify the controlling stmt is the same. */
5120 7627846 : gcond *last1 = as_a <gcond *> (*gsi_last_bb (idom1));
5121 7627846 : gcond *last2 = as_a <gcond *> (*gsi_last_bb (idom2));
5122 3813923 : bool inverted_p;
5123 3813923 : if (! cond_stmts_equal_p (last1, vp1->cclhs, vp1->ccrhs,
5124 3813923 : last2, vp2->cclhs, vp2->ccrhs,
5125 : &inverted_p))
5126 : return false;
5127 :
5128 : /* Get at true/false controlled edges into the PHI. */
5129 83793 : edge te1, te2, fe1, fe2;
5130 83793 : if (! extract_true_false_controlled_edges (idom1, vp1->block,
5131 : &te1, &fe1)
5132 83793 : || ! extract_true_false_controlled_edges (idom2, vp2->block,
5133 : &te2, &fe2))
5134 36098 : return false;
5135 :
5136 : /* Swap edges if the second condition is the inverted of the
5137 : first. */
5138 47695 : if (inverted_p)
5139 2038 : std::swap (te2, fe2);
5140 :
5141 : /* Since we do not know which edge will be executed we have
5142 : to be careful when matching VN_TOP. Be conservative and
5143 : only match VN_TOP == VN_TOP for now, we could allow
5144 : VN_TOP on the not prevailing PHI though. See for example
5145 : PR102920. */
5146 47695 : if (! expressions_equal_p (vp1->phiargs[te1->dest_idx],
5147 47695 : vp2->phiargs[te2->dest_idx], false)
5148 93577 : || ! expressions_equal_p (vp1->phiargs[fe1->dest_idx],
5149 45882 : vp2->phiargs[fe2->dest_idx], false))
5150 1813 : return false;
5151 :
5152 : return true;
5153 : }
5154 :
5155 : default:
5156 : return false;
5157 : }
5158 : }
5159 :
5160 : /* If the PHI nodes do not have compatible types
5161 : they are not the same. */
5162 9009389 : if (!types_compatible_p (vp1->type, vp2->type))
5163 : return false;
5164 :
5165 : /* Any phi in the same block will have it's arguments in the
5166 : same edge order, because of how we store phi nodes. */
5167 9008275 : unsigned nargs = EDGE_COUNT (vp1->block->preds);
5168 20884000 : for (unsigned i = 0; i < nargs; ++i)
5169 : {
5170 16704915 : tree phi1op = vp1->phiargs[i];
5171 16704915 : tree phi2op = vp2->phiargs[i];
5172 16704915 : if (phi1op == phi2op)
5173 11779928 : continue;
5174 4924987 : if (!expressions_equal_p (phi1op, phi2op, false))
5175 : return false;
5176 : }
5177 :
5178 : return true;
5179 : }
5180 :
5181 : /* Lookup PHI in the current hash table, and return the resulting
5182 : value number if it exists in the hash table. Return NULL_TREE if
5183 : it does not exist in the hash table. */
5184 :
5185 : static tree
5186 27761709 : vn_phi_lookup (gimple *phi, bool backedges_varying_p)
5187 : {
5188 27761709 : vn_phi_s **slot;
5189 27761709 : struct vn_phi_s *vp1;
5190 27761709 : edge e;
5191 27761709 : edge_iterator ei;
5192 :
5193 27761709 : vp1 = XALLOCAVAR (struct vn_phi_s,
5194 : sizeof (struct vn_phi_s)
5195 : + (gimple_phi_num_args (phi) - 1) * sizeof (tree));
5196 :
5197 : /* Canonicalize the SSA_NAME's to their value number. */
5198 95928591 : FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
5199 : {
5200 68166882 : tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
5201 68166882 : if (TREE_CODE (def) == SSA_NAME
5202 56778786 : && (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK)))
5203 : {
5204 54228478 : if (!virtual_operand_p (def)
5205 54228478 : && ssa_undefined_value_p (def, false))
5206 137440 : def = VN_TOP;
5207 : else
5208 54091038 : def = SSA_VAL (def);
5209 : }
5210 68166882 : vp1->phiargs[e->dest_idx] = def;
5211 : }
5212 27761709 : vp1->type = TREE_TYPE (gimple_phi_result (phi));
5213 27761709 : vp1->block = gimple_bb (phi);
5214 : /* Extract values of the controlling condition. */
5215 27761709 : vp1->cclhs = NULL_TREE;
5216 27761709 : vp1->ccrhs = NULL_TREE;
5217 27761709 : if (EDGE_COUNT (vp1->block->preds) == 2
5218 27761709 : && vp1->block->loop_father->header != vp1->block)
5219 : {
5220 8742884 : basic_block idom1 = get_immediate_dominator (CDI_DOMINATORS, vp1->block);
5221 8742884 : if (EDGE_COUNT (idom1->succs) == 2)
5222 17395292 : if (gcond *last1 = safe_dyn_cast <gcond *> (*gsi_last_bb (idom1)))
5223 : {
5224 : /* ??? We want to use SSA_VAL here. But possibly not
5225 : allow VN_TOP. */
5226 8461519 : vp1->cclhs = vn_valueize (gimple_cond_lhs (last1));
5227 8461519 : vp1->ccrhs = vn_valueize (gimple_cond_rhs (last1));
5228 : }
5229 : }
5230 27761709 : vp1->hashcode = vn_phi_compute_hash (vp1);
5231 27761709 : slot = valid_info->phis->find_slot_with_hash (vp1, vp1->hashcode, NO_INSERT);
5232 27761709 : if (!slot)
5233 : return NULL_TREE;
5234 4224967 : return (*slot)->result;
5235 : }
5236 :
5237 : /* Insert PHI into the current hash table with a value number of
5238 : RESULT. */
5239 :
5240 : static vn_phi_t
5241 22917080 : vn_phi_insert (gimple *phi, tree result, bool backedges_varying_p)
5242 : {
5243 22917080 : vn_phi_s **slot;
5244 22917080 : vn_phi_t vp1 = (vn_phi_t) obstack_alloc (&vn_tables_obstack,
5245 : sizeof (vn_phi_s)
5246 : + ((gimple_phi_num_args (phi) - 1)
5247 : * sizeof (tree)));
5248 22917080 : edge e;
5249 22917080 : edge_iterator ei;
5250 :
5251 : /* Canonicalize the SSA_NAME's to their value number. */
5252 80474929 : FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
5253 : {
5254 57557849 : tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
5255 57557849 : if (TREE_CODE (def) == SSA_NAME
5256 47264592 : && (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK)))
5257 : {
5258 44714718 : if (!virtual_operand_p (def)
5259 44714718 : && ssa_undefined_value_p (def, false))
5260 110354 : def = VN_TOP;
5261 : else
5262 44604364 : def = SSA_VAL (def);
5263 : }
5264 57557849 : vp1->phiargs[e->dest_idx] = def;
5265 : }
5266 22917080 : vp1->value_id = VN_INFO (result)->value_id;
5267 22917080 : vp1->type = TREE_TYPE (gimple_phi_result (phi));
5268 22917080 : vp1->block = gimple_bb (phi);
5269 : /* Extract values of the controlling condition. */
5270 22917080 : vp1->cclhs = NULL_TREE;
5271 22917080 : vp1->ccrhs = NULL_TREE;
5272 22917080 : if (EDGE_COUNT (vp1->block->preds) == 2
5273 22917080 : && vp1->block->loop_father->header != vp1->block)
5274 : {
5275 8374964 : basic_block idom1 = get_immediate_dominator (CDI_DOMINATORS, vp1->block);
5276 8374964 : if (EDGE_COUNT (idom1->succs) == 2)
5277 16663192 : if (gcond *last1 = safe_dyn_cast <gcond *> (*gsi_last_bb (idom1)))
5278 : {
5279 : /* ??? We want to use SSA_VAL here. But possibly not
5280 : allow VN_TOP. */
5281 8098378 : vp1->cclhs = vn_valueize (gimple_cond_lhs (last1));
5282 8098378 : vp1->ccrhs = vn_valueize (gimple_cond_rhs (last1));
5283 : }
5284 : }
5285 22917080 : vp1->result = result;
5286 22917080 : vp1->hashcode = vn_phi_compute_hash (vp1);
5287 :
5288 22917080 : slot = valid_info->phis->find_slot_with_hash (vp1, vp1->hashcode, INSERT);
5289 22917080 : gcc_assert (!*slot);
5290 :
5291 22917080 : *slot = vp1;
5292 22917080 : vp1->next = last_inserted_phi;
5293 22917080 : last_inserted_phi = vp1;
5294 22917080 : return vp1;
5295 : }
5296 :
5297 :
5298 : /* Return true if BB1 is dominated by BB2 taking into account edges
5299 : that are not executable. When ALLOW_BACK is false consider not
5300 : executable backedges as executable. */
5301 :
5302 : static bool
5303 74349632 : dominated_by_p_w_unex (basic_block bb1, basic_block bb2, bool allow_back)
5304 : {
5305 74349632 : edge_iterator ei;
5306 74349632 : edge e;
5307 :
5308 74349632 : if (dominated_by_p (CDI_DOMINATORS, bb1, bb2))
5309 : return true;
5310 :
5311 : /* Before iterating we'd like to know if there exists a
5312 : (executable) path from bb2 to bb1 at all, if not we can
5313 : directly return false. For now simply iterate once. */
5314 :
5315 : /* Iterate to the single executable bb1 predecessor. */
5316 21951424 : if (EDGE_COUNT (bb1->preds) > 1)
5317 : {
5318 3030140 : edge prede = NULL;
5319 6634969 : FOR_EACH_EDGE (e, ei, bb1->preds)
5320 6182968 : if ((e->flags & EDGE_EXECUTABLE)
5321 654531 : || (!allow_back && (e->flags & EDGE_DFS_BACK)))
5322 : {
5323 5608279 : if (prede)
5324 : {
5325 : prede = NULL;
5326 : break;
5327 : }
5328 : prede = e;
5329 : }
5330 3030140 : if (prede)
5331 : {
5332 452001 : bb1 = prede->src;
5333 :
5334 : /* Re-do the dominance check with changed bb1. */
5335 452001 : if (dominated_by_p (CDI_DOMINATORS, bb1, bb2))
5336 : return true;
5337 : }
5338 : }
5339 :
5340 : /* Iterate to the single executable bb2 successor. */
5341 21694275 : if (EDGE_COUNT (bb2->succs) > 1)
5342 : {
5343 6798955 : edge succe = NULL;
5344 13767158 : FOR_EACH_EDGE (e, ei, bb2->succs)
5345 13598163 : if ((e->flags & EDGE_EXECUTABLE)
5346 208218 : || (!allow_back && (e->flags & EDGE_DFS_BACK)))
5347 : {
5348 13389986 : if (succe)
5349 : {
5350 : succe = NULL;
5351 : break;
5352 : }
5353 : succe = e;
5354 : }
5355 6798955 : if (succe
5356 : /* Limit the number of edges we check, we should bring in
5357 : context from the iteration and compute the single
5358 : executable incoming edge when visiting a block. */
5359 6798955 : && EDGE_COUNT (succe->dest->preds) < 8)
5360 : {
5361 : /* Verify the reached block is only reached through succe.
5362 : If there is only one edge we can spare us the dominator
5363 : check and iterate directly. */
5364 129497 : if (EDGE_COUNT (succe->dest->preds) > 1)
5365 : {
5366 54831 : FOR_EACH_EDGE (e, ei, succe->dest->preds)
5367 42461 : if (e != succe
5368 27562 : && ((e->flags & EDGE_EXECUTABLE)
5369 18215 : || (!allow_back && (e->flags & EDGE_DFS_BACK))))
5370 : {
5371 : succe = NULL;
5372 : break;
5373 : }
5374 : }
5375 129497 : if (succe)
5376 : {
5377 120141 : bb2 = succe->dest;
5378 :
5379 : /* Re-do the dominance check with changed bb2. */
5380 120141 : if (dominated_by_p (CDI_DOMINATORS, bb1, bb2))
5381 : return true;
5382 : }
5383 : }
5384 : }
5385 : /* Iterate to the single successor of bb2 with only a single executable
5386 : incoming edge. */
5387 14895320 : else if (EDGE_COUNT (bb2->succs) == 1
5388 14323850 : && EDGE_COUNT (single_succ (bb2)->preds) > 1
5389 : /* Limit the number of edges we check, we should bring in
5390 : context from the iteration and compute the single
5391 : executable incoming edge when visiting a block. */
5392 28960409 : && EDGE_COUNT (single_succ (bb2)->preds) < 8)
5393 : {
5394 5084413 : edge prede = NULL;
5395 11499171 : FOR_EACH_EDGE (e, ei, single_succ (bb2)->preds)
5396 10914379 : if ((e->flags & EDGE_EXECUTABLE)
5397 1382167 : || (!allow_back && (e->flags & EDGE_DFS_BACK)))
5398 : {
5399 9536633 : if (prede)
5400 : {
5401 : prede = NULL;
5402 : break;
5403 : }
5404 : prede = e;
5405 : }
5406 : /* We might actually get to a query with BB2 not visited yet when
5407 : we're querying for a predicated value. */
5408 5084413 : if (prede && prede->src == bb2)
5409 : {
5410 522753 : bb2 = prede->dest;
5411 :
5412 : /* Re-do the dominance check with changed bb2. */
5413 522753 : if (dominated_by_p (CDI_DOMINATORS, bb1, bb2))
5414 : return true;
5415 : }
5416 : }
5417 :
5418 : /* We could now iterate updating bb1 / bb2. */
5419 : return false;
5420 : }
5421 :
5422 : /* Set the value number of FROM to TO, return true if it has changed
5423 : as a result. */
5424 :
5425 : static inline bool
5426 208864807 : set_ssa_val_to (tree from, tree to)
5427 : {
5428 208864807 : vn_ssa_aux_t from_info = VN_INFO (from);
5429 208864807 : tree currval = from_info->valnum; // SSA_VAL (from)
5430 208864807 : poly_int64 toff, coff;
5431 208864807 : bool curr_undefined = false;
5432 208864807 : bool curr_invariant = false;
5433 :
5434 : /* The only thing we allow as value numbers are ssa_names
5435 : and invariants. So assert that here. We don't allow VN_TOP
5436 : as visiting a stmt should produce a value-number other than
5437 : that.
5438 : ??? Still VN_TOP can happen for unreachable code, so force
5439 : it to varying in that case. Not all code is prepared to
5440 : get VN_TOP on valueization. */
5441 208864807 : if (to == VN_TOP)
5442 : {
5443 : /* ??? When iterating and visiting PHI <undef, backedge-value>
5444 : for the first time we rightfully get VN_TOP and we need to
5445 : preserve that to optimize for example gcc.dg/tree-ssa/ssa-sccvn-2.c.
5446 : With SCCVN we were simply lucky we iterated the other PHI
5447 : cycles first and thus visited the backedge-value DEF. */
5448 0 : if (currval == VN_TOP)
5449 0 : goto set_and_exit;
5450 0 : if (dump_file && (dump_flags & TDF_DETAILS))
5451 0 : fprintf (dump_file, "Forcing value number to varying on "
5452 : "receiving VN_TOP\n");
5453 : to = from;
5454 : }
5455 :
5456 208864807 : gcc_checking_assert (to != NULL_TREE
5457 : && ((TREE_CODE (to) == SSA_NAME
5458 : && (to == from || SSA_VAL (to) == to))
5459 : || is_gimple_min_invariant (to)));
5460 :
5461 208864807 : if (from != to)
5462 : {
5463 33346830 : if (currval == from)
5464 : {
5465 13454 : if (dump_file && (dump_flags & TDF_DETAILS))
5466 : {
5467 0 : fprintf (dump_file, "Not changing value number of ");
5468 0 : print_generic_expr (dump_file, from);
5469 0 : fprintf (dump_file, " from VARYING to ");
5470 0 : print_generic_expr (dump_file, to);
5471 0 : fprintf (dump_file, "\n");
5472 : }
5473 13454 : return false;
5474 : }
5475 33333376 : curr_invariant = is_gimple_min_invariant (currval);
5476 66666752 : curr_undefined = (TREE_CODE (currval) == SSA_NAME
5477 3904776 : && !virtual_operand_p (currval)
5478 37007735 : && ssa_undefined_value_p (currval, false));
5479 33333376 : if (currval != VN_TOP
5480 : && !curr_invariant
5481 5443046 : && !curr_undefined
5482 37224953 : && is_gimple_min_invariant (to))
5483 : {
5484 220 : if (dump_file && (dump_flags & TDF_DETAILS))
5485 : {
5486 0 : fprintf (dump_file, "Forcing VARYING instead of changing "
5487 : "value number of ");
5488 0 : print_generic_expr (dump_file, from);
5489 0 : fprintf (dump_file, " from ");
5490 0 : print_generic_expr (dump_file, currval);
5491 0 : fprintf (dump_file, " (non-constant) to ");
5492 0 : print_generic_expr (dump_file, to);
5493 0 : fprintf (dump_file, " (constant)\n");
5494 : }
5495 : to = from;
5496 : }
5497 33333156 : else if (currval != VN_TOP
5498 5442826 : && !curr_undefined
5499 5429627 : && TREE_CODE (to) == SSA_NAME
5500 4573709 : && !virtual_operand_p (to)
5501 37676448 : && ssa_undefined_value_p (to, false))
5502 : {
5503 6 : if (dump_file && (dump_flags & TDF_DETAILS))
5504 : {
5505 0 : fprintf (dump_file, "Forcing VARYING instead of changing "
5506 : "value number of ");
5507 0 : print_generic_expr (dump_file, from);
5508 0 : fprintf (dump_file, " from ");
5509 0 : print_generic_expr (dump_file, currval);
5510 0 : fprintf (dump_file, " (non-undefined) to ");
5511 0 : print_generic_expr (dump_file, to);
5512 0 : fprintf (dump_file, " (undefined)\n");
5513 : }
5514 : to = from;
5515 : }
5516 33333150 : else if (TREE_CODE (to) == SSA_NAME
5517 33333150 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (to))
5518 : to = from;
5519 : }
5520 :
5521 175517977 : set_and_exit:
5522 208851353 : if (dump_file && (dump_flags & TDF_DETAILS))
5523 : {
5524 399101 : fprintf (dump_file, "Setting value number of ");
5525 399101 : print_generic_expr (dump_file, from);
5526 399101 : fprintf (dump_file, " to ");
5527 399101 : print_generic_expr (dump_file, to);
5528 : }
5529 :
5530 208851353 : if (currval != to
5531 170524095 : && !operand_equal_p (currval, to, 0)
5532 : /* Different undefined SSA names are not actually different. See
5533 : PR82320 for a testcase were we'd otherwise not terminate iteration. */
5534 170454828 : && !(curr_undefined
5535 3425 : && TREE_CODE (to) == SSA_NAME
5536 608 : && !virtual_operand_p (to)
5537 608 : && ssa_undefined_value_p (to, false))
5538 : /* ??? For addresses involving volatile objects or types operand_equal_p
5539 : does not reliably detect ADDR_EXPRs as equal. We know we are only
5540 : getting invariant gimple addresses here, so can use
5541 : get_addr_base_and_unit_offset to do this comparison. */
5542 379305541 : && !(TREE_CODE (currval) == ADDR_EXPR
5543 467421 : && TREE_CODE (to) == ADDR_EXPR
5544 12 : && (get_addr_base_and_unit_offset (TREE_OPERAND (currval, 0), &coff)
5545 6 : == get_addr_base_and_unit_offset (TREE_OPERAND (to, 0), &toff))
5546 6 : && known_eq (coff, toff)))
5547 : {
5548 170454182 : if (to != from
5549 28913902 : && currval != VN_TOP
5550 1027161 : && !curr_undefined
5551 : /* We do not want to allow lattice transitions from one value
5552 : to another since that may lead to not terminating iteration
5553 : (see PR95049). Since there's no convenient way to check
5554 : for the allowed transition of VAL -> PHI (loop entry value,
5555 : same on two PHIs, to same PHI result) we restrict the check
5556 : to invariants. */
5557 1027161 : && curr_invariant
5558 171136528 : && is_gimple_min_invariant (to))
5559 : {
5560 0 : if (dump_file && (dump_flags & TDF_DETAILS))
5561 0 : fprintf (dump_file, " forced VARYING");
5562 : to = from;
5563 : }
5564 170454182 : if (dump_file && (dump_flags & TDF_DETAILS))
5565 398785 : fprintf (dump_file, " (changed)\n");
5566 170454182 : from_info->valnum = to;
5567 170454182 : return true;
5568 : }
5569 38397171 : if (dump_file && (dump_flags & TDF_DETAILS))
5570 316 : fprintf (dump_file, "\n");
5571 : return false;
5572 : }
5573 :
5574 : /* Set all definitions in STMT to value number to themselves.
5575 : Return true if a value number changed. */
5576 :
5577 : static bool
5578 304260206 : defs_to_varying (gimple *stmt)
5579 : {
5580 304260206 : bool changed = false;
5581 304260206 : ssa_op_iter iter;
5582 304260206 : def_operand_p defp;
5583 :
5584 334570186 : FOR_EACH_SSA_DEF_OPERAND (defp, stmt, iter, SSA_OP_ALL_DEFS)
5585 : {
5586 30309980 : tree def = DEF_FROM_PTR (defp);
5587 30309980 : changed |= set_ssa_val_to (def, def);
5588 : }
5589 304260206 : return changed;
5590 : }
5591 :
5592 : /* Visit a copy between LHS and RHS, return true if the value number
5593 : changed. */
5594 :
5595 : static bool
5596 8149758 : visit_copy (tree lhs, tree rhs)
5597 : {
5598 : /* Valueize. */
5599 8149758 : rhs = SSA_VAL (rhs);
5600 :
5601 8149758 : return set_ssa_val_to (lhs, rhs);
5602 : }
5603 :
5604 : /* Lookup a value for OP in type WIDE_TYPE where the value in type of OP
5605 : is the same. */
5606 :
5607 : static tree
5608 2486417 : valueized_wider_op (tree wide_type, tree op, bool allow_truncate)
5609 : {
5610 2486417 : if (TREE_CODE (op) == SSA_NAME)
5611 2184636 : op = vn_valueize (op);
5612 :
5613 : /* Either we have the op widened available. */
5614 2486417 : tree ops[3] = {};
5615 2486417 : ops[0] = op;
5616 2486417 : tree tem = vn_nary_op_lookup_pieces (1, NOP_EXPR,
5617 : wide_type, ops, NULL);
5618 2486417 : if (tem)
5619 : return tem;
5620 :
5621 : /* Or the op is truncated from some existing value. */
5622 2195840 : if (allow_truncate && TREE_CODE (op) == SSA_NAME)
5623 : {
5624 555154 : gimple *def = SSA_NAME_DEF_STMT (op);
5625 555154 : if (is_gimple_assign (def)
5626 555154 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
5627 : {
5628 297838 : tem = gimple_assign_rhs1 (def);
5629 297838 : if (useless_type_conversion_p (wide_type, TREE_TYPE (tem)))
5630 : {
5631 202668 : if (TREE_CODE (tem) == SSA_NAME)
5632 202668 : tem = vn_valueize (tem);
5633 202668 : return tem;
5634 : }
5635 : }
5636 : }
5637 :
5638 : /* For constants simply extend it. */
5639 1993172 : if (TREE_CODE (op) == INTEGER_CST)
5640 335123 : return wide_int_to_tree (wide_type, wi::to_widest (op));
5641 :
5642 : return NULL_TREE;
5643 : }
5644 :
5645 : /* Visit a nary operator RHS, value number it, and return true if the
5646 : value number of LHS has changed as a result. */
5647 :
5648 : static bool
5649 49840870 : visit_nary_op (tree lhs, gassign *stmt)
5650 : {
5651 49840870 : vn_nary_op_t vnresult;
5652 49840870 : tree result = vn_nary_op_lookup_stmt (stmt, &vnresult);
5653 49840870 : if (! result && vnresult)
5654 156775 : result = vn_nary_op_get_predicated_value (vnresult, gimple_bb (stmt));
5655 46101498 : if (result)
5656 3809353 : return set_ssa_val_to (lhs, result);
5657 :
5658 : /* Do some special pattern matching for redundancies of operations
5659 : in different types. */
5660 46031517 : enum tree_code code = gimple_assign_rhs_code (stmt);
5661 46031517 : tree type = TREE_TYPE (lhs);
5662 46031517 : tree rhs1 = gimple_assign_rhs1 (stmt);
5663 46031517 : switch (code)
5664 : {
5665 10172669 : CASE_CONVERT:
5666 : /* Match arithmetic done in a different type where we can easily
5667 : substitute the result from some earlier sign-changed or widened
5668 : operation. */
5669 10172669 : if (INTEGRAL_TYPE_P (type)
5670 9114318 : && TREE_CODE (rhs1) == SSA_NAME
5671 : /* We only handle sign-changes, zero-extension -> & mask or
5672 : sign-extension if we know the inner operation doesn't
5673 : overflow. */
5674 19049096 : && (((TYPE_UNSIGNED (TREE_TYPE (rhs1))
5675 5367791 : || (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
5676 5367014 : && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (rhs1))))
5677 8147849 : && TYPE_PRECISION (type) > TYPE_PRECISION (TREE_TYPE (rhs1)))
5678 6027117 : || TYPE_PRECISION (type) == TYPE_PRECISION (TREE_TYPE (rhs1))))
5679 : {
5680 7767900 : gassign *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (rhs1));
5681 5685364 : if (def
5682 5685364 : && (gimple_assign_rhs_code (def) == PLUS_EXPR
5683 4453268 : || gimple_assign_rhs_code (def) == MINUS_EXPR
5684 4299317 : || gimple_assign_rhs_code (def) == MULT_EXPR))
5685 : {
5686 2004103 : tree ops[3] = {};
5687 : /* When requiring a sign-extension we cannot model a
5688 : previous truncation with a single op so don't bother. */
5689 2004103 : bool allow_truncate = TYPE_UNSIGNED (TREE_TYPE (rhs1));
5690 : /* Either we have the op widened available. */
5691 2004103 : ops[0] = valueized_wider_op (type, gimple_assign_rhs1 (def),
5692 : allow_truncate);
5693 2004103 : if (ops[0])
5694 964628 : ops[1] = valueized_wider_op (type, gimple_assign_rhs2 (def),
5695 : allow_truncate);
5696 2004103 : if (ops[0] && ops[1])
5697 : {
5698 346054 : ops[0] = vn_nary_op_lookup_pieces
5699 346054 : (2, gimple_assign_rhs_code (def), type, ops, NULL);
5700 : /* We have wider operation available. */
5701 346054 : if (ops[0]
5702 : /* If the leader is a wrapping operation we can
5703 : insert it for code hoisting w/o introducing
5704 : undefined overflow. If it is not it has to
5705 : be available. See PR86554. */
5706 346054 : && (TYPE_OVERFLOW_WRAPS (TREE_TYPE (ops[0]))
5707 1849 : || (rpo_avail && vn_context_bb
5708 1849 : && rpo_avail->eliminate_avail (vn_context_bb,
5709 : ops[0]))))
5710 : {
5711 9759 : unsigned lhs_prec = TYPE_PRECISION (type);
5712 9759 : unsigned rhs_prec = TYPE_PRECISION (TREE_TYPE (rhs1));
5713 9759 : if (lhs_prec == rhs_prec
5714 9759 : || (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
5715 1804 : && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (rhs1))))
5716 : {
5717 9164 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5718 9164 : NOP_EXPR, type, ops[0]);
5719 9164 : result = vn_nary_build_or_lookup (&match_op);
5720 9164 : if (result)
5721 : {
5722 9164 : bool changed = set_ssa_val_to (lhs, result);
5723 9164 : if (TREE_CODE (result) == SSA_NAME)
5724 9164 : vn_nary_op_insert_stmt (stmt, result);
5725 9164 : return changed;
5726 : }
5727 : }
5728 : else
5729 : {
5730 595 : tree mask = wide_int_to_tree
5731 595 : (type, wi::mask (rhs_prec, false, lhs_prec));
5732 595 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5733 595 : BIT_AND_EXPR,
5734 595 : TREE_TYPE (lhs),
5735 595 : ops[0], mask);
5736 595 : result = vn_nary_build_or_lookup (&match_op);
5737 595 : if (result)
5738 : {
5739 595 : bool changed = set_ssa_val_to (lhs, result);
5740 595 : if (TREE_CODE (result) == SSA_NAME)
5741 595 : vn_nary_op_insert_stmt (stmt, result);
5742 595 : return changed;
5743 : }
5744 : }
5745 : }
5746 : }
5747 : }
5748 : }
5749 : break;
5750 1527823 : case BIT_AND_EXPR:
5751 1527823 : if (INTEGRAL_TYPE_P (type)
5752 1486625 : && TREE_CODE (rhs1) == SSA_NAME
5753 1486625 : && TREE_CODE (gimple_assign_rhs2 (stmt)) == INTEGER_CST
5754 902708 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)
5755 902590 : && default_vn_walk_kind != VN_NOWALK
5756 : && CHAR_BIT == 8
5757 : && BITS_PER_UNIT == 8
5758 : && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
5759 902381 : && TYPE_PRECISION (type) <= vn_walk_cb_data::bufsize * BITS_PER_UNIT
5760 902379 : && !integer_all_onesp (gimple_assign_rhs2 (stmt))
5761 2430202 : && !integer_zerop (gimple_assign_rhs2 (stmt)))
5762 : {
5763 902379 : gassign *ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (rhs1));
5764 663123 : if (ass
5765 663123 : && !gimple_has_volatile_ops (ass)
5766 661661 : && vn_get_stmt_kind (ass) == VN_REFERENCE)
5767 : {
5768 303292 : tree last_vuse = gimple_vuse (ass);
5769 303292 : tree op = gimple_assign_rhs1 (ass);
5770 909876 : tree result = vn_reference_lookup (op, gimple_vuse (ass),
5771 : default_vn_walk_kind,
5772 : NULL, true, &last_vuse,
5773 : gimple_assign_rhs2 (stmt));
5774 303292 : if (result
5775 303751 : && useless_type_conversion_p (TREE_TYPE (result),
5776 459 : TREE_TYPE (op)))
5777 459 : return set_ssa_val_to (lhs, result);
5778 : }
5779 : }
5780 : break;
5781 277112 : case BIT_FIELD_REF:
5782 277112 : if (TREE_CODE (TREE_OPERAND (rhs1, 0)) == SSA_NAME)
5783 : {
5784 277084 : tree op0 = vn_valueize (TREE_OPERAND (rhs1, 0));
5785 277084 : gassign *ass;
5786 277084 : if (TREE_CODE (op0) == SSA_NAME
5787 277084 : && (ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op0)))
5788 232580 : && !gimple_has_volatile_ops (ass)
5789 509581 : && vn_get_stmt_kind (ass) == VN_REFERENCE)
5790 : {
5791 97427 : tree last_vuse = gimple_vuse (ass);
5792 97427 : tree op = gimple_assign_rhs1 (ass);
5793 : /* Avoid building invalid and unexpected refs. */
5794 97427 : if (TREE_CODE (op) != TARGET_MEM_REF
5795 : && TREE_CODE (op) != BIT_FIELD_REF
5796 : && TREE_CODE (op) != REALPART_EXPR
5797 : && TREE_CODE (op) != IMAGPART_EXPR)
5798 : {
5799 90111 : tree op = build3 (BIT_FIELD_REF, TREE_TYPE (rhs1),
5800 : gimple_assign_rhs1 (ass),
5801 90111 : TREE_OPERAND (rhs1, 1),
5802 90111 : TREE_OPERAND (rhs1, 2));
5803 180222 : tree result = vn_reference_lookup (op, gimple_vuse (ass),
5804 : default_vn_walk_kind,
5805 : NULL, true, &last_vuse);
5806 90111 : if (result
5807 90111 : && useless_type_conversion_p (type, TREE_TYPE (result)))
5808 1521 : return set_ssa_val_to (lhs, result);
5809 88911 : else if (result
5810 321 : && TYPE_SIZE (type)
5811 321 : && TYPE_SIZE (TREE_TYPE (result))
5812 89232 : && operand_equal_p (TYPE_SIZE (type),
5813 321 : TYPE_SIZE (TREE_TYPE (result))))
5814 : {
5815 321 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5816 321 : VIEW_CONVERT_EXPR,
5817 321 : type, result);
5818 321 : result = vn_nary_build_or_lookup (&match_op);
5819 321 : if (result)
5820 : {
5821 321 : bool changed = set_ssa_val_to (lhs, result);
5822 321 : if (TREE_CODE (result) == SSA_NAME)
5823 309 : vn_nary_op_insert_stmt (stmt, result);
5824 321 : return changed;
5825 : }
5826 : }
5827 : }
5828 : }
5829 : }
5830 : break;
5831 344445 : case TRUNC_DIV_EXPR:
5832 344445 : if (TYPE_UNSIGNED (type))
5833 : break;
5834 : /* Fallthru. */
5835 5576311 : case RDIV_EXPR:
5836 5576311 : case MULT_EXPR:
5837 : /* Match up ([-]a){/,*}([-])b with v=a{/,*}b, replacing it with -v. */
5838 5576311 : if (! HONOR_SIGN_DEPENDENT_ROUNDING (type))
5839 : {
5840 5575401 : tree rhs[2];
5841 5575401 : rhs[0] = rhs1;
5842 5575401 : rhs[1] = gimple_assign_rhs2 (stmt);
5843 16719368 : for (unsigned i = 0; i <= 1; ++i)
5844 : {
5845 11149641 : unsigned j = i == 0 ? 1 : 0;
5846 11149641 : tree ops[2];
5847 11149641 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5848 11149641 : NEGATE_EXPR, type, rhs[i]);
5849 11149641 : ops[i] = vn_nary_build_or_lookup_1 (&match_op, false, true);
5850 11149641 : ops[j] = rhs[j];
5851 11149641 : if (ops[i]
5852 11149641 : && (ops[0] = vn_nary_op_lookup_pieces (2, code,
5853 : type, ops, NULL)))
5854 : {
5855 5674 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5856 5674 : NEGATE_EXPR, type, ops[0]);
5857 5674 : result = vn_nary_build_or_lookup_1 (&match_op, true, false);
5858 5674 : if (result)
5859 : {
5860 5674 : bool changed = set_ssa_val_to (lhs, result);
5861 5674 : if (TREE_CODE (result) == SSA_NAME)
5862 5674 : vn_nary_op_insert_stmt (stmt, result);
5863 5674 : return changed;
5864 : }
5865 : }
5866 : }
5867 : }
5868 : break;
5869 369875 : case LSHIFT_EXPR:
5870 : /* For X << C, use the value number of X * (1 << C). */
5871 369875 : if (INTEGRAL_TYPE_P (type)
5872 353887 : && TYPE_OVERFLOW_WRAPS (type)
5873 558420 : && !TYPE_SATURATING (type))
5874 : {
5875 188545 : tree rhs2 = gimple_assign_rhs2 (stmt);
5876 188545 : if (TREE_CODE (rhs2) == INTEGER_CST
5877 109234 : && tree_fits_uhwi_p (rhs2)
5878 297779 : && tree_to_uhwi (rhs2) < TYPE_PRECISION (type))
5879 : {
5880 109234 : wide_int w = wi::set_bit_in_zero (tree_to_uhwi (rhs2),
5881 109234 : TYPE_PRECISION (type));
5882 218468 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5883 109234 : MULT_EXPR, type, rhs1,
5884 109234 : wide_int_to_tree (type, w));
5885 109234 : result = vn_nary_build_or_lookup (&match_op);
5886 109234 : if (result)
5887 : {
5888 109234 : bool changed = set_ssa_val_to (lhs, result);
5889 109234 : if (TREE_CODE (result) == SSA_NAME)
5890 109233 : vn_nary_op_insert_stmt (stmt, result);
5891 109234 : return changed;
5892 : }
5893 109234 : }
5894 : }
5895 : break;
5896 : default:
5897 : break;
5898 : }
5899 :
5900 45904870 : bool changed = set_ssa_val_to (lhs, lhs);
5901 45904870 : vn_nary_op_insert_stmt (stmt, lhs);
5902 45904870 : return changed;
5903 : }
5904 :
5905 : /* Visit a call STMT storing into LHS. Return true if the value number
5906 : of the LHS has changed as a result. */
5907 :
5908 : static bool
5909 8784272 : visit_reference_op_call (tree lhs, gcall *stmt)
5910 : {
5911 8784272 : bool changed = false;
5912 8784272 : struct vn_reference_s vr1;
5913 8784272 : vn_reference_t vnresult = NULL;
5914 8784272 : tree vdef = gimple_vdef (stmt);
5915 8784272 : modref_summary *summary;
5916 :
5917 : /* Non-ssa lhs is handled in copy_reference_ops_from_call. */
5918 8784272 : if (lhs && TREE_CODE (lhs) != SSA_NAME)
5919 4692514 : lhs = NULL_TREE;
5920 :
5921 8784272 : vn_reference_lookup_call (stmt, &vnresult, &vr1);
5922 :
5923 : /* If the lookup did not succeed for pure functions try to use
5924 : modref info to find a candidate to CSE to. */
5925 8784272 : const unsigned accesses_limit = 8;
5926 8784272 : if (!vnresult
5927 8093292 : && !vdef
5928 8093292 : && lhs
5929 2834938 : && gimple_vuse (stmt)
5930 10372402 : && (((summary = get_modref_function_summary (stmt, NULL))
5931 228259 : && !summary->global_memory_read
5932 93216 : && summary->load_accesses < accesses_limit)
5933 1495258 : || gimple_call_flags (stmt) & ECF_CONST))
5934 : {
5935 : /* First search if we can do something useful and build a
5936 : vector of all loads we have to check. */
5937 93595 : bool unknown_memory_access = false;
5938 93595 : auto_vec<ao_ref, accesses_limit> accesses;
5939 93595 : unsigned load_accesses = summary ? summary->load_accesses : 0;
5940 93595 : if (!unknown_memory_access)
5941 : /* Add loads done as part of setting up the call arguments.
5942 : That's also necessary for CONST functions which will
5943 : not have a modref summary. */
5944 274761 : for (unsigned i = 0; i < gimple_call_num_args (stmt); ++i)
5945 : {
5946 181174 : tree arg = gimple_call_arg (stmt, i);
5947 181174 : if (TREE_CODE (arg) != SSA_NAME
5948 181174 : && !is_gimple_min_invariant (arg))
5949 : {
5950 64660 : if (accesses.length () >= accesses_limit - load_accesses)
5951 : {
5952 : unknown_memory_access = true;
5953 : break;
5954 : }
5955 32322 : accesses.quick_grow (accesses.length () + 1);
5956 32322 : ao_ref_init (&accesses.last (), arg);
5957 : }
5958 : }
5959 93595 : if (summary && !unknown_memory_access)
5960 : {
5961 : /* Add loads as analyzed by IPA modref. */
5962 320395 : for (auto base_node : summary->loads->bases)
5963 80486 : if (unknown_memory_access)
5964 : break;
5965 327351 : else for (auto ref_node : base_node->refs)
5966 86844 : if (unknown_memory_access)
5967 : break;
5968 363718 : else for (auto access_node : ref_node->accesses)
5969 : {
5970 239592 : accesses.quick_grow (accesses.length () + 1);
5971 119796 : ao_ref *r = &accesses.last ();
5972 119796 : if (!access_node.get_ao_ref (stmt, r))
5973 : {
5974 : /* Initialize a ref based on the argument and
5975 : unknown offset if possible. */
5976 16574 : tree arg = access_node.get_call_arg (stmt);
5977 16574 : if (arg && TREE_CODE (arg) == SSA_NAME)
5978 3107 : arg = SSA_VAL (arg);
5979 3107 : if (arg
5980 16564 : && TREE_CODE (arg) == ADDR_EXPR
5981 13463 : && (arg = get_base_address (arg))
5982 16570 : && DECL_P (arg))
5983 : {
5984 0 : ao_ref_init (r, arg);
5985 0 : r->ref = NULL_TREE;
5986 0 : r->base = arg;
5987 : }
5988 : else
5989 : {
5990 : unknown_memory_access = true;
5991 : break;
5992 : }
5993 : }
5994 103222 : r->base_alias_set = base_node->base;
5995 103222 : r->ref_alias_set = ref_node->ref;
5996 : }
5997 : }
5998 :
5999 : /* Walk the VUSE->VDEF chain optimistically trying to find an entry
6000 : for the call in the hashtable. */
6001 93595 : unsigned limit = (unknown_memory_access
6002 93595 : ? 0
6003 77013 : : (param_sccvn_max_alias_queries_per_access
6004 77013 : / (accesses.length () + 1)));
6005 93595 : tree saved_vuse = vr1.vuse;
6006 93595 : hashval_t saved_hashcode = vr1.hashcode;
6007 514700 : while (limit > 0 && !vnresult && !SSA_NAME_IS_DEFAULT_DEF (vr1.vuse))
6008 : {
6009 450054 : vr1.hashcode = vr1.hashcode - SSA_NAME_VERSION (vr1.vuse);
6010 450054 : gimple *def = SSA_NAME_DEF_STMT (vr1.vuse);
6011 : /* ??? We could use fancy stuff like in walk_non_aliased_vuses, but
6012 : do not bother for now. */
6013 450054 : if (is_a <gphi *> (def))
6014 : break;
6015 842210 : vr1.vuse = vuse_ssa_val (gimple_vuse (def));
6016 421105 : vr1.hashcode = vr1.hashcode + SSA_NAME_VERSION (vr1.vuse);
6017 421105 : vn_reference_lookup_1 (&vr1, &vnresult);
6018 421105 : limit--;
6019 : }
6020 :
6021 : /* If we found a candidate to CSE to verify it is valid. */
6022 93595 : if (vnresult && !accesses.is_empty ())
6023 : {
6024 1925 : tree vuse = vuse_ssa_val (gimple_vuse (stmt));
6025 7156 : while (vnresult && vuse != vr1.vuse)
6026 : {
6027 3306 : gimple *def = SSA_NAME_DEF_STMT (vuse);
6028 17361 : for (auto &ref : accesses)
6029 : {
6030 : /* ??? stmt_may_clobber_ref_p_1 does per stmt constant
6031 : analysis overhead that we might be able to cache. */
6032 9190 : if (stmt_may_clobber_ref_p_1 (def, &ref, true))
6033 : {
6034 1747 : vnresult = NULL;
6035 1747 : break;
6036 : }
6037 : }
6038 6612 : vuse = vuse_ssa_val (gimple_vuse (def));
6039 : }
6040 : }
6041 93595 : vr1.vuse = saved_vuse;
6042 93595 : vr1.hashcode = saved_hashcode;
6043 93595 : }
6044 :
6045 8784272 : if (vnresult)
6046 : {
6047 691186 : if (vdef)
6048 : {
6049 175173 : if (vnresult->result_vdef)
6050 175173 : changed |= set_ssa_val_to (vdef, vnresult->result_vdef);
6051 0 : else if (!lhs && gimple_call_lhs (stmt))
6052 : /* If stmt has non-SSA_NAME lhs, value number the vdef to itself,
6053 : as the call still acts as a lhs store. */
6054 0 : changed |= set_ssa_val_to (vdef, vdef);
6055 : else
6056 : /* If the call was discovered to be pure or const reflect
6057 : that as far as possible. */
6058 0 : changed |= set_ssa_val_to (vdef,
6059 : vuse_ssa_val (gimple_vuse (stmt)));
6060 : }
6061 :
6062 691186 : if (!vnresult->result && lhs)
6063 0 : vnresult->result = lhs;
6064 :
6065 691186 : if (vnresult->result && lhs)
6066 125278 : changed |= set_ssa_val_to (lhs, vnresult->result);
6067 : }
6068 : else
6069 : {
6070 8093086 : vn_reference_t vr2;
6071 8093086 : vn_reference_s **slot;
6072 8093086 : tree vdef_val = vdef;
6073 8093086 : if (vdef)
6074 : {
6075 : /* If we value numbered an indirect functions function to
6076 : one not clobbering memory value number its VDEF to its
6077 : VUSE. */
6078 4932970 : tree fn = gimple_call_fn (stmt);
6079 4932970 : if (fn && TREE_CODE (fn) == SSA_NAME)
6080 : {
6081 129494 : fn = SSA_VAL (fn);
6082 129494 : if (TREE_CODE (fn) == ADDR_EXPR
6083 1974 : && TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL
6084 1974 : && (flags_from_decl_or_type (TREE_OPERAND (fn, 0))
6085 1974 : & (ECF_CONST | ECF_PURE))
6086 : /* If stmt has non-SSA_NAME lhs, value number the
6087 : vdef to itself, as the call still acts as a lhs
6088 : store. */
6089 130869 : && (lhs || gimple_call_lhs (stmt) == NULL_TREE))
6090 2604 : vdef_val = vuse_ssa_val (gimple_vuse (stmt));
6091 : }
6092 4932970 : changed |= set_ssa_val_to (vdef, vdef_val);
6093 : }
6094 8093086 : if (lhs)
6095 3966480 : changed |= set_ssa_val_to (lhs, lhs);
6096 8093086 : vr2 = XOBNEW (&vn_tables_obstack, vn_reference_s);
6097 8093086 : vr2->vuse = vr1.vuse;
6098 : /* As we are not walking the virtual operand chain we know the
6099 : shared_lookup_references are still original so we can re-use
6100 : them here. */
6101 8093086 : vr2->operands = vr1.operands.copy ();
6102 8093086 : vr2->type = vr1.type;
6103 8093086 : vr2->punned = vr1.punned;
6104 8093086 : vr2->set = vr1.set;
6105 8093086 : vr2->offset = vr1.offset;
6106 8093086 : vr2->max_size = vr1.max_size;
6107 8093086 : vr2->base_set = vr1.base_set;
6108 8093086 : vr2->hashcode = vr1.hashcode;
6109 8093086 : vr2->result = lhs;
6110 8093086 : vr2->result_vdef = vdef_val;
6111 8093086 : vr2->value_id = 0;
6112 8093086 : slot = valid_info->references->find_slot_with_hash (vr2, vr2->hashcode,
6113 : INSERT);
6114 8093086 : gcc_assert (!*slot);
6115 8093086 : *slot = vr2;
6116 8093086 : vr2->next = last_inserted_ref;
6117 8093086 : last_inserted_ref = vr2;
6118 : }
6119 :
6120 8784272 : return changed;
6121 : }
6122 :
6123 : /* Visit a load from a reference operator RHS, part of STMT, value number it,
6124 : and return true if the value number of the LHS has changed as a result. */
6125 :
6126 : static bool
6127 35187668 : visit_reference_op_load (tree lhs, tree op, gimple *stmt)
6128 : {
6129 35187668 : bool changed = false;
6130 35187668 : tree result;
6131 35187668 : vn_reference_t res;
6132 :
6133 35187668 : tree vuse = gimple_vuse (stmt);
6134 35187668 : tree last_vuse = vuse;
6135 35187668 : result = vn_reference_lookup (op, vuse, default_vn_walk_kind, &res, true, &last_vuse);
6136 :
6137 : /* We handle type-punning through unions by value-numbering based
6138 : on offset and size of the access. Be prepared to handle a
6139 : type-mismatch here via creating a VIEW_CONVERT_EXPR. */
6140 35187668 : if (result
6141 35187668 : && !useless_type_conversion_p (TREE_TYPE (result), TREE_TYPE (op)))
6142 : {
6143 18562 : if (CONSTANT_CLASS_P (result))
6144 4205 : result = const_unop (VIEW_CONVERT_EXPR, TREE_TYPE (op), result);
6145 : else
6146 : {
6147 : /* We will be setting the value number of lhs to the value number
6148 : of VIEW_CONVERT_EXPR <TREE_TYPE (result)> (result).
6149 : So first simplify and lookup this expression to see if it
6150 : is already available. */
6151 14357 : gimple_match_op res_op (gimple_match_cond::UNCOND,
6152 14357 : VIEW_CONVERT_EXPR, TREE_TYPE (op), result);
6153 14357 : result = vn_nary_build_or_lookup (&res_op);
6154 14357 : if (result
6155 14351 : && TREE_CODE (result) == SSA_NAME
6156 27050 : && VN_INFO (result)->needs_insertion)
6157 : /* Track whether this is the canonical expression for different
6158 : typed loads. We use that as a stopgap measure for code
6159 : hoisting when dealing with floating point loads. */
6160 11435 : res->punned = true;
6161 : }
6162 :
6163 : /* When building the conversion fails avoid inserting the reference
6164 : again. */
6165 18562 : if (!result)
6166 6 : return set_ssa_val_to (lhs, lhs);
6167 : }
6168 :
6169 35169106 : if (result)
6170 5682962 : changed = set_ssa_val_to (lhs, result);
6171 : else
6172 : {
6173 29504700 : changed = set_ssa_val_to (lhs, lhs);
6174 29504700 : vn_reference_insert (op, lhs, last_vuse, NULL_TREE);
6175 29504700 : if (vuse && SSA_VAL (last_vuse) != SSA_VAL (vuse))
6176 : {
6177 9131902 : if (dump_file && (dump_flags & TDF_DETAILS))
6178 : {
6179 23143 : fprintf (dump_file, "Using extra use virtual operand ");
6180 23143 : print_generic_expr (dump_file, last_vuse);
6181 23143 : fprintf (dump_file, "\n");
6182 : }
6183 9131902 : vn_reference_insert (op, lhs, vuse, NULL_TREE);
6184 : }
6185 : }
6186 :
6187 : return changed;
6188 : }
6189 :
6190 :
6191 : /* Visit a store to a reference operator LHS, part of STMT, value number it,
6192 : and return true if the value number of the LHS has changed as a result. */
6193 :
6194 : static bool
6195 33557759 : visit_reference_op_store (tree lhs, tree op, gimple *stmt)
6196 : {
6197 33557759 : bool changed = false;
6198 33557759 : vn_reference_t vnresult = NULL;
6199 33557759 : tree assign;
6200 33557759 : bool resultsame = false;
6201 33557759 : tree vuse = gimple_vuse (stmt);
6202 33557759 : tree vdef = gimple_vdef (stmt);
6203 :
6204 33557759 : if (TREE_CODE (op) == SSA_NAME)
6205 15248735 : op = SSA_VAL (op);
6206 :
6207 : /* First we want to lookup using the *vuses* from the store and see
6208 : if there the last store to this location with the same address
6209 : had the same value.
6210 :
6211 : The vuses represent the memory state before the store. If the
6212 : memory state, address, and value of the store is the same as the
6213 : last store to this location, then this store will produce the
6214 : same memory state as that store.
6215 :
6216 : In this case the vdef versions for this store are value numbered to those
6217 : vuse versions, since they represent the same memory state after
6218 : this store.
6219 :
6220 : Otherwise, the vdefs for the store are used when inserting into
6221 : the table, since the store generates a new memory state. */
6222 :
6223 33557759 : vn_reference_lookup (lhs, vuse, VN_NOWALK, &vnresult, false);
6224 33557759 : if (vnresult
6225 1707322 : && vnresult->result)
6226 : {
6227 1707322 : tree result = vnresult->result;
6228 1707322 : gcc_checking_assert (TREE_CODE (result) != SSA_NAME
6229 : || result == SSA_VAL (result));
6230 1707322 : resultsame = expressions_equal_p (result, op);
6231 1707322 : if (resultsame)
6232 : {
6233 : /* If the TBAA state isn't compatible for downstream reads
6234 : we cannot value-number the VDEFs the same. */
6235 51977 : ao_ref lhs_ref;
6236 51977 : ao_ref_init (&lhs_ref, lhs);
6237 51977 : alias_set_type set = ao_ref_alias_set (&lhs_ref);
6238 51977 : alias_set_type base_set = ao_ref_base_alias_set (&lhs_ref);
6239 51977 : if ((vnresult->set != set
6240 955 : && ! alias_set_subset_of (set, vnresult->set))
6241 52587 : || (vnresult->base_set != base_set
6242 6353 : && ! alias_set_subset_of (base_set, vnresult->base_set)))
6243 873 : resultsame = false;
6244 : }
6245 : }
6246 :
6247 873 : if (!resultsame)
6248 : {
6249 33506655 : if (dump_file && (dump_flags & TDF_DETAILS))
6250 : {
6251 20357 : fprintf (dump_file, "No store match\n");
6252 20357 : fprintf (dump_file, "Value numbering store ");
6253 20357 : print_generic_expr (dump_file, lhs);
6254 20357 : fprintf (dump_file, " to ");
6255 20357 : print_generic_expr (dump_file, op);
6256 20357 : fprintf (dump_file, "\n");
6257 : }
6258 : /* Have to set value numbers before insert, since insert is
6259 : going to valueize the references in-place. */
6260 33506655 : if (vdef)
6261 33506655 : changed |= set_ssa_val_to (vdef, vdef);
6262 :
6263 : /* Do not insert structure copies into the tables. */
6264 33506655 : if (is_gimple_min_invariant (op)
6265 33506655 : || is_gimple_reg (op))
6266 29862729 : vn_reference_insert (lhs, op, vdef, NULL);
6267 :
6268 : /* Only perform the following when being called from PRE
6269 : which embeds tail merging. */
6270 33506655 : if (default_vn_walk_kind == VN_WALK)
6271 : {
6272 7596371 : assign = build2 (MODIFY_EXPR, TREE_TYPE (lhs), lhs, op);
6273 7596371 : vn_reference_lookup (assign, vuse, VN_NOWALK, &vnresult, false);
6274 7596371 : if (!vnresult)
6275 7554719 : vn_reference_insert (assign, lhs, vuse, vdef);
6276 : }
6277 : }
6278 : else
6279 : {
6280 : /* We had a match, so value number the vdef to have the value
6281 : number of the vuse it came from. */
6282 :
6283 51104 : if (dump_file && (dump_flags & TDF_DETAILS))
6284 9 : fprintf (dump_file, "Store matched earlier value, "
6285 : "value numbering store vdefs to matching vuses.\n");
6286 :
6287 51104 : changed |= set_ssa_val_to (vdef, SSA_VAL (vuse));
6288 : }
6289 :
6290 33557759 : return changed;
6291 : }
6292 :
6293 : /* Visit and value number PHI, return true if the value number
6294 : changed. When BACKEDGES_VARYING_P is true then assume all
6295 : backedge values are varying. When INSERTED is not NULL then
6296 : this is just a ahead query for a possible iteration, set INSERTED
6297 : to true if we'd insert into the hashtable. */
6298 :
6299 : static bool
6300 34798384 : visit_phi (gimple *phi, bool *inserted, bool backedges_varying_p)
6301 : {
6302 34798384 : tree result, sameval = VN_TOP, seen_undef = NULL_TREE;
6303 34798384 : bool seen_undef_visited = false;
6304 34798384 : tree backedge_val = NULL_TREE;
6305 34798384 : bool seen_non_backedge = false;
6306 34798384 : tree sameval_base = NULL_TREE;
6307 34798384 : poly_int64 soff, doff;
6308 34798384 : unsigned n_executable = 0;
6309 34798384 : edge sameval_e = NULL;
6310 :
6311 : /* TODO: We could check for this in initialization, and replace this
6312 : with a gcc_assert. */
6313 34798384 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)))
6314 31043 : return set_ssa_val_to (PHI_RESULT (phi), PHI_RESULT (phi));
6315 :
6316 : /* We track whether a PHI was CSEd to avoid excessive iterations
6317 : that would be necessary only because the PHI changed arguments
6318 : but not value. */
6319 34767341 : if (!inserted)
6320 27137193 : gimple_set_plf (phi, GF_PLF_1, false);
6321 :
6322 34767341 : basic_block bb = gimple_bb (phi);
6323 :
6324 : /* For the equivalence handling below make sure to first process an
6325 : edge with a non-constant. */
6326 34767341 : auto_vec<edge, 2> preds;
6327 69534682 : preds.reserve_exact (EDGE_COUNT (bb->preds));
6328 34767341 : bool seen_nonconstant = false;
6329 114952644 : for (unsigned i = 0; i < EDGE_COUNT (bb->preds); ++i)
6330 : {
6331 80185303 : edge e = EDGE_PRED (bb, i);
6332 80185303 : preds.quick_push (e);
6333 80185303 : if (!seen_nonconstant)
6334 : {
6335 42573633 : tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
6336 42573633 : if (TREE_CODE (def) == SSA_NAME)
6337 : {
6338 33017151 : seen_nonconstant = true;
6339 33017151 : if (i != 0)
6340 5813007 : std::swap (preds[0], preds[i]);
6341 : }
6342 : }
6343 : }
6344 :
6345 : /* See if all non-TOP arguments have the same value. TOP is
6346 : equivalent to everything, so we can ignore it. */
6347 146360778 : for (edge e : preds)
6348 69058509 : if (e->flags & EDGE_EXECUTABLE)
6349 : {
6350 63960481 : tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
6351 :
6352 63960481 : if (def == PHI_RESULT (phi))
6353 338645 : continue;
6354 63646971 : ++n_executable;
6355 63646971 : bool visited = true;
6356 63646971 : if (TREE_CODE (def) == SSA_NAME)
6357 : {
6358 51346478 : tree val = SSA_VAL (def, &visited);
6359 51346478 : if (SSA_NAME_IS_DEFAULT_DEF (def))
6360 2706735 : visited = true;
6361 51346478 : if (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK))
6362 48803657 : def = val;
6363 51346478 : if (e->flags & EDGE_DFS_BACK)
6364 15531632 : backedge_val = def;
6365 : }
6366 63646971 : if (!(e->flags & EDGE_DFS_BACK))
6367 47947962 : seen_non_backedge = true;
6368 63646971 : if (def == VN_TOP)
6369 : ;
6370 : /* Ignore undefined defs for sameval but record one. */
6371 63646971 : else if (TREE_CODE (def) == SSA_NAME
6372 47910278 : && ! virtual_operand_p (def)
6373 88024584 : && ssa_undefined_value_p (def, false))
6374 : {
6375 234996 : if (!seen_undef
6376 : /* Avoid having not visited undefined defs if we also have
6377 : a visited one. */
6378 35103 : || (!seen_undef_visited && visited))
6379 : {
6380 199897 : seen_undef = def;
6381 199897 : seen_undef_visited = visited;
6382 : }
6383 : }
6384 63411975 : else if (sameval == VN_TOP)
6385 : {
6386 : sameval = def;
6387 : sameval_e = e;
6388 : }
6389 28692068 : else if (expressions_equal_p (def, sameval))
6390 : sameval_e = NULL;
6391 45137642 : else if (virtual_operand_p (def))
6392 : {
6393 : sameval = NULL_TREE;
6394 26999754 : break;
6395 : }
6396 : else
6397 : {
6398 : /* We know we're arriving only with invariant addresses here,
6399 : try harder comparing them. We can do some caching here
6400 : which we cannot do in expressions_equal_p. */
6401 16869050 : if (TREE_CODE (def) == ADDR_EXPR
6402 391226 : && TREE_CODE (sameval) == ADDR_EXPR
6403 107802 : && sameval_base != (void *)-1)
6404 : {
6405 107802 : if (!sameval_base)
6406 107800 : sameval_base = get_addr_base_and_unit_offset
6407 107800 : (TREE_OPERAND (sameval, 0), &soff);
6408 107800 : if (!sameval_base)
6409 : sameval_base = (tree)(void *)-1;
6410 107807 : else if ((get_addr_base_and_unit_offset
6411 107802 : (TREE_OPERAND (def, 0), &doff) == sameval_base)
6412 107802 : && known_eq (soff, doff))
6413 5 : continue;
6414 : }
6415 : /* There's also the possibility to use equivalences. */
6416 32644378 : if (!FLOAT_TYPE_P (TREE_TYPE (def))
6417 : /* But only do this if we didn't force any of sameval or
6418 : val to VARYING because of backedge processing rules. */
6419 15669864 : && (TREE_CODE (sameval) != SSA_NAME
6420 12353102 : || SSA_VAL (sameval) == sameval)
6421 32538846 : && (TREE_CODE (def) != SSA_NAME || SSA_VAL (def) == def))
6422 : {
6423 15669789 : vn_nary_op_t vnresult;
6424 15669789 : tree ops[2];
6425 15669789 : ops[0] = def;
6426 15669789 : ops[1] = sameval;
6427 : /* Canonicalize the operands order for eq below. */
6428 15669789 : if (tree_swap_operands_p (ops[0], ops[1]))
6429 9449624 : std::swap (ops[0], ops[1]);
6430 15669789 : tree val = vn_nary_op_lookup_pieces (2, EQ_EXPR,
6431 : boolean_type_node,
6432 : ops, &vnresult);
6433 15669789 : if (! val && vnresult && vnresult->predicated_values)
6434 : {
6435 214908 : val = vn_nary_op_get_predicated_value (vnresult, e);
6436 121703 : if (val && integer_truep (val)
6437 240158 : && !(sameval_e && (sameval_e->flags & EDGE_DFS_BACK)))
6438 : {
6439 25130 : if (dump_file && (dump_flags & TDF_DETAILS))
6440 : {
6441 2 : fprintf (dump_file, "Predication says ");
6442 2 : print_generic_expr (dump_file, def, TDF_NONE);
6443 2 : fprintf (dump_file, " and ");
6444 2 : print_generic_expr (dump_file, sameval, TDF_NONE);
6445 2 : fprintf (dump_file, " are equal on edge %d -> %d\n",
6446 2 : e->src->index, e->dest->index);
6447 : }
6448 25130 : continue;
6449 : }
6450 : }
6451 : }
6452 : sameval = NULL_TREE;
6453 : break;
6454 : }
6455 : }
6456 :
6457 : /* If the value we want to use is flowing over the backedge and we
6458 : should take it as VARYING but it has a non-VARYING value drop to
6459 : VARYING.
6460 : If we value-number a virtual operand never value-number to the
6461 : value from the backedge as that confuses the alias-walking code.
6462 : See gcc.dg/torture/pr87176.c. If the value is the same on a
6463 : non-backedge everything is OK though. */
6464 34767341 : bool visited_p;
6465 34767341 : if ((backedge_val
6466 34767341 : && !seen_non_backedge
6467 2017 : && TREE_CODE (backedge_val) == SSA_NAME
6468 1750 : && sameval == backedge_val
6469 311 : && (SSA_NAME_IS_VIRTUAL_OPERAND (backedge_val)
6470 40 : || SSA_VAL (backedge_val) != backedge_val))
6471 : /* Do not value-number a virtual operand to sth not visited though
6472 : given that allows us to escape a region in alias walking. */
6473 34769087 : || (sameval
6474 7767316 : && TREE_CODE (sameval) == SSA_NAME
6475 4607463 : && !SSA_NAME_IS_DEFAULT_DEF (sameval)
6476 3893683 : && SSA_NAME_IS_VIRTUAL_OPERAND (sameval)
6477 1952808 : && (SSA_VAL (sameval, &visited_p), !visited_p)))
6478 : /* Note this just drops to VARYING without inserting the PHI into
6479 : the hashes. */
6480 300530 : result = PHI_RESULT (phi);
6481 : /* If none of the edges was executable keep the value-number at VN_TOP,
6482 : if only a single edge is executable use its value. */
6483 34466811 : else if (n_executable <= 1)
6484 6699579 : result = seen_undef ? seen_undef : sameval;
6485 : /* If we saw only undefined values and VN_TOP use one of the
6486 : undefined values. */
6487 27767232 : else if (sameval == VN_TOP)
6488 7282853 : result = (seen_undef && seen_undef_visited) ? seen_undef : sameval;
6489 : /* First see if it is equivalent to a phi node in this block. We prefer
6490 : this as it allows IV elimination - see PRs 66502 and 67167. */
6491 27761709 : else if ((result = vn_phi_lookup (phi, backedges_varying_p)))
6492 : {
6493 4224967 : if (!inserted
6494 70374 : && TREE_CODE (result) == SSA_NAME
6495 4295341 : && gimple_code (SSA_NAME_DEF_STMT (result)) == GIMPLE_PHI)
6496 : {
6497 70374 : gimple_set_plf (SSA_NAME_DEF_STMT (result), GF_PLF_1, true);
6498 70374 : if (dump_file && (dump_flags & TDF_DETAILS))
6499 : {
6500 6 : fprintf (dump_file, "Marking CSEd to PHI node ");
6501 6 : print_gimple_expr (dump_file, SSA_NAME_DEF_STMT (result),
6502 : 0, TDF_SLIM);
6503 6 : fprintf (dump_file, "\n");
6504 : }
6505 : }
6506 : }
6507 : /* If all values are the same use that, unless we've seen undefined
6508 : values as well and the value isn't constant.
6509 : CCP/copyprop have the same restriction to not remove uninit warnings. */
6510 23536742 : else if (sameval
6511 23536742 : && (! seen_undef || is_gimple_min_invariant (sameval)))
6512 : result = sameval;
6513 : else
6514 : {
6515 22917080 : result = PHI_RESULT (phi);
6516 : /* Only insert PHIs that are varying, for constant value numbers
6517 : we mess up equivalences otherwise as we are only comparing
6518 : the immediate controlling predicates. */
6519 22917080 : vn_phi_insert (phi, result, backedges_varying_p);
6520 22917080 : if (inserted)
6521 3318681 : *inserted = true;
6522 : }
6523 :
6524 34767341 : return set_ssa_val_to (PHI_RESULT (phi), result);
6525 34767341 : }
6526 :
6527 : /* Try to simplify RHS using equivalences and constant folding. */
6528 :
6529 : static tree
6530 129480341 : try_to_simplify (gassign *stmt)
6531 : {
6532 129480341 : enum tree_code code = gimple_assign_rhs_code (stmt);
6533 129480341 : tree tem;
6534 :
6535 : /* For stores we can end up simplifying a SSA_NAME rhs. Just return
6536 : in this case, there is no point in doing extra work. */
6537 129480341 : if (code == SSA_NAME)
6538 : return NULL_TREE;
6539 :
6540 : /* First try constant folding based on our current lattice. */
6541 114231311 : mprts_hook = vn_lookup_simplify_result;
6542 114231311 : tem = gimple_fold_stmt_to_constant_1 (stmt, vn_valueize, vn_valueize);
6543 114231311 : mprts_hook = NULL;
6544 114231311 : if (tem
6545 114231311 : && (TREE_CODE (tem) == SSA_NAME
6546 25384225 : || is_gimple_min_invariant (tem)))
6547 25456512 : return tem;
6548 :
6549 : return NULL_TREE;
6550 : }
6551 :
6552 : /* Visit and value number STMT, return true if the value number
6553 : changed. */
6554 :
6555 : static bool
6556 474758737 : visit_stmt (gimple *stmt, bool backedges_varying_p = false)
6557 : {
6558 474758737 : bool changed = false;
6559 :
6560 474758737 : if (dump_file && (dump_flags & TDF_DETAILS))
6561 : {
6562 411549 : fprintf (dump_file, "Value numbering stmt = ");
6563 411549 : print_gimple_stmt (dump_file, stmt, 0);
6564 : }
6565 :
6566 474758737 : if (gimple_code (stmt) == GIMPLE_PHI)
6567 27158457 : changed = visit_phi (stmt, NULL, backedges_varying_p);
6568 622477588 : else if (gimple_has_volatile_ops (stmt))
6569 9167217 : changed = defs_to_varying (stmt);
6570 438433063 : else if (gassign *ass = dyn_cast <gassign *> (stmt))
6571 : {
6572 134602183 : enum tree_code code = gimple_assign_rhs_code (ass);
6573 134602183 : tree lhs = gimple_assign_lhs (ass);
6574 134602183 : tree rhs1 = gimple_assign_rhs1 (ass);
6575 134602183 : tree simplified;
6576 :
6577 : /* Shortcut for copies. Simplifying copies is pointless,
6578 : since we copy the expression and value they represent. */
6579 134602183 : if (code == SSA_NAME
6580 20370872 : && TREE_CODE (lhs) == SSA_NAME)
6581 : {
6582 5121842 : changed = visit_copy (lhs, rhs1);
6583 5121842 : goto done;
6584 : }
6585 129480341 : simplified = try_to_simplify (ass);
6586 129480341 : if (simplified)
6587 : {
6588 25456512 : if (dump_file && (dump_flags & TDF_DETAILS))
6589 : {
6590 14771 : fprintf (dump_file, "RHS ");
6591 14771 : print_gimple_expr (dump_file, ass, 0);
6592 14771 : fprintf (dump_file, " simplified to ");
6593 14771 : print_generic_expr (dump_file, simplified);
6594 14771 : fprintf (dump_file, "\n");
6595 : }
6596 : }
6597 : /* Setting value numbers to constants will occasionally
6598 : screw up phi congruence because constants are not
6599 : uniquely associated with a single ssa name that can be
6600 : looked up. */
6601 25456512 : if (simplified
6602 25456512 : && is_gimple_min_invariant (simplified)
6603 22428889 : && TREE_CODE (lhs) == SSA_NAME)
6604 : {
6605 7758327 : changed = set_ssa_val_to (lhs, simplified);
6606 7758327 : goto done;
6607 : }
6608 121722014 : else if (simplified
6609 17698185 : && TREE_CODE (simplified) == SSA_NAME
6610 3027623 : && TREE_CODE (lhs) == SSA_NAME)
6611 : {
6612 3027623 : changed = visit_copy (lhs, simplified);
6613 3027623 : goto done;
6614 : }
6615 :
6616 118694391 : if ((TREE_CODE (lhs) == SSA_NAME
6617 : /* We can substitute SSA_NAMEs that are live over
6618 : abnormal edges with their constant value. */
6619 85136363 : && !(gimple_assign_copy_p (ass)
6620 26 : && is_gimple_min_invariant (rhs1))
6621 85136337 : && !(simplified
6622 0 : && is_gimple_min_invariant (simplified))
6623 85136337 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
6624 : /* Stores or copies from SSA_NAMEs that are live over
6625 : abnormal edges are a problem. */
6626 203829425 : || (code == SSA_NAME
6627 15249030 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
6628 1598 : changed = defs_to_varying (ass);
6629 118692793 : else if (REFERENCE_CLASS_P (lhs)
6630 118692793 : || DECL_P (lhs))
6631 33557759 : changed = visit_reference_op_store (lhs, rhs1, ass);
6632 85135034 : else if (TREE_CODE (lhs) == SSA_NAME)
6633 : {
6634 85135034 : if ((gimple_assign_copy_p (ass)
6635 26 : && is_gimple_min_invariant (rhs1))
6636 85135060 : || (simplified
6637 0 : && is_gimple_min_invariant (simplified)))
6638 : {
6639 0 : if (simplified)
6640 0 : changed = set_ssa_val_to (lhs, simplified);
6641 : else
6642 0 : changed = set_ssa_val_to (lhs, rhs1);
6643 : }
6644 : else
6645 : {
6646 : /* Visit the original statement. */
6647 85135034 : switch (vn_get_stmt_kind (ass))
6648 : {
6649 49840870 : case VN_NARY:
6650 49840870 : changed = visit_nary_op (lhs, ass);
6651 49840870 : break;
6652 35187668 : case VN_REFERENCE:
6653 35187668 : changed = visit_reference_op_load (lhs, rhs1, ass);
6654 35187668 : break;
6655 106496 : default:
6656 106496 : changed = defs_to_varying (ass);
6657 106496 : break;
6658 : }
6659 : }
6660 : }
6661 : else
6662 0 : changed = defs_to_varying (ass);
6663 : }
6664 303830880 : else if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
6665 : {
6666 25286056 : tree lhs = gimple_call_lhs (call_stmt);
6667 25286056 : if (lhs && TREE_CODE (lhs) == SSA_NAME)
6668 : {
6669 : /* Try constant folding based on our current lattice. */
6670 8482498 : tree simplified = gimple_fold_stmt_to_constant_1 (call_stmt,
6671 : vn_valueize);
6672 8482498 : if (simplified)
6673 : {
6674 67858 : if (dump_file && (dump_flags & TDF_DETAILS))
6675 : {
6676 1 : fprintf (dump_file, "call ");
6677 1 : print_gimple_expr (dump_file, call_stmt, 0);
6678 1 : fprintf (dump_file, " simplified to ");
6679 1 : print_generic_expr (dump_file, simplified);
6680 1 : fprintf (dump_file, "\n");
6681 : }
6682 : }
6683 : /* Setting value numbers to constants will occasionally
6684 : screw up phi congruence because constants are not
6685 : uniquely associated with a single ssa name that can be
6686 : looked up. */
6687 67858 : if (simplified
6688 67858 : && is_gimple_min_invariant (simplified))
6689 : {
6690 61420 : changed = set_ssa_val_to (lhs, simplified);
6691 122840 : if (gimple_vdef (call_stmt))
6692 740 : changed |= set_ssa_val_to (gimple_vdef (call_stmt),
6693 : SSA_VAL (gimple_vuse (call_stmt)));
6694 61420 : goto done;
6695 : }
6696 8421078 : else if (simplified
6697 6438 : && TREE_CODE (simplified) == SSA_NAME)
6698 : {
6699 293 : changed = visit_copy (lhs, simplified);
6700 586 : if (gimple_vdef (call_stmt))
6701 0 : changed |= set_ssa_val_to (gimple_vdef (call_stmt),
6702 : SSA_VAL (gimple_vuse (call_stmt)));
6703 293 : goto done;
6704 : }
6705 8420785 : else if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
6706 : {
6707 381 : changed = defs_to_varying (call_stmt);
6708 381 : goto done;
6709 : }
6710 : }
6711 :
6712 : /* Pick up flags from a devirtualization target. */
6713 25223962 : tree fn = gimple_call_fn (stmt);
6714 25223962 : int extra_fnflags = 0;
6715 25223962 : if (fn && TREE_CODE (fn) == SSA_NAME)
6716 : {
6717 537758 : fn = SSA_VAL (fn);
6718 537758 : if (TREE_CODE (fn) == ADDR_EXPR
6719 537758 : && TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL)
6720 5328 : extra_fnflags = flags_from_decl_or_type (TREE_OPERAND (fn, 0));
6721 : }
6722 25223962 : if ((/* Calls to the same function with the same vuse
6723 : and the same operands do not necessarily return the same
6724 : value, unless they're pure or const. */
6725 25223962 : ((gimple_call_flags (call_stmt) | extra_fnflags)
6726 25223962 : & (ECF_PURE | ECF_CONST))
6727 : /* If calls have a vdef, subsequent calls won't have
6728 : the same incoming vuse. So, if 2 calls with vdef have the
6729 : same vuse, we know they're not subsequent.
6730 : We can value number 2 calls to the same function with the
6731 : same vuse and the same operands which are not subsequent
6732 : the same, because there is no code in the program that can
6733 : compare the 2 values... */
6734 21206403 : || (gimple_vdef (call_stmt)
6735 : /* ... unless the call returns a pointer which does
6736 : not alias with anything else. In which case the
6737 : information that the values are distinct are encoded
6738 : in the IL. */
6739 21171571 : && !(gimple_call_return_flags (call_stmt) & ERF_NOALIAS)
6740 : /* Only perform the following when being called from PRE
6741 : which embeds tail merging. */
6742 20610585 : && default_vn_walk_kind == VN_WALK))
6743 : /* Do not process .DEFERRED_INIT since that confuses uninit
6744 : analysis. */
6745 30261888 : && !gimple_call_internal_p (call_stmt, IFN_DEFERRED_INIT))
6746 8784272 : changed = visit_reference_op_call (lhs, call_stmt);
6747 : else
6748 16439690 : changed = defs_to_varying (call_stmt);
6749 : }
6750 : else
6751 278544824 : changed = defs_to_varying (stmt);
6752 474758737 : done:
6753 474758737 : return changed;
6754 : }
6755 :
6756 :
6757 : /* Allocate a value number table. */
6758 :
6759 : static void
6760 6282091 : allocate_vn_table (vn_tables_t table, unsigned size)
6761 : {
6762 6282091 : table->phis = new vn_phi_table_type (size);
6763 6282091 : table->nary = new vn_nary_op_table_type (size);
6764 6282091 : table->references = new vn_reference_table_type (size);
6765 6282091 : }
6766 :
6767 : /* Free a value number table. */
6768 :
6769 : static void
6770 6282091 : free_vn_table (vn_tables_t table)
6771 : {
6772 : /* Walk over elements and release vectors. */
6773 6282091 : vn_reference_iterator_type hir;
6774 6282091 : vn_reference_t vr;
6775 149435483 : FOR_EACH_HASH_TABLE_ELEMENT (*table->references, vr, vn_reference_t, hir)
6776 71576696 : vr->operands.release ();
6777 6282091 : delete table->phis;
6778 6282091 : table->phis = NULL;
6779 6282091 : delete table->nary;
6780 6282091 : table->nary = NULL;
6781 6282091 : delete table->references;
6782 6282091 : table->references = NULL;
6783 6282091 : }
6784 :
6785 : /* Set *ID according to RESULT. */
6786 :
6787 : static void
6788 35020524 : set_value_id_for_result (tree result, unsigned int *id)
6789 : {
6790 35020524 : if (result && TREE_CODE (result) == SSA_NAME)
6791 21787294 : *id = VN_INFO (result)->value_id;
6792 9898264 : else if (result && is_gimple_min_invariant (result))
6793 3744730 : *id = get_or_alloc_constant_value_id (result);
6794 : else
6795 9488500 : *id = get_next_value_id ();
6796 35020524 : }
6797 :
6798 : /* Set the value ids in the valid hash tables. */
6799 :
6800 : static void
6801 976707 : set_hashtable_value_ids (void)
6802 : {
6803 976707 : vn_nary_op_iterator_type hin;
6804 976707 : vn_phi_iterator_type hip;
6805 976707 : vn_reference_iterator_type hir;
6806 976707 : vn_nary_op_t vno;
6807 976707 : vn_reference_t vr;
6808 976707 : vn_phi_t vp;
6809 :
6810 : /* Now set the value ids of the things we had put in the hash
6811 : table. */
6812 :
6813 49436571 : FOR_EACH_HASH_TABLE_ELEMENT (*valid_info->nary, vno, vn_nary_op_t, hin)
6814 24229932 : if (! vno->predicated_values)
6815 7905088 : set_value_id_for_result (vno->u.result, &vno->value_id);
6816 :
6817 9073207 : FOR_EACH_HASH_TABLE_ELEMENT (*valid_info->phis, vp, vn_phi_t, hip)
6818 4048250 : set_value_id_for_result (vp->result, &vp->value_id);
6819 :
6820 47111079 : FOR_EACH_HASH_TABLE_ELEMENT (*valid_info->references, vr, vn_reference_t,
6821 : hir)
6822 23067186 : set_value_id_for_result (vr->result, &vr->value_id);
6823 976707 : }
6824 :
6825 : /* Return the maximum value id we have ever seen. */
6826 :
6827 : unsigned int
6828 1953414 : get_max_value_id (void)
6829 : {
6830 1953414 : return next_value_id;
6831 : }
6832 :
6833 : /* Return the maximum constant value id we have ever seen. */
6834 :
6835 : unsigned int
6836 1953414 : get_max_constant_value_id (void)
6837 : {
6838 1953414 : return -next_constant_value_id;
6839 : }
6840 :
6841 : /* Return the next unique value id. */
6842 :
6843 : unsigned int
6844 49831359 : get_next_value_id (void)
6845 : {
6846 49831359 : gcc_checking_assert ((int)next_value_id > 0);
6847 49831359 : return next_value_id++;
6848 : }
6849 :
6850 : /* Return the next unique value id for constants. */
6851 :
6852 : unsigned int
6853 2563056 : get_next_constant_value_id (void)
6854 : {
6855 2563056 : gcc_checking_assert (next_constant_value_id < 0);
6856 2563056 : return next_constant_value_id--;
6857 : }
6858 :
6859 :
6860 : /* Compare two expressions E1 and E2 and return true if they are equal.
6861 : If match_vn_top_optimistically is true then VN_TOP is equal to anything,
6862 : otherwise VN_TOP only matches VN_TOP. */
6863 :
6864 : bool
6865 251064898 : expressions_equal_p (tree e1, tree e2, bool match_vn_top_optimistically)
6866 : {
6867 : /* The obvious case. */
6868 251064898 : if (e1 == e2)
6869 : return true;
6870 :
6871 : /* If either one is VN_TOP consider them equal. */
6872 71703199 : if (match_vn_top_optimistically
6873 66774612 : && (e1 == VN_TOP || e2 == VN_TOP))
6874 : return true;
6875 :
6876 : /* If only one of them is null, they cannot be equal. While in general
6877 : this should not happen for operations like TARGET_MEM_REF some
6878 : operands are optional and an identity value we could substitute
6879 : has differing semantics. */
6880 71703199 : if (!e1 || !e2)
6881 : return false;
6882 :
6883 : /* SSA_NAME compare pointer equal. */
6884 71703199 : if (TREE_CODE (e1) == SSA_NAME || TREE_CODE (e2) == SSA_NAME)
6885 : return false;
6886 :
6887 : /* Now perform the actual comparison. */
6888 35774647 : if (TREE_CODE (e1) == TREE_CODE (e2)
6889 35774647 : && operand_equal_p (e1, e2, OEP_PURE_SAME))
6890 : return true;
6891 :
6892 : return false;
6893 : }
6894 :
6895 :
6896 : /* Return true if the nary operation NARY may trap. This is a copy
6897 : of stmt_could_throw_1_p adjusted to the SCCVN IL. */
6898 :
6899 : bool
6900 5707873 : vn_nary_may_trap (vn_nary_op_t nary)
6901 : {
6902 5707873 : tree type;
6903 5707873 : tree rhs2 = NULL_TREE;
6904 5707873 : bool honor_nans = false;
6905 5707873 : bool honor_snans = false;
6906 5707873 : bool fp_operation = false;
6907 5707873 : bool honor_trapv = false;
6908 5707873 : bool handled, ret;
6909 5707873 : unsigned i;
6910 :
6911 5707873 : if (TREE_CODE_CLASS (nary->opcode) == tcc_comparison
6912 : || TREE_CODE_CLASS (nary->opcode) == tcc_unary
6913 5707873 : || TREE_CODE_CLASS (nary->opcode) == tcc_binary)
6914 : {
6915 5586717 : type = nary->type;
6916 5586717 : fp_operation = FLOAT_TYPE_P (type);
6917 5586717 : if (fp_operation)
6918 : {
6919 120272 : honor_nans = flag_trapping_math && !flag_finite_math_only;
6920 120272 : honor_snans = flag_signaling_nans != 0;
6921 : }
6922 5466445 : else if (INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type))
6923 : honor_trapv = true;
6924 : }
6925 5707873 : if (nary->length >= 2)
6926 2290145 : rhs2 = nary->op[1];
6927 5707873 : ret = operation_could_trap_helper_p (nary->opcode, fp_operation,
6928 : honor_trapv, honor_nans, honor_snans,
6929 : rhs2, &handled);
6930 5707873 : if (handled && ret)
6931 : return true;
6932 :
6933 13400316 : for (i = 0; i < nary->length; ++i)
6934 7811712 : if (tree_could_trap_p (nary->op[i]))
6935 : return true;
6936 :
6937 : return false;
6938 : }
6939 :
6940 : /* Return true if the reference operation REF may trap. */
6941 :
6942 : bool
6943 948162 : vn_reference_may_trap (vn_reference_t ref)
6944 : {
6945 948162 : switch (ref->operands[0].opcode)
6946 : {
6947 : case MODIFY_EXPR:
6948 : case CALL_EXPR:
6949 : /* We do not handle calls. */
6950 : return true;
6951 : case ADDR_EXPR:
6952 : /* And toplevel address computations never trap. */
6953 : return false;
6954 : default:;
6955 : }
6956 :
6957 : vn_reference_op_t op;
6958 : unsigned i;
6959 2629874 : FOR_EACH_VEC_ELT (ref->operands, i, op)
6960 : {
6961 2629619 : switch (op->opcode)
6962 : {
6963 : case WITH_SIZE_EXPR:
6964 : case TARGET_MEM_REF:
6965 : /* Always variable. */
6966 : return true;
6967 748107 : case COMPONENT_REF:
6968 748107 : if (op->op1 && TREE_CODE (op->op1) == SSA_NAME)
6969 : return true;
6970 : break;
6971 0 : case ARRAY_RANGE_REF:
6972 0 : if (TREE_CODE (op->op0) == SSA_NAME)
6973 : return true;
6974 : break;
6975 204920 : case ARRAY_REF:
6976 204920 : {
6977 204920 : if (TREE_CODE (op->op0) != INTEGER_CST)
6978 : return true;
6979 :
6980 : /* !in_array_bounds */
6981 184829 : tree domain_type = TYPE_DOMAIN (ref->operands[i+1].type);
6982 184829 : if (!domain_type)
6983 : return true;
6984 :
6985 184783 : tree min = op->op1;
6986 184783 : tree max = TYPE_MAX_VALUE (domain_type);
6987 184783 : if (!min
6988 184783 : || !max
6989 171858 : || TREE_CODE (min) != INTEGER_CST
6990 171858 : || TREE_CODE (max) != INTEGER_CST)
6991 : return true;
6992 :
6993 169203 : if (tree_int_cst_lt (op->op0, min)
6994 169203 : || tree_int_cst_lt (max, op->op0))
6995 325 : return true;
6996 :
6997 : break;
6998 : }
6999 : case MEM_REF:
7000 : /* Nothing interesting in itself, the base is separate. */
7001 : break;
7002 : /* The following are the address bases. */
7003 : case SSA_NAME:
7004 : return true;
7005 541475 : case ADDR_EXPR:
7006 541475 : if (op->op0)
7007 541475 : return tree_could_trap_p (TREE_OPERAND (op->op0, 0));
7008 : return false;
7009 1767635 : default:;
7010 : }
7011 : }
7012 : return false;
7013 : }
7014 :
7015 10656314 : eliminate_dom_walker::eliminate_dom_walker (cdi_direction direction,
7016 10656314 : bitmap inserted_exprs_)
7017 10656314 : : dom_walker (direction), do_pre (inserted_exprs_ != NULL),
7018 10656314 : el_todo (0), eliminations (0), insertions (0),
7019 10656314 : inserted_exprs (inserted_exprs_)
7020 : {
7021 10656314 : need_eh_cleanup = BITMAP_ALLOC (NULL);
7022 10656314 : need_ab_cleanup = BITMAP_ALLOC (NULL);
7023 10656314 : }
7024 :
7025 10656314 : eliminate_dom_walker::~eliminate_dom_walker ()
7026 : {
7027 10656314 : BITMAP_FREE (need_eh_cleanup);
7028 10656314 : BITMAP_FREE (need_ab_cleanup);
7029 10656314 : }
7030 :
7031 : /* Return a leader for OP that is available at the current point of the
7032 : eliminate domwalk. */
7033 :
7034 : tree
7035 185682424 : eliminate_dom_walker::eliminate_avail (basic_block, tree op)
7036 : {
7037 185682424 : tree valnum = VN_INFO (op)->valnum;
7038 185682424 : if (TREE_CODE (valnum) == SSA_NAME)
7039 : {
7040 180495281 : if (SSA_NAME_IS_DEFAULT_DEF (valnum))
7041 : return valnum;
7042 314147448 : if (avail.length () > SSA_NAME_VERSION (valnum))
7043 : {
7044 141507736 : tree av = avail[SSA_NAME_VERSION (valnum)];
7045 : /* When PRE discovers a new redundancy there's no way to unite
7046 : the value classes so it instead inserts a copy old-val = new-val.
7047 : Look through such copies here, providing one more level of
7048 : simplification at elimination time. */
7049 141507736 : gassign *ass;
7050 248902490 : if (av && (ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (av))))
7051 76426470 : if (gimple_assign_rhs_class (ass) == GIMPLE_SINGLE_RHS)
7052 : {
7053 40505369 : tree rhs1 = gimple_assign_rhs1 (ass);
7054 40505369 : if (CONSTANT_CLASS_P (rhs1)
7055 40505369 : || (TREE_CODE (rhs1) == SSA_NAME
7056 10743 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
7057 : av = rhs1;
7058 : }
7059 141507736 : return av;
7060 : }
7061 : }
7062 5187143 : else if (is_gimple_min_invariant (valnum))
7063 : return valnum;
7064 : return NULL_TREE;
7065 : }
7066 :
7067 : /* At the current point of the eliminate domwalk make OP available. */
7068 :
7069 : void
7070 51065463 : eliminate_dom_walker::eliminate_push_avail (basic_block, tree op)
7071 : {
7072 51065463 : tree valnum = VN_INFO (op)->valnum;
7073 51065463 : if (TREE_CODE (valnum) == SSA_NAME)
7074 : {
7075 98680082 : if (avail.length () <= SSA_NAME_VERSION (valnum))
7076 17308426 : avail.safe_grow_cleared (SSA_NAME_VERSION (valnum) + 1, true);
7077 51065463 : tree pushop = op;
7078 51065463 : if (avail[SSA_NAME_VERSION (valnum)])
7079 45002 : pushop = avail[SSA_NAME_VERSION (valnum)];
7080 51065463 : avail_stack.safe_push (pushop);
7081 51065463 : avail[SSA_NAME_VERSION (valnum)] = op;
7082 : }
7083 51065463 : }
7084 :
7085 : /* Insert the expression recorded by SCCVN for VAL at *GSI. Returns
7086 : the leader for the expression if insertion was successful. */
7087 :
7088 : tree
7089 125111 : eliminate_dom_walker::eliminate_insert (basic_block bb,
7090 : gimple_stmt_iterator *gsi, tree val)
7091 : {
7092 : /* We can insert a sequence with a single assignment only. */
7093 125111 : gimple_seq stmts = VN_INFO (val)->expr;
7094 125111 : if (!gimple_seq_singleton_p (stmts))
7095 : return NULL_TREE;
7096 227202 : gassign *stmt = dyn_cast <gassign *> (gimple_seq_first_stmt (stmts));
7097 125111 : if (!stmt
7098 125111 : || (!CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
7099 : && gimple_assign_rhs_code (stmt) != VIEW_CONVERT_EXPR
7100 : && gimple_assign_rhs_code (stmt) != NEGATE_EXPR
7101 : && gimple_assign_rhs_code (stmt) != BIT_FIELD_REF
7102 : && (gimple_assign_rhs_code (stmt) != BIT_AND_EXPR
7103 80 : || TREE_CODE (gimple_assign_rhs2 (stmt)) != INTEGER_CST)))
7104 : return NULL_TREE;
7105 :
7106 32735 : tree op = gimple_assign_rhs1 (stmt);
7107 32735 : if (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
7108 32735 : || gimple_assign_rhs_code (stmt) == BIT_FIELD_REF)
7109 20204 : op = TREE_OPERAND (op, 0);
7110 32735 : tree leader = TREE_CODE (op) == SSA_NAME ? eliminate_avail (bb, op) : op;
7111 32689 : if (!leader)
7112 : return NULL_TREE;
7113 :
7114 23024 : tree res;
7115 23024 : stmts = NULL;
7116 42222 : if (gimple_assign_rhs_code (stmt) == BIT_FIELD_REF)
7117 33990 : res = gimple_build (&stmts, BIT_FIELD_REF,
7118 16995 : TREE_TYPE (val), leader,
7119 16995 : TREE_OPERAND (gimple_assign_rhs1 (stmt), 1),
7120 16995 : TREE_OPERAND (gimple_assign_rhs1 (stmt), 2));
7121 6029 : else if (gimple_assign_rhs_code (stmt) == BIT_AND_EXPR)
7122 160 : res = gimple_build (&stmts, BIT_AND_EXPR,
7123 80 : TREE_TYPE (val), leader, gimple_assign_rhs2 (stmt));
7124 : else
7125 5949 : res = gimple_build (&stmts, gimple_assign_rhs_code (stmt),
7126 5949 : TREE_TYPE (val), leader);
7127 23024 : if (TREE_CODE (res) != SSA_NAME
7128 23023 : || SSA_NAME_IS_DEFAULT_DEF (res)
7129 46047 : || gimple_bb (SSA_NAME_DEF_STMT (res)))
7130 : {
7131 4 : gimple_seq_discard (stmts);
7132 :
7133 : /* During propagation we have to treat SSA info conservatively
7134 : and thus we can end up simplifying the inserted expression
7135 : at elimination time to sth not defined in stmts. */
7136 : /* But then this is a redundancy we failed to detect. Which means
7137 : res now has two values. That doesn't play well with how
7138 : we track availability here, so give up. */
7139 4 : if (dump_file && (dump_flags & TDF_DETAILS))
7140 : {
7141 0 : if (TREE_CODE (res) == SSA_NAME)
7142 0 : res = eliminate_avail (bb, res);
7143 0 : if (res)
7144 : {
7145 0 : fprintf (dump_file, "Failed to insert expression for value ");
7146 0 : print_generic_expr (dump_file, val);
7147 0 : fprintf (dump_file, " which is really fully redundant to ");
7148 0 : print_generic_expr (dump_file, res);
7149 0 : fprintf (dump_file, "\n");
7150 : }
7151 : }
7152 :
7153 4 : return NULL_TREE;
7154 : }
7155 : else
7156 : {
7157 23020 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
7158 23020 : vn_ssa_aux_t vn_info = VN_INFO (res);
7159 23020 : vn_info->valnum = val;
7160 23020 : vn_info->visited = true;
7161 : }
7162 :
7163 23020 : insertions++;
7164 23020 : if (dump_file && (dump_flags & TDF_DETAILS))
7165 : {
7166 501 : fprintf (dump_file, "Inserted ");
7167 501 : print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (res), 0);
7168 : }
7169 :
7170 : return res;
7171 : }
7172 :
7173 : void
7174 368164408 : eliminate_dom_walker::eliminate_stmt (basic_block b, gimple_stmt_iterator *gsi)
7175 : {
7176 368164408 : tree sprime = NULL_TREE;
7177 368164408 : gimple *stmt = gsi_stmt (*gsi);
7178 368164408 : tree lhs = gimple_get_lhs (stmt);
7179 122920664 : if (lhs && TREE_CODE (lhs) == SSA_NAME
7180 169906320 : && !gimple_has_volatile_ops (stmt)
7181 : /* See PR43491. Do not replace a global register variable when
7182 : it is a the RHS of an assignment. Do replace local register
7183 : variables since gcc does not guarantee a local variable will
7184 : be allocated in register.
7185 : ??? The fix isn't effective here. This should instead
7186 : be ensured by not value-numbering them the same but treating
7187 : them like volatiles? */
7188 452054262 : && !(gimple_assign_single_p (stmt)
7189 36259566 : && (TREE_CODE (gimple_assign_rhs1 (stmt)) == VAR_DECL
7190 2496733 : && DECL_HARD_REGISTER (gimple_assign_rhs1 (stmt))
7191 4184 : && is_global_var (gimple_assign_rhs1 (stmt)))))
7192 : {
7193 83889610 : sprime = eliminate_avail (b, lhs);
7194 83889610 : if (!sprime)
7195 : {
7196 : /* If there is no existing usable leader but SCCVN thinks
7197 : it has an expression it wants to use as replacement,
7198 : insert that. */
7199 70526270 : tree val = VN_INFO (lhs)->valnum;
7200 70526270 : vn_ssa_aux_t vn_info;
7201 70526270 : if (val != VN_TOP
7202 70526270 : && TREE_CODE (val) == SSA_NAME
7203 70526270 : && (vn_info = VN_INFO (val), true)
7204 70526270 : && vn_info->needs_insertion
7205 328148 : && vn_info->expr != NULL
7206 70651381 : && (sprime = eliminate_insert (b, gsi, val)) != NULL_TREE)
7207 23020 : eliminate_push_avail (b, sprime);
7208 : }
7209 :
7210 : /* If this now constitutes a copy duplicate points-to
7211 : and range info appropriately. This is especially
7212 : important for inserted code. */
7213 70526270 : if (sprime
7214 13386360 : && TREE_CODE (sprime) == SSA_NAME)
7215 9181370 : maybe_duplicate_ssa_info_at_copy (lhs, sprime);
7216 :
7217 : /* Inhibit the use of an inserted PHI on a loop header when
7218 : the address of the memory reference is a simple induction
7219 : variable. In other cases the vectorizer won't do anything
7220 : anyway (either it's loop invariant or a complicated
7221 : expression). */
7222 9181370 : if (sprime
7223 13386360 : && TREE_CODE (sprime) == SSA_NAME
7224 9181370 : && do_pre
7225 931290 : && (flag_tree_loop_vectorize || flag_tree_parallelize_loops > 1)
7226 912655 : && loop_outer (b->loop_father)
7227 391665 : && has_zero_uses (sprime)
7228 194593 : && bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (sprime))
7229 194379 : && gimple_assign_load_p (stmt))
7230 : {
7231 105306 : gimple *def_stmt = SSA_NAME_DEF_STMT (sprime);
7232 105306 : basic_block def_bb = gimple_bb (def_stmt);
7233 105306 : if (gimple_code (def_stmt) == GIMPLE_PHI
7234 105306 : && def_bb->loop_father->header == def_bb)
7235 : {
7236 66569 : loop_p loop = def_bb->loop_father;
7237 66569 : ssa_op_iter iter;
7238 66569 : tree op;
7239 66569 : bool found = false;
7240 84522 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
7241 : {
7242 62913 : affine_iv iv;
7243 62913 : def_bb = gimple_bb (SSA_NAME_DEF_STMT (op));
7244 62913 : if (def_bb
7245 56817 : && flow_bb_inside_loop_p (loop, def_bb)
7246 114585 : && simple_iv (loop, loop, op, &iv, true))
7247 : {
7248 44960 : found = true;
7249 44960 : break;
7250 : }
7251 : }
7252 21609 : if (found)
7253 : {
7254 44960 : if (dump_file && (dump_flags & TDF_DETAILS))
7255 : {
7256 3 : fprintf (dump_file, "Not replacing ");
7257 3 : print_gimple_expr (dump_file, stmt, 0);
7258 3 : fprintf (dump_file, " with ");
7259 3 : print_generic_expr (dump_file, sprime);
7260 3 : fprintf (dump_file, " which would add a loop"
7261 : " carried dependence to loop %d\n",
7262 : loop->num);
7263 : }
7264 : /* Don't keep sprime available. */
7265 44960 : sprime = NULL_TREE;
7266 : }
7267 : }
7268 : }
7269 :
7270 83889610 : if (sprime)
7271 : {
7272 : /* If we can propagate the value computed for LHS into
7273 : all uses don't bother doing anything with this stmt. */
7274 13341400 : if (may_propagate_copy (lhs, sprime))
7275 : {
7276 : /* Mark it for removal. */
7277 13339437 : to_remove.safe_push (stmt);
7278 :
7279 : /* ??? Don't count copy/constant propagations. */
7280 13339437 : if (gimple_assign_single_p (stmt)
7281 13339437 : && (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
7282 4674140 : || gimple_assign_rhs1 (stmt) == sprime))
7283 14192842 : return;
7284 :
7285 8120181 : if (dump_file && (dump_flags & TDF_DETAILS))
7286 : {
7287 19013 : fprintf (dump_file, "Replaced ");
7288 19013 : print_gimple_expr (dump_file, stmt, 0);
7289 19013 : fprintf (dump_file, " with ");
7290 19013 : print_generic_expr (dump_file, sprime);
7291 19013 : fprintf (dump_file, " in all uses of ");
7292 19013 : print_gimple_stmt (dump_file, stmt, 0);
7293 : }
7294 :
7295 8120181 : eliminations++;
7296 8120181 : return;
7297 : }
7298 :
7299 : /* If this is an assignment from our leader (which
7300 : happens in the case the value-number is a constant)
7301 : then there is nothing to do. Likewise if we run into
7302 : inserted code that needed a conversion because of
7303 : our type-agnostic value-numbering of loads. */
7304 1963 : if ((gimple_assign_single_p (stmt)
7305 1 : || (is_gimple_assign (stmt)
7306 1 : && (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
7307 0 : || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)))
7308 1964 : && sprime == gimple_assign_rhs1 (stmt))
7309 : return;
7310 :
7311 : /* Else replace its RHS. */
7312 719 : if (dump_file && (dump_flags & TDF_DETAILS))
7313 : {
7314 0 : fprintf (dump_file, "Replaced ");
7315 0 : print_gimple_expr (dump_file, stmt, 0);
7316 0 : fprintf (dump_file, " with ");
7317 0 : print_generic_expr (dump_file, sprime);
7318 0 : fprintf (dump_file, " in ");
7319 0 : print_gimple_stmt (dump_file, stmt, 0);
7320 : }
7321 719 : eliminations++;
7322 :
7323 719 : bool can_make_abnormal_goto = (is_gimple_call (stmt)
7324 719 : && stmt_can_make_abnormal_goto (stmt));
7325 719 : gimple *orig_stmt = stmt;
7326 719 : if (!useless_type_conversion_p (TREE_TYPE (lhs),
7327 719 : TREE_TYPE (sprime)))
7328 : {
7329 : /* We preserve conversions to but not from function or method
7330 : types. This asymmetry makes it necessary to re-instantiate
7331 : conversions here. */
7332 717 : if (POINTER_TYPE_P (TREE_TYPE (lhs))
7333 717 : && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (lhs))))
7334 717 : sprime = fold_convert (TREE_TYPE (lhs), sprime);
7335 : else
7336 0 : gcc_unreachable ();
7337 : }
7338 719 : tree vdef = gimple_vdef (stmt);
7339 719 : tree vuse = gimple_vuse (stmt);
7340 719 : propagate_tree_value_into_stmt (gsi, sprime);
7341 719 : stmt = gsi_stmt (*gsi);
7342 719 : update_stmt (stmt);
7343 : /* In case the VDEF on the original stmt was released, value-number
7344 : it to the VUSE. This is to make vuse_ssa_val able to skip
7345 : released virtual operands. */
7346 1438 : if (vdef != gimple_vdef (stmt))
7347 : {
7348 0 : gcc_assert (SSA_NAME_IN_FREE_LIST (vdef));
7349 0 : VN_INFO (vdef)->valnum = vuse;
7350 : }
7351 :
7352 : /* If we removed EH side-effects from the statement, clean
7353 : its EH information. */
7354 719 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
7355 : {
7356 0 : bitmap_set_bit (need_eh_cleanup,
7357 0 : gimple_bb (stmt)->index);
7358 0 : if (dump_file && (dump_flags & TDF_DETAILS))
7359 0 : fprintf (dump_file, " Removed EH side-effects.\n");
7360 : }
7361 :
7362 : /* Likewise for AB side-effects. */
7363 719 : if (can_make_abnormal_goto
7364 719 : && !stmt_can_make_abnormal_goto (stmt))
7365 : {
7366 0 : bitmap_set_bit (need_ab_cleanup,
7367 0 : gimple_bb (stmt)->index);
7368 0 : if (dump_file && (dump_flags & TDF_DETAILS))
7369 0 : fprintf (dump_file, " Removed AB side-effects.\n");
7370 : }
7371 :
7372 719 : return;
7373 : }
7374 : }
7375 :
7376 : /* If the statement is a scalar store, see if the expression
7377 : has the same value number as its rhs. If so, the store is
7378 : dead. */
7379 354823008 : if (gimple_assign_single_p (stmt)
7380 129967504 : && !gimple_has_volatile_ops (stmt)
7381 56475199 : && !is_gimple_reg (gimple_assign_lhs (stmt))
7382 29035479 : && (TREE_CODE (gimple_assign_lhs (stmt)) != VAR_DECL
7383 2849104 : || !DECL_HARD_REGISTER (gimple_assign_lhs (stmt)))
7384 383854478 : && (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
7385 16570919 : || is_gimple_min_invariant (gimple_assign_rhs1 (stmt))))
7386 : {
7387 25914503 : tree rhs = gimple_assign_rhs1 (stmt);
7388 25914503 : vn_reference_t vnresult;
7389 : /* ??? gcc.dg/torture/pr91445.c shows that we lookup a boolean
7390 : typed load of a byte known to be 0x11 as 1 so a store of
7391 : a boolean 1 is detected as redundant. Because of this we
7392 : have to make sure to lookup with a ref where its size
7393 : matches the precision. */
7394 25914503 : tree lookup_lhs = lhs;
7395 51562442 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
7396 13524504 : && (TREE_CODE (lhs) != COMPONENT_REF
7397 8190942 : || !DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
7398 39240498 : && !type_has_mode_precision_p (TREE_TYPE (lhs)))
7399 : {
7400 838822 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
7401 435662 : && TYPE_PRECISION (TREE_TYPE (lhs)) > MAX_FIXED_MODE_SIZE)
7402 : lookup_lhs = NULL_TREE;
7403 417617 : else if (TREE_CODE (lhs) == COMPONENT_REF
7404 417617 : || TREE_CODE (lhs) == MEM_REF)
7405 : {
7406 293401 : tree ltype = build_nonstandard_integer_type
7407 293401 : (TREE_INT_CST_LOW (TYPE_SIZE (TREE_TYPE (lhs))),
7408 293401 : TYPE_UNSIGNED (TREE_TYPE (lhs)));
7409 293401 : if (TREE_CODE (lhs) == COMPONENT_REF)
7410 : {
7411 225272 : tree foff = component_ref_field_offset (lhs);
7412 225272 : tree f = TREE_OPERAND (lhs, 1);
7413 225272 : if (!poly_int_tree_p (foff))
7414 : lookup_lhs = NULL_TREE;
7415 : else
7416 450544 : lookup_lhs = build3 (BIT_FIELD_REF, ltype,
7417 225272 : TREE_OPERAND (lhs, 0),
7418 225272 : TYPE_SIZE (TREE_TYPE (lhs)),
7419 : bit_from_pos
7420 225272 : (foff, DECL_FIELD_BIT_OFFSET (f)));
7421 : }
7422 : else
7423 68129 : lookup_lhs = build2 (MEM_REF, ltype,
7424 68129 : TREE_OPERAND (lhs, 0),
7425 68129 : TREE_OPERAND (lhs, 1));
7426 : }
7427 : else
7428 : lookup_lhs = NULL_TREE;
7429 : }
7430 25783080 : tree val = NULL_TREE, tem;
7431 25783080 : if (lookup_lhs)
7432 51566160 : val = vn_reference_lookup (lookup_lhs, gimple_vuse (stmt),
7433 : VN_WALKREWRITE, &vnresult, false,
7434 : NULL, NULL_TREE, true);
7435 25914503 : if (TREE_CODE (rhs) == SSA_NAME)
7436 12460551 : rhs = VN_INFO (rhs)->valnum;
7437 25914503 : gassign *ass;
7438 25914503 : if (val
7439 25914503 : && (operand_equal_p (val, rhs, 0)
7440 : /* Due to the bitfield lookups above we can get bit
7441 : interpretations of the same RHS as values here. Those
7442 : are redundant as well. */
7443 3160976 : || (TREE_CODE (val) == SSA_NAME
7444 1936232 : && gimple_assign_single_p (SSA_NAME_DEF_STMT (val))
7445 1759205 : && (tem = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (val)))
7446 1759205 : && TREE_CODE (tem) == VIEW_CONVERT_EXPR
7447 3534 : && TREE_OPERAND (tem, 0) == rhs)
7448 3160974 : || (TREE_CODE (rhs) == SSA_NAME
7449 26396313 : && (ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (rhs)))
7450 1507830 : && gimple_assign_rhs1 (ass) == val
7451 705883 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (ass))
7452 9 : && tree_nop_conversion_p (TREE_TYPE (rhs), TREE_TYPE (val)))))
7453 : {
7454 : /* We can only remove the later store if the former aliases
7455 : at least all accesses the later one does or if the store
7456 : was to readonly memory storing the same value. */
7457 244415 : ao_ref lhs_ref;
7458 244415 : ao_ref_init (&lhs_ref, lhs);
7459 244415 : alias_set_type set = ao_ref_alias_set (&lhs_ref);
7460 244415 : alias_set_type base_set = ao_ref_base_alias_set (&lhs_ref);
7461 244415 : if (! vnresult
7462 244415 : || ((vnresult->set == set
7463 52245 : || alias_set_subset_of (set, vnresult->set))
7464 226376 : && (vnresult->base_set == base_set
7465 21529 : || alias_set_subset_of (base_set, vnresult->base_set))))
7466 : {
7467 224073 : if (dump_file && (dump_flags & TDF_DETAILS))
7468 : {
7469 17 : fprintf (dump_file, "Deleted redundant store ");
7470 17 : print_gimple_stmt (dump_file, stmt, 0);
7471 : }
7472 :
7473 : /* Queue stmt for removal. */
7474 224073 : to_remove.safe_push (stmt);
7475 224073 : return;
7476 : }
7477 : }
7478 : }
7479 :
7480 : /* If this is a control statement value numbering left edges
7481 : unexecuted on force the condition in a way consistent with
7482 : that. */
7483 354598935 : if (gcond *cond = dyn_cast <gcond *> (stmt))
7484 : {
7485 19384206 : if ((EDGE_SUCC (b, 0)->flags & EDGE_EXECUTABLE)
7486 19384206 : ^ (EDGE_SUCC (b, 1)->flags & EDGE_EXECUTABLE))
7487 : {
7488 627369 : if (dump_file && (dump_flags & TDF_DETAILS))
7489 : {
7490 15 : fprintf (dump_file, "Removing unexecutable edge from ");
7491 15 : print_gimple_stmt (dump_file, stmt, 0);
7492 : }
7493 627369 : if (((EDGE_SUCC (b, 0)->flags & EDGE_TRUE_VALUE) != 0)
7494 627369 : == ((EDGE_SUCC (b, 0)->flags & EDGE_EXECUTABLE) != 0))
7495 247332 : gimple_cond_make_true (cond);
7496 : else
7497 380037 : gimple_cond_make_false (cond);
7498 627369 : update_stmt (cond);
7499 627369 : el_todo |= TODO_cleanup_cfg;
7500 627369 : return;
7501 : }
7502 : }
7503 :
7504 353971566 : bool can_make_abnormal_goto = stmt_can_make_abnormal_goto (stmt);
7505 353971566 : bool was_noreturn = (is_gimple_call (stmt)
7506 353971566 : && gimple_call_noreturn_p (stmt));
7507 353971566 : tree vdef = gimple_vdef (stmt);
7508 353971566 : tree vuse = gimple_vuse (stmt);
7509 :
7510 : /* If we didn't replace the whole stmt (or propagate the result
7511 : into all uses), replace all uses on this stmt with their
7512 : leaders. */
7513 353971566 : bool modified = false;
7514 353971566 : use_operand_p use_p;
7515 353971566 : ssa_op_iter iter;
7516 523153483 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
7517 : {
7518 169181917 : tree use = USE_FROM_PTR (use_p);
7519 : /* ??? The call code above leaves stmt operands un-updated. */
7520 169181917 : if (TREE_CODE (use) != SSA_NAME)
7521 0 : continue;
7522 169181917 : tree sprime;
7523 169181917 : if (SSA_NAME_IS_DEFAULT_DEF (use))
7524 : /* ??? For default defs BB shouldn't matter, but we have to
7525 : solve the inconsistency between rpo eliminate and
7526 : dom eliminate avail valueization first. */
7527 27116677 : sprime = eliminate_avail (b, use);
7528 : else
7529 : /* Look for sth available at the definition block of the argument.
7530 : This avoids inconsistencies between availability there which
7531 : decides if the stmt can be removed and availability at the
7532 : use site. The SSA property ensures that things available
7533 : at the definition are also available at uses. */
7534 142065240 : sprime = eliminate_avail (gimple_bb (SSA_NAME_DEF_STMT (use)), use);
7535 169181917 : if (sprime && sprime != use
7536 13455627 : && may_propagate_copy (use, sprime, true)
7537 : /* We substitute into debug stmts to avoid excessive
7538 : debug temporaries created by removed stmts, but we need
7539 : to avoid doing so for inserted sprimes as we never want
7540 : to create debug temporaries for them. */
7541 182636827 : && (!inserted_exprs
7542 1231451 : || TREE_CODE (sprime) != SSA_NAME
7543 1215664 : || !is_gimple_debug (stmt)
7544 397371 : || !bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (sprime))))
7545 : {
7546 13090818 : propagate_value (use_p, sprime);
7547 13090818 : modified = true;
7548 : }
7549 : }
7550 :
7551 : /* Fold the stmt if modified, this canonicalizes MEM_REFs we propagated
7552 : into which is a requirement for the IPA devirt machinery. */
7553 353971566 : gimple *old_stmt = stmt;
7554 353971566 : if (modified)
7555 : {
7556 : /* If a formerly non-invariant ADDR_EXPR is turned into an
7557 : invariant one it was on a separate stmt. */
7558 12175990 : if (gimple_assign_single_p (stmt)
7559 12175990 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == ADDR_EXPR)
7560 242683 : recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
7561 12175990 : gimple_stmt_iterator prev = *gsi;
7562 12175990 : gsi_prev (&prev);
7563 12175990 : if (fold_stmt (gsi, follow_all_ssa_edges))
7564 : {
7565 : /* fold_stmt may have created new stmts in between
7566 : the previous stmt and the folded stmt. Mark
7567 : all defs created there as varying to not confuse
7568 : the SCCVN machinery as we're using that even during
7569 : elimination. */
7570 1033295 : if (gsi_end_p (prev))
7571 222524 : prev = gsi_start_bb (b);
7572 : else
7573 922033 : gsi_next (&prev);
7574 1033295 : if (gsi_stmt (prev) != gsi_stmt (*gsi))
7575 100453 : do
7576 : {
7577 63022 : tree def;
7578 63022 : ssa_op_iter dit;
7579 121748 : FOR_EACH_SSA_TREE_OPERAND (def, gsi_stmt (prev),
7580 : dit, SSA_OP_ALL_DEFS)
7581 : /* As existing DEFs may move between stmts
7582 : only process new ones. */
7583 58726 : if (! has_VN_INFO (def))
7584 : {
7585 37329 : vn_ssa_aux_t vn_info = VN_INFO (def);
7586 37329 : vn_info->valnum = def;
7587 37329 : vn_info->visited = true;
7588 : }
7589 63022 : if (gsi_stmt (prev) == gsi_stmt (*gsi))
7590 : break;
7591 37431 : gsi_next (&prev);
7592 37431 : }
7593 : while (1);
7594 : }
7595 12175990 : stmt = gsi_stmt (*gsi);
7596 : /* In case we folded the stmt away schedule the NOP for removal. */
7597 12175990 : if (gimple_nop_p (stmt))
7598 823 : to_remove.safe_push (stmt);
7599 : }
7600 :
7601 : /* Visit indirect calls and turn them into direct calls if
7602 : possible using the devirtualization machinery. Do this before
7603 : checking for required EH/abnormal/noreturn cleanup as devird
7604 : may expose more of those. */
7605 353971566 : if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
7606 : {
7607 22793016 : tree fn = gimple_call_fn (call_stmt);
7608 22793016 : if (fn
7609 21980070 : && flag_devirtualize
7610 44029661 : && virtual_method_call_p (fn))
7611 : {
7612 186309 : tree otr_type = obj_type_ref_class (fn);
7613 186309 : unsigned HOST_WIDE_INT otr_tok
7614 186309 : = tree_to_uhwi (OBJ_TYPE_REF_TOKEN (fn));
7615 186309 : tree instance;
7616 186309 : ipa_polymorphic_call_context context (current_function_decl,
7617 186309 : fn, stmt, &instance);
7618 186309 : context.get_dynamic_type (instance, OBJ_TYPE_REF_OBJECT (fn),
7619 : otr_type, stmt, NULL);
7620 186309 : bool final;
7621 186309 : vec <cgraph_node *> targets
7622 186309 : = possible_polymorphic_call_targets (obj_type_ref_class (fn),
7623 : otr_tok, context, &final);
7624 186309 : if (dump_file)
7625 22 : dump_possible_polymorphic_call_targets (dump_file,
7626 : obj_type_ref_class (fn),
7627 : otr_tok, context);
7628 186603 : if (final && targets.length () <= 1 && dbg_cnt (devirt))
7629 : {
7630 72 : tree fn;
7631 72 : if (targets.length () == 1)
7632 72 : fn = targets[0]->decl;
7633 : else
7634 0 : fn = builtin_decl_unreachable ();
7635 72 : if (dump_enabled_p ())
7636 : {
7637 9 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, stmt,
7638 : "converting indirect call to "
7639 : "function %s\n",
7640 9 : lang_hooks.decl_printable_name (fn, 2));
7641 : }
7642 72 : gimple_call_set_fndecl (call_stmt, fn);
7643 : /* If changing the call to __builtin_unreachable
7644 : or similar noreturn function, adjust gimple_call_fntype
7645 : too. */
7646 72 : if (gimple_call_noreturn_p (call_stmt)
7647 0 : && VOID_TYPE_P (TREE_TYPE (TREE_TYPE (fn)))
7648 0 : && TYPE_ARG_TYPES (TREE_TYPE (fn))
7649 72 : && (TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fn)))
7650 0 : == void_type_node))
7651 0 : gimple_call_set_fntype (call_stmt, TREE_TYPE (fn));
7652 72 : maybe_remove_unused_call_args (cfun, call_stmt);
7653 72 : modified = true;
7654 : }
7655 : }
7656 : }
7657 :
7658 353971566 : if (modified)
7659 : {
7660 : /* When changing a call into a noreturn call, cfg cleanup
7661 : is needed to fix up the noreturn call. */
7662 12176011 : if (!was_noreturn
7663 12176011 : && is_gimple_call (stmt) && gimple_call_noreturn_p (stmt))
7664 56 : to_fixup.safe_push (stmt);
7665 : /* When changing a condition or switch into one we know what
7666 : edge will be executed, schedule a cfg cleanup. */
7667 12176011 : if ((gimple_code (stmt) == GIMPLE_COND
7668 1543760 : && (gimple_cond_true_p (as_a <gcond *> (stmt))
7669 1538220 : || gimple_cond_false_p (as_a <gcond *> (stmt))))
7670 13711453 : || (gimple_code (stmt) == GIMPLE_SWITCH
7671 7671 : && TREE_CODE (gimple_switch_index
7672 : (as_a <gswitch *> (stmt))) == INTEGER_CST))
7673 10120 : el_todo |= TODO_cleanup_cfg;
7674 : /* If we removed EH side-effects from the statement, clean
7675 : its EH information. */
7676 12176011 : if (maybe_clean_or_replace_eh_stmt (old_stmt, stmt))
7677 : {
7678 1958 : bitmap_set_bit (need_eh_cleanup,
7679 1958 : gimple_bb (stmt)->index);
7680 1958 : if (dump_file && (dump_flags & TDF_DETAILS))
7681 0 : fprintf (dump_file, " Removed EH side-effects.\n");
7682 : }
7683 : /* Likewise for AB side-effects. */
7684 12176011 : if (can_make_abnormal_goto
7685 12176011 : && !stmt_can_make_abnormal_goto (stmt))
7686 : {
7687 0 : bitmap_set_bit (need_ab_cleanup,
7688 0 : gimple_bb (stmt)->index);
7689 0 : if (dump_file && (dump_flags & TDF_DETAILS))
7690 0 : fprintf (dump_file, " Removed AB side-effects.\n");
7691 : }
7692 12176011 : update_stmt (stmt);
7693 : /* In case the VDEF on the original stmt was released, value-number
7694 : it to the VUSE. This is to make vuse_ssa_val able to skip
7695 : released virtual operands. */
7696 15490534 : if (vdef && SSA_NAME_IN_FREE_LIST (vdef))
7697 2143 : VN_INFO (vdef)->valnum = vuse;
7698 : }
7699 :
7700 : /* Make new values available - for fully redundant LHS we
7701 : continue with the next stmt above and skip this.
7702 : But avoid picking up dead defs. */
7703 353971566 : tree def;
7704 425850874 : FOR_EACH_SSA_TREE_OPERAND (def, stmt, iter, SSA_OP_DEF)
7705 71879308 : if (! has_zero_uses (def)
7706 71879308 : || (inserted_exprs
7707 212107 : && bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (def))))
7708 70447876 : eliminate_push_avail (b, def);
7709 : }
7710 :
7711 : /* Perform elimination for the basic-block B during the domwalk. */
7712 :
7713 : edge
7714 42111379 : eliminate_dom_walker::before_dom_children (basic_block b)
7715 : {
7716 : /* Mark new bb. */
7717 42111379 : avail_stack.safe_push (NULL_TREE);
7718 :
7719 : /* Skip unreachable blocks marked unreachable during the SCCVN domwalk. */
7720 42111379 : if (!(b->flags & BB_EXECUTABLE))
7721 : return NULL;
7722 :
7723 37184725 : vn_context_bb = b;
7724 :
7725 48820451 : for (gphi_iterator gsi = gsi_start_phis (b); !gsi_end_p (gsi);)
7726 : {
7727 11635726 : gphi *phi = gsi.phi ();
7728 11635726 : tree res = PHI_RESULT (phi);
7729 :
7730 23271452 : if (virtual_operand_p (res))
7731 : {
7732 5361886 : gsi_next (&gsi);
7733 5361886 : continue;
7734 : }
7735 :
7736 6273840 : tree sprime = eliminate_avail (b, res);
7737 6273840 : if (sprime
7738 6273840 : && sprime != res)
7739 : {
7740 442691 : if (dump_file && (dump_flags & TDF_DETAILS))
7741 : {
7742 20 : fprintf (dump_file, "Replaced redundant PHI node defining ");
7743 20 : print_generic_expr (dump_file, res);
7744 20 : fprintf (dump_file, " with ");
7745 20 : print_generic_expr (dump_file, sprime);
7746 20 : fprintf (dump_file, "\n");
7747 : }
7748 :
7749 : /* If we inserted this PHI node ourself, it's not an elimination. */
7750 442691 : if (! inserted_exprs
7751 561004 : || ! bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (res)))
7752 416774 : eliminations++;
7753 :
7754 : /* If we will propagate into all uses don't bother to do
7755 : anything. */
7756 442691 : if (may_propagate_copy (res, sprime))
7757 : {
7758 : /* Mark the PHI for removal. */
7759 442691 : to_remove.safe_push (phi);
7760 442691 : gsi_next (&gsi);
7761 442691 : continue;
7762 : }
7763 :
7764 0 : remove_phi_node (&gsi, false);
7765 :
7766 0 : if (!useless_type_conversion_p (TREE_TYPE (res), TREE_TYPE (sprime)))
7767 0 : sprime = fold_convert (TREE_TYPE (res), sprime);
7768 0 : gimple *stmt = gimple_build_assign (res, sprime);
7769 0 : gimple_stmt_iterator gsi2 = gsi_after_labels (b);
7770 0 : gsi_insert_before (&gsi2, stmt, GSI_NEW_STMT);
7771 0 : continue;
7772 0 : }
7773 :
7774 5831149 : eliminate_push_avail (b, res);
7775 5831149 : gsi_next (&gsi);
7776 : }
7777 :
7778 74369450 : for (gimple_stmt_iterator gsi = gsi_start_bb (b);
7779 293377207 : !gsi_end_p (gsi);
7780 256192482 : gsi_next (&gsi))
7781 256192482 : eliminate_stmt (b, &gsi);
7782 :
7783 : /* Replace destination PHI arguments. */
7784 37184725 : edge_iterator ei;
7785 37184725 : edge e;
7786 87751731 : FOR_EACH_EDGE (e, ei, b->succs)
7787 50567006 : if (e->flags & EDGE_EXECUTABLE)
7788 50005295 : for (gphi_iterator gsi = gsi_start_phis (e->dest);
7789 79937499 : !gsi_end_p (gsi);
7790 29932204 : gsi_next (&gsi))
7791 : {
7792 29932204 : gphi *phi = gsi.phi ();
7793 29932204 : use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
7794 29932204 : tree arg = USE_FROM_PTR (use_p);
7795 49513701 : if (TREE_CODE (arg) != SSA_NAME
7796 29932204 : || virtual_operand_p (arg))
7797 19581497 : continue;
7798 10350707 : tree sprime = eliminate_avail (b, arg);
7799 20701414 : if (sprime && may_propagate_copy (arg, sprime,
7800 10350707 : !(e->flags & EDGE_ABNORMAL)))
7801 10338417 : propagate_value (use_p, sprime);
7802 : }
7803 :
7804 37184725 : vn_context_bb = NULL;
7805 :
7806 37184725 : return NULL;
7807 : }
7808 :
7809 : /* Make no longer available leaders no longer available. */
7810 :
7811 : void
7812 42111379 : eliminate_dom_walker::after_dom_children (basic_block)
7813 : {
7814 42111379 : tree entry;
7815 93176842 : while ((entry = avail_stack.pop ()) != NULL_TREE)
7816 : {
7817 51065463 : tree valnum = VN_INFO (entry)->valnum;
7818 51065463 : tree old = avail[SSA_NAME_VERSION (valnum)];
7819 51065463 : if (old == entry)
7820 51020461 : avail[SSA_NAME_VERSION (valnum)] = NULL_TREE;
7821 : else
7822 45002 : avail[SSA_NAME_VERSION (valnum)] = entry;
7823 : }
7824 42111379 : }
7825 :
7826 : /* Remove queued stmts and perform delayed cleanups. */
7827 :
7828 : unsigned
7829 6262499 : eliminate_dom_walker::eliminate_cleanup (bool region_p)
7830 : {
7831 6262499 : statistics_counter_event (cfun, "Eliminated", eliminations);
7832 6262499 : statistics_counter_event (cfun, "Insertions", insertions);
7833 :
7834 : /* We cannot remove stmts during BB walk, especially not release SSA
7835 : names there as this confuses the VN machinery. The stmts ending
7836 : up in to_remove are either stores or simple copies.
7837 : Remove stmts in reverse order to make debug stmt creation possible. */
7838 34126247 : while (!to_remove.is_empty ())
7839 : {
7840 15338694 : bool do_release_defs = true;
7841 15338694 : gimple *stmt = to_remove.pop ();
7842 :
7843 : /* When we are value-numbering a region we do not require exit PHIs to
7844 : be present so we have to make sure to deal with uses outside of the
7845 : region of stmts that we thought are eliminated.
7846 : ??? Note we may be confused by uses in dead regions we didn't run
7847 : elimination on. Rather than checking individual uses we accept
7848 : dead copies to be generated here (gcc.c-torture/execute/20060905-1.c
7849 : contains such example). */
7850 15338694 : if (region_p)
7851 : {
7852 1810714 : if (gphi *phi = dyn_cast <gphi *> (stmt))
7853 : {
7854 1129476 : tree lhs = gimple_phi_result (phi);
7855 1129476 : if (!has_zero_uses (lhs))
7856 : {
7857 23799 : if (dump_file && (dump_flags & TDF_DETAILS))
7858 3 : fprintf (dump_file, "Keeping eliminated stmt live "
7859 : "as copy because of out-of-region uses\n");
7860 23799 : tree sprime = eliminate_avail (gimple_bb (stmt), lhs);
7861 23799 : gimple *copy = gimple_build_assign (lhs, sprime);
7862 23799 : gimple_stmt_iterator gsi
7863 23799 : = gsi_after_labels (gimple_bb (stmt));
7864 23799 : gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
7865 23799 : do_release_defs = false;
7866 : }
7867 : }
7868 681238 : else if (tree lhs = gimple_get_lhs (stmt))
7869 681238 : if (TREE_CODE (lhs) == SSA_NAME
7870 681238 : && !has_zero_uses (lhs))
7871 : {
7872 2043 : if (dump_file && (dump_flags & TDF_DETAILS))
7873 0 : fprintf (dump_file, "Keeping eliminated stmt live "
7874 : "as copy because of out-of-region uses\n");
7875 2043 : tree sprime = eliminate_avail (gimple_bb (stmt), lhs);
7876 2043 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
7877 2043 : if (is_gimple_assign (stmt))
7878 : {
7879 2043 : gimple_assign_set_rhs_from_tree (&gsi, sprime);
7880 2043 : stmt = gsi_stmt (gsi);
7881 2043 : update_stmt (stmt);
7882 2043 : if (maybe_clean_or_replace_eh_stmt (stmt, stmt))
7883 0 : bitmap_set_bit (need_eh_cleanup, gimple_bb (stmt)->index);
7884 2043 : continue;
7885 : }
7886 : else
7887 : {
7888 0 : gimple *copy = gimple_build_assign (lhs, sprime);
7889 0 : gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
7890 0 : do_release_defs = false;
7891 : }
7892 : }
7893 : }
7894 :
7895 15336651 : if (dump_file && (dump_flags & TDF_DETAILS))
7896 : {
7897 22045 : fprintf (dump_file, "Removing dead stmt ");
7898 22045 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
7899 : }
7900 :
7901 15336651 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
7902 15336651 : if (gimple_code (stmt) == GIMPLE_PHI)
7903 1774361 : remove_phi_node (&gsi, do_release_defs);
7904 : else
7905 : {
7906 13562290 : basic_block bb = gimple_bb (stmt);
7907 13562290 : unlink_stmt_vdef (stmt);
7908 13562290 : if (gsi_remove (&gsi, true))
7909 26582 : bitmap_set_bit (need_eh_cleanup, bb->index);
7910 13562290 : if (is_gimple_call (stmt) && stmt_can_make_abnormal_goto (stmt))
7911 2 : bitmap_set_bit (need_ab_cleanup, bb->index);
7912 13562290 : if (do_release_defs)
7913 13562290 : release_defs (stmt);
7914 : }
7915 :
7916 : /* Removing a stmt may expose a forwarder block. */
7917 15336651 : el_todo |= TODO_cleanup_cfg;
7918 : }
7919 :
7920 : /* Fixup stmts that became noreturn calls. This may require splitting
7921 : blocks and thus isn't possible during the dominator walk. Do this
7922 : in reverse order so we don't inadvertently remove a stmt we want to
7923 : fixup by visiting a dominating now noreturn call first. */
7924 6262555 : while (!to_fixup.is_empty ())
7925 : {
7926 56 : gimple *stmt = to_fixup.pop ();
7927 :
7928 56 : if (dump_file && (dump_flags & TDF_DETAILS))
7929 : {
7930 0 : fprintf (dump_file, "Fixing up noreturn call ");
7931 0 : print_gimple_stmt (dump_file, stmt, 0);
7932 : }
7933 :
7934 56 : if (fixup_noreturn_call (stmt))
7935 56 : el_todo |= TODO_cleanup_cfg;
7936 : }
7937 :
7938 6262499 : bool do_eh_cleanup = !bitmap_empty_p (need_eh_cleanup);
7939 6262499 : bool do_ab_cleanup = !bitmap_empty_p (need_ab_cleanup);
7940 :
7941 6262499 : if (do_eh_cleanup)
7942 10719 : gimple_purge_all_dead_eh_edges (need_eh_cleanup);
7943 :
7944 6262499 : if (do_ab_cleanup)
7945 2 : gimple_purge_all_dead_abnormal_call_edges (need_ab_cleanup);
7946 :
7947 6262499 : if (do_eh_cleanup || do_ab_cleanup)
7948 10721 : el_todo |= TODO_cleanup_cfg;
7949 :
7950 6262499 : return el_todo;
7951 : }
7952 :
7953 : /* Eliminate fully redundant computations. */
7954 :
7955 : unsigned
7956 4374223 : eliminate_with_rpo_vn (bitmap inserted_exprs)
7957 : {
7958 4374223 : eliminate_dom_walker walker (CDI_DOMINATORS, inserted_exprs);
7959 :
7960 4374223 : eliminate_dom_walker *saved_rpo_avail = rpo_avail;
7961 4374223 : rpo_avail = &walker;
7962 4374223 : walker.walk (cfun->cfg->x_entry_block_ptr);
7963 4374223 : rpo_avail = saved_rpo_avail;
7964 :
7965 4374223 : return walker.eliminate_cleanup ();
7966 4374223 : }
7967 :
7968 : static unsigned
7969 : do_rpo_vn_1 (function *fn, edge entry, bitmap exit_bbs,
7970 : bool iterate, bool eliminate, bool skip_entry_phis,
7971 : vn_lookup_kind kind);
7972 :
7973 : void
7974 976707 : run_rpo_vn (vn_lookup_kind kind)
7975 : {
7976 976707 : do_rpo_vn_1 (cfun, NULL, NULL, true, false, false, kind);
7977 :
7978 : /* ??? Prune requirement of these. */
7979 976707 : constant_to_value_id = new hash_table<vn_constant_hasher> (23);
7980 :
7981 : /* Initialize the value ids and prune out remaining VN_TOPs
7982 : from dead code. */
7983 976707 : tree name;
7984 976707 : unsigned i;
7985 48068582 : FOR_EACH_SSA_NAME (i, name, cfun)
7986 : {
7987 34114826 : vn_ssa_aux_t info = VN_INFO (name);
7988 34114826 : if (!info->visited
7989 34037002 : || info->valnum == VN_TOP)
7990 77824 : info->valnum = name;
7991 34114826 : if (info->valnum == name)
7992 32958714 : info->value_id = get_next_value_id ();
7993 1156112 : else if (is_gimple_min_invariant (info->valnum))
7994 39751 : info->value_id = get_or_alloc_constant_value_id (info->valnum);
7995 : }
7996 :
7997 : /* Propagate. */
7998 48068582 : FOR_EACH_SSA_NAME (i, name, cfun)
7999 : {
8000 34114826 : vn_ssa_aux_t info = VN_INFO (name);
8001 34114826 : if (TREE_CODE (info->valnum) == SSA_NAME
8002 34075075 : && info->valnum != name
8003 35231187 : && info->value_id != VN_INFO (info->valnum)->value_id)
8004 1116361 : info->value_id = VN_INFO (info->valnum)->value_id;
8005 : }
8006 :
8007 976707 : set_hashtable_value_ids ();
8008 :
8009 976707 : if (dump_file && (dump_flags & TDF_DETAILS))
8010 : {
8011 14 : fprintf (dump_file, "Value numbers:\n");
8012 406 : FOR_EACH_SSA_NAME (i, name, cfun)
8013 : {
8014 307 : if (VN_INFO (name)->visited
8015 307 : && SSA_VAL (name) != name)
8016 : {
8017 33 : print_generic_expr (dump_file, name);
8018 33 : fprintf (dump_file, " = ");
8019 33 : print_generic_expr (dump_file, SSA_VAL (name));
8020 33 : fprintf (dump_file, " (%04d)\n", VN_INFO (name)->value_id);
8021 : }
8022 : }
8023 : }
8024 976707 : }
8025 :
8026 : /* Free VN associated data structures. */
8027 :
8028 : void
8029 6282091 : free_rpo_vn (void)
8030 : {
8031 6282091 : free_vn_table (valid_info);
8032 6282091 : XDELETE (valid_info);
8033 6282091 : obstack_free (&vn_tables_obstack, NULL);
8034 6282091 : obstack_free (&vn_tables_insert_obstack, NULL);
8035 :
8036 6282091 : vn_ssa_aux_iterator_type it;
8037 6282091 : vn_ssa_aux_t info;
8038 358113017 : FOR_EACH_HASH_TABLE_ELEMENT (*vn_ssa_aux_hash, info, vn_ssa_aux_t, it)
8039 175915463 : if (info->needs_insertion)
8040 4180207 : release_ssa_name (info->name);
8041 6282091 : obstack_free (&vn_ssa_aux_obstack, NULL);
8042 6282091 : delete vn_ssa_aux_hash;
8043 :
8044 6282091 : delete constant_to_value_id;
8045 6282091 : constant_to_value_id = NULL;
8046 6282091 : }
8047 :
8048 : /* Hook for maybe_push_res_to_seq, lookup the expression in the VN tables. */
8049 :
8050 : static tree
8051 23555400 : vn_lookup_simplify_result (gimple_match_op *res_op)
8052 : {
8053 23555400 : if (!res_op->code.is_tree_code ())
8054 : return NULL_TREE;
8055 23552191 : tree *ops = res_op->ops;
8056 23552191 : unsigned int length = res_op->num_ops;
8057 23552191 : if (res_op->code == CONSTRUCTOR
8058 : /* ??? We're arriving here with SCCVNs view, decomposed CONSTRUCTOR
8059 : and GIMPLEs / match-and-simplifies, CONSTRUCTOR as GENERIC tree. */
8060 23552191 : && TREE_CODE (res_op->ops[0]) == CONSTRUCTOR)
8061 : {
8062 1052 : length = CONSTRUCTOR_NELTS (res_op->ops[0]);
8063 1052 : ops = XALLOCAVEC (tree, length);
8064 4752 : for (unsigned i = 0; i < length; ++i)
8065 3700 : ops[i] = CONSTRUCTOR_ELT (res_op->ops[0], i)->value;
8066 : }
8067 23552191 : vn_nary_op_t vnresult = NULL;
8068 23552191 : tree res = vn_nary_op_lookup_pieces (length, (tree_code) res_op->code,
8069 : res_op->type, ops, &vnresult);
8070 : /* If this is used from expression simplification make sure to
8071 : return an available expression. */
8072 23552191 : if (res && TREE_CODE (res) == SSA_NAME && mprts_hook && rpo_avail)
8073 2300902 : res = rpo_avail->eliminate_avail (vn_context_bb, res);
8074 : return res;
8075 : }
8076 :
8077 : /* Return a leader for OPs value that is valid at BB. */
8078 :
8079 : tree
8080 275365475 : rpo_elim::eliminate_avail (basic_block bb, tree op)
8081 : {
8082 275365475 : bool visited;
8083 275365475 : tree valnum = SSA_VAL (op, &visited);
8084 : /* If we didn't visit OP then it must be defined outside of the
8085 : region we process and also dominate it. So it is available. */
8086 275365475 : if (!visited)
8087 : return op;
8088 273163396 : if (TREE_CODE (valnum) == SSA_NAME)
8089 : {
8090 258644659 : if (SSA_NAME_IS_DEFAULT_DEF (valnum))
8091 : return valnum;
8092 251820865 : vn_ssa_aux_t valnum_info = VN_INFO (valnum);
8093 251820865 : vn_avail *av = valnum_info->avail;
8094 251820865 : if (!av)
8095 : {
8096 : /* See above. But when there's availability info prefer
8097 : what we recorded there for example to preserve LC SSA. */
8098 85266657 : if (!valnum_info->visited)
8099 : return valnum;
8100 : return NULL_TREE;
8101 : }
8102 166554208 : if (av->location == bb->index)
8103 : /* On tramp3d 90% of the cases are here. */
8104 110233836 : return ssa_name (av->leader);
8105 70563325 : do
8106 : {
8107 70563325 : basic_block abb = BASIC_BLOCK_FOR_FN (cfun, av->location);
8108 : /* ??? During elimination we have to use availability at the
8109 : definition site of a use we try to replace. This
8110 : is required to not run into inconsistencies because
8111 : of dominated_by_p_w_unex behavior and removing a definition
8112 : while not replacing all uses.
8113 : ??? We could try to consistently walk dominators
8114 : ignoring non-executable regions. The nearest common
8115 : dominator of bb and abb is where we can stop walking. We
8116 : may also be able to "pre-compute" (bits of) the next immediate
8117 : (non-)dominator during the RPO walk when marking edges as
8118 : executable. */
8119 70563325 : if (dominated_by_p_w_unex (bb, abb, true))
8120 : {
8121 52321997 : tree leader = ssa_name (av->leader);
8122 : /* Prevent eliminations that break loop-closed SSA. */
8123 52321997 : if (loops_state_satisfies_p (LOOP_CLOSED_SSA)
8124 3353761 : && ! SSA_NAME_IS_DEFAULT_DEF (leader)
8125 55675758 : && ! flow_bb_inside_loop_p (gimple_bb (SSA_NAME_DEF_STMT
8126 3353761 : (leader))->loop_father,
8127 : bb))
8128 : return NULL_TREE;
8129 52242341 : if (dump_file && (dump_flags & TDF_DETAILS))
8130 : {
8131 3538 : print_generic_expr (dump_file, leader);
8132 3538 : fprintf (dump_file, " is available for ");
8133 3538 : print_generic_expr (dump_file, valnum);
8134 3538 : fprintf (dump_file, "\n");
8135 : }
8136 : /* On tramp3d 99% of the _remaining_ cases succeed at
8137 : the first enty. */
8138 52242341 : return leader;
8139 : }
8140 : /* ??? Can we somehow skip to the immediate dominator
8141 : RPO index (bb_to_rpo)? Again, maybe not worth, on
8142 : tramp3d the worst number of elements in the vector is 9. */
8143 18241328 : av = av->next;
8144 : }
8145 18241328 : while (av);
8146 : /* While we prefer avail we have to fallback to using the value
8147 : directly if defined outside of the region when none of the
8148 : available defs suit. */
8149 3998375 : if (!valnum_info->visited)
8150 : return valnum;
8151 : }
8152 14518737 : else if (valnum != VN_TOP)
8153 : /* valnum is is_gimple_min_invariant. */
8154 : return valnum;
8155 : return NULL_TREE;
8156 : }
8157 :
8158 : /* Make LEADER a leader for its value at BB. */
8159 :
8160 : void
8161 98836317 : rpo_elim::eliminate_push_avail (basic_block bb, tree leader)
8162 : {
8163 98836317 : tree valnum = VN_INFO (leader)->valnum;
8164 98836317 : if (valnum == VN_TOP
8165 98836317 : || is_gimple_min_invariant (valnum))
8166 0 : return;
8167 98836317 : if (dump_file && (dump_flags & TDF_DETAILS))
8168 : {
8169 325025 : fprintf (dump_file, "Making available beyond BB%d ", bb->index);
8170 325025 : print_generic_expr (dump_file, leader);
8171 325025 : fprintf (dump_file, " for value ");
8172 325025 : print_generic_expr (dump_file, valnum);
8173 325025 : fprintf (dump_file, "\n");
8174 : }
8175 98836317 : vn_ssa_aux_t value = VN_INFO (valnum);
8176 98836317 : vn_avail *av;
8177 98836317 : if (m_avail_freelist)
8178 : {
8179 19004532 : av = m_avail_freelist;
8180 19004532 : m_avail_freelist = m_avail_freelist->next;
8181 : }
8182 : else
8183 79831785 : av = XOBNEW (&vn_ssa_aux_obstack, vn_avail);
8184 98836317 : av->location = bb->index;
8185 98836317 : av->leader = SSA_NAME_VERSION (leader);
8186 98836317 : av->next = value->avail;
8187 98836317 : av->next_undo = last_pushed_avail;
8188 98836317 : last_pushed_avail = value;
8189 98836317 : value->avail = av;
8190 : }
8191 :
8192 : /* Valueization hook for RPO VN plus required state. */
8193 :
8194 : tree
8195 2109728922 : rpo_vn_valueize (tree name)
8196 : {
8197 2109728922 : if (TREE_CODE (name) == SSA_NAME)
8198 : {
8199 2062940983 : vn_ssa_aux_t val = VN_INFO (name);
8200 2062940983 : if (val)
8201 : {
8202 2062940983 : tree tem = val->valnum;
8203 2062940983 : if (tem != VN_TOP && tem != name)
8204 : {
8205 112298400 : if (TREE_CODE (tem) != SSA_NAME)
8206 : return tem;
8207 : /* For all values we only valueize to an available leader
8208 : which means we can use SSA name info without restriction. */
8209 94891380 : tem = rpo_avail->eliminate_avail (vn_context_bb, tem);
8210 94891380 : if (tem)
8211 : return tem;
8212 : }
8213 : }
8214 : }
8215 : return name;
8216 : }
8217 :
8218 : /* Insert on PRED_E predicates derived from CODE OPS being true besides the
8219 : inverted condition. */
8220 :
8221 : static void
8222 27852955 : insert_related_predicates_on_edge (enum tree_code code, tree *ops, edge pred_e)
8223 : {
8224 27852955 : switch (code)
8225 : {
8226 1386348 : case LT_EXPR:
8227 : /* a < b -> a {!,<}= b */
8228 1386348 : vn_nary_op_insert_pieces_predicated (2, NE_EXPR, boolean_type_node,
8229 : ops, boolean_true_node, 0, pred_e);
8230 1386348 : vn_nary_op_insert_pieces_predicated (2, LE_EXPR, boolean_type_node,
8231 : ops, boolean_true_node, 0, pred_e);
8232 : /* a < b -> ! a {>,=} b */
8233 1386348 : vn_nary_op_insert_pieces_predicated (2, GT_EXPR, boolean_type_node,
8234 : ops, boolean_false_node, 0, pred_e);
8235 1386348 : vn_nary_op_insert_pieces_predicated (2, EQ_EXPR, boolean_type_node,
8236 : ops, boolean_false_node, 0, pred_e);
8237 1386348 : break;
8238 3521919 : case GT_EXPR:
8239 : /* a > b -> a {!,>}= b */
8240 3521919 : vn_nary_op_insert_pieces_predicated (2, NE_EXPR, boolean_type_node,
8241 : ops, boolean_true_node, 0, pred_e);
8242 3521919 : vn_nary_op_insert_pieces_predicated (2, GE_EXPR, boolean_type_node,
8243 : ops, boolean_true_node, 0, pred_e);
8244 : /* a > b -> ! a {<,=} b */
8245 3521919 : vn_nary_op_insert_pieces_predicated (2, LT_EXPR, boolean_type_node,
8246 : ops, boolean_false_node, 0, pred_e);
8247 3521919 : vn_nary_op_insert_pieces_predicated (2, EQ_EXPR, boolean_type_node,
8248 : ops, boolean_false_node, 0, pred_e);
8249 3521919 : break;
8250 9539551 : case EQ_EXPR:
8251 : /* a == b -> ! a {<,>} b */
8252 9539551 : vn_nary_op_insert_pieces_predicated (2, LT_EXPR, boolean_type_node,
8253 : ops, boolean_false_node, 0, pred_e);
8254 9539551 : vn_nary_op_insert_pieces_predicated (2, GT_EXPR, boolean_type_node,
8255 : ops, boolean_false_node, 0, pred_e);
8256 9539551 : break;
8257 : case LE_EXPR:
8258 : case GE_EXPR:
8259 : case NE_EXPR:
8260 : /* Nothing besides inverted condition. */
8261 : break;
8262 27852955 : default:;
8263 : }
8264 27852955 : }
8265 :
8266 : /* Insert on the TRUE_E true and FALSE_E false predicates
8267 : derived from LHS CODE RHS. */
8268 :
8269 : static void
8270 23832822 : insert_predicates_for_cond (tree_code code, tree lhs, tree rhs,
8271 : edge true_e, edge false_e)
8272 : {
8273 : /* If both edges are null, then there is nothing to be done. */
8274 23832822 : if (!true_e && !false_e)
8275 1363176 : return;
8276 :
8277 : /* Canonicalize the comparison if needed, putting
8278 : the constant in the rhs. */
8279 22473138 : if (tree_swap_operands_p (lhs, rhs))
8280 : {
8281 16879 : std::swap (lhs, rhs);
8282 16879 : code = swap_tree_comparison (code);
8283 : }
8284 :
8285 : /* If the lhs is not a ssa name, don't record anything. */
8286 22473138 : if (TREE_CODE (lhs) != SSA_NAME)
8287 : return;
8288 :
8289 22469646 : tree_code icode = invert_tree_comparison (code, HONOR_NANS (lhs));
8290 22469646 : tree ops[2];
8291 22469646 : ops[0] = lhs;
8292 22469646 : ops[1] = rhs;
8293 22469646 : if (true_e)
8294 18344188 : vn_nary_op_insert_pieces_predicated (2, code, boolean_type_node, ops,
8295 : boolean_true_node, 0, true_e);
8296 22469646 : if (false_e)
8297 17267112 : vn_nary_op_insert_pieces_predicated (2, code, boolean_type_node, ops,
8298 : boolean_false_node, 0, false_e);
8299 22469646 : if (icode != ERROR_MARK)
8300 : {
8301 22218375 : if (true_e)
8302 18187552 : vn_nary_op_insert_pieces_predicated (2, icode, boolean_type_node, ops,
8303 : boolean_false_node, 0, true_e);
8304 22218375 : if (false_e)
8305 17063987 : vn_nary_op_insert_pieces_predicated (2, icode, boolean_type_node, ops,
8306 : boolean_true_node, 0, false_e);
8307 : }
8308 : /* Relax for non-integers, inverted condition handled
8309 : above. */
8310 22469646 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
8311 : {
8312 17575391 : if (true_e)
8313 14413445 : insert_related_predicates_on_edge (code, ops, true_e);
8314 17575391 : if (false_e)
8315 13439510 : insert_related_predicates_on_edge (icode, ops, false_e);
8316 : }
8317 22469646 : if (integer_zerop (rhs)
8318 22469646 : && (code == NE_EXPR || code == EQ_EXPR))
8319 : {
8320 9378738 : gimple *def_stmt = SSA_NAME_DEF_STMT (lhs);
8321 : /* (A CMP B) != 0 is the same as (A CMP B).
8322 : (A CMP B) == 0 is just (A CMP B) with the edges swapped. */
8323 9378738 : if (is_gimple_assign (def_stmt)
8324 9378738 : && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_comparison)
8325 : {
8326 440680 : tree_code nc = gimple_assign_rhs_code (def_stmt);
8327 440680 : tree nlhs = vn_valueize (gimple_assign_rhs1 (def_stmt));
8328 440680 : tree nrhs = vn_valueize (gimple_assign_rhs2 (def_stmt));
8329 440680 : edge nt = true_e;
8330 440680 : edge nf = false_e;
8331 440680 : if (code == EQ_EXPR)
8332 314868 : std::swap (nt, nf);
8333 440680 : if (lhs != nlhs)
8334 440680 : insert_predicates_for_cond (nc, nlhs, nrhs, nt, nf);
8335 : }
8336 : /* (a | b) == 0 ->
8337 : on true edge assert: a == 0 & b == 0. */
8338 : /* (a | b) != 0 ->
8339 : on false edge assert: a == 0 & b == 0. */
8340 9378738 : if (is_gimple_assign (def_stmt)
8341 9378738 : && gimple_assign_rhs_code (def_stmt) == BIT_IOR_EXPR)
8342 : {
8343 263503 : edge e = code == EQ_EXPR ? true_e : false_e;
8344 263503 : tree nlhs;
8345 :
8346 263503 : nlhs = vn_valueize (gimple_assign_rhs1 (def_stmt));
8347 : /* A valueization of the `a` might return the old lhs
8348 : which is already handled above. */
8349 263503 : if (nlhs != lhs)
8350 263503 : insert_predicates_for_cond (EQ_EXPR, nlhs, rhs, e, nullptr);
8351 :
8352 : /* A valueization of the `b` might return the old lhs
8353 : which is already handled above. */
8354 263503 : nlhs = vn_valueize (gimple_assign_rhs2 (def_stmt));
8355 263503 : if (nlhs != lhs)
8356 263503 : insert_predicates_for_cond (EQ_EXPR, nlhs, rhs, e, nullptr);
8357 : }
8358 : }
8359 : }
8360 :
8361 : /* Main stmt worker for RPO VN, process BB. */
8362 :
8363 : static unsigned
8364 62581088 : process_bb (rpo_elim &avail, basic_block bb,
8365 : bool bb_visited, bool iterate_phis, bool iterate, bool eliminate,
8366 : bool do_region, bitmap exit_bbs, bool skip_phis)
8367 : {
8368 62581088 : unsigned todo = 0;
8369 62581088 : edge_iterator ei;
8370 62581088 : edge e;
8371 :
8372 62581088 : vn_context_bb = bb;
8373 :
8374 : /* If we are in loop-closed SSA preserve this state. This is
8375 : relevant when called on regions from outside of FRE/PRE. */
8376 62581088 : bool lc_phi_nodes = false;
8377 62581088 : if (!skip_phis
8378 62581088 : && loops_state_satisfies_p (LOOP_CLOSED_SSA))
8379 3829012 : FOR_EACH_EDGE (e, ei, bb->preds)
8380 2315005 : if (e->src->loop_father != e->dest->loop_father
8381 2315005 : && flow_loop_nested_p (e->dest->loop_father,
8382 : e->src->loop_father))
8383 : {
8384 : lc_phi_nodes = true;
8385 : break;
8386 : }
8387 :
8388 : /* When we visit a loop header substitute into loop info. */
8389 62581088 : if (!iterate && eliminate && bb->loop_father->header == bb)
8390 : {
8391 : /* Keep fields in sync with substitute_in_loop_info. */
8392 950289 : if (bb->loop_father->nb_iterations)
8393 155693 : bb->loop_father->nb_iterations
8394 155693 : = simplify_replace_tree (bb->loop_father->nb_iterations,
8395 : NULL_TREE, NULL_TREE, &vn_valueize_for_srt);
8396 : }
8397 :
8398 : /* Value-number all defs in the basic-block. */
8399 62581088 : if (!skip_phis)
8400 89710181 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
8401 27158457 : gsi_next (&gsi))
8402 : {
8403 27158457 : gphi *phi = gsi.phi ();
8404 27158457 : tree res = PHI_RESULT (phi);
8405 27158457 : vn_ssa_aux_t res_info = VN_INFO (res);
8406 27158457 : if (!bb_visited)
8407 : {
8408 19187172 : gcc_assert (!res_info->visited);
8409 19187172 : res_info->valnum = VN_TOP;
8410 19187172 : res_info->visited = true;
8411 : }
8412 :
8413 : /* When not iterating force backedge values to varying. */
8414 27158457 : visit_stmt (phi, !iterate_phis);
8415 54316914 : if (virtual_operand_p (res))
8416 10785008 : continue;
8417 :
8418 : /* Eliminate */
8419 : /* The interesting case is gcc.dg/tree-ssa/pr22230.c for correctness
8420 : how we handle backedges and availability.
8421 : And gcc.dg/tree-ssa/ssa-sccvn-2.c for optimization. */
8422 16373449 : tree val = res_info->valnum;
8423 16373449 : if (res != val && !iterate && eliminate)
8424 : {
8425 1452350 : if (tree leader = avail.eliminate_avail (bb, res))
8426 : {
8427 1332201 : if (leader != res
8428 : /* Preserve loop-closed SSA form. */
8429 1332201 : && (! lc_phi_nodes
8430 6400 : || is_gimple_min_invariant (leader)))
8431 : {
8432 1331670 : if (dump_file && (dump_flags & TDF_DETAILS))
8433 : {
8434 209 : fprintf (dump_file, "Replaced redundant PHI node "
8435 : "defining ");
8436 209 : print_generic_expr (dump_file, res);
8437 209 : fprintf (dump_file, " with ");
8438 209 : print_generic_expr (dump_file, leader);
8439 209 : fprintf (dump_file, "\n");
8440 : }
8441 1331670 : avail.eliminations++;
8442 :
8443 1331670 : if (may_propagate_copy (res, leader))
8444 : {
8445 : /* Schedule for removal. */
8446 1331670 : avail.to_remove.safe_push (phi);
8447 1331670 : continue;
8448 : }
8449 : /* ??? Else generate a copy stmt. */
8450 : }
8451 : }
8452 : }
8453 : /* Only make defs available that not already are. But make
8454 : sure loop-closed SSA PHI node defs are picked up for
8455 : downstream uses. */
8456 15041779 : if (lc_phi_nodes
8457 15041779 : || res == val
8458 15041779 : || ! avail.eliminate_avail (bb, res))
8459 11491478 : avail.eliminate_push_avail (bb, res);
8460 : }
8461 :
8462 : /* For empty BBs mark outgoing edges executable. For non-empty BBs
8463 : we do this when processing the last stmt as we have to do this
8464 : before elimination which otherwise forces GIMPLE_CONDs to
8465 : if (1 != 0) style when seeing non-executable edges. */
8466 125162176 : if (gsi_end_p (gsi_start_bb (bb)))
8467 : {
8468 14104070 : FOR_EACH_EDGE (e, ei, bb->succs)
8469 : {
8470 7052035 : if (!(e->flags & EDGE_EXECUTABLE))
8471 : {
8472 4827910 : if (dump_file && (dump_flags & TDF_DETAILS))
8473 6214 : fprintf (dump_file,
8474 : "marking outgoing edge %d -> %d executable\n",
8475 6214 : e->src->index, e->dest->index);
8476 4827910 : e->flags |= EDGE_EXECUTABLE;
8477 4827910 : e->dest->flags |= BB_EXECUTABLE;
8478 : }
8479 2224125 : else if (!(e->dest->flags & BB_EXECUTABLE))
8480 : {
8481 0 : if (dump_file && (dump_flags & TDF_DETAILS))
8482 0 : fprintf (dump_file,
8483 : "marking destination block %d reachable\n",
8484 : e->dest->index);
8485 0 : e->dest->flags |= BB_EXECUTABLE;
8486 : }
8487 : }
8488 : }
8489 125162176 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
8490 510181368 : !gsi_end_p (gsi); gsi_next (&gsi))
8491 : {
8492 447600280 : ssa_op_iter i;
8493 447600280 : tree op;
8494 447600280 : if (!bb_visited)
8495 : {
8496 508622856 : FOR_EACH_SSA_TREE_OPERAND (op, gsi_stmt (gsi), i, SSA_OP_ALL_DEFS)
8497 : {
8498 140797592 : vn_ssa_aux_t op_info = VN_INFO (op);
8499 140797592 : gcc_assert (!op_info->visited);
8500 140797592 : op_info->valnum = VN_TOP;
8501 140797592 : op_info->visited = true;
8502 : }
8503 :
8504 : /* We somehow have to deal with uses that are not defined
8505 : in the processed region. Forcing unvisited uses to
8506 : varying here doesn't play well with def-use following during
8507 : expression simplification, so we deal with this by checking
8508 : the visited flag in SSA_VAL. */
8509 : }
8510 :
8511 447600280 : visit_stmt (gsi_stmt (gsi));
8512 :
8513 447600280 : gimple *last = gsi_stmt (gsi);
8514 447600280 : e = NULL;
8515 447600280 : switch (gimple_code (last))
8516 : {
8517 114526 : case GIMPLE_SWITCH:
8518 114526 : e = find_taken_edge (bb, vn_valueize (gimple_switch_index
8519 114526 : (as_a <gswitch *> (last))));
8520 114526 : break;
8521 25123181 : case GIMPLE_COND:
8522 25123181 : {
8523 25123181 : tree lhs = vn_valueize (gimple_cond_lhs (last));
8524 25123181 : tree rhs = vn_valueize (gimple_cond_rhs (last));
8525 25123181 : tree_code cmpcode = gimple_cond_code (last);
8526 : /* Canonicalize the comparison if needed, putting
8527 : the constant in the rhs. */
8528 25123181 : if (tree_swap_operands_p (lhs, rhs))
8529 : {
8530 846208 : std::swap (lhs, rhs);
8531 846208 : cmpcode = swap_tree_comparison (cmpcode);
8532 : }
8533 25123181 : tree val = gimple_simplify (cmpcode,
8534 : boolean_type_node, lhs, rhs,
8535 : NULL, vn_valueize);
8536 : /* If the condition didn't simplify see if we have recorded
8537 : an expression from sofar taken edges. */
8538 25123181 : if (! val || TREE_CODE (val) != INTEGER_CST)
8539 : {
8540 23231980 : vn_nary_op_t vnresult;
8541 23231980 : tree ops[2];
8542 23231980 : ops[0] = lhs;
8543 23231980 : ops[1] = rhs;
8544 23231980 : val = vn_nary_op_lookup_pieces (2, cmpcode,
8545 : boolean_type_node, ops,
8546 : &vnresult);
8547 : /* Got back a ssa name, then try looking up `val != 0`
8548 : as it might have been recorded that way. */
8549 23231980 : if (val && TREE_CODE (val) == SSA_NAME)
8550 : {
8551 173491 : ops[0] = val;
8552 173491 : ops[1] = build_zero_cst (TREE_TYPE (val));
8553 173491 : val = vn_nary_op_lookup_pieces (2, NE_EXPR,
8554 : boolean_type_node, ops,
8555 : &vnresult);
8556 : }
8557 : /* Did we get a predicated value? */
8558 23231964 : if (! val && vnresult && vnresult->predicated_values)
8559 : {
8560 1420775 : val = vn_nary_op_get_predicated_value (vnresult, bb);
8561 1420775 : if (val && dump_file && (dump_flags & TDF_DETAILS))
8562 : {
8563 2 : fprintf (dump_file, "Got predicated value ");
8564 2 : print_generic_expr (dump_file, val, TDF_NONE);
8565 2 : fprintf (dump_file, " for ");
8566 2 : print_gimple_stmt (dump_file, last, TDF_SLIM);
8567 : }
8568 : }
8569 : }
8570 23231980 : if (val)
8571 2258045 : e = find_taken_edge (bb, val);
8572 25123181 : if (! e)
8573 : {
8574 : /* If we didn't manage to compute the taken edge then
8575 : push predicated expressions for the condition itself
8576 : and related conditions to the hashtables. This allows
8577 : simplification of redundant conditions which is
8578 : important as early cleanup. */
8579 22865136 : edge true_e, false_e;
8580 22865136 : extract_true_false_edges_from_block (bb, &true_e, &false_e);
8581 555468 : if ((do_region && bitmap_bit_p (exit_bbs, true_e->dest->index))
8582 23103892 : || !can_track_predicate_on_edge (true_e))
8583 5044302 : true_e = NULL;
8584 555468 : if ((do_region && bitmap_bit_p (exit_bbs, false_e->dest->index))
8585 23077890 : || !can_track_predicate_on_edge (false_e))
8586 6001046 : false_e = NULL;
8587 22865136 : insert_predicates_for_cond (cmpcode, lhs, rhs, true_e, false_e);
8588 : }
8589 : break;
8590 : }
8591 1436 : case GIMPLE_GOTO:
8592 1436 : e = find_taken_edge (bb, vn_valueize (gimple_goto_dest (last)));
8593 1436 : break;
8594 : default:
8595 : e = NULL;
8596 : }
8597 447600280 : if (e)
8598 : {
8599 2261677 : todo = TODO_cleanup_cfg;
8600 2261677 : if (!(e->flags & EDGE_EXECUTABLE))
8601 : {
8602 1787832 : if (dump_file && (dump_flags & TDF_DETAILS))
8603 35 : fprintf (dump_file,
8604 : "marking known outgoing %sedge %d -> %d executable\n",
8605 35 : e->flags & EDGE_DFS_BACK ? "back-" : "",
8606 35 : e->src->index, e->dest->index);
8607 1787832 : e->flags |= EDGE_EXECUTABLE;
8608 1787832 : e->dest->flags |= BB_EXECUTABLE;
8609 : }
8610 473845 : else if (!(e->dest->flags & BB_EXECUTABLE))
8611 : {
8612 27318 : if (dump_file && (dump_flags & TDF_DETAILS))
8613 1 : fprintf (dump_file,
8614 : "marking destination block %d reachable\n",
8615 : e->dest->index);
8616 27318 : e->dest->flags |= BB_EXECUTABLE;
8617 : }
8618 : }
8619 890677206 : else if (gsi_one_before_end_p (gsi))
8620 : {
8621 130645088 : FOR_EACH_EDGE (e, ei, bb->succs)
8622 : {
8623 77377712 : if (!(e->flags & EDGE_EXECUTABLE))
8624 : {
8625 56766564 : if (dump_file && (dump_flags & TDF_DETAILS))
8626 18522 : fprintf (dump_file,
8627 : "marking outgoing edge %d -> %d executable\n",
8628 18522 : e->src->index, e->dest->index);
8629 56766564 : e->flags |= EDGE_EXECUTABLE;
8630 56766564 : e->dest->flags |= BB_EXECUTABLE;
8631 : }
8632 20611148 : else if (!(e->dest->flags & BB_EXECUTABLE))
8633 : {
8634 2618709 : if (dump_file && (dump_flags & TDF_DETAILS))
8635 6013 : fprintf (dump_file,
8636 : "marking destination block %d reachable\n",
8637 : e->dest->index);
8638 2618709 : e->dest->flags |= BB_EXECUTABLE;
8639 : }
8640 : }
8641 : }
8642 :
8643 : /* Eliminate. That also pushes to avail. */
8644 447600280 : if (eliminate && ! iterate)
8645 111971926 : avail.eliminate_stmt (bb, &gsi);
8646 : else
8647 : /* If not eliminating, make all not already available defs
8648 : available. But avoid picking up dead defs. */
8649 417197716 : FOR_EACH_SSA_TREE_OPERAND (op, gsi_stmt (gsi), i, SSA_OP_DEF)
8650 81569362 : if (! has_zero_uses (op)
8651 81569362 : && ! avail.eliminate_avail (bb, op))
8652 62108257 : avail.eliminate_push_avail (bb, op);
8653 : }
8654 :
8655 : /* Eliminate in destination PHI arguments. Always substitute in dest
8656 : PHIs, even for non-executable edges. This handles region
8657 : exits PHIs. */
8658 62581088 : if (!iterate && eliminate)
8659 33477257 : FOR_EACH_EDGE (e, ei, bb->succs)
8660 19944348 : for (gphi_iterator gsi = gsi_start_phis (e->dest);
8661 38672862 : !gsi_end_p (gsi); gsi_next (&gsi))
8662 : {
8663 18728514 : gphi *phi = gsi.phi ();
8664 18728514 : use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
8665 18728514 : tree arg = USE_FROM_PTR (use_p);
8666 28468995 : if (TREE_CODE (arg) != SSA_NAME
8667 18728514 : || virtual_operand_p (arg))
8668 9740481 : continue;
8669 8988033 : tree sprime;
8670 8988033 : if (SSA_NAME_IS_DEFAULT_DEF (arg))
8671 : {
8672 118333 : sprime = SSA_VAL (arg);
8673 118333 : gcc_assert (TREE_CODE (sprime) != SSA_NAME
8674 : || SSA_NAME_IS_DEFAULT_DEF (sprime));
8675 : }
8676 : else
8677 : /* Look for sth available at the definition block of the argument.
8678 : This avoids inconsistencies between availability there which
8679 : decides if the stmt can be removed and availability at the
8680 : use site. The SSA property ensures that things available
8681 : at the definition are also available at uses. */
8682 8869700 : sprime = avail.eliminate_avail (gimple_bb (SSA_NAME_DEF_STMT (arg)),
8683 : arg);
8684 8988033 : if (sprime
8685 8988033 : && sprime != arg
8686 8988033 : && may_propagate_copy (arg, sprime, !(e->flags & EDGE_ABNORMAL)))
8687 1557881 : propagate_value (use_p, sprime);
8688 : }
8689 :
8690 62581088 : vn_context_bb = NULL;
8691 62581088 : return todo;
8692 : }
8693 :
8694 : /* Unwind state per basic-block. */
8695 :
8696 : struct unwind_state
8697 : {
8698 : /* Times this block has been visited. */
8699 : unsigned visited;
8700 : /* Whether to handle this as iteration point or whether to treat
8701 : incoming backedge PHI values as varying. */
8702 : bool iterate;
8703 : /* Maximum RPO index this block is reachable from. */
8704 : int max_rpo;
8705 : /* Unwind state. */
8706 : void *ob_top;
8707 : vn_reference_t ref_top;
8708 : vn_phi_t phi_top;
8709 : vn_nary_op_t nary_top;
8710 : vn_avail *avail_top;
8711 : };
8712 :
8713 : /* Unwind the RPO VN state for iteration. */
8714 :
8715 : static void
8716 1923707 : do_unwind (unwind_state *to, rpo_elim &avail)
8717 : {
8718 1923707 : gcc_assert (to->iterate);
8719 35333316 : for (; last_inserted_nary != to->nary_top;
8720 33409609 : last_inserted_nary = last_inserted_nary->next)
8721 : {
8722 33409609 : vn_nary_op_t *slot;
8723 33409609 : slot = valid_info->nary->find_slot_with_hash
8724 33409609 : (last_inserted_nary, last_inserted_nary->hashcode, NO_INSERT);
8725 : /* Predication causes the need to restore previous state. */
8726 33409609 : if ((*slot)->unwind_to)
8727 6783110 : *slot = (*slot)->unwind_to;
8728 : else
8729 26626499 : valid_info->nary->clear_slot (slot);
8730 : }
8731 7568905 : for (; last_inserted_phi != to->phi_top;
8732 5645198 : last_inserted_phi = last_inserted_phi->next)
8733 : {
8734 5645198 : vn_phi_t *slot;
8735 5645198 : slot = valid_info->phis->find_slot_with_hash
8736 5645198 : (last_inserted_phi, last_inserted_phi->hashcode, NO_INSERT);
8737 5645198 : valid_info->phis->clear_slot (slot);
8738 : }
8739 15475620 : for (; last_inserted_ref != to->ref_top;
8740 13551913 : last_inserted_ref = last_inserted_ref->next)
8741 : {
8742 13551913 : vn_reference_t *slot;
8743 13551913 : slot = valid_info->references->find_slot_with_hash
8744 13551913 : (last_inserted_ref, last_inserted_ref->hashcode, NO_INSERT);
8745 13551913 : (*slot)->operands.release ();
8746 13551913 : valid_info->references->clear_slot (slot);
8747 : }
8748 1923707 : obstack_free (&vn_tables_obstack, to->ob_top);
8749 :
8750 : /* Prune [rpo_idx, ] from avail. */
8751 20928239 : for (; last_pushed_avail && last_pushed_avail->avail != to->avail_top;)
8752 : {
8753 19004532 : vn_ssa_aux_t val = last_pushed_avail;
8754 19004532 : vn_avail *av = val->avail;
8755 19004532 : val->avail = av->next;
8756 19004532 : last_pushed_avail = av->next_undo;
8757 19004532 : av->next = avail.m_avail_freelist;
8758 19004532 : avail.m_avail_freelist = av;
8759 : }
8760 1923707 : }
8761 :
8762 : /* Do VN on a SEME region specified by ENTRY and EXIT_BBS in FN.
8763 : If ITERATE is true then treat backedges optimistically as not
8764 : executed and iterate. If ELIMINATE is true then perform
8765 : elimination, otherwise leave that to the caller. If SKIP_ENTRY_PHIS
8766 : is true then force PHI nodes in ENTRY->dest to VARYING. */
8767 :
8768 : static unsigned
8769 6282091 : do_rpo_vn_1 (function *fn, edge entry, bitmap exit_bbs,
8770 : bool iterate, bool eliminate, bool skip_entry_phis,
8771 : vn_lookup_kind kind)
8772 : {
8773 6282091 : unsigned todo = 0;
8774 6282091 : default_vn_walk_kind = kind;
8775 :
8776 : /* We currently do not support region-based iteration when
8777 : elimination is requested. */
8778 6282091 : gcc_assert (!entry || !iterate || !eliminate);
8779 : /* When iterating we need loop info up-to-date. */
8780 6282091 : gcc_assert (!iterate || !loops_state_satisfies_p (LOOPS_NEED_FIXUP));
8781 :
8782 6282091 : bool do_region = entry != NULL;
8783 6282091 : if (!do_region)
8784 : {
8785 5588223 : entry = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fn));
8786 5588223 : exit_bbs = BITMAP_ALLOC (NULL);
8787 5588223 : bitmap_set_bit (exit_bbs, EXIT_BLOCK);
8788 : }
8789 :
8790 : /* Clear EDGE_DFS_BACK on "all" entry edges, RPO order compute will
8791 : re-mark those that are contained in the region. */
8792 6282091 : edge_iterator ei;
8793 6282091 : edge e;
8794 12625091 : FOR_EACH_EDGE (e, ei, entry->dest->preds)
8795 6343000 : e->flags &= ~EDGE_DFS_BACK;
8796 :
8797 6282091 : int *rpo = XNEWVEC (int, n_basic_blocks_for_fn (fn) - NUM_FIXED_BLOCKS);
8798 6282091 : auto_vec<std::pair<int, int> > toplevel_scc_extents;
8799 6282091 : int n = rev_post_order_and_mark_dfs_back_seme
8800 8189959 : (fn, entry, exit_bbs, true, rpo, !iterate ? &toplevel_scc_extents : NULL);
8801 :
8802 6282091 : if (!do_region)
8803 5588223 : BITMAP_FREE (exit_bbs);
8804 :
8805 : /* If there are any non-DFS_BACK edges into entry->dest skip
8806 : processing PHI nodes for that block. This supports
8807 : value-numbering loop bodies w/o the actual loop. */
8808 12625090 : FOR_EACH_EDGE (e, ei, entry->dest->preds)
8809 6343000 : if (e != entry
8810 60909 : && !(e->flags & EDGE_DFS_BACK))
8811 : break;
8812 6282091 : if (e != NULL && dump_file && (dump_flags & TDF_DETAILS))
8813 0 : fprintf (dump_file, "Region does not contain all edges into "
8814 : "the entry block, skipping its PHIs.\n");
8815 6282091 : skip_entry_phis |= e != NULL;
8816 :
8817 6282091 : int *bb_to_rpo = XNEWVEC (int, last_basic_block_for_fn (fn));
8818 57715808 : for (int i = 0; i < n; ++i)
8819 51433717 : bb_to_rpo[rpo[i]] = i;
8820 6282091 : vn_bb_to_rpo = bb_to_rpo;
8821 :
8822 6282091 : unwind_state *rpo_state = XNEWVEC (unwind_state, n);
8823 :
8824 6282091 : rpo_elim avail (entry->dest);
8825 6282091 : rpo_avail = &avail;
8826 :
8827 : /* Verify we have no extra entries into the region. */
8828 6282091 : if (flag_checking && do_region)
8829 : {
8830 693862 : auto_bb_flag bb_in_region (fn);
8831 2127489 : for (int i = 0; i < n; ++i)
8832 : {
8833 1433627 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
8834 1433627 : bb->flags |= bb_in_region;
8835 : }
8836 : /* We can't merge the first two loops because we cannot rely
8837 : on EDGE_DFS_BACK for edges not within the region. But if
8838 : we decide to always have the bb_in_region flag we can
8839 : do the checking during the RPO walk itself (but then it's
8840 : also easy to handle MEME conservatively). */
8841 2127489 : for (int i = 0; i < n; ++i)
8842 : {
8843 1433627 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
8844 1433627 : edge e;
8845 1433627 : edge_iterator ei;
8846 3139078 : FOR_EACH_EDGE (e, ei, bb->preds)
8847 1705451 : gcc_assert (e == entry
8848 : || (skip_entry_phis && bb == entry->dest)
8849 : || (e->src->flags & bb_in_region));
8850 : }
8851 2127489 : for (int i = 0; i < n; ++i)
8852 : {
8853 1433627 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
8854 1433627 : bb->flags &= ~bb_in_region;
8855 : }
8856 693862 : }
8857 :
8858 : /* Create the VN state. For the initial size of the various hashtables
8859 : use a heuristic based on region size and number of SSA names. */
8860 6282091 : unsigned region_size = (((unsigned HOST_WIDE_INT)n * num_ssa_names)
8861 6282091 : / (n_basic_blocks_for_fn (fn) - NUM_FIXED_BLOCKS));
8862 6282091 : VN_TOP = create_tmp_var_raw (void_type_node, "vn_top");
8863 6282091 : next_value_id = 1;
8864 6282091 : next_constant_value_id = -1;
8865 :
8866 6282091 : vn_ssa_aux_hash = new hash_table <vn_ssa_aux_hasher> (region_size * 2);
8867 6282091 : gcc_obstack_init (&vn_ssa_aux_obstack);
8868 :
8869 6282091 : gcc_obstack_init (&vn_tables_obstack);
8870 6282091 : gcc_obstack_init (&vn_tables_insert_obstack);
8871 6282091 : valid_info = XCNEW (struct vn_tables_s);
8872 6282091 : allocate_vn_table (valid_info, region_size);
8873 6282091 : last_inserted_ref = NULL;
8874 6282091 : last_inserted_phi = NULL;
8875 6282091 : last_inserted_nary = NULL;
8876 6282091 : last_pushed_avail = NULL;
8877 :
8878 6282091 : vn_valueize = rpo_vn_valueize;
8879 :
8880 : /* Initialize the unwind state and edge/BB executable state. */
8881 6282091 : unsigned curr_scc = 0;
8882 57715808 : for (int i = 0; i < n; ++i)
8883 : {
8884 51433717 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
8885 51433717 : rpo_state[i].visited = 0;
8886 51433717 : rpo_state[i].max_rpo = i;
8887 60037829 : if (!iterate && curr_scc < toplevel_scc_extents.length ())
8888 : {
8889 7190398 : if (i >= toplevel_scc_extents[curr_scc].first
8890 7190398 : && i <= toplevel_scc_extents[curr_scc].second)
8891 3926803 : rpo_state[i].max_rpo = toplevel_scc_extents[curr_scc].second;
8892 7190398 : if (i == toplevel_scc_extents[curr_scc].second)
8893 736277 : curr_scc++;
8894 : }
8895 51433717 : bb->flags &= ~BB_EXECUTABLE;
8896 51433717 : bool has_backedges = false;
8897 51433717 : edge e;
8898 51433717 : edge_iterator ei;
8899 122024196 : FOR_EACH_EDGE (e, ei, bb->preds)
8900 : {
8901 70590479 : if (e->flags & EDGE_DFS_BACK)
8902 2871669 : has_backedges = true;
8903 70590479 : e->flags &= ~EDGE_EXECUTABLE;
8904 70590479 : if (iterate || e == entry || (skip_entry_phis && bb == entry->dest))
8905 70590479 : continue;
8906 : }
8907 51433717 : rpo_state[i].iterate = iterate && has_backedges;
8908 : }
8909 6282091 : entry->flags |= EDGE_EXECUTABLE;
8910 6282091 : entry->dest->flags |= BB_EXECUTABLE;
8911 :
8912 : /* As heuristic to improve compile-time we handle only the N innermost
8913 : loops and the outermost one optimistically. */
8914 6282091 : if (iterate)
8915 : {
8916 4374223 : unsigned max_depth = param_rpo_vn_max_loop_depth;
8917 14686725 : for (auto loop : loops_list (cfun, LI_ONLY_INNERMOST))
8918 1566465 : if (loop_depth (loop) > max_depth)
8919 2108 : for (unsigned i = 2;
8920 9034 : i < loop_depth (loop) - max_depth; ++i)
8921 : {
8922 2108 : basic_block header = superloop_at_depth (loop, i)->header;
8923 2108 : bool non_latch_backedge = false;
8924 2108 : edge e;
8925 2108 : edge_iterator ei;
8926 6355 : FOR_EACH_EDGE (e, ei, header->preds)
8927 4247 : if (e->flags & EDGE_DFS_BACK)
8928 : {
8929 : /* There can be a non-latch backedge into the header
8930 : which is part of an outer irreducible region. We
8931 : cannot avoid iterating this block then. */
8932 2139 : if (!dominated_by_p (CDI_DOMINATORS,
8933 2139 : e->src, e->dest))
8934 : {
8935 12 : if (dump_file && (dump_flags & TDF_DETAILS))
8936 0 : fprintf (dump_file, "non-latch backedge %d -> %d "
8937 : "forces iteration of loop %d\n",
8938 0 : e->src->index, e->dest->index, loop->num);
8939 : non_latch_backedge = true;
8940 : }
8941 : else
8942 2127 : e->flags |= EDGE_EXECUTABLE;
8943 : }
8944 2108 : rpo_state[bb_to_rpo[header->index]].iterate = non_latch_backedge;
8945 4374223 : }
8946 : }
8947 :
8948 6282091 : uint64_t nblk = 0;
8949 6282091 : int idx = 0;
8950 4374223 : if (iterate)
8951 : /* Go and process all blocks, iterating as necessary. */
8952 49910603 : do
8953 : {
8954 49910603 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[idx]);
8955 :
8956 : /* If the block has incoming backedges remember unwind state. This
8957 : is required even for non-executable blocks since in irreducible
8958 : regions we might reach them via the backedge and re-start iterating
8959 : from there.
8960 : Note we can individually mark blocks with incoming backedges to
8961 : not iterate where we then handle PHIs conservatively. We do that
8962 : heuristically to reduce compile-time for degenerate cases. */
8963 49910603 : if (rpo_state[idx].iterate)
8964 : {
8965 4431010 : rpo_state[idx].ob_top = obstack_alloc (&vn_tables_obstack, 0);
8966 4431010 : rpo_state[idx].ref_top = last_inserted_ref;
8967 4431010 : rpo_state[idx].phi_top = last_inserted_phi;
8968 4431010 : rpo_state[idx].nary_top = last_inserted_nary;
8969 4431010 : rpo_state[idx].avail_top
8970 4431010 : = last_pushed_avail ? last_pushed_avail->avail : NULL;
8971 : }
8972 :
8973 49910603 : if (!(bb->flags & BB_EXECUTABLE))
8974 : {
8975 971469 : if (dump_file && (dump_flags & TDF_DETAILS))
8976 2 : fprintf (dump_file, "Block %d: BB%d found not executable\n",
8977 : idx, bb->index);
8978 971469 : idx++;
8979 2895176 : continue;
8980 : }
8981 :
8982 48939134 : if (dump_file && (dump_flags & TDF_DETAILS))
8983 334 : fprintf (dump_file, "Processing block %d: BB%d\n", idx, bb->index);
8984 48939134 : nblk++;
8985 97878268 : todo |= process_bb (avail, bb,
8986 48939134 : rpo_state[idx].visited != 0,
8987 : rpo_state[idx].iterate,
8988 : iterate, eliminate, do_region, exit_bbs, false);
8989 48939134 : rpo_state[idx].visited++;
8990 :
8991 : /* Verify if changed values flow over executable outgoing backedges
8992 : and those change destination PHI values (that's the thing we
8993 : can easily verify). Reduce over all such edges to the farthest
8994 : away PHI. */
8995 48939134 : int iterate_to = -1;
8996 48939134 : edge_iterator ei;
8997 48939134 : edge e;
8998 117830348 : FOR_EACH_EDGE (e, ei, bb->succs)
8999 68891214 : if ((e->flags & (EDGE_DFS_BACK|EDGE_EXECUTABLE))
9000 : == (EDGE_DFS_BACK|EDGE_EXECUTABLE)
9001 4440976 : && rpo_state[bb_to_rpo[e->dest->index]].iterate)
9002 : {
9003 4438210 : int destidx = bb_to_rpo[e->dest->index];
9004 4438210 : if (!rpo_state[destidx].visited)
9005 : {
9006 134 : if (dump_file && (dump_flags & TDF_DETAILS))
9007 0 : fprintf (dump_file, "Unvisited destination %d\n",
9008 : e->dest->index);
9009 134 : if (iterate_to == -1 || destidx < iterate_to)
9010 134 : iterate_to = destidx;
9011 134 : continue;
9012 : }
9013 4438076 : if (dump_file && (dump_flags & TDF_DETAILS))
9014 53 : fprintf (dump_file, "Looking for changed values of backedge"
9015 : " %d->%d destination PHIs\n",
9016 53 : e->src->index, e->dest->index);
9017 4438076 : vn_context_bb = e->dest;
9018 4438076 : gphi_iterator gsi;
9019 4438076 : for (gsi = gsi_start_phis (e->dest);
9020 10154138 : !gsi_end_p (gsi); gsi_next (&gsi))
9021 : {
9022 7639927 : bool inserted = false;
9023 : /* While we'd ideally just iterate on value changes
9024 : we CSE PHIs and do that even across basic-block
9025 : boundaries. So even hashtable state changes can
9026 : be important (which is roughly equivalent to
9027 : PHI argument value changes). To not excessively
9028 : iterate because of that we track whether a PHI
9029 : was CSEd to with GF_PLF_1. */
9030 7639927 : bool phival_changed;
9031 7639927 : if ((phival_changed = visit_phi (gsi.phi (),
9032 : &inserted, false))
9033 9039382 : || (inserted && gimple_plf (gsi.phi (), GF_PLF_1)))
9034 : {
9035 1923865 : if (!phival_changed
9036 1923865 : && dump_file && (dump_flags & TDF_DETAILS))
9037 0 : fprintf (dump_file, "PHI was CSEd and hashtable "
9038 : "state (changed)\n");
9039 1923865 : if (iterate_to == -1 || destidx < iterate_to)
9040 1923780 : iterate_to = destidx;
9041 1923865 : break;
9042 : }
9043 : }
9044 4438076 : vn_context_bb = NULL;
9045 : }
9046 48939134 : if (iterate_to != -1)
9047 : {
9048 1923707 : do_unwind (&rpo_state[iterate_to], avail);
9049 1923707 : idx = iterate_to;
9050 1923707 : if (dump_file && (dump_flags & TDF_DETAILS))
9051 20 : fprintf (dump_file, "Iterating to %d BB%d\n",
9052 20 : iterate_to, rpo[iterate_to]);
9053 1923707 : continue;
9054 : }
9055 :
9056 47015427 : idx++;
9057 : }
9058 49910603 : while (idx < n);
9059 :
9060 : else /* !iterate */
9061 : {
9062 : /* Process all blocks greedily with a worklist that enforces RPO
9063 : processing of reachable blocks. */
9064 1907868 : auto_bitmap worklist;
9065 1907868 : bitmap_set_bit (worklist, 0);
9066 17457690 : while (!bitmap_empty_p (worklist))
9067 : {
9068 13641954 : int idx = bitmap_clear_first_set_bit (worklist);
9069 13641954 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[idx]);
9070 13641954 : gcc_assert ((bb->flags & BB_EXECUTABLE)
9071 : && !rpo_state[idx].visited);
9072 :
9073 13641954 : if (dump_file && (dump_flags & TDF_DETAILS))
9074 35239 : fprintf (dump_file, "Processing block %d: BB%d\n", idx, bb->index);
9075 :
9076 : /* When we run into predecessor edges where we cannot trust its
9077 : executable state mark them executable so PHI processing will
9078 : be conservative.
9079 : ??? Do we need to force arguments flowing over that edge
9080 : to be varying or will they even always be? */
9081 13641954 : edge_iterator ei;
9082 13641954 : edge e;
9083 33074141 : FOR_EACH_EDGE (e, ei, bb->preds)
9084 19432187 : if (!(e->flags & EDGE_EXECUTABLE)
9085 1028616 : && (bb == entry->dest
9086 971024 : || (!rpo_state[bb_to_rpo[e->src->index]].visited
9087 933580 : && (rpo_state[bb_to_rpo[e->src->index]].max_rpo
9088 : >= (int)idx))))
9089 : {
9090 967679 : if (dump_file && (dump_flags & TDF_DETAILS))
9091 11323 : fprintf (dump_file, "Cannot trust state of predecessor "
9092 : "edge %d -> %d, marking executable\n",
9093 11323 : e->src->index, e->dest->index);
9094 967679 : e->flags |= EDGE_EXECUTABLE;
9095 : }
9096 :
9097 13641954 : nblk++;
9098 13641954 : todo |= process_bb (avail, bb, false, false, false, eliminate,
9099 : do_region, exit_bbs,
9100 13641954 : skip_entry_phis && bb == entry->dest);
9101 13641954 : rpo_state[idx].visited++;
9102 :
9103 33718943 : FOR_EACH_EDGE (e, ei, bb->succs)
9104 20076989 : if ((e->flags & EDGE_EXECUTABLE)
9105 19998360 : && e->dest->index != EXIT_BLOCK
9106 18814253 : && (!do_region || !bitmap_bit_p (exit_bbs, e->dest->index))
9107 37534816 : && !rpo_state[bb_to_rpo[e->dest->index]].visited)
9108 16495703 : bitmap_set_bit (worklist, bb_to_rpo[e->dest->index]);
9109 : }
9110 1907868 : }
9111 :
9112 : /* If statistics or dump file active. */
9113 6282091 : int nex = 0;
9114 6282091 : unsigned max_visited = 1;
9115 57715808 : for (int i = 0; i < n; ++i)
9116 : {
9117 51433717 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
9118 51433717 : if (bb->flags & BB_EXECUTABLE)
9119 50815127 : nex++;
9120 51433717 : statistics_histogram_event (cfun, "RPO block visited times",
9121 51433717 : rpo_state[i].visited);
9122 51433717 : if (rpo_state[i].visited > max_visited)
9123 : max_visited = rpo_state[i].visited;
9124 : }
9125 6282091 : unsigned nvalues = 0, navail = 0;
9126 173316048 : for (hash_table<vn_ssa_aux_hasher>::iterator i = vn_ssa_aux_hash->begin ();
9127 340350005 : i != vn_ssa_aux_hash->end (); ++i)
9128 : {
9129 167033957 : nvalues++;
9130 167033957 : vn_avail *av = (*i)->avail;
9131 246865742 : while (av)
9132 : {
9133 79831785 : navail++;
9134 79831785 : av = av->next;
9135 : }
9136 : }
9137 6282091 : statistics_counter_event (cfun, "RPO blocks", n);
9138 6282091 : statistics_counter_event (cfun, "RPO blocks visited", nblk);
9139 6282091 : statistics_counter_event (cfun, "RPO blocks executable", nex);
9140 6282091 : statistics_histogram_event (cfun, "RPO iterations", 10*nblk / nex);
9141 6282091 : statistics_histogram_event (cfun, "RPO num values", nvalues);
9142 6282091 : statistics_histogram_event (cfun, "RPO num avail", navail);
9143 6282091 : statistics_histogram_event (cfun, "RPO num lattice",
9144 6282091 : vn_ssa_aux_hash->elements ());
9145 6282091 : if (dump_file && (dump_flags & (TDF_DETAILS|TDF_STATS)))
9146 : {
9147 11228 : fprintf (dump_file, "RPO iteration over %d blocks visited %" PRIu64
9148 : " blocks in total discovering %d executable blocks iterating "
9149 : "%d.%d times, a block was visited max. %u times\n",
9150 : n, nblk, nex,
9151 11228 : (int)((10*nblk / nex)/10), (int)((10*nblk / nex)%10),
9152 : max_visited);
9153 11228 : fprintf (dump_file, "RPO tracked %d values available at %d locations "
9154 : "and %" PRIu64 " lattice elements\n",
9155 11228 : nvalues, navail, (uint64_t) vn_ssa_aux_hash->elements ());
9156 : }
9157 :
9158 6282091 : if (eliminate)
9159 : {
9160 : /* When !iterate we already performed elimination during the RPO
9161 : walk. */
9162 5285792 : if (iterate)
9163 : {
9164 : /* Elimination for region-based VN needs to be done within the
9165 : RPO walk. */
9166 3397516 : gcc_assert (! do_region);
9167 : /* Note we can't use avail.walk here because that gets confused
9168 : by the existing availability and it will be less efficient
9169 : as well. */
9170 3397516 : todo |= eliminate_with_rpo_vn (NULL);
9171 : }
9172 : else
9173 1888276 : todo |= avail.eliminate_cleanup (do_region);
9174 : }
9175 :
9176 6282091 : vn_valueize = NULL;
9177 6282091 : rpo_avail = NULL;
9178 6282091 : vn_bb_to_rpo = NULL;
9179 :
9180 6282091 : XDELETEVEC (bb_to_rpo);
9181 6282091 : XDELETEVEC (rpo);
9182 6282091 : XDELETEVEC (rpo_state);
9183 :
9184 6282091 : return todo;
9185 6282091 : }
9186 :
9187 : /* Region-based entry for RPO VN. Performs value-numbering and elimination
9188 : on the SEME region specified by ENTRY and EXIT_BBS. If ENTRY is not
9189 : the only edge into the region at ENTRY->dest PHI nodes in ENTRY->dest
9190 : are not considered.
9191 : If ITERATE is true then treat backedges optimistically as not
9192 : executed and iterate. If ELIMINATE is true then perform
9193 : elimination, otherwise leave that to the caller.
9194 : If SKIP_ENTRY_PHIS is true then force PHI nodes in ENTRY->dest to VARYING.
9195 : KIND specifies the amount of work done for handling memory operations. */
9196 :
9197 : unsigned
9198 713460 : do_rpo_vn (function *fn, edge entry, bitmap exit_bbs,
9199 : bool iterate, bool eliminate, bool skip_entry_phis,
9200 : vn_lookup_kind kind)
9201 : {
9202 713460 : auto_timevar tv (TV_TREE_RPO_VN);
9203 713460 : unsigned todo = do_rpo_vn_1 (fn, entry, exit_bbs, iterate, eliminate,
9204 : skip_entry_phis, kind);
9205 713460 : free_rpo_vn ();
9206 1426920 : return todo;
9207 713460 : }
9208 :
9209 :
9210 : namespace {
9211 :
9212 : const pass_data pass_data_fre =
9213 : {
9214 : GIMPLE_PASS, /* type */
9215 : "fre", /* name */
9216 : OPTGROUP_NONE, /* optinfo_flags */
9217 : TV_TREE_FRE, /* tv_id */
9218 : ( PROP_cfg | PROP_ssa ), /* properties_required */
9219 : 0, /* properties_provided */
9220 : 0, /* properties_destroyed */
9221 : 0, /* todo_flags_start */
9222 : 0, /* todo_flags_finish */
9223 : };
9224 :
9225 : class pass_fre : public gimple_opt_pass
9226 : {
9227 : public:
9228 1461855 : pass_fre (gcc::context *ctxt)
9229 2923710 : : gimple_opt_pass (pass_data_fre, ctxt), may_iterate (true)
9230 : {}
9231 :
9232 : /* opt_pass methods: */
9233 1169484 : opt_pass * clone () final override { return new pass_fre (m_ctxt); }
9234 1461855 : void set_pass_param (unsigned int n, bool param) final override
9235 : {
9236 1461855 : gcc_assert (n == 0);
9237 1461855 : may_iterate = param;
9238 1461855 : }
9239 4671786 : bool gate (function *) final override
9240 : {
9241 4671786 : return flag_tree_fre != 0 && (may_iterate || optimize > 1);
9242 : }
9243 : unsigned int execute (function *) final override;
9244 :
9245 : private:
9246 : bool may_iterate;
9247 : }; // class pass_fre
9248 :
9249 : unsigned int
9250 4591924 : pass_fre::execute (function *fun)
9251 : {
9252 4591924 : unsigned todo = 0;
9253 :
9254 : /* At -O[1g] use the cheap non-iterating mode. */
9255 4591924 : bool iterate_p = may_iterate && (optimize > 1);
9256 4591924 : calculate_dominance_info (CDI_DOMINATORS);
9257 4591924 : if (iterate_p)
9258 3397516 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
9259 :
9260 4591924 : todo = do_rpo_vn_1 (fun, NULL, NULL, iterate_p, true, false, VN_WALKREWRITE);
9261 4591924 : free_rpo_vn ();
9262 :
9263 4591924 : if (iterate_p)
9264 3397516 : loop_optimizer_finalize ();
9265 :
9266 4591924 : if (scev_initialized_p ())
9267 32201 : scev_reset_htab ();
9268 :
9269 : /* For late FRE after IVOPTs and unrolling, see if we can
9270 : remove some TREE_ADDRESSABLE and rewrite stuff into SSA. */
9271 4591924 : if (!may_iterate)
9272 1008680 : todo |= TODO_update_address_taken;
9273 :
9274 4591924 : return todo;
9275 : }
9276 :
9277 : } // anon namespace
9278 :
9279 : gimple_opt_pass *
9280 292371 : make_pass_fre (gcc::context *ctxt)
9281 : {
9282 292371 : return new pass_fre (ctxt);
9283 : }
9284 :
9285 : #undef BB_EXECUTABLE
|
