Line data Source code
1 : /* SCC value numbering for trees
2 : Copyright (C) 2006-2026 Free Software Foundation, Inc.
3 : Contributed by Daniel Berlin <dan@dberlin.org>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify
8 : it under the terms of the GNU General Public License as published by
9 : the Free Software Foundation; either version 3, or (at your option)
10 : any later version.
11 :
12 : GCC is distributed in the hope that it will be useful,
13 : but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 : GNU General Public License for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "ssa.h"
29 : #include "expmed.h"
30 : #include "insn-config.h"
31 : #include "memmodel.h"
32 : #include "emit-rtl.h"
33 : #include "cgraph.h"
34 : #include "gimple-pretty-print.h"
35 : #include "splay-tree-utils.h"
36 : #include "alias.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "cfganal.h"
40 : #include "tree-inline.h"
41 : #include "internal-fn.h"
42 : #include "gimple-iterator.h"
43 : #include "gimple-fold.h"
44 : #include "tree-eh.h"
45 : #include "flags.h"
46 : #include "dojump.h"
47 : #include "explow.h"
48 : #include "calls.h"
49 : #include "varasm.h"
50 : #include "stmt.h"
51 : #include "expr.h"
52 : #include "tree-dfa.h"
53 : #include "tree-ssa.h"
54 : #include "dumpfile.h"
55 : #include "cfgloop.h"
56 : #include "tree-ssa-propagate.h"
57 : #include "tree-cfg.h"
58 : #include "domwalk.h"
59 : #include "gimple-match.h"
60 : #include "stringpool.h"
61 : #include "attribs.h"
62 : #include "tree-pass.h"
63 : #include "statistics.h"
64 : #include "langhooks.h"
65 : #include "ipa-utils.h"
66 : #include "dbgcnt.h"
67 : #include "tree-cfgcleanup.h"
68 : #include "tree-ssa-loop.h"
69 : #include "tree-scalar-evolution.h"
70 : #include "tree-ssa-loop-niter.h"
71 : #include "builtins.h"
72 : #include "fold-const-call.h"
73 : #include "ipa-modref-tree.h"
74 : #include "ipa-modref.h"
75 : #include "tree-ssa-sccvn.h"
76 : #include "alloc-pool.h"
77 : #include "symbol-summary.h"
78 : #include "sreal.h"
79 : #include "ipa-cp.h"
80 : #include "ipa-prop.h"
81 : #include "target.h"
82 :
83 : /* This algorithm is based on the SCC algorithm presented by Keith
84 : Cooper and L. Taylor Simpson in "SCC-Based Value numbering"
85 : (http://citeseer.ist.psu.edu/41805.html). In
86 : straight line code, it is equivalent to a regular hash based value
87 : numbering that is performed in reverse postorder.
88 :
89 : For code with cycles, there are two alternatives, both of which
90 : require keeping the hashtables separate from the actual list of
91 : value numbers for SSA names.
92 :
93 : 1. Iterate value numbering in an RPO walk of the blocks, removing
94 : all the entries from the hashtable after each iteration (but
95 : keeping the SSA name->value number mapping between iterations).
96 : Iterate until it does not change.
97 :
98 : 2. Perform value numbering as part of an SCC walk on the SSA graph,
99 : iterating only the cycles in the SSA graph until they do not change
100 : (using a separate, optimistic hashtable for value numbering the SCC
101 : operands).
102 :
103 : The second is not just faster in practice (because most SSA graph
104 : cycles do not involve all the variables in the graph), it also has
105 : some nice properties.
106 :
107 : One of these nice properties is that when we pop an SCC off the
108 : stack, we are guaranteed to have processed all the operands coming from
109 : *outside of that SCC*, so we do not need to do anything special to
110 : ensure they have value numbers.
111 :
112 : Another nice property is that the SCC walk is done as part of a DFS
113 : of the SSA graph, which makes it easy to perform combining and
114 : simplifying operations at the same time.
115 :
116 : The code below is deliberately written in a way that makes it easy
117 : to separate the SCC walk from the other work it does.
118 :
119 : In order to propagate constants through the code, we track which
120 : expressions contain constants, and use those while folding. In
121 : theory, we could also track expressions whose value numbers are
122 : replaced, in case we end up folding based on expression
123 : identities.
124 :
125 : In order to value number memory, we assign value numbers to vuses.
126 : This enables us to note that, for example, stores to the same
127 : address of the same value from the same starting memory states are
128 : equivalent.
129 : TODO:
130 :
131 : 1. We can iterate only the changing portions of the SCC's, but
132 : I have not seen an SCC big enough for this to be a win.
133 : 2. If you differentiate between phi nodes for loops and phi nodes
134 : for if-then-else, you can properly consider phi nodes in different
135 : blocks for equivalence.
136 : 3. We could value number vuses in more cases, particularly, whole
137 : structure copies.
138 : */
139 :
140 : /* There's no BB_EXECUTABLE but we can use BB_VISITED. */
141 : #define BB_EXECUTABLE BB_VISITED
142 :
143 : static vn_lookup_kind default_vn_walk_kind;
144 :
145 : /* vn_nary_op hashtable helpers. */
146 :
147 : struct vn_nary_op_hasher : nofree_ptr_hash <vn_nary_op_s>
148 : {
149 : typedef vn_nary_op_s *compare_type;
150 : static inline hashval_t hash (const vn_nary_op_s *);
151 : static inline bool equal (const vn_nary_op_s *, const vn_nary_op_s *);
152 : };
153 :
154 : /* Return the computed hashcode for nary operation P1. */
155 :
156 : inline hashval_t
157 765777572 : vn_nary_op_hasher::hash (const vn_nary_op_s *vno1)
158 : {
159 765777572 : return vno1->hashcode;
160 : }
161 :
162 : /* Compare nary operations P1 and P2 and return true if they are
163 : equivalent. */
164 :
165 : inline bool
166 970221843 : vn_nary_op_hasher::equal (const vn_nary_op_s *vno1, const vn_nary_op_s *vno2)
167 : {
168 970221843 : return vno1 == vno2 || vn_nary_op_eq (vno1, vno2);
169 : }
170 :
171 : typedef hash_table<vn_nary_op_hasher> vn_nary_op_table_type;
172 : typedef vn_nary_op_table_type::iterator vn_nary_op_iterator_type;
173 :
174 :
175 : /* vn_phi hashtable helpers. */
176 :
177 : static int
178 : vn_phi_eq (const_vn_phi_t const vp1, const_vn_phi_t const vp2);
179 :
180 : struct vn_phi_hasher : nofree_ptr_hash <vn_phi_s>
181 : {
182 : static inline hashval_t hash (const vn_phi_s *);
183 : static inline bool equal (const vn_phi_s *, const vn_phi_s *);
184 : };
185 :
186 : /* Return the computed hashcode for phi operation P1. */
187 :
188 : inline hashval_t
189 25354641 : vn_phi_hasher::hash (const vn_phi_s *vp1)
190 : {
191 25354641 : return vp1->hashcode;
192 : }
193 :
194 : /* Compare two phi entries for equality, ignoring VN_TOP arguments. */
195 :
196 : inline bool
197 46119405 : vn_phi_hasher::equal (const vn_phi_s *vp1, const vn_phi_s *vp2)
198 : {
199 46119405 : return vp1 == vp2 || vn_phi_eq (vp1, vp2);
200 : }
201 :
202 : typedef hash_table<vn_phi_hasher> vn_phi_table_type;
203 : typedef vn_phi_table_type::iterator vn_phi_iterator_type;
204 :
205 :
206 : /* Compare two reference operands P1 and P2 for equality. Return true if
207 : they are equal, and false otherwise. */
208 :
209 : static int
210 25412950 : vn_reference_op_eq (const void *p1, const void *p2)
211 : {
212 25412950 : const_vn_reference_op_t const vro1 = (const_vn_reference_op_t) p1;
213 25412950 : const_vn_reference_op_t const vro2 = (const_vn_reference_op_t) p2;
214 :
215 25412950 : return (vro1->opcode == vro2->opcode
216 : /* We do not care for differences in type qualification. */
217 25410997 : && (vro1->type == vro2->type
218 1163324 : || (vro1->type && vro2->type
219 1163324 : && types_compatible_p (TYPE_MAIN_VARIANT (vro1->type),
220 1163324 : TYPE_MAIN_VARIANT (vro2->type))))
221 24423940 : && expressions_equal_p (vro1->op0, vro2->op0)
222 24387330 : && expressions_equal_p (vro1->op1, vro2->op1)
223 24387330 : && expressions_equal_p (vro1->op2, vro2->op2)
224 49800280 : && (vro1->opcode != CALL_EXPR || vro1->clique == vro2->clique));
225 : }
226 :
227 : /* Free a reference operation structure VP. */
228 :
229 : static inline void
230 0 : free_reference (vn_reference_s *vr)
231 : {
232 0 : vr->operands.release ();
233 : }
234 :
235 :
236 : /* vn_reference hashtable helpers. */
237 :
238 : struct vn_reference_hasher : nofree_ptr_hash <vn_reference_s>
239 : {
240 : static inline hashval_t hash (const vn_reference_s *);
241 : static inline bool equal (const vn_reference_s *, const vn_reference_s *);
242 : };
243 :
244 : /* Return the hashcode for a given reference operation P1. */
245 :
246 : inline hashval_t
247 3790521900 : vn_reference_hasher::hash (const vn_reference_s *vr1)
248 : {
249 3790521900 : return vr1->hashcode;
250 : }
251 :
252 : inline bool
253 4505847558 : vn_reference_hasher::equal (const vn_reference_s *v, const vn_reference_s *c)
254 : {
255 4505847558 : return v == c || vn_reference_eq (v, c);
256 : }
257 :
258 : typedef hash_table<vn_reference_hasher> vn_reference_table_type;
259 : typedef vn_reference_table_type::iterator vn_reference_iterator_type;
260 :
261 : /* Pretty-print OPS to OUTFILE. */
262 :
263 : void
264 287 : print_vn_reference_ops (FILE *outfile, const vec<vn_reference_op_s> ops)
265 : {
266 287 : vn_reference_op_t vro;
267 287 : unsigned int i;
268 287 : fprintf (outfile, "{");
269 1304 : for (i = 0; ops.iterate (i, &vro); i++)
270 : {
271 1017 : bool closebrace = false;
272 1017 : if (vro->opcode != SSA_NAME
273 803 : && TREE_CODE_CLASS (vro->opcode) != tcc_declaration)
274 : {
275 803 : fprintf (outfile, "%s", get_tree_code_name (vro->opcode));
276 803 : if (vro->op0 || vro->opcode == CALL_EXPR)
277 : {
278 803 : fprintf (outfile, "<");
279 803 : closebrace = true;
280 : }
281 : }
282 1017 : if (vro->opcode == MEM_REF || vro->opcode == TARGET_MEM_REF)
283 275 : fprintf (outfile, "(A%d)", TYPE_ALIGN (vro->type));
284 1017 : if (vro->op0 || vro->opcode == CALL_EXPR)
285 : {
286 1017 : if (!vro->op0)
287 0 : fprintf (outfile, internal_fn_name ((internal_fn)vro->clique));
288 : else
289 : {
290 1017 : if (vro->opcode == MEM_REF || vro->opcode == TARGET_MEM_REF)
291 : {
292 275 : fprintf (outfile, "(");
293 275 : print_generic_expr (outfile, TREE_TYPE (vro->op0));
294 275 : fprintf (outfile, ")");
295 : }
296 1017 : print_generic_expr (outfile, vro->op0);
297 : }
298 1017 : if (vro->op1)
299 : {
300 185 : fprintf (outfile, ",");
301 185 : print_generic_expr (outfile, vro->op1);
302 : }
303 1017 : if (vro->op2)
304 : {
305 185 : fprintf (outfile, ",");
306 185 : print_generic_expr (outfile, vro->op2);
307 : }
308 : }
309 1017 : if (closebrace)
310 803 : fprintf (outfile, ">");
311 1017 : if (i != ops.length () - 1)
312 730 : fprintf (outfile, ",");
313 : }
314 287 : fprintf (outfile, "}");
315 287 : }
316 :
317 : DEBUG_FUNCTION void
318 0 : debug_vn_reference_ops (const vec<vn_reference_op_s> ops)
319 : {
320 0 : print_vn_reference_ops (stderr, ops);
321 0 : fputc ('\n', stderr);
322 0 : }
323 :
324 : /* The set of VN hashtables. */
325 :
326 : typedef struct vn_tables_s
327 : {
328 : vn_nary_op_table_type *nary;
329 : vn_phi_table_type *phis;
330 : vn_reference_table_type *references;
331 : } *vn_tables_t;
332 :
333 :
334 : /* vn_constant hashtable helpers. */
335 :
336 : struct vn_constant_hasher : free_ptr_hash <vn_constant_s>
337 : {
338 : static inline hashval_t hash (const vn_constant_s *);
339 : static inline bool equal (const vn_constant_s *, const vn_constant_s *);
340 : };
341 :
342 : /* Hash table hash function for vn_constant_t. */
343 :
344 : inline hashval_t
345 12136318 : vn_constant_hasher::hash (const vn_constant_s *vc1)
346 : {
347 12136318 : return vc1->hashcode;
348 : }
349 :
350 : /* Hash table equality function for vn_constant_t. */
351 :
352 : inline bool
353 14612771 : vn_constant_hasher::equal (const vn_constant_s *vc1, const vn_constant_s *vc2)
354 : {
355 14612771 : if (vc1->hashcode != vc2->hashcode)
356 : return false;
357 :
358 2194003 : return vn_constant_eq_with_type (vc1->constant, vc2->constant);
359 : }
360 :
361 : static hash_table<vn_constant_hasher> *constant_to_value_id;
362 :
363 :
364 : /* Obstack we allocate the vn-tables elements from. */
365 : static obstack vn_tables_obstack;
366 : /* Special obstack we never unwind. */
367 : static obstack vn_tables_insert_obstack;
368 :
369 : static vn_reference_t last_inserted_ref;
370 : static vn_phi_t last_inserted_phi;
371 : static vn_nary_op_t last_inserted_nary;
372 : static vn_ssa_aux_t last_pushed_avail;
373 :
374 : /* Valid hashtables storing information we have proven to be
375 : correct. */
376 : static vn_tables_t valid_info;
377 :
378 : /* Global RPO state for access from hooks. */
379 : static class eliminate_dom_walker *rpo_avail;
380 : basic_block vn_context_bb;
381 : int *vn_bb_to_rpo;
382 :
383 :
384 : /* Valueization hook for simplify_replace_tree. Valueize NAME if it is
385 : an SSA name, otherwise just return it. */
386 : tree (*vn_valueize) (tree);
387 : static tree
388 84171 : vn_valueize_for_srt (tree t, void* context ATTRIBUTE_UNUSED)
389 : {
390 84171 : basic_block saved_vn_context_bb = vn_context_bb;
391 : /* Look for sth available at the definition block of the argument.
392 : This avoids inconsistencies between availability there which
393 : decides if the stmt can be removed and availability at the
394 : use site. The SSA property ensures that things available
395 : at the definition are also available at uses. */
396 84171 : if (!SSA_NAME_IS_DEFAULT_DEF (t))
397 80413 : vn_context_bb = gimple_bb (SSA_NAME_DEF_STMT (t));
398 84171 : tree res = vn_valueize (t);
399 84171 : vn_context_bb = saved_vn_context_bb;
400 84171 : return res;
401 : }
402 :
403 :
404 : /* This represents the top of the VN lattice, which is the universal
405 : value. */
406 :
407 : tree VN_TOP;
408 :
409 : /* Unique counter for our value ids. */
410 :
411 : static unsigned int next_value_id;
412 : static int next_constant_value_id;
413 :
414 :
415 : /* Table of vn_ssa_aux_t's, one per ssa_name. The vn_ssa_aux_t objects
416 : are allocated on an obstack for locality reasons, and to free them
417 : without looping over the vec. */
418 :
419 : struct vn_ssa_aux_hasher : typed_noop_remove <vn_ssa_aux_t>
420 : {
421 : typedef vn_ssa_aux_t value_type;
422 : typedef tree compare_type;
423 : static inline hashval_t hash (const value_type &);
424 : static inline bool equal (const value_type &, const compare_type &);
425 : static inline void mark_deleted (value_type &) {}
426 : static const bool empty_zero_p = true;
427 0 : static inline void mark_empty (value_type &e) { e = NULL; }
428 : static inline bool is_deleted (value_type &) { return false; }
429 >13271*10^7 : static inline bool is_empty (value_type &e) { return e == NULL; }
430 : };
431 :
432 : hashval_t
433 43695395576 : vn_ssa_aux_hasher::hash (const value_type &entry)
434 : {
435 43695395576 : return SSA_NAME_VERSION (entry->name);
436 : }
437 :
438 : bool
439 49987260095 : vn_ssa_aux_hasher::equal (const value_type &entry, const compare_type &name)
440 : {
441 49987260095 : return name == entry->name;
442 : }
443 :
444 : static hash_table<vn_ssa_aux_hasher> *vn_ssa_aux_hash;
445 : typedef hash_table<vn_ssa_aux_hasher>::iterator vn_ssa_aux_iterator_type;
446 : static struct obstack vn_ssa_aux_obstack;
447 :
448 : static vn_nary_op_t vn_nary_op_insert_stmt (gimple *, tree);
449 : static vn_nary_op_t vn_nary_op_insert_into (vn_nary_op_t,
450 : vn_nary_op_table_type *);
451 : static void init_vn_nary_op_from_pieces (vn_nary_op_t, unsigned int,
452 : enum tree_code, tree, tree *);
453 : static tree vn_lookup_simplify_result (gimple_match_op *);
454 : static vn_reference_t vn_reference_lookup_or_insert_for_pieces
455 : (tree, alias_set_type, alias_set_type, poly_int64, poly_int64, tree,
456 : vec<vn_reference_op_s, va_heap>, tree);
457 :
458 : /* Return whether there is value numbering information for a given SSA name. */
459 :
460 : bool
461 5114832 : has_VN_INFO (tree name)
462 : {
463 5114832 : return vn_ssa_aux_hash->find_with_hash (name, SSA_NAME_VERSION (name));
464 : }
465 :
466 : vn_ssa_aux_t
467 3760753740 : VN_INFO (tree name)
468 : {
469 3760753740 : vn_ssa_aux_t *res
470 3760753740 : = vn_ssa_aux_hash->find_slot_with_hash (name, SSA_NAME_VERSION (name),
471 : INSERT);
472 3760753740 : if (*res != NULL)
473 : return *res;
474 :
475 171255438 : vn_ssa_aux_t newinfo = *res = XOBNEW (&vn_ssa_aux_obstack, struct vn_ssa_aux);
476 171255438 : memset (newinfo, 0, sizeof (struct vn_ssa_aux));
477 171255438 : newinfo->name = name;
478 171255438 : newinfo->valnum = VN_TOP;
479 : /* We are using the visited flag to handle uses with defs not within the
480 : region being value-numbered. */
481 171255438 : newinfo->visited = false;
482 :
483 : /* Given we create the VN_INFOs on-demand now we have to do initialization
484 : different than VN_TOP here. */
485 171255438 : if (SSA_NAME_IS_DEFAULT_DEF (name))
486 9256142 : switch (TREE_CODE (SSA_NAME_VAR (name)))
487 : {
488 1658357 : case VAR_DECL:
489 : /* All undefined vars are VARYING. */
490 1658357 : newinfo->valnum = name;
491 1658357 : newinfo->visited = true;
492 1658357 : break;
493 :
494 7538625 : case PARM_DECL:
495 : /* Parameters are VARYING but we can record a condition
496 : if we know it is a non-NULL pointer. */
497 7538625 : newinfo->visited = true;
498 7538625 : newinfo->valnum = name;
499 11584351 : if (POINTER_TYPE_P (TREE_TYPE (name))
500 8642553 : && nonnull_arg_p (SSA_NAME_VAR (name)))
501 : {
502 2281273 : tree ops[2];
503 2281273 : ops[0] = name;
504 2281273 : ops[1] = build_int_cst (TREE_TYPE (name), 0);
505 2281273 : vn_nary_op_t nary;
506 : /* Allocate from non-unwinding stack. */
507 2281273 : nary = alloc_vn_nary_op_noinit (2, &vn_tables_insert_obstack);
508 2281273 : init_vn_nary_op_from_pieces (nary, 2, NE_EXPR,
509 : boolean_type_node, ops);
510 2281273 : nary->predicated_values = 0;
511 2281273 : nary->u.result = boolean_true_node;
512 2281273 : vn_nary_op_insert_into (nary, valid_info->nary);
513 2281273 : gcc_assert (nary->unwind_to == NULL);
514 : /* Also do not link it into the undo chain. */
515 2281273 : last_inserted_nary = nary->next;
516 2281273 : nary->next = (vn_nary_op_t)(void *)-1;
517 2281273 : nary = alloc_vn_nary_op_noinit (2, &vn_tables_insert_obstack);
518 2281273 : init_vn_nary_op_from_pieces (nary, 2, EQ_EXPR,
519 : boolean_type_node, ops);
520 2281273 : nary->predicated_values = 0;
521 2281273 : nary->u.result = boolean_false_node;
522 2281273 : vn_nary_op_insert_into (nary, valid_info->nary);
523 2281273 : gcc_assert (nary->unwind_to == NULL);
524 2281273 : last_inserted_nary = nary->next;
525 2281273 : nary->next = (vn_nary_op_t)(void *)-1;
526 2281273 : if (dump_file && (dump_flags & TDF_DETAILS))
527 : {
528 38 : fprintf (dump_file, "Recording ");
529 38 : print_generic_expr (dump_file, name, TDF_SLIM);
530 38 : fprintf (dump_file, " != 0\n");
531 : }
532 : }
533 : break;
534 :
535 59160 : case RESULT_DECL:
536 : /* If the result is passed by invisible reference the default
537 : def is initialized, otherwise it's uninitialized. Still
538 : undefined is varying. */
539 59160 : newinfo->visited = true;
540 59160 : newinfo->valnum = name;
541 59160 : break;
542 :
543 0 : default:
544 0 : gcc_unreachable ();
545 : }
546 : return newinfo;
547 : }
548 :
549 : /* Return the SSA value of X. */
550 :
551 : inline tree
552 3428809954 : SSA_VAL (tree x, bool *visited = NULL)
553 : {
554 3428809954 : vn_ssa_aux_t tem = vn_ssa_aux_hash->find_with_hash (x, SSA_NAME_VERSION (x));
555 3428809954 : if (visited)
556 1392684567 : *visited = tem && tem->visited;
557 3428809954 : return tem && tem->visited ? tem->valnum : x;
558 : }
559 :
560 : /* Return the SSA value of the VUSE x, supporting released VDEFs
561 : during elimination which will value-number the VDEF to the
562 : associated VUSE (but not substitute in the whole lattice). */
563 :
564 : static inline tree
565 1268454788 : vuse_ssa_val (tree x)
566 : {
567 1268454788 : if (!x)
568 : return NULL_TREE;
569 :
570 1265111700 : do
571 : {
572 1265111700 : x = SSA_VAL (x);
573 1265111700 : gcc_assert (x != VN_TOP);
574 : }
575 1265111700 : while (SSA_NAME_IN_FREE_LIST (x));
576 :
577 : return x;
578 : }
579 :
580 : /* Similar to the above but used as callback for walk_non_aliased_vuses
581 : and thus should stop at unvisited VUSE to not walk across region
582 : boundaries. */
583 :
584 : static tree
585 1073741823 : vuse_valueize (tree vuse)
586 : {
587 1073741823 : do
588 : {
589 1073741823 : bool visited;
590 1073741823 : vuse = SSA_VAL (vuse, &visited);
591 1073741823 : if (!visited)
592 15966094 : return NULL_TREE;
593 1057775729 : gcc_assert (vuse != VN_TOP);
594 : }
595 1057775729 : while (SSA_NAME_IN_FREE_LIST (vuse));
596 : return vuse;
597 : }
598 :
599 :
600 : /* Return the vn_kind the expression computed by the stmt should be
601 : associated with. */
602 :
603 : enum vn_kind
604 101801746 : vn_get_stmt_kind (gimple *stmt)
605 : {
606 101801746 : switch (gimple_code (stmt))
607 : {
608 : case GIMPLE_CALL:
609 : return VN_REFERENCE;
610 : case GIMPLE_PHI:
611 : return VN_PHI;
612 101801746 : case GIMPLE_ASSIGN:
613 101801746 : {
614 101801746 : enum tree_code code = gimple_assign_rhs_code (stmt);
615 101801746 : tree rhs1 = gimple_assign_rhs1 (stmt);
616 101801746 : switch (get_gimple_rhs_class (code))
617 : {
618 : case GIMPLE_UNARY_RHS:
619 : case GIMPLE_BINARY_RHS:
620 : case GIMPLE_TERNARY_RHS:
621 : return VN_NARY;
622 47627545 : case GIMPLE_SINGLE_RHS:
623 47627545 : switch (TREE_CODE_CLASS (code))
624 : {
625 35867234 : case tcc_reference:
626 : /* VOP-less references can go through unary case. */
627 35867234 : if ((code == REALPART_EXPR
628 : || code == IMAGPART_EXPR
629 35867234 : || code == VIEW_CONVERT_EXPR
630 35867234 : || code == BIT_FIELD_REF)
631 35867234 : && (TREE_CODE (TREE_OPERAND (rhs1, 0)) == SSA_NAME
632 632326 : || is_gimple_min_invariant (TREE_OPERAND (rhs1, 0))))
633 2020642 : return VN_NARY;
634 :
635 : /* Fallthrough. */
636 : case tcc_declaration:
637 : return VN_REFERENCE;
638 :
639 : case tcc_constant:
640 : return VN_CONSTANT;
641 :
642 5884109 : default:
643 5884109 : if (code == ADDR_EXPR)
644 3195158 : return (is_gimple_min_invariant (rhs1)
645 3195158 : ? VN_CONSTANT : VN_REFERENCE);
646 2688951 : else if (code == CONSTRUCTOR)
647 : return VN_NARY;
648 : return VN_NONE;
649 : }
650 : default:
651 : return VN_NONE;
652 : }
653 : }
654 : default:
655 : return VN_NONE;
656 : }
657 : }
658 :
659 : /* Lookup a value id for CONSTANT and return it. If it does not
660 : exist returns 0. */
661 :
662 : unsigned int
663 0 : get_constant_value_id (tree constant)
664 : {
665 0 : vn_constant_s **slot;
666 0 : struct vn_constant_s vc;
667 :
668 0 : vc.hashcode = vn_hash_constant_with_type (constant);
669 0 : vc.constant = constant;
670 0 : slot = constant_to_value_id->find_slot (&vc, NO_INSERT);
671 0 : if (slot)
672 0 : return (*slot)->value_id;
673 : return 0;
674 : }
675 :
676 : /* Lookup a value id for CONSTANT, and if it does not exist, create a
677 : new one and return it. If it does exist, return it. */
678 :
679 : unsigned int
680 28475799 : get_or_alloc_constant_value_id (tree constant)
681 : {
682 28475799 : vn_constant_s **slot;
683 28475799 : struct vn_constant_s vc;
684 28475799 : vn_constant_t vcp;
685 :
686 : /* If the hashtable isn't initialized we're not running from PRE and thus
687 : do not need value-ids. */
688 28475799 : if (!constant_to_value_id)
689 : return 0;
690 :
691 4688417 : vc.hashcode = vn_hash_constant_with_type (constant);
692 4688417 : vc.constant = constant;
693 4688417 : slot = constant_to_value_id->find_slot (&vc, INSERT);
694 4688417 : if (*slot)
695 2176694 : return (*slot)->value_id;
696 :
697 2511723 : vcp = XNEW (struct vn_constant_s);
698 2511723 : vcp->hashcode = vc.hashcode;
699 2511723 : vcp->constant = constant;
700 2511723 : vcp->value_id = get_next_constant_value_id ();
701 2511723 : *slot = vcp;
702 2511723 : return vcp->value_id;
703 : }
704 :
705 : /* Compute the hash for a reference operand VRO1. */
706 :
707 : static void
708 135269475 : vn_reference_op_compute_hash (const vn_reference_op_t vro1, inchash::hash &hstate)
709 : {
710 135269475 : hstate.add_int (vro1->opcode);
711 135269475 : if (vro1->opcode == CALL_EXPR && !vro1->op0)
712 543508 : hstate.add_int (vro1->clique);
713 135269475 : if (vro1->op0)
714 128919697 : inchash::add_expr (vro1->op0, hstate);
715 135269475 : if (vro1->op1)
716 12082223 : inchash::add_expr (vro1->op1, hstate);
717 135269475 : if (vro1->op2)
718 13802846 : inchash::add_expr (vro1->op2, hstate);
719 135269475 : }
720 :
721 : /* Compute a hash for the reference operation VR1 and return it. */
722 :
723 : hashval_t
724 201460757 : vn_reference_compute_hash (const vn_reference_t vr1)
725 : {
726 201460757 : inchash::hash hstate;
727 201460757 : hashval_t result;
728 201460757 : int i;
729 201460757 : vn_reference_op_t vro;
730 201460757 : poly_offset_int off = -1;
731 201460757 : bool deref = false;
732 :
733 819233863 : FOR_EACH_VEC_ELT (vr1->operands, i, vro)
734 : {
735 617773106 : if (vro->opcode == MEM_REF)
736 : deref = true;
737 426947937 : else if (vro->opcode != ADDR_EXPR)
738 299087733 : deref = false;
739 617773106 : if (maybe_ne (vro->off, -1))
740 : {
741 362741010 : if (known_eq (off, -1))
742 193223908 : off = 0;
743 617773106 : off += vro->off;
744 : }
745 : else
746 : {
747 255032096 : if (maybe_ne (off, -1)
748 255032096 : && maybe_ne (off, 0))
749 102462131 : hstate.add_poly_hwi (off.force_shwi ());
750 255032096 : off = -1;
751 255032096 : if (deref
752 119975710 : && vro->opcode == ADDR_EXPR)
753 : {
754 119762621 : if (vro->op0)
755 : {
756 119762621 : tree op = TREE_OPERAND (vro->op0, 0);
757 119762621 : hstate.add_int (TREE_CODE (op));
758 119762621 : inchash::add_expr (op, hstate);
759 : }
760 : }
761 : else
762 135269475 : vn_reference_op_compute_hash (vro, hstate);
763 : }
764 : }
765 : /* Do not hash vr1->offset or vr1->max_size, we want to get collisions
766 : to be able to identify compatible results. */
767 201460757 : result = hstate.end ();
768 : /* ??? We would ICE later if we hash instead of adding that in. */
769 201460757 : if (vr1->vuse)
770 196550470 : result += SSA_NAME_VERSION (vr1->vuse);
771 :
772 201460757 : return result;
773 : }
774 :
775 : /* Return true if reference operations VR1 and VR2 are equivalent. This
776 : means they have the same set of operands and vuses. If LEXICAL
777 : is true then the full access path has to be the same. */
778 :
779 : bool
780 4501417946 : vn_reference_eq (const_vn_reference_t const vr1, const_vn_reference_t const vr2,
781 : bool lexical)
782 : {
783 4501417946 : unsigned i, j;
784 :
785 : /* Early out if this is not a hash collision. */
786 4501417946 : if (vr1->hashcode != vr2->hashcode)
787 : return false;
788 :
789 : /* The VOP needs to be the same. */
790 17556240 : if (vr1->vuse != vr2->vuse)
791 : return false;
792 :
793 : /* The offset/max_size used for the ao_ref during lookup has to be
794 : the same. */
795 17555837 : if (maybe_ne (vr1->offset, vr2->offset)
796 17555837 : || maybe_ne (vr1->max_size, vr2->max_size))
797 : {
798 : /* But nothing known in the prevailing entry is OK to be used. */
799 6792124 : if (maybe_ne (vr1->offset, 0) || known_size_p (vr1->max_size))
800 : return false;
801 : }
802 :
803 : /* If the operands are the same we are done. */
804 35020590 : if (vr1->operands == vr2->operands)
805 : return true;
806 :
807 17510295 : if (!vr1->type || !vr2->type)
808 : {
809 555381 : if (vr1->type != vr2->type)
810 : return false;
811 : }
812 16954914 : else if (vr1->type == vr2->type)
813 : ;
814 2127782 : else if (COMPLETE_TYPE_P (vr1->type) != COMPLETE_TYPE_P (vr2->type)
815 2127782 : || (COMPLETE_TYPE_P (vr1->type)
816 2127782 : && !expressions_equal_p (TYPE_SIZE (vr1->type),
817 2127782 : TYPE_SIZE (vr2->type))))
818 761805 : return false;
819 1365977 : else if (vr1->operands[0].opcode == CALL_EXPR
820 1365977 : && !types_compatible_p (vr1->type, vr2->type))
821 : return false;
822 1365977 : else if (INTEGRAL_TYPE_P (vr1->type)
823 542298 : && INTEGRAL_TYPE_P (vr2->type))
824 : {
825 502376 : if (TYPE_PRECISION (vr1->type) != TYPE_PRECISION (vr2->type))
826 : return false;
827 : }
828 863601 : else if (INTEGRAL_TYPE_P (vr1->type)
829 863601 : && (TYPE_PRECISION (vr1->type)
830 39922 : != TREE_INT_CST_LOW (TYPE_SIZE (vr1->type))))
831 : return false;
832 863555 : else if (INTEGRAL_TYPE_P (vr2->type)
833 863555 : && (TYPE_PRECISION (vr2->type)
834 9141 : != TREE_INT_CST_LOW (TYPE_SIZE (vr2->type))))
835 : return false;
836 13048 : else if (VECTOR_BOOLEAN_TYPE_P (vr1->type)
837 862960 : && VECTOR_BOOLEAN_TYPE_P (vr2->type))
838 : {
839 : /* Vector boolean types can have padding, verify we are dealing with
840 : the same number of elements, aka the precision of the types.
841 : For example, In most architecture the precision_size of vbool*_t
842 : types are caculated like below:
843 : precision_size = type_size * 8
844 :
845 : Unfortunately, the RISC-V will adjust the precision_size for the
846 : vbool*_t in order to align the ISA as below:
847 : type_size = [1, 1, 1, 1, 2, 4, 8]
848 : precision_size = [1, 2, 4, 8, 16, 32, 64]
849 :
850 : Then the precision_size of RISC-V vbool*_t will not be the multiple
851 : of the type_size. We take care of this case consolidated here. */
852 0 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vr1->type),
853 0 : TYPE_VECTOR_SUBPARTS (vr2->type)))
854 : return false;
855 : }
856 862960 : else if (TYPE_MODE (vr1->type) != TYPE_MODE (vr2->type)
857 862960 : && (!mode_can_transfer_bits (TYPE_MODE (vr1->type))
858 44754 : || !mode_can_transfer_bits (TYPE_MODE (vr2->type))))
859 1033 : return false;
860 :
861 : i = 0;
862 : j = 0;
863 21585910 : do
864 : {
865 21585910 : poly_offset_int off1 = 0, off2 = 0;
866 21585910 : vn_reference_op_t vro1, vro2;
867 21585910 : vn_reference_op_s tem1, tem2;
868 21585910 : bool deref1 = false, deref2 = false;
869 21585910 : bool reverse1 = false, reverse2 = false;
870 70157312 : for (; vr1->operands.iterate (i, &vro1); i++)
871 : {
872 48571402 : if (vro1->opcode == MEM_REF)
873 : deref1 = true;
874 : /* Do not look through a storage order barrier. */
875 33255938 : else if (vro1->opcode == VIEW_CONVERT_EXPR && vro1->reverse)
876 70594 : return false;
877 48571402 : reverse1 |= vro1->reverse;
878 48571402 : if (lexical || known_eq (vro1->off, -1))
879 : break;
880 26985492 : off1 += vro1->off;
881 : }
882 48701664 : for (; vr2->operands.iterate (j, &vro2); j++)
883 : {
884 48701664 : if (vro2->opcode == MEM_REF)
885 : deref2 = true;
886 : /* Do not look through a storage order barrier. */
887 33362539 : else if (vro2->opcode == VIEW_CONVERT_EXPR && vro2->reverse)
888 : return false;
889 48701664 : reverse2 |= vro2->reverse;
890 48701664 : if (lexical || known_eq (vro2->off, -1))
891 : break;
892 27115754 : off2 += vro2->off;
893 : }
894 21585910 : if (maybe_ne (off1, off2) || reverse1 != reverse2)
895 : return false;
896 21585772 : if (deref1 && vro1->opcode == ADDR_EXPR)
897 : {
898 8118089 : memset (&tem1, 0, sizeof (tem1));
899 8118089 : tem1.op0 = TREE_OPERAND (vro1->op0, 0);
900 8118089 : tem1.type = TREE_TYPE (tem1.op0);
901 8118089 : tem1.opcode = TREE_CODE (tem1.op0);
902 8118089 : vro1 = &tem1;
903 8118089 : deref1 = false;
904 : }
905 21585772 : if (deref2 && vro2->opcode == ADDR_EXPR)
906 : {
907 8118099 : memset (&tem2, 0, sizeof (tem2));
908 8118099 : tem2.op0 = TREE_OPERAND (vro2->op0, 0);
909 8118099 : tem2.type = TREE_TYPE (tem2.op0);
910 8118099 : tem2.opcode = TREE_CODE (tem2.op0);
911 8118099 : vro2 = &tem2;
912 8118099 : deref2 = false;
913 : }
914 21585772 : if (deref1 != deref2)
915 : return false;
916 21530539 : if (!vn_reference_op_eq (vro1, vro2))
917 : return false;
918 : /* Both alignment and alias set are not relevant for the produced
919 : value but need to be included when doing lexical comparison.
920 : We also need to make sure that the access path ends in an
921 : access of the same size as otherwise we might assume an access
922 : may not trap while in fact it might. */
923 21518558 : if (lexical
924 2202005 : && (vro1->opcode == MEM_REF
925 2202005 : || vro1->opcode == TARGET_MEM_REF)
926 22241703 : && (TYPE_ALIGN (vro1->type) != TYPE_ALIGN (vro2->type)
927 722964 : || (TYPE_SIZE (vro1->type) != TYPE_SIZE (vro2->type)
928 6 : && (! TYPE_SIZE (vro1->type)
929 6 : || ! TYPE_SIZE (vro2->type)
930 6 : || ! operand_equal_p (TYPE_SIZE (vro1->type),
931 6 : TYPE_SIZE (vro2->type))))
932 2168874 : || (get_deref_alias_set (vro1->opcode == MEM_REF
933 722958 : ? TREE_TYPE (vro1->op0)
934 0 : : TREE_TYPE (vro1->op2))
935 1445916 : != get_deref_alias_set (vro2->opcode == MEM_REF
936 722958 : ? TREE_TYPE (vro2->op0)
937 0 : : TREE_TYPE (vro2->op2)))))
938 3242 : return false;
939 21515316 : ++j;
940 21515316 : ++i;
941 : }
942 43030632 : while (vr1->operands.length () != i
943 64545948 : || vr2->operands.length () != j);
944 :
945 : return true;
946 : }
947 :
948 : /* Copy the operations present in load/store REF into RESULT, a vector of
949 : vn_reference_op_s's. */
950 :
951 : void
952 220242011 : copy_reference_ops_from_ref (tree ref, vec<vn_reference_op_s> *result)
953 : {
954 : /* For non-calls, store the information that makes up the address. */
955 220242011 : tree orig = ref;
956 763833328 : while (ref)
957 : {
958 543591317 : vn_reference_op_s temp;
959 :
960 543591317 : memset (&temp, 0, sizeof (temp));
961 543591317 : temp.type = TREE_TYPE (ref);
962 543591317 : temp.opcode = TREE_CODE (ref);
963 543591317 : temp.off = -1;
964 :
965 543591317 : switch (temp.opcode)
966 : {
967 14867688 : case MODIFY_EXPR:
968 14867688 : temp.op0 = TREE_OPERAND (ref, 1);
969 14867688 : break;
970 137 : case WITH_SIZE_EXPR:
971 137 : temp.op0 = TREE_OPERAND (ref, 1);
972 137 : temp.off = 0;
973 137 : break;
974 115974840 : case MEM_REF:
975 : /* The base address gets its own vn_reference_op_s structure. */
976 115974840 : temp.op0 = TREE_OPERAND (ref, 1);
977 115974840 : if (!mem_ref_offset (ref).to_shwi (&temp.off))
978 0 : temp.off = -1;
979 115974840 : temp.clique = MR_DEPENDENCE_CLIQUE (ref);
980 115974840 : temp.base = MR_DEPENDENCE_BASE (ref);
981 115974840 : temp.reverse = REF_REVERSE_STORAGE_ORDER (ref);
982 115974840 : break;
983 2541553 : case TARGET_MEM_REF:
984 : /* The base address gets its own vn_reference_op_s structure. */
985 2541553 : temp.op0 = TMR_INDEX (ref);
986 2541553 : temp.op1 = TMR_STEP (ref);
987 2541553 : temp.op2 = TMR_OFFSET (ref);
988 2541553 : temp.clique = MR_DEPENDENCE_CLIQUE (ref);
989 2541553 : temp.base = MR_DEPENDENCE_BASE (ref);
990 2541553 : result->safe_push (temp);
991 2541553 : memset (&temp, 0, sizeof (temp));
992 2541553 : temp.type = NULL_TREE;
993 2541553 : temp.opcode = ERROR_MARK;
994 2541553 : temp.op0 = TMR_INDEX2 (ref);
995 2541553 : temp.off = -1;
996 2541553 : break;
997 725315 : case BIT_FIELD_REF:
998 : /* Record bits, position and storage order. */
999 725315 : temp.op0 = TREE_OPERAND (ref, 1);
1000 725315 : temp.op1 = TREE_OPERAND (ref, 2);
1001 1449920 : if (!multiple_p (bit_field_offset (ref), BITS_PER_UNIT, &temp.off))
1002 710 : temp.off = -1;
1003 725315 : temp.reverse = REF_REVERSE_STORAGE_ORDER (ref);
1004 725315 : break;
1005 144575885 : case COMPONENT_REF:
1006 : /* The field decl is enough to unambiguously specify the field,
1007 : so use its type here. */
1008 144575885 : temp.type = TREE_TYPE (TREE_OPERAND (ref, 1));
1009 144575885 : temp.op0 = TREE_OPERAND (ref, 1);
1010 144575885 : temp.op1 = TREE_OPERAND (ref, 2);
1011 289149331 : temp.reverse = (AGGREGATE_TYPE_P (TREE_TYPE (TREE_OPERAND (ref, 0)))
1012 289149066 : && TYPE_REVERSE_STORAGE_ORDER
1013 : (TREE_TYPE (TREE_OPERAND (ref, 0))));
1014 144575885 : {
1015 144575885 : tree this_offset = component_ref_field_offset (ref);
1016 144575885 : if (this_offset
1017 144575885 : && poly_int_tree_p (this_offset))
1018 : {
1019 144573749 : tree bit_offset = DECL_FIELD_BIT_OFFSET (TREE_OPERAND (ref, 1));
1020 144573749 : if (TREE_INT_CST_LOW (bit_offset) % BITS_PER_UNIT == 0)
1021 : {
1022 144108245 : poly_offset_int off
1023 144108245 : = (wi::to_poly_offset (this_offset)
1024 144108245 : + (wi::to_offset (bit_offset) >> LOG2_BITS_PER_UNIT));
1025 : /* Prohibit value-numbering zero offset components
1026 : of addresses the same before the pass folding
1027 : __builtin_object_size had a chance to run. Likewise
1028 : for components of zero size at arbitrary offset. */
1029 144108245 : if (TREE_CODE (orig) != ADDR_EXPR
1030 4808504 : || (TYPE_SIZE (temp.type)
1031 4795502 : && integer_nonzerop (TYPE_SIZE (temp.type))
1032 6182872 : && maybe_ne (off, 0))
1033 147060164 : || (cfun->curr_properties & PROP_objsz))
1034 142718705 : off.to_shwi (&temp.off);
1035 : }
1036 : }
1037 : }
1038 : break;
1039 38126756 : case ARRAY_RANGE_REF:
1040 38126756 : case ARRAY_REF:
1041 38126756 : {
1042 38126756 : tree eltype = TREE_TYPE (TREE_TYPE (TREE_OPERAND (ref, 0)));
1043 : /* Record index as operand. */
1044 38126756 : temp.op0 = TREE_OPERAND (ref, 1);
1045 : /* Always record lower bounds and element size. */
1046 38126756 : temp.op1 = array_ref_low_bound (ref);
1047 : /* But record element size in units of the type alignment. */
1048 38126756 : temp.op2 = TREE_OPERAND (ref, 3);
1049 38126756 : temp.align = eltype->type_common.align;
1050 38126756 : if (! temp.op2)
1051 37916493 : temp.op2 = size_binop (EXACT_DIV_EXPR, TYPE_SIZE_UNIT (eltype),
1052 : size_int (TYPE_ALIGN_UNIT (eltype)));
1053 : /* Prohibit value-numbering addresses of one-after-the-last
1054 : element ARRAY_REFs the same as addresses of other components
1055 : before the pass folding __builtin_object_size had a chance
1056 : to run. */
1057 38126756 : bool avoid_oob = true;
1058 38126756 : if (TREE_CODE (orig) != ADDR_EXPR
1059 470210 : || cfun->curr_properties & PROP_objsz)
1060 : avoid_oob = false;
1061 222309 : else if (poly_int_tree_p (temp.op0))
1062 : {
1063 74372 : tree ub = array_ref_up_bound (ref);
1064 74372 : if (ub
1065 72734 : && poly_int_tree_p (ub)
1066 : /* ??? The C frontend for T[0] uses [0:] and the
1067 : C++ frontend [0:-1U]. See layout_type for how
1068 : awkward this is. */
1069 64519 : && !integer_minus_onep (ub)
1070 147106 : && known_le (wi::to_poly_offset (temp.op0),
1071 : wi::to_poly_offset (ub)))
1072 63672 : avoid_oob = false;
1073 : }
1074 38126756 : if (poly_int_tree_p (temp.op0)
1075 21867934 : && poly_int_tree_p (temp.op1)
1076 21867906 : && TREE_CODE (temp.op2) == INTEGER_CST
1077 59934054 : && !avoid_oob)
1078 : {
1079 43594906 : poly_offset_int off = ((wi::to_poly_offset (temp.op0)
1080 65392359 : - wi::to_poly_offset (temp.op1))
1081 43594906 : * wi::to_offset (temp.op2)
1082 21797453 : * vn_ref_op_align_unit (&temp));
1083 21797453 : off.to_shwi (&temp.off);
1084 : }
1085 38126756 : temp.reverse = (AGGREGATE_TYPE_P (TREE_TYPE (TREE_OPERAND (ref, 0)))
1086 38126756 : && TYPE_REVERSE_STORAGE_ORDER
1087 : (TREE_TYPE (TREE_OPERAND (ref, 0))));
1088 : }
1089 38126756 : break;
1090 81324543 : case VAR_DECL:
1091 81324543 : if (DECL_HARD_REGISTER (ref))
1092 : {
1093 20325 : temp.op0 = ref;
1094 20325 : break;
1095 : }
1096 : /* Fallthru. */
1097 84667084 : case PARM_DECL:
1098 84667084 : case CONST_DECL:
1099 84667084 : case RESULT_DECL:
1100 : /* Canonicalize decls to MEM[&decl] which is what we end up with
1101 : when valueizing MEM[ptr] with ptr = &decl. */
1102 84667084 : temp.opcode = MEM_REF;
1103 84667084 : temp.op0 = build_int_cst (build_pointer_type (TREE_TYPE (ref)), 0);
1104 84667084 : temp.off = 0;
1105 84667084 : result->safe_push (temp);
1106 84667084 : temp.opcode = ADDR_EXPR;
1107 84667084 : temp.op0 = build1 (ADDR_EXPR, TREE_TYPE (temp.op0), ref);
1108 84667084 : temp.type = TREE_TYPE (temp.op0);
1109 84667084 : temp.off = -1;
1110 84667084 : break;
1111 94960559 : case STRING_CST:
1112 94960559 : case INTEGER_CST:
1113 94960559 : case POLY_INT_CST:
1114 94960559 : case COMPLEX_CST:
1115 94960559 : case VECTOR_CST:
1116 94960559 : case REAL_CST:
1117 94960559 : case FIXED_CST:
1118 94960559 : case CONSTRUCTOR:
1119 94960559 : case SSA_NAME:
1120 94960559 : temp.op0 = ref;
1121 94960559 : break;
1122 44723113 : case ADDR_EXPR:
1123 44723113 : if (is_gimple_min_invariant (ref))
1124 : {
1125 40594043 : temp.op0 = ref;
1126 40594043 : break;
1127 : }
1128 : break;
1129 : /* These are only interesting for their operands, their
1130 : existence, and their type. They will never be the last
1131 : ref in the chain of references (IE they require an
1132 : operand), so we don't have to put anything
1133 : for op* as it will be handled by the iteration */
1134 484441 : case REALPART_EXPR:
1135 484441 : temp.off = 0;
1136 484441 : break;
1137 1434082 : case VIEW_CONVERT_EXPR:
1138 1434082 : temp.off = 0;
1139 1434082 : temp.reverse = storage_order_barrier_p (ref);
1140 1434082 : break;
1141 489539 : case IMAGPART_EXPR:
1142 : /* This is only interesting for its constant offset. */
1143 489539 : temp.off = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (ref)));
1144 489539 : break;
1145 0 : default:
1146 0 : gcc_unreachable ();
1147 : }
1148 543591317 : result->safe_push (temp);
1149 :
1150 543591317 : if (REFERENCE_CLASS_P (ref)
1151 239238906 : || TREE_CODE (ref) == MODIFY_EXPR
1152 224371218 : || TREE_CODE (ref) == WITH_SIZE_EXPR
1153 767962398 : || (TREE_CODE (ref) == ADDR_EXPR
1154 44723113 : && !is_gimple_min_invariant (ref)))
1155 323349306 : ref = TREE_OPERAND (ref, 0);
1156 : else
1157 : ref = NULL_TREE;
1158 : }
1159 220242011 : }
1160 :
1161 : /* Build a alias-oracle reference abstraction in *REF from the vn_reference
1162 : operands in *OPS, the reference alias set SET and the reference type TYPE.
1163 : Return true if something useful was produced. */
1164 :
1165 : bool
1166 14400191 : ao_ref_init_from_vn_reference (ao_ref *ref,
1167 : alias_set_type set, alias_set_type base_set,
1168 : tree type, const vec<vn_reference_op_s> &ops)
1169 : {
1170 14400191 : unsigned i;
1171 14400191 : tree base = NULL_TREE;
1172 14400191 : tree *op0_p = &base;
1173 14400191 : poly_offset_int offset = 0;
1174 14400191 : poly_offset_int max_size;
1175 14400191 : poly_offset_int size = -1;
1176 14400191 : tree size_tree = NULL_TREE;
1177 :
1178 : /* We don't handle calls. */
1179 14400191 : if (!type)
1180 : return false;
1181 :
1182 14400191 : machine_mode mode = TYPE_MODE (type);
1183 14400191 : if (mode == BLKmode)
1184 63765 : size_tree = TYPE_SIZE (type);
1185 : else
1186 28672852 : size = GET_MODE_BITSIZE (mode);
1187 14336426 : if (size_tree != NULL_TREE
1188 63765 : && poly_int_tree_p (size_tree))
1189 63765 : size = wi::to_poly_offset (size_tree);
1190 :
1191 : /* Lower the final access size from the outermost expression. */
1192 14400191 : const_vn_reference_op_t cst_op = &ops[0];
1193 : /* Cast away constness for the sake of the const-unsafe
1194 : FOR_EACH_VEC_ELT(). */
1195 14400191 : vn_reference_op_t op = const_cast<vn_reference_op_t>(cst_op);
1196 14400191 : size_tree = NULL_TREE;
1197 14400191 : if (op->opcode == COMPONENT_REF)
1198 4956765 : size_tree = DECL_SIZE (op->op0);
1199 9443426 : else if (op->opcode == BIT_FIELD_REF)
1200 58951 : size_tree = op->op0;
1201 5015716 : if (size_tree != NULL_TREE
1202 5015716 : && poly_int_tree_p (size_tree)
1203 10031432 : && (!known_size_p (size)
1204 14400191 : || known_lt (wi::to_poly_offset (size_tree), size)))
1205 35287 : size = wi::to_poly_offset (size_tree);
1206 :
1207 : /* Initially, maxsize is the same as the accessed element size.
1208 : In the following it will only grow (or become -1). */
1209 14400191 : max_size = size;
1210 :
1211 : /* Compute cumulative bit-offset for nested component-refs and array-refs,
1212 : and find the ultimate containing object. */
1213 55285434 : FOR_EACH_VEC_ELT (ops, i, op)
1214 : {
1215 41033784 : switch (op->opcode)
1216 : {
1217 : case CALL_EXPR:
1218 : return false;
1219 :
1220 : /* Record the base objects. */
1221 13940937 : case MEM_REF:
1222 13940937 : *op0_p = build2 (MEM_REF, op->type,
1223 : NULL_TREE, op->op0);
1224 13940937 : MR_DEPENDENCE_CLIQUE (*op0_p) = op->clique;
1225 13940937 : MR_DEPENDENCE_BASE (*op0_p) = op->base;
1226 13940937 : op0_p = &TREE_OPERAND (*op0_p, 0);
1227 13940937 : break;
1228 :
1229 310185 : case TARGET_MEM_REF:
1230 930555 : *op0_p = build5 (TARGET_MEM_REF, op->type,
1231 : NULL_TREE, op->op2, op->op0,
1232 310185 : op->op1, ops[i+1].op0);
1233 310185 : MR_DEPENDENCE_CLIQUE (*op0_p) = op->clique;
1234 310185 : MR_DEPENDENCE_BASE (*op0_p) = op->base;
1235 310185 : op0_p = &TREE_OPERAND (*op0_p, 0);
1236 310185 : ++i;
1237 310185 : break;
1238 :
1239 : /* Unwrap some of the wrapped decls. */
1240 6542861 : case ADDR_EXPR:
1241 : /* Apart from ADDR_EXPR arguments to MEM_REF. */
1242 6542861 : if (base != NULL_TREE
1243 6542861 : && TREE_CODE (base) == MEM_REF
1244 6506886 : && op->op0
1245 13049747 : && DECL_P (TREE_OPERAND (op->op0, 0)))
1246 : {
1247 6499927 : const_vn_reference_op_t pop = &ops[i-1];
1248 6499927 : base = TREE_OPERAND (op->op0, 0);
1249 6499927 : if (known_eq (pop->off, -1))
1250 : {
1251 25 : max_size = -1;
1252 25 : offset = 0;
1253 : }
1254 : else
1255 19499706 : offset += poly_offset_int (pop->off) * BITS_PER_UNIT;
1256 : op0_p = NULL;
1257 : break;
1258 : }
1259 : /* Fallthru. */
1260 7751723 : case PARM_DECL:
1261 7751723 : case CONST_DECL:
1262 7751723 : case RESULT_DECL:
1263 : /* ??? We shouldn't see these, but un-canonicalize what
1264 : copy_reference_ops_from_ref does when visiting MEM_REF. */
1265 7751723 : case VAR_DECL:
1266 : /* ??? And for this only have DECL_HARD_REGISTER. */
1267 7751723 : case STRING_CST:
1268 : /* This can show up in ARRAY_REF bases. */
1269 7751723 : case INTEGER_CST:
1270 7751723 : case SSA_NAME:
1271 7751723 : *op0_p = op->op0;
1272 7751723 : op0_p = NULL;
1273 7751723 : break;
1274 :
1275 : /* And now the usual component-reference style ops. */
1276 58951 : case BIT_FIELD_REF:
1277 58951 : offset += wi::to_poly_offset (op->op1);
1278 58951 : break;
1279 :
1280 8120079 : case COMPONENT_REF:
1281 8120079 : {
1282 8120079 : tree field = op->op0;
1283 : /* We do not have a complete COMPONENT_REF tree here so we
1284 : cannot use component_ref_field_offset. Do the interesting
1285 : parts manually. */
1286 8120079 : tree this_offset = DECL_FIELD_OFFSET (field);
1287 :
1288 8120079 : if (op->op1 || !poly_int_tree_p (this_offset))
1289 234 : max_size = -1;
1290 : else
1291 : {
1292 8119845 : poly_offset_int woffset = (wi::to_poly_offset (this_offset)
1293 8119845 : << LOG2_BITS_PER_UNIT);
1294 8119845 : woffset += wi::to_offset (DECL_FIELD_BIT_OFFSET (field));
1295 8119845 : offset += woffset;
1296 : }
1297 : break;
1298 : }
1299 :
1300 2999315 : case ARRAY_RANGE_REF:
1301 2999315 : case ARRAY_REF:
1302 : /* Use the recorded constant offset. */
1303 2999315 : if (maybe_eq (op->off, -1))
1304 1186842 : max_size = -1;
1305 : else
1306 5437419 : offset += poly_offset_int (op->off) * BITS_PER_UNIT;
1307 : break;
1308 :
1309 : case REALPART_EXPR:
1310 : break;
1311 :
1312 : case IMAGPART_EXPR:
1313 40885243 : offset += size;
1314 : break;
1315 :
1316 : case VIEW_CONVERT_EXPR:
1317 : break;
1318 :
1319 : case POLY_INT_CST:
1320 : case COMPLEX_CST:
1321 : case VECTOR_CST:
1322 : case REAL_CST:
1323 : case FIXED_CST:
1324 : case CONSTRUCTOR:
1325 : return false;
1326 :
1327 : default:
1328 : return false;
1329 : }
1330 : }
1331 :
1332 14251650 : if (base == NULL_TREE)
1333 : return false;
1334 :
1335 14251650 : ref->ref = NULL_TREE;
1336 14251650 : ref->base = base;
1337 14251650 : ref->ref_alias_set = set;
1338 14251650 : ref->base_alias_set = base_set;
1339 : /* We discount volatiles from value-numbering elsewhere. */
1340 14251650 : ref->volatile_p = false;
1341 :
1342 14251650 : if (!size.to_shwi (&ref->size) || maybe_lt (ref->size, 0))
1343 : {
1344 0 : ref->offset = 0;
1345 0 : ref->size = -1;
1346 0 : ref->max_size = -1;
1347 0 : return true;
1348 : }
1349 :
1350 14251650 : if (!offset.to_shwi (&ref->offset))
1351 : {
1352 26 : ref->offset = 0;
1353 26 : ref->max_size = -1;
1354 26 : return true;
1355 : }
1356 :
1357 14251624 : if (!max_size.to_shwi (&ref->max_size) || maybe_lt (ref->max_size, 0))
1358 1036044 : ref->max_size = -1;
1359 :
1360 : return true;
1361 : }
1362 :
1363 : /* Copy the operations present in load/store/call REF into RESULT, a vector of
1364 : vn_reference_op_s's. */
1365 :
1366 : static void
1367 9184515 : copy_reference_ops_from_call (gcall *call,
1368 : vec<vn_reference_op_s> *result)
1369 : {
1370 9184515 : vn_reference_op_s temp;
1371 9184515 : unsigned i;
1372 9184515 : tree lhs = gimple_call_lhs (call);
1373 9184515 : int lr;
1374 :
1375 : /* If 2 calls have a different non-ssa lhs, vdef value numbers should be
1376 : different. By adding the lhs here in the vector, we ensure that the
1377 : hashcode is different, guaranteeing a different value number. */
1378 9184515 : if (lhs && TREE_CODE (lhs) != SSA_NAME)
1379 : {
1380 438863 : memset (&temp, 0, sizeof (temp));
1381 438863 : temp.opcode = MODIFY_EXPR;
1382 438863 : temp.type = TREE_TYPE (lhs);
1383 438863 : temp.op0 = lhs;
1384 438863 : temp.off = -1;
1385 438863 : result->safe_push (temp);
1386 : }
1387 :
1388 : /* Copy the type, opcode, function, static chain and EH region, if any. */
1389 9184515 : memset (&temp, 0, sizeof (temp));
1390 9184515 : temp.type = gimple_call_fntype (call);
1391 9184515 : temp.opcode = CALL_EXPR;
1392 9184515 : temp.op0 = gimple_call_fn (call);
1393 9184515 : if (gimple_call_internal_p (call))
1394 528802 : temp.clique = gimple_call_internal_fn (call);
1395 9184515 : temp.op1 = gimple_call_chain (call);
1396 9184515 : if (stmt_could_throw_p (cfun, call) && (lr = lookup_stmt_eh_lp (call)) > 0)
1397 603965 : temp.op2 = size_int (lr);
1398 9184515 : temp.off = -1;
1399 9184515 : result->safe_push (temp);
1400 :
1401 : /* Copy the call arguments. As they can be references as well,
1402 : just chain them together. */
1403 27011223 : for (i = 0; i < gimple_call_num_args (call); ++i)
1404 : {
1405 17826708 : tree callarg = gimple_call_arg (call, i);
1406 17826708 : copy_reference_ops_from_ref (callarg, result);
1407 : }
1408 9184515 : }
1409 :
1410 : /* Fold *& at position *I_P in a vn_reference_op_s vector *OPS. Updates
1411 : *I_P to point to the last element of the replacement. */
1412 : static bool
1413 125771570 : vn_reference_fold_indirect (vec<vn_reference_op_s> *ops,
1414 : unsigned int *i_p)
1415 : {
1416 125771570 : unsigned int i = *i_p;
1417 125771570 : vn_reference_op_t op = &(*ops)[i];
1418 125771570 : vn_reference_op_t mem_op = &(*ops)[i - 1];
1419 125771570 : tree addr_base;
1420 125771570 : poly_int64 addr_offset = 0;
1421 :
1422 : /* The only thing we have to do is from &OBJ.foo.bar add the offset
1423 : from .foo.bar to the preceding MEM_REF offset and replace the
1424 : address with &OBJ. */
1425 125771570 : addr_base = get_addr_base_and_unit_offset_1 (TREE_OPERAND (op->op0, 0),
1426 : &addr_offset, vn_valueize);
1427 125771570 : gcc_checking_assert (addr_base && TREE_CODE (addr_base) != MEM_REF);
1428 125771570 : if (addr_base != TREE_OPERAND (op->op0, 0))
1429 : {
1430 657504 : poly_offset_int off
1431 657504 : = (poly_offset_int::from (wi::to_poly_wide (mem_op->op0),
1432 : SIGNED)
1433 657504 : + addr_offset);
1434 657504 : mem_op->op0 = wide_int_to_tree (TREE_TYPE (mem_op->op0), off);
1435 657504 : op->op0 = build_fold_addr_expr (addr_base);
1436 657504 : if (tree_fits_shwi_p (mem_op->op0))
1437 657437 : mem_op->off = tree_to_shwi (mem_op->op0);
1438 : else
1439 67 : mem_op->off = -1;
1440 657504 : return true;
1441 : }
1442 : return false;
1443 : }
1444 :
1445 : /* Fold *& at position *I_P in a vn_reference_op_s vector *OPS. Updates
1446 : *I_P to point to the last element of the replacement. */
1447 : static bool
1448 84459645 : vn_reference_maybe_forwprop_address (vec<vn_reference_op_s> *ops,
1449 : unsigned int *i_p)
1450 : {
1451 84459645 : bool changed = false;
1452 92322922 : vn_reference_op_t op;
1453 :
1454 92322922 : do
1455 : {
1456 92322922 : unsigned int i = *i_p;
1457 92322922 : op = &(*ops)[i];
1458 92322922 : vn_reference_op_t mem_op = &(*ops)[i - 1];
1459 92322922 : gimple *def_stmt;
1460 92322922 : enum tree_code code;
1461 92322922 : poly_offset_int off;
1462 :
1463 92322922 : def_stmt = SSA_NAME_DEF_STMT (op->op0);
1464 92322922 : if (!is_gimple_assign (def_stmt))
1465 84457694 : return changed;
1466 :
1467 37847160 : code = gimple_assign_rhs_code (def_stmt);
1468 37847160 : if (code != ADDR_EXPR
1469 37847160 : && code != POINTER_PLUS_EXPR)
1470 : return changed;
1471 :
1472 20386497 : off = poly_offset_int::from (wi::to_poly_wide (mem_op->op0), SIGNED);
1473 :
1474 : /* The only thing we have to do is from &OBJ.foo.bar add the offset
1475 : from .foo.bar to the preceding MEM_REF offset and replace the
1476 : address with &OBJ. */
1477 20386497 : if (code == ADDR_EXPR)
1478 : {
1479 939166 : tree addr, addr_base;
1480 939166 : poly_int64 addr_offset;
1481 :
1482 939166 : addr = gimple_assign_rhs1 (def_stmt);
1483 939166 : addr_base = get_addr_base_and_unit_offset_1 (TREE_OPERAND (addr, 0),
1484 : &addr_offset,
1485 : vn_valueize);
1486 : /* If that didn't work because the address isn't invariant propagate
1487 : the reference tree from the address operation in case the current
1488 : dereference isn't offsetted. */
1489 939166 : if (!addr_base
1490 279562 : && *i_p == ops->length () - 1
1491 139781 : && known_eq (off, 0)
1492 : /* This makes us disable this transform for PRE where the
1493 : reference ops might be also used for code insertion which
1494 : is invalid. */
1495 1023806 : && default_vn_walk_kind == VN_WALKREWRITE)
1496 : {
1497 84550 : auto_vec<vn_reference_op_s, 32> tem;
1498 84550 : copy_reference_ops_from_ref (TREE_OPERAND (addr, 0), &tem);
1499 : /* Make sure to preserve TBAA info. The only objects not
1500 : wrapped in MEM_REFs that can have their address taken are
1501 : STRING_CSTs. */
1502 84550 : if (tem.length () >= 2
1503 84550 : && tem[tem.length () - 2].opcode == MEM_REF)
1504 : {
1505 84535 : vn_reference_op_t new_mem_op = &tem[tem.length () - 2];
1506 84535 : new_mem_op->op0
1507 84535 : = wide_int_to_tree (TREE_TYPE (mem_op->op0),
1508 169070 : wi::to_poly_wide (new_mem_op->op0));
1509 : }
1510 : else
1511 15 : gcc_assert (tem.last ().opcode == STRING_CST);
1512 84550 : ops->pop ();
1513 84550 : ops->pop ();
1514 84550 : ops->safe_splice (tem);
1515 84550 : --*i_p;
1516 84550 : return true;
1517 84550 : }
1518 854616 : if (!addr_base
1519 799385 : || TREE_CODE (addr_base) != MEM_REF
1520 1652184 : || (TREE_CODE (TREE_OPERAND (addr_base, 0)) == SSA_NAME
1521 795707 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (TREE_OPERAND (addr_base,
1522 : 0))))
1523 : return changed;
1524 :
1525 797568 : off += addr_offset;
1526 797568 : off += mem_ref_offset (addr_base);
1527 797568 : op->op0 = TREE_OPERAND (addr_base, 0);
1528 : }
1529 : else
1530 : {
1531 19447331 : tree ptr, ptroff;
1532 19447331 : ptr = gimple_assign_rhs1 (def_stmt);
1533 19447331 : ptroff = gimple_assign_rhs2 (def_stmt);
1534 19447331 : if (TREE_CODE (ptr) != SSA_NAME
1535 17780460 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ptr)
1536 : /* Make sure to not endlessly recurse.
1537 : See gcc.dg/tree-ssa/20040408-1.c for an example. Can easily
1538 : happen when we value-number a PHI to its backedge value. */
1539 17779063 : || SSA_VAL (ptr) == op->op0
1540 37226394 : || !poly_int_tree_p (ptroff))
1541 12379671 : return changed;
1542 :
1543 7067660 : off += wi::to_poly_offset (ptroff);
1544 7067660 : op->op0 = ptr;
1545 : }
1546 :
1547 7865228 : mem_op->op0 = wide_int_to_tree (TREE_TYPE (mem_op->op0), off);
1548 7865228 : if (tree_fits_shwi_p (mem_op->op0))
1549 7561999 : mem_op->off = tree_to_shwi (mem_op->op0);
1550 : else
1551 303229 : mem_op->off = -1;
1552 : /* ??? Can end up with endless recursion here!?
1553 : gcc.c-torture/execute/strcmp-1.c */
1554 7865228 : if (TREE_CODE (op->op0) == SSA_NAME)
1555 7863367 : op->op0 = SSA_VAL (op->op0);
1556 7865228 : if (TREE_CODE (op->op0) != SSA_NAME)
1557 1951 : op->opcode = TREE_CODE (op->op0);
1558 :
1559 7865228 : changed = true;
1560 : }
1561 : /* Tail-recurse. */
1562 7865228 : while (TREE_CODE (op->op0) == SSA_NAME);
1563 :
1564 : /* Fold a remaining *&. */
1565 1951 : if (TREE_CODE (op->op0) == ADDR_EXPR)
1566 261 : vn_reference_fold_indirect (ops, i_p);
1567 :
1568 : return changed;
1569 : }
1570 :
1571 : /* Optimize the reference REF to a constant if possible or return
1572 : NULL_TREE if not. */
1573 :
1574 : tree
1575 108730960 : fully_constant_vn_reference_p (vn_reference_t ref)
1576 : {
1577 108730960 : vec<vn_reference_op_s> operands = ref->operands;
1578 108730960 : vn_reference_op_t op;
1579 :
1580 : /* Try to simplify the translated expression if it is
1581 : a call to a builtin function with at most two arguments. */
1582 108730960 : op = &operands[0];
1583 108730960 : if (op->opcode == CALL_EXPR
1584 89058 : && (!op->op0
1585 81679 : || (TREE_CODE (op->op0) == ADDR_EXPR
1586 81679 : && TREE_CODE (TREE_OPERAND (op->op0, 0)) == FUNCTION_DECL
1587 81679 : && fndecl_built_in_p (TREE_OPERAND (op->op0, 0),
1588 : BUILT_IN_NORMAL)))
1589 72096 : && operands.length () >= 2
1590 108803012 : && operands.length () <= 3)
1591 : {
1592 34210 : vn_reference_op_t arg0, arg1 = NULL;
1593 34210 : bool anyconst = false;
1594 34210 : arg0 = &operands[1];
1595 34210 : if (operands.length () > 2)
1596 5552 : arg1 = &operands[2];
1597 34210 : if (TREE_CODE_CLASS (arg0->opcode) == tcc_constant
1598 34210 : || (arg0->opcode == ADDR_EXPR
1599 13585 : && is_gimple_min_invariant (arg0->op0)))
1600 : anyconst = true;
1601 34210 : if (arg1
1602 34210 : && (TREE_CODE_CLASS (arg1->opcode) == tcc_constant
1603 4030 : || (arg1->opcode == ADDR_EXPR
1604 571 : && is_gimple_min_invariant (arg1->op0))))
1605 : anyconst = true;
1606 32117 : if (anyconst)
1607 : {
1608 21765 : combined_fn fn;
1609 21765 : if (op->op0)
1610 20813 : fn = as_combined_fn (DECL_FUNCTION_CODE
1611 20813 : (TREE_OPERAND (op->op0, 0)));
1612 : else
1613 952 : fn = as_combined_fn ((internal_fn) op->clique);
1614 21765 : tree folded;
1615 21765 : if (arg1)
1616 2701 : folded = fold_const_call (fn, ref->type, arg0->op0, arg1->op0);
1617 : else
1618 19064 : folded = fold_const_call (fn, ref->type, arg0->op0);
1619 21765 : if (folded
1620 21765 : && is_gimple_min_invariant (folded))
1621 : return folded;
1622 : }
1623 : }
1624 :
1625 : /* Simplify reads from constants or constant initializers. */
1626 108696750 : else if (BITS_PER_UNIT == 8
1627 108696750 : && ref->type
1628 108696750 : && COMPLETE_TYPE_P (ref->type)
1629 217393458 : && is_gimple_reg_type (ref->type))
1630 : {
1631 104393634 : poly_int64 off = 0;
1632 104393634 : HOST_WIDE_INT size;
1633 104393634 : if (INTEGRAL_TYPE_P (ref->type))
1634 53053647 : size = TYPE_PRECISION (ref->type);
1635 51339987 : else if (tree_fits_shwi_p (TYPE_SIZE (ref->type)))
1636 51339987 : size = tree_to_shwi (TYPE_SIZE (ref->type));
1637 : else
1638 108730960 : return NULL_TREE;
1639 104393634 : if (size % BITS_PER_UNIT != 0
1640 102637009 : || size > MAX_BITSIZE_MODE_ANY_MODE)
1641 : return NULL_TREE;
1642 102635682 : size /= BITS_PER_UNIT;
1643 102635682 : unsigned i;
1644 189709146 : for (i = 0; i < operands.length (); ++i)
1645 : {
1646 189709146 : if (TREE_CODE_CLASS (operands[i].opcode) == tcc_constant)
1647 : {
1648 309 : ++i;
1649 309 : break;
1650 : }
1651 189708837 : if (operands[i].reverse)
1652 : return NULL_TREE;
1653 189700479 : if (known_eq (operands[i].off, -1))
1654 : return NULL_TREE;
1655 175993088 : off += operands[i].off;
1656 175993088 : if (operands[i].opcode == MEM_REF)
1657 : {
1658 88919624 : ++i;
1659 88919624 : break;
1660 : }
1661 : }
1662 88919933 : vn_reference_op_t base = &operands[--i];
1663 88919933 : tree ctor = error_mark_node;
1664 88919933 : tree decl = NULL_TREE;
1665 88919933 : if (TREE_CODE_CLASS (base->opcode) == tcc_constant)
1666 309 : ctor = base->op0;
1667 88919624 : else if (base->opcode == MEM_REF
1668 88919624 : && base[1].opcode == ADDR_EXPR
1669 146181866 : && (VAR_P (TREE_OPERAND (base[1].op0, 0))
1670 3529092 : || TREE_CODE (TREE_OPERAND (base[1].op0, 0)) == CONST_DECL
1671 3529032 : || TREE_CODE (TREE_OPERAND (base[1].op0, 0)) == STRING_CST))
1672 : {
1673 53739065 : decl = TREE_OPERAND (base[1].op0, 0);
1674 53739065 : if (TREE_CODE (decl) == STRING_CST)
1675 : ctor = decl;
1676 : else
1677 53733210 : ctor = ctor_for_folding (decl);
1678 : }
1679 88914078 : if (ctor == NULL_TREE)
1680 386 : return build_zero_cst (ref->type);
1681 88919547 : else if (ctor != error_mark_node)
1682 : {
1683 100851 : HOST_WIDE_INT const_off;
1684 100851 : if (decl)
1685 : {
1686 201084 : tree res = fold_ctor_reference (ref->type, ctor,
1687 100542 : off * BITS_PER_UNIT,
1688 100542 : size * BITS_PER_UNIT, decl);
1689 100542 : if (res)
1690 : {
1691 57782 : STRIP_USELESS_TYPE_CONVERSION (res);
1692 57782 : if (is_gimple_min_invariant (res))
1693 108730960 : return res;
1694 : }
1695 : }
1696 309 : else if (off.is_constant (&const_off))
1697 : {
1698 309 : unsigned char buf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
1699 309 : int len = native_encode_expr (ctor, buf, size, const_off);
1700 309 : if (len > 0)
1701 139 : return native_interpret_expr (ref->type, buf, len);
1702 : }
1703 : }
1704 : }
1705 :
1706 : return NULL_TREE;
1707 : }
1708 :
1709 : /* Return true if OPS contain a storage order barrier. */
1710 :
1711 : static bool
1712 58693546 : contains_storage_order_barrier_p (vec<vn_reference_op_s> ops)
1713 : {
1714 58693546 : vn_reference_op_t op;
1715 58693546 : unsigned i;
1716 :
1717 229829454 : FOR_EACH_VEC_ELT (ops, i, op)
1718 171135908 : if (op->opcode == VIEW_CONVERT_EXPR && op->reverse)
1719 : return true;
1720 :
1721 : return false;
1722 : }
1723 :
1724 : /* Return true if OPS represent an access with reverse storage order. */
1725 :
1726 : static bool
1727 58701814 : reverse_storage_order_for_component_p (vec<vn_reference_op_s> ops)
1728 : {
1729 58701814 : unsigned i = 0;
1730 58701814 : if (ops[i].opcode == REALPART_EXPR || ops[i].opcode == IMAGPART_EXPR)
1731 : ++i;
1732 58701814 : switch (ops[i].opcode)
1733 : {
1734 56630676 : case ARRAY_REF:
1735 56630676 : case COMPONENT_REF:
1736 56630676 : case BIT_FIELD_REF:
1737 56630676 : case MEM_REF:
1738 56630676 : return ops[i].reverse;
1739 : default:
1740 : return false;
1741 : }
1742 : }
1743 :
1744 : /* Transform any SSA_NAME's in a vector of vn_reference_op_s
1745 : structures into their value numbers. This is done in-place, and
1746 : the vector passed in is returned. *VALUEIZED_ANYTHING will specify
1747 : whether any operands were valueized. */
1748 :
1749 : static void
1750 217423070 : valueize_refs_1 (vec<vn_reference_op_s> *orig, bool *valueized_anything,
1751 : bool with_avail = false)
1752 : {
1753 217423070 : *valueized_anything = false;
1754 :
1755 876689691 : for (unsigned i = 0; i < orig->length (); ++i)
1756 : {
1757 659266621 : re_valueize:
1758 663154972 : vn_reference_op_t vro = &(*orig)[i];
1759 663154972 : if (vro->opcode == SSA_NAME
1760 566945434 : || (vro->op0 && TREE_CODE (vro->op0) == SSA_NAME))
1761 : {
1762 120421901 : tree tem = with_avail ? vn_valueize (vro->op0) : SSA_VAL (vro->op0);
1763 120421901 : if (tem != vro->op0)
1764 : {
1765 17721985 : *valueized_anything = true;
1766 17721985 : vro->op0 = tem;
1767 : }
1768 : /* If it transforms from an SSA_NAME to a constant, update
1769 : the opcode. */
1770 120421901 : if (TREE_CODE (vro->op0) != SSA_NAME && vro->opcode == SSA_NAME)
1771 2041721 : vro->opcode = TREE_CODE (vro->op0);
1772 : }
1773 663154972 : if (vro->op1 && TREE_CODE (vro->op1) == SSA_NAME)
1774 : {
1775 26314 : tree tem = with_avail ? vn_valueize (vro->op1) : SSA_VAL (vro->op1);
1776 26314 : if (tem != vro->op1)
1777 : {
1778 607 : *valueized_anything = true;
1779 607 : vro->op1 = tem;
1780 : }
1781 : }
1782 663154972 : if (vro->op2 && TREE_CODE (vro->op2) == SSA_NAME)
1783 : {
1784 205463 : tree tem = with_avail ? vn_valueize (vro->op2) : SSA_VAL (vro->op2);
1785 205463 : if (tem != vro->op2)
1786 : {
1787 119592 : *valueized_anything = true;
1788 119592 : vro->op2 = tem;
1789 : }
1790 : }
1791 : /* If it transforms from an SSA_NAME to an address, fold with
1792 : a preceding indirect reference. */
1793 663154972 : if (i > 0
1794 445653638 : && vro->op0
1795 442146240 : && TREE_CODE (vro->op0) == ADDR_EXPR
1796 794750366 : && (*orig)[i - 1].opcode == MEM_REF)
1797 : {
1798 125771309 : if (vn_reference_fold_indirect (orig, &i))
1799 657504 : *valueized_anything = true;
1800 : }
1801 537383663 : else if (i > 0
1802 319882329 : && vro->opcode == SSA_NAME
1803 631551480 : && (*orig)[i - 1].opcode == MEM_REF)
1804 : {
1805 84459645 : if (vn_reference_maybe_forwprop_address (orig, &i))
1806 : {
1807 3888351 : *valueized_anything = true;
1808 : /* Re-valueize the current operand. */
1809 3888351 : goto re_valueize;
1810 : }
1811 : }
1812 : /* If it transforms a non-constant ARRAY_REF into a constant
1813 : one, adjust the constant offset. */
1814 452924018 : else if ((vro->opcode == ARRAY_REF
1815 452924018 : || vro->opcode == ARRAY_RANGE_REF)
1816 39216168 : && known_eq (vro->off, -1)
1817 17047883 : && poly_int_tree_p (vro->op0)
1818 4699382 : && poly_int_tree_p (vro->op1)
1819 457623400 : && TREE_CODE (vro->op2) == INTEGER_CST)
1820 : {
1821 : /* Prohibit value-numbering addresses of one-after-the-last
1822 : element ARRAY_REFs the same as addresses of other components
1823 : before the pass folding __builtin_object_size had a chance
1824 : to run. */
1825 4565687 : if (!(cfun->curr_properties & PROP_objsz)
1826 5803589 : && (*orig)[0].opcode == ADDR_EXPR)
1827 : {
1828 35482 : tree dom = TYPE_DOMAIN ((*orig)[i + 1].type);
1829 53870 : if (!dom
1830 35332 : || !TYPE_MAX_VALUE (dom)
1831 25448 : || !poly_int_tree_p (TYPE_MAX_VALUE (dom))
1832 52662 : || integer_minus_onep (TYPE_MAX_VALUE (dom)))
1833 19195 : continue;
1834 17094 : if (!known_le (wi::to_poly_offset (vro->op0),
1835 : wi::to_poly_offset (TYPE_MAX_VALUE (dom))))
1836 807 : continue;
1837 : }
1838 :
1839 9092984 : poly_offset_int off = ((wi::to_poly_offset (vro->op0)
1840 13639476 : - wi::to_poly_offset (vro->op1))
1841 9092984 : * wi::to_offset (vro->op2)
1842 4546492 : * vn_ref_op_align_unit (vro));
1843 4546492 : off.to_shwi (&vro->off);
1844 : }
1845 : }
1846 217423070 : }
1847 :
1848 : static void
1849 12602577 : valueize_refs (vec<vn_reference_op_s> *orig)
1850 : {
1851 12602577 : bool tem;
1852 0 : valueize_refs_1 (orig, &tem);
1853 0 : }
1854 :
1855 : static vec<vn_reference_op_s> shared_lookup_references;
1856 :
1857 : /* Create a vector of vn_reference_op_s structures from REF, a
1858 : REFERENCE_CLASS_P tree. The vector is shared among all callers of
1859 : this function. *VALUEIZED_ANYTHING will specify whether any
1860 : operands were valueized. */
1861 :
1862 : static vec<vn_reference_op_s>
1863 178952987 : valueize_shared_reference_ops_from_ref (tree ref, bool *valueized_anything)
1864 : {
1865 178952987 : if (!ref)
1866 0 : return vNULL;
1867 178952987 : shared_lookup_references.truncate (0);
1868 178952987 : copy_reference_ops_from_ref (ref, &shared_lookup_references);
1869 178952987 : valueize_refs_1 (&shared_lookup_references, valueized_anything);
1870 178952987 : return shared_lookup_references;
1871 : }
1872 :
1873 : /* Create a vector of vn_reference_op_s structures from CALL, a
1874 : call statement. The vector is shared among all callers of
1875 : this function. */
1876 :
1877 : static vec<vn_reference_op_s>
1878 9184515 : valueize_shared_reference_ops_from_call (gcall *call)
1879 : {
1880 9184515 : if (!call)
1881 0 : return vNULL;
1882 9184515 : shared_lookup_references.truncate (0);
1883 9184515 : copy_reference_ops_from_call (call, &shared_lookup_references);
1884 9184515 : valueize_refs (&shared_lookup_references);
1885 9184515 : return shared_lookup_references;
1886 : }
1887 :
1888 : /* Lookup a SCCVN reference operation VR in the current hash table.
1889 : Returns the resulting value number if it exists in the hash table,
1890 : NULL_TREE otherwise. VNRESULT will be filled in with the actual
1891 : vn_reference_t stored in the hashtable if something is found. */
1892 :
1893 : static tree
1894 64873802 : vn_reference_lookup_1 (vn_reference_t vr, vn_reference_t *vnresult)
1895 : {
1896 64873802 : vn_reference_s **slot;
1897 64873802 : hashval_t hash;
1898 :
1899 64873802 : hash = vr->hashcode;
1900 64873802 : slot = valid_info->references->find_slot_with_hash (vr, hash, NO_INSERT);
1901 64873802 : if (slot)
1902 : {
1903 8107525 : if (vnresult)
1904 8107525 : *vnresult = (vn_reference_t)*slot;
1905 8107525 : return ((vn_reference_t)*slot)->result;
1906 : }
1907 :
1908 : return NULL_TREE;
1909 : }
1910 :
1911 :
1912 : /* Partial definition tracking support. */
1913 :
1914 : struct pd_range
1915 : {
1916 : HOST_WIDE_INT offset;
1917 : HOST_WIDE_INT size;
1918 : pd_range *m_children[2];
1919 : };
1920 :
1921 : struct pd_data
1922 : {
1923 : tree rhs;
1924 : HOST_WIDE_INT rhs_off;
1925 : HOST_WIDE_INT offset;
1926 : HOST_WIDE_INT size;
1927 : };
1928 :
1929 : /* Context for alias walking. */
1930 :
1931 : struct vn_walk_cb_data
1932 : {
1933 60681916 : vn_walk_cb_data (vn_reference_t vr_, tree orig_ref_, tree *last_vuse_ptr_,
1934 : vn_lookup_kind vn_walk_kind_, bool tbaa_p_, tree mask_,
1935 : bool redundant_store_removal_p_)
1936 60681916 : : vr (vr_), last_vuse_ptr (last_vuse_ptr_), last_vuse (NULL_TREE),
1937 60681916 : mask (mask_), masked_result (NULL_TREE), same_val (NULL_TREE),
1938 60681916 : vn_walk_kind (vn_walk_kind_),
1939 60681916 : tbaa_p (tbaa_p_), redundant_store_removal_p (redundant_store_removal_p_),
1940 121363832 : saved_operands (vNULL), first_range (), first_set (-2),
1941 121363832 : first_base_set (-2)
1942 : {
1943 60681916 : if (!last_vuse_ptr)
1944 28142685 : last_vuse_ptr = &last_vuse;
1945 60681916 : ao_ref_init (&orig_ref, orig_ref_);
1946 60681916 : if (mask)
1947 : {
1948 300567 : wide_int w = wi::to_wide (mask);
1949 300567 : unsigned int pos = 0, prec = w.get_precision ();
1950 300567 : pd_data pd;
1951 300567 : pd.rhs = build_constructor (NULL_TREE, NULL);
1952 300567 : pd.rhs_off = 0;
1953 : /* When bitwise and with a constant is done on a memory load,
1954 : we don't really need all the bits to be defined or defined
1955 : to constants, we don't really care what is in the position
1956 : corresponding to 0 bits in the mask.
1957 : So, push the ranges of those 0 bits in the mask as artificial
1958 : zero stores and let the partial def handling code do the
1959 : rest. */
1960 644592 : while (pos < prec)
1961 : {
1962 624501 : int tz = wi::ctz (w);
1963 624501 : if (pos + tz > prec)
1964 280476 : tz = prec - pos;
1965 624501 : if (tz)
1966 : {
1967 474538 : if (BYTES_BIG_ENDIAN)
1968 : pd.offset = prec - pos - tz;
1969 : else
1970 474538 : pd.offset = pos;
1971 474538 : pd.size = tz;
1972 474538 : void *r = push_partial_def (pd, 0, 0, 0, prec);
1973 474538 : gcc_assert (r == NULL_TREE);
1974 : }
1975 624501 : pos += tz;
1976 624501 : if (pos == prec)
1977 : break;
1978 344025 : w = wi::lrshift (w, tz);
1979 344025 : tz = wi::ctz (wi::bit_not (w));
1980 344025 : if (pos + tz > prec)
1981 0 : tz = prec - pos;
1982 344025 : pos += tz;
1983 344025 : w = wi::lrshift (w, tz);
1984 : }
1985 300567 : }
1986 60681916 : }
1987 : ~vn_walk_cb_data ();
1988 : void *finish (alias_set_type, alias_set_type, tree);
1989 : void *push_partial_def (pd_data pd,
1990 : alias_set_type, alias_set_type, HOST_WIDE_INT,
1991 : HOST_WIDE_INT);
1992 :
1993 : vn_reference_t vr;
1994 : ao_ref orig_ref;
1995 : tree *last_vuse_ptr;
1996 : tree last_vuse;
1997 : tree mask;
1998 : tree masked_result;
1999 : tree same_val;
2000 : vn_lookup_kind vn_walk_kind;
2001 : bool tbaa_p;
2002 : bool redundant_store_removal_p;
2003 : vec<vn_reference_op_s> saved_operands;
2004 :
2005 : /* The VDEFs of partial defs we come along. */
2006 : auto_vec<pd_data, 2> partial_defs;
2007 : /* The first defs range to avoid splay tree setup in most cases. */
2008 : pd_range first_range;
2009 : alias_set_type first_set;
2010 : alias_set_type first_base_set;
2011 : default_splay_tree<pd_range *> known_ranges;
2012 : obstack ranges_obstack;
2013 : static constexpr HOST_WIDE_INT bufsize = 64;
2014 : };
2015 :
2016 60681916 : vn_walk_cb_data::~vn_walk_cb_data ()
2017 : {
2018 60681916 : if (known_ranges)
2019 169283 : obstack_free (&ranges_obstack, NULL);
2020 60681916 : saved_operands.release ();
2021 60681916 : }
2022 :
2023 : void *
2024 1534639 : vn_walk_cb_data::finish (alias_set_type set, alias_set_type base_set, tree val)
2025 : {
2026 1534639 : if (first_set != -2)
2027 : {
2028 430650 : set = first_set;
2029 430650 : base_set = first_base_set;
2030 : }
2031 1534639 : if (mask)
2032 : {
2033 453 : masked_result = val;
2034 453 : return (void *) -1;
2035 : }
2036 1534186 : if (same_val && !operand_equal_p (val, same_val))
2037 : return (void *) -1;
2038 1530361 : vec<vn_reference_op_s> &operands
2039 1530361 : = saved_operands.exists () ? saved_operands : vr->operands;
2040 1530361 : return vn_reference_lookup_or_insert_for_pieces (last_vuse, set, base_set,
2041 : vr->offset, vr->max_size,
2042 1530361 : vr->type, operands, val);
2043 : }
2044 :
2045 : /* Push PD to the vector of partial definitions returning a
2046 : value when we are ready to combine things with VUSE, SET and MAXSIZEI,
2047 : NULL when we want to continue looking for partial defs or -1
2048 : on failure. */
2049 :
2050 : void *
2051 554617 : vn_walk_cb_data::push_partial_def (pd_data pd,
2052 : alias_set_type set, alias_set_type base_set,
2053 : HOST_WIDE_INT offseti,
2054 : HOST_WIDE_INT maxsizei)
2055 : {
2056 : /* We're using a fixed buffer for encoding so fail early if the object
2057 : we want to interpret is bigger. */
2058 554617 : if (maxsizei > bufsize * BITS_PER_UNIT
2059 : || CHAR_BIT != 8
2060 : || BITS_PER_UNIT != 8
2061 : /* Not prepared to handle PDP endian. */
2062 : || BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
2063 : return (void *)-1;
2064 :
2065 : /* Turn too large constant stores into non-constant stores. */
2066 554546 : if (CONSTANT_CLASS_P (pd.rhs) && pd.size > bufsize * BITS_PER_UNIT)
2067 0 : pd.rhs = error_mark_node;
2068 :
2069 : /* And for non-constant or CONSTRUCTOR stores shrink them to only keep at
2070 : most a partial byte before and/or after the region. */
2071 554546 : if (!CONSTANT_CLASS_P (pd.rhs))
2072 : {
2073 514694 : if (pd.offset < offseti)
2074 : {
2075 8206 : HOST_WIDE_INT o = ROUND_DOWN (offseti - pd.offset, BITS_PER_UNIT);
2076 8206 : gcc_assert (pd.size > o);
2077 8206 : pd.size -= o;
2078 8206 : pd.offset += o;
2079 : }
2080 514694 : if (pd.size > maxsizei)
2081 7341 : pd.size = maxsizei + ((pd.size - maxsizei) % BITS_PER_UNIT);
2082 : }
2083 :
2084 554546 : pd.offset -= offseti;
2085 :
2086 1109092 : bool pd_constant_p = (TREE_CODE (pd.rhs) == CONSTRUCTOR
2087 554546 : || CONSTANT_CLASS_P (pd.rhs));
2088 554546 : pd_range *r;
2089 554546 : if (partial_defs.is_empty ())
2090 : {
2091 : /* If we get a clobber upfront, fail. */
2092 354058 : if (TREE_CLOBBER_P (pd.rhs))
2093 : return (void *)-1;
2094 353705 : if (!pd_constant_p)
2095 : return (void *)-1;
2096 324244 : partial_defs.safe_push (pd);
2097 324244 : first_range.offset = pd.offset;
2098 324244 : first_range.size = pd.size;
2099 324244 : first_set = set;
2100 324244 : first_base_set = base_set;
2101 324244 : last_vuse_ptr = NULL;
2102 324244 : r = &first_range;
2103 : /* Go check if the first partial definition was a full one in case
2104 : the caller didn't optimize for this. */
2105 : }
2106 : else
2107 : {
2108 200488 : if (!known_ranges)
2109 : {
2110 : /* ??? Optimize the case where the 2nd partial def completes
2111 : things. */
2112 169283 : gcc_obstack_init (&ranges_obstack);
2113 169283 : known_ranges.insert_max_node (&first_range);
2114 : }
2115 : /* Lookup the offset and see if we need to merge. */
2116 200488 : int comparison = known_ranges.lookup_le
2117 405272 : ([&] (pd_range *r) { return pd.offset < r->offset; },
2118 180237 : [&] (pd_range *r) { return pd.offset > r->offset; });
2119 200488 : r = known_ranges.root ();
2120 200488 : if (comparison >= 0
2121 200488 : && ranges_known_overlap_p (r->offset, r->size + 1,
2122 : pd.offset, pd.size))
2123 : {
2124 : /* Ignore partial defs already covered. Here we also drop shadowed
2125 : clobbers arriving here at the floor. */
2126 5836 : if (known_subrange_p (pd.offset, pd.size, r->offset, r->size))
2127 : return NULL;
2128 5001 : r->size = MAX (r->offset + r->size, pd.offset + pd.size) - r->offset;
2129 : }
2130 : else
2131 : {
2132 : /* pd.offset wasn't covered yet, insert the range. */
2133 194652 : void *addr = XOBNEW (&ranges_obstack, pd_range);
2134 194652 : r = new (addr) pd_range { pd.offset, pd.size, {} };
2135 194652 : known_ranges.insert_relative (comparison, r);
2136 : }
2137 : /* Merge r which now contains pd's range and is a member of the splay
2138 : tree with adjacent overlapping ranges. */
2139 199653 : if (known_ranges.splay_next_node ())
2140 22362 : do
2141 : {
2142 22362 : pd_range *rafter = known_ranges.root ();
2143 22362 : if (!ranges_known_overlap_p (r->offset, r->size + 1,
2144 22362 : rafter->offset, rafter->size))
2145 : break;
2146 22092 : r->size = MAX (r->offset + r->size,
2147 22092 : rafter->offset + rafter->size) - r->offset;
2148 : }
2149 22092 : while (known_ranges.remove_root_and_splay_next ());
2150 : /* If we get a clobber, fail. */
2151 199653 : if (TREE_CLOBBER_P (pd.rhs))
2152 : return (void *)-1;
2153 : /* Non-constants are OK as long as they are shadowed by a constant. */
2154 197459 : if (!pd_constant_p)
2155 : return (void *)-1;
2156 191343 : partial_defs.safe_push (pd);
2157 : }
2158 :
2159 : /* Now we have merged pd's range into the range tree. When we have covered
2160 : [offseti, sizei] then the tree will contain exactly one node which has
2161 : the desired properties and it will be 'r'. */
2162 515587 : if (!known_subrange_p (0, maxsizei, r->offset, r->size))
2163 : /* Continue looking for partial defs. */
2164 : return NULL;
2165 :
2166 : /* Now simply native encode all partial defs in reverse order. */
2167 8704 : unsigned ndefs = partial_defs.length ();
2168 : /* We support up to 512-bit values (for V8DFmode). */
2169 8704 : unsigned char buffer[bufsize + 1];
2170 8704 : unsigned char this_buffer[bufsize + 1];
2171 8704 : int len;
2172 :
2173 8704 : memset (buffer, 0, bufsize + 1);
2174 8704 : unsigned needed_len = ROUND_UP (maxsizei, BITS_PER_UNIT) / BITS_PER_UNIT;
2175 42995 : while (!partial_defs.is_empty ())
2176 : {
2177 25587 : pd_data pd = partial_defs.pop ();
2178 25587 : unsigned int amnt;
2179 25587 : if (TREE_CODE (pd.rhs) == CONSTRUCTOR)
2180 : {
2181 : /* Empty CONSTRUCTOR. */
2182 2073 : if (pd.size >= needed_len * BITS_PER_UNIT)
2183 2073 : len = needed_len;
2184 : else
2185 1786 : len = ROUND_UP (pd.size, BITS_PER_UNIT) / BITS_PER_UNIT;
2186 2073 : memset (this_buffer, 0, len);
2187 : }
2188 23514 : else if (pd.rhs_off >= 0)
2189 : {
2190 47028 : len = native_encode_expr (pd.rhs, this_buffer, bufsize,
2191 23514 : (MAX (0, -pd.offset)
2192 23514 : + pd.rhs_off) / BITS_PER_UNIT);
2193 23514 : if (len <= 0
2194 23514 : || len < (ROUND_UP (pd.size, BITS_PER_UNIT) / BITS_PER_UNIT
2195 23514 : - MAX (0, -pd.offset) / BITS_PER_UNIT))
2196 : {
2197 0 : if (dump_file && (dump_flags & TDF_DETAILS))
2198 0 : fprintf (dump_file, "Failed to encode %u "
2199 : "partial definitions\n", ndefs);
2200 0 : return (void *)-1;
2201 : }
2202 : }
2203 : else /* negative pd.rhs_off indicates we want to chop off first bits */
2204 : {
2205 0 : if (-pd.rhs_off >= bufsize)
2206 : return (void *)-1;
2207 0 : len = native_encode_expr (pd.rhs,
2208 0 : this_buffer + -pd.rhs_off / BITS_PER_UNIT,
2209 0 : bufsize - -pd.rhs_off / BITS_PER_UNIT,
2210 0 : MAX (0, -pd.offset) / BITS_PER_UNIT);
2211 0 : if (len <= 0
2212 0 : || len < (ROUND_UP (pd.size, BITS_PER_UNIT) / BITS_PER_UNIT
2213 0 : - MAX (0, -pd.offset) / BITS_PER_UNIT))
2214 : {
2215 0 : if (dump_file && (dump_flags & TDF_DETAILS))
2216 0 : fprintf (dump_file, "Failed to encode %u "
2217 : "partial definitions\n", ndefs);
2218 0 : return (void *)-1;
2219 : }
2220 : }
2221 :
2222 25587 : unsigned char *p = buffer;
2223 25587 : HOST_WIDE_INT size = pd.size;
2224 25587 : if (pd.offset < 0)
2225 263 : size -= ROUND_DOWN (-pd.offset, BITS_PER_UNIT);
2226 25587 : this_buffer[len] = 0;
2227 25587 : if (BYTES_BIG_ENDIAN)
2228 : {
2229 : /* LSB of this_buffer[len - 1] byte should be at
2230 : pd.offset + pd.size - 1 bits in buffer. */
2231 : amnt = ((unsigned HOST_WIDE_INT) pd.offset
2232 : + pd.size) % BITS_PER_UNIT;
2233 : if (amnt)
2234 : shift_bytes_in_array_right (this_buffer, len + 1, amnt);
2235 : unsigned char *q = this_buffer;
2236 : unsigned int off = 0;
2237 : if (pd.offset >= 0)
2238 : {
2239 : unsigned int msk;
2240 : off = pd.offset / BITS_PER_UNIT;
2241 : gcc_assert (off < needed_len);
2242 : p = buffer + off;
2243 : if (size <= amnt)
2244 : {
2245 : msk = ((1 << size) - 1) << (BITS_PER_UNIT - amnt);
2246 : *p = (*p & ~msk) | (this_buffer[len] & msk);
2247 : size = 0;
2248 : }
2249 : else
2250 : {
2251 : if (TREE_CODE (pd.rhs) != CONSTRUCTOR)
2252 : q = (this_buffer + len
2253 : - (ROUND_UP (size - amnt, BITS_PER_UNIT)
2254 : / BITS_PER_UNIT));
2255 : if (pd.offset % BITS_PER_UNIT)
2256 : {
2257 : msk = -1U << (BITS_PER_UNIT
2258 : - (pd.offset % BITS_PER_UNIT));
2259 : *p = (*p & msk) | (*q & ~msk);
2260 : p++;
2261 : q++;
2262 : off++;
2263 : size -= BITS_PER_UNIT - (pd.offset % BITS_PER_UNIT);
2264 : gcc_assert (size >= 0);
2265 : }
2266 : }
2267 : }
2268 : else if (TREE_CODE (pd.rhs) != CONSTRUCTOR)
2269 : {
2270 : q = (this_buffer + len
2271 : - (ROUND_UP (size - amnt, BITS_PER_UNIT)
2272 : / BITS_PER_UNIT));
2273 : if (pd.offset % BITS_PER_UNIT)
2274 : {
2275 : q++;
2276 : size -= BITS_PER_UNIT - ((unsigned HOST_WIDE_INT) pd.offset
2277 : % BITS_PER_UNIT);
2278 : gcc_assert (size >= 0);
2279 : }
2280 : }
2281 : if ((unsigned HOST_WIDE_INT) size / BITS_PER_UNIT + off
2282 : > needed_len)
2283 : size = (needed_len - off) * BITS_PER_UNIT;
2284 : memcpy (p, q, size / BITS_PER_UNIT);
2285 : if (size % BITS_PER_UNIT)
2286 : {
2287 : unsigned int msk
2288 : = -1U << (BITS_PER_UNIT - (size % BITS_PER_UNIT));
2289 : p += size / BITS_PER_UNIT;
2290 : q += size / BITS_PER_UNIT;
2291 : *p = (*q & msk) | (*p & ~msk);
2292 : }
2293 : }
2294 : else
2295 : {
2296 25587 : if (pd.offset >= 0)
2297 : {
2298 : /* LSB of this_buffer[0] byte should be at pd.offset bits
2299 : in buffer. */
2300 25324 : unsigned int msk;
2301 25324 : size = MIN (size, (HOST_WIDE_INT) needed_len * BITS_PER_UNIT);
2302 25324 : amnt = pd.offset % BITS_PER_UNIT;
2303 25324 : if (amnt)
2304 1507 : shift_bytes_in_array_left (this_buffer, len + 1, amnt);
2305 25324 : unsigned int off = pd.offset / BITS_PER_UNIT;
2306 25324 : gcc_assert (off < needed_len);
2307 25324 : size = MIN (size,
2308 : (HOST_WIDE_INT) (needed_len - off) * BITS_PER_UNIT);
2309 25324 : p = buffer + off;
2310 25324 : if (amnt + size < BITS_PER_UNIT)
2311 : {
2312 : /* Low amnt bits come from *p, then size bits
2313 : from this_buffer[0] and the remaining again from
2314 : *p. */
2315 1079 : msk = ((1 << size) - 1) << amnt;
2316 1079 : *p = (*p & ~msk) | (this_buffer[0] & msk);
2317 1079 : size = 0;
2318 : }
2319 24245 : else if (amnt)
2320 : {
2321 1133 : msk = -1U << amnt;
2322 1133 : *p = (*p & ~msk) | (this_buffer[0] & msk);
2323 1133 : p++;
2324 1133 : size -= (BITS_PER_UNIT - amnt);
2325 : }
2326 : }
2327 : else
2328 : {
2329 263 : amnt = (unsigned HOST_WIDE_INT) pd.offset % BITS_PER_UNIT;
2330 263 : if (amnt)
2331 16 : size -= BITS_PER_UNIT - amnt;
2332 263 : size = MIN (size, (HOST_WIDE_INT) needed_len * BITS_PER_UNIT);
2333 263 : if (amnt)
2334 16 : shift_bytes_in_array_left (this_buffer, len + 1, amnt);
2335 : }
2336 25587 : memcpy (p, this_buffer + (amnt != 0), size / BITS_PER_UNIT);
2337 25587 : p += size / BITS_PER_UNIT;
2338 25587 : if (size % BITS_PER_UNIT)
2339 : {
2340 625 : unsigned int msk = -1U << (size % BITS_PER_UNIT);
2341 625 : *p = (this_buffer[(amnt != 0) + size / BITS_PER_UNIT]
2342 625 : & ~msk) | (*p & msk);
2343 : }
2344 : }
2345 : }
2346 :
2347 8704 : tree type = vr->type;
2348 : /* Make sure to interpret in a type that has a range covering the whole
2349 : access size. */
2350 8704 : if (INTEGRAL_TYPE_P (vr->type) && maxsizei != TYPE_PRECISION (vr->type))
2351 : {
2352 0 : if (BITINT_TYPE_P (vr->type)
2353 26 : && maxsizei > MAX_FIXED_MODE_SIZE)
2354 13 : type = build_bitint_type (maxsizei, TYPE_UNSIGNED (type));
2355 : else
2356 0 : type = build_nonstandard_integer_type (maxsizei, TYPE_UNSIGNED (type));
2357 : }
2358 8704 : tree val;
2359 8704 : if (BYTES_BIG_ENDIAN)
2360 : {
2361 : unsigned sz = needed_len;
2362 : if (maxsizei % BITS_PER_UNIT)
2363 : shift_bytes_in_array_right (buffer, needed_len,
2364 : BITS_PER_UNIT
2365 : - (maxsizei % BITS_PER_UNIT));
2366 : if (INTEGRAL_TYPE_P (type))
2367 : {
2368 : if (TYPE_MODE (type) != BLKmode)
2369 : sz = GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (type));
2370 : else
2371 : sz = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (type));
2372 : }
2373 : if (sz > needed_len)
2374 : {
2375 : memcpy (this_buffer + (sz - needed_len), buffer, needed_len);
2376 : val = native_interpret_expr (type, this_buffer, sz);
2377 : }
2378 : else
2379 : val = native_interpret_expr (type, buffer, needed_len);
2380 : }
2381 : else
2382 8704 : val = native_interpret_expr (type, buffer, bufsize);
2383 : /* If we chop off bits because the types precision doesn't match the memory
2384 : access size this is ok when optimizing reads but not when called from
2385 : the DSE code during elimination. */
2386 8704 : if (val && type != vr->type)
2387 : {
2388 13 : if (! int_fits_type_p (val, vr->type))
2389 : val = NULL_TREE;
2390 : else
2391 13 : val = fold_convert (vr->type, val);
2392 : }
2393 :
2394 8700 : if (val)
2395 : {
2396 8700 : if (dump_file && (dump_flags & TDF_DETAILS))
2397 0 : fprintf (dump_file,
2398 : "Successfully combined %u partial definitions\n", ndefs);
2399 : /* We are using the alias-set of the first store we encounter which
2400 : should be appropriate here. */
2401 8700 : return finish (first_set, first_base_set, val);
2402 : }
2403 : else
2404 : {
2405 4 : if (dump_file && (dump_flags & TDF_DETAILS))
2406 0 : fprintf (dump_file,
2407 : "Failed to interpret %u encoded partial definitions\n", ndefs);
2408 4 : return (void *)-1;
2409 : }
2410 : }
2411 :
2412 : /* Callback for walk_non_aliased_vuses. Adjusts the vn_reference_t VR_
2413 : with the current VUSE and performs the expression lookup. */
2414 :
2415 : static void *
2416 1081475388 : vn_reference_lookup_2 (ao_ref *op, tree vuse, void *data_)
2417 : {
2418 1081475388 : vn_walk_cb_data *data = (vn_walk_cb_data *)data_;
2419 1081475388 : vn_reference_t vr = data->vr;
2420 1081475388 : vn_reference_s **slot;
2421 1081475388 : hashval_t hash;
2422 :
2423 : /* If we have partial definitions recorded we have to go through
2424 : vn_reference_lookup_3. */
2425 2155217211 : if (!data->partial_defs.is_empty ())
2426 : return NULL;
2427 :
2428 1080714871 : if (data->last_vuse_ptr)
2429 : {
2430 1058779775 : *data->last_vuse_ptr = vuse;
2431 1058779775 : data->last_vuse = vuse;
2432 : }
2433 :
2434 : /* Fixup vuse and hash. */
2435 1080714871 : if (vr->vuse)
2436 1080714871 : vr->hashcode = vr->hashcode - SSA_NAME_VERSION (vr->vuse);
2437 1080714871 : vr->vuse = vuse_ssa_val (vuse);
2438 1080714871 : if (vr->vuse)
2439 1080714871 : vr->hashcode = vr->hashcode + SSA_NAME_VERSION (vr->vuse);
2440 :
2441 1080714871 : hash = vr->hashcode;
2442 1080714871 : slot = valid_info->references->find_slot_with_hash (vr, hash, NO_INSERT);
2443 1080714871 : if (slot)
2444 : {
2445 7732359 : if ((*slot)->result && data->saved_operands.exists ())
2446 416783 : return data->finish (vr->set, vr->base_set, (*slot)->result);
2447 : return *slot;
2448 : }
2449 :
2450 1072982512 : if (SSA_NAME_IS_DEFAULT_DEF (vuse))
2451 : {
2452 17994998 : HOST_WIDE_INT op_offset, op_size;
2453 17994998 : tree v = NULL_TREE;
2454 17994998 : tree base = ao_ref_base (op);
2455 :
2456 17994998 : if (base
2457 17994998 : && op->offset.is_constant (&op_offset)
2458 17994998 : && op->size.is_constant (&op_size)
2459 17994998 : && op->max_size_known_p ()
2460 35538519 : && known_eq (op->size, op->max_size))
2461 : {
2462 17242502 : if (TREE_CODE (base) == PARM_DECL)
2463 658164 : v = ipcp_get_aggregate_const (cfun, base, false, op_offset,
2464 : op_size);
2465 16584338 : else if (TREE_CODE (base) == MEM_REF
2466 6811297 : && integer_zerop (TREE_OPERAND (base, 1))
2467 5470494 : && TREE_CODE (TREE_OPERAND (base, 0)) == SSA_NAME
2468 5465306 : && SSA_NAME_IS_DEFAULT_DEF (TREE_OPERAND (base, 0))
2469 20218160 : && (TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (base, 0)))
2470 : == PARM_DECL))
2471 3581583 : v = ipcp_get_aggregate_const (cfun,
2472 3581583 : SSA_NAME_VAR (TREE_OPERAND (base, 0)),
2473 : true, op_offset, op_size);
2474 : }
2475 4239747 : if (v)
2476 1206 : return data->finish (vr->set, vr->base_set, v);
2477 : }
2478 :
2479 : return NULL;
2480 : }
2481 :
2482 : /* Lookup an existing or insert a new vn_reference entry into the
2483 : value table for the VUSE, SET, TYPE, OPERANDS reference which
2484 : has the value VALUE which is either a constant or an SSA name. */
2485 :
2486 : static vn_reference_t
2487 1530361 : vn_reference_lookup_or_insert_for_pieces (tree vuse,
2488 : alias_set_type set,
2489 : alias_set_type base_set,
2490 : poly_int64 offset,
2491 : poly_int64 max_size,
2492 : tree type,
2493 : vec<vn_reference_op_s,
2494 : va_heap> operands,
2495 : tree value)
2496 : {
2497 1530361 : vn_reference_s vr1;
2498 1530361 : vn_reference_t result;
2499 1530361 : unsigned value_id;
2500 1530361 : vr1.vuse = vuse ? SSA_VAL (vuse) : NULL_TREE;
2501 1530361 : vr1.operands = operands;
2502 1530361 : vr1.type = type;
2503 1530361 : vr1.set = set;
2504 1530361 : vr1.base_set = base_set;
2505 1530361 : vr1.offset = offset;
2506 1530361 : vr1.max_size = max_size;
2507 1530361 : vr1.hashcode = vn_reference_compute_hash (&vr1);
2508 1530361 : if (vn_reference_lookup_1 (&vr1, &result))
2509 8897 : return result;
2510 :
2511 1521464 : if (TREE_CODE (value) == SSA_NAME)
2512 349087 : value_id = VN_INFO (value)->value_id;
2513 : else
2514 1172377 : value_id = get_or_alloc_constant_value_id (value);
2515 1521464 : return vn_reference_insert_pieces (vuse, set, base_set, offset, max_size,
2516 1521464 : type, operands.copy (), value, value_id);
2517 : }
2518 :
2519 : /* Return a value-number for RCODE OPS... either by looking up an existing
2520 : value-number for the possibly simplified result or by inserting the
2521 : operation if INSERT is true. If SIMPLIFY is false, return a value
2522 : number for the unsimplified expression. */
2523 :
2524 : static tree
2525 18479151 : vn_nary_build_or_lookup_1 (gimple_match_op *res_op, bool insert,
2526 : bool simplify)
2527 : {
2528 18479151 : tree result = NULL_TREE;
2529 : /* We will be creating a value number for
2530 : RCODE (OPS...).
2531 : So first simplify and lookup this expression to see if it
2532 : is already available. */
2533 : /* For simplification valueize. */
2534 18479151 : unsigned i = 0;
2535 18479151 : if (simplify)
2536 42919900 : for (i = 0; i < res_op->num_ops; ++i)
2537 24446529 : if (TREE_CODE (res_op->ops[i]) == SSA_NAME)
2538 : {
2539 15636959 : tree tem = vn_valueize (res_op->ops[i]);
2540 15636959 : if (!tem)
2541 : break;
2542 15636959 : res_op->ops[i] = tem;
2543 : }
2544 : /* If valueization of an operand fails (it is not available), skip
2545 : simplification. */
2546 18479151 : bool res = false;
2547 18479151 : if (i == res_op->num_ops)
2548 : {
2549 : /* Do not leak not available operands into the simplified expression
2550 : when called from PRE context. */
2551 18473371 : if (rpo_avail)
2552 11050351 : mprts_hook = vn_lookup_simplify_result;
2553 18473371 : res = res_op->resimplify (NULL, vn_valueize);
2554 18473371 : mprts_hook = NULL;
2555 : }
2556 31778593 : gimple *new_stmt = NULL;
2557 18473371 : if (res
2558 18473371 : && gimple_simplified_result_is_gimple_val (res_op))
2559 : {
2560 : /* The expression is already available. */
2561 5173929 : result = res_op->ops[0];
2562 : /* Valueize it, simplification returns sth in AVAIL only. */
2563 5173929 : if (TREE_CODE (result) == SSA_NAME)
2564 289560 : result = SSA_VAL (result);
2565 : }
2566 : else
2567 : {
2568 13305222 : tree val = vn_lookup_simplify_result (res_op);
2569 : /* ??? In weird cases we can end up with internal-fn calls,
2570 : but this isn't expected so throw the result away. See
2571 : PR123040 for an example. */
2572 13305222 : if (!val && insert && res_op->code.is_tree_code ())
2573 : {
2574 135161 : gimple_seq stmts = NULL;
2575 135161 : result = maybe_push_res_to_seq (res_op, &stmts);
2576 135161 : if (result)
2577 : {
2578 135155 : gcc_assert (gimple_seq_singleton_p (stmts));
2579 135155 : new_stmt = gimple_seq_first_stmt (stmts);
2580 : }
2581 : }
2582 : else
2583 : /* The expression is already available. */
2584 : result = val;
2585 : }
2586 289566 : if (new_stmt)
2587 : {
2588 : /* The expression is not yet available, value-number lhs to
2589 : the new SSA_NAME we created. */
2590 : /* Initialize value-number information properly. */
2591 135155 : vn_ssa_aux_t result_info = VN_INFO (result);
2592 135155 : result_info->valnum = result;
2593 135155 : result_info->value_id = get_next_value_id ();
2594 135155 : result_info->visited = 1;
2595 135155 : gimple_seq_add_stmt_without_update (&VN_INFO (result)->expr,
2596 : new_stmt);
2597 135155 : result_info->needs_insertion = true;
2598 : /* ??? PRE phi-translation inserts NARYs without corresponding
2599 : SSA name result. Re-use those but set their result according
2600 : to the stmt we just built. */
2601 135155 : vn_nary_op_t nary = NULL;
2602 135155 : vn_nary_op_lookup_stmt (new_stmt, &nary);
2603 135155 : if (nary)
2604 : {
2605 0 : gcc_assert (! nary->predicated_values && nary->u.result == NULL_TREE);
2606 0 : nary->u.result = gimple_assign_lhs (new_stmt);
2607 : }
2608 : /* As all "inserted" statements are singleton SCCs, insert
2609 : to the valid table. This is strictly needed to
2610 : avoid re-generating new value SSA_NAMEs for the same
2611 : expression during SCC iteration over and over (the
2612 : optimistic table gets cleared after each iteration).
2613 : We do not need to insert into the optimistic table, as
2614 : lookups there will fall back to the valid table. */
2615 : else
2616 : {
2617 135155 : unsigned int length = vn_nary_length_from_stmt (new_stmt);
2618 135155 : vn_nary_op_t vno1
2619 135155 : = alloc_vn_nary_op_noinit (length, &vn_tables_insert_obstack);
2620 135155 : vno1->value_id = result_info->value_id;
2621 135155 : vno1->length = length;
2622 135155 : vno1->predicated_values = 0;
2623 135155 : vno1->u.result = result;
2624 135155 : init_vn_nary_op_from_stmt (vno1, as_a <gassign *> (new_stmt));
2625 135155 : vn_nary_op_insert_into (vno1, valid_info->nary);
2626 : /* Also do not link it into the undo chain. */
2627 135155 : last_inserted_nary = vno1->next;
2628 135155 : vno1->next = (vn_nary_op_t)(void *)-1;
2629 : }
2630 135155 : if (dump_file && (dump_flags & TDF_DETAILS))
2631 : {
2632 590 : fprintf (dump_file, "Inserting name ");
2633 590 : print_generic_expr (dump_file, result);
2634 590 : fprintf (dump_file, " for expression ");
2635 590 : print_gimple_expr (dump_file, new_stmt, 0, TDF_SLIM);
2636 590 : fprintf (dump_file, "\n");
2637 : }
2638 : }
2639 18479151 : return result;
2640 : }
2641 :
2642 : /* Return a value-number for RCODE OPS... either by looking up an existing
2643 : value-number for the simplified result or by inserting the operation. */
2644 :
2645 : static tree
2646 179883 : vn_nary_build_or_lookup (gimple_match_op *res_op)
2647 : {
2648 0 : return vn_nary_build_or_lookup_1 (res_op, true, true);
2649 : }
2650 :
2651 : /* Try to simplify the expression RCODE OPS... of type TYPE and return
2652 : its value if present. Update NARY with a simplified expression if
2653 : it fits. */
2654 :
2655 : tree
2656 7418574 : vn_nary_simplify (vn_nary_op_t nary)
2657 : {
2658 7418574 : if (nary->length > gimple_match_op::MAX_NUM_OPS
2659 : /* For CONSTRUCTOR the vn_nary_op_t and gimple_match_op representation
2660 : does not match. */
2661 7418213 : || nary->opcode == CONSTRUCTOR)
2662 : return NULL_TREE;
2663 7416965 : gimple_match_op op (gimple_match_cond::UNCOND, nary->opcode,
2664 7416965 : nary->type, nary->length);
2665 7416965 : memcpy (op.ops, nary->op, sizeof (tree) * nary->length);
2666 7416965 : tree res = vn_nary_build_or_lookup_1 (&op, false, true);
2667 : /* Do not update *NARY with a simplified result that contains abnormals.
2668 : This matches what maybe_push_res_to_seq does when requesting insertion. */
2669 19468792 : for (unsigned i = 0; i < op.num_ops; ++i)
2670 12051908 : if (TREE_CODE (op.ops[i]) == SSA_NAME
2671 12051908 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op.ops[i]))
2672 : return res;
2673 7416884 : if (op.code.is_tree_code ()
2674 7416884 : && op.num_ops <= nary->length
2675 14833035 : && (tree_code) op.code != CONSTRUCTOR)
2676 : {
2677 7416150 : nary->opcode = (tree_code) op.code;
2678 7416150 : nary->length = op.num_ops;
2679 19466498 : for (unsigned i = 0; i < op.num_ops; ++i)
2680 12050348 : nary->op[i] = op.ops[i];
2681 : }
2682 : return res;
2683 : }
2684 :
2685 : /* Elimination engine. */
2686 :
2687 : class eliminate_dom_walker : public dom_walker
2688 : {
2689 : public:
2690 : eliminate_dom_walker (cdi_direction, bitmap);
2691 : ~eliminate_dom_walker ();
2692 :
2693 : edge before_dom_children (basic_block) final override;
2694 : void after_dom_children (basic_block) final override;
2695 :
2696 : virtual tree eliminate_avail (basic_block, tree op);
2697 : virtual void eliminate_push_avail (basic_block, tree op);
2698 : tree eliminate_insert (basic_block, gimple_stmt_iterator *gsi, tree val);
2699 :
2700 : void eliminate_stmt (basic_block, gimple_stmt_iterator *);
2701 :
2702 : unsigned eliminate_cleanup (bool region_p = false);
2703 :
2704 : bool do_pre;
2705 : unsigned int el_todo;
2706 : unsigned int eliminations;
2707 : unsigned int insertions;
2708 :
2709 : /* SSA names that had their defs inserted by PRE if do_pre. */
2710 : bitmap inserted_exprs;
2711 :
2712 : /* Blocks with statements that have had their EH properties changed. */
2713 : bitmap need_eh_cleanup;
2714 :
2715 : /* Blocks with statements that have had their AB properties changed. */
2716 : bitmap need_ab_cleanup;
2717 :
2718 : /* Local state for the eliminate domwalk. */
2719 : auto_vec<gimple *> to_remove;
2720 : auto_vec<gimple *> to_fixup;
2721 : auto_vec<tree> avail;
2722 : auto_vec<tree> avail_stack;
2723 : };
2724 :
2725 : /* Adaptor to the elimination engine using RPO availability. */
2726 :
2727 12295610 : class rpo_elim : public eliminate_dom_walker
2728 : {
2729 : public:
2730 6147805 : rpo_elim(basic_block entry_)
2731 12295610 : : eliminate_dom_walker (CDI_DOMINATORS, NULL), entry (entry_),
2732 12295610 : m_avail_freelist (NULL) {}
2733 :
2734 : tree eliminate_avail (basic_block, tree op) final override;
2735 :
2736 : void eliminate_push_avail (basic_block, tree) final override;
2737 :
2738 : basic_block entry;
2739 : /* Freelist of avail entries which are allocated from the vn_ssa_aux
2740 : obstack. */
2741 : vn_avail *m_avail_freelist;
2742 : };
2743 :
2744 : /* Return true if BASE1 and BASE2 can be adjusted so they have the
2745 : same address and adjust *OFFSET1 and *OFFSET2 accordingly.
2746 : Otherwise return false. */
2747 :
2748 : static bool
2749 6770929 : adjust_offsets_for_equal_base_address (tree base1, poly_int64 *offset1,
2750 : tree base2, poly_int64 *offset2)
2751 : {
2752 6770929 : poly_int64 soff;
2753 6770929 : if (TREE_CODE (base1) == MEM_REF
2754 3085905 : && TREE_CODE (base2) == MEM_REF)
2755 : {
2756 2474798 : if (mem_ref_offset (base1).to_shwi (&soff))
2757 : {
2758 2474798 : base1 = TREE_OPERAND (base1, 0);
2759 2474798 : *offset1 += soff * BITS_PER_UNIT;
2760 : }
2761 2474798 : if (mem_ref_offset (base2).to_shwi (&soff))
2762 : {
2763 2474798 : base2 = TREE_OPERAND (base2, 0);
2764 2474798 : *offset2 += soff * BITS_PER_UNIT;
2765 : }
2766 2474798 : return operand_equal_p (base1, base2, 0);
2767 : }
2768 4296131 : return operand_equal_p (base1, base2, OEP_ADDRESS_OF);
2769 : }
2770 :
2771 : /* Callback for walk_non_aliased_vuses. Tries to perform a lookup
2772 : from the statement defining VUSE and if not successful tries to
2773 : translate *REFP and VR_ through an aggregate copy at the definition
2774 : of VUSE. If *DISAMBIGUATE_ONLY is true then do not perform translation
2775 : of *REF and *VR. If only disambiguation was performed then
2776 : *DISAMBIGUATE_ONLY is set to true. */
2777 :
2778 : static void *
2779 42167877 : vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
2780 : translate_flags *disambiguate_only)
2781 : {
2782 42167877 : vn_walk_cb_data *data = (vn_walk_cb_data *)data_;
2783 42167877 : vn_reference_t vr = data->vr;
2784 42167877 : gimple *def_stmt = SSA_NAME_DEF_STMT (vuse);
2785 42167877 : tree base = ao_ref_base (ref);
2786 42167877 : HOST_WIDE_INT offseti = 0, maxsizei, sizei = 0;
2787 42167877 : static vec<vn_reference_op_s> lhs_ops;
2788 42167877 : ao_ref lhs_ref;
2789 42167877 : bool lhs_ref_ok = false;
2790 42167877 : poly_int64 copy_size;
2791 :
2792 : /* First try to disambiguate after value-replacing in the definitions LHS. */
2793 42167877 : if (is_gimple_assign (def_stmt))
2794 : {
2795 20755525 : tree lhs = gimple_assign_lhs (def_stmt);
2796 20755525 : bool valueized_anything = false;
2797 : /* Avoid re-allocation overhead. */
2798 20755525 : lhs_ops.truncate (0);
2799 20755525 : basic_block saved_rpo_bb = vn_context_bb;
2800 20755525 : vn_context_bb = gimple_bb (def_stmt);
2801 20755525 : if (*disambiguate_only <= TR_VALUEIZE_AND_DISAMBIGUATE)
2802 : {
2803 13514324 : copy_reference_ops_from_ref (lhs, &lhs_ops);
2804 13514324 : valueize_refs_1 (&lhs_ops, &valueized_anything, true);
2805 : }
2806 20755525 : vn_context_bb = saved_rpo_bb;
2807 20755525 : ao_ref_init (&lhs_ref, lhs);
2808 20755525 : lhs_ref_ok = true;
2809 20755525 : if (valueized_anything
2810 2004278 : && ao_ref_init_from_vn_reference
2811 2004278 : (&lhs_ref, ao_ref_alias_set (&lhs_ref),
2812 2004278 : ao_ref_base_alias_set (&lhs_ref), TREE_TYPE (lhs), lhs_ops)
2813 22759803 : && !refs_may_alias_p_1 (ref, &lhs_ref, data->tbaa_p))
2814 : {
2815 1715728 : *disambiguate_only = TR_VALUEIZE_AND_DISAMBIGUATE;
2816 8275933 : return NULL;
2817 : }
2818 :
2819 : /* When the def is a CLOBBER we can optimistically disambiguate
2820 : against it since any overlap it would be undefined behavior.
2821 : Avoid this for obvious must aliases to save compile-time though.
2822 : We also may not do this when the query is used for redundant
2823 : store removal. */
2824 19039797 : if (!data->redundant_store_removal_p
2825 10458943 : && gimple_clobber_p (def_stmt)
2826 19543221 : && !operand_equal_p (ao_ref_base (&lhs_ref), base, OEP_ADDRESS_OF))
2827 : {
2828 477914 : *disambiguate_only = TR_DISAMBIGUATE;
2829 477914 : return NULL;
2830 : }
2831 :
2832 : /* Besides valueizing the LHS we can also use access-path based
2833 : disambiguation on the original non-valueized ref. */
2834 18561883 : if (!ref->ref
2835 : && lhs_ref_ok
2836 2615149 : && data->orig_ref.ref)
2837 : {
2838 : /* We want to use the non-valueized LHS for this, but avoid redundant
2839 : work. */
2840 1814318 : ao_ref *lref = &lhs_ref;
2841 1814318 : ao_ref lref_alt;
2842 1814318 : if (valueized_anything)
2843 : {
2844 113014 : ao_ref_init (&lref_alt, lhs);
2845 113014 : lref = &lref_alt;
2846 : }
2847 1814318 : if (!refs_may_alias_p_1 (&data->orig_ref, lref, data->tbaa_p))
2848 : {
2849 234688 : *disambiguate_only = (valueized_anything
2850 117344 : ? TR_VALUEIZE_AND_DISAMBIGUATE
2851 : : TR_DISAMBIGUATE);
2852 117344 : return NULL;
2853 : }
2854 : }
2855 :
2856 : /* If we reach a clobbering statement try to skip it and see if
2857 : we find a VN result with exactly the same value as the
2858 : possible clobber. In this case we can ignore the clobber
2859 : and return the found value. */
2860 18444539 : if (!gimple_has_volatile_ops (def_stmt)
2861 17099708 : && ((is_gimple_reg_type (TREE_TYPE (lhs))
2862 12564288 : && types_compatible_p (TREE_TYPE (lhs), vr->type)
2863 9789155 : && !storage_order_barrier_p (lhs)
2864 9789155 : && !reverse_storage_order_for_component_p (lhs))
2865 7310557 : || TREE_CODE (gimple_assign_rhs1 (def_stmt)) == CONSTRUCTOR)
2866 10839849 : && (ref->ref || data->orig_ref.ref)
2867 10376346 : && !data->mask
2868 10354170 : && data->partial_defs.is_empty ()
2869 10351939 : && multiple_p (get_object_alignment
2870 : (ref->ref ? ref->ref : data->orig_ref.ref),
2871 : ref->size)
2872 41254197 : && multiple_p (get_object_alignment (lhs), ref->size))
2873 : {
2874 9959169 : HOST_WIDE_INT offset2i, size2i;
2875 9959169 : poly_int64 offset = ref->offset;
2876 9959169 : poly_int64 maxsize = ref->max_size;
2877 :
2878 9959169 : gcc_assert (lhs_ref_ok);
2879 9959169 : tree base2 = ao_ref_base (&lhs_ref);
2880 9959169 : poly_int64 offset2 = lhs_ref.offset;
2881 9959169 : poly_int64 size2 = lhs_ref.size;
2882 9959169 : poly_int64 maxsize2 = lhs_ref.max_size;
2883 :
2884 9959169 : tree rhs = gimple_assign_rhs1 (def_stmt);
2885 9959169 : if (TREE_CODE (rhs) == CONSTRUCTOR)
2886 1022052 : rhs = integer_zero_node;
2887 : /* ??? We may not compare to ahead values which might be from
2888 : a different loop iteration but only to loop invariants. Use
2889 : CONSTANT_CLASS_P (unvalueized!) as conservative approximation.
2890 : The one-hop lookup below doesn't have this issue since there's
2891 : a virtual PHI before we ever reach a backedge to cross.
2892 : We can skip multiple defs as long as they are from the same
2893 : value though. */
2894 9959169 : if (data->same_val
2895 9959169 : && !operand_equal_p (data->same_val, rhs))
2896 : ;
2897 : /* When this is a (partial) must-def, leave it to handling
2898 : below in case we are interested in the value. */
2899 9673086 : else if (!(*disambiguate_only > TR_TRANSLATE)
2900 3318067 : && base2
2901 3318067 : && known_eq (maxsize2, size2)
2902 2318963 : && adjust_offsets_for_equal_base_address (base, &offset,
2903 : base2, &offset2)
2904 1132097 : && offset2.is_constant (&offset2i)
2905 1132097 : && size2.is_constant (&size2i)
2906 1132097 : && maxsize.is_constant (&maxsizei)
2907 1132097 : && offset.is_constant (&offseti)
2908 10805183 : && ranges_known_overlap_p (offseti, maxsizei, offset2i,
2909 : size2i))
2910 : ;
2911 8627067 : else if (CONSTANT_CLASS_P (rhs))
2912 : {
2913 4185273 : if (dump_file && (dump_flags & TDF_DETAILS))
2914 : {
2915 2192 : fprintf (dump_file,
2916 : "Skipping possible redundant definition ");
2917 2192 : print_gimple_stmt (dump_file, def_stmt, 0);
2918 : }
2919 : /* Delay the actual compare of the values to the end of the walk
2920 : but do not update last_vuse from here. */
2921 4185273 : data->last_vuse_ptr = NULL;
2922 4185273 : data->same_val = rhs;
2923 4249219 : return NULL;
2924 : }
2925 : else
2926 : {
2927 4441794 : tree saved_vuse = vr->vuse;
2928 4441794 : hashval_t saved_hashcode = vr->hashcode;
2929 4441794 : if (vr->vuse)
2930 4441794 : vr->hashcode = vr->hashcode - SSA_NAME_VERSION (vr->vuse);
2931 8883588 : vr->vuse = vuse_ssa_val (gimple_vuse (def_stmt));
2932 4441794 : if (vr->vuse)
2933 4441794 : vr->hashcode = vr->hashcode + SSA_NAME_VERSION (vr->vuse);
2934 4441794 : vn_reference_t vnresult = NULL;
2935 : /* Do not use vn_reference_lookup_2 since that might perform
2936 : expression hashtable insertion but this lookup crosses
2937 : a possible may-alias making such insertion conditionally
2938 : invalid. */
2939 4441794 : vn_reference_lookup_1 (vr, &vnresult);
2940 : /* Need to restore vr->vuse and vr->hashcode. */
2941 4441794 : vr->vuse = saved_vuse;
2942 4441794 : vr->hashcode = saved_hashcode;
2943 4441794 : if (vnresult)
2944 : {
2945 241717 : if (TREE_CODE (rhs) == SSA_NAME)
2946 240198 : rhs = SSA_VAL (rhs);
2947 241717 : if (vnresult->result
2948 241717 : && operand_equal_p (vnresult->result, rhs, 0))
2949 63946 : return vnresult;
2950 : }
2951 : }
2952 : }
2953 : }
2954 21412352 : else if (*disambiguate_only <= TR_VALUEIZE_AND_DISAMBIGUATE
2955 19223381 : && gimple_call_builtin_p (def_stmt, BUILT_IN_NORMAL)
2956 23492864 : && gimple_call_num_args (def_stmt) <= 4)
2957 : {
2958 : /* For builtin calls valueize its arguments and call the
2959 : alias oracle again. Valueization may improve points-to
2960 : info of pointers and constify size and position arguments.
2961 : Originally this was motivated by PR61034 which has
2962 : conditional calls to free falsely clobbering ref because
2963 : of imprecise points-to info of the argument. */
2964 : tree oldargs[4];
2965 : bool valueized_anything = false;
2966 4875697 : for (unsigned i = 0; i < gimple_call_num_args (def_stmt); ++i)
2967 : {
2968 3350738 : oldargs[i] = gimple_call_arg (def_stmt, i);
2969 3350738 : tree val = vn_valueize (oldargs[i]);
2970 3350738 : if (val != oldargs[i])
2971 : {
2972 124008 : gimple_call_set_arg (def_stmt, i, val);
2973 124008 : valueized_anything = true;
2974 : }
2975 : }
2976 1524959 : if (valueized_anything)
2977 : {
2978 194176 : bool res = call_may_clobber_ref_p_1 (as_a <gcall *> (def_stmt),
2979 97088 : ref, data->tbaa_p);
2980 353857 : for (unsigned i = 0; i < gimple_call_num_args (def_stmt); ++i)
2981 256769 : gimple_call_set_arg (def_stmt, i, oldargs[i]);
2982 97088 : if (!res)
2983 : {
2984 31072 : *disambiguate_only = TR_VALUEIZE_AND_DISAMBIGUATE;
2985 31072 : return NULL;
2986 : }
2987 : }
2988 : }
2989 :
2990 35576600 : if (*disambiguate_only > TR_TRANSLATE)
2991 : return (void *)-1;
2992 :
2993 : /* If we cannot constrain the size of the reference we cannot
2994 : test if anything kills it. */
2995 23684955 : if (!ref->max_size_known_p ())
2996 : return (void *)-1;
2997 :
2998 23263693 : poly_int64 offset = ref->offset;
2999 23263693 : poly_int64 maxsize = ref->max_size;
3000 :
3001 : /* def_stmt may-defs *ref. See if we can derive a value for *ref
3002 : from that definition.
3003 : 1) Memset. */
3004 23263693 : if (is_gimple_reg_type (vr->type)
3005 23255764 : && (gimple_call_builtin_p (def_stmt, BUILT_IN_MEMSET)
3006 23166257 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMSET_CHK))
3007 90049 : && (integer_zerop (gimple_call_arg (def_stmt, 1))
3008 32457 : || ((TREE_CODE (gimple_call_arg (def_stmt, 1)) == INTEGER_CST
3009 9056 : || (INTEGRAL_TYPE_P (vr->type) && known_eq (ref->size, 8)))
3010 : && CHAR_BIT == 8
3011 : && BITS_PER_UNIT == 8
3012 : && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
3013 31191 : && offset.is_constant (&offseti)
3014 31191 : && ref->size.is_constant (&sizei)
3015 31191 : && (offseti % BITS_PER_UNIT == 0
3016 39 : || TREE_CODE (gimple_call_arg (def_stmt, 1)) == INTEGER_CST)))
3017 88783 : && (poly_int_tree_p (gimple_call_arg (def_stmt, 2))
3018 36266 : || (TREE_CODE (gimple_call_arg (def_stmt, 2)) == SSA_NAME
3019 36266 : && poly_int_tree_p (SSA_VAL (gimple_call_arg (def_stmt, 2)))))
3020 23316773 : && (TREE_CODE (gimple_call_arg (def_stmt, 0)) == ADDR_EXPR
3021 30371 : || TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME))
3022 : {
3023 53039 : tree base2;
3024 53039 : poly_int64 offset2, size2, maxsize2;
3025 53039 : bool reverse;
3026 53039 : tree ref2 = gimple_call_arg (def_stmt, 0);
3027 53039 : if (TREE_CODE (ref2) == SSA_NAME)
3028 : {
3029 30330 : ref2 = SSA_VAL (ref2);
3030 30330 : if (TREE_CODE (ref2) == SSA_NAME
3031 30330 : && (TREE_CODE (base) != MEM_REF
3032 19354 : || TREE_OPERAND (base, 0) != ref2))
3033 : {
3034 24002 : gimple *def_stmt = SSA_NAME_DEF_STMT (ref2);
3035 24002 : if (gimple_assign_single_p (def_stmt)
3036 24002 : && gimple_assign_rhs_code (def_stmt) == ADDR_EXPR)
3037 802 : ref2 = gimple_assign_rhs1 (def_stmt);
3038 : }
3039 : }
3040 53039 : if (TREE_CODE (ref2) == ADDR_EXPR)
3041 : {
3042 26496 : ref2 = TREE_OPERAND (ref2, 0);
3043 26496 : base2 = get_ref_base_and_extent (ref2, &offset2, &size2, &maxsize2,
3044 : &reverse);
3045 26496 : if (!known_size_p (maxsize2)
3046 26456 : || !known_eq (maxsize2, size2)
3047 52878 : || !operand_equal_p (base, base2, OEP_ADDRESS_OF))
3048 56541 : return (void *)-1;
3049 : }
3050 26543 : else if (TREE_CODE (ref2) == SSA_NAME)
3051 : {
3052 26543 : poly_int64 soff;
3053 26543 : if (TREE_CODE (base) != MEM_REF
3054 45314 : || !(mem_ref_offset (base)
3055 37542 : << LOG2_BITS_PER_UNIT).to_shwi (&soff))
3056 22598 : return (void *)-1;
3057 18771 : offset += soff;
3058 18771 : offset2 = 0;
3059 18771 : if (TREE_OPERAND (base, 0) != ref2)
3060 : {
3061 15428 : gimple *def = SSA_NAME_DEF_STMT (ref2);
3062 15428 : if (is_gimple_assign (def)
3063 14122 : && gimple_assign_rhs_code (def) == POINTER_PLUS_EXPR
3064 12148 : && gimple_assign_rhs1 (def) == TREE_OPERAND (base, 0)
3065 16060 : && poly_int_tree_p (gimple_assign_rhs2 (def)))
3066 : {
3067 602 : tree rhs2 = gimple_assign_rhs2 (def);
3068 602 : if (!(poly_offset_int::from (wi::to_poly_wide (rhs2),
3069 : SIGNED)
3070 602 : << LOG2_BITS_PER_UNIT).to_shwi (&offset2))
3071 : return (void *)-1;
3072 602 : ref2 = gimple_assign_rhs1 (def);
3073 602 : if (TREE_CODE (ref2) == SSA_NAME)
3074 602 : ref2 = SSA_VAL (ref2);
3075 : }
3076 : else
3077 : return (void *)-1;
3078 : }
3079 : }
3080 : else
3081 : return (void *)-1;
3082 26549 : tree len = gimple_call_arg (def_stmt, 2);
3083 26549 : HOST_WIDE_INT leni, offset2i;
3084 26549 : if (TREE_CODE (len) == SSA_NAME)
3085 255 : len = SSA_VAL (len);
3086 : /* Sometimes the above trickery is smarter than alias analysis. Take
3087 : advantage of that. */
3088 26549 : if (!ranges_maybe_overlap_p (offset, maxsize, offset2,
3089 53098 : (wi::to_poly_offset (len)
3090 26549 : << LOG2_BITS_PER_UNIT)))
3091 : return NULL;
3092 53041 : if (data->partial_defs.is_empty ()
3093 26492 : && known_subrange_p (offset, maxsize, offset2,
3094 26492 : wi::to_poly_offset (len) << LOG2_BITS_PER_UNIT))
3095 : {
3096 25997 : tree val;
3097 25997 : if (integer_zerop (gimple_call_arg (def_stmt, 1)))
3098 21164 : val = build_zero_cst (vr->type);
3099 4833 : else if (INTEGRAL_TYPE_P (vr->type)
3100 3693 : && known_eq (ref->size, 8)
3101 7771 : && offseti % BITS_PER_UNIT == 0)
3102 : {
3103 2938 : gimple_match_op res_op (gimple_match_cond::UNCOND, NOP_EXPR,
3104 2938 : vr->type, gimple_call_arg (def_stmt, 1));
3105 2938 : val = vn_nary_build_or_lookup (&res_op);
3106 2938 : if (!val
3107 2938 : || (TREE_CODE (val) == SSA_NAME
3108 626 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (val)))
3109 0 : return (void *)-1;
3110 : }
3111 : else
3112 : {
3113 1895 : unsigned buflen
3114 1895 : = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (vr->type)) + 1;
3115 1895 : if (INTEGRAL_TYPE_P (vr->type)
3116 1895 : && TYPE_MODE (vr->type) != BLKmode)
3117 1508 : buflen = GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (vr->type)) + 1;
3118 1895 : unsigned char *buf = XALLOCAVEC (unsigned char, buflen);
3119 1895 : memset (buf, TREE_INT_CST_LOW (gimple_call_arg (def_stmt, 1)),
3120 : buflen);
3121 1895 : if (BYTES_BIG_ENDIAN)
3122 : {
3123 : unsigned int amnt
3124 : = (((unsigned HOST_WIDE_INT) offseti + sizei)
3125 : % BITS_PER_UNIT);
3126 : if (amnt)
3127 : {
3128 : shift_bytes_in_array_right (buf, buflen,
3129 : BITS_PER_UNIT - amnt);
3130 : buf++;
3131 : buflen--;
3132 : }
3133 : }
3134 1895 : else if (offseti % BITS_PER_UNIT != 0)
3135 : {
3136 7 : unsigned int amnt
3137 : = BITS_PER_UNIT - ((unsigned HOST_WIDE_INT) offseti
3138 7 : % BITS_PER_UNIT);
3139 7 : shift_bytes_in_array_left (buf, buflen, amnt);
3140 7 : buf++;
3141 7 : buflen--;
3142 : }
3143 1895 : val = native_interpret_expr (vr->type, buf, buflen);
3144 1895 : if (!val)
3145 : return (void *)-1;
3146 : }
3147 25997 : return data->finish (0, 0, val);
3148 : }
3149 : /* For now handle clearing memory with partial defs. */
3150 552 : else if (known_eq (ref->size, maxsize)
3151 481 : && integer_zerop (gimple_call_arg (def_stmt, 1))
3152 166 : && tree_fits_poly_int64_p (len)
3153 162 : && tree_to_poly_int64 (len).is_constant (&leni)
3154 162 : && leni <= INTTYPE_MAXIMUM (HOST_WIDE_INT) / BITS_PER_UNIT
3155 162 : && offset.is_constant (&offseti)
3156 162 : && offset2.is_constant (&offset2i)
3157 162 : && maxsize.is_constant (&maxsizei)
3158 552 : && ranges_known_overlap_p (offseti, maxsizei, offset2i,
3159 552 : leni << LOG2_BITS_PER_UNIT))
3160 : {
3161 162 : pd_data pd;
3162 162 : pd.rhs = build_constructor (NULL_TREE, NULL);
3163 162 : pd.rhs_off = 0;
3164 162 : pd.offset = offset2i;
3165 162 : pd.size = leni << LOG2_BITS_PER_UNIT;
3166 162 : return data->push_partial_def (pd, 0, 0, offseti, maxsizei);
3167 : }
3168 : }
3169 :
3170 : /* 2) Assignment from an empty CONSTRUCTOR. */
3171 23210654 : else if (is_gimple_reg_type (vr->type)
3172 23202725 : && gimple_assign_single_p (def_stmt)
3173 7692812 : && gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR
3174 1938920 : && CONSTRUCTOR_NELTS (gimple_assign_rhs1 (def_stmt)) == 0
3175 25149574 : && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt)))
3176 : {
3177 1938888 : tree base2;
3178 1938888 : poly_int64 offset2, size2, maxsize2;
3179 1938888 : HOST_WIDE_INT offset2i, size2i;
3180 1938888 : gcc_assert (lhs_ref_ok);
3181 1938888 : base2 = ao_ref_base (&lhs_ref);
3182 1938888 : offset2 = lhs_ref.offset;
3183 1938888 : size2 = lhs_ref.size;
3184 1938888 : maxsize2 = lhs_ref.max_size;
3185 1938888 : if (known_size_p (maxsize2)
3186 1938850 : && known_eq (maxsize2, size2)
3187 3877692 : && adjust_offsets_for_equal_base_address (base, &offset,
3188 : base2, &offset2))
3189 : {
3190 1911780 : if (data->partial_defs.is_empty ()
3191 1908252 : && known_subrange_p (offset, maxsize, offset2, size2))
3192 : {
3193 : /* While technically undefined behavior do not optimize
3194 : a full read from a clobber. */
3195 1907381 : if (gimple_clobber_p (def_stmt))
3196 1911726 : return (void *)-1;
3197 972604 : tree val = build_zero_cst (vr->type);
3198 972604 : return data->finish (ao_ref_alias_set (&lhs_ref),
3199 972604 : ao_ref_base_alias_set (&lhs_ref), val);
3200 : }
3201 4399 : else if (known_eq (ref->size, maxsize)
3202 4345 : && maxsize.is_constant (&maxsizei)
3203 4345 : && offset.is_constant (&offseti)
3204 4345 : && offset2.is_constant (&offset2i)
3205 4345 : && size2.is_constant (&size2i)
3206 4399 : && ranges_known_overlap_p (offseti, maxsizei,
3207 : offset2i, size2i))
3208 : {
3209 : /* Let clobbers be consumed by the partial-def tracker
3210 : which can choose to ignore them if they are shadowed
3211 : by a later def. */
3212 4345 : pd_data pd;
3213 4345 : pd.rhs = gimple_assign_rhs1 (def_stmt);
3214 4345 : pd.rhs_off = 0;
3215 4345 : pd.offset = offset2i;
3216 4345 : pd.size = size2i;
3217 4345 : return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
3218 : ao_ref_base_alias_set (&lhs_ref),
3219 : offseti, maxsizei);
3220 : }
3221 : }
3222 : }
3223 :
3224 : /* 3) Assignment from a constant. We can use folds native encode/interpret
3225 : routines to extract the assigned bits. */
3226 21271766 : else if (known_eq (ref->size, maxsize)
3227 20753989 : && is_gimple_reg_type (vr->type)
3228 20746060 : && !reverse_storage_order_for_component_p (vr->operands)
3229 20743304 : && !contains_storage_order_barrier_p (vr->operands)
3230 20743304 : && gimple_assign_single_p (def_stmt)
3231 5432997 : && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt))
3232 : && CHAR_BIT == 8
3233 : && BITS_PER_UNIT == 8
3234 : && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
3235 : /* native_encode and native_decode operate on arrays of bytes
3236 : and so fundamentally need a compile-time size and offset. */
3237 5430036 : && maxsize.is_constant (&maxsizei)
3238 5430036 : && offset.is_constant (&offseti)
3239 26701802 : && (is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt))
3240 4599512 : || (TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME
3241 1858963 : && is_gimple_min_invariant (SSA_VAL (gimple_assign_rhs1 (def_stmt))))))
3242 : {
3243 846879 : tree lhs = gimple_assign_lhs (def_stmt);
3244 846879 : tree base2;
3245 846879 : poly_int64 offset2, size2, maxsize2;
3246 846879 : HOST_WIDE_INT offset2i, size2i;
3247 846879 : bool reverse;
3248 846879 : gcc_assert (lhs_ref_ok);
3249 846879 : base2 = ao_ref_base (&lhs_ref);
3250 846879 : offset2 = lhs_ref.offset;
3251 846879 : size2 = lhs_ref.size;
3252 846879 : maxsize2 = lhs_ref.max_size;
3253 846879 : reverse = reverse_storage_order_for_component_p (lhs);
3254 846879 : if (base2
3255 846879 : && !reverse
3256 846051 : && !storage_order_barrier_p (lhs)
3257 846051 : && known_eq (maxsize2, size2)
3258 814453 : && adjust_offsets_for_equal_base_address (base, &offset,
3259 : base2, &offset2)
3260 82760 : && offset.is_constant (&offseti)
3261 82760 : && offset2.is_constant (&offset2i)
3262 846879 : && size2.is_constant (&size2i))
3263 : {
3264 82760 : if (data->partial_defs.is_empty ()
3265 65925 : && known_subrange_p (offseti, maxsizei, offset2, size2))
3266 : {
3267 : /* We support up to 512-bit values (for V8DFmode). */
3268 42797 : unsigned char buffer[65];
3269 42797 : int len;
3270 :
3271 42797 : tree rhs = gimple_assign_rhs1 (def_stmt);
3272 42797 : if (TREE_CODE (rhs) == SSA_NAME)
3273 1657 : rhs = SSA_VAL (rhs);
3274 85594 : len = native_encode_expr (rhs,
3275 : buffer, sizeof (buffer) - 1,
3276 42797 : (offseti - offset2i) / BITS_PER_UNIT);
3277 42797 : if (len > 0 && len * BITS_PER_UNIT >= maxsizei)
3278 : {
3279 39775 : tree type = vr->type;
3280 39775 : unsigned char *buf = buffer;
3281 39775 : unsigned int amnt = 0;
3282 : /* Make sure to interpret in a type that has a range
3283 : covering the whole access size. */
3284 39775 : if (INTEGRAL_TYPE_P (vr->type)
3285 39775 : && maxsizei != TYPE_PRECISION (vr->type))
3286 : {
3287 884 : bool uns = TYPE_UNSIGNED (type);
3288 883 : if (BITINT_TYPE_P (vr->type)
3289 885 : && maxsizei > MAX_FIXED_MODE_SIZE)
3290 1 : type = build_bitint_type (maxsizei, uns);
3291 : else
3292 883 : type = build_nonstandard_integer_type (maxsizei, uns);
3293 : }
3294 39775 : if (BYTES_BIG_ENDIAN)
3295 : {
3296 : /* For big-endian native_encode_expr stored the rhs
3297 : such that the LSB of it is the LSB of buffer[len - 1].
3298 : That bit is stored into memory at position
3299 : offset2 + size2 - 1, i.e. in byte
3300 : base + (offset2 + size2 - 1) / BITS_PER_UNIT.
3301 : E.g. for offset2 1 and size2 14, rhs -1 and memory
3302 : previously cleared that is:
3303 : 0 1
3304 : 01111111|11111110
3305 : Now, if we want to extract offset 2 and size 12 from
3306 : it using native_interpret_expr (which actually works
3307 : for integral bitfield types in terms of byte size of
3308 : the mode), the native_encode_expr stored the value
3309 : into buffer as
3310 : XX111111|11111111
3311 : and returned len 2 (the X bits are outside of
3312 : precision).
3313 : Let sz be maxsize / BITS_PER_UNIT if not extracting
3314 : a bitfield, and GET_MODE_SIZE otherwise.
3315 : We need to align the LSB of the value we want to
3316 : extract as the LSB of buf[sz - 1].
3317 : The LSB from memory we need to read is at position
3318 : offset + maxsize - 1. */
3319 : HOST_WIDE_INT sz = maxsizei / BITS_PER_UNIT;
3320 : if (INTEGRAL_TYPE_P (type))
3321 : {
3322 : if (TYPE_MODE (type) != BLKmode)
3323 : sz = GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (type));
3324 : else
3325 : sz = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (type));
3326 : }
3327 : amnt = ((unsigned HOST_WIDE_INT) offset2i + size2i
3328 : - offseti - maxsizei) % BITS_PER_UNIT;
3329 : if (amnt)
3330 : shift_bytes_in_array_right (buffer, len, amnt);
3331 : amnt = ((unsigned HOST_WIDE_INT) offset2i + size2i
3332 : - offseti - maxsizei - amnt) / BITS_PER_UNIT;
3333 : if ((unsigned HOST_WIDE_INT) sz + amnt > (unsigned) len)
3334 : len = 0;
3335 : else
3336 : {
3337 : buf = buffer + len - sz - amnt;
3338 : len -= (buf - buffer);
3339 : }
3340 : }
3341 : else
3342 : {
3343 39775 : amnt = ((unsigned HOST_WIDE_INT) offset2i
3344 39775 : - offseti) % BITS_PER_UNIT;
3345 39775 : if (amnt)
3346 : {
3347 315 : buffer[len] = 0;
3348 315 : shift_bytes_in_array_left (buffer, len + 1, amnt);
3349 315 : buf = buffer + 1;
3350 : }
3351 : }
3352 39775 : tree val = native_interpret_expr (type, buf, len);
3353 : /* If we chop off bits because the types precision doesn't
3354 : match the memory access size this is ok when optimizing
3355 : reads but not when called from the DSE code during
3356 : elimination. */
3357 39775 : if (val
3358 39773 : && type != vr->type)
3359 : {
3360 884 : if (! int_fits_type_p (val, vr->type))
3361 : val = NULL_TREE;
3362 : else
3363 884 : val = fold_convert (vr->type, val);
3364 : }
3365 :
3366 39773 : if (val)
3367 39773 : return data->finish (ao_ref_alias_set (&lhs_ref),
3368 39773 : ao_ref_base_alias_set (&lhs_ref), val);
3369 : }
3370 : }
3371 39963 : else if (ranges_known_overlap_p (offseti, maxsizei, offset2i,
3372 : size2i))
3373 : {
3374 39963 : pd_data pd;
3375 39963 : tree rhs = gimple_assign_rhs1 (def_stmt);
3376 39963 : if (TREE_CODE (rhs) == SSA_NAME)
3377 2186 : rhs = SSA_VAL (rhs);
3378 39963 : pd.rhs = rhs;
3379 39963 : pd.rhs_off = 0;
3380 39963 : pd.offset = offset2i;
3381 39963 : pd.size = size2i;
3382 39963 : return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
3383 : ao_ref_base_alias_set (&lhs_ref),
3384 : offseti, maxsizei);
3385 : }
3386 : }
3387 : }
3388 :
3389 : /* 4) Assignment from an SSA name which definition we may be able
3390 : to access pieces from or we can combine to a larger entity. */
3391 20424887 : else if (known_eq (ref->size, maxsize)
3392 19907110 : && is_gimple_reg_type (vr->type)
3393 19899181 : && !reverse_storage_order_for_component_p (vr->operands)
3394 19896425 : && !contains_storage_order_barrier_p (vr->operands)
3395 19896425 : && gimple_assign_single_p (def_stmt)
3396 4586118 : && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt))
3397 25008044 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
3398 : {
3399 1842608 : tree lhs = gimple_assign_lhs (def_stmt);
3400 1842608 : tree base2;
3401 1842608 : poly_int64 offset2, size2, maxsize2;
3402 1842608 : HOST_WIDE_INT offset2i, size2i, offseti;
3403 1842608 : bool reverse;
3404 1842608 : gcc_assert (lhs_ref_ok);
3405 1842608 : base2 = ao_ref_base (&lhs_ref);
3406 1842608 : offset2 = lhs_ref.offset;
3407 1842608 : size2 = lhs_ref.size;
3408 1842608 : maxsize2 = lhs_ref.max_size;
3409 1842608 : reverse = reverse_storage_order_for_component_p (lhs);
3410 1842608 : tree def_rhs = gimple_assign_rhs1 (def_stmt);
3411 1842608 : if (!reverse
3412 1842396 : && !storage_order_barrier_p (lhs)
3413 1842396 : && known_size_p (maxsize2)
3414 1817588 : && known_eq (maxsize2, size2)
3415 3541303 : && adjust_offsets_for_equal_base_address (base, &offset,
3416 : base2, &offset2))
3417 : {
3418 80393 : if (data->partial_defs.is_empty ()
3419 74287 : && known_subrange_p (offset, maxsize, offset2, size2)
3420 : /* ??? We can't handle bitfield precision extracts without
3421 : either using an alternate type for the BIT_FIELD_REF and
3422 : then doing a conversion or possibly adjusting the offset
3423 : according to endianness. */
3424 50885 : && (! INTEGRAL_TYPE_P (vr->type)
3425 37221 : || known_eq (ref->size, TYPE_PRECISION (vr->type)))
3426 93031 : && multiple_p (ref->size, BITS_PER_UNIT))
3427 : {
3428 44795 : tree val = NULL_TREE;
3429 89584 : if (! INTEGRAL_TYPE_P (TREE_TYPE (def_rhs))
3430 49327 : || type_has_mode_precision_p (TREE_TYPE (def_rhs)))
3431 : {
3432 43741 : gimple_match_op op (gimple_match_cond::UNCOND,
3433 43741 : BIT_FIELD_REF, vr->type,
3434 : SSA_VAL (def_rhs),
3435 : bitsize_int (ref->size),
3436 43741 : bitsize_int (offset - offset2));
3437 43741 : val = vn_nary_build_or_lookup (&op);
3438 : }
3439 1054 : else if (known_eq (ref->size, size2))
3440 : {
3441 980 : gimple_match_op op (gimple_match_cond::UNCOND,
3442 980 : VIEW_CONVERT_EXPR, vr->type,
3443 980 : SSA_VAL (def_rhs));
3444 980 : val = vn_nary_build_or_lookup (&op);
3445 : }
3446 44721 : if (val
3447 44721 : && (TREE_CODE (val) != SSA_NAME
3448 43968 : || ! SSA_NAME_OCCURS_IN_ABNORMAL_PHI (val)))
3449 44702 : return data->finish (ao_ref_alias_set (&lhs_ref),
3450 80300 : ao_ref_base_alias_set (&lhs_ref), val);
3451 : }
3452 35598 : else if (maxsize.is_constant (&maxsizei)
3453 35598 : && offset.is_constant (&offseti)
3454 35598 : && offset2.is_constant (&offset2i)
3455 35598 : && size2.is_constant (&size2i)
3456 35598 : && ranges_known_overlap_p (offset, maxsize, offset2, size2))
3457 : {
3458 35598 : pd_data pd;
3459 35598 : pd.rhs = SSA_VAL (def_rhs);
3460 35598 : pd.rhs_off = 0;
3461 35598 : pd.offset = offset2i;
3462 35598 : pd.size = size2i;
3463 35598 : return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
3464 : ao_ref_base_alias_set (&lhs_ref),
3465 : offseti, maxsizei);
3466 : }
3467 : }
3468 : }
3469 :
3470 : /* 4b) Assignment done via one of the vectorizer internal store
3471 : functions where we may be able to access pieces from or we can
3472 : combine to a larger entity. */
3473 18582279 : else if (known_eq (ref->size, maxsize)
3474 18064502 : && is_gimple_reg_type (vr->type)
3475 18056573 : && !reverse_storage_order_for_component_p (vr->operands)
3476 18053817 : && !contains_storage_order_barrier_p (vr->operands)
3477 18053817 : && is_gimple_call (def_stmt)
3478 14519453 : && gimple_call_internal_p (def_stmt)
3479 18840301 : && internal_store_fn_p (gimple_call_internal_fn (def_stmt)))
3480 : {
3481 36 : gcall *call = as_a <gcall *> (def_stmt);
3482 36 : internal_fn fn = gimple_call_internal_fn (call);
3483 :
3484 36 : tree mask = NULL_TREE, len = NULL_TREE, bias = NULL_TREE;
3485 36 : switch (fn)
3486 : {
3487 36 : case IFN_MASK_STORE:
3488 36 : mask = gimple_call_arg (call, internal_fn_mask_index (fn));
3489 36 : mask = vn_valueize (mask);
3490 36 : if (TREE_CODE (mask) != VECTOR_CST)
3491 28 : return (void *)-1;
3492 : break;
3493 0 : case IFN_LEN_STORE:
3494 0 : {
3495 0 : int len_index = internal_fn_len_index (fn);
3496 0 : len = gimple_call_arg (call, len_index);
3497 0 : bias = gimple_call_arg (call, len_index + 1);
3498 0 : if (!tree_fits_uhwi_p (len) || !tree_fits_shwi_p (bias))
3499 : return (void *) -1;
3500 : break;
3501 : }
3502 : default:
3503 : return (void *)-1;
3504 : }
3505 14 : tree def_rhs = gimple_call_arg (call,
3506 14 : internal_fn_stored_value_index (fn));
3507 14 : def_rhs = vn_valueize (def_rhs);
3508 14 : if (TREE_CODE (def_rhs) != VECTOR_CST)
3509 : return (void *)-1;
3510 :
3511 14 : ao_ref_init_from_ptr_and_size (&lhs_ref,
3512 : vn_valueize (gimple_call_arg (call, 0)),
3513 14 : TYPE_SIZE_UNIT (TREE_TYPE (def_rhs)));
3514 14 : tree base2;
3515 14 : poly_int64 offset2, size2, maxsize2;
3516 14 : HOST_WIDE_INT offset2i, size2i, offseti;
3517 14 : base2 = ao_ref_base (&lhs_ref);
3518 14 : offset2 = lhs_ref.offset;
3519 14 : size2 = lhs_ref.size;
3520 14 : maxsize2 = lhs_ref.max_size;
3521 14 : if (known_size_p (maxsize2)
3522 14 : && known_eq (maxsize2, size2)
3523 14 : && adjust_offsets_for_equal_base_address (base, &offset,
3524 : base2, &offset2)
3525 6 : && maxsize.is_constant (&maxsizei)
3526 6 : && offset.is_constant (&offseti)
3527 6 : && offset2.is_constant (&offset2i)
3528 14 : && size2.is_constant (&size2i))
3529 : {
3530 6 : if (!ranges_maybe_overlap_p (offset, maxsize, offset2, size2))
3531 : /* Poor-mans disambiguation. */
3532 : return NULL;
3533 6 : else if (ranges_known_overlap_p (offset, maxsize, offset2, size2))
3534 : {
3535 6 : pd_data pd;
3536 6 : pd.rhs = def_rhs;
3537 6 : tree aa = gimple_call_arg (call, 1);
3538 6 : alias_set_type set = get_deref_alias_set (TREE_TYPE (aa));
3539 6 : tree vectype = TREE_TYPE (def_rhs);
3540 6 : unsigned HOST_WIDE_INT elsz
3541 6 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (vectype)));
3542 6 : if (mask)
3543 : {
3544 : HOST_WIDE_INT start = 0, length = 0;
3545 : unsigned mask_idx = 0;
3546 48 : do
3547 : {
3548 48 : if (integer_zerop (VECTOR_CST_ELT (mask, mask_idx)))
3549 : {
3550 24 : if (length != 0)
3551 : {
3552 18 : pd.rhs_off = start;
3553 18 : pd.offset = offset2i + start;
3554 18 : pd.size = length;
3555 18 : if (ranges_known_overlap_p
3556 18 : (offset, maxsize, pd.offset, pd.size))
3557 : {
3558 0 : void *res = data->push_partial_def
3559 0 : (pd, set, set, offseti, maxsizei);
3560 0 : if (res != NULL)
3561 6 : return res;
3562 : }
3563 : }
3564 24 : start = (mask_idx + 1) * elsz;
3565 24 : length = 0;
3566 : }
3567 : else
3568 24 : length += elsz;
3569 48 : mask_idx++;
3570 : }
3571 48 : while (known_lt (mask_idx, TYPE_VECTOR_SUBPARTS (vectype)));
3572 6 : if (length != 0)
3573 : {
3574 6 : pd.rhs_off = start;
3575 6 : pd.offset = offset2i + start;
3576 6 : pd.size = length;
3577 6 : if (ranges_known_overlap_p (offset, maxsize,
3578 : pd.offset, pd.size))
3579 2 : return data->push_partial_def (pd, set, set,
3580 2 : offseti, maxsizei);
3581 : }
3582 : }
3583 0 : else if (fn == IFN_LEN_STORE)
3584 : {
3585 0 : pd.offset = offset2i;
3586 0 : pd.size = (tree_to_uhwi (len)
3587 0 : + -tree_to_shwi (bias)) * BITS_PER_UNIT;
3588 0 : if (BYTES_BIG_ENDIAN)
3589 : pd.rhs_off = pd.size - tree_to_uhwi (TYPE_SIZE (vectype));
3590 : else
3591 0 : pd.rhs_off = 0;
3592 0 : if (ranges_known_overlap_p (offset, maxsize,
3593 : pd.offset, pd.size))
3594 0 : return data->push_partial_def (pd, set, set,
3595 0 : offseti, maxsizei);
3596 : }
3597 : else
3598 0 : gcc_unreachable ();
3599 4 : return NULL;
3600 : }
3601 : }
3602 : }
3603 :
3604 : /* 5) For aggregate copies translate the reference through them if
3605 : the copy kills ref. */
3606 18582243 : else if (data->vn_walk_kind == VN_WALKREWRITE
3607 15007602 : && gimple_assign_single_p (def_stmt)
3608 2532322 : && !gimple_has_volatile_ops (def_stmt)
3609 21112267 : && (DECL_P (gimple_assign_rhs1 (def_stmt))
3610 1979860 : || TREE_CODE (gimple_assign_rhs1 (def_stmt)) == MEM_REF
3611 1572309 : || handled_component_p (gimple_assign_rhs1 (def_stmt))))
3612 : {
3613 2321435 : tree base2;
3614 2321435 : int i, j, k;
3615 2321435 : auto_vec<vn_reference_op_s> rhs;
3616 2321435 : vn_reference_op_t vro;
3617 2321435 : ao_ref r;
3618 :
3619 2321435 : gcc_assert (lhs_ref_ok);
3620 :
3621 : /* See if the assignment kills REF. */
3622 2321435 : base2 = ao_ref_base (&lhs_ref);
3623 2321435 : if (!lhs_ref.max_size_known_p ()
3624 2320986 : || (base != base2
3625 86480 : && (TREE_CODE (base) != MEM_REF
3626 70939 : || TREE_CODE (base2) != MEM_REF
3627 55504 : || TREE_OPERAND (base, 0) != TREE_OPERAND (base2, 0)
3628 19666 : || !tree_int_cst_equal (TREE_OPERAND (base, 1),
3629 19666 : TREE_OPERAND (base2, 1))))
3630 4574088 : || !stmt_kills_ref_p (def_stmt, ref))
3631 393833 : return (void *)-1;
3632 :
3633 : /* Find the common base of ref and the lhs. lhs_ops already
3634 : contains valueized operands for the lhs. */
3635 1927602 : poly_int64 extra_off = 0;
3636 1927602 : i = vr->operands.length () - 1;
3637 1927602 : j = lhs_ops.length () - 1;
3638 :
3639 : /* The base should be always equal due to the above check. */
3640 1927602 : if (! vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
3641 : return (void *)-1;
3642 1927342 : i--, j--;
3643 :
3644 : /* The 2nd component should always exist and be a MEM_REF. */
3645 1927342 : if (!(i >= 0 && j >= 0))
3646 : ;
3647 1927342 : else if (vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
3648 915916 : i--, j--;
3649 1011426 : else if (vr->operands[i].opcode == MEM_REF
3650 1009908 : && lhs_ops[j].opcode == MEM_REF
3651 1009908 : && known_ne (lhs_ops[j].off, -1)
3652 2021334 : && known_ne (vr->operands[i].off, -1))
3653 : {
3654 1009908 : bool found = false;
3655 : /* When we ge a mismatch at a MEM_REF that is not the sole component
3656 : try finding a match in one of the outer components and continue
3657 : stripping there. This happens when addresses of components get
3658 : forwarded into dereferences. */
3659 1009908 : if (i > 0)
3660 : {
3661 107981 : int temi = i - 1;
3662 107981 : poly_int64 tem_extra_off = extra_off + vr->operands[i].off;
3663 107981 : while (temi >= 0
3664 234221 : && known_ne (vr->operands[temi].off, -1))
3665 : {
3666 127775 : if (vr->operands[temi].type
3667 127775 : && lhs_ops[j].type
3668 255550 : && (TYPE_MAIN_VARIANT (vr->operands[temi].type)
3669 127775 : == TYPE_MAIN_VARIANT (lhs_ops[j].type)))
3670 : {
3671 1535 : i = temi;
3672 : /* Strip the component that was type matched to
3673 : the MEM_REF. */
3674 1535 : extra_off = (tem_extra_off
3675 1535 : + vr->operands[i].off - lhs_ops[j].off);
3676 1535 : i--, j--;
3677 : /* Strip further equal components. */
3678 1535 : found = true;
3679 1535 : break;
3680 : }
3681 126240 : tem_extra_off += vr->operands[temi].off;
3682 126240 : temi--;
3683 : }
3684 : }
3685 1009908 : if (!found && j > 0)
3686 : {
3687 28998 : int temj = j - 1;
3688 28998 : poly_int64 tem_extra_off = extra_off - lhs_ops[j].off;
3689 28998 : while (temj >= 0
3690 56285 : && known_ne (lhs_ops[temj].off, -1))
3691 : {
3692 31157 : if (vr->operands[i].type
3693 31157 : && lhs_ops[temj].type
3694 62314 : && (TYPE_MAIN_VARIANT (vr->operands[i].type)
3695 31157 : == TYPE_MAIN_VARIANT (lhs_ops[temj].type)))
3696 : {
3697 3870 : j = temj;
3698 : /* Strip the component that was type matched to
3699 : the MEM_REF. */
3700 3870 : extra_off = (tem_extra_off
3701 3870 : + vr->operands[i].off - lhs_ops[j].off);
3702 3870 : i--, j--;
3703 : /* Strip further equal components. */
3704 3870 : found = true;
3705 3870 : break;
3706 : }
3707 27287 : tem_extra_off += -lhs_ops[temj].off;
3708 27287 : temj--;
3709 : }
3710 : }
3711 : /* When we cannot find a common base to reconstruct the full
3712 : reference instead try to reduce the lookup to the new
3713 : base plus a constant offset. */
3714 1009908 : if (!found)
3715 : {
3716 : while (j >= 0
3717 2035919 : && known_ne (lhs_ops[j].off, -1))
3718 : {
3719 1031416 : extra_off += -lhs_ops[j].off;
3720 1031416 : j--;
3721 : }
3722 1004503 : if (j != -1)
3723 : return (void *)-1;
3724 : while (i >= 0
3725 2129963 : && known_ne (vr->operands[i].off, -1))
3726 : {
3727 : /* Punt if the additional ops contain a storage order
3728 : barrier. */
3729 1125460 : if (vr->operands[i].opcode == VIEW_CONVERT_EXPR
3730 1125460 : && vr->operands[i].reverse)
3731 : break;
3732 1125460 : extra_off += vr->operands[i].off;
3733 1125460 : i--;
3734 : }
3735 1004503 : if (i != -1)
3736 : return (void *)-1;
3737 : found = true;
3738 : }
3739 : /* If we did find a match we'd eventually append a MEM_REF
3740 : as component. Don't. */
3741 : if (!found)
3742 : return (void *)-1;
3743 : }
3744 : else
3745 : return (void *)-1;
3746 :
3747 : /* Strip further common components, attempting to consume lhs_ops
3748 : in full. */
3749 1924863 : while (j >= 0 && i >= 0
3750 1924863 : && vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
3751 : {
3752 25514 : i--;
3753 25514 : j--;
3754 : }
3755 :
3756 : /* i now points to the first additional op.
3757 : ??? LHS may not be completely contained in VR, one or more
3758 : VIEW_CONVERT_EXPRs could be in its way. We could at least
3759 : try handling outermost VIEW_CONVERT_EXPRs. */
3760 1899349 : if (j != -1)
3761 : return (void *)-1;
3762 :
3763 : /* Punt if the additional ops contain a storage order barrier. */
3764 2968130 : for (k = i; k >= 0; k--)
3765 : {
3766 1071514 : vro = &vr->operands[k];
3767 1071514 : if (vro->opcode == VIEW_CONVERT_EXPR && vro->reverse)
3768 : return (void *)-1;
3769 : }
3770 :
3771 : /* Now re-write REF to be based on the rhs of the assignment. */
3772 1896616 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
3773 1896616 : copy_reference_ops_from_ref (rhs1, &rhs);
3774 :
3775 : /* Apply an extra offset to the inner MEM_REF of the RHS. */
3776 1896616 : bool force_no_tbaa = false;
3777 1896616 : if (maybe_ne (extra_off, 0))
3778 : {
3779 725455 : if (rhs.length () < 2)
3780 : return (void *)-1;
3781 725455 : int ix = rhs.length () - 2;
3782 725455 : if (rhs[ix].opcode != MEM_REF
3783 725455 : || known_eq (rhs[ix].off, -1))
3784 : return (void *)-1;
3785 725437 : rhs[ix].off += extra_off;
3786 725437 : rhs[ix].op0 = int_const_binop (PLUS_EXPR, rhs[ix].op0,
3787 725437 : build_int_cst (TREE_TYPE (rhs[ix].op0),
3788 : extra_off));
3789 : /* When we have offsetted the RHS, reading only parts of it,
3790 : we can no longer use the original TBAA type, force alias-set
3791 : zero. */
3792 725437 : force_no_tbaa = true;
3793 : }
3794 :
3795 : /* Save the operands since we need to use the original ones for
3796 : the hash entry we use. */
3797 1896598 : if (!data->saved_operands.exists ())
3798 1794822 : data->saved_operands = vr->operands.copy ();
3799 :
3800 : /* We need to pre-pend vr->operands[0..i] to rhs. */
3801 1896598 : vec<vn_reference_op_s> old = vr->operands;
3802 5689794 : if (i + 1 + rhs.length () > vr->operands.length ())
3803 1150703 : vr->operands.safe_grow (i + 1 + rhs.length (), true);
3804 : else
3805 745895 : vr->operands.truncate (i + 1 + rhs.length ());
3806 6887767 : FOR_EACH_VEC_ELT (rhs, j, vro)
3807 4991169 : vr->operands[i + 1 + j] = *vro;
3808 1896598 : valueize_refs (&vr->operands);
3809 3793196 : if (old == shared_lookup_references)
3810 1896598 : shared_lookup_references = vr->operands;
3811 1896598 : vr->hashcode = vn_reference_compute_hash (vr);
3812 :
3813 : /* Try folding the new reference to a constant. */
3814 1896598 : tree val = fully_constant_vn_reference_p (vr);
3815 1896598 : if (val)
3816 : {
3817 21914 : if (data->partial_defs.is_empty ())
3818 21905 : return data->finish (ao_ref_alias_set (&lhs_ref),
3819 21905 : ao_ref_base_alias_set (&lhs_ref), val);
3820 : /* This is the only interesting case for partial-def handling
3821 : coming from targets that like to gimplify init-ctors as
3822 : aggregate copies from constant data like aarch64 for
3823 : PR83518. */
3824 9 : if (maxsize.is_constant (&maxsizei) && known_eq (ref->size, maxsize))
3825 : {
3826 9 : pd_data pd;
3827 9 : pd.rhs = val;
3828 9 : pd.rhs_off = 0;
3829 9 : pd.offset = 0;
3830 9 : pd.size = maxsizei;
3831 9 : return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
3832 : ao_ref_base_alias_set (&lhs_ref),
3833 : 0, maxsizei);
3834 : }
3835 : }
3836 :
3837 : /* Continuing with partial defs isn't easily possible here, we
3838 : have to find a full def from further lookups from here. Probably
3839 : not worth the special-casing everywhere. */
3840 2305687 : if (!data->partial_defs.is_empty ())
3841 : return (void *)-1;
3842 :
3843 : /* Adjust *ref from the new operands. */
3844 1868518 : ao_ref rhs1_ref;
3845 1868518 : ao_ref_init (&rhs1_ref, rhs1);
3846 3026070 : if (!ao_ref_init_from_vn_reference (&r,
3847 : force_no_tbaa ? 0
3848 1157552 : : ao_ref_alias_set (&rhs1_ref),
3849 : force_no_tbaa ? 0
3850 1157552 : : ao_ref_base_alias_set (&rhs1_ref),
3851 : vr->type, vr->operands))
3852 : return (void *)-1;
3853 : /* This can happen with bitfields. */
3854 1868518 : if (maybe_ne (ref->size, r.size))
3855 : {
3856 : /* If the access lacks some subsetting simply apply that by
3857 : shortening it. That in the end can only be successful
3858 : if we can pun the lookup result which in turn requires
3859 : exact offsets. */
3860 1553 : if (known_eq (r.size, r.max_size)
3861 1553 : && known_lt (ref->size, r.size))
3862 1553 : r.size = r.max_size = ref->size;
3863 : else
3864 : return (void *)-1;
3865 : }
3866 1868518 : *ref = r;
3867 1868518 : vr->offset = r.offset;
3868 1868518 : vr->max_size = r.max_size;
3869 :
3870 : /* Do not update last seen VUSE after translating. */
3871 1868518 : data->last_vuse_ptr = NULL;
3872 : /* Invalidate the original access path since it now contains
3873 : the wrong base. */
3874 1868518 : data->orig_ref.ref = NULL_TREE;
3875 : /* Use the alias-set of this LHS for recording an eventual result. */
3876 1868518 : if (data->first_set == -2)
3877 : {
3878 1768262 : data->first_set = ao_ref_alias_set (&lhs_ref);
3879 1768262 : data->first_base_set = ao_ref_base_alias_set (&lhs_ref);
3880 : }
3881 :
3882 : /* Keep looking for the adjusted *REF / VR pair. */
3883 1868518 : return NULL;
3884 2321435 : }
3885 :
3886 : /* 6) For memcpy copies translate the reference through them if the copy
3887 : kills ref. But we cannot (easily) do this translation if the memcpy is
3888 : a storage order barrier, i.e. is equivalent to a VIEW_CONVERT_EXPR that
3889 : can modify the storage order of objects (see storage_order_barrier_p). */
3890 16260808 : else if (data->vn_walk_kind == VN_WALKREWRITE
3891 12686167 : && is_gimple_reg_type (vr->type)
3892 : /* ??? Handle BCOPY as well. */
3893 12678270 : && (gimple_call_builtin_p (def_stmt, BUILT_IN_MEMCPY)
3894 12610055 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMCPY_CHK)
3895 12609632 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMPCPY)
3896 12608456 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMPCPY_CHK)
3897 12608214 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMMOVE)
3898 12583061 : || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMMOVE_CHK))
3899 95537 : && (TREE_CODE (gimple_call_arg (def_stmt, 0)) == ADDR_EXPR
3900 84359 : || TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME)
3901 95503 : && (TREE_CODE (gimple_call_arg (def_stmt, 1)) == ADDR_EXPR
3902 67905 : || TREE_CODE (gimple_call_arg (def_stmt, 1)) == SSA_NAME)
3903 95488 : && (poly_int_tree_p (gimple_call_arg (def_stmt, 2), ©_size)
3904 54805 : || (TREE_CODE (gimple_call_arg (def_stmt, 2)) == SSA_NAME
3905 54805 : && poly_int_tree_p (SSA_VAL (gimple_call_arg (def_stmt, 2)),
3906 : ©_size)))
3907 : /* Handling this is more complicated, give up for now. */
3908 16303978 : && data->partial_defs.is_empty ())
3909 : {
3910 42559 : tree lhs, rhs;
3911 42559 : ao_ref r;
3912 42559 : poly_int64 rhs_offset, lhs_offset;
3913 42559 : vn_reference_op_s op;
3914 42559 : poly_uint64 mem_offset;
3915 42559 : poly_int64 at, byte_maxsize;
3916 :
3917 : /* Only handle non-variable, addressable refs. */
3918 42559 : if (maybe_ne (ref->size, maxsize)
3919 42068 : || !multiple_p (offset, BITS_PER_UNIT, &at)
3920 42559 : || !multiple_p (maxsize, BITS_PER_UNIT, &byte_maxsize))
3921 491 : return (void *)-1;
3922 :
3923 : /* Extract a pointer base and an offset for the destination. */
3924 42068 : lhs = gimple_call_arg (def_stmt, 0);
3925 42068 : lhs_offset = 0;
3926 42068 : if (TREE_CODE (lhs) == SSA_NAME)
3927 : {
3928 32490 : lhs = vn_valueize (lhs);
3929 32490 : if (TREE_CODE (lhs) == SSA_NAME)
3930 : {
3931 32170 : gimple *def_stmt = SSA_NAME_DEF_STMT (lhs);
3932 32170 : if (gimple_assign_single_p (def_stmt)
3933 32170 : && gimple_assign_rhs_code (def_stmt) == ADDR_EXPR)
3934 2373 : lhs = gimple_assign_rhs1 (def_stmt);
3935 : }
3936 : }
3937 42068 : if (TREE_CODE (lhs) == ADDR_EXPR)
3938 : {
3939 17244 : if (AGGREGATE_TYPE_P (TREE_TYPE (TREE_TYPE (lhs)))
3940 16947 : && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (TREE_TYPE (lhs))))
3941 : return (void *)-1;
3942 12131 : tree tem = get_addr_base_and_unit_offset (TREE_OPERAND (lhs, 0),
3943 : &lhs_offset);
3944 12131 : if (!tem)
3945 : return (void *)-1;
3946 11439 : if (TREE_CODE (tem) == MEM_REF
3947 11439 : && poly_int_tree_p (TREE_OPERAND (tem, 1), &mem_offset))
3948 : {
3949 1681 : lhs = TREE_OPERAND (tem, 0);
3950 1681 : if (TREE_CODE (lhs) == SSA_NAME)
3951 1681 : lhs = vn_valueize (lhs);
3952 1681 : lhs_offset += mem_offset;
3953 : }
3954 9758 : else if (DECL_P (tem))
3955 9758 : lhs = build_fold_addr_expr (tem);
3956 : else
3957 : return (void *)-1;
3958 : }
3959 41236 : if (TREE_CODE (lhs) != SSA_NAME
3960 9759 : && TREE_CODE (lhs) != ADDR_EXPR)
3961 : return (void *)-1;
3962 :
3963 : /* Extract a pointer base and an offset for the source. */
3964 41236 : rhs = gimple_call_arg (def_stmt, 1);
3965 41236 : rhs_offset = 0;
3966 41236 : if (TREE_CODE (rhs) == SSA_NAME)
3967 19284 : rhs = vn_valueize (rhs);
3968 41236 : if (TREE_CODE (rhs) == ADDR_EXPR)
3969 : {
3970 34591 : if (AGGREGATE_TYPE_P (TREE_TYPE (TREE_TYPE (rhs)))
3971 24129 : && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (TREE_TYPE (rhs))))
3972 : return (void *)-1;
3973 23533 : tree tem = get_addr_base_and_unit_offset (TREE_OPERAND (rhs, 0),
3974 : &rhs_offset);
3975 23533 : if (!tem)
3976 : return (void *)-1;
3977 23533 : if (TREE_CODE (tem) == MEM_REF
3978 23533 : && poly_int_tree_p (TREE_OPERAND (tem, 1), &mem_offset))
3979 : {
3980 0 : rhs = TREE_OPERAND (tem, 0);
3981 0 : rhs_offset += mem_offset;
3982 : }
3983 23533 : else if (DECL_P (tem)
3984 17332 : || TREE_CODE (tem) == STRING_CST)
3985 23533 : rhs = build_fold_addr_expr (tem);
3986 : else
3987 : return (void *)-1;
3988 : }
3989 41236 : if (TREE_CODE (rhs) == SSA_NAME)
3990 17703 : rhs = SSA_VAL (rhs);
3991 23533 : else if (TREE_CODE (rhs) != ADDR_EXPR)
3992 : return (void *)-1;
3993 :
3994 : /* The bases of the destination and the references have to agree. */
3995 41236 : if (TREE_CODE (base) == MEM_REF)
3996 : {
3997 15714 : if (TREE_OPERAND (base, 0) != lhs
3998 15714 : || !poly_int_tree_p (TREE_OPERAND (base, 1), &mem_offset))
3999 11977 : return (void *) -1;
4000 12294 : at += mem_offset;
4001 : }
4002 25522 : else if (!DECL_P (base)
4003 24612 : || TREE_CODE (lhs) != ADDR_EXPR
4004 34080 : || TREE_OPERAND (lhs, 0) != base)
4005 : return (void *)-1;
4006 :
4007 : /* If the access is completely outside of the memcpy destination
4008 : area there is no aliasing. */
4009 12294 : if (!ranges_maybe_overlap_p (lhs_offset, copy_size, at, byte_maxsize))
4010 : return NULL;
4011 : /* And the access has to be contained within the memcpy destination. */
4012 12261 : if (!known_subrange_p (at, byte_maxsize, lhs_offset, copy_size))
4013 : return (void *)-1;
4014 :
4015 : /* Save the operands since we need to use the original ones for
4016 : the hash entry we use. */
4017 11808 : if (!data->saved_operands.exists ())
4018 11312 : data->saved_operands = vr->operands.copy ();
4019 :
4020 : /* Make room for 2 operands in the new reference. */
4021 11808 : if (vr->operands.length () < 2)
4022 : {
4023 0 : vec<vn_reference_op_s> old = vr->operands;
4024 0 : vr->operands.safe_grow_cleared (2, true);
4025 0 : if (old == shared_lookup_references)
4026 0 : shared_lookup_references = vr->operands;
4027 : }
4028 : else
4029 11808 : vr->operands.truncate (2);
4030 :
4031 : /* The looked-through reference is a simple MEM_REF. */
4032 11808 : memset (&op, 0, sizeof (op));
4033 11808 : op.type = vr->type;
4034 11808 : op.opcode = MEM_REF;
4035 11808 : op.op0 = build_int_cst (ptr_type_node, at - lhs_offset + rhs_offset);
4036 11808 : op.off = at - lhs_offset + rhs_offset;
4037 11808 : vr->operands[0] = op;
4038 11808 : op.type = TREE_TYPE (rhs);
4039 11808 : op.opcode = TREE_CODE (rhs);
4040 11808 : op.op0 = rhs;
4041 11808 : op.off = -1;
4042 11808 : vr->operands[1] = op;
4043 11808 : vr->hashcode = vn_reference_compute_hash (vr);
4044 :
4045 : /* Try folding the new reference to a constant. */
4046 11808 : tree val = fully_constant_vn_reference_p (vr);
4047 11808 : if (val)
4048 2969 : return data->finish (0, 0, val);
4049 :
4050 : /* Adjust *ref from the new operands. */
4051 8839 : if (!ao_ref_init_from_vn_reference (&r, 0, 0, vr->type, vr->operands))
4052 : return (void *)-1;
4053 : /* This can happen with bitfields. */
4054 8839 : if (maybe_ne (ref->size, r.size))
4055 : return (void *)-1;
4056 8839 : *ref = r;
4057 8839 : vr->offset = r.offset;
4058 8839 : vr->max_size = r.max_size;
4059 :
4060 : /* Do not update last seen VUSE after translating. */
4061 8839 : data->last_vuse_ptr = NULL;
4062 : /* Invalidate the original access path since it now contains
4063 : the wrong base. */
4064 8839 : data->orig_ref.ref = NULL_TREE;
4065 : /* Use the alias-set of this stmt for recording an eventual result. */
4066 8839 : if (data->first_set == -2)
4067 : {
4068 8385 : data->first_set = 0;
4069 8385 : data->first_base_set = 0;
4070 : }
4071 :
4072 : /* Keep looking for the adjusted *REF / VR pair. */
4073 8839 : return NULL;
4074 : }
4075 :
4076 : /* Bail out and stop walking. */
4077 : return (void *)-1;
4078 : }
4079 :
4080 : /* Return true if E is a backedge with respect to our CFG walk order. */
4081 :
4082 : static bool
4083 120148455 : vn_is_backedge (edge e, void *)
4084 : {
4085 : /* During PRE elimination we no longer have access to this info. */
4086 120148455 : return (!vn_bb_to_rpo
4087 120148455 : || vn_bb_to_rpo[e->dest->index] <= vn_bb_to_rpo[e->src->index]);
4088 : }
4089 :
4090 : /* Return a reference op vector from OP that can be used for
4091 : vn_reference_lookup_pieces. The caller is responsible for releasing
4092 : the vector. */
4093 :
4094 : vec<vn_reference_op_s>
4095 4703971 : vn_reference_operands_for_lookup (tree op)
4096 : {
4097 4703971 : bool valueized;
4098 4703971 : return valueize_shared_reference_ops_from_ref (op, &valueized).copy ();
4099 : }
4100 :
4101 : /* Lookup a reference operation by it's parts, in the current hash table.
4102 : Returns the resulting value number if it exists in the hash table,
4103 : NULL_TREE otherwise. VNRESULT will be filled in with the actual
4104 : vn_reference_t stored in the hashtable if something is found. */
4105 :
4106 : tree
4107 7740703 : vn_reference_lookup_pieces (tree vuse, alias_set_type set,
4108 : alias_set_type base_set, tree type,
4109 : vec<vn_reference_op_s> operands,
4110 : vn_reference_t *vnresult, vn_lookup_kind kind)
4111 : {
4112 7740703 : struct vn_reference_s vr1;
4113 7740703 : vn_reference_t tmp;
4114 7740703 : tree cst;
4115 :
4116 7740703 : if (!vnresult)
4117 0 : vnresult = &tmp;
4118 7740703 : *vnresult = NULL;
4119 :
4120 7740703 : vr1.vuse = vuse_ssa_val (vuse);
4121 7740703 : shared_lookup_references.truncate (0);
4122 15481406 : shared_lookup_references.safe_grow (operands.length (), true);
4123 7740703 : memcpy (shared_lookup_references.address (),
4124 7740703 : operands.address (),
4125 : sizeof (vn_reference_op_s)
4126 7740703 : * operands.length ());
4127 7740703 : bool valueized_p;
4128 7740703 : valueize_refs_1 (&shared_lookup_references, &valueized_p);
4129 7740703 : vr1.operands = shared_lookup_references;
4130 7740703 : vr1.type = type;
4131 7740703 : vr1.set = set;
4132 7740703 : vr1.base_set = base_set;
4133 : /* We can pretend there's no extra info fed in since the ao_refs offset
4134 : and max_size are computed only from the VN reference ops. */
4135 7740703 : vr1.offset = 0;
4136 7740703 : vr1.max_size = -1;
4137 7740703 : vr1.hashcode = vn_reference_compute_hash (&vr1);
4138 7740703 : if ((cst = fully_constant_vn_reference_p (&vr1)))
4139 : return cst;
4140 :
4141 7721787 : vn_reference_lookup_1 (&vr1, vnresult);
4142 7721787 : if (!*vnresult
4143 2999827 : && kind != VN_NOWALK
4144 2999827 : && vr1.vuse)
4145 : {
4146 2971481 : ao_ref r;
4147 2971481 : unsigned limit = param_sccvn_max_alias_queries_per_access;
4148 2971481 : vn_walk_cb_data data (&vr1, NULL_TREE, NULL, kind, true, NULL_TREE,
4149 2971481 : false);
4150 2971481 : vec<vn_reference_op_s> ops_for_ref;
4151 2971481 : if (!valueized_p)
4152 2881449 : ops_for_ref = vr1.operands;
4153 : else
4154 : {
4155 : /* For ao_ref_from_mem we have to ensure only available SSA names
4156 : end up in base and the only convenient way to make this work
4157 : for PRE is to re-valueize with that in mind. */
4158 180064 : ops_for_ref.create (operands.length ());
4159 180064 : ops_for_ref.quick_grow (operands.length ());
4160 90032 : memcpy (ops_for_ref.address (),
4161 90032 : operands.address (),
4162 : sizeof (vn_reference_op_s)
4163 90032 : * operands.length ());
4164 90032 : valueize_refs_1 (&ops_for_ref, &valueized_p, true);
4165 : }
4166 2971481 : if (ao_ref_init_from_vn_reference (&r, set, base_set, type,
4167 : ops_for_ref))
4168 2901044 : *vnresult
4169 2901044 : = ((vn_reference_t)
4170 2901044 : walk_non_aliased_vuses (&r, vr1.vuse, true, vn_reference_lookup_2,
4171 : vn_reference_lookup_3, vn_is_backedge,
4172 : vuse_valueize, limit, &data));
4173 5942962 : if (ops_for_ref != shared_lookup_references)
4174 90032 : ops_for_ref.release ();
4175 5942962 : gcc_checking_assert (vr1.operands == shared_lookup_references);
4176 2971481 : if (*vnresult
4177 421389 : && data.same_val
4178 2971481 : && (!(*vnresult)->result
4179 0 : || !operand_equal_p ((*vnresult)->result, data.same_val)))
4180 : {
4181 0 : *vnresult = NULL;
4182 0 : return NULL_TREE;
4183 : }
4184 2971481 : }
4185 :
4186 7721787 : if (*vnresult)
4187 5143349 : return (*vnresult)->result;
4188 :
4189 : return NULL_TREE;
4190 : }
4191 :
4192 : /* When OPERANDS is an ADDR_EXPR that can be possibly expressed as a
4193 : POINTER_PLUS_EXPR return true and fill in its operands in OPS. */
4194 :
4195 : bool
4196 2205550 : vn_pp_nary_for_addr (const vec<vn_reference_op_s>& operands, tree ops[2])
4197 : {
4198 4411100 : gcc_assert (operands[0].opcode == ADDR_EXPR
4199 : && operands.last ().opcode == SSA_NAME);
4200 : poly_int64 off = 0;
4201 : vn_reference_op_t vro;
4202 : unsigned i;
4203 7133988 : for (i = 1; operands.iterate (i, &vro); ++i)
4204 : {
4205 7133988 : if (vro->opcode == SSA_NAME)
4206 : break;
4207 4978249 : else if (known_eq (vro->off, -1))
4208 : break;
4209 4928438 : off += vro->off;
4210 : }
4211 2205550 : if (i == operands.length () - 1
4212 2155739 : && maybe_ne (off, 0)
4213 : /* Make sure we the offset we accumulated in a 64bit int
4214 : fits the address computation carried out in target
4215 : offset precision. */
4216 3628283 : && (off.coeffs[0]
4217 1422733 : == sext_hwi (off.coeffs[0], TYPE_PRECISION (sizetype))))
4218 : {
4219 1421950 : gcc_assert (operands[i-1].opcode == MEM_REF);
4220 1421950 : ops[0] = operands[i].op0;
4221 1421950 : ops[1] = wide_int_to_tree (sizetype, off);
4222 1421950 : return true;
4223 : }
4224 : return false;
4225 : }
4226 :
4227 : /* Lookup OP in the current hash table, and return the resulting value
4228 : number if it exists in the hash table. Return NULL_TREE if it does
4229 : not exist in the hash table or if the result field of the structure
4230 : was NULL.. VNRESULT will be filled in with the vn_reference_t
4231 : stored in the hashtable if one exists. When TBAA_P is false assume
4232 : we are looking up a store and treat it as having alias-set zero.
4233 : *LAST_VUSE_PTR will be updated with the VUSE the value lookup succeeded.
4234 : MASK is either NULL_TREE, or can be an INTEGER_CST if the result of the
4235 : load is bitwise anded with MASK and so we are only interested in a subset
4236 : of the bits and can ignore if the other bits are uninitialized or
4237 : not initialized with constants. When doing redundant store removal
4238 : the caller has to set REDUNDANT_STORE_REMOVAL_P. */
4239 :
4240 : tree
4241 100075165 : vn_reference_lookup (tree op, tree vuse, vn_lookup_kind kind,
4242 : vn_reference_t *vnresult, bool tbaa_p,
4243 : tree *last_vuse_ptr, tree mask,
4244 : bool redundant_store_removal_p)
4245 : {
4246 100075165 : vec<vn_reference_op_s> operands;
4247 100075165 : struct vn_reference_s vr1;
4248 100075165 : bool valueized_anything;
4249 :
4250 100075165 : if (vnresult)
4251 99690593 : *vnresult = NULL;
4252 :
4253 100075165 : vr1.vuse = vuse_ssa_val (vuse);
4254 200150330 : vr1.operands = operands
4255 100075165 : = valueize_shared_reference_ops_from_ref (op, &valueized_anything);
4256 :
4257 : /* Handle &MEM[ptr + 5].b[1].c as POINTER_PLUS_EXPR. Avoid doing
4258 : this before the pass folding __builtin_object_size had a chance to run. */
4259 100075165 : if ((cfun->curr_properties & PROP_objsz)
4260 72643079 : && operands[0].opcode == ADDR_EXPR
4261 101182122 : && operands.last ().opcode == SSA_NAME)
4262 : {
4263 1073349 : tree ops[2];
4264 1073349 : if (vn_pp_nary_for_addr (operands, ops))
4265 : {
4266 692747 : tree res = vn_nary_op_lookup_pieces (2, POINTER_PLUS_EXPR,
4267 692747 : TREE_TYPE (op), ops, NULL);
4268 692747 : if (res)
4269 692747 : return res;
4270 692747 : return NULL_TREE;
4271 : }
4272 : }
4273 :
4274 99382418 : vr1.type = TREE_TYPE (op);
4275 99382418 : ao_ref op_ref;
4276 99382418 : ao_ref_init (&op_ref, op);
4277 99382418 : vr1.set = ao_ref_alias_set (&op_ref);
4278 99382418 : vr1.base_set = ao_ref_base_alias_set (&op_ref);
4279 99382418 : vr1.offset = 0;
4280 99382418 : vr1.max_size = -1;
4281 99382418 : vr1.hashcode = vn_reference_compute_hash (&vr1);
4282 99382418 : if (mask == NULL_TREE)
4283 99081851 : if (tree cst = fully_constant_vn_reference_p (&vr1))
4284 : return cst;
4285 :
4286 99367392 : if (kind != VN_NOWALK && vr1.vuse)
4287 : {
4288 57710435 : vn_reference_t wvnresult;
4289 57710435 : ao_ref r;
4290 57710435 : unsigned limit = param_sccvn_max_alias_queries_per_access;
4291 57710435 : auto_vec<vn_reference_op_s> ops_for_ref;
4292 57710435 : if (valueized_anything)
4293 : {
4294 4522447 : copy_reference_ops_from_ref (op, &ops_for_ref);
4295 4522447 : bool tem;
4296 4522447 : valueize_refs_1 (&ops_for_ref, &tem, true);
4297 : }
4298 : /* Make sure to use a valueized reference if we valueized anything.
4299 : Otherwise preserve the full reference for advanced TBAA. */
4300 57710435 : if (!valueized_anything
4301 57710435 : || !ao_ref_init_from_vn_reference (&r, vr1.set, vr1.base_set,
4302 : vr1.type, ops_for_ref))
4303 : {
4304 53187988 : ao_ref_init (&r, op);
4305 : /* Record the extra info we're getting from the full ref. */
4306 53187988 : ao_ref_base (&r);
4307 53187988 : vr1.offset = r.offset;
4308 53187988 : vr1.max_size = r.max_size;
4309 : }
4310 57710435 : vn_walk_cb_data data (&vr1, r.ref ? NULL_TREE : op,
4311 : last_vuse_ptr, kind, tbaa_p, mask,
4312 110898423 : redundant_store_removal_p);
4313 :
4314 57710435 : wvnresult
4315 : = ((vn_reference_t)
4316 57710435 : walk_non_aliased_vuses (&r, vr1.vuse, tbaa_p, vn_reference_lookup_2,
4317 : vn_reference_lookup_3, vn_is_backedge,
4318 : vuse_valueize, limit, &data));
4319 115420870 : gcc_checking_assert (vr1.operands == shared_lookup_references);
4320 57710435 : if (wvnresult)
4321 : {
4322 8484224 : gcc_assert (mask == NULL_TREE);
4323 8484224 : if (data.same_val
4324 8484224 : && (!wvnresult->result
4325 65298 : || !operand_equal_p (wvnresult->result, data.same_val)))
4326 45816 : return NULL_TREE;
4327 8438408 : if (vnresult)
4328 8436879 : *vnresult = wvnresult;
4329 8438408 : return wvnresult->result;
4330 : }
4331 49226211 : else if (mask)
4332 300567 : return data.masked_result;
4333 :
4334 : return NULL_TREE;
4335 57710435 : }
4336 :
4337 41656957 : if (last_vuse_ptr)
4338 1427422 : *last_vuse_ptr = vr1.vuse;
4339 41656957 : if (mask)
4340 : return NULL_TREE;
4341 41656957 : return vn_reference_lookup_1 (&vr1, vnresult);
4342 : }
4343 :
4344 : /* Lookup CALL in the current hash table and return the entry in
4345 : *VNRESULT if found. Populates *VR for the hashtable lookup. */
4346 :
4347 : void
4348 9184515 : vn_reference_lookup_call (gcall *call, vn_reference_t *vnresult,
4349 : vn_reference_t vr)
4350 : {
4351 9184515 : if (vnresult)
4352 9184515 : *vnresult = NULL;
4353 :
4354 9184515 : tree vuse = gimple_vuse (call);
4355 :
4356 9184515 : vr->vuse = vuse ? SSA_VAL (vuse) : NULL_TREE;
4357 9184515 : vr->operands = valueize_shared_reference_ops_from_call (call);
4358 9184515 : tree lhs = gimple_call_lhs (call);
4359 : /* For non-SSA return values the referece ops contain the LHS. */
4360 5020764 : vr->type = ((lhs && TREE_CODE (lhs) == SSA_NAME)
4361 13766416 : ? TREE_TYPE (lhs) : NULL_TREE);
4362 9184515 : vr->punned = false;
4363 9184515 : vr->set = 0;
4364 9184515 : vr->base_set = 0;
4365 9184515 : vr->offset = 0;
4366 9184515 : vr->max_size = -1;
4367 9184515 : vr->hashcode = vn_reference_compute_hash (vr);
4368 9184515 : vn_reference_lookup_1 (vr, vnresult);
4369 9184515 : }
4370 :
4371 : /* Insert OP into the current hash table with a value number of RESULT. */
4372 :
4373 : static void
4374 74173851 : vn_reference_insert (tree op, tree result, tree vuse, tree vdef)
4375 : {
4376 74173851 : vn_reference_s **slot;
4377 74173851 : vn_reference_t vr1;
4378 74173851 : bool tem;
4379 :
4380 74173851 : vec<vn_reference_op_s> operands
4381 74173851 : = valueize_shared_reference_ops_from_ref (op, &tem);
4382 : /* Handle &MEM[ptr + 5].b[1].c as POINTER_PLUS_EXPR. Avoid doing this
4383 : before the pass folding __builtin_object_size had a chance to run. */
4384 74173851 : if ((cfun->curr_properties & PROP_objsz)
4385 55680037 : && operands[0].opcode == ADDR_EXPR
4386 75079892 : && operands.last ().opcode == SSA_NAME)
4387 : {
4388 874518 : tree ops[2];
4389 874518 : if (vn_pp_nary_for_addr (operands, ops))
4390 : {
4391 557733 : vn_nary_op_insert_pieces (2, POINTER_PLUS_EXPR,
4392 557733 : TREE_TYPE (op), ops, result,
4393 557733 : VN_INFO (result)->value_id);
4394 557733 : return;
4395 : }
4396 : }
4397 :
4398 73616118 : vr1 = XOBNEW (&vn_tables_obstack, vn_reference_s);
4399 73616118 : if (TREE_CODE (result) == SSA_NAME)
4400 50843205 : vr1->value_id = VN_INFO (result)->value_id;
4401 : else
4402 22772913 : vr1->value_id = get_or_alloc_constant_value_id (result);
4403 73616118 : vr1->vuse = vuse_ssa_val (vuse);
4404 73616118 : vr1->operands = operands.copy ();
4405 73616118 : vr1->type = TREE_TYPE (op);
4406 73616118 : vr1->punned = false;
4407 73616118 : ao_ref op_ref;
4408 73616118 : ao_ref_init (&op_ref, op);
4409 73616118 : vr1->set = ao_ref_alias_set (&op_ref);
4410 73616118 : vr1->base_set = ao_ref_base_alias_set (&op_ref);
4411 : /* Specifically use an unknown extent here, we're not doing any lookup
4412 : and assume the caller didn't either (or it went VARYING). */
4413 73616118 : vr1->offset = 0;
4414 73616118 : vr1->max_size = -1;
4415 73616118 : vr1->hashcode = vn_reference_compute_hash (vr1);
4416 73616118 : vr1->result = TREE_CODE (result) == SSA_NAME ? SSA_VAL (result) : result;
4417 73616118 : vr1->result_vdef = vdef;
4418 :
4419 73616118 : slot = valid_info->references->find_slot_with_hash (vr1, vr1->hashcode,
4420 : INSERT);
4421 :
4422 : /* Because IL walking on reference lookup can end up visiting
4423 : a def that is only to be visited later in iteration order
4424 : when we are about to make an irreducible region reducible
4425 : the def can be effectively processed and its ref being inserted
4426 : by vn_reference_lookup_3 already. So we cannot assert (!*slot)
4427 : but save a lookup if we deal with already inserted refs here. */
4428 73616118 : if (*slot)
4429 : {
4430 : /* We cannot assert that we have the same value either because
4431 : when disentangling an irreducible region we may end up visiting
4432 : a use before the corresponding def. That's a missed optimization
4433 : only though. See gcc.dg/tree-ssa/pr87126.c for example. */
4434 0 : if (dump_file && (dump_flags & TDF_DETAILS)
4435 0 : && !operand_equal_p ((*slot)->result, vr1->result, 0))
4436 : {
4437 0 : fprintf (dump_file, "Keeping old value ");
4438 0 : print_generic_expr (dump_file, (*slot)->result);
4439 0 : fprintf (dump_file, " because of collision\n");
4440 : }
4441 0 : free_reference (vr1);
4442 0 : obstack_free (&vn_tables_obstack, vr1);
4443 0 : return;
4444 : }
4445 :
4446 73616118 : *slot = vr1;
4447 73616118 : vr1->next = last_inserted_ref;
4448 73616118 : last_inserted_ref = vr1;
4449 : }
4450 :
4451 : /* Insert a reference by it's pieces into the current hash table with
4452 : a value number of RESULT. Return the resulting reference
4453 : structure we created. */
4454 :
4455 : vn_reference_t
4456 1521464 : vn_reference_insert_pieces (tree vuse, alias_set_type set,
4457 : alias_set_type base_set,
4458 : poly_int64 offset, poly_int64 max_size, tree type,
4459 : vec<vn_reference_op_s> operands,
4460 : tree result, unsigned int value_id)
4461 :
4462 : {
4463 1521464 : vn_reference_s **slot;
4464 1521464 : vn_reference_t vr1;
4465 :
4466 1521464 : vr1 = XOBNEW (&vn_tables_obstack, vn_reference_s);
4467 1521464 : vr1->value_id = value_id;
4468 1521464 : vr1->vuse = vuse_ssa_val (vuse);
4469 1521464 : vr1->operands = operands;
4470 1521464 : valueize_refs (&vr1->operands);
4471 1521464 : vr1->type = type;
4472 1521464 : vr1->punned = false;
4473 1521464 : vr1->set = set;
4474 1521464 : vr1->base_set = base_set;
4475 1521464 : vr1->offset = offset;
4476 1521464 : vr1->max_size = max_size;
4477 1521464 : vr1->hashcode = vn_reference_compute_hash (vr1);
4478 1521464 : if (result && TREE_CODE (result) == SSA_NAME)
4479 349087 : result = SSA_VAL (result);
4480 1521464 : vr1->result = result;
4481 1521464 : vr1->result_vdef = NULL_TREE;
4482 :
4483 1521464 : slot = valid_info->references->find_slot_with_hash (vr1, vr1->hashcode,
4484 : INSERT);
4485 :
4486 : /* At this point we should have all the things inserted that we have
4487 : seen before, and we should never try inserting something that
4488 : already exists. */
4489 1521464 : gcc_assert (!*slot);
4490 :
4491 1521464 : *slot = vr1;
4492 1521464 : vr1->next = last_inserted_ref;
4493 1521464 : last_inserted_ref = vr1;
4494 1521464 : return vr1;
4495 : }
4496 :
4497 : /* Compute and return the hash value for nary operation VBO1. */
4498 :
4499 : hashval_t
4500 301466003 : vn_nary_op_compute_hash (const vn_nary_op_t vno1)
4501 : {
4502 301466003 : inchash::hash hstate;
4503 301466003 : unsigned i;
4504 :
4505 301466003 : if (((vno1->length == 2
4506 254026757 : && commutative_tree_code (vno1->opcode))
4507 137943945 : || (vno1->length == 3
4508 1547116 : && commutative_ternary_tree_code (vno1->opcode)))
4509 464990204 : && tree_swap_operands_p (vno1->op[0], vno1->op[1]))
4510 2392309 : std::swap (vno1->op[0], vno1->op[1]);
4511 299073694 : else if (TREE_CODE_CLASS (vno1->opcode) == tcc_comparison
4512 299073694 : && tree_swap_operands_p (vno1->op[0], vno1->op[1]))
4513 : {
4514 472305 : std::swap (vno1->op[0], vno1->op[1]);
4515 472305 : vno1->opcode = swap_tree_comparison (vno1->opcode);
4516 : }
4517 :
4518 301466003 : hstate.add_int (vno1->opcode);
4519 860783742 : for (i = 0; i < vno1->length; ++i)
4520 559317739 : inchash::add_expr (vno1->op[i], hstate);
4521 :
4522 301466003 : return hstate.end ();
4523 : }
4524 :
4525 : /* Compare nary operations VNO1 and VNO2 and return true if they are
4526 : equivalent. */
4527 :
4528 : bool
4529 958675462 : vn_nary_op_eq (const_vn_nary_op_t const vno1, const_vn_nary_op_t const vno2)
4530 : {
4531 958675462 : unsigned i;
4532 :
4533 958675462 : if (vno1->hashcode != vno2->hashcode)
4534 : return false;
4535 :
4536 49980193 : if (vno1->length != vno2->length)
4537 : return false;
4538 :
4539 49980193 : if (vno1->opcode != vno2->opcode
4540 49980193 : || !types_compatible_p (vno1->type, vno2->type))
4541 1142368 : return false;
4542 :
4543 141184704 : for (i = 0; i < vno1->length; ++i)
4544 92445097 : if (!expressions_equal_p (vno1->op[i], vno2->op[i]))
4545 : return false;
4546 :
4547 : /* BIT_INSERT_EXPR has an implict operand as the type precision
4548 : of op1. Need to check to make sure they are the same. */
4549 48739607 : if (vno1->opcode == BIT_INSERT_EXPR
4550 536 : && TREE_CODE (vno1->op[1]) == INTEGER_CST
4551 48739716 : && TYPE_PRECISION (TREE_TYPE (vno1->op[1]))
4552 109 : != TYPE_PRECISION (TREE_TYPE (vno2->op[1])))
4553 : return false;
4554 :
4555 : return true;
4556 : }
4557 :
4558 : /* Initialize VNO from the pieces provided. */
4559 :
4560 : static void
4561 186892726 : init_vn_nary_op_from_pieces (vn_nary_op_t vno, unsigned int length,
4562 : enum tree_code code, tree type, tree *ops)
4563 : {
4564 186892726 : vno->opcode = code;
4565 186892726 : vno->length = length;
4566 186892726 : vno->type = type;
4567 4562546 : memcpy (&vno->op[0], ops, sizeof (tree) * length);
4568 0 : }
4569 :
4570 : /* Return the number of operands for a vn_nary ops structure from STMT. */
4571 :
4572 : unsigned int
4573 108572541 : vn_nary_length_from_stmt (gimple *stmt)
4574 : {
4575 108572541 : switch (gimple_assign_rhs_code (stmt))
4576 : {
4577 : case REALPART_EXPR:
4578 : case IMAGPART_EXPR:
4579 : case VIEW_CONVERT_EXPR:
4580 : return 1;
4581 :
4582 603760 : case BIT_FIELD_REF:
4583 603760 : return 3;
4584 :
4585 508498 : case CONSTRUCTOR:
4586 508498 : return CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt));
4587 :
4588 104132707 : default:
4589 104132707 : return gimple_num_ops (stmt) - 1;
4590 : }
4591 : }
4592 :
4593 : /* Initialize VNO from STMT. */
4594 :
4595 : void
4596 108572541 : init_vn_nary_op_from_stmt (vn_nary_op_t vno, gassign *stmt)
4597 : {
4598 108572541 : unsigned i;
4599 :
4600 108572541 : vno->opcode = gimple_assign_rhs_code (stmt);
4601 108572541 : vno->type = TREE_TYPE (gimple_assign_lhs (stmt));
4602 108572541 : switch (vno->opcode)
4603 : {
4604 3327576 : case REALPART_EXPR:
4605 3327576 : case IMAGPART_EXPR:
4606 3327576 : case VIEW_CONVERT_EXPR:
4607 3327576 : vno->length = 1;
4608 3327576 : vno->op[0] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
4609 3327576 : break;
4610 :
4611 603760 : case BIT_FIELD_REF:
4612 603760 : vno->length = 3;
4613 603760 : vno->op[0] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
4614 603760 : vno->op[1] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 1);
4615 603760 : vno->op[2] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 2);
4616 603760 : break;
4617 :
4618 508498 : case CONSTRUCTOR:
4619 508498 : vno->length = CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt));
4620 2011657 : for (i = 0; i < vno->length; ++i)
4621 1503159 : vno->op[i] = CONSTRUCTOR_ELT (gimple_assign_rhs1 (stmt), i)->value;
4622 : break;
4623 :
4624 104132707 : default:
4625 104132707 : gcc_checking_assert (!gimple_assign_single_p (stmt));
4626 104132707 : vno->length = gimple_num_ops (stmt) - 1;
4627 285330733 : for (i = 0; i < vno->length; ++i)
4628 181198026 : vno->op[i] = gimple_op (stmt, i + 1);
4629 : }
4630 108572541 : }
4631 :
4632 : /* Compute the hashcode for VNO and look for it in the hash table;
4633 : return the resulting value number if it exists in the hash table.
4634 : Return NULL_TREE if it does not exist in the hash table or if the
4635 : result field of the operation is NULL. VNRESULT will contain the
4636 : vn_nary_op_t from the hashtable if it exists. */
4637 :
4638 : static tree
4639 130638425 : vn_nary_op_lookup_1 (vn_nary_op_t vno, vn_nary_op_t *vnresult)
4640 : {
4641 130638425 : vn_nary_op_s **slot;
4642 :
4643 130638425 : if (vnresult)
4644 123464965 : *vnresult = NULL;
4645 :
4646 363328253 : for (unsigned i = 0; i < vno->length; ++i)
4647 232689828 : if (TREE_CODE (vno->op[i]) == SSA_NAME)
4648 164757633 : vno->op[i] = SSA_VAL (vno->op[i]);
4649 :
4650 130638425 : vno->hashcode = vn_nary_op_compute_hash (vno);
4651 130638425 : slot = valid_info->nary->find_slot_with_hash (vno, vno->hashcode, NO_INSERT);
4652 130638425 : if (!slot)
4653 : return NULL_TREE;
4654 17566435 : if (vnresult)
4655 17129562 : *vnresult = *slot;
4656 17566435 : return (*slot)->predicated_values ? NULL_TREE : (*slot)->u.result;
4657 : }
4658 :
4659 : /* Lookup a n-ary operation by its pieces and return the resulting value
4660 : number if it exists in the hash table. Return NULL_TREE if it does
4661 : not exist in the hash table or if the result field of the operation
4662 : is NULL. VNRESULT will contain the vn_nary_op_t from the hashtable
4663 : if it exists. */
4664 :
4665 : tree
4666 74450155 : vn_nary_op_lookup_pieces (unsigned int length, enum tree_code code,
4667 : tree type, tree *ops, vn_nary_op_t *vnresult)
4668 : {
4669 74450155 : vn_nary_op_t vno1 = XALLOCAVAR (struct vn_nary_op_s,
4670 : sizeof_vn_nary_op (length));
4671 74450155 : init_vn_nary_op_from_pieces (vno1, length, code, type, ops);
4672 74450155 : return vn_nary_op_lookup_1 (vno1, vnresult);
4673 : }
4674 :
4675 : /* Lookup the rhs of STMT in the current hash table, and return the resulting
4676 : value number if it exists in the hash table. Return NULL_TREE if
4677 : it does not exist in the hash table. VNRESULT will contain the
4678 : vn_nary_op_t from the hashtable if it exists. */
4679 :
4680 : tree
4681 56188270 : vn_nary_op_lookup_stmt (gimple *stmt, vn_nary_op_t *vnresult)
4682 : {
4683 56188270 : vn_nary_op_t vno1
4684 56188270 : = XALLOCAVAR (struct vn_nary_op_s,
4685 : sizeof_vn_nary_op (vn_nary_length_from_stmt (stmt)));
4686 56188270 : init_vn_nary_op_from_stmt (vno1, as_a <gassign *> (stmt));
4687 56188270 : return vn_nary_op_lookup_1 (vno1, vnresult);
4688 : }
4689 :
4690 : /* Allocate a vn_nary_op_t with LENGTH operands on STACK. */
4691 :
4692 : vn_nary_op_t
4693 169864822 : alloc_vn_nary_op_noinit (unsigned int length, struct obstack *stack)
4694 : {
4695 169864822 : return (vn_nary_op_t) obstack_alloc (stack, sizeof_vn_nary_op (length));
4696 : }
4697 :
4698 : /* Allocate and initialize a vn_nary_op_t on CURRENT_INFO's
4699 : obstack. */
4700 :
4701 : static vn_nary_op_t
4702 152748610 : alloc_vn_nary_op (unsigned int length, tree result, unsigned int value_id)
4703 : {
4704 0 : vn_nary_op_t vno1 = alloc_vn_nary_op_noinit (length, &vn_tables_obstack);
4705 :
4706 152748610 : vno1->value_id = value_id;
4707 152748610 : vno1->length = length;
4708 152748610 : vno1->predicated_values = 0;
4709 152748610 : vno1->u.result = result;
4710 :
4711 152748610 : return vno1;
4712 : }
4713 :
4714 : /* Insert VNO into TABLE. */
4715 :
4716 : static vn_nary_op_t
4717 157446311 : vn_nary_op_insert_into (vn_nary_op_t vno, vn_nary_op_table_type *table)
4718 : {
4719 157446311 : vn_nary_op_s **slot;
4720 :
4721 157446311 : gcc_assert (! vno->predicated_values
4722 : || (! vno->u.values->next
4723 : && vno->u.values->n == 1));
4724 :
4725 460759252 : for (unsigned i = 0; i < vno->length; ++i)
4726 303312941 : if (TREE_CODE (vno->op[i]) == SSA_NAME)
4727 197778298 : vno->op[i] = SSA_VAL (vno->op[i]);
4728 :
4729 157446311 : vno->hashcode = vn_nary_op_compute_hash (vno);
4730 157446311 : slot = table->find_slot_with_hash (vno, vno->hashcode, INSERT);
4731 157446311 : vno->unwind_to = *slot;
4732 157446311 : if (*slot)
4733 : {
4734 : /* Prefer non-predicated values.
4735 : ??? Only if those are constant, otherwise, with constant predicated
4736 : value, turn them into predicated values with entry-block validity
4737 : (??? but we always find the first valid result currently). */
4738 30210416 : if ((*slot)->predicated_values
4739 29471466 : && ! vno->predicated_values)
4740 : {
4741 : /* ??? We cannot remove *slot from the unwind stack list.
4742 : For the moment we deal with this by skipping not found
4743 : entries but this isn't ideal ... */
4744 84172 : *slot = vno;
4745 : /* ??? Maintain a stack of states we can unwind in
4746 : vn_nary_op_s? But how far do we unwind? In reality
4747 : we need to push change records somewhere... Or not
4748 : unwind vn_nary_op_s and linking them but instead
4749 : unwind the results "list", linking that, which also
4750 : doesn't move on hashtable resize. */
4751 : /* We can also have a ->unwind_to recording *slot there.
4752 : That way we can make u.values a fixed size array with
4753 : recording the number of entries but of course we then
4754 : have always N copies for each unwind_to-state. Or we
4755 : make sure to only ever append and each unwinding will
4756 : pop off one entry (but how to deal with predicated
4757 : replaced with non-predicated here?) */
4758 84172 : vno->next = last_inserted_nary;
4759 84172 : last_inserted_nary = vno;
4760 84172 : return vno;
4761 : }
4762 30126244 : else if (vno->predicated_values
4763 30125888 : && ! (*slot)->predicated_values)
4764 : return *slot;
4765 29387650 : else if (vno->predicated_values
4766 29387294 : && (*slot)->predicated_values)
4767 : {
4768 : /* ??? Factor this all into a insert_single_predicated_value
4769 : routine. */
4770 29387294 : gcc_assert (!vno->u.values->next && vno->u.values->n == 1);
4771 29387294 : basic_block vno_bb
4772 29387294 : = BASIC_BLOCK_FOR_FN (cfun, vno->u.values->valid_dominated_by_p[0]);
4773 29387294 : vn_pval *nval = vno->u.values;
4774 29387294 : vn_pval **next = &vno->u.values;
4775 29387294 : vn_pval *ins = NULL;
4776 29387294 : vn_pval *ins_at = NULL;
4777 : /* Find an existing value to append to. */
4778 55271692 : for (vn_pval *val = (*slot)->u.values; val; val = val->next)
4779 : {
4780 30383088 : if (expressions_equal_p (val->result, nval->result))
4781 : {
4782 : /* Limit the number of places we register a predicate
4783 : as valid. */
4784 4498690 : if (val->n > 8)
4785 128302 : return *slot;
4786 11160409 : for (unsigned i = 0; i < val->n; ++i)
4787 : {
4788 7024225 : basic_block val_bb
4789 7024225 : = BASIC_BLOCK_FOR_FN (cfun,
4790 : val->valid_dominated_by_p[i]);
4791 7024225 : if (dominated_by_p (CDI_DOMINATORS, vno_bb, val_bb))
4792 : /* Value registered with more generic predicate. */
4793 234204 : return *slot;
4794 6790021 : else if (flag_checking)
4795 : /* Shouldn't happen, we insert in RPO order. */
4796 6790021 : gcc_assert (!dominated_by_p (CDI_DOMINATORS,
4797 : val_bb, vno_bb));
4798 : }
4799 : /* Append the location. */
4800 4136184 : ins_at = val;
4801 4136184 : ins = (vn_pval *) obstack_alloc (&vn_tables_obstack,
4802 : sizeof (vn_pval)
4803 : + val->n * sizeof (int));
4804 4136184 : ins->next = NULL;
4805 4136184 : ins->result = val->result;
4806 4136184 : ins->n = val->n + 1;
4807 4136184 : memcpy (ins->valid_dominated_by_p,
4808 4136184 : val->valid_dominated_by_p,
4809 4136184 : val->n * sizeof (int));
4810 4136184 : ins->valid_dominated_by_p[val->n] = vno_bb->index;
4811 4136184 : if (dump_file && (dump_flags & TDF_DETAILS))
4812 4 : fprintf (dump_file, "Appending predicate to value.\n");
4813 : break;
4814 : }
4815 : }
4816 : /* Copy the rest of the value chain. */
4817 59915790 : for (vn_pval *val = (*slot)->u.values; val; val = val->next)
4818 : {
4819 30891002 : if (val == ins_at)
4820 : /* Replace the node we appended to. */
4821 4136184 : *next = ins;
4822 : else
4823 : {
4824 : /* Copy other predicated values. */
4825 26754818 : *next = (vn_pval *) obstack_alloc (&vn_tables_obstack,
4826 : sizeof (vn_pval)
4827 : + ((val->n-1)
4828 : * sizeof (int)));
4829 26754818 : memcpy (*next, val,
4830 26754818 : sizeof (vn_pval) + (val->n-1) * sizeof (int));
4831 26754818 : (*next)->next = NULL;
4832 : }
4833 30891002 : next = &(*next)->next;
4834 : }
4835 : /* Append the value if we didn't find it. */
4836 29024788 : if (!ins_at)
4837 24888604 : *next = nval;
4838 29024788 : *slot = vno;
4839 29024788 : vno->next = last_inserted_nary;
4840 29024788 : last_inserted_nary = vno;
4841 29024788 : return vno;
4842 : }
4843 :
4844 : /* While we do not want to insert things twice it's awkward to
4845 : avoid it in the case where visit_nary_op pattern-matches stuff
4846 : and ends up simplifying the replacement to itself. We then
4847 : get two inserts, one from visit_nary_op and one from
4848 : vn_nary_build_or_lookup.
4849 : So allow inserts with the same value number. */
4850 356 : if ((*slot)->u.result == vno->u.result)
4851 : return *slot;
4852 : }
4853 :
4854 : /* ??? There's also optimistic vs. previous commited state merging
4855 : that is problematic for the case of unwinding. */
4856 :
4857 : /* ??? We should return NULL if we do not use 'vno' and have the
4858 : caller release it. */
4859 127235895 : gcc_assert (!*slot);
4860 :
4861 127235895 : *slot = vno;
4862 127235895 : vno->next = last_inserted_nary;
4863 127235895 : last_inserted_nary = vno;
4864 127235895 : return vno;
4865 : }
4866 :
4867 : /* Insert a n-ary operation into the current hash table using it's
4868 : pieces. Return the vn_nary_op_t structure we created and put in
4869 : the hashtable. */
4870 :
4871 : vn_nary_op_t
4872 557733 : vn_nary_op_insert_pieces (unsigned int length, enum tree_code code,
4873 : tree type, tree *ops,
4874 : tree result, unsigned int value_id)
4875 : {
4876 557733 : vn_nary_op_t vno1 = alloc_vn_nary_op (length, result, value_id);
4877 557733 : init_vn_nary_op_from_pieces (vno1, length, code, type, ops);
4878 557733 : return vn_nary_op_insert_into (vno1, valid_info->nary);
4879 : }
4880 :
4881 : /* Return whether we can track a predicate valid when PRED_E is executed. */
4882 :
4883 : static bool
4884 151389465 : can_track_predicate_on_edge (edge pred_e)
4885 : {
4886 : /* ??? As we are currently recording the destination basic-block index in
4887 : vn_pval.valid_dominated_by_p and using dominance for the
4888 : validity check we cannot track predicates on all edges. */
4889 151389465 : if (single_pred_p (pred_e->dest))
4890 : return true;
4891 : /* Never record for backedges. */
4892 11914211 : if (pred_e->flags & EDGE_DFS_BACK)
4893 : return false;
4894 : /* When there's more than one predecessor we cannot track
4895 : predicate validity based on the destination block. The
4896 : exception is when all other incoming edges sources are
4897 : dominated by the destination block. */
4898 11261954 : edge_iterator ei;
4899 11261954 : edge e;
4900 19298515 : FOR_EACH_EDGE (e, ei, pred_e->dest->preds)
4901 17466163 : if (e != pred_e && ! dominated_by_p (CDI_DOMINATORS, e->src, e->dest))
4902 : return false;
4903 : return true;
4904 : }
4905 :
4906 : static vn_nary_op_t
4907 107322292 : vn_nary_op_insert_pieces_predicated (unsigned int length, enum tree_code code,
4908 : tree type, tree *ops,
4909 : tree result, unsigned int value_id,
4910 : edge pred_e)
4911 : {
4912 107322292 : if (flag_checking)
4913 107321456 : gcc_assert (can_track_predicate_on_edge (pred_e));
4914 :
4915 75064 : if (dump_file && (dump_flags & TDF_DETAILS)
4916 : /* ??? Fix dumping, but currently we only get comparisons. */
4917 107393308 : && TREE_CODE_CLASS (code) == tcc_comparison)
4918 : {
4919 71016 : fprintf (dump_file, "Recording on edge %d->%d ", pred_e->src->index,
4920 71016 : pred_e->dest->index);
4921 71016 : print_generic_expr (dump_file, ops[0], TDF_SLIM);
4922 71016 : fprintf (dump_file, " %s ", get_tree_code_name (code));
4923 71016 : print_generic_expr (dump_file, ops[1], TDF_SLIM);
4924 106153 : fprintf (dump_file, " == %s\n",
4925 71016 : integer_zerop (result) ? "false" : "true");
4926 : }
4927 107322292 : vn_nary_op_t vno1 = alloc_vn_nary_op (length, NULL_TREE, value_id);
4928 107322292 : init_vn_nary_op_from_pieces (vno1, length, code, type, ops);
4929 107322292 : vno1->predicated_values = 1;
4930 107322292 : vno1->u.values = (vn_pval *) obstack_alloc (&vn_tables_obstack,
4931 : sizeof (vn_pval));
4932 107322292 : vno1->u.values->next = NULL;
4933 107322292 : vno1->u.values->result = result;
4934 107322292 : vno1->u.values->n = 1;
4935 107322292 : vno1->u.values->valid_dominated_by_p[0] = pred_e->dest->index;
4936 107322292 : return vn_nary_op_insert_into (vno1, valid_info->nary);
4937 : }
4938 :
4939 : static bool
4940 : dominated_by_p_w_unex (basic_block bb1, basic_block bb2, bool);
4941 :
4942 : static tree
4943 1737883 : vn_nary_op_get_predicated_value (vn_nary_op_t vno, basic_block bb,
4944 : edge e = NULL)
4945 : {
4946 1737883 : if (! vno->predicated_values)
4947 0 : return vno->u.result;
4948 3613456 : for (vn_pval *val = vno->u.values; val; val = val->next)
4949 5538283 : for (unsigned i = 0; i < val->n; ++i)
4950 : {
4951 3662710 : basic_block cand
4952 3662710 : = BASIC_BLOCK_FOR_FN (cfun, val->valid_dominated_by_p[i]);
4953 : /* Do not handle backedge executability optimistically since
4954 : when figuring out whether to iterate we do not consider
4955 : changed predication.
4956 : When asking for predicated values on an edge avoid looking
4957 : at edge executability for edges forward in our iteration
4958 : as well. */
4959 3662710 : if (e && (e->flags & EDGE_DFS_BACK))
4960 : {
4961 23362 : if (dominated_by_p (CDI_DOMINATORS, bb, cand))
4962 7769 : return val->result;
4963 : }
4964 3639348 : else if (dominated_by_p_w_unex (bb, cand, false))
4965 540155 : return val->result;
4966 : }
4967 : return NULL_TREE;
4968 : }
4969 :
4970 : static tree
4971 211651 : vn_nary_op_get_predicated_value (vn_nary_op_t vno, edge e)
4972 : {
4973 0 : return vn_nary_op_get_predicated_value (vno, e->src, e);
4974 : }
4975 :
4976 : /* Insert the rhs of STMT into the current hash table with a value number of
4977 : RESULT. */
4978 :
4979 : static vn_nary_op_t
4980 44868585 : vn_nary_op_insert_stmt (gimple *stmt, tree result)
4981 : {
4982 44868585 : vn_nary_op_t vno1
4983 44868585 : = alloc_vn_nary_op (vn_nary_length_from_stmt (stmt),
4984 44868585 : result, VN_INFO (result)->value_id);
4985 44868585 : init_vn_nary_op_from_stmt (vno1, as_a <gassign *> (stmt));
4986 44868585 : return vn_nary_op_insert_into (vno1, valid_info->nary);
4987 : }
4988 :
4989 : /* Compute a hashcode for PHI operation VP1 and return it. */
4990 :
4991 : static inline hashval_t
4992 49827994 : vn_phi_compute_hash (vn_phi_t vp1)
4993 : {
4994 49827994 : inchash::hash hstate;
4995 49827994 : tree phi1op;
4996 49827994 : tree type;
4997 49827994 : edge e;
4998 49827994 : edge_iterator ei;
4999 :
5000 99655988 : hstate.add_int (EDGE_COUNT (vp1->block->preds));
5001 49827994 : switch (EDGE_COUNT (vp1->block->preds))
5002 : {
5003 : case 1:
5004 : break;
5005 42919671 : case 2:
5006 : /* When this is a PHI node subject to CSE for different blocks
5007 : avoid hashing the block index. */
5008 42919671 : if (vp1->cclhs)
5009 : break;
5010 : /* Fallthru. */
5011 33634320 : default:
5012 33634320 : hstate.add_int (vp1->block->index);
5013 : }
5014 :
5015 : /* If all PHI arguments are constants we need to distinguish
5016 : the PHI node via its type. */
5017 49827994 : type = vp1->type;
5018 49827994 : hstate.merge_hash (vn_hash_type (type));
5019 :
5020 173178568 : FOR_EACH_EDGE (e, ei, vp1->block->preds)
5021 : {
5022 : /* Don't hash backedge values they need to be handled as VN_TOP
5023 : for optimistic value-numbering. */
5024 123350574 : if (e->flags & EDGE_DFS_BACK)
5025 27705900 : continue;
5026 :
5027 95644674 : phi1op = vp1->phiargs[e->dest_idx];
5028 95644674 : if (phi1op == VN_TOP)
5029 245427 : continue;
5030 95399247 : inchash::add_expr (phi1op, hstate);
5031 : }
5032 :
5033 49827994 : return hstate.end ();
5034 : }
5035 :
5036 :
5037 : /* Return true if COND1 and COND2 represent the same condition, set
5038 : *INVERTED_P if one needs to be inverted to make it the same as
5039 : the other. */
5040 :
5041 : static bool
5042 3773591 : cond_stmts_equal_p (gcond *cond1, tree lhs1, tree rhs1,
5043 : gcond *cond2, tree lhs2, tree rhs2, bool *inverted_p)
5044 : {
5045 3773591 : enum tree_code code1 = gimple_cond_code (cond1);
5046 3773591 : enum tree_code code2 = gimple_cond_code (cond2);
5047 :
5048 3773591 : *inverted_p = false;
5049 3773591 : if (code1 == code2)
5050 : ;
5051 299450 : else if (code1 == swap_tree_comparison (code2))
5052 : std::swap (lhs2, rhs2);
5053 263344 : else if (code1 == invert_tree_comparison (code2, HONOR_NANS (lhs2)))
5054 131157 : *inverted_p = true;
5055 132187 : else if (code1 == invert_tree_comparison
5056 132187 : (swap_tree_comparison (code2), HONOR_NANS (lhs2)))
5057 : {
5058 10348 : std::swap (lhs2, rhs2);
5059 10348 : *inverted_p = true;
5060 : }
5061 : else
5062 : return false;
5063 :
5064 3651752 : return ((expressions_equal_p (lhs1, lhs2)
5065 109257 : && expressions_equal_p (rhs1, rhs2))
5066 3676719 : || (commutative_tree_code (code1)
5067 1786946 : && expressions_equal_p (lhs1, rhs2)
5068 2370 : && expressions_equal_p (rhs1, lhs2)));
5069 : }
5070 :
5071 : /* Compare two phi entries for equality, ignoring VN_TOP arguments. */
5072 :
5073 : static int
5074 40547409 : vn_phi_eq (const_vn_phi_t const vp1, const_vn_phi_t const vp2)
5075 : {
5076 40547409 : if (vp1->hashcode != vp2->hashcode)
5077 : return false;
5078 :
5079 12667336 : if (vp1->block != vp2->block)
5080 : {
5081 11343774 : if (EDGE_COUNT (vp1->block->preds) != EDGE_COUNT (vp2->block->preds))
5082 : return false;
5083 :
5084 36414859 : switch (EDGE_COUNT (vp1->block->preds))
5085 : {
5086 : case 1:
5087 : /* Single-arg PHIs are just copies. */
5088 : break;
5089 :
5090 3781258 : case 2:
5091 3781258 : {
5092 : /* Make sure both PHIs are classified as CSEable. */
5093 3781258 : if (! vp1->cclhs || ! vp2->cclhs)
5094 : return false;
5095 :
5096 : /* Rule out backedges into the PHI. */
5097 3781258 : gcc_checking_assert
5098 : (vp1->block->loop_father->header != vp1->block
5099 : && vp2->block->loop_father->header != vp2->block);
5100 :
5101 : /* If the PHI nodes do not have compatible types
5102 : they are not the same. */
5103 3781258 : if (!types_compatible_p (vp1->type, vp2->type))
5104 : return false;
5105 :
5106 : /* If the immediate dominator end in switch stmts multiple
5107 : values may end up in the same PHI arg via intermediate
5108 : CFG merges. */
5109 3773591 : basic_block idom1
5110 3773591 : = get_immediate_dominator (CDI_DOMINATORS, vp1->block);
5111 3773591 : basic_block idom2
5112 3773591 : = get_immediate_dominator (CDI_DOMINATORS, vp2->block);
5113 3773591 : gcc_checking_assert (EDGE_COUNT (idom1->succs) == 2
5114 : && EDGE_COUNT (idom2->succs) == 2);
5115 :
5116 : /* Verify the controlling stmt is the same. */
5117 7547182 : gcond *last1 = as_a <gcond *> (*gsi_last_bb (idom1));
5118 7547182 : gcond *last2 = as_a <gcond *> (*gsi_last_bb (idom2));
5119 3773591 : bool inverted_p;
5120 3773591 : if (! cond_stmts_equal_p (last1, vp1->cclhs, vp1->ccrhs,
5121 3773591 : last2, vp2->cclhs, vp2->ccrhs,
5122 : &inverted_p))
5123 : return false;
5124 :
5125 : /* Get at true/false controlled edges into the PHI. */
5126 84373 : edge te1, te2, fe1, fe2;
5127 84373 : if (! extract_true_false_controlled_edges (idom1, vp1->block,
5128 : &te1, &fe1)
5129 84373 : || ! extract_true_false_controlled_edges (idom2, vp2->block,
5130 : &te2, &fe2))
5131 36979 : return false;
5132 :
5133 : /* Swap edges if the second condition is the inverted of the
5134 : first. */
5135 47394 : if (inverted_p)
5136 2038 : std::swap (te2, fe2);
5137 :
5138 : /* Since we do not know which edge will be executed we have
5139 : to be careful when matching VN_TOP. Be conservative and
5140 : only match VN_TOP == VN_TOP for now, we could allow
5141 : VN_TOP on the not prevailing PHI though. See for example
5142 : PR102920. */
5143 47394 : if (! expressions_equal_p (vp1->phiargs[te1->dest_idx],
5144 47394 : vp2->phiargs[te2->dest_idx], false)
5145 92975 : || ! expressions_equal_p (vp1->phiargs[fe1->dest_idx],
5146 45581 : vp2->phiargs[fe2->dest_idx], false))
5147 1813 : return false;
5148 :
5149 : return true;
5150 : }
5151 :
5152 : default:
5153 : return false;
5154 : }
5155 : }
5156 :
5157 : /* If the PHI nodes do not have compatible types
5158 : they are not the same. */
5159 8886078 : if (!types_compatible_p (vp1->type, vp2->type))
5160 : return false;
5161 :
5162 : /* Any phi in the same block will have it's arguments in the
5163 : same edge order, because of how we store phi nodes. */
5164 8885004 : unsigned nargs = EDGE_COUNT (vp1->block->preds);
5165 20611407 : for (unsigned i = 0; i < nargs; ++i)
5166 : {
5167 16478857 : tree phi1op = vp1->phiargs[i];
5168 16478857 : tree phi2op = vp2->phiargs[i];
5169 16478857 : if (phi1op == phi2op)
5170 11630938 : continue;
5171 4847919 : if (!expressions_equal_p (phi1op, phi2op, false))
5172 : return false;
5173 : }
5174 :
5175 : return true;
5176 : }
5177 :
5178 : /* Lookup PHI in the current hash table, and return the resulting
5179 : value number if it exists in the hash table. Return NULL_TREE if
5180 : it does not exist in the hash table. */
5181 :
5182 : static tree
5183 27306606 : vn_phi_lookup (gimple *phi, bool backedges_varying_p)
5184 : {
5185 27306606 : vn_phi_s **slot;
5186 27306606 : struct vn_phi_s *vp1;
5187 27306606 : edge e;
5188 27306606 : edge_iterator ei;
5189 :
5190 27306606 : vp1 = XALLOCAVAR (struct vn_phi_s,
5191 : sizeof (struct vn_phi_s)
5192 : + (gimple_phi_num_args (phi) - 1) * sizeof (tree));
5193 :
5194 : /* Canonicalize the SSA_NAME's to their value number. */
5195 94211644 : FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
5196 : {
5197 66905038 : tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
5198 66905038 : if (TREE_CODE (def) == SSA_NAME
5199 55784571 : && (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK)))
5200 : {
5201 53248029 : if (!virtual_operand_p (def)
5202 53248029 : && ssa_undefined_value_p (def, false))
5203 136136 : def = VN_TOP;
5204 : else
5205 53111893 : def = SSA_VAL (def);
5206 : }
5207 66905038 : vp1->phiargs[e->dest_idx] = def;
5208 : }
5209 27306606 : vp1->type = TREE_TYPE (gimple_phi_result (phi));
5210 27306606 : vp1->block = gimple_bb (phi);
5211 : /* Extract values of the controlling condition. */
5212 27306606 : vp1->cclhs = NULL_TREE;
5213 27306606 : vp1->ccrhs = NULL_TREE;
5214 27306606 : if (EDGE_COUNT (vp1->block->preds) == 2
5215 27306606 : && vp1->block->loop_father->header != vp1->block)
5216 : {
5217 8542950 : basic_block idom1 = get_immediate_dominator (CDI_DOMINATORS, vp1->block);
5218 8542950 : if (EDGE_COUNT (idom1->succs) == 2)
5219 16998144 : if (gcond *last1 = safe_dyn_cast <gcond *> (*gsi_last_bb (idom1)))
5220 : {
5221 : /* ??? We want to use SSA_VAL here. But possibly not
5222 : allow VN_TOP. */
5223 8275567 : vp1->cclhs = vn_valueize (gimple_cond_lhs (last1));
5224 8275567 : vp1->ccrhs = vn_valueize (gimple_cond_rhs (last1));
5225 : }
5226 : }
5227 27306606 : vp1->hashcode = vn_phi_compute_hash (vp1);
5228 27306606 : slot = valid_info->phis->find_slot_with_hash (vp1, vp1->hashcode, NO_INSERT);
5229 27306606 : if (!slot)
5230 : return NULL_TREE;
5231 4178131 : return (*slot)->result;
5232 : }
5233 :
5234 : /* Insert PHI into the current hash table with a value number of
5235 : RESULT. */
5236 :
5237 : static vn_phi_t
5238 22521388 : vn_phi_insert (gimple *phi, tree result, bool backedges_varying_p)
5239 : {
5240 22521388 : vn_phi_s **slot;
5241 22521388 : vn_phi_t vp1 = (vn_phi_t) obstack_alloc (&vn_tables_obstack,
5242 : sizeof (vn_phi_s)
5243 : + ((gimple_phi_num_args (phi) - 1)
5244 : * sizeof (tree)));
5245 22521388 : edge e;
5246 22521388 : edge_iterator ei;
5247 :
5248 : /* Canonicalize the SSA_NAME's to their value number. */
5249 78966924 : FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
5250 : {
5251 56445536 : tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
5252 56445536 : if (TREE_CODE (def) == SSA_NAME
5253 46402233 : && (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK)))
5254 : {
5255 43866107 : if (!virtual_operand_p (def)
5256 43866107 : && ssa_undefined_value_p (def, false))
5257 109535 : def = VN_TOP;
5258 : else
5259 43756572 : def = SSA_VAL (def);
5260 : }
5261 56445536 : vp1->phiargs[e->dest_idx] = def;
5262 : }
5263 22521388 : vp1->value_id = VN_INFO (result)->value_id;
5264 22521388 : vp1->type = TREE_TYPE (gimple_phi_result (phi));
5265 22521388 : vp1->block = gimple_bb (phi);
5266 : /* Extract values of the controlling condition. */
5267 22521388 : vp1->cclhs = NULL_TREE;
5268 22521388 : vp1->ccrhs = NULL_TREE;
5269 22521388 : if (EDGE_COUNT (vp1->block->preds) == 2
5270 22521388 : && vp1->block->loop_father->header != vp1->block)
5271 : {
5272 8181620 : basic_block idom1 = get_immediate_dominator (CDI_DOMINATORS, vp1->block);
5273 8181620 : if (EDGE_COUNT (idom1->succs) == 2)
5274 16278430 : if (gcond *last1 = safe_dyn_cast <gcond *> (*gsi_last_bb (idom1)))
5275 : {
5276 : /* ??? We want to use SSA_VAL here. But possibly not
5277 : allow VN_TOP. */
5278 7918107 : vp1->cclhs = vn_valueize (gimple_cond_lhs (last1));
5279 7918107 : vp1->ccrhs = vn_valueize (gimple_cond_rhs (last1));
5280 : }
5281 : }
5282 22521388 : vp1->result = result;
5283 22521388 : vp1->hashcode = vn_phi_compute_hash (vp1);
5284 :
5285 22521388 : slot = valid_info->phis->find_slot_with_hash (vp1, vp1->hashcode, INSERT);
5286 22521388 : gcc_assert (!*slot);
5287 :
5288 22521388 : *slot = vp1;
5289 22521388 : vp1->next = last_inserted_phi;
5290 22521388 : last_inserted_phi = vp1;
5291 22521388 : return vp1;
5292 : }
5293 :
5294 :
5295 : /* Return true if BB1 is dominated by BB2 taking into account edges
5296 : that are not executable. When ALLOW_BACK is false consider not
5297 : executable backedges as executable. */
5298 :
5299 : static bool
5300 72114353 : dominated_by_p_w_unex (basic_block bb1, basic_block bb2, bool allow_back)
5301 : {
5302 72114353 : edge_iterator ei;
5303 72114353 : edge e;
5304 :
5305 72114353 : if (dominated_by_p (CDI_DOMINATORS, bb1, bb2))
5306 : return true;
5307 :
5308 : /* Before iterating we'd like to know if there exists a
5309 : (executable) path from bb2 to bb1 at all, if not we can
5310 : directly return false. For now simply iterate once. */
5311 :
5312 : /* Iterate to the single executable bb1 predecessor. */
5313 21383067 : if (EDGE_COUNT (bb1->preds) > 1)
5314 : {
5315 2980000 : edge prede = NULL;
5316 6510113 : FOR_EACH_EDGE (e, ei, bb1->preds)
5317 6071015 : if ((e->flags & EDGE_EXECUTABLE)
5318 628258 : || (!allow_back && (e->flags & EDGE_DFS_BACK)))
5319 : {
5320 5520902 : if (prede)
5321 : {
5322 : prede = NULL;
5323 : break;
5324 : }
5325 : prede = e;
5326 : }
5327 2980000 : if (prede)
5328 : {
5329 439098 : bb1 = prede->src;
5330 :
5331 : /* Re-do the dominance check with changed bb1. */
5332 439098 : if (dominated_by_p (CDI_DOMINATORS, bb1, bb2))
5333 : return true;
5334 : }
5335 : }
5336 :
5337 : /* Iterate to the single executable bb2 successor. */
5338 21137201 : if (EDGE_COUNT (bb2->succs) > 1)
5339 : {
5340 6393585 : edge succe = NULL;
5341 12955078 : FOR_EACH_EDGE (e, ei, bb2->succs)
5342 12787419 : if ((e->flags & EDGE_EXECUTABLE)
5343 207491 : || (!allow_back && (e->flags & EDGE_DFS_BACK)))
5344 : {
5345 12579969 : if (succe)
5346 : {
5347 : succe = NULL;
5348 : break;
5349 : }
5350 : succe = e;
5351 : }
5352 6393585 : if (succe
5353 : /* Limit the number of edges we check, we should bring in
5354 : context from the iteration and compute the single
5355 : executable incoming edge when visiting a block. */
5356 6393585 : && EDGE_COUNT (succe->dest->preds) < 8)
5357 : {
5358 : /* Verify the reached block is only reached through succe.
5359 : If there is only one edge we can spare us the dominator
5360 : check and iterate directly. */
5361 127577 : if (EDGE_COUNT (succe->dest->preds) > 1)
5362 : {
5363 54147 : FOR_EACH_EDGE (e, ei, succe->dest->preds)
5364 41878 : if (e != succe
5365 27114 : && ((e->flags & EDGE_EXECUTABLE)
5366 18046 : || (!allow_back && (e->flags & EDGE_DFS_BACK))))
5367 : {
5368 : succe = NULL;
5369 : break;
5370 : }
5371 : }
5372 127577 : if (succe)
5373 : {
5374 118500 : bb2 = succe->dest;
5375 :
5376 : /* Re-do the dominance check with changed bb2. */
5377 118500 : if (dominated_by_p (CDI_DOMINATORS, bb1, bb2))
5378 : return true;
5379 : }
5380 : }
5381 : }
5382 : /* Iterate to the single successor of bb2 with only a single executable
5383 : incoming edge. */
5384 14743616 : else if (EDGE_COUNT (bb2->succs) == 1
5385 14182950 : && EDGE_COUNT (single_succ (bb2)->preds) > 1
5386 : /* Limit the number of edges we check, we should bring in
5387 : context from the iteration and compute the single
5388 : executable incoming edge when visiting a block. */
5389 28673470 : && EDGE_COUNT (single_succ (bb2)->preds) < 8)
5390 : {
5391 4986515 : edge prede = NULL;
5392 11274899 : FOR_EACH_EDGE (e, ei, single_succ (bb2)->preds)
5393 10705066 : if ((e->flags & EDGE_EXECUTABLE)
5394 1352032 : || (!allow_back && (e->flags & EDGE_DFS_BACK)))
5395 : {
5396 9357322 : if (prede)
5397 : {
5398 : prede = NULL;
5399 : break;
5400 : }
5401 : prede = e;
5402 : }
5403 : /* We might actually get to a query with BB2 not visited yet when
5404 : we're querying for a predicated value. */
5405 4986515 : if (prede && prede->src == bb2)
5406 : {
5407 509677 : bb2 = prede->dest;
5408 :
5409 : /* Re-do the dominance check with changed bb2. */
5410 509677 : if (dominated_by_p (CDI_DOMINATORS, bb1, bb2))
5411 : return true;
5412 : }
5413 : }
5414 :
5415 : /* We could now iterate updating bb1 / bb2. */
5416 : return false;
5417 : }
5418 :
5419 : /* Set the value number of FROM to TO, return true if it has changed
5420 : as a result. */
5421 :
5422 : static inline bool
5423 203791857 : set_ssa_val_to (tree from, tree to)
5424 : {
5425 203791857 : vn_ssa_aux_t from_info = VN_INFO (from);
5426 203791857 : tree currval = from_info->valnum; // SSA_VAL (from)
5427 203791857 : poly_int64 toff, coff;
5428 203791857 : bool curr_undefined = false;
5429 203791857 : bool curr_invariant = false;
5430 :
5431 : /* The only thing we allow as value numbers are ssa_names
5432 : and invariants. So assert that here. We don't allow VN_TOP
5433 : as visiting a stmt should produce a value-number other than
5434 : that.
5435 : ??? Still VN_TOP can happen for unreachable code, so force
5436 : it to varying in that case. Not all code is prepared to
5437 : get VN_TOP on valueization. */
5438 203791857 : if (to == VN_TOP)
5439 : {
5440 : /* ??? When iterating and visiting PHI <undef, backedge-value>
5441 : for the first time we rightfully get VN_TOP and we need to
5442 : preserve that to optimize for example gcc.dg/tree-ssa/ssa-sccvn-2.c.
5443 : With SCCVN we were simply lucky we iterated the other PHI
5444 : cycles first and thus visited the backedge-value DEF. */
5445 0 : if (currval == VN_TOP)
5446 0 : goto set_and_exit;
5447 0 : if (dump_file && (dump_flags & TDF_DETAILS))
5448 0 : fprintf (dump_file, "Forcing value number to varying on "
5449 : "receiving VN_TOP\n");
5450 : to = from;
5451 : }
5452 :
5453 203791857 : gcc_checking_assert (to != NULL_TREE
5454 : && ((TREE_CODE (to) == SSA_NAME
5455 : && (to == from || SSA_VAL (to) == to))
5456 : || is_gimple_min_invariant (to)));
5457 :
5458 203791857 : if (from != to)
5459 : {
5460 32401651 : if (currval == from)
5461 : {
5462 13834 : if (dump_file && (dump_flags & TDF_DETAILS))
5463 : {
5464 0 : fprintf (dump_file, "Not changing value number of ");
5465 0 : print_generic_expr (dump_file, from);
5466 0 : fprintf (dump_file, " from VARYING to ");
5467 0 : print_generic_expr (dump_file, to);
5468 0 : fprintf (dump_file, "\n");
5469 : }
5470 13834 : return false;
5471 : }
5472 32387817 : curr_invariant = is_gimple_min_invariant (currval);
5473 64775634 : curr_undefined = (TREE_CODE (currval) == SSA_NAME
5474 3852941 : && !virtual_operand_p (currval)
5475 36022142 : && ssa_undefined_value_p (currval, false));
5476 32387817 : if (currval != VN_TOP
5477 : && !curr_invariant
5478 5342943 : && !curr_undefined
5479 36227611 : && is_gimple_min_invariant (to))
5480 : {
5481 220 : if (dump_file && (dump_flags & TDF_DETAILS))
5482 : {
5483 0 : fprintf (dump_file, "Forcing VARYING instead of changing "
5484 : "value number of ");
5485 0 : print_generic_expr (dump_file, from);
5486 0 : fprintf (dump_file, " from ");
5487 0 : print_generic_expr (dump_file, currval);
5488 0 : fprintf (dump_file, " (non-constant) to ");
5489 0 : print_generic_expr (dump_file, to);
5490 0 : fprintf (dump_file, " (constant)\n");
5491 : }
5492 : to = from;
5493 : }
5494 32387597 : else if (currval != VN_TOP
5495 5342723 : && !curr_undefined
5496 5329576 : && TREE_CODE (to) == SSA_NAME
5497 4498064 : && !virtual_operand_p (to)
5498 36667045 : && ssa_undefined_value_p (to, false))
5499 : {
5500 6 : if (dump_file && (dump_flags & TDF_DETAILS))
5501 : {
5502 0 : fprintf (dump_file, "Forcing VARYING instead of changing "
5503 : "value number of ");
5504 0 : print_generic_expr (dump_file, from);
5505 0 : fprintf (dump_file, " from ");
5506 0 : print_generic_expr (dump_file, currval);
5507 0 : fprintf (dump_file, " (non-undefined) to ");
5508 0 : print_generic_expr (dump_file, to);
5509 0 : fprintf (dump_file, " (undefined)\n");
5510 : }
5511 : to = from;
5512 : }
5513 32387591 : else if (TREE_CODE (to) == SSA_NAME
5514 32387591 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (to))
5515 : to = from;
5516 : }
5517 :
5518 171390206 : set_and_exit:
5519 203778023 : if (dump_file && (dump_flags & TDF_DETAILS))
5520 : {
5521 397602 : fprintf (dump_file, "Setting value number of ");
5522 397602 : print_generic_expr (dump_file, from);
5523 397602 : fprintf (dump_file, " to ");
5524 397602 : print_generic_expr (dump_file, to);
5525 : }
5526 :
5527 203778023 : if (currval != to
5528 166040185 : && !operand_equal_p (currval, to, 0)
5529 : /* Different undefined SSA names are not actually different. See
5530 : PR82320 for a testcase were we'd otherwise not terminate iteration. */
5531 165971270 : && !(curr_undefined
5532 3385 : && TREE_CODE (to) == SSA_NAME
5533 574 : && !virtual_operand_p (to)
5534 574 : && ssa_undefined_value_p (to, false))
5535 : /* ??? For addresses involving volatile objects or types operand_equal_p
5536 : does not reliably detect ADDR_EXPRs as equal. We know we are only
5537 : getting invariant gimple addresses here, so can use
5538 : get_addr_base_and_unit_offset to do this comparison. */
5539 369748687 : && !(TREE_CODE (currval) == ADDR_EXPR
5540 455933 : && TREE_CODE (to) == ADDR_EXPR
5541 12 : && (get_addr_base_and_unit_offset (TREE_OPERAND (currval, 0), &coff)
5542 6 : == get_addr_base_and_unit_offset (TREE_OPERAND (to, 0), &toff))
5543 6 : && known_eq (coff, toff)))
5544 : {
5545 165970658 : if (to != from
5546 28040268 : && currval != VN_TOP
5547 998983 : && !curr_undefined
5548 : /* We do not want to allow lattice transitions from one value
5549 : to another since that may lead to not terminating iteration
5550 : (see PR95049). Since there's no convenient way to check
5551 : for the allowed transition of VAL -> PHI (loop entry value,
5552 : same on two PHIs, to same PHI result) we restrict the check
5553 : to invariants. */
5554 998983 : && curr_invariant
5555 166629142 : && is_gimple_min_invariant (to))
5556 : {
5557 0 : if (dump_file && (dump_flags & TDF_DETAILS))
5558 0 : fprintf (dump_file, " forced VARYING");
5559 : to = from;
5560 : }
5561 165970658 : if (dump_file && (dump_flags & TDF_DETAILS))
5562 397286 : fprintf (dump_file, " (changed)\n");
5563 165970658 : from_info->valnum = to;
5564 165970658 : return true;
5565 : }
5566 37807365 : if (dump_file && (dump_flags & TDF_DETAILS))
5567 316 : fprintf (dump_file, "\n");
5568 : return false;
5569 : }
5570 :
5571 : /* Set all definitions in STMT to value number to themselves.
5572 : Return true if a value number changed. */
5573 :
5574 : static bool
5575 287600662 : defs_to_varying (gimple *stmt)
5576 : {
5577 287600662 : bool changed = false;
5578 287600662 : ssa_op_iter iter;
5579 287600662 : def_operand_p defp;
5580 :
5581 317010846 : FOR_EACH_SSA_DEF_OPERAND (defp, stmt, iter, SSA_OP_ALL_DEFS)
5582 : {
5583 29410184 : tree def = DEF_FROM_PTR (defp);
5584 29410184 : changed |= set_ssa_val_to (def, def);
5585 : }
5586 287600662 : return changed;
5587 : }
5588 :
5589 : /* Visit a copy between LHS and RHS, return true if the value number
5590 : changed. */
5591 :
5592 : static bool
5593 7951190 : visit_copy (tree lhs, tree rhs)
5594 : {
5595 : /* Valueize. */
5596 7951190 : rhs = SSA_VAL (rhs);
5597 :
5598 7951190 : return set_ssa_val_to (lhs, rhs);
5599 : }
5600 :
5601 : /* Lookup a value for OP in type WIDE_TYPE where the value in type of OP
5602 : is the same. */
5603 :
5604 : static tree
5605 2444673 : valueized_wider_op (tree wide_type, tree op, bool allow_truncate)
5606 : {
5607 2444673 : if (TREE_CODE (op) == SSA_NAME)
5608 2147076 : op = vn_valueize (op);
5609 :
5610 : /* Either we have the op widened available. */
5611 2444673 : tree ops[3] = {};
5612 2444673 : ops[0] = op;
5613 2444673 : tree tem = vn_nary_op_lookup_pieces (1, NOP_EXPR,
5614 : wide_type, ops, NULL);
5615 2444673 : if (tem)
5616 : return tem;
5617 :
5618 : /* Or the op is truncated from some existing value. */
5619 2158591 : if (allow_truncate && TREE_CODE (op) == SSA_NAME)
5620 : {
5621 540634 : gimple *def = SSA_NAME_DEF_STMT (op);
5622 540634 : if (is_gimple_assign (def)
5623 540634 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
5624 : {
5625 290017 : tem = gimple_assign_rhs1 (def);
5626 290017 : if (useless_type_conversion_p (wide_type, TREE_TYPE (tem)))
5627 : {
5628 197988 : if (TREE_CODE (tem) == SSA_NAME)
5629 197988 : tem = vn_valueize (tem);
5630 197988 : return tem;
5631 : }
5632 : }
5633 : }
5634 :
5635 : /* For constants simply extend it. */
5636 1960603 : if (TREE_CODE (op) == INTEGER_CST)
5637 330716 : return wide_int_to_tree (wide_type, wi::to_widest (op));
5638 :
5639 : return NULL_TREE;
5640 : }
5641 :
5642 : /* Visit a nary operator RHS, value number it, and return true if the
5643 : value number of LHS has changed as a result. */
5644 :
5645 : static bool
5646 48565808 : visit_nary_op (tree lhs, gassign *stmt)
5647 : {
5648 48565808 : vn_nary_op_t vnresult;
5649 48565808 : tree result = vn_nary_op_lookup_stmt (stmt, &vnresult);
5650 48565808 : if (! result && vnresult)
5651 153963 : result = vn_nary_op_get_predicated_value (vnresult, gimple_bb (stmt));
5652 44940082 : if (result)
5653 3695517 : return set_ssa_val_to (lhs, result);
5654 :
5655 : /* Do some special pattern matching for redundancies of operations
5656 : in different types. */
5657 44870291 : enum tree_code code = gimple_assign_rhs_code (stmt);
5658 44870291 : tree type = TREE_TYPE (lhs);
5659 44870291 : tree rhs1 = gimple_assign_rhs1 (stmt);
5660 44870291 : switch (code)
5661 : {
5662 9974031 : CASE_CONVERT:
5663 : /* Match arithmetic done in a different type where we can easily
5664 : substitute the result from some earlier sign-changed or widened
5665 : operation. */
5666 9974031 : if (INTEGRAL_TYPE_P (type)
5667 8931254 : && TREE_CODE (rhs1) == SSA_NAME
5668 : /* We only handle sign-changes, zero-extension -> & mask or
5669 : sign-extension if we know the inner operation doesn't
5670 : overflow. */
5671 18669877 : && (((TYPE_UNSIGNED (TREE_TYPE (rhs1))
5672 5263103 : || (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
5673 5262305 : && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (rhs1))))
5674 7967223 : && TYPE_PRECISION (type) > TYPE_PRECISION (TREE_TYPE (rhs1)))
5675 5907111 : || TYPE_PRECISION (type) == TYPE_PRECISION (TREE_TYPE (rhs1))))
5676 : {
5677 7621865 : gassign *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (rhs1));
5678 5486112 : if (def
5679 5486112 : && (gimple_assign_rhs_code (def) == PLUS_EXPR
5680 4264460 : || gimple_assign_rhs_code (def) == MINUS_EXPR
5681 4122455 : || gimple_assign_rhs_code (def) == MULT_EXPR))
5682 : {
5683 1969973 : tree ops[3] = {};
5684 : /* When requiring a sign-extension we cannot model a
5685 : previous truncation with a single op so don't bother. */
5686 1969973 : bool allow_truncate = TYPE_UNSIGNED (TREE_TYPE (rhs1));
5687 : /* Either we have the op widened available. */
5688 1969973 : ops[0] = valueized_wider_op (type, gimple_assign_rhs1 (def),
5689 : allow_truncate);
5690 1969973 : if (ops[0])
5691 949400 : ops[1] = valueized_wider_op (type, gimple_assign_rhs2 (def),
5692 : allow_truncate);
5693 1969973 : if (ops[0] && ops[1])
5694 : {
5695 340086 : ops[0] = vn_nary_op_lookup_pieces
5696 340086 : (2, gimple_assign_rhs_code (def), type, ops, NULL);
5697 : /* We have wider operation available. */
5698 340086 : if (ops[0]
5699 : /* If the leader is a wrapping operation we can
5700 : insert it for code hoisting w/o introducing
5701 : undefined overflow. If it is not it has to
5702 : be available. See PR86554. */
5703 340086 : && (TYPE_OVERFLOW_WRAPS (TREE_TYPE (ops[0]))
5704 1846 : || (rpo_avail && vn_context_bb
5705 1846 : && rpo_avail->eliminate_avail (vn_context_bb,
5706 : ops[0]))))
5707 : {
5708 9595 : unsigned lhs_prec = TYPE_PRECISION (type);
5709 9595 : unsigned rhs_prec = TYPE_PRECISION (TREE_TYPE (rhs1));
5710 9595 : if (lhs_prec == rhs_prec
5711 9595 : || (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
5712 1767 : && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (rhs1))))
5713 : {
5714 9005 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5715 9005 : NOP_EXPR, type, ops[0]);
5716 9005 : result = vn_nary_build_or_lookup (&match_op);
5717 9005 : if (result)
5718 : {
5719 9005 : bool changed = set_ssa_val_to (lhs, result);
5720 9005 : if (TREE_CODE (result) == SSA_NAME)
5721 9005 : vn_nary_op_insert_stmt (stmt, result);
5722 9005 : return changed;
5723 : }
5724 : }
5725 : else
5726 : {
5727 590 : tree mask = wide_int_to_tree
5728 590 : (type, wi::mask (rhs_prec, false, lhs_prec));
5729 590 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5730 590 : BIT_AND_EXPR,
5731 590 : TREE_TYPE (lhs),
5732 590 : ops[0], mask);
5733 590 : result = vn_nary_build_or_lookup (&match_op);
5734 590 : if (result)
5735 : {
5736 590 : bool changed = set_ssa_val_to (lhs, result);
5737 590 : if (TREE_CODE (result) == SSA_NAME)
5738 590 : vn_nary_op_insert_stmt (stmt, result);
5739 590 : return changed;
5740 : }
5741 : }
5742 : }
5743 : }
5744 : }
5745 : }
5746 : break;
5747 1481429 : case BIT_AND_EXPR:
5748 1481429 : if (INTEGRAL_TYPE_P (type)
5749 1443321 : && TREE_CODE (rhs1) == SSA_NAME
5750 1443321 : && TREE_CODE (gimple_assign_rhs2 (stmt)) == INTEGER_CST
5751 891646 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)
5752 891528 : && default_vn_walk_kind != VN_NOWALK
5753 : && CHAR_BIT == 8
5754 : && BITS_PER_UNIT == 8
5755 : && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
5756 891319 : && TYPE_PRECISION (type) <= vn_walk_cb_data::bufsize * BITS_PER_UNIT
5757 891317 : && !integer_all_onesp (gimple_assign_rhs2 (stmt))
5758 2372746 : && !integer_zerop (gimple_assign_rhs2 (stmt)))
5759 : {
5760 891317 : gassign *ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (rhs1));
5761 654780 : if (ass
5762 654780 : && !gimple_has_volatile_ops (ass)
5763 653318 : && vn_get_stmt_kind (ass) == VN_REFERENCE)
5764 : {
5765 300567 : tree last_vuse = gimple_vuse (ass);
5766 300567 : tree op = gimple_assign_rhs1 (ass);
5767 901701 : tree result = vn_reference_lookup (op, gimple_vuse (ass),
5768 : default_vn_walk_kind,
5769 : NULL, true, &last_vuse,
5770 : gimple_assign_rhs2 (stmt));
5771 300567 : if (result
5772 301020 : && useless_type_conversion_p (TREE_TYPE (result),
5773 453 : TREE_TYPE (op)))
5774 453 : return set_ssa_val_to (lhs, result);
5775 : }
5776 : }
5777 : break;
5778 247501 : case BIT_FIELD_REF:
5779 247501 : if (TREE_CODE (TREE_OPERAND (rhs1, 0)) == SSA_NAME)
5780 : {
5781 247481 : tree op0 = TREE_OPERAND (rhs1, 0);
5782 247481 : gassign *ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op0));
5783 205204 : if (ass
5784 205204 : && !gimple_has_volatile_ops (ass)
5785 205121 : && vn_get_stmt_kind (ass) == VN_REFERENCE)
5786 : {
5787 91475 : tree last_vuse = gimple_vuse (ass);
5788 91475 : tree op = gimple_assign_rhs1 (ass);
5789 : /* Avoid building invalid and unexpected refs. */
5790 91475 : if (TREE_CODE (op) != TARGET_MEM_REF
5791 : && TREE_CODE (op) != BIT_FIELD_REF
5792 : && TREE_CODE (op) != REALPART_EXPR
5793 : && TREE_CODE (op) != IMAGPART_EXPR)
5794 : {
5795 84005 : tree op = build3 (BIT_FIELD_REF, TREE_TYPE (rhs1),
5796 : gimple_assign_rhs1 (ass),
5797 84005 : TREE_OPERAND (rhs1, 1),
5798 84005 : TREE_OPERAND (rhs1, 2));
5799 168010 : tree result = vn_reference_lookup (op, gimple_vuse (ass),
5800 : default_vn_walk_kind,
5801 : NULL, true, &last_vuse);
5802 84005 : if (result
5803 84005 : && useless_type_conversion_p (type, TREE_TYPE (result)))
5804 1529 : return set_ssa_val_to (lhs, result);
5805 82765 : else if (result
5806 289 : && TYPE_SIZE (type)
5807 289 : && TYPE_SIZE (TREE_TYPE (result))
5808 83054 : && operand_equal_p (TYPE_SIZE (type),
5809 289 : TYPE_SIZE (TREE_TYPE (result))))
5810 : {
5811 289 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5812 289 : VIEW_CONVERT_EXPR,
5813 289 : type, result);
5814 289 : result = vn_nary_build_or_lookup (&match_op);
5815 289 : if (result)
5816 : {
5817 289 : bool changed = set_ssa_val_to (lhs, result);
5818 289 : if (TREE_CODE (result) == SSA_NAME)
5819 277 : vn_nary_op_insert_stmt (stmt, result);
5820 289 : return changed;
5821 : }
5822 : }
5823 : }
5824 : }
5825 : }
5826 : break;
5827 321736 : case TRUNC_DIV_EXPR:
5828 321736 : if (TYPE_UNSIGNED (type))
5829 : break;
5830 : /* Fallthru. */
5831 5439748 : case RDIV_EXPR:
5832 5439748 : case MULT_EXPR:
5833 : /* Match up ([-]a){/,*}([-])b with v=a{/,*}b, replacing it with -v. */
5834 5439748 : if (! HONOR_SIGN_DEPENDENT_ROUNDING (type))
5835 : {
5836 5438838 : tree rhs[2];
5837 5438838 : rhs[0] = rhs1;
5838 5438838 : rhs[1] = gimple_assign_rhs2 (stmt);
5839 16309581 : for (unsigned i = 0; i <= 1; ++i)
5840 : {
5841 10876523 : unsigned j = i == 0 ? 1 : 0;
5842 10876523 : tree ops[2];
5843 10876523 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5844 10876523 : NEGATE_EXPR, type, rhs[i]);
5845 10876523 : ops[i] = vn_nary_build_or_lookup_1 (&match_op, false, true);
5846 10876523 : ops[j] = rhs[j];
5847 10876523 : if (ops[i]
5848 10876523 : && (ops[0] = vn_nary_op_lookup_pieces (2, code,
5849 : type, ops, NULL)))
5850 : {
5851 5780 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5852 5780 : NEGATE_EXPR, type, ops[0]);
5853 5780 : result = vn_nary_build_or_lookup_1 (&match_op, true, false);
5854 5780 : if (result)
5855 : {
5856 5780 : bool changed = set_ssa_val_to (lhs, result);
5857 5780 : if (TREE_CODE (result) == SSA_NAME)
5858 5780 : vn_nary_op_insert_stmt (stmt, result);
5859 5780 : return changed;
5860 : }
5861 : }
5862 : }
5863 : }
5864 : break;
5865 365917 : case LSHIFT_EXPR:
5866 : /* For X << C, use the value number of X * (1 << C). */
5867 365917 : if (INTEGRAL_TYPE_P (type)
5868 350979 : && TYPE_OVERFLOW_WRAPS (type)
5869 553197 : && !TYPE_SATURATING (type))
5870 : {
5871 187280 : tree rhs2 = gimple_assign_rhs2 (stmt);
5872 187280 : if (TREE_CODE (rhs2) == INTEGER_CST
5873 108472 : && tree_fits_uhwi_p (rhs2)
5874 295752 : && tree_to_uhwi (rhs2) < TYPE_PRECISION (type))
5875 : {
5876 108472 : wide_int w = wi::set_bit_in_zero (tree_to_uhwi (rhs2),
5877 108472 : TYPE_PRECISION (type));
5878 216944 : gimple_match_op match_op (gimple_match_cond::UNCOND,
5879 108472 : MULT_EXPR, type, rhs1,
5880 108472 : wide_int_to_tree (type, w));
5881 108472 : result = vn_nary_build_or_lookup (&match_op);
5882 108472 : if (result)
5883 : {
5884 108472 : bool changed = set_ssa_val_to (lhs, result);
5885 108472 : if (TREE_CODE (result) == SSA_NAME)
5886 108471 : vn_nary_op_insert_stmt (stmt, result);
5887 108472 : return changed;
5888 : }
5889 108472 : }
5890 : }
5891 : break;
5892 : default:
5893 : break;
5894 : }
5895 :
5896 44744462 : bool changed = set_ssa_val_to (lhs, lhs);
5897 44744462 : vn_nary_op_insert_stmt (stmt, lhs);
5898 44744462 : return changed;
5899 : }
5900 :
5901 : /* Visit a call STMT storing into LHS. Return true if the value number
5902 : of the LHS has changed as a result. */
5903 :
5904 : static bool
5905 8629207 : visit_reference_op_call (tree lhs, gcall *stmt)
5906 : {
5907 8629207 : bool changed = false;
5908 8629207 : struct vn_reference_s vr1;
5909 8629207 : vn_reference_t vnresult = NULL;
5910 8629207 : tree vdef = gimple_vdef (stmt);
5911 8629207 : modref_summary *summary;
5912 :
5913 : /* Non-ssa lhs is handled in copy_reference_ops_from_call. */
5914 8629207 : if (lhs && TREE_CODE (lhs) != SSA_NAME)
5915 4578343 : lhs = NULL_TREE;
5916 :
5917 8629207 : vn_reference_lookup_call (stmt, &vnresult, &vr1);
5918 :
5919 : /* If the lookup did not succeed for pure functions try to use
5920 : modref info to find a candidate to CSE to. */
5921 8629207 : const unsigned accesses_limit = 8;
5922 8629207 : if (!vnresult
5923 7975089 : && !vdef
5924 7975089 : && lhs
5925 2822002 : && gimple_vuse (stmt)
5926 10190740 : && (((summary = get_modref_function_summary (stmt, NULL))
5927 213000 : && !summary->global_memory_read
5928 83631 : && summary->load_accesses < accesses_limit)
5929 1478246 : || gimple_call_flags (stmt) & ECF_CONST))
5930 : {
5931 : /* First search if we can do someting useful and build a
5932 : vector of all loads we have to check. */
5933 84020 : bool unknown_memory_access = false;
5934 84020 : auto_vec<ao_ref, accesses_limit> accesses;
5935 84020 : unsigned load_accesses = summary ? summary->load_accesses : 0;
5936 84020 : if (!unknown_memory_access)
5937 : /* Add loads done as part of setting up the call arguments.
5938 : That's also necessary for CONST functions which will
5939 : not have a modref summary. */
5940 241919 : for (unsigned i = 0; i < gimple_call_num_args (stmt); ++i)
5941 : {
5942 157907 : tree arg = gimple_call_arg (stmt, i);
5943 157907 : if (TREE_CODE (arg) != SSA_NAME
5944 157907 : && !is_gimple_min_invariant (arg))
5945 : {
5946 61282 : if (accesses.length () >= accesses_limit - load_accesses)
5947 : {
5948 : unknown_memory_access = true;
5949 : break;
5950 : }
5951 30633 : accesses.quick_grow (accesses.length () + 1);
5952 30633 : ao_ref_init (&accesses.last (), arg);
5953 : }
5954 : }
5955 84020 : if (summary && !unknown_memory_access)
5956 : {
5957 : /* Add loads as analyzed by IPA modref. */
5958 287450 : for (auto base_node : summary->loads->bases)
5959 71995 : if (unknown_memory_access)
5960 : break;
5961 293435 : else for (auto ref_node : base_node->refs)
5962 78302 : if (unknown_memory_access)
5963 : break;
5964 325028 : else for (auto access_node : ref_node->accesses)
5965 : {
5966 212768 : accesses.quick_grow (accesses.length () + 1);
5967 106384 : ao_ref *r = &accesses.last ();
5968 106384 : if (!access_node.get_ao_ref (stmt, r))
5969 : {
5970 : /* Initialize a ref based on the argument and
5971 : unknown offset if possible. */
5972 16226 : tree arg = access_node.get_call_arg (stmt);
5973 16226 : if (arg && TREE_CODE (arg) == SSA_NAME)
5974 2766 : arg = SSA_VAL (arg);
5975 2766 : if (arg
5976 16216 : && TREE_CODE (arg) == ADDR_EXPR
5977 13456 : && (arg = get_base_address (arg))
5978 16222 : && DECL_P (arg))
5979 : {
5980 0 : ao_ref_init (r, arg);
5981 0 : r->ref = NULL_TREE;
5982 0 : r->base = arg;
5983 : }
5984 : else
5985 : {
5986 : unknown_memory_access = true;
5987 : break;
5988 : }
5989 : }
5990 90158 : r->base_alias_set = base_node->base;
5991 90158 : r->ref_alias_set = ref_node->ref;
5992 : }
5993 : }
5994 :
5995 : /* Walk the VUSE->VDEF chain optimistically trying to find an entry
5996 : for the call in the hashtable. */
5997 84020 : unsigned limit = (unknown_memory_access
5998 84020 : ? 0
5999 67786 : : (param_sccvn_max_alias_queries_per_access
6000 67786 : / (accesses.length () + 1)));
6001 84020 : tree saved_vuse = vr1.vuse;
6002 84020 : hashval_t saved_hashcode = vr1.hashcode;
6003 422408 : while (limit > 0 && !vnresult && !SSA_NAME_IS_DEFAULT_DEF (vr1.vuse))
6004 : {
6005 361243 : vr1.hashcode = vr1.hashcode - SSA_NAME_VERSION (vr1.vuse);
6006 361243 : gimple *def = SSA_NAME_DEF_STMT (vr1.vuse);
6007 : /* ??? We could use fancy stuff like in walk_non_aliased_vuses, but
6008 : do not bother for now. */
6009 361243 : if (is_a <gphi *> (def))
6010 : break;
6011 676776 : vr1.vuse = vuse_ssa_val (gimple_vuse (def));
6012 338388 : vr1.hashcode = vr1.hashcode + SSA_NAME_VERSION (vr1.vuse);
6013 338388 : vn_reference_lookup_1 (&vr1, &vnresult);
6014 338388 : limit--;
6015 : }
6016 :
6017 : /* If we found a candidate to CSE to verify it is valid. */
6018 84020 : if (vnresult && !accesses.is_empty ())
6019 : {
6020 1920 : tree vuse = vuse_ssa_val (gimple_vuse (stmt));
6021 7148 : while (vnresult && vuse != vr1.vuse)
6022 : {
6023 3308 : gimple *def = SSA_NAME_DEF_STMT (vuse);
6024 17407 : for (auto &ref : accesses)
6025 : {
6026 : /* ??? stmt_may_clobber_ref_p_1 does per stmt constant
6027 : analysis overhead that we might be able to cache. */
6028 9224 : if (stmt_may_clobber_ref_p_1 (def, &ref, true))
6029 : {
6030 1741 : vnresult = NULL;
6031 1741 : break;
6032 : }
6033 : }
6034 6616 : vuse = vuse_ssa_val (gimple_vuse (def));
6035 : }
6036 : }
6037 84020 : vr1.vuse = saved_vuse;
6038 84020 : vr1.hashcode = saved_hashcode;
6039 84020 : }
6040 :
6041 8629207 : if (vnresult)
6042 : {
6043 654325 : if (vdef)
6044 : {
6045 172508 : if (vnresult->result_vdef)
6046 172508 : changed |= set_ssa_val_to (vdef, vnresult->result_vdef);
6047 0 : else if (!lhs && gimple_call_lhs (stmt))
6048 : /* If stmt has non-SSA_NAME lhs, value number the vdef to itself,
6049 : as the call still acts as a lhs store. */
6050 0 : changed |= set_ssa_val_to (vdef, vdef);
6051 : else
6052 : /* If the call was discovered to be pure or const reflect
6053 : that as far as possible. */
6054 0 : changed |= set_ssa_val_to (vdef,
6055 : vuse_ssa_val (gimple_vuse (stmt)));
6056 : }
6057 :
6058 654325 : if (!vnresult->result && lhs)
6059 0 : vnresult->result = lhs;
6060 :
6061 654325 : if (vnresult->result && lhs)
6062 108235 : changed |= set_ssa_val_to (lhs, vnresult->result);
6063 : }
6064 : else
6065 : {
6066 7974882 : vn_reference_t vr2;
6067 7974882 : vn_reference_s **slot;
6068 7974882 : tree vdef_val = vdef;
6069 7974882 : if (vdef)
6070 : {
6071 : /* If we value numbered an indirect functions function to
6072 : one not clobbering memory value number its VDEF to its
6073 : VUSE. */
6074 4833289 : tree fn = gimple_call_fn (stmt);
6075 4833289 : if (fn && TREE_CODE (fn) == SSA_NAME)
6076 : {
6077 128427 : fn = SSA_VAL (fn);
6078 128427 : if (TREE_CODE (fn) == ADDR_EXPR
6079 1717 : && TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL
6080 1717 : && (flags_from_decl_or_type (TREE_OPERAND (fn, 0))
6081 1717 : & (ECF_CONST | ECF_PURE))
6082 : /* If stmt has non-SSA_NAME lhs, value number the
6083 : vdef to itself, as the call still acts as a lhs
6084 : store. */
6085 129550 : && (lhs || gimple_call_lhs (stmt) == NULL_TREE))
6086 2114 : vdef_val = vuse_ssa_val (gimple_vuse (stmt));
6087 : }
6088 4833289 : changed |= set_ssa_val_to (vdef, vdef_val);
6089 : }
6090 7974882 : if (lhs)
6091 3942629 : changed |= set_ssa_val_to (lhs, lhs);
6092 7974882 : vr2 = XOBNEW (&vn_tables_obstack, vn_reference_s);
6093 7974882 : vr2->vuse = vr1.vuse;
6094 : /* As we are not walking the virtual operand chain we know the
6095 : shared_lookup_references are still original so we can re-use
6096 : them here. */
6097 7974882 : vr2->operands = vr1.operands.copy ();
6098 7974882 : vr2->type = vr1.type;
6099 7974882 : vr2->punned = vr1.punned;
6100 7974882 : vr2->set = vr1.set;
6101 7974882 : vr2->offset = vr1.offset;
6102 7974882 : vr2->max_size = vr1.max_size;
6103 7974882 : vr2->base_set = vr1.base_set;
6104 7974882 : vr2->hashcode = vr1.hashcode;
6105 7974882 : vr2->result = lhs;
6106 7974882 : vr2->result_vdef = vdef_val;
6107 7974882 : vr2->value_id = 0;
6108 7974882 : slot = valid_info->references->find_slot_with_hash (vr2, vr2->hashcode,
6109 : INSERT);
6110 7974882 : gcc_assert (!*slot);
6111 7974882 : *slot = vr2;
6112 7974882 : vr2->next = last_inserted_ref;
6113 7974882 : last_inserted_ref = vr2;
6114 : }
6115 :
6116 8629207 : return changed;
6117 : }
6118 :
6119 : /* Visit a load from a reference operator RHS, part of STMT, value number it,
6120 : and return true if the value number of the LHS has changed as a result. */
6121 :
6122 : static bool
6123 34289529 : visit_reference_op_load (tree lhs, tree op, gimple *stmt)
6124 : {
6125 34289529 : bool changed = false;
6126 34289529 : tree result;
6127 34289529 : vn_reference_t res;
6128 :
6129 34289529 : tree vuse = gimple_vuse (stmt);
6130 34289529 : tree last_vuse = vuse;
6131 34289529 : result = vn_reference_lookup (op, vuse, default_vn_walk_kind, &res, true, &last_vuse);
6132 :
6133 : /* We handle type-punning through unions by value-numbering based
6134 : on offset and size of the access. Be prepared to handle a
6135 : type-mismatch here via creating a VIEW_CONVERT_EXPR. */
6136 34289529 : if (result
6137 34289529 : && !useless_type_conversion_p (TREE_TYPE (result), TREE_TYPE (op)))
6138 : {
6139 18025 : if (CONSTANT_CLASS_P (result))
6140 4157 : result = const_unop (VIEW_CONVERT_EXPR, TREE_TYPE (op), result);
6141 : else
6142 : {
6143 : /* We will be setting the value number of lhs to the value number
6144 : of VIEW_CONVERT_EXPR <TREE_TYPE (result)> (result).
6145 : So first simplify and lookup this expression to see if it
6146 : is already available. */
6147 13868 : gimple_match_op res_op (gimple_match_cond::UNCOND,
6148 13868 : VIEW_CONVERT_EXPR, TREE_TYPE (op), result);
6149 13868 : result = vn_nary_build_or_lookup (&res_op);
6150 13868 : if (result
6151 13862 : && TREE_CODE (result) == SSA_NAME
6152 26044 : && VN_INFO (result)->needs_insertion)
6153 : /* Track whether this is the canonical expression for different
6154 : typed loads. We use that as a stopgap measure for code
6155 : hoisting when dealing with floating point loads. */
6156 10938 : res->punned = true;
6157 : }
6158 :
6159 : /* When building the conversion fails avoid inserting the reference
6160 : again. */
6161 18025 : if (!result)
6162 6 : return set_ssa_val_to (lhs, lhs);
6163 : }
6164 :
6165 34271504 : if (result)
6166 5499578 : changed = set_ssa_val_to (lhs, result);
6167 : else
6168 : {
6169 28789945 : changed = set_ssa_val_to (lhs, lhs);
6170 28789945 : vn_reference_insert (op, lhs, last_vuse, NULL_TREE);
6171 28789945 : if (vuse && SSA_VAL (last_vuse) != SSA_VAL (vuse))
6172 : {
6173 8820536 : if (dump_file && (dump_flags & TDF_DETAILS))
6174 : {
6175 23131 : fprintf (dump_file, "Using extra use virtual operand ");
6176 23131 : print_generic_expr (dump_file, last_vuse);
6177 23131 : fprintf (dump_file, "\n");
6178 : }
6179 8820536 : vn_reference_insert (op, lhs, vuse, NULL_TREE);
6180 : }
6181 : }
6182 :
6183 : return changed;
6184 : }
6185 :
6186 :
6187 : /* Visit a store to a reference operator LHS, part of STMT, value number it,
6188 : and return true if the value number of the LHS has changed as a result. */
6189 :
6190 : static bool
6191 32775453 : visit_reference_op_store (tree lhs, tree op, gimple *stmt)
6192 : {
6193 32775453 : bool changed = false;
6194 32775453 : vn_reference_t vnresult = NULL;
6195 32775453 : tree assign;
6196 32775453 : bool resultsame = false;
6197 32775453 : tree vuse = gimple_vuse (stmt);
6198 32775453 : tree vdef = gimple_vdef (stmt);
6199 :
6200 32775453 : if (TREE_CODE (op) == SSA_NAME)
6201 14888238 : op = SSA_VAL (op);
6202 :
6203 : /* First we want to lookup using the *vuses* from the store and see
6204 : if there the last store to this location with the same address
6205 : had the same value.
6206 :
6207 : The vuses represent the memory state before the store. If the
6208 : memory state, address, and value of the store is the same as the
6209 : last store to this location, then this store will produce the
6210 : same memory state as that store.
6211 :
6212 : In this case the vdef versions for this store are value numbered to those
6213 : vuse versions, since they represent the same memory state after
6214 : this store.
6215 :
6216 : Otherwise, the vdefs for the store are used when inserting into
6217 : the table, since the store generates a new memory state. */
6218 :
6219 32775453 : vn_reference_lookup (lhs, vuse, VN_NOWALK, &vnresult, false);
6220 32775453 : if (vnresult
6221 1668243 : && vnresult->result)
6222 : {
6223 1668243 : tree result = vnresult->result;
6224 1668243 : gcc_checking_assert (TREE_CODE (result) != SSA_NAME
6225 : || result == SSA_VAL (result));
6226 1668243 : resultsame = expressions_equal_p (result, op);
6227 1668243 : if (resultsame)
6228 : {
6229 : /* If the TBAA state isn't compatible for downstream reads
6230 : we cannot value-number the VDEFs the same. */
6231 51746 : ao_ref lhs_ref;
6232 51746 : ao_ref_init (&lhs_ref, lhs);
6233 51746 : alias_set_type set = ao_ref_alias_set (&lhs_ref);
6234 51746 : alias_set_type base_set = ao_ref_base_alias_set (&lhs_ref);
6235 51746 : if ((vnresult->set != set
6236 910 : && ! alias_set_subset_of (set, vnresult->set))
6237 52340 : || (vnresult->base_set != base_set
6238 6344 : && ! alias_set_subset_of (base_set, vnresult->base_set)))
6239 844 : resultsame = false;
6240 : }
6241 : }
6242 :
6243 844 : if (!resultsame)
6244 : {
6245 32724551 : if (dump_file && (dump_flags & TDF_DETAILS))
6246 : {
6247 20435 : fprintf (dump_file, "No store match\n");
6248 20435 : fprintf (dump_file, "Value numbering store ");
6249 20435 : print_generic_expr (dump_file, lhs);
6250 20435 : fprintf (dump_file, " to ");
6251 20435 : print_generic_expr (dump_file, op);
6252 20435 : fprintf (dump_file, "\n");
6253 : }
6254 : /* Have to set value numbers before insert, since insert is
6255 : going to valueize the references in-place. */
6256 32724551 : if (vdef)
6257 32724551 : changed |= set_ssa_val_to (vdef, vdef);
6258 :
6259 : /* Do not insert structure copies into the tables. */
6260 32724551 : if (is_gimple_min_invariant (op)
6261 32724551 : || is_gimple_reg (op))
6262 29149937 : vn_reference_insert (lhs, op, vdef, NULL);
6263 :
6264 : /* Only perform the following when being called from PRE
6265 : which embeds tail merging. */
6266 32724551 : if (default_vn_walk_kind == VN_WALK)
6267 : {
6268 7454255 : assign = build2 (MODIFY_EXPR, TREE_TYPE (lhs), lhs, op);
6269 7454255 : vn_reference_lookup (assign, vuse, VN_NOWALK, &vnresult, false);
6270 7454255 : if (!vnresult)
6271 7413433 : vn_reference_insert (assign, lhs, vuse, vdef);
6272 : }
6273 : }
6274 : else
6275 : {
6276 : /* We had a match, so value number the vdef to have the value
6277 : number of the vuse it came from. */
6278 :
6279 50902 : if (dump_file && (dump_flags & TDF_DETAILS))
6280 9 : fprintf (dump_file, "Store matched earlier value, "
6281 : "value numbering store vdefs to matching vuses.\n");
6282 :
6283 50902 : changed |= set_ssa_val_to (vdef, SSA_VAL (vuse));
6284 : }
6285 :
6286 32775453 : return changed;
6287 : }
6288 :
6289 : /* Visit and value number PHI, return true if the value number
6290 : changed. When BACKEDGES_VARYING_P is true then assume all
6291 : backedge values are varying. When INSERTED is not NULL then
6292 : this is just a ahead query for a possible iteration, set INSERTED
6293 : to true if we'd insert into the hashtable. */
6294 :
6295 : static bool
6296 34154119 : visit_phi (gimple *phi, bool *inserted, bool backedges_varying_p)
6297 : {
6298 34154119 : tree result, sameval = VN_TOP, seen_undef = NULL_TREE;
6299 34154119 : bool seen_undef_visited = false;
6300 34154119 : tree backedge_val = NULL_TREE;
6301 34154119 : bool seen_non_backedge = false;
6302 34154119 : tree sameval_base = NULL_TREE;
6303 34154119 : poly_int64 soff, doff;
6304 34154119 : unsigned n_executable = 0;
6305 34154119 : edge sameval_e = NULL;
6306 :
6307 : /* TODO: We could check for this in initialization, and replace this
6308 : with a gcc_assert. */
6309 34154119 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)))
6310 30809 : return set_ssa_val_to (PHI_RESULT (phi), PHI_RESULT (phi));
6311 :
6312 : /* We track whether a PHI was CSEd to avoid excessive iterations
6313 : that would be necessary only because the PHI changed arguments
6314 : but not value. */
6315 34123310 : if (!inserted)
6316 26593776 : gimple_set_plf (phi, GF_PLF_1, false);
6317 :
6318 34123310 : basic_block bb = gimple_bb (phi);
6319 :
6320 : /* For the equivalence handling below make sure to first process an
6321 : edge with a non-constant. */
6322 34123310 : auto_vec<edge, 2> preds;
6323 68246620 : preds.reserve_exact (EDGE_COUNT (bb->preds));
6324 34123310 : bool seen_nonconstant = false;
6325 112764143 : for (unsigned i = 0; i < EDGE_COUNT (bb->preds); ++i)
6326 : {
6327 78640833 : edge e = EDGE_PRED (bb, i);
6328 78640833 : preds.quick_push (e);
6329 78640833 : if (!seen_nonconstant)
6330 : {
6331 41725957 : tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
6332 41725957 : if (TREE_CODE (def) == SSA_NAME)
6333 : {
6334 32429227 : seen_nonconstant = true;
6335 32429227 : if (i != 0)
6336 5709613 : std::swap (preds[0], preds[i]);
6337 : }
6338 : }
6339 : }
6340 :
6341 : /* See if all non-TOP arguments have the same value. TOP is
6342 : equivalent to everything, so we can ignore it. */
6343 143643724 : for (edge e : preds)
6344 67832082 : if (e->flags & EDGE_EXECUTABLE)
6345 : {
6346 62822588 : tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
6347 :
6348 62822588 : if (def == PHI_RESULT (phi))
6349 322009 : continue;
6350 62524959 : ++n_executable;
6351 62524959 : bool visited = true;
6352 62524959 : if (TREE_CODE (def) == SSA_NAME)
6353 : {
6354 50461972 : tree val = SSA_VAL (def, &visited);
6355 50461972 : if (SSA_NAME_IS_DEFAULT_DEF (def))
6356 2662912 : visited = true;
6357 50461972 : if (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK))
6358 47930271 : def = val;
6359 50461972 : if (e->flags & EDGE_DFS_BACK)
6360 15357209 : backedge_val = def;
6361 : }
6362 62524959 : if (!(e->flags & EDGE_DFS_BACK))
6363 47000666 : seen_non_backedge = true;
6364 62524959 : if (def == VN_TOP)
6365 : ;
6366 : /* Ignore undefined defs for sameval but record one. */
6367 62524959 : else if (TREE_CODE (def) == SSA_NAME
6368 47077202 : && ! virtual_operand_p (def)
6369 86537966 : && ssa_undefined_value_p (def, false))
6370 : {
6371 233555 : if (!seen_undef
6372 : /* Avoid having not visited undefined defs if we also have
6373 : a visited one. */
6374 35060 : || (!seen_undef_visited && visited))
6375 : {
6376 198498 : seen_undef = def;
6377 198498 : seen_undef_visited = visited;
6378 : }
6379 : }
6380 62291404 : else if (sameval == VN_TOP)
6381 : {
6382 : sameval = def;
6383 : sameval_e = e;
6384 : }
6385 28215247 : else if (expressions_equal_p (def, sameval))
6386 : sameval_e = NULL;
6387 44407781 : else if (virtual_operand_p (def))
6388 : {
6389 : sameval = NULL_TREE;
6390 26558288 : break;
6391 : }
6392 : else
6393 : {
6394 : /* We know we're arriving only with invariant addresses here,
6395 : try harder comparing them. We can do some caching here
6396 : which we cannot do in expressions_equal_p. */
6397 16605633 : if (TREE_CODE (def) == ADDR_EXPR
6398 370690 : && TREE_CODE (sameval) == ADDR_EXPR
6399 102899 : && sameval_base != (void *)-1)
6400 : {
6401 102899 : if (!sameval_base)
6402 102897 : sameval_base = get_addr_base_and_unit_offset
6403 102897 : (TREE_OPERAND (sameval, 0), &soff);
6404 102897 : if (!sameval_base)
6405 : sameval_base = (tree)(void *)-1;
6406 102908 : else if ((get_addr_base_and_unit_offset
6407 102899 : (TREE_OPERAND (def, 0), &doff) == sameval_base)
6408 102899 : && known_eq (soff, doff))
6409 9 : continue;
6410 : }
6411 : /* There's also the possibility to use equivalences. */
6412 32124094 : if (!FLOAT_TYPE_P (TREE_TYPE (def))
6413 : /* But only do this if we didn't force any of sameval or
6414 : val to VARYING because of backedge processing rules. */
6415 15414615 : && (TREE_CODE (sameval) != SSA_NAME
6416 12154772 : || SSA_VAL (sameval) == sameval)
6417 32020176 : && (TREE_CODE (def) != SSA_NAME || SSA_VAL (def) == def))
6418 : {
6419 15414538 : vn_nary_op_t vnresult;
6420 15414538 : tree ops[2];
6421 15414538 : ops[0] = def;
6422 15414538 : ops[1] = sameval;
6423 : /* Canonicalize the operands order for eq below. */
6424 15414538 : if (tree_swap_operands_p (ops[0], ops[1]))
6425 9293891 : std::swap (ops[0], ops[1]);
6426 15414538 : tree val = vn_nary_op_lookup_pieces (2, EQ_EXPR,
6427 : boolean_type_node,
6428 : ops, &vnresult);
6429 15414538 : if (! val && vnresult && vnresult->predicated_values)
6430 : {
6431 211651 : val = vn_nary_op_get_predicated_value (vnresult, e);
6432 119926 : if (val && integer_truep (val)
6433 236144 : && !(sameval_e && (sameval_e->flags & EDGE_DFS_BACK)))
6434 : {
6435 24371 : if (dump_file && (dump_flags & TDF_DETAILS))
6436 : {
6437 2 : fprintf (dump_file, "Predication says ");
6438 2 : print_generic_expr (dump_file, def, TDF_NONE);
6439 2 : fprintf (dump_file, " and ");
6440 2 : print_generic_expr (dump_file, sameval, TDF_NONE);
6441 2 : fprintf (dump_file, " are equal on edge %d -> %d\n",
6442 2 : e->src->index, e->dest->index);
6443 : }
6444 24371 : continue;
6445 : }
6446 : }
6447 : }
6448 : sameval = NULL_TREE;
6449 : break;
6450 : }
6451 : }
6452 :
6453 : /* If the value we want to use is flowing over the backedge and we
6454 : should take it as VARYING but it has a non-VARYING value drop to
6455 : VARYING.
6456 : If we value-number a virtual operand never value-number to the
6457 : value from the backedge as that confuses the alias-walking code.
6458 : See gcc.dg/torture/pr87176.c. If the value is the same on a
6459 : non-backedge everything is OK though. */
6460 34123310 : bool visited_p;
6461 34123310 : if ((backedge_val
6462 34123310 : && !seen_non_backedge
6463 1998 : && TREE_CODE (backedge_val) == SSA_NAME
6464 1731 : && sameval == backedge_val
6465 311 : && (SSA_NAME_IS_VIRTUAL_OPERAND (backedge_val)
6466 40 : || SSA_VAL (backedge_val) != backedge_val))
6467 : /* Do not value-number a virtual operand to sth not visited though
6468 : given that allows us to escape a region in alias walking. */
6469 34125037 : || (sameval
6470 7564751 : && TREE_CODE (sameval) == SSA_NAME
6471 4473056 : && !SSA_NAME_IS_DEFAULT_DEF (sameval)
6472 3778739 : && SSA_NAME_IS_VIRTUAL_OPERAND (sameval)
6473 1881127 : && (SSA_VAL (sameval, &visited_p), !visited_p)))
6474 : /* Note this just drops to VARYING without inserting the PHI into
6475 : the hashes. */
6476 294541 : result = PHI_RESULT (phi);
6477 : /* If none of the edges was executable keep the value-number at VN_TOP,
6478 : if only a single edge is exectuable use its value. */
6479 33828769 : else if (n_executable <= 1)
6480 6516649 : result = seen_undef ? seen_undef : sameval;
6481 : /* If we saw only undefined values and VN_TOP use one of the
6482 : undefined values. */
6483 27312120 : else if (sameval == VN_TOP)
6484 7087611 : result = (seen_undef && seen_undef_visited) ? seen_undef : sameval;
6485 : /* First see if it is equivalent to a phi node in this block. We prefer
6486 : this as it allows IV elimination - see PRs 66502 and 67167. */
6487 27306606 : else if ((result = vn_phi_lookup (phi, backedges_varying_p)))
6488 : {
6489 4178131 : if (!inserted
6490 69366 : && TREE_CODE (result) == SSA_NAME
6491 4247497 : && gimple_code (SSA_NAME_DEF_STMT (result)) == GIMPLE_PHI)
6492 : {
6493 69366 : gimple_set_plf (SSA_NAME_DEF_STMT (result), GF_PLF_1, true);
6494 69366 : if (dump_file && (dump_flags & TDF_DETAILS))
6495 : {
6496 6 : fprintf (dump_file, "Marking CSEd to PHI node ");
6497 6 : print_gimple_expr (dump_file, SSA_NAME_DEF_STMT (result),
6498 : 0, TDF_SLIM);
6499 6 : fprintf (dump_file, "\n");
6500 : }
6501 : }
6502 : }
6503 : /* If all values are the same use that, unless we've seen undefined
6504 : values as well and the value isn't constant.
6505 : CCP/copyprop have the same restriction to not remove uninit warnings. */
6506 23128475 : else if (sameval
6507 23128475 : && (! seen_undef || is_gimple_min_invariant (sameval)))
6508 : result = sameval;
6509 : else
6510 : {
6511 22521388 : result = PHI_RESULT (phi);
6512 : /* Only insert PHIs that are varying, for constant value numbers
6513 : we mess up equivalences otherwise as we are only comparing
6514 : the immediate controlling predicates. */
6515 22521388 : vn_phi_insert (phi, result, backedges_varying_p);
6516 22521388 : if (inserted)
6517 3270091 : *inserted = true;
6518 : }
6519 :
6520 34123310 : return set_ssa_val_to (PHI_RESULT (phi), result);
6521 34123310 : }
6522 :
6523 : /* Try to simplify RHS using equivalences and constant folding. */
6524 :
6525 : static tree
6526 126229430 : try_to_simplify (gassign *stmt)
6527 : {
6528 126229430 : enum tree_code code = gimple_assign_rhs_code (stmt);
6529 126229430 : tree tem;
6530 :
6531 : /* For stores we can end up simplifying a SSA_NAME rhs. Just return
6532 : in this case, there is no point in doing extra work. */
6533 126229430 : if (code == SSA_NAME)
6534 : return NULL_TREE;
6535 :
6536 : /* First try constant folding based on our current lattice. */
6537 111340897 : mprts_hook = vn_lookup_simplify_result;
6538 111340897 : tem = gimple_fold_stmt_to_constant_1 (stmt, vn_valueize, vn_valueize);
6539 111340897 : mprts_hook = NULL;
6540 111340897 : if (tem
6541 111340897 : && (TREE_CODE (tem) == SSA_NAME
6542 24709968 : || is_gimple_min_invariant (tem)))
6543 24808756 : return tem;
6544 :
6545 : return NULL_TREE;
6546 : }
6547 :
6548 : /* Visit and value number STMT, return true if the value number
6549 : changed. */
6550 :
6551 : static bool
6552 454014899 : visit_stmt (gimple *stmt, bool backedges_varying_p = false)
6553 : {
6554 454014899 : bool changed = false;
6555 :
6556 454014899 : if (dump_file && (dump_flags & TDF_DETAILS))
6557 : {
6558 410029 : fprintf (dump_file, "Value numbering stmt = ");
6559 410029 : print_gimple_stmt (dump_file, stmt, 0);
6560 : }
6561 :
6562 454014899 : if (gimple_code (stmt) == GIMPLE_PHI)
6563 26614877 : changed = visit_phi (stmt, NULL, backedges_varying_p);
6564 597785652 : else if (gimple_has_volatile_ops (stmt))
6565 8743045 : changed = defs_to_varying (stmt);
6566 418656977 : else if (gassign *ass = dyn_cast <gassign *> (stmt))
6567 : {
6568 131214482 : enum tree_code code = gimple_assign_rhs_code (ass);
6569 131214482 : tree lhs = gimple_assign_lhs (ass);
6570 131214482 : tree rhs1 = gimple_assign_rhs1 (ass);
6571 131214482 : tree simplified;
6572 :
6573 : /* Shortcut for copies. Simplifying copies is pointless,
6574 : since we copy the expression and value they represent. */
6575 131214482 : if (code == SSA_NAME
6576 19873585 : && TREE_CODE (lhs) == SSA_NAME)
6577 : {
6578 4985052 : changed = visit_copy (lhs, rhs1);
6579 4985052 : goto done;
6580 : }
6581 126229430 : simplified = try_to_simplify (ass);
6582 126229430 : if (simplified)
6583 : {
6584 24808756 : if (dump_file && (dump_flags & TDF_DETAILS))
6585 : {
6586 14436 : fprintf (dump_file, "RHS ");
6587 14436 : print_gimple_expr (dump_file, ass, 0);
6588 14436 : fprintf (dump_file, " simplified to ");
6589 14436 : print_generic_expr (dump_file, simplified);
6590 14436 : fprintf (dump_file, "\n");
6591 : }
6592 : }
6593 : /* Setting value numbers to constants will occasionally
6594 : screw up phi congruence because constants are not
6595 : uniquely associated with a single ssa name that can be
6596 : looked up. */
6597 24808756 : if (simplified
6598 24808756 : && is_gimple_min_invariant (simplified)
6599 21842911 : && TREE_CODE (lhs) == SSA_NAME)
6600 : {
6601 7530290 : changed = set_ssa_val_to (lhs, simplified);
6602 7530290 : goto done;
6603 : }
6604 118699140 : else if (simplified
6605 17278466 : && TREE_CODE (simplified) == SSA_NAME
6606 2965845 : && TREE_CODE (lhs) == SSA_NAME)
6607 : {
6608 2965845 : changed = visit_copy (lhs, simplified);
6609 2965845 : goto done;
6610 : }
6611 :
6612 115733295 : if ((TREE_CODE (lhs) == SSA_NAME
6613 : /* We can substitute SSA_NAMEs that are live over
6614 : abnormal edges with their constant value. */
6615 82957573 : && !(gimple_assign_copy_p (ass)
6616 26 : && is_gimple_min_invariant (rhs1))
6617 82957547 : && !(simplified
6618 0 : && is_gimple_min_invariant (simplified))
6619 82957547 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
6620 : /* Stores or copies from SSA_NAMEs that are live over
6621 : abnormal edges are a problem. */
6622 198689558 : || (code == SSA_NAME
6623 14888533 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
6624 1579 : changed = defs_to_varying (ass);
6625 115731716 : else if (REFERENCE_CLASS_P (lhs)
6626 115731716 : || DECL_P (lhs))
6627 32775453 : changed = visit_reference_op_store (lhs, rhs1, ass);
6628 82956263 : else if (TREE_CODE (lhs) == SSA_NAME)
6629 : {
6630 82956263 : if ((gimple_assign_copy_p (ass)
6631 26 : && is_gimple_min_invariant (rhs1))
6632 82956289 : || (simplified
6633 0 : && is_gimple_min_invariant (simplified)))
6634 : {
6635 0 : if (simplified)
6636 0 : changed = set_ssa_val_to (lhs, simplified);
6637 : else
6638 0 : changed = set_ssa_val_to (lhs, rhs1);
6639 : }
6640 : else
6641 : {
6642 : /* Visit the original statement. */
6643 82956263 : switch (vn_get_stmt_kind (ass))
6644 : {
6645 48565808 : case VN_NARY:
6646 48565808 : changed = visit_nary_op (lhs, ass);
6647 48565808 : break;
6648 34289529 : case VN_REFERENCE:
6649 34289529 : changed = visit_reference_op_load (lhs, rhs1, ass);
6650 34289529 : break;
6651 100926 : default:
6652 100926 : changed = defs_to_varying (ass);
6653 100926 : break;
6654 : }
6655 : }
6656 : }
6657 : else
6658 0 : changed = defs_to_varying (ass);
6659 : }
6660 287442495 : else if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
6661 : {
6662 24736651 : tree lhs = gimple_call_lhs (call_stmt);
6663 24736651 : if (lhs && TREE_CODE (lhs) == SSA_NAME)
6664 : {
6665 : /* Try constant folding based on our current lattice. */
6666 8346393 : tree simplified = gimple_fold_stmt_to_constant_1 (call_stmt,
6667 : vn_valueize);
6668 8346393 : if (simplified)
6669 : {
6670 64295 : if (dump_file && (dump_flags & TDF_DETAILS))
6671 : {
6672 1 : fprintf (dump_file, "call ");
6673 1 : print_gimple_expr (dump_file, call_stmt, 0);
6674 1 : fprintf (dump_file, " simplified to ");
6675 1 : print_generic_expr (dump_file, simplified);
6676 1 : fprintf (dump_file, "\n");
6677 : }
6678 : }
6679 : /* Setting value numbers to constants will occasionally
6680 : screw up phi congruence because constants are not
6681 : uniquely associated with a single ssa name that can be
6682 : looked up. */
6683 64295 : if (simplified
6684 64295 : && is_gimple_min_invariant (simplified))
6685 : {
6686 57883 : changed = set_ssa_val_to (lhs, simplified);
6687 115766 : if (gimple_vdef (call_stmt))
6688 740 : changed |= set_ssa_val_to (gimple_vdef (call_stmt),
6689 : SSA_VAL (gimple_vuse (call_stmt)));
6690 57883 : goto done;
6691 : }
6692 8288510 : else if (simplified
6693 6412 : && TREE_CODE (simplified) == SSA_NAME)
6694 : {
6695 293 : changed = visit_copy (lhs, simplified);
6696 586 : if (gimple_vdef (call_stmt))
6697 0 : changed |= set_ssa_val_to (gimple_vdef (call_stmt),
6698 : SSA_VAL (gimple_vuse (call_stmt)));
6699 293 : goto done;
6700 : }
6701 8288217 : else if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
6702 : {
6703 381 : changed = defs_to_varying (call_stmt);
6704 381 : goto done;
6705 : }
6706 : }
6707 :
6708 : /* Pick up flags from a devirtualization target. */
6709 24678094 : tree fn = gimple_call_fn (stmt);
6710 24678094 : int extra_fnflags = 0;
6711 24678094 : if (fn && TREE_CODE (fn) == SSA_NAME)
6712 : {
6713 533669 : fn = SSA_VAL (fn);
6714 533669 : if (TREE_CODE (fn) == ADDR_EXPR
6715 533669 : && TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL)
6716 4848 : extra_fnflags = flags_from_decl_or_type (TREE_OPERAND (fn, 0));
6717 : }
6718 24678094 : if ((/* Calls to the same function with the same vuse
6719 : and the same operands do not necessarily return the same
6720 : value, unless they're pure or const. */
6721 24678094 : ((gimple_call_flags (call_stmt) | extra_fnflags)
6722 24678094 : & (ECF_PURE | ECF_CONST))
6723 : /* If calls have a vdef, subsequent calls won't have
6724 : the same incoming vuse. So, if 2 calls with vdef have the
6725 : same vuse, we know they're not subsequent.
6726 : We can value number 2 calls to the same function with the
6727 : same vuse and the same operands which are not subsequent
6728 : the same, because there is no code in the program that can
6729 : compare the 2 values... */
6730 20783265 : || (gimple_vdef (call_stmt)
6731 : /* ... unless the call returns a pointer which does
6732 : not alias with anything else. In which case the
6733 : information that the values are distinct are encoded
6734 : in the IL. */
6735 20748393 : && !(gimple_call_return_flags (call_stmt) & ERF_NOALIAS)
6736 : /* Only perform the following when being called from PRE
6737 : which embeds tail merging. */
6738 20207099 : && default_vn_walk_kind == VN_WALK))
6739 : /* Do not process .DEFERRED_INIT since that confuses uninit
6740 : analysis. */
6741 29618352 : && !gimple_call_internal_p (call_stmt, IFN_DEFERRED_INIT))
6742 8629207 : changed = visit_reference_op_call (lhs, call_stmt);
6743 : else
6744 16048887 : changed = defs_to_varying (call_stmt);
6745 : }
6746 : else
6747 262705844 : changed = defs_to_varying (stmt);
6748 454014899 : done:
6749 454014899 : return changed;
6750 : }
6751 :
6752 :
6753 : /* Allocate a value number table. */
6754 :
6755 : static void
6756 6147805 : allocate_vn_table (vn_tables_t table, unsigned size)
6757 : {
6758 6147805 : table->phis = new vn_phi_table_type (size);
6759 6147805 : table->nary = new vn_nary_op_table_type (size);
6760 6147805 : table->references = new vn_reference_table_type (size);
6761 6147805 : }
6762 :
6763 : /* Free a value number table. */
6764 :
6765 : static void
6766 6147805 : free_vn_table (vn_tables_t table)
6767 : {
6768 : /* Walk over elements and release vectors. */
6769 6147805 : vn_reference_iterator_type hir;
6770 6147805 : vn_reference_t vr;
6771 145799855 : FOR_EACH_HASH_TABLE_ELEMENT (*table->references, vr, vn_reference_t, hir)
6772 69826025 : vr->operands.release ();
6773 6147805 : delete table->phis;
6774 6147805 : table->phis = NULL;
6775 6147805 : delete table->nary;
6776 6147805 : table->nary = NULL;
6777 6147805 : delete table->references;
6778 6147805 : table->references = NULL;
6779 6147805 : }
6780 :
6781 : /* Set *ID according to RESULT. */
6782 :
6783 : static void
6784 34340064 : set_value_id_for_result (tree result, unsigned int *id)
6785 : {
6786 34340064 : if (result && TREE_CODE (result) == SSA_NAME)
6787 21370841 : *id = VN_INFO (result)->value_id;
6788 9709447 : else if (result && is_gimple_min_invariant (result))
6789 3667157 : *id = get_or_alloc_constant_value_id (result);
6790 : else
6791 9302066 : *id = get_next_value_id ();
6792 34340064 : }
6793 :
6794 : /* Set the value ids in the valid hash tables. */
6795 :
6796 : static void
6797 966285 : set_hashtable_value_ids (void)
6798 : {
6799 966285 : vn_nary_op_iterator_type hin;
6800 966285 : vn_phi_iterator_type hip;
6801 966285 : vn_reference_iterator_type hir;
6802 966285 : vn_nary_op_t vno;
6803 966285 : vn_reference_t vr;
6804 966285 : vn_phi_t vp;
6805 :
6806 : /* Now set the value ids of the things we had put in the hash
6807 : table. */
6808 :
6809 48556097 : FOR_EACH_HASH_TABLE_ELEMENT (*valid_info->nary, vno, vn_nary_op_t, hin)
6810 23794906 : if (! vno->predicated_values)
6811 7737840 : set_value_id_for_result (vno->u.result, &vno->value_id);
6812 :
6813 8898719 : FOR_EACH_HASH_TABLE_ELEMENT (*valid_info->phis, vp, vn_phi_t, hip)
6814 3966217 : set_value_id_for_result (vp->result, &vp->value_id);
6815 :
6816 46238299 : FOR_EACH_HASH_TABLE_ELEMENT (*valid_info->references, vr, vn_reference_t,
6817 : hir)
6818 22636007 : set_value_id_for_result (vr->result, &vr->value_id);
6819 966285 : }
6820 :
6821 : /* Return the maximum value id we have ever seen. */
6822 :
6823 : unsigned int
6824 1932570 : get_max_value_id (void)
6825 : {
6826 1932570 : return next_value_id;
6827 : }
6828 :
6829 : /* Return the maximum constant value id we have ever seen. */
6830 :
6831 : unsigned int
6832 1932570 : get_max_constant_value_id (void)
6833 : {
6834 1932570 : return -next_constant_value_id;
6835 : }
6836 :
6837 : /* Return the next unique value id. */
6838 :
6839 : unsigned int
6840 48865449 : get_next_value_id (void)
6841 : {
6842 48865449 : gcc_checking_assert ((int)next_value_id > 0);
6843 48865449 : return next_value_id++;
6844 : }
6845 :
6846 : /* Return the next unique value id for constants. */
6847 :
6848 : unsigned int
6849 2511723 : get_next_constant_value_id (void)
6850 : {
6851 2511723 : gcc_checking_assert (next_constant_value_id < 0);
6852 2511723 : return next_constant_value_id--;
6853 : }
6854 :
6855 :
6856 : /* Compare two expressions E1 and E2 and return true if they are equal.
6857 : If match_vn_top_optimistically is true then VN_TOP is equal to anything,
6858 : otherwise VN_TOP only matches VN_TOP. */
6859 :
6860 : bool
6861 245427560 : expressions_equal_p (tree e1, tree e2, bool match_vn_top_optimistically)
6862 : {
6863 : /* The obvious case. */
6864 245427560 : if (e1 == e2)
6865 : return true;
6866 :
6867 : /* If either one is VN_TOP consider them equal. */
6868 70385053 : if (match_vn_top_optimistically
6869 65533657 : && (e1 == VN_TOP || e2 == VN_TOP))
6870 : return true;
6871 :
6872 : /* If only one of them is null, they cannot be equal. While in general
6873 : this should not happen for operations like TARGET_MEM_REF some
6874 : operands are optional and an identity value we could substitute
6875 : has differing semantics. */
6876 70385053 : if (!e1 || !e2)
6877 : return false;
6878 :
6879 : /* SSA_NAME compare pointer equal. */
6880 70385053 : if (TREE_CODE (e1) == SSA_NAME || TREE_CODE (e2) == SSA_NAME)
6881 : return false;
6882 :
6883 : /* Now perform the actual comparison. */
6884 35038514 : if (TREE_CODE (e1) == TREE_CODE (e2)
6885 35038514 : && operand_equal_p (e1, e2, OEP_PURE_SAME))
6886 : return true;
6887 :
6888 : return false;
6889 : }
6890 :
6891 :
6892 : /* Return true if the nary operation NARY may trap. This is a copy
6893 : of stmt_could_throw_1_p adjusted to the SCCVN IL. */
6894 :
6895 : bool
6896 5634542 : vn_nary_may_trap (vn_nary_op_t nary)
6897 : {
6898 5634542 : tree type;
6899 5634542 : tree rhs2 = NULL_TREE;
6900 5634542 : bool honor_nans = false;
6901 5634542 : bool honor_snans = false;
6902 5634542 : bool fp_operation = false;
6903 5634542 : bool honor_trapv = false;
6904 5634542 : bool handled, ret;
6905 5634542 : unsigned i;
6906 :
6907 5634542 : if (TREE_CODE_CLASS (nary->opcode) == tcc_comparison
6908 : || TREE_CODE_CLASS (nary->opcode) == tcc_unary
6909 5634542 : || TREE_CODE_CLASS (nary->opcode) == tcc_binary)
6910 : {
6911 5533696 : type = nary->type;
6912 5533696 : fp_operation = FLOAT_TYPE_P (type);
6913 5533696 : if (fp_operation)
6914 : {
6915 118034 : honor_nans = flag_trapping_math && !flag_finite_math_only;
6916 118034 : honor_snans = flag_signaling_nans != 0;
6917 : }
6918 5415662 : else if (INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type))
6919 : honor_trapv = true;
6920 : }
6921 5634542 : if (nary->length >= 2)
6922 2247277 : rhs2 = nary->op[1];
6923 5634542 : ret = operation_could_trap_helper_p (nary->opcode, fp_operation,
6924 : honor_trapv, honor_nans, honor_snans,
6925 : rhs2, &handled);
6926 5634542 : if (handled && ret)
6927 : return true;
6928 :
6929 13205474 : for (i = 0; i < nary->length; ++i)
6930 7687965 : if (tree_could_trap_p (nary->op[i]))
6931 : return true;
6932 :
6933 : return false;
6934 : }
6935 :
6936 : /* Return true if the reference operation REF may trap. */
6937 :
6938 : bool
6939 922935 : vn_reference_may_trap (vn_reference_t ref)
6940 : {
6941 922935 : switch (ref->operands[0].opcode)
6942 : {
6943 : case MODIFY_EXPR:
6944 : case CALL_EXPR:
6945 : /* We do not handle calls. */
6946 : return true;
6947 : case ADDR_EXPR:
6948 : /* And toplevel address computations never trap. */
6949 : return false;
6950 : default:;
6951 : }
6952 :
6953 : vn_reference_op_t op;
6954 : unsigned i;
6955 2559774 : FOR_EACH_VEC_ELT (ref->operands, i, op)
6956 : {
6957 2559539 : switch (op->opcode)
6958 : {
6959 : case WITH_SIZE_EXPR:
6960 : case TARGET_MEM_REF:
6961 : /* Always variable. */
6962 : return true;
6963 724552 : case COMPONENT_REF:
6964 724552 : if (op->op1 && TREE_CODE (op->op1) == SSA_NAME)
6965 : return true;
6966 : break;
6967 0 : case ARRAY_RANGE_REF:
6968 0 : if (TREE_CODE (op->op0) == SSA_NAME)
6969 : return true;
6970 : break;
6971 204211 : case ARRAY_REF:
6972 204211 : {
6973 204211 : if (TREE_CODE (op->op0) != INTEGER_CST)
6974 : return true;
6975 :
6976 : /* !in_array_bounds */
6977 184147 : tree domain_type = TYPE_DOMAIN (ref->operands[i+1].type);
6978 184147 : if (!domain_type)
6979 : return true;
6980 :
6981 184101 : tree min = op->op1;
6982 184101 : tree max = TYPE_MAX_VALUE (domain_type);
6983 184101 : if (!min
6984 184101 : || !max
6985 171469 : || TREE_CODE (min) != INTEGER_CST
6986 171469 : || TREE_CODE (max) != INTEGER_CST)
6987 : return true;
6988 :
6989 168853 : if (tree_int_cst_lt (op->op0, min)
6990 168853 : || tree_int_cst_lt (max, op->op0))
6991 316 : return true;
6992 :
6993 : break;
6994 : }
6995 : case MEM_REF:
6996 : /* Nothing interesting in itself, the base is separate. */
6997 : break;
6998 : /* The following are the address bases. */
6999 : case SSA_NAME:
7000 : return true;
7001 528885 : case ADDR_EXPR:
7002 528885 : if (op->op0)
7003 528885 : return tree_could_trap_p (TREE_OPERAND (op->op0, 0));
7004 : return false;
7005 1720005 : default:;
7006 : }
7007 : }
7008 : return false;
7009 : }
7010 :
7011 10408773 : eliminate_dom_walker::eliminate_dom_walker (cdi_direction direction,
7012 10408773 : bitmap inserted_exprs_)
7013 10408773 : : dom_walker (direction), do_pre (inserted_exprs_ != NULL),
7014 10408773 : el_todo (0), eliminations (0), insertions (0),
7015 10408773 : inserted_exprs (inserted_exprs_)
7016 : {
7017 10408773 : need_eh_cleanup = BITMAP_ALLOC (NULL);
7018 10408773 : need_ab_cleanup = BITMAP_ALLOC (NULL);
7019 10408773 : }
7020 :
7021 10408773 : eliminate_dom_walker::~eliminate_dom_walker ()
7022 : {
7023 10408773 : BITMAP_FREE (need_eh_cleanup);
7024 10408773 : BITMAP_FREE (need_ab_cleanup);
7025 10408773 : }
7026 :
7027 : /* Return a leader for OP that is available at the current point of the
7028 : eliminate domwalk. */
7029 :
7030 : tree
7031 180542569 : eliminate_dom_walker::eliminate_avail (basic_block, tree op)
7032 : {
7033 180542569 : tree valnum = VN_INFO (op)->valnum;
7034 180542569 : if (TREE_CODE (valnum) == SSA_NAME)
7035 : {
7036 175543113 : if (SSA_NAME_IS_DEFAULT_DEF (valnum))
7037 : return valnum;
7038 305669375 : if (avail.length () > SSA_NAME_VERSION (valnum))
7039 : {
7040 137675236 : tree av = avail[SSA_NAME_VERSION (valnum)];
7041 : /* When PRE discovers a new redundancy there's no way to unite
7042 : the value classes so it instead inserts a copy old-val = new-val.
7043 : Look through such copies here, providing one more level of
7044 : simplification at elimination time. */
7045 137675236 : gassign *ass;
7046 242049650 : if (av && (ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (av))))
7047 74093563 : if (gimple_assign_rhs_class (ass) == GIMPLE_SINGLE_RHS)
7048 : {
7049 39161751 : tree rhs1 = gimple_assign_rhs1 (ass);
7050 39161751 : if (CONSTANT_CLASS_P (rhs1)
7051 39161751 : || (TREE_CODE (rhs1) == SSA_NAME
7052 9737 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
7053 : av = rhs1;
7054 : }
7055 137675236 : return av;
7056 : }
7057 : }
7058 4999456 : else if (is_gimple_min_invariant (valnum))
7059 : return valnum;
7060 : return NULL_TREE;
7061 : }
7062 :
7063 : /* At the current point of the eliminate domwalk make OP available. */
7064 :
7065 : void
7066 49864094 : eliminate_dom_walker::eliminate_push_avail (basic_block, tree op)
7067 : {
7068 49864094 : tree valnum = VN_INFO (op)->valnum;
7069 49864094 : if (TREE_CODE (valnum) == SSA_NAME)
7070 : {
7071 96362362 : if (avail.length () <= SSA_NAME_VERSION (valnum))
7072 16889173 : avail.safe_grow_cleared (SSA_NAME_VERSION (valnum) + 1, true);
7073 49864094 : tree pushop = op;
7074 49864094 : if (avail[SSA_NAME_VERSION (valnum)])
7075 44352 : pushop = avail[SSA_NAME_VERSION (valnum)];
7076 49864094 : avail_stack.safe_push (pushop);
7077 49864094 : avail[SSA_NAME_VERSION (valnum)] = op;
7078 : }
7079 49864094 : }
7080 :
7081 : /* Insert the expression recorded by SCCVN for VAL at *GSI. Returns
7082 : the leader for the expression if insertion was successful. */
7083 :
7084 : tree
7085 123471 : eliminate_dom_walker::eliminate_insert (basic_block bb,
7086 : gimple_stmt_iterator *gsi, tree val)
7087 : {
7088 : /* We can insert a sequence with a single assignment only. */
7089 123471 : gimple_seq stmts = VN_INFO (val)->expr;
7090 123471 : if (!gimple_seq_singleton_p (stmts))
7091 : return NULL_TREE;
7092 224364 : gassign *stmt = dyn_cast <gassign *> (gimple_seq_first_stmt (stmts));
7093 123471 : if (!stmt
7094 123471 : || (!CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
7095 : && gimple_assign_rhs_code (stmt) != VIEW_CONVERT_EXPR
7096 : && gimple_assign_rhs_code (stmt) != NEGATE_EXPR
7097 : && gimple_assign_rhs_code (stmt) != BIT_FIELD_REF
7098 : && (gimple_assign_rhs_code (stmt) != BIT_AND_EXPR
7099 75 : || TREE_CODE (gimple_assign_rhs2 (stmt)) != INTEGER_CST)))
7100 : return NULL_TREE;
7101 :
7102 32262 : tree op = gimple_assign_rhs1 (stmt);
7103 32262 : if (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
7104 32262 : || gimple_assign_rhs_code (stmt) == BIT_FIELD_REF)
7105 19852 : op = TREE_OPERAND (op, 0);
7106 32262 : tree leader = TREE_CODE (op) == SSA_NAME ? eliminate_avail (bb, op) : op;
7107 32216 : if (!leader)
7108 : return NULL_TREE;
7109 :
7110 22582 : tree res;
7111 22582 : stmts = NULL;
7112 41422 : if (gimple_assign_rhs_code (stmt) == BIT_FIELD_REF)
7113 33746 : res = gimple_build (&stmts, BIT_FIELD_REF,
7114 16873 : TREE_TYPE (val), leader,
7115 16873 : TREE_OPERAND (gimple_assign_rhs1 (stmt), 1),
7116 16873 : TREE_OPERAND (gimple_assign_rhs1 (stmt), 2));
7117 5709 : else if (gimple_assign_rhs_code (stmt) == BIT_AND_EXPR)
7118 150 : res = gimple_build (&stmts, BIT_AND_EXPR,
7119 75 : TREE_TYPE (val), leader, gimple_assign_rhs2 (stmt));
7120 : else
7121 5634 : res = gimple_build (&stmts, gimple_assign_rhs_code (stmt),
7122 5634 : TREE_TYPE (val), leader);
7123 22582 : if (TREE_CODE (res) != SSA_NAME
7124 22581 : || SSA_NAME_IS_DEFAULT_DEF (res)
7125 45163 : || gimple_bb (SSA_NAME_DEF_STMT (res)))
7126 : {
7127 4 : gimple_seq_discard (stmts);
7128 :
7129 : /* During propagation we have to treat SSA info conservatively
7130 : and thus we can end up simplifying the inserted expression
7131 : at elimination time to sth not defined in stmts. */
7132 : /* But then this is a redundancy we failed to detect. Which means
7133 : res now has two values. That doesn't play well with how
7134 : we track availability here, so give up. */
7135 4 : if (dump_file && (dump_flags & TDF_DETAILS))
7136 : {
7137 0 : if (TREE_CODE (res) == SSA_NAME)
7138 0 : res = eliminate_avail (bb, res);
7139 0 : if (res)
7140 : {
7141 0 : fprintf (dump_file, "Failed to insert expression for value ");
7142 0 : print_generic_expr (dump_file, val);
7143 0 : fprintf (dump_file, " which is really fully redundant to ");
7144 0 : print_generic_expr (dump_file, res);
7145 0 : fprintf (dump_file, "\n");
7146 : }
7147 : }
7148 :
7149 4 : return NULL_TREE;
7150 : }
7151 : else
7152 : {
7153 22578 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
7154 22578 : vn_ssa_aux_t vn_info = VN_INFO (res);
7155 22578 : vn_info->valnum = val;
7156 22578 : vn_info->visited = true;
7157 : }
7158 :
7159 22578 : insertions++;
7160 22578 : if (dump_file && (dump_flags & TDF_DETAILS))
7161 : {
7162 499 : fprintf (dump_file, "Inserted ");
7163 499 : print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (res), 0);
7164 : }
7165 :
7166 : return res;
7167 : }
7168 :
7169 : void
7170 351178100 : eliminate_dom_walker::eliminate_stmt (basic_block b, gimple_stmt_iterator *gsi)
7171 : {
7172 351178100 : tree sprime = NULL_TREE;
7173 351178100 : gimple *stmt = gsi_stmt (*gsi);
7174 351178100 : tree lhs = gimple_get_lhs (stmt);
7175 119405939 : if (lhs && TREE_CODE (lhs) == SSA_NAME
7176 165140622 : && !gimple_has_volatile_ops (stmt)
7177 : /* See PR43491. Do not replace a global register variable when
7178 : it is a the RHS of an assignment. Do replace local register
7179 : variables since gcc does not guarantee a local variable will
7180 : be allocated in register.
7181 : ??? The fix isn't effective here. This should instead
7182 : be ensured by not value-numbering them the same but treating
7183 : them like volatiles? */
7184 432685586 : && !(gimple_assign_single_p (stmt)
7185 35110350 : && (TREE_CODE (gimple_assign_rhs1 (stmt)) == VAR_DECL
7186 2439844 : && DECL_HARD_REGISTER (gimple_assign_rhs1 (stmt))
7187 4184 : && is_global_var (gimple_assign_rhs1 (stmt)))))
7188 : {
7189 81507242 : sprime = eliminate_avail (b, lhs);
7190 81507242 : if (!sprime)
7191 : {
7192 : /* If there is no existing usable leader but SCCVN thinks
7193 : it has an expression it wants to use as replacement,
7194 : insert that. */
7195 68624893 : tree val = VN_INFO (lhs)->valnum;
7196 68624893 : vn_ssa_aux_t vn_info;
7197 68624893 : if (val != VN_TOP
7198 68624893 : && TREE_CODE (val) == SSA_NAME
7199 68624893 : && (vn_info = VN_INFO (val), true)
7200 68624893 : && vn_info->needs_insertion
7201 318568 : && vn_info->expr != NULL
7202 68748364 : && (sprime = eliminate_insert (b, gsi, val)) != NULL_TREE)
7203 22578 : eliminate_push_avail (b, sprime);
7204 : }
7205 :
7206 : /* If this now constitutes a copy duplicate points-to
7207 : and range info appropriately. This is especially
7208 : important for inserted code. */
7209 68624893 : if (sprime
7210 12904927 : && TREE_CODE (sprime) == SSA_NAME)
7211 8888245 : maybe_duplicate_ssa_info_at_copy (lhs, sprime);
7212 :
7213 : /* Inhibit the use of an inserted PHI on a loop header when
7214 : the address of the memory reference is a simple induction
7215 : variable. In other cases the vectorizer won't do anything
7216 : anyway (either it's loop invariant or a complicated
7217 : expression). */
7218 8888245 : if (sprime
7219 12904927 : && TREE_CODE (sprime) == SSA_NAME
7220 8888245 : && do_pre
7221 910349 : && (flag_tree_loop_vectorize || flag_tree_parallelize_loops > 1)
7222 891680 : && loop_outer (b->loop_father)
7223 384962 : && has_zero_uses (sprime)
7224 191016 : && bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (sprime))
7225 190864 : && gimple_assign_load_p (stmt))
7226 : {
7227 103262 : gimple *def_stmt = SSA_NAME_DEF_STMT (sprime);
7228 103262 : basic_block def_bb = gimple_bb (def_stmt);
7229 103262 : if (gimple_code (def_stmt) == GIMPLE_PHI
7230 103262 : && def_bb->loop_father->header == def_bb)
7231 : {
7232 65344 : loop_p loop = def_bb->loop_father;
7233 65344 : ssa_op_iter iter;
7234 65344 : tree op;
7235 65344 : bool found = false;
7236 82895 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
7237 : {
7238 61861 : affine_iv iv;
7239 61861 : def_bb = gimple_bb (SSA_NAME_DEF_STMT (op));
7240 61861 : if (def_bb
7241 56012 : && flow_bb_inside_loop_p (loop, def_bb)
7242 112811 : && simple_iv (loop, loop, op, &iv, true))
7243 : {
7244 44310 : found = true;
7245 44310 : break;
7246 : }
7247 : }
7248 21034 : if (found)
7249 : {
7250 44310 : if (dump_file && (dump_flags & TDF_DETAILS))
7251 : {
7252 3 : fprintf (dump_file, "Not replacing ");
7253 3 : print_gimple_expr (dump_file, stmt, 0);
7254 3 : fprintf (dump_file, " with ");
7255 3 : print_generic_expr (dump_file, sprime);
7256 3 : fprintf (dump_file, " which would add a loop"
7257 : " carried dependence to loop %d\n",
7258 : loop->num);
7259 : }
7260 : /* Don't keep sprime available. */
7261 44310 : sprime = NULL_TREE;
7262 : }
7263 : }
7264 : }
7265 :
7266 81507242 : if (sprime)
7267 : {
7268 : /* If we can propagate the value computed for LHS into
7269 : all uses don't bother doing anything with this stmt. */
7270 12860617 : if (may_propagate_copy (lhs, sprime))
7271 : {
7272 : /* Mark it for removal. */
7273 12858654 : to_remove.safe_push (stmt);
7274 :
7275 : /* ??? Don't count copy/constant propagations. */
7276 12858654 : if (gimple_assign_single_p (stmt)
7277 12858654 : && (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
7278 4503993 : || gimple_assign_rhs1 (stmt) == sprime))
7279 13685116 : return;
7280 :
7281 7799109 : if (dump_file && (dump_flags & TDF_DETAILS))
7282 : {
7283 18756 : fprintf (dump_file, "Replaced ");
7284 18756 : print_gimple_expr (dump_file, stmt, 0);
7285 18756 : fprintf (dump_file, " with ");
7286 18756 : print_generic_expr (dump_file, sprime);
7287 18756 : fprintf (dump_file, " in all uses of ");
7288 18756 : print_gimple_stmt (dump_file, stmt, 0);
7289 : }
7290 :
7291 7799109 : eliminations++;
7292 7799109 : return;
7293 : }
7294 :
7295 : /* If this is an assignment from our leader (which
7296 : happens in the case the value-number is a constant)
7297 : then there is nothing to do. Likewise if we run into
7298 : inserted code that needed a conversion because of
7299 : our type-agnostic value-numbering of loads. */
7300 1963 : if ((gimple_assign_single_p (stmt)
7301 1 : || (is_gimple_assign (stmt)
7302 1 : && (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
7303 0 : || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)))
7304 1964 : && sprime == gimple_assign_rhs1 (stmt))
7305 : return;
7306 :
7307 : /* Else replace its RHS. */
7308 719 : if (dump_file && (dump_flags & TDF_DETAILS))
7309 : {
7310 0 : fprintf (dump_file, "Replaced ");
7311 0 : print_gimple_expr (dump_file, stmt, 0);
7312 0 : fprintf (dump_file, " with ");
7313 0 : print_generic_expr (dump_file, sprime);
7314 0 : fprintf (dump_file, " in ");
7315 0 : print_gimple_stmt (dump_file, stmt, 0);
7316 : }
7317 719 : eliminations++;
7318 :
7319 719 : bool can_make_abnormal_goto = (is_gimple_call (stmt)
7320 719 : && stmt_can_make_abnormal_goto (stmt));
7321 719 : gimple *orig_stmt = stmt;
7322 719 : if (!useless_type_conversion_p (TREE_TYPE (lhs),
7323 719 : TREE_TYPE (sprime)))
7324 : {
7325 : /* We preserve conversions to but not from function or method
7326 : types. This asymmetry makes it necessary to re-instantiate
7327 : conversions here. */
7328 717 : if (POINTER_TYPE_P (TREE_TYPE (lhs))
7329 717 : && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (lhs))))
7330 717 : sprime = fold_convert (TREE_TYPE (lhs), sprime);
7331 : else
7332 0 : gcc_unreachable ();
7333 : }
7334 719 : tree vdef = gimple_vdef (stmt);
7335 719 : tree vuse = gimple_vuse (stmt);
7336 719 : propagate_tree_value_into_stmt (gsi, sprime);
7337 719 : stmt = gsi_stmt (*gsi);
7338 719 : update_stmt (stmt);
7339 : /* In case the VDEF on the original stmt was released, value-number
7340 : it to the VUSE. This is to make vuse_ssa_val able to skip
7341 : released virtual operands. */
7342 1438 : if (vdef != gimple_vdef (stmt))
7343 : {
7344 0 : gcc_assert (SSA_NAME_IN_FREE_LIST (vdef));
7345 0 : VN_INFO (vdef)->valnum = vuse;
7346 : }
7347 :
7348 : /* If we removed EH side-effects from the statement, clean
7349 : its EH information. */
7350 719 : if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
7351 : {
7352 0 : bitmap_set_bit (need_eh_cleanup,
7353 0 : gimple_bb (stmt)->index);
7354 0 : if (dump_file && (dump_flags & TDF_DETAILS))
7355 0 : fprintf (dump_file, " Removed EH side-effects.\n");
7356 : }
7357 :
7358 : /* Likewise for AB side-effects. */
7359 719 : if (can_make_abnormal_goto
7360 719 : && !stmt_can_make_abnormal_goto (stmt))
7361 : {
7362 0 : bitmap_set_bit (need_ab_cleanup,
7363 0 : gimple_bb (stmt)->index);
7364 0 : if (dump_file && (dump_flags & TDF_DETAILS))
7365 0 : fprintf (dump_file, " Removed AB side-effects.\n");
7366 : }
7367 :
7368 719 : return;
7369 : }
7370 : }
7371 :
7372 : /* If the statement is a scalar store, see if the expression
7373 : has the same value number as its rhs. If so, the store is
7374 : dead. */
7375 338317483 : if (gimple_assign_single_p (stmt)
7376 126136036 : && !gimple_has_volatile_ops (stmt)
7377 54948549 : && !is_gimple_reg (gimple_assign_lhs (stmt))
7378 28359712 : && (TREE_CODE (gimple_assign_lhs (stmt)) != VAR_DECL
7379 2755987 : || !DECL_HARD_REGISTER (gimple_assign_lhs (stmt)))
7380 366673186 : && (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
7381 16198701 : || is_gimple_min_invariant (gimple_assign_rhs1 (stmt))))
7382 : {
7383 25300873 : tree rhs = gimple_assign_rhs1 (stmt);
7384 25300873 : vn_reference_t vnresult;
7385 : /* ??? gcc.dg/torture/pr91445.c shows that we lookup a boolean
7386 : typed load of a byte known to be 0x11 as 1 so a store of
7387 : a boolean 1 is detected as redundant. Because of this we
7388 : have to make sure to lookup with a ref where its size
7389 : matches the precision. */
7390 25300873 : tree lookup_lhs = lhs;
7391 50343067 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
7392 13202562 : && (TREE_CODE (lhs) != COMPONENT_REF
7393 8037305 : || !DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
7394 38308512 : && !type_has_mode_precision_p (TREE_TYPE (lhs)))
7395 : {
7396 822268 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
7397 427346 : && TYPE_PRECISION (TREE_TYPE (lhs)) > MAX_FIXED_MODE_SIZE)
7398 : lookup_lhs = NULL_TREE;
7399 409353 : else if (TREE_CODE (lhs) == COMPONENT_REF
7400 409353 : || TREE_CODE (lhs) == MEM_REF)
7401 : {
7402 287017 : tree ltype = build_nonstandard_integer_type
7403 287017 : (TREE_INT_CST_LOW (TYPE_SIZE (TREE_TYPE (lhs))),
7404 287017 : TYPE_UNSIGNED (TREE_TYPE (lhs)));
7405 287017 : if (TREE_CODE (lhs) == COMPONENT_REF)
7406 : {
7407 219380 : tree foff = component_ref_field_offset (lhs);
7408 219380 : tree f = TREE_OPERAND (lhs, 1);
7409 219380 : if (!poly_int_tree_p (foff))
7410 : lookup_lhs = NULL_TREE;
7411 : else
7412 438760 : lookup_lhs = build3 (BIT_FIELD_REF, ltype,
7413 219380 : TREE_OPERAND (lhs, 0),
7414 219380 : TYPE_SIZE (TREE_TYPE (lhs)),
7415 : bit_from_pos
7416 219380 : (foff, DECL_FIELD_BIT_OFFSET (f)));
7417 : }
7418 : else
7419 67637 : lookup_lhs = build2 (MEM_REF, ltype,
7420 67637 : TREE_OPERAND (lhs, 0),
7421 67637 : TREE_OPERAND (lhs, 1));
7422 : }
7423 : else
7424 : lookup_lhs = NULL_TREE;
7425 : }
7426 25171356 : tree val = NULL_TREE, tem;
7427 25171356 : if (lookup_lhs)
7428 50342712 : val = vn_reference_lookup (lookup_lhs, gimple_vuse (stmt),
7429 : VN_WALKREWRITE, &vnresult, false,
7430 : NULL, NULL_TREE, true);
7431 25300873 : if (TREE_CODE (rhs) == SSA_NAME)
7432 12157002 : rhs = VN_INFO (rhs)->valnum;
7433 25300873 : gassign *ass;
7434 25300873 : if (val
7435 25300873 : && (operand_equal_p (val, rhs, 0)
7436 : /* Due to the bitfield lookups above we can get bit
7437 : interpretations of the same RHS as values here. Those
7438 : are redundant as well. */
7439 3080588 : || (TREE_CODE (val) == SSA_NAME
7440 1881849 : && gimple_assign_single_p (SSA_NAME_DEF_STMT (val))
7441 1712817 : && (tem = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (val)))
7442 1712817 : && TREE_CODE (tem) == VIEW_CONVERT_EXPR
7443 3455 : && TREE_OPERAND (tem, 0) == rhs)
7444 3080586 : || (TREE_CODE (rhs) == SSA_NAME
7445 25758998 : && (ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (rhs)))
7446 1470555 : && gimple_assign_rhs1 (ass) == val
7447 679539 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (ass))
7448 9 : && tree_nop_conversion_p (TREE_TYPE (rhs), TREE_TYPE (val)))))
7449 : {
7450 : /* We can only remove the later store if the former aliases
7451 : at least all accesses the later one does or if the store
7452 : was to readonly memory storing the same value. */
7453 238619 : ao_ref lhs_ref;
7454 238619 : ao_ref_init (&lhs_ref, lhs);
7455 238619 : alias_set_type set = ao_ref_alias_set (&lhs_ref);
7456 238619 : alias_set_type base_set = ao_ref_base_alias_set (&lhs_ref);
7457 238619 : if (! vnresult
7458 238619 : || ((vnresult->set == set
7459 48695 : || alias_set_subset_of (set, vnresult->set))
7460 223552 : && (vnresult->base_set == base_set
7461 21203 : || alias_set_subset_of (base_set, vnresult->base_set))))
7462 : {
7463 221414 : if (dump_file && (dump_flags & TDF_DETAILS))
7464 : {
7465 17 : fprintf (dump_file, "Deleted redundant store ");
7466 17 : print_gimple_stmt (dump_file, stmt, 0);
7467 : }
7468 :
7469 : /* Queue stmt for removal. */
7470 221414 : to_remove.safe_push (stmt);
7471 221414 : return;
7472 : }
7473 : }
7474 : }
7475 :
7476 : /* If this is a control statement value numbering left edges
7477 : unexecuted on force the condition in a way consistent with
7478 : that. */
7479 338096069 : if (gcond *cond = dyn_cast <gcond *> (stmt))
7480 : {
7481 18919270 : if ((EDGE_SUCC (b, 0)->flags & EDGE_EXECUTABLE)
7482 18919270 : ^ (EDGE_SUCC (b, 1)->flags & EDGE_EXECUTABLE))
7483 : {
7484 603085 : if (dump_file && (dump_flags & TDF_DETAILS))
7485 : {
7486 15 : fprintf (dump_file, "Removing unexecutable edge from ");
7487 15 : print_gimple_stmt (dump_file, stmt, 0);
7488 : }
7489 603085 : if (((EDGE_SUCC (b, 0)->flags & EDGE_TRUE_VALUE) != 0)
7490 603085 : == ((EDGE_SUCC (b, 0)->flags & EDGE_EXECUTABLE) != 0))
7491 234370 : gimple_cond_make_true (cond);
7492 : else
7493 368715 : gimple_cond_make_false (cond);
7494 603085 : update_stmt (cond);
7495 603085 : el_todo |= TODO_cleanup_cfg;
7496 603085 : return;
7497 : }
7498 : }
7499 :
7500 337492984 : bool can_make_abnormal_goto = stmt_can_make_abnormal_goto (stmt);
7501 337492984 : bool was_noreturn = (is_gimple_call (stmt)
7502 337492984 : && gimple_call_noreturn_p (stmt));
7503 337492984 : tree vdef = gimple_vdef (stmt);
7504 337492984 : tree vuse = gimple_vuse (stmt);
7505 :
7506 : /* If we didn't replace the whole stmt (or propagate the result
7507 : into all uses), replace all uses on this stmt with their
7508 : leaders. */
7509 337492984 : bool modified = false;
7510 337492984 : use_operand_p use_p;
7511 337492984 : ssa_op_iter iter;
7512 501387494 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
7513 : {
7514 163894510 : tree use = USE_FROM_PTR (use_p);
7515 : /* ??? The call code above leaves stmt operands un-updated. */
7516 163894510 : if (TREE_CODE (use) != SSA_NAME)
7517 0 : continue;
7518 163894510 : tree sprime;
7519 163894510 : if (SSA_NAME_IS_DEFAULT_DEF (use))
7520 : /* ??? For default defs BB shouldn't matter, but we have to
7521 : solve the inconsistency between rpo eliminate and
7522 : dom eliminate avail valueization first. */
7523 26330494 : sprime = eliminate_avail (b, use);
7524 : else
7525 : /* Look for sth available at the definition block of the argument.
7526 : This avoids inconsistencies between availability there which
7527 : decides if the stmt can be removed and availability at the
7528 : use site. The SSA property ensures that things available
7529 : at the definition are also available at uses. */
7530 137564016 : sprime = eliminate_avail (gimple_bb (SSA_NAME_DEF_STMT (use)), use);
7531 163894510 : if (sprime && sprime != use
7532 12809770 : && may_propagate_copy (use, sprime, true)
7533 : /* We substitute into debug stmts to avoid excessive
7534 : debug temporaries created by removed stmts, but we need
7535 : to avoid doing so for inserted sprimes as we never want
7536 : to create debug temporaries for them. */
7537 176703563 : && (!inserted_exprs
7538 1178165 : || TREE_CODE (sprime) != SSA_NAME
7539 1163430 : || !is_gimple_debug (stmt)
7540 371860 : || !bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (sprime))))
7541 : {
7542 12467398 : propagate_value (use_p, sprime);
7543 12467398 : modified = true;
7544 : }
7545 : }
7546 :
7547 : /* Fold the stmt if modified, this canonicalizes MEM_REFs we propagated
7548 : into which is a requirement for the IPA devirt machinery. */
7549 337492984 : gimple *old_stmt = stmt;
7550 337492984 : if (modified)
7551 : {
7552 : /* If a formerly non-invariant ADDR_EXPR is turned into an
7553 : invariant one it was on a separate stmt. */
7554 11583482 : if (gimple_assign_single_p (stmt)
7555 11583482 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == ADDR_EXPR)
7556 232048 : recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
7557 11583482 : gimple_stmt_iterator prev = *gsi;
7558 11583482 : gsi_prev (&prev);
7559 11583482 : if (fold_stmt (gsi, follow_all_ssa_edges))
7560 : {
7561 : /* fold_stmt may have created new stmts inbetween
7562 : the previous stmt and the folded stmt. Mark
7563 : all defs created there as varying to not confuse
7564 : the SCCVN machinery as we're using that even during
7565 : elimination. */
7566 969399 : if (gsi_end_p (prev))
7567 217520 : prev = gsi_start_bb (b);
7568 : else
7569 860639 : gsi_next (&prev);
7570 969399 : if (gsi_stmt (prev) != gsi_stmt (*gsi))
7571 88205 : do
7572 : {
7573 54948 : tree def;
7574 54948 : ssa_op_iter dit;
7575 105792 : FOR_EACH_SSA_TREE_OPERAND (def, gsi_stmt (prev),
7576 : dit, SSA_OP_ALL_DEFS)
7577 : /* As existing DEFs may move between stmts
7578 : only process new ones. */
7579 50844 : if (! has_VN_INFO (def))
7580 : {
7581 33155 : vn_ssa_aux_t vn_info = VN_INFO (def);
7582 33155 : vn_info->valnum = def;
7583 33155 : vn_info->visited = true;
7584 : }
7585 54948 : if (gsi_stmt (prev) == gsi_stmt (*gsi))
7586 : break;
7587 33257 : gsi_next (&prev);
7588 33257 : }
7589 : while (1);
7590 : }
7591 11583482 : stmt = gsi_stmt (*gsi);
7592 : /* In case we folded the stmt away schedule the NOP for removal. */
7593 11583482 : if (gimple_nop_p (stmt))
7594 811 : to_remove.safe_push (stmt);
7595 : }
7596 :
7597 : /* Visit indirect calls and turn them into direct calls if
7598 : possible using the devirtualization machinery. Do this before
7599 : checking for required EH/abnormal/noreturn cleanup as devird
7600 : may expose more of those. */
7601 337492984 : if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
7602 : {
7603 22280062 : tree fn = gimple_call_fn (call_stmt);
7604 22280062 : if (fn
7605 21536243 : && flag_devirtualize
7606 43073988 : && virtual_method_call_p (fn))
7607 : {
7608 180796 : tree otr_type = obj_type_ref_class (fn);
7609 180796 : unsigned HOST_WIDE_INT otr_tok
7610 180796 : = tree_to_uhwi (OBJ_TYPE_REF_TOKEN (fn));
7611 180796 : tree instance;
7612 180796 : ipa_polymorphic_call_context context (current_function_decl,
7613 180796 : fn, stmt, &instance);
7614 180796 : context.get_dynamic_type (instance, OBJ_TYPE_REF_OBJECT (fn),
7615 : otr_type, stmt, NULL);
7616 180796 : bool final;
7617 180796 : vec <cgraph_node *> targets
7618 180796 : = possible_polymorphic_call_targets (obj_type_ref_class (fn),
7619 : otr_tok, context, &final);
7620 180796 : if (dump_file)
7621 22 : dump_possible_polymorphic_call_targets (dump_file,
7622 : obj_type_ref_class (fn),
7623 : otr_tok, context);
7624 181082 : if (final && targets.length () <= 1 && dbg_cnt (devirt))
7625 : {
7626 64 : tree fn;
7627 64 : if (targets.length () == 1)
7628 64 : fn = targets[0]->decl;
7629 : else
7630 0 : fn = builtin_decl_unreachable ();
7631 64 : if (dump_enabled_p ())
7632 : {
7633 9 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, stmt,
7634 : "converting indirect call to "
7635 : "function %s\n",
7636 9 : lang_hooks.decl_printable_name (fn, 2));
7637 : }
7638 64 : gimple_call_set_fndecl (call_stmt, fn);
7639 : /* If changing the call to __builtin_unreachable
7640 : or similar noreturn function, adjust gimple_call_fntype
7641 : too. */
7642 64 : if (gimple_call_noreturn_p (call_stmt)
7643 0 : && VOID_TYPE_P (TREE_TYPE (TREE_TYPE (fn)))
7644 0 : && TYPE_ARG_TYPES (TREE_TYPE (fn))
7645 64 : && (TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fn)))
7646 0 : == void_type_node))
7647 0 : gimple_call_set_fntype (call_stmt, TREE_TYPE (fn));
7648 64 : maybe_remove_unused_call_args (cfun, call_stmt);
7649 64 : modified = true;
7650 : }
7651 : }
7652 : }
7653 :
7654 337492984 : if (modified)
7655 : {
7656 : /* When changing a call into a noreturn call, cfg cleanup
7657 : is needed to fix up the noreturn call. */
7658 11583503 : if (!was_noreturn
7659 11583503 : && is_gimple_call (stmt) && gimple_call_noreturn_p (stmt))
7660 56 : to_fixup.safe_push (stmt);
7661 : /* When changing a condition or switch into one we know what
7662 : edge will be executed, schedule a cfg cleanup. */
7663 11583503 : if ((gimple_code (stmt) == GIMPLE_COND
7664 1486591 : && (gimple_cond_true_p (as_a <gcond *> (stmt))
7665 1481307 : || gimple_cond_false_p (as_a <gcond *> (stmt))))
7666 13062493 : || (gimple_code (stmt) == GIMPLE_SWITCH
7667 7378 : && TREE_CODE (gimple_switch_index
7668 : (as_a <gswitch *> (stmt))) == INTEGER_CST))
7669 9386 : el_todo |= TODO_cleanup_cfg;
7670 : /* If we removed EH side-effects from the statement, clean
7671 : its EH information. */
7672 11583503 : if (maybe_clean_or_replace_eh_stmt (old_stmt, stmt))
7673 : {
7674 1649 : bitmap_set_bit (need_eh_cleanup,
7675 1649 : gimple_bb (stmt)->index);
7676 1649 : if (dump_file && (dump_flags & TDF_DETAILS))
7677 0 : fprintf (dump_file, " Removed EH side-effects.\n");
7678 : }
7679 : /* Likewise for AB side-effects. */
7680 11583503 : if (can_make_abnormal_goto
7681 11583503 : && !stmt_can_make_abnormal_goto (stmt))
7682 : {
7683 0 : bitmap_set_bit (need_ab_cleanup,
7684 0 : gimple_bb (stmt)->index);
7685 0 : if (dump_file && (dump_flags & TDF_DETAILS))
7686 0 : fprintf (dump_file, " Removed AB side-effects.\n");
7687 : }
7688 11583503 : update_stmt (stmt);
7689 : /* In case the VDEF on the original stmt was released, value-number
7690 : it to the VUSE. This is to make vuse_ssa_val able to skip
7691 : released virtual operands. */
7692 14779775 : if (vdef && SSA_NAME_IN_FREE_LIST (vdef))
7693 1857 : VN_INFO (vdef)->valnum = vuse;
7694 : }
7695 :
7696 : /* Make new values available - for fully redundant LHS we
7697 : continue with the next stmt above and skip this.
7698 : But avoid picking up dead defs. */
7699 337492984 : tree def;
7700 407462203 : FOR_EACH_SSA_TREE_OPERAND (def, stmt, iter, SSA_OP_DEF)
7701 69969219 : if (! has_zero_uses (def)
7702 69969219 : || (inserted_exprs
7703 206718 : && bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (def))))
7704 68593392 : eliminate_push_avail (b, def);
7705 : }
7706 :
7707 : /* Perform elimination for the basic-block B during the domwalk. */
7708 :
7709 : edge
7710 41047760 : eliminate_dom_walker::before_dom_children (basic_block b)
7711 : {
7712 : /* Mark new bb. */
7713 41047760 : avail_stack.safe_push (NULL_TREE);
7714 :
7715 : /* Skip unreachable blocks marked unreachable during the SCCVN domwalk. */
7716 41047760 : if (!(b->flags & BB_EXECUTABLE))
7717 : return NULL;
7718 :
7719 36257278 : vn_context_bb = b;
7720 :
7721 47644673 : for (gphi_iterator gsi = gsi_start_phis (b); !gsi_end_p (gsi);)
7722 : {
7723 11387395 : gphi *phi = gsi.phi ();
7724 11387395 : tree res = PHI_RESULT (phi);
7725 :
7726 22774790 : if (virtual_operand_p (res))
7727 : {
7728 5235858 : gsi_next (&gsi);
7729 5235858 : continue;
7730 : }
7731 :
7732 6151537 : tree sprime = eliminate_avail (b, res);
7733 6151537 : if (sprime
7734 6151537 : && sprime != res)
7735 : {
7736 431616 : if (dump_file && (dump_flags & TDF_DETAILS))
7737 : {
7738 20 : fprintf (dump_file, "Replaced redundant PHI node defining ");
7739 20 : print_generic_expr (dump_file, res);
7740 20 : fprintf (dump_file, " with ");
7741 20 : print_generic_expr (dump_file, sprime);
7742 20 : fprintf (dump_file, "\n");
7743 : }
7744 :
7745 : /* If we inserted this PHI node ourself, it's not an elimination. */
7746 431616 : if (! inserted_exprs
7747 547138 : || ! bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (res)))
7748 406003 : eliminations++;
7749 :
7750 : /* If we will propagate into all uses don't bother to do
7751 : anything. */
7752 431616 : if (may_propagate_copy (res, sprime))
7753 : {
7754 : /* Mark the PHI for removal. */
7755 431616 : to_remove.safe_push (phi);
7756 431616 : gsi_next (&gsi);
7757 431616 : continue;
7758 : }
7759 :
7760 0 : remove_phi_node (&gsi, false);
7761 :
7762 0 : if (!useless_type_conversion_p (TREE_TYPE (res), TREE_TYPE (sprime)))
7763 0 : sprime = fold_convert (TREE_TYPE (res), sprime);
7764 0 : gimple *stmt = gimple_build_assign (res, sprime);
7765 0 : gimple_stmt_iterator gsi2 = gsi_after_labels (b);
7766 0 : gsi_insert_before (&gsi2, stmt, GSI_NEW_STMT);
7767 0 : continue;
7768 0 : }
7769 :
7770 5719921 : eliminate_push_avail (b, res);
7771 5719921 : gsi_next (&gsi);
7772 : }
7773 :
7774 72514556 : for (gimple_stmt_iterator gsi = gsi_start_bb (b);
7775 280627379 : !gsi_end_p (gsi);
7776 244370101 : gsi_next (&gsi))
7777 244370101 : eliminate_stmt (b, &gsi);
7778 :
7779 : /* Replace destination PHI arguments. */
7780 36257278 : edge_iterator ei;
7781 36257278 : edge e;
7782 85549888 : FOR_EACH_EDGE (e, ei, b->succs)
7783 49292610 : if (e->flags & EDGE_EXECUTABLE)
7784 48753244 : for (gphi_iterator gsi = gsi_start_phis (e->dest);
7785 78039364 : !gsi_end_p (gsi);
7786 29286120 : gsi_next (&gsi))
7787 : {
7788 29286120 : gphi *phi = gsi.phi ();
7789 29286120 : use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
7790 29286120 : tree arg = USE_FROM_PTR (use_p);
7791 48434200 : if (TREE_CODE (arg) != SSA_NAME
7792 29286120 : || virtual_operand_p (arg))
7793 19148080 : continue;
7794 10138040 : tree sprime = eliminate_avail (b, arg);
7795 20276080 : if (sprime && may_propagate_copy (arg, sprime,
7796 10138040 : !(e->flags & EDGE_ABNORMAL)))
7797 10125814 : propagate_value (use_p, sprime);
7798 : }
7799 :
7800 36257278 : vn_context_bb = NULL;
7801 :
7802 36257278 : return NULL;
7803 : }
7804 :
7805 : /* Make no longer available leaders no longer available. */
7806 :
7807 : void
7808 41047760 : eliminate_dom_walker::after_dom_children (basic_block)
7809 : {
7810 41047760 : tree entry;
7811 90911854 : while ((entry = avail_stack.pop ()) != NULL_TREE)
7812 : {
7813 49864094 : tree valnum = VN_INFO (entry)->valnum;
7814 49864094 : tree old = avail[SSA_NAME_VERSION (valnum)];
7815 49864094 : if (old == entry)
7816 49819742 : avail[SSA_NAME_VERSION (valnum)] = NULL_TREE;
7817 : else
7818 44352 : avail[SSA_NAME_VERSION (valnum)] = entry;
7819 : }
7820 41047760 : }
7821 :
7822 : /* Remove queued stmts and perform delayed cleanups. */
7823 :
7824 : unsigned
7825 6128251 : eliminate_dom_walker::eliminate_cleanup (bool region_p)
7826 : {
7827 6128251 : statistics_counter_event (cfun, "Eliminated", eliminations);
7828 6128251 : statistics_counter_event (cfun, "Insertions", insertions);
7829 :
7830 : /* We cannot remove stmts during BB walk, especially not release SSA
7831 : names there as this confuses the VN machinery. The stmts ending
7832 : up in to_remove are either stores or simple copies.
7833 : Remove stmts in reverse order to make debug stmt creation possible. */
7834 33164868 : while (!to_remove.is_empty ())
7835 : {
7836 14780059 : bool do_release_defs = true;
7837 14780059 : gimple *stmt = to_remove.pop ();
7838 :
7839 : /* When we are value-numbering a region we do not require exit PHIs to
7840 : be present so we have to make sure to deal with uses outside of the
7841 : region of stmts that we thought are eliminated.
7842 : ??? Note we may be confused by uses in dead regions we didn't run
7843 : elimination on. Rather than checking individual uses we accept
7844 : dead copies to be generated here (gcc.c-torture/execute/20060905-1.c
7845 : contains such example). */
7846 14780059 : if (region_p)
7847 : {
7848 1705457 : if (gphi *phi = dyn_cast <gphi *> (stmt))
7849 : {
7850 1096220 : tree lhs = gimple_phi_result (phi);
7851 1096220 : if (!has_zero_uses (lhs))
7852 : {
7853 22793 : if (dump_file && (dump_flags & TDF_DETAILS))
7854 3 : fprintf (dump_file, "Keeping eliminated stmt live "
7855 : "as copy because of out-of-region uses\n");
7856 22793 : tree sprime = eliminate_avail (gimple_bb (stmt), lhs);
7857 22793 : gimple *copy = gimple_build_assign (lhs, sprime);
7858 22793 : gimple_stmt_iterator gsi
7859 22793 : = gsi_after_labels (gimple_bb (stmt));
7860 22793 : gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
7861 22793 : do_release_defs = false;
7862 : }
7863 : }
7864 609237 : else if (tree lhs = gimple_get_lhs (stmt))
7865 609237 : if (TREE_CODE (lhs) == SSA_NAME
7866 609237 : && !has_zero_uses (lhs))
7867 : {
7868 1652 : if (dump_file && (dump_flags & TDF_DETAILS))
7869 0 : fprintf (dump_file, "Keeping eliminated stmt live "
7870 : "as copy because of out-of-region uses\n");
7871 1652 : tree sprime = eliminate_avail (gimple_bb (stmt), lhs);
7872 1652 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
7873 1652 : if (is_gimple_assign (stmt))
7874 : {
7875 1652 : gimple_assign_set_rhs_from_tree (&gsi, sprime);
7876 1652 : stmt = gsi_stmt (gsi);
7877 1652 : update_stmt (stmt);
7878 1652 : if (maybe_clean_or_replace_eh_stmt (stmt, stmt))
7879 0 : bitmap_set_bit (need_eh_cleanup, gimple_bb (stmt)->index);
7880 1652 : continue;
7881 : }
7882 : else
7883 : {
7884 0 : gimple *copy = gimple_build_assign (lhs, sprime);
7885 0 : gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
7886 0 : do_release_defs = false;
7887 : }
7888 : }
7889 : }
7890 :
7891 14778407 : if (dump_file && (dump_flags & TDF_DETAILS))
7892 : {
7893 21791 : fprintf (dump_file, "Removing dead stmt ");
7894 21791 : print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
7895 : }
7896 :
7897 14778407 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
7898 14778407 : if (gimple_code (stmt) == GIMPLE_PHI)
7899 1699180 : remove_phi_node (&gsi, do_release_defs);
7900 : else
7901 : {
7902 13079227 : basic_block bb = gimple_bb (stmt);
7903 13079227 : unlink_stmt_vdef (stmt);
7904 13079227 : if (gsi_remove (&gsi, true))
7905 26543 : bitmap_set_bit (need_eh_cleanup, bb->index);
7906 13079227 : if (is_gimple_call (stmt) && stmt_can_make_abnormal_goto (stmt))
7907 2 : bitmap_set_bit (need_ab_cleanup, bb->index);
7908 13079227 : if (do_release_defs)
7909 13079227 : release_defs (stmt);
7910 : }
7911 :
7912 : /* Removing a stmt may expose a forwarder block. */
7913 14778407 : el_todo |= TODO_cleanup_cfg;
7914 : }
7915 :
7916 : /* Fixup stmts that became noreturn calls. This may require splitting
7917 : blocks and thus isn't possible during the dominator walk. Do this
7918 : in reverse order so we don't inadvertedly remove a stmt we want to
7919 : fixup by visiting a dominating now noreturn call first. */
7920 6128307 : while (!to_fixup.is_empty ())
7921 : {
7922 56 : gimple *stmt = to_fixup.pop ();
7923 :
7924 56 : if (dump_file && (dump_flags & TDF_DETAILS))
7925 : {
7926 0 : fprintf (dump_file, "Fixing up noreturn call ");
7927 0 : print_gimple_stmt (dump_file, stmt, 0);
7928 : }
7929 :
7930 56 : if (fixup_noreturn_call (stmt))
7931 56 : el_todo |= TODO_cleanup_cfg;
7932 : }
7933 :
7934 6128251 : bool do_eh_cleanup = !bitmap_empty_p (need_eh_cleanup);
7935 6128251 : bool do_ab_cleanup = !bitmap_empty_p (need_ab_cleanup);
7936 :
7937 6128251 : if (do_eh_cleanup)
7938 10640 : gimple_purge_all_dead_eh_edges (need_eh_cleanup);
7939 :
7940 6128251 : if (do_ab_cleanup)
7941 2 : gimple_purge_all_dead_abnormal_call_edges (need_ab_cleanup);
7942 :
7943 6128251 : if (do_eh_cleanup || do_ab_cleanup)
7944 10642 : el_todo |= TODO_cleanup_cfg;
7945 :
7946 6128251 : return el_todo;
7947 : }
7948 :
7949 : /* Eliminate fully redundant computations. */
7950 :
7951 : unsigned
7952 4260968 : eliminate_with_rpo_vn (bitmap inserted_exprs)
7953 : {
7954 4260968 : eliminate_dom_walker walker (CDI_DOMINATORS, inserted_exprs);
7955 :
7956 4260968 : eliminate_dom_walker *saved_rpo_avail = rpo_avail;
7957 4260968 : rpo_avail = &walker;
7958 4260968 : walker.walk (cfun->cfg->x_entry_block_ptr);
7959 4260968 : rpo_avail = saved_rpo_avail;
7960 :
7961 4260968 : return walker.eliminate_cleanup ();
7962 4260968 : }
7963 :
7964 : static unsigned
7965 : do_rpo_vn_1 (function *fn, edge entry, bitmap exit_bbs,
7966 : bool iterate, bool eliminate, bool skip_entry_phis,
7967 : vn_lookup_kind kind);
7968 :
7969 : void
7970 966285 : run_rpo_vn (vn_lookup_kind kind)
7971 : {
7972 966285 : do_rpo_vn_1 (cfun, NULL, NULL, true, false, false, kind);
7973 :
7974 : /* ??? Prune requirement of these. */
7975 966285 : constant_to_value_id = new hash_table<vn_constant_hasher> (23);
7976 :
7977 : /* Initialize the value ids and prune out remaining VN_TOPs
7978 : from dead code. */
7979 966285 : tree name;
7980 966285 : unsigned i;
7981 46894443 : FOR_EACH_SSA_NAME (i, name, cfun)
7982 : {
7983 33429444 : vn_ssa_aux_t info = VN_INFO (name);
7984 33429444 : if (!info->visited
7985 33353476 : || info->valnum == VN_TOP)
7986 75968 : info->valnum = name;
7987 33429444 : if (info->valnum == name)
7988 32292535 : info->value_id = get_next_value_id ();
7989 1136909 : else if (is_gimple_min_invariant (info->valnum))
7990 38309 : info->value_id = get_or_alloc_constant_value_id (info->valnum);
7991 : }
7992 :
7993 : /* Propagate. */
7994 46894443 : FOR_EACH_SSA_NAME (i, name, cfun)
7995 : {
7996 33429444 : vn_ssa_aux_t info = VN_INFO (name);
7997 33429444 : if (TREE_CODE (info->valnum) == SSA_NAME
7998 33391135 : && info->valnum != name
7999 34528044 : && info->value_id != VN_INFO (info->valnum)->value_id)
8000 1098600 : info->value_id = VN_INFO (info->valnum)->value_id;
8001 : }
8002 :
8003 966285 : set_hashtable_value_ids ();
8004 :
8005 966285 : if (dump_file && (dump_flags & TDF_DETAILS))
8006 : {
8007 14 : fprintf (dump_file, "Value numbers:\n");
8008 406 : FOR_EACH_SSA_NAME (i, name, cfun)
8009 : {
8010 307 : if (VN_INFO (name)->visited
8011 307 : && SSA_VAL (name) != name)
8012 : {
8013 33 : print_generic_expr (dump_file, name);
8014 33 : fprintf (dump_file, " = ");
8015 33 : print_generic_expr (dump_file, SSA_VAL (name));
8016 33 : fprintf (dump_file, " (%04d)\n", VN_INFO (name)->value_id);
8017 : }
8018 : }
8019 : }
8020 966285 : }
8021 :
8022 : /* Free VN associated data structures. */
8023 :
8024 : void
8025 6147805 : free_rpo_vn (void)
8026 : {
8027 6147805 : free_vn_table (valid_info);
8028 6147805 : XDELETE (valid_info);
8029 6147805 : obstack_free (&vn_tables_obstack, NULL);
8030 6147805 : obstack_free (&vn_tables_insert_obstack, NULL);
8031 :
8032 6147805 : vn_ssa_aux_iterator_type it;
8033 6147805 : vn_ssa_aux_t info;
8034 348658681 : FOR_EACH_HASH_TABLE_ELEMENT (*vn_ssa_aux_hash, info, vn_ssa_aux_t, it)
8035 171255438 : if (info->needs_insertion)
8036 4113833 : release_ssa_name (info->name);
8037 6147805 : obstack_free (&vn_ssa_aux_obstack, NULL);
8038 6147805 : delete vn_ssa_aux_hash;
8039 :
8040 6147805 : delete constant_to_value_id;
8041 6147805 : constant_to_value_id = NULL;
8042 6147805 : }
8043 :
8044 : /* Hook for maybe_push_res_to_seq, lookup the expression in the VN tables. */
8045 :
8046 : static tree
8047 22964896 : vn_lookup_simplify_result (gimple_match_op *res_op)
8048 : {
8049 22964896 : if (!res_op->code.is_tree_code ())
8050 : return NULL_TREE;
8051 22961736 : tree *ops = res_op->ops;
8052 22961736 : unsigned int length = res_op->num_ops;
8053 22961736 : if (res_op->code == CONSTRUCTOR
8054 : /* ??? We're arriving here with SCCVNs view, decomposed CONSTRUCTOR
8055 : and GIMPLEs / match-and-simplifies, CONSTRUCTOR as GENERIC tree. */
8056 22961736 : && TREE_CODE (res_op->ops[0]) == CONSTRUCTOR)
8057 : {
8058 1002 : length = CONSTRUCTOR_NELTS (res_op->ops[0]);
8059 1002 : ops = XALLOCAVEC (tree, length);
8060 4574 : for (unsigned i = 0; i < length; ++i)
8061 3572 : ops[i] = CONSTRUCTOR_ELT (res_op->ops[0], i)->value;
8062 : }
8063 22961736 : vn_nary_op_t vnresult = NULL;
8064 22961736 : tree res = vn_nary_op_lookup_pieces (length, (tree_code) res_op->code,
8065 : res_op->type, ops, &vnresult);
8066 : /* If this is used from expression simplification make sure to
8067 : return an available expression. */
8068 22961736 : if (res && TREE_CODE (res) == SSA_NAME && mprts_hook && rpo_avail)
8069 2231167 : res = rpo_avail->eliminate_avail (vn_context_bb, res);
8070 : return res;
8071 : }
8072 :
8073 : /* Return a leader for OPs value that is valid at BB. */
8074 :
8075 : tree
8076 266599645 : rpo_elim::eliminate_avail (basic_block bb, tree op)
8077 : {
8078 266599645 : bool visited;
8079 266599645 : tree valnum = SSA_VAL (op, &visited);
8080 : /* If we didn't visit OP then it must be defined outside of the
8081 : region we process and also dominate it. So it is available. */
8082 266599645 : if (!visited)
8083 : return op;
8084 264451970 : if (TREE_CODE (valnum) == SSA_NAME)
8085 : {
8086 250481181 : if (SSA_NAME_IS_DEFAULT_DEF (valnum))
8087 : return valnum;
8088 243802519 : vn_ssa_aux_t valnum_info = VN_INFO (valnum);
8089 243802519 : vn_avail *av = valnum_info->avail;
8090 243802519 : if (!av)
8091 : {
8092 : /* See above. But when there's availability info prefer
8093 : what we recorded there for example to preserve LC SSA. */
8094 83280580 : if (!valnum_info->visited)
8095 : return valnum;
8096 : return NULL_TREE;
8097 : }
8098 160521939 : if (av->location == bb->index)
8099 : /* On tramp3d 90% of the cases are here. */
8100 106013978 : return ssa_name (av->leader);
8101 68475005 : do
8102 : {
8103 68475005 : basic_block abb = BASIC_BLOCK_FOR_FN (cfun, av->location);
8104 : /* ??? During elimination we have to use availability at the
8105 : definition site of a use we try to replace. This
8106 : is required to not run into inconsistencies because
8107 : of dominated_by_p_w_unex behavior and removing a definition
8108 : while not replacing all uses.
8109 : ??? We could try to consistently walk dominators
8110 : ignoring non-executable regions. The nearest common
8111 : dominator of bb and abb is where we can stop walking. We
8112 : may also be able to "pre-compute" (bits of) the next immediate
8113 : (non-)dominator during the RPO walk when marking edges as
8114 : executable. */
8115 68475005 : if (dominated_by_p_w_unex (bb, abb, true))
8116 : {
8117 50643121 : tree leader = ssa_name (av->leader);
8118 : /* Prevent eliminations that break loop-closed SSA. */
8119 50643121 : if (loops_state_satisfies_p (LOOP_CLOSED_SSA)
8120 3103176 : && ! SSA_NAME_IS_DEFAULT_DEF (leader)
8121 53746297 : && ! flow_bb_inside_loop_p (gimple_bb (SSA_NAME_DEF_STMT
8122 3103176 : (leader))->loop_father,
8123 : bb))
8124 : return NULL_TREE;
8125 50564556 : if (dump_file && (dump_flags & TDF_DETAILS))
8126 : {
8127 3540 : print_generic_expr (dump_file, leader);
8128 3540 : fprintf (dump_file, " is available for ");
8129 3540 : print_generic_expr (dump_file, valnum);
8130 3540 : fprintf (dump_file, "\n");
8131 : }
8132 : /* On tramp3d 99% of the _remaining_ cases succeed at
8133 : the first enty. */
8134 50564556 : return leader;
8135 : }
8136 : /* ??? Can we somehow skip to the immediate dominator
8137 : RPO index (bb_to_rpo)? Again, maybe not worth, on
8138 : tramp3d the worst number of elements in the vector is 9. */
8139 17831884 : av = av->next;
8140 : }
8141 17831884 : while (av);
8142 : /* While we prefer avail we have to fallback to using the value
8143 : directly if defined outside of the region when none of the
8144 : available defs suit. */
8145 3864840 : if (!valnum_info->visited)
8146 : return valnum;
8147 : }
8148 13970789 : else if (valnum != VN_TOP)
8149 : /* valnum is is_gimple_min_invariant. */
8150 : return valnum;
8151 : return NULL_TREE;
8152 : }
8153 :
8154 : /* Make LEADER a leader for its value at BB. */
8155 :
8156 : void
8157 96578182 : rpo_elim::eliminate_push_avail (basic_block bb, tree leader)
8158 : {
8159 96578182 : tree valnum = VN_INFO (leader)->valnum;
8160 96578182 : if (valnum == VN_TOP
8161 96578182 : || is_gimple_min_invariant (valnum))
8162 0 : return;
8163 96578182 : if (dump_file && (dump_flags & TDF_DETAILS))
8164 : {
8165 323998 : fprintf (dump_file, "Making available beyond BB%d ", bb->index);
8166 323998 : print_generic_expr (dump_file, leader);
8167 323998 : fprintf (dump_file, " for value ");
8168 323998 : print_generic_expr (dump_file, valnum);
8169 323998 : fprintf (dump_file, "\n");
8170 : }
8171 96578182 : vn_ssa_aux_t value = VN_INFO (valnum);
8172 96578182 : vn_avail *av;
8173 96578182 : if (m_avail_freelist)
8174 : {
8175 18761539 : av = m_avail_freelist;
8176 18761539 : m_avail_freelist = m_avail_freelist->next;
8177 : }
8178 : else
8179 77816643 : av = XOBNEW (&vn_ssa_aux_obstack, vn_avail);
8180 96578182 : av->location = bb->index;
8181 96578182 : av->leader = SSA_NAME_VERSION (leader);
8182 96578182 : av->next = value->avail;
8183 96578182 : av->next_undo = last_pushed_avail;
8184 96578182 : last_pushed_avail = value;
8185 96578182 : value->avail = av;
8186 : }
8187 :
8188 : /* Valueization hook for RPO VN plus required state. */
8189 :
8190 : tree
8191 2052832946 : rpo_vn_valueize (tree name)
8192 : {
8193 2052832946 : if (TREE_CODE (name) == SSA_NAME)
8194 : {
8195 2007090344 : vn_ssa_aux_t val = VN_INFO (name);
8196 2007090344 : if (val)
8197 : {
8198 2007090344 : tree tem = val->valnum;
8199 2007090344 : if (tem != VN_TOP && tem != name)
8200 : {
8201 107985355 : if (TREE_CODE (tem) != SSA_NAME)
8202 : return tem;
8203 : /* For all values we only valueize to an available leader
8204 : which means we can use SSA name info without restriction. */
8205 91106749 : tem = rpo_avail->eliminate_avail (vn_context_bb, tem);
8206 91106749 : if (tem)
8207 : return tem;
8208 : }
8209 : }
8210 : }
8211 : return name;
8212 : }
8213 :
8214 : /* Insert on PRED_E predicates derived from CODE OPS being true besides the
8215 : inverted condition. */
8216 :
8217 : static void
8218 27304212 : insert_related_predicates_on_edge (enum tree_code code, tree *ops, edge pred_e)
8219 : {
8220 27304212 : switch (code)
8221 : {
8222 1368398 : case LT_EXPR:
8223 : /* a < b -> a {!,<}= b */
8224 1368398 : vn_nary_op_insert_pieces_predicated (2, NE_EXPR, boolean_type_node,
8225 : ops, boolean_true_node, 0, pred_e);
8226 1368398 : vn_nary_op_insert_pieces_predicated (2, LE_EXPR, boolean_type_node,
8227 : ops, boolean_true_node, 0, pred_e);
8228 : /* a < b -> ! a {>,=} b */
8229 1368398 : vn_nary_op_insert_pieces_predicated (2, GT_EXPR, boolean_type_node,
8230 : ops, boolean_false_node, 0, pred_e);
8231 1368398 : vn_nary_op_insert_pieces_predicated (2, EQ_EXPR, boolean_type_node,
8232 : ops, boolean_false_node, 0, pred_e);
8233 1368398 : break;
8234 3436990 : case GT_EXPR:
8235 : /* a > b -> a {!,>}= b */
8236 3436990 : vn_nary_op_insert_pieces_predicated (2, NE_EXPR, boolean_type_node,
8237 : ops, boolean_true_node, 0, pred_e);
8238 3436990 : vn_nary_op_insert_pieces_predicated (2, GE_EXPR, boolean_type_node,
8239 : ops, boolean_true_node, 0, pred_e);
8240 : /* a > b -> ! a {<,=} b */
8241 3436990 : vn_nary_op_insert_pieces_predicated (2, LT_EXPR, boolean_type_node,
8242 : ops, boolean_false_node, 0, pred_e);
8243 3436990 : vn_nary_op_insert_pieces_predicated (2, EQ_EXPR, boolean_type_node,
8244 : ops, boolean_false_node, 0, pred_e);
8245 3436990 : break;
8246 9334699 : case EQ_EXPR:
8247 : /* a == b -> ! a {<,>} b */
8248 9334699 : vn_nary_op_insert_pieces_predicated (2, LT_EXPR, boolean_type_node,
8249 : ops, boolean_false_node, 0, pred_e);
8250 9334699 : vn_nary_op_insert_pieces_predicated (2, GT_EXPR, boolean_type_node,
8251 : ops, boolean_false_node, 0, pred_e);
8252 9334699 : break;
8253 : case LE_EXPR:
8254 : case GE_EXPR:
8255 : case NE_EXPR:
8256 : /* Nothing besides inverted condition. */
8257 : break;
8258 27304212 : default:;
8259 : }
8260 27304212 : }
8261 :
8262 : /* Insert on the TRUE_E true and FALSE_E false predicates
8263 : derived from LHS CODE RHS. */
8264 :
8265 : static void
8266 23303817 : insert_predicates_for_cond (tree_code code, tree lhs, tree rhs,
8267 : edge true_e, edge false_e)
8268 : {
8269 : /* If both edges are null, then there is nothing to be done. */
8270 23303817 : if (!true_e && !false_e)
8271 1314519 : return;
8272 :
8273 : /* Canonicalize the comparison if needed, putting
8274 : the constant in the rhs. */
8275 21992795 : if (tree_swap_operands_p (lhs, rhs))
8276 : {
8277 16632 : std::swap (lhs, rhs);
8278 16632 : code = swap_tree_comparison (code);
8279 : }
8280 :
8281 : /* If the lhs is not a ssa name, don't record anything. */
8282 21992795 : if (TREE_CODE (lhs) != SSA_NAME)
8283 : return;
8284 :
8285 21989298 : tree_code icode = invert_tree_comparison (code, HONOR_NANS (lhs));
8286 21989298 : tree ops[2];
8287 21989298 : ops[0] = lhs;
8288 21989298 : ops[1] = rhs;
8289 21989298 : if (true_e)
8290 17958669 : vn_nary_op_insert_pieces_predicated (2, code, boolean_type_node, ops,
8291 : boolean_true_node, 0, true_e);
8292 21989298 : if (false_e)
8293 16934090 : vn_nary_op_insert_pieces_predicated (2, code, boolean_type_node, ops,
8294 : boolean_false_node, 0, false_e);
8295 21989298 : if (icode != ERROR_MARK)
8296 : {
8297 21741889 : if (true_e)
8298 17804492 : vn_nary_op_insert_pieces_predicated (2, icode, boolean_type_node, ops,
8299 : boolean_false_node, 0, true_e);
8300 21741889 : if (false_e)
8301 16734091 : vn_nary_op_insert_pieces_predicated (2, icode, boolean_type_node, ops,
8302 : boolean_true_node, 0, false_e);
8303 : }
8304 : /* Relax for non-integers, inverted condition handled
8305 : above. */
8306 21989298 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
8307 : {
8308 17197959 : if (true_e)
8309 14119464 : insert_related_predicates_on_edge (code, ops, true_e);
8310 17197959 : if (false_e)
8311 13184748 : insert_related_predicates_on_edge (icode, ops, false_e);
8312 : }
8313 21989298 : if (integer_zerop (rhs)
8314 21989298 : && (code == NE_EXPR || code == EQ_EXPR))
8315 : {
8316 9160094 : gimple *def_stmt = SSA_NAME_DEF_STMT (lhs);
8317 : /* (A CMP B) != 0 is the same as (A CMP B).
8318 : (A CMP B) == 0 is just (A CMP B) with the edges swapped. */
8319 9160094 : if (is_gimple_assign (def_stmt)
8320 9160094 : && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_comparison)
8321 : {
8322 432630 : tree_code nc = gimple_assign_rhs_code (def_stmt);
8323 432630 : tree nlhs = vn_valueize (gimple_assign_rhs1 (def_stmt));
8324 432630 : tree nrhs = vn_valueize (gimple_assign_rhs2 (def_stmt));
8325 432630 : edge nt = true_e;
8326 432630 : edge nf = false_e;
8327 432630 : if (code == EQ_EXPR)
8328 306666 : std::swap (nt, nf);
8329 432630 : if (lhs != nlhs)
8330 432630 : insert_predicates_for_cond (nc, nlhs, nrhs, nt, nf);
8331 : }
8332 : /* (a | b) == 0 ->
8333 : on true edge assert: a == 0 & b == 0. */
8334 : /* (a | b) != 0 ->
8335 : on false edge assert: a == 0 & b == 0. */
8336 9160094 : if (is_gimple_assign (def_stmt)
8337 9160094 : && gimple_assign_rhs_code (def_stmt) == BIT_IOR_EXPR)
8338 : {
8339 256275 : edge e = code == EQ_EXPR ? true_e : false_e;
8340 256275 : tree nlhs;
8341 :
8342 256275 : nlhs = vn_valueize (gimple_assign_rhs1 (def_stmt));
8343 : /* A valueization of the `a` might return the old lhs
8344 : which is already handled above. */
8345 256275 : if (nlhs != lhs)
8346 256275 : insert_predicates_for_cond (EQ_EXPR, nlhs, rhs, e, nullptr);
8347 :
8348 : /* A valueization of the `b` might return the old lhs
8349 : which is already handled above. */
8350 256275 : nlhs = vn_valueize (gimple_assign_rhs2 (def_stmt));
8351 256275 : if (nlhs != lhs)
8352 256275 : insert_predicates_for_cond (EQ_EXPR, nlhs, rhs, e, nullptr);
8353 : }
8354 : }
8355 : }
8356 :
8357 : /* Main stmt worker for RPO VN, process BB. */
8358 :
8359 : static unsigned
8360 61197303 : process_bb (rpo_elim &avail, basic_block bb,
8361 : bool bb_visited, bool iterate_phis, bool iterate, bool eliminate,
8362 : bool do_region, bitmap exit_bbs, bool skip_phis)
8363 : {
8364 61197303 : unsigned todo = 0;
8365 61197303 : edge_iterator ei;
8366 61197303 : edge e;
8367 :
8368 61197303 : vn_context_bb = bb;
8369 :
8370 : /* If we are in loop-closed SSA preserve this state. This is
8371 : relevant when called on regions from outside of FRE/PRE. */
8372 61197303 : bool lc_phi_nodes = false;
8373 61197303 : if (!skip_phis
8374 61197303 : && loops_state_satisfies_p (LOOP_CLOSED_SSA))
8375 3693169 : FOR_EACH_EDGE (e, ei, bb->preds)
8376 2230491 : if (e->src->loop_father != e->dest->loop_father
8377 2230491 : && flow_loop_nested_p (e->dest->loop_father,
8378 : e->src->loop_father))
8379 : {
8380 : lc_phi_nodes = true;
8381 : break;
8382 : }
8383 :
8384 : /* When we visit a loop header substitute into loop info. */
8385 61197303 : if (!iterate && eliminate && bb->loop_father->header == bb)
8386 : {
8387 : /* Keep fields in sync with substitute_in_loop_info. */
8388 944310 : if (bb->loop_father->nb_iterations)
8389 156228 : bb->loop_father->nb_iterations
8390 156228 : = simplify_replace_tree (bb->loop_father->nb_iterations,
8391 : NULL_TREE, NULL_TREE, &vn_valueize_for_srt);
8392 : }
8393 :
8394 : /* Value-number all defs in the basic-block. */
8395 61197303 : if (!skip_phis)
8396 87782969 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
8397 26614877 : gsi_next (&gsi))
8398 : {
8399 26614877 : gphi *phi = gsi.phi ();
8400 26614877 : tree res = PHI_RESULT (phi);
8401 26614877 : vn_ssa_aux_t res_info = VN_INFO (res);
8402 26614877 : if (!bb_visited)
8403 : {
8404 18755577 : gcc_assert (!res_info->visited);
8405 18755577 : res_info->valnum = VN_TOP;
8406 18755577 : res_info->visited = true;
8407 : }
8408 :
8409 : /* When not iterating force backedge values to varying. */
8410 26614877 : visit_stmt (phi, !iterate_phis);
8411 53229754 : if (virtual_operand_p (res))
8412 10545836 : continue;
8413 :
8414 : /* Eliminate */
8415 : /* The interesting case is gcc.dg/tree-ssa/pr22230.c for correctness
8416 : how we handle backedges and availability.
8417 : And gcc.dg/tree-ssa/ssa-sccvn-2.c for optimization. */
8418 16069041 : tree val = res_info->valnum;
8419 16069041 : if (res != val && !iterate && eliminate)
8420 : {
8421 1392040 : if (tree leader = avail.eliminate_avail (bb, res))
8422 : {
8423 1268134 : if (leader != res
8424 : /* Preserve loop-closed SSA form. */
8425 1268134 : && (! lc_phi_nodes
8426 6386 : || is_gimple_min_invariant (leader)))
8427 : {
8428 1267564 : if (dump_file && (dump_flags & TDF_DETAILS))
8429 : {
8430 203 : fprintf (dump_file, "Replaced redundant PHI node "
8431 : "defining ");
8432 203 : print_generic_expr (dump_file, res);
8433 203 : fprintf (dump_file, " with ");
8434 203 : print_generic_expr (dump_file, leader);
8435 203 : fprintf (dump_file, "\n");
8436 : }
8437 1267564 : avail.eliminations++;
8438 :
8439 1267564 : if (may_propagate_copy (res, leader))
8440 : {
8441 : /* Schedule for removal. */
8442 1267564 : avail.to_remove.safe_push (phi);
8443 1267564 : continue;
8444 : }
8445 : /* ??? Else generate a copy stmt. */
8446 : }
8447 : }
8448 : }
8449 : /* Only make defs available that not already are. But make
8450 : sure loop-closed SSA PHI node defs are picked up for
8451 : downstream uses. */
8452 14801477 : if (lc_phi_nodes
8453 14801477 : || res == val
8454 14801477 : || ! avail.eliminate_avail (bb, res))
8455 11303245 : avail.eliminate_push_avail (bb, res);
8456 : }
8457 :
8458 : /* For empty BBs mark outgoing edges executable. For non-empty BBs
8459 : we do this when processing the last stmt as we have to do this
8460 : before elimination which otherwise forces GIMPLE_CONDs to
8461 : if (1 != 0) style when seeing non-executable edges. */
8462 122394606 : if (gsi_end_p (gsi_start_bb (bb)))
8463 : {
8464 13982096 : FOR_EACH_EDGE (e, ei, bb->succs)
8465 : {
8466 6991048 : if (!(e->flags & EDGE_EXECUTABLE))
8467 : {
8468 4750011 : if (dump_file && (dump_flags & TDF_DETAILS))
8469 6187 : fprintf (dump_file,
8470 : "marking outgoing edge %d -> %d executable\n",
8471 6187 : e->src->index, e->dest->index);
8472 4750011 : e->flags |= EDGE_EXECUTABLE;
8473 4750011 : e->dest->flags |= BB_EXECUTABLE;
8474 : }
8475 2241037 : else if (!(e->dest->flags & BB_EXECUTABLE))
8476 : {
8477 0 : if (dump_file && (dump_flags & TDF_DETAILS))
8478 0 : fprintf (dump_file,
8479 : "marking destination block %d reachable\n",
8480 : e->dest->index);
8481 0 : e->dest->flags |= BB_EXECUTABLE;
8482 : }
8483 : }
8484 : }
8485 122394606 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
8486 488597325 : !gsi_end_p (gsi); gsi_next (&gsi))
8487 : {
8488 427400022 : ssa_op_iter i;
8489 427400022 : tree op;
8490 427400022 : if (!bb_visited)
8491 : {
8492 487777179 : FOR_EACH_SSA_TREE_OPERAND (op, gsi_stmt (gsi), i, SSA_OP_ALL_DEFS)
8493 : {
8494 136919576 : vn_ssa_aux_t op_info = VN_INFO (op);
8495 136919576 : gcc_assert (!op_info->visited);
8496 136919576 : op_info->valnum = VN_TOP;
8497 136919576 : op_info->visited = true;
8498 : }
8499 :
8500 : /* We somehow have to deal with uses that are not defined
8501 : in the processed region. Forcing unvisited uses to
8502 : varying here doesn't play well with def-use following during
8503 : expression simplification, so we deal with this by checking
8504 : the visited flag in SSA_VAL. */
8505 : }
8506 :
8507 427400022 : visit_stmt (gsi_stmt (gsi));
8508 :
8509 427400022 : gimple *last = gsi_stmt (gsi);
8510 427400022 : e = NULL;
8511 427400022 : switch (gimple_code (last))
8512 : {
8513 108384 : case GIMPLE_SWITCH:
8514 108384 : e = find_taken_edge (bb, vn_valueize (gimple_switch_index
8515 108384 : (as_a <gswitch *> (last))));
8516 108384 : break;
8517 24562883 : case GIMPLE_COND:
8518 24562883 : {
8519 24562883 : tree lhs = vn_valueize (gimple_cond_lhs (last));
8520 24562883 : tree rhs = vn_valueize (gimple_cond_rhs (last));
8521 24562883 : tree_code cmpcode = gimple_cond_code (last);
8522 : /* Canonicalize the comparison if needed, putting
8523 : the constant in the rhs. */
8524 24562883 : if (tree_swap_operands_p (lhs, rhs))
8525 : {
8526 830490 : std::swap (lhs, rhs);
8527 830490 : cmpcode = swap_tree_comparison (cmpcode);
8528 : }
8529 24562883 : tree val = gimple_simplify (cmpcode,
8530 : boolean_type_node, lhs, rhs,
8531 : NULL, vn_valueize);
8532 : /* If the condition didn't simplfy see if we have recorded
8533 : an expression from sofar taken edges. */
8534 24562883 : if (! val || TREE_CODE (val) != INTEGER_CST)
8535 : {
8536 22716860 : vn_nary_op_t vnresult;
8537 22716860 : tree ops[2];
8538 22716860 : ops[0] = lhs;
8539 22716860 : ops[1] = rhs;
8540 22716860 : val = vn_nary_op_lookup_pieces (2, cmpcode,
8541 : boolean_type_node, ops,
8542 : &vnresult);
8543 : /* Got back a ssa name, then try looking up `val != 0`
8544 : as it might have been recorded that way. */
8545 22716860 : if (val && TREE_CODE (val) == SSA_NAME)
8546 : {
8547 153706 : ops[0] = val;
8548 153706 : ops[1] = build_zero_cst (TREE_TYPE (val));
8549 153706 : val = vn_nary_op_lookup_pieces (2, NE_EXPR,
8550 : boolean_type_node, ops,
8551 : &vnresult);
8552 : }
8553 : /* Did we get a predicated value? */
8554 22716844 : if (! val && vnresult && vnresult->predicated_values)
8555 : {
8556 1372269 : val = vn_nary_op_get_predicated_value (vnresult, bb);
8557 1372269 : if (val && dump_file && (dump_flags & TDF_DETAILS))
8558 : {
8559 2 : fprintf (dump_file, "Got predicated value ");
8560 2 : print_generic_expr (dump_file, val, TDF_NONE);
8561 2 : fprintf (dump_file, " for ");
8562 2 : print_gimple_stmt (dump_file, last, TDF_SLIM);
8563 : }
8564 : }
8565 : }
8566 22716860 : if (val)
8567 2204246 : e = find_taken_edge (bb, val);
8568 24562883 : if (! e)
8569 : {
8570 : /* If we didn't manage to compute the taken edge then
8571 : push predicated expressions for the condition itself
8572 : and related conditions to the hashtables. This allows
8573 : simplification of redundant conditions which is
8574 : important as early cleanup. */
8575 22358637 : edge true_e, false_e;
8576 22358637 : extract_true_false_edges_from_block (bb, &true_e, &false_e);
8577 540797 : if ((do_region && bitmap_bit_p (exit_bbs, true_e->dest->index))
8578 22587852 : || !can_track_predicate_on_edge (true_e))
8579 4911763 : true_e = NULL;
8580 540797 : if ((do_region && bitmap_bit_p (exit_bbs, false_e->dest->index))
8581 22561751 : || !can_track_predicate_on_edge (false_e))
8582 5819361 : false_e = NULL;
8583 22358637 : insert_predicates_for_cond (cmpcode, lhs, rhs, true_e, false_e);
8584 : }
8585 : break;
8586 : }
8587 1394 : case GIMPLE_GOTO:
8588 1394 : e = find_taken_edge (bb, vn_valueize (gimple_goto_dest (last)));
8589 1394 : break;
8590 : default:
8591 : e = NULL;
8592 : }
8593 427400022 : if (e)
8594 : {
8595 2207837 : todo = TODO_cleanup_cfg;
8596 2207837 : if (!(e->flags & EDGE_EXECUTABLE))
8597 : {
8598 1744042 : if (dump_file && (dump_flags & TDF_DETAILS))
8599 35 : fprintf (dump_file,
8600 : "marking known outgoing %sedge %d -> %d executable\n",
8601 35 : e->flags & EDGE_DFS_BACK ? "back-" : "",
8602 35 : e->src->index, e->dest->index);
8603 1744042 : e->flags |= EDGE_EXECUTABLE;
8604 1744042 : e->dest->flags |= BB_EXECUTABLE;
8605 : }
8606 463795 : else if (!(e->dest->flags & BB_EXECUTABLE))
8607 : {
8608 27237 : if (dump_file && (dump_flags & TDF_DETAILS))
8609 1 : fprintf (dump_file,
8610 : "marking destination block %d reachable\n",
8611 : e->dest->index);
8612 27237 : e->dest->flags |= BB_EXECUTABLE;
8613 : }
8614 : }
8615 850384370 : else if (gsi_one_before_end_p (gsi))
8616 : {
8617 127528684 : FOR_EACH_EDGE (e, ei, bb->succs)
8618 : {
8619 75530266 : if (!(e->flags & EDGE_EXECUTABLE))
8620 : {
8621 55379178 : if (dump_file && (dump_flags & TDF_DETAILS))
8622 18532 : fprintf (dump_file,
8623 : "marking outgoing edge %d -> %d executable\n",
8624 18532 : e->src->index, e->dest->index);
8625 55379178 : e->flags |= EDGE_EXECUTABLE;
8626 55379178 : e->dest->flags |= BB_EXECUTABLE;
8627 : }
8628 20151088 : else if (!(e->dest->flags & BB_EXECUTABLE))
8629 : {
8630 2524761 : if (dump_file && (dump_flags & TDF_DETAILS))
8631 5984 : fprintf (dump_file,
8632 : "marking destination block %d reachable\n",
8633 : e->dest->index);
8634 2524761 : e->dest->flags |= BB_EXECUTABLE;
8635 : }
8636 : }
8637 : }
8638 :
8639 : /* Eliminate. That also pushes to avail. */
8640 427400022 : if (eliminate && ! iterate)
8641 106807999 : avail.eliminate_stmt (bb, &gsi);
8642 : else
8643 : /* If not eliminating, make all not already available defs
8644 : available. But avoid picking up dead defs. */
8645 400379923 : FOR_EACH_SSA_TREE_OPERAND (op, gsi_stmt (gsi), i, SSA_OP_DEF)
8646 79787900 : if (! has_zero_uses (op)
8647 79787900 : && ! avail.eliminate_avail (bb, op))
8648 60803140 : avail.eliminate_push_avail (bb, op);
8649 : }
8650 :
8651 : /* Eliminate in destination PHI arguments. Always substitute in dest
8652 : PHIs, even for non-executable edges. This handles region
8653 : exits PHIs. */
8654 61197303 : if (!iterate && eliminate)
8655 32834198 : FOR_EACH_EDGE (e, ei, bb->succs)
8656 19545263 : for (gphi_iterator gsi = gsi_start_phis (e->dest);
8657 37845240 : !gsi_end_p (gsi); gsi_next (&gsi))
8658 : {
8659 18299977 : gphi *phi = gsi.phi ();
8660 18299977 : use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
8661 18299977 : tree arg = USE_FROM_PTR (use_p);
8662 27813914 : if (TREE_CODE (arg) != SSA_NAME
8663 18299977 : || virtual_operand_p (arg))
8664 9513937 : continue;
8665 8786040 : tree sprime;
8666 8786040 : if (SSA_NAME_IS_DEFAULT_DEF (arg))
8667 : {
8668 117707 : sprime = SSA_VAL (arg);
8669 117707 : gcc_assert (TREE_CODE (sprime) != SSA_NAME
8670 : || SSA_NAME_IS_DEFAULT_DEF (sprime));
8671 : }
8672 : else
8673 : /* Look for sth available at the definition block of the argument.
8674 : This avoids inconsistencies between availability there which
8675 : decides if the stmt can be removed and availability at the
8676 : use site. The SSA property ensures that things available
8677 : at the definition are also available at uses. */
8678 8668333 : sprime = avail.eliminate_avail (gimple_bb (SSA_NAME_DEF_STMT (arg)),
8679 : arg);
8680 8786040 : if (sprime
8681 8786040 : && sprime != arg
8682 8786040 : && may_propagate_copy (arg, sprime, !(e->flags & EDGE_ABNORMAL)))
8683 1500853 : propagate_value (use_p, sprime);
8684 : }
8685 :
8686 61197303 : vn_context_bb = NULL;
8687 61197303 : return todo;
8688 : }
8689 :
8690 : /* Unwind state per basic-block. */
8691 :
8692 : struct unwind_state
8693 : {
8694 : /* Times this block has been visited. */
8695 : unsigned visited;
8696 : /* Whether to handle this as iteration point or whether to treat
8697 : incoming backedge PHI values as varying. */
8698 : bool iterate;
8699 : /* Maximum RPO index this block is reachable from. */
8700 : int max_rpo;
8701 : /* Unwind state. */
8702 : void *ob_top;
8703 : vn_reference_t ref_top;
8704 : vn_phi_t phi_top;
8705 : vn_nary_op_t nary_top;
8706 : vn_avail *avail_top;
8707 : };
8708 :
8709 : /* Unwind the RPO VN state for iteration. */
8710 :
8711 : static void
8712 1896769 : do_unwind (unwind_state *to, rpo_elim &avail)
8713 : {
8714 1896769 : gcc_assert (to->iterate);
8715 34814895 : for (; last_inserted_nary != to->nary_top;
8716 32918126 : last_inserted_nary = last_inserted_nary->next)
8717 : {
8718 32918126 : vn_nary_op_t *slot;
8719 32918126 : slot = valid_info->nary->find_slot_with_hash
8720 32918126 : (last_inserted_nary, last_inserted_nary->hashcode, NO_INSERT);
8721 : /* Predication causes the need to restore previous state. */
8722 32918126 : if ((*slot)->unwind_to)
8723 6667255 : *slot = (*slot)->unwind_to;
8724 : else
8725 26250871 : valid_info->nary->clear_slot (slot);
8726 : }
8727 7468765 : for (; last_inserted_phi != to->phi_top;
8728 5571996 : last_inserted_phi = last_inserted_phi->next)
8729 : {
8730 5571996 : vn_phi_t *slot;
8731 5571996 : slot = valid_info->phis->find_slot_with_hash
8732 5571996 : (last_inserted_phi, last_inserted_phi->hashcode, NO_INSERT);
8733 5571996 : valid_info->phis->clear_slot (slot);
8734 : }
8735 15183208 : for (; last_inserted_ref != to->ref_top;
8736 13286439 : last_inserted_ref = last_inserted_ref->next)
8737 : {
8738 13286439 : vn_reference_t *slot;
8739 13286439 : slot = valid_info->references->find_slot_with_hash
8740 13286439 : (last_inserted_ref, last_inserted_ref->hashcode, NO_INSERT);
8741 13286439 : (*slot)->operands.release ();
8742 13286439 : valid_info->references->clear_slot (slot);
8743 : }
8744 1896769 : obstack_free (&vn_tables_obstack, to->ob_top);
8745 :
8746 : /* Prune [rpo_idx, ] from avail. */
8747 20658308 : for (; last_pushed_avail && last_pushed_avail->avail != to->avail_top;)
8748 : {
8749 18761539 : vn_ssa_aux_t val = last_pushed_avail;
8750 18761539 : vn_avail *av = val->avail;
8751 18761539 : val->avail = av->next;
8752 18761539 : last_pushed_avail = av->next_undo;
8753 18761539 : av->next = avail.m_avail_freelist;
8754 18761539 : avail.m_avail_freelist = av;
8755 : }
8756 1896769 : }
8757 :
8758 : /* Do VN on a SEME region specified by ENTRY and EXIT_BBS in FN.
8759 : If ITERATE is true then treat backedges optimistically as not
8760 : executed and iterate. If ELIMINATE is true then perform
8761 : elimination, otherwise leave that to the caller. If SKIP_ENTRY_PHIS
8762 : is true then force PHI nodes in ENTRY->dest to VARYING. */
8763 :
8764 : static unsigned
8765 6147805 : do_rpo_vn_1 (function *fn, edge entry, bitmap exit_bbs,
8766 : bool iterate, bool eliminate, bool skip_entry_phis,
8767 : vn_lookup_kind kind)
8768 : {
8769 6147805 : unsigned todo = 0;
8770 6147805 : default_vn_walk_kind = kind;
8771 :
8772 : /* We currently do not support region-based iteration when
8773 : elimination is requested. */
8774 6147805 : gcc_assert (!entry || !iterate || !eliminate);
8775 : /* When iterating we need loop info up-to-date. */
8776 6147805 : gcc_assert (!iterate || !loops_state_satisfies_p (LOOPS_NEED_FIXUP));
8777 :
8778 6147805 : bool do_region = entry != NULL;
8779 6147805 : if (!do_region)
8780 : {
8781 5463505 : entry = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fn));
8782 5463505 : exit_bbs = BITMAP_ALLOC (NULL);
8783 5463505 : bitmap_set_bit (exit_bbs, EXIT_BLOCK);
8784 : }
8785 :
8786 : /* Clear EDGE_DFS_BACK on "all" entry edges, RPO order compute will
8787 : re-mark those that are contained in the region. */
8788 6147805 : edge_iterator ei;
8789 6147805 : edge e;
8790 12355535 : FOR_EACH_EDGE (e, ei, entry->dest->preds)
8791 6207730 : e->flags &= ~EDGE_DFS_BACK;
8792 :
8793 6147805 : int *rpo = XNEWVEC (int, n_basic_blocks_for_fn (fn) - NUM_FIXED_BLOCKS);
8794 6147805 : auto_vec<std::pair<int, int> > toplevel_scc_extents;
8795 6147805 : int n = rev_post_order_and_mark_dfs_back_seme
8796 8034642 : (fn, entry, exit_bbs, true, rpo, !iterate ? &toplevel_scc_extents : NULL);
8797 :
8798 6147805 : if (!do_region)
8799 5463505 : BITMAP_FREE (exit_bbs);
8800 :
8801 : /* If there are any non-DFS_BACK edges into entry->dest skip
8802 : processing PHI nodes for that block. This supports
8803 : value-numbering loop bodies w/o the actual loop. */
8804 12355534 : FOR_EACH_EDGE (e, ei, entry->dest->preds)
8805 6207730 : if (e != entry
8806 59925 : && !(e->flags & EDGE_DFS_BACK))
8807 : break;
8808 6147805 : if (e != NULL && dump_file && (dump_flags & TDF_DETAILS))
8809 0 : fprintf (dump_file, "Region does not contain all edges into "
8810 : "the entry block, skipping its PHIs.\n");
8811 6147805 : skip_entry_phis |= e != NULL;
8812 :
8813 6147805 : int *bb_to_rpo = XNEWVEC (int, last_basic_block_for_fn (fn));
8814 56386170 : for (int i = 0; i < n; ++i)
8815 50238365 : bb_to_rpo[rpo[i]] = i;
8816 6147805 : vn_bb_to_rpo = bb_to_rpo;
8817 :
8818 6147805 : unwind_state *rpo_state = XNEWVEC (unwind_state, n);
8819 :
8820 6147805 : rpo_elim avail (entry->dest);
8821 6147805 : rpo_avail = &avail;
8822 :
8823 : /* Verify we have no extra entries into the region. */
8824 6147805 : if (flag_checking && do_region)
8825 : {
8826 684294 : auto_bb_flag bb_in_region (fn);
8827 2078209 : for (int i = 0; i < n; ++i)
8828 : {
8829 1393915 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
8830 1393915 : bb->flags |= bb_in_region;
8831 : }
8832 : /* We can't merge the first two loops because we cannot rely
8833 : on EDGE_DFS_BACK for edges not within the region. But if
8834 : we decide to always have the bb_in_region flag we can
8835 : do the checking during the RPO walk itself (but then it's
8836 : also easy to handle MEME conservatively). */
8837 2078209 : for (int i = 0; i < n; ++i)
8838 : {
8839 1393915 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
8840 1393915 : edge e;
8841 1393915 : edge_iterator ei;
8842 3049511 : FOR_EACH_EDGE (e, ei, bb->preds)
8843 1655596 : gcc_assert (e == entry
8844 : || (skip_entry_phis && bb == entry->dest)
8845 : || (e->src->flags & bb_in_region));
8846 : }
8847 2078209 : for (int i = 0; i < n; ++i)
8848 : {
8849 1393915 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
8850 1393915 : bb->flags &= ~bb_in_region;
8851 : }
8852 684294 : }
8853 :
8854 : /* Create the VN state. For the initial size of the various hashtables
8855 : use a heuristic based on region size and number of SSA names. */
8856 6147805 : unsigned region_size = (((unsigned HOST_WIDE_INT)n * num_ssa_names)
8857 6147805 : / (n_basic_blocks_for_fn (fn) - NUM_FIXED_BLOCKS));
8858 6147805 : VN_TOP = create_tmp_var_raw (void_type_node, "vn_top");
8859 6147805 : next_value_id = 1;
8860 6147805 : next_constant_value_id = -1;
8861 :
8862 6147805 : vn_ssa_aux_hash = new hash_table <vn_ssa_aux_hasher> (region_size * 2);
8863 6147805 : gcc_obstack_init (&vn_ssa_aux_obstack);
8864 :
8865 6147805 : gcc_obstack_init (&vn_tables_obstack);
8866 6147805 : gcc_obstack_init (&vn_tables_insert_obstack);
8867 6147805 : valid_info = XCNEW (struct vn_tables_s);
8868 6147805 : allocate_vn_table (valid_info, region_size);
8869 6147805 : last_inserted_ref = NULL;
8870 6147805 : last_inserted_phi = NULL;
8871 6147805 : last_inserted_nary = NULL;
8872 6147805 : last_pushed_avail = NULL;
8873 :
8874 6147805 : vn_valueize = rpo_vn_valueize;
8875 :
8876 : /* Initialize the unwind state and edge/BB executable state. */
8877 6147805 : unsigned curr_scc = 0;
8878 56386170 : for (int i = 0; i < n; ++i)
8879 : {
8880 50238365 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
8881 50238365 : rpo_state[i].visited = 0;
8882 50238365 : rpo_state[i].max_rpo = i;
8883 58683401 : if (!iterate && curr_scc < toplevel_scc_extents.length ())
8884 : {
8885 7056764 : if (i >= toplevel_scc_extents[curr_scc].first
8886 7056764 : && i <= toplevel_scc_extents[curr_scc].second)
8887 3893437 : rpo_state[i].max_rpo = toplevel_scc_extents[curr_scc].second;
8888 7056764 : if (i == toplevel_scc_extents[curr_scc].second)
8889 730793 : curr_scc++;
8890 : }
8891 50238365 : bb->flags &= ~BB_EXECUTABLE;
8892 50238365 : bool has_backedges = false;
8893 50238365 : edge e;
8894 50238365 : edge_iterator ei;
8895 119143997 : FOR_EACH_EDGE (e, ei, bb->preds)
8896 : {
8897 68905632 : if (e->flags & EDGE_DFS_BACK)
8898 2838184 : has_backedges = true;
8899 68905632 : e->flags &= ~EDGE_EXECUTABLE;
8900 68905632 : if (iterate || e == entry || (skip_entry_phis && bb == entry->dest))
8901 68905632 : continue;
8902 : }
8903 50238365 : rpo_state[i].iterate = iterate && has_backedges;
8904 : }
8905 6147805 : entry->flags |= EDGE_EXECUTABLE;
8906 6147805 : entry->dest->flags |= BB_EXECUTABLE;
8907 :
8908 : /* As heuristic to improve compile-time we handle only the N innermost
8909 : loops and the outermost one optimistically. */
8910 6147805 : if (iterate)
8911 : {
8912 4260968 : unsigned max_depth = param_rpo_vn_max_loop_depth;
8913 14322609 : for (auto loop : loops_list (cfun, LI_ONLY_INNERMOST))
8914 1542098 : if (loop_depth (loop) > max_depth)
8915 2108 : for (unsigned i = 2;
8916 9002 : i < loop_depth (loop) - max_depth; ++i)
8917 : {
8918 2108 : basic_block header = superloop_at_depth (loop, i)->header;
8919 2108 : bool non_latch_backedge = false;
8920 2108 : edge e;
8921 2108 : edge_iterator ei;
8922 6355 : FOR_EACH_EDGE (e, ei, header->preds)
8923 4247 : if (e->flags & EDGE_DFS_BACK)
8924 : {
8925 : /* There can be a non-latch backedge into the header
8926 : which is part of an outer irreducible region. We
8927 : cannot avoid iterating this block then. */
8928 2139 : if (!dominated_by_p (CDI_DOMINATORS,
8929 2139 : e->src, e->dest))
8930 : {
8931 12 : if (dump_file && (dump_flags & TDF_DETAILS))
8932 0 : fprintf (dump_file, "non-latch backedge %d -> %d "
8933 : "forces iteration of loop %d\n",
8934 0 : e->src->index, e->dest->index, loop->num);
8935 : non_latch_backedge = true;
8936 : }
8937 : else
8938 2127 : e->flags |= EDGE_EXECUTABLE;
8939 : }
8940 2108 : rpo_state[bb_to_rpo[header->index]].iterate = non_latch_backedge;
8941 4260968 : }
8942 : }
8943 :
8944 6147805 : uint64_t nblk = 0;
8945 6147805 : int idx = 0;
8946 4260968 : if (iterate)
8947 : /* Go and process all blocks, iterating as necessary. */
8948 48738931 : do
8949 : {
8950 48738931 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[idx]);
8951 :
8952 : /* If the block has incoming backedges remember unwind state. This
8953 : is required even for non-executable blocks since in irreducible
8954 : regions we might reach them via the backedge and re-start iterating
8955 : from there.
8956 : Note we can individually mark blocks with incoming backedges to
8957 : not iterate where we then handle PHIs conservatively. We do that
8958 : heuristically to reduce compile-time for degenerate cases. */
8959 48738931 : if (rpo_state[idx].iterate)
8960 : {
8961 4373401 : rpo_state[idx].ob_top = obstack_alloc (&vn_tables_obstack, 0);
8962 4373401 : rpo_state[idx].ref_top = last_inserted_ref;
8963 4373401 : rpo_state[idx].phi_top = last_inserted_phi;
8964 4373401 : rpo_state[idx].nary_top = last_inserted_nary;
8965 4373401 : rpo_state[idx].avail_top
8966 4373401 : = last_pushed_avail ? last_pushed_avail->avail : NULL;
8967 : }
8968 :
8969 48738931 : if (!(bb->flags & BB_EXECUTABLE))
8970 : {
8971 939274 : if (dump_file && (dump_flags & TDF_DETAILS))
8972 2 : fprintf (dump_file, "Block %d: BB%d found not executable\n",
8973 : idx, bb->index);
8974 939274 : idx++;
8975 2836043 : continue;
8976 : }
8977 :
8978 47799657 : if (dump_file && (dump_flags & TDF_DETAILS))
8979 334 : fprintf (dump_file, "Processing block %d: BB%d\n", idx, bb->index);
8980 47799657 : nblk++;
8981 95599314 : todo |= process_bb (avail, bb,
8982 47799657 : rpo_state[idx].visited != 0,
8983 : rpo_state[idx].iterate,
8984 : iterate, eliminate, do_region, exit_bbs, false);
8985 47799657 : rpo_state[idx].visited++;
8986 :
8987 : /* Verify if changed values flow over executable outgoing backedges
8988 : and those change destination PHI values (that's the thing we
8989 : can easily verify). Reduce over all such edges to the farthest
8990 : away PHI. */
8991 47799657 : int iterate_to = -1;
8992 47799657 : edge_iterator ei;
8993 47799657 : edge e;
8994 115074078 : FOR_EACH_EDGE (e, ei, bb->succs)
8995 67274421 : if ((e->flags & (EDGE_DFS_BACK|EDGE_EXECUTABLE))
8996 : == (EDGE_DFS_BACK|EDGE_EXECUTABLE)
8997 4383080 : && rpo_state[bb_to_rpo[e->dest->index]].iterate)
8998 : {
8999 4380314 : int destidx = bb_to_rpo[e->dest->index];
9000 4380314 : if (!rpo_state[destidx].visited)
9001 : {
9002 134 : if (dump_file && (dump_flags & TDF_DETAILS))
9003 0 : fprintf (dump_file, "Unvisited destination %d\n",
9004 : e->dest->index);
9005 134 : if (iterate_to == -1 || destidx < iterate_to)
9006 134 : iterate_to = destidx;
9007 134 : continue;
9008 : }
9009 4380180 : if (dump_file && (dump_flags & TDF_DETAILS))
9010 53 : fprintf (dump_file, "Looking for changed values of backedge"
9011 : " %d->%d destination PHIs\n",
9012 53 : e->src->index, e->dest->index);
9013 4380180 : vn_context_bb = e->dest;
9014 4380180 : gphi_iterator gsi;
9015 4380180 : for (gsi = gsi_start_phis (e->dest);
9016 10022495 : !gsi_end_p (gsi); gsi_next (&gsi))
9017 : {
9018 7539242 : bool inserted = false;
9019 : /* While we'd ideally just iterate on value changes
9020 : we CSE PHIs and do that even across basic-block
9021 : boundaries. So even hashtable state changes can
9022 : be important (which is roughly equivalent to
9023 : PHI argument value changes). To not excessively
9024 : iterate because of that we track whether a PHI
9025 : was CSEd to with GF_PLF_1. */
9026 7539242 : bool phival_changed;
9027 7539242 : if ((phival_changed = visit_phi (gsi.phi (),
9028 : &inserted, false))
9029 8917002 : || (inserted && gimple_plf (gsi.phi (), GF_PLF_1)))
9030 : {
9031 1896927 : if (!phival_changed
9032 1896927 : && dump_file && (dump_flags & TDF_DETAILS))
9033 0 : fprintf (dump_file, "PHI was CSEd and hashtable "
9034 : "state (changed)\n");
9035 1896927 : if (iterate_to == -1 || destidx < iterate_to)
9036 1896842 : iterate_to = destidx;
9037 1896927 : break;
9038 : }
9039 : }
9040 4380180 : vn_context_bb = NULL;
9041 : }
9042 47799657 : if (iterate_to != -1)
9043 : {
9044 1896769 : do_unwind (&rpo_state[iterate_to], avail);
9045 1896769 : idx = iterate_to;
9046 1896769 : if (dump_file && (dump_flags & TDF_DETAILS))
9047 20 : fprintf (dump_file, "Iterating to %d BB%d\n",
9048 20 : iterate_to, rpo[iterate_to]);
9049 1896769 : continue;
9050 : }
9051 :
9052 45902888 : idx++;
9053 : }
9054 48738931 : while (idx < n);
9055 :
9056 : else /* !iterate */
9057 : {
9058 : /* Process all blocks greedily with a worklist that enforces RPO
9059 : processing of reachable blocks. */
9060 1886837 : auto_bitmap worklist;
9061 1886837 : bitmap_set_bit (worklist, 0);
9062 17171320 : while (!bitmap_empty_p (worklist))
9063 : {
9064 13397646 : int idx = bitmap_clear_first_set_bit (worklist);
9065 13397646 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[idx]);
9066 13397646 : gcc_assert ((bb->flags & BB_EXECUTABLE)
9067 : && !rpo_state[idx].visited);
9068 :
9069 13397646 : if (dump_file && (dump_flags & TDF_DETAILS))
9070 35154 : fprintf (dump_file, "Processing block %d: BB%d\n", idx, bb->index);
9071 :
9072 : /* When we run into predecessor edges where we cannot trust its
9073 : executable state mark them executable so PHI processing will
9074 : be conservative.
9075 : ??? Do we need to force arguments flowing over that edge
9076 : to be varying or will they even always be? */
9077 13397646 : edge_iterator ei;
9078 13397646 : edge e;
9079 32443536 : FOR_EACH_EDGE (e, ei, bb->preds)
9080 19045890 : if (!(e->flags & EDGE_EXECUTABLE)
9081 1020349 : && (bb == entry->dest
9082 963644 : || (!rpo_state[bb_to_rpo[e->src->index]].visited
9083 927065 : && (rpo_state[bb_to_rpo[e->src->index]].max_rpo
9084 : >= (int)idx))))
9085 : {
9086 960488 : if (dump_file && (dump_flags & TDF_DETAILS))
9087 11280 : fprintf (dump_file, "Cannot trust state of predecessor "
9088 : "edge %d -> %d, marking executable\n",
9089 11280 : e->src->index, e->dest->index);
9090 960488 : e->flags |= EDGE_EXECUTABLE;
9091 : }
9092 :
9093 13397646 : nblk++;
9094 13397646 : todo |= process_bb (avail, bb, false, false, false, eliminate,
9095 : do_region, exit_bbs,
9096 13397646 : skip_entry_phis && bb == entry->dest);
9097 13397646 : rpo_state[idx].visited++;
9098 :
9099 33075159 : FOR_EACH_EDGE (e, ei, bb->succs)
9100 19677513 : if ((e->flags & EDGE_EXECUTABLE)
9101 19600937 : && e->dest->index != EXIT_BLOCK
9102 18428014 : && (!do_region || !bitmap_bit_p (exit_bbs, e->dest->index))
9103 36771445 : && !rpo_state[bb_to_rpo[e->dest->index]].visited)
9104 16138704 : bitmap_set_bit (worklist, bb_to_rpo[e->dest->index]);
9105 : }
9106 1886837 : }
9107 :
9108 : /* If statistics or dump file active. */
9109 6147805 : int nex = 0;
9110 6147805 : unsigned max_visited = 1;
9111 56386170 : for (int i = 0; i < n; ++i)
9112 : {
9113 50238365 : basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
9114 50238365 : if (bb->flags & BB_EXECUTABLE)
9115 49643370 : nex++;
9116 50238365 : statistics_histogram_event (cfun, "RPO block visited times",
9117 50238365 : rpo_state[i].visited);
9118 50238365 : if (rpo_state[i].visited > max_visited)
9119 : max_visited = rpo_state[i].visited;
9120 : }
9121 6147805 : unsigned nvalues = 0, navail = 0;
9122 168699998 : for (hash_table<vn_ssa_aux_hasher>::iterator i = vn_ssa_aux_hash->begin ();
9123 331252191 : i != vn_ssa_aux_hash->end (); ++i)
9124 : {
9125 162552193 : nvalues++;
9126 162552193 : vn_avail *av = (*i)->avail;
9127 240368836 : while (av)
9128 : {
9129 77816643 : navail++;
9130 77816643 : av = av->next;
9131 : }
9132 : }
9133 6147805 : statistics_counter_event (cfun, "RPO blocks", n);
9134 6147805 : statistics_counter_event (cfun, "RPO blocks visited", nblk);
9135 6147805 : statistics_counter_event (cfun, "RPO blocks executable", nex);
9136 6147805 : statistics_histogram_event (cfun, "RPO iterations", 10*nblk / nex);
9137 6147805 : statistics_histogram_event (cfun, "RPO num values", nvalues);
9138 6147805 : statistics_histogram_event (cfun, "RPO num avail", navail);
9139 6147805 : statistics_histogram_event (cfun, "RPO num lattice",
9140 6147805 : vn_ssa_aux_hash->elements ());
9141 6147805 : if (dump_file && (dump_flags & (TDF_DETAILS|TDF_STATS)))
9142 : {
9143 11185 : fprintf (dump_file, "RPO iteration over %d blocks visited %" PRIu64
9144 : " blocks in total discovering %d executable blocks iterating "
9145 : "%d.%d times, a block was visited max. %u times\n",
9146 : n, nblk, nex,
9147 11185 : (int)((10*nblk / nex)/10), (int)((10*nblk / nex)%10),
9148 : max_visited);
9149 11185 : fprintf (dump_file, "RPO tracked %d values available at %d locations "
9150 : "and %" PRIu64 " lattice elements\n",
9151 11185 : nvalues, navail, (uint64_t) vn_ssa_aux_hash->elements ());
9152 : }
9153 :
9154 6147805 : if (eliminate)
9155 : {
9156 : /* When !iterate we already performed elimination during the RPO
9157 : walk. */
9158 5161966 : if (iterate)
9159 : {
9160 : /* Elimination for region-based VN needs to be done within the
9161 : RPO walk. */
9162 3294683 : gcc_assert (! do_region);
9163 : /* Note we can't use avail.walk here because that gets confused
9164 : by the existing availability and it will be less efficient
9165 : as well. */
9166 3294683 : todo |= eliminate_with_rpo_vn (NULL);
9167 : }
9168 : else
9169 1867283 : todo |= avail.eliminate_cleanup (do_region);
9170 : }
9171 :
9172 6147805 : vn_valueize = NULL;
9173 6147805 : rpo_avail = NULL;
9174 6147805 : vn_bb_to_rpo = NULL;
9175 :
9176 6147805 : XDELETEVEC (bb_to_rpo);
9177 6147805 : XDELETEVEC (rpo);
9178 6147805 : XDELETEVEC (rpo_state);
9179 :
9180 6147805 : return todo;
9181 6147805 : }
9182 :
9183 : /* Region-based entry for RPO VN. Performs value-numbering and elimination
9184 : on the SEME region specified by ENTRY and EXIT_BBS. If ENTRY is not
9185 : the only edge into the region at ENTRY->dest PHI nodes in ENTRY->dest
9186 : are not considered.
9187 : If ITERATE is true then treat backedges optimistically as not
9188 : executed and iterate. If ELIMINATE is true then perform
9189 : elimination, otherwise leave that to the caller.
9190 : If SKIP_ENTRY_PHIS is true then force PHI nodes in ENTRY->dest to VARYING.
9191 : KIND specifies the amount of work done for handling memory operations. */
9192 :
9193 : unsigned
9194 703854 : do_rpo_vn (function *fn, edge entry, bitmap exit_bbs,
9195 : bool iterate, bool eliminate, bool skip_entry_phis,
9196 : vn_lookup_kind kind)
9197 : {
9198 703854 : auto_timevar tv (TV_TREE_RPO_VN);
9199 703854 : unsigned todo = do_rpo_vn_1 (fn, entry, exit_bbs, iterate, eliminate,
9200 : skip_entry_phis, kind);
9201 703854 : free_rpo_vn ();
9202 1407708 : return todo;
9203 703854 : }
9204 :
9205 :
9206 : namespace {
9207 :
9208 : const pass_data pass_data_fre =
9209 : {
9210 : GIMPLE_PASS, /* type */
9211 : "fre", /* name */
9212 : OPTGROUP_NONE, /* optinfo_flags */
9213 : TV_TREE_FRE, /* tv_id */
9214 : ( PROP_cfg | PROP_ssa ), /* properties_required */
9215 : 0, /* properties_provided */
9216 : 0, /* properties_destroyed */
9217 : 0, /* todo_flags_start */
9218 : 0, /* todo_flags_finish */
9219 : };
9220 :
9221 : class pass_fre : public gimple_opt_pass
9222 : {
9223 : public:
9224 1443835 : pass_fre (gcc::context *ctxt)
9225 2887670 : : gimple_opt_pass (pass_data_fre, ctxt), may_iterate (true)
9226 : {}
9227 :
9228 : /* opt_pass methods: */
9229 1155068 : opt_pass * clone () final override { return new pass_fre (m_ctxt); }
9230 1443835 : void set_pass_param (unsigned int n, bool param) final override
9231 : {
9232 1443835 : gcc_assert (n == 0);
9233 1443835 : may_iterate = param;
9234 1443835 : }
9235 4557138 : bool gate (function *) final override
9236 : {
9237 4557138 : return flag_tree_fre != 0 && (may_iterate || optimize > 1);
9238 : }
9239 : unsigned int execute (function *) final override;
9240 :
9241 : private:
9242 : bool may_iterate;
9243 : }; // class pass_fre
9244 :
9245 : unsigned int
9246 4477666 : pass_fre::execute (function *fun)
9247 : {
9248 4477666 : unsigned todo = 0;
9249 :
9250 : /* At -O[1g] use the cheap non-iterating mode. */
9251 4477666 : bool iterate_p = may_iterate && (optimize > 1);
9252 4477666 : calculate_dominance_info (CDI_DOMINATORS);
9253 4477666 : if (iterate_p)
9254 3294683 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
9255 :
9256 4477666 : todo = do_rpo_vn_1 (fun, NULL, NULL, iterate_p, true, false, VN_WALKREWRITE);
9257 4477666 : free_rpo_vn ();
9258 :
9259 4477666 : if (iterate_p)
9260 3294683 : loop_optimizer_finalize ();
9261 :
9262 4477666 : if (scev_initialized_p ())
9263 31997 : scev_reset_htab ();
9264 :
9265 : /* For late FRE after IVOPTs and unrolling, see if we can
9266 : remove some TREE_ADDRESSABLE and rewrite stuff into SSA. */
9267 4477666 : if (!may_iterate)
9268 998057 : todo |= TODO_update_address_taken;
9269 :
9270 4477666 : return todo;
9271 : }
9272 :
9273 : } // anon namespace
9274 :
9275 : gimple_opt_pass *
9276 288767 : make_pass_fre (gcc::context *ctxt)
9277 : {
9278 288767 : return new pass_fre (ctxt);
9279 : }
9280 :
9281 : #undef BB_EXECUTABLE
|
