Branch data Line data Source code
1 : : /* Scalar Replacement of Aggregates (SRA) converts some structure
2 : : references into scalar references, exposing them to the scalar
3 : : optimizers.
4 : : Copyright (C) 2008-2025 Free Software Foundation, Inc.
5 : : Contributed by Martin Jambor <mjambor@suse.cz>
6 : :
7 : : This file is part of GCC.
8 : :
9 : : GCC is free software; you can redistribute it and/or modify it under
10 : : the terms of the GNU General Public License as published by the Free
11 : : Software Foundation; either version 3, or (at your option) any later
12 : : version.
13 : :
14 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 : : for more details.
18 : :
19 : : You should have received a copy of the GNU General Public License
20 : : along with GCC; see the file COPYING3. If not see
21 : : <http://www.gnu.org/licenses/>. */
22 : :
23 : : /* This file implements Scalar Reduction of Aggregates (SRA). SRA is run
24 : : twice, once in the early stages of compilation (early SRA) and once in the
25 : : late stages (late SRA). The aim of both is to turn references to scalar
26 : : parts of aggregates into uses of independent scalar variables.
27 : :
28 : : The two passes are nearly identical, the only difference is that early SRA
29 : : does not scalarize unions which are used as the result in a GIMPLE_RETURN
30 : : statement because together with inlining this can lead to weird type
31 : : conversions.
32 : :
33 : : Both passes operate in four stages:
34 : :
35 : : 1. The declarations that have properties which make them candidates for
36 : : scalarization are identified in function find_var_candidates(). The
37 : : candidates are stored in candidate_bitmap.
38 : :
39 : : 2. The function body is scanned. In the process, declarations which are
40 : : used in a manner that prevent their scalarization are removed from the
41 : : candidate bitmap. More importantly, for every access into an aggregate,
42 : : an access structure (struct access) is created by create_access() and
43 : : stored in a vector associated with the aggregate. Among other
44 : : information, the aggregate declaration, the offset and size of the access
45 : : and its type are stored in the structure.
46 : :
47 : : On a related note, assign_link structures are created for every assign
48 : : statement between candidate aggregates and attached to the related
49 : : accesses.
50 : :
51 : : 3. The vectors of accesses are analyzed. They are first sorted according to
52 : : their offset and size and then scanned for partially overlapping accesses
53 : : (i.e. those which overlap but one is not entirely within another). Such
54 : : an access disqualifies the whole aggregate from being scalarized.
55 : :
56 : : If there is no such inhibiting overlap, a representative access structure
57 : : is chosen for every unique combination of offset and size. Afterwards,
58 : : the pass builds a set of trees from these structures, in which children
59 : : of an access are within their parent (in terms of offset and size).
60 : :
61 : : Then accesses are propagated whenever possible (i.e. in cases when it
62 : : does not create a partially overlapping access) across assign_links from
63 : : the right hand side to the left hand side.
64 : :
65 : : Then the set of trees for each declaration is traversed again and those
66 : : accesses which should be replaced by a scalar are identified.
67 : :
68 : : 4. The function is traversed again, and for every reference into an
69 : : aggregate that has some component which is about to be scalarized,
70 : : statements are amended and new statements are created as necessary.
71 : : Finally, if a parameter got scalarized, the scalar replacements are
72 : : initialized with values from respective parameter aggregates. */
73 : :
74 : : #include "config.h"
75 : : #include "system.h"
76 : : #include "coretypes.h"
77 : : #include "backend.h"
78 : : #include "target.h"
79 : : #include "rtl.h"
80 : : #include "tree.h"
81 : : #include "gimple.h"
82 : : #include "predict.h"
83 : : #include "alloc-pool.h"
84 : : #include "tree-pass.h"
85 : : #include "ssa.h"
86 : : #include "cgraph.h"
87 : : #include "gimple-pretty-print.h"
88 : : #include "alias.h"
89 : : #include "fold-const.h"
90 : : #include "tree-eh.h"
91 : : #include "stor-layout.h"
92 : : #include "gimplify.h"
93 : : #include "gimple-iterator.h"
94 : : #include "gimplify-me.h"
95 : : #include "gimple-walk.h"
96 : : #include "tree-cfg.h"
97 : : #include "tree-dfa.h"
98 : : #include "tree-ssa.h"
99 : : #include "dbgcnt.h"
100 : : #include "builtins.h"
101 : : #include "tree-sra.h"
102 : : #include "opts.h"
103 : :
104 : : /* Enumeration of all aggregate reductions we can do. */
105 : : enum sra_mode { SRA_MODE_EARLY_IPA, /* early call regularization */
106 : : SRA_MODE_EARLY_INTRA, /* early intraprocedural SRA */
107 : : SRA_MODE_INTRA }; /* late intraprocedural SRA */
108 : :
109 : : /* Global variable describing which aggregate reduction we are performing at
110 : : the moment. */
111 : : static enum sra_mode sra_mode;
112 : :
113 : : struct assign_link;
114 : :
115 : : /* ACCESS represents each access to an aggregate variable (as a whole or a
116 : : part). It can also represent a group of accesses that refer to exactly the
117 : : same fragment of an aggregate (i.e. those that have exactly the same offset
118 : : and size). Such representatives for a single aggregate, once determined,
119 : : are linked in a linked list and have the group fields set.
120 : :
121 : : Moreover, when doing intraprocedural SRA, a tree is built from those
122 : : representatives (by the means of first_child and next_sibling pointers), in
123 : : which all items in a subtree are "within" the root, i.e. their offset is
124 : : greater or equal to offset of the root and offset+size is smaller or equal
125 : : to offset+size of the root. Children of an access are sorted by offset.
126 : :
127 : : Note that accesses to parts of vector and complex number types always
128 : : represented by an access to the whole complex number or a vector. It is a
129 : : duty of the modifying functions to replace them appropriately. */
130 : :
131 : : struct access
132 : : {
133 : : /* Values returned by `get_ref_base_and_extent' for each component reference
134 : : If EXPR isn't a component reference just set `BASE = EXPR', `OFFSET = 0',
135 : : `SIZE = TREE_SIZE (TREE_TYPE (expr))'. */
136 : : HOST_WIDE_INT offset;
137 : : HOST_WIDE_INT size;
138 : : tree base;
139 : :
140 : : /* Expression. It is context dependent so do not use it to create new
141 : : expressions to access the original aggregate. See PR 42154 for a
142 : : testcase. */
143 : : tree expr;
144 : : /* Type. */
145 : : tree type;
146 : :
147 : : /* The statement this access belongs to. */
148 : : gimple *stmt;
149 : :
150 : : /* Next group representative for this aggregate. */
151 : : struct access *next_grp;
152 : :
153 : : /* Pointer to the group representative. Pointer to itself if the struct is
154 : : the representative. */
155 : : struct access *group_representative;
156 : :
157 : : /* After access tree has been constructed, this points to the parent of the
158 : : current access, if there is one. NULL for roots. */
159 : : struct access *parent;
160 : :
161 : : /* If this access has any children (in terms of the definition above), this
162 : : points to the first one. */
163 : : struct access *first_child;
164 : :
165 : : /* In intraprocedural SRA, pointer to the next sibling in the access tree as
166 : : described above. */
167 : : struct access *next_sibling;
168 : :
169 : : /* Pointers to the first and last element in the linked list of assign
170 : : links for propagation from LHS to RHS. */
171 : : struct assign_link *first_rhs_link, *last_rhs_link;
172 : :
173 : : /* Pointers to the first and last element in the linked list of assign
174 : : links for propagation from LHS to RHS. */
175 : : struct assign_link *first_lhs_link, *last_lhs_link;
176 : :
177 : : /* Pointer to the next access in the work queues. */
178 : : struct access *next_rhs_queued, *next_lhs_queued;
179 : :
180 : : /* Replacement variable for this access "region." Never to be accessed
181 : : directly, always only by the means of get_access_replacement() and only
182 : : when grp_to_be_replaced flag is set. */
183 : : tree replacement_decl;
184 : :
185 : : /* Is this access made in reverse storage order? */
186 : : unsigned reverse : 1;
187 : :
188 : : /* Is this particular access write access? */
189 : : unsigned write : 1;
190 : :
191 : : /* Is this access currently in the rhs work queue? */
192 : : unsigned grp_rhs_queued : 1;
193 : :
194 : : /* Is this access currently in the lhs work queue? */
195 : : unsigned grp_lhs_queued : 1;
196 : :
197 : : /* Does this group contain a write access? This flag is propagated down the
198 : : access tree. */
199 : : unsigned grp_write : 1;
200 : :
201 : : /* Does this group contain a read access? This flag is propagated down the
202 : : access tree. */
203 : : unsigned grp_read : 1;
204 : :
205 : : /* Does this group contain a read access that comes from an assignment
206 : : statement? This flag is propagated down the access tree. */
207 : : unsigned grp_assignment_read : 1;
208 : :
209 : : /* Does this group contain a write access that comes from an assignment
210 : : statement? This flag is propagated down the access tree. */
211 : : unsigned grp_assignment_write : 1;
212 : :
213 : : /* Does this group contain a read access through a scalar type? This flag is
214 : : not propagated in the access tree in any direction. */
215 : : unsigned grp_scalar_read : 1;
216 : :
217 : : /* Does this group contain a write access through a scalar type? This flag
218 : : is not propagated in the access tree in any direction. */
219 : : unsigned grp_scalar_write : 1;
220 : :
221 : : /* In a root of an access tree, true means that the entire tree should be
222 : : totally scalarized - that all scalar leafs should be scalarized and
223 : : non-root grp_total_scalarization accesses should be honored. Otherwise,
224 : : non-root accesses with grp_total_scalarization should never get scalar
225 : : replacements. */
226 : : unsigned grp_total_scalarization : 1;
227 : :
228 : : /* Other passes of the analysis use this bit to make function
229 : : analyze_access_subtree create scalar replacements for this group if
230 : : possible. */
231 : : unsigned grp_hint : 1;
232 : :
233 : : /* Is the subtree rooted in this access fully covered by scalar
234 : : replacements? */
235 : : unsigned grp_covered : 1;
236 : :
237 : : /* If set to true, this access and all below it in an access tree must not be
238 : : scalarized. */
239 : : unsigned grp_unscalarizable_region : 1;
240 : :
241 : : /* Whether data have been written to parts of the aggregate covered by this
242 : : access which is not to be scalarized. This flag is propagated up in the
243 : : access tree. */
244 : : unsigned grp_unscalarized_data : 1;
245 : :
246 : : /* Set if all accesses in the group consist of the same chain of
247 : : COMPONENT_REFs and ARRAY_REFs. */
248 : : unsigned grp_same_access_path : 1;
249 : :
250 : : /* Does this access and/or group contain a write access through a
251 : : BIT_FIELD_REF? */
252 : : unsigned grp_partial_lhs : 1;
253 : :
254 : : /* Set when a scalar replacement should be created for this variable. */
255 : : unsigned grp_to_be_replaced : 1;
256 : :
257 : : /* Set when we want a replacement for the sole purpose of having it in
258 : : generated debug statements. */
259 : : unsigned grp_to_be_debug_replaced : 1;
260 : :
261 : : /* Should TREE_NO_WARNING of a replacement be set? */
262 : : unsigned grp_no_warning : 1;
263 : :
264 : : /* Result of propagation accross link from LHS to RHS. */
265 : : unsigned grp_result_of_prop_from_lhs : 1;
266 : : };
267 : :
268 : : typedef struct access *access_p;
269 : :
270 : :
271 : : /* Alloc pool for allocating access structures. */
272 : : static object_allocator<struct access> access_pool ("SRA accesses");
273 : :
274 : : /* A structure linking lhs and rhs accesses from an aggregate assignment. They
275 : : are used to propagate subaccesses from rhs to lhs and vice versa as long as
276 : : they don't conflict with what is already there. In the RHS->LHS direction,
277 : : we also propagate grp_write flag to lazily mark that the access contains any
278 : : meaningful data. */
279 : : struct assign_link
280 : : {
281 : : struct access *lacc, *racc;
282 : : struct assign_link *next_rhs, *next_lhs;
283 : : };
284 : :
285 : : /* Alloc pool for allocating assign link structures. */
286 : : static object_allocator<assign_link> assign_link_pool ("SRA links");
287 : :
288 : : /* Base (tree) -> Vector (vec<access_p> *) map. */
289 : : static hash_map<tree, auto_vec<access_p> > *base_access_vec;
290 : :
291 : : /* Hash to limit creation of artificial accesses */
292 : : static hash_map<tree, unsigned> *propagation_budget;
293 : :
294 : : /* Candidate hash table helpers. */
295 : :
296 : : struct uid_decl_hasher : nofree_ptr_hash <tree_node>
297 : : {
298 : : static inline hashval_t hash (const tree_node *);
299 : : static inline bool equal (const tree_node *, const tree_node *);
300 : : };
301 : :
302 : : /* Hash a tree in a uid_decl_map. */
303 : :
304 : : inline hashval_t
305 : 81461246 : uid_decl_hasher::hash (const tree_node *item)
306 : : {
307 : 81461246 : return item->decl_minimal.uid;
308 : : }
309 : :
310 : : /* Return true if the DECL_UID in both trees are equal. */
311 : :
312 : : inline bool
313 : 93644780 : uid_decl_hasher::equal (const tree_node *a, const tree_node *b)
314 : : {
315 : 93644780 : return (a->decl_minimal.uid == b->decl_minimal.uid);
316 : : }
317 : :
318 : : /* Set of candidates. */
319 : : static bitmap candidate_bitmap;
320 : : static hash_table<uid_decl_hasher> *candidates;
321 : :
322 : : /* For a candidate UID return the candidates decl. */
323 : :
324 : : static inline tree
325 : 14785075 : candidate (unsigned uid)
326 : : {
327 : 14785075 : tree_node t;
328 : 14785075 : t.decl_minimal.uid = uid;
329 : 14785075 : return candidates->find_with_hash (&t, static_cast <hashval_t> (uid));
330 : : }
331 : :
332 : : /* Bitmap of candidates which we should try to entirely scalarize away and
333 : : those which cannot be (because they are and need be used as a whole). */
334 : : static bitmap should_scalarize_away_bitmap, cannot_scalarize_away_bitmap;
335 : :
336 : : /* Bitmap of candidates in the constant pool, which cannot be scalarized
337 : : because this would produce non-constant expressions (e.g. Ada). */
338 : : static bitmap disqualified_constants;
339 : :
340 : : /* Bitmap of candidates which are passed by reference in call arguments. */
341 : : static bitmap passed_by_ref_in_call;
342 : :
343 : : /* Obstack for creation of fancy names. */
344 : : static struct obstack name_obstack;
345 : :
346 : : /* Head of a linked list of accesses that need to have its subaccesses
347 : : propagated to their assignment counterparts. */
348 : : static struct access *rhs_work_queue_head, *lhs_work_queue_head;
349 : :
350 : : /* Dump contents of ACCESS to file F in a human friendly way. If GRP is true,
351 : : representative fields are dumped, otherwise those which only describe the
352 : : individual access are. */
353 : :
354 : : static struct
355 : : {
356 : : /* Number of processed aggregates is readily available in
357 : : analyze_all_variable_accesses and so is not stored here. */
358 : :
359 : : /* Number of created scalar replacements. */
360 : : int replacements;
361 : :
362 : : /* Number of times sra_modify_expr or sra_modify_assign themselves changed an
363 : : expression. */
364 : : int exprs;
365 : :
366 : : /* Number of statements created by generate_subtree_copies. */
367 : : int subtree_copies;
368 : :
369 : : /* Number of statements created by load_assign_lhs_subreplacements. */
370 : : int subreplacements;
371 : :
372 : : /* Number of times sra_modify_assign has deleted a statement. */
373 : : int deleted;
374 : :
375 : : /* Number of times sra_modify_assign has to deal with subaccesses of LHS and
376 : : RHS reparately due to type conversions or nonexistent matching
377 : : references. */
378 : : int separate_lhs_rhs_handling;
379 : :
380 : : /* Number of parameters that were removed because they were unused. */
381 : : int deleted_unused_parameters;
382 : :
383 : : /* Number of scalars passed as parameters by reference that have been
384 : : converted to be passed by value. */
385 : : int scalar_by_ref_to_by_val;
386 : :
387 : : /* Number of aggregate parameters that were replaced by one or more of their
388 : : components. */
389 : : int aggregate_params_reduced;
390 : :
391 : : /* Numbber of components created when splitting aggregate parameters. */
392 : : int param_reductions_created;
393 : :
394 : : /* Number of deferred_init calls that are modified. */
395 : : int deferred_init;
396 : :
397 : : /* Number of deferred_init calls that are created by
398 : : generate_subtree_deferred_init. */
399 : : int subtree_deferred_init;
400 : : } sra_stats;
401 : :
402 : : static void
403 : 36 : dump_access (FILE *f, struct access *access, bool grp)
404 : : {
405 : 36 : fprintf (f, "access { ");
406 : 36 : fprintf (f, "base = (%d)'", DECL_UID (access->base));
407 : 36 : print_generic_expr (f, access->base);
408 : 36 : fprintf (f, "', offset = " HOST_WIDE_INT_PRINT_DEC, access->offset);
409 : 36 : fprintf (f, ", size = " HOST_WIDE_INT_PRINT_DEC, access->size);
410 : 36 : fprintf (f, ", expr = ");
411 : 36 : print_generic_expr (f, access->expr);
412 : 36 : fprintf (f, ", type = ");
413 : 36 : print_generic_expr (f, access->type);
414 : 36 : fprintf (f, ", reverse = %d", access->reverse);
415 : 36 : if (grp)
416 : 36 : fprintf (f, ", grp_read = %d, grp_write = %d, grp_assignment_read = %d, "
417 : : "grp_assignment_write = %d, grp_scalar_read = %d, "
418 : : "grp_scalar_write = %d, grp_total_scalarization = %d, "
419 : : "grp_hint = %d, grp_covered = %d, "
420 : : "grp_unscalarizable_region = %d, grp_unscalarized_data = %d, "
421 : : "grp_same_access_path = %d, grp_partial_lhs = %d, "
422 : : "grp_to_be_replaced = %d, grp_to_be_debug_replaced = %d}\n",
423 : 36 : access->grp_read, access->grp_write, access->grp_assignment_read,
424 : 36 : access->grp_assignment_write, access->grp_scalar_read,
425 : 36 : access->grp_scalar_write, access->grp_total_scalarization,
426 : 36 : access->grp_hint, access->grp_covered,
427 : 36 : access->grp_unscalarizable_region, access->grp_unscalarized_data,
428 : 36 : access->grp_same_access_path, access->grp_partial_lhs,
429 : 36 : access->grp_to_be_replaced, access->grp_to_be_debug_replaced);
430 : : else
431 : 0 : fprintf (f, ", write = %d, grp_total_scalarization = %d, "
432 : : "grp_partial_lhs = %d}\n",
433 : 0 : access->write, access->grp_total_scalarization,
434 : 0 : access->grp_partial_lhs);
435 : 36 : }
436 : :
437 : : /* Dump a subtree rooted in ACCESS to file F, indent by LEVEL. */
438 : :
439 : : static void
440 : 18 : dump_access_tree_1 (FILE *f, struct access *access, int level)
441 : : {
442 : 36 : do
443 : : {
444 : 36 : int i;
445 : :
446 : 62 : for (i = 0; i < level; i++)
447 : 26 : fputs ("* ", f);
448 : :
449 : 36 : dump_access (f, access, true);
450 : :
451 : 36 : if (access->first_child)
452 : 8 : dump_access_tree_1 (f, access->first_child, level + 1);
453 : :
454 : 36 : access = access->next_sibling;
455 : : }
456 : 36 : while (access);
457 : 18 : }
458 : :
459 : : /* Dump all access trees for a variable, given the pointer to the first root in
460 : : ACCESS. */
461 : :
462 : : static void
463 : 9 : dump_access_tree (FILE *f, struct access *access)
464 : : {
465 : 19 : for (; access; access = access->next_grp)
466 : 10 : dump_access_tree_1 (f, access, 0);
467 : 9 : }
468 : :
469 : : /* Return true iff ACC is non-NULL and has subaccesses. */
470 : :
471 : : static inline bool
472 : 15782500 : access_has_children_p (struct access *acc)
473 : : {
474 : 8600204 : return acc && acc->first_child;
475 : : }
476 : :
477 : : /* Return true iff ACC is (partly) covered by at least one replacement. */
478 : :
479 : : static bool
480 : 672 : access_has_replacements_p (struct access *acc)
481 : : {
482 : 672 : struct access *child;
483 : 672 : if (acc->grp_to_be_replaced)
484 : : return true;
485 : 599 : for (child = acc->first_child; child; child = child->next_sibling)
486 : 85 : if (access_has_replacements_p (child))
487 : : return true;
488 : : return false;
489 : : }
490 : :
491 : : /* Return a vector of pointers to accesses for the variable given in BASE or
492 : : NULL if there is none. */
493 : :
494 : : static vec<access_p> *
495 : 24245909 : get_base_access_vector (tree base)
496 : : {
497 : 0 : return base_access_vec->get (base);
498 : : }
499 : :
500 : : /* Find an access with required OFFSET and SIZE in a subtree of accesses rooted
501 : : in ACCESS. Return NULL if it cannot be found. */
502 : :
503 : : static struct access *
504 : 10728372 : find_access_in_subtree (struct access *access, HOST_WIDE_INT offset,
505 : : HOST_WIDE_INT size)
506 : : {
507 : 27125785 : while (access && (access->offset != offset || access->size != size))
508 : : {
509 : 5669041 : struct access *child = access->first_child;
510 : :
511 : 11185197 : while (child && (child->offset + child->size <= offset))
512 : 5516156 : child = child->next_sibling;
513 : : access = child;
514 : : }
515 : :
516 : : /* Total scalarization does not replace single field structures with their
517 : : single field but rather creates an access for them underneath. Look for
518 : : it. */
519 : 10728372 : if (access)
520 : 10865109 : while (access->first_child
521 : 3358065 : && access->first_child->offset == offset
522 : 14150130 : && access->first_child->size == size)
523 : : access = access->first_child;
524 : :
525 : 10728372 : return access;
526 : : }
527 : :
528 : : /* Return the first group representative for DECL or NULL if none exists. */
529 : :
530 : : static struct access *
531 : 20252930 : get_first_repr_for_decl (tree base)
532 : : {
533 : 20252930 : vec<access_p> *access_vec;
534 : :
535 : 20252930 : access_vec = get_base_access_vector (base);
536 : 20252930 : if (!access_vec)
537 : : return NULL;
538 : :
539 : 20252930 : return (*access_vec)[0];
540 : : }
541 : :
542 : : /* Find an access representative for the variable BASE and given OFFSET and
543 : : SIZE. Requires that access trees have already been built. Return NULL if
544 : : it cannot be found. */
545 : :
546 : : static struct access *
547 : 9370718 : get_var_base_offset_size_access (tree base, HOST_WIDE_INT offset,
548 : : HOST_WIDE_INT size)
549 : : {
550 : 9370718 : struct access *access;
551 : :
552 : 9370718 : access = get_first_repr_for_decl (base);
553 : 21592393 : while (access && (access->offset + access->size <= offset))
554 : 2850957 : access = access->next_grp;
555 : 9370718 : if (!access)
556 : : return NULL;
557 : :
558 : 9370705 : return find_access_in_subtree (access, offset, size);
559 : : }
560 : :
561 : : /* Add LINK to the linked list of assign links of RACC. */
562 : :
563 : : static void
564 : 1253122 : add_link_to_rhs (struct access *racc, struct assign_link *link)
565 : : {
566 : 1253122 : gcc_assert (link->racc == racc);
567 : :
568 : 1253122 : if (!racc->first_rhs_link)
569 : : {
570 : 1253122 : gcc_assert (!racc->last_rhs_link);
571 : 1253122 : racc->first_rhs_link = link;
572 : : }
573 : : else
574 : 0 : racc->last_rhs_link->next_rhs = link;
575 : :
576 : 1253122 : racc->last_rhs_link = link;
577 : 1253122 : link->next_rhs = NULL;
578 : 1253122 : }
579 : :
580 : : /* Add LINK to the linked list of lhs assign links of LACC. */
581 : :
582 : : static void
583 : 1253122 : add_link_to_lhs (struct access *lacc, struct assign_link *link)
584 : : {
585 : 1253122 : gcc_assert (link->lacc == lacc);
586 : :
587 : 1253122 : if (!lacc->first_lhs_link)
588 : : {
589 : 1253122 : gcc_assert (!lacc->last_lhs_link);
590 : 1253122 : lacc->first_lhs_link = link;
591 : : }
592 : : else
593 : 0 : lacc->last_lhs_link->next_lhs = link;
594 : :
595 : 1253122 : lacc->last_lhs_link = link;
596 : 1253122 : link->next_lhs = NULL;
597 : 1253122 : }
598 : :
599 : : /* Move all link structures in their linked list in OLD_ACC to the linked list
600 : : in NEW_ACC. */
601 : : static void
602 : 5127233 : relink_to_new_repr (struct access *new_acc, struct access *old_acc)
603 : : {
604 : 5127233 : if (old_acc->first_rhs_link)
605 : : {
606 : :
607 : 898530 : if (new_acc->first_rhs_link)
608 : : {
609 : 151835 : gcc_assert (!new_acc->last_rhs_link->next_rhs);
610 : 151835 : gcc_assert (!old_acc->last_rhs_link
611 : : || !old_acc->last_rhs_link->next_rhs);
612 : :
613 : 151835 : new_acc->last_rhs_link->next_rhs = old_acc->first_rhs_link;
614 : 151835 : new_acc->last_rhs_link = old_acc->last_rhs_link;
615 : : }
616 : : else
617 : : {
618 : 746695 : gcc_assert (!new_acc->last_rhs_link);
619 : :
620 : 746695 : new_acc->first_rhs_link = old_acc->first_rhs_link;
621 : 746695 : new_acc->last_rhs_link = old_acc->last_rhs_link;
622 : : }
623 : 898530 : old_acc->first_rhs_link = old_acc->last_rhs_link = NULL;
624 : : }
625 : : else
626 : 4228703 : gcc_assert (!old_acc->last_rhs_link);
627 : :
628 : 5127233 : if (old_acc->first_lhs_link)
629 : : {
630 : :
631 : 365944 : if (new_acc->first_lhs_link)
632 : : {
633 : 147658 : gcc_assert (!new_acc->last_lhs_link->next_lhs);
634 : 147658 : gcc_assert (!old_acc->last_lhs_link
635 : : || !old_acc->last_lhs_link->next_lhs);
636 : :
637 : 147658 : new_acc->last_lhs_link->next_lhs = old_acc->first_lhs_link;
638 : 147658 : new_acc->last_lhs_link = old_acc->last_lhs_link;
639 : : }
640 : : else
641 : : {
642 : 218286 : gcc_assert (!new_acc->last_lhs_link);
643 : :
644 : 218286 : new_acc->first_lhs_link = old_acc->first_lhs_link;
645 : 218286 : new_acc->last_lhs_link = old_acc->last_lhs_link;
646 : : }
647 : 365944 : old_acc->first_lhs_link = old_acc->last_lhs_link = NULL;
648 : : }
649 : : else
650 : 4761289 : gcc_assert (!old_acc->last_lhs_link);
651 : :
652 : 5127233 : }
653 : :
654 : : /* Add ACCESS to the work to queue for propagation of subaccesses from RHS to
655 : : LHS (which is actually a stack). */
656 : :
657 : : static void
658 : 4098662 : add_access_to_rhs_work_queue (struct access *access)
659 : : {
660 : 4098662 : if (access->first_rhs_link && !access->grp_rhs_queued)
661 : : {
662 : 1613417 : gcc_assert (!access->next_rhs_queued);
663 : 1613417 : access->next_rhs_queued = rhs_work_queue_head;
664 : 1613417 : access->grp_rhs_queued = 1;
665 : 1613417 : rhs_work_queue_head = access;
666 : : }
667 : 4098662 : }
668 : :
669 : : /* Add ACCESS to the work to queue for propagation of subaccesses from LHS to
670 : : RHS (which is actually a stack). */
671 : :
672 : : static void
673 : 1496806 : add_access_to_lhs_work_queue (struct access *access)
674 : : {
675 : 1496806 : if (access->first_lhs_link && !access->grp_lhs_queued)
676 : : {
677 : 1255566 : gcc_assert (!access->next_lhs_queued);
678 : 1255566 : access->next_lhs_queued = lhs_work_queue_head;
679 : 1255566 : access->grp_lhs_queued = 1;
680 : 1255566 : lhs_work_queue_head = access;
681 : : }
682 : 1496806 : }
683 : :
684 : : /* Pop an access from the work queue for propagating from RHS to LHS, and
685 : : return it, assuming there is one. */
686 : :
687 : : static struct access *
688 : 1613417 : pop_access_from_rhs_work_queue (void)
689 : : {
690 : 1613417 : struct access *access = rhs_work_queue_head;
691 : :
692 : 1613417 : rhs_work_queue_head = access->next_rhs_queued;
693 : 1613417 : access->next_rhs_queued = NULL;
694 : 1613417 : access->grp_rhs_queued = 0;
695 : 1613417 : return access;
696 : : }
697 : :
698 : : /* Pop an access from the work queue for propagating from LHS to RHS, and
699 : : return it, assuming there is one. */
700 : :
701 : : static struct access *
702 : 1255566 : pop_access_from_lhs_work_queue (void)
703 : : {
704 : 1255566 : struct access *access = lhs_work_queue_head;
705 : :
706 : 1255566 : lhs_work_queue_head = access->next_lhs_queued;
707 : 1255566 : access->next_lhs_queued = NULL;
708 : 1255566 : access->grp_lhs_queued = 0;
709 : 1255566 : return access;
710 : : }
711 : :
712 : : /* Allocate necessary structures. */
713 : :
714 : : static void
715 : 3311876 : sra_initialize (void)
716 : : {
717 : 3311876 : candidate_bitmap = BITMAP_ALLOC (NULL);
718 : 6623752 : candidates = new hash_table<uid_decl_hasher>
719 : 6155480 : (vec_safe_length (cfun->local_decls) / 2);
720 : 3311876 : should_scalarize_away_bitmap = BITMAP_ALLOC (NULL);
721 : 3311876 : cannot_scalarize_away_bitmap = BITMAP_ALLOC (NULL);
722 : 3311876 : disqualified_constants = BITMAP_ALLOC (NULL);
723 : 3311876 : passed_by_ref_in_call = BITMAP_ALLOC (NULL);
724 : 3311876 : gcc_obstack_init (&name_obstack);
725 : 3311876 : base_access_vec = new hash_map<tree, auto_vec<access_p> >;
726 : 3311876 : memset (&sra_stats, 0, sizeof (sra_stats));
727 : 3311876 : }
728 : :
729 : : /* Deallocate all general structures. */
730 : :
731 : : static void
732 : 3311876 : sra_deinitialize (void)
733 : : {
734 : 3311876 : BITMAP_FREE (candidate_bitmap);
735 : 3311876 : delete candidates;
736 : 3311876 : candidates = NULL;
737 : 3311876 : BITMAP_FREE (should_scalarize_away_bitmap);
738 : 3311876 : BITMAP_FREE (cannot_scalarize_away_bitmap);
739 : 3311876 : BITMAP_FREE (disqualified_constants);
740 : 3311876 : BITMAP_FREE (passed_by_ref_in_call);
741 : 3311876 : access_pool.release ();
742 : 3311876 : assign_link_pool.release ();
743 : 3311876 : obstack_free (&name_obstack, NULL);
744 : :
745 : 6623752 : delete base_access_vec;
746 : 3311876 : }
747 : :
748 : : /* Return true if DECL is a VAR_DECL in the constant pool, false otherwise. */
749 : :
750 : 40085002 : static bool constant_decl_p (tree decl)
751 : : {
752 : 34896566 : return VAR_P (decl) && DECL_IN_CONSTANT_POOL (decl);
753 : : }
754 : :
755 : : /* Remove DECL from candidates for SRA and write REASON to the dump file if
756 : : there is one. */
757 : :
758 : : static void
759 : 4034265 : disqualify_candidate (tree decl, const char *reason)
760 : : {
761 : 4034265 : if (bitmap_clear_bit (candidate_bitmap, DECL_UID (decl)))
762 : 2048253 : candidates->remove_elt_with_hash (decl, DECL_UID (decl));
763 : 4034265 : if (constant_decl_p (decl))
764 : 4069 : bitmap_set_bit (disqualified_constants, DECL_UID (decl));
765 : :
766 : 4034265 : if (dump_file && (dump_flags & TDF_DETAILS))
767 : : {
768 : 8 : fprintf (dump_file, "! Disqualifying ");
769 : 8 : print_generic_expr (dump_file, decl);
770 : 8 : fprintf (dump_file, " - %s\n", reason);
771 : : }
772 : 4034265 : }
773 : :
774 : : /* Return true iff the type contains a field or an element which does not allow
775 : : scalarization. Use VISITED_TYPES to avoid re-checking already checked
776 : : (sub-)types. */
777 : :
778 : : static bool
779 : 7858413 : type_internals_preclude_sra_p_1 (tree type, const char **msg,
780 : : hash_set<tree> *visited_types)
781 : : {
782 : 7858413 : tree fld;
783 : 7858413 : tree et;
784 : :
785 : 7858413 : if (visited_types->contains (type))
786 : : return false;
787 : 7577002 : visited_types->add (type);
788 : :
789 : 7577002 : switch (TREE_CODE (type))
790 : : {
791 : 6950514 : case RECORD_TYPE:
792 : 6950514 : case UNION_TYPE:
793 : 6950514 : case QUAL_UNION_TYPE:
794 : 128703358 : for (fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
795 : 121763171 : if (TREE_CODE (fld) == FIELD_DECL)
796 : : {
797 : 16271198 : if (TREE_CODE (fld) == FUNCTION_DECL)
798 : : continue;
799 : 16271198 : tree ft = TREE_TYPE (fld);
800 : :
801 : 16271198 : if (TREE_THIS_VOLATILE (fld))
802 : : {
803 : 867 : *msg = "volatile structure field";
804 : 867 : return true;
805 : : }
806 : 16270331 : if (!DECL_FIELD_OFFSET (fld))
807 : : {
808 : 0 : *msg = "no structure field offset";
809 : 0 : return true;
810 : : }
811 : 16270331 : if (!DECL_SIZE (fld))
812 : : {
813 : 8335 : *msg = "zero structure field size";
814 : 8335 : return true;
815 : : }
816 : 16261996 : if (!tree_fits_uhwi_p (DECL_FIELD_OFFSET (fld)))
817 : : {
818 : 0 : *msg = "structure field offset not fixed";
819 : 0 : return true;
820 : : }
821 : 16261996 : if (!tree_fits_uhwi_p (DECL_SIZE (fld)))
822 : : {
823 : 0 : *msg = "structure field size not fixed";
824 : 0 : return true;
825 : : }
826 : 16261996 : if (!tree_fits_shwi_p (bit_position (fld)))
827 : : {
828 : 0 : *msg = "structure field size too big";
829 : 0 : return true;
830 : : }
831 : 16261996 : if (AGGREGATE_TYPE_P (ft)
832 : 16261996 : && int_bit_position (fld) % BITS_PER_UNIT != 0)
833 : : {
834 : 0 : *msg = "structure field is bit field";
835 : 0 : return true;
836 : : }
837 : :
838 : 16261996 : if (AGGREGATE_TYPE_P (ft)
839 : 16261996 : && type_internals_preclude_sra_p_1 (ft, msg, visited_types))
840 : : return true;
841 : : }
842 : :
843 : : return false;
844 : :
845 : 515895 : case ARRAY_TYPE:
846 : 515895 : et = TREE_TYPE (type);
847 : :
848 : 515895 : if (TYPE_VOLATILE (et))
849 : : {
850 : 0 : *msg = "element type is volatile";
851 : 0 : return true;
852 : : }
853 : :
854 : 515895 : if (AGGREGATE_TYPE_P (et)
855 : 515895 : && type_internals_preclude_sra_p_1 (et, msg, visited_types))
856 : : return true;
857 : :
858 : : return false;
859 : :
860 : : default:
861 : : return false;
862 : : }
863 : : }
864 : :
865 : : /* Return true iff the type contains a field or an element which does not allow
866 : : scalarization. */
867 : :
868 : : bool
869 : 4631034 : type_internals_preclude_sra_p (tree type, const char **msg)
870 : : {
871 : 4631034 : hash_set<tree> visited_types;
872 : 4631034 : return type_internals_preclude_sra_p_1 (type, msg, &visited_types);
873 : 4631034 : }
874 : :
875 : :
876 : : /* Allocate an access structure for BASE, OFFSET and SIZE, clear it, fill in
877 : : the three fields. Also add it to the vector of accesses corresponding to
878 : : the base. Finally, return the new access. */
879 : :
880 : : static struct access *
881 : 14406375 : create_access_1 (tree base, HOST_WIDE_INT offset, HOST_WIDE_INT size)
882 : : {
883 : 14406375 : struct access *access = access_pool.allocate ();
884 : :
885 : 14406375 : memset (access, 0, sizeof (struct access));
886 : 14406375 : access->base = base;
887 : 14406375 : access->offset = offset;
888 : 14406375 : access->size = size;
889 : :
890 : 14406375 : base_access_vec->get_or_insert (base).safe_push (access);
891 : :
892 : 14406375 : return access;
893 : : }
894 : :
895 : : static bool maybe_add_sra_candidate (tree);
896 : :
897 : : /* Create and insert access for EXPR. Return created access, or NULL if it is
898 : : not possible. Also scan for uses of constant pool as we go along and add
899 : : to candidates. */
900 : :
901 : : static struct access *
902 : 26903123 : create_access (tree expr, gimple *stmt, bool write)
903 : : {
904 : 26903123 : struct access *access;
905 : 26903123 : poly_int64 poffset, psize, pmax_size;
906 : 26903123 : tree base = expr;
907 : 26903123 : bool reverse, unscalarizable_region = false;
908 : :
909 : 26903123 : base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size,
910 : : &reverse);
911 : :
912 : : /* For constant-pool entries, check we can substitute the constant value. */
913 : 26903123 : if (constant_decl_p (base)
914 : 2978 : && !bitmap_bit_p (disqualified_constants, DECL_UID (base)))
915 : : {
916 : 2978 : if (expr != base
917 : 7 : && !is_gimple_reg_type (TREE_TYPE (expr))
918 : 2978 : && dump_file && (dump_flags & TDF_DETAILS))
919 : : {
920 : : /* This occurs in Ada with accesses to ARRAY_RANGE_REFs,
921 : : and elements of multidimensional arrays (which are
922 : : multi-element arrays in their own right). */
923 : 0 : fprintf (dump_file, "Allowing non-reg-type load of part"
924 : : " of constant-pool entry: ");
925 : 0 : print_generic_expr (dump_file, expr);
926 : : }
927 : 2978 : maybe_add_sra_candidate (base);
928 : : }
929 : :
930 : 26903123 : if (!DECL_P (base) || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
931 : 12490448 : return NULL;
932 : :
933 : 14412675 : if (write && TREE_READONLY (base))
934 : : {
935 : 5608 : disqualify_candidate (base, "Encountered a store to a read-only decl.");
936 : 5608 : return NULL;
937 : : }
938 : :
939 : 14407067 : HOST_WIDE_INT offset, size, max_size;
940 : 14407067 : if (!poffset.is_constant (&offset)
941 : 14407067 : || !psize.is_constant (&size)
942 : 14407067 : || !pmax_size.is_constant (&max_size))
943 : : {
944 : : disqualify_candidate (base, "Encountered a polynomial-sized access.");
945 : : return NULL;
946 : : }
947 : :
948 : 14407067 : if (size != max_size)
949 : : {
950 : 363595 : size = max_size;
951 : 363595 : unscalarizable_region = true;
952 : : }
953 : 14407067 : if (size == 0)
954 : : return NULL;
955 : 14407065 : if (offset < 0)
956 : : {
957 : 34 : disqualify_candidate (base, "Encountered a negative offset access.");
958 : 34 : return NULL;
959 : : }
960 : 14407031 : if (size < 0)
961 : : {
962 : 24 : disqualify_candidate (base, "Encountered an unconstrained access.");
963 : 24 : return NULL;
964 : : }
965 : 14407007 : if (offset + size > tree_to_shwi (DECL_SIZE (base)))
966 : : {
967 : 631 : disqualify_candidate (base, "Encountered an access beyond the base.");
968 : 631 : return NULL;
969 : : }
970 : 14406376 : if (TREE_CODE (TREE_TYPE (expr)) == BITINT_TYPE
971 : 14406376 : && size > WIDE_INT_MAX_PRECISION - 1)
972 : : {
973 : 1 : disqualify_candidate (base, "Encountered too large _BitInt access.");
974 : 1 : return NULL;
975 : : }
976 : :
977 : 14406375 : access = create_access_1 (base, offset, size);
978 : 14406375 : access->expr = expr;
979 : 14406375 : access->type = TREE_TYPE (expr);
980 : 14406375 : access->write = write;
981 : 14406375 : access->grp_unscalarizable_region = unscalarizable_region;
982 : 14406375 : access->stmt = stmt;
983 : 14406375 : access->reverse = reverse;
984 : :
985 : 14406375 : return access;
986 : : }
987 : :
988 : : /* Given an array type TYPE, extract element size to *EL_SIZE, minimum index to
989 : : *IDX and maximum index to *MAX so that the caller can iterate over all
990 : : elements and return true, except if the array is known to be zero-length,
991 : : then return false. */
992 : :
993 : : static bool
994 : 17486 : prepare_iteration_over_array_elts (tree type, HOST_WIDE_INT *el_size,
995 : : offset_int *idx, offset_int *max)
996 : : {
997 : 17486 : tree elem_size = TYPE_SIZE (TREE_TYPE (type));
998 : 17486 : gcc_assert (elem_size && tree_fits_shwi_p (elem_size));
999 : 17486 : *el_size = tree_to_shwi (elem_size);
1000 : 17486 : gcc_assert (*el_size > 0);
1001 : :
1002 : 17486 : tree minidx = TYPE_MIN_VALUE (TYPE_DOMAIN (type));
1003 : 17486 : gcc_assert (TREE_CODE (minidx) == INTEGER_CST);
1004 : 17486 : tree maxidx = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
1005 : : /* Skip (some) zero-length arrays; others have MAXIDX == MINIDX - 1. */
1006 : 17486 : if (!maxidx)
1007 : : return false;
1008 : 17486 : gcc_assert (TREE_CODE (maxidx) == INTEGER_CST);
1009 : 17486 : tree domain = TYPE_DOMAIN (type);
1010 : : /* MINIDX and MAXIDX are inclusive, and must be interpreted in
1011 : : DOMAIN (e.g. signed int, whereas min/max may be size_int). */
1012 : 17486 : *idx = wi::to_offset (minidx);
1013 : 17486 : *max = wi::to_offset (maxidx);
1014 : 17486 : if (!TYPE_UNSIGNED (domain))
1015 : : {
1016 : 17486 : *idx = wi::sext (*idx, TYPE_PRECISION (domain));
1017 : 17486 : *max = wi::sext (*max, TYPE_PRECISION (domain));
1018 : : }
1019 : : return true;
1020 : : }
1021 : :
1022 : : /* A structure to track collecting padding and hold collected padding
1023 : : information. */
1024 : :
1025 : 71296 : class sra_padding_collecting
1026 : : {
1027 : : public:
1028 : : /* Given that there won't be any data until at least OFFSET, add an
1029 : : appropriate entry to the list of paddings or extend the last one. */
1030 : : void record_padding (HOST_WIDE_INT offset);
1031 : : /* Vector of pairs describing contiguous pieces of padding, each pair
1032 : : consisting of offset and length. */
1033 : : auto_vec<std::pair<HOST_WIDE_INT, HOST_WIDE_INT>, 10> m_padding;
1034 : : /* Offset where data should continue after the last seen actual bit of data
1035 : : if there was no padding. */
1036 : : HOST_WIDE_INT m_data_until = 0;
1037 : : };
1038 : :
1039 : : /* Given that there won't be any data until at least OFFSET, add an appropriate
1040 : : entry to the list of paddings or extend the last one. */
1041 : :
1042 : 208378 : void sra_padding_collecting::record_padding (HOST_WIDE_INT offset)
1043 : : {
1044 : 208378 : if (offset > m_data_until)
1045 : : {
1046 : 14026 : HOST_WIDE_INT psz = offset - m_data_until;
1047 : 14026 : if (!m_padding.is_empty ()
1048 : 582 : && ((m_padding[m_padding.length () - 1].first
1049 : 582 : + m_padding[m_padding.length () - 1].second) == offset))
1050 : 0 : m_padding[m_padding.length () - 1].second += psz;
1051 : : else
1052 : 14026 : m_padding.safe_push (std::make_pair (m_data_until, psz));
1053 : : }
1054 : 208378 : }
1055 : :
1056 : : /* Return true iff TYPE is totally scalarizable - i.e. a RECORD_TYPE or
1057 : : fixed-length ARRAY_TYPE with fields that are either of gimple register types
1058 : : (excluding bit-fields) or (recursively) scalarizable types. CONST_DECL must
1059 : : be true if we are considering a decl from constant pool. If it is false,
1060 : : char arrays will be refused.
1061 : :
1062 : : TOTAL_OFFSET is the offset of TYPE within any outer type that is being
1063 : : examined.
1064 : :
1065 : : If PC is non-NULL, collect padding information into the vector within the
1066 : : structure. The information is however only complete if the function returns
1067 : : true and does not contain any padding at its end. */
1068 : :
1069 : : static bool
1070 : 3647589 : totally_scalarizable_type_p (tree type, bool const_decl,
1071 : : HOST_WIDE_INT total_offset,
1072 : : sra_padding_collecting *pc)
1073 : : {
1074 : 3647589 : if (is_gimple_reg_type (type))
1075 : : {
1076 : 2359472 : if (pc)
1077 : : {
1078 : 136216 : pc->record_padding (total_offset);
1079 : 136216 : pc->m_data_until = total_offset + tree_to_shwi (TYPE_SIZE (type));
1080 : : }
1081 : 2359472 : return true;
1082 : : }
1083 : 1288117 : if (type_contains_placeholder_p (type))
1084 : : return false;
1085 : :
1086 : 1288117 : bool have_predecessor_field = false;
1087 : 1288117 : HOST_WIDE_INT prev_pos = 0;
1088 : :
1089 : 1288117 : switch (TREE_CODE (type))
1090 : : {
1091 : 1240072 : case RECORD_TYPE:
1092 : 19489429 : for (tree fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
1093 : 18282163 : if (TREE_CODE (fld) == FIELD_DECL)
1094 : : {
1095 : 2563827 : tree ft = TREE_TYPE (fld);
1096 : :
1097 : 2563827 : if (!DECL_SIZE (fld))
1098 : : return false;
1099 : 2563827 : if (zerop (DECL_SIZE (fld)))
1100 : 60982 : continue;
1101 : :
1102 : 2502845 : HOST_WIDE_INT pos = int_bit_position (fld);
1103 : 2502845 : if (have_predecessor_field
1104 : 2502845 : && pos <= prev_pos)
1105 : : return false;
1106 : :
1107 : 2502845 : have_predecessor_field = true;
1108 : 2502845 : prev_pos = pos;
1109 : :
1110 : 2502845 : if (DECL_BIT_FIELD (fld))
1111 : : return false;
1112 : :
1113 : 2501476 : if (!totally_scalarizable_type_p (ft, const_decl, total_offset + pos,
1114 : : pc))
1115 : : return false;
1116 : : }
1117 : :
1118 : : return true;
1119 : :
1120 : 28791 : case ARRAY_TYPE:
1121 : 28791 : {
1122 : 28791 : HOST_WIDE_INT min_elem_size;
1123 : 28791 : if (const_decl)
1124 : : min_elem_size = 0;
1125 : : else
1126 : 20875 : min_elem_size = BITS_PER_UNIT;
1127 : :
1128 : 28791 : if (TYPE_DOMAIN (type) == NULL_TREE
1129 : 28791 : || !tree_fits_shwi_p (TYPE_SIZE (type))
1130 : 28791 : || !tree_fits_shwi_p (TYPE_SIZE (TREE_TYPE (type)))
1131 : 28791 : || (tree_to_shwi (TYPE_SIZE (TREE_TYPE (type))) <= min_elem_size)
1132 : 50959 : || !tree_fits_shwi_p (TYPE_MIN_VALUE (TYPE_DOMAIN (type))))
1133 : : return false;
1134 : 22168 : if (tree_to_shwi (TYPE_SIZE (type)) == 0
1135 : 22168 : && TYPE_MAX_VALUE (TYPE_DOMAIN (type)) == NULL_TREE)
1136 : : /* Zero-element array, should not prevent scalarization. */
1137 : : ;
1138 : 22168 : else if ((tree_to_shwi (TYPE_SIZE (type)) <= 0)
1139 : 22168 : || !tree_fits_shwi_p (TYPE_MAX_VALUE (TYPE_DOMAIN (type))))
1140 : : /* Variable-length array, do not allow scalarization. */
1141 : : return false;
1142 : :
1143 : 22132 : unsigned old_padding_len = 0;
1144 : 22132 : if (pc)
1145 : 7820 : old_padding_len = pc->m_padding.length ();
1146 : 22132 : tree elem = TREE_TYPE (type);
1147 : 22132 : if (!totally_scalarizable_type_p (elem, const_decl, total_offset, pc))
1148 : : return false;
1149 : 17573 : if (pc)
1150 : : {
1151 : 7820 : unsigned new_padding_len = pc->m_padding.length ();
1152 : 7820 : HOST_WIDE_INT el_size;
1153 : 7820 : offset_int idx, max;
1154 : 7820 : if (!prepare_iteration_over_array_elts (type, &el_size, &idx, &max))
1155 : 0 : return true;
1156 : 7820 : pc->record_padding (total_offset + el_size);
1157 : 7820 : ++idx;
1158 : 7820 : for (HOST_WIDE_INT pos = total_offset + el_size;
1159 : 36076 : idx <= max;
1160 : 28256 : pos += el_size, ++idx)
1161 : : {
1162 : 28283 : for (unsigned i = old_padding_len; i < new_padding_len; i++)
1163 : : {
1164 : 27 : HOST_WIDE_INT pp
1165 : 27 : = pos + pc->m_padding[i].first - total_offset;
1166 : 27 : HOST_WIDE_INT psz = pc->m_padding[i].second;
1167 : 27 : pc->m_padding.safe_push (std::make_pair (pp, psz));
1168 : : }
1169 : : }
1170 : 7820 : pc->m_data_until = total_offset + tree_to_shwi (TYPE_SIZE (type));
1171 : : }
1172 : : return true;
1173 : : }
1174 : : default:
1175 : : return false;
1176 : : }
1177 : : }
1178 : :
1179 : : /* Return true if REF has an VIEW_CONVERT_EXPR somewhere in it. */
1180 : :
1181 : : static inline bool
1182 : 54732223 : contains_view_convert_expr_p (const_tree ref)
1183 : : {
1184 : 75965322 : while (handled_component_p (ref))
1185 : : {
1186 : 21242464 : if (TREE_CODE (ref) == VIEW_CONVERT_EXPR)
1187 : : return true;
1188 : 21233099 : ref = TREE_OPERAND (ref, 0);
1189 : : }
1190 : :
1191 : : return false;
1192 : : }
1193 : :
1194 : : /* Return true if REF contains a VIEW_CONVERT_EXPR or a COMPONENT_REF with a
1195 : : bit-field field declaration. If TYPE_CHANGING_P is non-NULL, set the bool
1196 : : it points to will be set if REF contains any of the above or a MEM_REF
1197 : : expression that effectively performs type conversion. */
1198 : :
1199 : : static bool
1200 : 7857409 : contains_vce_or_bfcref_p (const_tree ref, bool *type_changing_p = NULL)
1201 : : {
1202 : 9848887 : while (handled_component_p (ref))
1203 : : {
1204 : 2372977 : if (TREE_CODE (ref) == VIEW_CONVERT_EXPR
1205 : 2372977 : || (TREE_CODE (ref) == COMPONENT_REF
1206 : 1664216 : && DECL_BIT_FIELD (TREE_OPERAND (ref, 1))))
1207 : : {
1208 : 381499 : if (type_changing_p)
1209 : 200049 : *type_changing_p = true;
1210 : 381499 : return true;
1211 : : }
1212 : 1991478 : ref = TREE_OPERAND (ref, 0);
1213 : : }
1214 : :
1215 : 7475910 : if (!type_changing_p
1216 : 3657457 : || TREE_CODE (ref) != MEM_REF
1217 : 7586829 : || TREE_CODE (TREE_OPERAND (ref, 0)) != ADDR_EXPR)
1218 : : return false;
1219 : :
1220 : 110919 : tree mem = TREE_OPERAND (TREE_OPERAND (ref, 0), 0);
1221 : 110919 : if (TYPE_MAIN_VARIANT (TREE_TYPE (ref))
1222 : 110919 : != TYPE_MAIN_VARIANT (TREE_TYPE (mem)))
1223 : 84720 : *type_changing_p = true;
1224 : :
1225 : : return false;
1226 : : }
1227 : :
1228 : : /* Search the given tree for a declaration by skipping handled components and
1229 : : exclude it from the candidates. */
1230 : :
1231 : : static void
1232 : 909708 : disqualify_base_of_expr (tree t, const char *reason)
1233 : : {
1234 : 909708 : t = get_base_address (t);
1235 : 909708 : if (t && DECL_P (t))
1236 : 849243 : disqualify_candidate (t, reason);
1237 : 909708 : }
1238 : :
1239 : : /* Return true if the BIT_FIELD_REF read EXPR is handled by SRA. */
1240 : :
1241 : : static bool
1242 : 86530 : sra_handled_bf_read_p (tree expr)
1243 : : {
1244 : 86530 : uint64_t size, offset;
1245 : 86530 : if (bit_field_size (expr).is_constant (&size)
1246 : 86530 : && bit_field_offset (expr).is_constant (&offset)
1247 : 86530 : && size % BITS_PER_UNIT == 0
1248 : 86530 : && offset % BITS_PER_UNIT == 0
1249 : 86598 : && pow2p_hwi (size))
1250 : 86434 : return true;
1251 : : return false;
1252 : : }
1253 : :
1254 : : /* Scan expression EXPR and create access structures for all accesses to
1255 : : candidates for scalarization. Return the created access or NULL if none is
1256 : : created. */
1257 : :
1258 : : static struct access *
1259 : 56591054 : build_access_from_expr_1 (tree expr, gimple *stmt, bool write)
1260 : : {
1261 : : /* We only allow ADDR_EXPRs in arguments of function calls and those must
1262 : : have been dealt with in build_access_from_call_arg. Any other address
1263 : : taking should have been caught by scan_visit_addr. */
1264 : 56591054 : if (TREE_CODE (expr) == ADDR_EXPR)
1265 : : {
1266 : 1858830 : tree base = get_base_address (TREE_OPERAND (expr, 0));
1267 : 1858830 : gcc_assert (!DECL_P (base)
1268 : : || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)));
1269 : 1858830 : return NULL;
1270 : : }
1271 : :
1272 : 54732224 : struct access *ret = NULL;
1273 : 54732224 : bool partial_ref;
1274 : :
1275 : 54732224 : if ((TREE_CODE (expr) == BIT_FIELD_REF
1276 : 51877 : && (write || !sra_handled_bf_read_p (expr)))
1277 : 54731106 : || TREE_CODE (expr) == IMAGPART_EXPR
1278 : 109439717 : || TREE_CODE (expr) == REALPART_EXPR)
1279 : : {
1280 : 47152 : expr = TREE_OPERAND (expr, 0);
1281 : 47152 : partial_ref = true;
1282 : : }
1283 : : else
1284 : : partial_ref = false;
1285 : :
1286 : 54732224 : if (storage_order_barrier_p (expr))
1287 : : {
1288 : 1 : disqualify_base_of_expr (expr, "storage order barrier.");
1289 : 1 : return NULL;
1290 : : }
1291 : :
1292 : : /* We need to dive through V_C_Es in order to get the size of its parameter
1293 : : and not the result type. Ada produces such statements. We are also
1294 : : capable of handling the topmost V_C_E but not any of those buried in other
1295 : : handled components. */
1296 : 54732223 : if (TREE_CODE (expr) == VIEW_CONVERT_EXPR)
1297 : 276271 : expr = TREE_OPERAND (expr, 0);
1298 : :
1299 : 54732223 : if (contains_view_convert_expr_p (expr))
1300 : : {
1301 : 9365 : disqualify_base_of_expr (expr, "V_C_E under a different handled "
1302 : : "component.");
1303 : 9365 : return NULL;
1304 : : }
1305 : 54722858 : if (TREE_THIS_VOLATILE (expr))
1306 : : {
1307 : 21333 : disqualify_base_of_expr (expr, "part of a volatile reference.");
1308 : 21333 : return NULL;
1309 : : }
1310 : :
1311 : 54701525 : switch (TREE_CODE (expr))
1312 : : {
1313 : 2923922 : case MEM_REF:
1314 : 2923922 : if (TREE_CODE (TREE_OPERAND (expr, 0)) != ADDR_EXPR)
1315 : : return NULL;
1316 : : /* fall through */
1317 : 26903123 : case VAR_DECL:
1318 : 26903123 : case PARM_DECL:
1319 : 26903123 : case RESULT_DECL:
1320 : 26903123 : case COMPONENT_REF:
1321 : 26903123 : case ARRAY_REF:
1322 : 26903123 : case ARRAY_RANGE_REF:
1323 : 26903123 : case BIT_FIELD_REF:
1324 : 26903123 : ret = create_access (expr, stmt, write);
1325 : 26903123 : break;
1326 : :
1327 : : default:
1328 : : break;
1329 : : }
1330 : :
1331 : 53070265 : if (write && partial_ref && ret)
1332 : 4229 : ret->grp_partial_lhs = 1;
1333 : :
1334 : : return ret;
1335 : : }
1336 : :
1337 : : /* Scan expression EXPR and create access structures for all accesses to
1338 : : candidates for scalarization. Return true if any access has been inserted.
1339 : : STMT must be the statement from which the expression is taken, WRITE must be
1340 : : true if the expression is a store and false otherwise. */
1341 : :
1342 : : static bool
1343 : 15334375 : build_access_from_expr (tree expr, gimple *stmt, bool write)
1344 : : {
1345 : 15334375 : struct access *access;
1346 : :
1347 : 15334375 : access = build_access_from_expr_1 (expr, stmt, write);
1348 : 15334375 : if (access)
1349 : : {
1350 : : /* This means the aggregate is accesses as a whole in a way other than an
1351 : : assign statement and thus cannot be removed even if we had a scalar
1352 : : replacement for everything. */
1353 : 2415620 : if (cannot_scalarize_away_bitmap)
1354 : 2415620 : bitmap_set_bit (cannot_scalarize_away_bitmap, DECL_UID (access->base));
1355 : 2415620 : return true;
1356 : : }
1357 : : return false;
1358 : : }
1359 : :
1360 : : enum out_edge_check { SRA_OUTGOING_EDGES_UNCHECKED, SRA_OUTGOING_EDGES_OK,
1361 : : SRA_OUTGOING_EDGES_FAIL };
1362 : :
1363 : : /* Return true if STMT terminates BB and there is an abnormal edge going out of
1364 : : the BB and remember the decision in OE_CHECK. */
1365 : :
1366 : : static bool
1367 : 2746090 : abnormal_edge_after_stmt_p (gimple *stmt, enum out_edge_check *oe_check)
1368 : : {
1369 : 2746090 : if (*oe_check == SRA_OUTGOING_EDGES_OK)
1370 : : return false;
1371 : 1632593 : if (*oe_check == SRA_OUTGOING_EDGES_FAIL)
1372 : : return true;
1373 : 1632372 : if (stmt_ends_bb_p (stmt))
1374 : : {
1375 : 669933 : edge e;
1376 : 669933 : edge_iterator ei;
1377 : 1738291 : FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
1378 : 1068837 : if (e->flags & EDGE_ABNORMAL)
1379 : : {
1380 : 479 : *oe_check = SRA_OUTGOING_EDGES_FAIL;
1381 : 479 : return true;
1382 : : }
1383 : : }
1384 : 1631893 : *oe_check = SRA_OUTGOING_EDGES_OK;
1385 : 1631893 : return false;
1386 : : }
1387 : :
1388 : : /* Scan expression EXPR which is an argument of a call and create access
1389 : : structures for all accesses to candidates for scalarization. Return true
1390 : : if any access has been inserted. STMT must be the statement from which the
1391 : : expression is taken. CAN_BE_RETURNED must be true if call argument flags
1392 : : do not rule out that the argument is directly returned. OE_CHECK is used
1393 : : to remember result of a test for abnormal outgoing edges after this
1394 : : statement. */
1395 : :
1396 : : static bool
1397 : 10816965 : build_access_from_call_arg (tree expr, gimple *stmt, bool can_be_returned,
1398 : : enum out_edge_check *oe_check)
1399 : : {
1400 : 10816965 : if (gimple_call_flags (stmt) & ECF_RETURNS_TWICE)
1401 : : {
1402 : 49 : tree base = expr;
1403 : 49 : if (TREE_CODE (expr) == ADDR_EXPR)
1404 : 2 : base = get_base_address (TREE_OPERAND (expr, 0));
1405 : 49 : disqualify_base_of_expr (base, "Passed to a returns_twice call.");
1406 : 49 : return false;
1407 : : }
1408 : :
1409 : 10816916 : if (TREE_CODE (expr) == ADDR_EXPR)
1410 : : {
1411 : 3623802 : tree base = get_base_address (TREE_OPERAND (expr, 0));
1412 : :
1413 : 3623802 : if (can_be_returned)
1414 : : {
1415 : 877712 : disqualify_base_of_expr (base, "Address possibly returned, "
1416 : : "leading to an alis SRA may not know.");
1417 : 877712 : return false;
1418 : : }
1419 : 2746090 : if (abnormal_edge_after_stmt_p (stmt, oe_check))
1420 : : {
1421 : 700 : disqualify_base_of_expr (base, "May lead to need to add statements "
1422 : : "to abnormal edge.");
1423 : 700 : return false;
1424 : : }
1425 : :
1426 : 2745390 : bool read = build_access_from_expr (base, stmt, false);
1427 : 2745390 : bool write = build_access_from_expr (base, stmt, true);
1428 : 2745390 : if (read || write)
1429 : : {
1430 : 247149 : if (dump_file && (dump_flags & TDF_DETAILS))
1431 : : {
1432 : 0 : fprintf (dump_file, "Allowed ADDR_EXPR of ");
1433 : 0 : print_generic_expr (dump_file, base);
1434 : 0 : fprintf (dump_file, " because of ");
1435 : 0 : print_gimple_stmt (dump_file, stmt, 0);
1436 : 0 : fprintf (dump_file, "\n");
1437 : : }
1438 : 247149 : bitmap_set_bit (passed_by_ref_in_call, DECL_UID (base));
1439 : 247149 : return true;
1440 : : }
1441 : : else
1442 : : return false;
1443 : : }
1444 : :
1445 : 7193114 : return build_access_from_expr (expr, stmt, false);
1446 : : }
1447 : :
1448 : :
1449 : : /* Return the single non-EH successor edge of BB or NULL if there is none or
1450 : : more than one. */
1451 : :
1452 : : static edge
1453 : 1525916 : single_non_eh_succ (basic_block bb)
1454 : : {
1455 : 1525916 : edge e, res = NULL;
1456 : 1525916 : edge_iterator ei;
1457 : :
1458 : 4576298 : FOR_EACH_EDGE (e, ei, bb->succs)
1459 : 3050769 : if (!(e->flags & EDGE_EH))
1460 : : {
1461 : 1526186 : if (res)
1462 : : return NULL;
1463 : : res = e;
1464 : : }
1465 : :
1466 : : return res;
1467 : : }
1468 : :
1469 : : /* Disqualify LHS and RHS for scalarization if STMT has to terminate its BB and
1470 : : there is no alternative spot where to put statements SRA might need to
1471 : : generate after it. The spot we are looking for is an edge leading to a
1472 : : single non-EH successor, if it exists and is indeed single. RHS may be
1473 : : NULL, in that case ignore it. */
1474 : :
1475 : : static bool
1476 : 22824824 : disqualify_if_bad_bb_terminating_stmt (gimple *stmt, tree lhs, tree rhs)
1477 : : {
1478 : 22824824 : if (stmt_ends_bb_p (stmt))
1479 : : {
1480 : 1408547 : if (single_non_eh_succ (gimple_bb (stmt)))
1481 : : return false;
1482 : :
1483 : 504 : disqualify_base_of_expr (lhs, "LHS of a throwing stmt.");
1484 : 504 : if (rhs)
1485 : 0 : disqualify_base_of_expr (rhs, "RHS of a throwing stmt.");
1486 : 504 : return true;
1487 : : }
1488 : : return false;
1489 : : }
1490 : :
1491 : : /* Return true if the nature of BASE is such that it contains data even if
1492 : : there is no write to it in the function. */
1493 : :
1494 : : static bool
1495 : 3696558 : comes_initialized_p (tree base)
1496 : : {
1497 : 0 : return TREE_CODE (base) == PARM_DECL || constant_decl_p (base);
1498 : : }
1499 : :
1500 : : /* Scan expressions occurring in STMT, create access structures for all accesses
1501 : : to candidates for scalarization and remove those candidates which occur in
1502 : : statements or expressions that prevent them from being split apart. Return
1503 : : true if any access has been inserted. */
1504 : :
1505 : : static bool
1506 : 31500559 : build_accesses_from_assign (gimple *stmt)
1507 : : {
1508 : 31500559 : tree lhs, rhs;
1509 : 31500559 : struct access *lacc, *racc;
1510 : :
1511 : 31500559 : if (!gimple_assign_single_p (stmt)
1512 : : /* Scope clobbers don't influence scalarization. */
1513 : 31500559 : || gimple_clobber_p (stmt))
1514 : : return false;
1515 : :
1516 : 20628323 : lhs = gimple_assign_lhs (stmt);
1517 : 20628323 : rhs = gimple_assign_rhs1 (stmt);
1518 : :
1519 : 20628323 : if (disqualify_if_bad_bb_terminating_stmt (stmt, lhs, rhs))
1520 : : return false;
1521 : :
1522 : 20628323 : racc = build_access_from_expr_1 (rhs, stmt, false);
1523 : 20628323 : lacc = build_access_from_expr_1 (lhs, stmt, true);
1524 : :
1525 : 20628323 : if (lacc)
1526 : : {
1527 : 6499347 : lacc->grp_assignment_write = 1;
1528 : 6499347 : if (storage_order_barrier_p (rhs))
1529 : 1 : lacc->grp_unscalarizable_region = 1;
1530 : :
1531 : 6499347 : if (should_scalarize_away_bitmap && !is_gimple_reg_type (lacc->type))
1532 : : {
1533 : 2012999 : bool type_changing_p = false;
1534 : 2012999 : contains_vce_or_bfcref_p (lhs, &type_changing_p);
1535 : 2012999 : if (type_changing_p)
1536 : 140356 : bitmap_set_bit (cannot_scalarize_away_bitmap,
1537 : 70178 : DECL_UID (lacc->base));
1538 : : }
1539 : : }
1540 : :
1541 : 20628323 : if (racc)
1542 : : {
1543 : 5491382 : racc->grp_assignment_read = 1;
1544 : 5491382 : if (should_scalarize_away_bitmap && !is_gimple_reg_type (racc->type))
1545 : : {
1546 : 1844507 : bool type_changing_p = false;
1547 : 1844507 : contains_vce_or_bfcref_p (rhs, &type_changing_p);
1548 : :
1549 : 3474423 : if (type_changing_p || gimple_has_volatile_ops (stmt))
1550 : 429900 : bitmap_set_bit (cannot_scalarize_away_bitmap,
1551 : 214950 : DECL_UID (racc->base));
1552 : : else
1553 : 3259114 : bitmap_set_bit (should_scalarize_away_bitmap,
1554 : 1629557 : DECL_UID (racc->base));
1555 : : }
1556 : 5491382 : if (storage_order_barrier_p (lhs))
1557 : 0 : racc->grp_unscalarizable_region = 1;
1558 : : }
1559 : :
1560 : 20628323 : if (lacc && racc
1561 : 1447720 : && (sra_mode == SRA_MODE_EARLY_INTRA || sra_mode == SRA_MODE_INTRA)
1562 : 1447720 : && !lacc->grp_unscalarizable_region
1563 : 1447235 : && !racc->grp_unscalarizable_region
1564 : 1446696 : && AGGREGATE_TYPE_P (TREE_TYPE (lhs))
1565 : 1446696 : && lacc->size == racc->size
1566 : 22074804 : && useless_type_conversion_p (lacc->type, racc->type))
1567 : : {
1568 : 1253122 : struct assign_link *link;
1569 : :
1570 : 1253122 : link = assign_link_pool.allocate ();
1571 : 1253122 : memset (link, 0, sizeof (struct assign_link));
1572 : :
1573 : 1253122 : link->lacc = lacc;
1574 : 1253122 : link->racc = racc;
1575 : 1253122 : add_link_to_rhs (racc, link);
1576 : 1253122 : add_link_to_lhs (lacc, link);
1577 : 1253122 : add_access_to_rhs_work_queue (racc);
1578 : 1253122 : add_access_to_lhs_work_queue (lacc);
1579 : :
1580 : : /* Let's delay marking the areas as written until propagation of accesses
1581 : : across link, unless the nature of rhs tells us that its data comes
1582 : : from elsewhere. */
1583 : 1253122 : if (!comes_initialized_p (racc->base))
1584 : 1157165 : lacc->write = false;
1585 : : }
1586 : :
1587 : 20628323 : return lacc || racc;
1588 : : }
1589 : :
1590 : : /* Callback of walk_stmt_load_store_addr_ops visit_addr used to detect taking
1591 : : addresses of candidates a places which are not call arguments. Such
1592 : : candidates are disqalified from SRA. This also applies to GIMPLE_ASM
1593 : : operands with memory constrains which cannot be scalarized. */
1594 : :
1595 : : static bool
1596 : 2178925 : scan_visit_addr (gimple *, tree op, tree, void *)
1597 : : {
1598 : 2178925 : op = get_base_address (op);
1599 : 2178925 : if (op
1600 : 2178925 : && DECL_P (op))
1601 : 1182470 : disqualify_candidate (op, "Address taken in a non-call-argument context.");
1602 : :
1603 : 2178925 : return false;
1604 : : }
1605 : :
1606 : : /* Scan function and look for interesting expressions and create access
1607 : : structures for them. Return true iff any access is created. */
1608 : :
1609 : : static bool
1610 : 738411 : scan_function (void)
1611 : : {
1612 : 738411 : basic_block bb;
1613 : 738411 : bool ret = false;
1614 : :
1615 : 12628741 : FOR_EACH_BB_FN (bb, cfun)
1616 : : {
1617 : 11890330 : gimple_stmt_iterator gsi;
1618 : 16364010 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1619 : 4473680 : walk_stmt_load_store_addr_ops (gsi_stmt (gsi), NULL, NULL, NULL,
1620 : : scan_visit_addr);
1621 : :
1622 : 109724330 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1623 : : {
1624 : 85943670 : gimple *stmt = gsi_stmt (gsi);
1625 : 85943670 : tree t;
1626 : 85943670 : unsigned i;
1627 : :
1628 : 85943670 : if (gimple_code (stmt) != GIMPLE_CALL)
1629 : 80519232 : walk_stmt_load_store_addr_ops (stmt, NULL, NULL, NULL,
1630 : : scan_visit_addr);
1631 : :
1632 : 85943670 : switch (gimple_code (stmt))
1633 : : {
1634 : 727698 : case GIMPLE_RETURN:
1635 : 727698 : t = gimple_return_retval (as_a <greturn *> (stmt));
1636 : 727698 : if (t != NULL_TREE)
1637 : 439568 : ret |= build_access_from_expr (t, stmt, false);
1638 : : break;
1639 : :
1640 : 31500559 : case GIMPLE_ASSIGN:
1641 : 31500559 : ret |= build_accesses_from_assign (stmt);
1642 : 31500559 : break;
1643 : :
1644 : 5424438 : case GIMPLE_CALL:
1645 : 5424438 : {
1646 : 5424438 : enum out_edge_check oe_check = SRA_OUTGOING_EDGES_UNCHECKED;
1647 : 5424438 : gcall *call = as_a <gcall *> (stmt);
1648 : 16200830 : for (i = 0; i < gimple_call_num_args (call); i++)
1649 : : {
1650 : 10776392 : bool can_be_returned;
1651 : 10776392 : if (gimple_call_lhs (call))
1652 : : {
1653 : 4224483 : int af = gimple_call_arg_flags (call, i);
1654 : 4224483 : can_be_returned = !(af & EAF_NOT_RETURNED_DIRECTLY);
1655 : : }
1656 : : else
1657 : : can_be_returned = false;
1658 : 10776392 : ret |= build_access_from_call_arg (gimple_call_arg (call,
1659 : : i),
1660 : : stmt, can_be_returned,
1661 : : &oe_check);
1662 : : }
1663 : 5424438 : if (gimple_call_chain(stmt))
1664 : 40573 : ret |= build_access_from_call_arg (gimple_call_chain(call),
1665 : : stmt, false, &oe_check);
1666 : : }
1667 : :
1668 : 5424438 : t = gimple_call_lhs (stmt);
1669 : 5424438 : if (t && !disqualify_if_bad_bb_terminating_stmt (stmt, t, NULL))
1670 : : {
1671 : : /* If the STMT is a call to DEFERRED_INIT, avoid setting
1672 : : cannot_scalarize_away_bitmap. */
1673 : 2195997 : if (gimple_call_internal_p (stmt, IFN_DEFERRED_INIT))
1674 : 33 : ret |= !!build_access_from_expr_1 (t, stmt, true);
1675 : : else
1676 : 2195964 : ret |= build_access_from_expr (t, stmt, true);
1677 : : }
1678 : : break;
1679 : :
1680 : 8940 : case GIMPLE_ASM:
1681 : 8940 : {
1682 : 8940 : gasm *asm_stmt = as_a <gasm *> (stmt);
1683 : 8940 : if (stmt_ends_bb_p (asm_stmt)
1684 : 8955 : && !single_succ_p (gimple_bb (asm_stmt)))
1685 : : {
1686 : 32 : for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
1687 : : {
1688 : 17 : t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
1689 : 17 : disqualify_base_of_expr (t, "OP of asm goto.");
1690 : : }
1691 : 42 : for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
1692 : : {
1693 : 27 : t = TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
1694 : 27 : disqualify_base_of_expr (t, "OP of asm goto.");
1695 : : }
1696 : : }
1697 : : else
1698 : : {
1699 : 16803 : for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
1700 : : {
1701 : 7878 : t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
1702 : 7878 : ret |= build_access_from_expr (t, asm_stmt, false);
1703 : : }
1704 : 15996 : for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
1705 : : {
1706 : 7071 : t = TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
1707 : 7071 : ret |= build_access_from_expr (t, asm_stmt, true);
1708 : : }
1709 : : }
1710 : : }
1711 : : break;
1712 : :
1713 : : default:
1714 : : break;
1715 : : }
1716 : : }
1717 : : }
1718 : :
1719 : 738411 : return ret;
1720 : : }
1721 : :
1722 : : /* Helper of QSORT function. There are pointers to accesses in the array. An
1723 : : access is considered smaller than another if it has smaller offset or if the
1724 : : offsets are the same but is size is bigger. */
1725 : :
1726 : : static int
1727 : 122504701 : compare_access_positions (const void *a, const void *b)
1728 : : {
1729 : 122504701 : const access_p *fp1 = (const access_p *) a;
1730 : 122504701 : const access_p *fp2 = (const access_p *) b;
1731 : 122504701 : const access_p f1 = *fp1;
1732 : 122504701 : const access_p f2 = *fp2;
1733 : :
1734 : 122504701 : if (f1->offset != f2->offset)
1735 : 109493686 : return f1->offset < f2->offset ? -1 : 1;
1736 : :
1737 : 48177643 : if (f1->size == f2->size)
1738 : : {
1739 : 33194954 : if (f1->type == f2->type)
1740 : : return 0;
1741 : : /* Put any non-aggregate type before any aggregate type. */
1742 : 4345758 : else if (!is_gimple_reg_type (f1->type)
1743 : 4345758 : && is_gimple_reg_type (f2->type))
1744 : : return 1;
1745 : 3134316 : else if (is_gimple_reg_type (f1->type)
1746 : 3134316 : && !is_gimple_reg_type (f2->type))
1747 : : return -1;
1748 : : /* Put any complex or vector type before any other scalar type. */
1749 : 1508611 : else if (TREE_CODE (f1->type) != COMPLEX_TYPE
1750 : 1508611 : && TREE_CODE (f1->type) != VECTOR_TYPE
1751 : 1441505 : && (TREE_CODE (f2->type) == COMPLEX_TYPE
1752 : 1441505 : || VECTOR_TYPE_P (f2->type)))
1753 : : return 1;
1754 : 1470065 : else if ((TREE_CODE (f1->type) == COMPLEX_TYPE
1755 : : || VECTOR_TYPE_P (f1->type))
1756 : 67106 : && TREE_CODE (f2->type) != COMPLEX_TYPE
1757 : 65617 : && TREE_CODE (f2->type) != VECTOR_TYPE)
1758 : : return -1;
1759 : : /* Put any integral type before any non-integral type. When splicing, we
1760 : : make sure that those with insufficient precision and occupying the
1761 : : same space are not scalarized. */
1762 : 1419122 : else if (INTEGRAL_TYPE_P (f1->type)
1763 : 335675 : && !INTEGRAL_TYPE_P (f2->type))
1764 : : return -1;
1765 : 1314616 : else if (!INTEGRAL_TYPE_P (f1->type)
1766 : 1083447 : && INTEGRAL_TYPE_P (f2->type))
1767 : : return 1;
1768 : : /* Put the integral type with the bigger precision first. */
1769 : 1208783 : else if (INTEGRAL_TYPE_P (f1->type)
1770 : 231169 : && INTEGRAL_TYPE_P (f2->type)
1771 : 1439952 : && (TYPE_PRECISION (f2->type) != TYPE_PRECISION (f1->type)))
1772 : 12472 : return TYPE_PRECISION (f2->type) - TYPE_PRECISION (f1->type);
1773 : : /* Stabilize the sort. */
1774 : 1196311 : return TYPE_UID (f1->type) - TYPE_UID (f2->type);
1775 : : }
1776 : :
1777 : : /* We want the bigger accesses first, thus the opposite operator in the next
1778 : : line: */
1779 : 14982689 : return f1->size > f2->size ? -1 : 1;
1780 : : }
1781 : :
1782 : :
1783 : : /* Append a name of the declaration to the name obstack. A helper function for
1784 : : make_fancy_name. */
1785 : :
1786 : : static void
1787 : 2123817 : make_fancy_decl_name (tree decl)
1788 : : {
1789 : 2123817 : char buffer[32];
1790 : :
1791 : 2123817 : tree name = DECL_NAME (decl);
1792 : 2123817 : if (name)
1793 : 2054506 : obstack_grow (&name_obstack, IDENTIFIER_POINTER (name),
1794 : : IDENTIFIER_LENGTH (name));
1795 : : else
1796 : : {
1797 : 69311 : sprintf (buffer, "D%u", DECL_UID (decl));
1798 : 69311 : obstack_grow (&name_obstack, buffer, strlen (buffer));
1799 : : }
1800 : 2123817 : }
1801 : :
1802 : : /* Helper for make_fancy_name. */
1803 : :
1804 : : static void
1805 : 2331103 : make_fancy_name_1 (tree expr)
1806 : : {
1807 : 2488733 : char buffer[32];
1808 : 2488733 : tree index;
1809 : :
1810 : 2488733 : if (DECL_P (expr))
1811 : : {
1812 : 1029099 : make_fancy_decl_name (expr);
1813 : 1029099 : return;
1814 : : }
1815 : :
1816 : 1459634 : switch (TREE_CODE (expr))
1817 : : {
1818 : 1094718 : case COMPONENT_REF:
1819 : 1094718 : make_fancy_name_1 (TREE_OPERAND (expr, 0));
1820 : 1094718 : obstack_1grow (&name_obstack, '$');
1821 : 1094718 : make_fancy_decl_name (TREE_OPERAND (expr, 1));
1822 : 1094718 : break;
1823 : :
1824 : 50786 : case ARRAY_REF:
1825 : 50786 : make_fancy_name_1 (TREE_OPERAND (expr, 0));
1826 : 50786 : obstack_1grow (&name_obstack, '$');
1827 : : /* Arrays with only one element may not have a constant as their
1828 : : index. */
1829 : 50786 : index = TREE_OPERAND (expr, 1);
1830 : 50786 : if (TREE_CODE (index) != INTEGER_CST)
1831 : : break;
1832 : 50672 : sprintf (buffer, HOST_WIDE_INT_PRINT_DEC, TREE_INT_CST_LOW (index));
1833 : 50672 : obstack_grow (&name_obstack, buffer, strlen (buffer));
1834 : 50672 : break;
1835 : :
1836 : 157630 : case BIT_FIELD_REF:
1837 : 157630 : case ADDR_EXPR:
1838 : 157630 : make_fancy_name_1 (TREE_OPERAND (expr, 0));
1839 : 157630 : break;
1840 : :
1841 : 156423 : case MEM_REF:
1842 : 156423 : make_fancy_name_1 (TREE_OPERAND (expr, 0));
1843 : 156423 : if (!integer_zerop (TREE_OPERAND (expr, 1)))
1844 : : {
1845 : 44892 : obstack_1grow (&name_obstack, '$');
1846 : 89784 : sprintf (buffer, HOST_WIDE_INT_PRINT_DEC,
1847 : 44892 : TREE_INT_CST_LOW (TREE_OPERAND (expr, 1)));
1848 : 44892 : obstack_grow (&name_obstack, buffer, strlen (buffer));
1849 : : }
1850 : : break;
1851 : :
1852 : 0 : case REALPART_EXPR:
1853 : 0 : case IMAGPART_EXPR:
1854 : 0 : gcc_unreachable (); /* we treat these as scalars. */
1855 : : break;
1856 : : default:
1857 : : break;
1858 : : }
1859 : : }
1860 : :
1861 : : /* Create a human readable name for replacement variable of ACCESS. */
1862 : :
1863 : : static char *
1864 : 1029176 : make_fancy_name (tree expr)
1865 : : {
1866 : 1029176 : make_fancy_name_1 (expr);
1867 : 1029176 : obstack_1grow (&name_obstack, '\0');
1868 : 1029176 : return XOBFINISH (&name_obstack, char *);
1869 : : }
1870 : :
1871 : : /* Construct a MEM_REF that would reference a part of aggregate BASE of type
1872 : : EXP_TYPE at the given OFFSET and with storage order REVERSE. If BASE is
1873 : : something for which get_addr_base_and_unit_offset returns NULL, gsi must
1874 : : be non-NULL and is used to insert new statements either before or below
1875 : : the current one as specified by INSERT_AFTER. This function is not capable
1876 : : of handling bitfields. */
1877 : :
1878 : : tree
1879 : 1170569 : build_ref_for_offset (location_t loc, tree base, poly_int64 offset,
1880 : : bool reverse, tree exp_type, gimple_stmt_iterator *gsi,
1881 : : bool insert_after)
1882 : : {
1883 : 1170569 : tree prev_base = base;
1884 : 1170569 : tree off;
1885 : 1170569 : tree mem_ref;
1886 : 1170569 : poly_int64 base_offset;
1887 : 1170569 : unsigned HOST_WIDE_INT misalign;
1888 : 1170569 : unsigned int align;
1889 : :
1890 : : /* Preserve address-space information. */
1891 : 1170569 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (base));
1892 : 1170569 : if (as != TYPE_ADDR_SPACE (exp_type))
1893 : 8 : exp_type = build_qualified_type (exp_type,
1894 : 4 : TYPE_QUALS (exp_type)
1895 : 4 : | ENCODE_QUAL_ADDR_SPACE (as));
1896 : :
1897 : 1170569 : poly_int64 byte_offset = exact_div (offset, BITS_PER_UNIT);
1898 : 1170569 : get_object_alignment_1 (base, &align, &misalign);
1899 : 1170569 : base = get_addr_base_and_unit_offset (base, &base_offset);
1900 : :
1901 : : /* get_addr_base_and_unit_offset returns NULL for references with a variable
1902 : : offset such as array[var_index]. */
1903 : 1170569 : if (!base)
1904 : : {
1905 : 10467 : gassign *stmt;
1906 : 10467 : tree tmp, addr;
1907 : :
1908 : 10467 : gcc_checking_assert (gsi);
1909 : 10467 : tmp = make_ssa_name (build_pointer_type (TREE_TYPE (prev_base)));
1910 : 10467 : addr = build_fold_addr_expr (unshare_expr (prev_base));
1911 : 10467 : STRIP_USELESS_TYPE_CONVERSION (addr);
1912 : 10467 : stmt = gimple_build_assign (tmp, addr);
1913 : 10467 : gimple_set_location (stmt, loc);
1914 : 10467 : if (insert_after)
1915 : 5366 : gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
1916 : : else
1917 : 5101 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1918 : :
1919 : 10467 : off = build_int_cst (reference_alias_ptr_type (prev_base), byte_offset);
1920 : 10467 : base = tmp;
1921 : : }
1922 : 1160102 : else if (TREE_CODE (base) == MEM_REF)
1923 : : {
1924 : 72010 : off = build_int_cst (TREE_TYPE (TREE_OPERAND (base, 1)),
1925 : : base_offset + byte_offset);
1926 : 72010 : off = int_const_binop (PLUS_EXPR, TREE_OPERAND (base, 1), off);
1927 : 72010 : base = unshare_expr (TREE_OPERAND (base, 0));
1928 : : }
1929 : : else
1930 : : {
1931 : 1088092 : off = build_int_cst (reference_alias_ptr_type (prev_base),
1932 : : base_offset + byte_offset);
1933 : 1088092 : base = build_fold_addr_expr (unshare_expr (base));
1934 : : }
1935 : :
1936 : 1170569 : unsigned int align_bound = known_alignment (misalign + offset);
1937 : 1170569 : if (align_bound != 0)
1938 : 791903 : align = MIN (align, align_bound);
1939 : 1170569 : if (align != TYPE_ALIGN (exp_type))
1940 : 299683 : exp_type = build_aligned_type (exp_type, align);
1941 : :
1942 : 1170569 : mem_ref = fold_build2_loc (loc, MEM_REF, exp_type, base, off);
1943 : 1170569 : REF_REVERSE_STORAGE_ORDER (mem_ref) = reverse;
1944 : 1170569 : if (TREE_THIS_VOLATILE (prev_base))
1945 : 6 : TREE_THIS_VOLATILE (mem_ref) = 1;
1946 : 1170569 : if (TREE_SIDE_EFFECTS (prev_base))
1947 : 62 : TREE_SIDE_EFFECTS (mem_ref) = 1;
1948 : 1170569 : return mem_ref;
1949 : : }
1950 : :
1951 : : /* Construct and return a memory reference that is equal to a portion of
1952 : : MODEL->expr but is based on BASE. If this cannot be done, return NULL. */
1953 : :
1954 : : static tree
1955 : 2566308 : build_reconstructed_reference (location_t, tree base, struct access *model)
1956 : : {
1957 : 2566308 : tree expr = model->expr;
1958 : : /* We have to make sure to start just below the outermost union. */
1959 : 2566308 : tree start_expr = expr;
1960 : 5778800 : while (handled_component_p (expr))
1961 : : {
1962 : 3212492 : if (TREE_CODE (TREE_TYPE (TREE_OPERAND (expr, 0))) == UNION_TYPE)
1963 : 15892 : start_expr = expr;
1964 : 3212492 : expr = TREE_OPERAND (expr, 0);
1965 : : }
1966 : :
1967 : : expr = start_expr;
1968 : : tree prev_expr = NULL_TREE;
1969 : 5706962 : while (!types_compatible_p (TREE_TYPE (expr), TREE_TYPE (base)))
1970 : : {
1971 : 3305455 : if (!handled_component_p (expr))
1972 : : return NULL_TREE;
1973 : 3140654 : prev_expr = expr;
1974 : 3140654 : expr = TREE_OPERAND (expr, 0);
1975 : : }
1976 : :
1977 : : /* Guard against broken VIEW_CONVERT_EXPRs... */
1978 : 2401507 : if (!prev_expr)
1979 : : return NULL_TREE;
1980 : :
1981 : 2400411 : TREE_OPERAND (prev_expr, 0) = base;
1982 : 2400411 : tree ref = unshare_expr (model->expr);
1983 : 2400411 : TREE_OPERAND (prev_expr, 0) = expr;
1984 : 2400411 : return ref;
1985 : : }
1986 : :
1987 : : /* Construct a memory reference to a part of an aggregate BASE at the given
1988 : : OFFSET and of the same type as MODEL. In case this is a reference to a
1989 : : bit-field, the function will replicate the last component_ref of model's
1990 : : expr to access it. INSERT_AFTER and GSI have the same meaning as in
1991 : : build_ref_for_offset, furthermore, when GSI is NULL, the function expects
1992 : : that it re-builds the entire reference from a DECL to the final access and
1993 : : so will create a MEM_REF when OFFSET does not exactly match offset of
1994 : : MODEL. */
1995 : :
1996 : : static tree
1997 : 3510894 : build_ref_for_model (location_t loc, tree base, HOST_WIDE_INT offset,
1998 : : struct access *model, gimple_stmt_iterator *gsi,
1999 : : bool insert_after)
2000 : : {
2001 : 3510894 : gcc_assert (offset >= 0);
2002 : 3510894 : if (TREE_CODE (model->expr) == COMPONENT_REF
2003 : 3510894 : && DECL_BIT_FIELD (TREE_OPERAND (model->expr, 1)))
2004 : : {
2005 : : /* This access represents a bit-field. */
2006 : 40017 : tree t, exp_type, fld = TREE_OPERAND (model->expr, 1);
2007 : :
2008 : 40017 : offset -= int_bit_position (fld);
2009 : 40017 : exp_type = TREE_TYPE (TREE_OPERAND (model->expr, 0));
2010 : 40017 : t = build_ref_for_offset (loc, base, offset, model->reverse, exp_type,
2011 : : gsi, insert_after);
2012 : : /* The flag will be set on the record type. */
2013 : 40017 : REF_REVERSE_STORAGE_ORDER (t) = 0;
2014 : 40017 : return fold_build3_loc (loc, COMPONENT_REF, TREE_TYPE (fld), t, fld,
2015 : 40017 : NULL_TREE);
2016 : : }
2017 : : else
2018 : : {
2019 : 3470877 : tree res;
2020 : 3470877 : if (model->grp_same_access_path
2021 : 2566379 : && !TREE_THIS_VOLATILE (base)
2022 : 2566373 : && (TYPE_ADDR_SPACE (TREE_TYPE (base))
2023 : 2566373 : == TYPE_ADDR_SPACE (TREE_TYPE (model->expr)))
2024 : 2566369 : && (offset == model->offset
2025 : 21926 : || (gsi && offset <= model->offset))
2026 : : /* build_reconstructed_reference can still fail if we have already
2027 : : massaged BASE because of another type incompatibility. */
2028 : 6037185 : && (res = build_reconstructed_reference (loc, base, model)))
2029 : : return res;
2030 : : else
2031 : 1070466 : return build_ref_for_offset (loc, base, offset, model->reverse,
2032 : : model->type, gsi, insert_after);
2033 : : }
2034 : : }
2035 : :
2036 : : /* Attempt to build a memory reference that we could but into a gimple
2037 : : debug_bind statement. Similar to build_ref_for_model but punts if it has to
2038 : : create statements and return s NULL instead. This function also ignores
2039 : : alignment issues and so its results should never end up in non-debug
2040 : : statements. */
2041 : :
2042 : : static tree
2043 : 867 : build_debug_ref_for_model (location_t loc, tree base, HOST_WIDE_INT offset,
2044 : : struct access *model)
2045 : : {
2046 : 867 : poly_int64 base_offset;
2047 : 867 : tree off;
2048 : :
2049 : 867 : if (TREE_CODE (model->expr) == COMPONENT_REF
2050 : 867 : && DECL_BIT_FIELD (TREE_OPERAND (model->expr, 1)))
2051 : : return NULL_TREE;
2052 : :
2053 : 867 : base = get_addr_base_and_unit_offset (base, &base_offset);
2054 : 867 : if (!base)
2055 : : return NULL_TREE;
2056 : 867 : if (TREE_CODE (base) == MEM_REF)
2057 : : {
2058 : 176 : off = build_int_cst (TREE_TYPE (TREE_OPERAND (base, 1)),
2059 : 176 : base_offset + offset / BITS_PER_UNIT);
2060 : 176 : off = int_const_binop (PLUS_EXPR, TREE_OPERAND (base, 1), off);
2061 : 176 : base = unshare_expr (TREE_OPERAND (base, 0));
2062 : : }
2063 : : else
2064 : : {
2065 : 691 : off = build_int_cst (reference_alias_ptr_type (base),
2066 : 691 : base_offset + offset / BITS_PER_UNIT);
2067 : 691 : base = build_fold_addr_expr (unshare_expr (base));
2068 : : }
2069 : :
2070 : 867 : return fold_build2_loc (loc, MEM_REF, model->type, base, off);
2071 : : }
2072 : :
2073 : : /* Construct a memory reference consisting of component_refs and array_refs to
2074 : : a part of an aggregate *RES (which is of type TYPE). The requested part
2075 : : should have type EXP_TYPE at be the given OFFSET. This function might not
2076 : : succeed, it returns true when it does and only then *RES points to something
2077 : : meaningful. This function should be used only to build expressions that we
2078 : : might need to present to user (e.g. in warnings). In all other situations,
2079 : : build_ref_for_model or build_ref_for_offset should be used instead. */
2080 : :
2081 : : static bool
2082 : 3157664 : build_user_friendly_ref_for_offset (tree *res, tree type, HOST_WIDE_INT offset,
2083 : : tree exp_type)
2084 : : {
2085 : 3207110 : while (1)
2086 : : {
2087 : 3182387 : tree fld;
2088 : 3182387 : tree tr_size, index, minidx;
2089 : 3182387 : HOST_WIDE_INT el_size;
2090 : :
2091 : 3182387 : if (offset == 0 && exp_type
2092 : 3182387 : && types_compatible_p (exp_type, type))
2093 : : return true;
2094 : :
2095 : 1900267 : switch (TREE_CODE (type))
2096 : : {
2097 : 1815194 : case UNION_TYPE:
2098 : 1815194 : case QUAL_UNION_TYPE:
2099 : 1815194 : case RECORD_TYPE:
2100 : 14720017 : for (fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
2101 : : {
2102 : 14667573 : HOST_WIDE_INT pos, size;
2103 : 14667573 : tree tr_pos, expr, *expr_ptr;
2104 : :
2105 : 14667573 : if (TREE_CODE (fld) != FIELD_DECL)
2106 : 12800215 : continue;
2107 : :
2108 : 3028517 : tr_pos = bit_position (fld);
2109 : 3028517 : if (!tr_pos || !tree_fits_uhwi_p (tr_pos))
2110 : 0 : continue;
2111 : 3028517 : pos = tree_to_uhwi (tr_pos);
2112 : 3028517 : gcc_assert (TREE_CODE (type) == RECORD_TYPE || pos == 0);
2113 : 3028517 : tr_size = DECL_SIZE (fld);
2114 : 3028517 : if (!tr_size || !tree_fits_uhwi_p (tr_size))
2115 : 0 : continue;
2116 : 3028517 : size = tree_to_uhwi (tr_size);
2117 : 3028517 : if (size == 0)
2118 : : {
2119 : 53375 : if (pos != offset)
2120 : 22951 : continue;
2121 : : }
2122 : 2975142 : else if (pos > offset || (pos + size) <= offset)
2123 : 1138208 : continue;
2124 : :
2125 : 1867358 : expr = build3 (COMPONENT_REF, TREE_TYPE (fld), *res, fld,
2126 : : NULL_TREE);
2127 : 1867358 : expr_ptr = &expr;
2128 : 1867358 : if (build_user_friendly_ref_for_offset (expr_ptr, TREE_TYPE (fld),
2129 : : offset - pos, exp_type))
2130 : : {
2131 : 1762750 : *res = expr;
2132 : 1762750 : return true;
2133 : : }
2134 : : }
2135 : : return false;
2136 : :
2137 : 24723 : case ARRAY_TYPE:
2138 : 24723 : tr_size = TYPE_SIZE (TREE_TYPE (type));
2139 : 24723 : if (!tr_size || !tree_fits_uhwi_p (tr_size))
2140 : : return false;
2141 : 24723 : el_size = tree_to_uhwi (tr_size);
2142 : :
2143 : 24723 : minidx = TYPE_MIN_VALUE (TYPE_DOMAIN (type));
2144 : 24723 : if (TREE_CODE (minidx) != INTEGER_CST || el_size == 0)
2145 : : return false;
2146 : 24723 : index = build_int_cst (TYPE_DOMAIN (type), offset / el_size);
2147 : 24723 : if (!integer_zerop (minidx))
2148 : 570 : index = int_const_binop (PLUS_EXPR, index, minidx);
2149 : 24723 : *res = build4 (ARRAY_REF, TREE_TYPE (type), *res, index,
2150 : : NULL_TREE, NULL_TREE);
2151 : 24723 : offset = offset % el_size;
2152 : 24723 : type = TREE_TYPE (type);
2153 : 24723 : break;
2154 : :
2155 : 60350 : default:
2156 : 60350 : if (offset != 0)
2157 : : return false;
2158 : :
2159 : 56663 : if (exp_type)
2160 : : return false;
2161 : : else
2162 : : return true;
2163 : : }
2164 : 24723 : }
2165 : : }
2166 : :
2167 : : /* Print message to dump file why a variable was rejected. */
2168 : :
2169 : : static void
2170 : 14223323 : reject (tree var, const char *msg)
2171 : : {
2172 : 14223323 : if (dump_file && (dump_flags & TDF_DETAILS))
2173 : : {
2174 : 28 : fprintf (dump_file, "Rejected (%d): %s: ", DECL_UID (var), msg);
2175 : 28 : print_generic_expr (dump_file, var);
2176 : 28 : fprintf (dump_file, "\n");
2177 : : }
2178 : 14223323 : }
2179 : :
2180 : : /* Return true if VAR is a candidate for SRA. */
2181 : :
2182 : : static bool
2183 : 18266799 : maybe_add_sra_candidate (tree var)
2184 : : {
2185 : 18266799 : tree type = TREE_TYPE (var);
2186 : 18266799 : const char *msg;
2187 : 18266799 : tree_node **slot;
2188 : :
2189 : 18266799 : if (!AGGREGATE_TYPE_P (type))
2190 : : {
2191 : 12771927 : reject (var, "not aggregate");
2192 : 12771927 : return false;
2193 : : }
2194 : :
2195 : 5494872 : if ((is_global_var (var)
2196 : : /* There are cases where non-addressable variables fail the
2197 : : pt_solutions_check test, e.g in gcc.dg/uninit-40.c. */
2198 : 5282111 : || (TREE_ADDRESSABLE (var)
2199 : 1422379 : && pt_solution_includes (&cfun->gimple_df->escaped_return, var))
2200 : 4048050 : || (TREE_CODE (var) == RESULT_DECL
2201 : 0 : && !DECL_BY_REFERENCE (var)
2202 : 0 : && aggregate_value_p (var, current_function_decl)))
2203 : : /* Allow constant-pool entries that "need to live in memory". */
2204 : 6728933 : && !constant_decl_p (var))
2205 : : {
2206 : 1444987 : reject (var, "needs to live in memory and escapes or global");
2207 : 1444987 : return false;
2208 : : }
2209 : 4049885 : if (TREE_THIS_VOLATILE (var))
2210 : : {
2211 : 564 : reject (var, "is volatile");
2212 : 564 : return false;
2213 : : }
2214 : 4049321 : if (!COMPLETE_TYPE_P (type))
2215 : : {
2216 : 0 : reject (var, "has incomplete type");
2217 : 0 : return false;
2218 : : }
2219 : 4049321 : if (!tree_fits_shwi_p (TYPE_SIZE (type)))
2220 : : {
2221 : 44 : reject (var, "type size not fixed");
2222 : 44 : return false;
2223 : : }
2224 : 4049277 : if (tree_to_shwi (TYPE_SIZE (type)) == 0)
2225 : : {
2226 : 5556 : reject (var, "type size is zero");
2227 : 5556 : return false;
2228 : : }
2229 : 4043721 : if (type_internals_preclude_sra_p (type, &msg))
2230 : : {
2231 : 245 : reject (var, msg);
2232 : 245 : return false;
2233 : : }
2234 : 4043476 : if (/* Fix for PR 41089. tree-stdarg.cc needs to have va_lists intact but
2235 : : we also want to schedule it rather late. Thus we ignore it in
2236 : : the early pass. */
2237 : 4043476 : (sra_mode == SRA_MODE_EARLY_INTRA
2238 : 4043476 : && is_va_list_type (type)))
2239 : : {
2240 : 0 : reject (var, "is va_list");
2241 : 0 : return false;
2242 : : }
2243 : :
2244 : 4043476 : bitmap_set_bit (candidate_bitmap, DECL_UID (var));
2245 : 4043476 : slot = candidates->find_slot_with_hash (var, DECL_UID (var), INSERT);
2246 : 4043476 : *slot = var;
2247 : :
2248 : 4043476 : if (dump_file && (dump_flags & TDF_DETAILS))
2249 : : {
2250 : 14 : fprintf (dump_file, "Candidate (%d): ", DECL_UID (var));
2251 : 14 : print_generic_expr (dump_file, var);
2252 : 14 : fprintf (dump_file, "\n");
2253 : : }
2254 : :
2255 : : return true;
2256 : : }
2257 : :
2258 : : /* The very first phase of intraprocedural SRA. It marks in candidate_bitmap
2259 : : those with type which is suitable for scalarization. */
2260 : :
2261 : : static bool
2262 : 3311876 : find_var_candidates (void)
2263 : : {
2264 : 3311876 : tree var, parm;
2265 : 3311876 : unsigned int i;
2266 : 3311876 : bool ret = false;
2267 : :
2268 : 3311876 : for (parm = DECL_ARGUMENTS (current_function_decl);
2269 : 10307255 : parm;
2270 : 6995379 : parm = DECL_CHAIN (parm))
2271 : 6995379 : ret |= maybe_add_sra_candidate (parm);
2272 : :
2273 : 17423922 : FOR_EACH_LOCAL_DECL (cfun, i, var)
2274 : : {
2275 : 11268442 : if (!VAR_P (var))
2276 : 0 : continue;
2277 : :
2278 : 11268442 : ret |= maybe_add_sra_candidate (var);
2279 : : }
2280 : :
2281 : 3311876 : return ret;
2282 : : }
2283 : :
2284 : : /* Return true if EXP is a reference chain of COMPONENT_REFs and AREAY_REFs
2285 : : ending either with a DECL or a MEM_REF with zero offset. */
2286 : :
2287 : : static bool
2288 : 9791375 : path_comparable_for_same_access (tree expr)
2289 : : {
2290 : 17737188 : while (handled_component_p (expr))
2291 : : {
2292 : 8069076 : if (TREE_CODE (expr) == ARRAY_REF)
2293 : : {
2294 : : /* SSA name indices can occur here too when the array is of sie one.
2295 : : But we cannot just re-use array_refs with SSA names elsewhere in
2296 : : the function, so disallow non-constant indices. TODO: Remove this
2297 : : limitation after teaching build_reconstructed_reference to replace
2298 : : the index with the index type lower bound. */
2299 : 652250 : if (TREE_CODE (TREE_OPERAND (expr, 1)) != INTEGER_CST)
2300 : : return false;
2301 : : }
2302 : 7945813 : expr = TREE_OPERAND (expr, 0);
2303 : : }
2304 : :
2305 : 9668112 : if (TREE_CODE (expr) == MEM_REF)
2306 : : {
2307 : 862759 : if (!zerop (TREE_OPERAND (expr, 1)))
2308 : : return false;
2309 : : }
2310 : : else
2311 : 8805353 : gcc_assert (DECL_P (expr));
2312 : :
2313 : : return true;
2314 : : }
2315 : :
2316 : : /* Assuming that EXP1 consists of only COMPONENT_REFs and ARRAY_REFs, return
2317 : : true if the chain of these handled components are exactly the same as EXP2
2318 : : and the expression under them is the same DECL or an equivalent MEM_REF.
2319 : : The reference picked by compare_access_positions must go to EXP1. */
2320 : :
2321 : : static bool
2322 : 4711977 : same_access_path_p (tree exp1, tree exp2)
2323 : : {
2324 : 4711977 : if (TREE_CODE (exp1) != TREE_CODE (exp2))
2325 : : {
2326 : : /* Special case single-field structures loaded sometimes as the field
2327 : : and sometimes as the structure. If the field is of a scalar type,
2328 : : compare_access_positions will put it into exp1.
2329 : :
2330 : : TODO: The gimple register type condition can be removed if teach
2331 : : compare_access_positions to put inner types first. */
2332 : 471136 : if (is_gimple_reg_type (TREE_TYPE (exp1))
2333 : 455800 : && TREE_CODE (exp1) == COMPONENT_REF
2334 : 848125 : && (TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (exp1, 0)))
2335 : 376989 : == TYPE_MAIN_VARIANT (TREE_TYPE (exp2))))
2336 : 332224 : exp1 = TREE_OPERAND (exp1, 0);
2337 : : else
2338 : : return false;
2339 : : }
2340 : :
2341 : 4573065 : if (!operand_equal_p (exp1, exp2, OEP_ADDRESS_OF))
2342 : : return false;
2343 : :
2344 : : return true;
2345 : : }
2346 : :
2347 : : /* Return true when either T1 is a type that, when loaded into a register and
2348 : : stored back to memory will yield the same bits or when both T1 and T2 are
2349 : : compatible. */
2350 : :
2351 : : static bool
2352 : 5315789 : types_risk_mangled_binary_repr_p (tree t1, tree t2)
2353 : : {
2354 : 5315789 : if (mode_can_transfer_bits (TYPE_MODE (t1)))
2355 : : return false;
2356 : :
2357 : 4423 : return !types_compatible_p (t1, t2);
2358 : : }
2359 : :
2360 : : /* Sort all accesses for the given variable, check for partial overlaps and
2361 : : return NULL if there are any. If there are none, pick a representative for
2362 : : each combination of offset and size and create a linked list out of them.
2363 : : Return the pointer to the first representative and make sure it is the first
2364 : : one in the vector of accesses. */
2365 : :
2366 : : static struct access *
2367 : 3930900 : sort_and_splice_var_accesses (tree var)
2368 : : {
2369 : 3930900 : int i, j, access_count;
2370 : 3930900 : struct access *res, **prev_acc_ptr = &res;
2371 : 3930900 : vec<access_p> *access_vec;
2372 : 3930900 : bool first = true;
2373 : 3930900 : HOST_WIDE_INT low = -1, high = 0;
2374 : :
2375 : 3930900 : access_vec = get_base_access_vector (var);
2376 : 3930900 : if (!access_vec)
2377 : : return NULL;
2378 : 3894371 : access_count = access_vec->length ();
2379 : :
2380 : : /* Sort by <OFFSET, SIZE>. */
2381 : 3894371 : access_vec->qsort (compare_access_positions);
2382 : :
2383 : : i = 0;
2384 : 12991123 : while (i < access_count)
2385 : : {
2386 : 9101385 : struct access *access = (*access_vec)[i];
2387 : 9101385 : bool grp_write = access->write;
2388 : 9101385 : bool grp_read = !access->write;
2389 : 9101385 : bool grp_scalar_write = access->write
2390 : 9101385 : && is_gimple_reg_type (access->type);
2391 : 9101385 : bool grp_scalar_read = !access->write
2392 : 9101385 : && is_gimple_reg_type (access->type);
2393 : 9101385 : bool grp_assignment_read = access->grp_assignment_read;
2394 : 9101385 : bool grp_assignment_write = access->grp_assignment_write;
2395 : 9101385 : bool multiple_scalar_reads = false;
2396 : 9101385 : bool grp_partial_lhs = access->grp_partial_lhs;
2397 : 9101385 : bool first_scalar = is_gimple_reg_type (access->type);
2398 : 9101385 : bool unscalarizable_region = access->grp_unscalarizable_region;
2399 : 9101385 : bool grp_same_access_path = true;
2400 : 9101385 : bool bf_non_full_precision
2401 : 9101385 : = (INTEGRAL_TYPE_P (access->type)
2402 : 2994802 : && TYPE_PRECISION (access->type) != access->size
2403 : 148989 : && TREE_CODE (access->expr) == COMPONENT_REF
2404 : 9163286 : && DECL_BIT_FIELD (TREE_OPERAND (access->expr, 1)));
2405 : :
2406 : 9101385 : if (first || access->offset >= high)
2407 : : {
2408 : 4304305 : first = false;
2409 : 4304305 : low = access->offset;
2410 : 4304305 : high = access->offset + access->size;
2411 : : }
2412 : 4797080 : else if (access->offset > low && access->offset + access->size > high)
2413 : : return NULL;
2414 : : else
2415 : 4796529 : gcc_assert (access->offset >= low
2416 : : && access->offset + access->size <= high);
2417 : :
2418 : 9100834 : if (INTEGRAL_TYPE_P (access->type)
2419 : 2994358 : && TYPE_PRECISION (access->type) != access->size
2420 : 9249425 : && bitmap_bit_p (passed_by_ref_in_call, DECL_UID (access->base)))
2421 : : {
2422 : : /* This can lead to performance regressions because we can generate
2423 : : excessive zero extensions. */
2424 : 4082 : if (dump_file && (dump_flags & TDF_DETAILS))
2425 : : {
2426 : 0 : fprintf (dump_file, "Won't scalarize ");
2427 : 0 : print_generic_expr (dump_file, access->base);
2428 : 0 : fprintf (dump_file, "(%d), it is passed by reference to a call "
2429 : : "and there are accesses with precision not covering "
2430 : 0 : "their type size.", DECL_UID (access->base));
2431 : : }
2432 : 4082 : return NULL;
2433 : : }
2434 : :
2435 : 9096752 : grp_same_access_path = path_comparable_for_same_access (access->expr);
2436 : :
2437 : 9096752 : j = i + 1;
2438 : 14223985 : while (j < access_count)
2439 : : {
2440 : 10334247 : struct access *ac2 = (*access_vec)[j];
2441 : 10334247 : if (ac2->offset != access->offset || ac2->size != access->size)
2442 : : break;
2443 : 5127233 : if (ac2->write)
2444 : : {
2445 : 1104762 : grp_write = true;
2446 : 1104762 : grp_scalar_write = (grp_scalar_write
2447 : 1104762 : || is_gimple_reg_type (ac2->type));
2448 : : }
2449 : : else
2450 : : {
2451 : 4022471 : grp_read = true;
2452 : 4022471 : if (is_gimple_reg_type (ac2->type))
2453 : : {
2454 : 1669859 : if (grp_scalar_read)
2455 : : multiple_scalar_reads = true;
2456 : : else
2457 : 357281 : grp_scalar_read = true;
2458 : : }
2459 : : }
2460 : 5127233 : grp_assignment_read |= ac2->grp_assignment_read;
2461 : 5127233 : grp_assignment_write |= ac2->grp_assignment_write;
2462 : 5127233 : grp_partial_lhs |= ac2->grp_partial_lhs;
2463 : 5127233 : unscalarizable_region |= ac2->grp_unscalarizable_region;
2464 : 5127233 : relink_to_new_repr (access, ac2);
2465 : :
2466 : : /* If there are both aggregate-type and scalar-type accesses with
2467 : : this combination of size and offset, the comparison function
2468 : : should have put the scalars first. */
2469 : 5127233 : gcc_assert (first_scalar || !is_gimple_reg_type (ac2->type));
2470 : : /* It also prefers integral types to non-integral. However, when the
2471 : : precision of the selected type does not span the entire area and
2472 : : should also be used for a non-integer (i.e. float), we must not
2473 : : let that happen. Normally analyze_access_subtree expands the type
2474 : : to cover the entire area but for bit-fields it doesn't. */
2475 : 5127233 : if (bf_non_full_precision && !INTEGRAL_TYPE_P (ac2->type))
2476 : : {
2477 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
2478 : : {
2479 : 0 : fprintf (dump_file, "Cannot scalarize the following access "
2480 : : "because insufficient precision integer type was "
2481 : : "selected.\n ");
2482 : 0 : dump_access (dump_file, access, false);
2483 : : }
2484 : : unscalarizable_region = true;
2485 : : }
2486 : 5127233 : else if (types_risk_mangled_binary_repr_p (access->type, ac2->type))
2487 : : {
2488 : 1560 : if (dump_file && (dump_flags & TDF_DETAILS))
2489 : : {
2490 : 0 : fprintf (dump_file, "Cannot scalarize the following access "
2491 : : "because data would be held in a mode which is not "
2492 : : "guaranteed to preserve all bits.\n ");
2493 : 0 : dump_access (dump_file, access, false);
2494 : : }
2495 : : unscalarizable_region = true;
2496 : : }
2497 : :
2498 : 5127233 : if (grp_same_access_path
2499 : 5127233 : && !same_access_path_p (access->expr, ac2->expr))
2500 : : grp_same_access_path = false;
2501 : :
2502 : 5127233 : ac2->group_representative = access;
2503 : 5127233 : j++;
2504 : : }
2505 : :
2506 : 9096752 : i = j;
2507 : :
2508 : 9096752 : access->group_representative = access;
2509 : 9096752 : access->grp_write = grp_write;
2510 : 9096752 : access->grp_read = grp_read;
2511 : 9096752 : access->grp_scalar_read = grp_scalar_read;
2512 : 9096752 : access->grp_scalar_write = grp_scalar_write;
2513 : 9096752 : access->grp_assignment_read = grp_assignment_read;
2514 : 9096752 : access->grp_assignment_write = grp_assignment_write;
2515 : 9096752 : access->grp_hint = multiple_scalar_reads && !constant_decl_p (var);
2516 : 9096752 : access->grp_partial_lhs = grp_partial_lhs;
2517 : 9096752 : access->grp_unscalarizable_region = unscalarizable_region;
2518 : 9096752 : access->grp_same_access_path = grp_same_access_path;
2519 : :
2520 : 9096752 : *prev_acc_ptr = access;
2521 : 9096752 : prev_acc_ptr = &access->next_grp;
2522 : : }
2523 : :
2524 : 3889738 : gcc_assert (res == (*access_vec)[0]);
2525 : : return res;
2526 : : }
2527 : :
2528 : : /* Create a variable for the given ACCESS which determines the type, name and a
2529 : : few other properties. Return the variable declaration and store it also to
2530 : : ACCESS->replacement. REG_TREE is used when creating a declaration to base a
2531 : : default-definition SSA name on in order to facilitate an uninitialized
2532 : : warning. It is used instead of the actual ACCESS type if that is not of a
2533 : : gimple register type. */
2534 : :
2535 : : static tree
2536 : 4130712 : create_access_replacement (struct access *access, tree reg_type = NULL_TREE)
2537 : : {
2538 : 4130712 : tree repl;
2539 : :
2540 : 4130712 : tree type = access->type;
2541 : 4130712 : if (reg_type && !is_gimple_reg_type (type))
2542 : : type = reg_type;
2543 : :
2544 : 4130712 : if (access->grp_to_be_debug_replaced)
2545 : : {
2546 : 125406 : repl = create_tmp_var_raw (access->type);
2547 : 125406 : DECL_CONTEXT (repl) = current_function_decl;
2548 : : }
2549 : : else
2550 : : /* Drop any special alignment on the type if it's not on the main
2551 : : variant. This avoids issues with weirdo ABIs like AAPCS. */
2552 : 4005306 : repl = create_tmp_var (build_qualified_type (TYPE_MAIN_VARIANT (type),
2553 : 4005306 : TYPE_QUALS (type)), "SR");
2554 : 4130712 : if (access->grp_partial_lhs
2555 : 4130712 : && is_gimple_reg_type (type))
2556 : 623 : DECL_NOT_GIMPLE_REG_P (repl) = 1;
2557 : :
2558 : 4130712 : DECL_SOURCE_LOCATION (repl) = DECL_SOURCE_LOCATION (access->base);
2559 : 4130712 : DECL_ARTIFICIAL (repl) = 1;
2560 : 4130712 : DECL_IGNORED_P (repl) = DECL_IGNORED_P (access->base);
2561 : :
2562 : 4130712 : if (DECL_NAME (access->base)
2563 : 3114924 : && !DECL_IGNORED_P (access->base)
2564 : 5168630 : && !DECL_ARTIFICIAL (access->base))
2565 : : {
2566 : 1029176 : char *pretty_name = make_fancy_name (access->expr);
2567 : 1029176 : tree debug_expr = unshare_expr_without_location (access->expr), d;
2568 : 1029176 : bool fail = false;
2569 : :
2570 : 1029176 : DECL_NAME (repl) = get_identifier (pretty_name);
2571 : 1029176 : DECL_NAMELESS (repl) = 1;
2572 : 1029176 : obstack_free (&name_obstack, pretty_name);
2573 : :
2574 : : /* Get rid of any SSA_NAMEs embedded in debug_expr,
2575 : : as DECL_DEBUG_EXPR isn't considered when looking for still
2576 : : used SSA_NAMEs and thus they could be freed. All debug info
2577 : : generation cares is whether something is constant or variable
2578 : : and that get_ref_base_and_extent works properly on the
2579 : : expression. It cannot handle accesses at a non-constant offset
2580 : : though, so just give up in those cases. */
2581 : 1303360 : for (d = debug_expr;
2582 : 3518022 : !fail && (handled_component_p (d) || TREE_CODE (d) == MEM_REF);
2583 : 1303360 : d = TREE_OPERAND (d, 0))
2584 : 1303360 : switch (TREE_CODE (d))
2585 : : {
2586 : 50862 : case ARRAY_REF:
2587 : 50862 : case ARRAY_RANGE_REF:
2588 : 50862 : if (TREE_OPERAND (d, 1)
2589 : 50862 : && TREE_CODE (TREE_OPERAND (d, 1)) != INTEGER_CST)
2590 : : fail = true;
2591 : 50862 : if (TREE_OPERAND (d, 3)
2592 : 50862 : && TREE_CODE (TREE_OPERAND (d, 3)) != INTEGER_CST)
2593 : : fail = true;
2594 : : /* FALLTHRU */
2595 : 1145730 : case COMPONENT_REF:
2596 : 1145730 : if (TREE_OPERAND (d, 2)
2597 : 1145730 : && TREE_CODE (TREE_OPERAND (d, 2)) != INTEGER_CST)
2598 : : fail = true;
2599 : : break;
2600 : 156423 : case MEM_REF:
2601 : 156423 : if (TREE_CODE (TREE_OPERAND (d, 0)) != ADDR_EXPR)
2602 : : fail = true;
2603 : : else
2604 : 156423 : d = TREE_OPERAND (d, 0);
2605 : : break;
2606 : : default:
2607 : : break;
2608 : : }
2609 : 1029176 : if (!fail)
2610 : : {
2611 : 1029063 : SET_DECL_DEBUG_EXPR (repl, debug_expr);
2612 : 1029063 : DECL_HAS_DEBUG_EXPR_P (repl) = 1;
2613 : : }
2614 : 1029176 : if (access->grp_no_warning)
2615 : 451 : suppress_warning (repl /* Be more selective! */);
2616 : : else
2617 : 1028725 : copy_warning (repl, access->base);
2618 : : }
2619 : : else
2620 : 3101536 : suppress_warning (repl /* Be more selective! */);
2621 : :
2622 : 4130712 : if (dump_file)
2623 : : {
2624 : 157 : if (access->grp_to_be_debug_replaced)
2625 : : {
2626 : 4 : fprintf (dump_file, "Created a debug-only replacement for ");
2627 : 4 : print_generic_expr (dump_file, access->base);
2628 : 4 : fprintf (dump_file, " offset: %u, size: %u\n",
2629 : 4 : (unsigned) access->offset, (unsigned) access->size);
2630 : : }
2631 : : else
2632 : : {
2633 : 153 : fprintf (dump_file, "Created a replacement for ");
2634 : 153 : print_generic_expr (dump_file, access->base);
2635 : 153 : fprintf (dump_file, " offset: %u, size: %u: ",
2636 : 153 : (unsigned) access->offset, (unsigned) access->size);
2637 : 153 : print_generic_expr (dump_file, repl, TDF_UID);
2638 : 153 : fprintf (dump_file, "\n");
2639 : : }
2640 : : }
2641 : 4130712 : sra_stats.replacements++;
2642 : :
2643 : 4130712 : return repl;
2644 : : }
2645 : :
2646 : : /* Return ACCESS scalar replacement, which must exist. */
2647 : :
2648 : : static inline tree
2649 : 13459080 : get_access_replacement (struct access *access)
2650 : : {
2651 : 13459080 : gcc_checking_assert (access->replacement_decl);
2652 : 13459080 : return access->replacement_decl;
2653 : : }
2654 : :
2655 : :
2656 : : /* Build a subtree of accesses rooted in *ACCESS, and move the pointer in the
2657 : : linked list along the way. Stop when *ACCESS is NULL or the access pointed
2658 : : to it is not "within" the root. Return false iff some accesses partially
2659 : : overlap. */
2660 : :
2661 : : static bool
2662 : 9078282 : build_access_subtree (struct access **access)
2663 : : {
2664 : 9078282 : struct access *root = *access, *last_child = NULL;
2665 : 9078282 : HOST_WIDE_INT limit = root->offset + root->size;
2666 : :
2667 : 9078282 : *access = (*access)->next_grp;
2668 : 13854181 : while (*access && (*access)->offset + (*access)->size <= limit)
2669 : : {
2670 : 4778627 : if (!last_child)
2671 : 1917463 : root->first_child = *access;
2672 : : else
2673 : 2861164 : last_child->next_sibling = *access;
2674 : 4778627 : last_child = *access;
2675 : 4778627 : (*access)->parent = root;
2676 : 4778627 : (*access)->grp_write |= root->grp_write;
2677 : :
2678 : 4778627 : if (!build_access_subtree (access))
2679 : : return false;
2680 : : }
2681 : :
2682 : 9075554 : if (*access && (*access)->offset < limit)
2683 : : return false;
2684 : :
2685 : : return true;
2686 : : }
2687 : :
2688 : : /* Build a tree of access representatives, ACCESS is the pointer to the first
2689 : : one, others are linked in a list by the next_grp field. Return false iff
2690 : : some accesses partially overlap. */
2691 : :
2692 : : static bool
2693 : 3889738 : build_access_trees (struct access *access)
2694 : : {
2695 : 8186814 : while (access)
2696 : : {
2697 : 4299655 : struct access *root = access;
2698 : :
2699 : 4299655 : if (!build_access_subtree (&access))
2700 : : return false;
2701 : 4297076 : root->next_grp = access;
2702 : : }
2703 : : return true;
2704 : : }
2705 : :
2706 : : /* Traverse the access forest where ROOT is the first root and verify that
2707 : : various important invariants hold true. */
2708 : :
2709 : : DEBUG_FUNCTION void
2710 : 3887159 : verify_sra_access_forest (struct access *root)
2711 : : {
2712 : 3887159 : struct access *access = root;
2713 : 3887159 : tree first_base = root->base;
2714 : 3887159 : gcc_assert (DECL_P (first_base));
2715 : 10868179 : do
2716 : : {
2717 : 10868179 : gcc_assert (access->base == first_base);
2718 : 10868179 : if (access->parent)
2719 : 6571118 : gcc_assert (access->offset >= access->parent->offset
2720 : : && access->size <= access->parent->size);
2721 : 10868179 : if (access->next_sibling)
2722 : 3826100 : gcc_assert (access->next_sibling->offset
2723 : : >= access->offset + access->size);
2724 : :
2725 : 10868179 : poly_int64 poffset, psize, pmax_size;
2726 : 10868179 : bool reverse;
2727 : 10868179 : tree base = get_ref_base_and_extent (access->expr, &poffset, &psize,
2728 : : &pmax_size, &reverse);
2729 : 10868179 : HOST_WIDE_INT offset, size, max_size;
2730 : 10868179 : if (!poffset.is_constant (&offset)
2731 : 10868179 : || !psize.is_constant (&size)
2732 : 10868179 : || !pmax_size.is_constant (&max_size))
2733 : : gcc_unreachable ();
2734 : 10868179 : gcc_assert (base == first_base);
2735 : 10868179 : gcc_assert (offset == access->offset);
2736 : 10868179 : gcc_assert (access->grp_unscalarizable_region
2737 : : || access->grp_total_scalarization
2738 : : || size == max_size);
2739 : 10868179 : gcc_assert (access->grp_unscalarizable_region
2740 : : || !is_gimple_reg_type (access->type)
2741 : : || size == access->size);
2742 : 10868179 : gcc_assert (reverse == access->reverse);
2743 : :
2744 : 10868179 : if (access->first_child)
2745 : : {
2746 : 2745018 : gcc_assert (access->first_child->parent == access);
2747 : : access = access->first_child;
2748 : : }
2749 : 8123161 : else if (access->next_sibling)
2750 : : {
2751 : 3655489 : gcc_assert (access->next_sibling->parent == access->parent);
2752 : : access = access->next_sibling;
2753 : : }
2754 : : else
2755 : : {
2756 : 7212690 : while (access->parent && !access->next_sibling)
2757 : : access = access->parent;
2758 : 4467672 : if (access->next_sibling)
2759 : : access = access->next_sibling;
2760 : : else
2761 : : {
2762 : 4297061 : gcc_assert (access == root);
2763 : 4297061 : root = root->next_grp;
2764 : 4297061 : access = root;
2765 : : }
2766 : : }
2767 : : }
2768 : 10868179 : while (access);
2769 : 3887159 : }
2770 : :
2771 : : /* Verify access forests of all candidates with accesses by calling
2772 : : verify_access_forest on each on them. */
2773 : :
2774 : : DEBUG_FUNCTION void
2775 : 684075 : verify_all_sra_access_forests (void)
2776 : : {
2777 : 684075 : bitmap_iterator bi;
2778 : 684075 : unsigned i;
2779 : 4571234 : EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
2780 : : {
2781 : 3887159 : tree var = candidate (i);
2782 : 3887159 : struct access *access = get_first_repr_for_decl (var);
2783 : 3887159 : if (access)
2784 : : {
2785 : 3887159 : gcc_assert (access->base == var);
2786 : 3887159 : verify_sra_access_forest (access);
2787 : : }
2788 : : }
2789 : 684075 : }
2790 : :
2791 : : /* Return true if expr contains some ARRAY_REFs into a variable bounded
2792 : : array. */
2793 : :
2794 : : static bool
2795 : 10591670 : expr_with_var_bounded_array_refs_p (tree expr)
2796 : : {
2797 : 20034798 : while (handled_component_p (expr))
2798 : : {
2799 : 9443128 : if (TREE_CODE (expr) == ARRAY_REF
2800 : 9443128 : && !tree_fits_shwi_p (array_ref_low_bound (expr)))
2801 : : return true;
2802 : 9443128 : expr = TREE_OPERAND (expr, 0);
2803 : : }
2804 : : return false;
2805 : : }
2806 : :
2807 : : /* Analyze the subtree of accesses rooted in ROOT, scheduling replacements when
2808 : : both seeming beneficial and when ALLOW_REPLACEMENTS allows it. If TOTALLY
2809 : : is set, we are totally scalarizing the aggregate. Also set all sorts of
2810 : : access flags appropriately along the way, notably always set grp_read and
2811 : : grp_assign_read according to MARK_READ and grp_write when MARK_WRITE is
2812 : : true.
2813 : :
2814 : : Creating a replacement for a scalar access is considered beneficial if its
2815 : : grp_hint ot TOTALLY is set (this means either that there is more than one
2816 : : direct read access or that we are attempting total scalarization) or
2817 : : according to the following table:
2818 : :
2819 : : Access written to through a scalar type (once or more times)
2820 : : |
2821 : : | Written to in an assignment statement
2822 : : | |
2823 : : | | Access read as scalar _once_
2824 : : | | |
2825 : : | | | Read in an assignment statement
2826 : : | | | |
2827 : : | | | | Scalarize Comment
2828 : : -----------------------------------------------------------------------------
2829 : : 0 0 0 0 No access for the scalar
2830 : : 0 0 0 1 No access for the scalar
2831 : : 0 0 1 0 No Single read - won't help
2832 : : 0 0 1 1 No The same case
2833 : : 0 1 0 0 No access for the scalar
2834 : : 0 1 0 1 No access for the scalar
2835 : : 0 1 1 0 Yes s = *g; return s.i;
2836 : : 0 1 1 1 Yes The same case as above
2837 : : 1 0 0 0 No Won't help
2838 : : 1 0 0 1 Yes s.i = 1; *g = s;
2839 : : 1 0 1 0 Yes s.i = 5; g = s.i;
2840 : : 1 0 1 1 Yes The same case as above
2841 : : 1 1 0 0 No Won't help.
2842 : : 1 1 0 1 Yes s.i = 1; *g = s;
2843 : : 1 1 1 0 Yes s = *g; return s.i;
2844 : : 1 1 1 1 Yes Any of the above yeses */
2845 : :
2846 : : static bool
2847 : 10868179 : analyze_access_subtree (struct access *root, struct access *parent,
2848 : : bool allow_replacements, bool totally)
2849 : : {
2850 : 10868179 : struct access *child;
2851 : 10868179 : HOST_WIDE_INT limit = root->offset + root->size;
2852 : 10868179 : HOST_WIDE_INT covered_to = root->offset;
2853 : 10868179 : bool scalar = is_gimple_reg_type (root->type);
2854 : 10868179 : bool hole = false, sth_created = false;
2855 : :
2856 : 10868179 : if (parent)
2857 : : {
2858 : 6571118 : if (parent->grp_read)
2859 : 5386295 : root->grp_read = 1;
2860 : 6571118 : if (parent->grp_assignment_read)
2861 : 2715294 : root->grp_assignment_read = 1;
2862 : 6571118 : if (parent->grp_write)
2863 : 3785432 : root->grp_write = 1;
2864 : 6571118 : if (parent->grp_assignment_write)
2865 : 3213214 : root->grp_assignment_write = 1;
2866 : 6571118 : if (!parent->grp_same_access_path)
2867 : 320480 : root->grp_same_access_path = 0;
2868 : : }
2869 : :
2870 : 10868179 : if (root->grp_unscalarizable_region)
2871 : : allow_replacements = false;
2872 : :
2873 : 10745381 : if (allow_replacements && expr_with_var_bounded_array_refs_p (root->expr))
2874 : : allow_replacements = false;
2875 : :
2876 : 10868179 : if (!totally && root->grp_result_of_prop_from_lhs)
2877 : 10868179 : allow_replacements = false;
2878 : :
2879 : 17439297 : for (child = root->first_child; child; child = child->next_sibling)
2880 : : {
2881 : 6571118 : hole |= covered_to < child->offset;
2882 : 6571118 : sth_created |= analyze_access_subtree (child, root,
2883 : 6571118 : allow_replacements && !scalar
2884 : 6571118 : && !root->grp_partial_lhs,
2885 : : totally);
2886 : :
2887 : 6571118 : root->grp_unscalarized_data |= child->grp_unscalarized_data;
2888 : 6571118 : if (child->grp_covered)
2889 : 3610931 : covered_to += child->size;
2890 : : else
2891 : : hole = true;
2892 : : }
2893 : :
2894 : 10868179 : if (allow_replacements && scalar && !root->first_child
2895 : 7014344 : && (totally || !root->grp_total_scalarization)
2896 : : && (totally
2897 : 4784419 : || root->grp_hint
2898 : 4102606 : || ((root->grp_scalar_read || root->grp_assignment_read)
2899 : 1507116 : && (root->grp_scalar_write || root->grp_assignment_write))))
2900 : : {
2901 : : /* Always create access replacements that cover the whole access.
2902 : : For integral types this means the precision has to match.
2903 : : Avoid assumptions based on the integral type kind, too. */
2904 : 4004798 : if (INTEGRAL_TYPE_P (root->type)
2905 : 1723790 : && ((TREE_CODE (root->type) != INTEGER_TYPE
2906 : 1723790 : && TREE_CODE (root->type) != BITINT_TYPE)
2907 : 1663386 : || TYPE_PRECISION (root->type) != root->size)
2908 : : /* But leave bitfield accesses alone. */
2909 : 4065209 : && (TREE_CODE (root->expr) != COMPONENT_REF
2910 : 59555 : || !DECL_BIT_FIELD (TREE_OPERAND (root->expr, 1))))
2911 : : {
2912 : 60086 : tree rt = root->type;
2913 : 60086 : gcc_assert ((root->offset % BITS_PER_UNIT) == 0
2914 : : && (root->size % BITS_PER_UNIT) == 0);
2915 : 60086 : if (TREE_CODE (root->type) == BITINT_TYPE)
2916 : 7 : root->type = build_bitint_type (root->size, TYPE_UNSIGNED (rt));
2917 : : else
2918 : 60079 : root->type = build_nonstandard_integer_type (root->size,
2919 : 60079 : TYPE_UNSIGNED (rt));
2920 : 120172 : root->expr = build_ref_for_offset (UNKNOWN_LOCATION, root->base,
2921 : 60086 : root->offset, root->reverse,
2922 : : root->type, NULL, false);
2923 : :
2924 : 60086 : if (dump_file && (dump_flags & TDF_DETAILS))
2925 : : {
2926 : 0 : fprintf (dump_file, "Changing the type of a replacement for ");
2927 : 0 : print_generic_expr (dump_file, root->base);
2928 : 0 : fprintf (dump_file, " offset: %u, size: %u ",
2929 : 0 : (unsigned) root->offset, (unsigned) root->size);
2930 : 0 : fprintf (dump_file, " to an integer.\n");
2931 : : }
2932 : : }
2933 : :
2934 : 4004798 : root->grp_to_be_replaced = 1;
2935 : 4004798 : root->replacement_decl = create_access_replacement (root);
2936 : 4004798 : sth_created = true;
2937 : 4004798 : hole = false;
2938 : 4004798 : }
2939 : : else
2940 : : {
2941 : 6863381 : if (allow_replacements
2942 : 3017516 : && scalar && !root->first_child
2943 : 3009546 : && !root->grp_total_scalarization
2944 : 3009504 : && (root->grp_scalar_write || root->grp_assignment_write)
2945 : 9458871 : && !bitmap_bit_p (cannot_scalarize_away_bitmap,
2946 : 2595490 : DECL_UID (root->base)))
2947 : : {
2948 : 334251 : gcc_checking_assert (!root->grp_scalar_read
2949 : : && !root->grp_assignment_read);
2950 : 334251 : sth_created = true;
2951 : 334251 : if (MAY_HAVE_DEBUG_BIND_STMTS)
2952 : : {
2953 : 125406 : root->grp_to_be_debug_replaced = 1;
2954 : 125406 : root->replacement_decl = create_access_replacement (root);
2955 : : }
2956 : : }
2957 : :
2958 : 6863381 : if (covered_to < limit)
2959 : 5505122 : hole = true;
2960 : 6863381 : if (scalar || !allow_replacements)
2961 : 3430512 : root->grp_total_scalarization = 0;
2962 : : }
2963 : :
2964 : 10868179 : if (!hole || totally)
2965 : 5441941 : root->grp_covered = 1;
2966 : 5426238 : else if (root->grp_write || comes_initialized_p (root->base))
2967 : 4681885 : root->grp_unscalarized_data = 1; /* not covered and written to */
2968 : 10868179 : return sth_created;
2969 : : }
2970 : :
2971 : : /* Analyze all access trees linked by next_grp by the means of
2972 : : analyze_access_subtree. */
2973 : : static bool
2974 : 3887159 : analyze_access_trees (struct access *access)
2975 : : {
2976 : 3887159 : bool ret = false;
2977 : :
2978 : 8184220 : while (access)
2979 : : {
2980 : 4297061 : if (analyze_access_subtree (access, NULL, true,
2981 : 4297061 : access->grp_total_scalarization))
2982 : 2271353 : ret = true;
2983 : 4297061 : access = access->next_grp;
2984 : : }
2985 : :
2986 : 3887159 : return ret;
2987 : : }
2988 : :
2989 : : /* Return true iff a potential new child of ACC at offset OFFSET and with size
2990 : : SIZE would conflict with an already existing one. If exactly such a child
2991 : : already exists in ACC, store a pointer to it in EXACT_MATCH. */
2992 : :
2993 : : static bool
2994 : 6537415 : child_would_conflict_in_acc (struct access *acc, HOST_WIDE_INT norm_offset,
2995 : : HOST_WIDE_INT size, struct access **exact_match)
2996 : : {
2997 : 6537415 : struct access *child;
2998 : :
2999 : 11390755 : for (child = acc->first_child; child; child = child->next_sibling)
3000 : : {
3001 : 10284977 : if (child->offset == norm_offset && child->size == size)
3002 : : {
3003 : 5398059 : *exact_match = child;
3004 : 5398059 : return true;
3005 : : }
3006 : :
3007 : 4886918 : if (child->offset < norm_offset + size
3008 : 4812238 : && child->offset + child->size > norm_offset)
3009 : : return true;
3010 : : }
3011 : :
3012 : : return false;
3013 : : }
3014 : :
3015 : : /* Create a new child access of PARENT, with all properties just like MODEL
3016 : : except for its offset and with its grp_write false and grp_read true.
3017 : : Return the new access or NULL if it cannot be created. Note that this
3018 : : access is created long after all splicing and sorting, it's not located in
3019 : : any access vector and is automatically a representative of its group. Set
3020 : : the gpr_write flag of the new accesss if SET_GRP_WRITE is true. */
3021 : :
3022 : : static struct access *
3023 : 1101750 : create_artificial_child_access (struct access *parent, struct access *model,
3024 : : HOST_WIDE_INT new_offset,
3025 : : bool set_grp_read, bool set_grp_write)
3026 : : {
3027 : 1101750 : struct access **child;
3028 : 1101750 : tree expr = parent->base;
3029 : :
3030 : 1101750 : gcc_assert (!model->grp_unscalarizable_region);
3031 : :
3032 : 1101750 : struct access *access = access_pool.allocate ();
3033 : 1101750 : memset (access, 0, sizeof (struct access));
3034 : 1101750 : if (!build_user_friendly_ref_for_offset (&expr, TREE_TYPE (expr), new_offset,
3035 : : model->type))
3036 : : {
3037 : 8097 : access->grp_no_warning = true;
3038 : 8097 : expr = build_ref_for_model (EXPR_LOCATION (parent->base), parent->base,
3039 : : new_offset, model, NULL, false);
3040 : : }
3041 : :
3042 : 1101750 : access->base = parent->base;
3043 : 1101750 : access->expr = expr;
3044 : 1101750 : access->offset = new_offset;
3045 : 1101750 : access->size = model->size;
3046 : 1101750 : access->type = model->type;
3047 : 1101750 : access->parent = parent;
3048 : 1101750 : access->grp_read = set_grp_read;
3049 : 1101750 : access->grp_write = set_grp_write;
3050 : 1101750 : access->reverse = model->reverse;
3051 : :
3052 : 1101750 : child = &parent->first_child;
3053 : 1904816 : while (*child && (*child)->offset < new_offset)
3054 : 803066 : child = &(*child)->next_sibling;
3055 : :
3056 : 1101750 : access->next_sibling = *child;
3057 : 1101750 : *child = access;
3058 : :
3059 : 1101750 : return access;
3060 : : }
3061 : :
3062 : :
3063 : : /* Beginning with ACCESS, traverse its whole access subtree and mark all
3064 : : sub-trees as written to. If any of them has not been marked so previously
3065 : : and has assignment links leading from it, re-enqueue it. */
3066 : :
3067 : : static void
3068 : 1081234 : subtree_mark_written_and_rhs_enqueue (struct access *access)
3069 : : {
3070 : 1081234 : if (access->grp_write)
3071 : : return;
3072 : 1040511 : access->grp_write = true;
3073 : 1040511 : add_access_to_rhs_work_queue (access);
3074 : :
3075 : 1040511 : struct access *child;
3076 : 1265659 : for (child = access->first_child; child; child = child->next_sibling)
3077 : 225148 : subtree_mark_written_and_rhs_enqueue (child);
3078 : : }
3079 : :
3080 : : /* If there is still budget to create a propagation access for DECL, return
3081 : : true and decrement the budget. Otherwise return false. */
3082 : :
3083 : : static bool
3084 : 1105347 : budget_for_propagation_access (tree decl)
3085 : : {
3086 : 1105347 : unsigned b, *p = propagation_budget->get (decl);
3087 : 1105347 : if (p)
3088 : 614196 : b = *p;
3089 : : else
3090 : 491151 : b = param_sra_max_propagations;
3091 : :
3092 : 1105347 : if (b == 0)
3093 : : return false;
3094 : 1101753 : b--;
3095 : :
3096 : 1101753 : if (b == 0 && dump_file && (dump_flags & TDF_DETAILS))
3097 : : {
3098 : 0 : fprintf (dump_file, "The propagation budget of ");
3099 : 0 : print_generic_expr (dump_file, decl);
3100 : 0 : fprintf (dump_file, " (UID: %u) has been exhausted.\n", DECL_UID (decl));
3101 : : }
3102 : 1101753 : propagation_budget->put (decl, b);
3103 : 1101753 : return true;
3104 : : }
3105 : :
3106 : : /* Return true if ACC or any of its subaccesses has grp_child set. */
3107 : :
3108 : : static bool
3109 : 1739 : access_or_its_child_written (struct access *acc)
3110 : : {
3111 : 1739 : if (acc->grp_write)
3112 : : return true;
3113 : 1793 : for (struct access *sub = acc->first_child; sub; sub = sub->next_sibling)
3114 : 417 : if (access_or_its_child_written (sub))
3115 : : return true;
3116 : : return false;
3117 : : }
3118 : :
3119 : : /* Propagate subaccesses and grp_write flags of RACC across an assignment link
3120 : : to LACC. Enqueue sub-accesses as necessary so that the write flag is
3121 : : propagated transitively. Return true if anything changed. Additionally, if
3122 : : RACC is a scalar access but LACC is not, change the type of the latter, if
3123 : : possible. */
3124 : :
3125 : : static bool
3126 : 3477876 : propagate_subaccesses_from_rhs (struct access *lacc, struct access *racc)
3127 : : {
3128 : 3477876 : struct access *rchild;
3129 : 3477876 : HOST_WIDE_INT norm_delta = lacc->offset - racc->offset;
3130 : 3477876 : bool ret = false;
3131 : :
3132 : : /* IF the LHS is still not marked as being written to, we only need to do so
3133 : : if the RHS at this level actually was. */
3134 : 3477876 : if (!lacc->grp_write)
3135 : : {
3136 : 1339524 : gcc_checking_assert (!comes_initialized_p (racc->base));
3137 : 1339524 : if (racc->grp_write)
3138 : : {
3139 : 640341 : subtree_mark_written_and_rhs_enqueue (lacc);
3140 : 640341 : ret = true;
3141 : : }
3142 : : }
3143 : :
3144 : 3477876 : if (is_gimple_reg_type (lacc->type)
3145 : 2534260 : || lacc->grp_unscalarizable_region
3146 : 6011467 : || racc->grp_unscalarizable_region)
3147 : : {
3148 : 946504 : if (!lacc->grp_write)
3149 : : {
3150 : 76327 : ret = true;
3151 : 76327 : subtree_mark_written_and_rhs_enqueue (lacc);
3152 : : }
3153 : 946504 : return ret;
3154 : : }
3155 : :
3156 : 2531372 : if (is_gimple_reg_type (racc->type))
3157 : : {
3158 : 188933 : if (!lacc->grp_write)
3159 : : {
3160 : 1705 : ret = true;
3161 : 1705 : subtree_mark_written_and_rhs_enqueue (lacc);
3162 : : }
3163 : 188933 : if (!lacc->first_child
3164 : 188811 : && !racc->first_child
3165 : 377489 : && !types_risk_mangled_binary_repr_p (racc->type, lacc->type))
3166 : : {
3167 : : /* We are about to change the access type from aggregate to scalar,
3168 : : so we need to put the reverse flag onto the access, if any. */
3169 : 188556 : const bool reverse
3170 : 188556 : = TYPE_REVERSE_STORAGE_ORDER (lacc->type)
3171 : 1 : && !POINTER_TYPE_P (racc->type)
3172 : 188556 : && !VECTOR_TYPE_P (racc->type);
3173 : 188556 : tree t = lacc->base;
3174 : :
3175 : 188556 : lacc->type = racc->type;
3176 : 188556 : if (build_user_friendly_ref_for_offset (&t, TREE_TYPE (t),
3177 : : lacc->offset, racc->type))
3178 : : {
3179 : 188467 : lacc->expr = t;
3180 : 188467 : lacc->grp_same_access_path = true;
3181 : : }
3182 : : else
3183 : : {
3184 : 89 : lacc->expr = build_ref_for_model (EXPR_LOCATION (lacc->base),
3185 : : lacc->base, lacc->offset,
3186 : : racc, NULL, false);
3187 : 89 : if (TREE_CODE (lacc->expr) == MEM_REF)
3188 : 89 : REF_REVERSE_STORAGE_ORDER (lacc->expr) = reverse;
3189 : 89 : lacc->grp_no_warning = true;
3190 : 89 : lacc->grp_same_access_path = false;
3191 : : }
3192 : 188556 : lacc->reverse = reverse;
3193 : : }
3194 : 188933 : return ret;
3195 : : }
3196 : :
3197 : 5674461 : for (rchild = racc->first_child; rchild; rchild = rchild->next_sibling)
3198 : : {
3199 : 3332022 : struct access *new_acc = NULL;
3200 : 3332022 : HOST_WIDE_INT norm_offset = rchild->offset + norm_delta;
3201 : :
3202 : 3332022 : if (child_would_conflict_in_acc (lacc, norm_offset, rchild->size,
3203 : : &new_acc))
3204 : : {
3205 : 2707179 : if (new_acc)
3206 : : {
3207 : 2687563 : if (!new_acc->grp_write && rchild->grp_write)
3208 : : {
3209 : 131266 : gcc_assert (!lacc->grp_write);
3210 : 131266 : subtree_mark_written_and_rhs_enqueue (new_acc);
3211 : 131266 : ret = true;
3212 : : }
3213 : :
3214 : 2687563 : rchild->grp_hint = 1;
3215 : 2687563 : new_acc->grp_hint |= new_acc->grp_read;
3216 : 2687563 : if (rchild->first_child
3217 : 2687563 : && propagate_subaccesses_from_rhs (new_acc, rchild))
3218 : : {
3219 : 1298 : ret = 1;
3220 : 1298 : add_access_to_rhs_work_queue (new_acc);
3221 : : }
3222 : : }
3223 : : else
3224 : : {
3225 : 19616 : if (!lacc->grp_write)
3226 : : {
3227 : 5458 : ret = true;
3228 : 5458 : subtree_mark_written_and_rhs_enqueue (lacc);
3229 : : }
3230 : : }
3231 : 2711188 : continue;
3232 : : }
3233 : :
3234 : 628852 : if (rchild->grp_unscalarizable_region
3235 : 624843 : || !budget_for_propagation_access (lacc->base))
3236 : : {
3237 : 4009 : if (!lacc->grp_write && access_or_its_child_written (rchild))
3238 : : {
3239 : 330 : ret = true;
3240 : 330 : subtree_mark_written_and_rhs_enqueue (lacc);
3241 : : }
3242 : 4009 : continue;
3243 : : }
3244 : :
3245 : 620834 : rchild->grp_hint = 1;
3246 : : /* Because get_ref_base_and_extent always includes padding in size for
3247 : : accesses to DECLs but not necessarily for COMPONENT_REFs of the same
3248 : : type, we might be actually attempting to here to create a child of the
3249 : : same type as the parent. */
3250 : 620834 : if (!types_compatible_p (lacc->type, rchild->type))
3251 : 620834 : new_acc = create_artificial_child_access (lacc, rchild, norm_offset,
3252 : : false,
3253 : 620834 : (lacc->grp_write
3254 : 620834 : || rchild->grp_write));
3255 : : else
3256 : 0 : new_acc = lacc;
3257 : 620834 : gcc_checking_assert (new_acc);
3258 : 620834 : if (racc->first_child)
3259 : 620834 : propagate_subaccesses_from_rhs (new_acc, rchild);
3260 : :
3261 : 620834 : add_access_to_rhs_work_queue (lacc);
3262 : 620834 : ret = true;
3263 : : }
3264 : :
3265 : : return ret;
3266 : : }
3267 : :
3268 : : /* Propagate subaccesses of LACC across an assignment link to RACC if they
3269 : : should inhibit total scalarization of the corresponding area. No flags are
3270 : : being propagated in the process. Return true if anything changed. */
3271 : :
3272 : : static bool
3273 : 5370567 : propagate_subaccesses_from_lhs (struct access *lacc, struct access *racc)
3274 : : {
3275 : 5370567 : if (is_gimple_reg_type (racc->type)
3276 : 1922430 : || lacc->grp_unscalarizable_region
3277 : 7291305 : || racc->grp_unscalarizable_region)
3278 : : return false;
3279 : :
3280 : : /* TODO: Do we want set some new racc flag to stop potential total
3281 : : scalarization if lacc is a scalar access (and none fo the two have
3282 : : children)? */
3283 : :
3284 : 1920718 : bool ret = false;
3285 : 1920718 : HOST_WIDE_INT norm_delta = racc->offset - lacc->offset;
3286 : 1920718 : for (struct access *lchild = lacc->first_child;
3287 : 5129906 : lchild;
3288 : 3209188 : lchild = lchild->next_sibling)
3289 : : {
3290 : 3209188 : struct access *matching_acc = NULL;
3291 : 3209188 : HOST_WIDE_INT norm_offset = lchild->offset + norm_delta;
3292 : :
3293 : 5937457 : if (lchild->grp_unscalarizable_region
3294 : 3205393 : || child_would_conflict_in_acc (racc, norm_offset, lchild->size,
3295 : : &matching_acc)
3296 : 3690123 : || !budget_for_propagation_access (racc->base))
3297 : : {
3298 : 2728269 : if (matching_acc
3299 : 2728269 : && propagate_subaccesses_from_lhs (lchild, matching_acc))
3300 : 277 : add_access_to_lhs_work_queue (matching_acc);
3301 : 2728269 : continue;
3302 : : }
3303 : :
3304 : : /* Because get_ref_base_and_extent always includes padding in size for
3305 : : accesses to DECLs but not necessarily for COMPONENT_REFs of the same
3306 : : type, we might be actually attempting to here to create a child of the
3307 : : same type as the parent. */
3308 : 480919 : if (!types_compatible_p (racc->type, lchild->type))
3309 : : {
3310 : 480916 : struct access *new_acc
3311 : 480916 : = create_artificial_child_access (racc, lchild, norm_offset,
3312 : : true, false);
3313 : 480916 : new_acc->grp_result_of_prop_from_lhs = 1;
3314 : 480916 : propagate_subaccesses_from_lhs (lchild, new_acc);
3315 : : }
3316 : : else
3317 : 3 : propagate_subaccesses_from_lhs (lchild, racc);
3318 : 480919 : ret = true;
3319 : : }
3320 : : return ret;
3321 : : }
3322 : :
3323 : : /* Propagate all subaccesses across assignment links. */
3324 : :
3325 : : static void
3326 : 684075 : propagate_all_subaccesses (void)
3327 : : {
3328 : 684075 : propagation_budget = new hash_map<tree, unsigned>;
3329 : 2297492 : while (rhs_work_queue_head)
3330 : : {
3331 : 1613417 : struct access *racc = pop_access_from_rhs_work_queue ();
3332 : 1613417 : struct assign_link *link;
3333 : :
3334 : 1613417 : if (racc->group_representative)
3335 : 1612644 : racc= racc->group_representative;
3336 : 1613417 : gcc_assert (racc->first_rhs_link);
3337 : :
3338 : 4457888 : for (link = racc->first_rhs_link; link; link = link->next_rhs)
3339 : : {
3340 : 2844471 : struct access *lacc = link->lacc;
3341 : :
3342 : 2844471 : if (!bitmap_bit_p (candidate_bitmap, DECL_UID (lacc->base)))
3343 : 5133 : continue;
3344 : 2839338 : lacc = lacc->group_representative;
3345 : :
3346 : 2839338 : bool reque_parents = false;
3347 : 2839338 : if (!bitmap_bit_p (candidate_bitmap, DECL_UID (racc->base)))
3348 : : {
3349 : 1790 : if (!lacc->grp_write)
3350 : : {
3351 : 659 : subtree_mark_written_and_rhs_enqueue (lacc);
3352 : 659 : reque_parents = true;
3353 : : }
3354 : : }
3355 : 2837548 : else if (propagate_subaccesses_from_rhs (lacc, racc))
3356 : : reque_parents = true;
3357 : :
3358 : : if (reque_parents)
3359 : 1182897 : do
3360 : : {
3361 : 1182897 : add_access_to_rhs_work_queue (lacc);
3362 : 1182897 : lacc = lacc->parent;
3363 : : }
3364 : 1182897 : while (lacc);
3365 : : }
3366 : : }
3367 : :
3368 : 1939641 : while (lhs_work_queue_head)
3369 : : {
3370 : 1255566 : struct access *lacc = pop_access_from_lhs_work_queue ();
3371 : 1255566 : struct assign_link *link;
3372 : :
3373 : 1255566 : if (lacc->group_representative)
3374 : 1252450 : lacc = lacc->group_representative;
3375 : 1255566 : gcc_assert (lacc->first_lhs_link);
3376 : :
3377 : 1255566 : if (!bitmap_bit_p (candidate_bitmap, DECL_UID (lacc->base)))
3378 : 3448 : continue;
3379 : :
3380 : 3432195 : for (link = lacc->first_lhs_link; link; link = link->next_lhs)
3381 : : {
3382 : 2180077 : struct access *racc = link->racc;
3383 : :
3384 : 2180077 : if (racc->group_representative)
3385 : 2179479 : racc = racc->group_representative;
3386 : 2180077 : if (!bitmap_bit_p (candidate_bitmap, DECL_UID (racc->base)))
3387 : 925 : continue;
3388 : 2179152 : if (propagate_subaccesses_from_lhs (lacc, racc))
3389 : 243407 : add_access_to_lhs_work_queue (racc);
3390 : : }
3391 : : }
3392 : 1368150 : delete propagation_budget;
3393 : 684075 : }
3394 : :
3395 : : /* Return true if the forest beginning with ROOT does not contain
3396 : : unscalarizable regions or non-byte aligned accesses. */
3397 : :
3398 : : static bool
3399 : 1058815 : can_totally_scalarize_forest_p (struct access *root)
3400 : : {
3401 : 1058815 : struct access *access = root;
3402 : 2701673 : do
3403 : : {
3404 : 2701673 : if (access->grp_unscalarizable_region
3405 : 2699182 : || (access->offset % BITS_PER_UNIT) != 0
3406 : 2698417 : || (access->size % BITS_PER_UNIT) != 0
3407 : 5397379 : || (is_gimple_reg_type (access->type)
3408 : 1718024 : && access->first_child))
3409 : : return false;
3410 : :
3411 : 2695543 : if (access->first_child)
3412 : : access = access->first_child;
3413 : 1946676 : else if (access->next_sibling)
3414 : : access = access->next_sibling;
3415 : : else
3416 : : {
3417 : 1872352 : while (access->parent && !access->next_sibling)
3418 : : access = access->parent;
3419 : 1127402 : if (access->next_sibling)
3420 : : access = access->next_sibling;
3421 : : else
3422 : : {
3423 : 1072364 : gcc_assert (access == root);
3424 : 1072364 : root = root->next_grp;
3425 : 1072364 : access = root;
3426 : : }
3427 : : }
3428 : : }
3429 : 2695543 : while (access);
3430 : : return true;
3431 : : }
3432 : :
3433 : : /* Create and return an ACCESS in PARENT spanning from POS with SIZE, TYPE and
3434 : : reference EXPR for total scalarization purposes and mark it as such. Within
3435 : : the children of PARENT, link it in between PTR and NEXT_SIBLING. */
3436 : :
3437 : : static struct access *
3438 : 694623 : create_total_scalarization_access (struct access *parent, HOST_WIDE_INT pos,
3439 : : HOST_WIDE_INT size, tree type, tree expr,
3440 : : struct access **ptr,
3441 : : struct access *next_sibling)
3442 : : {
3443 : 694623 : struct access *access = access_pool.allocate ();
3444 : 694623 : memset (access, 0, sizeof (struct access));
3445 : 694623 : access->base = parent->base;
3446 : 694623 : access->offset = pos;
3447 : 694623 : access->size = size;
3448 : 694623 : access->expr = expr;
3449 : 694623 : access->type = type;
3450 : 694623 : access->parent = parent;
3451 : 694623 : access->grp_write = parent->grp_write;
3452 : 694623 : access->grp_total_scalarization = 1;
3453 : 694623 : access->grp_hint = 1;
3454 : 694623 : access->grp_same_access_path = path_comparable_for_same_access (expr);
3455 : 694623 : access->reverse = reverse_storage_order_for_component_p (expr);
3456 : :
3457 : 694623 : access->next_sibling = next_sibling;
3458 : 694623 : *ptr = access;
3459 : 694623 : return access;
3460 : : }
3461 : :
3462 : : /* Create and return an ACCESS in PARENT spanning from POS with SIZE, TYPE and
3463 : : reference EXPR for total scalarization purposes and mark it as such, link it
3464 : : at *PTR and reshape the tree so that those elements at *PTR and their
3465 : : siblings which fall within the part described by POS and SIZE are moved to
3466 : : be children of the new access. If a partial overlap is detected, return
3467 : : NULL. */
3468 : :
3469 : : static struct access *
3470 : 694623 : create_total_access_and_reshape (struct access *parent, HOST_WIDE_INT pos,
3471 : : HOST_WIDE_INT size, tree type, tree expr,
3472 : : struct access **ptr)
3473 : : {
3474 : 694623 : struct access **p = ptr;
3475 : :
3476 : 839858 : while (*p && (*p)->offset < pos + size)
3477 : : {
3478 : 145235 : if ((*p)->offset + (*p)->size > pos + size)
3479 : : return NULL;
3480 : 145235 : p = &(*p)->next_sibling;
3481 : : }
3482 : :
3483 : 694623 : struct access *next_child = *ptr;
3484 : 694623 : struct access *new_acc
3485 : 694623 : = create_total_scalarization_access (parent, pos, size, type, expr,
3486 : : ptr, *p);
3487 : 694623 : if (p != ptr)
3488 : : {
3489 : 60026 : new_acc->first_child = next_child;
3490 : 60026 : *p = NULL;
3491 : 205261 : for (struct access *a = next_child; a; a = a->next_sibling)
3492 : 145235 : a->parent = new_acc;
3493 : : }
3494 : : return new_acc;
3495 : : }
3496 : :
3497 : : static bool totally_scalarize_subtree (struct access *root);
3498 : :
3499 : : /* Return true if INNER is either the same type as OUTER or if it is the type
3500 : : of a record field in OUTER at offset zero, possibly in nested
3501 : : sub-records. */
3502 : :
3503 : : static bool
3504 : 192661 : access_and_field_type_match_p (tree outer, tree inner)
3505 : : {
3506 : 192661 : if (TYPE_MAIN_VARIANT (outer) == TYPE_MAIN_VARIANT (inner))
3507 : : return true;
3508 : 235 : if (TREE_CODE (outer) != RECORD_TYPE)
3509 : : return false;
3510 : 221 : tree fld = TYPE_FIELDS (outer);
3511 : 4758 : while (fld)
3512 : : {
3513 : 4685 : if (TREE_CODE (fld) == FIELD_DECL)
3514 : : {
3515 : 252 : if (!zerop (DECL_FIELD_OFFSET (fld)))
3516 : : return false;
3517 : 252 : if (TYPE_MAIN_VARIANT (TREE_TYPE (fld)) == inner)
3518 : : return true;
3519 : 132 : if (TREE_CODE (TREE_TYPE (fld)) == RECORD_TYPE)
3520 : 104 : fld = TYPE_FIELDS (TREE_TYPE (fld));
3521 : : else
3522 : : return false;
3523 : : }
3524 : : else
3525 : 4433 : fld = DECL_CHAIN (fld);
3526 : : }
3527 : : return false;
3528 : : }
3529 : :
3530 : : /* Return type of total_should_skip_creating_access indicating whether a total
3531 : : scalarization access for a field/element should be created, whether it
3532 : : already exists or whether the entire total scalarization has to fail. */
3533 : :
3534 : : enum total_sra_field_state {TOTAL_FLD_CREATE, TOTAL_FLD_DONE, TOTAL_FLD_FAILED};
3535 : :
3536 : : /* Do all the necessary steps in total scalarization when the given aggregate
3537 : : type has a TYPE at POS with the given SIZE should be put into PARENT and
3538 : : when we have processed all its siblings with smaller offsets up until and
3539 : : including LAST_SEEN_SIBLING (which can be NULL).
3540 : :
3541 : : If some further siblings are to be skipped, set *LAST_SEEN_SIBLING as
3542 : : appropriate. Return TOTAL_FLD_CREATE id the caller should carry on with
3543 : : creating a new access, TOTAL_FLD_DONE if access or accesses capable of
3544 : : representing the described part of the aggregate for the purposes of total
3545 : : scalarization already exist or TOTAL_FLD_FAILED if there is a problem which
3546 : : prevents total scalarization from happening at all. */
3547 : :
3548 : : static enum total_sra_field_state
3549 : 2268714 : total_should_skip_creating_access (struct access *parent,
3550 : : struct access **last_seen_sibling,
3551 : : tree type, HOST_WIDE_INT pos,
3552 : : HOST_WIDE_INT size)
3553 : : {
3554 : 2268714 : struct access *next_child;
3555 : 2268714 : if (!*last_seen_sibling)
3556 : 1068891 : next_child = parent->first_child;
3557 : : else
3558 : 1199823 : next_child = (*last_seen_sibling)->next_sibling;
3559 : :
3560 : : /* First, traverse the chain of siblings until it points to an access with
3561 : : offset at least equal to POS. Check all skipped accesses whether they
3562 : : span the POS boundary and if so, return with a failure. */
3563 : 2268747 : while (next_child && next_child->offset < pos)
3564 : : {
3565 : 33 : if (next_child->offset + next_child->size > pos)
3566 : : return TOTAL_FLD_FAILED;
3567 : 33 : *last_seen_sibling = next_child;
3568 : 33 : next_child = next_child->next_sibling;
3569 : : }
3570 : :
3571 : : /* Now check whether next_child has exactly the right POS and SIZE and if so,
3572 : : whether it can represent what we need and can be totally scalarized
3573 : : itself. */
3574 : 2268714 : if (next_child && next_child->offset == pos
3575 : 1633508 : && next_child->size == size)
3576 : : {
3577 : 1573917 : if (!is_gimple_reg_type (next_child->type)
3578 : 1573917 : && (!access_and_field_type_match_p (type, next_child->type)
3579 : 192546 : || !totally_scalarize_subtree (next_child)))
3580 : 115 : return TOTAL_FLD_FAILED;
3581 : :
3582 : 1573802 : *last_seen_sibling = next_child;
3583 : 1573802 : return TOTAL_FLD_DONE;
3584 : : }
3585 : :
3586 : : /* If the child we're looking at would partially overlap, we just cannot
3587 : : totally scalarize. */
3588 : : if (next_child
3589 : 85324 : && next_child->offset < pos + size
3590 : 60200 : && next_child->offset + next_child->size > pos + size)
3591 : : return TOTAL_FLD_FAILED;
3592 : :
3593 : 694674 : if (is_gimple_reg_type (type))
3594 : : {
3595 : : /* We don't scalarize accesses that are children of other scalar type
3596 : : accesses, so if we go on and create an access for a register type,
3597 : : there should not be any pre-existing children. There are rare cases
3598 : : where the requested type is a vector but we already have register
3599 : : accesses for all its elements which is equally good. Detect that
3600 : : situation or whether we need to bail out. */
3601 : :
3602 : : HOST_WIDE_INT covered = pos;
3603 : : bool skipping = false;
3604 : : while (next_child
3605 : 569848 : && next_child->offset + next_child->size <= pos + size)
3606 : : {
3607 : 177 : if (next_child->offset != covered
3608 : 177 : || !is_gimple_reg_type (next_child->type))
3609 : : return TOTAL_FLD_FAILED;
3610 : :
3611 : 177 : covered += next_child->size;
3612 : 177 : *last_seen_sibling = next_child;
3613 : 177 : next_child = next_child->next_sibling;
3614 : 177 : skipping = true;
3615 : : }
3616 : :
3617 : 569671 : if (skipping)
3618 : : {
3619 : 51 : if (covered != pos + size)
3620 : : return TOTAL_FLD_FAILED;
3621 : : else
3622 : : return TOTAL_FLD_DONE;
3623 : : }
3624 : : }
3625 : :
3626 : : return TOTAL_FLD_CREATE;
3627 : : }
3628 : :
3629 : : /* Go over sub-tree rooted in ROOT and attempt to create scalar accesses
3630 : : spanning all uncovered areas covered by ROOT, return false if the attempt
3631 : : failed. All created accesses will have grp_unscalarizable_region set (and
3632 : : should be ignored if the function returns false). */
3633 : :
3634 : : static bool
3635 : 1070852 : totally_scalarize_subtree (struct access *root)
3636 : : {
3637 : 1070852 : gcc_checking_assert (!root->grp_unscalarizable_region);
3638 : 1070852 : gcc_checking_assert (!is_gimple_reg_type (root->type));
3639 : :
3640 : 1070852 : struct access *last_seen_sibling = NULL;
3641 : :
3642 : 1070852 : switch (TREE_CODE (root->type))
3643 : : {
3644 : 1061186 : case RECORD_TYPE:
3645 : 14053336 : for (tree fld = TYPE_FIELDS (root->type); fld; fld = DECL_CHAIN (fld))
3646 : 12992415 : if (TREE_CODE (fld) == FIELD_DECL)
3647 : : {
3648 : 2289622 : tree ft = TREE_TYPE (fld);
3649 : 2289622 : HOST_WIDE_INT fsize = tree_to_uhwi (DECL_SIZE (fld));
3650 : 2289622 : if (!fsize)
3651 : 39900 : continue;
3652 : :
3653 : 2249722 : HOST_WIDE_INT pos = root->offset + int_bit_position (fld);
3654 : 2249722 : if (pos + fsize > root->offset + root->size)
3655 : : return false;
3656 : 2249722 : enum total_sra_field_state
3657 : 2249722 : state = total_should_skip_creating_access (root,
3658 : : &last_seen_sibling,
3659 : : ft, pos, fsize);
3660 : 2249722 : switch (state)
3661 : : {
3662 : : case TOTAL_FLD_FAILED:
3663 : : return false;
3664 : 1570952 : case TOTAL_FLD_DONE:
3665 : 1570952 : continue;
3666 : 678550 : case TOTAL_FLD_CREATE:
3667 : 678550 : break;
3668 : 0 : default:
3669 : 0 : gcc_unreachable ();
3670 : : }
3671 : :
3672 : 1357100 : struct access **p = (last_seen_sibling
3673 : 678550 : ? &last_seen_sibling->next_sibling
3674 : : : &root->first_child);
3675 : 678550 : tree nref = build3 (COMPONENT_REF, ft, root->expr, fld, NULL_TREE);
3676 : 678550 : struct access *new_child
3677 : 678550 : = create_total_access_and_reshape (root, pos, fsize, ft, nref, p);
3678 : 678550 : if (!new_child)
3679 : : return false;
3680 : :
3681 : 678550 : if (!is_gimple_reg_type (ft)
3682 : 678550 : && !totally_scalarize_subtree (new_child))
3683 : : return false;
3684 : 678505 : last_seen_sibling = new_child;
3685 : : }
3686 : : break;
3687 : 9666 : case ARRAY_TYPE:
3688 : 9666 : {
3689 : 9666 : tree elemtype = TREE_TYPE (root->type);
3690 : 9666 : HOST_WIDE_INT el_size;
3691 : 9666 : offset_int idx, max;
3692 : 9666 : if (!prepare_iteration_over_array_elts (root->type, &el_size,
3693 : : &idx, &max))
3694 : : break;
3695 : :
3696 : 9666 : for (HOST_WIDE_INT pos = root->offset;
3697 : 28628 : idx <= max;
3698 : 18962 : pos += el_size, ++idx)
3699 : : {
3700 : 18992 : enum total_sra_field_state
3701 : 18992 : state = total_should_skip_creating_access (root,
3702 : : &last_seen_sibling,
3703 : : elemtype, pos,
3704 : : el_size);
3705 : 18992 : switch (state)
3706 : : {
3707 : : case TOTAL_FLD_FAILED:
3708 : 30 : return false;
3709 : 2892 : case TOTAL_FLD_DONE:
3710 : 2892 : continue;
3711 : 16073 : case TOTAL_FLD_CREATE:
3712 : 16073 : break;
3713 : 0 : default:
3714 : 0 : gcc_unreachable ();
3715 : : }
3716 : :
3717 : 32146 : struct access **p = (last_seen_sibling
3718 : 16073 : ? &last_seen_sibling->next_sibling
3719 : : : &root->first_child);
3720 : 32146 : tree nref = build4 (ARRAY_REF, elemtype, root->expr,
3721 : 16073 : wide_int_to_tree (TYPE_DOMAIN (root->type),
3722 : : idx),
3723 : : NULL_TREE, NULL_TREE);
3724 : 16073 : struct access *new_child
3725 : 16073 : = create_total_access_and_reshape (root, pos, el_size, elemtype,
3726 : : nref, p);
3727 : 16073 : if (!new_child)
3728 : : return false;
3729 : :
3730 : 16073 : if (!is_gimple_reg_type (elemtype)
3731 : 16073 : && !totally_scalarize_subtree (new_child))
3732 : : return false;
3733 : 16070 : last_seen_sibling = new_child;
3734 : : }
3735 : : }
3736 : 9636 : break;
3737 : 0 : default:
3738 : 0 : gcc_unreachable ();
3739 : : }
3740 : : return true;
3741 : : }
3742 : :
3743 : : /* Get the total total scalarization size limit in the current function. */
3744 : :
3745 : : unsigned HOST_WIDE_INT
3746 : 689999 : sra_get_max_scalarization_size (void)
3747 : : {
3748 : 689999 : bool optimize_speed_p = !optimize_function_for_size_p (cfun);
3749 : : /* If the user didn't set PARAM_SRA_MAX_SCALARIZATION_SIZE_<...>,
3750 : : fall back to a target default. */
3751 : 689999 : unsigned HOST_WIDE_INT max_scalarization_size
3752 : 689999 : = get_move_ratio (optimize_speed_p) * UNITS_PER_WORD;
3753 : :
3754 : 689999 : if (optimize_speed_p)
3755 : : {
3756 : 671705 : if (OPTION_SET_P (param_sra_max_scalarization_size_speed))
3757 : 9 : max_scalarization_size = param_sra_max_scalarization_size_speed;
3758 : : }
3759 : : else
3760 : : {
3761 : 18294 : if (OPTION_SET_P (param_sra_max_scalarization_size_size))
3762 : 0 : max_scalarization_size = param_sra_max_scalarization_size_size;
3763 : : }
3764 : 689999 : max_scalarization_size *= BITS_PER_UNIT;
3765 : 689999 : return max_scalarization_size;
3766 : : }
3767 : :
3768 : : /* Go through all accesses collected throughout the (intraprocedural) analysis
3769 : : stage, exclude overlapping ones, identify representatives and build trees
3770 : : out of them, making decisions about scalarization on the way. Return true
3771 : : iff there are any to-be-scalarized variables after this stage. */
3772 : :
3773 : : static bool
3774 : 684075 : analyze_all_variable_accesses (void)
3775 : : {
3776 : 684075 : int res = 0;
3777 : 684075 : bitmap tmp = BITMAP_ALLOC (NULL);
3778 : 684075 : bitmap_iterator bi;
3779 : 684075 : unsigned i;
3780 : :
3781 : 684075 : bitmap_copy (tmp, candidate_bitmap);
3782 : 4614975 : EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
3783 : : {
3784 : 3930900 : tree var = candidate (i);
3785 : 3930900 : struct access *access;
3786 : :
3787 : 3930900 : access = sort_and_splice_var_accesses (var);
3788 : 3930900 : if (!access || !build_access_trees (access))
3789 : 43741 : disqualify_candidate (var,
3790 : : "No or inhibitingly overlapping accesses.");
3791 : : }
3792 : :
3793 : 684075 : propagate_all_subaccesses ();
3794 : :
3795 : 684075 : unsigned HOST_WIDE_INT max_scalarization_size
3796 : 684075 : = sra_get_max_scalarization_size ();
3797 : 4571234 : EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
3798 : 3887159 : if (bitmap_bit_p (should_scalarize_away_bitmap, i)
3799 : 3887159 : && !bitmap_bit_p (cannot_scalarize_away_bitmap, i))
3800 : : {
3801 : 1145211 : tree var = candidate (i);
3802 : 1145211 : if (!VAR_P (var))
3803 : 79849 : continue;
3804 : :
3805 : 1065362 : if (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))) > max_scalarization_size)
3806 : : {
3807 : 11682 : if (dump_file && (dump_flags & TDF_DETAILS))
3808 : : {
3809 : 0 : fprintf (dump_file, "Too big to totally scalarize: ");
3810 : 0 : print_generic_expr (dump_file, var);
3811 : 0 : fprintf (dump_file, " (UID: %u)\n", DECL_UID (var));
3812 : : }
3813 : 11682 : continue;
3814 : : }
3815 : :
3816 : 1053680 : bool all_types_ok = true;
3817 : 1053680 : for (struct access *access = get_first_repr_for_decl (var);
3818 : 2086037 : access;
3819 : 1032357 : access = access->next_grp)
3820 : 1058815 : if (!can_totally_scalarize_forest_p (access)
3821 : 2111500 : || !totally_scalarizable_type_p (access->type,
3822 : 1052685 : constant_decl_p (var),
3823 : : 0, nullptr))
3824 : : {
3825 : : all_types_ok = false;
3826 : : break;
3827 : : }
3828 : 1053680 : if (!all_types_ok)
3829 : 26458 : continue;
3830 : :
3831 : 1027222 : if (dump_file && (dump_flags & TDF_DETAILS))
3832 : : {
3833 : 4 : fprintf (dump_file, "Will attempt to totally scalarize ");
3834 : 4 : print_generic_expr (dump_file, var);
3835 : 4 : fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
3836 : : }
3837 : 1027222 : bool scalarized = true;
3838 : 1027222 : for (struct access *access = get_first_repr_for_decl (var);
3839 : 2059328 : access;
3840 : 1032106 : access = access->next_grp)
3841 : 1032353 : if (!is_gimple_reg_type (access->type)
3842 : 1032353 : && !totally_scalarize_subtree (access))
3843 : : {
3844 : : scalarized = false;
3845 : : break;
3846 : : }
3847 : :
3848 : 1027222 : if (scalarized)
3849 : 1026975 : for (struct access *access = get_first_repr_for_decl (var);
3850 : 2059081 : access;
3851 : 1032106 : access = access->next_grp)
3852 : 1032106 : access->grp_total_scalarization = true;
3853 : : }
3854 : :
3855 : 684075 : if (flag_checking)
3856 : 684075 : verify_all_sra_access_forests ();
3857 : :
3858 : 684075 : bitmap_copy (tmp, candidate_bitmap);
3859 : 4571234 : EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
3860 : : {
3861 : 3887159 : tree var = candidate (i);
3862 : 3887159 : struct access *access = get_first_repr_for_decl (var);
3863 : :
3864 : 3887159 : if (analyze_access_trees (access))
3865 : : {
3866 : 1934646 : res++;
3867 : 1934646 : if (dump_file && (dump_flags & TDF_DETAILS))
3868 : : {
3869 : 9 : fprintf (dump_file, "\nAccess trees for ");
3870 : 9 : print_generic_expr (dump_file, var);
3871 : 9 : fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
3872 : 9 : dump_access_tree (dump_file, access);
3873 : 9 : fprintf (dump_file, "\n");
3874 : : }
3875 : : }
3876 : : else
3877 : 1952513 : disqualify_candidate (var, "No scalar replacements to be created.");
3878 : : }
3879 : :
3880 : 684075 : BITMAP_FREE (tmp);
3881 : :
3882 : 684075 : if (res)
3883 : : {
3884 : 428278 : statistics_counter_event (cfun, "Scalarized aggregates", res);
3885 : 428278 : return true;
3886 : : }
3887 : : else
3888 : : return false;
3889 : : }
3890 : :
3891 : : /* Generate statements copying scalar replacements of accesses within a subtree
3892 : : into or out of AGG. ACCESS, all its children, siblings and their children
3893 : : are to be processed. AGG is an aggregate type expression (can be a
3894 : : declaration but does not have to be, it can for example also be a mem_ref or
3895 : : a series of handled components). TOP_OFFSET is the offset of the processed
3896 : : subtree which has to be subtracted from offsets of individual accesses to
3897 : : get corresponding offsets for AGG. If CHUNK_SIZE is non-null, copy only
3898 : : replacements in the interval <start_offset, start_offset + chunk_size>,
3899 : : otherwise copy all. GSI is a statement iterator used to place the new
3900 : : statements. WRITE should be true when the statements should write from AGG
3901 : : to the replacement and false if vice versa. if INSERT_AFTER is true, new
3902 : : statements will be added after the current statement in GSI, they will be
3903 : : added before the statement otherwise. */
3904 : :
3905 : : static void
3906 : 1722746 : generate_subtree_copies (struct access *access, tree agg,
3907 : : HOST_WIDE_INT top_offset,
3908 : : HOST_WIDE_INT start_offset, HOST_WIDE_INT chunk_size,
3909 : : gimple_stmt_iterator *gsi, bool write,
3910 : : bool insert_after, location_t loc)
3911 : : {
3912 : : /* Never write anything into constant pool decls. See PR70602. */
3913 : 3445492 : if (!write && constant_decl_p (agg))
3914 : : return;
3915 : 4077887 : do
3916 : : {
3917 : 4077887 : if (chunk_size && access->offset >= start_offset + chunk_size)
3918 : : return;
3919 : :
3920 : 4077887 : if (access->grp_to_be_replaced
3921 : 3245810 : && (chunk_size == 0
3922 : 0 : || access->offset + access->size > start_offset))
3923 : : {
3924 : 3245810 : tree expr, repl = get_access_replacement (access);
3925 : 3245810 : gassign *stmt;
3926 : :
3927 : 3245810 : expr = build_ref_for_model (loc, agg, access->offset - top_offset,
3928 : : access, gsi, insert_after);
3929 : :
3930 : 3245810 : if (write)
3931 : : {
3932 : 1618974 : if (access->grp_partial_lhs)
3933 : 26 : expr = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE,
3934 : 26 : !insert_after,
3935 : : insert_after ? GSI_NEW_STMT
3936 : : : GSI_SAME_STMT);
3937 : 1618974 : stmt = gimple_build_assign (repl, expr);
3938 : : }
3939 : : else
3940 : : {
3941 : 1626836 : suppress_warning (repl /* Be more selective! */);
3942 : 1626836 : if (access->grp_partial_lhs)
3943 : 86 : repl = force_gimple_operand_gsi (gsi, repl, true, NULL_TREE,
3944 : 86 : !insert_after,
3945 : : insert_after ? GSI_NEW_STMT
3946 : : : GSI_SAME_STMT);
3947 : 1626836 : stmt = gimple_build_assign (expr, repl);
3948 : : }
3949 : 3245810 : gimple_set_location (stmt, loc);
3950 : :
3951 : 3245810 : if (insert_after)
3952 : 1618974 : gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
3953 : : else
3954 : 1626836 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
3955 : 3245810 : update_stmt (stmt);
3956 : 3245810 : sra_stats.subtree_copies++;
3957 : 3245810 : }
3958 : 832077 : else if (write
3959 : 323543 : && access->grp_to_be_debug_replaced
3960 : 842 : && (chunk_size == 0
3961 : 0 : || access->offset + access->size > start_offset))
3962 : : {
3963 : 842 : gdebug *ds;
3964 : 1684 : tree drhs = build_debug_ref_for_model (loc, agg,
3965 : 842 : access->offset - top_offset,
3966 : : access);
3967 : 842 : ds = gimple_build_debug_bind (get_access_replacement (access),
3968 : : drhs, gsi_stmt (*gsi));
3969 : 842 : if (insert_after)
3970 : 842 : gsi_insert_after (gsi, ds, GSI_NEW_STMT);
3971 : : else
3972 : 0 : gsi_insert_before (gsi, ds, GSI_SAME_STMT);
3973 : : }
3974 : :
3975 : 4077887 : if (access->first_child)
3976 : 404899 : generate_subtree_copies (access->first_child, agg, top_offset,
3977 : : start_offset, chunk_size, gsi,
3978 : : write, insert_after, loc);
3979 : :
3980 : 4077887 : access = access->next_sibling;
3981 : : }
3982 : 4077887 : while (access);
3983 : : }
3984 : :
3985 : : /* Assign zero to all scalar replacements in an access subtree. ACCESS is the
3986 : : root of the subtree to be processed. GSI is the statement iterator used
3987 : : for inserting statements which are added after the current statement if
3988 : : INSERT_AFTER is true or before it otherwise. */
3989 : :
3990 : : static void
3991 : 582118 : init_subtree_with_zero (struct access *access, gimple_stmt_iterator *gsi,
3992 : : bool insert_after, location_t loc)
3993 : :
3994 : : {
3995 : 582118 : struct access *child;
3996 : :
3997 : 582118 : if (access->grp_to_be_replaced)
3998 : : {
3999 : 282714 : gassign *stmt;
4000 : :
4001 : 282714 : stmt = gimple_build_assign (get_access_replacement (access),
4002 : : build_zero_cst (access->type));
4003 : 282714 : if (insert_after)
4004 : 32414 : gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
4005 : : else
4006 : 250300 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
4007 : 282714 : update_stmt (stmt);
4008 : 282714 : gimple_set_location (stmt, loc);
4009 : : }
4010 : 299404 : else if (access->grp_to_be_debug_replaced)
4011 : : {
4012 : 24435 : gdebug *ds
4013 : 24435 : = gimple_build_debug_bind (get_access_replacement (access),
4014 : : build_zero_cst (access->type),
4015 : : gsi_stmt (*gsi));
4016 : 24435 : if (insert_after)
4017 : 24435 : gsi_insert_after (gsi, ds, GSI_NEW_STMT);
4018 : : else
4019 : 0 : gsi_insert_before (gsi, ds, GSI_SAME_STMT);
4020 : : }
4021 : :
4022 : 1005436 : for (child = access->first_child; child; child = child->next_sibling)
4023 : 423318 : init_subtree_with_zero (child, gsi, insert_after, loc);
4024 : 582118 : }
4025 : :
4026 : : /* Clobber all scalar replacements in an access subtree. ACCESS is the
4027 : : root of the subtree to be processed. GSI is the statement iterator used
4028 : : for inserting statements which are added after the current statement if
4029 : : INSERT_AFTER is true or before it otherwise. */
4030 : :
4031 : : static void
4032 : 2917300 : clobber_subtree (struct access *access, gimple_stmt_iterator *gsi,
4033 : : bool insert_after, location_t loc)
4034 : :
4035 : : {
4036 : 2917300 : struct access *child;
4037 : :
4038 : 2917300 : if (access->grp_to_be_replaced)
4039 : : {
4040 : 1970973 : tree rep = get_access_replacement (access);
4041 : 1970973 : tree clobber = build_clobber (access->type);
4042 : 1970973 : gimple *stmt = gimple_build_assign (rep, clobber);
4043 : :
4044 : 1970973 : if (insert_after)
4045 : 310847 : gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
4046 : : else
4047 : 1660126 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
4048 : 1970973 : update_stmt (stmt);
4049 : 1970973 : gimple_set_location (stmt, loc);
4050 : : }
4051 : :
4052 : 4731410 : for (child = access->first_child; child; child = child->next_sibling)
4053 : 1814110 : clobber_subtree (child, gsi, insert_after, loc);
4054 : 2917300 : }
4055 : :
4056 : : /* Search for an access representative for the given expression EXPR and
4057 : : return it or NULL if it cannot be found. */
4058 : :
4059 : : static struct access *
4060 : 45436586 : get_access_for_expr (tree expr)
4061 : : {
4062 : 45436586 : poly_int64 poffset, psize, pmax_size;
4063 : 45436586 : HOST_WIDE_INT offset, max_size;
4064 : 45436586 : tree base;
4065 : 45436586 : bool reverse;
4066 : :
4067 : : /* FIXME: This should not be necessary but Ada produces V_C_Es with a type of
4068 : : a different size than the size of its argument and we need the latter
4069 : : one. */
4070 : 45436586 : if (TREE_CODE (expr) == VIEW_CONVERT_EXPR)
4071 : 250255 : expr = TREE_OPERAND (expr, 0);
4072 : :
4073 : 45436586 : base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size,
4074 : : &reverse);
4075 : 45436586 : if (!known_size_p (pmax_size)
4076 : 45311509 : || !pmax_size.is_constant (&max_size)
4077 : 45311509 : || !poffset.is_constant (&offset)
4078 : 45436586 : || !DECL_P (base))
4079 : : return NULL;
4080 : :
4081 : 21663169 : if (tree basesize = DECL_SIZE (base))
4082 : : {
4083 : 21622313 : poly_int64 sz;
4084 : 21622313 : if (offset < 0
4085 : 21622297 : || !poly_int_tree_p (basesize, &sz)
4086 : 43244610 : || known_le (sz, offset))
4087 : 7105 : return NULL;
4088 : : }
4089 : :
4090 : 21656064 : if (max_size == 0
4091 : 21656064 : || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
4092 : 12285346 : return NULL;
4093 : :
4094 : 9370718 : return get_var_base_offset_size_access (base, offset, max_size);
4095 : : }
4096 : :
4097 : : /* Replace the expression EXPR with a scalar replacement if there is one and
4098 : : generate other statements to do type conversion or subtree copying if
4099 : : necessary. WRITE is true if the expression is being written to (it is on a
4100 : : LHS of a statement or output in an assembly statement). STMT_GSI is used to
4101 : : place newly created statements before the processed statement, REFRESH_GSI
4102 : : is used to place them afterwards - unless the processed statement must end a
4103 : : BB in which case it is placed on the outgoing non-EH edge. REFRESH_GSI and
4104 : : is then used to continue iteration over the BB. If sra_modify_expr is
4105 : : called only once with WRITE equal to true on a given statement, both
4106 : : iterator parameters can point to the same one. */
4107 : :
4108 : : static bool
4109 : 7362805 : sra_modify_expr (tree *expr, bool write, gimple_stmt_iterator *stmt_gsi,
4110 : : gimple_stmt_iterator *refresh_gsi)
4111 : : {
4112 : 7362805 : location_t loc;
4113 : 7362805 : struct access *access;
4114 : 7362805 : tree type, bfr, orig_expr;
4115 : 7362805 : bool partial_cplx_access = false;
4116 : :
4117 : 7362805 : if (TREE_CODE (*expr) == BIT_FIELD_REF
4118 : 7362805 : && (write || !sra_handled_bf_read_p (*expr)))
4119 : : {
4120 : 597 : bfr = *expr;
4121 : 597 : expr = &TREE_OPERAND (*expr, 0);
4122 : : }
4123 : : else
4124 : : bfr = NULL_TREE;
4125 : :
4126 : 7362805 : if (TREE_CODE (*expr) == REALPART_EXPR || TREE_CODE (*expr) == IMAGPART_EXPR)
4127 : : {
4128 : 23902 : expr = &TREE_OPERAND (*expr, 0);
4129 : 23902 : partial_cplx_access = true;
4130 : : }
4131 : 7362805 : access = get_access_for_expr (*expr);
4132 : 7362805 : if (!access)
4133 : : return false;
4134 : 154535 : type = TREE_TYPE (*expr);
4135 : 154535 : orig_expr = *expr;
4136 : :
4137 : 154535 : loc = gimple_location (gsi_stmt (*stmt_gsi));
4138 : 154535 : gimple_stmt_iterator alt_gsi = gsi_none ();
4139 : 154535 : if (write && stmt_ends_bb_p (gsi_stmt (*stmt_gsi)))
4140 : : {
4141 : 33948 : alt_gsi = gsi_start_edge (single_non_eh_succ (gsi_bb (*stmt_gsi)));
4142 : 33948 : refresh_gsi = &alt_gsi;
4143 : : }
4144 : :
4145 : 154535 : if (access->grp_to_be_replaced)
4146 : : {
4147 : 27922 : tree repl = get_access_replacement (access);
4148 : : /* If we replace a non-register typed access simply use the original
4149 : : access expression to extract the scalar component afterwards.
4150 : : This happens if scalarizing a function return value or parameter
4151 : : like in gcc.c-torture/execute/20041124-1.c, 20050316-1.c and
4152 : : gcc.c-torture/compile/20011217-1.c.
4153 : :
4154 : : We also want to use this when accessing a complex or vector which can
4155 : : be accessed as a different type too, potentially creating a need for
4156 : : type conversion (see PR42196) and when scalarized unions are involved
4157 : : in assembler statements (see PR42398). */
4158 : 27922 : if (!bfr && !useless_type_conversion_p (type, access->type))
4159 : : {
4160 : 24734 : tree ref;
4161 : :
4162 : 24734 : ref = build_ref_for_model (loc, orig_expr, 0, access, stmt_gsi,
4163 : : false);
4164 : :
4165 : 24734 : if (partial_cplx_access)
4166 : : {
4167 : : /* VIEW_CONVERT_EXPRs in partial complex access are always fine in
4168 : : the case of a write because in such case the replacement cannot
4169 : : be a gimple register. In the case of a load, we have to
4170 : : differentiate in between a register an non-register
4171 : : replacement. */
4172 : 15 : tree t = build1 (VIEW_CONVERT_EXPR, type, repl);
4173 : 15 : gcc_checking_assert (!write || access->grp_partial_lhs);
4174 : 15 : if (!access->grp_partial_lhs)
4175 : : {
4176 : 12 : tree tmp = make_ssa_name (type);
4177 : 12 : gassign *stmt = gimple_build_assign (tmp, t);
4178 : : /* This is always a read. */
4179 : 12 : gsi_insert_before (stmt_gsi, stmt, GSI_SAME_STMT);
4180 : 12 : t = tmp;
4181 : : }
4182 : 15 : *expr = t;
4183 : : }
4184 : 24719 : else if (write)
4185 : : {
4186 : 8596 : gassign *stmt;
4187 : :
4188 : 8596 : if (access->grp_partial_lhs)
4189 : 6 : ref = force_gimple_operand_gsi (refresh_gsi, ref, true,
4190 : : NULL_TREE, false, GSI_NEW_STMT);
4191 : 8596 : stmt = gimple_build_assign (repl, ref);
4192 : 8596 : gimple_set_location (stmt, loc);
4193 : 8596 : gsi_insert_after (refresh_gsi, stmt, GSI_NEW_STMT);
4194 : : }
4195 : : else
4196 : : {
4197 : 16123 : gassign *stmt;
4198 : :
4199 : 16123 : if (access->grp_partial_lhs)
4200 : 6 : repl = force_gimple_operand_gsi (stmt_gsi, repl, true,
4201 : : NULL_TREE, true,
4202 : : GSI_SAME_STMT);
4203 : 16123 : stmt = gimple_build_assign (ref, repl);
4204 : 16123 : gimple_set_location (stmt, loc);
4205 : 16123 : gsi_insert_before (stmt_gsi, stmt, GSI_SAME_STMT);
4206 : : }
4207 : : }
4208 : : else
4209 : : {
4210 : : /* If we are going to replace a scalar field in a structure with
4211 : : reverse storage order by a stand-alone scalar, we are going to
4212 : : effectively byte-swap the scalar and we also need to byte-swap
4213 : : the portion of it represented by the bit-field. */
4214 : 3188 : if (bfr && REF_REVERSE_STORAGE_ORDER (bfr))
4215 : : {
4216 : 0 : REF_REVERSE_STORAGE_ORDER (bfr) = 0;
4217 : 0 : TREE_OPERAND (bfr, 2)
4218 : 0 : = size_binop (MINUS_EXPR, TYPE_SIZE (TREE_TYPE (repl)),
4219 : : size_binop (PLUS_EXPR, TREE_OPERAND (bfr, 1),
4220 : : TREE_OPERAND (bfr, 2)));
4221 : : }
4222 : :
4223 : 3188 : *expr = repl;
4224 : : }
4225 : :
4226 : 27922 : sra_stats.exprs++;
4227 : : }
4228 : 126613 : else if (write && access->grp_to_be_debug_replaced)
4229 : : {
4230 : 12 : gdebug *ds = gimple_build_debug_bind (get_access_replacement (access),
4231 : : NULL_TREE,
4232 : : gsi_stmt (*stmt_gsi));
4233 : 12 : gsi_insert_after (stmt_gsi, ds, GSI_NEW_STMT);
4234 : : }
4235 : :
4236 : 154535 : if (access->first_child && !TREE_READONLY (access->base))
4237 : : {
4238 : 124187 : HOST_WIDE_INT start_offset, chunk_size;
4239 : 124187 : if (bfr
4240 : 0 : && tree_fits_uhwi_p (TREE_OPERAND (bfr, 1))
4241 : 124187 : && tree_fits_uhwi_p (TREE_OPERAND (bfr, 2)))
4242 : : {
4243 : 0 : chunk_size = tree_to_uhwi (TREE_OPERAND (bfr, 1));
4244 : 0 : start_offset = access->offset
4245 : 0 : + tree_to_uhwi (TREE_OPERAND (bfr, 2));
4246 : : }
4247 : : else
4248 : : start_offset = chunk_size = 0;
4249 : :
4250 : 201552 : generate_subtree_copies (access->first_child, orig_expr, access->offset,
4251 : : start_offset, chunk_size,
4252 : : write ? refresh_gsi : stmt_gsi,
4253 : : write, write, loc);
4254 : : }
4255 : : return true;
4256 : : }
4257 : :
4258 : : /* If EXPR, which must be a call argument, is an ADDR_EXPR, generate writes and
4259 : : reads from its base before and after the call statement given in CALL_GSI
4260 : : and return true if any copying took place. Otherwise call sra_modify_expr
4261 : : on EXPR and return its value. FLAGS is what the gimple_call_arg_flags
4262 : : return for the given parameter. */
4263 : :
4264 : : static bool
4265 : 7993056 : sra_modify_call_arg (tree *expr, gimple_stmt_iterator *call_gsi,
4266 : : gimple_stmt_iterator *refresh_gsi, int flags)
4267 : : {
4268 : 7993056 : if (TREE_CODE (*expr) != ADDR_EXPR)
4269 : 5320987 : return sra_modify_expr (expr, false, call_gsi, refresh_gsi);
4270 : :
4271 : 2672069 : if (flags & EAF_UNUSED)
4272 : : return false;
4273 : :
4274 : 2668532 : tree base = get_base_address (TREE_OPERAND (*expr, 0));
4275 : 2668532 : if (!DECL_P (base))
4276 : : return false;
4277 : 2068546 : struct access *access = get_access_for_expr (base);
4278 : 2068546 : if (!access)
4279 : : return false;
4280 : :
4281 : 39310 : gimple *stmt = gsi_stmt (*call_gsi);
4282 : 39310 : location_t loc = gimple_location (stmt);
4283 : 39310 : generate_subtree_copies (access, base, 0, 0, 0, call_gsi, false, false,
4284 : : loc);
4285 : :
4286 : 39310 : if (flags & EAF_NO_DIRECT_CLOBBER)
4287 : : return true;
4288 : :
4289 : 28581 : if (!stmt_ends_bb_p (stmt))
4290 : 20337 : generate_subtree_copies (access, base, 0, 0, 0, refresh_gsi, true,
4291 : : true, loc);
4292 : : else
4293 : : {
4294 : 8244 : edge e;
4295 : 8244 : edge_iterator ei;
4296 : 24731 : FOR_EACH_EDGE (e, ei, gsi_bb (*call_gsi)->succs)
4297 : : {
4298 : 16487 : gimple_stmt_iterator alt_gsi = gsi_start_edge (e);
4299 : 16487 : generate_subtree_copies (access, base, 0, 0, 0, &alt_gsi, true,
4300 : : true, loc);
4301 : : }
4302 : : }
4303 : : return true;
4304 : : }
4305 : :
4306 : : /* Where scalar replacements of the RHS have been written to when a replacement
4307 : : of a LHS of an assigments cannot be direclty loaded from a replacement of
4308 : : the RHS. */
4309 : : enum unscalarized_data_handling { SRA_UDH_NONE, /* Nothing done so far. */
4310 : : SRA_UDH_RIGHT, /* Data flushed to the RHS. */
4311 : : SRA_UDH_LEFT }; /* Data flushed to the LHS. */
4312 : :
4313 : : struct subreplacement_assignment_data
4314 : : {
4315 : : /* Offset of the access representing the lhs of the assignment. */
4316 : : HOST_WIDE_INT left_offset;
4317 : :
4318 : : /* LHS and RHS of the original assignment. */
4319 : : tree assignment_lhs, assignment_rhs;
4320 : :
4321 : : /* Access representing the rhs of the whole assignment. */
4322 : : struct access *top_racc;
4323 : :
4324 : : /* Stmt iterator used for statement insertions after the original assignment.
4325 : : It points to the main GSI used to traverse a BB during function body
4326 : : modification. */
4327 : : gimple_stmt_iterator *new_gsi;
4328 : :
4329 : : /* Stmt iterator used for statement insertions before the original
4330 : : assignment. Keeps on pointing to the original statement. */
4331 : : gimple_stmt_iterator old_gsi;
4332 : :
4333 : : /* Location of the assignment. */
4334 : : location_t loc;
4335 : :
4336 : : /* Keeps the information whether we have needed to refresh replacements of
4337 : : the LHS and from which side of the assignments this takes place. */
4338 : : enum unscalarized_data_handling refreshed;
4339 : : };
4340 : :
4341 : : /* Store all replacements in the access tree rooted in TOP_RACC either to their
4342 : : base aggregate if there are unscalarized data or directly to LHS of the
4343 : : statement that is pointed to by GSI otherwise. */
4344 : :
4345 : : static void
4346 : 76738 : handle_unscalarized_data_in_subtree (struct subreplacement_assignment_data *sad)
4347 : : {
4348 : 76738 : tree src;
4349 : : /* If the RHS is a load from a constant, we do not need to (and must not)
4350 : : flush replacements to it and can use it directly as if we did. */
4351 : 76738 : if (TREE_READONLY (sad->top_racc->base))
4352 : : {
4353 : 11 : sad->refreshed = SRA_UDH_RIGHT;
4354 : 11 : return;
4355 : : }
4356 : 76727 : if (sad->top_racc->grp_unscalarized_data)
4357 : : {
4358 : 24515 : src = sad->assignment_rhs;
4359 : 24515 : sad->refreshed = SRA_UDH_RIGHT;
4360 : : }
4361 : : else
4362 : : {
4363 : 52212 : src = sad->assignment_lhs;
4364 : 52212 : sad->refreshed = SRA_UDH_LEFT;
4365 : : }
4366 : 76727 : generate_subtree_copies (sad->top_racc->first_child, src,
4367 : : sad->top_racc->offset, 0, 0,
4368 : : &sad->old_gsi, false, false, sad->loc);
4369 : : }
4370 : :
4371 : : /* Try to generate statements to load all sub-replacements in an access subtree
4372 : : formed by children of LACC from scalar replacements in the SAD->top_racc
4373 : : subtree. If that is not possible, refresh the SAD->top_racc base aggregate
4374 : : and load the accesses from it. */
4375 : :
4376 : : static void
4377 : 641619 : load_assign_lhs_subreplacements (struct access *lacc,
4378 : : struct subreplacement_assignment_data *sad)
4379 : : {
4380 : 2104763 : for (lacc = lacc->first_child; lacc; lacc = lacc->next_sibling)
4381 : : {
4382 : 1463144 : HOST_WIDE_INT offset;
4383 : 1463144 : offset = lacc->offset - sad->left_offset + sad->top_racc->offset;
4384 : :
4385 : 1463144 : if (lacc->grp_to_be_replaced)
4386 : : {
4387 : 1317937 : struct access *racc;
4388 : 1317937 : gassign *stmt;
4389 : 1317937 : tree rhs;
4390 : :
4391 : 1317937 : racc = find_access_in_subtree (sad->top_racc, offset, lacc->size);
4392 : 1317937 : if (racc && racc->grp_to_be_replaced)
4393 : : {
4394 : 1291517 : rhs = get_access_replacement (racc);
4395 : 1291517 : bool vce = false;
4396 : 1291517 : if (!useless_type_conversion_p (lacc->type, racc->type))
4397 : : {
4398 : 28 : rhs = fold_build1_loc (sad->loc, VIEW_CONVERT_EXPR,
4399 : : lacc->type, rhs);
4400 : 28 : vce = true;
4401 : : }
4402 : :
4403 : 1291517 : if (lacc->grp_partial_lhs && (vce || racc->grp_partial_lhs))
4404 : 0 : rhs = force_gimple_operand_gsi (&sad->old_gsi, rhs, true,
4405 : : NULL_TREE, true, GSI_SAME_STMT);
4406 : : }
4407 : : else
4408 : : {
4409 : : /* No suitable access on the right hand side, need to load from
4410 : : the aggregate. See if we have to update it first... */
4411 : 26420 : if (sad->refreshed == SRA_UDH_NONE)
4412 : 14313 : handle_unscalarized_data_in_subtree (sad);
4413 : :
4414 : 26420 : if (sad->refreshed == SRA_UDH_LEFT)
4415 : 240 : rhs = build_ref_for_model (sad->loc, sad->assignment_lhs,
4416 : 240 : lacc->offset - sad->left_offset,
4417 : : lacc, sad->new_gsi, true);
4418 : : else
4419 : 26180 : rhs = build_ref_for_model (sad->loc, sad->assignment_rhs,
4420 : 26180 : lacc->offset - sad->left_offset,
4421 : : lacc, sad->new_gsi, true);
4422 : 26420 : if (lacc->grp_partial_lhs)
4423 : 4 : rhs = force_gimple_operand_gsi (sad->new_gsi,
4424 : : rhs, true, NULL_TREE,
4425 : : false, GSI_NEW_STMT);
4426 : : }
4427 : :
4428 : 1317937 : stmt = gimple_build_assign (get_access_replacement (lacc), rhs);
4429 : 1317937 : gsi_insert_after (sad->new_gsi, stmt, GSI_NEW_STMT);
4430 : 1317937 : gimple_set_location (stmt, sad->loc);
4431 : 1317937 : update_stmt (stmt);
4432 : 1317937 : sra_stats.subreplacements++;
4433 : : }
4434 : : else
4435 : : {
4436 : 145207 : if (sad->refreshed == SRA_UDH_NONE
4437 : 48998 : && lacc->grp_read && !lacc->grp_covered)
4438 : 10 : handle_unscalarized_data_in_subtree (sad);
4439 : :
4440 : 145207 : if (lacc && lacc->grp_to_be_debug_replaced)
4441 : : {
4442 : 39730 : gdebug *ds;
4443 : 39730 : tree drhs;
4444 : 39730 : struct access *racc = find_access_in_subtree (sad->top_racc,
4445 : : offset,
4446 : : lacc->size);
4447 : :
4448 : 39730 : if (racc && racc->grp_to_be_replaced)
4449 : : {
4450 : 39726 : if (racc->grp_write || constant_decl_p (racc->base))
4451 : 38418 : drhs = get_access_replacement (racc);
4452 : : else
4453 : : drhs = NULL;
4454 : : }
4455 : 4 : else if (sad->refreshed == SRA_UDH_LEFT)
4456 : 0 : drhs = build_debug_ref_for_model (sad->loc, lacc->base,
4457 : : lacc->offset, lacc);
4458 : 4 : else if (sad->refreshed == SRA_UDH_RIGHT)
4459 : 4 : drhs = build_debug_ref_for_model (sad->loc, sad->top_racc->base,
4460 : : offset, lacc);
4461 : : else
4462 : : drhs = NULL_TREE;
4463 : 38418 : if (drhs
4464 : 38422 : && !useless_type_conversion_p (lacc->type, TREE_TYPE (drhs)))
4465 : 378 : drhs = fold_build1_loc (sad->loc, VIEW_CONVERT_EXPR,
4466 : : lacc->type, drhs);
4467 : 39730 : ds = gimple_build_debug_bind (get_access_replacement (lacc),
4468 : : drhs, gsi_stmt (sad->old_gsi));
4469 : 39730 : gsi_insert_after (sad->new_gsi, ds, GSI_NEW_STMT);
4470 : : }
4471 : : }
4472 : :
4473 : 1463144 : if (lacc->first_child)
4474 : 59656 : load_assign_lhs_subreplacements (lacc, sad);
4475 : : }
4476 : 641619 : }
4477 : :
4478 : : /* Result code for SRA assignment modification. */
4479 : : enum assignment_mod_result { SRA_AM_NONE, /* nothing done for the stmt */
4480 : : SRA_AM_MODIFIED, /* stmt changed but not
4481 : : removed */
4482 : : SRA_AM_REMOVED }; /* stmt eliminated */
4483 : :
4484 : : /* Modify assignments with a CONSTRUCTOR on their RHS. STMT contains a pointer
4485 : : to the assignment and GSI is the statement iterator pointing at it. Returns
4486 : : the same values as sra_modify_assign. */
4487 : :
4488 : : static enum assignment_mod_result
4489 : 2909924 : sra_modify_constructor_assign (gimple *stmt, gimple_stmt_iterator *gsi)
4490 : : {
4491 : 2909924 : tree lhs = gimple_assign_lhs (stmt);
4492 : 2909924 : struct access *acc = get_access_for_expr (lhs);
4493 : 2909924 : if (!acc)
4494 : : return SRA_AM_NONE;
4495 : 1261990 : location_t loc = gimple_location (stmt);
4496 : :
4497 : 1261990 : if (gimple_clobber_p (stmt))
4498 : : {
4499 : : /* Clobber the replacement variable. */
4500 : 1103190 : clobber_subtree (acc, gsi, !acc->grp_covered, loc);
4501 : : /* Remove clobbers of fully scalarized variables, they are dead. */
4502 : 1103190 : if (acc->grp_covered)
4503 : : {
4504 : 927758 : unlink_stmt_vdef (stmt);
4505 : 927758 : gsi_remove (gsi, true);
4506 : 927758 : release_defs (stmt);
4507 : 927758 : return SRA_AM_REMOVED;
4508 : : }
4509 : : else
4510 : : return SRA_AM_MODIFIED;
4511 : : }
4512 : :
4513 : 158800 : if (CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt)) > 0)
4514 : : {
4515 : : /* I have never seen this code path trigger but if it can happen the
4516 : : following should handle it gracefully. */
4517 : 0 : if (access_has_children_p (acc))
4518 : 0 : generate_subtree_copies (acc->first_child, lhs, acc->offset, 0, 0, gsi,
4519 : : true, true, loc);
4520 : 0 : return SRA_AM_MODIFIED;
4521 : : }
4522 : :
4523 : 158800 : if (acc->grp_covered)
4524 : : {
4525 : 94987 : init_subtree_with_zero (acc, gsi, false, loc);
4526 : 94987 : unlink_stmt_vdef (stmt);
4527 : 94987 : gsi_remove (gsi, true);
4528 : 94987 : release_defs (stmt);
4529 : 94987 : return SRA_AM_REMOVED;
4530 : : }
4531 : : else
4532 : : {
4533 : 63813 : init_subtree_with_zero (acc, gsi, true, loc);
4534 : 63813 : return SRA_AM_MODIFIED;
4535 : : }
4536 : : }
4537 : :
4538 : : /* Create and return a new suitable default definition SSA_NAME for RACC which
4539 : : is an access describing an uninitialized part of an aggregate that is being
4540 : : loaded. REG_TREE is used instead of the actual RACC type if that is not of
4541 : : a gimple register type. */
4542 : :
4543 : : static tree
4544 : 508 : get_repl_default_def_ssa_name (struct access *racc, tree reg_type)
4545 : : {
4546 : 508 : gcc_checking_assert (!racc->grp_to_be_replaced
4547 : : && !racc->grp_to_be_debug_replaced);
4548 : 508 : if (!racc->replacement_decl)
4549 : 508 : racc->replacement_decl = create_access_replacement (racc, reg_type);
4550 : 508 : return get_or_create_ssa_default_def (cfun, racc->replacement_decl);
4551 : : }
4552 : :
4553 : :
4554 : : /* Generate statements to call .DEFERRED_INIT to initialize scalar replacements
4555 : : of accesses within a subtree ACCESS; all its children, siblings and their
4556 : : children are to be processed.
4557 : : GSI is a statement iterator used to place the new statements. */
4558 : : static void
4559 : 9 : generate_subtree_deferred_init (struct access *access,
4560 : : tree init_type,
4561 : : tree decl_name,
4562 : : gimple_stmt_iterator *gsi,
4563 : : location_t loc)
4564 : : {
4565 : 28 : do
4566 : : {
4567 : 28 : if (access->grp_to_be_replaced)
4568 : : {
4569 : 27 : tree repl = get_access_replacement (access);
4570 : 27 : gimple *call
4571 : 27 : = gimple_build_call_internal (IFN_DEFERRED_INIT, 3,
4572 : 27 : TYPE_SIZE_UNIT (TREE_TYPE (repl)),
4573 : : init_type, decl_name);
4574 : 27 : gimple_call_set_lhs (call, repl);
4575 : 27 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4576 : 27 : update_stmt (call);
4577 : 27 : gimple_set_location (call, loc);
4578 : 27 : sra_stats.subtree_deferred_init++;
4579 : : }
4580 : 28 : if (access->first_child)
4581 : 1 : generate_subtree_deferred_init (access->first_child, init_type,
4582 : : decl_name, gsi, loc);
4583 : :
4584 : 28 : access = access ->next_sibling;
4585 : : }
4586 : 28 : while (access);
4587 : 9 : }
4588 : :
4589 : : /* For a call to .DEFERRED_INIT:
4590 : : var = .DEFERRED_INIT (size_of_var, init_type, name_of_var);
4591 : : examine the LHS variable VAR and replace it with a scalar replacement if
4592 : : there is one, also replace the RHS call to a call to .DEFERRED_INIT of
4593 : : the corresponding scalar relacement variable. Examine the subtree and
4594 : : do the scalar replacements in the subtree too. STMT is the call, GSI is
4595 : : the statment iterator to place newly created statement. */
4596 : :
4597 : : static enum assignment_mod_result
4598 : 11 : sra_modify_deferred_init (gimple *stmt, gimple_stmt_iterator *gsi)
4599 : : {
4600 : 11 : tree lhs = gimple_call_lhs (stmt);
4601 : 11 : tree init_type = gimple_call_arg (stmt, 1);
4602 : 11 : tree decl_name = gimple_call_arg (stmt, 2);
4603 : :
4604 : 11 : struct access *lhs_access = get_access_for_expr (lhs);
4605 : 11 : if (!lhs_access)
4606 : : return SRA_AM_NONE;
4607 : :
4608 : 11 : location_t loc = gimple_location (stmt);
4609 : :
4610 : 11 : if (lhs_access->grp_to_be_replaced)
4611 : : {
4612 : 3 : tree lhs_repl = get_access_replacement (lhs_access);
4613 : 3 : gimple_call_set_lhs (stmt, lhs_repl);
4614 : 3 : tree arg0_repl = TYPE_SIZE_UNIT (TREE_TYPE (lhs_repl));
4615 : 3 : gimple_call_set_arg (stmt, 0, arg0_repl);
4616 : 3 : sra_stats.deferred_init++;
4617 : 3 : gcc_assert (!lhs_access->first_child);
4618 : : return SRA_AM_MODIFIED;
4619 : : }
4620 : :
4621 : 8 : if (lhs_access->first_child)
4622 : 8 : generate_subtree_deferred_init (lhs_access->first_child,
4623 : : init_type, decl_name, gsi, loc);
4624 : 8 : if (lhs_access->grp_covered)
4625 : : {
4626 : 7 : unlink_stmt_vdef (stmt);
4627 : 7 : gsi_remove (gsi, true);
4628 : 7 : release_defs (stmt);
4629 : 7 : return SRA_AM_REMOVED;
4630 : : }
4631 : :
4632 : : return SRA_AM_MODIFIED;
4633 : : }
4634 : :
4635 : : /* Examine both sides of the assignment statement pointed to by STMT, replace
4636 : : them with a scalare replacement if there is one and generate copying of
4637 : : replacements if scalarized aggregates have been used in the assignment. GSI
4638 : : is used to hold generated statements for type conversions and subtree
4639 : : copying. */
4640 : :
4641 : : static enum assignment_mod_result
4642 : 24704244 : sra_modify_assign (gimple *stmt, gimple_stmt_iterator *gsi)
4643 : : {
4644 : 24704244 : struct access *lacc, *racc;
4645 : 24704244 : tree lhs, rhs;
4646 : 24704244 : bool modify_this_stmt = false;
4647 : 24704244 : bool force_gimple_rhs = false;
4648 : 24704244 : location_t loc;
4649 : 24704244 : gimple_stmt_iterator orig_gsi = *gsi;
4650 : :
4651 : 24704244 : if (!gimple_assign_single_p (stmt))
4652 : : return SRA_AM_NONE;
4653 : 19482073 : lhs = gimple_assign_lhs (stmt);
4654 : 19482073 : rhs = gimple_assign_rhs1 (stmt);
4655 : :
4656 : 19482073 : if (TREE_CODE (rhs) == CONSTRUCTOR)
4657 : 2909924 : return sra_modify_constructor_assign (stmt, gsi);
4658 : :
4659 : 16563622 : if (TREE_CODE (rhs) == REALPART_EXPR || TREE_CODE (lhs) == REALPART_EXPR
4660 : 16560561 : || TREE_CODE (rhs) == IMAGPART_EXPR || TREE_CODE (lhs) == IMAGPART_EXPR
4661 : 16548247 : || (TREE_CODE (rhs) == BIT_FIELD_REF && !sra_handled_bf_read_p (rhs))
4662 : 33120392 : || TREE_CODE (lhs) == BIT_FIELD_REF)
4663 : : {
4664 : 24499 : modify_this_stmt = sra_modify_expr (gimple_assign_rhs1_ptr (stmt),
4665 : : false, gsi, gsi);
4666 : 24499 : modify_this_stmt |= sra_modify_expr (gimple_assign_lhs_ptr (stmt),
4667 : : true, gsi, gsi);
4668 : 45904 : return modify_this_stmt ? SRA_AM_MODIFIED : SRA_AM_NONE;
4669 : : }
4670 : :
4671 : 16547650 : lacc = get_access_for_expr (lhs);
4672 : 16547650 : racc = get_access_for_expr (rhs);
4673 : 16547650 : if (!lacc && !racc)
4674 : : return SRA_AM_NONE;
4675 : : /* Avoid modifying initializations of constant-pool replacements. */
4676 : 6945214 : if (racc && (racc->replacement_decl == lhs))
4677 : : return SRA_AM_NONE;
4678 : :
4679 : 6943957 : loc = gimple_location (stmt);
4680 : 6943957 : if (lacc && lacc->grp_to_be_replaced)
4681 : : {
4682 : 1961299 : lhs = get_access_replacement (lacc);
4683 : 1961299 : gimple_assign_set_lhs (stmt, lhs);
4684 : 1961299 : modify_this_stmt = true;
4685 : 1961299 : if (lacc->grp_partial_lhs)
4686 : 73 : force_gimple_rhs = true;
4687 : 1961299 : sra_stats.exprs++;
4688 : : }
4689 : :
4690 : 6943957 : if (racc && racc->grp_to_be_replaced)
4691 : : {
4692 : 3161452 : rhs = get_access_replacement (racc);
4693 : 3161452 : modify_this_stmt = true;
4694 : 3161452 : if (racc->grp_partial_lhs)
4695 : 518 : force_gimple_rhs = true;
4696 : 3161452 : sra_stats.exprs++;
4697 : : }
4698 : 1151647 : else if (racc
4699 : : && !racc->grp_unscalarized_data
4700 : 1151647 : && !racc->grp_unscalarizable_region
4701 : 949657 : && TREE_CODE (lhs) == SSA_NAME
4702 : 587 : && !access_has_replacements_p (racc))
4703 : : {
4704 : 508 : rhs = get_repl_default_def_ssa_name (racc, TREE_TYPE (lhs));
4705 : 508 : modify_this_stmt = true;
4706 : 508 : sra_stats.exprs++;
4707 : : }
4708 : :
4709 : 3161960 : if (modify_this_stmt
4710 : 6943957 : && !useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
4711 : : {
4712 : : /* If we can avoid creating a VIEW_CONVERT_EXPR, then do so.
4713 : : ??? This should move to fold_stmt which we simply should
4714 : : call after building a VIEW_CONVERT_EXPR here. */
4715 : 502477 : if (AGGREGATE_TYPE_P (TREE_TYPE (lhs))
4716 : 128183 : && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (lhs)) == racc->reverse
4717 : 379934 : && !contains_bitfld_component_ref_p (lhs))
4718 : : {
4719 : 128182 : lhs = build_ref_for_model (loc, lhs, 0, racc, gsi, false);
4720 : 128182 : gimple_assign_set_lhs (stmt, lhs);
4721 : : }
4722 : 123570 : else if (lacc
4723 : 98599 : && AGGREGATE_TYPE_P (TREE_TYPE (rhs))
4724 : 77630 : && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (rhs)) == lacc->reverse
4725 : 201200 : && !contains_vce_or_bfcref_p (rhs))
4726 : 77562 : rhs = build_ref_for_model (loc, rhs, 0, lacc, gsi, false);
4727 : :
4728 : 251752 : if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
4729 : : {
4730 : 46008 : rhs = fold_build1_loc (loc, VIEW_CONVERT_EXPR, TREE_TYPE (lhs), rhs);
4731 : 46008 : if (is_gimple_reg_type (TREE_TYPE (lhs))
4732 : 46008 : && TREE_CODE (lhs) != SSA_NAME)
4733 : 6943957 : force_gimple_rhs = true;
4734 : : }
4735 : : }
4736 : :
4737 : 6943957 : if (lacc && lacc->grp_to_be_debug_replaced)
4738 : : {
4739 : 94732 : tree dlhs = get_access_replacement (lacc);
4740 : 94732 : tree drhs = unshare_expr (rhs);
4741 : 94732 : if (!useless_type_conversion_p (TREE_TYPE (dlhs), TREE_TYPE (drhs)))
4742 : : {
4743 : 44 : if (AGGREGATE_TYPE_P (TREE_TYPE (drhs))
4744 : 43 : && !contains_vce_or_bfcref_p (drhs))
4745 : 21 : drhs = build_debug_ref_for_model (loc, drhs, 0, lacc);
4746 : 22 : if (drhs
4747 : 44 : && !useless_type_conversion_p (TREE_TYPE (dlhs),
4748 : 22 : TREE_TYPE (drhs)))
4749 : 1 : drhs = fold_build1_loc (loc, VIEW_CONVERT_EXPR,
4750 : 1 : TREE_TYPE (dlhs), drhs);
4751 : : }
4752 : 94732 : gdebug *ds = gimple_build_debug_bind (dlhs, drhs, stmt);
4753 : 94732 : gsi_insert_before (gsi, ds, GSI_SAME_STMT);
4754 : : }
4755 : :
4756 : : /* From this point on, the function deals with assignments in between
4757 : : aggregates when at least one has scalar reductions of some of its
4758 : : components. There are three possible scenarios: Both the LHS and RHS have
4759 : : to-be-scalarized components, 2) only the RHS has or 3) only the LHS has.
4760 : :
4761 : : In the first case, we would like to load the LHS components from RHS
4762 : : components whenever possible. If that is not possible, we would like to
4763 : : read it directly from the RHS (after updating it by storing in it its own
4764 : : components). If there are some necessary unscalarized data in the LHS,
4765 : : those will be loaded by the original assignment too. If neither of these
4766 : : cases happen, the original statement can be removed. Most of this is done
4767 : : by load_assign_lhs_subreplacements.
4768 : :
4769 : : In the second case, we would like to store all RHS scalarized components
4770 : : directly into LHS and if they cover the aggregate completely, remove the
4771 : : statement too. In the third case, we want the LHS components to be loaded
4772 : : directly from the RHS (DSE will remove the original statement if it
4773 : : becomes redundant).
4774 : :
4775 : : This is a bit complex but manageable when types match and when unions do
4776 : : not cause confusion in a way that we cannot really load a component of LHS
4777 : : from the RHS or vice versa (the access representing this level can have
4778 : : subaccesses that are accessible only through a different union field at a
4779 : : higher level - different from the one used in the examined expression).
4780 : : Unions are fun.
4781 : :
4782 : : Therefore, I specially handle a fourth case, happening when there is a
4783 : : specific type cast or it is impossible to locate a scalarized subaccess on
4784 : : the other side of the expression. If that happens, I simply "refresh" the
4785 : : RHS by storing in it is scalarized components leave the original statement
4786 : : there to do the copying and then load the scalar replacements of the LHS.
4787 : : This is what the first branch does. */
4788 : :
4789 : 6943957 : if (modify_this_stmt
4790 : 2051509 : || gimple_has_volatile_ops (stmt)
4791 : 2051155 : || contains_vce_or_bfcref_p (rhs)
4792 : 1871097 : || contains_vce_or_bfcref_p (lhs)
4793 : 8813730 : || stmt_ends_bb_p (stmt))
4794 : : {
4795 : : /* No need to copy into a constant, it comes pre-initialized. */
4796 : 5173388 : if (access_has_children_p (racc) && !TREE_READONLY (racc->base))
4797 : 15193 : generate_subtree_copies (racc->first_child, rhs, racc->offset, 0, 0,
4798 : : gsi, false, false, loc);
4799 : 5158195 : if (access_has_children_p (lacc))
4800 : : {
4801 : 260050 : gimple_stmt_iterator alt_gsi = gsi_none ();
4802 : 260050 : if (stmt_ends_bb_p (stmt))
4803 : : {
4804 : 83421 : alt_gsi = gsi_start_edge (single_non_eh_succ (gsi_bb (*gsi)));
4805 : 83421 : gsi = &alt_gsi;
4806 : : }
4807 : 260050 : generate_subtree_copies (lacc->first_child, lhs, lacc->offset, 0, 0,
4808 : : gsi, true, true, loc);
4809 : : }
4810 : 5158195 : sra_stats.separate_lhs_rhs_handling++;
4811 : :
4812 : : /* This gimplification must be done after generate_subtree_copies,
4813 : : lest we insert the subtree copies in the middle of the gimplified
4814 : : sequence. */
4815 : 5158195 : if (force_gimple_rhs)
4816 : 21615 : rhs = force_gimple_operand_gsi (&orig_gsi, rhs, true, NULL_TREE,
4817 : : true, GSI_SAME_STMT);
4818 : 5158195 : if (gimple_assign_rhs1 (stmt) != rhs)
4819 : : {
4820 : 3260474 : modify_this_stmt = true;
4821 : 3260474 : gimple_assign_set_rhs_from_tree (&orig_gsi, rhs);
4822 : 3260474 : gcc_assert (stmt == gsi_stmt (orig_gsi));
4823 : : }
4824 : :
4825 : 6790169 : return modify_this_stmt ? SRA_AM_MODIFIED : SRA_AM_NONE;
4826 : : }
4827 : : else
4828 : : {
4829 : 2609933 : if (access_has_children_p (lacc)
4830 : 1785771 : && access_has_children_p (racc)
4831 : : /* When an access represents an unscalarizable region, it usually
4832 : : represents accesses with variable offset and thus must not be used
4833 : : to generate new memory accesses. */
4834 : 581972 : && !lacc->grp_unscalarizable_region
4835 : 581969 : && !racc->grp_unscalarizable_region)
4836 : : {
4837 : 581963 : struct subreplacement_assignment_data sad;
4838 : :
4839 : 581963 : sad.left_offset = lacc->offset;
4840 : 581963 : sad.assignment_lhs = lhs;
4841 : 581963 : sad.assignment_rhs = rhs;
4842 : 581963 : sad.top_racc = racc;
4843 : 581963 : sad.old_gsi = *gsi;
4844 : 581963 : sad.new_gsi = gsi;
4845 : 581963 : sad.loc = gimple_location (stmt);
4846 : 581963 : sad.refreshed = SRA_UDH_NONE;
4847 : :
4848 : 581963 : if (lacc->grp_read && !lacc->grp_covered)
4849 : 62415 : handle_unscalarized_data_in_subtree (&sad);
4850 : :
4851 : 581963 : load_assign_lhs_subreplacements (lacc, &sad);
4852 : 581963 : if (sad.refreshed != SRA_UDH_RIGHT)
4853 : : {
4854 : 557437 : gsi_next (gsi);
4855 : 557437 : unlink_stmt_vdef (stmt);
4856 : 557437 : gsi_remove (&sad.old_gsi, true);
4857 : 557437 : release_defs (stmt);
4858 : 557437 : sra_stats.deleted++;
4859 : 557437 : return SRA_AM_REMOVED;
4860 : : }
4861 : : }
4862 : : else
4863 : : {
4864 : 1203799 : if (access_has_children_p (racc)
4865 : 425968 : && !racc->grp_unscalarized_data
4866 : 377611 : && TREE_CODE (lhs) != SSA_NAME)
4867 : : {
4868 : 377610 : if (dump_file)
4869 : : {
4870 : 4 : fprintf (dump_file, "Removing load: ");
4871 : 4 : print_gimple_stmt (dump_file, stmt, 0);
4872 : : }
4873 : 377610 : generate_subtree_copies (racc->first_child, lhs,
4874 : : racc->offset, 0, 0, gsi,
4875 : : false, false, loc);
4876 : 377610 : gcc_assert (stmt == gsi_stmt (*gsi));
4877 : 377610 : unlink_stmt_vdef (stmt);
4878 : 377610 : gsi_remove (gsi, true);
4879 : 377610 : release_defs (stmt);
4880 : 377610 : sra_stats.deleted++;
4881 : 377610 : return SRA_AM_REMOVED;
4882 : : }
4883 : : /* Restore the aggregate RHS from its components so the
4884 : : prevailing aggregate copy does the right thing. */
4885 : 874547 : if (access_has_children_p (racc) && !TREE_READONLY (racc->base))
4886 : 48333 : generate_subtree_copies (racc->first_child, rhs, racc->offset, 0, 0,
4887 : : gsi, false, false, loc);
4888 : : /* Re-load the components of the aggregate copy destination.
4889 : : But use the RHS aggregate to load from to expose more
4890 : : optimization opportunities. */
4891 : 826189 : if (access_has_children_p (lacc))
4892 : : {
4893 : 242205 : generate_subtree_copies (lacc->first_child, rhs, lacc->offset,
4894 : : 0, 0, gsi, true, true, loc);
4895 : 242205 : if (lacc->grp_covered)
4896 : : {
4897 : 190739 : unlink_stmt_vdef (stmt);
4898 : 190739 : gsi_remove (& orig_gsi, true);
4899 : 190739 : release_defs (stmt);
4900 : 190739 : sra_stats.deleted++;
4901 : 190739 : return SRA_AM_REMOVED;
4902 : : }
4903 : : }
4904 : : }
4905 : :
4906 : 659976 : return SRA_AM_NONE;
4907 : : }
4908 : : }
4909 : :
4910 : : /* Set any scalar replacements of values in the constant pool to the initial
4911 : : value of the constant. (Constant-pool decls like *.LC0 have effectively
4912 : : been initialized before the program starts, we must do the same for their
4913 : : replacements.) Thus, we output statements like 'SR.1 = *.LC0[0];' into
4914 : : the function's entry block. */
4915 : :
4916 : : static void
4917 : 428278 : initialize_constant_pool_replacements (void)
4918 : : {
4919 : 428278 : gimple_seq seq = NULL;
4920 : 428278 : gimple_stmt_iterator gsi = gsi_start (seq);
4921 : 428278 : bitmap_iterator bi;
4922 : 428278 : unsigned i;
4923 : :
4924 : 2362924 : EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
4925 : : {
4926 : 1934646 : tree var = candidate (i);
4927 : 1934646 : if (!constant_decl_p (var))
4928 : 1934629 : continue;
4929 : :
4930 : 17 : struct access *access = get_first_repr_for_decl (var);
4931 : :
4932 : 1308 : while (access)
4933 : : {
4934 : 1274 : if (access->replacement_decl)
4935 : : {
4936 : 1257 : gassign *stmt
4937 : 1257 : = gimple_build_assign (get_access_replacement (access),
4938 : : unshare_expr (access->expr));
4939 : 1257 : if (dump_file && (dump_flags & TDF_DETAILS))
4940 : : {
4941 : 0 : fprintf (dump_file, "Generating constant initializer: ");
4942 : 0 : print_gimple_stmt (dump_file, stmt, 0);
4943 : 0 : fprintf (dump_file, "\n");
4944 : : }
4945 : 1257 : gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
4946 : 1257 : update_stmt (stmt);
4947 : : }
4948 : :
4949 : 1274 : if (access->first_child)
4950 : : access = access->first_child;
4951 : 1257 : else if (access->next_sibling)
4952 : : access = access->next_sibling;
4953 : : else
4954 : : {
4955 : 34 : while (access->parent && !access->next_sibling)
4956 : : access = access->parent;
4957 : 17 : if (access->next_sibling)
4958 : : access = access->next_sibling;
4959 : : else
4960 : 17 : access = access->next_grp;
4961 : : }
4962 : : }
4963 : : }
4964 : :
4965 : 428278 : seq = gsi_seq (gsi);
4966 : 428278 : if (seq)
4967 : 17 : gsi_insert_seq_on_edge_immediate (
4968 : 17 : single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
4969 : 428278 : }
4970 : :
4971 : : /* Traverse the function body and all modifications as decided in
4972 : : analyze_all_variable_accesses. Return true iff the CFG has been
4973 : : changed. */
4974 : :
4975 : : static bool
4976 : 428278 : sra_modify_function_body (void)
4977 : : {
4978 : 428278 : bool cfg_changed = false;
4979 : 428278 : basic_block bb;
4980 : :
4981 : 428278 : initialize_constant_pool_replacements ();
4982 : :
4983 : 9587883 : FOR_EACH_BB_FN (bb, cfun)
4984 : : {
4985 : 9159605 : gimple_stmt_iterator gsi = gsi_start_bb (bb);
4986 : 74473643 : while (!gsi_end_p (gsi))
4987 : : {
4988 : 65314038 : gimple *stmt = gsi_stmt (gsi);
4989 : 65314038 : enum assignment_mod_result assign_result;
4990 : 65314038 : bool modified = false, deleted = false;
4991 : 65314038 : tree *t;
4992 : 65314038 : unsigned i;
4993 : :
4994 : 65314038 : switch (gimple_code (stmt))
4995 : : {
4996 : 425881 : case GIMPLE_RETURN:
4997 : 425881 : t = gimple_return_retval_ptr (as_a <greturn *> (stmt));
4998 : 425881 : if (*t != NULL_TREE)
4999 : 283346 : modified |= sra_modify_expr (t, false, &gsi, &gsi);
5000 : : break;
5001 : :
5002 : 24704244 : case GIMPLE_ASSIGN:
5003 : 24704244 : assign_result = sra_modify_assign (stmt, &gsi);
5004 : 24704244 : modified |= assign_result == SRA_AM_MODIFIED;
5005 : 24704244 : deleted = assign_result == SRA_AM_REMOVED;
5006 : 24704244 : break;
5007 : :
5008 : 4000538 : case GIMPLE_CALL:
5009 : : /* Handle calls to .DEFERRED_INIT specially. */
5010 : 4000538 : if (gimple_call_internal_p (stmt, IFN_DEFERRED_INIT))
5011 : : {
5012 : 11 : assign_result = sra_modify_deferred_init (stmt, &gsi);
5013 : 11 : modified |= assign_result == SRA_AM_MODIFIED;
5014 : 11 : deleted = assign_result == SRA_AM_REMOVED;
5015 : : }
5016 : : else
5017 : : {
5018 : 4000527 : gcall *call = as_a <gcall *> (stmt);
5019 : 4000527 : gimple_stmt_iterator call_gsi = gsi;
5020 : :
5021 : : /* Operands must be processed before the lhs. */
5022 : 11964104 : for (i = 0; i < gimple_call_num_args (call); i++)
5023 : : {
5024 : 7963577 : int flags = gimple_call_arg_flags (call, i);
5025 : 7963577 : t = gimple_call_arg_ptr (call, i);
5026 : 7963577 : modified |= sra_modify_call_arg (t, &call_gsi, &gsi, flags);
5027 : : }
5028 : 4000527 : if (gimple_call_chain (call))
5029 : : {
5030 : 29479 : t = gimple_call_chain_ptr (call);
5031 : 29479 : int flags = gimple_call_static_chain_flags (call);
5032 : 29479 : modified |= sra_modify_call_arg (t, &call_gsi, &gsi,
5033 : : flags);
5034 : : }
5035 : 4000527 : if (gimple_call_lhs (call))
5036 : : {
5037 : 1706414 : t = gimple_call_lhs_ptr (call);
5038 : 1706414 : modified |= sra_modify_expr (t, true, &call_gsi, &gsi);
5039 : : }
5040 : : }
5041 : : break;
5042 : :
5043 : 4322 : case GIMPLE_ASM:
5044 : 4322 : {
5045 : 4322 : gimple_stmt_iterator stmt_gsi = gsi;
5046 : 4322 : gasm *asm_stmt = as_a <gasm *> (stmt);
5047 : 10012 : for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
5048 : : {
5049 : 1368 : t = &TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
5050 : 1368 : modified |= sra_modify_expr (t, false, &stmt_gsi, &gsi);
5051 : : }
5052 : 6014 : for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
5053 : : {
5054 : 1692 : t = &TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
5055 : 1692 : modified |= sra_modify_expr (t, true, &stmt_gsi, &gsi);
5056 : : }
5057 : : }
5058 : 4322 : break;
5059 : :
5060 : : default:
5061 : : break;
5062 : : }
5063 : :
5064 : 28992450 : if (modified)
5065 : : {
5066 : 5303084 : update_stmt (stmt);
5067 : 5303084 : if (maybe_clean_eh_stmt (stmt)
5068 : 5303084 : && gimple_purge_dead_eh_edges (gimple_bb (stmt)))
5069 : : cfg_changed = true;
5070 : : }
5071 : 65314038 : if (!deleted)
5072 : 63165500 : gsi_next (&gsi);
5073 : : }
5074 : : }
5075 : :
5076 : 428278 : gsi_commit_edge_inserts ();
5077 : 428278 : return cfg_changed;
5078 : : }
5079 : :
5080 : : /* Generate statements initializing scalar replacements of parts of function
5081 : : parameters. */
5082 : :
5083 : : static void
5084 : 428278 : initialize_parameter_reductions (void)
5085 : : {
5086 : 428278 : gimple_stmt_iterator gsi;
5087 : 428278 : gimple_seq seq = NULL;
5088 : 428278 : tree parm;
5089 : :
5090 : 428278 : gsi = gsi_start (seq);
5091 : 428278 : for (parm = DECL_ARGUMENTS (current_function_decl);
5092 : 1279506 : parm;
5093 : 851228 : parm = DECL_CHAIN (parm))
5094 : : {
5095 : 851228 : vec<access_p> *access_vec;
5096 : 851228 : struct access *access;
5097 : :
5098 : 851228 : if (!bitmap_bit_p (candidate_bitmap, DECL_UID (parm)))
5099 : 789149 : continue;
5100 : 62079 : access_vec = get_base_access_vector (parm);
5101 : 62079 : if (!access_vec)
5102 : 0 : continue;
5103 : :
5104 : 62079 : for (access = (*access_vec)[0];
5105 : 159487 : access;
5106 : 97408 : access = access->next_grp)
5107 : 97408 : generate_subtree_copies (access, parm, 0, 0, 0, &gsi, true, true,
5108 : 97408 : EXPR_LOCATION (parm));
5109 : : }
5110 : :
5111 : 428278 : seq = gsi_seq (gsi);
5112 : 428278 : if (seq)
5113 : 49496 : gsi_insert_seq_on_edge_immediate (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
5114 : 428278 : }
5115 : :
5116 : : /* The "main" function of intraprocedural SRA passes. Runs the analysis and if
5117 : : it reveals there are components of some aggregates to be scalarized, it runs
5118 : : the required transformations. */
5119 : : static unsigned int
5120 : 3311876 : perform_intra_sra (void)
5121 : : {
5122 : 3311876 : int ret = 0;
5123 : 3311876 : sra_initialize ();
5124 : :
5125 : 3311876 : if (!find_var_candidates ())
5126 : 2573465 : goto out;
5127 : :
5128 : 738411 : if (!scan_function ())
5129 : 54336 : goto out;
5130 : :
5131 : 684075 : if (!analyze_all_variable_accesses ())
5132 : 255797 : goto out;
5133 : :
5134 : 428278 : if (sra_modify_function_body ())
5135 : : ret = TODO_update_ssa | TODO_cleanup_cfg;
5136 : : else
5137 : 428256 : ret = TODO_update_ssa;
5138 : 428278 : initialize_parameter_reductions ();
5139 : :
5140 : 428278 : statistics_counter_event (cfun, "Scalar replacements created",
5141 : : sra_stats.replacements);
5142 : 428278 : statistics_counter_event (cfun, "Modified expressions", sra_stats.exprs);
5143 : 428278 : statistics_counter_event (cfun, "Subtree copy stmts",
5144 : : sra_stats.subtree_copies);
5145 : 428278 : statistics_counter_event (cfun, "Subreplacement stmts",
5146 : : sra_stats.subreplacements);
5147 : 428278 : statistics_counter_event (cfun, "Deleted stmts", sra_stats.deleted);
5148 : 428278 : statistics_counter_event (cfun, "Separate LHS and RHS handling",
5149 : : sra_stats.separate_lhs_rhs_handling);
5150 : :
5151 : 3311876 : out:
5152 : 3311876 : sra_deinitialize ();
5153 : 3311876 : return ret;
5154 : : }
5155 : :
5156 : : /* Perform early intraprocedural SRA. */
5157 : : static unsigned int
5158 : 2305947 : early_intra_sra (void)
5159 : : {
5160 : 2305947 : sra_mode = SRA_MODE_EARLY_INTRA;
5161 : 0 : return perform_intra_sra ();
5162 : : }
5163 : :
5164 : : /* Perform "late" intraprocedural SRA. */
5165 : : static unsigned int
5166 : 1005929 : late_intra_sra (void)
5167 : : {
5168 : 1005929 : sra_mode = SRA_MODE_INTRA;
5169 : 0 : return perform_intra_sra ();
5170 : : }
5171 : :
5172 : :
5173 : : static bool
5174 : 3316048 : gate_intra_sra (void)
5175 : : {
5176 : 3316048 : return flag_tree_sra != 0 && dbg_cnt (tree_sra);
5177 : : }
5178 : :
5179 : :
5180 : : namespace {
5181 : :
5182 : : const pass_data pass_data_sra_early =
5183 : : {
5184 : : GIMPLE_PASS, /* type */
5185 : : "esra", /* name */
5186 : : OPTGROUP_NONE, /* optinfo_flags */
5187 : : TV_TREE_SRA, /* tv_id */
5188 : : ( PROP_cfg | PROP_ssa ), /* properties_required */
5189 : : 0, /* properties_provided */
5190 : : 0, /* properties_destroyed */
5191 : : 0, /* todo_flags_start */
5192 : : TODO_update_ssa, /* todo_flags_finish */
5193 : : };
5194 : :
5195 : : class pass_sra_early : public gimple_opt_pass
5196 : : {
5197 : : public:
5198 : 283157 : pass_sra_early (gcc::context *ctxt)
5199 : 566314 : : gimple_opt_pass (pass_data_sra_early, ctxt)
5200 : : {}
5201 : :
5202 : : /* opt_pass methods: */
5203 : 2309698 : bool gate (function *) final override { return gate_intra_sra (); }
5204 : 2305947 : unsigned int execute (function *) final override
5205 : : {
5206 : 2305947 : return early_intra_sra ();
5207 : : }
5208 : :
5209 : : }; // class pass_sra_early
5210 : :
5211 : : } // anon namespace
5212 : :
5213 : : gimple_opt_pass *
5214 : 283157 : make_pass_sra_early (gcc::context *ctxt)
5215 : : {
5216 : 283157 : return new pass_sra_early (ctxt);
5217 : : }
5218 : :
5219 : : namespace {
5220 : :
5221 : : const pass_data pass_data_sra =
5222 : : {
5223 : : GIMPLE_PASS, /* type */
5224 : : "sra", /* name */
5225 : : OPTGROUP_NONE, /* optinfo_flags */
5226 : : TV_TREE_SRA, /* tv_id */
5227 : : ( PROP_cfg | PROP_ssa ), /* properties_required */
5228 : : 0, /* properties_provided */
5229 : : 0, /* properties_destroyed */
5230 : : TODO_update_address_taken, /* todo_flags_start */
5231 : : TODO_update_ssa, /* todo_flags_finish */
5232 : : };
5233 : :
5234 : : class pass_sra : public gimple_opt_pass
5235 : : {
5236 : : public:
5237 : 283157 : pass_sra (gcc::context *ctxt)
5238 : 566314 : : gimple_opt_pass (pass_data_sra, ctxt)
5239 : : {}
5240 : :
5241 : : /* opt_pass methods: */
5242 : 1006350 : bool gate (function *) final override { return gate_intra_sra (); }
5243 : 1005929 : unsigned int execute (function *) final override { return late_intra_sra (); }
5244 : :
5245 : : }; // class pass_sra
5246 : :
5247 : : } // anon namespace
5248 : :
5249 : : gimple_opt_pass *
5250 : 283157 : make_pass_sra (gcc::context *ctxt)
5251 : : {
5252 : 283157 : return new pass_sra (ctxt);
5253 : : }
5254 : :
5255 : :
5256 : : /* If type T cannot be totally scalarized, return false. Otherwise return true
5257 : : and push to the vector within PC offsets and lengths of all padding in the
5258 : : type as total scalarization would encounter it. */
5259 : :
5260 : : static bool
5261 : 71296 : check_ts_and_push_padding_to_vec (tree type, sra_padding_collecting *pc)
5262 : : {
5263 : 71296 : if (!totally_scalarizable_type_p (type, true /* optimistic value */,
5264 : : 0, pc))
5265 : : return false;
5266 : :
5267 : 64342 : pc->record_padding (tree_to_shwi (TYPE_SIZE (type)));
5268 : 64342 : return true;
5269 : : }
5270 : :
5271 : : /* Given two types in an assignment, return true either if any one cannot be
5272 : : totally scalarized or if they have padding (i.e. not copied bits) */
5273 : :
5274 : : bool
5275 : 39125 : sra_total_scalarization_would_copy_same_data_p (tree t1, tree t2)
5276 : : {
5277 : 39125 : sra_padding_collecting p1;
5278 : 39125 : if (!check_ts_and_push_padding_to_vec (t1, &p1))
5279 : : return true;
5280 : :
5281 : 32171 : sra_padding_collecting p2;
5282 : 32171 : if (!check_ts_and_push_padding_to_vec (t2, &p2))
5283 : : return true;
5284 : :
5285 : 32171 : unsigned l = p1.m_padding.length ();
5286 : 64342 : if (l != p2.m_padding.length ())
5287 : : return false;
5288 : 39169 : for (unsigned i = 0; i < l; i++)
5289 : 7001 : if (p1.m_padding[i].first != p2.m_padding[i].first
5290 : 7001 : || p1.m_padding[i].second != p2.m_padding[i].second)
5291 : : return false;
5292 : :
5293 : : return true;
5294 : 32171 : }
5295 : :
|
