Branch data Line data Source code
1 : : /* Scalar Replacement of Aggregates (SRA) converts some structure
2 : : references into scalar references, exposing them to the scalar
3 : : optimizers.
4 : : Copyright (C) 2008-2025 Free Software Foundation, Inc.
5 : : Contributed by Martin Jambor <mjambor@suse.cz>
6 : :
7 : : This file is part of GCC.
8 : :
9 : : GCC is free software; you can redistribute it and/or modify it under
10 : : the terms of the GNU General Public License as published by the Free
11 : : Software Foundation; either version 3, or (at your option) any later
12 : : version.
13 : :
14 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 : : for more details.
18 : :
19 : : You should have received a copy of the GNU General Public License
20 : : along with GCC; see the file COPYING3. If not see
21 : : <http://www.gnu.org/licenses/>. */
22 : :
23 : : /* This file implements Scalar Reduction of Aggregates (SRA). SRA is run
24 : : twice, once in the early stages of compilation (early SRA) and once in the
25 : : late stages (late SRA). The aim of both is to turn references to scalar
26 : : parts of aggregates into uses of independent scalar variables.
27 : :
28 : : The two passes are nearly identical, the only difference is that early SRA
29 : : does not scalarize unions which are used as the result in a GIMPLE_RETURN
30 : : statement because together with inlining this can lead to weird type
31 : : conversions.
32 : :
33 : : Both passes operate in four stages:
34 : :
35 : : 1. The declarations that have properties which make them candidates for
36 : : scalarization are identified in function find_var_candidates(). The
37 : : candidates are stored in candidate_bitmap.
38 : :
39 : : 2. The function body is scanned. In the process, declarations which are
40 : : used in a manner that prevent their scalarization are removed from the
41 : : candidate bitmap. More importantly, for every access into an aggregate,
42 : : an access structure (struct access) is created by create_access() and
43 : : stored in a vector associated with the aggregate. Among other
44 : : information, the aggregate declaration, the offset and size of the access
45 : : and its type are stored in the structure.
46 : :
47 : : On a related note, assign_link structures are created for every assign
48 : : statement between candidate aggregates and attached to the related
49 : : accesses.
50 : :
51 : : 3. The vectors of accesses are analyzed. They are first sorted according to
52 : : their offset and size and then scanned for partially overlapping accesses
53 : : (i.e. those which overlap but one is not entirely within another). Such
54 : : an access disqualifies the whole aggregate from being scalarized.
55 : :
56 : : If there is no such inhibiting overlap, a representative access structure
57 : : is chosen for every unique combination of offset and size. Afterwards,
58 : : the pass builds a set of trees from these structures, in which children
59 : : of an access are within their parent (in terms of offset and size).
60 : :
61 : : Then accesses are propagated whenever possible (i.e. in cases when it
62 : : does not create a partially overlapping access) across assign_links from
63 : : the right hand side to the left hand side.
64 : :
65 : : Then the set of trees for each declaration is traversed again and those
66 : : accesses which should be replaced by a scalar are identified.
67 : :
68 : : 4. The function is traversed again, and for every reference into an
69 : : aggregate that has some component which is about to be scalarized,
70 : : statements are amended and new statements are created as necessary.
71 : : Finally, if a parameter got scalarized, the scalar replacements are
72 : : initialized with values from respective parameter aggregates. */
73 : :
74 : : #include "config.h"
75 : : #include "system.h"
76 : : #include "coretypes.h"
77 : : #include "backend.h"
78 : : #include "target.h"
79 : : #include "rtl.h"
80 : : #include "tree.h"
81 : : #include "gimple.h"
82 : : #include "predict.h"
83 : : #include "alloc-pool.h"
84 : : #include "tree-pass.h"
85 : : #include "ssa.h"
86 : : #include "cgraph.h"
87 : : #include "gimple-pretty-print.h"
88 : : #include "alias.h"
89 : : #include "fold-const.h"
90 : : #include "tree-eh.h"
91 : : #include "stor-layout.h"
92 : : #include "gimplify.h"
93 : : #include "gimple-iterator.h"
94 : : #include "gimplify-me.h"
95 : : #include "gimple-walk.h"
96 : : #include "tree-cfg.h"
97 : : #include "tree-dfa.h"
98 : : #include "tree-ssa.h"
99 : : #include "dbgcnt.h"
100 : : #include "builtins.h"
101 : : #include "tree-sra.h"
102 : : #include "opts.h"
103 : : #include "tree-ssa-alias-compare.h"
104 : :
105 : : /* Enumeration of all aggregate reductions we can do. */
106 : : enum sra_mode { SRA_MODE_EARLY_IPA, /* early call regularization */
107 : : SRA_MODE_EARLY_INTRA, /* early intraprocedural SRA */
108 : : SRA_MODE_INTRA }; /* late intraprocedural SRA */
109 : :
110 : : /* Global variable describing which aggregate reduction we are performing at
111 : : the moment. */
112 : : static enum sra_mode sra_mode;
113 : :
114 : : struct assign_link;
115 : :
116 : : /* ACCESS represents each access to an aggregate variable (as a whole or a
117 : : part). It can also represent a group of accesses that refer to exactly the
118 : : same fragment of an aggregate (i.e. those that have exactly the same offset
119 : : and size). Such representatives for a single aggregate, once determined,
120 : : are linked in a linked list and have the group fields set.
121 : :
122 : : Moreover, when doing intraprocedural SRA, a tree is built from those
123 : : representatives (by the means of first_child and next_sibling pointers), in
124 : : which all items in a subtree are "within" the root, i.e. their offset is
125 : : greater or equal to offset of the root and offset+size is smaller or equal
126 : : to offset+size of the root. Children of an access are sorted by offset.
127 : :
128 : : Note that accesses to parts of vector and complex number types always
129 : : represented by an access to the whole complex number or a vector. It is a
130 : : duty of the modifying functions to replace them appropriately. */
131 : :
132 : : struct access
133 : : {
134 : : /* Values returned by `get_ref_base_and_extent' for each component reference
135 : : If EXPR isn't a component reference just set `BASE = EXPR', `OFFSET = 0',
136 : : `SIZE = TREE_SIZE (TREE_TYPE (expr))'. */
137 : : HOST_WIDE_INT offset;
138 : : HOST_WIDE_INT size;
139 : : tree base;
140 : :
141 : : /* Expression. It is context dependent so do not use it to create new
142 : : expressions to access the original aggregate. See PR 42154 for a
143 : : testcase. */
144 : : tree expr;
145 : : /* Type. */
146 : : tree type;
147 : :
148 : : /* The statement this access belongs to. */
149 : : gimple *stmt;
150 : :
151 : : /* Next group representative for this aggregate. */
152 : : struct access *next_grp;
153 : :
154 : : /* Pointer to the group representative. Pointer to itself if the struct is
155 : : the representative. */
156 : : struct access *group_representative;
157 : :
158 : : /* After access tree has been constructed, this points to the parent of the
159 : : current access, if there is one. NULL for roots. */
160 : : struct access *parent;
161 : :
162 : : /* If this access has any children (in terms of the definition above), this
163 : : points to the first one. */
164 : : struct access *first_child;
165 : :
166 : : /* In intraprocedural SRA, pointer to the next sibling in the access tree as
167 : : described above. */
168 : : struct access *next_sibling;
169 : :
170 : : /* Pointers to the first and last element in the linked list of assign
171 : : links for propagation from LHS to RHS. */
172 : : struct assign_link *first_rhs_link, *last_rhs_link;
173 : :
174 : : /* Pointers to the first and last element in the linked list of assign
175 : : links for propagation from LHS to RHS. */
176 : : struct assign_link *first_lhs_link, *last_lhs_link;
177 : :
178 : : /* Pointer to the next access in the work queues. */
179 : : struct access *next_rhs_queued, *next_lhs_queued;
180 : :
181 : : /* Replacement variable for this access "region." Never to be accessed
182 : : directly, always only by the means of get_access_replacement() and only
183 : : when grp_to_be_replaced flag is set. */
184 : : tree replacement_decl;
185 : :
186 : : /* Is this access made in reverse storage order? */
187 : : unsigned reverse : 1;
188 : :
189 : : /* Is this particular access write access? */
190 : : unsigned write : 1;
191 : :
192 : : /* Is this access currently in the rhs work queue? */
193 : : unsigned grp_rhs_queued : 1;
194 : :
195 : : /* Is this access currently in the lhs work queue? */
196 : : unsigned grp_lhs_queued : 1;
197 : :
198 : : /* Does this group contain a write access? This flag is propagated down the
199 : : access tree. */
200 : : unsigned grp_write : 1;
201 : :
202 : : /* Does this group contain a read access? This flag is propagated down the
203 : : access tree. */
204 : : unsigned grp_read : 1;
205 : :
206 : : /* Does this group contain a read access that comes from an assignment
207 : : statement? This flag is propagated down the access tree. */
208 : : unsigned grp_assignment_read : 1;
209 : :
210 : : /* Does this group contain a write access that comes from an assignment
211 : : statement? This flag is propagated down the access tree. */
212 : : unsigned grp_assignment_write : 1;
213 : :
214 : : /* Does this group contain a read access through a scalar type? This flag is
215 : : not propagated in the access tree in any direction. */
216 : : unsigned grp_scalar_read : 1;
217 : :
218 : : /* Does this group contain a write access through a scalar type? This flag
219 : : is not propagated in the access tree in any direction. */
220 : : unsigned grp_scalar_write : 1;
221 : :
222 : : /* In a root of an access tree, true means that the entire tree should be
223 : : totally scalarized - that all scalar leafs should be scalarized and
224 : : non-root grp_total_scalarization accesses should be honored. Otherwise,
225 : : non-root accesses with grp_total_scalarization should never get scalar
226 : : replacements. */
227 : : unsigned grp_total_scalarization : 1;
228 : :
229 : : /* Other passes of the analysis use this bit to make function
230 : : analyze_access_subtree create scalar replacements for this group if
231 : : possible. */
232 : : unsigned grp_hint : 1;
233 : :
234 : : /* Is the subtree rooted in this access fully covered by scalar
235 : : replacements? */
236 : : unsigned grp_covered : 1;
237 : :
238 : : /* If set to true, this access and all below it in an access tree must not be
239 : : scalarized. */
240 : : unsigned grp_unscalarizable_region : 1;
241 : :
242 : : /* Whether data have been written to parts of the aggregate covered by this
243 : : access which is not to be scalarized. This flag is propagated up in the
244 : : access tree. */
245 : : unsigned grp_unscalarized_data : 1;
246 : :
247 : : /* Set if all accesses in the group consist of the same chain of
248 : : COMPONENT_REFs and ARRAY_REFs. */
249 : : unsigned grp_same_access_path : 1;
250 : :
251 : : /* Does this access and/or group contain a write access through a
252 : : BIT_FIELD_REF? */
253 : : unsigned grp_partial_lhs : 1;
254 : :
255 : : /* Set when a scalar replacement should be created for this variable. */
256 : : unsigned grp_to_be_replaced : 1;
257 : :
258 : : /* Set when we want a replacement for the sole purpose of having it in
259 : : generated debug statements. */
260 : : unsigned grp_to_be_debug_replaced : 1;
261 : :
262 : : /* Should TREE_NO_WARNING of a replacement be set? */
263 : : unsigned grp_no_warning : 1;
264 : :
265 : : /* Result of propagation accross link from LHS to RHS. */
266 : : unsigned grp_result_of_prop_from_lhs : 1;
267 : : };
268 : :
269 : : typedef struct access *access_p;
270 : :
271 : :
272 : : /* Alloc pool for allocating access structures. */
273 : : static object_allocator<struct access> access_pool ("SRA accesses");
274 : :
275 : : /* A structure linking lhs and rhs accesses from an aggregate assignment. They
276 : : are used to propagate subaccesses from rhs to lhs and vice versa as long as
277 : : they don't conflict with what is already there. In the RHS->LHS direction,
278 : : we also propagate grp_write flag to lazily mark that the access contains any
279 : : meaningful data. */
280 : : struct assign_link
281 : : {
282 : : struct access *lacc, *racc;
283 : : struct assign_link *next_rhs, *next_lhs;
284 : : };
285 : :
286 : : /* Alloc pool for allocating assign link structures. */
287 : : static object_allocator<assign_link> assign_link_pool ("SRA links");
288 : :
289 : : /* Base (tree) -> Vector (vec<access_p> *) map. */
290 : : static hash_map<tree, auto_vec<access_p> > *base_access_vec;
291 : :
292 : : /* Hash to limit creation of artificial accesses */
293 : : static hash_map<tree, unsigned> *propagation_budget;
294 : :
295 : : /* Candidate hash table helpers. */
296 : :
297 : : struct uid_decl_hasher : nofree_ptr_hash <tree_node>
298 : : {
299 : : static inline hashval_t hash (const tree_node *);
300 : : static inline bool equal (const tree_node *, const tree_node *);
301 : : };
302 : :
303 : : /* Hash a tree in a uid_decl_map. */
304 : :
305 : : inline hashval_t
306 : 86691581 : uid_decl_hasher::hash (const tree_node *item)
307 : : {
308 : 86691581 : return item->decl_minimal.uid;
309 : : }
310 : :
311 : : /* Return true if the DECL_UID in both trees are equal. */
312 : :
313 : : inline bool
314 : 99837043 : uid_decl_hasher::equal (const tree_node *a, const tree_node *b)
315 : : {
316 : 99837043 : return (a->decl_minimal.uid == b->decl_minimal.uid);
317 : : }
318 : :
319 : : /* Set of candidates. */
320 : : static bitmap candidate_bitmap;
321 : : static hash_table<uid_decl_hasher> *candidates;
322 : :
323 : : /* For a candidate UID return the candidates decl. */
324 : :
325 : : static inline tree
326 : 15560288 : candidate (unsigned uid)
327 : : {
328 : 15560288 : tree_node t;
329 : 15560288 : t.decl_minimal.uid = uid;
330 : 15560288 : return candidates->find_with_hash (&t, static_cast <hashval_t> (uid));
331 : : }
332 : :
333 : : /* Bitmap of candidates which we should try to entirely scalarize away and
334 : : those which cannot be (because they are and need be used as a whole). */
335 : : static bitmap should_scalarize_away_bitmap, cannot_scalarize_away_bitmap;
336 : :
337 : : /* Bitmap of candidates in the constant pool, which cannot be scalarized
338 : : because this would produce non-constant expressions (e.g. Ada). */
339 : : static bitmap disqualified_constants;
340 : :
341 : : /* Bitmap of candidates which are passed by reference in call arguments. */
342 : : static bitmap passed_by_ref_in_call;
343 : :
344 : : /* Obstack for creation of fancy names. */
345 : : static struct obstack name_obstack;
346 : :
347 : : /* Head of a linked list of accesses that need to have its subaccesses
348 : : propagated to their assignment counterparts. */
349 : : static struct access *rhs_work_queue_head, *lhs_work_queue_head;
350 : :
351 : : /* Dump contents of ACCESS to file F in a human friendly way. If GRP is true,
352 : : representative fields are dumped, otherwise those which only describe the
353 : : individual access are. */
354 : :
355 : : static struct
356 : : {
357 : : /* Number of processed aggregates is readily available in
358 : : analyze_all_variable_accesses and so is not stored here. */
359 : :
360 : : /* Number of created scalar replacements. */
361 : : int replacements;
362 : :
363 : : /* Number of times sra_modify_expr or sra_modify_assign themselves changed an
364 : : expression. */
365 : : int exprs;
366 : :
367 : : /* Number of statements created by generate_subtree_copies. */
368 : : int subtree_copies;
369 : :
370 : : /* Number of statements created by load_assign_lhs_subreplacements. */
371 : : int subreplacements;
372 : :
373 : : /* Number of times sra_modify_assign has deleted a statement. */
374 : : int deleted;
375 : :
376 : : /* Number of times sra_modify_assign has to deal with subaccesses of LHS and
377 : : RHS reparately due to type conversions or nonexistent matching
378 : : references. */
379 : : int separate_lhs_rhs_handling;
380 : :
381 : : /* Number of parameters that were removed because they were unused. */
382 : : int deleted_unused_parameters;
383 : :
384 : : /* Number of scalars passed as parameters by reference that have been
385 : : converted to be passed by value. */
386 : : int scalar_by_ref_to_by_val;
387 : :
388 : : /* Number of aggregate parameters that were replaced by one or more of their
389 : : components. */
390 : : int aggregate_params_reduced;
391 : :
392 : : /* Numbber of components created when splitting aggregate parameters. */
393 : : int param_reductions_created;
394 : :
395 : : /* Number of deferred_init calls that are modified. */
396 : : int deferred_init;
397 : :
398 : : /* Number of deferred_init calls that are created by
399 : : generate_subtree_deferred_init. */
400 : : int subtree_deferred_init;
401 : : } sra_stats;
402 : :
403 : : static void
404 : 26 : dump_access (FILE *f, struct access *access, bool grp)
405 : : {
406 : 26 : fprintf (f, "access { ");
407 : 26 : fprintf (f, "base = (%d)'", DECL_UID (access->base));
408 : 26 : print_generic_expr (f, access->base);
409 : 26 : fprintf (f, "', offset = " HOST_WIDE_INT_PRINT_DEC, access->offset);
410 : 26 : fprintf (f, ", size = " HOST_WIDE_INT_PRINT_DEC, access->size);
411 : 26 : fprintf (f, ", expr = ");
412 : 26 : print_generic_expr (f, access->expr);
413 : 26 : fprintf (f, ", type = ");
414 : 26 : print_generic_expr (f, access->type);
415 : 26 : fprintf (f, ", reverse = %d", access->reverse);
416 : 26 : if (grp)
417 : 26 : fprintf (f, ", grp_read = %d, grp_write = %d, grp_assignment_read = %d, "
418 : : "grp_assignment_write = %d, grp_scalar_read = %d, "
419 : : "grp_scalar_write = %d, grp_total_scalarization = %d, "
420 : : "grp_hint = %d, grp_covered = %d, "
421 : : "grp_unscalarizable_region = %d, grp_unscalarized_data = %d, "
422 : : "grp_same_access_path = %d, grp_partial_lhs = %d, "
423 : : "grp_to_be_replaced = %d, grp_to_be_debug_replaced = %d}\n",
424 : 26 : access->grp_read, access->grp_write, access->grp_assignment_read,
425 : 26 : access->grp_assignment_write, access->grp_scalar_read,
426 : 26 : access->grp_scalar_write, access->grp_total_scalarization,
427 : 26 : access->grp_hint, access->grp_covered,
428 : 26 : access->grp_unscalarizable_region, access->grp_unscalarized_data,
429 : 26 : access->grp_same_access_path, access->grp_partial_lhs,
430 : 26 : access->grp_to_be_replaced, access->grp_to_be_debug_replaced);
431 : : else
432 : 0 : fprintf (f, ", write = %d, grp_total_scalarization = %d, "
433 : : "grp_partial_lhs = %d}\n",
434 : 0 : access->write, access->grp_total_scalarization,
435 : 0 : access->grp_partial_lhs);
436 : 26 : }
437 : :
438 : : /* Dump a subtree rooted in ACCESS to file F, indent by LEVEL. */
439 : :
440 : : static void
441 : 16 : dump_access_tree_1 (FILE *f, struct access *access, int level)
442 : : {
443 : 26 : do
444 : : {
445 : 26 : int i;
446 : :
447 : 43 : for (i = 0; i < level; i++)
448 : 17 : fputs ("* ", f);
449 : :
450 : 26 : dump_access (f, access, true);
451 : :
452 : 26 : if (access->first_child)
453 : 7 : dump_access_tree_1 (f, access->first_child, level + 1);
454 : :
455 : 26 : access = access->next_sibling;
456 : : }
457 : 26 : while (access);
458 : 16 : }
459 : :
460 : : /* Dump all access trees for a variable, given the pointer to the first root in
461 : : ACCESS. */
462 : :
463 : : static void
464 : 8 : dump_access_tree (FILE *f, struct access *access)
465 : : {
466 : 17 : for (; access; access = access->next_grp)
467 : 9 : dump_access_tree_1 (f, access, 0);
468 : 8 : }
469 : :
470 : : /* Return true iff ACC is non-NULL and has subaccesses. */
471 : :
472 : : static inline bool
473 : 16216244 : access_has_children_p (struct access *acc)
474 : : {
475 : 8704038 : return acc && acc->first_child;
476 : : }
477 : :
478 : : /* Return true iff ACC is (partly) covered by at least one replacement. */
479 : :
480 : : static bool
481 : 672 : access_has_replacements_p (struct access *acc)
482 : : {
483 : 672 : struct access *child;
484 : 672 : if (acc->grp_to_be_replaced)
485 : : return true;
486 : 599 : for (child = acc->first_child; child; child = child->next_sibling)
487 : 85 : if (access_has_replacements_p (child))
488 : : return true;
489 : : return false;
490 : : }
491 : :
492 : : /* Return a vector of pointers to accesses for the variable given in BASE or
493 : : NULL if there is none. */
494 : :
495 : : static vec<access_p> *
496 : 24822136 : get_base_access_vector (tree base)
497 : : {
498 : 0 : return base_access_vec->get (base);
499 : : }
500 : :
501 : : /* Find an access with required OFFSET and SIZE in a subtree of accesses rooted
502 : : in ACCESS. Return NULL if it cannot be found. */
503 : :
504 : : static struct access *
505 : 10626236 : find_access_in_subtree (struct access *access, HOST_WIDE_INT offset,
506 : : HOST_WIDE_INT size)
507 : : {
508 : 26869793 : while (access && (access->offset != offset || access->size != size))
509 : : {
510 : 5617321 : struct access *child = access->first_child;
511 : :
512 : 11074258 : while (child && (child->offset + child->size <= offset))
513 : 5456937 : child = child->next_sibling;
514 : : access = child;
515 : : }
516 : :
517 : : /* Total scalarization does not replace single field structures with their
518 : : single field but rather creates an access for them underneath. Look for
519 : : it. */
520 : 10626236 : if (access)
521 : 10676557 : while (access->first_child
522 : 3170363 : && access->first_child->offset == offset
523 : 13741899 : && access->first_child->size == size)
524 : : access = access->first_child;
525 : :
526 : 10626236 : return access;
527 : : }
528 : :
529 : : /* Return the first group representative for DECL or NULL if none exists. */
530 : :
531 : : static struct access *
532 : 20463165 : get_first_repr_for_decl (tree base)
533 : : {
534 : 20463165 : vec<access_p> *access_vec;
535 : :
536 : 20463165 : access_vec = get_base_access_vector (base);
537 : 20463165 : if (!access_vec)
538 : : return NULL;
539 : :
540 : 20463165 : return (*access_vec)[0];
541 : : }
542 : :
543 : : /* Find an access representative for the variable BASE and given OFFSET and
544 : : SIZE. Requires that access trees have already been built. Return NULL if
545 : : it cannot be found. */
546 : :
547 : : static struct access *
548 : 9559653 : get_var_base_offset_size_access (tree base, HOST_WIDE_INT offset,
549 : : HOST_WIDE_INT size)
550 : : {
551 : 9559653 : struct access *access;
552 : :
553 : 9559653 : access = get_first_repr_for_decl (base);
554 : 22009027 : while (access && (access->offset + access->size <= offset))
555 : 2889721 : access = access->next_grp;
556 : 9559653 : if (!access)
557 : : return NULL;
558 : :
559 : 9559644 : return find_access_in_subtree (access, offset, size);
560 : : }
561 : :
562 : : /* Add LINK to the linked list of assign links of RACC. */
563 : :
564 : : static void
565 : 1319510 : add_link_to_rhs (struct access *racc, struct assign_link *link)
566 : : {
567 : 1319510 : gcc_assert (link->racc == racc);
568 : :
569 : 1319510 : if (!racc->first_rhs_link)
570 : : {
571 : 1319510 : gcc_assert (!racc->last_rhs_link);
572 : 1319510 : racc->first_rhs_link = link;
573 : : }
574 : : else
575 : 0 : racc->last_rhs_link->next_rhs = link;
576 : :
577 : 1319510 : racc->last_rhs_link = link;
578 : 1319510 : link->next_rhs = NULL;
579 : 1319510 : }
580 : :
581 : : /* Add LINK to the linked list of lhs assign links of LACC. */
582 : :
583 : : static void
584 : 1319510 : add_link_to_lhs (struct access *lacc, struct assign_link *link)
585 : : {
586 : 1319510 : gcc_assert (link->lacc == lacc);
587 : :
588 : 1319510 : if (!lacc->first_lhs_link)
589 : : {
590 : 1319510 : gcc_assert (!lacc->last_lhs_link);
591 : 1319510 : lacc->first_lhs_link = link;
592 : : }
593 : : else
594 : 0 : lacc->last_lhs_link->next_lhs = link;
595 : :
596 : 1319510 : lacc->last_lhs_link = link;
597 : 1319510 : link->next_lhs = NULL;
598 : 1319510 : }
599 : :
600 : : /* Move all link structures in their linked list in OLD_ACC to the linked list
601 : : in NEW_ACC. */
602 : : static void
603 : 5520196 : relink_to_new_repr (struct access *new_acc, struct access *old_acc)
604 : : {
605 : 5520196 : if (old_acc->first_rhs_link)
606 : : {
607 : :
608 : 930043 : if (new_acc->first_rhs_link)
609 : : {
610 : 376058 : gcc_assert (!new_acc->last_rhs_link->next_rhs);
611 : 376058 : gcc_assert (!old_acc->last_rhs_link
612 : : || !old_acc->last_rhs_link->next_rhs);
613 : :
614 : 376058 : new_acc->last_rhs_link->next_rhs = old_acc->first_rhs_link;
615 : 376058 : new_acc->last_rhs_link = old_acc->last_rhs_link;
616 : : }
617 : : else
618 : : {
619 : 553985 : gcc_assert (!new_acc->last_rhs_link);
620 : :
621 : 553985 : new_acc->first_rhs_link = old_acc->first_rhs_link;
622 : 553985 : new_acc->last_rhs_link = old_acc->last_rhs_link;
623 : : }
624 : 930043 : old_acc->first_rhs_link = old_acc->last_rhs_link = NULL;
625 : : }
626 : : else
627 : 4590153 : gcc_assert (!old_acc->last_rhs_link);
628 : :
629 : 5520196 : if (old_acc->first_lhs_link)
630 : : {
631 : :
632 : 380769 : if (new_acc->first_lhs_link)
633 : : {
634 : 156646 : gcc_assert (!new_acc->last_lhs_link->next_lhs);
635 : 156646 : gcc_assert (!old_acc->last_lhs_link
636 : : || !old_acc->last_lhs_link->next_lhs);
637 : :
638 : 156646 : new_acc->last_lhs_link->next_lhs = old_acc->first_lhs_link;
639 : 156646 : new_acc->last_lhs_link = old_acc->last_lhs_link;
640 : : }
641 : : else
642 : : {
643 : 224123 : gcc_assert (!new_acc->last_lhs_link);
644 : :
645 : 224123 : new_acc->first_lhs_link = old_acc->first_lhs_link;
646 : 224123 : new_acc->last_lhs_link = old_acc->last_lhs_link;
647 : : }
648 : 380769 : old_acc->first_lhs_link = old_acc->last_lhs_link = NULL;
649 : : }
650 : : else
651 : 5139427 : gcc_assert (!old_acc->last_lhs_link);
652 : :
653 : 5520196 : }
654 : :
655 : : /* Add ACCESS to the work to queue for propagation of subaccesses from RHS to
656 : : LHS (which is actually a stack). */
657 : :
658 : : static void
659 : 4189085 : add_access_to_rhs_work_queue (struct access *access)
660 : : {
661 : 4189085 : if (access->first_rhs_link && !access->grp_rhs_queued)
662 : : {
663 : 1479260 : gcc_assert (!access->next_rhs_queued);
664 : 1479260 : access->next_rhs_queued = rhs_work_queue_head;
665 : 1479260 : access->grp_rhs_queued = 1;
666 : 1479260 : rhs_work_queue_head = access;
667 : : }
668 : 4189085 : }
669 : :
670 : : /* Add ACCESS to the work to queue for propagation of subaccesses from LHS to
671 : : RHS (which is actually a stack). */
672 : :
673 : : static void
674 : 1485676 : add_access_to_lhs_work_queue (struct access *access)
675 : : {
676 : 1485676 : if (access->first_lhs_link && !access->grp_lhs_queued)
677 : : {
678 : 1320995 : gcc_assert (!access->next_lhs_queued);
679 : 1320995 : access->next_lhs_queued = lhs_work_queue_head;
680 : 1320995 : access->grp_lhs_queued = 1;
681 : 1320995 : lhs_work_queue_head = access;
682 : : }
683 : 1485676 : }
684 : :
685 : : /* Pop an access from the work queue for propagating from RHS to LHS, and
686 : : return it, assuming there is one. */
687 : :
688 : : static struct access *
689 : 1479260 : pop_access_from_rhs_work_queue (void)
690 : : {
691 : 1479260 : struct access *access = rhs_work_queue_head;
692 : :
693 : 1479260 : rhs_work_queue_head = access->next_rhs_queued;
694 : 1479260 : access->next_rhs_queued = NULL;
695 : 1479260 : access->grp_rhs_queued = 0;
696 : 1479260 : return access;
697 : : }
698 : :
699 : : /* Pop an access from the work queue for propagating from LHS to RHS, and
700 : : return it, assuming there is one. */
701 : :
702 : : static struct access *
703 : 1320995 : pop_access_from_lhs_work_queue (void)
704 : : {
705 : 1320995 : struct access *access = lhs_work_queue_head;
706 : :
707 : 1320995 : lhs_work_queue_head = access->next_lhs_queued;
708 : 1320995 : access->next_lhs_queued = NULL;
709 : 1320995 : access->grp_lhs_queued = 0;
710 : 1320995 : return access;
711 : : }
712 : :
713 : : /* Allocate necessary structures. */
714 : :
715 : : static void
716 : 3458716 : sra_initialize (void)
717 : : {
718 : 3458716 : candidate_bitmap = BITMAP_ALLOC (NULL);
719 : 6917432 : candidates = new hash_table<uid_decl_hasher>
720 : 6434905 : (vec_safe_length (cfun->local_decls) / 2);
721 : 3458716 : should_scalarize_away_bitmap = BITMAP_ALLOC (NULL);
722 : 3458716 : cannot_scalarize_away_bitmap = BITMAP_ALLOC (NULL);
723 : 3458716 : disqualified_constants = BITMAP_ALLOC (NULL);
724 : 3458716 : passed_by_ref_in_call = BITMAP_ALLOC (NULL);
725 : 3458716 : gcc_obstack_init (&name_obstack);
726 : 3458716 : base_access_vec = new hash_map<tree, auto_vec<access_p> >;
727 : 3458716 : memset (&sra_stats, 0, sizeof (sra_stats));
728 : 3458716 : }
729 : :
730 : : /* Deallocate all general structures. */
731 : :
732 : : static void
733 : 3458716 : sra_deinitialize (void)
734 : : {
735 : 3458716 : BITMAP_FREE (candidate_bitmap);
736 : 3458716 : delete candidates;
737 : 3458716 : candidates = NULL;
738 : 3458716 : BITMAP_FREE (should_scalarize_away_bitmap);
739 : 3458716 : BITMAP_FREE (cannot_scalarize_away_bitmap);
740 : 3458716 : BITMAP_FREE (disqualified_constants);
741 : 3458716 : BITMAP_FREE (passed_by_ref_in_call);
742 : 3458716 : access_pool.release ();
743 : 3458716 : assign_link_pool.release ();
744 : 3458716 : obstack_free (&name_obstack, NULL);
745 : :
746 : 6917432 : delete base_access_vec;
747 : 3458716 : }
748 : :
749 : : /* Return true if DECL is a VAR_DECL in the constant pool, false otherwise. */
750 : :
751 : 42619645 : static bool constant_decl_p (tree decl)
752 : : {
753 : 36875487 : return VAR_P (decl) && DECL_IN_CONSTANT_POOL (decl);
754 : : }
755 : :
756 : : /* Remove DECL from candidates for SRA and write REASON to the dump file if
757 : : there is one. */
758 : :
759 : : static void
760 : 4634494 : disqualify_candidate (tree decl, const char *reason)
761 : : {
762 : 4634494 : if (bitmap_clear_bit (candidate_bitmap, DECL_UID (decl)))
763 : 2492267 : candidates->remove_elt_with_hash (decl, DECL_UID (decl));
764 : 4634494 : if (constant_decl_p (decl))
765 : 4040 : bitmap_set_bit (disqualified_constants, DECL_UID (decl));
766 : :
767 : 4634494 : if (dump_file && (dump_flags & TDF_DETAILS))
768 : : {
769 : 9 : fprintf (dump_file, "! Disqualifying ");
770 : 9 : print_generic_expr (dump_file, decl);
771 : 9 : fprintf (dump_file, " - %s\n", reason);
772 : : }
773 : 4634494 : }
774 : :
775 : : /* Return true iff the type contains a field or an element which does not allow
776 : : scalarization. Use VISITED_TYPES to avoid re-checking already checked
777 : : (sub-)types. */
778 : :
779 : : static bool
780 : 8575823 : type_internals_preclude_sra_p_1 (tree type, const char **msg,
781 : : hash_set<tree> *visited_types)
782 : : {
783 : 8575823 : tree fld;
784 : 8575823 : tree et;
785 : :
786 : 8575823 : if (visited_types->contains (type))
787 : : return false;
788 : 8289299 : visited_types->add (type);
789 : :
790 : 8289299 : switch (TREE_CODE (type))
791 : : {
792 : 7648844 : case RECORD_TYPE:
793 : 7648844 : case UNION_TYPE:
794 : 7648844 : case QUAL_UNION_TYPE:
795 : 154534717 : for (fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
796 : 146896234 : if (TREE_CODE (fld) == FIELD_DECL)
797 : : {
798 : 17330789 : if (TREE_CODE (fld) == FUNCTION_DECL)
799 : : continue;
800 : 17330789 : tree ft = TREE_TYPE (fld);
801 : :
802 : 17330789 : if (TREE_THIS_VOLATILE (fld))
803 : : {
804 : 855 : *msg = "volatile structure field";
805 : 855 : return true;
806 : : }
807 : 17329934 : if (!DECL_FIELD_OFFSET (fld))
808 : : {
809 : 0 : *msg = "no structure field offset";
810 : 0 : return true;
811 : : }
812 : 17329934 : if (!DECL_SIZE (fld))
813 : : {
814 : 8381 : *msg = "zero structure field size";
815 : 8381 : return true;
816 : : }
817 : 17321553 : if (!tree_fits_uhwi_p (DECL_FIELD_OFFSET (fld)))
818 : : {
819 : 0 : *msg = "structure field offset not fixed";
820 : 0 : return true;
821 : : }
822 : 17321553 : if (!tree_fits_uhwi_p (DECL_SIZE (fld)))
823 : : {
824 : 0 : *msg = "structure field size not fixed";
825 : 0 : return true;
826 : : }
827 : 17321553 : if (!tree_fits_shwi_p (bit_position (fld)))
828 : : {
829 : 0 : *msg = "structure field size too big";
830 : 0 : return true;
831 : : }
832 : 17321553 : if (AGGREGATE_TYPE_P (ft)
833 : 17321553 : && int_bit_position (fld) % BITS_PER_UNIT != 0)
834 : : {
835 : 0 : *msg = "structure field is bit field";
836 : 0 : return true;
837 : : }
838 : :
839 : 17321553 : if (AGGREGATE_TYPE_P (ft)
840 : 17321553 : && type_internals_preclude_sra_p_1 (ft, msg, visited_types))
841 : : return true;
842 : : }
843 : :
844 : : return false;
845 : :
846 : 523892 : case ARRAY_TYPE:
847 : 523892 : et = TREE_TYPE (type);
848 : :
849 : 523892 : if (TYPE_VOLATILE (et))
850 : : {
851 : 0 : *msg = "element type is volatile";
852 : 0 : return true;
853 : : }
854 : :
855 : 523892 : if (AGGREGATE_TYPE_P (et)
856 : 523892 : && type_internals_preclude_sra_p_1 (et, msg, visited_types))
857 : : return true;
858 : :
859 : : return false;
860 : :
861 : : default:
862 : : return false;
863 : : }
864 : : }
865 : :
866 : : /* Return true iff the type contains a field or an element which does not allow
867 : : scalarization. */
868 : :
869 : : bool
870 : 5055326 : type_internals_preclude_sra_p (tree type, const char **msg)
871 : : {
872 : 5055326 : hash_set<tree> visited_types;
873 : 5055326 : return type_internals_preclude_sra_p_1 (type, msg, &visited_types);
874 : 5055326 : }
875 : :
876 : :
877 : : /* Allocate an access structure for BASE, OFFSET and SIZE, clear it, fill in
878 : : the three fields. Also add it to the vector of accesses corresponding to
879 : : the base. Finally, return the new access. */
880 : :
881 : : static struct access *
882 : 15284198 : create_access_1 (tree base, HOST_WIDE_INT offset, HOST_WIDE_INT size)
883 : : {
884 : 15284198 : struct access *access = access_pool.allocate ();
885 : :
886 : 15284198 : memset (access, 0, sizeof (struct access));
887 : 15284198 : access->base = base;
888 : 15284198 : access->offset = offset;
889 : 15284198 : access->size = size;
890 : :
891 : 15284198 : base_access_vec->get_or_insert (base).safe_push (access);
892 : :
893 : 15284198 : return access;
894 : : }
895 : :
896 : : static bool maybe_add_sra_candidate (tree);
897 : :
898 : : /* Create and insert access for EXPR. Return created access, or NULL if it is
899 : : not possible. Also scan for uses of constant pool as we go along and add
900 : : to candidates. */
901 : :
902 : : static struct access *
903 : 28820324 : create_access (tree expr, gimple *stmt, bool write)
904 : : {
905 : 28820324 : struct access *access;
906 : 28820324 : poly_int64 poffset, psize, pmax_size;
907 : 28820324 : tree base = expr;
908 : 28820324 : bool reverse, unscalarizable_region = false;
909 : :
910 : 28820324 : base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size,
911 : : &reverse);
912 : :
913 : : /* For constant-pool entries, check we can substitute the constant value. */
914 : 28820324 : if (constant_decl_p (base)
915 : 3563 : && !bitmap_bit_p (disqualified_constants, DECL_UID (base)))
916 : : {
917 : 3563 : if (expr != base
918 : 263 : && !is_gimple_reg_type (TREE_TYPE (expr))
919 : 3563 : && dump_file && (dump_flags & TDF_DETAILS))
920 : : {
921 : : /* This occurs in Ada with accesses to ARRAY_RANGE_REFs,
922 : : and elements of multidimensional arrays (which are
923 : : multi-element arrays in their own right). */
924 : 0 : fprintf (dump_file, "Allowing non-reg-type load of part"
925 : : " of constant-pool entry: ");
926 : 0 : print_generic_expr (dump_file, expr);
927 : : }
928 : 3563 : maybe_add_sra_candidate (base);
929 : : }
930 : :
931 : 28820324 : if (!DECL_P (base) || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
932 : 13525689 : return NULL;
933 : :
934 : 15294635 : if (write && TREE_READONLY (base))
935 : : {
936 : 9717 : disqualify_candidate (base, "Encountered a store to a read-only decl.");
937 : 9717 : return NULL;
938 : : }
939 : :
940 : 15284918 : HOST_WIDE_INT offset, size, max_size;
941 : 15284918 : if (!poffset.is_constant (&offset)
942 : 15284918 : || !psize.is_constant (&size)
943 : 15284918 : || !pmax_size.is_constant (&max_size))
944 : : {
945 : : disqualify_candidate (base, "Encountered a polynomial-sized access.");
946 : : return NULL;
947 : : }
948 : :
949 : 15284918 : if (size != max_size)
950 : : {
951 : 385194 : size = max_size;
952 : 385194 : unscalarizable_region = true;
953 : : }
954 : 15284918 : if (size == 0)
955 : : return NULL;
956 : 15284916 : if (offset < 0)
957 : : {
958 : 34 : disqualify_candidate (base, "Encountered a negative offset access.");
959 : 34 : return NULL;
960 : : }
961 : 15284882 : if (size < 0)
962 : : {
963 : 24 : disqualify_candidate (base, "Encountered an unconstrained access.");
964 : 24 : return NULL;
965 : : }
966 : 15284858 : if (offset + size > tree_to_shwi (DECL_SIZE (base)))
967 : : {
968 : 659 : disqualify_candidate (base, "Encountered an access beyond the base.");
969 : 659 : return NULL;
970 : : }
971 : 15284199 : if (TREE_CODE (TREE_TYPE (expr)) == BITINT_TYPE
972 : 15284199 : && size > WIDE_INT_MAX_PRECISION - 1)
973 : : {
974 : 1 : disqualify_candidate (base, "Encountered too large _BitInt access.");
975 : 1 : return NULL;
976 : : }
977 : :
978 : 15284198 : access = create_access_1 (base, offset, size);
979 : 15284198 : access->expr = expr;
980 : 15284198 : access->type = TREE_TYPE (expr);
981 : 15284198 : access->write = write;
982 : 15284198 : access->grp_unscalarizable_region = unscalarizable_region;
983 : 15284198 : access->grp_same_access_path = true;
984 : 15284198 : access->stmt = stmt;
985 : 15284198 : access->reverse = reverse;
986 : :
987 : 15284198 : return access;
988 : : }
989 : :
990 : : /* Given an array type TYPE, extract element size to *EL_SIZE, minimum index to
991 : : *IDX and maximum index to *MAX so that the caller can iterate over all
992 : : elements and return true, except if the array is known to be zero-length,
993 : : then return false. */
994 : :
995 : : static bool
996 : 20736 : prepare_iteration_over_array_elts (tree type, HOST_WIDE_INT *el_size,
997 : : offset_int *idx, offset_int *max)
998 : : {
999 : 20736 : tree elem_size = TYPE_SIZE (TREE_TYPE (type));
1000 : 20736 : gcc_assert (elem_size && tree_fits_shwi_p (elem_size));
1001 : 20736 : *el_size = tree_to_shwi (elem_size);
1002 : 20736 : gcc_assert (*el_size > 0);
1003 : :
1004 : 20736 : tree minidx = TYPE_MIN_VALUE (TYPE_DOMAIN (type));
1005 : 20736 : gcc_assert (TREE_CODE (minidx) == INTEGER_CST);
1006 : 20736 : tree maxidx = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
1007 : : /* Skip (some) zero-length arrays; others have MAXIDX == MINIDX - 1. */
1008 : 20736 : if (!maxidx)
1009 : : return false;
1010 : 20736 : gcc_assert (TREE_CODE (maxidx) == INTEGER_CST);
1011 : 20736 : tree domain = TYPE_DOMAIN (type);
1012 : : /* MINIDX and MAXIDX are inclusive, and must be interpreted in
1013 : : DOMAIN (e.g. signed int, whereas min/max may be size_int). */
1014 : 20736 : *idx = wi::to_offset (minidx);
1015 : 20736 : *max = wi::to_offset (maxidx);
1016 : 20736 : if (!TYPE_UNSIGNED (domain))
1017 : : {
1018 : 20736 : *idx = wi::sext (*idx, TYPE_PRECISION (domain));
1019 : 20736 : *max = wi::sext (*max, TYPE_PRECISION (domain));
1020 : : }
1021 : : return true;
1022 : : }
1023 : :
1024 : : /* A structure to track collecting padding and hold collected padding
1025 : : information. */
1026 : :
1027 : 121522 : class sra_padding_collecting
1028 : : {
1029 : : public:
1030 : : /* Given that there won't be any data until at least OFFSET, add an
1031 : : appropriate entry to the list of paddings or extend the last one. */
1032 : : void record_padding (HOST_WIDE_INT offset);
1033 : : /* Vector of pairs describing contiguous pieces of padding, each pair
1034 : : consisting of offset and length. */
1035 : : auto_vec<std::pair<HOST_WIDE_INT, HOST_WIDE_INT>, 10> m_padding;
1036 : : /* Offset where data should continue after the last seen actual bit of data
1037 : : if there was no padding. */
1038 : : HOST_WIDE_INT m_data_until = 0;
1039 : : };
1040 : :
1041 : : /* Given that there won't be any data until at least OFFSET, add an appropriate
1042 : : entry to the list of paddings or extend the last one. */
1043 : :
1044 : 376780 : void sra_padding_collecting::record_padding (HOST_WIDE_INT offset)
1045 : : {
1046 : 376780 : if (offset > m_data_until)
1047 : : {
1048 : 19526 : HOST_WIDE_INT psz = offset - m_data_until;
1049 : 19526 : if (!m_padding.is_empty ()
1050 : 670 : && ((m_padding[m_padding.length () - 1].first
1051 : 670 : + m_padding[m_padding.length () - 1].second) == offset))
1052 : 0 : m_padding[m_padding.length () - 1].second += psz;
1053 : : else
1054 : 19526 : m_padding.safe_push (std::make_pair (m_data_until, psz));
1055 : : }
1056 : 376780 : }
1057 : :
1058 : : /* Return true iff TYPE is totally scalarizable - i.e. a RECORD_TYPE or
1059 : : fixed-length ARRAY_TYPE with fields that are either of gimple register types
1060 : : (excluding bit-fields) or (recursively) scalarizable types. CONST_DECL must
1061 : : be true if we are considering a decl from constant pool. If it is false,
1062 : : char arrays will be refused.
1063 : :
1064 : : TOTAL_OFFSET is the offset of TYPE within any outer type that is being
1065 : : examined.
1066 : :
1067 : : If PC is non-NULL, collect padding information into the vector within the
1068 : : structure. The information is however only complete if the function returns
1069 : : true and does not contain any padding at its end. */
1070 : :
1071 : : static bool
1072 : 3174484 : totally_scalarizable_type_p (tree type, bool const_decl,
1073 : : HOST_WIDE_INT total_offset,
1074 : : sra_padding_collecting *pc)
1075 : : {
1076 : 3174484 : if (is_gimple_reg_type (type))
1077 : : {
1078 : 2034241 : if (pc)
1079 : : {
1080 : 251470 : pc->record_padding (total_offset);
1081 : 251470 : pc->m_data_until = total_offset + tree_to_shwi (TYPE_SIZE (type));
1082 : : }
1083 : 2034241 : return true;
1084 : : }
1085 : 1140243 : if (type_contains_placeholder_p (type))
1086 : : return false;
1087 : :
1088 : 1140243 : bool have_predecessor_field = false;
1089 : 1140243 : HOST_WIDE_INT prev_pos = 0;
1090 : :
1091 : 1140243 : switch (TREE_CODE (type))
1092 : : {
1093 : 1100194 : case RECORD_TYPE:
1094 : 21513196 : for (tree fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
1095 : 20436405 : if (TREE_CODE (fld) == FIELD_DECL)
1096 : : {
1097 : 2275207 : tree ft = TREE_TYPE (fld);
1098 : :
1099 : 2275207 : if (!DECL_SIZE (fld))
1100 : : return false;
1101 : 2275207 : if (zerop (DECL_SIZE (fld)))
1102 : 57379 : continue;
1103 : :
1104 : 2217828 : HOST_WIDE_INT pos = int_bit_position (fld);
1105 : 2217828 : if (have_predecessor_field
1106 : 2217828 : && pos <= prev_pos)
1107 : : return false;
1108 : :
1109 : 2217828 : have_predecessor_field = true;
1110 : 2217828 : prev_pos = pos;
1111 : :
1112 : 2217828 : if (DECL_BIT_FIELD (fld))
1113 : : return false;
1114 : :
1115 : 2216190 : if (!totally_scalarizable_type_p (ft, const_decl, total_offset + pos,
1116 : : pc))
1117 : : return false;
1118 : : }
1119 : :
1120 : : return true;
1121 : :
1122 : 27217 : case ARRAY_TYPE:
1123 : 27217 : {
1124 : 27217 : HOST_WIDE_INT min_elem_size;
1125 : 27217 : if (const_decl)
1126 : : min_elem_size = 0;
1127 : : else
1128 : 16301 : min_elem_size = BITS_PER_UNIT;
1129 : :
1130 : 27217 : if (TYPE_DOMAIN (type) == NULL_TREE
1131 : 27217 : || !tree_fits_shwi_p (TYPE_SIZE (type))
1132 : 27217 : || !tree_fits_shwi_p (TYPE_SIZE (TREE_TYPE (type)))
1133 : 27217 : || (tree_to_shwi (TYPE_SIZE (TREE_TYPE (type))) <= min_elem_size)
1134 : 48758 : || !tree_fits_shwi_p (TYPE_MIN_VALUE (TYPE_DOMAIN (type))))
1135 : : return false;
1136 : 21541 : if (tree_to_shwi (TYPE_SIZE (type)) == 0
1137 : 21541 : && TYPE_MAX_VALUE (TYPE_DOMAIN (type)) == NULL_TREE)
1138 : : /* Zero-element array, should not prevent scalarization. */
1139 : : ;
1140 : 21541 : else if ((tree_to_shwi (TYPE_SIZE (type)) <= 0)
1141 : 21541 : || !tree_fits_shwi_p (TYPE_MAX_VALUE (TYPE_DOMAIN (type))))
1142 : : /* Variable-length array, do not allow scalarization. */
1143 : : return false;
1144 : :
1145 : 21505 : unsigned old_padding_len = 0;
1146 : 21505 : if (pc)
1147 : 10760 : old_padding_len = pc->m_padding.length ();
1148 : 21505 : tree elem = TREE_TYPE (type);
1149 : 21505 : if (!totally_scalarizable_type_p (elem, const_decl, total_offset, pc))
1150 : : return false;
1151 : 21366 : if (pc)
1152 : : {
1153 : 10760 : unsigned new_padding_len = pc->m_padding.length ();
1154 : 10760 : HOST_WIDE_INT el_size;
1155 : 10760 : offset_int idx, max;
1156 : 10760 : if (!prepare_iteration_over_array_elts (type, &el_size, &idx, &max))
1157 : 0 : return true;
1158 : 10760 : pc->record_padding (total_offset + el_size);
1159 : 10760 : ++idx;
1160 : 10760 : for (HOST_WIDE_INT pos = total_offset + el_size;
1161 : 275244 : idx <= max;
1162 : 264484 : pos += el_size, ++idx)
1163 : : {
1164 : 264511 : for (unsigned i = old_padding_len; i < new_padding_len; i++)
1165 : : {
1166 : 27 : HOST_WIDE_INT pp
1167 : 27 : = pos + pc->m_padding[i].first - total_offset;
1168 : 27 : HOST_WIDE_INT psz = pc->m_padding[i].second;
1169 : 27 : pc->m_padding.safe_push (std::make_pair (pp, psz));
1170 : : }
1171 : : }
1172 : 10760 : pc->m_data_until = total_offset + tree_to_shwi (TYPE_SIZE (type));
1173 : : }
1174 : : return true;
1175 : : }
1176 : : default:
1177 : : return false;
1178 : : }
1179 : : }
1180 : :
1181 : : /* Return true if REF has an VIEW_CONVERT_EXPR somewhere in it. */
1182 : :
1183 : : static inline bool
1184 : 58570136 : contains_view_convert_expr_p (const_tree ref)
1185 : : {
1186 : 80328438 : while (handled_component_p (ref))
1187 : : {
1188 : 21767743 : if (TREE_CODE (ref) == VIEW_CONVERT_EXPR)
1189 : : return true;
1190 : 21758302 : ref = TREE_OPERAND (ref, 0);
1191 : : }
1192 : :
1193 : : return false;
1194 : : }
1195 : :
1196 : : /* Return true if REF contains a VIEW_CONVERT_EXPR or a COMPONENT_REF with a
1197 : : bit-field field declaration. If TYPE_CHANGING_P is non-NULL, set the bool
1198 : : it points to will be set if REF contains any of the above or a MEM_REF
1199 : : expression that effectively performs type conversion. */
1200 : :
1201 : : static bool
1202 : 8199948 : contains_vce_or_bfcref_p (const_tree ref, bool *type_changing_p = NULL)
1203 : : {
1204 : 10424596 : while (handled_component_p (ref))
1205 : : {
1206 : 2608038 : if (TREE_CODE (ref) == VIEW_CONVERT_EXPR
1207 : 2608038 : || (TREE_CODE (ref) == COMPONENT_REF
1208 : 1893836 : && DECL_BIT_FIELD (TREE_OPERAND (ref, 1))))
1209 : : {
1210 : 383390 : if (type_changing_p)
1211 : 201392 : *type_changing_p = true;
1212 : 383390 : return true;
1213 : : }
1214 : 2224648 : ref = TREE_OPERAND (ref, 0);
1215 : : }
1216 : :
1217 : 7816558 : if (!type_changing_p
1218 : 4048123 : || TREE_CODE (ref) != MEM_REF
1219 : 7936582 : || TREE_CODE (TREE_OPERAND (ref, 0)) != ADDR_EXPR)
1220 : : return false;
1221 : :
1222 : 120024 : tree mem = TREE_OPERAND (TREE_OPERAND (ref, 0), 0);
1223 : 120024 : if (TYPE_MAIN_VARIANT (TREE_TYPE (ref))
1224 : 120024 : != TYPE_MAIN_VARIANT (TREE_TYPE (mem)))
1225 : 89257 : *type_changing_p = true;
1226 : :
1227 : : return false;
1228 : : }
1229 : :
1230 : : /* Search the given tree for a declaration by skipping handled components and
1231 : : exclude it from the candidates. */
1232 : :
1233 : : static void
1234 : 946828 : disqualify_base_of_expr (tree t, const char *reason)
1235 : : {
1236 : 946828 : t = get_base_address (t);
1237 : 946828 : if (t && DECL_P (t))
1238 : 884213 : disqualify_candidate (t, reason);
1239 : 946828 : }
1240 : :
1241 : : /* Return true if the BIT_FIELD_REF read EXPR is handled by SRA. */
1242 : :
1243 : : static bool
1244 : 124234 : sra_handled_bf_read_p (tree expr)
1245 : : {
1246 : 124234 : uint64_t size, offset;
1247 : 124234 : if (bit_field_size (expr).is_constant (&size)
1248 : 124234 : && bit_field_offset (expr).is_constant (&offset)
1249 : 124234 : && size % BITS_PER_UNIT == 0
1250 : 124234 : && offset % BITS_PER_UNIT == 0
1251 : 124302 : && pow2p_hwi (size))
1252 : 124138 : return true;
1253 : : return false;
1254 : : }
1255 : :
1256 : : /* Scan expression EXPR and create access structures for all accesses to
1257 : : candidates for scalarization. Return the created access or NULL if none is
1258 : : created. */
1259 : :
1260 : : static struct access *
1261 : 60583087 : build_access_from_expr_1 (tree expr, gimple *stmt, bool write)
1262 : : {
1263 : : /* We only allow ADDR_EXPRs in arguments of function calls and those must
1264 : : have been dealt with in build_access_from_call_arg. Any other address
1265 : : taking should have been caught by scan_visit_addr. */
1266 : 60583087 : if (TREE_CODE (expr) == ADDR_EXPR)
1267 : : {
1268 : 2012950 : tree base = get_base_address (TREE_OPERAND (expr, 0));
1269 : 2012950 : gcc_assert (!DECL_P (base)
1270 : : || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)));
1271 : 2012950 : return NULL;
1272 : : }
1273 : :
1274 : 58570137 : struct access *ret = NULL;
1275 : 58570137 : bool partial_ref;
1276 : :
1277 : 58570137 : if ((TREE_CODE (expr) == BIT_FIELD_REF
1278 : 72636 : && (write || !sra_handled_bf_read_p (expr)))
1279 : 58569019 : || TREE_CODE (expr) == IMAGPART_EXPR
1280 : 117114437 : || TREE_CODE (expr) == REALPART_EXPR)
1281 : : {
1282 : 49390 : expr = TREE_OPERAND (expr, 0);
1283 : 49390 : partial_ref = true;
1284 : : }
1285 : : else
1286 : : partial_ref = false;
1287 : :
1288 : 58570137 : if (storage_order_barrier_p (expr))
1289 : : {
1290 : 1 : disqualify_base_of_expr (expr, "storage order barrier.");
1291 : 1 : return NULL;
1292 : : }
1293 : :
1294 : : /* We need to dive through V_C_Es in order to get the size of its parameter
1295 : : and not the result type. Ada produces such statements. We are also
1296 : : capable of handling the topmost V_C_E but not any of those buried in other
1297 : : handled components. */
1298 : 58570136 : if (TREE_CODE (expr) == VIEW_CONVERT_EXPR)
1299 : 278724 : expr = TREE_OPERAND (expr, 0);
1300 : :
1301 : 58570136 : if (contains_view_convert_expr_p (expr))
1302 : : {
1303 : 9441 : disqualify_base_of_expr (expr, "V_C_E under a different handled "
1304 : : "component.");
1305 : 9441 : return NULL;
1306 : : }
1307 : 58560695 : if (TREE_THIS_VOLATILE (expr))
1308 : : {
1309 : 21645 : disqualify_base_of_expr (expr, "part of a volatile reference.");
1310 : 21645 : return NULL;
1311 : : }
1312 : :
1313 : 58539050 : switch (TREE_CODE (expr))
1314 : : {
1315 : 3657301 : case MEM_REF:
1316 : 3657301 : if (TREE_CODE (TREE_OPERAND (expr, 0)) != ADDR_EXPR)
1317 : : return NULL;
1318 : : /* fall through */
1319 : 28820324 : case VAR_DECL:
1320 : 28820324 : case PARM_DECL:
1321 : 28820324 : case RESULT_DECL:
1322 : 28820324 : case COMPONENT_REF:
1323 : 28820324 : case ARRAY_REF:
1324 : 28820324 : case ARRAY_RANGE_REF:
1325 : 28820324 : case BIT_FIELD_REF:
1326 : 28820324 : ret = create_access (expr, stmt, write);
1327 : 28820324 : break;
1328 : :
1329 : : default:
1330 : : break;
1331 : : }
1332 : :
1333 : 56658902 : if (write && partial_ref && ret)
1334 : 4315 : ret->grp_partial_lhs = 1;
1335 : :
1336 : : return ret;
1337 : : }
1338 : :
1339 : : /* Scan expression EXPR and create access structures for all accesses to
1340 : : candidates for scalarization. Return true if any access has been inserted.
1341 : : STMT must be the statement from which the expression is taken, WRITE must be
1342 : : true if the expression is a store and false otherwise. */
1343 : :
1344 : : static bool
1345 : 16472331 : build_access_from_expr (tree expr, gimple *stmt, bool write)
1346 : : {
1347 : 16472331 : struct access *access;
1348 : :
1349 : 16472331 : access = build_access_from_expr_1 (expr, stmt, write);
1350 : 16472331 : if (access)
1351 : : {
1352 : : /* This means the aggregate is accesses as a whole in a way other than an
1353 : : assign statement and thus cannot be removed even if we had a scalar
1354 : : replacement for everything. */
1355 : 2640394 : if (cannot_scalarize_away_bitmap)
1356 : 2640394 : bitmap_set_bit (cannot_scalarize_away_bitmap, DECL_UID (access->base));
1357 : 2640394 : return true;
1358 : : }
1359 : : return false;
1360 : : }
1361 : :
1362 : : enum out_edge_check { SRA_OUTGOING_EDGES_UNCHECKED, SRA_OUTGOING_EDGES_OK,
1363 : : SRA_OUTGOING_EDGES_FAIL };
1364 : :
1365 : : /* Return true if STMT terminates BB and there is an abnormal edge going out of
1366 : : the BB and remember the decision in OE_CHECK. */
1367 : :
1368 : : static bool
1369 : 2993987 : abnormal_edge_after_stmt_p (gimple *stmt, enum out_edge_check *oe_check)
1370 : : {
1371 : 2993987 : if (*oe_check == SRA_OUTGOING_EDGES_OK)
1372 : : return false;
1373 : 1782030 : if (*oe_check == SRA_OUTGOING_EDGES_FAIL)
1374 : : return true;
1375 : 1781809 : if (stmt_ends_bb_p (stmt))
1376 : : {
1377 : 738718 : edge e;
1378 : 738718 : edge_iterator ei;
1379 : 1914644 : FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
1380 : 1176405 : if (e->flags & EDGE_ABNORMAL)
1381 : : {
1382 : 479 : *oe_check = SRA_OUTGOING_EDGES_FAIL;
1383 : 479 : return true;
1384 : : }
1385 : : }
1386 : 1781330 : *oe_check = SRA_OUTGOING_EDGES_OK;
1387 : 1781330 : return false;
1388 : : }
1389 : :
1390 : : /* Scan expression EXPR which is an argument of a call and create access
1391 : : structures for all accesses to candidates for scalarization. Return true
1392 : : if any access has been inserted. STMT must be the statement from which the
1393 : : expression is taken. CAN_BE_RETURNED must be true if call argument flags
1394 : : do not rule out that the argument is directly returned. OE_CHECK is used
1395 : : to remember result of a test for abnormal outgoing edges after this
1396 : : statement. */
1397 : :
1398 : : static bool
1399 : 11534875 : build_access_from_call_arg (tree expr, gimple *stmt, bool can_be_returned,
1400 : : enum out_edge_check *oe_check)
1401 : : {
1402 : 11534875 : if (gimple_call_flags (stmt) & ECF_RETURNS_TWICE)
1403 : : {
1404 : 49 : tree base = expr;
1405 : 49 : if (TREE_CODE (expr) == ADDR_EXPR)
1406 : 2 : base = get_base_address (TREE_OPERAND (expr, 0));
1407 : 49 : disqualify_base_of_expr (base, "Passed to a returns_twice call.");
1408 : 49 : return false;
1409 : : }
1410 : :
1411 : 11534826 : if (TREE_CODE (expr) == ADDR_EXPR)
1412 : : {
1413 : 3908428 : tree base = get_base_address (TREE_OPERAND (expr, 0));
1414 : :
1415 : 3908428 : if (can_be_returned)
1416 : : {
1417 : 914441 : disqualify_base_of_expr (base, "Address possibly returned, "
1418 : : "leading to an alis SRA may not know.");
1419 : 914441 : return false;
1420 : : }
1421 : 2993987 : if (abnormal_edge_after_stmt_p (stmt, oe_check))
1422 : : {
1423 : 700 : disqualify_base_of_expr (base, "May lead to need to add statements "
1424 : : "to abnormal edge.");
1425 : 700 : return false;
1426 : : }
1427 : :
1428 : 2993287 : bool read = build_access_from_expr (base, stmt, false);
1429 : 2993287 : bool write = build_access_from_expr (base, stmt, true);
1430 : 2993287 : if (read || write)
1431 : : {
1432 : 264671 : if (dump_file && (dump_flags & TDF_DETAILS))
1433 : : {
1434 : 0 : fprintf (dump_file, "Allowed ADDR_EXPR of ");
1435 : 0 : print_generic_expr (dump_file, base);
1436 : 0 : fprintf (dump_file, " because of ");
1437 : 0 : print_gimple_stmt (dump_file, stmt, 0);
1438 : 0 : fprintf (dump_file, "\n");
1439 : : }
1440 : 264671 : bitmap_set_bit (passed_by_ref_in_call, DECL_UID (base));
1441 : 264671 : return true;
1442 : : }
1443 : : else
1444 : : return false;
1445 : : }
1446 : :
1447 : 7626398 : return build_access_from_expr (expr, stmt, false);
1448 : : }
1449 : :
1450 : :
1451 : : /* Return the single non-EH successor edge of BB or NULL if there is none or
1452 : : more than one. */
1453 : :
1454 : : static edge
1455 : 1556939 : single_non_eh_succ (basic_block bb)
1456 : : {
1457 : 1556939 : edge e, res = NULL;
1458 : 1556939 : edge_iterator ei;
1459 : :
1460 : 4669354 : FOR_EACH_EDGE (e, ei, bb->succs)
1461 : 3112803 : if (!(e->flags & EDGE_EH))
1462 : : {
1463 : 1557208 : if (res)
1464 : : return NULL;
1465 : : res = e;
1466 : : }
1467 : :
1468 : : return res;
1469 : : }
1470 : :
1471 : : /* Disqualify LHS and RHS for scalarization if STMT has to terminate its BB and
1472 : : there is no alternative spot where to put statements SRA might need to
1473 : : generate after it. The spot we are looking for is an edge leading to a
1474 : : single non-EH successor, if it exists and is indeed single. RHS may be
1475 : : NULL, in that case ignore it. */
1476 : :
1477 : : static bool
1478 : 24420785 : disqualify_if_bad_bb_terminating_stmt (gimple *stmt, tree lhs, tree rhs)
1479 : : {
1480 : 24420785 : if (stmt_ends_bb_p (stmt))
1481 : : {
1482 : 1463653 : if (single_non_eh_succ (gimple_bb (stmt)))
1483 : : return false;
1484 : :
1485 : 507 : disqualify_base_of_expr (lhs, "LHS of a throwing stmt.");
1486 : 507 : if (rhs)
1487 : 0 : disqualify_base_of_expr (rhs, "RHS of a throwing stmt.");
1488 : 507 : return true;
1489 : : }
1490 : : return false;
1491 : : }
1492 : :
1493 : : /* Return true if the nature of BASE is such that it contains data even if
1494 : : there is no write to it in the function. */
1495 : :
1496 : : static bool
1497 : 3830559 : comes_initialized_p (tree base)
1498 : : {
1499 : 0 : return TREE_CODE (base) == PARM_DECL || constant_decl_p (base);
1500 : : }
1501 : :
1502 : : /* Scan expressions occurring in STMT, create access structures for all accesses
1503 : : to candidates for scalarization and remove those candidates which occur in
1504 : : statements or expressions that prevent them from being split apart. Return
1505 : : true if any access has been inserted. */
1506 : :
1507 : : static bool
1508 : 34120407 : build_accesses_from_assign (gimple *stmt)
1509 : : {
1510 : 34120407 : tree lhs, rhs;
1511 : 34120407 : struct access *lacc, *racc;
1512 : :
1513 : 34120407 : if (!gimple_assign_single_p (stmt)
1514 : : /* Scope clobbers don't influence scalarization. */
1515 : 34120407 : || gimple_clobber_p (stmt))
1516 : : return false;
1517 : :
1518 : 22055360 : lhs = gimple_assign_lhs (stmt);
1519 : 22055360 : rhs = gimple_assign_rhs1 (stmt);
1520 : :
1521 : 22055360 : if (disqualify_if_bad_bb_terminating_stmt (stmt, lhs, rhs))
1522 : : return false;
1523 : :
1524 : 22055360 : racc = build_access_from_expr_1 (rhs, stmt, false);
1525 : 22055360 : lacc = build_access_from_expr_1 (lhs, stmt, true);
1526 : :
1527 : 22055360 : bool tbaa_hazard
1528 : 22055360 : = !types_equal_for_same_type_for_tbaa_p (TREE_TYPE (lhs), TREE_TYPE (rhs));
1529 : :
1530 : 22055360 : if (lacc)
1531 : : {
1532 : 6973330 : lacc->grp_assignment_write = 1;
1533 : 6973330 : if (storage_order_barrier_p (rhs))
1534 : 1 : lacc->grp_unscalarizable_region = 1;
1535 : :
1536 : 6973330 : if (should_scalarize_away_bitmap && !is_gimple_reg_type (lacc->type))
1537 : : {
1538 : 2259652 : bool type_changing_p = false;
1539 : 2259652 : contains_vce_or_bfcref_p (lhs, &type_changing_p);
1540 : 2259652 : if (type_changing_p)
1541 : 146160 : bitmap_set_bit (cannot_scalarize_away_bitmap,
1542 : 73080 : DECL_UID (lacc->base));
1543 : : }
1544 : 6973330 : if (tbaa_hazard)
1545 : 850240 : lacc->grp_same_access_path = false;
1546 : : }
1547 : :
1548 : 22055360 : if (racc)
1549 : : {
1550 : 5670446 : racc->grp_assignment_read = 1;
1551 : 5670446 : if (should_scalarize_away_bitmap && !is_gimple_reg_type (racc->type))
1552 : : {
1553 : 1989863 : bool type_changing_p = false;
1554 : 1989863 : contains_vce_or_bfcref_p (rhs, &type_changing_p);
1555 : :
1556 : 3762157 : if (type_changing_p || gimple_has_volatile_ops (stmt))
1557 : 435856 : bitmap_set_bit (cannot_scalarize_away_bitmap,
1558 : 217928 : DECL_UID (racc->base));
1559 : : else
1560 : 3543870 : bitmap_set_bit (should_scalarize_away_bitmap,
1561 : 1771935 : DECL_UID (racc->base));
1562 : : }
1563 : 5670446 : if (storage_order_barrier_p (lhs))
1564 : 0 : racc->grp_unscalarizable_region = 1;
1565 : 5670446 : if (tbaa_hazard)
1566 : 66389 : racc->grp_same_access_path = false;
1567 : : }
1568 : :
1569 : 22055360 : if (lacc && racc
1570 : 1516053 : && (sra_mode == SRA_MODE_EARLY_INTRA || sra_mode == SRA_MODE_INTRA)
1571 : 1516053 : && !lacc->grp_unscalarizable_region
1572 : 1515553 : && !racc->grp_unscalarizable_region
1573 : 1514202 : && AGGREGATE_TYPE_P (TREE_TYPE (lhs))
1574 : 1514202 : && lacc->size == racc->size
1575 : 23569289 : && useless_type_conversion_p (lacc->type, racc->type))
1576 : : {
1577 : 1319510 : struct assign_link *link;
1578 : :
1579 : 1319510 : link = assign_link_pool.allocate ();
1580 : 1319510 : memset (link, 0, sizeof (struct assign_link));
1581 : :
1582 : 1319510 : link->lacc = lacc;
1583 : 1319510 : link->racc = racc;
1584 : 1319510 : add_link_to_rhs (racc, link);
1585 : 1319510 : add_link_to_lhs (lacc, link);
1586 : 1319510 : add_access_to_rhs_work_queue (racc);
1587 : 1319510 : add_access_to_lhs_work_queue (lacc);
1588 : :
1589 : : /* Let's delay marking the areas as written until propagation of accesses
1590 : : across link, unless the nature of rhs tells us that its data comes
1591 : : from elsewhere. */
1592 : 1319510 : if (!comes_initialized_p (racc->base))
1593 : 1189308 : lacc->write = false;
1594 : : }
1595 : :
1596 : 22055360 : return lacc || racc;
1597 : : }
1598 : :
1599 : : /* Callback of walk_stmt_load_store_addr_ops visit_addr used to detect taking
1600 : : addresses of candidates a places which are not call arguments. Such
1601 : : candidates are disqalified from SRA. This also applies to GIMPLE_ASM
1602 : : operands with memory constrains which cannot be scalarized. */
1603 : :
1604 : : static bool
1605 : 2387554 : scan_visit_addr (gimple *, tree op, tree, void *)
1606 : : {
1607 : 2387554 : op = get_base_address (op);
1608 : 2387554 : if (op
1609 : 2387554 : && DECL_P (op))
1610 : 1307964 : disqualify_candidate (op, "Address taken in a non-call-argument context.");
1611 : :
1612 : 2387554 : return false;
1613 : : }
1614 : :
1615 : : /* Scan function and look for interesting expressions and create access
1616 : : structures for them. Return true iff any access is created. */
1617 : :
1618 : : static bool
1619 : 796104 : scan_function (void)
1620 : : {
1621 : 796104 : basic_block bb;
1622 : 796104 : bool ret = false;
1623 : :
1624 : 13677681 : FOR_EACH_BB_FN (bb, cfun)
1625 : : {
1626 : 12881577 : gimple_stmt_iterator gsi;
1627 : 17738173 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1628 : 4856596 : walk_stmt_load_store_addr_ops (gsi_stmt (gsi), NULL, NULL, NULL,
1629 : : scan_visit_addr);
1630 : :
1631 : 122970843 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1632 : : {
1633 : 97207689 : gimple *stmt = gsi_stmt (gsi);
1634 : 97207689 : tree t;
1635 : 97207689 : unsigned i;
1636 : :
1637 : 97207689 : if (gimple_code (stmt) != GIMPLE_CALL)
1638 : 91404510 : walk_stmt_load_store_addr_ops (stmt, NULL, NULL, NULL,
1639 : : scan_visit_addr);
1640 : :
1641 : 97207689 : switch (gimple_code (stmt))
1642 : : {
1643 : 782821 : case GIMPLE_RETURN:
1644 : 782821 : t = gimple_return_retval (as_a <greturn *> (stmt));
1645 : 782821 : if (t != NULL_TREE)
1646 : 479443 : ret |= build_access_from_expr (t, stmt, false);
1647 : : break;
1648 : :
1649 : 34120407 : case GIMPLE_ASSIGN:
1650 : 34120407 : ret |= build_accesses_from_assign (stmt);
1651 : 34120407 : break;
1652 : :
1653 : 5803179 : case GIMPLE_CALL:
1654 : 5803179 : {
1655 : 5803179 : enum out_edge_check oe_check = SRA_OUTGOING_EDGES_UNCHECKED;
1656 : 5803179 : gcall *call = as_a <gcall *> (stmt);
1657 : 17297531 : for (i = 0; i < gimple_call_num_args (call); i++)
1658 : : {
1659 : 11494352 : bool can_be_returned;
1660 : 11494352 : if (gimple_call_lhs (call))
1661 : : {
1662 : 4616392 : int af = gimple_call_arg_flags (call, i);
1663 : 4616392 : can_be_returned = !(af & EAF_NOT_RETURNED_DIRECTLY);
1664 : : }
1665 : : else
1666 : : can_be_returned = false;
1667 : 11494352 : ret |= build_access_from_call_arg (gimple_call_arg (call,
1668 : : i),
1669 : : stmt, can_be_returned,
1670 : : &oe_check);
1671 : : }
1672 : 5803179 : if (gimple_call_chain(stmt))
1673 : 40523 : ret |= build_access_from_call_arg (gimple_call_chain(call),
1674 : : stmt, false, &oe_check);
1675 : : }
1676 : :
1677 : 5803179 : t = gimple_call_lhs (stmt);
1678 : 5803179 : if (t && !disqualify_if_bad_bb_terminating_stmt (stmt, t, NULL))
1679 : : {
1680 : : /* If the STMT is a call to DEFERRED_INIT, avoid setting
1681 : : cannot_scalarize_away_bitmap. */
1682 : 2364918 : if (gimple_call_internal_p (stmt, IFN_DEFERRED_INIT))
1683 : 36 : ret |= !!build_access_from_expr_1 (t, stmt, true);
1684 : : else
1685 : 2364882 : ret |= build_access_from_expr (t, stmt, true);
1686 : : }
1687 : : break;
1688 : :
1689 : 9023 : case GIMPLE_ASM:
1690 : 9023 : {
1691 : 9023 : gasm *asm_stmt = as_a <gasm *> (stmt);
1692 : 9023 : if (stmt_ends_bb_p (asm_stmt)
1693 : 9038 : && !single_succ_p (gimple_bb (asm_stmt)))
1694 : : {
1695 : 32 : for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
1696 : : {
1697 : 17 : t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
1698 : 17 : disqualify_base_of_expr (t, "OP of asm goto.");
1699 : : }
1700 : 42 : for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
1701 : : {
1702 : 27 : t = TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
1703 : 27 : disqualify_base_of_expr (t, "OP of asm goto.");
1704 : : }
1705 : : }
1706 : : else
1707 : : {
1708 : 16947 : for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
1709 : : {
1710 : 7939 : t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
1711 : 7939 : ret |= build_access_from_expr (t, asm_stmt, false);
1712 : : }
1713 : 16103 : for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
1714 : : {
1715 : 7095 : t = TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
1716 : 7095 : ret |= build_access_from_expr (t, asm_stmt, true);
1717 : : }
1718 : : }
1719 : : }
1720 : : break;
1721 : :
1722 : : default:
1723 : : break;
1724 : : }
1725 : : }
1726 : : }
1727 : :
1728 : 796104 : return ret;
1729 : : }
1730 : :
1731 : : /* Helper of QSORT function. There are pointers to accesses in the array. An
1732 : : access is considered smaller than another if it has smaller offset or if the
1733 : : offsets are the same but is size is bigger. */
1734 : :
1735 : : static int
1736 : 128506714 : compare_access_positions (const void *a, const void *b)
1737 : : {
1738 : 128506714 : const access_p *fp1 = (const access_p *) a;
1739 : 128506714 : const access_p *fp2 = (const access_p *) b;
1740 : 128506714 : const access_p f1 = *fp1;
1741 : 128506714 : const access_p f2 = *fp2;
1742 : :
1743 : 128506714 : if (f1->offset != f2->offset)
1744 : 113759579 : return f1->offset < f2->offset ? -1 : 1;
1745 : :
1746 : 51317158 : if (f1->size == f2->size)
1747 : : {
1748 : 35845114 : if (f1->type == f2->type)
1749 : : return 0;
1750 : : /* Put any non-aggregate type before any aggregate type. */
1751 : 5091304 : else if (!is_gimple_reg_type (f1->type)
1752 : 5091304 : && is_gimple_reg_type (f2->type))
1753 : : return 1;
1754 : 3612697 : else if (is_gimple_reg_type (f1->type)
1755 : 3612697 : && !is_gimple_reg_type (f2->type))
1756 : : return -1;
1757 : : /* Put any complex or vector type before any other scalar type. */
1758 : 1730506 : else if (TREE_CODE (f1->type) != COMPLEX_TYPE
1759 : 1730506 : && TREE_CODE (f1->type) != VECTOR_TYPE
1760 : 1663593 : && (TREE_CODE (f2->type) == COMPLEX_TYPE
1761 : 1663593 : || VECTOR_TYPE_P (f2->type)))
1762 : : return 1;
1763 : 1692284 : else if ((TREE_CODE (f1->type) == COMPLEX_TYPE
1764 : : || VECTOR_TYPE_P (f1->type))
1765 : 66913 : && TREE_CODE (f2->type) != COMPLEX_TYPE
1766 : 64686 : && TREE_CODE (f2->type) != VECTOR_TYPE)
1767 : : return -1;
1768 : : /* Put any integral type before any non-integral type. When splicing, we
1769 : : make sure that those with insufficient precision and occupying the
1770 : : same space are not scalarized. */
1771 : 1642272 : else if (INTEGRAL_TYPE_P (f1->type)
1772 : 401051 : && !INTEGRAL_TYPE_P (f2->type))
1773 : : return -1;
1774 : 1537752 : else if (!INTEGRAL_TYPE_P (f1->type)
1775 : 1241221 : && INTEGRAL_TYPE_P (f2->type))
1776 : : return 1;
1777 : : /* Put the integral type with the bigger precision first. */
1778 : 1431911 : else if (INTEGRAL_TYPE_P (f1->type)
1779 : 296531 : && INTEGRAL_TYPE_P (f2->type)
1780 : 1728442 : && (TYPE_PRECISION (f2->type) != TYPE_PRECISION (f1->type)))
1781 : 12225 : return TYPE_PRECISION (f2->type) - TYPE_PRECISION (f1->type);
1782 : : /* Stabilize the sort. */
1783 : 1419686 : return TYPE_UID (f1->type) - TYPE_UID (f2->type);
1784 : : }
1785 : :
1786 : : /* We want the bigger accesses first, thus the opposite operator in the next
1787 : : line: */
1788 : 15472044 : return f1->size > f2->size ? -1 : 1;
1789 : : }
1790 : :
1791 : :
1792 : : /* Append a name of the declaration to the name obstack. A helper function for
1793 : : make_fancy_name. */
1794 : :
1795 : : static void
1796 : 2126950 : make_fancy_decl_name (tree decl)
1797 : : {
1798 : 2126950 : char buffer[32];
1799 : :
1800 : 2126950 : tree name = DECL_NAME (decl);
1801 : 2126950 : if (name)
1802 : 2060294 : obstack_grow (&name_obstack, IDENTIFIER_POINTER (name),
1803 : : IDENTIFIER_LENGTH (name));
1804 : : else
1805 : : {
1806 : 66656 : sprintf (buffer, "D%u", DECL_UID (decl));
1807 : 66656 : obstack_grow (&name_obstack, buffer, strlen (buffer));
1808 : : }
1809 : 2126950 : }
1810 : :
1811 : : /* Helper for make_fancy_name. */
1812 : :
1813 : : static void
1814 : 2400979 : make_fancy_name_1 (tree expr)
1815 : : {
1816 : 2623986 : char buffer[32];
1817 : 2623986 : tree index;
1818 : :
1819 : 2623986 : if (DECL_P (expr))
1820 : : {
1821 : 1050702 : make_fancy_decl_name (expr);
1822 : 1050702 : return;
1823 : : }
1824 : :
1825 : 1573284 : switch (TREE_CODE (expr))
1826 : : {
1827 : 1076248 : case COMPONENT_REF:
1828 : 1076248 : make_fancy_name_1 (TREE_OPERAND (expr, 0));
1829 : 1076248 : obstack_1grow (&name_obstack, '$');
1830 : 1076248 : make_fancy_decl_name (TREE_OPERAND (expr, 1));
1831 : 1076248 : break;
1832 : :
1833 : 53130 : case ARRAY_REF:
1834 : 53130 : make_fancy_name_1 (TREE_OPERAND (expr, 0));
1835 : 53130 : obstack_1grow (&name_obstack, '$');
1836 : : /* Arrays with only one element may not have a constant as their
1837 : : index. */
1838 : 53130 : index = TREE_OPERAND (expr, 1);
1839 : 53130 : if (TREE_CODE (index) != INTEGER_CST)
1840 : : break;
1841 : 53008 : sprintf (buffer, HOST_WIDE_INT_PRINT_DEC, TREE_INT_CST_LOW (index));
1842 : 53008 : obstack_grow (&name_obstack, buffer, strlen (buffer));
1843 : 53008 : break;
1844 : :
1845 : 223007 : case BIT_FIELD_REF:
1846 : 223007 : case ADDR_EXPR:
1847 : 223007 : make_fancy_name_1 (TREE_OPERAND (expr, 0));
1848 : 223007 : break;
1849 : :
1850 : 220822 : case MEM_REF:
1851 : 220822 : make_fancy_name_1 (TREE_OPERAND (expr, 0));
1852 : 220822 : if (!integer_zerop (TREE_OPERAND (expr, 1)))
1853 : : {
1854 : 68656 : obstack_1grow (&name_obstack, '$');
1855 : 137312 : sprintf (buffer, HOST_WIDE_INT_PRINT_DEC,
1856 : 68656 : TREE_INT_CST_LOW (TREE_OPERAND (expr, 1)));
1857 : 68656 : obstack_grow (&name_obstack, buffer, strlen (buffer));
1858 : : }
1859 : : break;
1860 : :
1861 : 0 : case REALPART_EXPR:
1862 : 0 : case IMAGPART_EXPR:
1863 : 0 : gcc_unreachable (); /* we treat these as scalars. */
1864 : : break;
1865 : : default:
1866 : : break;
1867 : : }
1868 : : }
1869 : :
1870 : : /* Create a human readable name for replacement variable of ACCESS. */
1871 : :
1872 : : static char *
1873 : 1050779 : make_fancy_name (tree expr)
1874 : : {
1875 : 1050779 : make_fancy_name_1 (expr);
1876 : 1050779 : obstack_1grow (&name_obstack, '\0');
1877 : 1050779 : return XOBFINISH (&name_obstack, char *);
1878 : : }
1879 : :
1880 : : /* Construct a MEM_REF that would reference a part of aggregate BASE of type
1881 : : EXP_TYPE at the given OFFSET and with storage order REVERSE. If BASE is
1882 : : something for which get_addr_base_and_unit_offset returns NULL, gsi must
1883 : : be non-NULL and is used to insert new statements either before or below
1884 : : the current one as specified by INSERT_AFTER. This function is not capable
1885 : : of handling bitfields. */
1886 : :
1887 : : tree
1888 : 2191578 : build_ref_for_offset (location_t loc, tree base, poly_int64 offset,
1889 : : bool reverse, tree exp_type, gimple_stmt_iterator *gsi,
1890 : : bool insert_after)
1891 : : {
1892 : 2191578 : tree prev_base = base;
1893 : 2191578 : tree off;
1894 : 2191578 : tree mem_ref;
1895 : 2191578 : poly_int64 base_offset;
1896 : 2191578 : unsigned HOST_WIDE_INT misalign;
1897 : 2191578 : unsigned int align;
1898 : :
1899 : : /* Preserve address-space information. */
1900 : 2191578 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (base));
1901 : 2191578 : if (as != TYPE_ADDR_SPACE (exp_type))
1902 : 6 : exp_type = build_qualified_type (exp_type,
1903 : 3 : TYPE_QUALS (exp_type)
1904 : 3 : | ENCODE_QUAL_ADDR_SPACE (as));
1905 : :
1906 : 2191578 : poly_int64 byte_offset = exact_div (offset, BITS_PER_UNIT);
1907 : 2191578 : get_object_alignment_1 (base, &align, &misalign);
1908 : 2191578 : base = get_addr_base_and_unit_offset (base, &base_offset);
1909 : :
1910 : : /* get_addr_base_and_unit_offset returns NULL for references with a variable
1911 : : offset such as array[var_index]. */
1912 : 2191578 : if (!base)
1913 : : {
1914 : 21376 : gassign *stmt;
1915 : 21376 : tree tmp, addr;
1916 : :
1917 : 21376 : gcc_checking_assert (gsi);
1918 : 21376 : tmp = make_ssa_name (build_pointer_type (TREE_TYPE (prev_base)));
1919 : 21376 : addr = build_fold_addr_expr (unshare_expr (prev_base));
1920 : 21376 : STRIP_USELESS_TYPE_CONVERSION (addr);
1921 : 21376 : stmt = gimple_build_assign (tmp, addr);
1922 : 21376 : gimple_set_location (stmt, loc);
1923 : 21376 : if (insert_after)
1924 : 2495 : gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
1925 : : else
1926 : 18881 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1927 : :
1928 : 21376 : off = build_int_cst (reference_alias_ptr_type (prev_base), byte_offset);
1929 : 21376 : base = tmp;
1930 : : }
1931 : 2170202 : else if (TREE_CODE (base) == MEM_REF)
1932 : : {
1933 : 132602 : off = build_int_cst (TREE_TYPE (TREE_OPERAND (base, 1)),
1934 : : base_offset + byte_offset);
1935 : 132602 : off = int_const_binop (PLUS_EXPR, TREE_OPERAND (base, 1), off);
1936 : 132602 : base = unshare_expr (TREE_OPERAND (base, 0));
1937 : : }
1938 : : else
1939 : : {
1940 : 2037600 : off = build_int_cst (reference_alias_ptr_type (prev_base),
1941 : : base_offset + byte_offset);
1942 : 2037600 : base = build_fold_addr_expr (unshare_expr (base));
1943 : : }
1944 : :
1945 : 2191578 : unsigned int align_bound = known_alignment (misalign + offset);
1946 : 2191578 : if (align_bound != 0)
1947 : 1388564 : align = MIN (align, align_bound);
1948 : 2191578 : if (align != TYPE_ALIGN (exp_type))
1949 : 415759 : exp_type = build_aligned_type (exp_type, align);
1950 : :
1951 : 2191578 : mem_ref = fold_build2_loc (loc, MEM_REF, exp_type, base, off);
1952 : 2191578 : REF_REVERSE_STORAGE_ORDER (mem_ref) = reverse;
1953 : 2191578 : if (TREE_THIS_VOLATILE (prev_base))
1954 : 6 : TREE_THIS_VOLATILE (mem_ref) = 1;
1955 : 2191578 : if (TREE_SIDE_EFFECTS (prev_base))
1956 : 62 : TREE_SIDE_EFFECTS (mem_ref) = 1;
1957 : 2191578 : return mem_ref;
1958 : : }
1959 : :
1960 : : /* Construct and return a memory reference that is equal to a portion of
1961 : : MODEL->expr but is based on BASE. If this cannot be done, return NULL. */
1962 : :
1963 : : static tree
1964 : 1925587 : build_reconstructed_reference (location_t, tree base, struct access *model)
1965 : : {
1966 : 1925587 : tree expr = model->expr;
1967 : : /* We have to make sure to start just below the outermost union. */
1968 : 1925587 : tree start_expr = expr;
1969 : 3973889 : while (handled_component_p (expr))
1970 : : {
1971 : 2048302 : if (TREE_CODE (TREE_TYPE (TREE_OPERAND (expr, 0))) == UNION_TYPE)
1972 : 7783 : start_expr = expr;
1973 : 2048302 : expr = TREE_OPERAND (expr, 0);
1974 : : }
1975 : :
1976 : : expr = start_expr;
1977 : : tree prev_expr = NULL_TREE;
1978 : 3949460 : while (!types_compatible_p (TREE_TYPE (expr), TREE_TYPE (base)))
1979 : : {
1980 : 2195117 : if (!handled_component_p (expr))
1981 : : return NULL_TREE;
1982 : 2023873 : prev_expr = expr;
1983 : 2023873 : expr = TREE_OPERAND (expr, 0);
1984 : : }
1985 : :
1986 : : /* Guard against broken VIEW_CONVERT_EXPRs... */
1987 : 1754343 : if (!prev_expr)
1988 : : return NULL_TREE;
1989 : :
1990 : 1753364 : TREE_OPERAND (prev_expr, 0) = base;
1991 : 1753364 : tree ref = unshare_expr (model->expr);
1992 : 1753364 : TREE_OPERAND (prev_expr, 0) = expr;
1993 : 1753364 : return ref;
1994 : : }
1995 : :
1996 : : /* Construct a memory reference to a part of an aggregate BASE at the given
1997 : : OFFSET and of the same type as MODEL. In case this is a reference to a
1998 : : bit-field, the function will replicate the last component_ref of model's
1999 : : expr to access it. INSERT_AFTER and GSI have the same meaning as in
2000 : : build_ref_for_offset, furthermore, when GSI is NULL, the function expects
2001 : : that it re-builds the entire reference from a DECL to the final access and
2002 : : so will create a MEM_REF when OFFSET does not exactly match offset of
2003 : : MODEL. */
2004 : :
2005 : : static tree
2006 : 3888543 : build_ref_for_model (location_t loc, tree base, HOST_WIDE_INT offset,
2007 : : struct access *model, gimple_stmt_iterator *gsi,
2008 : : bool insert_after)
2009 : : {
2010 : 3888543 : gcc_assert (offset >= 0);
2011 : 3888543 : if (TREE_CODE (model->expr) == COMPONENT_REF
2012 : 3888543 : && DECL_BIT_FIELD (TREE_OPERAND (model->expr, 1)))
2013 : : {
2014 : : /* This access represents a bit-field. */
2015 : 64861 : tree t, exp_type, fld = TREE_OPERAND (model->expr, 1);
2016 : :
2017 : 64861 : offset -= int_bit_position (fld);
2018 : 64861 : exp_type = TREE_TYPE (TREE_OPERAND (model->expr, 0));
2019 : 64861 : t = build_ref_for_offset (loc, base, offset, model->reverse, exp_type,
2020 : : gsi, insert_after);
2021 : : /* The flag will be set on the record type. */
2022 : 64861 : REF_REVERSE_STORAGE_ORDER (t) = 0;
2023 : 64861 : return fold_build3_loc (loc, COMPONENT_REF, TREE_TYPE (fld), t, fld,
2024 : 64861 : NULL_TREE);
2025 : : }
2026 : : else
2027 : : {
2028 : 3823682 : tree res;
2029 : 3823682 : if (model->grp_same_access_path
2030 : 1925668 : && !TREE_THIS_VOLATILE (base)
2031 : 1925662 : && (TYPE_ADDR_SPACE (TREE_TYPE (base))
2032 : 1925662 : == TYPE_ADDR_SPACE (TREE_TYPE (model->expr)))
2033 : 1925660 : && (offset == model->offset
2034 : 9327 : || (gsi && offset <= model->offset))
2035 : : /* build_reconstructed_reference can still fail if we have already
2036 : : massaged BASE because of another type incompatibility. */
2037 : 5749269 : && (res = build_reconstructed_reference (loc, base, model)))
2038 : : return res;
2039 : : else
2040 : 2070318 : return build_ref_for_offset (loc, base, offset, model->reverse,
2041 : : model->type, gsi, insert_after);
2042 : : }
2043 : : }
2044 : :
2045 : : /* Attempt to build a memory reference that we could but into a gimple
2046 : : debug_bind statement. Similar to build_ref_for_model but punts if it has to
2047 : : create statements and return s NULL instead. This function also ignores
2048 : : alignment issues and so its results should never end up in non-debug
2049 : : statements. */
2050 : :
2051 : : static tree
2052 : 5475 : build_debug_ref_for_model (location_t loc, tree base, HOST_WIDE_INT offset,
2053 : : struct access *model)
2054 : : {
2055 : 5475 : poly_int64 base_offset;
2056 : 5475 : tree off;
2057 : :
2058 : 5475 : if (TREE_CODE (model->expr) == COMPONENT_REF
2059 : 5475 : && DECL_BIT_FIELD (TREE_OPERAND (model->expr, 1)))
2060 : : return NULL_TREE;
2061 : :
2062 : 5473 : base = get_addr_base_and_unit_offset (base, &base_offset);
2063 : 5473 : if (!base)
2064 : : return NULL_TREE;
2065 : 5473 : if (TREE_CODE (base) == MEM_REF)
2066 : : {
2067 : 256 : off = build_int_cst (TREE_TYPE (TREE_OPERAND (base, 1)),
2068 : 256 : base_offset + offset / BITS_PER_UNIT);
2069 : 256 : off = int_const_binop (PLUS_EXPR, TREE_OPERAND (base, 1), off);
2070 : 256 : base = unshare_expr (TREE_OPERAND (base, 0));
2071 : : }
2072 : : else
2073 : : {
2074 : 5217 : off = build_int_cst (reference_alias_ptr_type (base),
2075 : 5217 : base_offset + offset / BITS_PER_UNIT);
2076 : 5217 : base = build_fold_addr_expr (unshare_expr (base));
2077 : : }
2078 : :
2079 : 5473 : return fold_build2_loc (loc, MEM_REF, model->type, base, off);
2080 : : }
2081 : :
2082 : : /* Construct a memory reference consisting of component_refs and array_refs to
2083 : : a part of an aggregate *RES (which is of type TYPE). The requested part
2084 : : should have type EXP_TYPE at be the given OFFSET. This function might not
2085 : : succeed, it returns true when it does and only then *RES points to something
2086 : : meaningful. This function should be used only to build expressions that we
2087 : : might need to present to user (e.g. in warnings). In all other situations,
2088 : : build_ref_for_model or build_ref_for_offset should be used instead. */
2089 : :
2090 : : static bool
2091 : 2826182 : build_user_friendly_ref_for_offset (tree *res, tree type, HOST_WIDE_INT offset,
2092 : : tree exp_type)
2093 : : {
2094 : 2865138 : while (1)
2095 : : {
2096 : 2845660 : tree fld;
2097 : 2845660 : tree tr_size, index, minidx;
2098 : 2845660 : HOST_WIDE_INT el_size;
2099 : :
2100 : 2845660 : if (offset == 0 && exp_type
2101 : 2845660 : && types_compatible_p (exp_type, type))
2102 : : return true;
2103 : :
2104 : 1697472 : switch (TREE_CODE (type))
2105 : : {
2106 : 1666029 : case UNION_TYPE:
2107 : 1666029 : case QUAL_UNION_TYPE:
2108 : 1666029 : case RECORD_TYPE:
2109 : 17342173 : for (fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
2110 : : {
2111 : 17298520 : HOST_WIDE_INT pos, size;
2112 : 17298520 : tree tr_pos, expr, *expr_ptr;
2113 : :
2114 : 17298520 : if (TREE_CODE (fld) != FIELD_DECL)
2115 : 15629397 : continue;
2116 : :
2117 : 2646428 : tr_pos = bit_position (fld);
2118 : 2646428 : if (!tr_pos || !tree_fits_uhwi_p (tr_pos))
2119 : 0 : continue;
2120 : 2646428 : pos = tree_to_uhwi (tr_pos);
2121 : 2646428 : gcc_assert (TREE_CODE (type) == RECORD_TYPE || pos == 0);
2122 : 2646428 : tr_size = DECL_SIZE (fld);
2123 : 2646428 : if (!tr_size || !tree_fits_uhwi_p (tr_size))
2124 : 0 : continue;
2125 : 2646428 : size = tree_to_uhwi (tr_size);
2126 : 2646428 : if (size == 0)
2127 : : {
2128 : 53838 : if (pos != offset)
2129 : 22912 : continue;
2130 : : }
2131 : 2592590 : else if (pos > offset || (pos + size) <= offset)
2132 : 954393 : continue;
2133 : :
2134 : 1669123 : expr = build3 (COMPONENT_REF, TREE_TYPE (fld), *res, fld,
2135 : : NULL_TREE);
2136 : 1669123 : expr_ptr = &expr;
2137 : 1669123 : if (build_user_friendly_ref_for_offset (expr_ptr, TREE_TYPE (fld),
2138 : : offset - pos, exp_type))
2139 : : {
2140 : 1622376 : *res = expr;
2141 : 1622376 : return true;
2142 : : }
2143 : : }
2144 : : return false;
2145 : :
2146 : 19478 : case ARRAY_TYPE:
2147 : 19478 : tr_size = TYPE_SIZE (TREE_TYPE (type));
2148 : 19478 : if (!tr_size || !tree_fits_uhwi_p (tr_size))
2149 : : return false;
2150 : 19478 : el_size = tree_to_uhwi (tr_size);
2151 : :
2152 : 19478 : minidx = TYPE_MIN_VALUE (TYPE_DOMAIN (type));
2153 : 19478 : if (TREE_CODE (minidx) != INTEGER_CST || el_size == 0)
2154 : : return false;
2155 : 19478 : index = build_int_cst (TYPE_DOMAIN (type), offset / el_size);
2156 : 19478 : if (!integer_zerop (minidx))
2157 : 557 : index = int_const_binop (PLUS_EXPR, index, minidx);
2158 : 19478 : *res = build4 (ARRAY_REF, TREE_TYPE (type), *res, index,
2159 : : NULL_TREE, NULL_TREE);
2160 : 19478 : offset = offset % el_size;
2161 : 19478 : type = TREE_TYPE (type);
2162 : 19478 : break;
2163 : :
2164 : 11965 : default:
2165 : 11965 : if (offset != 0)
2166 : : return false;
2167 : :
2168 : 11842 : if (exp_type)
2169 : : return false;
2170 : : else
2171 : : return true;
2172 : : }
2173 : 19478 : }
2174 : : }
2175 : :
2176 : : /* Print message to dump file why a variable was rejected. */
2177 : :
2178 : : static void
2179 : 14761456 : reject (tree var, const char *msg)
2180 : : {
2181 : 14761456 : if (dump_file && (dump_flags & TDF_DETAILS))
2182 : : {
2183 : 28 : fprintf (dump_file, "Rejected (%d): %s: ", DECL_UID (var), msg);
2184 : 28 : print_generic_expr (dump_file, var);
2185 : 28 : fprintf (dump_file, "\n");
2186 : : }
2187 : 14761456 : }
2188 : :
2189 : : /* Return true if VAR is a candidate for SRA. */
2190 : :
2191 : : static bool
2192 : 19181109 : maybe_add_sra_candidate (tree var)
2193 : : {
2194 : 19181109 : tree type = TREE_TYPE (var);
2195 : 19181109 : const char *msg;
2196 : 19181109 : tree_node **slot;
2197 : :
2198 : 19181109 : if (!AGGREGATE_TYPE_P (type))
2199 : : {
2200 : 13222980 : reject (var, "not aggregate");
2201 : 13222980 : return false;
2202 : : }
2203 : :
2204 : 5958129 : if ((is_global_var (var)
2205 : : /* There are cases where non-addressable variables fail the
2206 : : pt_solutions_check test, e.g in gcc.dg/uninit-40.c. */
2207 : 5743012 : || (TREE_ADDRESSABLE (var)
2208 : 1524043 : && pt_solution_includes (&cfun->gimple_df->escaped_return, var))
2209 : 4423741 : || (TREE_CODE (var) == RESULT_DECL
2210 : 0 : && !DECL_BY_REFERENCE (var)
2211 : 0 : && aggregate_value_p (var, current_function_decl)))
2212 : : /* Allow constant-pool entries that "need to live in memory". */
2213 : 7277400 : && !constant_decl_p (var))
2214 : : {
2215 : 1531969 : reject (var, "needs to live in memory and escapes or global");
2216 : 1531969 : return false;
2217 : : }
2218 : 4426160 : if (TREE_THIS_VOLATILE (var))
2219 : : {
2220 : 564 : reject (var, "is volatile");
2221 : 564 : return false;
2222 : : }
2223 : 4425596 : if (!COMPLETE_TYPE_P (type))
2224 : : {
2225 : 0 : reject (var, "has incomplete type");
2226 : 0 : return false;
2227 : : }
2228 : 4425596 : if (!tree_fits_shwi_p (TYPE_SIZE (type)))
2229 : : {
2230 : 44 : reject (var, "type size not fixed");
2231 : 44 : return false;
2232 : : }
2233 : 4425552 : if (tree_to_shwi (TYPE_SIZE (type)) == 0)
2234 : : {
2235 : 5670 : reject (var, "type size is zero");
2236 : 5670 : return false;
2237 : : }
2238 : 4419882 : if (type_internals_preclude_sra_p (type, &msg))
2239 : : {
2240 : 229 : reject (var, msg);
2241 : 229 : return false;
2242 : : }
2243 : 4419653 : if (/* Fix for PR 41089. tree-stdarg.cc needs to have va_lists intact but
2244 : : we also want to schedule it rather late. Thus we ignore it in
2245 : : the early pass. */
2246 : 4419653 : (sra_mode == SRA_MODE_EARLY_INTRA
2247 : 4419653 : && is_va_list_type (type)))
2248 : : {
2249 : 0 : reject (var, "is va_list");
2250 : 0 : return false;
2251 : : }
2252 : :
2253 : 4419653 : bitmap_set_bit (candidate_bitmap, DECL_UID (var));
2254 : 4419653 : slot = candidates->find_slot_with_hash (var, DECL_UID (var), INSERT);
2255 : 4419653 : *slot = var;
2256 : :
2257 : 4419653 : if (dump_file && (dump_flags & TDF_DETAILS))
2258 : : {
2259 : 14 : fprintf (dump_file, "Candidate (%d): ", DECL_UID (var));
2260 : 14 : print_generic_expr (dump_file, var);
2261 : 14 : fprintf (dump_file, "\n");
2262 : : }
2263 : :
2264 : : return true;
2265 : : }
2266 : :
2267 : : /* The very first phase of intraprocedural SRA. It marks in candidate_bitmap
2268 : : those with type which is suitable for scalarization. */
2269 : :
2270 : : static bool
2271 : 3458716 : find_var_candidates (void)
2272 : : {
2273 : 3458716 : tree var, parm;
2274 : 3458716 : unsigned int i;
2275 : 3458716 : bool ret = false;
2276 : :
2277 : 3458716 : for (parm = DECL_ARGUMENTS (current_function_decl);
2278 : 10730442 : parm;
2279 : 7271726 : parm = DECL_CHAIN (parm))
2280 : 7271726 : ret |= maybe_add_sra_candidate (parm);
2281 : :
2282 : 18340725 : FOR_EACH_LOCAL_DECL (cfun, i, var)
2283 : : {
2284 : 11905820 : if (!VAR_P (var))
2285 : 0 : continue;
2286 : :
2287 : 11905820 : ret |= maybe_add_sra_candidate (var);
2288 : : }
2289 : :
2290 : 3458716 : return ret;
2291 : : }
2292 : :
2293 : : /* Return true if EXP is a reference chain of COMPONENT_REFs and AREAY_REFs
2294 : : ending either with a DECL or a MEM_REF with zero offset. */
2295 : :
2296 : : static bool
2297 : 8766926 : path_comparable_for_same_access (tree expr)
2298 : : {
2299 : 14667556 : while (handled_component_p (expr))
2300 : : {
2301 : 6028716 : if (TREE_CODE (expr) == ARRAY_REF)
2302 : : {
2303 : : /* SSA name indices can occur here too when the array is of sie one.
2304 : : But we cannot just re-use array_refs with SSA names elsewhere in
2305 : : the function, so disallow non-constant indices. TODO: Remove this
2306 : : limitation after teaching build_reconstructed_reference to replace
2307 : : the index with the index type lower bound. */
2308 : 628233 : if (TREE_CODE (TREE_OPERAND (expr, 1)) != INTEGER_CST)
2309 : : return false;
2310 : : }
2311 : 5900630 : expr = TREE_OPERAND (expr, 0);
2312 : : }
2313 : :
2314 : 8638840 : if (TREE_CODE (expr) == MEM_REF)
2315 : : {
2316 : 1028953 : if (!zerop (TREE_OPERAND (expr, 1)))
2317 : : return false;
2318 : : }
2319 : : else
2320 : 7609887 : gcc_assert (DECL_P (expr));
2321 : :
2322 : : return true;
2323 : : }
2324 : :
2325 : : /* Assuming that EXP1 consists of only COMPONENT_REFs and ARRAY_REFs, return
2326 : : true if the chain of these handled components are exactly the same as EXP2
2327 : : and the expression under them is the same DECL or an equivalent MEM_REF.
2328 : : The reference picked by compare_access_positions must go to EXP1. */
2329 : :
2330 : : static bool
2331 : 4708703 : same_access_path_p (tree exp1, tree exp2)
2332 : : {
2333 : 4708703 : if (TREE_CODE (exp1) != TREE_CODE (exp2))
2334 : : {
2335 : : /* Special case single-field structures loaded sometimes as the field
2336 : : and sometimes as the structure. If the field is of a scalar type,
2337 : : compare_access_positions will put it into exp1.
2338 : :
2339 : : TODO: The gimple register type condition can be removed if teach
2340 : : compare_access_positions to put inner types first. */
2341 : 529093 : if (is_gimple_reg_type (TREE_TYPE (exp1))
2342 : 507331 : && TREE_CODE (exp1) == COMPONENT_REF
2343 : 936228 : && (TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (exp1, 0)))
2344 : 407135 : == TYPE_MAIN_VARIANT (TREE_TYPE (exp2))))
2345 : 359431 : exp1 = TREE_OPERAND (exp1, 0);
2346 : : else
2347 : : return false;
2348 : : }
2349 : :
2350 : 4539041 : if (!operand_equal_p (exp1, exp2, OEP_ADDRESS_OF))
2351 : : return false;
2352 : :
2353 : : return true;
2354 : : }
2355 : :
2356 : : /* Return true when either T1 is a type that, when loaded into a register and
2357 : : stored back to memory will yield the same bits or when both T1 and T2 are
2358 : : compatible. */
2359 : :
2360 : : static bool
2361 : 5730993 : types_risk_mangled_binary_repr_p (tree t1, tree t2)
2362 : : {
2363 : 5730993 : if (mode_can_transfer_bits (TYPE_MODE (t1)))
2364 : : return false;
2365 : :
2366 : 4116 : return !types_compatible_p (t1, t2);
2367 : : }
2368 : :
2369 : : /* Sort all accesses for the given variable, check for partial overlaps and
2370 : : return NULL if there are any. If there are none, pick a representative for
2371 : : each combination of offset and size and create a linked list out of them.
2372 : : Return the pointer to the first representative and make sure it is the first
2373 : : one in the vector of accesses. */
2374 : :
2375 : : static struct access *
2376 : 4293832 : sort_and_splice_var_accesses (tree var)
2377 : : {
2378 : 4293832 : int i, j, access_count;
2379 : 4293832 : struct access *res, **prev_acc_ptr = &res;
2380 : 4293832 : vec<access_p> *access_vec;
2381 : 4293832 : bool first = true;
2382 : 4293832 : HOST_WIDE_INT low = -1, high = 0;
2383 : :
2384 : 4293832 : access_vec = get_base_access_vector (var);
2385 : 4293832 : if (!access_vec)
2386 : : return NULL;
2387 : 4253821 : access_count = access_vec->length ();
2388 : :
2389 : : /* Sort by <OFFSET, SIZE>. */
2390 : 4253821 : access_vec->qsort (compare_access_positions);
2391 : :
2392 : : i = 0;
2393 : 13814840 : while (i < access_count)
2394 : : {
2395 : 9565830 : struct access *access = (*access_vec)[i];
2396 : 9565830 : bool grp_write = access->write;
2397 : 9565830 : bool grp_read = !access->write;
2398 : 9565830 : bool grp_scalar_write = access->write
2399 : 9565830 : && is_gimple_reg_type (access->type);
2400 : 9565830 : bool grp_scalar_read = !access->write
2401 : 9565830 : && is_gimple_reg_type (access->type);
2402 : 9565830 : bool grp_assignment_read = access->grp_assignment_read;
2403 : 9565830 : bool grp_assignment_write = access->grp_assignment_write;
2404 : 9565830 : bool multiple_scalar_reads = false;
2405 : 9565830 : bool grp_partial_lhs = access->grp_partial_lhs;
2406 : 9565830 : bool first_scalar = is_gimple_reg_type (access->type);
2407 : 9565830 : bool unscalarizable_region = access->grp_unscalarizable_region;
2408 : 9565830 : bool grp_same_access_path = access->grp_same_access_path;
2409 : 9565830 : bool bf_non_full_precision
2410 : 9565830 : = (INTEGRAL_TYPE_P (access->type)
2411 : 3088011 : && TYPE_PRECISION (access->type) != access->size
2412 : 156493 : && TREE_CODE (access->expr) == COMPONENT_REF
2413 : 9628948 : && DECL_BIT_FIELD (TREE_OPERAND (access->expr, 1)));
2414 : :
2415 : 9565830 : if (first || access->offset >= high)
2416 : : {
2417 : 4679767 : first = false;
2418 : 4679767 : low = access->offset;
2419 : 4679767 : high = access->offset + access->size;
2420 : : }
2421 : 4886063 : else if (access->offset > low && access->offset + access->size > high)
2422 : : return NULL;
2423 : : else
2424 : 4885492 : gcc_assert (access->offset >= low
2425 : : && access->offset + access->size <= high);
2426 : :
2427 : 9565259 : if (INTEGRAL_TYPE_P (access->type)
2428 : 3087549 : && TYPE_PRECISION (access->type) != access->size
2429 : 9721338 : && bitmap_bit_p (passed_by_ref_in_call, DECL_UID (access->base)))
2430 : : {
2431 : : /* This can lead to performance regressions because we can generate
2432 : : excessive zero extensions. */
2433 : 4240 : if (dump_file && (dump_flags & TDF_DETAILS))
2434 : : {
2435 : 0 : fprintf (dump_file, "Won't scalarize ");
2436 : 0 : print_generic_expr (dump_file, access->base);
2437 : 0 : fprintf (dump_file, "(%d), it is passed by reference to a call "
2438 : : "and there are accesses with precision not covering "
2439 : 0 : "their type size.", DECL_UID (access->base));
2440 : : }
2441 : 4240 : return NULL;
2442 : : }
2443 : :
2444 : 9561019 : if (grp_same_access_path)
2445 : 8766926 : grp_same_access_path = path_comparable_for_same_access (access->expr);
2446 : :
2447 : 9561019 : j = i + 1;
2448 : 15081215 : while (j < access_count)
2449 : : {
2450 : 10832205 : struct access *ac2 = (*access_vec)[j];
2451 : 10832205 : if (ac2->offset != access->offset || ac2->size != access->size)
2452 : : break;
2453 : 5520196 : if (ac2->write)
2454 : : {
2455 : 1214646 : grp_write = true;
2456 : 1214646 : grp_scalar_write = (grp_scalar_write
2457 : 1214646 : || is_gimple_reg_type (ac2->type));
2458 : : }
2459 : : else
2460 : : {
2461 : 4305550 : grp_read = true;
2462 : 4305550 : if (is_gimple_reg_type (ac2->type))
2463 : : {
2464 : 1732631 : if (grp_scalar_read)
2465 : : multiple_scalar_reads = true;
2466 : : else
2467 : 378256 : grp_scalar_read = true;
2468 : : }
2469 : : }
2470 : 5520196 : grp_assignment_read |= ac2->grp_assignment_read;
2471 : 5520196 : grp_assignment_write |= ac2->grp_assignment_write;
2472 : 5520196 : grp_partial_lhs |= ac2->grp_partial_lhs;
2473 : 5520196 : unscalarizable_region |= ac2->grp_unscalarizable_region;
2474 : 5520196 : relink_to_new_repr (access, ac2);
2475 : :
2476 : : /* If there are both aggregate-type and scalar-type accesses with
2477 : : this combination of size and offset, the comparison function
2478 : : should have put the scalars first. */
2479 : 5520196 : gcc_assert (first_scalar || !is_gimple_reg_type (ac2->type));
2480 : : /* It also prefers integral types to non-integral. However, when the
2481 : : precision of the selected type does not span the entire area and
2482 : : should also be used for a non-integer (i.e. float), we must not
2483 : : let that happen. Normally analyze_access_subtree expands the type
2484 : : to cover the entire area but for bit-fields it doesn't. */
2485 : 5520196 : if (bf_non_full_precision && !INTEGRAL_TYPE_P (ac2->type))
2486 : : {
2487 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
2488 : : {
2489 : 0 : fprintf (dump_file, "Cannot scalarize the following access "
2490 : : "because insufficient precision integer type was "
2491 : : "selected.\n ");
2492 : 0 : dump_access (dump_file, access, false);
2493 : : }
2494 : : unscalarizable_region = true;
2495 : : }
2496 : 5520196 : else if (types_risk_mangled_binary_repr_p (access->type, ac2->type))
2497 : : {
2498 : 1635 : if (dump_file && (dump_flags & TDF_DETAILS))
2499 : : {
2500 : 0 : fprintf (dump_file, "Cannot scalarize the following access "
2501 : : "because data would be held in a mode which is not "
2502 : : "guaranteed to preserve all bits.\n ");
2503 : 0 : dump_access (dump_file, access, false);
2504 : : }
2505 : : unscalarizable_region = true;
2506 : : }
2507 : :
2508 : 5520196 : if (grp_same_access_path
2509 : 5520196 : && (!ac2->grp_same_access_path
2510 : 4708703 : || !same_access_path_p (access->expr, ac2->expr)))
2511 : : grp_same_access_path = false;
2512 : :
2513 : 5520196 : ac2->group_representative = access;
2514 : 5520196 : j++;
2515 : : }
2516 : :
2517 : 9561019 : i = j;
2518 : :
2519 : 9561019 : access->group_representative = access;
2520 : 9561019 : access->grp_write = grp_write;
2521 : 9561019 : access->grp_read = grp_read;
2522 : 9561019 : access->grp_scalar_read = grp_scalar_read;
2523 : 9561019 : access->grp_scalar_write = grp_scalar_write;
2524 : 9561019 : access->grp_assignment_read = grp_assignment_read;
2525 : 9561019 : access->grp_assignment_write = grp_assignment_write;
2526 : 9561019 : access->grp_hint = multiple_scalar_reads && !constant_decl_p (var);
2527 : 9561019 : access->grp_partial_lhs = grp_partial_lhs;
2528 : 9561019 : access->grp_unscalarizable_region = unscalarizable_region;
2529 : 9561019 : access->grp_same_access_path = grp_same_access_path;
2530 : :
2531 : 9561019 : *prev_acc_ptr = access;
2532 : 9561019 : prev_acc_ptr = &access->next_grp;
2533 : : }
2534 : :
2535 : 4249010 : gcc_assert (res == (*access_vec)[0]);
2536 : : return res;
2537 : : }
2538 : :
2539 : : /* Create a variable for the given ACCESS which determines the type, name and a
2540 : : few other properties. Return the variable declaration and store it also to
2541 : : ACCESS->replacement. REG_TREE is used when creating a declaration to base a
2542 : : default-definition SSA name on in order to facilitate an uninitialized
2543 : : warning. It is used instead of the actual ACCESS type if that is not of a
2544 : : gimple register type. */
2545 : :
2546 : : static tree
2547 : 3879620 : create_access_replacement (struct access *access, tree reg_type = NULL_TREE)
2548 : : {
2549 : 3879620 : tree repl;
2550 : :
2551 : 3879620 : tree type = access->type;
2552 : 3879620 : if (reg_type && !is_gimple_reg_type (type))
2553 : : type = reg_type;
2554 : :
2555 : 3879620 : if (access->grp_to_be_debug_replaced)
2556 : : {
2557 : 269452 : repl = create_tmp_var_raw (access->type);
2558 : 269452 : DECL_CONTEXT (repl) = current_function_decl;
2559 : : }
2560 : : else
2561 : : /* Drop any special alignment on the type if it's not on the main
2562 : : variant. This avoids issues with weirdo ABIs like AAPCS. */
2563 : 3610168 : repl = create_tmp_var (build_qualified_type (TYPE_MAIN_VARIANT (type),
2564 : 3610168 : TYPE_QUALS (type)), "SR");
2565 : 3879620 : if (access->grp_partial_lhs
2566 : 3879620 : && is_gimple_reg_type (type))
2567 : 666 : DECL_NOT_GIMPLE_REG_P (repl) = 1;
2568 : :
2569 : 3879620 : DECL_SOURCE_LOCATION (repl) = DECL_SOURCE_LOCATION (access->base);
2570 : 3879620 : DECL_ARTIFICIAL (repl) = 1;
2571 : 3879620 : DECL_IGNORED_P (repl) = DECL_IGNORED_P (access->base);
2572 : :
2573 : 3879620 : if (DECL_NAME (access->base)
2574 : 2786892 : && !DECL_IGNORED_P (access->base)
2575 : 4939881 : && !DECL_ARTIFICIAL (access->base))
2576 : : {
2577 : 1050779 : char *pretty_name = make_fancy_name (access->expr);
2578 : 1050779 : tree debug_expr = unshare_expr_without_location (access->expr), d;
2579 : 1050779 : bool fail = false;
2580 : :
2581 : 1050779 : DECL_NAME (repl) = get_identifier (pretty_name);
2582 : 1050779 : DECL_NAMELESS (repl) = 1;
2583 : 1050779 : obstack_free (&name_obstack, pretty_name);
2584 : :
2585 : : /* Get rid of any SSA_NAMEs embedded in debug_expr,
2586 : : as DECL_DEBUG_EXPR isn't considered when looking for still
2587 : : used SSA_NAMEs and thus they could be freed. All debug info
2588 : : generation cares is whether something is constant or variable
2589 : : and that get_ref_base_and_extent works properly on the
2590 : : expression. It cannot handle accesses at a non-constant offset
2591 : : though, so just give up in those cases. */
2592 : 1352607 : for (d = debug_expr;
2593 : 3674866 : !fail && (handled_component_p (d) || TREE_CODE (d) == MEM_REF);
2594 : 1352607 : d = TREE_OPERAND (d, 0))
2595 : 1352607 : switch (TREE_CODE (d))
2596 : : {
2597 : 53206 : case ARRAY_REF:
2598 : 53206 : case ARRAY_RANGE_REF:
2599 : 53206 : if (TREE_OPERAND (d, 1)
2600 : 53206 : && TREE_CODE (TREE_OPERAND (d, 1)) != INTEGER_CST)
2601 : : fail = true;
2602 : 53206 : if (TREE_OPERAND (d, 3)
2603 : 53206 : && TREE_CODE (TREE_OPERAND (d, 3)) != INTEGER_CST)
2604 : : fail = true;
2605 : : /* FALLTHRU */
2606 : 1129600 : case COMPONENT_REF:
2607 : 1129600 : if (TREE_OPERAND (d, 2)
2608 : 1129600 : && TREE_CODE (TREE_OPERAND (d, 2)) != INTEGER_CST)
2609 : : fail = true;
2610 : : break;
2611 : 220822 : case MEM_REF:
2612 : 220822 : if (TREE_CODE (TREE_OPERAND (d, 0)) != ADDR_EXPR)
2613 : : fail = true;
2614 : : else
2615 : 220822 : d = TREE_OPERAND (d, 0);
2616 : : break;
2617 : : default:
2618 : : break;
2619 : : }
2620 : 1050779 : if (!fail)
2621 : : {
2622 : 1050658 : SET_DECL_DEBUG_EXPR (repl, debug_expr);
2623 : 1050658 : DECL_HAS_DEBUG_EXPR_P (repl) = 1;
2624 : : }
2625 : 1050779 : if (access->grp_no_warning)
2626 : 542 : suppress_warning (repl /* Be more selective! */);
2627 : : else
2628 : 1050237 : copy_warning (repl, access->base);
2629 : : }
2630 : : else
2631 : 2828841 : suppress_warning (repl /* Be more selective! */);
2632 : :
2633 : 3879620 : if (dump_file)
2634 : : {
2635 : 140 : if (access->grp_to_be_debug_replaced)
2636 : : {
2637 : 4 : fprintf (dump_file, "Created a debug-only replacement for ");
2638 : 4 : print_generic_expr (dump_file, access->base);
2639 : 4 : fprintf (dump_file, " offset: %u, size: %u\n",
2640 : 4 : (unsigned) access->offset, (unsigned) access->size);
2641 : : }
2642 : : else
2643 : : {
2644 : 136 : fprintf (dump_file, "Created a replacement for ");
2645 : 136 : print_generic_expr (dump_file, access->base);
2646 : 136 : fprintf (dump_file, " offset: %u, size: %u: ",
2647 : 136 : (unsigned) access->offset, (unsigned) access->size);
2648 : 136 : print_generic_expr (dump_file, repl, TDF_UID);
2649 : 136 : fprintf (dump_file, "\n");
2650 : : }
2651 : : }
2652 : 3879620 : sra_stats.replacements++;
2653 : :
2654 : 3879620 : return repl;
2655 : : }
2656 : :
2657 : : /* Return ACCESS scalar replacement, which must exist. */
2658 : :
2659 : : static inline tree
2660 : 13355425 : get_access_replacement (struct access *access)
2661 : : {
2662 : 13355425 : gcc_checking_assert (access->replacement_decl);
2663 : 13355425 : return access->replacement_decl;
2664 : : }
2665 : :
2666 : :
2667 : : /* Build a subtree of accesses rooted in *ACCESS, and move the pointer in the
2668 : : linked list along the way. Stop when *ACCESS is NULL or the access pointed
2669 : : to it is not "within" the root. Return false iff some accesses partially
2670 : : overlap. */
2671 : :
2672 : : static bool
2673 : 9541829 : build_access_subtree (struct access **access)
2674 : : {
2675 : 9541829 : struct access *root = *access, *last_child = NULL;
2676 : 9541829 : HOST_WIDE_INT limit = root->offset + root->size;
2677 : :
2678 : 9541829 : *access = (*access)->next_grp;
2679 : 14405861 : while (*access && (*access)->offset + (*access)->size <= limit)
2680 : : {
2681 : 4866890 : if (!last_child)
2682 : 1964811 : root->first_child = *access;
2683 : : else
2684 : 2902079 : last_child->next_sibling = *access;
2685 : 4866890 : last_child = *access;
2686 : 4866890 : (*access)->parent = root;
2687 : 4866890 : (*access)->grp_write |= root->grp_write;
2688 : :
2689 : 4866890 : if (!build_access_subtree (access))
2690 : : return false;
2691 : : }
2692 : :
2693 : 9538971 : if (*access && (*access)->offset < limit)
2694 : : return false;
2695 : :
2696 : : return true;
2697 : : }
2698 : :
2699 : : /* Build a tree of access representatives, ACCESS is the pointer to the first
2700 : : one, others are linked in a list by the next_grp field. Return false iff
2701 : : some accesses partially overlap. */
2702 : :
2703 : : static bool
2704 : 4249010 : build_access_trees (struct access *access)
2705 : : {
2706 : 8921233 : while (access)
2707 : : {
2708 : 4674939 : struct access *root = access;
2709 : :
2710 : 4674939 : if (!build_access_subtree (&access))
2711 : : return false;
2712 : 4672223 : root->next_grp = access;
2713 : : }
2714 : : return true;
2715 : : }
2716 : :
2717 : : /* Traverse the access forest where ROOT is the first root and verify that
2718 : : various important invariants hold true. */
2719 : :
2720 : : DEBUG_FUNCTION void
2721 : 4246294 : verify_sra_access_forest (struct access *root)
2722 : : {
2723 : 4246294 : struct access *access = root;
2724 : 4246294 : tree first_base = root->base;
2725 : 4246294 : gcc_assert (DECL_P (first_base));
2726 : 10954037 : do
2727 : : {
2728 : 10954037 : gcc_assert (access->base == first_base);
2729 : 10954037 : if (access->parent)
2730 : 6281829 : gcc_assert (access->offset >= access->parent->offset
2731 : : && access->size <= access->parent->size);
2732 : 10954037 : if (access->next_sibling)
2733 : 3671120 : gcc_assert (access->next_sibling->offset
2734 : : >= access->offset + access->size);
2735 : :
2736 : 10954037 : poly_int64 poffset, psize, pmax_size;
2737 : 10954037 : bool reverse;
2738 : 10954037 : tree base = get_ref_base_and_extent (access->expr, &poffset, &psize,
2739 : : &pmax_size, &reverse);
2740 : 10954037 : HOST_WIDE_INT offset, size, max_size;
2741 : 10954037 : if (!poffset.is_constant (&offset)
2742 : 10954037 : || !psize.is_constant (&size)
2743 : 10954037 : || !pmax_size.is_constant (&max_size))
2744 : : gcc_unreachable ();
2745 : 10954037 : gcc_assert (base == first_base);
2746 : 10954037 : gcc_assert (offset == access->offset);
2747 : 10954037 : gcc_assert (access->grp_unscalarizable_region
2748 : : || access->grp_total_scalarization
2749 : : || size == max_size);
2750 : 10954037 : gcc_assert (access->grp_unscalarizable_region
2751 : : || !is_gimple_reg_type (access->type)
2752 : : || size == access->size);
2753 : 10954037 : gcc_assert (reverse == access->reverse);
2754 : :
2755 : 10954037 : if (access->first_child)
2756 : : {
2757 : 2610709 : gcc_assert (access->first_child->parent == access);
2758 : : access = access->first_child;
2759 : : }
2760 : 8343328 : else if (access->next_sibling)
2761 : : {
2762 : 3516086 : gcc_assert (access->next_sibling->parent == access->parent);
2763 : : access = access->next_sibling;
2764 : : }
2765 : : else
2766 : : {
2767 : 7437951 : while (access->parent && !access->next_sibling)
2768 : : access = access->parent;
2769 : 4827242 : if (access->next_sibling)
2770 : : access = access->next_sibling;
2771 : : else
2772 : : {
2773 : 4672208 : gcc_assert (access == root);
2774 : 4672208 : root = root->next_grp;
2775 : 4672208 : access = root;
2776 : : }
2777 : : }
2778 : : }
2779 : 10954037 : while (access);
2780 : 4246294 : }
2781 : :
2782 : : /* Verify access forests of all candidates with accesses by calling
2783 : : verify_access_forest on each on them. */
2784 : :
2785 : : DEBUG_FUNCTION void
2786 : 738647 : verify_all_sra_access_forests (void)
2787 : : {
2788 : 738647 : bitmap_iterator bi;
2789 : 738647 : unsigned i;
2790 : 4984941 : EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
2791 : : {
2792 : 4246294 : tree var = candidate (i);
2793 : 4246294 : struct access *access = get_first_repr_for_decl (var);
2794 : 4246294 : if (access)
2795 : : {
2796 : 4246294 : gcc_assert (access->base == var);
2797 : 4246294 : verify_sra_access_forest (access);
2798 : : }
2799 : : }
2800 : 738647 : }
2801 : :
2802 : : /* Return true if expr contains some ARRAY_REFs into a variable bounded
2803 : : array. */
2804 : :
2805 : : static bool
2806 : 10668617 : expr_with_var_bounded_array_refs_p (tree expr)
2807 : : {
2808 : 19541914 : while (handled_component_p (expr))
2809 : : {
2810 : 8873297 : if (TREE_CODE (expr) == ARRAY_REF
2811 : 8873297 : && !tree_fits_shwi_p (array_ref_low_bound (expr)))
2812 : : return true;
2813 : 8873297 : expr = TREE_OPERAND (expr, 0);
2814 : : }
2815 : : return false;
2816 : : }
2817 : :
2818 : : /* Analyze the subtree of accesses rooted in ROOT, scheduling replacements when
2819 : : both seeming beneficial and when ALLOW_REPLACEMENTS allows it. If TOTALLY
2820 : : is set, we are totally scalarizing the aggregate. Also set all sorts of
2821 : : access flags appropriately along the way, notably always set grp_read and
2822 : : grp_assign_read according to MARK_READ and grp_write when MARK_WRITE is
2823 : : true.
2824 : :
2825 : : Creating a replacement for a scalar access is considered beneficial if its
2826 : : grp_hint ot TOTALLY is set (this means either that there is more than one
2827 : : direct read access or that we are attempting total scalarization) or
2828 : : according to the following table:
2829 : :
2830 : : Access written to through a scalar type (once or more times)
2831 : : |
2832 : : | Written to in an assignment statement
2833 : : | |
2834 : : | | Access read as scalar _once_
2835 : : | | |
2836 : : | | | Read in an assignment statement
2837 : : | | | |
2838 : : | | | | Scalarize Comment
2839 : : -----------------------------------------------------------------------------
2840 : : 0 0 0 0 No access for the scalar
2841 : : 0 0 0 1 No access for the scalar
2842 : : 0 0 1 0 No Single read - won't help
2843 : : 0 0 1 1 No The same case
2844 : : 0 1 0 0 No access for the scalar
2845 : : 0 1 0 1 No access for the scalar
2846 : : 0 1 1 0 Yes s = *g; return s.i;
2847 : : 0 1 1 1 Yes The same case as above
2848 : : 1 0 0 0 No Won't help
2849 : : 1 0 0 1 Yes s.i = 1; *g = s;
2850 : : 1 0 1 0 Yes s.i = 5; g = s.i;
2851 : : 1 0 1 1 Yes The same case as above
2852 : : 1 1 0 0 No Won't help.
2853 : : 1 1 0 1 Yes s.i = 1; *g = s;
2854 : : 1 1 1 0 Yes s = *g; return s.i;
2855 : : 1 1 1 1 Yes Any of the above yeses */
2856 : :
2857 : : static bool
2858 : 10954037 : analyze_access_subtree (struct access *root, struct access *parent,
2859 : : bool allow_replacements, bool totally)
2860 : : {
2861 : 10954037 : struct access *child;
2862 : 10954037 : HOST_WIDE_INT limit = root->offset + root->size;
2863 : 10954037 : HOST_WIDE_INT covered_to = root->offset;
2864 : 10954037 : bool scalar = is_gimple_reg_type (root->type);
2865 : 10954037 : bool hole = false, sth_created = false;
2866 : :
2867 : 10954037 : if (parent)
2868 : : {
2869 : 6281829 : if (parent->grp_read)
2870 : 5129975 : root->grp_read = 1;
2871 : 6281829 : if (parent->grp_assignment_read)
2872 : 2312844 : root->grp_assignment_read = 1;
2873 : 6281829 : if (parent->grp_write)
2874 : 3326469 : root->grp_write = 1;
2875 : 6281829 : if (parent->grp_assignment_write)
2876 : 2809301 : root->grp_assignment_write = 1;
2877 : 6281829 : if (!parent->grp_same_access_path)
2878 : 528214 : root->grp_same_access_path = 0;
2879 : : }
2880 : :
2881 : 10954037 : if (root->grp_unscalarizable_region)
2882 : : allow_replacements = false;
2883 : :
2884 : 10827462 : if (allow_replacements && expr_with_var_bounded_array_refs_p (root->expr))
2885 : : allow_replacements = false;
2886 : :
2887 : 10954037 : if (!totally && root->grp_result_of_prop_from_lhs)
2888 : 10954037 : allow_replacements = false;
2889 : :
2890 : 17235866 : for (child = root->first_child; child; child = child->next_sibling)
2891 : : {
2892 : 6281829 : hole |= covered_to < child->offset;
2893 : 6281829 : sth_created |= analyze_access_subtree (child, root,
2894 : 6281829 : allow_replacements && !scalar
2895 : 6281829 : && !root->grp_partial_lhs,
2896 : : totally);
2897 : :
2898 : 6281829 : root->grp_unscalarized_data |= child->grp_unscalarized_data;
2899 : 6281829 : if (child->grp_covered)
2900 : 3199213 : covered_to += child->size;
2901 : : else
2902 : : hole = true;
2903 : : }
2904 : :
2905 : 10954037 : if (allow_replacements && scalar && !root->first_child
2906 : 6813913 : && (totally || !root->grp_total_scalarization)
2907 : : && (totally
2908 : 5019754 : || root->grp_hint
2909 : 4317040 : || ((root->grp_scalar_read || root->grp_assignment_read)
2910 : 1470711 : && (root->grp_scalar_write || root->grp_assignment_write))))
2911 : : {
2912 : : /* Always create access replacements that cover the whole access.
2913 : : For integral types this means the precision has to match.
2914 : : Avoid assumptions based on the integral type kind, too. */
2915 : 3609660 : if (INTEGRAL_TYPE_P (root->type)
2916 : 1595388 : && ((TREE_CODE (root->type) != INTEGER_TYPE
2917 : 1595388 : && TREE_CODE (root->type) != BITINT_TYPE)
2918 : 1538707 : || TYPE_PRECISION (root->type) != root->size)
2919 : : /* But leave bitfield accesses alone. */
2920 : 3666348 : && (TREE_CODE (root->expr) != COMPONENT_REF
2921 : 55816 : || !DECL_BIT_FIELD (TREE_OPERAND (root->expr, 1))))
2922 : : {
2923 : 56399 : tree rt = root->type;
2924 : 56399 : gcc_assert ((root->offset % BITS_PER_UNIT) == 0
2925 : : && (root->size % BITS_PER_UNIT) == 0);
2926 : 56399 : if (TREE_CODE (root->type) == BITINT_TYPE)
2927 : 7 : root->type = build_bitint_type (root->size, TYPE_UNSIGNED (rt));
2928 : : else
2929 : 56392 : root->type = build_nonstandard_integer_type (root->size,
2930 : 56392 : TYPE_UNSIGNED (rt));
2931 : 112798 : root->expr = build_ref_for_offset (UNKNOWN_LOCATION, root->base,
2932 : 56399 : root->offset, root->reverse,
2933 : : root->type, NULL, false);
2934 : :
2935 : 56399 : if (dump_file && (dump_flags & TDF_DETAILS))
2936 : : {
2937 : 0 : fprintf (dump_file, "Changing the type of a replacement for ");
2938 : 0 : print_generic_expr (dump_file, root->base);
2939 : 0 : fprintf (dump_file, " offset: %u, size: %u ",
2940 : 0 : (unsigned) root->offset, (unsigned) root->size);
2941 : 0 : fprintf (dump_file, " to an integer.\n");
2942 : : }
2943 : : }
2944 : :
2945 : 3609660 : root->grp_to_be_replaced = 1;
2946 : 3609660 : root->replacement_decl = create_access_replacement (root);
2947 : 3609660 : sth_created = true;
2948 : 3609660 : hole = false;
2949 : 3609660 : }
2950 : : else
2951 : : {
2952 : 7344377 : if (allow_replacements
2953 : 3212601 : && scalar && !root->first_child
2954 : 3204253 : && !root->grp_total_scalarization
2955 : 3204147 : && (root->grp_scalar_write || root->grp_assignment_write)
2956 : 10190706 : && !bitmap_bit_p (cannot_scalarize_away_bitmap,
2957 : 2846329 : DECL_UID (root->base)))
2958 : : {
2959 : 483797 : gcc_checking_assert (!root->grp_scalar_read
2960 : : && !root->grp_assignment_read);
2961 : 483797 : sth_created = true;
2962 : 483797 : if (MAY_HAVE_DEBUG_BIND_STMTS)
2963 : : {
2964 : 269452 : root->grp_to_be_debug_replaced = 1;
2965 : 269452 : root->replacement_decl = create_access_replacement (root);
2966 : : }
2967 : : }
2968 : :
2969 : 7344377 : if (covered_to < limit)
2970 : 6191833 : hole = true;
2971 : 7344377 : if (scalar || !allow_replacements)
2972 : 3656812 : root->grp_total_scalarization = 0;
2973 : : }
2974 : :
2975 : 10954037 : if (!hole || totally)
2976 : 4831517 : root->grp_covered = 1;
2977 : 6122520 : else if (root->grp_write || comes_initialized_p (root->base))
2978 : 5320256 : root->grp_unscalarized_data = 1; /* not covered and written to */
2979 : 10954037 : return sth_created;
2980 : : }
2981 : :
2982 : : /* Analyze all access trees linked by next_grp by the means of
2983 : : analyze_access_subtree. */
2984 : : static bool
2985 : 4246294 : analyze_access_trees (struct access *access)
2986 : : {
2987 : 4246294 : bool ret = false;
2988 : :
2989 : 8918502 : while (access)
2990 : : {
2991 : 4672208 : if (analyze_access_subtree (access, NULL, true,
2992 : 4672208 : access->grp_total_scalarization))
2993 : 2212532 : ret = true;
2994 : 4672208 : access = access->next_grp;
2995 : : }
2996 : :
2997 : 4246294 : return ret;
2998 : : }
2999 : :
3000 : : /* Return true iff a potential new child of ACC at offset OFFSET and with size
3001 : : SIZE would conflict with an already existing one. If exactly such a child
3002 : : already exists in ACC, store a pointer to it in EXACT_MATCH. */
3003 : :
3004 : : static bool
3005 : 6320998 : child_would_conflict_in_acc (struct access *acc, HOST_WIDE_INT norm_offset,
3006 : : HOST_WIDE_INT size, struct access **exact_match)
3007 : : {
3008 : 6320998 : struct access *child;
3009 : :
3010 : 10812464 : for (child = acc->first_child; child; child = child->next_sibling)
3011 : : {
3012 : 9862163 : if (child->offset == norm_offset && child->size == size)
3013 : : {
3014 : 5329058 : *exact_match = child;
3015 : 5329058 : return true;
3016 : : }
3017 : :
3018 : 4533105 : if (child->offset < norm_offset + size
3019 : 4478239 : && child->offset + child->size > norm_offset)
3020 : : return true;
3021 : : }
3022 : :
3023 : : return false;
3024 : : }
3025 : :
3026 : : /* Create a new child access of PARENT, with all properties just like MODEL
3027 : : except for its offset and with its grp_write false and grp_read true.
3028 : : Return the new access or NULL if it cannot be created. Note that this
3029 : : access is created long after all splicing and sorting, it's not located in
3030 : : any access vector and is automatically a representative of its group. Set
3031 : : the gpr_write flag of the new accesss if SET_GRP_WRITE is true. */
3032 : :
3033 : : static struct access *
3034 : 946262 : create_artificial_child_access (struct access *parent, struct access *model,
3035 : : HOST_WIDE_INT new_offset,
3036 : : bool set_grp_read, bool set_grp_write)
3037 : : {
3038 : 946262 : struct access **child;
3039 : 946262 : tree expr = parent->base;
3040 : :
3041 : 946262 : gcc_assert (!model->grp_unscalarizable_region);
3042 : :
3043 : 946262 : struct access *access = access_pool.allocate ();
3044 : 946262 : memset (access, 0, sizeof (struct access));
3045 : 946262 : if (!build_user_friendly_ref_for_offset (&expr, TREE_TYPE (expr), new_offset,
3046 : : model->type))
3047 : : {
3048 : 8790 : access->grp_no_warning = true;
3049 : 8790 : expr = build_ref_for_model (EXPR_LOCATION (parent->base), parent->base,
3050 : : new_offset, model, NULL, false);
3051 : : }
3052 : :
3053 : 946262 : access->base = parent->base;
3054 : 946262 : access->expr = expr;
3055 : 946262 : access->offset = new_offset;
3056 : 946262 : access->size = model->size;
3057 : 946262 : access->type = model->type;
3058 : 946262 : access->parent = parent;
3059 : 946262 : access->grp_read = set_grp_read;
3060 : 946262 : access->grp_write = set_grp_write;
3061 : 946262 : access->reverse = model->reverse;
3062 : :
3063 : 946262 : child = &parent->first_child;
3064 : 1632254 : while (*child && (*child)->offset < new_offset)
3065 : 685992 : child = &(*child)->next_sibling;
3066 : :
3067 : 946262 : access->next_sibling = *child;
3068 : 946262 : *child = access;
3069 : :
3070 : 946262 : return access;
3071 : : }
3072 : :
3073 : :
3074 : : /* Beginning with ACCESS, traverse its whole access subtree and mark all
3075 : : sub-trees as written to. If any of them has not been marked so previously
3076 : : and has assignment links leading from it, re-enqueue it. */
3077 : :
3078 : : static void
3079 : 1090519 : subtree_mark_written_and_rhs_enqueue (struct access *access)
3080 : : {
3081 : 1090519 : if (access->grp_write)
3082 : : return;
3083 : 1037067 : access->grp_write = true;
3084 : 1037067 : add_access_to_rhs_work_queue (access);
3085 : :
3086 : 1037067 : struct access *child;
3087 : 1258398 : for (child = access->first_child; child; child = child->next_sibling)
3088 : 221331 : subtree_mark_written_and_rhs_enqueue (child);
3089 : : }
3090 : :
3091 : : /* If there is still budget to create a propagation access for DECL, return
3092 : : true and decrement the budget. Otherwise return false. */
3093 : :
3094 : : static bool
3095 : 949856 : budget_for_propagation_access (tree decl)
3096 : : {
3097 : 949856 : unsigned b, *p = propagation_budget->get (decl);
3098 : 949856 : if (p)
3099 : 527229 : b = *p;
3100 : : else
3101 : 422627 : b = param_sra_max_propagations;
3102 : :
3103 : 949856 : if (b == 0)
3104 : : return false;
3105 : 946268 : b--;
3106 : :
3107 : 946268 : if (b == 0 && dump_file && (dump_flags & TDF_DETAILS))
3108 : : {
3109 : 0 : fprintf (dump_file, "The propagation budget of ");
3110 : 0 : print_generic_expr (dump_file, decl);
3111 : 0 : fprintf (dump_file, " (UID: %u) has been exhausted.\n", DECL_UID (decl));
3112 : : }
3113 : 946268 : propagation_budget->put (decl, b);
3114 : 946268 : return true;
3115 : : }
3116 : :
3117 : : /* Return true if ACC or any of its subaccesses has grp_child set. */
3118 : :
3119 : : static bool
3120 : 1731 : access_or_its_child_written (struct access *acc)
3121 : : {
3122 : 1731 : if (acc->grp_write)
3123 : : return true;
3124 : 1788 : for (struct access *sub = acc->first_child; sub; sub = sub->next_sibling)
3125 : 412 : if (access_or_its_child_written (sub))
3126 : : return true;
3127 : : return false;
3128 : : }
3129 : :
3130 : : /* Propagate subaccesses and grp_write flags of RACC across an assignment link
3131 : : to LACC. Enqueue sub-accesses as necessary so that the write flag is
3132 : : propagated transitively. Return true if anything changed. Additionally, if
3133 : : RACC is a scalar access but LACC is not, change the type of the latter, if
3134 : : possible. */
3135 : :
3136 : : static bool
3137 : 3905836 : propagate_subaccesses_from_rhs (struct access *lacc, struct access *racc)
3138 : : {
3139 : 3905836 : struct access *rchild;
3140 : 3905836 : HOST_WIDE_INT norm_delta = lacc->offset - racc->offset;
3141 : 3905836 : bool ret = false;
3142 : :
3143 : : /* IF the LHS is still not marked as being written to, we only need to do so
3144 : : if the RHS at this level actually was. */
3145 : 3905836 : if (!lacc->grp_write)
3146 : : {
3147 : 1329093 : gcc_checking_assert (!comes_initialized_p (racc->base));
3148 : 1329093 : if (racc->grp_write)
3149 : : {
3150 : 698107 : subtree_mark_written_and_rhs_enqueue (lacc);
3151 : 698107 : ret = true;
3152 : : }
3153 : : }
3154 : :
3155 : 3905836 : if (is_gimple_reg_type (lacc->type)
3156 : 2746131 : || lacc->grp_unscalarizable_region
3157 : 6651316 : || racc->grp_unscalarizable_region)
3158 : : {
3159 : 1163596 : if (!lacc->grp_write)
3160 : : {
3161 : 20405 : ret = true;
3162 : 20405 : subtree_mark_written_and_rhs_enqueue (lacc);
3163 : : }
3164 : 1163596 : return ret;
3165 : : }
3166 : :
3167 : 2742240 : if (is_gimple_reg_type (racc->type))
3168 : : {
3169 : 211291 : if (!lacc->grp_write)
3170 : : {
3171 : 1806 : ret = true;
3172 : 1806 : subtree_mark_written_and_rhs_enqueue (lacc);
3173 : : }
3174 : 211291 : if (!lacc->first_child
3175 : 211091 : && !racc->first_child
3176 : 422088 : && !types_risk_mangled_binary_repr_p (racc->type, lacc->type))
3177 : : {
3178 : : /* We are about to change the access type from aggregate to scalar,
3179 : : so we need to put the reverse flag onto the access, if any. */
3180 : 210797 : const bool reverse
3181 : 210797 : = TYPE_REVERSE_STORAGE_ORDER (lacc->type)
3182 : 1 : && !POINTER_TYPE_P (racc->type)
3183 : 210797 : && !VECTOR_TYPE_P (racc->type);
3184 : 210797 : tree t = lacc->base;
3185 : :
3186 : 210797 : lacc->type = racc->type;
3187 : 210797 : if (build_user_friendly_ref_for_offset (&t, TREE_TYPE (t),
3188 : : lacc->offset, racc->type))
3189 : : {
3190 : 210716 : lacc->expr = t;
3191 : 210716 : lacc->grp_same_access_path = true;
3192 : : }
3193 : : else
3194 : : {
3195 : 81 : lacc->expr = build_ref_for_model (EXPR_LOCATION (lacc->base),
3196 : : lacc->base, lacc->offset,
3197 : : racc, NULL, false);
3198 : 81 : if (TREE_CODE (lacc->expr) == MEM_REF)
3199 : 81 : REF_REVERSE_STORAGE_ORDER (lacc->expr) = reverse;
3200 : 81 : lacc->grp_no_warning = true;
3201 : 81 : lacc->grp_same_access_path = false;
3202 : : }
3203 : 210797 : lacc->reverse = reverse;
3204 : : }
3205 : 211291 : return ret;
3206 : : }
3207 : :
3208 : 5789457 : for (rchild = racc->first_child; rchild; rchild = rchild->next_sibling)
3209 : : {
3210 : 3258508 : struct access *new_acc = NULL;
3211 : 3258508 : HOST_WIDE_INT norm_offset = rchild->offset + norm_delta;
3212 : :
3213 : 3258508 : if (child_would_conflict_in_acc (lacc, norm_offset, rchild->size,
3214 : : &new_acc))
3215 : : {
3216 : 2623679 : if (new_acc)
3217 : : {
3218 : 2601231 : if (!new_acc->grp_write && rchild->grp_write)
3219 : : {
3220 : 140387 : gcc_assert (!lacc->grp_write);
3221 : 140387 : subtree_mark_written_and_rhs_enqueue (new_acc);
3222 : 140387 : ret = true;
3223 : : }
3224 : :
3225 : 2601231 : rchild->grp_hint = 1;
3226 : 2601231 : new_acc->grp_hint |= new_acc->grp_read;
3227 : 2601231 : if (rchild->first_child
3228 : 2601231 : && propagate_subaccesses_from_rhs (new_acc, rchild))
3229 : : {
3230 : 1166 : ret = 1;
3231 : 1166 : add_access_to_rhs_work_queue (new_acc);
3232 : : }
3233 : : }
3234 : : else
3235 : : {
3236 : 22448 : if (!lacc->grp_write)
3237 : : {
3238 : 7541 : ret = true;
3239 : 7541 : subtree_mark_written_and_rhs_enqueue (lacc);
3240 : : }
3241 : : }
3242 : 2627702 : continue;
3243 : : }
3244 : :
3245 : 638852 : if (rchild->grp_unscalarizable_region
3246 : 634829 : || !budget_for_propagation_access (lacc->base))
3247 : : {
3248 : 4023 : if (!lacc->grp_write && access_or_its_child_written (rchild))
3249 : : {
3250 : 327 : ret = true;
3251 : 327 : subtree_mark_written_and_rhs_enqueue (lacc);
3252 : : }
3253 : 4023 : continue;
3254 : : }
3255 : :
3256 : 630806 : rchild->grp_hint = 1;
3257 : : /* Because get_ref_base_and_extent always includes padding in size for
3258 : : accesses to DECLs but not necessarily for COMPONENT_REFs of the same
3259 : : type, we might be actually attempting to here to create a child of the
3260 : : same type as the parent. */
3261 : 630806 : if (!types_compatible_p (lacc->type, rchild->type))
3262 : 630806 : new_acc = create_artificial_child_access (lacc, rchild, norm_offset,
3263 : : false,
3264 : 630806 : (lacc->grp_write
3265 : 630806 : || rchild->grp_write));
3266 : : else
3267 : 0 : new_acc = lacc;
3268 : 630806 : gcc_checking_assert (new_acc);
3269 : 630806 : if (racc->first_child)
3270 : 630806 : propagate_subaccesses_from_rhs (new_acc, rchild);
3271 : :
3272 : 630806 : add_access_to_rhs_work_queue (lacc);
3273 : 630806 : ret = true;
3274 : : }
3275 : :
3276 : : return ret;
3277 : : }
3278 : :
3279 : : /* Propagate subaccesses of LACC across an assignment link to RACC if they
3280 : : should inhibit total scalarization of the corresponding area. No flags are
3281 : : being propagated in the process. Return true if anything changed. */
3282 : :
3283 : : static bool
3284 : 5329201 : propagate_subaccesses_from_lhs (struct access *lacc, struct access *racc)
3285 : : {
3286 : 5329201 : if (is_gimple_reg_type (racc->type)
3287 : 1944465 : || lacc->grp_unscalarizable_region
3288 : 7271490 : || racc->grp_unscalarizable_region)
3289 : : return false;
3290 : :
3291 : : /* TODO: Do we want set some new racc flag to stop potential total
3292 : : scalarization if lacc is a scalar access (and none fo the two have
3293 : : children)? */
3294 : :
3295 : 1942269 : bool ret = false;
3296 : 1942269 : HOST_WIDE_INT norm_delta = racc->offset - lacc->offset;
3297 : 1942269 : for (struct access *lchild = lacc->first_child;
3298 : 5008544 : lchild;
3299 : 3066275 : lchild = lchild->next_sibling)
3300 : : {
3301 : 3066275 : struct access *matching_acc = NULL;
3302 : 3066275 : HOST_WIDE_INT norm_offset = lchild->offset + norm_delta;
3303 : :
3304 : 5817088 : if (lchild->grp_unscalarizable_region
3305 : 3062490 : || child_would_conflict_in_acc (racc, norm_offset, lchild->size,
3306 : : &matching_acc)
3307 : 3381747 : || !budget_for_propagation_access (racc->base))
3308 : : {
3309 : 2750813 : if (matching_acc
3310 : 2750813 : && propagate_subaccesses_from_lhs (lchild, matching_acc))
3311 : 199 : add_access_to_lhs_work_queue (matching_acc);
3312 : 2750813 : continue;
3313 : : }
3314 : :
3315 : : /* Because get_ref_base_and_extent always includes padding in size for
3316 : : accesses to DECLs but not necessarily for COMPONENT_REFs of the same
3317 : : type, we might be actually attempting to here to create a child of the
3318 : : same type as the parent. */
3319 : 315462 : if (!types_compatible_p (racc->type, lchild->type))
3320 : : {
3321 : 315456 : struct access *new_acc
3322 : 315456 : = create_artificial_child_access (racc, lchild, norm_offset,
3323 : : true, false);
3324 : 315456 : new_acc->grp_result_of_prop_from_lhs = 1;
3325 : 315456 : propagate_subaccesses_from_lhs (lchild, new_acc);
3326 : : }
3327 : : else
3328 : 6 : propagate_subaccesses_from_lhs (lchild, racc);
3329 : 315462 : ret = true;
3330 : : }
3331 : : return ret;
3332 : : }
3333 : :
3334 : : /* Propagate all subaccesses across assignment links. */
3335 : :
3336 : : static void
3337 : 738647 : propagate_all_subaccesses (void)
3338 : : {
3339 : 738647 : propagation_budget = new hash_map<tree, unsigned>;
3340 : 2217907 : while (rhs_work_queue_head)
3341 : : {
3342 : 1479260 : struct access *racc = pop_access_from_rhs_work_queue ();
3343 : 1479260 : struct assign_link *link;
3344 : :
3345 : 1479260 : if (racc->group_representative)
3346 : 1478296 : racc= racc->group_representative;
3347 : 1479260 : gcc_assert (racc->first_rhs_link);
3348 : :
3349 : 4742621 : for (link = racc->first_rhs_link; link; link = link->next_rhs)
3350 : : {
3351 : 3263361 : struct access *lacc = link->lacc;
3352 : :
3353 : 3263361 : if (!bitmap_bit_p (candidate_bitmap, DECL_UID (lacc->base)))
3354 : 5674 : continue;
3355 : 3257687 : lacc = lacc->group_representative;
3356 : :
3357 : 3257687 : bool reque_parents = false;
3358 : 3257687 : if (!bitmap_bit_p (candidate_bitmap, DECL_UID (racc->base)))
3359 : : {
3360 : 2195 : if (!lacc->grp_write)
3361 : : {
3362 : 615 : subtree_mark_written_and_rhs_enqueue (lacc);
3363 : 615 : reque_parents = true;
3364 : : }
3365 : : }
3366 : 3255492 : else if (propagate_subaccesses_from_rhs (lacc, racc))
3367 : : reque_parents = true;
3368 : :
3369 : : if (reque_parents)
3370 : 1200536 : do
3371 : : {
3372 : 1200536 : add_access_to_rhs_work_queue (lacc);
3373 : 1200536 : lacc = lacc->parent;
3374 : : }
3375 : 1200536 : while (lacc);
3376 : : }
3377 : : }
3378 : :
3379 : 2059642 : while (lhs_work_queue_head)
3380 : : {
3381 : 1320995 : struct access *lacc = pop_access_from_lhs_work_queue ();
3382 : 1320995 : struct assign_link *link;
3383 : :
3384 : 1320995 : if (lacc->group_representative)
3385 : 1317570 : lacc = lacc->group_representative;
3386 : 1320995 : gcc_assert (lacc->first_lhs_link);
3387 : :
3388 : 1320995 : if (!bitmap_bit_p (candidate_bitmap, DECL_UID (lacc->base)))
3389 : 3751 : continue;
3390 : :
3391 : 3604016 : for (link = lacc->first_lhs_link; link; link = link->next_lhs)
3392 : : {
3393 : 2286772 : struct access *racc = link->racc;
3394 : :
3395 : 2286772 : if (racc->group_representative)
3396 : 2286230 : racc = racc->group_representative;
3397 : 2286772 : if (!bitmap_bit_p (candidate_bitmap, DECL_UID (racc->base)))
3398 : 860 : continue;
3399 : 2285912 : if (propagate_subaccesses_from_lhs (lacc, racc))
3400 : 165967 : add_access_to_lhs_work_queue (racc);
3401 : : }
3402 : : }
3403 : 1477294 : delete propagation_budget;
3404 : 738647 : }
3405 : :
3406 : : /* Return true if the forest beginning with ROOT does not contain
3407 : : unscalarizable regions or non-byte aligned accesses. */
3408 : :
3409 : : static bool
3410 : 822565 : can_totally_scalarize_forest_p (struct access *root)
3411 : : {
3412 : 822565 : struct access *access = root;
3413 : 2268179 : do
3414 : : {
3415 : 2268179 : if (access->grp_unscalarizable_region
3416 : 2265300 : || (access->offset % BITS_PER_UNIT) != 0
3417 : 2264925 : || (access->size % BITS_PER_UNIT) != 0
3418 : 4529222 : || (is_gimple_reg_type (access->type)
3419 : 1486814 : && access->first_child))
3420 : : return false;
3421 : :
3422 : 2260881 : if (access->first_child)
3423 : : access = access->first_child;
3424 : 1611960 : else if (access->next_sibling)
3425 : : access = access->next_sibling;
3426 : : else
3427 : : {
3428 : 1530688 : while (access->parent && !access->next_sibling)
3429 : : access = access->parent;
3430 : 886485 : if (access->next_sibling)
3431 : : access = access->next_sibling;
3432 : : else
3433 : : {
3434 : 836653 : gcc_assert (access == root);
3435 : 836653 : root = root->next_grp;
3436 : 836653 : access = root;
3437 : : }
3438 : : }
3439 : : }
3440 : 2260881 : while (access);
3441 : : return true;
3442 : : }
3443 : :
3444 : : /* Create and return an ACCESS in PARENT spanning from POS with SIZE, TYPE and
3445 : : reference EXPR for total scalarization purposes and mark it as such. Within
3446 : : the children of PARENT, link it in between PTR and NEXT_SIBLING. */
3447 : :
3448 : : static struct access *
3449 : 472648 : create_total_scalarization_access (struct access *parent, HOST_WIDE_INT pos,
3450 : : HOST_WIDE_INT size, tree type, tree expr,
3451 : : struct access **ptr,
3452 : : struct access *next_sibling)
3453 : : {
3454 : 472648 : struct access *access = access_pool.allocate ();
3455 : 472648 : memset (access, 0, sizeof (struct access));
3456 : 472648 : access->base = parent->base;
3457 : 472648 : access->offset = pos;
3458 : 472648 : access->size = size;
3459 : 472648 : access->expr = expr;
3460 : 472648 : access->type = type;
3461 : 472648 : access->parent = parent;
3462 : 472648 : access->grp_write = parent->grp_write;
3463 : 472648 : access->grp_total_scalarization = 1;
3464 : 472648 : access->grp_hint = 1;
3465 : 472648 : access->grp_same_access_path = 0;
3466 : 472648 : access->reverse = reverse_storage_order_for_component_p (expr);
3467 : :
3468 : 472648 : access->next_sibling = next_sibling;
3469 : 472648 : *ptr = access;
3470 : 472648 : return access;
3471 : : }
3472 : :
3473 : : /* Create and return an ACCESS in PARENT spanning from POS with SIZE, TYPE and
3474 : : reference EXPR for total scalarization purposes and mark it as such, link it
3475 : : at *PTR and reshape the tree so that those elements at *PTR and their
3476 : : siblings which fall within the part described by POS and SIZE are moved to
3477 : : be children of the new access. If a partial overlap is detected, return
3478 : : NULL. */
3479 : :
3480 : : static struct access *
3481 : 472648 : create_total_access_and_reshape (struct access *parent, HOST_WIDE_INT pos,
3482 : : HOST_WIDE_INT size, tree type, tree expr,
3483 : : struct access **ptr)
3484 : : {
3485 : 472648 : struct access **p = ptr;
3486 : :
3487 : 643029 : while (*p && (*p)->offset < pos + size)
3488 : : {
3489 : 170381 : if ((*p)->offset + (*p)->size > pos + size)
3490 : : return NULL;
3491 : 170381 : p = &(*p)->next_sibling;
3492 : : }
3493 : :
3494 : 472648 : struct access *next_child = *ptr;
3495 : 472648 : struct access *new_acc
3496 : 472648 : = create_total_scalarization_access (parent, pos, size, type, expr,
3497 : : ptr, *p);
3498 : 472648 : if (p != ptr)
3499 : : {
3500 : 70941 : new_acc->first_child = next_child;
3501 : 70941 : *p = NULL;
3502 : 241322 : for (struct access *a = next_child; a; a = a->next_sibling)
3503 : 170381 : a->parent = new_acc;
3504 : : }
3505 : : return new_acc;
3506 : : }
3507 : :
3508 : : static bool totally_scalarize_subtree (struct access *root);
3509 : :
3510 : : /* Return true if INNER is either the same type as OUTER or if it is the type
3511 : : of a record field in OUTER at offset zero, possibly in nested
3512 : : sub-records. */
3513 : :
3514 : : static bool
3515 : 178560 : access_and_field_type_match_p (tree outer, tree inner)
3516 : : {
3517 : 178560 : if (TYPE_MAIN_VARIANT (outer) == TYPE_MAIN_VARIANT (inner))
3518 : : return true;
3519 : 301 : if (TREE_CODE (outer) != RECORD_TYPE)
3520 : : return false;
3521 : 297 : tree fld = TYPE_FIELDS (outer);
3522 : 7107 : while (fld)
3523 : : {
3524 : 6977 : if (TREE_CODE (fld) == FIELD_DECL)
3525 : : {
3526 : 333 : if (!zerop (DECL_FIELD_OFFSET (fld)))
3527 : : return false;
3528 : 333 : if (TYPE_MAIN_VARIANT (TREE_TYPE (fld)) == inner)
3529 : : return true;
3530 : 194 : if (TREE_CODE (TREE_TYPE (fld)) == RECORD_TYPE)
3531 : 166 : fld = TYPE_FIELDS (TREE_TYPE (fld));
3532 : : else
3533 : : return false;
3534 : : }
3535 : : else
3536 : 6644 : fld = DECL_CHAIN (fld);
3537 : : }
3538 : : return false;
3539 : : }
3540 : :
3541 : : /* Return type of total_should_skip_creating_access indicating whether a total
3542 : : scalarization access for a field/element should be created, whether it
3543 : : already exists or whether the entire total scalarization has to fail. */
3544 : :
3545 : : enum total_sra_field_state {TOTAL_FLD_CREATE, TOTAL_FLD_DONE, TOTAL_FLD_FAILED};
3546 : :
3547 : : /* Do all the necessary steps in total scalarization when the given aggregate
3548 : : type has a TYPE at POS with the given SIZE should be put into PARENT and
3549 : : when we have processed all its siblings with smaller offsets up until and
3550 : : including LAST_SEEN_SIBLING (which can be NULL).
3551 : :
3552 : : If some further siblings are to be skipped, set *LAST_SEEN_SIBLING as
3553 : : appropriate. Return TOTAL_FLD_CREATE id the caller should carry on with
3554 : : creating a new access, TOTAL_FLD_DONE if access or accesses capable of
3555 : : representing the described part of the aggregate for the purposes of total
3556 : : scalarization already exist or TOTAL_FLD_FAILED if there is a problem which
3557 : : prevents total scalarization from happening at all. */
3558 : :
3559 : : static enum total_sra_field_state
3560 : 1854967 : total_should_skip_creating_access (struct access *parent,
3561 : : struct access **last_seen_sibling,
3562 : : tree type, HOST_WIDE_INT pos,
3563 : : HOST_WIDE_INT size)
3564 : : {
3565 : 1854967 : struct access *next_child;
3566 : 1854967 : if (!*last_seen_sibling)
3567 : 860831 : next_child = parent->first_child;
3568 : : else
3569 : 994136 : next_child = (*last_seen_sibling)->next_sibling;
3570 : :
3571 : : /* First, traverse the chain of siblings until it points to an access with
3572 : : offset at least equal to POS. Check all skipped accesses whether they
3573 : : span the POS boundary and if so, return with a failure. */
3574 : 1854968 : while (next_child && next_child->offset < pos)
3575 : : {
3576 : 1 : if (next_child->offset + next_child->size > pos)
3577 : : return TOTAL_FLD_FAILED;
3578 : 1 : *last_seen_sibling = next_child;
3579 : 1 : next_child = next_child->next_sibling;
3580 : : }
3581 : :
3582 : : /* Now check whether next_child has exactly the right POS and SIZE and if so,
3583 : : whether it can represent what we need and can be totally scalarized
3584 : : itself. */
3585 : 1854967 : if (next_child && next_child->offset == pos
3586 : 1452648 : && next_child->size == size)
3587 : : {
3588 : 1382255 : if (!is_gimple_reg_type (next_child->type)
3589 : 1382255 : && (!access_and_field_type_match_p (type, next_child->type)
3590 : 178398 : || !totally_scalarize_subtree (next_child)))
3591 : 170 : return TOTAL_FLD_FAILED;
3592 : :
3593 : 1382085 : *last_seen_sibling = next_child;
3594 : 1382085 : return TOTAL_FLD_DONE;
3595 : : }
3596 : :
3597 : : /* If the child we're looking at would partially overlap, we just cannot
3598 : : totally scalarize. */
3599 : : if (next_child
3600 : 92192 : && next_child->offset < pos + size
3601 : 71005 : && next_child->offset + next_child->size > pos + size)
3602 : : return TOTAL_FLD_FAILED;
3603 : :
3604 : 472696 : if (is_gimple_reg_type (type))
3605 : : {
3606 : : /* We don't scalarize accesses that are children of other scalar type
3607 : : accesses, so if we go on and create an access for a register type,
3608 : : there should not be any pre-existing children. There are rare cases
3609 : : where the requested type is a vector but we already have register
3610 : : accesses for all its elements which is equally good. Detect that
3611 : : situation or whether we need to bail out. */
3612 : :
3613 : : HOST_WIDE_INT covered = pos;
3614 : : bool skipping = false;
3615 : : while (next_child
3616 : 358909 : && next_child->offset + next_child->size <= pos + size)
3617 : : {
3618 : 132 : if (next_child->offset != covered
3619 : 132 : || !is_gimple_reg_type (next_child->type))
3620 : : return TOTAL_FLD_FAILED;
3621 : :
3622 : 132 : covered += next_child->size;
3623 : 132 : *last_seen_sibling = next_child;
3624 : 132 : next_child = next_child->next_sibling;
3625 : 132 : skipping = true;
3626 : : }
3627 : :
3628 : 358777 : if (skipping)
3629 : : {
3630 : 48 : if (covered != pos + size)
3631 : : return TOTAL_FLD_FAILED;
3632 : : else
3633 : : return TOTAL_FLD_DONE;
3634 : : }
3635 : : }
3636 : :
3637 : : return TOTAL_FLD_CREATE;
3638 : : }
3639 : :
3640 : : /* Go over sub-tree rooted in ROOT and attempt to create scalar accesses
3641 : : spanning all uncovered areas covered by ROOT, return false if the attempt
3642 : : failed. All created accesses will have grp_unscalarizable_region set (and
3643 : : should be ignored if the function returns false). */
3644 : :
3645 : : static bool
3646 : 862623 : totally_scalarize_subtree (struct access *root)
3647 : : {
3648 : 862623 : gcc_checking_assert (!root->grp_unscalarizable_region);
3649 : 862623 : gcc_checking_assert (!is_gimple_reg_type (root->type));
3650 : :
3651 : 862623 : struct access *last_seen_sibling = NULL;
3652 : :
3653 : 862623 : switch (TREE_CODE (root->type))
3654 : : {
3655 : 852647 : case RECORD_TYPE:
3656 : 13056280 : for (tree fld = TYPE_FIELDS (root->type); fld; fld = DECL_CHAIN (fld))
3657 : 12203855 : if (TREE_CODE (fld) == FIELD_DECL)
3658 : : {
3659 : 1866581 : tree ft = TREE_TYPE (fld);
3660 : 1866581 : HOST_WIDE_INT fsize = tree_to_uhwi (DECL_SIZE (fld));
3661 : 1866581 : if (!fsize)
3662 : 34194 : continue;
3663 : :
3664 : 1832387 : HOST_WIDE_INT pos = root->offset + int_bit_position (fld);
3665 : 1832387 : if (pos + fsize > root->offset + root->size)
3666 : : return false;
3667 : 1832387 : enum total_sra_field_state
3668 : 1832387 : state = total_should_skip_creating_access (root,
3669 : : &last_seen_sibling,
3670 : : ft, pos, fsize);
3671 : 1832387 : switch (state)
3672 : : {
3673 : : case TOTAL_FLD_FAILED:
3674 : : return false;
3675 : 1379258 : case TOTAL_FLD_DONE:
3676 : 1379258 : continue;
3677 : 452968 : case TOTAL_FLD_CREATE:
3678 : 452968 : break;
3679 : 0 : default:
3680 : 0 : gcc_unreachable ();
3681 : : }
3682 : :
3683 : 452968 : struct access **p = (last_seen_sibling
3684 : 452968 : ? &last_seen_sibling->next_sibling
3685 : : : &root->first_child);
3686 : 452968 : tree nref = build3 (COMPONENT_REF, ft, root->expr, fld, NULL_TREE);
3687 : 452968 : struct access *new_child
3688 : 452968 : = create_total_access_and_reshape (root, pos, fsize, ft, nref, p);
3689 : 452968 : if (!new_child)
3690 : : return false;
3691 : :
3692 : 452968 : if (!is_gimple_reg_type (ft)
3693 : 452968 : && !totally_scalarize_subtree (new_child))
3694 : : return false;
3695 : 452907 : last_seen_sibling = new_child;
3696 : : }
3697 : : break;
3698 : 9976 : case ARRAY_TYPE:
3699 : 9976 : {
3700 : 9976 : tree elemtype = TREE_TYPE (root->type);
3701 : 9976 : HOST_WIDE_INT el_size;
3702 : 9976 : offset_int idx, max;
3703 : 9976 : if (!prepare_iteration_over_array_elts (root->type, &el_size,
3704 : : &idx, &max))
3705 : : break;
3706 : :
3707 : 9976 : for (HOST_WIDE_INT pos = root->offset;
3708 : 32511 : idx <= max;
3709 : 22535 : pos += el_size, ++idx)
3710 : : {
3711 : 22580 : enum total_sra_field_state
3712 : 22580 : state = total_should_skip_creating_access (root,
3713 : : &last_seen_sibling,
3714 : : elemtype, pos,
3715 : : el_size);
3716 : 22580 : switch (state)
3717 : : {
3718 : : case TOTAL_FLD_FAILED:
3719 : 45 : return false;
3720 : 2855 : case TOTAL_FLD_DONE:
3721 : 2855 : continue;
3722 : 19680 : case TOTAL_FLD_CREATE:
3723 : 19680 : break;
3724 : 0 : default:
3725 : 0 : gcc_unreachable ();
3726 : : }
3727 : :
3728 : 19680 : struct access **p = (last_seen_sibling
3729 : 19680 : ? &last_seen_sibling->next_sibling
3730 : : : &root->first_child);
3731 : 39360 : tree nref = build4 (ARRAY_REF, elemtype, root->expr,
3732 : 19680 : wide_int_to_tree (TYPE_DOMAIN (root->type),
3733 : 19680 : idx),
3734 : : NULL_TREE, NULL_TREE);
3735 : 19680 : struct access *new_child
3736 : 19680 : = create_total_access_and_reshape (root, pos, el_size, elemtype,
3737 : : nref, p);
3738 : 19680 : if (!new_child)
3739 : : return false;
3740 : :
3741 : 19680 : if (!is_gimple_reg_type (elemtype)
3742 : 19680 : && !totally_scalarize_subtree (new_child))
3743 : : return false;
3744 : 19680 : last_seen_sibling = new_child;
3745 : : }
3746 : : }
3747 : 9931 : break;
3748 : 0 : default:
3749 : 0 : gcc_unreachable ();
3750 : : }
3751 : : return true;
3752 : : }
3753 : :
3754 : : /* Get the total total scalarization size limit in the current function. */
3755 : :
3756 : : unsigned HOST_WIDE_INT
3757 : 745903 : sra_get_max_scalarization_size (void)
3758 : : {
3759 : 745903 : bool optimize_speed_p = !optimize_function_for_size_p (cfun);
3760 : : /* If the user didn't set PARAM_SRA_MAX_SCALARIZATION_SIZE_<...>,
3761 : : fall back to a target default. */
3762 : 745903 : unsigned HOST_WIDE_INT max_scalarization_size
3763 : 745903 : = get_move_ratio (optimize_speed_p) * MOVE_MAX;
3764 : :
3765 : 745903 : if (optimize_speed_p)
3766 : : {
3767 : 727469 : if (OPTION_SET_P (param_sra_max_scalarization_size_speed))
3768 : 9 : max_scalarization_size = param_sra_max_scalarization_size_speed;
3769 : : }
3770 : : else
3771 : : {
3772 : 18434 : if (OPTION_SET_P (param_sra_max_scalarization_size_size))
3773 : 0 : max_scalarization_size = param_sra_max_scalarization_size_size;
3774 : : }
3775 : 745903 : max_scalarization_size *= BITS_PER_UNIT;
3776 : 745903 : return max_scalarization_size;
3777 : : }
3778 : :
3779 : : /* Go through all accesses collected throughout the (intraprocedural) analysis
3780 : : stage, exclude overlapping ones, identify representatives and build trees
3781 : : out of them, making decisions about scalarization on the way. Return true
3782 : : iff there are any to-be-scalarized variables after this stage. */
3783 : :
3784 : : static bool
3785 : 738647 : analyze_all_variable_accesses (void)
3786 : : {
3787 : 738647 : int res = 0;
3788 : 738647 : bitmap tmp = BITMAP_ALLOC (NULL);
3789 : 738647 : bitmap_iterator bi;
3790 : 738647 : unsigned i;
3791 : :
3792 : 738647 : bitmap_copy (tmp, candidate_bitmap);
3793 : 5032479 : EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
3794 : : {
3795 : 4293832 : tree var = candidate (i);
3796 : 4293832 : struct access *access;
3797 : :
3798 : 4293832 : access = sort_and_splice_var_accesses (var);
3799 : 4293832 : if (!access || !build_access_trees (access))
3800 : 47538 : disqualify_candidate (var,
3801 : : "No or inhibitingly overlapping accesses.");
3802 : : }
3803 : :
3804 : 738647 : propagate_all_subaccesses ();
3805 : :
3806 : 738647 : unsigned HOST_WIDE_INT max_scalarization_size
3807 : 738647 : = sra_get_max_scalarization_size ();
3808 : 4984941 : EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
3809 : 4246294 : if (bitmap_bit_p (should_scalarize_away_bitmap, i)
3810 : 4246294 : && !bitmap_bit_p (cannot_scalarize_away_bitmap, i))
3811 : : {
3812 : 911918 : tree var = candidate (i);
3813 : 911918 : if (!VAR_P (var))
3814 : 87832 : continue;
3815 : :
3816 : 824086 : if (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))) > max_scalarization_size)
3817 : : {
3818 : 6732 : if (dump_file && (dump_flags & TDF_DETAILS))
3819 : : {
3820 : 0 : fprintf (dump_file, "Too big to totally scalarize: ");
3821 : 0 : print_generic_expr (dump_file, var);
3822 : 0 : fprintf (dump_file, " (UID: %u)\n", DECL_UID (var));
3823 : : }
3824 : 6732 : continue;
3825 : : }
3826 : :
3827 : 817354 : bool all_types_ok = true;
3828 : 817354 : for (struct access *access = get_first_repr_for_decl (var);
3829 : 1619411 : access;
3830 : 802057 : access = access->next_grp)
3831 : 822565 : if (!can_totally_scalarize_forest_p (access)
3832 : 1637832 : || !totally_scalarizable_type_p (access->type,
3833 : 815267 : constant_decl_p (var),
3834 : : 0, nullptr))
3835 : : {
3836 : : all_types_ok = false;
3837 : : break;
3838 : : }
3839 : 817354 : if (!all_types_ok)
3840 : 20508 : continue;
3841 : :
3842 : 796846 : if (dump_file && (dump_flags & TDF_DETAILS))
3843 : : {
3844 : 1 : fprintf (dump_file, "Will attempt to totally scalarize ");
3845 : 1 : print_generic_expr (dump_file, var);
3846 : 1 : fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
3847 : : }
3848 : 796846 : bool scalarized = true;
3849 : 796846 : for (struct access *access = get_first_repr_for_decl (var);
3850 : 1598702 : access;
3851 : 801856 : access = access->next_grp)
3852 : 802054 : if (!is_gimple_reg_type (access->type)
3853 : 802054 : && !totally_scalarize_subtree (access))
3854 : : {
3855 : : scalarized = false;
3856 : : break;
3857 : : }
3858 : :
3859 : 796846 : if (scalarized)
3860 : 796648 : for (struct access *access = get_first_repr_for_decl (var);
3861 : 1598504 : access;
3862 : 801856 : access = access->next_grp)
3863 : 801856 : access->grp_total_scalarization = true;
3864 : : }
3865 : :
3866 : 738647 : if (flag_checking)
3867 : 738647 : verify_all_sra_access_forests ();
3868 : :
3869 : 738647 : bitmap_copy (tmp, candidate_bitmap);
3870 : 4984941 : EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
3871 : : {
3872 : 4246294 : tree var = candidate (i);
3873 : 4246294 : struct access *access = get_first_repr_for_decl (var);
3874 : :
3875 : 4246294 : if (analyze_access_trees (access))
3876 : : {
3877 : 1861950 : res++;
3878 : 1861950 : if (dump_file && (dump_flags & TDF_DETAILS))
3879 : : {
3880 : 8 : fprintf (dump_file, "\nAccess trees for ");
3881 : 8 : print_generic_expr (dump_file, var);
3882 : 8 : fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
3883 : 8 : dump_access_tree (dump_file, access);
3884 : 8 : fprintf (dump_file, "\n");
3885 : : }
3886 : : }
3887 : : else
3888 : 2384344 : disqualify_candidate (var, "No scalar replacements to be created.");
3889 : : }
3890 : :
3891 : 738647 : BITMAP_FREE (tmp);
3892 : :
3893 : 738647 : if (res)
3894 : : {
3895 : 435984 : statistics_counter_event (cfun, "Scalarized aggregates", res);
3896 : 435984 : return true;
3897 : : }
3898 : : else
3899 : : return false;
3900 : : }
3901 : :
3902 : : /* Generate statements copying scalar replacements of accesses within a subtree
3903 : : into or out of AGG. ACCESS, all its children, siblings and their children
3904 : : are to be processed. AGG is an aggregate type expression (can be a
3905 : : declaration but does not have to be, it can for example also be a mem_ref or
3906 : : a series of handled components). TOP_OFFSET is the offset of the processed
3907 : : subtree which has to be subtracted from offsets of individual accesses to
3908 : : get corresponding offsets for AGG. If CHUNK_SIZE is non-null, copy only
3909 : : replacements in the interval <start_offset, start_offset + chunk_size>,
3910 : : otherwise copy all. GSI is a statement iterator used to place the new
3911 : : statements. WRITE should be true when the statements should write from AGG
3912 : : to the replacement and false if vice versa. if INSERT_AFTER is true, new
3913 : : statements will be added after the current statement in GSI, they will be
3914 : : added before the statement otherwise. */
3915 : :
3916 : : static void
3917 : 1860193 : generate_subtree_copies (struct access *access, tree agg,
3918 : : HOST_WIDE_INT top_offset,
3919 : : HOST_WIDE_INT start_offset, HOST_WIDE_INT chunk_size,
3920 : : gimple_stmt_iterator *gsi, bool write,
3921 : : bool insert_after, location_t loc)
3922 : : {
3923 : : /* Never write anything into constant pool decls. See PR70602. */
3924 : 3720386 : if (!write && constant_decl_p (agg))
3925 : : return;
3926 : 4437656 : do
3927 : : {
3928 : 4437656 : if (chunk_size && access->offset >= start_offset + chunk_size)
3929 : : return;
3930 : :
3931 : 4437656 : if (access->grp_to_be_replaced
3932 : 3517559 : && (chunk_size == 0
3933 : 0 : || access->offset + access->size > start_offset))
3934 : : {
3935 : 3517559 : tree expr, repl = get_access_replacement (access);
3936 : 3517559 : gassign *stmt;
3937 : :
3938 : 3517559 : expr = build_ref_for_model (loc, agg, access->offset - top_offset,
3939 : : access, gsi, insert_after);
3940 : :
3941 : 3517559 : if (write)
3942 : : {
3943 : 1580275 : if (access->grp_partial_lhs)
3944 : 8 : expr = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE,
3945 : 8 : !insert_after,
3946 : : insert_after ? GSI_NEW_STMT
3947 : : : GSI_SAME_STMT);
3948 : 1580275 : stmt = gimple_build_assign (repl, expr);
3949 : : }
3950 : : else
3951 : : {
3952 : 1937284 : suppress_warning (repl /* Be more selective! */);
3953 : 1937284 : if (access->grp_partial_lhs)
3954 : 68 : repl = force_gimple_operand_gsi (gsi, repl, true, NULL_TREE,
3955 : 68 : !insert_after,
3956 : : insert_after ? GSI_NEW_STMT
3957 : : : GSI_SAME_STMT);
3958 : 1937284 : stmt = gimple_build_assign (expr, repl);
3959 : : }
3960 : 3517559 : gimple_set_location (stmt, loc);
3961 : :
3962 : 3517559 : if (insert_after)
3963 : 1580275 : gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
3964 : : else
3965 : 1937284 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
3966 : 3517559 : update_stmt (stmt);
3967 : 3517559 : sra_stats.subtree_copies++;
3968 : 3517559 : }
3969 : 920097 : else if (write
3970 : 363176 : && access->grp_to_be_debug_replaced
3971 : 5240 : && (chunk_size == 0
3972 : 0 : || access->offset + access->size > start_offset))
3973 : : {
3974 : 5240 : gdebug *ds;
3975 : 10480 : tree drhs = build_debug_ref_for_model (loc, agg,
3976 : 5240 : access->offset - top_offset,
3977 : : access);
3978 : 5240 : ds = gimple_build_debug_bind (get_access_replacement (access),
3979 : : drhs, gsi_stmt (*gsi));
3980 : 5240 : if (insert_after)
3981 : 5240 : gsi_insert_after (gsi, ds, GSI_NEW_STMT);
3982 : : else
3983 : 0 : gsi_insert_before (gsi, ds, GSI_SAME_STMT);
3984 : : }
3985 : :
3986 : 4437656 : if (access->first_child)
3987 : 421581 : generate_subtree_copies (access->first_child, agg, top_offset,
3988 : : start_offset, chunk_size, gsi,
3989 : : write, insert_after, loc);
3990 : :
3991 : 4437656 : access = access->next_sibling;
3992 : : }
3993 : 4437656 : while (access);
3994 : : }
3995 : :
3996 : : /* Assign zero to all scalar replacements in an access subtree. ACCESS is the
3997 : : root of the subtree to be processed. GSI is the statement iterator used
3998 : : for inserting statements which are added after the current statement if
3999 : : INSERT_AFTER is true or before it otherwise. */
4000 : :
4001 : : static void
4002 : 539942 : init_subtree_with_zero (struct access *access, gimple_stmt_iterator *gsi,
4003 : : bool insert_after, location_t loc)
4004 : :
4005 : : {
4006 : 539942 : struct access *child;
4007 : :
4008 : 539942 : if (access->grp_to_be_replaced)
4009 : : {
4010 : 239397 : gassign *stmt;
4011 : :
4012 : 239397 : stmt = gimple_build_assign (get_access_replacement (access),
4013 : : build_zero_cst (access->type));
4014 : 239397 : if (insert_after)
4015 : 35519 : gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
4016 : : else
4017 : 203878 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
4018 : 239397 : update_stmt (stmt);
4019 : 239397 : gimple_set_location (stmt, loc);
4020 : : }
4021 : 300545 : else if (access->grp_to_be_debug_replaced)
4022 : : {
4023 : 36727 : gdebug *ds
4024 : 36727 : = gimple_build_debug_bind (get_access_replacement (access),
4025 : : build_zero_cst (access->type),
4026 : : gsi_stmt (*gsi));
4027 : 36727 : if (insert_after)
4028 : 36727 : gsi_insert_after (gsi, ds, GSI_NEW_STMT);
4029 : : else
4030 : 0 : gsi_insert_before (gsi, ds, GSI_SAME_STMT);
4031 : : }
4032 : :
4033 : 932708 : for (child = access->first_child; child; child = child->next_sibling)
4034 : 392766 : init_subtree_with_zero (child, gsi, insert_after, loc);
4035 : 539942 : }
4036 : :
4037 : : /* Clobber all scalar replacements in an access subtree. ACCESS is the
4038 : : root of the subtree to be processed. GSI is the statement iterator used
4039 : : for inserting statements which are added after the current statement if
4040 : : INSERT_AFTER is true or before it otherwise. */
4041 : :
4042 : : static void
4043 : 3120520 : clobber_subtree (struct access *access, gimple_stmt_iterator *gsi,
4044 : : bool insert_after, location_t loc)
4045 : :
4046 : : {
4047 : 3120520 : struct access *child;
4048 : :
4049 : 3120520 : if (access->grp_to_be_replaced)
4050 : : {
4051 : 1974580 : tree rep = get_access_replacement (access);
4052 : 1974580 : tree clobber = build_clobber (access->type);
4053 : 1974580 : gimple *stmt = gimple_build_assign (rep, clobber);
4054 : :
4055 : 1974580 : if (insert_after)
4056 : 369918 : gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
4057 : : else
4058 : 1604662 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
4059 : 1974580 : update_stmt (stmt);
4060 : 1974580 : gimple_set_location (stmt, loc);
4061 : : }
4062 : :
4063 : 5049182 : for (child = access->first_child; child; child = child->next_sibling)
4064 : 1928662 : clobber_subtree (child, gsi, insert_after, loc);
4065 : 3120520 : }
4066 : :
4067 : : /* Search for an access representative for the given expression EXPR and
4068 : : return it or NULL if it cannot be found. */
4069 : :
4070 : : static struct access *
4071 : 48074784 : get_access_for_expr (tree expr)
4072 : : {
4073 : 48074784 : poly_int64 poffset, psize, pmax_size;
4074 : 48074784 : HOST_WIDE_INT offset, max_size;
4075 : 48074784 : tree base;
4076 : 48074784 : bool reverse;
4077 : :
4078 : : /* FIXME: This should not be necessary but Ada produces V_C_Es with a type of
4079 : : a different size than the size of its argument and we need the latter
4080 : : one. */
4081 : 48074784 : if (TREE_CODE (expr) == VIEW_CONVERT_EXPR)
4082 : 251265 : expr = TREE_OPERAND (expr, 0);
4083 : :
4084 : 48074784 : base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size,
4085 : : &reverse);
4086 : 48074784 : if (!known_size_p (pmax_size)
4087 : 47946791 : || !pmax_size.is_constant (&max_size)
4088 : 47946791 : || !poffset.is_constant (&offset)
4089 : 48074784 : || !DECL_P (base))
4090 : : return NULL;
4091 : :
4092 : 22902414 : if (tree basesize = DECL_SIZE (base))
4093 : : {
4094 : 22860351 : poly_int64 sz;
4095 : 22860351 : if (offset < 0
4096 : 22860335 : || !poly_int_tree_p (basesize, &sz)
4097 : 45720686 : || known_le (sz, offset))
4098 : 7027 : return NULL;
4099 : : }
4100 : :
4101 : 22895387 : if (max_size == 0
4102 : 22895387 : || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
4103 : 13335734 : return NULL;
4104 : :
4105 : 9559653 : return get_var_base_offset_size_access (base, offset, max_size);
4106 : : }
4107 : :
4108 : : /* Replace the expression EXPR with a scalar replacement if there is one and
4109 : : generate other statements to do type conversion or subtree copying if
4110 : : necessary. WRITE is true if the expression is being written to (it is on a
4111 : : LHS of a statement or output in an assembly statement). STMT_GSI is used to
4112 : : place newly created statements before the processed statement, REFRESH_GSI
4113 : : is used to place them afterwards - unless the processed statement must end a
4114 : : BB in which case it is placed on the outgoing non-EH edge. REFRESH_GSI and
4115 : : is then used to continue iteration over the BB. If sra_modify_expr is
4116 : : called only once with WRITE equal to true on a given statement, both
4117 : : iterator parameters can point to the same one. */
4118 : :
4119 : : static bool
4120 : 7686082 : sra_modify_expr (tree *expr, bool write, gimple_stmt_iterator *stmt_gsi,
4121 : : gimple_stmt_iterator *refresh_gsi)
4122 : : {
4123 : 7686082 : location_t loc;
4124 : 7686082 : struct access *access;
4125 : 7686082 : tree type, bfr, orig_expr;
4126 : 7686082 : bool partial_cplx_access = false;
4127 : :
4128 : 7686082 : if (TREE_CODE (*expr) == BIT_FIELD_REF
4129 : 7686082 : && (write || !sra_handled_bf_read_p (*expr)))
4130 : : {
4131 : 597 : bfr = *expr;
4132 : 597 : expr = &TREE_OPERAND (*expr, 0);
4133 : : }
4134 : : else
4135 : : bfr = NULL_TREE;
4136 : :
4137 : 7686082 : if (TREE_CODE (*expr) == REALPART_EXPR || TREE_CODE (*expr) == IMAGPART_EXPR)
4138 : : {
4139 : 24173 : expr = &TREE_OPERAND (*expr, 0);
4140 : 24173 : partial_cplx_access = true;
4141 : : }
4142 : 7686082 : access = get_access_for_expr (*expr);
4143 : 7686082 : if (!access)
4144 : : return false;
4145 : 170764 : type = TREE_TYPE (*expr);
4146 : 170764 : orig_expr = *expr;
4147 : :
4148 : 170764 : loc = gimple_location (gsi_stmt (*stmt_gsi));
4149 : 170764 : gimple_stmt_iterator alt_gsi = gsi_none ();
4150 : 170764 : if (write && stmt_ends_bb_p (gsi_stmt (*stmt_gsi)))
4151 : : {
4152 : 34446 : alt_gsi = gsi_start_edge (single_non_eh_succ (gsi_bb (*stmt_gsi)));
4153 : 34446 : refresh_gsi = &alt_gsi;
4154 : : }
4155 : :
4156 : 170764 : if (access->grp_to_be_replaced)
4157 : : {
4158 : 42578 : tree repl = get_access_replacement (access);
4159 : : /* If we replace a non-register typed access simply use the original
4160 : : access expression to extract the scalar component afterwards.
4161 : : This happens if scalarizing a function return value or parameter
4162 : : like in gcc.c-torture/execute/20041124-1.c, 20050316-1.c and
4163 : : gcc.c-torture/compile/20011217-1.c.
4164 : :
4165 : : We also want to use this when accessing a complex or vector which can
4166 : : be accessed as a different type too, potentially creating a need for
4167 : : type conversion (see PR42196) and when scalarized unions are involved
4168 : : in assembler statements (see PR42398). */
4169 : 42578 : if (!bfr && !useless_type_conversion_p (type, access->type))
4170 : : {
4171 : 39378 : tree ref;
4172 : :
4173 : 39378 : ref = build_ref_for_model (loc, orig_expr, 0, access, stmt_gsi,
4174 : : false);
4175 : :
4176 : 39378 : if (partial_cplx_access)
4177 : : {
4178 : : /* VIEW_CONVERT_EXPRs in partial complex access are always fine in
4179 : : the case of a write because in such case the replacement cannot
4180 : : be a gimple register. In the case of a load, we have to
4181 : : differentiate in between a register an non-register
4182 : : replacement. */
4183 : 15 : tree t = build1 (VIEW_CONVERT_EXPR, type, repl);
4184 : 15 : gcc_checking_assert (!write || access->grp_partial_lhs);
4185 : 15 : if (!access->grp_partial_lhs)
4186 : : {
4187 : 12 : tree tmp = make_ssa_name (type);
4188 : 12 : gassign *stmt = gimple_build_assign (tmp, t);
4189 : : /* This is always a read. */
4190 : 12 : gsi_insert_before (stmt_gsi, stmt, GSI_SAME_STMT);
4191 : 12 : t = tmp;
4192 : : }
4193 : 15 : *expr = t;
4194 : : }
4195 : 39363 : else if (write)
4196 : : {
4197 : 15425 : gassign *stmt;
4198 : :
4199 : 15425 : if (access->grp_partial_lhs)
4200 : 6 : ref = force_gimple_operand_gsi (refresh_gsi, ref, true,
4201 : : NULL_TREE, false, GSI_NEW_STMT);
4202 : 15425 : stmt = gimple_build_assign (repl, ref);
4203 : 15425 : gimple_set_location (stmt, loc);
4204 : 15425 : gsi_insert_after (refresh_gsi, stmt, GSI_NEW_STMT);
4205 : : }
4206 : : else
4207 : : {
4208 : 23938 : if (TREE_READONLY (access->base))
4209 : : return false;
4210 : :
4211 : 23933 : gassign *stmt;
4212 : 23933 : if (access->grp_partial_lhs)
4213 : 6 : repl = force_gimple_operand_gsi (stmt_gsi, repl, true,
4214 : : NULL_TREE, true,
4215 : : GSI_SAME_STMT);
4216 : 23933 : stmt = gimple_build_assign (ref, repl);
4217 : 23933 : gimple_set_location (stmt, loc);
4218 : 23933 : gsi_insert_before (stmt_gsi, stmt, GSI_SAME_STMT);
4219 : : }
4220 : : }
4221 : : else
4222 : : {
4223 : : /* If we are going to replace a scalar field in a structure with
4224 : : reverse storage order by a stand-alone scalar, we are going to
4225 : : effectively byte-swap the scalar and we also need to byte-swap
4226 : : the portion of it represented by the bit-field. */
4227 : 3200 : if (bfr && REF_REVERSE_STORAGE_ORDER (bfr))
4228 : : {
4229 : 0 : REF_REVERSE_STORAGE_ORDER (bfr) = 0;
4230 : 0 : TREE_OPERAND (bfr, 2)
4231 : 0 : = size_binop (MINUS_EXPR, TYPE_SIZE (TREE_TYPE (repl)),
4232 : : size_binop (PLUS_EXPR, TREE_OPERAND (bfr, 1),
4233 : : TREE_OPERAND (bfr, 2)));
4234 : : }
4235 : :
4236 : 3200 : *expr = repl;
4237 : : }
4238 : :
4239 : 42573 : sra_stats.exprs++;
4240 : : }
4241 : 128186 : else if (write && access->grp_to_be_debug_replaced)
4242 : : {
4243 : 12 : gdebug *ds = gimple_build_debug_bind (get_access_replacement (access),
4244 : : NULL_TREE,
4245 : : gsi_stmt (*stmt_gsi));
4246 : 12 : gsi_insert_after (stmt_gsi, ds, GSI_NEW_STMT);
4247 : : }
4248 : :
4249 : 170759 : if (access->first_child && !TREE_READONLY (access->base))
4250 : : {
4251 : 124796 : HOST_WIDE_INT start_offset, chunk_size;
4252 : 124796 : if (bfr
4253 : 0 : && tree_fits_uhwi_p (TREE_OPERAND (bfr, 1))
4254 : 124796 : && tree_fits_uhwi_p (TREE_OPERAND (bfr, 2)))
4255 : : {
4256 : 0 : chunk_size = tree_to_uhwi (TREE_OPERAND (bfr, 1));
4257 : 0 : start_offset = access->offset
4258 : 0 : + tree_to_uhwi (TREE_OPERAND (bfr, 2));
4259 : : }
4260 : : else
4261 : : start_offset = chunk_size = 0;
4262 : :
4263 : 201715 : generate_subtree_copies (access->first_child, orig_expr, access->offset,
4264 : : start_offset, chunk_size,
4265 : : write ? refresh_gsi : stmt_gsi,
4266 : : write, write, loc);
4267 : : }
4268 : : return true;
4269 : : }
4270 : :
4271 : : /* If EXPR, which must be a call argument, is an ADDR_EXPR, generate writes and
4272 : : reads from its base before and after the call statement given in CALL_GSI
4273 : : and return true if any copying took place. Otherwise call sra_modify_expr
4274 : : on EXPR and return its value. FLAGS is what the gimple_call_arg_flags
4275 : : return for the given parameter. */
4276 : :
4277 : : static bool
4278 : 8401527 : sra_modify_call_arg (tree *expr, gimple_stmt_iterator *call_gsi,
4279 : : gimple_stmt_iterator *refresh_gsi, int flags)
4280 : : {
4281 : 8401527 : if (TREE_CODE (*expr) != ADDR_EXPR)
4282 : 5556176 : return sra_modify_expr (expr, false, call_gsi, refresh_gsi);
4283 : :
4284 : 2845351 : if (flags & EAF_UNUSED)
4285 : : return false;
4286 : :
4287 : 2841883 : tree base = get_base_address (TREE_OPERAND (*expr, 0));
4288 : 2841883 : if (!DECL_P (base))
4289 : : return false;
4290 : 2163311 : struct access *access = get_access_for_expr (base);
4291 : 2163311 : if (!access)
4292 : : return false;
4293 : :
4294 : 48784 : gimple *stmt = gsi_stmt (*call_gsi);
4295 : 48784 : location_t loc = gimple_location (stmt);
4296 : 48784 : generate_subtree_copies (access, base, 0, 0, 0, call_gsi, false, false,
4297 : : loc);
4298 : :
4299 : 48784 : if (flags & EAF_NO_DIRECT_CLOBBER)
4300 : : return true;
4301 : :
4302 : 35645 : if (!stmt_ends_bb_p (stmt))
4303 : 25678 : generate_subtree_copies (access, base, 0, 0, 0, refresh_gsi, true,
4304 : : true, loc);
4305 : : else
4306 : : {
4307 : 9967 : edge e;
4308 : 9967 : edge_iterator ei;
4309 : 29900 : FOR_EACH_EDGE (e, ei, gsi_bb (*call_gsi)->succs)
4310 : : {
4311 : 19933 : gimple_stmt_iterator alt_gsi = gsi_start_edge (e);
4312 : 19933 : generate_subtree_copies (access, base, 0, 0, 0, &alt_gsi, true,
4313 : : true, loc);
4314 : : }
4315 : : }
4316 : : return true;
4317 : : }
4318 : :
4319 : : /* Where scalar replacements of the RHS have been written to when a replacement
4320 : : of a LHS of an assigments cannot be direclty loaded from a replacement of
4321 : : the RHS. */
4322 : : enum unscalarized_data_handling { SRA_UDH_NONE, /* Nothing done so far. */
4323 : : SRA_UDH_RIGHT, /* Data flushed to the RHS. */
4324 : : SRA_UDH_LEFT }; /* Data flushed to the LHS. */
4325 : :
4326 : : struct subreplacement_assignment_data
4327 : : {
4328 : : /* Offset of the access representing the lhs of the assignment. */
4329 : : HOST_WIDE_INT left_offset;
4330 : :
4331 : : /* LHS and RHS of the original assignment. */
4332 : : tree assignment_lhs, assignment_rhs;
4333 : :
4334 : : /* Access representing the rhs of the whole assignment. */
4335 : : struct access *top_racc;
4336 : :
4337 : : /* Stmt iterator used for statement insertions after the original assignment.
4338 : : It points to the main GSI used to traverse a BB during function body
4339 : : modification. */
4340 : : gimple_stmt_iterator *new_gsi;
4341 : :
4342 : : /* Stmt iterator used for statement insertions before the original
4343 : : assignment. Keeps on pointing to the original statement. */
4344 : : gimple_stmt_iterator old_gsi;
4345 : :
4346 : : /* Location of the assignment. */
4347 : : location_t loc;
4348 : :
4349 : : /* Keeps the information whether we have needed to refresh replacements of
4350 : : the LHS and from which side of the assignments this takes place. */
4351 : : enum unscalarized_data_handling refreshed;
4352 : : };
4353 : :
4354 : : /* Store all replacements in the access tree rooted in TOP_RACC either to their
4355 : : base aggregate if there are unscalarized data or directly to LHS of the
4356 : : statement that is pointed to by GSI otherwise. */
4357 : :
4358 : : static void
4359 : 110692 : handle_unscalarized_data_in_subtree (struct subreplacement_assignment_data *sad)
4360 : : {
4361 : 110692 : tree src;
4362 : : /* If the RHS is a load from a constant, we do not need to (and must not)
4363 : : flush replacements to it and can use it directly as if we did. */
4364 : 110692 : if (TREE_READONLY (sad->top_racc->base))
4365 : : {
4366 : 5 : sad->refreshed = SRA_UDH_RIGHT;
4367 : 5 : return;
4368 : : }
4369 : 110687 : if (sad->top_racc->grp_unscalarized_data)
4370 : : {
4371 : 27003 : src = sad->assignment_rhs;
4372 : 27003 : sad->refreshed = SRA_UDH_RIGHT;
4373 : : }
4374 : : else
4375 : : {
4376 : 83684 : src = sad->assignment_lhs;
4377 : 83684 : sad->refreshed = SRA_UDH_LEFT;
4378 : : }
4379 : 110687 : generate_subtree_copies (sad->top_racc->first_child, src,
4380 : : sad->top_racc->offset, 0, 0,
4381 : : &sad->old_gsi, false, false, sad->loc);
4382 : : }
4383 : :
4384 : : /* Try to generate statements to load all sub-replacements in an access subtree
4385 : : formed by children of LACC from scalar replacements in the SAD->top_racc
4386 : : subtree. If that is not possible, refresh the SAD->top_racc base aggregate
4387 : : and load the accesses from it. */
4388 : :
4389 : : static void
4390 : 504130 : load_assign_lhs_subreplacements (struct access *lacc,
4391 : : struct subreplacement_assignment_data *sad)
4392 : : {
4393 : 1645529 : for (lacc = lacc->first_child; lacc; lacc = lacc->next_sibling)
4394 : : {
4395 : 1141399 : HOST_WIDE_INT offset;
4396 : 1141399 : offset = lacc->offset - sad->left_offset + sad->top_racc->offset;
4397 : :
4398 : 1141399 : if (lacc->grp_to_be_replaced)
4399 : : {
4400 : 957954 : struct access *racc;
4401 : 957954 : gassign *stmt;
4402 : 957954 : tree rhs;
4403 : :
4404 : 957954 : racc = find_access_in_subtree (sad->top_racc, offset, lacc->size);
4405 : 957954 : if (racc && racc->grp_to_be_replaced)
4406 : : {
4407 : 931910 : rhs = get_access_replacement (racc);
4408 : 931910 : bool vce = false;
4409 : 931910 : if (!useless_type_conversion_p (lacc->type, racc->type))
4410 : : {
4411 : 31 : rhs = fold_build1_loc (sad->loc, VIEW_CONVERT_EXPR,
4412 : : lacc->type, rhs);
4413 : 31 : vce = true;
4414 : : }
4415 : :
4416 : 931910 : if (lacc->grp_partial_lhs && (vce || racc->grp_partial_lhs))
4417 : 3 : rhs = force_gimple_operand_gsi (&sad->old_gsi, rhs, true,
4418 : : NULL_TREE, true, GSI_SAME_STMT);
4419 : : }
4420 : : else
4421 : : {
4422 : : /* No suitable access on the right hand side, need to load from
4423 : : the aggregate. See if we have to update it first... */
4424 : 26044 : if (sad->refreshed == SRA_UDH_NONE)
4425 : 15099 : handle_unscalarized_data_in_subtree (sad);
4426 : :
4427 : 26044 : if (sad->refreshed == SRA_UDH_LEFT)
4428 : 508 : rhs = build_ref_for_model (sad->loc, sad->assignment_lhs,
4429 : 508 : lacc->offset - sad->left_offset,
4430 : : lacc, sad->new_gsi, true);
4431 : : else
4432 : 25536 : rhs = build_ref_for_model (sad->loc, sad->assignment_rhs,
4433 : 25536 : lacc->offset - sad->left_offset,
4434 : : lacc, sad->new_gsi, true);
4435 : 26044 : if (lacc->grp_partial_lhs)
4436 : 1 : rhs = force_gimple_operand_gsi (sad->new_gsi,
4437 : : rhs, true, NULL_TREE,
4438 : : false, GSI_NEW_STMT);
4439 : : }
4440 : :
4441 : 957954 : stmt = gimple_build_assign (get_access_replacement (lacc), rhs);
4442 : 957954 : gsi_insert_after (sad->new_gsi, stmt, GSI_NEW_STMT);
4443 : 957954 : gimple_set_location (stmt, sad->loc);
4444 : 957954 : update_stmt (stmt);
4445 : 957954 : sra_stats.subreplacements++;
4446 : : }
4447 : : else
4448 : : {
4449 : 183445 : if (sad->refreshed == SRA_UDH_NONE
4450 : 24915 : && lacc->grp_read && !lacc->grp_covered)
4451 : 12 : handle_unscalarized_data_in_subtree (sad);
4452 : :
4453 : 183445 : if (lacc && lacc->grp_to_be_debug_replaced)
4454 : : {
4455 : 108638 : gdebug *ds;
4456 : 108638 : tree drhs;
4457 : 108638 : struct access *racc = find_access_in_subtree (sad->top_racc,
4458 : : offset,
4459 : : lacc->size);
4460 : :
4461 : 108638 : if (racc && racc->grp_to_be_replaced)
4462 : : {
4463 : 108528 : if (racc->grp_write || constant_decl_p (racc->base))
4464 : 106220 : drhs = get_access_replacement (racc);
4465 : : else
4466 : : drhs = NULL;
4467 : : }
4468 : 110 : else if (sad->refreshed == SRA_UDH_LEFT)
4469 : 0 : drhs = build_debug_ref_for_model (sad->loc, lacc->base,
4470 : : lacc->offset, lacc);
4471 : 110 : else if (sad->refreshed == SRA_UDH_RIGHT)
4472 : 108 : drhs = build_debug_ref_for_model (sad->loc, sad->top_racc->base,
4473 : : offset, lacc);
4474 : : else
4475 : : drhs = NULL_TREE;
4476 : 106220 : if (drhs
4477 : 106328 : && !useless_type_conversion_p (lacc->type, TREE_TYPE (drhs)))
4478 : 354 : drhs = fold_build1_loc (sad->loc, VIEW_CONVERT_EXPR,
4479 : : lacc->type, drhs);
4480 : 108638 : ds = gimple_build_debug_bind (get_access_replacement (lacc),
4481 : : drhs, gsi_stmt (sad->old_gsi));
4482 : 108638 : gsi_insert_after (sad->new_gsi, ds, GSI_NEW_STMT);
4483 : : }
4484 : : }
4485 : :
4486 : 1141399 : if (lacc->first_child)
4487 : 32945 : load_assign_lhs_subreplacements (lacc, sad);
4488 : : }
4489 : 504130 : }
4490 : :
4491 : : /* Result code for SRA assignment modification. */
4492 : : enum assignment_mod_result { SRA_AM_NONE, /* nothing done for the stmt */
4493 : : SRA_AM_MODIFIED, /* stmt changed but not
4494 : : removed */
4495 : : SRA_AM_REMOVED }; /* stmt eliminated */
4496 : :
4497 : : /* Modify assignments with a CONSTRUCTOR on their RHS. STMT contains a pointer
4498 : : to the assignment and GSI is the statement iterator pointing at it. Returns
4499 : : the same values as sra_modify_assign. */
4500 : :
4501 : : static enum assignment_mod_result
4502 : 3310560 : sra_modify_constructor_assign (gimple *stmt, gimple_stmt_iterator *gsi)
4503 : : {
4504 : 3310560 : tree lhs = gimple_assign_lhs (stmt);
4505 : 3310560 : struct access *acc = get_access_for_expr (lhs);
4506 : 3310560 : if (!acc)
4507 : : return SRA_AM_NONE;
4508 : 1339034 : location_t loc = gimple_location (stmt);
4509 : :
4510 : 1339034 : if (gimple_clobber_p (stmt))
4511 : : {
4512 : : /* Clobber the replacement variable. */
4513 : 1191858 : clobber_subtree (acc, gsi, !acc->grp_covered, loc);
4514 : : /* Remove clobbers of fully scalarized variables, they are dead. */
4515 : 1191858 : if (acc->grp_covered)
4516 : : {
4517 : 889121 : unlink_stmt_vdef (stmt);
4518 : 889121 : gsi_remove (gsi, true);
4519 : 889121 : release_defs (stmt);
4520 : 889121 : return SRA_AM_REMOVED;
4521 : : }
4522 : : else
4523 : : return SRA_AM_MODIFIED;
4524 : : }
4525 : :
4526 : 147176 : if (CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt)) > 0)
4527 : : {
4528 : : /* I have never seen this code path trigger but if it can happen the
4529 : : following should handle it gracefully. */
4530 : 0 : if (access_has_children_p (acc))
4531 : 0 : generate_subtree_copies (acc->first_child, lhs, acc->offset, 0, 0, gsi,
4532 : : true, true, loc);
4533 : 0 : return SRA_AM_MODIFIED;
4534 : : }
4535 : :
4536 : 147176 : if (acc->grp_covered)
4537 : : {
4538 : 73505 : init_subtree_with_zero (acc, gsi, false, loc);
4539 : 73505 : unlink_stmt_vdef (stmt);
4540 : 73505 : gsi_remove (gsi, true);
4541 : 73505 : release_defs (stmt);
4542 : 73505 : return SRA_AM_REMOVED;
4543 : : }
4544 : : else
4545 : : {
4546 : 73671 : init_subtree_with_zero (acc, gsi, true, loc);
4547 : 73671 : return SRA_AM_MODIFIED;
4548 : : }
4549 : : }
4550 : :
4551 : : /* Create and return a new suitable default definition SSA_NAME for RACC which
4552 : : is an access describing an uninitialized part of an aggregate that is being
4553 : : loaded. REG_TREE is used instead of the actual RACC type if that is not of
4554 : : a gimple register type. */
4555 : :
4556 : : static tree
4557 : 508 : get_repl_default_def_ssa_name (struct access *racc, tree reg_type)
4558 : : {
4559 : 508 : gcc_checking_assert (!racc->grp_to_be_replaced
4560 : : && !racc->grp_to_be_debug_replaced);
4561 : 508 : if (!racc->replacement_decl)
4562 : 508 : racc->replacement_decl = create_access_replacement (racc, reg_type);
4563 : 508 : return get_or_create_ssa_default_def (cfun, racc->replacement_decl);
4564 : : }
4565 : :
4566 : :
4567 : : /* Generate statements to call .DEFERRED_INIT to initialize scalar replacements
4568 : : of accesses within a subtree ACCESS; all its children, siblings and their
4569 : : children are to be processed.
4570 : : GSI is a statement iterator used to place the new statements. */
4571 : : static void
4572 : 9 : generate_subtree_deferred_init (struct access *access,
4573 : : tree init_type,
4574 : : tree decl_name,
4575 : : gimple_stmt_iterator *gsi,
4576 : : location_t loc)
4577 : : {
4578 : 28 : do
4579 : : {
4580 : 28 : if (access->grp_to_be_replaced)
4581 : : {
4582 : 27 : tree repl = get_access_replacement (access);
4583 : 27 : gimple *call
4584 : 27 : = gimple_build_call_internal (IFN_DEFERRED_INIT, 3,
4585 : 27 : TYPE_SIZE_UNIT (TREE_TYPE (repl)),
4586 : : init_type, decl_name);
4587 : 27 : gimple_call_set_lhs (call, repl);
4588 : 27 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
4589 : 27 : update_stmt (call);
4590 : 27 : gimple_set_location (call, loc);
4591 : 27 : sra_stats.subtree_deferred_init++;
4592 : : }
4593 : 28 : if (access->first_child)
4594 : 1 : generate_subtree_deferred_init (access->first_child, init_type,
4595 : : decl_name, gsi, loc);
4596 : :
4597 : 28 : access = access ->next_sibling;
4598 : : }
4599 : 28 : while (access);
4600 : 9 : }
4601 : :
4602 : : /* For a call to .DEFERRED_INIT:
4603 : : var = .DEFERRED_INIT (size_of_var, init_type, name_of_var);
4604 : : examine the LHS variable VAR and replace it with a scalar replacement if
4605 : : there is one, also replace the RHS call to a call to .DEFERRED_INIT of
4606 : : the corresponding scalar relacement variable. Examine the subtree and
4607 : : do the scalar replacements in the subtree too. STMT is the call, GSI is
4608 : : the statment iterator to place newly created statement. */
4609 : :
4610 : : static enum assignment_mod_result
4611 : 11 : sra_modify_deferred_init (gimple *stmt, gimple_stmt_iterator *gsi)
4612 : : {
4613 : 11 : tree lhs = gimple_call_lhs (stmt);
4614 : 11 : tree init_type = gimple_call_arg (stmt, 1);
4615 : 11 : tree decl_name = gimple_call_arg (stmt, 2);
4616 : :
4617 : 11 : struct access *lhs_access = get_access_for_expr (lhs);
4618 : 11 : if (!lhs_access)
4619 : : return SRA_AM_NONE;
4620 : :
4621 : 11 : location_t loc = gimple_location (stmt);
4622 : :
4623 : 11 : if (lhs_access->grp_to_be_replaced)
4624 : : {
4625 : 3 : tree lhs_repl = get_access_replacement (lhs_access);
4626 : 3 : gimple_call_set_lhs (stmt, lhs_repl);
4627 : 3 : tree arg0_repl = TYPE_SIZE_UNIT (TREE_TYPE (lhs_repl));
4628 : 3 : gimple_call_set_arg (stmt, 0, arg0_repl);
4629 : 3 : sra_stats.deferred_init++;
4630 : 3 : gcc_assert (!lhs_access->first_child);
4631 : : return SRA_AM_MODIFIED;
4632 : : }
4633 : :
4634 : 8 : if (lhs_access->first_child)
4635 : 8 : generate_subtree_deferred_init (lhs_access->first_child,
4636 : : init_type, decl_name, gsi, loc);
4637 : 8 : if (lhs_access->grp_covered)
4638 : : {
4639 : 7 : unlink_stmt_vdef (stmt);
4640 : 7 : gsi_remove (gsi, true);
4641 : 7 : release_defs (stmt);
4642 : 7 : return SRA_AM_REMOVED;
4643 : : }
4644 : :
4645 : : return SRA_AM_MODIFIED;
4646 : : }
4647 : :
4648 : : /* Examine both sides of the assignment statement pointed to by STMT, replace
4649 : : them with a scalare replacement if there is one and generate copying of
4650 : : replacements if scalarized aggregates have been used in the assignment. GSI
4651 : : is used to hold generated statements for type conversions and subtree
4652 : : copying. */
4653 : :
4654 : : static enum assignment_mod_result
4655 : 26539545 : sra_modify_assign (gimple *stmt, gimple_stmt_iterator *gsi)
4656 : : {
4657 : 26539545 : struct access *lacc, *racc;
4658 : 26539545 : tree lhs, rhs;
4659 : 26539545 : bool modify_this_stmt = false;
4660 : 26539545 : bool force_gimple_rhs = false;
4661 : 26539545 : location_t loc;
4662 : 26539545 : gimple_stmt_iterator orig_gsi = *gsi;
4663 : :
4664 : 26539545 : if (!gimple_assign_single_p (stmt))
4665 : : return SRA_AM_NONE;
4666 : 20792740 : lhs = gimple_assign_lhs (stmt);
4667 : 20792740 : rhs = gimple_assign_rhs1 (stmt);
4668 : :
4669 : 20792740 : if (TREE_CODE (rhs) == CONSTRUCTOR)
4670 : 3310560 : return sra_modify_constructor_assign (stmt, gsi);
4671 : :
4672 : 17473217 : if (TREE_CODE (rhs) == REALPART_EXPR || TREE_CODE (lhs) == REALPART_EXPR
4673 : 17470451 : || TREE_CODE (rhs) == IMAGPART_EXPR || TREE_CODE (lhs) == IMAGPART_EXPR
4674 : 17458007 : || (TREE_CODE (rhs) == BIT_FIELD_REF && !sra_handled_bf_read_p (rhs))
4675 : 34940183 : || TREE_CODE (lhs) == BIT_FIELD_REF)
4676 : : {
4677 : 24770 : modify_this_stmt = sra_modify_expr (gimple_assign_rhs1_ptr (stmt),
4678 : : false, gsi, gsi);
4679 : 24770 : modify_this_stmt |= sra_modify_expr (gimple_assign_lhs_ptr (stmt),
4680 : : true, gsi, gsi);
4681 : 46434 : return modify_this_stmt ? SRA_AM_MODIFIED : SRA_AM_NONE;
4682 : : }
4683 : :
4684 : 17457410 : lacc = get_access_for_expr (lhs);
4685 : 17457410 : racc = get_access_for_expr (rhs);
4686 : 17457410 : if (!lacc && !racc)
4687 : : return SRA_AM_NONE;
4688 : : /* Avoid modifying initializations of constant-pool replacements. */
4689 : 7126658 : if (racc && (racc->replacement_decl == lhs))
4690 : : return SRA_AM_NONE;
4691 : :
4692 : 7122452 : loc = gimple_location (stmt);
4693 : 7122452 : if (lacc && lacc->grp_to_be_replaced)
4694 : : {
4695 : 2020882 : lhs = get_access_replacement (lacc);
4696 : 2020882 : gimple_assign_set_lhs (stmt, lhs);
4697 : 2020882 : modify_this_stmt = true;
4698 : 2020882 : if (lacc->grp_partial_lhs)
4699 : 85 : force_gimple_rhs = true;
4700 : 2020882 : sra_stats.exprs++;
4701 : : }
4702 : :
4703 : 7122452 : if (racc && racc->grp_to_be_replaced)
4704 : : {
4705 : 3234237 : rhs = get_access_replacement (racc);
4706 : 3234237 : modify_this_stmt = true;
4707 : 3234237 : if (racc->grp_partial_lhs)
4708 : 661 : force_gimple_rhs = true;
4709 : 3234237 : sra_stats.exprs++;
4710 : : }
4711 : 1124701 : else if (racc
4712 : 1124701 : && !racc->grp_unscalarized_data
4713 : 907418 : && !racc->grp_unscalarizable_region
4714 : 907416 : && TREE_CODE (lhs) == SSA_NAME
4715 : 587 : && !access_has_replacements_p (racc))
4716 : : {
4717 : 508 : rhs = get_repl_default_def_ssa_name (racc, TREE_TYPE (lhs));
4718 : 508 : modify_this_stmt = true;
4719 : 508 : sra_stats.exprs++;
4720 : : }
4721 : :
4722 : 3234745 : if (modify_this_stmt
4723 : 7122452 : && !useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
4724 : : {
4725 : : /* If we can avoid creating a VIEW_CONVERT_EXPR, then do so.
4726 : : ??? This should move to fold_stmt which we simply should
4727 : : call after building a VIEW_CONVERT_EXPR here. */
4728 : 686390 : if (AGGREGATE_TYPE_P (TREE_TYPE (lhs))
4729 : 211747 : && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (lhs)) == racc->reverse
4730 : 555465 : && !contains_bitfld_component_ref_p (lhs))
4731 : : {
4732 : 211746 : lhs = build_ref_for_model (loc, lhs, 0, racc, gsi, false);
4733 : 211746 : gimple_assign_set_lhs (stmt, lhs);
4734 : : }
4735 : 131973 : else if (lacc
4736 : 106778 : && AGGREGATE_TYPE_P (TREE_TYPE (rhs))
4737 : 85019 : && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (rhs)) == lacc->reverse
4738 : 216992 : && !contains_vce_or_bfcref_p (rhs))
4739 : 84945 : rhs = build_ref_for_model (loc, rhs, 0, lacc, gsi, false);
4740 : :
4741 : 343719 : if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
4742 : : {
4743 : 47028 : rhs = fold_build1_loc (loc, VIEW_CONVERT_EXPR, TREE_TYPE (lhs), rhs);
4744 : 47028 : if (is_gimple_reg_type (TREE_TYPE (lhs))
4745 : 47028 : && TREE_CODE (lhs) != SSA_NAME)
4746 : 7122452 : force_gimple_rhs = true;
4747 : : }
4748 : : }
4749 : :
4750 : 7122452 : if (lacc && lacc->grp_to_be_debug_replaced)
4751 : : {
4752 : 175255 : tree dlhs = get_access_replacement (lacc);
4753 : 175255 : tree drhs = unshare_expr (rhs);
4754 : 175255 : if (!useless_type_conversion_p (TREE_TYPE (dlhs), TREE_TYPE (drhs)))
4755 : : {
4756 : 272 : if (AGGREGATE_TYPE_P (TREE_TYPE (drhs))
4757 : 263 : && !contains_vce_or_bfcref_p (drhs))
4758 : 127 : drhs = build_debug_ref_for_model (loc, drhs, 0, lacc);
4759 : 136 : if (drhs
4760 : 272 : && !useless_type_conversion_p (TREE_TYPE (dlhs),
4761 : 136 : TREE_TYPE (drhs)))
4762 : 9 : drhs = fold_build1_loc (loc, VIEW_CONVERT_EXPR,
4763 : 9 : TREE_TYPE (dlhs), drhs);
4764 : : }
4765 : 175255 : gdebug *ds = gimple_build_debug_bind (dlhs, drhs, stmt);
4766 : 175255 : gsi_insert_before (gsi, ds, GSI_SAME_STMT);
4767 : : }
4768 : :
4769 : : /* From this point on, the function deals with assignments in between
4770 : : aggregates when at least one has scalar reductions of some of its
4771 : : components. There are three possible scenarios: Both the LHS and RHS have
4772 : : to-be-scalarized components, 2) only the RHS has or 3) only the LHS has.
4773 : :
4774 : : In the first case, we would like to load the LHS components from RHS
4775 : : components whenever possible. If that is not possible, we would like to
4776 : : read it directly from the RHS (after updating it by storing in it its own
4777 : : components). If there are some necessary unscalarized data in the LHS,
4778 : : those will be loaded by the original assignment too. If neither of these
4779 : : cases happen, the original statement can be removed. Most of this is done
4780 : : by load_assign_lhs_subreplacements.
4781 : :
4782 : : In the second case, we would like to store all RHS scalarized components
4783 : : directly into LHS and if they cover the aggregate completely, remove the
4784 : : statement too. In the third case, we want the LHS components to be loaded
4785 : : directly from the RHS (DSE will remove the original statement if it
4786 : : becomes redundant).
4787 : :
4788 : : This is a bit complex but manageable when types match and when unions do
4789 : : not cause confusion in a way that we cannot really load a component of LHS
4790 : : from the RHS or vice versa (the access representing this level can have
4791 : : subaccesses that are accessible only through a different union field at a
4792 : : higher level - different from the one used in the examined expression).
4793 : : Unions are fun.
4794 : :
4795 : : Therefore, I specially handle a fourth case, happening when there is a
4796 : : specific type cast or it is impossible to locate a scalarized subaccess on
4797 : : the other side of the expression. If that happens, I simply "refresh" the
4798 : : RHS by storing in it is scalarized components leave the original statement
4799 : : there to do the copying and then load the scalar replacements of the LHS.
4800 : : This is what the first branch does. */
4801 : :
4802 : 7122452 : if (modify_this_stmt
4803 : 2023199 : || gimple_has_volatile_ops (stmt)
4804 : 2022845 : || contains_vce_or_bfcref_p (rhs)
4805 : 1842442 : || contains_vce_or_bfcref_p (lhs)
4806 : 8963373 : || stmt_ends_bb_p (stmt))
4807 : : {
4808 : : /* No need to copy into a constant, it comes pre-initialized. */
4809 : 5356287 : if (access_has_children_p (racc) && !TREE_READONLY (racc->base))
4810 : 15372 : generate_subtree_copies (racc->first_child, rhs, racc->offset, 0, 0,
4811 : : gsi, false, false, loc);
4812 : 5340915 : if (access_has_children_p (lacc))
4813 : : {
4814 : 235580 : gimple_stmt_iterator alt_gsi = gsi_none ();
4815 : 235580 : if (stmt_ends_bb_p (stmt))
4816 : : {
4817 : 58840 : alt_gsi = gsi_start_edge (single_non_eh_succ (gsi_bb (*gsi)));
4818 : 58840 : gsi = &alt_gsi;
4819 : : }
4820 : 235580 : generate_subtree_copies (lacc->first_child, lhs, lacc->offset, 0, 0,
4821 : : gsi, true, true, loc);
4822 : : }
4823 : 5340915 : sra_stats.separate_lhs_rhs_handling++;
4824 : :
4825 : : /* This gimplification must be done after generate_subtree_copies,
4826 : : lest we insert the subtree copies in the middle of the gimplified
4827 : : sequence. */
4828 : 5340915 : if (force_gimple_rhs)
4829 : 22563 : rhs = force_gimple_operand_gsi (&orig_gsi, rhs, true, NULL_TREE,
4830 : : true, GSI_SAME_STMT);
4831 : 5340915 : if (gimple_assign_rhs1 (stmt) != rhs)
4832 : : {
4833 : 3341424 : modify_this_stmt = true;
4834 : 3341424 : gimple_assign_set_rhs_from_tree (&orig_gsi, rhs);
4835 : 3341424 : gcc_assert (stmt == gsi_stmt (orig_gsi));
4836 : : }
4837 : :
4838 : 7098744 : return modify_this_stmt ? SRA_AM_MODIFIED : SRA_AM_NONE;
4839 : : }
4840 : : else
4841 : : {
4842 : 2501048 : if (access_has_children_p (lacc)
4843 : 1781550 : && access_has_children_p (racc)
4844 : : /* When an access represents an unscalarizable region, it usually
4845 : : represents accesses with variable offset and thus must not be used
4846 : : to generate new memory accesses. */
4847 : 471198 : && !lacc->grp_unscalarizable_region
4848 : 471191 : && !racc->grp_unscalarizable_region)
4849 : : {
4850 : 471185 : struct subreplacement_assignment_data sad;
4851 : :
4852 : 471185 : sad.left_offset = lacc->offset;
4853 : 471185 : sad.assignment_lhs = lhs;
4854 : 471185 : sad.assignment_rhs = rhs;
4855 : 471185 : sad.top_racc = racc;
4856 : 471185 : sad.old_gsi = *gsi;
4857 : 471185 : sad.new_gsi = gsi;
4858 : 471185 : sad.loc = gimple_location (stmt);
4859 : 471185 : sad.refreshed = SRA_UDH_NONE;
4860 : :
4861 : 471185 : if (lacc->grp_read && !lacc->grp_covered)
4862 : 95581 : handle_unscalarized_data_in_subtree (&sad);
4863 : :
4864 : 471185 : load_assign_lhs_subreplacements (lacc, &sad);
4865 : 471185 : if (sad.refreshed != SRA_UDH_RIGHT)
4866 : : {
4867 : 444177 : gsi_next (gsi);
4868 : 444177 : unlink_stmt_vdef (stmt);
4869 : 444177 : gsi_remove (&sad.old_gsi, true);
4870 : 444177 : release_defs (stmt);
4871 : 444177 : sra_stats.deleted++;
4872 : 444177 : return SRA_AM_REMOVED;
4873 : : }
4874 : : }
4875 : : else
4876 : : {
4877 : 1310352 : if (access_has_children_p (racc)
4878 : 508203 : && !racc->grp_unscalarized_data
4879 : 448846 : && TREE_CODE (lhs) != SSA_NAME)
4880 : : {
4881 : 448845 : if (dump_file)
4882 : : {
4883 : 4 : fprintf (dump_file, "Removing load: ");
4884 : 4 : print_gimple_stmt (dump_file, stmt, 0);
4885 : : }
4886 : 448845 : generate_subtree_copies (racc->first_child, lhs,
4887 : : racc->offset, 0, 0, gsi,
4888 : : false, false, loc);
4889 : 448845 : gcc_assert (stmt == gsi_stmt (*gsi));
4890 : 448845 : unlink_stmt_vdef (stmt);
4891 : 448845 : gsi_remove (gsi, true);
4892 : 448845 : release_defs (stmt);
4893 : 448845 : sra_stats.deleted++;
4894 : 448845 : return SRA_AM_REMOVED;
4895 : : }
4896 : : /* Restore the aggregate RHS from its components so the
4897 : : prevailing aggregate copy does the right thing. */
4898 : 920865 : if (access_has_children_p (racc) && !TREE_READONLY (racc->base))
4899 : 59331 : generate_subtree_copies (racc->first_child, rhs, racc->offset, 0, 0,
4900 : : gsi, false, false, loc);
4901 : : /* Re-load the components of the aggregate copy destination.
4902 : : But use the RHS aggregate to load from to expose more
4903 : : optimization opportunities. */
4904 : 861507 : if (access_has_children_p (lacc))
4905 : : {
4906 : 248319 : generate_subtree_copies (lacc->first_child, rhs, lacc->offset,
4907 : : 0, 0, gsi, true, true, loc);
4908 : 248319 : if (lacc->grp_covered)
4909 : : {
4910 : 173533 : unlink_stmt_vdef (stmt);
4911 : 173533 : gsi_remove (& orig_gsi, true);
4912 : 173533 : release_defs (stmt);
4913 : 173533 : sra_stats.deleted++;
4914 : 173533 : return SRA_AM_REMOVED;
4915 : : }
4916 : : }
4917 : : }
4918 : :
4919 : 714982 : return SRA_AM_NONE;
4920 : : }
4921 : : }
4922 : :
4923 : : /* Set any scalar replacements of values in the constant pool to the initial
4924 : : value of the constant. (Constant-pool decls like *.LC0 have effectively
4925 : : been initialized before the program starts, we must do the same for their
4926 : : replacements.) Thus, we output statements like 'SR.1 = *.LC0[0];' into
4927 : : the function's entry block. */
4928 : :
4929 : : static void
4930 : 435984 : initialize_constant_pool_replacements (void)
4931 : : {
4932 : 435984 : gimple_seq seq = NULL;
4933 : 435984 : gimple_stmt_iterator gsi = gsi_start (seq);
4934 : 435984 : bitmap_iterator bi;
4935 : 435984 : unsigned i;
4936 : :
4937 : 2297934 : EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
4938 : : {
4939 : 1861950 : tree var = candidate (i);
4940 : 1861950 : if (!constant_decl_p (var))
4941 : 1861874 : continue;
4942 : :
4943 : 76 : struct access *access = get_first_repr_for_decl (var);
4944 : :
4945 : 5596 : while (access)
4946 : : {
4947 : 5444 : if (access->replacement_decl)
4948 : : {
4949 : 4206 : gassign *stmt
4950 : 4206 : = gimple_build_assign (get_access_replacement (access),
4951 : : unshare_expr (access->expr));
4952 : 4206 : if (dump_file && (dump_flags & TDF_DETAILS))
4953 : : {
4954 : 0 : fprintf (dump_file, "Generating constant initializer: ");
4955 : 0 : print_gimple_stmt (dump_file, stmt, 0);
4956 : 0 : fprintf (dump_file, "\n");
4957 : : }
4958 : 4206 : gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
4959 : 4206 : update_stmt (stmt);
4960 : : }
4961 : :
4962 : 5444 : if (access->first_child)
4963 : : access = access->first_child;
4964 : 4206 : else if (access->next_sibling)
4965 : : access = access->next_sibling;
4966 : : else
4967 : : {
4968 : 2341 : while (access->parent && !access->next_sibling)
4969 : : access = access->parent;
4970 : 1103 : if (access->next_sibling)
4971 : : access = access->next_sibling;
4972 : : else
4973 : 76 : access = access->next_grp;
4974 : : }
4975 : : }
4976 : : }
4977 : :
4978 : 435984 : seq = gsi_seq (gsi);
4979 : 435984 : if (seq)
4980 : 71 : gsi_insert_seq_on_edge_immediate (
4981 : 71 : single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
4982 : 435984 : }
4983 : :
4984 : : /* Traverse the function body and all modifications as decided in
4985 : : analyze_all_variable_accesses. Return true iff the CFG has been
4986 : : changed. */
4987 : :
4988 : : static bool
4989 : 435984 : sra_modify_function_body (void)
4990 : : {
4991 : 435984 : bool cfg_changed = false;
4992 : 435984 : basic_block bb;
4993 : :
4994 : 435984 : initialize_constant_pool_replacements ();
4995 : :
4996 : 10223675 : FOR_EACH_BB_FN (bb, cfun)
4997 : : {
4998 : 9787691 : gimple_stmt_iterator gsi = gsi_start_bb (bb);
4999 : 83987768 : while (!gsi_end_p (gsi))
5000 : : {
5001 : 74200077 : gimple *stmt = gsi_stmt (gsi);
5002 : 74200077 : enum assignment_mod_result assign_result;
5003 : 74200077 : bool modified = false, deleted = false;
5004 : 74200077 : tree *t;
5005 : 74200077 : unsigned i;
5006 : :
5007 : 74200077 : switch (gimple_code (stmt))
5008 : : {
5009 : 433449 : case GIMPLE_RETURN:
5010 : 433449 : t = gimple_return_retval_ptr (as_a <greturn *> (stmt));
5011 : 433449 : if (*t != NULL_TREE)
5012 : 287040 : modified |= sra_modify_expr (t, false, &gsi, &gsi);
5013 : : break;
5014 : :
5015 : 26539545 : case GIMPLE_ASSIGN:
5016 : 26539545 : assign_result = sra_modify_assign (stmt, &gsi);
5017 : 26539545 : modified |= assign_result == SRA_AM_MODIFIED;
5018 : 26539545 : deleted = assign_result == SRA_AM_REMOVED;
5019 : 26539545 : break;
5020 : :
5021 : 4228222 : case GIMPLE_CALL:
5022 : : /* Handle calls to .DEFERRED_INIT specially. */
5023 : 4228222 : if (gimple_call_internal_p (stmt, IFN_DEFERRED_INIT))
5024 : : {
5025 : 11 : assign_result = sra_modify_deferred_init (stmt, &gsi);
5026 : 11 : modified |= assign_result == SRA_AM_MODIFIED;
5027 : 11 : deleted = assign_result == SRA_AM_REMOVED;
5028 : : }
5029 : : else
5030 : : {
5031 : 4228211 : gcall *call = as_a <gcall *> (stmt);
5032 : 4228211 : gimple_stmt_iterator call_gsi = gsi;
5033 : :
5034 : : /* Operands must be processed before the lhs. */
5035 : 12601096 : for (i = 0; i < gimple_call_num_args (call); i++)
5036 : : {
5037 : 8372885 : int flags = gimple_call_arg_flags (call, i);
5038 : 8372885 : t = gimple_call_arg_ptr (call, i);
5039 : 8372885 : modified |= sra_modify_call_arg (t, &call_gsi, &gsi, flags);
5040 : : }
5041 : 4228211 : if (gimple_call_chain (call))
5042 : : {
5043 : 28642 : t = gimple_call_chain_ptr (call);
5044 : 28642 : int flags = gimple_call_static_chain_flags (call);
5045 : 28642 : modified |= sra_modify_call_arg (t, &call_gsi, &gsi,
5046 : : flags);
5047 : : }
5048 : 4228211 : if (gimple_call_lhs (call))
5049 : : {
5050 : 1790182 : t = gimple_call_lhs_ptr (call);
5051 : 1790182 : modified |= sra_modify_expr (t, true, &call_gsi, &gsi);
5052 : : }
5053 : : }
5054 : : break;
5055 : :
5056 : 4402 : case GIMPLE_ASM:
5057 : 4402 : {
5058 : 4402 : gimple_stmt_iterator stmt_gsi = gsi;
5059 : 4402 : gasm *asm_stmt = as_a <gasm *> (stmt);
5060 : 10232 : for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
5061 : : {
5062 : 1428 : t = &TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
5063 : 1428 : modified |= sra_modify_expr (t, false, &stmt_gsi, &gsi);
5064 : : }
5065 : 6118 : for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
5066 : : {
5067 : 1716 : t = &TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
5068 : 1716 : modified |= sra_modify_expr (t, true, &stmt_gsi, &gsi);
5069 : : }
5070 : : }
5071 : 4402 : break;
5072 : :
5073 : : default:
5074 : : break;
5075 : : }
5076 : :
5077 : 31059209 : if (modified)
5078 : : {
5079 : 5666556 : update_stmt (stmt);
5080 : 5666556 : if (maybe_clean_eh_stmt (stmt)
5081 : 5666556 : && gimple_purge_dead_eh_edges (gimple_bb (stmt)))
5082 : : cfg_changed = true;
5083 : : }
5084 : 74200077 : if (!deleted)
5085 : 72170889 : gsi_next (&gsi);
5086 : : }
5087 : : }
5088 : :
5089 : 435984 : gsi_commit_edge_inserts ();
5090 : 435984 : return cfg_changed;
5091 : : }
5092 : :
5093 : : /* Generate statements initializing scalar replacements of parts of function
5094 : : parameters. */
5095 : :
5096 : : static void
5097 : 435984 : initialize_parameter_reductions (void)
5098 : : {
5099 : 435984 : gimple_stmt_iterator gsi;
5100 : 435984 : gimple_seq seq = NULL;
5101 : 435984 : tree parm;
5102 : :
5103 : 435984 : gsi = gsi_start (seq);
5104 : 435984 : for (parm = DECL_ARGUMENTS (current_function_decl);
5105 : 1326376 : parm;
5106 : 890392 : parm = DECL_CHAIN (parm))
5107 : : {
5108 : 890392 : vec<access_p> *access_vec;
5109 : 890392 : struct access *access;
5110 : :
5111 : 890392 : if (!bitmap_bit_p (candidate_bitmap, DECL_UID (parm)))
5112 : 825253 : continue;
5113 : 65139 : access_vec = get_base_access_vector (parm);
5114 : 65139 : if (!access_vec)
5115 : 0 : continue;
5116 : :
5117 : 65139 : for (access = (*access_vec)[0];
5118 : 166426 : access;
5119 : 101287 : access = access->next_grp)
5120 : 101287 : generate_subtree_copies (access, parm, 0, 0, 0, &gsi, true, true,
5121 : 101287 : EXPR_LOCATION (parm));
5122 : : }
5123 : :
5124 : 435984 : seq = gsi_seq (gsi);
5125 : 435984 : if (seq)
5126 : 52477 : gsi_insert_seq_on_edge_immediate (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
5127 : 435984 : }
5128 : :
5129 : : /* The "main" function of intraprocedural SRA passes. Runs the analysis and if
5130 : : it reveals there are components of some aggregates to be scalarized, it runs
5131 : : the required transformations. */
5132 : : static unsigned int
5133 : 3458716 : perform_intra_sra (void)
5134 : : {
5135 : 3458716 : int ret = 0;
5136 : 3458716 : sra_initialize ();
5137 : :
5138 : 3458716 : if (!find_var_candidates ())
5139 : 2662612 : goto out;
5140 : :
5141 : 796104 : if (!scan_function ())
5142 : 57457 : goto out;
5143 : :
5144 : 738647 : if (!analyze_all_variable_accesses ())
5145 : 302663 : goto out;
5146 : :
5147 : 435984 : if (sra_modify_function_body ())
5148 : : ret = TODO_update_ssa | TODO_cleanup_cfg;
5149 : : else
5150 : 435962 : ret = TODO_update_ssa;
5151 : 435984 : initialize_parameter_reductions ();
5152 : :
5153 : 435984 : statistics_counter_event (cfun, "Scalar replacements created",
5154 : : sra_stats.replacements);
5155 : 435984 : statistics_counter_event (cfun, "Modified expressions", sra_stats.exprs);
5156 : 435984 : statistics_counter_event (cfun, "Subtree copy stmts",
5157 : : sra_stats.subtree_copies);
5158 : 435984 : statistics_counter_event (cfun, "Subreplacement stmts",
5159 : : sra_stats.subreplacements);
5160 : 435984 : statistics_counter_event (cfun, "Deleted stmts", sra_stats.deleted);
5161 : 435984 : statistics_counter_event (cfun, "Separate LHS and RHS handling",
5162 : : sra_stats.separate_lhs_rhs_handling);
5163 : :
5164 : 3458716 : out:
5165 : 3458716 : sra_deinitialize ();
5166 : 3458716 : return ret;
5167 : : }
5168 : :
5169 : : /* Perform early intraprocedural SRA. */
5170 : : static unsigned int
5171 : 2437556 : early_intra_sra (void)
5172 : : {
5173 : 2437556 : sra_mode = SRA_MODE_EARLY_INTRA;
5174 : 0 : return perform_intra_sra ();
5175 : : }
5176 : :
5177 : : /* Perform "late" intraprocedural SRA. */
5178 : : static unsigned int
5179 : 1021160 : late_intra_sra (void)
5180 : : {
5181 : 1021160 : sra_mode = SRA_MODE_INTRA;
5182 : 0 : return perform_intra_sra ();
5183 : : }
5184 : :
5185 : :
5186 : : static bool
5187 : 3462900 : gate_intra_sra (void)
5188 : : {
5189 : 3462900 : return flag_tree_sra != 0 && dbg_cnt (tree_sra);
5190 : : }
5191 : :
5192 : :
5193 : : namespace {
5194 : :
5195 : : const pass_data pass_data_sra_early =
5196 : : {
5197 : : GIMPLE_PASS, /* type */
5198 : : "esra", /* name */
5199 : : OPTGROUP_NONE, /* optinfo_flags */
5200 : : TV_TREE_SRA, /* tv_id */
5201 : : ( PROP_cfg | PROP_ssa ), /* properties_required */
5202 : : 0, /* properties_provided */
5203 : : 0, /* properties_destroyed */
5204 : : 0, /* todo_flags_start */
5205 : : TODO_update_ssa, /* todo_flags_finish */
5206 : : };
5207 : :
5208 : : class pass_sra_early : public gimple_opt_pass
5209 : : {
5210 : : public:
5211 : 285081 : pass_sra_early (gcc::context *ctxt)
5212 : 570162 : : gimple_opt_pass (pass_data_sra_early, ctxt)
5213 : : {}
5214 : :
5215 : : /* opt_pass methods: */
5216 : 2441318 : bool gate (function *) final override { return gate_intra_sra (); }
5217 : 2437556 : unsigned int execute (function *) final override
5218 : : {
5219 : 2437556 : return early_intra_sra ();
5220 : : }
5221 : :
5222 : : }; // class pass_sra_early
5223 : :
5224 : : } // anon namespace
5225 : :
5226 : : gimple_opt_pass *
5227 : 285081 : make_pass_sra_early (gcc::context *ctxt)
5228 : : {
5229 : 285081 : return new pass_sra_early (ctxt);
5230 : : }
5231 : :
5232 : : namespace {
5233 : :
5234 : : const pass_data pass_data_sra =
5235 : : {
5236 : : GIMPLE_PASS, /* type */
5237 : : "sra", /* name */
5238 : : OPTGROUP_NONE, /* optinfo_flags */
5239 : : TV_TREE_SRA, /* tv_id */
5240 : : ( PROP_cfg | PROP_ssa ), /* properties_required */
5241 : : 0, /* properties_provided */
5242 : : 0, /* properties_destroyed */
5243 : : TODO_update_address_taken, /* todo_flags_start */
5244 : : TODO_update_ssa, /* todo_flags_finish */
5245 : : };
5246 : :
5247 : : class pass_sra : public gimple_opt_pass
5248 : : {
5249 : : public:
5250 : 285081 : pass_sra (gcc::context *ctxt)
5251 : 570162 : : gimple_opt_pass (pass_data_sra, ctxt)
5252 : : {}
5253 : :
5254 : : /* opt_pass methods: */
5255 : 1021582 : bool gate (function *) final override { return gate_intra_sra (); }
5256 : 1021160 : unsigned int execute (function *) final override { return late_intra_sra (); }
5257 : :
5258 : : }; // class pass_sra
5259 : :
5260 : : } // anon namespace
5261 : :
5262 : : gimple_opt_pass *
5263 : 285081 : make_pass_sra (gcc::context *ctxt)
5264 : : {
5265 : 285081 : return new pass_sra (ctxt);
5266 : : }
5267 : :
5268 : :
5269 : : /* If type T cannot be totally scalarized, return false. Otherwise return true
5270 : : and push to the vector within PC offsets and lengths of all padding in the
5271 : : type as total scalarization would encounter it. */
5272 : :
5273 : : static bool
5274 : 121522 : check_ts_and_push_padding_to_vec (tree type, sra_padding_collecting *pc)
5275 : : {
5276 : 121522 : if (!totally_scalarizable_type_p (type, true /* optimistic value */,
5277 : : 0, pc))
5278 : : return false;
5279 : :
5280 : 114550 : pc->record_padding (tree_to_shwi (TYPE_SIZE (type)));
5281 : 114550 : return true;
5282 : : }
5283 : :
5284 : : /* Given two types in an assignment, return true either if any one cannot be
5285 : : totally scalarized or if they have padding (i.e. not copied bits) */
5286 : :
5287 : : bool
5288 : 64247 : sra_total_scalarization_would_copy_same_data_p (tree t1, tree t2)
5289 : : {
5290 : 64247 : sra_padding_collecting p1;
5291 : 64247 : if (!check_ts_and_push_padding_to_vec (t1, &p1))
5292 : : return true;
5293 : :
5294 : 57275 : sra_padding_collecting p2;
5295 : 57275 : if (!check_ts_and_push_padding_to_vec (t2, &p2))
5296 : : return true;
5297 : :
5298 : 57275 : unsigned l = p1.m_padding.length ();
5299 : 114550 : if (l != p2.m_padding.length ())
5300 : : return false;
5301 : 67019 : for (unsigned i = 0; i < l; i++)
5302 : 9747 : if (p1.m_padding[i].first != p2.m_padding[i].first
5303 : 9747 : || p1.m_padding[i].second != p2.m_padding[i].second)
5304 : : return false;
5305 : :
5306 : : return true;
5307 : 57275 : }
5308 : :
|
