Line data Source code
1 : /* Interprocedural scalar replacement of aggregates
2 : Copyright (C) 2019-2026 Free Software Foundation, Inc.
3 : Contributed by Martin Jambor <mjambor@suse.cz>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* IPA-SRA is an interprocedural pass that removes unused function return
22 : values (turning functions returning a value which is never used into void
23 : functions) and removes unused function parameters. It can also replace an
24 : aggregate parameter by a set of other parameters representing part of the
25 : original, turning those passed by reference into new ones which pass the
26 : value directly.
27 :
28 : The pass is a true IPA one, which means that it works in three stages in
29 : order to be able to take advantage of LTO. First, summaries about functions
30 : and each calls are generated. Function summaries (often called call graph
31 : node summaries) contain mainly information about which parameters are
32 : potential transformation candidates and which bits of candidates are
33 : accessed. We differentiate between accesses done as a part of a call
34 : statement (which might be not necessary if the callee is also transformed)
35 : and others (which are mandatory). Call summaries (often called call graph
36 : edge summaries) contain information about which function formal parameters
37 : feed into which actual call arguments so that if two parameters are only
38 : used in a sum which is then passed to another function which then however
39 : does not use this parameter, all three parameters of the two functions can
40 : be eliminated. Edge summaries also have flags whether the return value is
41 : used or if it is only returned in the caller too. In LTO mode these
42 : summaries are then streamed to the object file in the compilation phase and
43 : streamed back in in the WPA analysis stage.
44 :
45 : The interprocedural analysis phase traverses the graph in topological order
46 : in two sweeps, one in each direction. First, from callees to callers for
47 : parameter removal and splitting. Each strongly-connected component is
48 : processed iteratively until the situation in it stabilizes. The pass from
49 : callers to callees is then carried out to remove unused return values in a
50 : very similar fashion.
51 :
52 : Because parameter manipulation has big implications for call redirection
53 : which is done only after all call graph nodes materialize, the
54 : transformation phase is not part of this patch but is carried out by the
55 : clone materialization and edge redirection itself, see comments in
56 : ipa-param-manipulation.h for more details. */
57 :
58 :
59 : #include "config.h"
60 : #include "system.h"
61 : #include "coretypes.h"
62 : #include "backend.h"
63 : #include "tree.h"
64 : #include "gimple.h"
65 : #include "predict.h"
66 : #include "tree-pass.h"
67 : #include "ssa.h"
68 : #include "cgraph.h"
69 : #include "gimple-pretty-print.h"
70 : #include "alias.h"
71 : #include "tree-eh.h"
72 : #include "gimple-iterator.h"
73 : #include "gimple-walk.h"
74 : #include "tree-dfa.h"
75 : #include "tree-sra.h"
76 : #include "alloc-pool.h"
77 : #include "symbol-summary.h"
78 : #include "dbgcnt.h"
79 : #include "tree-inline.h"
80 : #include "ipa-utils.h"
81 : #include "builtins.h"
82 : #include "cfganal.h"
83 : #include "tree-streamer.h"
84 : #include "internal-fn.h"
85 : #include "symtab-clones.h"
86 : #include "attribs.h"
87 : #include "sreal.h"
88 : #include "ipa-cp.h"
89 : #include "ipa-prop.h"
90 :
91 : static void ipa_sra_summarize_function (cgraph_node *);
92 :
93 : /* Bits used to track size of an aggregate in bytes interprocedurally. */
94 : #define ISRA_ARG_SIZE_LIMIT_BITS 16
95 : #define ISRA_ARG_SIZE_LIMIT (1 << ISRA_ARG_SIZE_LIMIT_BITS)
96 : /* How many parameters can feed into a call actual argument and still be
97 : tracked. */
98 : #define IPA_SRA_MAX_PARAM_FLOW_LEN 7
99 :
100 : /* Structure describing accesses to a specific portion of an aggregate
101 : parameter, as given by the offset and size. Any smaller accesses that occur
102 : within a function that fall within another access form a tree. The pass
103 : cannot analyze parameters with only partially overlapping accesses. */
104 :
105 : struct GTY(()) param_access
106 : {
107 : /* Type that a potential replacement should have. This field only has
108 : meaning in the summary building and transformation phases, when it is
109 : reconstructed from the body. Must not be touched in IPA analysis
110 : stage. */
111 : tree type;
112 :
113 : /* Alias reference type to be used in MEM_REFs when adjusting caller
114 : arguments. */
115 : tree alias_ptr_type;
116 :
117 : /* Values returned by get_ref_base_and_extent but converted to bytes and
118 : stored as unsigned ints. */
119 : unsigned unit_offset;
120 : unsigned unit_size : ISRA_ARG_SIZE_LIMIT_BITS;
121 :
122 : /* Set once we are sure that the access will really end up in a potentially
123 : transformed function - initially not set for portions of formal parameters
124 : that are only used as actual function arguments passed to callees. */
125 : unsigned certain : 1;
126 : /* Set if the access has reverse scalar storage order. */
127 : unsigned reverse : 1;
128 : };
129 :
130 : /* This structure has the same purpose as the one above and additionally it
131 : contains some fields that are only necessary in the summary generation
132 : phase. */
133 :
134 : struct gensum_param_access
135 : {
136 : /* Values returned by get_ref_base_and_extent. */
137 : HOST_WIDE_INT offset;
138 : HOST_WIDE_INT size;
139 :
140 : /* if this access has any children (in terms of the definition above), this
141 : points to the first one. */
142 : struct gensum_param_access *first_child;
143 : /* In intraprocedural SRA, pointer to the next sibling in the access tree as
144 : described above. */
145 : struct gensum_param_access *next_sibling;
146 :
147 : /* Type that a potential replacement should have. This field only has
148 : meaning in the summary building and transformation phases, when it is
149 : reconstructed from the body. Must not be touched in IPA analysis
150 : stage. */
151 : tree type;
152 : /* Alias reference type to be used in MEM_REFs when adjusting caller
153 : arguments. */
154 : tree alias_ptr_type;
155 :
156 : /* Cumulative count of all loads. */
157 : profile_count load_count;
158 : /* Have there been writes to or reads from this exact location except for as
159 : arguments to a function call that can be tracked. */
160 : bool nonarg;
161 :
162 : /* Set if the access has reverse scalar storage order. */
163 : bool reverse;
164 : };
165 :
166 : /* Summary describing a parameter in the IPA stages. */
167 :
168 : struct GTY(()) isra_param_desc
169 : {
170 : /* List of access representatives to the parameters, sorted according to
171 : their offset. */
172 : vec <param_access *, va_gc> *accesses;
173 :
174 : /* Unit size limit of total size of all replacements. */
175 : unsigned param_size_limit : ISRA_ARG_SIZE_LIMIT_BITS;
176 : /* Sum of unit sizes of all certain replacements. */
177 : unsigned size_reached : ISRA_ARG_SIZE_LIMIT_BITS;
178 : /* Minimum offset that is known to be safe to dereference because of callers
179 : pass pointers to DECLs of at least this size or because of dereferences in
180 : callers. */
181 : unsigned safe_size : ISRA_ARG_SIZE_LIMIT_BITS;
182 :
183 : /* A parameter that is used only in call arguments and can be removed if all
184 : concerned actual arguments are removed. */
185 : unsigned locally_unused : 1;
186 : /* An aggregate that is a candidate for breaking up or complete removal. */
187 : unsigned split_candidate : 1;
188 : /* Is this a parameter passing stuff by reference? */
189 : unsigned by_ref : 1;
190 : /* If set, this parameter can only be a candidate for removal if the function
191 : is going to loose its return value. */
192 : unsigned remove_only_when_retval_removed : 1;
193 : /* If set, this parameter can only be a candidate for splitting if the
194 : function is going to loose its return value. Can only be meaningfully set
195 : for by_ref parameters. */
196 : unsigned split_only_when_retval_removed : 1;
197 : /* Parameter hint set during IPA analysis when there is a caller which does
198 : not construct the argument just to pass it to calls. Only meaningful for
199 : by_ref parameters. */
200 : unsigned not_specially_constructed : 1;
201 : /* Only meaningful for by_ref parameters. If set, this parameter can only be
202 : a split candidate if all callers pass pointers that are known to point to
203 : a chunk of memory large enough to contain all accesses. */
204 : unsigned conditionally_dereferenceable : 1;
205 : /* Set when safe_size has been updated from at least one caller. */
206 : unsigned safe_size_set : 1;
207 : };
208 :
209 : /* Structure used when generating summaries that describes a parameter. */
210 :
211 : struct gensum_param_desc
212 : {
213 : /* Roots of param_accesses. */
214 : gensum_param_access *accesses;
215 : /* Number of accesses in the access tree rooted in field accesses. */
216 : unsigned access_count;
217 :
218 : /* If the below is non-zero, this is the number of uses as actual
219 : arguments. */
220 : int call_uses;
221 : /* Number of times this parameter has been directly passed to. */
222 : unsigned ptr_pt_count;
223 :
224 : /* Size limit of total size of all replacements. */
225 : unsigned param_size_limit;
226 : /* Sum of sizes of nonarg accesses. */
227 : unsigned nonarg_acc_size;
228 :
229 : /* A parameter that is used only in call arguments and can be removed if all
230 : concerned actual arguments are removed. */
231 : bool locally_unused;
232 : /* An aggregate that is a candidate for breaking up or a pointer passing data
233 : by reference that is a candidate for being converted to a set of
234 : parameters passing those data by value. */
235 : bool split_candidate;
236 : /* Is this a parameter passing stuff by reference (either a pointer or a
237 : source language reference type)? */
238 : bool by_ref;
239 : /* If this parameter passes stuff by reference, can it be safely dereferenced
240 : without performing further checks (for example because it is a
241 : REFERENCE_TYPE)? */
242 : bool safe_ref;
243 : /* If set, this parameter can only be a candidate for removal if the function
244 : is going to loose its return value. */
245 : bool remove_only_when_retval_removed;
246 : /* If set, this parameter can only be a candidate for splitting if the
247 : function is going to loose its return value. Can only be meaningfully set
248 : for by_ref parameters. */
249 : bool split_only_when_retval_removed;
250 : /* Only meaningful for by_ref parameters. If set, this parameter can only be
251 : a split candidate if all callers pass pointers that are known to point to
252 : a chunk of memory large enough to contain all accesses. */
253 : bool conditionally_dereferenceable;
254 :
255 : /* The number of this parameter as they are ordered in function decl. */
256 : int param_number;
257 : /* For parameters passing data by reference, this is parameter index to
258 : compute indices to bb_dereferences. */
259 : int deref_index;
260 : };
261 :
262 : /* Properly deallocate accesses of DESC. TODO: Since this data structure is
263 : allocated in GC memory, this is not necessary and we can consider removing
264 : the function. */
265 :
266 : static void
267 2788136 : free_param_decl_accesses (isra_param_desc *desc)
268 : {
269 2788136 : unsigned len = vec_safe_length (desc->accesses);
270 3379594 : for (unsigned i = 0; i < len; ++i)
271 591458 : ggc_free ((*desc->accesses)[i]);
272 2788136 : vec_free (desc->accesses);
273 2788136 : }
274 :
275 : /* Class used to convey information about functions from the
276 : intra-procedural analysis stage to inter-procedural one. */
277 :
278 : class GTY((for_user)) isra_func_summary
279 : {
280 : public:
281 : /* initialize the object. */
282 :
283 1203812 : isra_func_summary ()
284 1203812 : : m_parameters (NULL), m_candidate (false), m_returns_value (false),
285 1203812 : m_return_ignored (false), m_queued (false)
286 : {}
287 :
288 : /* Destroy m_parameters. */
289 :
290 : ~isra_func_summary ();
291 :
292 : /* Mark the function as not a candidate for any IPA-SRA transformation.
293 : Return true if it was a candidate until now. */
294 :
295 : bool zap ();
296 :
297 : /* Vector of parameter descriptors corresponding to the function being
298 : analyzed. */
299 : vec<isra_param_desc, va_gc> *m_parameters;
300 :
301 : /* Whether the node is even a candidate for any IPA-SRA transformation at
302 : all. */
303 : unsigned m_candidate : 1;
304 :
305 : /* Whether the original function returns any value. */
306 : unsigned m_returns_value : 1;
307 :
308 : /* Set to true if all call statements do not actually use the returned
309 : value. */
310 :
311 : unsigned m_return_ignored : 1;
312 :
313 : /* Whether the node is already queued in IPA SRA stack during processing of
314 : call graphs SCCs. */
315 :
316 : unsigned m_queued : 1;
317 : };
318 :
319 : /* Deallocate the memory pointed to by isra_func_summary. TODO: Since this
320 : data structure is allocated in GC memory, this is not necessary and we can
321 : consider removing the destructor. */
322 :
323 1203811 : isra_func_summary::~isra_func_summary ()
324 : {
325 1203811 : unsigned len = vec_safe_length (m_parameters);
326 2480139 : for (unsigned i = 0; i < len; ++i)
327 1276328 : free_param_decl_accesses (&(*m_parameters)[i]);
328 1203811 : vec_free (m_parameters);
329 1203811 : }
330 :
331 : /* Mark the function as not a candidate for any IPA-SRA transformation. Return
332 : true if it was a candidate until now. */
333 :
334 : bool
335 658914 : isra_func_summary::zap ()
336 : {
337 658914 : bool ret = m_candidate;
338 658914 : m_candidate = false;
339 :
340 : /* TODO: see the destructor above. */
341 658914 : unsigned len = vec_safe_length (m_parameters);
342 2170722 : for (unsigned i = 0; i < len; ++i)
343 1511808 : free_param_decl_accesses (&(*m_parameters)[i]);
344 658914 : vec_free (m_parameters);
345 :
346 658914 : return ret;
347 : }
348 :
349 : /* Structure to describe which formal parameters feed into a particular actual
350 : argument. */
351 :
352 : struct isra_param_flow
353 : {
354 : /* Number of elements in array inputs that contain valid data. */
355 : char length;
356 : /* Indices of formal parameters that feed into the described actual argument.
357 : If aggregate_pass_through or pointer_pass_through below are true, it must
358 : contain exactly one element which is passed through from a formal
359 : parameter if the given number. Otherwise, the array contains indices of
360 : callee's formal parameters which are used to calculate value of this
361 : actual argument. */
362 : unsigned char inputs[IPA_SRA_MAX_PARAM_FLOW_LEN];
363 :
364 : /* Offset within the formal parameter. */
365 : unsigned unit_offset;
366 : /* When aggregate_pass_through is set, this is the size of the portion of an
367 : aggregate formal parameter that is being passed. Otherwise, this is size
368 : of pointed to memory that is known to be valid be dereferenced. */
369 : unsigned unit_size : ISRA_ARG_SIZE_LIMIT_BITS;
370 :
371 : /* True when the value of this actual argument is a portion of a formal
372 : parameter. */
373 : unsigned aggregate_pass_through : 1;
374 : /* True when the value of this actual copy is a verbatim pass through of an
375 : obtained pointer. */
376 : unsigned pointer_pass_through : 1;
377 : /* True when it is safe to copy access candidates here from the callee, which
378 : would mean introducing dereferences into callers of the caller. */
379 : unsigned safe_to_import_accesses : 1;
380 : /* True when the passed value is an address of a structure that has been
381 : constructed in the caller just to be passed by reference to functions
382 : (i.e. is never read). */
383 : unsigned constructed_for_calls : 1;
384 : };
385 :
386 : /* Structure used to convey information about calls from the intra-procedural
387 : analysis stage to inter-procedural one. */
388 :
389 6070116 : class isra_call_summary
390 : {
391 : public:
392 6070116 : isra_call_summary ()
393 0 : : m_arg_flow (), m_return_ignored (false), m_return_returned (false),
394 0 : m_bit_aligned_arg (false), m_before_any_store (false)
395 : {}
396 :
397 : void init_inputs (unsigned arg_count);
398 : void dump (FILE *f);
399 :
400 : /* Information about what formal parameters of the caller are used to compute
401 : individual actual arguments of this call. */
402 : auto_vec <isra_param_flow> m_arg_flow;
403 :
404 : /* Set to true if the call statement does not have a LHS. */
405 : unsigned m_return_ignored : 1;
406 :
407 : /* Set to true if the LHS of call statement is only used to construct the
408 : return value of the caller. */
409 : unsigned m_return_returned : 1;
410 :
411 : /* Set when any of the call arguments are not byte-aligned. */
412 : unsigned m_bit_aligned_arg : 1;
413 :
414 : /* Set to true if the call happend before any (other) store to memory in the
415 : caller. */
416 : unsigned m_before_any_store : 1;
417 : };
418 :
419 : /* Class to manage function summaries. */
420 :
421 : class GTY((user)) ipa_sra_function_summaries
422 : : public function_summary <isra_func_summary *>
423 : {
424 : public:
425 135817 : ipa_sra_function_summaries (symbol_table *table, bool ggc):
426 271634 : function_summary<isra_func_summary *> (table, ggc) { }
427 :
428 : void duplicate (cgraph_node *, cgraph_node *,
429 : isra_func_summary *old_sum,
430 : isra_func_summary *new_sum) final override;
431 : void insert (cgraph_node *, isra_func_summary *) final override;
432 : };
433 :
434 : /* Hook that is called by summary when a node is duplicated. */
435 :
436 : void
437 133675 : ipa_sra_function_summaries::duplicate (cgraph_node *, cgraph_node *,
438 : isra_func_summary *old_sum,
439 : isra_func_summary *new_sum)
440 : {
441 : /* TODO: Somehow stop copying when ISRA is doing the cloning, it is
442 : useless. */
443 133675 : new_sum->m_candidate = old_sum->m_candidate;
444 133675 : new_sum->m_returns_value = old_sum->m_returns_value;
445 133675 : new_sum->m_return_ignored = old_sum->m_return_ignored;
446 133675 : gcc_assert (!old_sum->m_queued);
447 133675 : new_sum->m_queued = false;
448 :
449 133675 : unsigned param_count = vec_safe_length (old_sum->m_parameters);
450 132809 : if (!param_count)
451 : return;
452 132809 : vec_safe_reserve_exact (new_sum->m_parameters, param_count);
453 132809 : new_sum->m_parameters->quick_grow_cleared (param_count);
454 508840 : for (unsigned i = 0; i < param_count; i++)
455 : {
456 376031 : isra_param_desc *s = &(*old_sum->m_parameters)[i];
457 376031 : isra_param_desc *d = &(*new_sum->m_parameters)[i];
458 :
459 376031 : d->param_size_limit = s->param_size_limit;
460 376031 : d->size_reached = s->size_reached;
461 376031 : d->safe_size = s->safe_size;
462 376031 : d->locally_unused = s->locally_unused;
463 376031 : d->split_candidate = s->split_candidate;
464 376031 : d->by_ref = s->by_ref;
465 376031 : d->remove_only_when_retval_removed = s->remove_only_when_retval_removed;
466 376031 : d->split_only_when_retval_removed = s->split_only_when_retval_removed;
467 376031 : d->not_specially_constructed = s->not_specially_constructed;
468 376031 : d->conditionally_dereferenceable = s->conditionally_dereferenceable;
469 376031 : d->safe_size_set = s->safe_size_set;
470 :
471 376031 : unsigned acc_count = vec_safe_length (s->accesses);
472 376031 : vec_safe_reserve_exact (d->accesses, acc_count);
473 528805 : for (unsigned j = 0; j < acc_count; j++)
474 : {
475 152774 : param_access *from = (*s->accesses)[j];
476 152774 : param_access *to = ggc_cleared_alloc<param_access> ();
477 152774 : to->type = from->type;
478 152774 : to->alias_ptr_type = from->alias_ptr_type;
479 152774 : to->unit_offset = from->unit_offset;
480 152774 : to->unit_size = from->unit_size;
481 152774 : to->certain = from->certain;
482 152774 : to->reverse = from->reverse;
483 152774 : d->accesses->quick_push (to);
484 : }
485 : }
486 : }
487 :
488 : /* Pointer to the pass function summary holder. */
489 :
490 : static GTY(()) ipa_sra_function_summaries *func_sums;
491 :
492 : /* Hook that is called by summary when new node appears. */
493 :
494 : void
495 18050 : ipa_sra_function_summaries::insert (cgraph_node *node, isra_func_summary *)
496 : {
497 18050 : if (opt_for_fn (node->decl, flag_ipa_sra))
498 : {
499 18050 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
500 18050 : ipa_sra_summarize_function (node);
501 18050 : pop_cfun ();
502 : }
503 : else
504 0 : func_sums->remove (node);
505 18050 : }
506 :
507 : /* Class to manage call summaries. */
508 :
509 : class ipa_sra_call_summaries: public call_summary <isra_call_summary *>
510 : {
511 : public:
512 135817 : ipa_sra_call_summaries (symbol_table *table):
513 271634 : call_summary<isra_call_summary *> (table) { }
514 :
515 : /* Duplicate info when an edge is cloned. */
516 : void duplicate (cgraph_edge *, cgraph_edge *,
517 : isra_call_summary *old_sum,
518 : isra_call_summary *new_sum) final override;
519 : };
520 :
521 : static ipa_sra_call_summaries *call_sums;
522 :
523 :
524 : /* Initialize m_arg_flow of a particular instance of isra_call_summary.
525 : ARG_COUNT is the number of actual arguments passed. */
526 :
527 : void
528 6997393 : isra_call_summary::init_inputs (unsigned arg_count)
529 : {
530 6997393 : if (arg_count == 0)
531 : {
532 352968 : gcc_checking_assert (m_arg_flow.length () == 0);
533 : return;
534 : }
535 6644425 : if (m_arg_flow.length () == 0)
536 : {
537 3284916 : m_arg_flow.reserve_exact (arg_count);
538 3284916 : m_arg_flow.quick_grow_cleared (arg_count);
539 : }
540 : else
541 3359509 : gcc_checking_assert (arg_count == m_arg_flow.length ());
542 : }
543 :
544 : /* Dump all information in call summary to F. */
545 :
546 : void
547 180 : isra_call_summary::dump (FILE *f)
548 : {
549 180 : if (m_return_ignored)
550 128 : fprintf (f, " return value ignored\n");
551 180 : if (m_return_returned)
552 38 : fprintf (f, " return value used only to compute caller return value\n");
553 180 : if (m_before_any_store)
554 17 : fprintf (f, " happens before any store to memory\n");
555 409 : for (unsigned i = 0; i < m_arg_flow.length (); i++)
556 : {
557 229 : fprintf (f, " Parameter %u:\n", i);
558 229 : isra_param_flow *ipf = &m_arg_flow[i];
559 :
560 229 : if (ipf->length)
561 : {
562 158 : bool first = true;
563 158 : fprintf (f, " Scalar param sources: ");
564 319 : for (int j = 0; j < ipf->length; j++)
565 : {
566 161 : if (!first)
567 3 : fprintf (f, ", ");
568 : else
569 : first = false;
570 161 : fprintf (f, "%i", (int) ipf->inputs[j]);
571 : }
572 158 : fprintf (f, "\n");
573 : }
574 229 : if (ipf->aggregate_pass_through)
575 12 : fprintf (f, " Aggregate pass through from the param given above, "
576 : "unit offset: %u , unit size: %u\n",
577 12 : ipf->unit_offset, ipf->unit_size);
578 217 : else if (ipf->unit_size > 0)
579 52 : fprintf (f, " Known dereferenceable size: %u\n", ipf->unit_size);
580 229 : if (ipf->pointer_pass_through)
581 19 : fprintf (f, " Pointer pass through from the param given above, "
582 19 : "safe_to_import_accesses: %u\n", ipf->safe_to_import_accesses);
583 229 : if (ipf->constructed_for_calls)
584 28 : fprintf (f, " Variable constructed just to be passed to "
585 : "calls.\n");
586 : }
587 180 : }
588 :
589 : /* Duplicate edge summary when an edge is cloned. */
590 :
591 : void
592 651653 : ipa_sra_call_summaries::duplicate (cgraph_edge *, cgraph_edge *,
593 : isra_call_summary *old_sum,
594 : isra_call_summary *new_sum)
595 : {
596 651653 : unsigned arg_count = old_sum->m_arg_flow.length ();
597 651653 : new_sum->init_inputs (arg_count);
598 1892255 : for (unsigned i = 0; i < arg_count; i++)
599 1240602 : new_sum->m_arg_flow[i] = old_sum->m_arg_flow[i];
600 :
601 651653 : new_sum->m_return_ignored = old_sum->m_return_ignored;
602 651653 : new_sum->m_return_returned = old_sum->m_return_returned;
603 651653 : new_sum->m_bit_aligned_arg = old_sum->m_bit_aligned_arg;
604 651653 : new_sum->m_before_any_store = old_sum->m_before_any_store;
605 651653 : }
606 :
607 :
608 : /* With all GTY stuff done, we can move to anonymous namespace. */
609 : namespace {
610 : /* Quick mapping from a decl to its param descriptor. */
611 :
612 : hash_map<tree, gensum_param_desc *> *decl2desc;
613 :
614 : /* All local DECLs ever loaded from of and of those that have their address
615 : assigned to a variable. */
616 :
617 : hash_set <tree> *loaded_decls;
618 :
619 : /* Countdown of allowed Alias Analysis steps during summary building. */
620 :
621 : int aa_walking_limit;
622 :
623 : /* This is a table in which for each basic block and parameter there is a
624 : distance (offset + size) in that parameter which is dereferenced and
625 : accessed in that BB. */
626 : HOST_WIDE_INT *bb_dereferences = NULL;
627 : /* How many by-reference parameters there are in the current function. */
628 : int unsafe_by_ref_count;
629 :
630 : /* Bitmap of BBs that can cause the function to "stop" progressing by
631 : returning, throwing externally, looping infinitely or calling a function
632 : which might abort etc.. */
633 : bitmap final_bbs;
634 :
635 : /* Obstack to allocate various small structures required only when generating
636 : summary for a function. */
637 : struct obstack gensum_obstack;
638 :
639 : /* Return false the function is apparently unsuitable for IPA-SRA based on it's
640 : attributes, return true otherwise. NODE is the cgraph node of the current
641 : function. */
642 :
643 : static bool
644 1267514 : ipa_sra_preliminary_function_checks (cgraph_node *node)
645 : {
646 1267514 : if (!node->can_change_signature)
647 : {
648 35674 : if (dump_file)
649 0 : fprintf (dump_file, "Function cannot change signature.\n");
650 35674 : return false;
651 : }
652 :
653 1231840 : if (!tree_versionable_function_p (node->decl))
654 : {
655 118483 : if (dump_file)
656 7 : fprintf (dump_file, "Function is not versionable.\n");
657 118483 : return false;
658 : }
659 :
660 1113357 : if (!opt_for_fn (node->decl, optimize)
661 1113357 : || !opt_for_fn (node->decl, flag_ipa_sra))
662 : {
663 240 : if (dump_file)
664 0 : fprintf (dump_file, "Not optimizing or IPA-SRA turned off for this "
665 : "function.\n");
666 240 : return false;
667 : }
668 :
669 1113117 : if (DECL_VIRTUAL_P (node->decl))
670 : {
671 31315 : if (dump_file)
672 0 : fprintf (dump_file, "Function is a virtual method.\n");
673 31315 : return false;
674 : }
675 :
676 1081802 : struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
677 1081802 : if (fun->stdarg)
678 : {
679 124 : if (dump_file)
680 0 : fprintf (dump_file, "Function uses stdarg. \n");
681 124 : return false;
682 : }
683 :
684 1081678 : if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
685 : {
686 49323 : if (dump_file)
687 0 : fprintf (dump_file, "Always inline function will be inlined "
688 : "anyway. \n");
689 49323 : return false;
690 : }
691 :
692 : return true;
693 : }
694 :
695 : /* Print access tree starting at ACCESS to F. */
696 :
697 : static void
698 69 : dump_gensum_access (FILE *f, gensum_param_access *access, unsigned indent)
699 : {
700 69 : fprintf (f, " ");
701 211 : for (unsigned i = 0; i < indent; i++)
702 142 : fprintf (f, " ");
703 69 : fprintf (f, " * Access to offset: " HOST_WIDE_INT_PRINT_DEC,
704 : access->offset);
705 69 : fprintf (f, ", size: " HOST_WIDE_INT_PRINT_DEC, access->size);
706 69 : fprintf (f, ", type: ");
707 69 : print_generic_expr (f, access->type);
708 69 : fprintf (f, ", alias_ptr_type: ");
709 69 : print_generic_expr (f, access->alias_ptr_type);
710 69 : fprintf (f, ", load_count: ");
711 69 : access->load_count.dump (f);
712 69 : fprintf (f, ", nonarg: %u, reverse: %u\n", access->nonarg, access->reverse);
713 69 : for (gensum_param_access *ch = access->first_child;
714 71 : ch;
715 2 : ch = ch->next_sibling)
716 2 : dump_gensum_access (f, ch, indent + 2);
717 69 : }
718 :
719 :
720 : /* Print access tree starting at ACCESS to F. */
721 :
722 : static void
723 152 : dump_isra_access (FILE *f, param_access *access)
724 : {
725 152 : fprintf (f, " * Access to unit offset: %u", access->unit_offset);
726 152 : fprintf (f, ", unit size: %u", access->unit_size);
727 152 : fprintf (f, ", type: ");
728 152 : print_generic_expr (f, access->type);
729 152 : fprintf (f, ", alias_ptr_type: ");
730 152 : print_generic_expr (f, access->alias_ptr_type);
731 152 : if (access->certain)
732 130 : fprintf (f, ", certain");
733 : else
734 22 : fprintf (f, ", not certain");
735 152 : if (access->reverse)
736 0 : fprintf (f, ", reverse");
737 152 : fprintf (f, "\n");
738 152 : }
739 :
740 : /* Dump access tree starting at ACCESS to stderr. */
741 :
742 : DEBUG_FUNCTION void
743 0 : debug_isra_access (param_access *access)
744 : {
745 0 : dump_isra_access (stderr, access);
746 0 : }
747 :
748 : /* Dump DESC to F. */
749 :
750 : static void
751 332 : dump_gensum_param_descriptor (FILE *f, gensum_param_desc *desc)
752 : {
753 332 : if (desc->locally_unused)
754 166 : fprintf (f, " unused with %i call_uses%s\n", desc->call_uses,
755 166 : desc->remove_only_when_retval_removed ?
756 : " remove_only_when_retval_removed" : "");
757 332 : if (!desc->split_candidate)
758 : {
759 244 : fprintf (f, " not a candidate\n");
760 244 : return;
761 : }
762 88 : if (desc->by_ref)
763 73 : fprintf (f, " %s%s%s by_ref with %u pass throughs\n",
764 73 : desc->safe_ref ? "safe" : "unsafe",
765 73 : desc->conditionally_dereferenceable
766 : ? " conditionally_dereferenceable" : "",
767 73 : desc->split_only_when_retval_removed
768 : ? " split_only_when_retval_removed" : "",
769 : desc->ptr_pt_count);
770 :
771 155 : for (gensum_param_access *acc = desc->accesses; acc; acc = acc->next_sibling)
772 67 : dump_gensum_access (f, acc, 2);
773 : }
774 :
775 : /* Dump all parameter descriptors in IFS, assuming it describes FNDECL, to
776 : F. */
777 :
778 : static void
779 135 : dump_gensum_param_descriptors (FILE *f, tree fndecl,
780 : vec<gensum_param_desc> *param_descriptions)
781 : {
782 135 : tree parm = DECL_ARGUMENTS (fndecl);
783 135 : for (unsigned i = 0;
784 467 : i < param_descriptions->length ();
785 332 : ++i, parm = DECL_CHAIN (parm))
786 : {
787 332 : fprintf (f, " Descriptor for parameter %i ", i);
788 332 : print_generic_expr (f, parm, TDF_UID);
789 332 : fprintf (f, "\n");
790 332 : dump_gensum_param_descriptor (f, &(*param_descriptions)[i]);
791 : }
792 135 : }
793 :
794 :
795 : /* Dump DESC to F. If HINTS is true, also dump IPA-analysis computed
796 : hints. */
797 :
798 : static void
799 646 : dump_isra_param_descriptor (FILE *f, isra_param_desc *desc, bool hints)
800 : {
801 646 : if (desc->locally_unused)
802 : {
803 335 : fprintf (f, " (locally) unused\n");
804 : }
805 646 : if (!desc->split_candidate)
806 : {
807 471 : fprintf (f, " not a candidate for splitting");
808 471 : if (hints && desc->by_ref && desc->safe_size_set)
809 9 : fprintf (f, ", safe_size: %u", (unsigned) desc->safe_size);
810 471 : fprintf (f, "\n");
811 471 : return;
812 : }
813 350 : fprintf (f, " param_size_limit: %u, size_reached: %u%s",
814 175 : desc->param_size_limit, desc->size_reached,
815 175 : desc->by_ref ? ", by_ref" : "");
816 175 : if (desc->remove_only_when_retval_removed)
817 12 : fprintf (f, ", remove_only_when_retval_removed");
818 175 : if (desc->split_only_when_retval_removed)
819 4 : fprintf (f, ", split_only_when_retval_removed");
820 175 : if (desc->by_ref && desc->conditionally_dereferenceable)
821 33 : fprintf (f, ", conditionally_dereferenceable");
822 175 : if (hints)
823 : {
824 6 : if (desc->by_ref && !desc->not_specially_constructed)
825 3 : fprintf (f, ", args_specially_constructed");
826 6 : if (desc->by_ref && desc->safe_size_set)
827 5 : fprintf (f, ", safe_size: %u", (unsigned) desc->safe_size);
828 : }
829 175 : fprintf (f, "\n");
830 :
831 327 : for (unsigned i = 0; i < vec_safe_length (desc->accesses); ++i)
832 : {
833 152 : param_access *access = (*desc->accesses)[i];
834 152 : dump_isra_access (f, access);
835 : }
836 : }
837 :
838 : /* Dump all parameter descriptors in IFS, assuming it describes FNDECL, to F.
839 : If HINTS is true, also dump IPA-analysis computed hints. */
840 :
841 : static void
842 135 : dump_isra_param_descriptors (FILE *f, tree fndecl, isra_func_summary *ifs,
843 : bool hints)
844 : {
845 135 : tree parm = DECL_ARGUMENTS (fndecl);
846 135 : if (!ifs->m_parameters)
847 : {
848 0 : fprintf (f, " parameter descriptors not available\n");
849 0 : return;
850 : }
851 :
852 : for (unsigned i = 0;
853 467 : i < ifs->m_parameters->length ();
854 332 : ++i, parm = DECL_CHAIN (parm))
855 : {
856 332 : fprintf (f, " Descriptor for parameter %i ", i);
857 332 : print_generic_expr (f, parm, TDF_UID);
858 332 : fprintf (f, "\n");
859 332 : dump_isra_param_descriptor (f, &(*ifs->m_parameters)[i], hints);
860 : }
861 : }
862 :
863 : /* Add SRC to inputs of PARAM_FLOW, unless it would exceed storage. If the
864 : function fails return false, otherwise return true. SRC must fit into an
865 : unsigned char. Used for purposes of transitive unused parameter
866 : removal. */
867 :
868 : static bool
869 1318723 : add_src_to_param_flow (isra_param_flow *param_flow, int src)
870 : {
871 1318723 : gcc_checking_assert (src >= 0 && src <= UCHAR_MAX);
872 1318723 : if (param_flow->length == IPA_SRA_MAX_PARAM_FLOW_LEN)
873 : return false;
874 :
875 1318720 : param_flow->inputs[(int) param_flow->length] = src;
876 1318720 : param_flow->length++;
877 1318720 : return true;
878 : }
879 :
880 : /* Add a SRC to the inputs of PARAM_FLOW unless it is already there and assert
881 : it is the only input. Used for purposes of transitive parameter
882 : splitting. */
883 :
884 : static void
885 787073 : set_single_param_flow_source (isra_param_flow *param_flow, int src)
886 : {
887 787073 : gcc_checking_assert (src >= 0 && src <= UCHAR_MAX);
888 787073 : if (param_flow->length == 0)
889 : {
890 120856 : param_flow->inputs[0] = src;
891 120856 : param_flow->length = 1;
892 : }
893 666217 : else if (param_flow->length == 1)
894 666217 : gcc_assert (param_flow->inputs[0] == src);
895 : else
896 0 : gcc_unreachable ();
897 787073 : }
898 :
899 : /* Assert that there is only a single value in PARAM_FLOW's inputs and return
900 : it. */
901 :
902 : static unsigned
903 1289726 : get_single_param_flow_source (const isra_param_flow *param_flow)
904 : {
905 1289726 : gcc_assert (param_flow->length == 1);
906 1289726 : return param_flow->inputs[0];
907 : }
908 :
909 : /* Inspect all uses of NAME and simple arithmetic calculations involving NAME
910 : in FUN represented with NODE and return a negative number if any of them is
911 : used for something else than either an actual call argument, simple return,
912 : simple arithmetic operation or debug statement. If there are no such uses,
913 : return the number of actual arguments that this parameter eventually feeds
914 : to (or zero if there is none). If there are any simple return uses, set
915 : DESC->remove_only_when_retval_removed. For any such parameter, mark
916 : PARM_NUM as one of its sources. ANALYZED is a bitmap that tracks which SSA
917 : names we have already started investigating. */
918 :
919 : static int
920 3381211 : isra_track_scalar_value_uses (function *fun, cgraph_node *node, tree name,
921 : int parm_num, bitmap analyzed,
922 : gensum_param_desc *desc)
923 : {
924 3381211 : int res = 0;
925 3381211 : imm_use_iterator imm_iter;
926 3381211 : gimple *stmt;
927 :
928 10342965 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
929 : {
930 5204390 : if (is_gimple_debug (stmt)
931 5204390 : || gimple_clobber_p (stmt))
932 872942 : continue;
933 :
934 : /* TODO: We could handle at least const builtin functions like arithmetic
935 : operations below. */
936 4331448 : if (is_gimple_call (stmt))
937 : {
938 1331223 : int all_uses = 0;
939 1331223 : use_operand_p use_p;
940 2676100 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
941 1344877 : all_uses++;
942 :
943 1331223 : gcall *call = as_a <gcall *> (stmt);
944 1331223 : unsigned arg_count;
945 1331223 : if (gimple_call_internal_p (call)
946 1331223 : || (arg_count = gimple_call_num_args (call)) == 0)
947 : {
948 : res = -1;
949 : break;
950 : }
951 :
952 1326477 : cgraph_edge *cs = node->get_edge (stmt);
953 1326477 : gcc_checking_assert (cs);
954 1326477 : isra_call_summary *csum = call_sums->get_create (cs);
955 1326477 : csum->init_inputs (arg_count);
956 :
957 1326477 : int simple_uses = 0;
958 6024998 : for (unsigned i = 0; i < arg_count; i++)
959 4698524 : if (gimple_call_arg (call, i) == name)
960 : {
961 1318723 : if (!add_src_to_param_flow (&csum->m_arg_flow[i], parm_num))
962 : {
963 : simple_uses = -1;
964 : break;
965 : }
966 1318720 : simple_uses++;
967 : }
968 :
969 1326477 : if (simple_uses < 0
970 1326477 : || all_uses != simple_uses)
971 : {
972 : res = -1;
973 : break;
974 : }
975 1305313 : res += all_uses;
976 : }
977 3000225 : else if (!stmt_unremovable_because_of_non_call_eh_p (fun, stmt)
978 3000225 : && ((is_gimple_assign (stmt) && !gimple_has_volatile_ops (stmt))
979 673072 : || gimple_code (stmt) == GIMPLE_PHI))
980 : {
981 2393939 : tree lhs;
982 2393939 : if (gimple_code (stmt) == GIMPLE_PHI)
983 174073 : lhs = gimple_phi_result (stmt);
984 : else
985 2219866 : lhs = gimple_assign_lhs (stmt);
986 :
987 2393939 : if (TREE_CODE (lhs) != SSA_NAME)
988 : {
989 : res = -1;
990 : break;
991 : }
992 1998633 : gcc_assert (!gimple_vdef (stmt));
993 1998633 : if (bitmap_set_bit (analyzed, SSA_NAME_VERSION (lhs)))
994 : {
995 1904963 : int tmp = isra_track_scalar_value_uses (fun, node, lhs, parm_num,
996 : analyzed, desc);
997 1904963 : if (tmp < 0)
998 : {
999 : res = tmp;
1000 : break;
1001 : }
1002 1140629 : res += tmp;
1003 : }
1004 : }
1005 606286 : else if (greturn *gr = dyn_cast<greturn *>(stmt))
1006 : {
1007 167989 : tree rv = gimple_return_retval (gr);
1008 167989 : if (rv != name)
1009 : {
1010 : res = -1;
1011 : break;
1012 : }
1013 167989 : desc->remove_only_when_retval_removed = true;
1014 : }
1015 : else
1016 : {
1017 : res = -1;
1018 : break;
1019 : }
1020 3381211 : }
1021 3381211 : return res;
1022 : }
1023 :
1024 : /* Inspect all uses of PARM, which must be a gimple register, in FUN (which is
1025 : also described by NODE) and simple arithmetic calculations involving PARM
1026 : and return false if any of them is used for something else than either an
1027 : actual call argument, simple return, simple arithmetic operation or debug
1028 : statement. If there are no such uses, return true and store the number of
1029 : actual arguments that this parameter eventually feeds to (or zero if there
1030 : is none) to DESC->call_uses and set DESC->remove_only_when_retval_removed if
1031 : there are any uses in return statemens. For any such parameter, mark
1032 : PARM_NUM as one of its sources.
1033 :
1034 : This function is similar to ptr_parm_has_nonarg_uses but its results are
1035 : meant for unused parameter removal, as opposed to splitting of parameters
1036 : passed by reference or converting them to passed by value. */
1037 :
1038 : static bool
1039 2171783 : isra_track_scalar_param_local_uses (function *fun, cgraph_node *node, tree parm,
1040 : int parm_num, gensum_param_desc *desc)
1041 : {
1042 2171783 : gcc_checking_assert (is_gimple_reg (parm));
1043 :
1044 2171783 : tree name = ssa_default_def (fun, parm);
1045 2171783 : if (!name || has_zero_uses (name))
1046 : {
1047 695535 : desc->call_uses = 0;
1048 695535 : return false;
1049 : }
1050 :
1051 : /* Edge summaries can only handle callers with fewer than 256 parameters. */
1052 1476248 : if (parm_num > UCHAR_MAX)
1053 : return true;
1054 :
1055 1476248 : bitmap analyzed = BITMAP_ALLOC (NULL);
1056 1476248 : int call_uses = isra_track_scalar_value_uses (fun, node, name, parm_num,
1057 : analyzed, desc);
1058 1476248 : BITMAP_FREE (analyzed);
1059 1476248 : if (call_uses < 0)
1060 : return true;
1061 616735 : desc->call_uses = call_uses;
1062 616735 : return false;
1063 : }
1064 :
1065 : /* Scan immediate uses of a default definition SSA name of a parameter PARM and
1066 : examine whether there are any nonarg uses that are not actual arguments or
1067 : otherwise infeasible uses. If so, return true, otherwise return false.
1068 : Create pass-through IPA flow records for any direct uses as argument calls
1069 : and if returning false, store their number into DESC->ptr_pt_count. If
1070 : removal of return value would still allow splitting, return true but set
1071 : DESC->split_only_when_retval_removed. NODE and FUN must represent the
1072 : function that is currently analyzed, PARM_NUM must be the index of the
1073 : analyzed parameter.
1074 :
1075 : This function is similar to isra_track_scalar_param_local_uses but its
1076 : results are meant for splitting of parameters passed by reference or turning
1077 : them into bits passed by value, as opposed to generic unused parameter
1078 : removal. */
1079 :
1080 : static bool
1081 1145772 : ptr_parm_has_nonarg_uses (cgraph_node *node, function *fun, tree parm,
1082 : int parm_num, gensum_param_desc *desc)
1083 : {
1084 1145772 : imm_use_iterator ui;
1085 1145772 : gimple *stmt;
1086 1145772 : tree name = ssa_default_def (fun, parm);
1087 1145772 : bool ret = false;
1088 1145772 : unsigned pt_count = 0;
1089 :
1090 1145772 : if (!name || has_zero_uses (name))
1091 : return false;
1092 :
1093 : /* Edge summaries can only handle callers with fewer than 256 parameters. */
1094 1033722 : if (parm_num > UCHAR_MAX)
1095 : return true;
1096 :
1097 3869440 : FOR_EACH_IMM_USE_STMT (stmt, ui, name)
1098 : {
1099 2335420 : unsigned uses_ok = 0;
1100 2335420 : use_operand_p use_p;
1101 :
1102 2335420 : if (is_gimple_debug (stmt)
1103 2335420 : || gimple_clobber_p (stmt))
1104 594249 : continue;
1105 :
1106 1741171 : if (gimple_assign_single_p (stmt))
1107 : {
1108 879574 : tree rhs = gimple_assign_rhs1 (stmt);
1109 879574 : if (!TREE_THIS_VOLATILE (rhs))
1110 : {
1111 1659810 : while (handled_component_p (rhs))
1112 780312 : rhs = TREE_OPERAND (rhs, 0);
1113 879498 : if (TREE_CODE (rhs) == MEM_REF
1114 602828 : && TREE_OPERAND (rhs, 0) == name
1115 598955 : && integer_zerop (TREE_OPERAND (rhs, 1))
1116 1434865 : && types_compatible_p (TREE_TYPE (rhs),
1117 555367 : TREE_TYPE (TREE_TYPE (name))))
1118 : uses_ok++;
1119 : }
1120 : }
1121 861597 : else if (is_gimple_call (stmt))
1122 : {
1123 715239 : gcall *call = as_a <gcall *> (stmt);
1124 715239 : unsigned arg_count;
1125 715239 : if (gimple_call_internal_p (call)
1126 715239 : || (arg_count = gimple_call_num_args (call)) == 0)
1127 : {
1128 : ret = true;
1129 : break;
1130 : }
1131 :
1132 714466 : cgraph_edge *cs = node->get_edge (stmt);
1133 714466 : gcc_checking_assert (cs);
1134 714466 : isra_call_summary *csum = call_sums->get_create (cs);
1135 714466 : csum->init_inputs (arg_count);
1136 :
1137 2731001 : for (unsigned i = 0; i < arg_count; ++i)
1138 : {
1139 2016535 : tree arg = gimple_call_arg (stmt, i);
1140 :
1141 2016535 : if (arg == name)
1142 : {
1143 : /* TODO: Allow &MEM_REF[name + offset] here,
1144 : ipa_param_body_adjustments::modify_call_stmt has to be
1145 : adjusted too. */
1146 702995 : csum->m_arg_flow[i].pointer_pass_through = true;
1147 702995 : set_single_param_flow_source (&csum->m_arg_flow[i], parm_num);
1148 702995 : pt_count++;
1149 702995 : uses_ok++;
1150 702995 : continue;
1151 : }
1152 :
1153 1313540 : if (!TREE_THIS_VOLATILE (arg))
1154 : {
1155 1340757 : while (handled_component_p (arg))
1156 27217 : arg = TREE_OPERAND (arg, 0);
1157 1313540 : if (TREE_CODE (arg) == MEM_REF
1158 20861 : && TREE_OPERAND (arg, 0) == name
1159 11325 : && integer_zerop (TREE_OPERAND (arg, 1))
1160 1324863 : && types_compatible_p (TREE_TYPE (arg),
1161 11323 : TREE_TYPE (TREE_TYPE (name))))
1162 8490 : uses_ok++;
1163 : }
1164 : }
1165 : }
1166 146358 : else if (greturn *gr = dyn_cast<greturn *>(stmt))
1167 : {
1168 9655 : tree rv = gimple_return_retval (gr);
1169 9655 : if (rv == name)
1170 : {
1171 9655 : uses_ok++;
1172 : /* Analysis for feasibility of removal must have already reached
1173 : the conclusion that the flag must be set if it completed. */
1174 9655 : gcc_assert (!desc->locally_unused
1175 : || desc->remove_only_when_retval_removed);
1176 9655 : desc->split_only_when_retval_removed = true;
1177 : }
1178 : }
1179 :
1180 : /* If the number of valid uses does not match the number of
1181 : uses in this stmt there is an unhandled use. */
1182 1740398 : unsigned all_uses = 0;
1183 3487807 : FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
1184 1747409 : all_uses++;
1185 :
1186 1740398 : gcc_checking_assert (uses_ok <= all_uses);
1187 1740398 : if (uses_ok != all_uses)
1188 : {
1189 : ret = true;
1190 : break;
1191 : }
1192 1033722 : }
1193 :
1194 1033722 : desc->ptr_pt_count = pt_count;
1195 1033722 : return ret;
1196 : }
1197 :
1198 : /* Initialize vector of parameter descriptors of NODE. Return true if there
1199 : are any candidates for splitting or unused aggregate parameter removal (the
1200 : function may return false if there are candidates for removal of register
1201 : parameters). */
1202 :
1203 : static bool
1204 837454 : create_parameter_descriptors (cgraph_node *node,
1205 : vec<gensum_param_desc> *param_descriptions)
1206 : {
1207 837454 : function *fun = DECL_STRUCT_FUNCTION (node->decl);
1208 837454 : bool ret = false;
1209 :
1210 837454 : int num = 0;
1211 837454 : for (tree parm = DECL_ARGUMENTS (node->decl);
1212 3221632 : parm;
1213 2384178 : parm = DECL_CHAIN (parm), num++)
1214 : {
1215 2384178 : const char *msg;
1216 2384178 : gensum_param_desc *desc = &(*param_descriptions)[num];
1217 : /* param_descriptions vector is grown cleared in the caller. */
1218 2384178 : desc->param_number = num;
1219 2384178 : decl2desc->put (parm, desc);
1220 :
1221 2384178 : if (dump_file && (dump_flags & TDF_DETAILS))
1222 39 : print_generic_expr (dump_file, parm, TDF_UID);
1223 :
1224 2384178 : tree type = TREE_TYPE (parm);
1225 2384178 : if (TREE_THIS_VOLATILE (parm))
1226 : {
1227 51 : if (dump_file && (dump_flags & TDF_DETAILS))
1228 0 : fprintf (dump_file, " not a candidate, is volatile\n");
1229 1780258 : continue;
1230 : }
1231 2384127 : if (!is_gimple_reg_type (type) && is_va_list_type (type))
1232 : {
1233 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1234 0 : fprintf (dump_file, " not a candidate, is a va_list type\n");
1235 0 : continue;
1236 : }
1237 2384127 : if (TREE_ADDRESSABLE (parm))
1238 : {
1239 36280 : if (dump_file && (dump_flags & TDF_DETAILS))
1240 0 : fprintf (dump_file, " not a candidate, is addressable\n");
1241 36280 : continue;
1242 : }
1243 2347847 : if (TREE_ADDRESSABLE (type))
1244 : {
1245 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1246 0 : fprintf (dump_file, " not a candidate, type cannot be split\n");
1247 0 : continue;
1248 : }
1249 :
1250 2347847 : if (is_gimple_reg (parm)
1251 2347847 : && !isra_track_scalar_param_local_uses (fun, node, parm, num, desc))
1252 : {
1253 1312270 : desc->locally_unused = true;
1254 :
1255 1312270 : if (dump_file && (dump_flags & TDF_DETAILS))
1256 19 : fprintf (dump_file, " is a scalar with only %i call uses%s\n",
1257 : desc->call_uses,
1258 19 : desc->remove_only_when_retval_removed
1259 : ? " and return uses" : "");
1260 : }
1261 :
1262 2347847 : if (POINTER_TYPE_P (type))
1263 : {
1264 1152313 : desc->by_ref = true;
1265 1152313 : if (TREE_CODE (type) == REFERENCE_TYPE
1266 1152313 : || (num == 0
1267 559843 : && TREE_CODE (TREE_TYPE (node->decl)) == METHOD_TYPE))
1268 491214 : desc->safe_ref = true;
1269 : else
1270 661099 : desc->safe_ref = false;
1271 1152313 : type = TREE_TYPE (type);
1272 :
1273 1157335 : if (TREE_CODE (type) == FUNCTION_TYPE
1274 1152313 : || TREE_CODE (type) == METHOD_TYPE)
1275 : {
1276 5022 : if (dump_file && (dump_flags & TDF_DETAILS))
1277 0 : fprintf (dump_file, " not a candidate, reference to "
1278 : "a function\n");
1279 5022 : continue;
1280 : }
1281 1147291 : if (TYPE_VOLATILE (type))
1282 : {
1283 1128 : if (dump_file && (dump_flags & TDF_DETAILS))
1284 0 : fprintf (dump_file, " not a candidate, reference to "
1285 : "a volatile type\n");
1286 1128 : continue;
1287 : }
1288 1146163 : if (TREE_CODE (type) == ARRAY_TYPE
1289 1146163 : && TYPE_NONALIASED_COMPONENT (type))
1290 : {
1291 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1292 0 : fprintf (dump_file, " not a candidate, reference to "
1293 : "a nonaliased component array\n");
1294 0 : continue;
1295 : }
1296 1146163 : if (!is_gimple_reg (parm))
1297 : {
1298 6 : if (dump_file && (dump_flags & TDF_DETAILS))
1299 0 : fprintf (dump_file, " not a candidate, a reference which is "
1300 : "not a gimple register (probably addressable)\n");
1301 6 : continue;
1302 : }
1303 1146157 : if (is_va_list_type (type))
1304 : {
1305 385 : if (dump_file && (dump_flags & TDF_DETAILS))
1306 0 : fprintf (dump_file, " not a candidate, reference to "
1307 : "a va list\n");
1308 385 : continue;
1309 : }
1310 1145772 : if (ptr_parm_has_nonarg_uses (node, fun, parm, num, desc))
1311 : {
1312 533424 : if (dump_file && (dump_flags & TDF_DETAILS))
1313 7 : fprintf (dump_file, " not a candidate, reference has "
1314 : "nonarg uses\n");
1315 533424 : continue;
1316 : }
1317 : }
1318 1195534 : else if (!AGGREGATE_TYPE_P (type))
1319 : {
1320 : /* This is in an else branch because scalars passed by reference are
1321 : still candidates to be passed by value. */
1322 1019671 : if (dump_file && (dump_flags & TDF_DETAILS))
1323 17 : fprintf (dump_file, " not a candidate, not an aggregate\n");
1324 1019671 : continue;
1325 : }
1326 :
1327 788211 : if (!COMPLETE_TYPE_P (type))
1328 : {
1329 170950 : if (dump_file && (dump_flags & TDF_DETAILS))
1330 0 : fprintf (dump_file, " not a candidate, not a complete type\n");
1331 170950 : continue;
1332 : }
1333 617261 : if (!tree_fits_uhwi_p (TYPE_SIZE (type)))
1334 : {
1335 2 : if (dump_file && (dump_flags & TDF_DETAILS))
1336 0 : fprintf (dump_file, " not a candidate, size not representable\n");
1337 2 : continue;
1338 : }
1339 617259 : unsigned HOST_WIDE_INT type_size
1340 617259 : = tree_to_uhwi (TYPE_SIZE (type)) / BITS_PER_UNIT;
1341 621506 : if (type_size == 0
1342 617259 : || type_size >= ISRA_ARG_SIZE_LIMIT)
1343 : {
1344 4247 : if (dump_file && (dump_flags & TDF_DETAILS))
1345 0 : fprintf (dump_file, " not a candidate, has zero or huge size\n");
1346 4247 : continue;
1347 : }
1348 613012 : if (type_internals_preclude_sra_p (type, &msg))
1349 : {
1350 9092 : if (dump_file && (dump_flags & TDF_DETAILS))
1351 0 : fprintf (dump_file, " not a candidate, %s\n", msg);
1352 9092 : continue;
1353 : }
1354 :
1355 603920 : if (dump_file && (dump_flags & TDF_DETAILS))
1356 15 : fprintf (dump_file, " is a candidate\n");
1357 :
1358 603920 : ret = true;
1359 603920 : desc->split_candidate = true;
1360 603920 : if (desc->by_ref && !desc->safe_ref)
1361 195238 : desc->deref_index = unsafe_by_ref_count++;
1362 : }
1363 837454 : return ret;
1364 : }
1365 :
1366 : /* Return pointer to descriptor of parameter DECL or NULL if it cannot be
1367 : found, which happens if DECL is for a static chain. */
1368 :
1369 : static gensum_param_desc *
1370 2442237 : get_gensum_param_desc (tree decl)
1371 : {
1372 2442237 : if (!decl2desc)
1373 : return NULL;
1374 2269791 : gcc_checking_assert (TREE_CODE (decl) == PARM_DECL);
1375 2269791 : gensum_param_desc **slot = decl2desc->get (decl);
1376 2269791 : if (!slot)
1377 : /* This can happen for static chains which we cannot handle so far. */
1378 : return NULL;
1379 2244374 : gcc_checking_assert (*slot);
1380 : return *slot;
1381 : }
1382 :
1383 :
1384 : /* Remove parameter described by DESC from candidates for IPA-SRA splitting and
1385 : write REASON to the dump file if there is one. */
1386 :
1387 : static void
1388 160854 : disqualify_split_candidate (gensum_param_desc *desc, const char *reason)
1389 : {
1390 160854 : if (!desc->split_candidate)
1391 : return;
1392 :
1393 160825 : if (dump_file && (dump_flags & TDF_DETAILS))
1394 0 : fprintf (dump_file, "! Disqualifying parameter number %i - %s\n",
1395 : desc->param_number, reason);
1396 :
1397 160825 : desc->split_candidate = false;
1398 : }
1399 :
1400 : /* Remove DECL from candidates for IPA-SRA and write REASON to the dump file if
1401 : there is one. */
1402 :
1403 : static void
1404 177 : disqualify_split_candidate (tree decl, const char *reason)
1405 : {
1406 177 : gensum_param_desc *desc = get_gensum_param_desc (decl);
1407 177 : if (desc)
1408 29 : disqualify_split_candidate (desc, reason);
1409 177 : }
1410 :
1411 : /* Allocate a new access to DESC and fill it in with OFFSET and SIZE. But
1412 : first, check that there are not too many of them already. If so, do not
1413 : allocate anything and return NULL. */
1414 :
1415 : static gensum_param_access *
1416 432459 : allocate_access (gensum_param_desc *desc,
1417 : HOST_WIDE_INT offset, HOST_WIDE_INT size)
1418 : {
1419 432459 : if (desc->access_count
1420 432459 : == (unsigned) param_ipa_sra_max_replacements)
1421 : {
1422 0 : disqualify_split_candidate (desc, "Too many replacement candidates");
1423 0 : return NULL;
1424 : }
1425 :
1426 432459 : gensum_param_access *access
1427 432459 : = (gensum_param_access *) obstack_alloc (&gensum_obstack,
1428 : sizeof (gensum_param_access));
1429 432459 : memset (access, 0, sizeof (*access));
1430 432459 : access->offset = offset;
1431 432459 : access->size = size;
1432 432459 : access->load_count = profile_count::zero ();
1433 432459 : return access;
1434 : }
1435 :
1436 : /* In what context scan_expr_access has been called, whether it deals with a
1437 : load, a function argument, or a store. Please note that in rare
1438 : circumstances when it is not clear if the access is a load or store,
1439 : ISRA_CTX_STORE is used too. */
1440 :
1441 : enum isra_scan_context {ISRA_CTX_LOAD, ISRA_CTX_ARG, ISRA_CTX_STORE};
1442 :
1443 : /* Return an access describing memory access to the variable described by DESC
1444 : at OFFSET with SIZE in context CTX, starting at pointer to the linked list
1445 : at a certain tree level FIRST. Attempt to create it and put into the
1446 : appropriate place in the access tree if does not exist, but fail and return
1447 : NULL if there are already too many accesses, if it would create a partially
1448 : overlapping access or if an access would end up within a pre-existing
1449 : non-call access. */
1450 :
1451 : static gensum_param_access *
1452 508242 : get_access_1 (gensum_param_desc *desc, gensum_param_access **first,
1453 : HOST_WIDE_INT offset, HOST_WIDE_INT size, isra_scan_context ctx)
1454 : {
1455 518370 : gensum_param_access *access = *first, **ptr = first;
1456 :
1457 518370 : if (!access)
1458 : {
1459 : /* No pre-existing access at this level, just create it. */
1460 282141 : gensum_param_access *a = allocate_access (desc, offset, size);
1461 282141 : if (!a)
1462 : return NULL;
1463 282141 : *first = a;
1464 282141 : return *first;
1465 : }
1466 :
1467 236229 : if (access->offset >= offset + size)
1468 : {
1469 : /* We want to squeeze it in front of the very first access, just do
1470 : it. */
1471 44178 : gensum_param_access *r = allocate_access (desc, offset, size);
1472 44178 : if (!r)
1473 : return NULL;
1474 44178 : r->next_sibling = access;
1475 44178 : *first = r;
1476 44178 : return r;
1477 : }
1478 :
1479 : /* Skip all accesses that have to come before us until the next sibling is
1480 : already too far. */
1481 280936 : while (offset >= access->offset + access->size
1482 183677 : && access->next_sibling
1483 380272 : && access->next_sibling->offset < offset + size)
1484 : {
1485 88885 : ptr = &access->next_sibling;
1486 88885 : access = access->next_sibling;
1487 : }
1488 :
1489 : /* At this point we know we do not belong before access. */
1490 192051 : gcc_assert (access->offset < offset + size);
1491 :
1492 192051 : if (access->offset == offset && access->size == size)
1493 : /* We found what we were looking for. */
1494 : return access;
1495 :
1496 121159 : if (access->offset <= offset
1497 118037 : && access->offset + access->size >= offset + size)
1498 : {
1499 : /* We fit into access which is larger than us. We need to find/create
1500 : something below access. But we only allow nesting in call
1501 : arguments. */
1502 11738 : if (access->nonarg)
1503 : return NULL;
1504 :
1505 10128 : return get_access_1 (desc, &access->first_child, offset, size, ctx);
1506 : }
1507 :
1508 109421 : if (offset <= access->offset
1509 14629 : && offset + size >= access->offset + access->size)
1510 : /* We are actually bigger than access, which fully fits into us, take its
1511 : place and make all accesses fitting into it its children. */
1512 : {
1513 : /* But first, we only allow nesting in call arguments so check if that is
1514 : what we are trying to represent. */
1515 14629 : if (ctx != ISRA_CTX_ARG)
1516 : return NULL;
1517 :
1518 11348 : gensum_param_access *r = allocate_access (desc, offset, size);
1519 11348 : if (!r)
1520 : return NULL;
1521 11348 : r->first_child = access;
1522 :
1523 11348 : while (access->next_sibling
1524 19986 : && access->next_sibling->offset < offset + size)
1525 : access = access->next_sibling;
1526 11348 : if (access->offset + access->size > offset + size)
1527 : {
1528 : /* This must be a different access, which are sorted, so the
1529 : following must be true and this signals a partial overlap. */
1530 0 : gcc_assert (access->offset > offset);
1531 : return NULL;
1532 : }
1533 :
1534 11348 : r->next_sibling = access->next_sibling;
1535 11348 : access->next_sibling = NULL;
1536 11348 : *ptr = r;
1537 11348 : return r;
1538 : }
1539 :
1540 94792 : if (offset >= access->offset + access->size)
1541 : {
1542 : /* We belong after access. */
1543 94792 : gensum_param_access *r = allocate_access (desc, offset, size);
1544 94792 : if (!r)
1545 : return NULL;
1546 94792 : r->next_sibling = access->next_sibling;
1547 94792 : access->next_sibling = r;
1548 94792 : return r;
1549 : }
1550 :
1551 0 : if (offset < access->offset)
1552 : {
1553 : /* We know the following, otherwise we would have created a
1554 : super-access. */
1555 0 : gcc_checking_assert (offset + size < access->offset + access->size);
1556 : return NULL;
1557 : }
1558 :
1559 0 : if (offset + size > access->offset + access->size)
1560 : {
1561 : /* Likewise. */
1562 0 : gcc_checking_assert (offset > access->offset);
1563 : return NULL;
1564 : }
1565 :
1566 0 : gcc_unreachable ();
1567 : }
1568 :
1569 : /* Return an access describing memory access to the variable described by DESC
1570 : at OFFSET with SIZE in context CTX, mark it as used in context CTX. Attempt
1571 : to create if it does not exist, but fail and return NULL if there are
1572 : already too many accesses, if it would create a partially overlapping access
1573 : or if an access would end up in a non-call access. */
1574 :
1575 : static gensum_param_access *
1576 508242 : get_access (gensum_param_desc *desc, HOST_WIDE_INT offset, HOST_WIDE_INT size,
1577 : isra_scan_context ctx)
1578 : {
1579 508242 : gcc_checking_assert (desc->split_candidate);
1580 :
1581 508242 : gensum_param_access *access = get_access_1 (desc, &desc->accesses, offset,
1582 : size, ctx);
1583 508242 : if (!access)
1584 : {
1585 4891 : disqualify_split_candidate (desc,
1586 : "Bad access overlap or too many accesses");
1587 4891 : return NULL;
1588 : }
1589 :
1590 503351 : switch (ctx)
1591 : {
1592 13251 : case ISRA_CTX_STORE:
1593 13251 : gcc_assert (!desc->by_ref);
1594 : /* Fall-through */
1595 416361 : case ISRA_CTX_LOAD:
1596 416361 : access->nonarg = true;
1597 416361 : break;
1598 : case ISRA_CTX_ARG:
1599 : break;
1600 : }
1601 :
1602 : return access;
1603 : }
1604 :
1605 : /* Verify that parameter access tree starting with ACCESS is in good shape.
1606 : PARENT_OFFSET and PARENT_SIZE are the corresponding fields of parent of
1607 : ACCESS or zero if there is none. */
1608 :
1609 : static bool
1610 909537 : verify_access_tree_1 (gensum_param_access *access, HOST_WIDE_INT parent_offset,
1611 : HOST_WIDE_INT parent_size)
1612 : {
1613 1294805 : while (access)
1614 : {
1615 385268 : gcc_assert (access->offset >= 0 && access->size >= 0);
1616 :
1617 385268 : if (parent_size != 0)
1618 : {
1619 22623 : if (access->offset < parent_offset)
1620 : {
1621 0 : error ("Access offset before parent offset");
1622 0 : return true;
1623 : }
1624 22623 : if (access->size >= parent_size)
1625 : {
1626 0 : error ("Access size greater or equal to its parent size");
1627 0 : return true;
1628 : }
1629 22623 : if (access->offset + access->size > parent_offset + parent_size)
1630 : {
1631 0 : error ("Access terminates outside of its parent");
1632 0 : return true;
1633 : }
1634 : }
1635 :
1636 385268 : if (verify_access_tree_1 (access->first_child, access->offset,
1637 : access->size))
1638 : return true;
1639 :
1640 385268 : if (access->next_sibling
1641 127047 : && (access->next_sibling->offset < access->offset + access->size))
1642 : {
1643 0 : error ("Access overlaps with its sibling");
1644 0 : return true;
1645 : }
1646 :
1647 : access = access->next_sibling;
1648 : }
1649 : return false;
1650 : }
1651 :
1652 : /* Verify that parameter access tree starting with ACCESS is in good shape,
1653 : halt compilation and dump the tree to stderr if not. */
1654 :
1655 : DEBUG_FUNCTION void
1656 524269 : isra_verify_access_tree (gensum_param_access *access)
1657 : {
1658 524269 : if (verify_access_tree_1 (access, 0, 0))
1659 : {
1660 0 : for (; access; access = access->next_sibling)
1661 0 : dump_gensum_access (stderr, access, 2);
1662 0 : internal_error ("IPA-SRA access verification failed");
1663 : }
1664 524269 : }
1665 :
1666 :
1667 : /* Callback of walk_stmt_load_store_addr_ops visit_addr used to determine
1668 : GIMPLE_ASM operands with memory constrains which cannot be scalarized. */
1669 :
1670 : static bool
1671 9111 : asm_visit_addr (gimple *, tree op, tree, void *)
1672 : {
1673 9111 : op = get_base_address (op);
1674 9111 : if (op
1675 9111 : && TREE_CODE (op) == PARM_DECL)
1676 177 : disqualify_split_candidate (op, "Non-scalarizable GIMPLE_ASM operand.");
1677 :
1678 9111 : return false;
1679 : }
1680 :
1681 : /* Mark a dereference of parameter identified by DESC of distance DIST in a
1682 : basic block BB, unless the BB has already been marked as a potentially
1683 : final. */
1684 :
1685 : static void
1686 237933 : mark_param_dereference (gensum_param_desc *desc, HOST_WIDE_INT dist,
1687 : basic_block bb)
1688 : {
1689 237933 : gcc_assert (desc->by_ref);
1690 237933 : gcc_checking_assert (desc->split_candidate);
1691 :
1692 237933 : if (desc->safe_ref
1693 237933 : || bitmap_bit_p (final_bbs, bb->index))
1694 212111 : return;
1695 :
1696 25822 : int idx = bb->index * unsafe_by_ref_count + desc->deref_index;
1697 25822 : if (bb_dereferences[idx] < dist)
1698 24541 : bb_dereferences[idx] = dist;
1699 : }
1700 :
1701 : /* Return true, if any potential replacements should use NEW_TYPE as opposed to
1702 : previously recorded OLD_TYPE. */
1703 :
1704 : static bool
1705 60886 : type_prevails_p (tree old_type, tree new_type)
1706 : {
1707 60886 : if (old_type == new_type)
1708 : return false;
1709 :
1710 : /* Non-aggregates are always better. */
1711 281 : if (!is_gimple_reg_type (old_type)
1712 281 : && is_gimple_reg_type (new_type))
1713 : return true;
1714 269 : if (is_gimple_reg_type (old_type)
1715 269 : && !is_gimple_reg_type (new_type))
1716 : return false;
1717 :
1718 : /* Prefer any complex or vector type over any other scalar type. */
1719 243 : if (TREE_CODE (old_type) != COMPLEX_TYPE
1720 243 : && TREE_CODE (old_type) != VECTOR_TYPE
1721 230 : && (TREE_CODE (new_type) == COMPLEX_TYPE
1722 230 : || VECTOR_TYPE_P (new_type)))
1723 : return true;
1724 243 : if ((TREE_CODE (old_type) == COMPLEX_TYPE
1725 : || VECTOR_TYPE_P (old_type))
1726 13 : && TREE_CODE (new_type) != COMPLEX_TYPE
1727 0 : && TREE_CODE (new_type) != VECTOR_TYPE)
1728 : return false;
1729 :
1730 : /* Use the integral type with the bigger precision. */
1731 243 : if (INTEGRAL_TYPE_P (old_type)
1732 12 : && INTEGRAL_TYPE_P (new_type))
1733 12 : return (TYPE_PRECISION (new_type) > TYPE_PRECISION (old_type));
1734 :
1735 : /* Attempt to disregard any integral type with non-full precision. */
1736 231 : if (INTEGRAL_TYPE_P (old_type)
1737 231 : && (TREE_INT_CST_LOW (TYPE_SIZE (old_type))
1738 0 : != TYPE_PRECISION (old_type)))
1739 : return true;
1740 231 : if (INTEGRAL_TYPE_P (new_type)
1741 231 : && (TREE_INT_CST_LOW (TYPE_SIZE (new_type))
1742 6 : != TYPE_PRECISION (new_type)))
1743 : return false;
1744 : /* Stabilize the selection. */
1745 231 : return TYPE_UID (old_type) < TYPE_UID (new_type);
1746 : }
1747 :
1748 : /* When scanning an expression which is a call argument, this structure
1749 : specifies the call and the position of the argument. */
1750 :
1751 : struct scan_call_info
1752 : {
1753 : /* Call graph edge representing the call. */
1754 : cgraph_edge *cs;
1755 : /* Total number of arguments in the call. */
1756 : unsigned argument_count;
1757 : /* Number of the actual argument being scanned. */
1758 : unsigned arg_idx;
1759 : };
1760 :
1761 : /* Record use of ACCESS which belongs to a parameter described by DESC in a
1762 : call argument described by CALL_INFO. */
1763 :
1764 : static void
1765 84078 : record_nonregister_call_use (gensum_param_desc *desc,
1766 : scan_call_info *call_info,
1767 : unsigned unit_offset, unsigned unit_size)
1768 : {
1769 84078 : isra_call_summary *csum = call_sums->get_create (call_info->cs);
1770 84078 : csum->init_inputs (call_info->argument_count);
1771 :
1772 84078 : isra_param_flow *param_flow = &csum->m_arg_flow[call_info->arg_idx];
1773 84078 : param_flow->aggregate_pass_through = true;
1774 84078 : set_single_param_flow_source (param_flow, desc->param_number);
1775 84078 : param_flow->unit_offset = unit_offset;
1776 84078 : param_flow->unit_size = unit_size;
1777 84078 : desc->call_uses++;
1778 84078 : }
1779 :
1780 : /* Callback of walk_aliased_vdefs, just mark that there was a possible
1781 : modification. */
1782 :
1783 : static bool
1784 63738 : mark_maybe_modified (ao_ref *, tree, void *data)
1785 : {
1786 63738 : bool *maybe_modified = (bool *) data;
1787 63738 : *maybe_modified = true;
1788 63738 : return true;
1789 : }
1790 :
1791 : /* Analyze expression EXPR from GIMPLE for accesses to parameters. CTX
1792 : specifies whether EXPR is used in a load, store or as an argument call. BB
1793 : must be the basic block in which expr resides. If CTX specifies call
1794 : argument context, CALL_INFO must describe that call and argument position,
1795 : otherwise it is ignored. */
1796 :
1797 : static void
1798 35753377 : scan_expr_access (tree expr, gimple *stmt, isra_scan_context ctx,
1799 : basic_block bb, scan_call_info *call_info = NULL)
1800 : {
1801 35753377 : poly_int64 poffset, psize, pmax_size;
1802 35753377 : HOST_WIDE_INT offset, size, max_size;
1803 35753377 : tree base;
1804 35753377 : bool deref = false;
1805 35753377 : bool reverse;
1806 :
1807 35753377 : if (TREE_CODE (expr) == ADDR_EXPR)
1808 : {
1809 4410494 : if (ctx == ISRA_CTX_ARG)
1810 : return;
1811 1212949 : tree t = get_base_address (TREE_OPERAND (expr, 0));
1812 1212949 : if (VAR_P (t) && !TREE_STATIC (t))
1813 236473 : loaded_decls->add (t);
1814 1212949 : return;
1815 : }
1816 31342883 : if (TREE_CODE (expr) == SSA_NAME
1817 17357678 : || CONSTANT_CLASS_P (expr))
1818 : return;
1819 :
1820 12439133 : if (TREE_CODE (expr) == BIT_FIELD_REF
1821 12439133 : || TREE_CODE (expr) == IMAGPART_EXPR
1822 12325742 : || TREE_CODE (expr) == REALPART_EXPR)
1823 165939 : expr = TREE_OPERAND (expr, 0);
1824 :
1825 12439133 : base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size, &reverse);
1826 :
1827 12439133 : if (TREE_CODE (base) == MEM_REF)
1828 : {
1829 3939378 : tree op = TREE_OPERAND (base, 0);
1830 3939378 : if (TREE_CODE (op) != SSA_NAME
1831 3939378 : || !SSA_NAME_IS_DEFAULT_DEF (op))
1832 : return;
1833 2075244 : base = SSA_NAME_VAR (op);
1834 2075244 : if (!base)
1835 : return;
1836 : deref = true;
1837 : }
1838 8499755 : else if (VAR_P (base)
1839 7350956 : && !TREE_STATIC (base)
1840 6370720 : && (ctx == ISRA_CTX_ARG
1841 : || ctx == ISRA_CTX_LOAD))
1842 2387132 : loaded_decls->add (base);
1843 :
1844 10574999 : if (TREE_CODE (base) != PARM_DECL)
1845 : return;
1846 :
1847 2442060 : gensum_param_desc *desc = get_gensum_param_desc (base);
1848 2442060 : if (!desc || !desc->split_candidate)
1849 : return;
1850 :
1851 572990 : if (storage_order_barrier_p (expr))
1852 : {
1853 0 : disqualify_split_candidate (desc, "Encountered a storage order barrier.");
1854 0 : return;
1855 : }
1856 :
1857 572990 : if (!poffset.is_constant (&offset)
1858 572990 : || !psize.is_constant (&size)
1859 572990 : || !pmax_size.is_constant (&max_size))
1860 : {
1861 : disqualify_split_candidate (desc, "Encountered a polynomial-sized "
1862 : "access.");
1863 : return;
1864 : }
1865 572990 : if (size < 0 || size != max_size)
1866 : {
1867 6401 : disqualify_split_candidate (desc, "Encountered a variable sized access.");
1868 6401 : return;
1869 : }
1870 566589 : if (TREE_CODE (expr) == COMPONENT_REF
1871 566589 : && DECL_BIT_FIELD (TREE_OPERAND (expr, 1)))
1872 : {
1873 876 : disqualify_split_candidate (desc, "Encountered a bit-field access.");
1874 876 : return;
1875 : }
1876 565713 : if (offset < 0)
1877 : {
1878 4 : disqualify_split_candidate (desc, "Encountered an access at a "
1879 : "negative offset.");
1880 4 : return;
1881 : }
1882 565709 : gcc_assert ((offset % BITS_PER_UNIT) == 0);
1883 565709 : gcc_assert ((size % BITS_PER_UNIT) == 0);
1884 565709 : if ((offset / BITS_PER_UNIT) >= (UINT_MAX - ISRA_ARG_SIZE_LIMIT)
1885 565709 : || (size / BITS_PER_UNIT) >= ISRA_ARG_SIZE_LIMIT)
1886 : {
1887 0 : disqualify_split_candidate (desc, "Encountered an access with too big "
1888 : "offset or size");
1889 0 : return;
1890 : }
1891 :
1892 565709 : tree type = TREE_TYPE (expr);
1893 565709 : unsigned int exp_align = get_object_alignment (expr);
1894 :
1895 565709 : if (exp_align < TYPE_ALIGN (type))
1896 : {
1897 1372 : disqualify_split_candidate (desc, "Underaligned access.");
1898 1372 : return;
1899 : }
1900 :
1901 564337 : if (deref)
1902 : {
1903 294028 : if (!desc->by_ref)
1904 : {
1905 0 : disqualify_split_candidate (desc, "Dereferencing a non-reference.");
1906 56095 : return;
1907 : }
1908 294028 : else if (ctx == ISRA_CTX_STORE)
1909 : {
1910 0 : disqualify_split_candidate (desc, "Storing to data passed by "
1911 : "reference.");
1912 0 : return;
1913 : }
1914 :
1915 294028 : if (!aa_walking_limit)
1916 : {
1917 0 : disqualify_split_candidate (desc, "Out of alias analysis step "
1918 : "limit.");
1919 0 : return;
1920 : }
1921 :
1922 588056 : gcc_checking_assert (gimple_vuse (stmt));
1923 294028 : bool maybe_modified = false;
1924 294028 : ao_ref ar;
1925 :
1926 294028 : ao_ref_init (&ar, expr);
1927 294028 : bitmap visited = BITMAP_ALLOC (NULL);
1928 588056 : int walked = walk_aliased_vdefs (&ar, gimple_vuse (stmt),
1929 : mark_maybe_modified, &maybe_modified,
1930 : &visited, NULL, aa_walking_limit);
1931 294028 : BITMAP_FREE (visited);
1932 294028 : if (walked > 0)
1933 : {
1934 96896 : gcc_assert (aa_walking_limit > walked);
1935 96896 : aa_walking_limit = aa_walking_limit - walked;
1936 : }
1937 294028 : if (walked < 0)
1938 0 : aa_walking_limit = 0;
1939 294028 : if (maybe_modified || walked < 0)
1940 : {
1941 56095 : disqualify_split_candidate (desc, "Data passed by reference possibly "
1942 : "modified through an alias.");
1943 56095 : return;
1944 : }
1945 : else
1946 237933 : mark_param_dereference (desc, offset + size, bb);
1947 : }
1948 : else
1949 : /* Pointer parameters with direct uses should have been ruled out by
1950 : analyzing SSA default def when looking at the parameters. */
1951 270309 : gcc_assert (!desc->by_ref);
1952 :
1953 508242 : gensum_param_access *access = get_access (desc, offset, size, ctx);
1954 508242 : if (!access)
1955 : return;
1956 :
1957 503351 : if (ctx == ISRA_CTX_ARG)
1958 : {
1959 86990 : gcc_checking_assert (call_info);
1960 :
1961 86990 : if (!deref)
1962 84078 : record_nonregister_call_use (desc, call_info, offset / BITS_PER_UNIT,
1963 84078 : size / BITS_PER_UNIT);
1964 : else
1965 : /* This is not a pass-through of a pointer, this is a use like any
1966 : other. */
1967 2912 : access->nonarg = true;
1968 : }
1969 416361 : else if (ctx == ISRA_CTX_LOAD && bb->count.initialized_p ())
1970 403022 : access->load_count += bb->count;
1971 :
1972 503351 : if (!access->type)
1973 : {
1974 432459 : access->type = type;
1975 432459 : access->alias_ptr_type = reference_alias_ptr_type (expr);
1976 432459 : access->reverse = reverse;
1977 : }
1978 : else
1979 : {
1980 70892 : if (exp_align < TYPE_ALIGN (access->type))
1981 : {
1982 0 : disqualify_split_candidate (desc, "Reference has lower alignment "
1983 : "than a previous one.");
1984 0 : return;
1985 : }
1986 70892 : if (access->alias_ptr_type != reference_alias_ptr_type (expr))
1987 : {
1988 4426 : disqualify_split_candidate (desc, "Multiple alias pointer types.");
1989 4426 : return;
1990 : }
1991 66466 : if (access->reverse != reverse)
1992 : {
1993 0 : disqualify_split_candidate (desc, "Both normal and reverse "
1994 : "scalar storage order.");
1995 0 : return;
1996 : }
1997 66466 : if (!deref
1998 51746 : && (AGGREGATE_TYPE_P (type) || AGGREGATE_TYPE_P (access->type))
1999 104614 : && (TYPE_MAIN_VARIANT (access->type) != TYPE_MAIN_VARIANT (type)))
2000 : {
2001 : /* We need the same aggregate type on all accesses to be able to
2002 : distinguish transformation spots from pass-through arguments in
2003 : the transformation phase. */
2004 5580 : disqualify_split_candidate (desc, "We do not support aggregate "
2005 : "type punning.");
2006 5580 : return;
2007 : }
2008 :
2009 60886 : if (type_prevails_p (access->type, type))
2010 99 : access->type = type;
2011 : }
2012 : }
2013 :
2014 : /* Scan body function described by NODE and FUN and create access trees for
2015 : parameters. */
2016 :
2017 : static void
2018 1267514 : scan_function (cgraph_node *node, struct function *fun)
2019 : {
2020 1267514 : basic_block bb;
2021 :
2022 10230522 : FOR_EACH_BB_FN (bb, fun)
2023 : {
2024 8963008 : gimple_stmt_iterator gsi;
2025 69907609 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2026 : {
2027 51981593 : gimple *stmt = gsi_stmt (gsi);
2028 :
2029 51981593 : if (final_bbs && stmt_can_throw_external (fun, stmt))
2030 1347149 : bitmap_set_bit (final_bbs, bb->index);
2031 51981593 : switch (gimple_code (stmt))
2032 : {
2033 1235741 : case GIMPLE_RETURN:
2034 1235741 : {
2035 1235741 : tree t = gimple_return_retval (as_a <greturn *> (stmt));
2036 1235741 : if (t != NULL_TREE)
2037 677422 : scan_expr_access (t, stmt, ISRA_CTX_LOAD, bb);
2038 1235741 : if (final_bbs)
2039 411679 : bitmap_set_bit (final_bbs, bb->index);
2040 : }
2041 : break;
2042 :
2043 18085416 : case GIMPLE_ASSIGN:
2044 18085416 : if (gimple_assign_single_p (stmt)
2045 18085416 : && !gimple_clobber_p (stmt))
2046 : {
2047 11034314 : tree rhs = gimple_assign_rhs1 (stmt);
2048 11034314 : scan_expr_access (rhs, stmt, ISRA_CTX_LOAD, bb);
2049 11034314 : tree lhs = gimple_assign_lhs (stmt);
2050 11034314 : scan_expr_access (lhs, stmt, ISRA_CTX_STORE, bb);
2051 : }
2052 : break;
2053 :
2054 5331707 : case GIMPLE_CALL:
2055 5331707 : {
2056 5331707 : unsigned argument_count = gimple_call_num_args (stmt);
2057 5331707 : isra_scan_context ctx = ISRA_CTX_ARG;
2058 5331707 : scan_call_info call_info, *call_info_p = &call_info;
2059 5331707 : if (gimple_call_internal_p (stmt))
2060 : {
2061 179926 : call_info_p = NULL;
2062 179926 : ctx = ISRA_CTX_LOAD;
2063 179926 : internal_fn ifn = gimple_call_internal_fn (stmt);
2064 179926 : if (internal_store_fn_p (ifn))
2065 0 : ctx = ISRA_CTX_STORE;
2066 : }
2067 : else
2068 : {
2069 5151781 : call_info.cs = node->get_edge (stmt);
2070 5151781 : call_info.argument_count = argument_count;
2071 : }
2072 :
2073 16105035 : for (unsigned i = 0; i < argument_count; i++)
2074 : {
2075 10773328 : call_info.arg_idx = i;
2076 10773328 : scan_expr_access (gimple_call_arg (stmt, i), stmt,
2077 : ctx, bb, call_info_p);
2078 : }
2079 :
2080 5331707 : tree lhs = gimple_call_lhs (stmt);
2081 5331707 : if (lhs)
2082 2143973 : scan_expr_access (lhs, stmt, ISRA_CTX_STORE, bb);
2083 5331707 : int flags = gimple_call_flags (stmt);
2084 5331707 : if (final_bbs
2085 2065367 : && (((flags & (ECF_CONST | ECF_PURE)) == 0)
2086 340491 : || (flags & ECF_LOOPING_CONST_OR_PURE)))
2087 1782451 : bitmap_set_bit (final_bbs, bb->index);
2088 : }
2089 5331707 : break;
2090 :
2091 50259 : case GIMPLE_ASM:
2092 50259 : {
2093 50259 : gasm *asm_stmt = as_a <gasm *> (stmt);
2094 50259 : walk_stmt_load_store_addr_ops (asm_stmt, NULL, NULL, NULL,
2095 : asm_visit_addr);
2096 50259 : if (final_bbs)
2097 1941 : bitmap_set_bit (final_bbs, bb->index);
2098 :
2099 87546 : for (unsigned i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
2100 : {
2101 37287 : tree t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
2102 37287 : scan_expr_access (t, stmt, ISRA_CTX_LOAD, bb);
2103 : }
2104 102998 : for (unsigned i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
2105 : {
2106 52739 : tree t = TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
2107 52739 : scan_expr_access (t, stmt, ISRA_CTX_STORE, bb);
2108 : }
2109 : }
2110 : break;
2111 :
2112 : default:
2113 : break;
2114 : }
2115 : }
2116 : }
2117 1267514 : }
2118 :
2119 : /* Return true if SSA_NAME NAME of function described by FUN is only used in
2120 : return statements, or if results of any operations it is involved in are
2121 : only used in return statements. ANALYZED is a bitmap that tracks which SSA
2122 : names we have already started investigating. */
2123 :
2124 : static bool
2125 2435631 : ssa_name_only_returned_p (function *fun, tree name, bitmap analyzed)
2126 : {
2127 2435631 : bool res = true;
2128 2435631 : imm_use_iterator imm_iter;
2129 2435631 : gimple *stmt;
2130 :
2131 5517328 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
2132 : {
2133 2660884 : if (is_gimple_debug (stmt))
2134 184463 : continue;
2135 :
2136 2476421 : if (gimple_code (stmt) == GIMPLE_RETURN)
2137 : {
2138 253918 : tree t = gimple_return_retval (as_a <greturn *> (stmt));
2139 253918 : if (t != name)
2140 : {
2141 : res = false;
2142 : break;
2143 : }
2144 : }
2145 2222503 : else if (!stmt_unremovable_because_of_non_call_eh_p (fun, stmt)
2146 2222503 : && ((is_gimple_assign (stmt) && !gimple_has_volatile_ops (stmt))
2147 1169210 : || gimple_code (stmt) == GIMPLE_PHI))
2148 : {
2149 : /* TODO: And perhaps for const function calls too? */
2150 1166250 : tree lhs;
2151 1166250 : if (gimple_code (stmt) == GIMPLE_PHI)
2152 209220 : lhs = gimple_phi_result (stmt);
2153 : else
2154 957030 : lhs = gimple_assign_lhs (stmt);
2155 :
2156 1166250 : if (TREE_CODE (lhs) != SSA_NAME)
2157 : {
2158 : res = false;
2159 : break;
2160 : }
2161 822184 : gcc_assert (!gimple_vdef (stmt));
2162 822184 : if (bitmap_set_bit (analyzed, SSA_NAME_VERSION (lhs))
2163 822184 : && !ssa_name_only_returned_p (fun, lhs, analyzed))
2164 : {
2165 : res = false;
2166 : break;
2167 : }
2168 : }
2169 : else
2170 : {
2171 : res = false;
2172 : break;
2173 : }
2174 2435631 : }
2175 2435631 : return res;
2176 : }
2177 :
2178 : /* Inspect the uses of the return value of the call associated with CS, and if
2179 : it is not used or if it is only used to construct the return value of the
2180 : caller, mark it as such in call or caller summary. Also check for
2181 : misaligned arguments. */
2182 :
2183 : static void
2184 5165277 : isra_analyze_call (cgraph_edge *cs)
2185 : {
2186 5165277 : gcall *call_stmt = cs->call_stmt;
2187 5165277 : unsigned count = gimple_call_num_args (call_stmt);
2188 5165277 : isra_call_summary *csum = call_sums->get_create (cs);
2189 :
2190 15412499 : for (unsigned i = 0; i < count; i++)
2191 : {
2192 10247222 : tree arg = gimple_call_arg (call_stmt, i);
2193 10247222 : if (TREE_CODE (arg) == ADDR_EXPR)
2194 : {
2195 3222525 : poly_int64 poffset, psize, pmax_size;
2196 3222525 : bool reverse;
2197 3222525 : tree base = get_ref_base_and_extent (TREE_OPERAND (arg, 0), &poffset,
2198 3222525 : &psize, &pmax_size, &reverse);
2199 3222525 : HOST_WIDE_INT offset;
2200 3222525 : unsigned HOST_WIDE_INT ds;
2201 3222525 : if (DECL_P (base)
2202 2307114 : && (poffset.is_constant (&offset))
2203 2307114 : && tree_fits_uhwi_p (DECL_SIZE (base))
2204 5388015 : && ((ds = tree_to_uhwi (DECL_SIZE (base)) - offset)
2205 : < ISRA_ARG_SIZE_LIMIT * BITS_PER_UNIT))
2206 : {
2207 2032264 : csum->init_inputs (count);
2208 2032264 : gcc_assert (!csum->m_arg_flow[i].aggregate_pass_through);
2209 2032264 : csum->m_arg_flow[i].unit_size = ds / BITS_PER_UNIT;
2210 : }
2211 :
2212 3222525 : if (TREE_CODE (base) == VAR_DECL
2213 2125983 : && !TREE_STATIC (base)
2214 4600938 : && !loaded_decls->contains (base))
2215 : {
2216 909468 : csum->init_inputs (count);
2217 909468 : csum->m_arg_flow[i].constructed_for_calls = true;
2218 : }
2219 : }
2220 :
2221 10247222 : if (is_gimple_reg (arg))
2222 4030719 : continue;
2223 :
2224 6216503 : tree offset;
2225 6216503 : poly_int64 bitsize, bitpos;
2226 6216503 : machine_mode mode;
2227 6216503 : int unsignedp, reversep, volatilep = 0;
2228 6216503 : get_inner_reference (arg, &bitsize, &bitpos, &offset, &mode,
2229 : &unsignedp, &reversep, &volatilep);
2230 12433006 : if (!multiple_p (bitpos, BITS_PER_UNIT))
2231 : {
2232 0 : csum->m_bit_aligned_arg = true;
2233 0 : break;
2234 : }
2235 : }
2236 :
2237 5165277 : tree lhs = gimple_call_lhs (call_stmt);
2238 5165277 : if (lhs)
2239 : {
2240 : /* TODO: Also detect aggregates on a LHS of a call that are only returned
2241 : from this function (without being read anywhere). */
2242 1982291 : if (TREE_CODE (lhs) == SSA_NAME)
2243 : {
2244 1633594 : bitmap analyzed = BITMAP_ALLOC (NULL);
2245 1633594 : if (ssa_name_only_returned_p (DECL_STRUCT_FUNCTION (cs->caller->decl),
2246 : lhs, analyzed))
2247 233275 : csum->m_return_returned = true;
2248 1633594 : BITMAP_FREE (analyzed);
2249 : }
2250 : }
2251 : /* Don't set m_return_ignored for musttail calls. The tailc/musttail passes
2252 : compare the returned value against the IPA-VRP return value range if
2253 : it is a singleton, but if the call is changed to something which doesn't
2254 : return anything, it will always fail. */
2255 3182986 : else if (!gimple_call_must_tail_p (call_stmt))
2256 3182885 : csum->m_return_ignored = true;
2257 5165277 : }
2258 :
2259 : /* Look at all calls going out of NODE, described also by IFS and perform all
2260 : analyses necessary for IPA-SRA that are not done at body scan time or done
2261 : even when body is not scanned because the function is not a candidate. */
2262 :
2263 : static void
2264 1267514 : isra_analyze_all_outgoing_calls (cgraph_node *node)
2265 : {
2266 6298281 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
2267 5030767 : isra_analyze_call (cs);
2268 1402024 : for (cgraph_edge *cs = node->indirect_calls; cs; cs = cs->next_callee)
2269 134510 : isra_analyze_call (cs);
2270 1267514 : }
2271 :
2272 : /* Dump a dereferences table with heading STR to file F. */
2273 :
2274 : static void
2275 10 : dump_dereferences_table (FILE *f, struct function *fun, const char *str)
2276 : {
2277 10 : basic_block bb;
2278 :
2279 10 : fprintf (dump_file, "%s", str);
2280 52 : FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun),
2281 : EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
2282 : {
2283 42 : fprintf (f, "%4i %i ", bb->index, bitmap_bit_p (final_bbs, bb->index));
2284 42 : if (bb != EXIT_BLOCK_PTR_FOR_FN (fun))
2285 : {
2286 : int i;
2287 104 : for (i = 0; i < unsafe_by_ref_count; i++)
2288 : {
2289 62 : int idx = bb->index * unsafe_by_ref_count + i;
2290 62 : fprintf (f, " %4" HOST_WIDE_INT_PRINT "d", bb_dereferences[idx]);
2291 : }
2292 : }
2293 42 : fprintf (f, "\n");
2294 : }
2295 10 : fprintf (dump_file, "\n");
2296 10 : }
2297 :
2298 : /* Propagate distances in bb_dereferences in the opposite direction than the
2299 : control flow edges, in each step storing the maximum of the current value
2300 : and the minimum of all successors. These steps are repeated until the table
2301 : stabilizes. Note that BBs which might terminate the functions (according to
2302 : final_bbs bitmap) never updated in this way. */
2303 :
2304 : static void
2305 27079 : propagate_dereference_distances (struct function *fun)
2306 : {
2307 27079 : basic_block bb;
2308 :
2309 27079 : if (dump_file && (dump_flags & TDF_DETAILS))
2310 5 : dump_dereferences_table (dump_file, fun,
2311 : "Dereference table before propagation:\n");
2312 :
2313 27079 : auto_vec<basic_block> queue (last_basic_block_for_fn (fun));
2314 27079 : queue.quick_push (ENTRY_BLOCK_PTR_FOR_FN (fun));
2315 234676 : FOR_EACH_BB_FN (bb, fun)
2316 : {
2317 207597 : queue.quick_push (bb);
2318 207597 : bb->aux = bb;
2319 : }
2320 :
2321 262636 : while (!queue.is_empty ())
2322 : {
2323 235557 : edge_iterator ei;
2324 235557 : edge e;
2325 235557 : bool change = false;
2326 235557 : int i;
2327 :
2328 235557 : bb = queue.pop ();
2329 235557 : bb->aux = NULL;
2330 :
2331 235557 : if (bitmap_bit_p (final_bbs, bb->index))
2332 73269 : continue;
2333 :
2334 6202884 : for (i = 0; i < unsafe_by_ref_count; i++)
2335 : {
2336 6040596 : int idx = bb->index * unsafe_by_ref_count + i;
2337 6040596 : bool first = true;
2338 6040596 : HOST_WIDE_INT inh = 0;
2339 :
2340 14229756 : FOR_EACH_EDGE (e, ei, bb->succs)
2341 : {
2342 8189160 : int succ_idx = e->dest->index * unsafe_by_ref_count + i;
2343 :
2344 8189160 : if (e->dest == EXIT_BLOCK_PTR_FOR_FN (fun))
2345 0 : continue;
2346 :
2347 8189160 : if (first)
2348 : {
2349 6040591 : first = false;
2350 6040591 : inh = bb_dereferences [succ_idx];
2351 : }
2352 2148569 : else if (bb_dereferences [succ_idx] < inh)
2353 : inh = bb_dereferences [succ_idx];
2354 : }
2355 :
2356 6040596 : if (!first && bb_dereferences[idx] < inh)
2357 : {
2358 10890 : bb_dereferences[idx] = inh;
2359 10890 : change = true;
2360 : }
2361 : }
2362 :
2363 162288 : if (change)
2364 11849 : FOR_EACH_EDGE (e, ei, bb->preds)
2365 : {
2366 1872 : if (e->src->aux)
2367 991 : continue;
2368 :
2369 881 : e->src->aux = e->src;
2370 881 : queue.quick_push (e->src);
2371 : }
2372 : }
2373 :
2374 27079 : if (dump_file && (dump_flags & TDF_DETAILS))
2375 5 : dump_dereferences_table (dump_file, fun,
2376 : "Dereference table after propagation:\n");
2377 27079 : }
2378 :
2379 : /* Return true if the ACCESS loads happen frequently enough in FUN to risk
2380 : moving them to the caller and only pass the result. */
2381 :
2382 : static bool
2383 171427 : dereference_probable_p (struct function *fun, gensum_param_access *access)
2384 : {
2385 171427 : int threshold = opt_for_fn (fun->decl, param_ipa_sra_deref_prob_threshold);
2386 171427 : return access->load_count
2387 171427 : >= ENTRY_BLOCK_PTR_FOR_FN (fun)->count.apply_scale (threshold, 100);
2388 : }
2389 :
2390 : /* Perform basic checks on ACCESS to PARM (of FUN) described by DESC and all
2391 : its children, return true if the parameter cannot be split, otherwise return
2392 : false and update *NONARG_ACC_SIZE and *ONLY_CALLS. ENTRY_BB_INDEX must be
2393 : the index of the entry BB in the function of PARM. */
2394 :
2395 : static bool
2396 377678 : check_gensum_access (struct function *fun, tree parm, gensum_param_desc *desc,
2397 : gensum_param_access *access,
2398 : HOST_WIDE_INT *nonarg_acc_size, bool *only_calls,
2399 : int entry_bb_index)
2400 : {
2401 377678 : if (access->nonarg)
2402 : {
2403 326588 : *only_calls = false;
2404 326588 : *nonarg_acc_size += access->size;
2405 :
2406 326588 : if (access->first_child)
2407 : {
2408 379 : disqualify_split_candidate (desc, "Overlapping non-call uses.");
2409 379 : return true;
2410 : }
2411 : }
2412 : /* Do not decompose a non-BLKmode param in a way that would create
2413 : BLKmode params. Especially for by-reference passing (thus,
2414 : pointer-type param) this is hardly worthwhile. */
2415 377299 : if (DECL_MODE (parm) != BLKmode
2416 377299 : && TYPE_MODE (access->type) == BLKmode)
2417 : {
2418 3037 : disqualify_split_candidate (desc, "Would convert a non-BLK to a BLK.");
2419 3037 : return true;
2420 : }
2421 :
2422 374262 : if (desc->by_ref)
2423 : {
2424 182102 : if (desc->safe_ref)
2425 : {
2426 146529 : if (!dereference_probable_p (fun, access))
2427 : {
2428 6563 : disqualify_split_candidate (desc, "Dereferences in callers "
2429 : "would happen much more frequently.");
2430 6563 : return true;
2431 : }
2432 : }
2433 : else
2434 : {
2435 35573 : int idx = (entry_bb_index * unsafe_by_ref_count + desc->deref_index);
2436 35573 : if ((access->offset + access->size) > bb_dereferences[idx])
2437 : {
2438 24898 : if (!dereference_probable_p (fun, access))
2439 : {
2440 2177 : disqualify_split_candidate (desc, "Would create a possibly "
2441 : "illegal dereference in a "
2442 : "caller.");
2443 2177 : return true;
2444 : }
2445 22721 : desc->conditionally_dereferenceable = true;
2446 : }
2447 : }
2448 : }
2449 :
2450 365522 : for (gensum_param_access *ch = access->first_child;
2451 387432 : ch;
2452 21910 : ch = ch->next_sibling)
2453 21912 : if (check_gensum_access (fun, parm, desc, ch, nonarg_acc_size, only_calls,
2454 : entry_bb_index))
2455 : return true;
2456 :
2457 : return false;
2458 : }
2459 :
2460 : /* Copy data from FROM and all of its children to a vector of accesses in IPA
2461 : descriptor DESC. */
2462 :
2463 : static void
2464 432459 : copy_accesses_to_ipa_desc (gensum_param_access *from, isra_param_desc *desc)
2465 : {
2466 432459 : param_access *to = ggc_cleared_alloc<param_access> ();
2467 432459 : gcc_checking_assert ((from->offset % BITS_PER_UNIT) == 0);
2468 432459 : gcc_checking_assert ((from->size % BITS_PER_UNIT) == 0);
2469 432459 : to->unit_offset = from->offset / BITS_PER_UNIT;
2470 432459 : to->unit_size = from->size / BITS_PER_UNIT;
2471 432459 : to->type = from->type;
2472 432459 : to->alias_ptr_type = from->alias_ptr_type;
2473 432459 : to->certain = from->nonarg;
2474 432459 : to->reverse = from->reverse;
2475 432459 : vec_safe_push (desc->accesses, to);
2476 :
2477 432459 : for (gensum_param_access *ch = from->first_child;
2478 456484 : ch;
2479 24025 : ch = ch->next_sibling)
2480 24025 : copy_accesses_to_ipa_desc (ch, desc);
2481 432459 : }
2482 :
2483 : /* Analyze function body scan results stored in param_accesses and
2484 : param_accesses, detect possible transformations and store information of
2485 : those in function summary. NODE, FUN and IFS are all various structures
2486 : describing the currently analyzed function. */
2487 :
2488 : static void
2489 837454 : process_scan_results (cgraph_node *node, struct function *fun,
2490 : isra_func_summary *ifs,
2491 : vec<gensum_param_desc> *param_descriptions)
2492 : {
2493 837454 : bool check_pass_throughs = false;
2494 837454 : bool dereferences_propagated = false;
2495 837454 : tree parm = DECL_ARGUMENTS (node->decl);
2496 837454 : unsigned param_count = param_descriptions->length();
2497 :
2498 837454 : for (unsigned desc_index = 0;
2499 3221632 : desc_index < param_count;
2500 2384178 : desc_index++, parm = DECL_CHAIN (parm))
2501 : {
2502 2384178 : gensum_param_desc *desc = &(*param_descriptions)[desc_index];
2503 2384178 : if (!desc->split_candidate)
2504 1872059 : continue;
2505 :
2506 524275 : if (flag_checking)
2507 524269 : isra_verify_access_tree (desc->accesses);
2508 :
2509 524275 : if (!dereferences_propagated
2510 518512 : && desc->by_ref
2511 361971 : && !desc->safe_ref
2512 164674 : && desc->accesses)
2513 : {
2514 27079 : propagate_dereference_distances (fun);
2515 27079 : dereferences_propagated = true;
2516 : }
2517 :
2518 524275 : HOST_WIDE_INT nonarg_acc_size = 0;
2519 524275 : bool only_calls = true;
2520 524275 : bool check_failed = false;
2521 :
2522 524275 : int entry_bb_index = ENTRY_BLOCK_PTR_FOR_FN (fun)->index;
2523 524275 : for (gensum_param_access *acc = desc->accesses;
2524 867885 : acc;
2525 343610 : acc = acc->next_sibling)
2526 355766 : if (check_gensum_access (fun, parm, desc, acc, &nonarg_acc_size,
2527 : &only_calls, entry_bb_index))
2528 : {
2529 : check_failed = true;
2530 : break;
2531 : }
2532 524275 : if (check_failed)
2533 12156 : continue;
2534 :
2535 512119 : if (only_calls)
2536 317405 : desc->locally_unused = true;
2537 :
2538 512119 : HOST_WIDE_INT cur_param_size
2539 512119 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm)));
2540 512119 : HOST_WIDE_INT param_size_limit, optimistic_limit;
2541 512119 : if (!desc->by_ref || optimize_function_for_size_p (fun))
2542 : {
2543 : param_size_limit = cur_param_size;
2544 : optimistic_limit = cur_param_size;
2545 : }
2546 : else
2547 : {
2548 333244 : param_size_limit
2549 333244 : = opt_for_fn (node->decl,
2550 : param_ipa_sra_ptr_growth_factor) * cur_param_size;
2551 333244 : optimistic_limit
2552 333244 : = (opt_for_fn (node->decl, param_ipa_sra_ptrwrap_growth_factor)
2553 : * param_size_limit);
2554 : }
2555 :
2556 512119 : if (nonarg_acc_size > optimistic_limit
2557 509737 : || (!desc->by_ref && nonarg_acc_size == param_size_limit))
2558 : {
2559 69024 : disqualify_split_candidate (desc, "Would result into a too big set "
2560 : "of replacements even in best "
2561 : "scenarios.");
2562 : }
2563 : else
2564 : {
2565 : /* create_parameter_descriptors makes sure unit sizes of all
2566 : candidate parameters fit unsigned integers restricted to
2567 : ISRA_ARG_SIZE_LIMIT. */
2568 443095 : desc->param_size_limit = param_size_limit / BITS_PER_UNIT;
2569 443095 : desc->nonarg_acc_size = nonarg_acc_size / BITS_PER_UNIT;
2570 443095 : if (desc->split_candidate && desc->ptr_pt_count)
2571 : {
2572 184905 : gcc_assert (desc->by_ref);
2573 : check_pass_throughs = true;
2574 : }
2575 : }
2576 : }
2577 :
2578 : /* When a pointer parameter is passed-through to a callee, in which it is
2579 : only used to read only one or a few items, we can attempt to transform it
2580 : to obtaining and passing through the items instead of the pointer. But we
2581 : must take extra care that 1) we do not introduce any segfault by moving
2582 : dereferences above control flow and that 2) the data is not modified
2583 : through an alias in this function. The IPA analysis must not introduce
2584 : any accesses candidates unless it can prove both.
2585 :
2586 : The current solution is very crude as it consists of ensuring that the
2587 : call postdominates entry BB and that the definition of VUSE of the call is
2588 : default definition. TODO: For non-recursive callees in the same
2589 : compilation unit we could do better by doing analysis in topological order
2590 : an looking into access candidates of callees, using their alias_ptr_types
2591 : to attempt real AA. We could also use the maximum known dereferenced
2592 : offset in this function at IPA level.
2593 :
2594 : TODO: Measure the overhead and the effect of just being pessimistic.
2595 : Maybe this is only -O3 material? */
2596 :
2597 837454 : hash_map<gimple *, bool> analyzed_stmts;
2598 837454 : bitmap always_executed_bbs = NULL;
2599 837454 : if (check_pass_throughs)
2600 1175081 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
2601 : {
2602 1025801 : gcall *call_stmt = cs->call_stmt;
2603 1025801 : tree vuse = gimple_vuse (call_stmt);
2604 :
2605 : /* If the callee is a const function, we don't get a VUSE. In such
2606 : case there will be no memory accesses in the called function (or the
2607 : const attribute is wrong) and then we just don't care. */
2608 2010326 : bool uses_memory_as_obtained = vuse && SSA_NAME_IS_DEFAULT_DEF (vuse);
2609 :
2610 1025801 : unsigned count = gimple_call_num_args (call_stmt);
2611 1025801 : isra_call_summary *csum = call_sums->get_create (cs);
2612 1025801 : csum->init_inputs (count);
2613 1025801 : csum->m_before_any_store = uses_memory_as_obtained;
2614 3300327 : for (unsigned argidx = 0; argidx < count; argidx++)
2615 : {
2616 2274526 : if (!csum->m_arg_flow[argidx].pointer_pass_through)
2617 4128355 : continue;
2618 420697 : unsigned pidx
2619 420697 : = get_single_param_flow_source (&csum->m_arg_flow[argidx]);
2620 420697 : gensum_param_desc *desc = &(*param_descriptions)[pidx];
2621 420697 : if (!desc->split_candidate)
2622 : {
2623 34955 : csum->m_arg_flow[argidx].pointer_pass_through = false;
2624 34955 : continue;
2625 : }
2626 385742 : if (!uses_memory_as_obtained)
2627 141548 : continue;
2628 :
2629 244194 : if (desc->safe_ref)
2630 : {
2631 45105 : csum->m_arg_flow[argidx].safe_to_import_accesses = true;
2632 45105 : continue;
2633 : }
2634 :
2635 : /* Walk basic block and see if its execution can terminate earlier.
2636 : Keep the info for later re-use to avoid quadratic behavoiur here. */
2637 199089 : gimple_stmt_iterator gsi = gsi_for_stmt (call_stmt);
2638 199089 : bool safe = true;
2639 199089 : int n = 0;
2640 628948 : for (gsi_prev (&gsi); !gsi_end_p (gsi); gsi_prev (&gsi))
2641 : {
2642 128281 : bool *b = analyzed_stmts.get (gsi_stmt (gsi));
2643 128281 : if (b)
2644 : {
2645 9454 : safe = *b;
2646 9454 : gsi_next (&gsi);
2647 9454 : break;
2648 : }
2649 118827 : n++;
2650 118827 : if (stmt_may_terminate_function_p (fun, gsi_stmt (gsi), false))
2651 : {
2652 : safe = false;
2653 : break;
2654 : }
2655 : }
2656 199089 : if (n)
2657 : {
2658 33350 : if (gsi_end_p (gsi))
2659 59816 : gsi = gsi_start_bb (gimple_bb (call_stmt));
2660 152177 : for (; gsi_stmt (gsi) != call_stmt; gsi_next (&gsi))
2661 118827 : analyzed_stmts.get_or_insert (gsi_stmt (gsi)) = safe;
2662 : }
2663 :
2664 199089 : if (safe && !always_executed_bbs)
2665 : {
2666 57267 : mark_dfs_back_edges ();
2667 57267 : always_executed_bbs = find_always_executed_bbs (fun, false);
2668 : }
2669 199089 : if (safe && bitmap_bit_p (always_executed_bbs, gimple_bb (call_stmt)->index))
2670 45877 : csum->m_arg_flow[argidx].safe_to_import_accesses = true;
2671 : }
2672 :
2673 : }
2674 837454 : BITMAP_FREE (always_executed_bbs);
2675 :
2676 : /* TODO: Add early exit if we disqualified everything. This also requires
2677 : that we either relax the restriction that
2678 : ipa_param_adjustments.m_always_copy_start must be the number of PARM_DECLs
2679 : or store the number of parameters to IPA-SRA function summary and use that
2680 : when just removing params. */
2681 :
2682 837454 : vec_safe_reserve_exact (ifs->m_parameters, param_count);
2683 837454 : ifs->m_parameters->quick_grow_cleared (param_count);
2684 3221632 : for (unsigned desc_index = 0; desc_index < param_count; desc_index++)
2685 : {
2686 2384178 : gensum_param_desc *s = &(*param_descriptions)[desc_index];
2687 2384178 : isra_param_desc *d = &(*ifs->m_parameters)[desc_index];
2688 :
2689 2384178 : d->param_size_limit = s->param_size_limit;
2690 2384178 : d->size_reached = s->nonarg_acc_size;
2691 2384178 : d->locally_unused = s->locally_unused;
2692 2384178 : d->split_candidate = s->split_candidate;
2693 2384178 : d->by_ref = s->by_ref;
2694 2384178 : d->remove_only_when_retval_removed = s->remove_only_when_retval_removed;
2695 2384178 : d->split_only_when_retval_removed = s->split_only_when_retval_removed;
2696 2384178 : d->conditionally_dereferenceable = s->conditionally_dereferenceable;
2697 :
2698 2384178 : for (gensum_param_access *acc = s->accesses;
2699 2792612 : acc;
2700 408434 : acc = acc->next_sibling)
2701 408434 : copy_accesses_to_ipa_desc (acc, d);
2702 : }
2703 :
2704 837454 : if (dump_file)
2705 135 : dump_isra_param_descriptors (dump_file, node->decl, ifs, false);
2706 837454 : }
2707 :
2708 : /* Return true if there are any overlaps among certain accesses of DESC. If
2709 : non-NULL, set *CERTAIN_ACCESS_PRESENT_P upon encountering a certain access
2710 : too. DESC is assumed to be a split candidate that is not locally
2711 : unused. */
2712 :
2713 : static bool
2714 171581 : overlapping_certain_accesses_p (isra_param_desc *desc,
2715 : bool *certain_access_present_p)
2716 : {
2717 171581 : unsigned pclen = vec_safe_length (desc->accesses);
2718 447557 : for (unsigned i = 0; i < pclen; i++)
2719 : {
2720 278125 : param_access *a1 = (*desc->accesses)[i];
2721 :
2722 278125 : if (!a1->certain)
2723 6176 : continue;
2724 271949 : if (certain_access_present_p)
2725 257005 : *certain_access_present_p = true;
2726 437543 : for (unsigned j = i + 1; j < pclen; j++)
2727 : {
2728 167743 : param_access *a2 = (*desc->accesses)[j];
2729 167743 : if (a2->certain
2730 167669 : && a1->unit_offset < a2->unit_offset + a2->unit_size
2731 167335 : && a1->unit_offset + a1->unit_size > a2->unit_offset)
2732 : return true;
2733 : }
2734 : }
2735 : return false;
2736 : }
2737 :
2738 : /* Check for any overlaps of certain param accesses among splitting candidates
2739 : and signal an ICE if there are any. If CERTAIN_MUST_EXIST is set, also
2740 : check that used splitting candidates have at least one certain access. */
2741 :
2742 : static void
2743 2614037 : verify_splitting_accesses (cgraph_node *node, bool certain_must_exist)
2744 : {
2745 2614037 : isra_func_summary *ifs = func_sums->get (node);
2746 2614037 : if (!ifs || !ifs->m_candidate)
2747 : return;
2748 1402287 : unsigned param_count = vec_safe_length (ifs->m_parameters);
2749 4548394 : for (unsigned pidx = 0; pidx < param_count; pidx++)
2750 : {
2751 3146107 : isra_param_desc *desc = &(*ifs->m_parameters)[pidx];
2752 3146107 : if (!desc->split_candidate || desc->locally_unused)
2753 2989401 : continue;
2754 :
2755 156706 : bool certain_access_present = !certain_must_exist;
2756 156706 : if (overlapping_certain_accesses_p (desc, &certain_access_present))
2757 0 : internal_error ("function %qs, parameter %u, has IPA-SRA accesses "
2758 : "which overlap", node->dump_name (), pidx);
2759 156706 : if (!certain_access_present)
2760 0 : internal_error ("function %qs, parameter %u, is used but does not "
2761 : "have any certain IPA-SRA access",
2762 : node->dump_name (), pidx);
2763 : }
2764 : }
2765 :
2766 : /* Intraprocedural part of IPA-SRA analysis. Scan bodies of all functions in
2767 : this compilation unit and create summary structures describing IPA-SRA
2768 : opportunities and constraints in them. */
2769 :
2770 : static void
2771 125617 : ipa_sra_generate_summary (void)
2772 : {
2773 125617 : struct cgraph_node *node;
2774 :
2775 125617 : gcc_checking_assert (!func_sums);
2776 125617 : gcc_checking_assert (!call_sums);
2777 125617 : func_sums
2778 125617 : = (new (ggc_alloc_no_dtor <ipa_sra_function_summaries> ())
2779 125617 : ipa_sra_function_summaries (symtab, true));
2780 125617 : call_sums = new ipa_sra_call_summaries (symtab);
2781 :
2782 1375081 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
2783 1249464 : ipa_sra_summarize_function (node);
2784 125617 : return;
2785 : }
2786 :
2787 : /* Write intraprocedural analysis information about E and all of its outgoing
2788 : edges into a stream for LTO WPA. */
2789 :
2790 : static void
2791 281358 : isra_write_edge_summary (output_block *ob, cgraph_edge *e)
2792 : {
2793 281358 : isra_call_summary *csum = call_sums->get (e);
2794 281358 : unsigned input_count = csum->m_arg_flow.length ();
2795 281358 : streamer_write_uhwi (ob, input_count);
2796 754143 : for (unsigned i = 0; i < input_count; i++)
2797 : {
2798 472785 : isra_param_flow *ipf = &csum->m_arg_flow[i];
2799 472785 : streamer_write_hwi (ob, ipf->length);
2800 472785 : bitpack_d bp = bitpack_create (ob->main_stream);
2801 547612 : for (int j = 0; j < ipf->length; j++)
2802 74827 : bp_pack_value (&bp, ipf->inputs[j], 8);
2803 472785 : bp_pack_value (&bp, ipf->aggregate_pass_through, 1);
2804 472785 : bp_pack_value (&bp, ipf->pointer_pass_through, 1);
2805 472785 : bp_pack_value (&bp, ipf->safe_to_import_accesses, 1);
2806 472785 : bp_pack_value (&bp, ipf->constructed_for_calls, 1);
2807 472785 : streamer_write_bitpack (&bp);
2808 472785 : streamer_write_uhwi (ob, ipf->unit_offset);
2809 472785 : streamer_write_uhwi (ob, ipf->unit_size);
2810 : }
2811 281358 : bitpack_d bp = bitpack_create (ob->main_stream);
2812 281358 : bp_pack_value (&bp, csum->m_return_ignored, 1);
2813 281358 : bp_pack_value (&bp, csum->m_return_returned, 1);
2814 281358 : bp_pack_value (&bp, csum->m_bit_aligned_arg, 1);
2815 281358 : bp_pack_value (&bp, csum->m_before_any_store, 1);
2816 281358 : streamer_write_bitpack (&bp);
2817 281358 : }
2818 :
2819 : /* Write intraprocedural analysis information about NODE and all of its outgoing
2820 : edges into a stream for LTO WPA. */
2821 :
2822 : static void
2823 49197 : isra_write_node_summary (output_block *ob, cgraph_node *node)
2824 : {
2825 49197 : isra_func_summary *ifs = func_sums->get (node);
2826 49197 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
2827 49197 : int node_ref = lto_symtab_encoder_encode (encoder, node);
2828 49197 : streamer_write_uhwi (ob, node_ref);
2829 :
2830 49197 : unsigned param_desc_count = vec_safe_length (ifs->m_parameters);
2831 49197 : streamer_write_uhwi (ob, param_desc_count);
2832 298908 : for (unsigned i = 0; i < param_desc_count; i++)
2833 : {
2834 249711 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
2835 249711 : unsigned access_count = vec_safe_length (desc->accesses);
2836 249711 : streamer_write_uhwi (ob, access_count);
2837 252980 : for (unsigned j = 0; j < access_count; j++)
2838 : {
2839 3269 : param_access *acc = (*desc->accesses)[j];
2840 3269 : stream_write_tree (ob, acc->type, true);
2841 3269 : stream_write_tree (ob, acc->alias_ptr_type, true);
2842 3269 : streamer_write_uhwi (ob, acc->unit_offset);
2843 3269 : streamer_write_uhwi (ob, acc->unit_size);
2844 3269 : bitpack_d bp = bitpack_create (ob->main_stream);
2845 3269 : bp_pack_value (&bp, acc->certain, 1);
2846 3269 : bp_pack_value (&bp, acc->reverse, 1);
2847 3269 : streamer_write_bitpack (&bp);
2848 : }
2849 249711 : streamer_write_uhwi (ob, desc->param_size_limit);
2850 249711 : streamer_write_uhwi (ob, desc->size_reached);
2851 249711 : gcc_assert (desc->safe_size == 0);
2852 249711 : bitpack_d bp = bitpack_create (ob->main_stream);
2853 249711 : bp_pack_value (&bp, desc->locally_unused, 1);
2854 249711 : bp_pack_value (&bp, desc->split_candidate, 1);
2855 249711 : bp_pack_value (&bp, desc->by_ref, 1);
2856 249711 : gcc_assert (!desc->not_specially_constructed);
2857 249711 : bp_pack_value (&bp, desc->remove_only_when_retval_removed, 1);
2858 249711 : bp_pack_value (&bp, desc->split_only_when_retval_removed, 1);
2859 249711 : bp_pack_value (&bp, desc->conditionally_dereferenceable, 1);
2860 249711 : gcc_assert (!desc->safe_size_set);
2861 249711 : streamer_write_bitpack (&bp);
2862 : }
2863 49197 : bitpack_d bp = bitpack_create (ob->main_stream);
2864 49197 : bp_pack_value (&bp, ifs->m_candidate, 1);
2865 49197 : bp_pack_value (&bp, ifs->m_returns_value, 1);
2866 49197 : bp_pack_value (&bp, ifs->m_return_ignored, 1);
2867 49197 : gcc_assert (!ifs->m_queued);
2868 49197 : streamer_write_bitpack (&bp);
2869 :
2870 328104 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
2871 278907 : isra_write_edge_summary (ob, e);
2872 51648 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
2873 2451 : isra_write_edge_summary (ob, e);
2874 49197 : }
2875 :
2876 : /* Write intraprocedural analysis information into a stream for LTO WPA. */
2877 :
2878 : static void
2879 19876 : ipa_sra_write_summary (void)
2880 : {
2881 19876 : if (!func_sums || !call_sums)
2882 0 : return;
2883 :
2884 19876 : struct output_block *ob = create_output_block (LTO_section_ipa_sra);
2885 19876 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
2886 19876 : ob->symbol = NULL;
2887 :
2888 19876 : unsigned int count = 0;
2889 19876 : lto_symtab_encoder_iterator lsei;
2890 19876 : for (lsei = lsei_start_function_in_partition (encoder);
2891 113318 : !lsei_end_p (lsei);
2892 93442 : lsei_next_function_in_partition (&lsei))
2893 : {
2894 93442 : cgraph_node *node = lsei_cgraph_node (lsei);
2895 93442 : if (node->has_gimple_body_p ()
2896 93442 : && func_sums->get (node) != NULL)
2897 49197 : count++;
2898 : }
2899 19876 : streamer_write_uhwi (ob, count);
2900 :
2901 : /* Process all of the functions. */
2902 113318 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
2903 93442 : lsei_next_function_in_partition (&lsei))
2904 : {
2905 93442 : cgraph_node *node = lsei_cgraph_node (lsei);
2906 93442 : if (node->has_gimple_body_p ()
2907 93442 : && func_sums->get (node) != NULL)
2908 49197 : isra_write_node_summary (ob, node);
2909 : }
2910 19876 : streamer_write_char_stream (ob->main_stream, 0);
2911 19876 : produce_asm (ob);
2912 19876 : destroy_output_block (ob);
2913 : }
2914 :
2915 : /* Read intraprocedural analysis information about E and all of its outgoing
2916 : edges into a stream for LTO WPA. */
2917 :
2918 : static void
2919 253186 : isra_read_edge_summary (struct lto_input_block *ib, cgraph_edge *cs)
2920 : {
2921 253186 : isra_call_summary *csum = call_sums->get_create (cs);
2922 253186 : unsigned input_count = streamer_read_uhwi (ib);
2923 253186 : csum->init_inputs (input_count);
2924 700162 : for (unsigned i = 0; i < input_count; i++)
2925 : {
2926 446976 : isra_param_flow *ipf = &csum->m_arg_flow[i];
2927 446976 : ipf->length = streamer_read_hwi (ib);
2928 446976 : bitpack_d bp = streamer_read_bitpack (ib);
2929 514364 : for (int j = 0; j < ipf->length; j++)
2930 67388 : ipf->inputs[j] = bp_unpack_value (&bp, 8);
2931 446976 : ipf->aggregate_pass_through = bp_unpack_value (&bp, 1);
2932 446976 : ipf->pointer_pass_through = bp_unpack_value (&bp, 1);
2933 446976 : ipf->safe_to_import_accesses = bp_unpack_value (&bp, 1);
2934 446976 : ipf->constructed_for_calls = bp_unpack_value (&bp, 1);
2935 446976 : ipf->unit_offset = streamer_read_uhwi (ib);
2936 446976 : ipf->unit_size = streamer_read_uhwi (ib);
2937 : }
2938 253186 : bitpack_d bp = streamer_read_bitpack (ib);
2939 253186 : csum->m_return_ignored = bp_unpack_value (&bp, 1);
2940 253186 : csum->m_return_returned = bp_unpack_value (&bp, 1);
2941 253186 : csum->m_bit_aligned_arg = bp_unpack_value (&bp, 1);
2942 253186 : csum->m_before_any_store = bp_unpack_value (&bp, 1);
2943 253186 : }
2944 :
2945 : /* Read intraprocedural analysis information about NODE and all of its outgoing
2946 : edges into a stream for LTO WPA. */
2947 :
2948 : static void
2949 34780 : isra_read_node_info (struct lto_input_block *ib, cgraph_node *node,
2950 : struct data_in *data_in)
2951 : {
2952 34780 : isra_func_summary *ifs = func_sums->get_create (node);
2953 34780 : unsigned param_desc_count = streamer_read_uhwi (ib);
2954 34780 : if (param_desc_count > 0)
2955 : {
2956 17722 : vec_safe_reserve_exact (ifs->m_parameters, param_desc_count);
2957 17722 : ifs->m_parameters->quick_grow_cleared (param_desc_count);
2958 : }
2959 62708 : for (unsigned i = 0; i < param_desc_count; i++)
2960 : {
2961 27928 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
2962 27928 : unsigned access_count = streamer_read_uhwi (ib);
2963 29675 : for (unsigned j = 0; j < access_count; j++)
2964 : {
2965 1747 : param_access *acc = ggc_cleared_alloc<param_access> ();
2966 1747 : acc->type = stream_read_tree (ib, data_in);
2967 1747 : acc->alias_ptr_type = stream_read_tree (ib, data_in);
2968 1747 : acc->unit_offset = streamer_read_uhwi (ib);
2969 1747 : acc->unit_size = streamer_read_uhwi (ib);
2970 1747 : bitpack_d bp = streamer_read_bitpack (ib);
2971 1747 : acc->certain = bp_unpack_value (&bp, 1);
2972 1747 : acc->reverse = bp_unpack_value (&bp, 1);
2973 1747 : vec_safe_push (desc->accesses, acc);
2974 : }
2975 27928 : desc->param_size_limit = streamer_read_uhwi (ib);
2976 27928 : desc->size_reached = streamer_read_uhwi (ib);
2977 27928 : desc->safe_size = 0;
2978 27928 : bitpack_d bp = streamer_read_bitpack (ib);
2979 27928 : desc->locally_unused = bp_unpack_value (&bp, 1);
2980 27928 : desc->split_candidate = bp_unpack_value (&bp, 1);
2981 27928 : desc->by_ref = bp_unpack_value (&bp, 1);
2982 27928 : desc->not_specially_constructed = 0;
2983 27928 : desc->remove_only_when_retval_removed = bp_unpack_value (&bp, 1);
2984 27928 : desc->split_only_when_retval_removed = bp_unpack_value (&bp, 1);
2985 27928 : desc->conditionally_dereferenceable = bp_unpack_value (&bp, 1);
2986 27928 : desc->safe_size_set = 0;
2987 : }
2988 34780 : bitpack_d bp = streamer_read_bitpack (ib);
2989 34780 : ifs->m_candidate = bp_unpack_value (&bp, 1);
2990 34780 : ifs->m_returns_value = bp_unpack_value (&bp, 1);
2991 34780 : ifs->m_return_ignored = bp_unpack_value (&bp, 1);
2992 34780 : ifs->m_queued = 0;
2993 :
2994 286641 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
2995 251861 : isra_read_edge_summary (ib, e);
2996 36105 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
2997 1325 : isra_read_edge_summary (ib, e);
2998 34780 : }
2999 :
3000 : /* Read IPA-SRA summaries from a section in file FILE_DATA of length LEN with
3001 : data DATA. TODO: This function was copied almost verbatim from ipa-prop.cc,
3002 : it should be possible to unify them somehow. */
3003 :
3004 : static void
3005 10875 : isra_read_summary_section (struct lto_file_decl_data *file_data,
3006 : const char *data, size_t len)
3007 : {
3008 10875 : const struct lto_function_header *header =
3009 : (const struct lto_function_header *) data;
3010 10875 : const int cfg_offset = sizeof (struct lto_function_header);
3011 10875 : const int main_offset = cfg_offset + header->cfg_size;
3012 10875 : const int string_offset = main_offset + header->main_size;
3013 10875 : struct data_in *data_in;
3014 10875 : unsigned int i;
3015 10875 : unsigned int count;
3016 :
3017 10875 : lto_input_block ib_main ((const char *) data + main_offset,
3018 10875 : header->main_size, file_data);
3019 :
3020 10875 : data_in =
3021 21750 : lto_data_in_create (file_data, (const char *) data + string_offset,
3022 10875 : header->string_size, vNULL);
3023 10875 : count = streamer_read_uhwi (&ib_main);
3024 :
3025 45655 : for (i = 0; i < count; i++)
3026 : {
3027 34780 : unsigned int index;
3028 34780 : struct cgraph_node *node;
3029 34780 : lto_symtab_encoder_t encoder;
3030 :
3031 34780 : index = streamer_read_uhwi (&ib_main);
3032 34780 : encoder = file_data->symtab_node_encoder;
3033 34780 : node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
3034 : index));
3035 34780 : gcc_assert (node->definition);
3036 34780 : isra_read_node_info (&ib_main, node, data_in);
3037 : }
3038 10875 : lto_free_section_data (file_data, LTO_section_ipa_sra, NULL, data,
3039 : len);
3040 10875 : lto_data_in_delete (data_in);
3041 10875 : }
3042 :
3043 : /* Read intraprocedural analysis information into a stream for LTO WPA. */
3044 :
3045 : static void
3046 10200 : ipa_sra_read_summary (void)
3047 : {
3048 10200 : struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
3049 10200 : struct lto_file_decl_data *file_data;
3050 10200 : unsigned int j = 0;
3051 :
3052 10200 : gcc_checking_assert (!func_sums);
3053 10200 : gcc_checking_assert (!call_sums);
3054 10200 : func_sums
3055 10200 : = (new (ggc_alloc_no_dtor <ipa_sra_function_summaries> ())
3056 10200 : ipa_sra_function_summaries (symtab, true));
3057 10200 : call_sums = new ipa_sra_call_summaries (symtab);
3058 :
3059 21086 : while ((file_data = file_data_vec[j++]))
3060 : {
3061 10886 : size_t len;
3062 10886 : const char *data
3063 10886 : = lto_get_summary_section_data (file_data, LTO_section_ipa_sra, &len);
3064 10886 : if (data)
3065 10875 : isra_read_summary_section (file_data, data, len);
3066 : }
3067 10200 : }
3068 :
3069 : /* Dump all IPA-SRA summary data for all cgraph nodes and edges to file F. If
3070 : HINTS is true, also dump IPA-analysis computed hints. */
3071 :
3072 : static void
3073 63 : ipa_sra_dump_all_summaries (FILE *f, bool hints)
3074 : {
3075 63 : cgraph_node *node;
3076 258 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
3077 : {
3078 195 : fprintf (f, "\nSummary for node %s:\n", node->dump_name ());
3079 :
3080 195 : isra_func_summary *ifs = func_sums->get (node);
3081 195 : if (!ifs)
3082 10 : fprintf (f, " Function does not have any associated IPA-SRA "
3083 : "summary\n");
3084 185 : else if (!ifs->m_candidate)
3085 12 : fprintf (f, " Not a candidate function\n");
3086 : else
3087 : {
3088 173 : if (ifs->m_returns_value)
3089 95 : fprintf (f, " Returns value\n");
3090 173 : if (vec_safe_is_empty (ifs->m_parameters))
3091 41 : fprintf (f, " No parameter information. \n");
3092 : else
3093 446 : for (unsigned i = 0; i < ifs->m_parameters->length (); ++i)
3094 : {
3095 314 : fprintf (f, " Descriptor for parameter %i:\n", i);
3096 314 : dump_isra_param_descriptor (f, &(*ifs->m_parameters)[i], hints);
3097 : }
3098 173 : fprintf (f, "\n");
3099 : }
3100 :
3101 195 : struct cgraph_edge *cs;
3102 375 : for (cs = node->callees; cs; cs = cs->next_callee)
3103 : {
3104 180 : fprintf (f, " Summary for edge %s->%s:\n", cs->caller->dump_name (),
3105 180 : cs->callee->dump_name ());
3106 180 : isra_call_summary *csum = call_sums->get (cs);
3107 180 : if (csum)
3108 180 : csum->dump (f);
3109 : else
3110 0 : fprintf (f, " Call summary is MISSING!\n");
3111 : }
3112 :
3113 : }
3114 63 : fprintf (f, "\n\n");
3115 63 : }
3116 :
3117 : /* Perform function-scope viability tests that can be only made at IPA level
3118 : and return false if the function is deemed unsuitable for IPA-SRA. */
3119 :
3120 : static bool
3121 1028864 : ipa_sra_ipa_function_checks (cgraph_node *node)
3122 : {
3123 1028864 : if (!node->can_be_local_p ())
3124 : {
3125 658671 : if (dump_file)
3126 87 : fprintf (dump_file, "Function %s disqualified because it cannot be "
3127 : "made local.\n", node->dump_name ());
3128 658671 : return false;
3129 : }
3130 370193 : if (!node->can_change_signature)
3131 : {
3132 0 : if (dump_file)
3133 0 : fprintf (dump_file, "Function can not change signature.\n");
3134 0 : return false;
3135 : }
3136 :
3137 : return true;
3138 : }
3139 :
3140 : /* Issues found out by check_callers_for_issues. */
3141 :
3142 : struct caller_issues
3143 : {
3144 : /* The candidate being considered. */
3145 : cgraph_node *candidate;
3146 : /* There is a thunk among callers. */
3147 : bool thunk;
3148 : /* Set if there is at least one caller that is OK. */
3149 : bool there_is_one;
3150 : /* Call site with no available information. */
3151 : bool unknown_callsite;
3152 : /* Call from outside the candidate's comdat group. */
3153 : bool call_from_outside_comdat;
3154 : /* There is a bit-aligned load into one of non-gimple-typed arguments. */
3155 : bool bit_aligned_aggregate_argument;
3156 : };
3157 :
3158 : /* Worker for call_for_symbol_and_aliases, set any flags of passed caller_issues
3159 : that apply. */
3160 :
3161 : static bool
3162 375461 : check_for_caller_issues (struct cgraph_node *node, void *data)
3163 : {
3164 375461 : struct caller_issues *issues = (struct caller_issues *) data;
3165 :
3166 1432410 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
3167 : {
3168 1057191 : if (cs->caller->thunk)
3169 : {
3170 0 : issues->thunk = true;
3171 : /* TODO: We should be able to process at least some types of
3172 : thunks. */
3173 0 : return true;
3174 : }
3175 1057191 : if (issues->candidate->calls_comdat_local
3176 10 : && issues->candidate->same_comdat_group
3177 1057201 : && !issues->candidate->in_same_comdat_group_p (cs->caller))
3178 : {
3179 0 : issues->call_from_outside_comdat = true;
3180 0 : return true;
3181 : }
3182 :
3183 1057191 : isra_call_summary *csum = call_sums->get (cs);
3184 1057191 : if (!csum)
3185 : {
3186 242 : issues->unknown_callsite = true;
3187 242 : return true;
3188 : }
3189 :
3190 1056949 : if (csum->m_bit_aligned_arg)
3191 0 : issues->bit_aligned_aggregate_argument = true;
3192 :
3193 1056949 : issues->there_is_one = true;
3194 : }
3195 : return false;
3196 : }
3197 :
3198 : /* Look at all incoming edges to NODE, including aliases and thunks and look
3199 : for problems. Return true if NODE type should not be modified at all. */
3200 :
3201 : static bool
3202 370193 : check_all_callers_for_issues (cgraph_node *node)
3203 : {
3204 370193 : struct caller_issues issues;
3205 370193 : memset (&issues, 0, sizeof (issues));
3206 370193 : issues.candidate = node;
3207 :
3208 370193 : node->call_for_symbol_and_aliases (check_for_caller_issues, &issues, true);
3209 370193 : if (issues.unknown_callsite)
3210 : {
3211 242 : if (dump_file && (dump_flags & TDF_DETAILS))
3212 0 : fprintf (dump_file, "A call of %s has not been analyzed. Disabling "
3213 : "all modifications.\n", node->dump_name ());
3214 242 : return true;
3215 : }
3216 : /* TODO: We should be able to process at least some types of thunks. */
3217 369951 : if (issues.thunk)
3218 : {
3219 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3220 0 : fprintf (dump_file, "A call of %s is through thunk, which are not"
3221 : " handled yet. Disabling all modifications.\n",
3222 : node->dump_name ());
3223 0 : return true;
3224 : }
3225 369951 : if (issues.call_from_outside_comdat)
3226 : {
3227 0 : if (dump_file)
3228 0 : fprintf (dump_file, "Function would become private comdat called "
3229 : "outside of its comdat group.\n");
3230 0 : return true;
3231 : }
3232 :
3233 369951 : if (issues.bit_aligned_aggregate_argument)
3234 : {
3235 : /* Let's only remove parameters/return values from such functions.
3236 : TODO: We could only prevent splitting the problematic parameters if
3237 : anybody thinks it is worth it. */
3238 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3239 0 : fprintf (dump_file, "A call of %s has bit-aligned aggregate argument,"
3240 : " disabling parameter splitting.\n", node->dump_name ());
3241 :
3242 0 : isra_func_summary *ifs = func_sums->get (node);
3243 0 : gcc_checking_assert (ifs);
3244 0 : unsigned param_count = vec_safe_length (ifs->m_parameters);
3245 0 : for (unsigned i = 0; i < param_count; i++)
3246 0 : (*ifs->m_parameters)[i].split_candidate = false;
3247 : }
3248 369951 : if (!issues.there_is_one)
3249 : {
3250 1 : if (dump_file && (dump_flags & TDF_DETAILS))
3251 0 : fprintf (dump_file, "There is no call to %s that we can modify. "
3252 : "Disabling all modifications.\n", node->dump_name ());
3253 1 : return true;
3254 : }
3255 : return false;
3256 : }
3257 :
3258 : /* Find the access with corresponding OFFSET and SIZE among accesses in
3259 : PARAM_DESC and return it or NULL if such an access is not there. */
3260 :
3261 : static param_access *
3262 16019 : find_param_access (isra_param_desc *param_desc, unsigned offset, unsigned size)
3263 : {
3264 16019 : unsigned pclen = vec_safe_length (param_desc->accesses);
3265 :
3266 : /* The search is linear but the number of stored accesses is bound by
3267 : PARAM_IPA_SRA_MAX_REPLACEMENTS, so most probably 8. */
3268 :
3269 16096 : for (unsigned i = 0; i < pclen; i++)
3270 16051 : if ((*param_desc->accesses)[i]->unit_offset == offset
3271 16051 : && (*param_desc->accesses)[i]->unit_size == size)
3272 : return (*param_desc->accesses)[i];
3273 :
3274 : return NULL;
3275 : }
3276 :
3277 : /* Return iff the total size of definite replacement SIZE would violate the
3278 : limit set for it in PARAM. */
3279 :
3280 : static bool
3281 216053 : size_would_violate_limit_p (isra_param_desc *desc, unsigned size)
3282 : {
3283 216053 : unsigned limit = desc->param_size_limit;
3284 0 : if (size > limit
3285 213349 : || (!desc->by_ref && size == limit))
3286 0 : return true;
3287 : return false;
3288 : }
3289 :
3290 : /* Increase reached size of DESC by SIZE or disqualify it if it would violate
3291 : the set limit. IDX is the parameter number which is dumped when
3292 : disqualifying. */
3293 :
3294 : static void
3295 12726 : bump_reached_size (isra_param_desc *desc, unsigned size, unsigned idx)
3296 : {
3297 12726 : unsigned after = desc->size_reached + size;
3298 12726 : if (size_would_violate_limit_p (desc, after))
3299 : {
3300 12694 : if (dump_file && (dump_flags & TDF_DETAILS))
3301 0 : fprintf (dump_file, " ...size limit reached, disqualifying "
3302 : "candidate parameter %u\n", idx);
3303 12694 : desc->split_candidate = false;
3304 12694 : return;
3305 : }
3306 32 : desc->size_reached = after;
3307 : }
3308 :
3309 : /* Take all actions required to deal with an edge CS that represents a call to
3310 : an unknown or un-analyzed function, for both parameter removal and
3311 : splitting. */
3312 :
3313 : static void
3314 510268 : process_edge_to_unknown_caller (cgraph_edge *cs)
3315 : {
3316 510268 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3317 510268 : gcc_checking_assert (from_ifs);
3318 510268 : isra_call_summary *csum = call_sums->get (cs);
3319 :
3320 510268 : if (dump_file && (dump_flags & TDF_DETAILS))
3321 4 : fprintf (dump_file, "Processing an edge to an unknown caller from %s:\n",
3322 : cs->caller->dump_name ());
3323 :
3324 510268 : unsigned args_count = csum->m_arg_flow.length ();
3325 1243676 : for (unsigned i = 0; i < args_count; i++)
3326 : {
3327 733408 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3328 :
3329 733408 : if (ipf->pointer_pass_through)
3330 : {
3331 192474 : isra_param_desc *param_desc
3332 192474 : = &(*from_ifs->m_parameters)[get_single_param_flow_source (ipf)];
3333 192474 : param_desc->locally_unused = false;
3334 192474 : param_desc->split_candidate = false;
3335 192474 : continue;
3336 192474 : }
3337 540934 : if (ipf->aggregate_pass_through)
3338 : {
3339 5719 : unsigned idx = get_single_param_flow_source (ipf);
3340 5719 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3341 :
3342 5719 : param_desc->locally_unused = false;
3343 5719 : if (!param_desc->split_candidate)
3344 1780 : continue;
3345 3939 : gcc_assert (!param_desc->by_ref);
3346 7878 : param_access *pacc = find_param_access (param_desc, ipf->unit_offset,
3347 3939 : ipf->unit_size);
3348 3939 : gcc_checking_assert (pacc);
3349 3939 : pacc->certain = true;
3350 3939 : if (overlapping_certain_accesses_p (param_desc, NULL))
3351 : {
3352 619 : if (dump_file && (dump_flags & TDF_DETAILS))
3353 0 : fprintf (dump_file, " ...leading to overlap, "
3354 : " disqualifying candidate parameter %u\n",
3355 : idx);
3356 619 : param_desc->split_candidate = false;
3357 : }
3358 : else
3359 3320 : bump_reached_size (param_desc, pacc->unit_size, idx);
3360 3939 : ipf->aggregate_pass_through = false;
3361 3939 : continue;
3362 3939 : }
3363 :
3364 633009 : for (int j = 0; j < ipf->length; j++)
3365 : {
3366 97794 : int input_idx = ipf->inputs[j];
3367 97794 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3368 : }
3369 : }
3370 510268 : }
3371 :
3372 : /* Propagate parameter removal information through cross-SCC edge CS,
3373 : i.e. decrease the use count in the caller parameter descriptor for each use
3374 : in this call. */
3375 :
3376 : static void
3377 821882 : param_removal_cross_scc_edge (cgraph_edge *cs)
3378 : {
3379 821882 : enum availability availability;
3380 821882 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3381 821882 : isra_func_summary *to_ifs = func_sums->get (callee);
3382 380487 : if (!to_ifs || !to_ifs->m_candidate
3383 335076 : || (availability < AVAIL_AVAILABLE)
3384 1156958 : || vec_safe_is_empty (to_ifs->m_parameters))
3385 : {
3386 496737 : process_edge_to_unknown_caller (cs);
3387 496737 : return;
3388 : }
3389 325145 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3390 325145 : gcc_checking_assert (from_ifs);
3391 :
3392 325145 : isra_call_summary *csum = call_sums->get (cs);
3393 325145 : unsigned args_count = csum->m_arg_flow.length ();
3394 325145 : unsigned param_count = vec_safe_length (to_ifs->m_parameters);
3395 :
3396 1143975 : for (unsigned i = 0; i < args_count; i++)
3397 : {
3398 818830 : bool unused_in_callee;
3399 818830 : if (i < param_count)
3400 818826 : unused_in_callee = (*to_ifs->m_parameters)[i].locally_unused;
3401 : else
3402 : unused_in_callee = false;
3403 :
3404 818826 : if (!unused_in_callee)
3405 : {
3406 769488 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3407 1001893 : for (int j = 0; j < ipf->length; j++)
3408 : {
3409 232405 : int input_idx = ipf->inputs[j];
3410 232405 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3411 : }
3412 : }
3413 : }
3414 : }
3415 :
3416 : /* Unless it is already there, push NODE which is also described by IFS to
3417 : STACK. */
3418 :
3419 : static void
3420 151152 : isra_push_node_to_stack (cgraph_node *node, isra_func_summary *ifs,
3421 : vec<cgraph_node *> *stack)
3422 : {
3423 0 : if (!ifs->m_queued)
3424 : {
3425 150768 : ifs->m_queued = true;
3426 150768 : stack->safe_push (node);
3427 : }
3428 0 : }
3429 :
3430 : /* If parameter with index INPUT_IDX is marked as locally unused, mark it as
3431 : used and push CALLER on STACK. */
3432 :
3433 : static void
3434 11692 : isra_mark_caller_param_used (isra_func_summary *from_ifs, int input_idx,
3435 : cgraph_node *caller, vec<cgraph_node *> *stack)
3436 : {
3437 11692 : if ((*from_ifs->m_parameters)[input_idx].locally_unused)
3438 : {
3439 1095 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3440 1853 : isra_push_node_to_stack (caller, from_ifs, stack);
3441 : }
3442 11692 : }
3443 :
3444 : /* Combine safe_size of DESC with SIZE and return true if it has changed. */
3445 :
3446 : static bool
3447 2382940 : update_safe_size (isra_param_desc *desc, unsigned size)
3448 : {
3449 0 : if (!desc->safe_size_set)
3450 : {
3451 760364 : desc->safe_size_set = 1;
3452 760364 : desc->safe_size = size;
3453 760364 : return true;
3454 : }
3455 1622576 : if (desc->safe_size <= size)
3456 : return false;
3457 19767 : desc->safe_size = size;
3458 19767 : return true;
3459 : }
3460 :
3461 : /* Set all param hints in DESC to the pessimistic values. Return true if any
3462 : hints that need to potentially trigger further propagation have changed. */
3463 :
3464 : static bool
3465 711676 : flip_all_hints_pessimistic (isra_param_desc *desc)
3466 : {
3467 711676 : desc->not_specially_constructed = true;
3468 0 : return update_safe_size (desc, 0);
3469 : }
3470 :
3471 : /* Because we have not analyzed or otherwise problematic caller, go over all
3472 : parameter int flags of IFS describing a call graph node of a calllee and
3473 : turn them pessimistic. Return true if any hints that need to potentially
3474 : trigger further propagation have changed. */
3475 :
3476 : static bool
3477 424745 : flip_all_param_hints_pessimistic (isra_func_summary *ifs)
3478 : {
3479 424745 : if (!ifs || !ifs->m_candidate)
3480 : return false;
3481 :
3482 25063 : bool ret = false;
3483 25063 : unsigned param_count = vec_safe_length (ifs->m_parameters);
3484 :
3485 70470 : for (unsigned i = 0; i < param_count; i++)
3486 90814 : ret |= flip_all_hints_pessimistic (&(*ifs->m_parameters)[i]);
3487 :
3488 : return ret;
3489 : }
3490 :
3491 : /* Propagate hints accross edge CS which ultimately leads to a node described
3492 : by TO_IFS. Return true if any hints of the callee which should potentially
3493 : trigger further propagation have changed. */
3494 :
3495 : static bool
3496 4847171 : propagate_param_hints_accross_call (cgraph_edge *cs, isra_func_summary *to_ifs)
3497 : {
3498 4847171 : if (!to_ifs || !to_ifs->m_candidate)
3499 : return false;
3500 :
3501 1610414 : isra_call_summary *csum = call_sums->get (cs);
3502 1610414 : bool ret = false;
3503 1610414 : unsigned args_count = csum->m_arg_flow.length ();
3504 1610414 : unsigned param_count = vec_safe_length (to_ifs->m_parameters);
3505 :
3506 5138448 : for (unsigned i = 0; i < param_count; i++)
3507 : {
3508 3528034 : isra_param_desc *desc = &(*to_ifs->m_parameters)[i];
3509 3528034 : if (i >= args_count)
3510 : {
3511 666269 : ret |= flip_all_hints_pessimistic (desc);
3512 666269 : continue;
3513 : }
3514 :
3515 2861765 : if (desc->by_ref)
3516 : {
3517 1671282 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3518 :
3519 1671282 : if (!ipf->constructed_for_calls)
3520 1248046 : desc->not_specially_constructed = true;
3521 :
3522 1671282 : if (ipf->pointer_pass_through)
3523 : {
3524 270945 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3525 270945 : int srcidx = get_single_param_flow_source (ipf);
3526 270945 : if (vec_safe_length (from_ifs->m_parameters) > (unsigned) srcidx)
3527 : {
3528 162472 : isra_param_desc *src_d = &(*from_ifs->m_parameters)[srcidx];
3529 162472 : if (src_d->safe_size_set)
3530 324908 : ret |= update_safe_size (desc, src_d->safe_size);
3531 : }
3532 : else
3533 216946 : ret |= update_safe_size (desc, 0);
3534 : }
3535 1400337 : else if (!ipf->aggregate_pass_through)
3536 2800672 : ret |= update_safe_size (desc, ipf->unit_size);
3537 : else
3538 : /* LTOing type-mismatched programs can end up here. */
3539 2 : ret |= update_safe_size (desc, 0);
3540 : }
3541 : }
3542 : return ret;
3543 : }
3544 :
3545 : /* Propagate hints from NODE described by FROM_IFS to all its (dorect) callees,
3546 : push those that may need re-visiting onto STACK. */
3547 :
3548 : static void
3549 1349049 : propagate_hints_to_all_callees (cgraph_node *node, isra_func_summary *from_ifs,
3550 : vec<cgraph_node *> *stack)
3551 : {
3552 6620965 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
3553 : {
3554 5271916 : enum availability availability;
3555 5271916 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3556 5271916 : isra_func_summary *to_ifs = func_sums->get (callee);
3557 5271916 : if (!from_ifs)
3558 : {
3559 424745 : if (flip_all_param_hints_pessimistic (to_ifs)
3560 424745 : && ipa_edge_within_scc (cs))
3561 230 : isra_push_node_to_stack (callee, to_ifs, stack);
3562 : }
3563 4847171 : else if (propagate_param_hints_accross_call (cs, to_ifs)
3564 4847171 : && ipa_edge_within_scc (cs))
3565 4799 : isra_push_node_to_stack (callee, to_ifs, stack);
3566 : }
3567 1349049 : }
3568 :
3569 : /* Propagate information that any parameter is not used only locally within a
3570 : SCC across CS to the caller, which must be in the same SCC as the
3571 : callee. Push any callers that need to be re-processed to STACK. */
3572 :
3573 : static void
3574 28515 : propagate_used_across_scc_edge (cgraph_edge *cs, vec<cgraph_node *> *stack)
3575 : {
3576 28515 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3577 28515 : if (!from_ifs || vec_safe_is_empty (from_ifs->m_parameters))
3578 15444 : return;
3579 :
3580 13071 : isra_call_summary *csum = call_sums->get (cs);
3581 13071 : gcc_checking_assert (csum);
3582 13071 : unsigned args_count = csum->m_arg_flow.length ();
3583 13071 : enum availability availability;
3584 13071 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3585 13071 : isra_func_summary *to_ifs = func_sums->get (callee);
3586 :
3587 13071 : unsigned param_count
3588 12867 : = (to_ifs && (availability >= AVAIL_AVAILABLE))
3589 25672 : ? vec_safe_length (to_ifs->m_parameters) : 0;
3590 36958 : for (unsigned i = 0; i < args_count; i++)
3591 : {
3592 27193 : if (i < param_count
3593 23887 : && (*to_ifs->m_parameters)[i].locally_unused)
3594 3306 : continue;
3595 :
3596 : /* The argument is needed in the callee it, we must mark the parameter as
3597 : used also in the caller and its callers within this SCC. */
3598 20581 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3599 32273 : for (int j = 0; j < ipf->length; j++)
3600 : {
3601 11692 : int input_idx = ipf->inputs[j];
3602 11692 : isra_mark_caller_param_used (from_ifs, input_idx, cs->caller, stack);
3603 : }
3604 : }
3605 : }
3606 :
3607 : /* Propagate information that any parameter is not used only locally within a
3608 : SCC (i.e. is used also elsewhere) to all callers of NODE that are in the
3609 : same SCC. Push any callers that need to be re-processed to STACK. */
3610 :
3611 : static bool
3612 1424121 : propagate_used_to_scc_callers (cgraph_node *node, void *data)
3613 : {
3614 1424121 : vec<cgraph_node *> *stack = (vec<cgraph_node *> *) data;
3615 1424121 : cgraph_edge *cs;
3616 3457389 : for (cs = node->callers; cs; cs = cs->next_caller)
3617 2033268 : if (ipa_edge_within_scc (cs))
3618 28515 : propagate_used_across_scc_edge (cs, stack);
3619 1424121 : return false;
3620 : }
3621 :
3622 : /* Return true iff all certain accesses in ARG_DESC are also present as
3623 : certain accesses in PARAM_DESC. */
3624 :
3625 : static bool
3626 70 : all_callee_accesses_present_p (isra_param_desc *param_desc,
3627 : isra_param_desc *arg_desc)
3628 : {
3629 70 : unsigned aclen = vec_safe_length (arg_desc->accesses);
3630 74 : for (unsigned j = 0; j < aclen; j++)
3631 : {
3632 50 : param_access *argacc = (*arg_desc->accesses)[j];
3633 50 : if (!argacc->certain)
3634 0 : continue;
3635 100 : param_access *pacc = find_param_access (param_desc, argacc->unit_offset,
3636 50 : argacc->unit_size);
3637 50 : if (!pacc
3638 5 : || !pacc->certain
3639 55 : || !types_compatible_p (argacc->type, pacc->type))
3640 46 : return false;
3641 : }
3642 : return true;
3643 : }
3644 :
3645 : /* Type internal to function pull_accesses_from_callee. Unfortunately gcc 4.8
3646 : does not allow instantiating an auto_vec with a type defined within a
3647 : function so it is a global type. */
3648 : enum acc_prop_kind {ACC_PROP_DONT, ACC_PROP_COPY, ACC_PROP_CERTAIN};
3649 :
3650 :
3651 : /* Attempt to propagate all definite accesses from ARG_DESC to PARAM_DESC,
3652 : (which belongs to CALLER) if they would not violate some constraint there.
3653 : CALLER_IPCP_TS describes the caller, PARAM_IDX is the index of the parameter
3654 : described by PARAM_DESC. If successful, return NULL, otherwise return the
3655 : string reason for failure (which can be written to the dump file).
3656 : DELTA_OFFSET is the known offset of the actual argument withing the formal
3657 : parameter (so of ARG_DESCS within PARAM_DESCS), ARG_SIZE is the size of the
3658 : actual argument or zero, if not known. In case of success, set *CHANGE_P to
3659 : true if propagation actually changed anything. */
3660 :
3661 : static const char *
3662 5225 : pull_accesses_from_callee (cgraph_node *caller,
3663 : ipcp_transformation *caller_ipcp_ts,
3664 : int param_idx,
3665 : isra_param_desc *param_desc,
3666 : isra_param_desc *arg_desc,
3667 : unsigned delta_offset, unsigned arg_size,
3668 : bool *change_p)
3669 : {
3670 5225 : unsigned pclen = vec_safe_length (param_desc->accesses);
3671 5225 : unsigned aclen = vec_safe_length (arg_desc->accesses);
3672 5225 : unsigned prop_count = 0;
3673 5225 : unsigned prop_size = 0;
3674 5225 : bool change = false;
3675 :
3676 5225 : auto_vec <enum acc_prop_kind, 8> prop_kinds (aclen);
3677 12979 : for (unsigned j = 0; j < aclen; j++)
3678 : {
3679 7960 : param_access *argacc = (*arg_desc->accesses)[j];
3680 7960 : prop_kinds.safe_push (ACC_PROP_DONT);
3681 :
3682 7960 : if (arg_size > 0
3683 2712 : && argacc->unit_offset + argacc->unit_size > arg_size)
3684 : return "callee access outsize size boundary";
3685 :
3686 7960 : if (!argacc->certain)
3687 382 : continue;
3688 :
3689 7578 : unsigned offset = argacc->unit_offset + delta_offset;
3690 :
3691 7578 : if (caller_ipcp_ts && !AGGREGATE_TYPE_P (argacc->type))
3692 : {
3693 399 : ipa_argagg_value_list avl (caller_ipcp_ts);
3694 399 : tree value = avl.get_value (param_idx, offset);
3695 399 : if (value && ((tree_to_uhwi (TYPE_SIZE (TREE_TYPE (value)))
3696 48 : / BITS_PER_UNIT)
3697 48 : != argacc->unit_size))
3698 1 : return " propagated access would conflict with an IPA-CP constant";
3699 : }
3700 :
3701 : /* Given that accesses are initially stored according to increasing
3702 : offset and decreasing size in case of equal offsets, the following
3703 : searches could be written more efficiently if we kept the ordering
3704 : when copying. But the number of accesses is capped at
3705 : PARAM_IPA_SRA_MAX_REPLACEMENTS (so most likely 8) and the code gets
3706 : messy quickly, so let's improve on that only if necessary. */
3707 :
3708 : bool exact_match = false;
3709 15855 : for (unsigned i = 0; i < pclen; i++)
3710 : {
3711 : /* Check for overlaps. */
3712 8483 : param_access *pacc = (*param_desc->accesses)[i];
3713 8483 : if (pacc->unit_offset == offset
3714 4224 : && pacc->unit_size == argacc->unit_size)
3715 : {
3716 3065 : if (argacc->alias_ptr_type != pacc->alias_ptr_type
3717 2861 : || !types_compatible_p (argacc->type, pacc->type)
3718 5926 : || argacc->reverse != pacc->reverse)
3719 204 : return "propagated access types would not match existing ones";
3720 :
3721 2861 : exact_match = true;
3722 2861 : if (!pacc->certain)
3723 : {
3724 0 : prop_kinds[j] = ACC_PROP_CERTAIN;
3725 0 : prop_size += argacc->unit_size;
3726 0 : change = true;
3727 : }
3728 2861 : continue;
3729 : }
3730 :
3731 5418 : if (offset < pacc->unit_offset + pacc->unit_size
3732 3628 : && offset + argacc->unit_size > pacc->unit_offset)
3733 : {
3734 : /* None permissible with load accesses, possible to fit into
3735 : argument ones. */
3736 2331 : if (pacc->certain
3737 2330 : || offset < pacc->unit_offset
3738 2330 : || (offset + argacc->unit_size
3739 : > pacc->unit_offset + pacc->unit_size))
3740 : return "a propagated access would conflict in caller";
3741 : }
3742 : }
3743 :
3744 7372 : if (!exact_match)
3745 : {
3746 4511 : prop_kinds[j] = ACC_PROP_COPY;
3747 4511 : prop_count++;
3748 4511 : prop_size += argacc->unit_size;
3749 4511 : change = true;
3750 : }
3751 : }
3752 :
3753 5019 : if (!change)
3754 : return NULL;
3755 :
3756 2916 : if ((prop_count + pclen
3757 2916 : > (unsigned) opt_for_fn (caller->decl, param_ipa_sra_max_replacements))
3758 8141 : || size_would_violate_limit_p (param_desc,
3759 2916 : param_desc->size_reached + prop_size))
3760 : return "propagating accesses would violate the count or size limit";
3761 :
3762 2897 : *change_p = true;
3763 7765 : for (unsigned j = 0; j < aclen; j++)
3764 : {
3765 4868 : if (prop_kinds[j] == ACC_PROP_COPY)
3766 : {
3767 4478 : param_access *argacc = (*arg_desc->accesses)[j];
3768 :
3769 4478 : param_access *copy = ggc_cleared_alloc<param_access> ();
3770 4478 : copy->unit_offset = argacc->unit_offset + delta_offset;
3771 4478 : copy->unit_size = argacc->unit_size;
3772 4478 : copy->type = argacc->type;
3773 4478 : copy->alias_ptr_type = argacc->alias_ptr_type;
3774 4478 : copy->certain = true;
3775 4478 : copy->reverse = argacc->reverse;
3776 4478 : vec_safe_push (param_desc->accesses, copy);
3777 : }
3778 390 : else if (prop_kinds[j] == ACC_PROP_CERTAIN)
3779 : {
3780 0 : param_access *argacc = (*arg_desc->accesses)[j];
3781 0 : param_access *csp
3782 0 : = find_param_access (param_desc, argacc->unit_offset + delta_offset,
3783 0 : argacc->unit_size);
3784 0 : csp->certain = true;
3785 : }
3786 : }
3787 :
3788 2897 : param_desc->size_reached += prop_size;
3789 :
3790 2897 : return NULL;
3791 5225 : }
3792 :
3793 : /* Propagate parameter splitting information through call graph edge CS.
3794 : Return true if any changes that might need to be propagated within SCCs have
3795 : been made. The function also clears the aggregate_pass_through and
3796 : pointer_pass_through in call summaries which do not need to be processed
3797 : again if this CS is revisited when iterating while changes are propagated
3798 : within an SCC. */
3799 :
3800 : static bool
3801 1530983 : param_splitting_across_edge (cgraph_edge *cs)
3802 : {
3803 1530983 : bool res = false;
3804 1530983 : bool cross_scc = !ipa_edge_within_scc (cs);
3805 1530983 : enum availability availability;
3806 1530983 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3807 1530983 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3808 1530983 : gcc_checking_assert (from_ifs && from_ifs->m_parameters);
3809 1530983 : ipcp_transformation *caller_ipcp_ts
3810 1530983 : = ipcp_get_transformation_summary (cs->caller);
3811 :
3812 1530983 : isra_call_summary *csum = call_sums->get (cs);
3813 1530983 : gcc_checking_assert (csum);
3814 1530983 : unsigned args_count = csum->m_arg_flow.length ();
3815 1530983 : isra_func_summary *to_ifs = func_sums->get (callee);
3816 1530983 : unsigned param_count
3817 788917 : = ((to_ifs && to_ifs->m_candidate && (availability >= AVAIL_AVAILABLE))
3818 2226754 : ? vec_safe_length (to_ifs->m_parameters)
3819 : : 0);
3820 :
3821 1530983 : if (dump_file && (dump_flags & TDF_DETAILS))
3822 18 : fprintf (dump_file, "Splitting across %s->%s:\n",
3823 9 : cs->caller->dump_name (), callee->dump_name ());
3824 :
3825 : unsigned i;
3826 2954752 : for (i = 0; (i < args_count) && (i < param_count); i++)
3827 : {
3828 1423769 : isra_param_desc *arg_desc = &(*to_ifs->m_parameters)[i];
3829 1423769 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3830 :
3831 1423769 : if (arg_desc->locally_unused)
3832 : {
3833 86679 : if (dump_file && (dump_flags & TDF_DETAILS))
3834 0 : fprintf (dump_file, " ->%u: unused in callee\n", i);
3835 86679 : ipf->pointer_pass_through = false;
3836 86679 : continue;
3837 : }
3838 :
3839 1337090 : if (ipf->pointer_pass_through)
3840 : {
3841 166227 : int idx = get_single_param_flow_source (ipf);
3842 166227 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3843 166227 : if (!param_desc->split_candidate)
3844 117136 : continue;
3845 49091 : gcc_assert (param_desc->by_ref);
3846 :
3847 49091 : if (!arg_desc->split_candidate || !arg_desc->by_ref)
3848 : {
3849 43292 : if (dump_file && (dump_flags & TDF_DETAILS))
3850 2 : fprintf (dump_file, " %u->%u: not candidate or not by "
3851 : "reference in callee\n", idx, i);
3852 43292 : param_desc->split_candidate = false;
3853 43292 : ipf->pointer_pass_through = false;
3854 43292 : res = true;
3855 : }
3856 5799 : else if (!ipf->safe_to_import_accesses)
3857 : {
3858 1879 : if (!csum->m_before_any_store
3859 1879 : || !all_callee_accesses_present_p (param_desc, arg_desc))
3860 : {
3861 1855 : if (dump_file && (dump_flags & TDF_DETAILS))
3862 0 : fprintf (dump_file, " %u->%u: cannot import accesses.\n",
3863 : idx, i);
3864 1855 : param_desc->split_candidate = false;
3865 1855 : ipf->pointer_pass_through = false;
3866 1855 : res = true;
3867 :
3868 : }
3869 : else
3870 : {
3871 24 : if (dump_file && (dump_flags & TDF_DETAILS))
3872 0 : fprintf (dump_file, " %u->%u: verified callee accesses "
3873 : "present.\n", idx, i);
3874 24 : if (cross_scc)
3875 0 : ipf->pointer_pass_through = false;
3876 : }
3877 : }
3878 : else
3879 : {
3880 3920 : const char *pull_failure
3881 3920 : = pull_accesses_from_callee (cs->caller, caller_ipcp_ts, idx,
3882 : param_desc, arg_desc, 0, 0, &res);
3883 3920 : if (pull_failure)
3884 : {
3885 13 : if (dump_file && (dump_flags & TDF_DETAILS))
3886 0 : fprintf (dump_file, " %u->%u: by_ref access pull "
3887 : "failed: %s.\n", idx, i, pull_failure);
3888 13 : param_desc->split_candidate = false;
3889 13 : ipf->pointer_pass_through = false;
3890 13 : res = true;
3891 : }
3892 : else
3893 : {
3894 3907 : if (dump_file && (dump_flags & TDF_DETAILS))
3895 1 : fprintf (dump_file, " %u->%u: by_ref access pull "
3896 : "succeeded.\n", idx, i);
3897 3907 : if (cross_scc)
3898 3832 : ipf->pointer_pass_through = false;
3899 : }
3900 : }
3901 : }
3902 1170863 : else if (ipf->aggregate_pass_through)
3903 : {
3904 33579 : int idx = get_single_param_flow_source (ipf);
3905 33579 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3906 33579 : if (!param_desc->split_candidate)
3907 21549 : continue;
3908 12030 : gcc_assert (!param_desc->by_ref);
3909 24060 : param_access *pacc = find_param_access (param_desc, ipf->unit_offset,
3910 12030 : ipf->unit_size);
3911 12030 : gcc_checking_assert (pacc);
3912 :
3913 12030 : if (pacc->certain)
3914 : {
3915 1 : if (dump_file && (dump_flags & TDF_DETAILS))
3916 0 : fprintf (dump_file, " %u->%u: already certain\n", idx, i);
3917 1 : ipf->aggregate_pass_through = false;
3918 : }
3919 12029 : else if (!arg_desc->split_candidate || arg_desc->by_ref)
3920 : {
3921 10724 : if (dump_file && (dump_flags & TDF_DETAILS))
3922 0 : fprintf (dump_file, " %u->%u: not candidate or by "
3923 : "reference in callee\n", idx, i);
3924 :
3925 10724 : pacc->certain = true;
3926 10724 : if (overlapping_certain_accesses_p (param_desc, NULL))
3927 : {
3928 1318 : if (dump_file && (dump_flags & TDF_DETAILS))
3929 0 : fprintf (dump_file, " ...leading to overlap, "
3930 : " disqualifying candidate parameter %u\n",
3931 : idx);
3932 1318 : param_desc->split_candidate = false;
3933 : }
3934 : else
3935 9406 : bump_reached_size (param_desc, pacc->unit_size, idx);
3936 :
3937 10724 : ipf->aggregate_pass_through = false;
3938 10724 : res = true;
3939 : }
3940 : else
3941 : {
3942 1305 : const char *pull_failure
3943 1305 : = pull_accesses_from_callee (cs->caller, caller_ipcp_ts, idx,
3944 : param_desc, arg_desc,
3945 : ipf->unit_offset,
3946 : ipf->unit_size, &res);
3947 1305 : if (pull_failure)
3948 : {
3949 212 : if (dump_file && (dump_flags & TDF_DETAILS))
3950 0 : fprintf (dump_file, " %u->%u: arg access pull "
3951 : "failed: %s.\n", idx, i, pull_failure);
3952 :
3953 212 : ipf->aggregate_pass_through = false;
3954 212 : pacc->certain = true;
3955 :
3956 212 : if (overlapping_certain_accesses_p (param_desc, NULL))
3957 : {
3958 212 : if (dump_file && (dump_flags & TDF_DETAILS))
3959 0 : fprintf (dump_file, " ...leading to overlap, "
3960 : " disqualifying candidate parameter %u\n",
3961 : idx);
3962 212 : param_desc->split_candidate = false;
3963 : }
3964 : else
3965 0 : bump_reached_size (param_desc, pacc->unit_size, idx);
3966 :
3967 212 : res = true;
3968 : }
3969 : else
3970 : {
3971 1093 : if (dump_file && (dump_flags & TDF_DETAILS))
3972 0 : fprintf (dump_file, " %u->%u: arg access pull "
3973 : "succeeded.\n", idx, i);
3974 1093 : if (cross_scc)
3975 954 : ipf->aggregate_pass_through = false;
3976 : }
3977 : }
3978 : }
3979 : }
3980 :
3981 : /* Handle argument-parameter count mismatches. */
3982 2511365 : for (; (i < args_count); i++)
3983 : {
3984 980382 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3985 :
3986 980382 : if (ipf->pointer_pass_through || ipf->aggregate_pass_through)
3987 : {
3988 200085 : int idx = get_single_param_flow_source (ipf);
3989 200085 : ipf->pointer_pass_through = false;
3990 200085 : ipf->aggregate_pass_through = false;
3991 200085 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3992 200085 : if (!param_desc->split_candidate)
3993 200002 : continue;
3994 :
3995 83 : if (dump_file && (dump_flags & TDF_DETAILS))
3996 0 : fprintf (dump_file, " %u->%u: no corresponding formal parameter\n",
3997 : idx, i);
3998 83 : param_desc->split_candidate = false;
3999 83 : res = true;
4000 : }
4001 : }
4002 1530983 : return res;
4003 : }
4004 :
4005 : /* Worker for call_for_symbol_and_aliases, look at all callers and if all their
4006 : callers ignore the return value, or come from the same SCC and use the
4007 : return value only to compute their return value, return false, otherwise
4008 : return true. */
4009 :
4010 : static bool
4011 179373 : retval_used_p (cgraph_node *node, void *)
4012 : {
4013 272308 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4014 : {
4015 239951 : isra_call_summary *csum = call_sums->get (cs);
4016 239951 : gcc_checking_assert (csum);
4017 239951 : if (csum->m_return_ignored)
4018 87120 : continue;
4019 152831 : if (!csum->m_return_returned)
4020 : return true;
4021 :
4022 28366 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
4023 28366 : if (!from_ifs || !from_ifs->m_candidate)
4024 : return true;
4025 :
4026 21660 : if (!ipa_edge_within_scc (cs)
4027 21660 : && !from_ifs->m_return_ignored)
4028 : return true;
4029 : }
4030 :
4031 : return false;
4032 : }
4033 :
4034 : /* Push into NEW_PARAMS all required parameter adjustment entries to copy or
4035 : modify parameter which originally had index BASE_INDEX, in the adjustment
4036 : vector of parent clone (if any) had PREV_CLONE_INDEX and was described by
4037 : PREV_ADJUSTMENT. If IPA-CP has created a transformation summary for the
4038 : original node, it needs to be passed in IPCP_TS, otherwise it should be
4039 : NULL. If the parent clone is the original function, PREV_ADJUSTMENT is NULL
4040 : and PREV_CLONE_INDEX is equal to BASE_INDEX. */
4041 :
4042 : static void
4043 316731 : push_param_adjustments_for_index (isra_func_summary *ifs, unsigned base_index,
4044 : unsigned prev_clone_index,
4045 : ipa_adjusted_param *prev_adjustment,
4046 : ipcp_transformation *ipcp_ts,
4047 : vec<ipa_adjusted_param, va_gc> **new_params)
4048 : {
4049 316731 : isra_param_desc *desc = &(*ifs->m_parameters)[base_index];
4050 316731 : if (desc->locally_unused)
4051 : {
4052 64216 : if (dump_file)
4053 27 : fprintf (dump_file, " Will remove parameter %u\n", base_index);
4054 64216 : return;
4055 : }
4056 :
4057 252515 : if (!desc->split_candidate)
4058 : {
4059 203902 : ipa_adjusted_param adj;
4060 203902 : if (prev_adjustment)
4061 : {
4062 1883 : adj = *prev_adjustment;
4063 1883 : adj.prev_clone_adjustment = true;
4064 1883 : adj.prev_clone_index = prev_clone_index;
4065 : }
4066 : else
4067 : {
4068 202019 : memset (&adj, 0, sizeof (adj));
4069 202019 : adj.op = IPA_PARAM_OP_COPY;
4070 202019 : adj.base_index = base_index;
4071 202019 : adj.prev_clone_index = prev_clone_index;
4072 : }
4073 203902 : vec_safe_push ((*new_params), adj);
4074 203902 : return;
4075 : }
4076 :
4077 48613 : if (dump_file)
4078 39 : fprintf (dump_file, " Will split parameter %u\n", base_index);
4079 :
4080 48613 : gcc_assert (!prev_adjustment || prev_adjustment->op == IPA_PARAM_OP_COPY);
4081 48613 : unsigned aclen = vec_safe_length (desc->accesses);
4082 133797 : for (unsigned j = 0; j < aclen; j++)
4083 : {
4084 85184 : param_access *pa = (*desc->accesses)[j];
4085 85184 : if (!pa->certain)
4086 7654 : continue;
4087 :
4088 84328 : if (ipcp_ts)
4089 : {
4090 14857 : ipa_argagg_value_list avl (ipcp_ts);
4091 14857 : tree value = avl.get_value (base_index, pa->unit_offset);
4092 14857 : if (value && !AGGREGATE_TYPE_P (pa->type))
4093 : {
4094 6798 : if (dump_file)
4095 20 : fprintf (dump_file, " - omitting component at byte "
4096 : "offset %u which is known to have a constant value\n ",
4097 : pa->unit_offset);
4098 6798 : continue;
4099 : }
4100 : }
4101 :
4102 77530 : if (dump_file)
4103 38 : fprintf (dump_file, " - component at byte offset %u, "
4104 38 : "size %u\n", pa->unit_offset, pa->unit_size);
4105 :
4106 77530 : ipa_adjusted_param adj;
4107 77530 : memset (&adj, 0, sizeof (adj));
4108 77530 : adj.op = IPA_PARAM_OP_SPLIT;
4109 77530 : adj.base_index = base_index;
4110 77530 : adj.prev_clone_index = prev_clone_index;
4111 77530 : adj.param_prefix_index = IPA_PARAM_PREFIX_ISRA;
4112 77530 : adj.reverse = pa->reverse;
4113 77530 : adj.type = pa->type;
4114 77530 : adj.alias_ptr_type = pa->alias_ptr_type;
4115 77530 : adj.unit_offset = pa->unit_offset;
4116 77530 : vec_safe_push ((*new_params), adj);
4117 : }
4118 : }
4119 :
4120 : /* Worker for all call_for_symbol_thunks_and_aliases. Set calls_comdat_local
4121 : flag of all callers of NODE. */
4122 :
4123 : static bool
4124 0 : mark_callers_calls_comdat_local (struct cgraph_node *node, void *)
4125 : {
4126 0 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4127 0 : cs->caller->calls_comdat_local = true;
4128 0 : return false;
4129 : }
4130 :
4131 : /* Remove any IPA-CP results stored in TS that are associated with removed
4132 : parameters as marked in IFS. */
4133 :
4134 : static void
4135 8388 : zap_useless_ipcp_results (const isra_func_summary *ifs, ipcp_transformation *ts)
4136 : {
4137 8388 : ts->remove_argaggs_if ([ifs](const ipa_argagg_value &v)
4138 : {
4139 15479 : return (*ifs->m_parameters)[v.index].locally_unused;
4140 : });
4141 :
4142 8388 : bool useful_vr = false;
4143 8388 : unsigned count = vec_safe_length (ts->m_vr);
4144 29774 : for (unsigned i = 0; i < count; i++)
4145 21386 : if ((*ts->m_vr)[i].known_p ())
4146 : {
4147 14864 : const isra_param_desc *desc = &(*ifs->m_parameters)[i];
4148 14864 : if (desc->locally_unused)
4149 1956 : (*ts->m_vr)[i].set_unknown ();
4150 : else
4151 : useful_vr = true;
4152 : }
4153 8388 : if (!useful_vr)
4154 1261 : ts->m_vr = NULL;
4155 8388 : }
4156 :
4157 : /* Do final processing of results of IPA propagation regarding NODE, clone it
4158 : if appropriate. */
4159 :
4160 : static void
4161 1270911 : process_isra_node_results (cgraph_node *node,
4162 : hash_map<const char *, unsigned> *clone_num_suffixes)
4163 : {
4164 1270911 : isra_func_summary *ifs = func_sums->get (node);
4165 1270911 : if (!ifs || !ifs->m_candidate)
4166 1157612 : return;
4167 :
4168 369950 : auto_vec<bool, 16> surviving_params;
4169 369950 : bool check_surviving = false;
4170 369950 : clone_info *cinfo = clone_info::get (node);
4171 369950 : if (cinfo && cinfo->param_adjustments)
4172 : {
4173 15439 : check_surviving = true;
4174 15439 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
4175 : }
4176 :
4177 369950 : unsigned param_count = vec_safe_length (ifs->m_parameters);
4178 369950 : bool will_change_function = false;
4179 369950 : if (ifs->m_returns_value && ifs->m_return_ignored)
4180 : will_change_function = true;
4181 : else
4182 851895 : for (unsigned i = 0; i < param_count; i++)
4183 : {
4184 595244 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
4185 541433 : if ((desc->locally_unused || desc->split_candidate)
4186 : /* Make sure we do not clone just to attempt to remove an already
4187 : removed unused argument. */
4188 631140 : && (!check_surviving
4189 516369 : || (i < surviving_params.length ()
4190 2587 : && surviving_params[i])))
4191 : {
4192 : will_change_function = true;
4193 : break;
4194 : }
4195 : }
4196 338113 : if (!will_change_function)
4197 256651 : return;
4198 :
4199 113299 : if (dump_file)
4200 : {
4201 61 : fprintf (dump_file, "\nEvaluating analysis results for %s\n",
4202 : node->dump_name ());
4203 61 : if (ifs->m_returns_value && ifs->m_return_ignored)
4204 15 : fprintf (dump_file, " Will remove return value.\n");
4205 : }
4206 :
4207 113299 : ipcp_transformation *ipcp_ts = ipcp_get_transformation_summary (node);
4208 113299 : if (ipcp_ts)
4209 8388 : zap_useless_ipcp_results (ifs, ipcp_ts);
4210 113299 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
4211 113299 : if (ipa_param_adjustments *old_adjustments
4212 113299 : = cinfo ? cinfo->param_adjustments : NULL)
4213 : {
4214 2612 : unsigned old_adj_len = vec_safe_length (old_adjustments->m_adj_params);
4215 6381 : for (unsigned i = 0; i < old_adj_len; i++)
4216 : {
4217 3769 : ipa_adjusted_param *old_adj = &(*old_adjustments->m_adj_params)[i];
4218 3769 : push_param_adjustments_for_index (ifs, old_adj->base_index, i,
4219 : old_adj, ipcp_ts, &new_params);
4220 : }
4221 : }
4222 : else
4223 423649 : for (unsigned i = 0; i < param_count; i++)
4224 312962 : push_param_adjustments_for_index (ifs, i, i, NULL, ipcp_ts, &new_params);
4225 :
4226 113299 : ipa_param_adjustments *new_adjustments
4227 113299 : = (new (ggc_alloc <ipa_param_adjustments> ())
4228 : ipa_param_adjustments (new_params, param_count,
4229 194761 : ifs->m_returns_value && ifs->m_return_ignored));
4230 :
4231 113299 : if (dump_file && (dump_flags & TDF_DETAILS))
4232 : {
4233 9 : fprintf (dump_file, "\n Created adjustments:\n");
4234 9 : new_adjustments->dump (dump_file);
4235 : }
4236 :
4237 339897 : unsigned &suffix_counter = clone_num_suffixes->get_or_insert (
4238 113299 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (
4239 : node->decl)));
4240 113299 : auto_vec<cgraph_edge *> callers = node->collect_callers ();
4241 113299 : cgraph_node *new_node
4242 113299 : = node->create_virtual_clone (callers, NULL, new_adjustments, "isra",
4243 : suffix_counter);
4244 113299 : suffix_counter++;
4245 113299 : if (node->calls_comdat_local && node->same_comdat_group)
4246 : {
4247 0 : new_node->add_to_same_comdat_group (node);
4248 0 : new_node->call_for_symbol_and_aliases (mark_callers_calls_comdat_local,
4249 : NULL, true);
4250 : }
4251 113299 : new_node->calls_comdat_local = node->calls_comdat_local;
4252 :
4253 113299 : if (dump_file)
4254 61 : fprintf (dump_file, " Created new node %s\n", new_node->dump_name ());
4255 113299 : callers.release ();
4256 369950 : }
4257 :
4258 : /* If INDICES is not empty, dump a combination of NODE's dump_name and MSG
4259 : followed by the list of numbers in INDICES. */
4260 :
4261 : static void
4262 632354 : dump_list_of_param_indices (const cgraph_node *node, const char* msg,
4263 : const vec<unsigned> &indices)
4264 : {
4265 632354 : if (indices.is_empty ())
4266 : return;
4267 1 : fprintf (dump_file, "The following parameters of %s %s:", node->dump_name (),
4268 : msg);
4269 4 : for (unsigned i : indices)
4270 1 : fprintf (dump_file, " %u", i);
4271 1 : fprintf (dump_file, "\n");
4272 : }
4273 :
4274 : /* Check which parameters of NODE described by IFS have survived until IPA-SRA
4275 : and disable transformations for those which have not or which should not
4276 : transformed because the associated debug counter reached its limit. Return
4277 : true if none survived or if there were no candidates to begin with.
4278 : Additionally, also adjust parameter descriptions based on debug counters and
4279 : hints propagated earlier. */
4280 :
4281 : static bool
4282 369950 : adjust_parameter_descriptions (cgraph_node *node, isra_func_summary *ifs)
4283 : {
4284 369950 : bool ret = true;
4285 369950 : unsigned len = vec_safe_length (ifs->m_parameters);
4286 316177 : if (!len)
4287 : return true;
4288 :
4289 316177 : auto_vec<bool, 16> surviving_params;
4290 316177 : bool check_surviving = false;
4291 316177 : clone_info *cinfo = clone_info::get (node);
4292 316177 : if (cinfo && cinfo->param_adjustments)
4293 : {
4294 15439 : check_surviving = true;
4295 15439 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
4296 : }
4297 316177 : ipcp_transformation *ipcp_ts = ipcp_get_transformation_summary (node);
4298 316177 : auto_vec <unsigned> dump_dead_indices;
4299 316177 : auto_vec <unsigned> dump_bad_cond_indices;
4300 1078121 : for (unsigned i = 0; i < len; i++)
4301 : {
4302 761944 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
4303 761944 : if (!dbg_cnt (ipa_sra_params))
4304 : {
4305 0 : desc->locally_unused = false;
4306 0 : desc->split_candidate = false;
4307 0 : continue;
4308 : }
4309 :
4310 761944 : if (desc->split_only_when_retval_removed
4311 7221 : && !ifs->m_return_ignored)
4312 : {
4313 0 : if (dump_file && (dump_flags & TDF_DETAILS)
4314 1835 : && (desc->locally_unused || desc->split_candidate))
4315 0 : dump_bad_cond_indices.safe_push (i);
4316 :
4317 1835 : gcc_checking_assert (!desc->locally_unused
4318 : || desc->remove_only_when_retval_removed);
4319 1835 : desc->locally_unused = false;
4320 1835 : desc->split_candidate = false;
4321 1835 : continue;
4322 : }
4323 760109 : if (desc->remove_only_when_retval_removed
4324 43674 : && !ifs->m_return_ignored)
4325 : {
4326 6 : if (dump_file && (dump_flags & TDF_DETAILS)
4327 34302 : && (desc->locally_unused || desc->split_candidate))
4328 0 : dump_bad_cond_indices.safe_push (i);
4329 :
4330 34302 : desc->locally_unused = false;
4331 : }
4332 760109 : if (check_surviving
4333 805041 : && (i >= surviving_params.length ()
4334 21057 : || !surviving_params[i]))
4335 : {
4336 : /* Even if the parameter was removed by a previous IPA pass, we do
4337 : not clear locally_unused because if it really is unused, this
4338 : information might be useful in callers. */
4339 28419 : desc->split_candidate = false;
4340 :
4341 28419 : if (dump_file && (dump_flags & TDF_DETAILS))
4342 0 : dump_dead_indices.safe_push (i);
4343 : }
4344 :
4345 760109 : if (desc->split_candidate && desc->conditionally_dereferenceable)
4346 : {
4347 931 : gcc_assert (desc->safe_size_set);
4348 1556 : for (param_access *pa : *desc->accesses)
4349 1393 : if ((pa->unit_offset + pa->unit_size) > desc->safe_size)
4350 : {
4351 768 : if (dump_file && (dump_flags & TDF_DETAILS))
4352 1 : dump_bad_cond_indices.safe_push (i);
4353 768 : desc->split_candidate = false;
4354 768 : break;
4355 : }
4356 : }
4357 :
4358 760109 : if (desc->split_candidate)
4359 : {
4360 200411 : if (desc->by_ref && !desc->not_specially_constructed)
4361 : {
4362 29331 : int extra_factor
4363 29331 : = opt_for_fn (node->decl,
4364 : param_ipa_sra_ptrwrap_growth_factor);
4365 29331 : desc->param_size_limit = extra_factor * desc->param_size_limit;
4366 : }
4367 200411 : if (size_would_violate_limit_p (desc, desc->size_reached))
4368 2693 : desc->split_candidate = false;
4369 : }
4370 :
4371 : /* Avoid ICEs on size-mismatched VIEW_CONVERT_EXPRs when callers and
4372 : callees don't agree on types in aggregates and we try to do both
4373 : IPA-CP and IPA-SRA. */
4374 760109 : if (ipcp_ts && desc->split_candidate)
4375 : {
4376 22829 : ipa_argagg_value_list avl (ipcp_ts);
4377 55881 : for (const param_access *pa : desc->accesses)
4378 : {
4379 16467 : if (!pa->certain)
4380 1009 : continue;
4381 15458 : tree value = avl.get_value (i, pa->unit_offset);
4382 15458 : if (value
4383 15458 : && ((tree_to_uhwi (TYPE_SIZE (TREE_TYPE (value)))
4384 6908 : / BITS_PER_UNIT)
4385 6908 : != pa->unit_size))
4386 : {
4387 43 : desc->split_candidate = false;
4388 43 : if (dump_file && (dump_flags & TDF_DETAILS))
4389 0 : dump_dead_indices.safe_push (i);
4390 : break;
4391 : }
4392 : }
4393 : }
4394 :
4395 760109 : if (desc->locally_unused || desc->split_candidate)
4396 761944 : ret = false;
4397 : }
4398 :
4399 316177 : dump_list_of_param_indices (node, "are dead on arrival or have a type "
4400 : "mismatch with IPA-CP", dump_dead_indices);
4401 316177 : dump_list_of_param_indices (node, "fail additional requirements ",
4402 : dump_bad_cond_indices);
4403 :
4404 316177 : return ret;
4405 316177 : }
4406 :
4407 :
4408 : /* Run the interprocedural part of IPA-SRA. */
4409 :
4410 : static unsigned int
4411 126744 : ipa_sra_analysis (void)
4412 : {
4413 126744 : if (dump_file)
4414 : {
4415 54 : fprintf (dump_file, "\n========== IPA-SRA IPA stage ==========\n");
4416 54 : ipa_sra_dump_all_summaries (dump_file, false);
4417 : }
4418 :
4419 126744 : gcc_checking_assert (func_sums);
4420 126744 : gcc_checking_assert (call_sums);
4421 126744 : cgraph_node **order = XCNEWVEC (cgraph_node *, symtab->cgraph_count);
4422 126744 : auto_vec <cgraph_node *, 16> stack;
4423 126744 : int node_scc_count = ipa_reduced_postorder (order, true, NULL);
4424 :
4425 : /* One sweep from callers to callees for return value removal. */
4426 1467554 : for (int i = node_scc_count - 1; i >= 0 ; i--)
4427 : {
4428 1340810 : cgraph_node *scc_rep = order[i];
4429 1340810 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (scc_rep);
4430 :
4431 : /* Preliminary IPA function level checks. */
4432 5368988 : for (cgraph_node *v : cycle_nodes)
4433 : {
4434 1346558 : isra_func_summary *ifs = func_sums->get (v);
4435 1346558 : if (!ifs || !ifs->m_candidate)
4436 317694 : continue;
4437 1028864 : if (!ipa_sra_ipa_function_checks (v)
4438 1028864 : || check_all_callers_for_issues (v))
4439 658914 : ifs->zap ();
4440 : }
4441 :
4442 4028178 : for (cgraph_node *v : cycle_nodes)
4443 : {
4444 1346558 : isra_func_summary *ifs = func_sums->get (v);
4445 1346558 : if (!ifs || !ifs->m_candidate)
4446 976608 : continue;
4447 369950 : bool return_needed
4448 369950 : = (ifs->m_returns_value
4449 369950 : && (!dbg_cnt (ipa_sra_retvalues)
4450 179356 : || v->call_for_symbol_and_aliases (retval_used_p,
4451 369950 : NULL, true)));
4452 369950 : ifs->m_return_ignored = !return_needed;
4453 369950 : if (return_needed)
4454 294032 : isra_push_node_to_stack (v, ifs, &stack);
4455 : }
4456 :
4457 1488329 : while (!stack.is_empty ())
4458 : {
4459 147519 : cgraph_node *node = stack.pop ();
4460 147519 : isra_func_summary *ifs = func_sums->get (node);
4461 147519 : gcc_checking_assert (ifs && ifs->m_queued);
4462 147519 : ifs->m_queued = false;
4463 :
4464 819576 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
4465 672057 : if (ipa_edge_within_scc (cs)
4466 672057 : && call_sums->get (cs)->m_return_returned)
4467 : {
4468 1126 : enum availability av;
4469 1126 : cgraph_node *callee = cs->callee->function_symbol (&av);
4470 1126 : isra_func_summary *to_ifs = func_sums->get (callee);
4471 1126 : if (to_ifs && to_ifs->m_return_ignored)
4472 : {
4473 503 : to_ifs->m_return_ignored = false;
4474 1006 : isra_push_node_to_stack (callee, to_ifs, &stack);
4475 : }
4476 : }
4477 : }
4478 :
4479 : /* Parameter hint propagation. */
4480 4028178 : for (cgraph_node *v : cycle_nodes)
4481 : {
4482 1346558 : isra_func_summary *ifs = func_sums->get (v);
4483 1346558 : propagate_hints_to_all_callees (v, ifs, &stack);
4484 : }
4485 :
4486 1343301 : while (!stack.is_empty ())
4487 : {
4488 2491 : cgraph_node *node = stack.pop ();
4489 2491 : isra_func_summary *ifs = func_sums->get (node);
4490 2491 : gcc_checking_assert (ifs && ifs->m_queued);
4491 2491 : ifs->m_queued = false;
4492 2491 : propagate_hints_to_all_callees (node, ifs, &stack);
4493 : }
4494 :
4495 1340810 : cycle_nodes.release ();
4496 : }
4497 :
4498 : /* One sweep from callees to callers for parameter removal and splitting. */
4499 1467554 : for (int i = 0; i < node_scc_count; i++)
4500 : {
4501 1340810 : cgraph_node *scc_rep = order[i];
4502 1340810 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (scc_rep);
4503 :
4504 : /* First step of parameter removal. */
4505 5368988 : for (cgraph_node *v : cycle_nodes)
4506 : {
4507 1346558 : isra_func_summary *ifs = func_sums->get (v);
4508 1346558 : if (!ifs || !ifs->m_candidate)
4509 976608 : continue;
4510 369950 : if (adjust_parameter_descriptions (v, ifs))
4511 192098 : continue;
4512 191383 : for (cgraph_edge *cs = v->indirect_calls; cs; cs = cs->next_callee)
4513 13531 : process_edge_to_unknown_caller (cs);
4514 1005098 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
4515 827246 : if (!ipa_edge_within_scc (cs))
4516 821882 : param_removal_cross_scc_edge (cs);
4517 : }
4518 :
4519 : /* Look at edges within the current SCC and propagate used-ness across
4520 : them, pushing onto the stack all notes which might need to be
4521 : revisited. */
4522 4028178 : for (cgraph_node *v : cycle_nodes)
4523 1346558 : v->call_for_symbol_thunks_and_aliases (propagate_used_to_scc_callers,
4524 : &stack, true);
4525 :
4526 : /* Keep revisiting and pushing until nothing changes. */
4527 1341568 : while (!stack.is_empty ())
4528 : {
4529 758 : cgraph_node *v = stack.pop ();
4530 758 : isra_func_summary *ifs = func_sums->get (v);
4531 758 : gcc_checking_assert (ifs && ifs->m_queued);
4532 758 : ifs->m_queued = false;
4533 :
4534 758 : v->call_for_symbol_thunks_and_aliases (propagate_used_to_scc_callers,
4535 : &stack, true);
4536 : }
4537 :
4538 : /* Parameter splitting. */
4539 1386446 : bool repeat_scc_access_propagation;
4540 1386446 : do
4541 : {
4542 1386446 : repeat_scc_access_propagation = false;
4543 4167781 : for (cgraph_node *v : cycle_nodes)
4544 : {
4545 1394889 : isra_func_summary *ifs = func_sums->get (v);
4546 2425898 : if (!ifs
4547 1080187 : || !ifs->m_candidate
4548 1812542 : || vec_safe_is_empty (ifs->m_parameters))
4549 1031009 : continue;
4550 1894863 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
4551 1530983 : if (param_splitting_across_edge (cs))
4552 48962 : repeat_scc_access_propagation = true;
4553 : }
4554 : }
4555 : while (repeat_scc_access_propagation);
4556 :
4557 1340810 : if (flag_checking)
4558 4028127 : for (cgraph_node *v : cycle_nodes)
4559 1346541 : verify_splitting_accesses (v, true);
4560 :
4561 1340810 : cycle_nodes.release ();
4562 : }
4563 :
4564 126744 : ipa_free_postorder_info ();
4565 126744 : free (order);
4566 :
4567 126744 : if (dump_file)
4568 : {
4569 54 : if (dump_flags & TDF_DETAILS)
4570 : {
4571 9 : fprintf (dump_file, "\n========== IPA-SRA propagation final state "
4572 : " ==========\n");
4573 9 : ipa_sra_dump_all_summaries (dump_file, true);
4574 : }
4575 54 : fprintf (dump_file, "\n========== IPA-SRA decisions ==========\n");
4576 : }
4577 :
4578 126744 : hash_map<const char *, unsigned> *clone_num_suffixes
4579 126744 : = new hash_map<const char *, unsigned>;
4580 :
4581 126744 : cgraph_node *node;
4582 1397655 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
4583 1270911 : process_isra_node_results (node, clone_num_suffixes);
4584 :
4585 126744 : delete clone_num_suffixes;
4586 126744 : ggc_delete (func_sums);
4587 126744 : func_sums = NULL;
4588 126744 : delete call_sums;
4589 126744 : call_sums = NULL;
4590 :
4591 126744 : if (dump_file)
4592 54 : fprintf (dump_file, "\n========== IPA SRA IPA analysis done "
4593 : "==========\n\n");
4594 126744 : return 0;
4595 126744 : }
4596 :
4597 :
4598 : const pass_data pass_data_ipa_sra =
4599 : {
4600 : IPA_PASS, /* type */
4601 : "sra", /* name */
4602 : OPTGROUP_NONE, /* optinfo_flags */
4603 : TV_IPA_SRA, /* tv_id */
4604 : 0, /* properties_required */
4605 : 0, /* properties_provided */
4606 : 0, /* properties_destroyed */
4607 : 0, /* todo_flags_start */
4608 : ( TODO_dump_symtab | TODO_remove_functions ), /* todo_flags_finish */
4609 : };
4610 :
4611 : class pass_ipa_sra : public ipa_opt_pass_d
4612 : {
4613 : public:
4614 288775 : pass_ipa_sra (gcc::context *ctxt)
4615 : : ipa_opt_pass_d (pass_data_ipa_sra, ctxt,
4616 : ipa_sra_generate_summary, /* generate_summary */
4617 : ipa_sra_write_summary, /* write_summary */
4618 : ipa_sra_read_summary, /* read_summary */
4619 : NULL , /* write_optimization_summary */
4620 : NULL, /* read_optimization_summary */
4621 : NULL, /* stmt_fixup */
4622 : 0, /* function_transform_todo_flags_start */
4623 : NULL, /* function_transform */
4624 288775 : NULL) /* variable_transform */
4625 288775 : {}
4626 :
4627 : /* opt_pass methods: */
4628 592729 : bool gate (function *) final override
4629 : {
4630 : /* TODO: We should remove the optimize check after we ensure we never run
4631 : IPA passes when not optimizing. */
4632 592729 : return (flag_ipa_sra && optimize);
4633 : }
4634 :
4635 126744 : unsigned int execute (function *) final override
4636 : {
4637 126744 : return ipa_sra_analysis ();
4638 : }
4639 :
4640 : }; // class pass_ipa_sra
4641 :
4642 : } // anon namespace
4643 :
4644 : /* Intraprocedural part of IPA-SRA analysis. Scan function body of NODE and
4645 : create a summary structure describing IPA-SRA opportunities and constraints
4646 : in it. */
4647 :
4648 : static void
4649 1267514 : ipa_sra_summarize_function (cgraph_node *node)
4650 : {
4651 1267514 : if (dump_file)
4652 187 : fprintf (dump_file, "Creating summary for %s/%i:\n", node->name (),
4653 : node->get_uid ());
4654 1267514 : gcc_obstack_init (&gensum_obstack);
4655 1267514 : loaded_decls = new hash_set<tree>;
4656 :
4657 1267514 : isra_func_summary *ifs = NULL;
4658 1267514 : unsigned count = 0;
4659 1267514 : if (ipa_sra_preliminary_function_checks (node))
4660 : {
4661 1032355 : ifs = func_sums->get_create (node);
4662 1032355 : ifs->m_candidate = true;
4663 1032355 : tree ret = TREE_TYPE (TREE_TYPE (node->decl));
4664 1032355 : ifs->m_returns_value = (TREE_CODE (ret) != VOID_TYPE);
4665 1032355 : for (tree parm = DECL_ARGUMENTS (node->decl);
4666 3416533 : parm;
4667 2384178 : parm = DECL_CHAIN (parm))
4668 2384178 : count++;
4669 : }
4670 1267514 : auto_vec<gensum_param_desc, 16> param_descriptions (count);
4671 :
4672 1267514 : struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
4673 1267514 : bool cfun_pushed = false;
4674 1267514 : if (count > 0)
4675 : {
4676 837454 : decl2desc = new hash_map<tree, gensum_param_desc *>;
4677 837454 : param_descriptions.reserve_exact (count);
4678 837454 : param_descriptions.quick_grow_cleared (count);
4679 :
4680 837454 : if (create_parameter_descriptors (node, ¶m_descriptions))
4681 : {
4682 415162 : push_cfun (fun);
4683 415162 : cfun_pushed = true;
4684 415162 : final_bbs = BITMAP_ALLOC (NULL);
4685 415162 : bb_dereferences = XCNEWVEC (HOST_WIDE_INT,
4686 : unsafe_by_ref_count
4687 : * last_basic_block_for_fn (fun));
4688 415162 : aa_walking_limit = opt_for_fn (node->decl, param_ipa_max_aa_steps);
4689 : }
4690 : }
4691 : /* Scan function is run even when there are no removal or splitting
4692 : candidates so that we can calculate hints on call edges which can be
4693 : useful in callees. */
4694 1267514 : scan_function (node, fun);
4695 :
4696 1267514 : if (count > 0)
4697 : {
4698 837454 : if (dump_file)
4699 : {
4700 135 : dump_gensum_param_descriptors (dump_file, node->decl,
4701 : ¶m_descriptions);
4702 135 : fprintf (dump_file, "----------------------------------------\n");
4703 : }
4704 :
4705 837454 : process_scan_results (node, fun, ifs, ¶m_descriptions);
4706 :
4707 837454 : if (cfun_pushed)
4708 415162 : pop_cfun ();
4709 837454 : if (bb_dereferences)
4710 : {
4711 415162 : free (bb_dereferences);
4712 415162 : bb_dereferences = NULL;
4713 415162 : BITMAP_FREE (final_bbs);
4714 415162 : final_bbs = NULL;
4715 : }
4716 : }
4717 1267514 : isra_analyze_all_outgoing_calls (node);
4718 :
4719 2535028 : delete loaded_decls;
4720 1267514 : loaded_decls = NULL;
4721 1267514 : if (decl2desc)
4722 : {
4723 837454 : delete decl2desc;
4724 837454 : decl2desc = NULL;
4725 : }
4726 1267514 : obstack_free (&gensum_obstack, NULL);
4727 1267514 : if (dump_file)
4728 187 : fprintf (dump_file, "\n\n");
4729 1267514 : if (flag_checking)
4730 1267496 : verify_splitting_accesses (node, false);
4731 1267514 : return;
4732 1267514 : }
4733 :
4734 : ipa_opt_pass_d *
4735 288775 : make_pass_ipa_sra (gcc::context *ctxt)
4736 : {
4737 288775 : return new pass_ipa_sra (ctxt);
4738 : }
4739 :
4740 : /* Reset all state within ipa-sra.cc so that we can rerun the compiler
4741 : within the same process. For use by toplev::finalize. */
4742 :
4743 : void
4744 259496 : ipa_sra_cc_finalize (void)
4745 : {
4746 259496 : if (func_sums)
4747 9072 : ggc_delete (func_sums);
4748 259496 : func_sums = NULL;
4749 259496 : delete call_sums;
4750 259496 : call_sums = NULL;
4751 259496 : }
4752 :
4753 : #include "gt-ipa-sra.h"
|
