Line data Source code
1 : /* Interprocedural scalar replacement of aggregates
2 : Copyright (C) 2019-2026 Free Software Foundation, Inc.
3 : Contributed by Martin Jambor <mjambor@suse.cz>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* IPA-SRA is an interprocedural pass that removes unused function return
22 : values (turning functions returning a value which is never used into void
23 : functions) and removes unused function parameters. It can also replace an
24 : aggregate parameter by a set of other parameters representing part of the
25 : original, turning those passed by reference into new ones which pass the
26 : value directly.
27 :
28 : The pass is a true IPA one, which means that it works in three stages in
29 : order to be able to take advantage of LTO. First, summaries about functions
30 : and each calls are generated. Function summaries (often called call graph
31 : node summaries) contain mainly information about which parameters are
32 : potential transformation candidates and which bits of candidates are
33 : accessed. We differentiate between accesses done as a part of a call
34 : statement (which might be not necessary if the callee is also transformed)
35 : and others (which are mandatory). Call summaries (often called call graph
36 : edge summaries) contain information about which function formal parameters
37 : feed into which actual call arguments so that if two parameters are only
38 : used in a sum which is then passed to another function which then however
39 : does not use this parameter, all three parameters of the two functions can
40 : be eliminated. Edge summaries also have flags whether the return value is
41 : used or if it is only returned in the caller too. In LTO mode these
42 : summaries are then streamed to the object file in the compilation phase and
43 : streamed back in in the WPA analysis stage.
44 :
45 : The interprocedural analysis phase traverses the graph in topological order
46 : in two sweeps, one in each direction. First, from callees to callers for
47 : parameter removal and splitting. Each strongly-connected component is
48 : processed iteratively until the situation in it stabilizes. The pass from
49 : callers to callees is then carried out to remove unused return values in a
50 : very similar fashion.
51 :
52 : Because parameter manipulation has big implications for call redirection
53 : which is done only after all call graph nodes materialize, the
54 : transformation phase is not part of this patch but is carried out by the
55 : clone materialization and edge redirection itself, see comments in
56 : ipa-param-manipulation.h for more details. */
57 :
58 :
59 : #include "config.h"
60 : #include "system.h"
61 : #include "coretypes.h"
62 : #include "backend.h"
63 : #include "tree.h"
64 : #include "gimple.h"
65 : #include "predict.h"
66 : #include "tree-pass.h"
67 : #include "ssa.h"
68 : #include "cgraph.h"
69 : #include "gimple-pretty-print.h"
70 : #include "alias.h"
71 : #include "tree-eh.h"
72 : #include "gimple-iterator.h"
73 : #include "gimple-walk.h"
74 : #include "tree-dfa.h"
75 : #include "tree-sra.h"
76 : #include "alloc-pool.h"
77 : #include "symbol-summary.h"
78 : #include "dbgcnt.h"
79 : #include "tree-inline.h"
80 : #include "ipa-utils.h"
81 : #include "builtins.h"
82 : #include "cfganal.h"
83 : #include "tree-streamer.h"
84 : #include "internal-fn.h"
85 : #include "symtab-clones.h"
86 : #include "attribs.h"
87 : #include "sreal.h"
88 : #include "ipa-cp.h"
89 : #include "ipa-prop.h"
90 : #include "attr-callback.h"
91 : #include "callback-info.h"
92 :
93 : static void ipa_sra_summarize_function (cgraph_node *);
94 :
95 : /* Bits used to track size of an aggregate in bytes interprocedurally. */
96 : #define ISRA_ARG_SIZE_LIMIT_BITS 16
97 : #define ISRA_ARG_SIZE_LIMIT (1 << ISRA_ARG_SIZE_LIMIT_BITS)
98 : /* How many parameters can feed into a call actual argument and still be
99 : tracked. */
100 : #define IPA_SRA_MAX_PARAM_FLOW_LEN 7
101 :
102 : /* Structure describing accesses to a specific portion of an aggregate
103 : parameter, as given by the offset and size. Any smaller accesses that occur
104 : within a function that fall within another access form a tree. The pass
105 : cannot analyze parameters with only partially overlapping accesses. */
106 :
107 : struct GTY(()) param_access
108 : {
109 : /* Type that a potential replacement should have. This field only has
110 : meaning in the summary building and transformation phases, when it is
111 : reconstructed from the body. Must not be touched in IPA analysis
112 : stage. */
113 : tree type;
114 :
115 : /* Alias reference type to be used in MEM_REFs when adjusting caller
116 : arguments. */
117 : tree alias_ptr_type;
118 :
119 : /* Values returned by get_ref_base_and_extent but converted to bytes and
120 : stored as unsigned ints. */
121 : unsigned unit_offset;
122 : unsigned unit_size : ISRA_ARG_SIZE_LIMIT_BITS;
123 :
124 : /* Set once we are sure that the access will really end up in a potentially
125 : transformed function - initially not set for portions of formal parameters
126 : that are only used as actual function arguments passed to callees. */
127 : unsigned certain : 1;
128 : /* Set if the access has reverse scalar storage order. */
129 : unsigned reverse : 1;
130 : };
131 :
132 : /* This structure has the same purpose as the one above and additionally it
133 : contains some fields that are only necessary in the summary generation
134 : phase. */
135 :
136 : struct gensum_param_access
137 : {
138 : /* Values returned by get_ref_base_and_extent. */
139 : HOST_WIDE_INT offset;
140 : HOST_WIDE_INT size;
141 :
142 : /* if this access has any children (in terms of the definition above), this
143 : points to the first one. */
144 : struct gensum_param_access *first_child;
145 : /* In intraprocedural SRA, pointer to the next sibling in the access tree as
146 : described above. */
147 : struct gensum_param_access *next_sibling;
148 :
149 : /* Type that a potential replacement should have. This field only has
150 : meaning in the summary building and transformation phases, when it is
151 : reconstructed from the body. Must not be touched in IPA analysis
152 : stage. */
153 : tree type;
154 : /* Alias reference type to be used in MEM_REFs when adjusting caller
155 : arguments. */
156 : tree alias_ptr_type;
157 :
158 : /* Cumulative count of all loads. */
159 : profile_count load_count;
160 : /* Have there been writes to or reads from this exact location except for as
161 : arguments to a function call that can be tracked. */
162 : bool nonarg;
163 :
164 : /* Set if the access has reverse scalar storage order. */
165 : bool reverse;
166 : };
167 :
168 : /* Summary describing a parameter in the IPA stages. */
169 :
170 : struct GTY(()) isra_param_desc
171 : {
172 : /* List of access representatives to the parameters, sorted according to
173 : their offset. */
174 : vec <param_access *, va_gc> *accesses;
175 :
176 : /* Unit size limit of total size of all replacements. */
177 : unsigned param_size_limit : ISRA_ARG_SIZE_LIMIT_BITS;
178 : /* Sum of unit sizes of all certain replacements. */
179 : unsigned size_reached : ISRA_ARG_SIZE_LIMIT_BITS;
180 : /* Minimum offset that is known to be safe to dereference because of callers
181 : pass pointers to DECLs of at least this size or because of dereferences in
182 : callers. */
183 : unsigned safe_size : ISRA_ARG_SIZE_LIMIT_BITS;
184 :
185 : /* A parameter that is used only in call arguments and can be removed if all
186 : concerned actual arguments are removed. */
187 : unsigned locally_unused : 1;
188 : /* An aggregate that is a candidate for breaking up or complete removal. */
189 : unsigned split_candidate : 1;
190 : /* Is this a parameter passing stuff by reference? */
191 : unsigned by_ref : 1;
192 : /* If set, this parameter can only be a candidate for removal if the function
193 : is going to loose its return value. */
194 : unsigned remove_only_when_retval_removed : 1;
195 : /* If set, this parameter can only be a candidate for splitting if the
196 : function is going to loose its return value. Can only be meaningfully set
197 : for by_ref parameters. */
198 : unsigned split_only_when_retval_removed : 1;
199 : /* Parameter hint set during IPA analysis when there is a caller which does
200 : not construct the argument just to pass it to calls. Only meaningful for
201 : by_ref parameters. */
202 : unsigned not_specially_constructed : 1;
203 : /* Only meaningful for by_ref parameters. If set, this parameter can only be
204 : a split candidate if all callers pass pointers that are known to point to
205 : a chunk of memory large enough to contain all accesses. */
206 : unsigned conditionally_dereferenceable : 1;
207 : /* Set when safe_size has been updated from at least one caller. */
208 : unsigned safe_size_set : 1;
209 : };
210 :
211 : /* Structure used when generating summaries that describes a parameter. */
212 :
213 : struct gensum_param_desc
214 : {
215 : /* Roots of param_accesses. */
216 : gensum_param_access *accesses;
217 : /* Number of accesses in the access tree rooted in field accesses. */
218 : unsigned access_count;
219 :
220 : /* If the below is non-zero, this is the number of uses as actual
221 : arguments. */
222 : int call_uses;
223 : /* Number of times this parameter has been directly passed to. */
224 : unsigned ptr_pt_count;
225 :
226 : /* Size limit of total size of all replacements. */
227 : unsigned param_size_limit;
228 : /* Sum of sizes of nonarg accesses. */
229 : unsigned nonarg_acc_size;
230 :
231 : /* A parameter that is used only in call arguments and can be removed if all
232 : concerned actual arguments are removed. */
233 : bool locally_unused;
234 : /* An aggregate that is a candidate for breaking up or a pointer passing data
235 : by reference that is a candidate for being converted to a set of
236 : parameters passing those data by value. */
237 : bool split_candidate;
238 : /* Is this a parameter passing stuff by reference (either a pointer or a
239 : source language reference type)? */
240 : bool by_ref;
241 : /* If this parameter passes stuff by reference, can it be safely dereferenced
242 : without performing further checks (for example because it is a
243 : REFERENCE_TYPE)? */
244 : bool safe_ref;
245 : /* If set, this parameter can only be a candidate for removal if the function
246 : is going to loose its return value. */
247 : bool remove_only_when_retval_removed;
248 : /* If set, this parameter can only be a candidate for splitting if the
249 : function is going to loose its return value. Can only be meaningfully set
250 : for by_ref parameters. */
251 : bool split_only_when_retval_removed;
252 : /* Only meaningful for by_ref parameters. If set, this parameter can only be
253 : a split candidate if all callers pass pointers that are known to point to
254 : a chunk of memory large enough to contain all accesses. */
255 : bool conditionally_dereferenceable;
256 :
257 : /* The number of this parameter as they are ordered in function decl. */
258 : int param_number;
259 : /* For parameters passing data by reference, this is parameter index to
260 : compute indices to bb_dereferences. */
261 : int deref_index;
262 : };
263 :
264 : /* Properly deallocate accesses of DESC. TODO: Since this data structure is
265 : allocated in GC memory, this is not necessary and we can consider removing
266 : the function. */
267 :
268 : static void
269 2872725 : free_param_decl_accesses (isra_param_desc *desc)
270 : {
271 2872725 : unsigned len = vec_safe_length (desc->accesses);
272 3491073 : for (unsigned i = 0; i < len; ++i)
273 618348 : ggc_free ((*desc->accesses)[i]);
274 2872725 : vec_free (desc->accesses);
275 2872725 : }
276 :
277 : /* Class used to convey information about functions from the
278 : intra-procedural analysis stage to inter-procedural one. */
279 :
280 : class GTY((for_user)) isra_func_summary
281 : {
282 : public:
283 : /* initialize the object. */
284 :
285 1244490 : isra_func_summary ()
286 1244490 : : m_parameters (NULL), m_candidate (false), m_returns_value (false),
287 1244490 : m_return_ignored (false), m_queued (false)
288 : {}
289 :
290 : /* Destroy m_parameters. */
291 :
292 : ~isra_func_summary ();
293 :
294 : /* Mark the function as not a candidate for any IPA-SRA transformation.
295 : Return true if it was a candidate until now. */
296 :
297 : bool zap ();
298 :
299 : /* Vector of parameter descriptors corresponding to the function being
300 : analyzed. */
301 : vec<isra_param_desc, va_gc> *m_parameters;
302 :
303 : /* Whether the node is even a candidate for any IPA-SRA transformation at
304 : all. */
305 : unsigned m_candidate : 1;
306 :
307 : /* Whether the original function returns any value. */
308 : unsigned m_returns_value : 1;
309 :
310 : /* Set to true if all call statements do not actually use the returned
311 : value. */
312 :
313 : unsigned m_return_ignored : 1;
314 :
315 : /* Whether the node is already queued in IPA SRA stack during processing of
316 : call graphs SCCs. */
317 :
318 : unsigned m_queued : 1;
319 : };
320 :
321 : /* Deallocate the memory pointed to by isra_func_summary. TODO: Since this
322 : data structure is allocated in GC memory, this is not necessary and we can
323 : consider removing the destructor. */
324 :
325 1244489 : isra_func_summary::~isra_func_summary ()
326 : {
327 1244489 : unsigned len = vec_safe_length (m_parameters);
328 2588768 : for (unsigned i = 0; i < len; ++i)
329 1344279 : free_param_decl_accesses (&(*m_parameters)[i]);
330 1244489 : vec_free (m_parameters);
331 1244489 : }
332 :
333 : /* Mark the function as not a candidate for any IPA-SRA transformation. Return
334 : true if it was a candidate until now. */
335 :
336 : bool
337 670426 : isra_func_summary::zap ()
338 : {
339 670426 : bool ret = m_candidate;
340 670426 : m_candidate = false;
341 :
342 : /* TODO: see the destructor above. */
343 670426 : unsigned len = vec_safe_length (m_parameters);
344 2198872 : for (unsigned i = 0; i < len; ++i)
345 1528446 : free_param_decl_accesses (&(*m_parameters)[i]);
346 670426 : vec_free (m_parameters);
347 :
348 670426 : return ret;
349 : }
350 :
351 : /* Structure to describe which formal parameters feed into a particular actual
352 : argument. */
353 :
354 : struct isra_param_flow
355 : {
356 : /* Number of elements in array inputs that contain valid data. */
357 : char length;
358 : /* Indices of formal parameters that feed into the described actual argument.
359 : If aggregate_pass_through or pointer_pass_through below are true, it must
360 : contain exactly one element which is passed through from a formal
361 : parameter if the given number. Otherwise, the array contains indices of
362 : callee's formal parameters which are used to calculate value of this
363 : actual argument. */
364 : unsigned char inputs[IPA_SRA_MAX_PARAM_FLOW_LEN];
365 :
366 : /* Offset within the formal parameter. */
367 : unsigned unit_offset;
368 : /* When aggregate_pass_through is set, this is the size of the portion of an
369 : aggregate formal parameter that is being passed. Otherwise, this is size
370 : of pointed to memory that is known to be valid be dereferenced. */
371 : unsigned unit_size : ISRA_ARG_SIZE_LIMIT_BITS;
372 :
373 : /* True when the value of this actual argument is a portion of a formal
374 : parameter. */
375 : unsigned aggregate_pass_through : 1;
376 : /* True when the value of this actual copy is a verbatim pass through of an
377 : obtained pointer. */
378 : unsigned pointer_pass_through : 1;
379 : /* True when it is safe to copy access candidates here from the callee, which
380 : would mean introducing dereferences into callers of the caller. */
381 : unsigned safe_to_import_accesses : 1;
382 : /* True when the passed value is an address of a structure that has been
383 : constructed in the caller just to be passed by reference to functions
384 : (i.e. is never read). */
385 : unsigned constructed_for_calls : 1;
386 : };
387 :
388 : /* Structure used to convey information about calls from the intra-procedural
389 : analysis stage to inter-procedural one. */
390 :
391 6317071 : class isra_call_summary
392 : {
393 : public:
394 6317071 : isra_call_summary ()
395 0 : : m_arg_flow (), m_return_ignored (false), m_return_returned (false),
396 0 : m_bit_aligned_arg (false), m_before_any_store (false)
397 : {}
398 :
399 : void init_inputs (unsigned arg_count);
400 : void dump (FILE *f);
401 :
402 : /* Information about what formal parameters of the caller are used to compute
403 : individual actual arguments of this call. */
404 : auto_vec <isra_param_flow> m_arg_flow;
405 :
406 : /* Set to true if the call statement does not have a LHS. */
407 : unsigned m_return_ignored : 1;
408 :
409 : /* Set to true if the LHS of call statement is only used to construct the
410 : return value of the caller. */
411 : unsigned m_return_returned : 1;
412 :
413 : /* Set when any of the call arguments are not byte-aligned. */
414 : unsigned m_bit_aligned_arg : 1;
415 :
416 : /* Set to true if the call happened before any (other) store to memory in the
417 : caller. */
418 : unsigned m_before_any_store : 1;
419 : };
420 :
421 : /* Class to manage function summaries. */
422 :
423 : class GTY((user)) ipa_sra_function_summaries
424 : : public function_summary <isra_func_summary *>
425 : {
426 : public:
427 140040 : ipa_sra_function_summaries (symbol_table *table, bool ggc):
428 280080 : function_summary<isra_func_summary *> (table, ggc) { }
429 :
430 : void duplicate (cgraph_node *, cgraph_node *,
431 : isra_func_summary *old_sum,
432 : isra_func_summary *new_sum) final override;
433 : void insert (cgraph_node *, isra_func_summary *) final override;
434 : };
435 :
436 : /* Hook that is called by summary when a node is duplicated. */
437 :
438 : void
439 142038 : ipa_sra_function_summaries::duplicate (cgraph_node *, cgraph_node *,
440 : isra_func_summary *old_sum,
441 : isra_func_summary *new_sum)
442 : {
443 : /* TODO: Somehow stop copying when ISRA is doing the cloning, it is
444 : useless. */
445 142038 : new_sum->m_candidate = old_sum->m_candidate;
446 142038 : new_sum->m_returns_value = old_sum->m_returns_value;
447 142038 : new_sum->m_return_ignored = old_sum->m_return_ignored;
448 142038 : gcc_assert (!old_sum->m_queued);
449 142038 : new_sum->m_queued = false;
450 :
451 142038 : unsigned param_count = vec_safe_length (old_sum->m_parameters);
452 140472 : if (!param_count)
453 : return;
454 140472 : vec_safe_reserve_exact (new_sum->m_parameters, param_count);
455 140472 : new_sum->m_parameters->quick_grow_cleared (param_count);
456 676997 : for (unsigned i = 0; i < param_count; i++)
457 : {
458 396053 : isra_param_desc *s = &(*old_sum->m_parameters)[i];
459 396053 : isra_param_desc *d = &(*new_sum->m_parameters)[i];
460 :
461 396053 : d->param_size_limit = s->param_size_limit;
462 396053 : d->size_reached = s->size_reached;
463 396053 : d->safe_size = s->safe_size;
464 396053 : d->locally_unused = s->locally_unused;
465 396053 : d->split_candidate = s->split_candidate;
466 396053 : d->by_ref = s->by_ref;
467 396053 : d->remove_only_when_retval_removed = s->remove_only_when_retval_removed;
468 396053 : d->split_only_when_retval_removed = s->split_only_when_retval_removed;
469 396053 : d->not_specially_constructed = s->not_specially_constructed;
470 396053 : d->conditionally_dereferenceable = s->conditionally_dereferenceable;
471 396053 : d->safe_size_set = s->safe_size_set;
472 :
473 396053 : unsigned acc_count = vec_safe_length (s->accesses);
474 396053 : vec_safe_reserve_exact (d->accesses, acc_count);
475 953730 : for (unsigned j = 0; j < acc_count; j++)
476 : {
477 161624 : param_access *from = (*s->accesses)[j];
478 161624 : param_access *to = ggc_cleared_alloc<param_access> ();
479 161624 : to->type = from->type;
480 161624 : to->alias_ptr_type = from->alias_ptr_type;
481 161624 : to->unit_offset = from->unit_offset;
482 161624 : to->unit_size = from->unit_size;
483 161624 : to->certain = from->certain;
484 161624 : to->reverse = from->reverse;
485 161624 : d->accesses->quick_push (to);
486 : }
487 : }
488 : }
489 :
490 : /* Pointer to the pass function summary holder. */
491 :
492 : static GTY(()) ipa_sra_function_summaries *func_sums;
493 :
494 : /* Hook that is called by summary when new node appears. */
495 :
496 : void
497 18871 : ipa_sra_function_summaries::insert (cgraph_node *node, isra_func_summary *)
498 : {
499 18871 : if (opt_for_fn (node->decl, flag_ipa_sra))
500 : {
501 18871 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
502 18871 : ipa_sra_summarize_function (node);
503 18871 : pop_cfun ();
504 : }
505 : else
506 0 : func_sums->remove (node);
507 18871 : }
508 :
509 : /* Class to manage call summaries. */
510 :
511 : class ipa_sra_call_summaries: public call_summary <isra_call_summary *>
512 : {
513 : public:
514 140040 : ipa_sra_call_summaries (symbol_table *table):
515 280080 : call_summary<isra_call_summary *> (table) { }
516 :
517 : /* Duplicate info when an edge is cloned. */
518 : void duplicate (cgraph_edge *, cgraph_edge *,
519 : isra_call_summary *old_sum,
520 : isra_call_summary *new_sum) final override;
521 : };
522 :
523 : static ipa_sra_call_summaries *call_sums;
524 :
525 :
526 : /* Initialize m_arg_flow of a particular instance of isra_call_summary.
527 : ARG_COUNT is the number of actual arguments passed. */
528 :
529 : void
530 7323386 : isra_call_summary::init_inputs (unsigned arg_count)
531 : {
532 7323386 : if (arg_count == 0)
533 : {
534 370748 : gcc_checking_assert (m_arg_flow.length () == 0);
535 : return;
536 : }
537 6952638 : if (m_arg_flow.length () == 0)
538 : {
539 3438195 : m_arg_flow.reserve_exact (arg_count);
540 3438195 : m_arg_flow.quick_grow_cleared (arg_count);
541 : }
542 : else
543 3514443 : gcc_checking_assert (arg_count == m_arg_flow.length ());
544 : }
545 :
546 : /* Dump all information in call summary to F. */
547 :
548 : void
549 296 : isra_call_summary::dump (FILE *f)
550 : {
551 296 : if (m_return_ignored)
552 188 : fprintf (f, " return value ignored\n");
553 296 : if (m_return_returned)
554 38 : fprintf (f, " return value used only to compute caller return value\n");
555 296 : if (m_before_any_store)
556 57 : fprintf (f, " happens before any store to memory\n");
557 627 : for (unsigned i = 0; i < m_arg_flow.length (); i++)
558 : {
559 331 : fprintf (f, " Parameter %u:\n", i);
560 331 : isra_param_flow *ipf = &m_arg_flow[i];
561 :
562 331 : if (ipf->length)
563 : {
564 206 : bool first = true;
565 206 : fprintf (f, " Scalar param sources: ");
566 621 : for (int j = 0; j < ipf->length; j++)
567 : {
568 209 : if (!first)
569 3 : fprintf (f, ", ");
570 : else
571 : first = false;
572 209 : fprintf (f, "%i", (int) ipf->inputs[j]);
573 : }
574 206 : fprintf (f, "\n");
575 : }
576 331 : if (ipf->aggregate_pass_through)
577 26 : fprintf (f, " Aggregate pass through from the param given above, "
578 : "unit offset: %u , unit size: %u\n",
579 26 : ipf->unit_offset, ipf->unit_size);
580 305 : else if (ipf->unit_size > 0)
581 72 : fprintf (f, " Known dereferenceable size: %u\n", ipf->unit_size);
582 331 : if (ipf->pointer_pass_through)
583 30 : fprintf (f, " Pointer pass through from the param given above, "
584 30 : "safe_to_import_accesses: %u\n", ipf->safe_to_import_accesses);
585 331 : if (ipf->constructed_for_calls)
586 38 : fprintf (f, " Variable constructed just to be passed to "
587 : "calls.\n");
588 : }
589 296 : }
590 :
591 : /* Duplicate edge summary when an edge is cloned. */
592 :
593 : void
594 722932 : ipa_sra_call_summaries::duplicate (cgraph_edge *, cgraph_edge *,
595 : isra_call_summary *old_sum,
596 : isra_call_summary *new_sum)
597 : {
598 722932 : unsigned arg_count = old_sum->m_arg_flow.length ();
599 722932 : new_sum->init_inputs (arg_count);
600 2800364 : for (unsigned i = 0; i < arg_count; i++)
601 1354500 : new_sum->m_arg_flow[i] = old_sum->m_arg_flow[i];
602 :
603 722932 : new_sum->m_return_ignored = old_sum->m_return_ignored;
604 722932 : new_sum->m_return_returned = old_sum->m_return_returned;
605 722932 : new_sum->m_bit_aligned_arg = old_sum->m_bit_aligned_arg;
606 722932 : new_sum->m_before_any_store = old_sum->m_before_any_store;
607 722932 : }
608 :
609 :
610 : /* With all GTY stuff done, we can move to anonymous namespace. */
611 : namespace {
612 : /* Quick mapping from a decl to its param descriptor. */
613 :
614 : hash_map<tree, gensum_param_desc *> *decl2desc;
615 :
616 : /* All local DECLs ever loaded from of and of those that have their address
617 : assigned to a variable. */
618 :
619 : hash_set <tree> *loaded_decls;
620 :
621 : /* Countdown of allowed Alias Analysis steps during summary building. */
622 :
623 : int aa_walking_limit;
624 :
625 : /* This is a table in which for each basic block and parameter there is a
626 : distance (offset + size) in that parameter which is dereferenced and
627 : accessed in that BB. */
628 : HOST_WIDE_INT *bb_dereferences = NULL;
629 : /* How many by-reference parameters there are in the current function. */
630 : int unsafe_by_ref_count;
631 :
632 : /* Bitmap of BBs that can cause the function to "stop" progressing by
633 : returning, throwing externally, looping infinitely or calling a function
634 : which might abort etc.. */
635 : bitmap final_bbs;
636 :
637 : /* Obstack to allocate various small structures required only when generating
638 : summary for a function. */
639 : struct obstack gensum_obstack;
640 :
641 : /* Return false the function is apparently unsuitable for IPA-SRA based on it's
642 : attributes, return true otherwise. NODE is the cgraph node of the current
643 : function. */
644 :
645 : static bool
646 1304084 : ipa_sra_preliminary_function_checks (cgraph_node *node)
647 : {
648 1304084 : if (!node->can_change_signature)
649 : {
650 35928 : if (dump_file)
651 0 : fprintf (dump_file, "Function cannot change signature.\n");
652 : return false;
653 : }
654 :
655 1268156 : if (!tree_versionable_function_p (node->decl))
656 : {
657 122233 : if (dump_file)
658 7 : fprintf (dump_file, "Function is not versionable.\n");
659 : return false;
660 : }
661 :
662 1145923 : if (!opt_for_fn (node->decl, optimize)
663 1145923 : || !opt_for_fn (node->decl, flag_ipa_sra))
664 : {
665 301 : if (dump_file)
666 0 : fprintf (dump_file, "Not optimizing or IPA-SRA turned off for this "
667 : "function.\n");
668 : return false;
669 : }
670 :
671 1145622 : if (DECL_VIRTUAL_P (node->decl))
672 : {
673 31086 : if (dump_file)
674 0 : fprintf (dump_file, "Function is a virtual method.\n");
675 : return false;
676 : }
677 :
678 1114536 : struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
679 1114536 : if (fun->stdarg)
680 : {
681 124 : if (dump_file)
682 0 : fprintf (dump_file, "Function uses stdarg. \n");
683 : return false;
684 : }
685 :
686 1114412 : if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
687 : {
688 51103 : if (dump_file)
689 0 : fprintf (dump_file, "Always inline function will be inlined "
690 : "anyway. \n");
691 : return false;
692 : }
693 :
694 : /* A special case. If the callback edge pointing to the assume body is not
695 : created before ipa-sra analysis is run, the newly created edge won't be
696 : analyzed and will thus lack the appropriate summaries. Be conservative and
697 : skip the function entirely. We aren't missing anything, because we can't
698 : use an unsummarized call anyway. */
699 1063309 : if (node->assume_function && !node->callers)
700 : {
701 0 : if (dump_file)
702 0 : fprintf (dump_file,
703 : "Assume body without a callback edge, skipping. \n");
704 : return false;
705 : }
706 :
707 : return true;
708 : }
709 :
710 : /* Print access tree starting at ACCESS to F. */
711 :
712 : static void
713 117 : dump_gensum_access (FILE *f, gensum_param_access *access, unsigned indent)
714 : {
715 117 : fprintf (f, " ");
716 488 : for (unsigned i = 0; i < indent; i++)
717 254 : fprintf (f, " ");
718 117 : fprintf (f, " * Access to offset: " HOST_WIDE_INT_PRINT_DEC,
719 : access->offset);
720 117 : fprintf (f, ", size: " HOST_WIDE_INT_PRINT_DEC, access->size);
721 117 : fprintf (f, ", type: ");
722 117 : print_generic_expr (f, access->type);
723 117 : fprintf (f, ", alias_ptr_type: ");
724 117 : print_generic_expr (f, access->alias_ptr_type);
725 117 : fprintf (f, ", load_count: ");
726 117 : access->load_count.dump (f);
727 117 : fprintf (f, ", nonarg: %u, reverse: %u\n", access->nonarg, access->reverse);
728 117 : for (gensum_param_access *ch = access->first_child;
729 127 : ch;
730 10 : ch = ch->next_sibling)
731 10 : dump_gensum_access (f, ch, indent + 2);
732 117 : }
733 :
734 :
735 : /* Print access tree starting at ACCESS to F. */
736 :
737 : static void
738 281 : dump_isra_access (FILE *f, param_access *access)
739 : {
740 281 : fprintf (f, " * Access to unit offset: %u", access->unit_offset);
741 281 : fprintf (f, ", unit size: %u", access->unit_size);
742 281 : fprintf (f, ", type: ");
743 281 : print_generic_expr (f, access->type);
744 281 : fprintf (f, ", alias_ptr_type: ");
745 281 : print_generic_expr (f, access->alias_ptr_type);
746 281 : if (access->certain)
747 241 : fprintf (f, ", certain");
748 : else
749 40 : fprintf (f, ", not certain");
750 281 : if (access->reverse)
751 0 : fprintf (f, ", reverse");
752 281 : fprintf (f, "\n");
753 281 : }
754 :
755 : /* Dump access tree starting at ACCESS to stderr. */
756 :
757 : DEBUG_FUNCTION void
758 0 : debug_isra_access (param_access *access)
759 : {
760 0 : dump_isra_access (stderr, access);
761 0 : }
762 :
763 : /* Dump DESC to F. */
764 :
765 : static void
766 369 : dump_gensum_param_descriptor (FILE *f, gensum_param_desc *desc)
767 : {
768 369 : if (desc->locally_unused)
769 175 : fprintf (f, " unused with %i call_uses%s\n", desc->call_uses,
770 175 : desc->remove_only_when_retval_removed ?
771 : " remove_only_when_retval_removed" : "");
772 369 : if (!desc->split_candidate)
773 : {
774 248 : fprintf (f, " not a candidate\n");
775 248 : return;
776 : }
777 121 : if (desc->by_ref)
778 90 : fprintf (f, " %s%s%s by_ref with %u pass throughs\n",
779 90 : desc->safe_ref ? "safe" : "unsafe",
780 90 : desc->conditionally_dereferenceable
781 : ? " conditionally_dereferenceable" : "",
782 90 : desc->split_only_when_retval_removed
783 : ? " split_only_when_retval_removed" : "",
784 : desc->ptr_pt_count);
785 :
786 228 : for (gensum_param_access *acc = desc->accesses; acc; acc = acc->next_sibling)
787 107 : dump_gensum_access (f, acc, 2);
788 : }
789 :
790 : /* Dump all parameter descriptors in IFS, assuming it describes FNDECL, to
791 : F. */
792 :
793 : static void
794 172 : dump_gensum_param_descriptors (FILE *f, tree fndecl,
795 : vec<gensum_param_desc> *param_descriptions)
796 : {
797 172 : tree parm = DECL_ARGUMENTS (fndecl);
798 172 : for (unsigned i = 0;
799 541 : i < param_descriptions->length ();
800 369 : ++i, parm = DECL_CHAIN (parm))
801 : {
802 369 : fprintf (f, " Descriptor for parameter %i ", i);
803 369 : print_generic_expr (f, parm, TDF_UID);
804 369 : fprintf (f, "\n");
805 369 : dump_gensum_param_descriptor (f, &(*param_descriptions)[i]);
806 : }
807 172 : }
808 :
809 :
810 : /* Dump DESC to F. If HINTS is true, also dump IPA-analysis computed
811 : hints. */
812 :
813 : static void
814 743 : dump_isra_param_descriptor (FILE *f, isra_param_desc *desc, bool hints)
815 : {
816 743 : if (desc->locally_unused)
817 : {
818 352 : fprintf (f, " (locally) unused\n");
819 : }
820 743 : if (!desc->split_candidate)
821 : {
822 480 : fprintf (f, " not a candidate for splitting");
823 480 : if (hints && desc->by_ref && desc->safe_size_set)
824 9 : fprintf (f, ", safe_size: %u", (unsigned) desc->safe_size);
825 480 : fprintf (f, "\n");
826 480 : return;
827 : }
828 526 : fprintf (f, " param_size_limit: %u, size_reached: %u%s",
829 263 : desc->param_size_limit, desc->size_reached,
830 263 : desc->by_ref ? ", by_ref" : "");
831 263 : if (desc->remove_only_when_retval_removed)
832 48 : fprintf (f, ", remove_only_when_retval_removed");
833 263 : if (desc->split_only_when_retval_removed)
834 4 : fprintf (f, ", split_only_when_retval_removed");
835 263 : if (desc->by_ref && desc->conditionally_dereferenceable)
836 54 : fprintf (f, ", conditionally_dereferenceable");
837 263 : if (hints)
838 : {
839 29 : if (desc->by_ref && !desc->not_specially_constructed)
840 8 : fprintf (f, ", args_specially_constructed");
841 29 : if (desc->by_ref && desc->safe_size_set)
842 18 : fprintf (f, ", safe_size: %u", (unsigned) desc->safe_size);
843 : }
844 263 : fprintf (f, "\n");
845 :
846 807 : for (unsigned i = 0; i < vec_safe_length (desc->accesses); ++i)
847 : {
848 281 : param_access *access = (*desc->accesses)[i];
849 281 : dump_isra_access (f, access);
850 : }
851 : }
852 :
853 : /* Dump all parameter descriptors in IFS, assuming it describes FNDECL, to F.
854 : If HINTS is true, also dump IPA-analysis computed hints. */
855 :
856 : static void
857 172 : dump_isra_param_descriptors (FILE *f, tree fndecl, isra_func_summary *ifs,
858 : bool hints)
859 : {
860 172 : tree parm = DECL_ARGUMENTS (fndecl);
861 172 : if (!ifs->m_parameters)
862 : {
863 0 : fprintf (f, " parameter descriptors not available\n");
864 0 : return;
865 : }
866 :
867 : for (unsigned i = 0;
868 541 : i < ifs->m_parameters->length ();
869 369 : ++i, parm = DECL_CHAIN (parm))
870 : {
871 369 : fprintf (f, " Descriptor for parameter %i ", i);
872 369 : print_generic_expr (f, parm, TDF_UID);
873 369 : fprintf (f, "\n");
874 369 : dump_isra_param_descriptor (f, &(*ifs->m_parameters)[i], hints);
875 : }
876 : }
877 :
878 : /* Add SRC to inputs of PARAM_FLOW, unless it would exceed storage. If the
879 : function fails return false, otherwise return true. SRC must fit into an
880 : unsigned char. Used for purposes of transitive unused parameter
881 : removal. */
882 :
883 : static bool
884 1366204 : add_src_to_param_flow (isra_param_flow *param_flow, int src)
885 : {
886 1366204 : gcc_checking_assert (src >= 0 && src <= UCHAR_MAX);
887 1366204 : if (param_flow->length == IPA_SRA_MAX_PARAM_FLOW_LEN)
888 : return false;
889 :
890 1366202 : param_flow->inputs[(int) param_flow->length] = src;
891 1366202 : param_flow->length++;
892 1366202 : return true;
893 : }
894 :
895 : /* Add a SRC to the inputs of PARAM_FLOW unless it is already there and assert
896 : it is the only input. Used for purposes of transitive parameter
897 : splitting. */
898 :
899 : static void
900 809997 : set_single_param_flow_source (isra_param_flow *param_flow, int src)
901 : {
902 809997 : gcc_checking_assert (src >= 0 && src <= UCHAR_MAX);
903 809997 : if (param_flow->length == 0)
904 : {
905 125375 : param_flow->inputs[0] = src;
906 125375 : param_flow->length = 1;
907 : }
908 684622 : else if (param_flow->length == 1)
909 684622 : gcc_assert (param_flow->inputs[0] == src);
910 : else
911 0 : gcc_unreachable ();
912 809997 : }
913 :
914 : /* Assert that there is only a single value in PARAM_FLOW's inputs and return
915 : it. */
916 :
917 : static unsigned
918 1343958 : get_single_param_flow_source (const isra_param_flow *param_flow)
919 : {
920 1343958 : gcc_assert (param_flow->length == 1);
921 1343958 : return param_flow->inputs[0];
922 : }
923 :
924 : /* Inspect all uses of NAME and simple arithmetic calculations involving NAME
925 : in FUN represented with NODE and return a negative number if any of them is
926 : used for something else than either an actual call argument, simple return,
927 : simple arithmetic operation or debug statement. If there are no such uses,
928 : return the number of actual arguments that this parameter eventually feeds
929 : to (or zero if there is none). If there are any simple return uses, set
930 : DESC->remove_only_when_retval_removed. For any such parameter, mark
931 : PARM_NUM as one of its sources. ANALYZED is a bitmap that tracks which SSA
932 : names we have already started investigating. */
933 :
934 : static int
935 3538057 : isra_track_scalar_value_uses (function *fun, cgraph_node *node, tree name,
936 : int parm_num, bitmap analyzed,
937 : gensum_param_desc *desc)
938 : {
939 3538057 : int res = 0;
940 3538057 : imm_use_iterator imm_iter;
941 3538057 : gimple *stmt;
942 :
943 7332623 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
944 : {
945 5502486 : if (is_gimple_debug (stmt)
946 5502486 : || gimple_clobber_p (stmt))
947 963432 : continue;
948 :
949 : /* TODO: We could handle at least const builtin functions like arithmetic
950 : operations below. */
951 4539054 : if (is_gimple_call (stmt))
952 : {
953 1379760 : int all_uses = 0;
954 1379760 : use_operand_p use_p;
955 2773329 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
956 1393569 : all_uses++;
957 :
958 1379760 : gcall *call = as_a <gcall *> (stmt);
959 1379760 : unsigned arg_count;
960 1379760 : if (gimple_call_internal_p (call)
961 1379760 : || (arg_count = gimple_call_num_args (call)) == 0)
962 : {
963 : res = -1;
964 : break;
965 : }
966 :
967 1374479 : cgraph_edge *cs = node->get_edge (stmt);
968 1374479 : gcc_checking_assert (cs);
969 1374479 : isra_call_summary *csum = call_sums->get_create (cs);
970 1374479 : csum->init_inputs (arg_count);
971 :
972 1374479 : int simple_uses = 0;
973 7614307 : for (unsigned i = 0; i < arg_count; i++)
974 4865351 : if (gimple_call_arg (call, i) == name)
975 : {
976 1366204 : if (!add_src_to_param_flow (&csum->m_arg_flow[i], parm_num))
977 : {
978 : simple_uses = -1;
979 : break;
980 : }
981 1366202 : simple_uses++;
982 : }
983 :
984 1374479 : if (simple_uses < 0
985 1374479 : || all_uses != simple_uses)
986 : {
987 : res = -1;
988 : break;
989 : }
990 1352654 : res += all_uses;
991 : }
992 3159294 : else if (!stmt_unremovable_because_of_non_call_eh_p (fun, stmt)
993 3159294 : && ((is_gimple_assign (stmt) && !gimple_has_volatile_ops (stmt))
994 731351 : || gimple_code (stmt) == GIMPLE_PHI))
995 : {
996 2540292 : tree lhs;
997 2540292 : if (gimple_code (stmt) == GIMPLE_PHI)
998 219939 : lhs = gimple_phi_result (stmt);
999 : else
1000 2320353 : lhs = gimple_assign_lhs (stmt);
1001 :
1002 2540292 : if (TREE_CODE (lhs) != SSA_NAME)
1003 : {
1004 : res = -1;
1005 : break;
1006 : }
1007 2121662 : gcc_assert (!gimple_vdef (stmt));
1008 2121662 : if (bitmap_set_bit (analyzed, SSA_NAME_VERSION (lhs)))
1009 : {
1010 2011173 : int tmp = isra_track_scalar_value_uses (fun, node, lhs, parm_num,
1011 : analyzed, desc);
1012 2011173 : if (tmp < 0)
1013 : {
1014 : res = tmp;
1015 : break;
1016 : }
1017 1194876 : res += tmp;
1018 : }
1019 : }
1020 619002 : else if (greturn *gr = dyn_cast<greturn *>(stmt))
1021 : {
1022 173115 : tree rv = gimple_return_retval (gr);
1023 173115 : if (rv != name)
1024 : {
1025 : res = -1;
1026 : break;
1027 : }
1028 173115 : desc->remove_only_when_retval_removed = true;
1029 : }
1030 : else
1031 : {
1032 : res = -1;
1033 : break;
1034 : }
1035 3538057 : }
1036 3538057 : return res;
1037 : }
1038 :
1039 : /* Inspect all uses of PARM, which must be a gimple register, in FUN (which is
1040 : also described by NODE) and simple arithmetic calculations involving PARM
1041 : and return false if any of them is used for something else than either an
1042 : actual call argument, simple return, simple arithmetic operation or debug
1043 : statement. If there are no such uses, return true and store the number of
1044 : actual arguments that this parameter eventually feeds to (or zero if there
1045 : is none) to DESC->call_uses and set DESC->remove_only_when_retval_removed if
1046 : there are any uses in return statements. For any such parameter, mark
1047 : PARM_NUM as one of its sources.
1048 :
1049 : This function is similar to ptr_parm_has_nonarg_uses but its results are
1050 : meant for unused parameter removal, as opposed to splitting of parameters
1051 : passed by reference or converting them to passed by value. */
1052 :
1053 : static bool
1054 2225070 : isra_track_scalar_param_local_uses (function *fun, cgraph_node *node, tree parm,
1055 : int parm_num, gensum_param_desc *desc)
1056 : {
1057 2225070 : gcc_checking_assert (is_gimple_reg (parm));
1058 :
1059 2225070 : tree name = ssa_default_def (fun, parm);
1060 2225070 : if (!name || has_zero_uses (name))
1061 : {
1062 698186 : desc->call_uses = 0;
1063 698186 : return false;
1064 : }
1065 :
1066 : /* Edge summaries can only handle callers with fewer than 256 parameters. */
1067 1526884 : if (parm_num > UCHAR_MAX)
1068 : return true;
1069 :
1070 1526884 : bitmap analyzed = BITMAP_ALLOC (NULL);
1071 1526884 : int call_uses = isra_track_scalar_value_uses (fun, node, name, parm_num,
1072 : analyzed, desc);
1073 1526884 : BITMAP_FREE (analyzed);
1074 1526884 : if (call_uses < 0)
1075 : return true;
1076 635261 : desc->call_uses = call_uses;
1077 635261 : return false;
1078 : }
1079 :
1080 : /* Scan immediate uses of a default definition SSA name of a parameter PARM and
1081 : examine whether there are any nonarg uses that are not actual arguments or
1082 : otherwise infeasible uses. If so, return true, otherwise return false.
1083 : Create pass-through IPA flow records for any direct uses as argument calls
1084 : and if returning false, store their number into DESC->ptr_pt_count. If
1085 : removal of return value would still allow splitting, return true but set
1086 : DESC->split_only_when_retval_removed. NODE and FUN must represent the
1087 : function that is currently analyzed, PARM_NUM must be the index of the
1088 : analyzed parameter.
1089 :
1090 : This function is similar to isra_track_scalar_param_local_uses but its
1091 : results are meant for splitting of parameters passed by reference or turning
1092 : them into bits passed by value, as opposed to generic unused parameter
1093 : removal. */
1094 :
1095 : static bool
1096 1175852 : ptr_parm_has_nonarg_uses (cgraph_node *node, function *fun, tree parm,
1097 : int parm_num, gensum_param_desc *desc)
1098 : {
1099 1175852 : imm_use_iterator ui;
1100 1175852 : gimple *stmt;
1101 1175852 : tree name = ssa_default_def (fun, parm);
1102 1175852 : bool ret = false;
1103 1175852 : unsigned pt_count = 0;
1104 :
1105 1175852 : if (!name || has_zero_uses (name))
1106 : return false;
1107 :
1108 : /* Edge summaries can only handle callers with fewer than 256 parameters. */
1109 1061587 : if (parm_num > UCHAR_MAX)
1110 : return true;
1111 :
1112 2960934 : FOR_EACH_IMM_USE_STMT (stmt, ui, name)
1113 : {
1114 2448784 : unsigned uses_ok = 0;
1115 2448784 : use_operand_p use_p;
1116 :
1117 2448784 : if (is_gimple_debug (stmt)
1118 2448784 : || gimple_clobber_p (stmt))
1119 632043 : continue;
1120 :
1121 1816741 : if (gimple_assign_single_p (stmt))
1122 : {
1123 932853 : tree rhs = gimple_assign_rhs1 (stmt);
1124 932853 : if (!TREE_THIS_VOLATILE (rhs))
1125 : {
1126 1773217 : while (handled_component_p (rhs))
1127 840448 : rhs = TREE_OPERAND (rhs, 0);
1128 932769 : if (TREE_CODE (rhs) == MEM_REF
1129 646910 : && TREE_OPERAND (rhs, 0) == name
1130 642656 : && integer_zerop (TREE_OPERAND (rhs, 1))
1131 1530005 : && types_compatible_p (TREE_TYPE (rhs),
1132 597236 : TREE_TYPE (TREE_TYPE (name))))
1133 : uses_ok++;
1134 : }
1135 : }
1136 883888 : else if (is_gimple_call (stmt))
1137 : {
1138 736863 : gcall *call = as_a<gcall *> (stmt);
1139 736863 : unsigned arg_count = gimple_call_num_args (call);
1140 736863 : if ((gimple_call_internal_p (call)
1141 526 : && !callback_ifn_assume_call_with_edge_p (node, stmt))
1142 736874 : || arg_count == 0)
1143 : {
1144 : ret = true;
1145 : break;
1146 : }
1147 :
1148 736074 : cgraph_edge *cs = node->get_edge (stmt);
1149 736074 : gcc_checking_assert (cs);
1150 736074 : isra_call_summary *csum = call_sums->get_create (cs);
1151 736074 : csum->init_inputs (arg_count);
1152 :
1153 3573242 : for (unsigned i = 0; i < arg_count; ++i)
1154 : {
1155 2101094 : tree arg = gimple_call_arg (stmt, i);
1156 :
1157 2101094 : if (arg == name)
1158 : {
1159 : /* TODO: Allow &MEM_REF[name + offset] here,
1160 : ipa_param_body_adjustments::modify_call_stmt has to be
1161 : adjusted too. */
1162 724197 : csum->m_arg_flow[i].pointer_pass_through = true;
1163 724197 : set_single_param_flow_source (&csum->m_arg_flow[i], parm_num);
1164 724197 : pt_count++;
1165 724197 : uses_ok++;
1166 724197 : continue;
1167 : }
1168 :
1169 1376897 : if (!TREE_THIS_VOLATILE (arg))
1170 : {
1171 1406977 : while (handled_component_p (arg))
1172 30080 : arg = TREE_OPERAND (arg, 0);
1173 1376897 : if (TREE_CODE (arg) == MEM_REF
1174 23115 : && TREE_OPERAND (arg, 0) == name
1175 11861 : && integer_zerop (TREE_OPERAND (arg, 1))
1176 1388756 : && types_compatible_p (TREE_TYPE (arg),
1177 11859 : TREE_TYPE (TREE_TYPE (name))))
1178 9017 : uses_ok++;
1179 : }
1180 : }
1181 : }
1182 147025 : else if (greturn *gr = dyn_cast<greturn *>(stmt))
1183 : {
1184 9390 : tree rv = gimple_return_retval (gr);
1185 9390 : if (rv == name)
1186 : {
1187 9390 : uses_ok++;
1188 : /* Analysis for feasibility of removal must have already reached
1189 : the conclusion that the flag must be set if it completed. */
1190 9390 : gcc_assert (!desc->locally_unused
1191 : || desc->remove_only_when_retval_removed);
1192 9390 : desc->split_only_when_retval_removed = true;
1193 : }
1194 : }
1195 :
1196 : /* If the number of valid uses does not match the number of
1197 : uses in this stmt there is an unhandled use. */
1198 1815952 : unsigned all_uses = 0;
1199 3639254 : FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
1200 1823302 : all_uses++;
1201 :
1202 1815952 : gcc_checking_assert (uses_ok <= all_uses);
1203 1815952 : if (uses_ok != all_uses)
1204 : {
1205 : ret = true;
1206 : break;
1207 : }
1208 1061587 : }
1209 :
1210 1061587 : desc->ptr_pt_count = pt_count;
1211 1061587 : return ret;
1212 : }
1213 :
1214 : /* Initialize vector of parameter descriptors of NODE. Return true if there
1215 : are any candidates for splitting or unused aggregate parameter removal (the
1216 : function may return false if there are candidates for removal of register
1217 : parameters). */
1218 :
1219 : static bool
1220 861037 : create_parameter_descriptors (cgraph_node *node,
1221 : vec<gensum_param_desc> *param_descriptions)
1222 : {
1223 861037 : function *fun = DECL_STRUCT_FUNCTION (node->decl);
1224 861037 : bool ret = false;
1225 :
1226 861037 : int num = 0;
1227 861037 : for (tree parm = DECL_ARGUMENTS (node->decl);
1228 3308445 : parm;
1229 2447408 : parm = DECL_CHAIN (parm), num++)
1230 : {
1231 2447408 : const char *msg;
1232 2447408 : gensum_param_desc *desc = &(*param_descriptions)[num];
1233 : /* param_descriptions vector is grown cleared in the caller. */
1234 2447408 : desc->param_number = num;
1235 2447408 : decl2desc->put (parm, desc);
1236 :
1237 2447408 : if (dump_file && (dump_flags & TDF_DETAILS))
1238 76 : print_generic_expr (dump_file, parm, TDF_UID);
1239 :
1240 2447408 : tree type = TREE_TYPE (parm);
1241 2447408 : if (TREE_THIS_VOLATILE (parm))
1242 : {
1243 52 : if (dump_file && (dump_flags & TDF_DETAILS))
1244 0 : fprintf (dump_file, " not a candidate, is volatile\n");
1245 1820117 : continue;
1246 : }
1247 2447356 : if (!is_gimple_reg_type (type) && is_va_list_type (type))
1248 : {
1249 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1250 0 : fprintf (dump_file, " not a candidate, is a va_list type\n");
1251 0 : continue;
1252 : }
1253 2447356 : if (TREE_ADDRESSABLE (parm))
1254 : {
1255 36338 : if (dump_file && (dump_flags & TDF_DETAILS))
1256 0 : fprintf (dump_file, " not a candidate, is addressable\n");
1257 36338 : continue;
1258 : }
1259 2411018 : if (TREE_ADDRESSABLE (type))
1260 : {
1261 3 : if (dump_file && (dump_flags & TDF_DETAILS))
1262 0 : fprintf (dump_file, " not a candidate, type cannot be split\n");
1263 3 : continue;
1264 : }
1265 :
1266 2411015 : if (is_gimple_reg (parm)
1267 2411015 : && !isra_track_scalar_param_local_uses (fun, node, parm, num, desc))
1268 : {
1269 1333447 : desc->locally_unused = true;
1270 :
1271 1333447 : if (dump_file && (dump_flags & TDF_DETAILS))
1272 28 : fprintf (dump_file, " is a scalar with only %i call uses%s\n",
1273 : desc->call_uses,
1274 28 : desc->remove_only_when_retval_removed
1275 : ? " and return uses" : "");
1276 : }
1277 :
1278 2411015 : if (POINTER_TYPE_P (type))
1279 : {
1280 1182446 : desc->by_ref = true;
1281 1182446 : if (TREE_CODE (type) == REFERENCE_TYPE
1282 1182446 : || (num == 0
1283 570228 : && TREE_CODE (TREE_TYPE (node->decl)) == METHOD_TYPE))
1284 515257 : desc->safe_ref = true;
1285 : else
1286 667189 : desc->safe_ref = false;
1287 1182446 : type = TREE_TYPE (type);
1288 :
1289 1187519 : if (TREE_CODE (type) == FUNCTION_TYPE
1290 1182446 : || TREE_CODE (type) == METHOD_TYPE)
1291 : {
1292 5073 : if (dump_file && (dump_flags & TDF_DETAILS))
1293 0 : fprintf (dump_file, " not a candidate, reference to "
1294 : "a function\n");
1295 5073 : continue;
1296 : }
1297 1177373 : if (TYPE_VOLATILE (type))
1298 : {
1299 1130 : if (dump_file && (dump_flags & TDF_DETAILS))
1300 0 : fprintf (dump_file, " not a candidate, reference to "
1301 : "a volatile type\n");
1302 1130 : continue;
1303 : }
1304 1176243 : if (TREE_CODE (type) == ARRAY_TYPE
1305 1176243 : && TYPE_NONALIASED_COMPONENT (type))
1306 : {
1307 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1308 0 : fprintf (dump_file, " not a candidate, reference to "
1309 : "a nonaliased component array\n");
1310 0 : continue;
1311 : }
1312 1176243 : if (!is_gimple_reg (parm))
1313 : {
1314 6 : if (dump_file && (dump_flags & TDF_DETAILS))
1315 0 : fprintf (dump_file, " not a candidate, a reference which is "
1316 : "not a gimple register (probably addressable)\n");
1317 6 : continue;
1318 : }
1319 1176237 : if (is_va_list_type (type))
1320 : {
1321 385 : if (dump_file && (dump_flags & TDF_DETAILS))
1322 0 : fprintf (dump_file, " not a candidate, reference to "
1323 : "a va list\n");
1324 385 : continue;
1325 : }
1326 1175852 : if (ptr_parm_has_nonarg_uses (node, fun, parm, num, desc))
1327 : {
1328 549437 : if (dump_file && (dump_flags & TDF_DETAILS))
1329 7 : fprintf (dump_file, " not a candidate, reference has "
1330 : "nonarg uses\n");
1331 549437 : continue;
1332 : }
1333 : }
1334 1228569 : else if (!AGGREGATE_TYPE_P (type))
1335 : {
1336 : /* This is in an else branch because scalars passed by reference are
1337 : still candidates to be passed by value. */
1338 1042832 : if (dump_file && (dump_flags & TDF_DETAILS))
1339 21 : fprintf (dump_file, " not a candidate, not an aggregate\n");
1340 1042832 : continue;
1341 : }
1342 :
1343 812152 : if (!COMPLETE_TYPE_P (type))
1344 : {
1345 171303 : if (dump_file && (dump_flags & TDF_DETAILS))
1346 0 : fprintf (dump_file, " not a candidate, not a complete type\n");
1347 171303 : continue;
1348 : }
1349 640849 : if (!tree_fits_uhwi_p (TYPE_SIZE (type)))
1350 : {
1351 2 : if (dump_file && (dump_flags & TDF_DETAILS))
1352 0 : fprintf (dump_file, " not a candidate, size not representable\n");
1353 2 : continue;
1354 : }
1355 640847 : unsigned HOST_WIDE_INT type_size
1356 640847 : = tree_to_uhwi (TYPE_SIZE (type)) / BITS_PER_UNIT;
1357 645305 : if (type_size == 0
1358 640847 : || type_size >= ISRA_ARG_SIZE_LIMIT)
1359 : {
1360 4458 : if (dump_file && (dump_flags & TDF_DETAILS))
1361 0 : fprintf (dump_file, " not a candidate, has zero or huge size\n");
1362 4458 : continue;
1363 : }
1364 636389 : if (type_internals_preclude_sra_p (type, &msg))
1365 : {
1366 9098 : if (dump_file && (dump_flags & TDF_DETAILS))
1367 0 : fprintf (dump_file, " not a candidate, %s\n", msg);
1368 9098 : continue;
1369 : }
1370 :
1371 627291 : if (dump_file && (dump_flags & TDF_DETAILS))
1372 48 : fprintf (dump_file, " is a candidate\n");
1373 :
1374 627291 : ret = true;
1375 627291 : desc->split_candidate = true;
1376 627291 : if (desc->by_ref && !desc->safe_ref)
1377 196840 : desc->deref_index = unsafe_by_ref_count++;
1378 : }
1379 861037 : return ret;
1380 : }
1381 :
1382 : /* Return pointer to descriptor of parameter DECL or NULL if it cannot be
1383 : found, which happens if DECL is for a static chain. */
1384 :
1385 : static gensum_param_desc *
1386 2517067 : get_gensum_param_desc (tree decl)
1387 : {
1388 2517067 : if (!decl2desc)
1389 : return NULL;
1390 2337697 : gcc_checking_assert (TREE_CODE (decl) == PARM_DECL);
1391 2337697 : gensum_param_desc **slot = decl2desc->get (decl);
1392 2337697 : if (!slot)
1393 : /* This can happen for static chains which we cannot handle so far. */
1394 : return NULL;
1395 2312255 : gcc_checking_assert (*slot);
1396 : return *slot;
1397 : }
1398 :
1399 :
1400 : /* Remove parameter described by DESC from candidates for IPA-SRA splitting and
1401 : write REASON to the dump file if there is one. */
1402 :
1403 : static void
1404 168617 : disqualify_split_candidate (gensum_param_desc *desc, const char *reason)
1405 : {
1406 168617 : if (!desc->split_candidate)
1407 : return;
1408 :
1409 168588 : if (dump_file && (dump_flags & TDF_DETAILS))
1410 0 : fprintf (dump_file, "! Disqualifying parameter number %i - %s\n",
1411 : desc->param_number, reason);
1412 :
1413 168588 : desc->split_candidate = false;
1414 : }
1415 :
1416 : /* Remove DECL from candidates for IPA-SRA and write REASON to the dump file if
1417 : there is one. */
1418 :
1419 : static void
1420 177 : disqualify_split_candidate (tree decl, const char *reason)
1421 : {
1422 177 : gensum_param_desc *desc = get_gensum_param_desc (decl);
1423 177 : if (desc)
1424 29 : disqualify_split_candidate (desc, reason);
1425 177 : }
1426 :
1427 : /* Allocate a new access to DESC and fill it in with OFFSET and SIZE. But
1428 : first, check that there are not too many of them already. If so, do not
1429 : allocate anything and return NULL. */
1430 :
1431 : static gensum_param_access *
1432 450149 : allocate_access (gensum_param_desc *desc,
1433 : HOST_WIDE_INT offset, HOST_WIDE_INT size)
1434 : {
1435 450149 : if (desc->access_count
1436 450149 : == (unsigned) param_ipa_sra_max_replacements)
1437 : {
1438 0 : disqualify_split_candidate (desc, "Too many replacement candidates");
1439 0 : return NULL;
1440 : }
1441 :
1442 450149 : gensum_param_access *access
1443 450149 : = (gensum_param_access *) obstack_alloc (&gensum_obstack,
1444 : sizeof (gensum_param_access));
1445 450149 : memset (access, 0, sizeof (*access));
1446 450149 : access->offset = offset;
1447 450149 : access->size = size;
1448 450149 : access->load_count = profile_count::zero ();
1449 450149 : return access;
1450 : }
1451 :
1452 : /* In what context scan_expr_access has been called, whether it deals with a
1453 : load, a function argument, or a store. Please note that in rare
1454 : circumstances when it is not clear if the access is a load or store,
1455 : ISRA_CTX_STORE is used too. */
1456 :
1457 : enum isra_scan_context {ISRA_CTX_LOAD, ISRA_CTX_ARG, ISRA_CTX_STORE};
1458 :
1459 : /* Return an access describing memory access to the variable described by DESC
1460 : at OFFSET with SIZE in context CTX, starting at pointer to the linked list
1461 : at a certain tree level FIRST. Attempt to create it and put into the
1462 : appropriate place in the access tree if does not exist, but fail and return
1463 : NULL if there are already too many accesses, if it would create a partially
1464 : overlapping access or if an access would end up within a pre-existing
1465 : non-call access. */
1466 :
1467 : static gensum_param_access *
1468 528348 : get_access_1 (gensum_param_desc *desc, gensum_param_access **first,
1469 : HOST_WIDE_INT offset, HOST_WIDE_INT size, isra_scan_context ctx)
1470 : {
1471 538488 : gensum_param_access *access = *first, **ptr = first;
1472 :
1473 538488 : if (!access)
1474 : {
1475 : /* No pre-existing access at this level, just create it. */
1476 292957 : gensum_param_access *a = allocate_access (desc, offset, size);
1477 292957 : if (!a)
1478 : return NULL;
1479 292957 : *first = a;
1480 292957 : return *first;
1481 : }
1482 :
1483 245531 : if (access->offset >= offset + size)
1484 : {
1485 : /* We want to squeeze it in front of the very first access, just do
1486 : it. */
1487 45595 : gensum_param_access *r = allocate_access (desc, offset, size);
1488 45595 : if (!r)
1489 : return NULL;
1490 45595 : r->next_sibling = access;
1491 45595 : *first = r;
1492 45595 : return r;
1493 : }
1494 :
1495 : /* Skip all accesses that have to come before us until the next sibling is
1496 : already too far. */
1497 292907 : while (offset >= access->offset + access->size
1498 193078 : && access->next_sibling
1499 398084 : && access->next_sibling->offset < offset + size)
1500 : {
1501 92971 : ptr = &access->next_sibling;
1502 92971 : access = access->next_sibling;
1503 : }
1504 :
1505 : /* At this point we know we do not belong before access. */
1506 199936 : gcc_assert (access->offset < offset + size);
1507 :
1508 199936 : if (access->offset == offset && access->size == size)
1509 : /* We found what we were looking for. */
1510 : return access;
1511 :
1512 126824 : if (access->offset <= offset
1513 123687 : && access->offset + access->size >= offset + size)
1514 : {
1515 : /* We fit into access which is larger than us. We need to find/create
1516 : something below access. But we only allow nesting in call
1517 : arguments. */
1518 11903 : if (access->nonarg)
1519 : return NULL;
1520 :
1521 10140 : return get_access_1 (desc, &access->first_child, offset, size, ctx);
1522 : }
1523 :
1524 114921 : if (offset <= access->offset
1525 14814 : && offset + size >= access->offset + access->size)
1526 : /* We are actually bigger than access, which fully fits into us, take its
1527 : place and make all accesses fitting into it its children. */
1528 : {
1529 : /* But first, we only allow nesting in call arguments so check if that is
1530 : what we are trying to represent. */
1531 14814 : if (ctx != ISRA_CTX_ARG)
1532 : return NULL;
1533 :
1534 11490 : gensum_param_access *r = allocate_access (desc, offset, size);
1535 11490 : if (!r)
1536 : return NULL;
1537 11490 : r->first_child = access;
1538 :
1539 11490 : while (access->next_sibling
1540 20202 : && access->next_sibling->offset < offset + size)
1541 : access = access->next_sibling;
1542 11490 : if (access->offset + access->size > offset + size)
1543 : {
1544 : /* This must be a different access, which are sorted, so the
1545 : following must be true and this signals a partial overlap. */
1546 0 : gcc_assert (access->offset > offset);
1547 : return NULL;
1548 : }
1549 :
1550 11490 : r->next_sibling = access->next_sibling;
1551 11490 : access->next_sibling = NULL;
1552 11490 : *ptr = r;
1553 11490 : return r;
1554 : }
1555 :
1556 100107 : if (offset >= access->offset + access->size)
1557 : {
1558 : /* We belong after access. */
1559 100107 : gensum_param_access *r = allocate_access (desc, offset, size);
1560 100107 : if (!r)
1561 : return NULL;
1562 100107 : r->next_sibling = access->next_sibling;
1563 100107 : access->next_sibling = r;
1564 100107 : return r;
1565 : }
1566 :
1567 0 : if (offset < access->offset)
1568 : {
1569 : /* We know the following, otherwise we would have created a
1570 : super-access. */
1571 0 : gcc_checking_assert (offset + size < access->offset + access->size);
1572 : return NULL;
1573 : }
1574 :
1575 0 : if (offset + size > access->offset + access->size)
1576 : {
1577 : /* Likewise. */
1578 0 : gcc_checking_assert (offset > access->offset);
1579 : return NULL;
1580 : }
1581 :
1582 0 : gcc_unreachable ();
1583 : }
1584 :
1585 : /* Return an access describing memory access to the variable described by DESC
1586 : at OFFSET with SIZE in context CTX, mark it as used in context CTX. Attempt
1587 : to create if it does not exist, but fail and return NULL if there are
1588 : already too many accesses, if it would create a partially overlapping access
1589 : or if an access would end up in a non-call access. */
1590 :
1591 : static gensum_param_access *
1592 528348 : get_access (gensum_param_desc *desc, HOST_WIDE_INT offset, HOST_WIDE_INT size,
1593 : isra_scan_context ctx)
1594 : {
1595 528348 : gcc_checking_assert (desc->split_candidate);
1596 :
1597 528348 : gensum_param_access *access = get_access_1 (desc, &desc->accesses, offset,
1598 : size, ctx);
1599 528348 : if (!access)
1600 : {
1601 5087 : disqualify_split_candidate (desc,
1602 : "Bad access overlap or too many accesses");
1603 5087 : return NULL;
1604 : }
1605 :
1606 523261 : switch (ctx)
1607 : {
1608 13341 : case ISRA_CTX_STORE:
1609 13341 : gcc_assert (!desc->by_ref);
1610 : /* Fall-through */
1611 434457 : case ISRA_CTX_LOAD:
1612 434457 : access->nonarg = true;
1613 434457 : break;
1614 : case ISRA_CTX_ARG:
1615 : break;
1616 : }
1617 :
1618 : return access;
1619 : }
1620 :
1621 : /* Verify that parameter access tree starting with ACCESS is in good shape.
1622 : PARENT_OFFSET and PARENT_SIZE are the corresponding fields of parent of
1623 : ACCESS or zero if there is none. */
1624 :
1625 : static bool
1626 944983 : verify_access_tree_1 (gensum_param_access *access, HOST_WIDE_INT parent_offset,
1627 : HOST_WIDE_INT parent_size)
1628 : {
1629 1345499 : while (access)
1630 : {
1631 400516 : gcc_assert (access->offset >= 0 && access->size >= 0);
1632 :
1633 400516 : if (parent_size != 0)
1634 : {
1635 22847 : if (access->offset < parent_offset)
1636 : {
1637 0 : error ("Access offset before parent offset");
1638 0 : return true;
1639 : }
1640 22847 : if (access->size >= parent_size)
1641 : {
1642 0 : error ("Access size greater or equal to its parent size");
1643 0 : return true;
1644 : }
1645 22847 : if (access->offset + access->size > parent_offset + parent_size)
1646 : {
1647 0 : error ("Access terminates outside of its parent");
1648 0 : return true;
1649 : }
1650 : }
1651 :
1652 400516 : if (verify_access_tree_1 (access->first_child, access->offset,
1653 : access->size))
1654 : return true;
1655 :
1656 400516 : if (access->next_sibling
1657 132871 : && (access->next_sibling->offset < access->offset + access->size))
1658 : {
1659 0 : error ("Access overlaps with its sibling");
1660 0 : return true;
1661 : }
1662 :
1663 400516 : access = access->next_sibling;
1664 : }
1665 : return false;
1666 : }
1667 :
1668 : /* Verify that parameter access tree starting with ACCESS is in good shape,
1669 : halt compilation and dump the tree to stderr if not. */
1670 :
1671 : DEBUG_FUNCTION void
1672 544467 : isra_verify_access_tree (gensum_param_access *access)
1673 : {
1674 544467 : if (verify_access_tree_1 (access, 0, 0))
1675 : {
1676 0 : for (; access; access = access->next_sibling)
1677 0 : dump_gensum_access (stderr, access, 2);
1678 0 : internal_error ("IPA-SRA access verification failed");
1679 : }
1680 544467 : }
1681 :
1682 :
1683 : /* Callback of walk_stmt_load_store_addr_ops visit_addr used to determine
1684 : GIMPLE_ASM operands with memory constrains which cannot be scalarized. */
1685 :
1686 : static bool
1687 10867 : asm_visit_addr (gimple *, tree op, tree, void *)
1688 : {
1689 10867 : op = get_base_address (op);
1690 10867 : if (op
1691 10867 : && TREE_CODE (op) == PARM_DECL)
1692 177 : disqualify_split_candidate (op, "Non-scalarizable GIMPLE_ASM operand.");
1693 :
1694 10867 : return false;
1695 : }
1696 :
1697 : /* Mark a dereference of parameter identified by DESC of distance DIST in a
1698 : basic block BB, unless the BB has already been marked as a potentially
1699 : final. */
1700 :
1701 : static void
1702 249561 : mark_param_dereference (gensum_param_desc *desc, HOST_WIDE_INT dist,
1703 : basic_block bb)
1704 : {
1705 249561 : gcc_assert (desc->by_ref);
1706 249561 : gcc_checking_assert (desc->split_candidate);
1707 :
1708 249561 : if (desc->safe_ref
1709 249561 : || bitmap_bit_p (final_bbs, bb->index))
1710 : return;
1711 :
1712 26098 : int idx = bb->index * unsafe_by_ref_count + desc->deref_index;
1713 26098 : if (bb_dereferences[idx] < dist)
1714 24804 : bb_dereferences[idx] = dist;
1715 : }
1716 :
1717 : /* Return true, if any potential replacements should use NEW_TYPE as opposed to
1718 : previously recorded OLD_TYPE. */
1719 :
1720 : static bool
1721 62786 : type_prevails_p (tree old_type, tree new_type)
1722 : {
1723 62786 : if (old_type == new_type)
1724 : return false;
1725 :
1726 : /* Non-aggregates are always better. */
1727 362 : if (!is_gimple_reg_type (old_type)
1728 362 : && is_gimple_reg_type (new_type))
1729 : return true;
1730 362 : if (is_gimple_reg_type (old_type)
1731 362 : && !is_gimple_reg_type (new_type))
1732 : return false;
1733 :
1734 : /* Prefer any complex or vector type over any other scalar type. */
1735 296 : if (TREE_CODE (old_type) != COMPLEX_TYPE
1736 296 : && TREE_CODE (old_type) != VECTOR_TYPE
1737 231 : && (TREE_CODE (new_type) == COMPLEX_TYPE
1738 231 : || VECTOR_TYPE_P (new_type)))
1739 : return true;
1740 296 : if ((TREE_CODE (old_type) == COMPLEX_TYPE
1741 : || VECTOR_TYPE_P (old_type))
1742 65 : && TREE_CODE (new_type) != COMPLEX_TYPE
1743 52 : && TREE_CODE (new_type) != VECTOR_TYPE)
1744 : return false;
1745 :
1746 : /* Use the integral type with the bigger precision. */
1747 296 : if (INTEGRAL_TYPE_P (old_type)
1748 12 : && INTEGRAL_TYPE_P (new_type))
1749 12 : return (TYPE_PRECISION (new_type) > TYPE_PRECISION (old_type));
1750 :
1751 : /* Attempt to disregard any integral type with non-full precision. */
1752 284 : if (INTEGRAL_TYPE_P (old_type)
1753 284 : && (TREE_INT_CST_LOW (TYPE_SIZE (old_type))
1754 0 : != TYPE_PRECISION (old_type)))
1755 : return true;
1756 284 : if (INTEGRAL_TYPE_P (new_type)
1757 284 : && (TREE_INT_CST_LOW (TYPE_SIZE (new_type))
1758 6 : != TYPE_PRECISION (new_type)))
1759 : return false;
1760 : /* Stabilize the selection. */
1761 284 : return TYPE_UID (old_type) < TYPE_UID (new_type);
1762 : }
1763 :
1764 : /* When scanning an expression which is a call argument, this structure
1765 : specifies the call and the position of the argument. */
1766 :
1767 : struct scan_call_info
1768 : {
1769 : /* Call graph edge representing the call. */
1770 : cgraph_edge *cs;
1771 : /* Total number of arguments in the call. */
1772 : unsigned argument_count;
1773 : /* Number of the actual argument being scanned. */
1774 : unsigned arg_idx;
1775 : };
1776 :
1777 : /* Record use of ACCESS which belongs to a parameter described by DESC in a
1778 : call argument described by CALL_INFO. */
1779 :
1780 : static void
1781 85800 : record_nonregister_call_use (gensum_param_desc *desc,
1782 : scan_call_info *call_info,
1783 : unsigned unit_offset, unsigned unit_size)
1784 : {
1785 85800 : isra_call_summary *csum = call_sums->get_create (call_info->cs);
1786 85800 : csum->init_inputs (call_info->argument_count);
1787 :
1788 85800 : isra_param_flow *param_flow = &csum->m_arg_flow[call_info->arg_idx];
1789 85800 : param_flow->aggregate_pass_through = true;
1790 85800 : set_single_param_flow_source (param_flow, desc->param_number);
1791 85800 : param_flow->unit_offset = unit_offset;
1792 85800 : param_flow->unit_size = unit_size;
1793 85800 : desc->call_uses++;
1794 85800 : }
1795 :
1796 : /* Callback of walk_aliased_vdefs, just mark that there was a possible
1797 : modification. */
1798 :
1799 : static bool
1800 66256 : mark_maybe_modified (ao_ref *, tree, void *data)
1801 : {
1802 66256 : bool *maybe_modified = (bool *) data;
1803 66256 : *maybe_modified = true;
1804 66256 : return true;
1805 : }
1806 :
1807 : /* Analyze expression EXPR from GIMPLE for accesses to parameters. CTX
1808 : specifies whether EXPR is used in a load, store or as an argument call. BB
1809 : must be the basic block in which expr resides. If CTX specifies call
1810 : argument context, CALL_INFO must describe that call and argument position,
1811 : otherwise it is ignored. */
1812 :
1813 : static void
1814 37148668 : scan_expr_access (tree expr, gimple *stmt, isra_scan_context ctx,
1815 : basic_block bb, scan_call_info *call_info = NULL)
1816 : {
1817 37148668 : poly_int64 poffset, psize, pmax_size;
1818 37148668 : HOST_WIDE_INT offset, size, max_size;
1819 37148668 : tree base;
1820 37148668 : bool deref = false;
1821 37148668 : bool reverse;
1822 :
1823 37148668 : if (TREE_CODE (expr) == ADDR_EXPR)
1824 : {
1825 4606097 : if (ctx == ISRA_CTX_ARG)
1826 : return;
1827 1265240 : tree t = get_base_address (TREE_OPERAND (expr, 0));
1828 1265240 : if (VAR_P (t) && !TREE_STATIC (t))
1829 247510 : loaded_decls->add (t);
1830 : return;
1831 : }
1832 32542571 : if (TREE_CODE (expr) == SSA_NAME
1833 18074750 : || CONSTANT_CLASS_P (expr))
1834 : return;
1835 :
1836 12971189 : if (TREE_CODE (expr) == BIT_FIELD_REF
1837 12971189 : || TREE_CODE (expr) == IMAGPART_EXPR
1838 12825890 : || TREE_CODE (expr) == REALPART_EXPR)
1839 210837 : expr = TREE_OPERAND (expr, 0);
1840 :
1841 12971189 : base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size, &reverse);
1842 :
1843 12971189 : if (TREE_CODE (base) == MEM_REF)
1844 : {
1845 4074818 : tree op = TREE_OPERAND (base, 0);
1846 4074818 : if (TREE_CODE (op) != SSA_NAME
1847 4074818 : || !SSA_NAME_IS_DEFAULT_DEF (op))
1848 : return;
1849 2144206 : base = SSA_NAME_VAR (op);
1850 2144206 : if (!base)
1851 : return;
1852 : deref = true;
1853 : }
1854 8896371 : else if (VAR_P (base)
1855 7679713 : && !TREE_STATIC (base)
1856 6618087 : && (ctx == ISRA_CTX_ARG
1857 : || ctx == ISRA_CTX_LOAD))
1858 2476573 : loaded_decls->add (base);
1859 :
1860 11040577 : if (TREE_CODE (base) != PARM_DECL)
1861 : return;
1862 :
1863 2516890 : gensum_param_desc *desc = get_gensum_param_desc (base);
1864 2516890 : if (!desc || !desc->split_candidate)
1865 : return;
1866 :
1867 595750 : if (storage_order_barrier_p (expr))
1868 : {
1869 0 : disqualify_split_candidate (desc, "Encountered a storage order barrier.");
1870 0 : return;
1871 : }
1872 :
1873 595750 : if (!poffset.is_constant (&offset)
1874 595750 : || !psize.is_constant (&size)
1875 595750 : || !pmax_size.is_constant (&max_size))
1876 : {
1877 : disqualify_split_candidate (desc, "Encountered a polynomial-sized "
1878 : "access.");
1879 : return;
1880 : }
1881 595750 : if (size < 0 || size != max_size)
1882 : {
1883 6583 : disqualify_split_candidate (desc, "Encountered a variable sized access.");
1884 6583 : return;
1885 : }
1886 589167 : if (TREE_CODE (expr) == COMPONENT_REF
1887 589167 : && DECL_BIT_FIELD (TREE_OPERAND (expr, 1)))
1888 : {
1889 939 : disqualify_split_candidate (desc, "Encountered a bit-field access.");
1890 939 : return;
1891 : }
1892 588228 : if (offset < 0)
1893 : {
1894 4 : disqualify_split_candidate (desc, "Encountered an access at a "
1895 : "negative offset.");
1896 4 : return;
1897 : }
1898 588224 : gcc_assert ((offset % BITS_PER_UNIT) == 0);
1899 588224 : gcc_assert ((size % BITS_PER_UNIT) == 0);
1900 588224 : if ((offset / BITS_PER_UNIT) >= (UINT_MAX - ISRA_ARG_SIZE_LIMIT)
1901 588224 : || (size / BITS_PER_UNIT) >= ISRA_ARG_SIZE_LIMIT)
1902 : {
1903 0 : disqualify_split_candidate (desc, "Encountered an access with too big "
1904 : "offset or size");
1905 0 : return;
1906 : }
1907 :
1908 588224 : tree type = TREE_TYPE (expr);
1909 588224 : unsigned int exp_align = get_object_alignment (expr);
1910 :
1911 588224 : if (exp_align < TYPE_ALIGN (type))
1912 : {
1913 1397 : disqualify_split_candidate (desc, "Underaligned access.");
1914 1397 : return;
1915 : }
1916 :
1917 586827 : if (deref)
1918 : {
1919 308040 : if (!desc->by_ref)
1920 : {
1921 0 : disqualify_split_candidate (desc, "Dereferencing a non-reference.");
1922 58479 : return;
1923 : }
1924 308040 : else if (ctx == ISRA_CTX_STORE)
1925 : {
1926 0 : disqualify_split_candidate (desc, "Storing to data passed by "
1927 : "reference.");
1928 0 : return;
1929 : }
1930 :
1931 308040 : if (!aa_walking_limit)
1932 : {
1933 0 : disqualify_split_candidate (desc, "Out of alias analysis step "
1934 : "limit.");
1935 0 : return;
1936 : }
1937 :
1938 616080 : gcc_checking_assert (gimple_vuse (stmt));
1939 308040 : bool maybe_modified = false;
1940 308040 : ao_ref ar;
1941 :
1942 308040 : ao_ref_init (&ar, expr);
1943 308040 : bitmap visited = BITMAP_ALLOC (NULL);
1944 616080 : int walked = walk_aliased_vdefs (&ar, gimple_vuse (stmt),
1945 : mark_maybe_modified, &maybe_modified,
1946 : &visited, NULL, aa_walking_limit);
1947 308040 : BITMAP_FREE (visited);
1948 308040 : if (walked > 0)
1949 : {
1950 102440 : gcc_assert (aa_walking_limit > walked);
1951 102440 : aa_walking_limit = aa_walking_limit - walked;
1952 : }
1953 308040 : if (walked < 0)
1954 0 : aa_walking_limit = 0;
1955 308040 : if (maybe_modified || walked < 0)
1956 : {
1957 58479 : disqualify_split_candidate (desc, "Data passed by reference possibly "
1958 : "modified through an alias.");
1959 58479 : return;
1960 : }
1961 : else
1962 249561 : mark_param_dereference (desc, offset + size, bb);
1963 : }
1964 : else
1965 : /* Pointer parameters with direct uses should have been ruled out by
1966 : analyzing SSA default def when looking at the parameters. */
1967 278787 : gcc_assert (!desc->by_ref);
1968 :
1969 528348 : gensum_param_access *access = get_access (desc, offset, size, ctx);
1970 528348 : if (!access)
1971 : return;
1972 :
1973 523261 : if (ctx == ISRA_CTX_ARG)
1974 : {
1975 88804 : gcc_checking_assert (call_info);
1976 :
1977 88804 : if (desc->param_number > UCHAR_MAX)
1978 : {
1979 3 : disqualify_split_candidate (desc, "Caller parameter with a large "
1980 : "index passed in a call.");
1981 3 : return;
1982 : }
1983 :
1984 88801 : if (!deref)
1985 85800 : record_nonregister_call_use (desc, call_info, offset / BITS_PER_UNIT,
1986 85800 : size / BITS_PER_UNIT);
1987 : else
1988 : /* This is not a pass-through of a pointer, this is a use like any
1989 : other. */
1990 3001 : access->nonarg = true;
1991 : }
1992 434457 : else if (ctx == ISRA_CTX_LOAD && bb->count.initialized_p ())
1993 421028 : access->load_count += bb->count;
1994 :
1995 523258 : if (!access->type)
1996 : {
1997 450146 : access->type = type;
1998 450146 : access->alias_ptr_type = reference_alias_ptr_type (expr);
1999 450146 : access->reverse = reverse;
2000 : }
2001 : else
2002 : {
2003 73112 : if (exp_align < TYPE_ALIGN (access->type))
2004 : {
2005 0 : disqualify_split_candidate (desc, "Reference has lower alignment "
2006 : "than a previous one.");
2007 0 : return;
2008 : }
2009 73112 : if (access->alias_ptr_type != reference_alias_ptr_type (expr))
2010 : {
2011 4572 : disqualify_split_candidate (desc, "Multiple alias pointer types.");
2012 4572 : return;
2013 : }
2014 68540 : if (access->reverse != reverse)
2015 : {
2016 0 : disqualify_split_candidate (desc, "Both normal and reverse "
2017 : "scalar storage order.");
2018 0 : return;
2019 : }
2020 68540 : if (!deref
2021 52587 : && (AGGREGATE_TYPE_P (type) || AGGREGATE_TYPE_P (access->type))
2022 107121 : && (TYPE_MAIN_VARIANT (access->type) != TYPE_MAIN_VARIANT (type)))
2023 : {
2024 : /* We need the same aggregate type on all accesses to be able to
2025 : distinguish transformation spots from pass-through arguments in
2026 : the transformation phase. */
2027 5754 : disqualify_split_candidate (desc, "We do not support aggregate "
2028 : "type punning.");
2029 5754 : return;
2030 : }
2031 :
2032 62786 : if (type_prevails_p (access->type, type))
2033 140 : access->type = type;
2034 : }
2035 : }
2036 :
2037 : /* Scan body function described by NODE and FUN and create access trees for
2038 : parameters. */
2039 :
2040 : static void
2041 1304084 : scan_function (cgraph_node *node, struct function *fun)
2042 : {
2043 1304084 : basic_block bb;
2044 :
2045 10565861 : FOR_EACH_BB_FN (bb, fun)
2046 : {
2047 9261777 : gimple_stmt_iterator gsi;
2048 74392693 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2049 : {
2050 55869139 : gimple *stmt = gsi_stmt (gsi);
2051 :
2052 55869139 : if (final_bbs && stmt_can_throw_external (fun, stmt))
2053 1372560 : bitmap_set_bit (final_bbs, bb->index);
2054 55869139 : switch (gimple_code (stmt))
2055 : {
2056 1270821 : case GIMPLE_RETURN:
2057 1270821 : {
2058 1270821 : tree t = gimple_return_retval (as_a <greturn *> (stmt));
2059 1270821 : if (t != NULL_TREE)
2060 700268 : scan_expr_access (t, stmt, ISRA_CTX_LOAD, bb);
2061 1270821 : if (final_bbs)
2062 423163 : bitmap_set_bit (final_bbs, bb->index);
2063 : }
2064 : break;
2065 :
2066 18860213 : case GIMPLE_ASSIGN:
2067 18860213 : if (gimple_assign_single_p (stmt)
2068 18860213 : && !gimple_clobber_p (stmt))
2069 : {
2070 11477179 : tree rhs = gimple_assign_rhs1 (stmt);
2071 11477179 : scan_expr_access (rhs, stmt, ISRA_CTX_LOAD, bb);
2072 11477179 : tree lhs = gimple_assign_lhs (stmt);
2073 11477179 : scan_expr_access (lhs, stmt, ISRA_CTX_STORE, bb);
2074 : }
2075 : break;
2076 :
2077 5513550 : case GIMPLE_CALL:
2078 5513550 : {
2079 5513550 : unsigned argument_count = gimple_call_num_args (stmt);
2080 5513550 : isra_scan_context ctx = ISRA_CTX_ARG;
2081 5513550 : scan_call_info call_info, *call_info_p = &call_info;
2082 5513550 : if (gimple_call_internal_p (stmt)
2083 5513550 : && !callback_ifn_assume_call_with_edge_p (node, stmt))
2084 : {
2085 189491 : call_info_p = NULL;
2086 189491 : ctx = ISRA_CTX_LOAD;
2087 189491 : internal_fn ifn = gimple_call_internal_fn (stmt);
2088 189491 : if (internal_store_fn_p (ifn))
2089 0 : ctx = ISRA_CTX_STORE;
2090 : }
2091 : else
2092 : {
2093 5324059 : call_info.cs = node->get_edge (stmt);
2094 5324059 : call_info.argument_count = argument_count;
2095 : }
2096 :
2097 16697872 : for (unsigned i = 0; i < argument_count; i++)
2098 : {
2099 11184322 : call_info.arg_idx = i;
2100 11184322 : scan_expr_access (gimple_call_arg (stmt, i), stmt,
2101 : ctx, bb, call_info_p);
2102 : }
2103 :
2104 5513550 : tree lhs = gimple_call_lhs (stmt);
2105 5513550 : if (lhs)
2106 2214369 : scan_expr_access (lhs, stmt, ISRA_CTX_STORE, bb);
2107 5513550 : int flags = gimple_call_flags (stmt);
2108 5513550 : if (final_bbs
2109 2162997 : && (((flags & (ECF_CONST | ECF_PURE)) == 0)
2110 357421 : || (flags & ECF_LOOPING_CONST_OR_PURE)))
2111 1865277 : bitmap_set_bit (final_bbs, bb->index);
2112 : }
2113 5513550 : break;
2114 :
2115 44197 : case GIMPLE_ASM:
2116 44197 : {
2117 44197 : gasm *asm_stmt = as_a <gasm *> (stmt);
2118 44197 : walk_stmt_load_store_addr_ops (asm_stmt, NULL, NULL, NULL,
2119 : asm_visit_addr);
2120 44197 : if (final_bbs)
2121 1931 : bitmap_set_bit (final_bbs, bb->index);
2122 :
2123 83919 : for (unsigned i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
2124 : {
2125 39722 : tree t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
2126 39722 : scan_expr_access (t, stmt, ISRA_CTX_LOAD, bb);
2127 : }
2128 99826 : for (unsigned i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
2129 : {
2130 55629 : tree t = TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
2131 55629 : scan_expr_access (t, stmt, ISRA_CTX_STORE, bb);
2132 : }
2133 : }
2134 : break;
2135 :
2136 : default:
2137 : break;
2138 : }
2139 : }
2140 : }
2141 1304084 : }
2142 :
2143 : /* Return true if SSA_NAME NAME of function described by FUN is only used in
2144 : return statements, or if results of any operations it is involved in are
2145 : only used in return statements. ANALYZED is a bitmap that tracks which SSA
2146 : names we have already started investigating. */
2147 :
2148 : static bool
2149 2433078 : ssa_name_only_returned_p (function *fun, tree name, bitmap analyzed)
2150 : {
2151 2433078 : bool res = true;
2152 2433078 : imm_use_iterator imm_iter;
2153 2433078 : gimple *stmt;
2154 :
2155 3088723 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
2156 : {
2157 2660843 : if (is_gimple_debug (stmt))
2158 185229 : continue;
2159 :
2160 2475614 : if (gimple_code (stmt) == GIMPLE_RETURN)
2161 : {
2162 254148 : tree t = gimple_return_retval (as_a <greturn *> (stmt));
2163 254148 : if (t != name)
2164 : {
2165 : res = false;
2166 : break;
2167 : }
2168 : }
2169 2221466 : else if (!stmt_unremovable_because_of_non_call_eh_p (fun, stmt)
2170 2221466 : && ((is_gimple_assign (stmt) && !gimple_has_volatile_ops (stmt))
2171 1213245 : || gimple_code (stmt) == GIMPLE_PHI))
2172 : {
2173 : /* TODO: And perhaps for const function calls too? */
2174 1121458 : tree lhs;
2175 1121458 : if (gimple_code (stmt) == GIMPLE_PHI)
2176 209193 : lhs = gimple_phi_result (stmt);
2177 : else
2178 912265 : lhs = gimple_assign_lhs (stmt);
2179 :
2180 1121458 : if (TREE_CODE (lhs) != SSA_NAME)
2181 : {
2182 : res = false;
2183 : break;
2184 : }
2185 786045 : gcc_assert (!gimple_vdef (stmt));
2186 786045 : if (bitmap_set_bit (analyzed, SSA_NAME_VERSION (lhs))
2187 786045 : && !ssa_name_only_returned_p (fun, lhs, analyzed))
2188 : {
2189 : res = false;
2190 : break;
2191 : }
2192 : }
2193 : else
2194 : {
2195 : res = false;
2196 : break;
2197 : }
2198 2433078 : }
2199 2433078 : return res;
2200 : }
2201 :
2202 : /* Inspect the uses of the return value of the call associated with CS, and if
2203 : it is not used or if it is only used to construct the return value of the
2204 : caller, mark it as such in call or caller summary. Also check for
2205 : misaligned arguments. */
2206 :
2207 : static void
2208 5337640 : isra_analyze_call (cgraph_edge *cs)
2209 : {
2210 5337640 : gcall *call_stmt = cs->call_stmt;
2211 5337640 : unsigned count = gimple_call_num_args (call_stmt);
2212 5337640 : isra_call_summary *csum = call_sums->get_create (cs);
2213 :
2214 21306966 : for (unsigned i = 0; i < count; i++)
2215 : {
2216 10631686 : tree arg = gimple_call_arg (call_stmt, i);
2217 10631686 : if (TREE_CODE (arg) == ADDR_EXPR)
2218 : {
2219 3365989 : poly_int64 poffset, psize, pmax_size;
2220 3365989 : bool reverse;
2221 3365989 : tree base = get_ref_base_and_extent (TREE_OPERAND (arg, 0), &poffset,
2222 3365989 : &psize, &pmax_size, &reverse);
2223 3365989 : HOST_WIDE_INT offset;
2224 3365989 : unsigned HOST_WIDE_INT ds;
2225 3365989 : if (DECL_P (base)
2226 2400664 : && (poffset.is_constant (&offset))
2227 2400664 : && tree_fits_uhwi_p (DECL_SIZE (base))
2228 5620674 : && ((ds = tree_to_uhwi (DECL_SIZE (base)) - offset)
2229 : < ISRA_ARG_SIZE_LIMIT * BITS_PER_UNIT))
2230 : {
2231 2121425 : csum->init_inputs (count);
2232 2121425 : gcc_assert (!csum->m_arg_flow[i].aggregate_pass_through);
2233 2121425 : csum->m_arg_flow[i].unit_size = ds / BITS_PER_UNIT;
2234 : }
2235 :
2236 3365989 : if (TREE_CODE (base) == VAR_DECL
2237 2212739 : && !TREE_STATIC (base)
2238 4820018 : && !loaded_decls->contains (base))
2239 : {
2240 966438 : csum->init_inputs (count);
2241 966438 : csum->m_arg_flow[i].constructed_for_calls = true;
2242 : }
2243 : }
2244 :
2245 10631686 : if (is_gimple_reg (arg))
2246 4176977 : continue;
2247 :
2248 6454709 : tree offset;
2249 6454709 : poly_int64 bitsize, bitpos;
2250 6454709 : machine_mode mode;
2251 6454709 : int unsignedp, reversep, volatilep = 0;
2252 6454709 : get_inner_reference (arg, &bitsize, &bitpos, &offset, &mode,
2253 : &unsignedp, &reversep, &volatilep);
2254 12909418 : if (!multiple_p (bitpos, BITS_PER_UNIT))
2255 : {
2256 0 : csum->m_bit_aligned_arg = true;
2257 0 : break;
2258 : }
2259 : }
2260 :
2261 5337640 : tree lhs = gimple_call_lhs (call_stmt);
2262 5337640 : if (lhs)
2263 : {
2264 : /* TODO: Also detect aggregates on a LHS of a call that are only returned
2265 : from this function (without being read anywhere). */
2266 2043125 : if (TREE_CODE (lhs) == SSA_NAME)
2267 : {
2268 1668393 : bitmap analyzed = BITMAP_ALLOC (NULL);
2269 1668393 : if (ssa_name_only_returned_p (DECL_STRUCT_FUNCTION (cs->caller->decl),
2270 : lhs, analyzed))
2271 232972 : csum->m_return_returned = true;
2272 1668393 : BITMAP_FREE (analyzed);
2273 : }
2274 : }
2275 :
2276 : /* The return value of an ASSUME body is not ignored, but we cannot detect
2277 : this, as the function is never called directly. */
2278 3294515 : else if (cs->callee && cs->callee->assume_function)
2279 166 : csum->m_return_ignored = false;
2280 : /* Don't set m_return_ignored for musttail calls. The tailc/musttail passes
2281 : compare the returned value against the IPA-VRP return value range if
2282 : it is a singleton, but if the call is changed to something which doesn't
2283 : return anything, it will always fail. */
2284 3294349 : else if (!gimple_call_must_tail_p (call_stmt))
2285 3294248 : csum->m_return_ignored = true;
2286 5337640 : }
2287 :
2288 : /* Look at all calls going out of NODE, described also by IFS and perform all
2289 : analyses necessary for IPA-SRA that are not done at body scan time or done
2290 : even when body is not scanned because the function is not a candidate. */
2291 :
2292 : static void
2293 1304084 : isra_analyze_all_outgoing_calls (cgraph_node *node)
2294 : {
2295 6504870 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
2296 5200786 : isra_analyze_call (cs);
2297 1440938 : for (cgraph_edge *cs = node->indirect_calls; cs; cs = cs->next_callee)
2298 136854 : isra_analyze_call (cs);
2299 1304084 : }
2300 :
2301 : /* Dump a dereferences table with heading STR to file F. */
2302 :
2303 : static void
2304 42 : dump_dereferences_table (FILE *f, struct function *fun, const char *str)
2305 : {
2306 42 : basic_block bb;
2307 :
2308 42 : fprintf (dump_file, "%s", str);
2309 196 : FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun),
2310 : EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
2311 : {
2312 154 : fprintf (f, "%4i %i ", bb->index, bitmap_bit_p (final_bbs, bb->index));
2313 154 : if (bb != EXIT_BLOCK_PTR_FOR_FN (fun))
2314 : {
2315 : int i;
2316 408 : for (i = 0; i < unsafe_by_ref_count; i++)
2317 : {
2318 254 : int idx = bb->index * unsafe_by_ref_count + i;
2319 254 : fprintf (f, " %4" HOST_WIDE_INT_PRINT "d", bb_dereferences[idx]);
2320 : }
2321 : }
2322 154 : fprintf (f, "\n");
2323 : }
2324 42 : fprintf (dump_file, "\n");
2325 42 : }
2326 :
2327 : /* Propagate distances in bb_dereferences in the opposite direction than the
2328 : control flow edges, in each step storing the maximum of the current value
2329 : and the minimum of all successors. These steps are repeated until the table
2330 : stabilizes. Note that BBs which might terminate the functions (according to
2331 : final_bbs bitmap) never updated in this way. */
2332 :
2333 : static void
2334 27106 : propagate_dereference_distances (struct function *fun)
2335 : {
2336 27106 : basic_block bb;
2337 :
2338 27106 : if (dump_file && (dump_flags & TDF_DETAILS))
2339 21 : dump_dereferences_table (dump_file, fun,
2340 : "Dereference table before propagation:\n");
2341 :
2342 27106 : auto_vec<basic_block> queue (last_basic_block_for_fn (fun));
2343 27106 : queue.quick_push (ENTRY_BLOCK_PTR_FOR_FN (fun));
2344 226902 : FOR_EACH_BB_FN (bb, fun)
2345 : {
2346 199796 : queue.quick_push (bb);
2347 199796 : bb->aux = bb;
2348 : }
2349 :
2350 254901 : while (!queue.is_empty ())
2351 : {
2352 227795 : edge_iterator ei;
2353 227795 : edge e;
2354 227795 : bool change = false;
2355 227795 : int i;
2356 :
2357 227795 : bb = queue.pop ();
2358 227795 : bb->aux = NULL;
2359 :
2360 227795 : if (bitmap_bit_p (final_bbs, bb->index))
2361 73680 : continue;
2362 :
2363 6228235 : for (i = 0; i < unsafe_by_ref_count; i++)
2364 : {
2365 6074120 : int idx = bb->index * unsafe_by_ref_count + i;
2366 6074120 : bool first = true;
2367 6074120 : HOST_WIDE_INT inh = 0;
2368 :
2369 14383752 : FOR_EACH_EDGE (e, ei, bb->succs)
2370 : {
2371 8309632 : int succ_idx = e->dest->index * unsafe_by_ref_count + i;
2372 :
2373 8309632 : if (e->dest == EXIT_BLOCK_PTR_FOR_FN (fun))
2374 0 : continue;
2375 :
2376 8309632 : if (first)
2377 : {
2378 6074115 : first = false;
2379 6074115 : inh = bb_dereferences [succ_idx];
2380 : }
2381 2235517 : else if (bb_dereferences [succ_idx] < inh)
2382 : inh = bb_dereferences [succ_idx];
2383 : }
2384 :
2385 6074120 : if (!first && bb_dereferences[idx] < inh)
2386 : {
2387 10922 : bb_dereferences[idx] = inh;
2388 10922 : change = true;
2389 : }
2390 : }
2391 :
2392 154115 : if (change)
2393 11875 : FOR_EACH_EDGE (e, ei, bb->preds)
2394 : {
2395 1882 : if (e->src->aux)
2396 989 : continue;
2397 :
2398 893 : e->src->aux = e->src;
2399 893 : queue.quick_push (e->src);
2400 : }
2401 : }
2402 :
2403 27106 : if (dump_file && (dump_flags & TDF_DETAILS))
2404 21 : dump_dereferences_table (dump_file, fun,
2405 : "Dereference table after propagation:\n");
2406 27106 : }
2407 :
2408 : /* Return true if the ACCESS loads happen frequently enough in FUN to risk
2409 : moving them to the caller and only pass the result. */
2410 :
2411 : static bool
2412 179154 : dereference_probable_p (struct function *fun, gensum_param_access *access)
2413 : {
2414 179154 : int threshold = opt_for_fn (fun->decl, param_ipa_sra_deref_prob_threshold);
2415 179154 : return access->load_count
2416 179154 : >= ENTRY_BLOCK_PTR_FOR_FN (fun)->count.apply_scale (threshold, 100);
2417 : }
2418 :
2419 : /* Perform basic checks on ACCESS to PARM (of FUN) described by DESC and all
2420 : its children, return true if the parameter cannot be split, otherwise return
2421 : false and update *NONARG_ACC_SIZE and *ONLY_CALLS. ENTRY_BB_INDEX must be
2422 : the index of the entry BB in the function of PARM. */
2423 :
2424 : static bool
2425 393027 : check_gensum_access (struct function *fun, tree parm, gensum_param_desc *desc,
2426 : gensum_param_access *access,
2427 : HOST_WIDE_INT *nonarg_acc_size, bool *only_calls,
2428 : int entry_bb_index)
2429 : {
2430 393027 : if (access->nonarg)
2431 : {
2432 340606 : *only_calls = false;
2433 340606 : *nonarg_acc_size += access->size;
2434 :
2435 340606 : if (access->first_child)
2436 : {
2437 384 : disqualify_split_candidate (desc, "Overlapping non-call uses.");
2438 384 : return true;
2439 : }
2440 : }
2441 : /* Do not decompose a non-BLKmode param in a way that would create
2442 : BLKmode params. Especially for by-reference passing (thus,
2443 : pointer-type param) this is hardly worthwhile. */
2444 392643 : if (DECL_MODE (parm) != BLKmode
2445 392643 : && TYPE_MODE (access->type) == BLKmode)
2446 : {
2447 3586 : disqualify_split_candidate (desc, "Would convert a non-BLK to a BLK.");
2448 3586 : return true;
2449 : }
2450 :
2451 389057 : if (desc->by_ref)
2452 : {
2453 189941 : if (desc->safe_ref)
2454 : {
2455 154238 : if (!dereference_probable_p (fun, access))
2456 : {
2457 6417 : disqualify_split_candidate (desc, "Dereferences in callers "
2458 : "would happen much more frequently.");
2459 6417 : return true;
2460 : }
2461 : }
2462 : else
2463 : {
2464 35703 : int idx = (entry_bb_index * unsafe_by_ref_count + desc->deref_index);
2465 35703 : if ((access->offset + access->size) > bb_dereferences[idx])
2466 : {
2467 24916 : if (!dereference_probable_p (fun, access))
2468 : {
2469 2218 : disqualify_split_candidate (desc, "Would create a possibly "
2470 : "illegal dereference in a "
2471 : "caller.");
2472 2218 : return true;
2473 : }
2474 22698 : desc->conditionally_dereferenceable = true;
2475 : }
2476 : }
2477 : }
2478 :
2479 380422 : for (gensum_param_access *ch = access->first_child;
2480 402544 : ch;
2481 22122 : ch = ch->next_sibling)
2482 22124 : if (check_gensum_access (fun, parm, desc, ch, nonarg_acc_size, only_calls,
2483 : entry_bb_index))
2484 : return true;
2485 :
2486 : return false;
2487 : }
2488 :
2489 : /* Copy data from FROM and all of its children to a vector of accesses in IPA
2490 : descriptor DESC. */
2491 :
2492 : static void
2493 450149 : copy_accesses_to_ipa_desc (gensum_param_access *from, isra_param_desc *desc)
2494 : {
2495 450149 : param_access *to = ggc_cleared_alloc<param_access> ();
2496 450149 : gcc_checking_assert ((from->offset % BITS_PER_UNIT) == 0);
2497 450149 : gcc_checking_assert ((from->size % BITS_PER_UNIT) == 0);
2498 450149 : to->unit_offset = from->offset / BITS_PER_UNIT;
2499 450149 : to->unit_size = from->size / BITS_PER_UNIT;
2500 450149 : to->type = from->type;
2501 450149 : to->alias_ptr_type = from->alias_ptr_type;
2502 450149 : to->certain = from->nonarg;
2503 450149 : to->reverse = from->reverse;
2504 450149 : vec_safe_push (desc->accesses, to);
2505 :
2506 450149 : for (gensum_param_access *ch = from->first_child;
2507 474398 : ch;
2508 24249 : ch = ch->next_sibling)
2509 24249 : copy_accesses_to_ipa_desc (ch, desc);
2510 450149 : }
2511 :
2512 : /* Analyze function body scan results stored in param_accesses and
2513 : param_accesses, detect possible transformations and store information of
2514 : those in function summary. NODE, FUN and IFS are all various structures
2515 : describing the currently analyzed function. */
2516 :
2517 : static void
2518 861037 : process_scan_results (cgraph_node *node, struct function *fun,
2519 : isra_func_summary *ifs,
2520 : vec<gensum_param_desc> *param_descriptions)
2521 : {
2522 861037 : bool check_pass_throughs = false;
2523 861037 : bool dereferences_propagated = false;
2524 861037 : tree parm = DECL_ARGUMENTS (node->decl);
2525 861037 : unsigned param_count = param_descriptions->length();
2526 :
2527 861037 : for (unsigned desc_index = 0;
2528 3308445 : desc_index < param_count;
2529 2447408 : desc_index++, parm = DECL_CHAIN (parm))
2530 : {
2531 2447408 : gensum_param_desc *desc = &(*param_descriptions)[desc_index];
2532 2447408 : if (!desc->split_candidate)
2533 1915540 : continue;
2534 :
2535 544473 : if (flag_checking)
2536 544467 : isra_verify_access_tree (desc->accesses);
2537 :
2538 544473 : if (!dereferences_propagated
2539 538606 : && desc->by_ref
2540 372657 : && !desc->safe_ref
2541 166058 : && desc->accesses)
2542 : {
2543 27106 : propagate_dereference_distances (fun);
2544 27106 : dereferences_propagated = true;
2545 : }
2546 :
2547 544473 : HOST_WIDE_INT nonarg_acc_size = 0;
2548 544473 : bool only_calls = true;
2549 544473 : bool check_failed = false;
2550 :
2551 544473 : int entry_bb_index = ENTRY_BLOCK_PTR_FOR_FN (fun)->index;
2552 544473 : for (gensum_param_access *acc = desc->accesses;
2553 902771 : acc;
2554 358298 : acc = acc->next_sibling)
2555 370903 : if (check_gensum_access (fun, parm, desc, acc, &nonarg_acc_size,
2556 : &only_calls, entry_bb_index))
2557 : {
2558 : check_failed = true;
2559 : break;
2560 : }
2561 544473 : if (check_failed)
2562 12605 : continue;
2563 :
2564 531868 : if (only_calls)
2565 329515 : desc->locally_unused = true;
2566 :
2567 531868 : HOST_WIDE_INT cur_param_size
2568 531868 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm)));
2569 531868 : HOST_WIDE_INT param_size_limit, optimistic_limit;
2570 531868 : if (!desc->by_ref || optimize_function_for_size_p (fun))
2571 : {
2572 : param_size_limit = cur_param_size;
2573 : optimistic_limit = cur_param_size;
2574 : }
2575 : else
2576 : {
2577 342878 : param_size_limit
2578 342878 : = opt_for_fn (node->decl,
2579 : param_ipa_sra_ptr_growth_factor) * cur_param_size;
2580 342878 : optimistic_limit
2581 342878 : = (opt_for_fn (node->decl, param_ipa_sra_ptrwrap_growth_factor)
2582 : * param_size_limit);
2583 : }
2584 :
2585 : /* Assume functions have no real replacements, no calls to them will make
2586 : it into the binary, so limits don't matter. */
2587 531868 : if (!node->assume_function
2588 531827 : && (nonarg_acc_size > optimistic_limit
2589 529331 : || (!desc->by_ref && nonarg_acc_size == param_size_limit)))
2590 : {
2591 73165 : disqualify_split_candidate (desc, "Would result into a too big set "
2592 : "of replacements even in best "
2593 : "scenarios.");
2594 : }
2595 : else
2596 : {
2597 : /* create_parameter_descriptors makes sure unit sizes of all
2598 : candidate parameters fit unsigned integers restricted to
2599 : ISRA_ARG_SIZE_LIMIT. */
2600 458703 : desc->param_size_limit = param_size_limit / BITS_PER_UNIT;
2601 458703 : desc->nonarg_acc_size = nonarg_acc_size / BITS_PER_UNIT;
2602 458703 : if (desc->split_candidate && desc->ptr_pt_count)
2603 : {
2604 190532 : gcc_assert (desc->by_ref);
2605 : check_pass_throughs = true;
2606 : }
2607 : }
2608 : }
2609 :
2610 : /* When a pointer parameter is passed-through to a callee, in which it is
2611 : only used to read only one or a few items, we can attempt to transform it
2612 : to obtaining and passing through the items instead of the pointer. But we
2613 : must take extra care that 1) we do not introduce any segfault by moving
2614 : dereferences above control flow and that 2) the data is not modified
2615 : through an alias in this function. The IPA analysis must not introduce
2616 : any accesses candidates unless it can prove both.
2617 :
2618 : The current solution is very crude as it consists of ensuring that the
2619 : call postdominates entry BB and that the definition of VUSE of the call is
2620 : default definition. TODO: For non-recursive callees in the same
2621 : compilation unit we could do better by doing analysis in topological order
2622 : an looking into access candidates of callees, using their alias_ptr_types
2623 : to attempt real AA. We could also use the maximum known dereferenced
2624 : offset in this function at IPA level.
2625 :
2626 : TODO: Measure the overhead and the effect of just being pessimistic.
2627 : Maybe this is only -O3 material? */
2628 :
2629 861037 : hash_map<gimple *, bool> analyzed_stmts;
2630 861037 : bitmap always_executed_bbs = NULL;
2631 861037 : if (check_pass_throughs)
2632 1213147 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
2633 : {
2634 1059739 : gcall *call_stmt = cs->call_stmt;
2635 1059739 : tree vuse = gimple_vuse (call_stmt);
2636 :
2637 : /* If the callee is a const function, we don't get a VUSE. In such
2638 : case there will be no memory accesses in the called function (or the
2639 : const attribute is wrong) and then we just don't care. */
2640 2068974 : bool uses_memory_as_obtained = vuse && SSA_NAME_IS_DEFAULT_DEF (vuse);
2641 :
2642 1059739 : unsigned count = gimple_call_num_args (call_stmt);
2643 1059739 : isra_call_summary *csum = call_sums->get_create (cs);
2644 1059739 : csum->init_inputs (count);
2645 1059739 : csum->m_before_any_store = uses_memory_as_obtained;
2646 3421565 : for (unsigned argidx = 0; argidx < count; argidx++)
2647 : {
2648 2361826 : if (!csum->m_arg_flow[argidx].pointer_pass_through)
2649 1928131 : continue;
2650 433695 : unsigned pidx
2651 433695 : = get_single_param_flow_source (&csum->m_arg_flow[argidx]);
2652 433695 : gensum_param_desc *desc = &(*param_descriptions)[pidx];
2653 433695 : if (!desc->split_candidate)
2654 : {
2655 36752 : csum->m_arg_flow[argidx].pointer_pass_through = false;
2656 36752 : continue;
2657 : }
2658 396943 : if (!uses_memory_as_obtained)
2659 150955 : continue;
2660 :
2661 245988 : if (desc->safe_ref)
2662 : {
2663 45890 : csum->m_arg_flow[argidx].safe_to_import_accesses = true;
2664 45890 : continue;
2665 : }
2666 :
2667 : /* Walk basic block and see if its execution can terminate earlier.
2668 : Keep the info for later re-use to avoid quadratic behaviour here. */
2669 200098 : gimple_stmt_iterator gsi = gsi_for_stmt (call_stmt);
2670 200098 : bool safe = true;
2671 200098 : int n = 0;
2672 500277 : for (gsi_prev (&gsi); !gsi_end_p (gsi); gsi_prev (&gsi))
2673 : {
2674 127121 : bool *b = analyzed_stmts.get (gsi_stmt (gsi));
2675 127121 : if (b)
2676 : {
2677 9879 : safe = *b;
2678 9879 : gsi_next (&gsi);
2679 9879 : break;
2680 : }
2681 117242 : n++;
2682 117242 : if (stmt_may_terminate_function_p (fun, gsi_stmt (gsi), false))
2683 : {
2684 : safe = false;
2685 : break;
2686 : }
2687 : }
2688 200098 : if (n)
2689 : {
2690 33261 : if (gsi_end_p (gsi))
2691 59240 : gsi = gsi_start_bb (gimple_bb (call_stmt));
2692 150503 : for (; gsi_stmt (gsi) != call_stmt; gsi_next (&gsi))
2693 117242 : analyzed_stmts.get_or_insert (gsi_stmt (gsi)) = safe;
2694 : }
2695 :
2696 200098 : if (safe && !always_executed_bbs)
2697 : {
2698 57558 : mark_dfs_back_edges ();
2699 57558 : always_executed_bbs = find_always_executed_bbs (fun, false);
2700 : }
2701 200098 : if (safe && bitmap_bit_p (always_executed_bbs, gimple_bb (call_stmt)->index))
2702 46557 : csum->m_arg_flow[argidx].safe_to_import_accesses = true;
2703 : }
2704 :
2705 : }
2706 861037 : BITMAP_FREE (always_executed_bbs);
2707 :
2708 : /* TODO: Add early exit if we disqualified everything. This also requires
2709 : that we either relax the restriction that
2710 : ipa_param_adjustments.m_always_copy_start must be the number of PARM_DECLs
2711 : or store the number of parameters to IPA-SRA function summary and use that
2712 : when just removing params. */
2713 :
2714 861037 : vec_safe_reserve_exact (ifs->m_parameters, param_count);
2715 861037 : ifs->m_parameters->quick_grow_cleared (param_count);
2716 4169482 : for (unsigned desc_index = 0; desc_index < param_count; desc_index++)
2717 : {
2718 2447408 : gensum_param_desc *s = &(*param_descriptions)[desc_index];
2719 2447408 : isra_param_desc *d = &(*ifs->m_parameters)[desc_index];
2720 :
2721 2447408 : d->param_size_limit = s->param_size_limit;
2722 2447408 : d->size_reached = s->nonarg_acc_size;
2723 2447408 : d->locally_unused = s->locally_unused;
2724 2447408 : d->split_candidate = s->split_candidate;
2725 2447408 : d->by_ref = s->by_ref;
2726 2447408 : d->remove_only_when_retval_removed = s->remove_only_when_retval_removed;
2727 2447408 : d->split_only_when_retval_removed = s->split_only_when_retval_removed;
2728 2447408 : d->conditionally_dereferenceable = s->conditionally_dereferenceable;
2729 :
2730 2447408 : for (gensum_param_access *acc = s->accesses;
2731 2873308 : acc;
2732 425900 : acc = acc->next_sibling)
2733 425900 : copy_accesses_to_ipa_desc (acc, d);
2734 : }
2735 :
2736 861037 : if (dump_file)
2737 172 : dump_isra_param_descriptors (dump_file, node->decl, ifs, false);
2738 861037 : }
2739 :
2740 : /* Return true if there are any overlaps among certain accesses of DESC. If
2741 : non-NULL, set *CERTAIN_ACCESS_PRESENT_P upon encountering a certain access
2742 : too. DESC is assumed to be a split candidate that is not locally
2743 : unused. */
2744 :
2745 : static bool
2746 176733 : overlapping_certain_accesses_p (isra_param_desc *desc,
2747 : bool *certain_access_present_p)
2748 : {
2749 176733 : unsigned pclen = vec_safe_length (desc->accesses);
2750 463557 : for (unsigned i = 0; i < pclen; i++)
2751 : {
2752 289043 : param_access *a1 = (*desc->accesses)[i];
2753 :
2754 289043 : if (!a1->certain)
2755 6280 : continue;
2756 282763 : if (certain_access_present_p)
2757 266712 : *certain_access_present_p = true;
2758 460674 : for (unsigned j = i + 1; j < pclen; j++)
2759 : {
2760 180130 : param_access *a2 = (*desc->accesses)[j];
2761 180130 : if (a2->certain
2762 180056 : && a1->unit_offset < a2->unit_offset + a2->unit_size
2763 179691 : && a1->unit_offset + a1->unit_size > a2->unit_offset)
2764 : return true;
2765 : }
2766 : }
2767 : return false;
2768 : }
2769 :
2770 : /* Check for any overlaps of certain param accesses among splitting candidates
2771 : and signal an ICE if there are any. If CERTAIN_MUST_EXIST is set, also
2772 : check that used splitting candidates have at least one certain access. */
2773 :
2774 : static void
2775 2685922 : verify_splitting_accesses (cgraph_node *node, bool certain_must_exist)
2776 : {
2777 2685922 : isra_func_summary *ifs = func_sums->get (node);
2778 2685922 : if (!ifs || !ifs->m_candidate)
2779 : return;
2780 1449573 : unsigned param_count = vec_safe_length (ifs->m_parameters);
2781 4697540 : for (unsigned pidx = 0; pidx < param_count; pidx++)
2782 : {
2783 3247967 : isra_param_desc *desc = &(*ifs->m_parameters)[pidx];
2784 3247967 : if (!desc->split_candidate || desc->locally_unused)
2785 3087216 : continue;
2786 :
2787 160751 : bool certain_access_present = !certain_must_exist;
2788 160751 : if (overlapping_certain_accesses_p (desc, &certain_access_present))
2789 0 : internal_error ("function %qs, parameter %u, has IPA-SRA accesses "
2790 : "which overlap", node->dump_name (), pidx);
2791 160751 : if (!certain_access_present)
2792 0 : internal_error ("function %qs, parameter %u, is used but does not "
2793 : "have any certain IPA-SRA access",
2794 : node->dump_name (), pidx);
2795 : }
2796 : }
2797 :
2798 : /* Intraprocedural part of IPA-SRA analysis. Scan bodies of all functions in
2799 : this compilation unit and create summary structures describing IPA-SRA
2800 : opportunities and constraints in them. */
2801 :
2802 : static void
2803 129344 : ipa_sra_generate_summary (void)
2804 : {
2805 129344 : struct cgraph_node *node;
2806 :
2807 129344 : gcc_checking_assert (!func_sums);
2808 129344 : gcc_checking_assert (!call_sums);
2809 129344 : func_sums
2810 129344 : = (new (ggc_alloc_no_dtor <ipa_sra_function_summaries> ())
2811 129344 : ipa_sra_function_summaries (symtab, true));
2812 129344 : call_sums = new ipa_sra_call_summaries (symtab);
2813 129344 : callback_info_sum_t::check_create_info_sum ();
2814 :
2815 1414557 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
2816 1285213 : ipa_sra_summarize_function (node);
2817 129344 : return;
2818 : }
2819 :
2820 : /* Write intraprocedural analysis information about E and all of its outgoing
2821 : edges into a stream for LTO WPA. */
2822 :
2823 : static void
2824 285084 : isra_write_edge_summary (output_block *ob, cgraph_edge *e)
2825 : {
2826 285084 : isra_call_summary *csum = call_sums->get (e);
2827 285084 : unsigned input_count = csum->m_arg_flow.length ();
2828 285084 : streamer_write_uhwi (ob, input_count);
2829 1045914 : for (unsigned i = 0; i < input_count; i++)
2830 : {
2831 475746 : isra_param_flow *ipf = &csum->m_arg_flow[i];
2832 475746 : streamer_write_hwi (ob, ipf->length);
2833 475746 : bitpack_d bp = bitpack_create (ob->main_stream);
2834 551284 : for (int j = 0; j < ipf->length; j++)
2835 75538 : bp_pack_value (&bp, ipf->inputs[j], 8);
2836 475746 : bp_pack_value (&bp, ipf->aggregate_pass_through, 1);
2837 475746 : bp_pack_value (&bp, ipf->pointer_pass_through, 1);
2838 475746 : bp_pack_value (&bp, ipf->safe_to_import_accesses, 1);
2839 475746 : bp_pack_value (&bp, ipf->constructed_for_calls, 1);
2840 475746 : streamer_write_bitpack (&bp);
2841 475746 : streamer_write_uhwi (ob, ipf->unit_offset);
2842 475746 : streamer_write_uhwi (ob, ipf->unit_size);
2843 : }
2844 285084 : bitpack_d bp = bitpack_create (ob->main_stream);
2845 285084 : bp_pack_value (&bp, csum->m_return_ignored, 1);
2846 285084 : bp_pack_value (&bp, csum->m_return_returned, 1);
2847 285084 : bp_pack_value (&bp, csum->m_bit_aligned_arg, 1);
2848 285084 : bp_pack_value (&bp, csum->m_before_any_store, 1);
2849 285084 : streamer_write_bitpack (&bp);
2850 285084 : }
2851 :
2852 : /* Write intraprocedural analysis information about NODE and all of its outgoing
2853 : edges into a stream for LTO WPA. */
2854 :
2855 : static void
2856 50863 : isra_write_node_summary (output_block *ob, cgraph_node *node)
2857 : {
2858 50863 : isra_func_summary *ifs = func_sums->get (node);
2859 50863 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
2860 50863 : int node_ref = lto_symtab_encoder_encode (encoder, node);
2861 50863 : streamer_write_uhwi (ob, node_ref);
2862 :
2863 50863 : unsigned param_desc_count = vec_safe_length (ifs->m_parameters);
2864 50863 : streamer_write_uhwi (ob, param_desc_count);
2865 353426 : for (unsigned i = 0; i < param_desc_count; i++)
2866 : {
2867 251700 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
2868 251700 : unsigned access_count = vec_safe_length (desc->accesses);
2869 251700 : streamer_write_uhwi (ob, access_count);
2870 506876 : for (unsigned j = 0; j < access_count; j++)
2871 : {
2872 3476 : param_access *acc = (*desc->accesses)[j];
2873 3476 : stream_write_tree (ob, acc->type, true);
2874 3476 : stream_write_tree (ob, acc->alias_ptr_type, true);
2875 3476 : streamer_write_uhwi (ob, acc->unit_offset);
2876 3476 : streamer_write_uhwi (ob, acc->unit_size);
2877 3476 : bitpack_d bp = bitpack_create (ob->main_stream);
2878 3476 : bp_pack_value (&bp, acc->certain, 1);
2879 3476 : bp_pack_value (&bp, acc->reverse, 1);
2880 3476 : streamer_write_bitpack (&bp);
2881 : }
2882 251700 : streamer_write_uhwi (ob, desc->param_size_limit);
2883 251700 : streamer_write_uhwi (ob, desc->size_reached);
2884 251700 : gcc_assert (desc->safe_size == 0);
2885 251700 : bitpack_d bp = bitpack_create (ob->main_stream);
2886 251700 : bp_pack_value (&bp, desc->locally_unused, 1);
2887 251700 : bp_pack_value (&bp, desc->split_candidate, 1);
2888 251700 : bp_pack_value (&bp, desc->by_ref, 1);
2889 251700 : gcc_assert (!desc->not_specially_constructed);
2890 251700 : bp_pack_value (&bp, desc->remove_only_when_retval_removed, 1);
2891 251700 : bp_pack_value (&bp, desc->split_only_when_retval_removed, 1);
2892 251700 : bp_pack_value (&bp, desc->conditionally_dereferenceable, 1);
2893 251700 : gcc_assert (!desc->safe_size_set);
2894 251700 : streamer_write_bitpack (&bp);
2895 : }
2896 50863 : bitpack_d bp = bitpack_create (ob->main_stream);
2897 50863 : bp_pack_value (&bp, ifs->m_candidate, 1);
2898 50863 : bp_pack_value (&bp, ifs->m_returns_value, 1);
2899 50863 : bp_pack_value (&bp, ifs->m_return_ignored, 1);
2900 50863 : gcc_assert (!ifs->m_queued);
2901 50863 : streamer_write_bitpack (&bp);
2902 :
2903 333472 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
2904 282609 : isra_write_edge_summary (ob, e);
2905 53338 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
2906 2475 : isra_write_edge_summary (ob, e);
2907 50863 : }
2908 :
2909 : /* Write intraprocedural analysis information into a stream for LTO WPA. */
2910 :
2911 : static void
2912 20640 : ipa_sra_write_summary (void)
2913 : {
2914 20640 : if (!func_sums || !call_sums)
2915 0 : return;
2916 :
2917 20640 : struct output_block *ob = create_output_block (LTO_section_ipa_sra);
2918 20640 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
2919 20640 : ob->symbol = NULL;
2920 :
2921 20640 : unsigned int count = 0;
2922 20640 : lto_symtab_encoder_iterator lsei;
2923 20640 : for (lsei = lsei_start_function_in_partition (encoder);
2924 116630 : !lsei_end_p (lsei);
2925 95990 : lsei_next_function_in_partition (&lsei))
2926 : {
2927 95990 : cgraph_node *node = lsei_cgraph_node (lsei);
2928 95990 : if (node->has_gimple_body_p ()
2929 95990 : && func_sums->get (node) != NULL)
2930 50863 : count++;
2931 : }
2932 20640 : streamer_write_uhwi (ob, count);
2933 :
2934 : /* Process all of the functions. */
2935 116630 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
2936 95990 : lsei_next_function_in_partition (&lsei))
2937 : {
2938 95990 : cgraph_node *node = lsei_cgraph_node (lsei);
2939 95990 : if (node->has_gimple_body_p ()
2940 95990 : && func_sums->get (node) != NULL)
2941 50863 : isra_write_node_summary (ob, node);
2942 : }
2943 20640 : streamer_write_char_stream (ob->main_stream, 0);
2944 20640 : produce_asm (ob);
2945 20640 : destroy_output_block (ob);
2946 : }
2947 :
2948 : /* Read intraprocedural analysis information about E and all of its outgoing
2949 : edges into a stream for LTO WPA. */
2950 :
2951 : static void
2952 256499 : isra_read_edge_summary (struct lto_input_block *ib, cgraph_edge *cs)
2953 : {
2954 256499 : isra_call_summary *csum = call_sums->get_create (cs);
2955 256499 : unsigned input_count = streamer_read_uhwi (ib);
2956 256499 : csum->init_inputs (input_count);
2957 962641 : for (unsigned i = 0; i < input_count; i++)
2958 : {
2959 449643 : isra_param_flow *ipf = &csum->m_arg_flow[i];
2960 449643 : ipf->length = streamer_read_hwi (ib);
2961 449643 : bitpack_d bp = streamer_read_bitpack (ib);
2962 517505 : for (int j = 0; j < ipf->length; j++)
2963 67862 : ipf->inputs[j] = bp_unpack_value (&bp, 8);
2964 449643 : ipf->aggregate_pass_through = bp_unpack_value (&bp, 1);
2965 449643 : ipf->pointer_pass_through = bp_unpack_value (&bp, 1);
2966 449643 : ipf->safe_to_import_accesses = bp_unpack_value (&bp, 1);
2967 449643 : ipf->constructed_for_calls = bp_unpack_value (&bp, 1);
2968 449643 : ipf->unit_offset = streamer_read_uhwi (ib);
2969 449643 : ipf->unit_size = streamer_read_uhwi (ib);
2970 : }
2971 256499 : bitpack_d bp = streamer_read_bitpack (ib);
2972 256499 : csum->m_return_ignored = bp_unpack_value (&bp, 1);
2973 256499 : csum->m_return_returned = bp_unpack_value (&bp, 1);
2974 256499 : csum->m_bit_aligned_arg = bp_unpack_value (&bp, 1);
2975 256499 : csum->m_before_any_store = bp_unpack_value (&bp, 1);
2976 256499 : }
2977 :
2978 : /* Read intraprocedural analysis information about NODE and all of its outgoing
2979 : edges into a stream for LTO WPA. */
2980 :
2981 : static void
2982 36085 : isra_read_node_info (struct lto_input_block *ib, cgraph_node *node,
2983 : struct data_in *data_in)
2984 : {
2985 36085 : isra_func_summary *ifs = func_sums->get_create (node);
2986 36085 : unsigned param_desc_count = streamer_read_uhwi (ib);
2987 36085 : if (param_desc_count > 0)
2988 : {
2989 18370 : vec_safe_reserve_exact (ifs->m_parameters, param_desc_count);
2990 18370 : ifs->m_parameters->quick_grow_cleared (param_desc_count);
2991 : }
2992 65350 : for (unsigned i = 0; i < param_desc_count; i++)
2993 : {
2994 29265 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
2995 29265 : unsigned access_count = streamer_read_uhwi (ib);
2996 31181 : for (unsigned j = 0; j < access_count; j++)
2997 : {
2998 1916 : param_access *acc = ggc_cleared_alloc<param_access> ();
2999 1916 : acc->type = stream_read_tree (ib, data_in);
3000 1916 : acc->alias_ptr_type = stream_read_tree (ib, data_in);
3001 1916 : acc->unit_offset = streamer_read_uhwi (ib);
3002 1916 : acc->unit_size = streamer_read_uhwi (ib);
3003 1916 : bitpack_d bp = streamer_read_bitpack (ib);
3004 1916 : acc->certain = bp_unpack_value (&bp, 1);
3005 1916 : acc->reverse = bp_unpack_value (&bp, 1);
3006 1916 : vec_safe_push (desc->accesses, acc);
3007 : }
3008 29265 : desc->param_size_limit = streamer_read_uhwi (ib);
3009 29265 : desc->size_reached = streamer_read_uhwi (ib);
3010 29265 : desc->safe_size = 0;
3011 29265 : bitpack_d bp = streamer_read_bitpack (ib);
3012 29265 : desc->locally_unused = bp_unpack_value (&bp, 1);
3013 29265 : desc->split_candidate = bp_unpack_value (&bp, 1);
3014 29265 : desc->by_ref = bp_unpack_value (&bp, 1);
3015 29265 : desc->not_specially_constructed = 0;
3016 29265 : desc->remove_only_when_retval_removed = bp_unpack_value (&bp, 1);
3017 29265 : desc->split_only_when_retval_removed = bp_unpack_value (&bp, 1);
3018 29265 : desc->conditionally_dereferenceable = bp_unpack_value (&bp, 1);
3019 29265 : desc->safe_size_set = 0;
3020 : }
3021 36085 : bitpack_d bp = streamer_read_bitpack (ib);
3022 36085 : ifs->m_candidate = bp_unpack_value (&bp, 1);
3023 36085 : ifs->m_returns_value = bp_unpack_value (&bp, 1);
3024 36085 : ifs->m_return_ignored = bp_unpack_value (&bp, 1);
3025 36085 : ifs->m_queued = 0;
3026 :
3027 291241 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
3028 255156 : isra_read_edge_summary (ib, e);
3029 37428 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
3030 1343 : isra_read_edge_summary (ib, e);
3031 36085 : }
3032 :
3033 : /* Read IPA-SRA summaries from a section in file FILE_DATA of length LEN with
3034 : data DATA. TODO: This function was copied almost verbatim from ipa-prop.cc,
3035 : it should be possible to unify them somehow. */
3036 :
3037 : static void
3038 11415 : isra_read_summary_section (struct lto_file_decl_data *file_data,
3039 : const char *data, size_t len)
3040 : {
3041 11415 : const struct lto_function_header *header =
3042 : (const struct lto_function_header *) data;
3043 11415 : const int cfg_offset = sizeof (struct lto_function_header);
3044 11415 : const int main_offset = cfg_offset + header->cfg_size;
3045 11415 : const int string_offset = main_offset + header->main_size;
3046 11415 : struct data_in *data_in;
3047 11415 : unsigned int i;
3048 11415 : unsigned int count;
3049 :
3050 11415 : lto_input_block ib_main ((const char *) data + main_offset,
3051 11415 : header->main_size, file_data);
3052 :
3053 11415 : data_in =
3054 22830 : lto_data_in_create (file_data, (const char *) data + string_offset,
3055 11415 : header->string_size, vNULL);
3056 11415 : count = streamer_read_uhwi (&ib_main);
3057 :
3058 47500 : for (i = 0; i < count; i++)
3059 : {
3060 36085 : unsigned int index;
3061 36085 : struct cgraph_node *node;
3062 36085 : lto_symtab_encoder_t encoder;
3063 :
3064 36085 : index = streamer_read_uhwi (&ib_main);
3065 36085 : encoder = file_data->symtab_node_encoder;
3066 36085 : node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
3067 : index));
3068 36085 : gcc_assert (node->definition);
3069 36085 : isra_read_node_info (&ib_main, node, data_in);
3070 : }
3071 11415 : lto_free_section_data (file_data, LTO_section_ipa_sra, NULL, data,
3072 : len);
3073 11415 : lto_data_in_delete (data_in);
3074 11415 : }
3075 :
3076 : /* Read intraprocedural analysis information into a stream for LTO WPA. */
3077 :
3078 : static void
3079 10696 : ipa_sra_read_summary (void)
3080 : {
3081 10696 : struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
3082 10696 : struct lto_file_decl_data *file_data;
3083 10696 : unsigned int j = 0;
3084 :
3085 10696 : gcc_checking_assert (!func_sums);
3086 10696 : gcc_checking_assert (!call_sums);
3087 10696 : func_sums
3088 10696 : = (new (ggc_alloc_no_dtor <ipa_sra_function_summaries> ())
3089 10696 : ipa_sra_function_summaries (symtab, true));
3090 10696 : call_sums = new ipa_sra_call_summaries (symtab);
3091 :
3092 22122 : while ((file_data = file_data_vec[j++]))
3093 : {
3094 11426 : size_t len;
3095 11426 : const char *data
3096 11426 : = lto_get_summary_section_data (file_data, LTO_section_ipa_sra, &len);
3097 11426 : if (data)
3098 11415 : isra_read_summary_section (file_data, data, len);
3099 : }
3100 10696 : }
3101 :
3102 : /* Dump all IPA-SRA summary data for all cgraph nodes and edges to file F. If
3103 : HINTS is true, also dump IPA-analysis computed hints. */
3104 :
3105 : static void
3106 79 : ipa_sra_dump_all_summaries (FILE *f, bool hints)
3107 : {
3108 79 : cgraph_node *node;
3109 350 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
3110 : {
3111 271 : fprintf (f, "\nSummary for node %s:\n", node->dump_name ());
3112 :
3113 271 : isra_func_summary *ifs = func_sums->get (node);
3114 271 : if (!ifs)
3115 10 : fprintf (f, " Function does not have any associated IPA-SRA "
3116 : "summary\n");
3117 261 : else if (!ifs->m_candidate)
3118 26 : fprintf (f, " Not a candidate function\n");
3119 : else
3120 : {
3121 235 : if (ifs->m_returns_value)
3122 137 : fprintf (f, " Returns value\n");
3123 235 : if (vec_safe_is_empty (ifs->m_parameters))
3124 43 : fprintf (f, " No parameter information. \n");
3125 : else
3126 566 : for (unsigned i = 0; i < ifs->m_parameters->length (); ++i)
3127 : {
3128 374 : fprintf (f, " Descriptor for parameter %i:\n", i);
3129 374 : dump_isra_param_descriptor (f, &(*ifs->m_parameters)[i], hints);
3130 : }
3131 235 : fprintf (f, "\n");
3132 : }
3133 :
3134 271 : struct cgraph_edge *cs;
3135 567 : for (cs = node->callees; cs; cs = cs->next_callee)
3136 : {
3137 296 : fprintf (f, " Summary for edge %s->%s:\n", cs->caller->dump_name (),
3138 296 : cs->callee->dump_name ());
3139 296 : isra_call_summary *csum = call_sums->get (cs);
3140 296 : if (csum)
3141 296 : csum->dump (f);
3142 : else
3143 0 : fprintf (f, " Call summary is MISSING!\n");
3144 : }
3145 :
3146 : }
3147 79 : fprintf (f, "\n\n");
3148 79 : }
3149 :
3150 : /* Perform function-scope viability tests that can be only made at IPA level
3151 : and return false if the function is deemed unsuitable for IPA-SRA. */
3152 :
3153 : static bool
3154 1056708 : ipa_sra_ipa_function_checks (cgraph_node *node)
3155 : {
3156 : /* Assume bodies cannot be local, because their address is taken, but we want
3157 : to consider them anyway. */
3158 1056708 : if (!node->can_be_local_p () && !node->assume_function)
3159 : {
3160 670180 : if (dump_file)
3161 101 : fprintf (dump_file, "Function %s disqualified because it cannot be "
3162 : "made local.\n", node->dump_name ());
3163 : return false;
3164 : }
3165 386528 : if (!node->can_change_signature)
3166 : {
3167 0 : if (dump_file)
3168 0 : fprintf (dump_file, "Function can not change signature.\n");
3169 : return false;
3170 : }
3171 :
3172 : return true;
3173 : }
3174 :
3175 : /* Issues found out by check_callers_for_issues. */
3176 :
3177 : struct caller_issues
3178 : {
3179 : /* The candidate being considered. */
3180 : cgraph_node *candidate;
3181 : /* There is a thunk among callers. */
3182 : bool thunk;
3183 : /* Set if there is at least one caller that is OK. */
3184 : bool there_is_one;
3185 : /* Call site with no available information. */
3186 : bool unknown_callsite;
3187 : /* Call from outside the candidate's comdat group. */
3188 : bool call_from_outside_comdat;
3189 : /* There is a bit-aligned load into one of non-gimple-typed arguments. */
3190 : bool bit_aligned_aggregate_argument;
3191 : };
3192 :
3193 : /* Worker for call_for_symbol_and_aliases, set any flags of passed caller_issues
3194 : that apply. */
3195 :
3196 : static bool
3197 392214 : check_for_caller_issues (struct cgraph_node *node, void *data)
3198 : {
3199 392214 : struct caller_issues *issues = (struct caller_issues *) data;
3200 :
3201 1499713 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
3202 : {
3203 1107744 : if (cs->caller->thunk)
3204 : {
3205 0 : issues->thunk = true;
3206 : /* TODO: We should be able to process at least some types of
3207 : thunks. */
3208 0 : return true;
3209 : }
3210 1107744 : if (issues->candidate->calls_comdat_local
3211 10 : && issues->candidate->same_comdat_group
3212 1107754 : && !issues->candidate->in_same_comdat_group_p (cs->caller))
3213 : {
3214 0 : issues->call_from_outside_comdat = true;
3215 0 : return true;
3216 : }
3217 :
3218 1107744 : isra_call_summary *csum = call_sums->get (cs);
3219 1107744 : if (!csum)
3220 : {
3221 245 : issues->unknown_callsite = true;
3222 245 : return true;
3223 : }
3224 :
3225 1107499 : if (csum->m_bit_aligned_arg)
3226 0 : issues->bit_aligned_aggregate_argument = true;
3227 :
3228 1107499 : issues->there_is_one = true;
3229 : }
3230 : return false;
3231 : }
3232 :
3233 : /* Look at all incoming edges to NODE, including aliases and thunks and look
3234 : for problems. Return true if NODE type should not be modified at all. */
3235 :
3236 : static bool
3237 386528 : check_all_callers_for_issues (cgraph_node *node)
3238 : {
3239 386528 : struct caller_issues issues;
3240 386528 : memset (&issues, 0, sizeof (issues));
3241 386528 : issues.candidate = node;
3242 :
3243 386528 : node->call_for_symbol_and_aliases (check_for_caller_issues, &issues, true);
3244 386528 : if (issues.unknown_callsite)
3245 : {
3246 245 : if (dump_file && (dump_flags & TDF_DETAILS))
3247 0 : fprintf (dump_file, "A call of %s has not been analyzed. Disabling "
3248 : "all modifications.\n", node->dump_name ());
3249 : return true;
3250 : }
3251 : /* TODO: We should be able to process at least some types of thunks. */
3252 386283 : if (issues.thunk)
3253 : {
3254 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3255 0 : fprintf (dump_file, "A call of %s is through thunk, which are not"
3256 : " handled yet. Disabling all modifications.\n",
3257 : node->dump_name ());
3258 : return true;
3259 : }
3260 386283 : if (issues.call_from_outside_comdat)
3261 : {
3262 0 : if (dump_file)
3263 0 : fprintf (dump_file, "Function would become private comdat called "
3264 : "outside of its comdat group.\n");
3265 : return true;
3266 : }
3267 :
3268 386283 : if (issues.bit_aligned_aggregate_argument)
3269 : {
3270 : /* Let's only remove parameters/return values from such functions.
3271 : TODO: We could only prevent splitting the problematic parameters if
3272 : anybody thinks it is worth it. */
3273 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3274 0 : fprintf (dump_file, "A call of %s has bit-aligned aggregate argument,"
3275 : " disabling parameter splitting.\n", node->dump_name ());
3276 :
3277 0 : isra_func_summary *ifs = func_sums->get (node);
3278 0 : gcc_checking_assert (ifs);
3279 0 : unsigned param_count = vec_safe_length (ifs->m_parameters);
3280 0 : for (unsigned i = 0; i < param_count; i++)
3281 0 : (*ifs->m_parameters)[i].split_candidate = false;
3282 : }
3283 386283 : if (!issues.there_is_one)
3284 : {
3285 1 : if (dump_file && (dump_flags & TDF_DETAILS))
3286 0 : fprintf (dump_file, "There is no call to %s that we can modify. "
3287 : "Disabling all modifications.\n", node->dump_name ());
3288 : return true;
3289 : }
3290 : return false;
3291 : }
3292 :
3293 : /* Find the access with corresponding OFFSET and SIZE among accesses in
3294 : PARAM_DESC and return it or NULL if such an access is not there. */
3295 :
3296 : static param_access *
3297 17148 : find_param_access (isra_param_desc *param_desc, unsigned offset, unsigned size)
3298 : {
3299 17148 : unsigned pclen = vec_safe_length (param_desc->accesses);
3300 :
3301 : /* The search is linear but the number of stored accesses is bound by
3302 : PARAM_IPA_SRA_MAX_REPLACEMENTS, so most probably 8. */
3303 :
3304 17225 : for (unsigned i = 0; i < pclen; i++)
3305 17180 : if ((*param_desc->accesses)[i]->unit_offset == offset
3306 17180 : && (*param_desc->accesses)[i]->unit_size == size)
3307 : return (*param_desc->accesses)[i];
3308 :
3309 : return NULL;
3310 : }
3311 :
3312 : /* Return iff the total size of definite replacement SIZE would violate the
3313 : limit set for it in PARAM. */
3314 :
3315 : static bool
3316 229576 : size_would_violate_limit_p (isra_param_desc *desc, unsigned size)
3317 : {
3318 229576 : unsigned limit = desc->param_size_limit;
3319 0 : if (size > limit
3320 226510 : || (!desc->by_ref && size == limit))
3321 0 : return true;
3322 : return false;
3323 : }
3324 :
3325 : /* Increase reached size of DESC by SIZE or disqualify it if it would violate
3326 : the set limit. IDX is the parameter number which is dumped when
3327 : disqualifying. */
3328 :
3329 : static void
3330 13763 : bump_reached_size (isra_param_desc *desc, unsigned size, unsigned idx)
3331 : {
3332 13763 : unsigned after = desc->size_reached + size;
3333 13763 : if (size_would_violate_limit_p (desc, after))
3334 : {
3335 13731 : if (dump_file && (dump_flags & TDF_DETAILS))
3336 0 : fprintf (dump_file, " ...size limit reached, disqualifying "
3337 : "candidate parameter %u\n", idx);
3338 13731 : desc->split_candidate = false;
3339 13731 : return;
3340 : }
3341 32 : desc->size_reached = after;
3342 : }
3343 :
3344 : /* Take all actions required to deal with an edge CS that represents a call to
3345 : an unknown or un-analyzed function, for both parameter removal and
3346 : splitting. */
3347 :
3348 : static void
3349 546113 : process_edge_to_unknown_caller (cgraph_edge *cs)
3350 : {
3351 546113 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3352 546113 : gcc_checking_assert (from_ifs);
3353 546113 : isra_call_summary *csum = call_sums->get (cs);
3354 :
3355 546113 : if (dump_file && (dump_flags & TDF_DETAILS))
3356 23 : fprintf (dump_file, "Processing an edge to an unknown caller from %s:\n",
3357 : cs->caller->dump_name ());
3358 :
3359 546113 : unsigned args_count = csum->m_arg_flow.length ();
3360 1316901 : for (unsigned i = 0; i < args_count; i++)
3361 : {
3362 770788 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3363 :
3364 770788 : if (ipf->pointer_pass_through)
3365 : {
3366 193948 : isra_param_desc *param_desc
3367 193948 : = &(*from_ifs->m_parameters)[get_single_param_flow_source (ipf)];
3368 193948 : param_desc->locally_unused = false;
3369 193948 : param_desc->split_candidate = false;
3370 193948 : continue;
3371 193948 : }
3372 576840 : if (ipf->aggregate_pass_through)
3373 : {
3374 6502 : unsigned idx = get_single_param_flow_source (ipf);
3375 6502 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3376 :
3377 6502 : param_desc->locally_unused = false;
3378 6502 : if (!param_desc->split_candidate)
3379 1786 : continue;
3380 4716 : gcc_assert (!param_desc->by_ref);
3381 9432 : param_access *pacc = find_param_access (param_desc, ipf->unit_offset,
3382 4716 : ipf->unit_size);
3383 4716 : gcc_checking_assert (pacc);
3384 4716 : pacc->certain = true;
3385 4716 : if (overlapping_certain_accesses_p (param_desc, NULL))
3386 : {
3387 627 : if (dump_file && (dump_flags & TDF_DETAILS))
3388 0 : fprintf (dump_file, " ...leading to overlap, "
3389 : " disqualifying candidate parameter %u\n",
3390 : idx);
3391 627 : param_desc->split_candidate = false;
3392 : }
3393 : else
3394 4089 : bump_reached_size (param_desc, pacc->unit_size, idx);
3395 4716 : ipf->aggregate_pass_through = false;
3396 4716 : continue;
3397 4716 : }
3398 :
3399 665838 : for (int j = 0; j < ipf->length; j++)
3400 : {
3401 95500 : int input_idx = ipf->inputs[j];
3402 95500 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3403 : }
3404 : }
3405 546113 : }
3406 :
3407 : /* Propagate parameter removal information through cross-SCC edge CS,
3408 : i.e. decrease the use count in the caller parameter descriptor for each use
3409 : in this call. */
3410 :
3411 : static void
3412 907024 : param_removal_cross_scc_edge (cgraph_edge *cs)
3413 : {
3414 907024 : enum availability availability;
3415 907024 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3416 907024 : isra_func_summary *to_ifs = func_sums->get (callee);
3417 438752 : if (!to_ifs || !to_ifs->m_candidate
3418 385656 : || (availability < AVAIL_AVAILABLE)
3419 1292680 : || vec_safe_is_empty (to_ifs->m_parameters))
3420 : {
3421 531956 : process_edge_to_unknown_caller (cs);
3422 531956 : return;
3423 : }
3424 375068 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3425 375068 : gcc_checking_assert (from_ifs);
3426 :
3427 375068 : isra_call_summary *csum = call_sums->get (cs);
3428 375068 : unsigned args_count = csum->m_arg_flow.length ();
3429 375068 : unsigned param_count = vec_safe_length (to_ifs->m_parameters);
3430 :
3431 1295097 : for (unsigned i = 0; i < args_count; i++)
3432 : {
3433 920029 : bool unused_in_callee = false;
3434 :
3435 920029 : isra_param_desc *descriptor = NULL;
3436 920029 : if (cs->callback)
3437 : {
3438 14 : callback_info *ci = callback_info_sum->get (cs);
3439 14 : int remapped_idx = ci->inverted_mapping[i];
3440 14 : if (remapped_idx != ARG_MAPPING_UNKNOWN_IDX)
3441 7 : descriptor = &(*to_ifs->m_parameters)[remapped_idx];
3442 : }
3443 920015 : else if (i < param_count)
3444 920011 : descriptor = &(*to_ifs->m_parameters)[i];
3445 :
3446 920018 : if (descriptor)
3447 920018 : unused_in_callee = descriptor->locally_unused;
3448 :
3449 920018 : if (!unused_in_callee)
3450 : {
3451 860605 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3452 1979595 : for (int j = 0; j < ipf->length; j++)
3453 : {
3454 258385 : int input_idx = ipf->inputs[j];
3455 258385 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3456 : }
3457 : }
3458 : }
3459 : }
3460 :
3461 : /* Unless it is already there, push NODE which is also described by IFS to
3462 : STACK. */
3463 :
3464 : static void
3465 161143 : isra_push_node_to_stack (cgraph_node *node, isra_func_summary *ifs,
3466 : vec<cgraph_node *> *stack)
3467 : {
3468 0 : if (!ifs->m_queued)
3469 : {
3470 160721 : ifs->m_queued = true;
3471 160721 : stack->safe_push (node);
3472 : }
3473 0 : }
3474 :
3475 : /* If parameter with index INPUT_IDX is marked as locally unused, mark it as
3476 : used and push CALLER on STACK. */
3477 :
3478 : static void
3479 13580 : isra_mark_caller_param_used (isra_func_summary *from_ifs, int input_idx,
3480 : cgraph_node *caller, vec<cgraph_node *> *stack)
3481 : {
3482 13580 : if ((*from_ifs->m_parameters)[input_idx].locally_unused)
3483 : {
3484 1153 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3485 1933 : isra_push_node_to_stack (caller, from_ifs, stack);
3486 : }
3487 13580 : }
3488 :
3489 : /* Combine safe_size of DESC with SIZE and return true if it has changed. */
3490 :
3491 : static bool
3492 2526302 : update_safe_size (isra_param_desc *desc, unsigned size)
3493 : {
3494 0 : if (!desc->safe_size_set)
3495 : {
3496 796597 : desc->safe_size_set = 1;
3497 796597 : desc->safe_size = size;
3498 796597 : return true;
3499 : }
3500 1729705 : if (desc->safe_size <= size)
3501 : return false;
3502 20005 : desc->safe_size = size;
3503 20005 : return true;
3504 : }
3505 :
3506 : /* Set all param hints in DESC to the pessimistic values. Return true if any
3507 : hints that need to potentially trigger further propagation have changed. */
3508 :
3509 : static bool
3510 744375 : flip_all_hints_pessimistic (isra_param_desc *desc)
3511 : {
3512 744375 : desc->not_specially_constructed = true;
3513 0 : return update_safe_size (desc, 0);
3514 : }
3515 :
3516 : /* Because we have not analyzed or otherwise problematic caller, go over all
3517 : parameter int flags of IFS describing a call graph node of a callee and
3518 : turn them pessimistic. Return true if any hints that need to potentially
3519 : trigger further propagation have changed. */
3520 :
3521 : static bool
3522 420567 : flip_all_param_hints_pessimistic (isra_func_summary *ifs)
3523 : {
3524 420567 : if (!ifs || !ifs->m_candidate)
3525 : return false;
3526 :
3527 26199 : bool ret = false;
3528 26199 : unsigned param_count = vec_safe_length (ifs->m_parameters);
3529 :
3530 74637 : for (unsigned i = 0; i < param_count; i++)
3531 96876 : ret |= flip_all_hints_pessimistic (&(*ifs->m_parameters)[i]);
3532 :
3533 : return ret;
3534 : }
3535 :
3536 : /* Propagate hints across edge CS which ultimately leads to a node described
3537 : by TO_IFS. Return true if any hints of the callee which should potentially
3538 : trigger further propagation have changed. */
3539 :
3540 : static bool
3541 5021899 : propagate_param_hints_accross_call (cgraph_edge *cs, isra_func_summary *to_ifs)
3542 : {
3543 5021899 : if (!to_ifs || !to_ifs->m_candidate)
3544 : return false;
3545 :
3546 1696212 : isra_call_summary *csum = call_sums->get (cs);
3547 1696212 : bool ret = false;
3548 1696212 : unsigned args_count = csum->m_arg_flow.length ();
3549 1696212 : unsigned param_count = vec_safe_length (to_ifs->m_parameters);
3550 :
3551 5415410 : for (unsigned i = 0; i < param_count; i++)
3552 : {
3553 3719198 : isra_param_desc *desc = &(*to_ifs->m_parameters)[i];
3554 3719198 : if (i >= args_count)
3555 : {
3556 695937 : ret |= flip_all_hints_pessimistic (desc);
3557 695937 : continue;
3558 : }
3559 :
3560 3023261 : if (desc->by_ref)
3561 : {
3562 1781945 : unsigned idx = i;
3563 1781945 : if (cs->callback)
3564 : {
3565 15 : gcc_assert (cs->callee->assume_function);
3566 15 : callback_info *ci = callback_info_sum->get (cs);
3567 15 : int remapped = ci->arg_mapping[i];
3568 15 : if (remapped == ARG_MAPPING_UNKNOWN_IDX)
3569 : /* In the case of assume outlined functions, the only skipped
3570 : argument is the address to be called, which is fine. In
3571 : other cases, if they ever arise, processing of the call to
3572 : callback dispatching functions will need to handle unmapped
3573 : arguments as necessary. */
3574 0 : continue;
3575 15 : idx = (unsigned) remapped;
3576 : }
3577 1781945 : isra_param_flow *ipf = &csum->m_arg_flow[idx];
3578 :
3579 1781945 : if (!ipf->constructed_for_calls)
3580 1310548 : desc->not_specially_constructed = true;
3581 :
3582 1781945 : if (ipf->pointer_pass_through)
3583 : {
3584 286928 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3585 286928 : int srcidx = get_single_param_flow_source (ipf);
3586 286928 : if (vec_safe_length (from_ifs->m_parameters) > (unsigned) srcidx)
3587 : {
3588 178821 : isra_param_desc *src_d = &(*from_ifs->m_parameters)[srcidx];
3589 178821 : if (src_d->safe_size_set)
3590 357606 : ret |= update_safe_size (desc, src_d->safe_size);
3591 : }
3592 : else
3593 216214 : ret |= update_safe_size (desc, 0);
3594 : }
3595 1495017 : else if (!ipf->aggregate_pass_through)
3596 2990032 : ret |= update_safe_size (desc, ipf->unit_size);
3597 : else
3598 : /* LTOing type-mismatched programs can end up here. */
3599 2 : ret |= update_safe_size (desc, 0);
3600 : }
3601 : }
3602 : return ret;
3603 : }
3604 :
3605 : /* Propagate hints from NODE described by FROM_IFS to all its (dorect) callees,
3606 : push those that may need re-visiting onto STACK. */
3607 :
3608 : static void
3609 1384629 : propagate_hints_to_all_callees (cgraph_node *node, isra_func_summary *from_ifs,
3610 : vec<cgraph_node *> *stack)
3611 : {
3612 6827095 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
3613 : {
3614 5442466 : enum availability availability;
3615 5442466 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3616 5442466 : isra_func_summary *to_ifs = func_sums->get (callee);
3617 : /* Skip over regular callbacks, as we cannot perform SRA on them. We
3618 : don't zap them early on, because the callback call might get optimized
3619 : out, so don't want to miss that opportunity. Skipping them here is OK,
3620 : as they will get disqualified later on in the IPA stage. */
3621 5442466 : if (!from_ifs || (cs->callback && !cs->callee->assume_function))
3622 : {
3623 420567 : if (flip_all_param_hints_pessimistic (to_ifs)
3624 420567 : && ipa_edge_within_scc (cs))
3625 228 : isra_push_node_to_stack (callee, to_ifs, stack);
3626 : }
3627 5021899 : else if (propagate_param_hints_accross_call (cs, to_ifs)
3628 5021899 : && ipa_edge_within_scc (cs))
3629 5333 : isra_push_node_to_stack (callee, to_ifs, stack);
3630 : }
3631 1384629 : }
3632 :
3633 : /* Propagate information that any parameter is not used only locally within a
3634 : SCC across CS to the caller, which must be in the same SCC as the
3635 : callee. Push any callers that need to be re-processed to STACK. */
3636 :
3637 : static void
3638 29644 : propagate_used_across_scc_edge (cgraph_edge *cs, vec<cgraph_node *> *stack)
3639 : {
3640 29644 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3641 29644 : if (!from_ifs || vec_safe_is_empty (from_ifs->m_parameters))
3642 15488 : return;
3643 :
3644 14156 : isra_call_summary *csum = call_sums->get (cs);
3645 14156 : gcc_checking_assert (csum);
3646 14156 : unsigned args_count = csum->m_arg_flow.length ();
3647 14156 : enum availability availability;
3648 14156 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3649 14156 : isra_func_summary *to_ifs = func_sums->get (callee);
3650 :
3651 14156 : unsigned param_count
3652 13953 : = (to_ifs && (availability >= AVAIL_AVAILABLE))
3653 27843 : ? vec_safe_length (to_ifs->m_parameters) : 0;
3654 42084 : for (unsigned i = 0; i < args_count; i++)
3655 : {
3656 27928 : unsigned idx = i;
3657 :
3658 27928 : if (cs->callback)
3659 : {
3660 0 : gcc_assert (cs->callee->assume_function);
3661 0 : callback_info *ci = callback_info_sum->get (cs);
3662 0 : int remapped = ci->inverted_mapping[i];
3663 0 : if (remapped == ARG_MAPPING_UNKNOWN_IDX)
3664 : /* In the case of assume outlined functions, the only skipped
3665 : argument is the address to be called, which is fine. In other
3666 : cases, if they ever arise, processing of the call to callback
3667 : dispatching functions will need to handle unmapped arguments as
3668 : necessary. */
3669 0 : continue;
3670 0 : idx = (unsigned) remapped;
3671 : }
3672 :
3673 31813 : if (idx < param_count
3674 27928 : && (*to_ifs->m_parameters)[idx].locally_unused)
3675 3885 : continue;
3676 :
3677 : /* The argument is needed in the callee it, we must mark the parameter as
3678 : used also in the caller and its callers within this SCC. */
3679 24043 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3680 61666 : for (int j = 0; j < ipf->length; j++)
3681 : {
3682 13580 : int input_idx = ipf->inputs[j];
3683 13580 : isra_mark_caller_param_used (from_ifs, input_idx, cs->caller, stack);
3684 : }
3685 : }
3686 : }
3687 :
3688 : /* Propagate information that any parameter is not used only locally within a
3689 : SCC (i.e. is used also elsewhere) to all callers of NODE that are in the
3690 : same SCC. Push any callers that need to be re-processed to STACK. */
3691 :
3692 : static bool
3693 1461308 : propagate_used_to_scc_callers (cgraph_node *node, void *data)
3694 : {
3695 1461308 : vec<cgraph_node *> *stack = (vec<cgraph_node *> *) data;
3696 1461308 : cgraph_edge *cs;
3697 3605638 : for (cs = node->callers; cs; cs = cs->next_caller)
3698 2144330 : if (ipa_edge_within_scc (cs))
3699 29644 : propagate_used_across_scc_edge (cs, stack);
3700 1461308 : return false;
3701 : }
3702 :
3703 : /* Return true iff all certain accesses in ARG_DESC are also present as
3704 : certain accesses in PARAM_DESC. */
3705 :
3706 : static bool
3707 82 : all_callee_accesses_present_p (isra_param_desc *param_desc,
3708 : isra_param_desc *arg_desc)
3709 : {
3710 82 : unsigned aclen = vec_safe_length (arg_desc->accesses);
3711 86 : for (unsigned j = 0; j < aclen; j++)
3712 : {
3713 50 : param_access *argacc = (*arg_desc->accesses)[j];
3714 50 : if (!argacc->certain)
3715 0 : continue;
3716 100 : param_access *pacc = find_param_access (param_desc, argacc->unit_offset,
3717 50 : argacc->unit_size);
3718 50 : if (!pacc
3719 5 : || !pacc->certain
3720 55 : || !types_compatible_p (argacc->type, pacc->type))
3721 : return false;
3722 : }
3723 : return true;
3724 : }
3725 :
3726 : /* Type internal to function pull_accesses_from_callee. Unfortunately gcc 4.8
3727 : does not allow instantiating an auto_vec with a type defined within a
3728 : function so it is a global type. */
3729 : enum acc_prop_kind {ACC_PROP_DONT, ACC_PROP_COPY, ACC_PROP_CERTAIN};
3730 :
3731 :
3732 : /* Attempt to propagate all definite accesses from ARG_DESC to PARAM_DESC,
3733 : (which belongs to CALLER) if they would not violate some constraint there.
3734 : CALLER_IPCP_TS describes the caller, PARAM_IDX is the index of the parameter
3735 : described by PARAM_DESC. If successful, return NULL, otherwise return the
3736 : string reason for failure (which can be written to the dump file).
3737 : DELTA_OFFSET is the known offset of the actual argument withing the formal
3738 : parameter (so of ARG_DESCS within PARAM_DESCS), ARG_SIZE is the size of the
3739 : actual argument or zero, if not known. In case of success, set *CHANGE_P to
3740 : true if propagation actually changed anything. */
3741 :
3742 : static const char *
3743 5525 : pull_accesses_from_callee (cgraph_node *caller,
3744 : ipcp_transformation *caller_ipcp_ts,
3745 : int param_idx,
3746 : isra_param_desc *param_desc,
3747 : isra_param_desc *arg_desc,
3748 : unsigned delta_offset, unsigned arg_size,
3749 : bool *change_p)
3750 : {
3751 5525 : unsigned pclen = vec_safe_length (param_desc->accesses);
3752 5525 : unsigned aclen = vec_safe_length (arg_desc->accesses);
3753 5525 : unsigned prop_count = 0;
3754 5525 : unsigned prop_size = 0;
3755 5525 : bool change = false;
3756 :
3757 5525 : auto_vec <enum acc_prop_kind, 8> prop_kinds (aclen);
3758 19217 : for (unsigned j = 0; j < aclen; j++)
3759 : {
3760 8372 : param_access *argacc = (*arg_desc->accesses)[j];
3761 8372 : prop_kinds.safe_push (ACC_PROP_DONT);
3762 :
3763 8372 : if (arg_size > 0
3764 2747 : && argacc->unit_offset + argacc->unit_size > arg_size)
3765 : return "callee access outsize size boundary";
3766 :
3767 8372 : if (!argacc->certain)
3768 379 : continue;
3769 :
3770 7993 : unsigned offset = argacc->unit_offset + delta_offset;
3771 :
3772 7993 : if (caller_ipcp_ts && !AGGREGATE_TYPE_P (argacc->type))
3773 : {
3774 407 : ipa_argagg_value_list avl (caller_ipcp_ts);
3775 407 : tree value = avl.get_value (param_idx, offset);
3776 407 : if (value && ((tree_to_uhwi (TYPE_SIZE (TREE_TYPE (value)))
3777 51 : / BITS_PER_UNIT)
3778 51 : != argacc->unit_size))
3779 0 : return " propagated access would conflict with an IPA-CP constant";
3780 : }
3781 :
3782 : /* Given that accesses are initially stored according to increasing
3783 : offset and decreasing size in case of equal offsets, the following
3784 : searches could be written more efficiently if we kept the ordering
3785 : when copying. But the number of accesses is capped at
3786 : PARAM_IPA_SRA_MAX_REPLACEMENTS (so most likely 8) and the code gets
3787 : messy quickly, so let's improve on that only if necessary. */
3788 :
3789 7993 : bool exact_match = false;
3790 16819 : for (unsigned i = 0; i < pclen; i++)
3791 : {
3792 : /* Check for overlaps. */
3793 9031 : param_access *pacc = (*param_desc->accesses)[i];
3794 9031 : if (pacc->unit_offset == offset
3795 4472 : && pacc->unit_size == argacc->unit_size)
3796 : {
3797 3288 : if (argacc->alias_ptr_type != pacc->alias_ptr_type
3798 3084 : || !types_compatible_p (argacc->type, pacc->type)
3799 6372 : || argacc->reverse != pacc->reverse)
3800 : return "propagated access types would not match existing ones";
3801 :
3802 3084 : exact_match = true;
3803 3084 : if (!pacc->certain)
3804 : {
3805 0 : prop_kinds[j] = ACC_PROP_CERTAIN;
3806 0 : prop_size += argacc->unit_size;
3807 0 : change = true;
3808 : }
3809 3084 : continue;
3810 : }
3811 :
3812 5743 : if (offset < pacc->unit_offset + pacc->unit_size
3813 3823 : && offset + argacc->unit_size > pacc->unit_offset)
3814 : {
3815 : /* None permissible with load accesses, possible to fit into
3816 : argument ones. */
3817 2369 : if (pacc->certain
3818 2368 : || offset < pacc->unit_offset
3819 2368 : || (offset + argacc->unit_size
3820 : > pacc->unit_offset + pacc->unit_size))
3821 : return "a propagated access would conflict in caller";
3822 : }
3823 : }
3824 :
3825 7788 : if (!exact_match)
3826 : {
3827 4704 : prop_kinds[j] = ACC_PROP_COPY;
3828 4704 : prop_count++;
3829 4704 : prop_size += argacc->unit_size;
3830 4704 : change = true;
3831 : }
3832 : }
3833 :
3834 5320 : if (!change)
3835 : return NULL;
3836 :
3837 3047 : if ((prop_count + pclen
3838 3047 : > (unsigned) opt_for_fn (caller->decl, param_ipa_sra_max_replacements))
3839 8572 : || size_would_violate_limit_p (param_desc,
3840 3047 : param_desc->size_reached + prop_size))
3841 : return "propagating accesses would violate the count or size limit";
3842 :
3843 3022 : *change_p = true;
3844 8095 : for (unsigned j = 0; j < aclen; j++)
3845 : {
3846 5073 : if (prop_kinds[j] == ACC_PROP_COPY)
3847 : {
3848 4659 : param_access *argacc = (*arg_desc->accesses)[j];
3849 :
3850 4659 : param_access *copy = ggc_cleared_alloc<param_access> ();
3851 4659 : copy->unit_offset = argacc->unit_offset + delta_offset;
3852 4659 : copy->unit_size = argacc->unit_size;
3853 4659 : copy->type = argacc->type;
3854 4659 : copy->alias_ptr_type = argacc->alias_ptr_type;
3855 4659 : copy->certain = true;
3856 4659 : copy->reverse = argacc->reverse;
3857 4659 : vec_safe_push (param_desc->accesses, copy);
3858 : }
3859 414 : else if (prop_kinds[j] == ACC_PROP_CERTAIN)
3860 : {
3861 0 : param_access *argacc = (*arg_desc->accesses)[j];
3862 0 : param_access *csp
3863 0 : = find_param_access (param_desc, argacc->unit_offset + delta_offset,
3864 0 : argacc->unit_size);
3865 0 : csp->certain = true;
3866 : }
3867 : }
3868 :
3869 3022 : param_desc->size_reached += prop_size;
3870 :
3871 3022 : return NULL;
3872 5525 : }
3873 :
3874 : /* Propagate parameter splitting information through call graph edge CS.
3875 : Return true if any changes that might need to be propagated within SCCs have
3876 : been made. The function also clears the aggregate_pass_through and
3877 : pointer_pass_through in call summaries which do not need to be processed
3878 : again if this CS is revisited when iterating while changes are propagated
3879 : within an SCC. */
3880 :
3881 : static bool
3882 1704255 : param_splitting_across_edge (cgraph_edge *cs)
3883 : {
3884 1704255 : bool res = false;
3885 1704255 : bool cross_scc = !ipa_edge_within_scc (cs);
3886 1704255 : enum availability availability;
3887 1704255 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3888 1704255 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3889 1704255 : gcc_checking_assert (from_ifs && from_ifs->m_parameters);
3890 1704255 : ipcp_transformation *caller_ipcp_ts
3891 1704255 : = ipcp_get_transformation_summary (cs->caller);
3892 :
3893 1704255 : isra_call_summary *csum = call_sums->get (cs);
3894 : /* The summary may not exist if the edge is a callback edge created after SRA
3895 : summaries are generated (when ipa-cp is disabled for example). Bail if
3896 : that happens. */
3897 1704255 : if (!csum)
3898 : {
3899 0 : gcc_checking_assert (callback_ifn_assume_call_p (cs->call_stmt));
3900 : return false;
3901 : }
3902 1704255 : unsigned args_count = csum->m_arg_flow.length ();
3903 1704255 : isra_func_summary *to_ifs = func_sums->get (callee);
3904 1704255 : unsigned param_count
3905 899225 : = ((to_ifs && to_ifs->m_candidate && (availability >= AVAIL_AVAILABLE))
3906 2500705 : ? vec_safe_length (to_ifs->m_parameters)
3907 : : 0);
3908 :
3909 1704255 : if (dump_file && (dump_flags & TDF_DETAILS))
3910 74 : fprintf (dump_file, "Splitting across %s->%s:\n",
3911 37 : cs->caller->dump_name (), callee->dump_name ());
3912 :
3913 1704255 : unsigned i;
3914 3340500 : for (i = 0; (i < args_count) && (i < param_count); i++)
3915 : {
3916 1636245 : isra_param_desc *arg_desc = &(*to_ifs->m_parameters)[i];
3917 1636245 : unsigned ipf_idx = i;
3918 1636245 : if (cs->callback)
3919 : {
3920 8 : gcc_assert (cs->callee->assume_function);
3921 8 : callback_info *ci = callback_info_sum->get (cs);
3922 8 : int remapped = ci->arg_mapping[i];
3923 8 : if (remapped == ARG_MAPPING_UNKNOWN_IDX)
3924 : /* In the case of assume outlined functions, the only skipped
3925 : argument is the address to be called, which is fine. In other
3926 : cases, if they ever arise, processing of the call to callback
3927 : dispatching functions will need to handle unmapped arguments as
3928 : necessary. */
3929 0 : continue;
3930 8 : ipf_idx = (unsigned) remapped;
3931 : }
3932 1636245 : isra_param_flow *ipf = &csum->m_arg_flow[ipf_idx];
3933 :
3934 1636245 : if (arg_desc->locally_unused)
3935 : {
3936 98295 : if (dump_file && (dump_flags & TDF_DETAILS))
3937 0 : fprintf (dump_file, " ->%u: unused in callee\n", i);
3938 98295 : ipf->pointer_pass_through = false;
3939 98295 : continue;
3940 : }
3941 :
3942 1537950 : if (ipf->pointer_pass_through)
3943 : {
3944 185339 : int idx = get_single_param_flow_source (ipf);
3945 185339 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3946 185339 : if (!param_desc->split_candidate)
3947 132684 : continue;
3948 52655 : gcc_assert (param_desc->by_ref);
3949 :
3950 52655 : if (!arg_desc->split_candidate || !arg_desc->by_ref)
3951 : {
3952 46280 : if (dump_file && (dump_flags & TDF_DETAILS))
3953 2 : fprintf (dump_file, " %u->%u: not candidate or not by "
3954 : "reference in callee\n", idx, i);
3955 46280 : param_desc->split_candidate = false;
3956 46280 : ipf->pointer_pass_through = false;
3957 46280 : res = true;
3958 : }
3959 6375 : else if (!ipf->safe_to_import_accesses)
3960 : {
3961 2177 : if (!csum->m_before_any_store
3962 2177 : || !all_callee_accesses_present_p (param_desc, arg_desc))
3963 : {
3964 2141 : if (dump_file && (dump_flags & TDF_DETAILS))
3965 0 : fprintf (dump_file, " %u->%u: cannot import accesses.\n",
3966 : idx, i);
3967 2141 : param_desc->split_candidate = false;
3968 2141 : ipf->pointer_pass_through = false;
3969 2141 : res = true;
3970 :
3971 : }
3972 : else
3973 : {
3974 36 : if (dump_file && (dump_flags & TDF_DETAILS))
3975 0 : fprintf (dump_file, " %u->%u: verified callee accesses "
3976 : "present.\n", idx, i);
3977 36 : if (cross_scc)
3978 0 : ipf->pointer_pass_through = false;
3979 : }
3980 : }
3981 : else
3982 : {
3983 4198 : const char *pull_failure
3984 4198 : = pull_accesses_from_callee (cs->caller, caller_ipcp_ts, idx,
3985 : param_desc, arg_desc, 0, 0, &res);
3986 4198 : if (pull_failure)
3987 : {
3988 18 : if (dump_file && (dump_flags & TDF_DETAILS))
3989 0 : fprintf (dump_file, " %u->%u: by_ref access pull "
3990 : "failed: %s.\n", idx, i, pull_failure);
3991 18 : param_desc->split_candidate = false;
3992 18 : ipf->pointer_pass_through = false;
3993 18 : res = true;
3994 : }
3995 : else
3996 : {
3997 4180 : if (dump_file && (dump_flags & TDF_DETAILS))
3998 6 : fprintf (dump_file, " %u->%u: by_ref access pull "
3999 : "succeeded.\n", idx, i);
4000 4180 : if (cross_scc)
4001 4023 : ipf->pointer_pass_through = false;
4002 : }
4003 : }
4004 : }
4005 1352611 : else if (ipf->aggregate_pass_through)
4006 : {
4007 34382 : int idx = get_single_param_flow_source (ipf);
4008 34382 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
4009 34382 : if (!param_desc->split_candidate)
4010 22000 : continue;
4011 12382 : gcc_assert (!param_desc->by_ref);
4012 24764 : param_access *pacc = find_param_access (param_desc, ipf->unit_offset,
4013 12382 : ipf->unit_size);
4014 12382 : gcc_checking_assert (pacc);
4015 :
4016 12382 : if (pacc->certain)
4017 : {
4018 1 : if (dump_file && (dump_flags & TDF_DETAILS))
4019 0 : fprintf (dump_file, " %u->%u: already certain\n", idx, i);
4020 1 : ipf->aggregate_pass_through = false;
4021 : }
4022 12381 : else if (!arg_desc->split_candidate || arg_desc->by_ref)
4023 : {
4024 11054 : if (dump_file && (dump_flags & TDF_DETAILS))
4025 0 : fprintf (dump_file, " %u->%u: not candidate or by "
4026 : "reference in callee\n", idx, i);
4027 :
4028 11054 : pacc->certain = true;
4029 11054 : if (overlapping_certain_accesses_p (param_desc, NULL))
4030 : {
4031 1380 : if (dump_file && (dump_flags & TDF_DETAILS))
4032 0 : fprintf (dump_file, " ...leading to overlap, "
4033 : " disqualifying candidate parameter %u\n",
4034 : idx);
4035 1380 : param_desc->split_candidate = false;
4036 : }
4037 : else
4038 9674 : bump_reached_size (param_desc, pacc->unit_size, idx);
4039 :
4040 11054 : ipf->aggregate_pass_through = false;
4041 11054 : res = true;
4042 : }
4043 : else
4044 : {
4045 1327 : const char *pull_failure
4046 1327 : = pull_accesses_from_callee (cs->caller, caller_ipcp_ts, idx,
4047 : param_desc, arg_desc,
4048 : ipf->unit_offset,
4049 : ipf->unit_size, &res);
4050 1327 : if (pull_failure)
4051 : {
4052 212 : if (dump_file && (dump_flags & TDF_DETAILS))
4053 0 : fprintf (dump_file, " %u->%u: arg access pull "
4054 : "failed: %s.\n", idx, i, pull_failure);
4055 :
4056 212 : ipf->aggregate_pass_through = false;
4057 212 : pacc->certain = true;
4058 :
4059 212 : if (overlapping_certain_accesses_p (param_desc, NULL))
4060 : {
4061 212 : if (dump_file && (dump_flags & TDF_DETAILS))
4062 0 : fprintf (dump_file, " ...leading to overlap, "
4063 : " disqualifying candidate parameter %u\n",
4064 : idx);
4065 212 : param_desc->split_candidate = false;
4066 : }
4067 : else
4068 0 : bump_reached_size (param_desc, pacc->unit_size, idx);
4069 :
4070 212 : res = true;
4071 : }
4072 : else
4073 : {
4074 1115 : if (dump_file && (dump_flags & TDF_DETAILS))
4075 2 : fprintf (dump_file, " %u->%u: arg access pull "
4076 : "succeeded.\n", idx, i);
4077 1115 : if (cross_scc)
4078 980 : ipf->aggregate_pass_through = false;
4079 : }
4080 : }
4081 : }
4082 : }
4083 :
4084 : /* Handle argument-parameter count mismatches. */
4085 2758658 : for (; (i < args_count); i++)
4086 : {
4087 1054403 : isra_param_flow *ipf = &csum->m_arg_flow[i];
4088 :
4089 1054403 : if (ipf->pointer_pass_through || ipf->aggregate_pass_through)
4090 : {
4091 203164 : int idx = get_single_param_flow_source (ipf);
4092 203164 : ipf->pointer_pass_through = false;
4093 203164 : ipf->aggregate_pass_through = false;
4094 203164 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
4095 203164 : if (!param_desc->split_candidate)
4096 203081 : continue;
4097 :
4098 83 : if (dump_file && (dump_flags & TDF_DETAILS))
4099 0 : fprintf (dump_file, " %u->%u: no corresponding formal parameter\n",
4100 : idx, i);
4101 83 : param_desc->split_candidate = false;
4102 83 : res = true;
4103 : }
4104 : }
4105 1704255 : return res;
4106 : }
4107 :
4108 : /* Worker for call_for_symbol_and_aliases, look at all callers and if all their
4109 : callers ignore the return value, or come from the same SCC and use the
4110 : return value only to compute their return value, return false, otherwise
4111 : return true. */
4112 :
4113 : static bool
4114 191900 : retval_used_p (cgraph_node *node, void *)
4115 : {
4116 299921 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4117 : {
4118 264662 : isra_call_summary *csum = call_sums->get (cs);
4119 264662 : gcc_checking_assert (csum);
4120 264662 : if (csum->m_return_ignored)
4121 102172 : continue;
4122 162490 : if (!csum->m_return_returned)
4123 : return true;
4124 :
4125 27628 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
4126 27628 : if (!from_ifs || !from_ifs->m_candidate)
4127 : return true;
4128 :
4129 20970 : if (!ipa_edge_within_scc (cs)
4130 20970 : && !from_ifs->m_return_ignored)
4131 : return true;
4132 : }
4133 :
4134 : /* Keep the return value, even though it's not used in the code directly,
4135 : because it's used by vrp later. */
4136 35259 : if (node->assume_function)
4137 0 : return true;
4138 :
4139 : return false;
4140 : }
4141 :
4142 : /* Push into NEW_PARAMS all required parameter adjustment entries to copy or
4143 : modify parameter which originally had index BASE_INDEX, in the adjustment
4144 : vector of parent clone (if any) had PREV_CLONE_INDEX and was described by
4145 : PREV_ADJUSTMENT. If IPA-CP has created a transformation summary for the
4146 : original node, it needs to be passed in IPCP_TS, otherwise it should be
4147 : NULL. If the parent clone is the original function, PREV_ADJUSTMENT is NULL
4148 : and PREV_CLONE_INDEX is equal to BASE_INDEX. */
4149 :
4150 : static void
4151 344372 : push_param_adjustments_for_index (isra_func_summary *ifs, unsigned base_index,
4152 : unsigned prev_clone_index,
4153 : ipa_adjusted_param *prev_adjustment,
4154 : ipcp_transformation *ipcp_ts,
4155 : vec<ipa_adjusted_param, va_gc> **new_params)
4156 : {
4157 344372 : isra_param_desc *desc = &(*ifs->m_parameters)[base_index];
4158 344372 : if (desc->locally_unused)
4159 : {
4160 73621 : if (dump_file)
4161 27 : fprintf (dump_file, " Will remove parameter %u\n", base_index);
4162 : return;
4163 : }
4164 :
4165 270751 : if (!desc->split_candidate)
4166 : {
4167 221061 : ipa_adjusted_param adj;
4168 221061 : if (prev_adjustment)
4169 : {
4170 1678 : adj = *prev_adjustment;
4171 1678 : adj.prev_clone_adjustment = true;
4172 1678 : adj.prev_clone_index = prev_clone_index;
4173 : }
4174 : else
4175 : {
4176 219383 : memset (&adj, 0, sizeof (adj));
4177 219383 : adj.op = IPA_PARAM_OP_COPY;
4178 219383 : adj.base_index = base_index;
4179 219383 : adj.prev_clone_index = prev_clone_index;
4180 : }
4181 221061 : vec_safe_push ((*new_params), adj);
4182 221061 : return;
4183 : }
4184 :
4185 49690 : if (dump_file)
4186 62 : fprintf (dump_file, " Will split parameter %u\n", base_index);
4187 :
4188 49690 : gcc_assert (!prev_adjustment || prev_adjustment->op == IPA_PARAM_OP_COPY);
4189 49690 : unsigned aclen = vec_safe_length (desc->accesses);
4190 138587 : for (unsigned j = 0; j < aclen; j++)
4191 : {
4192 88897 : param_access *pa = (*desc->accesses)[j];
4193 88897 : if (!pa->certain)
4194 7996 : continue;
4195 :
4196 88015 : if (ipcp_ts)
4197 : {
4198 15432 : ipa_argagg_value_list avl (ipcp_ts);
4199 15432 : tree value = avl.get_value (base_index, pa->unit_offset);
4200 15432 : if (value && !AGGREGATE_TYPE_P (pa->type))
4201 : {
4202 7114 : if (dump_file)
4203 28 : fprintf (dump_file, " - omitting component at byte "
4204 : "offset %u which is known to have a constant value\n ",
4205 : pa->unit_offset);
4206 7114 : continue;
4207 : }
4208 : }
4209 :
4210 80901 : if (dump_file)
4211 61 : fprintf (dump_file, " - component at byte offset %u, "
4212 61 : "size %u\n", pa->unit_offset, pa->unit_size);
4213 :
4214 80901 : ipa_adjusted_param adj;
4215 80901 : memset (&adj, 0, sizeof (adj));
4216 80901 : adj.op = IPA_PARAM_OP_SPLIT;
4217 80901 : adj.base_index = base_index;
4218 80901 : adj.prev_clone_index = prev_clone_index;
4219 80901 : adj.param_prefix_index = IPA_PARAM_PREFIX_ISRA;
4220 80901 : adj.reverse = pa->reverse;
4221 80901 : adj.type = pa->type;
4222 80901 : adj.alias_ptr_type = pa->alias_ptr_type;
4223 80901 : adj.unit_offset = pa->unit_offset;
4224 80901 : vec_safe_push ((*new_params), adj);
4225 : }
4226 : }
4227 :
4228 : /* Worker for all call_for_symbol_thunks_and_aliases. Set calls_comdat_local
4229 : flag of all callers of NODE. */
4230 :
4231 : static bool
4232 0 : mark_callers_calls_comdat_local (struct cgraph_node *node, void *)
4233 : {
4234 0 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4235 0 : cs->caller->calls_comdat_local = true;
4236 0 : return false;
4237 : }
4238 :
4239 : /* Remove any IPA-CP results stored in TS that are associated with removed
4240 : parameters as marked in IFS. */
4241 :
4242 : static void
4243 8583 : zap_useless_ipcp_results (const isra_func_summary *ifs, ipcp_transformation *ts)
4244 : {
4245 8583 : ts->remove_argaggs_if ([ifs](const ipa_argagg_value &v)
4246 : {
4247 15944 : return (*ifs->m_parameters)[v.index].locally_unused;
4248 : });
4249 :
4250 8583 : bool useful_vr = false;
4251 8583 : unsigned count = vec_safe_length (ts->m_vr);
4252 30309 : for (unsigned i = 0; i < count; i++)
4253 21726 : if ((*ts->m_vr)[i].known_p ())
4254 : {
4255 15139 : const isra_param_desc *desc = &(*ifs->m_parameters)[i];
4256 15139 : if (desc->locally_unused)
4257 1942 : (*ts->m_vr)[i].set_unknown ();
4258 : else
4259 : useful_vr = true;
4260 : }
4261 8583 : if (!useful_vr)
4262 1224 : ts->m_vr = NULL;
4263 8583 : }
4264 :
4265 : /* Do final processing of results of IPA propagation regarding NODE, clone it
4266 : if appropriate. */
4267 :
4268 : static void
4269 1304448 : process_isra_node_results (cgraph_node *node,
4270 : hash_map<const char *, unsigned> *clone_num_suffixes)
4271 : {
4272 1304448 : isra_func_summary *ifs = func_sums->get (node);
4273 1304448 : if (!ifs || !ifs->m_candidate)
4274 1180368 : return;
4275 :
4276 386282 : auto_vec<bool, 16> surviving_params;
4277 386282 : bool check_surviving = false;
4278 386282 : clone_info *cinfo = clone_info::get (node);
4279 386282 : if (cinfo && cinfo->param_adjustments)
4280 : {
4281 13205 : check_surviving = true;
4282 13205 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
4283 : }
4284 :
4285 386282 : unsigned param_count = vec_safe_length (ifs->m_parameters);
4286 386282 : bool will_change_function = false;
4287 386282 : if (ifs->m_returns_value && ifs->m_return_ignored)
4288 : will_change_function = true;
4289 : else
4290 879723 : for (unsigned i = 0; i < param_count; i++)
4291 : {
4292 617521 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
4293 562000 : if ((desc->locally_unused || desc->split_candidate)
4294 : /* Make sure we do not clone just to attempt to remove an already
4295 : removed unused argument. */
4296 654143 : && (!check_surviving
4297 530197 : || (i < surviving_params.length ()
4298 2063 : && surviving_params[i])))
4299 : {
4300 : will_change_function = true;
4301 : break;
4302 : }
4303 : }
4304 351589 : if (!will_change_function)
4305 262202 : return;
4306 :
4307 124080 : if (dump_file)
4308 : {
4309 84 : fprintf (dump_file, "\nEvaluating analysis results for %s\n",
4310 : node->dump_name ());
4311 84 : if (ifs->m_returns_value && ifs->m_return_ignored)
4312 15 : fprintf (dump_file, " Will remove return value.\n");
4313 : }
4314 :
4315 124080 : ipcp_transformation *ipcp_ts = ipcp_get_transformation_summary (node);
4316 124080 : if (ipcp_ts)
4317 8583 : zap_useless_ipcp_results (ifs, ipcp_ts);
4318 124080 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
4319 124080 : if (ipa_param_adjustments *old_adjustments
4320 124080 : = cinfo ? cinfo->param_adjustments : NULL)
4321 : {
4322 2496 : unsigned old_adj_len = vec_safe_length (old_adjustments->m_adj_params);
4323 5896 : for (unsigned i = 0; i < old_adj_len; i++)
4324 : {
4325 3400 : ipa_adjusted_param *old_adj = &(*old_adjustments->m_adj_params)[i];
4326 3400 : push_param_adjustments_for_index (ifs, old_adj->base_index, i,
4327 : old_adj, ipcp_ts, &new_params);
4328 : }
4329 : }
4330 : else
4331 462556 : for (unsigned i = 0; i < param_count; i++)
4332 340972 : push_param_adjustments_for_index (ifs, i, i, NULL, ipcp_ts, &new_params);
4333 :
4334 124080 : ipa_param_adjustments *new_adjustments
4335 124080 : = (new (ggc_alloc <ipa_param_adjustments> ())
4336 : ipa_param_adjustments (new_params, param_count,
4337 213467 : ifs->m_returns_value && ifs->m_return_ignored));
4338 :
4339 124080 : if (dump_file && (dump_flags & TDF_DETAILS))
4340 : {
4341 32 : fprintf (dump_file, "\n Created adjustments:\n");
4342 32 : new_adjustments->dump (dump_file);
4343 : }
4344 :
4345 372240 : unsigned &suffix_counter = clone_num_suffixes->get_or_insert (
4346 124080 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (
4347 : node->decl)));
4348 124080 : auto_vec<cgraph_edge *> callers = node->collect_callers ();
4349 124080 : cgraph_node *new_node
4350 124080 : = node->create_virtual_clone (callers, NULL, new_adjustments, "isra",
4351 : suffix_counter);
4352 124080 : suffix_counter++;
4353 124080 : if (node->calls_comdat_local && node->same_comdat_group)
4354 : {
4355 0 : new_node->add_to_same_comdat_group (node);
4356 0 : new_node->call_for_symbol_and_aliases (mark_callers_calls_comdat_local,
4357 : NULL, true);
4358 : }
4359 124080 : new_node->calls_comdat_local = node->calls_comdat_local;
4360 :
4361 124080 : if (dump_file)
4362 84 : fprintf (dump_file, " Created new node %s\n", new_node->dump_name ());
4363 124080 : callers.release ();
4364 386282 : }
4365 :
4366 : /* If INDICES is not empty, dump a combination of NODE's dump_name and MSG
4367 : followed by the list of numbers in INDICES. */
4368 :
4369 : static void
4370 660162 : dump_list_of_param_indices (const cgraph_node *node, const char* msg,
4371 : const vec<unsigned> &indices)
4372 : {
4373 660162 : if (indices.is_empty ())
4374 : return;
4375 13 : fprintf (dump_file, "The following parameters of %s %s:", node->dump_name (),
4376 : msg);
4377 52 : for (unsigned i : indices)
4378 13 : fprintf (dump_file, " %u", i);
4379 13 : fprintf (dump_file, "\n");
4380 : }
4381 :
4382 : /* Check which parameters of NODE described by IFS have survived until IPA-SRA
4383 : and disable transformations for those which have not or which should not
4384 : transformed because the associated debug counter reached its limit. Return
4385 : true if none survived or if there were no candidates to begin with.
4386 : Additionally, also adjust parameter descriptions based on debug counters and
4387 : hints propagated earlier. */
4388 :
4389 : static bool
4390 386282 : adjust_parameter_descriptions (cgraph_node *node, isra_func_summary *ifs)
4391 : {
4392 386282 : bool ret = true;
4393 386282 : unsigned len = vec_safe_length (ifs->m_parameters);
4394 330081 : if (!len)
4395 : return true;
4396 :
4397 330081 : auto_vec<bool, 16> surviving_params;
4398 330081 : bool check_surviving = false;
4399 330081 : clone_info *cinfo = clone_info::get (node);
4400 330081 : if (cinfo && cinfo->param_adjustments)
4401 : {
4402 13205 : check_surviving = true;
4403 13205 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
4404 : }
4405 330081 : ipcp_transformation *ipcp_ts = ipcp_get_transformation_summary (node);
4406 330081 : auto_vec <unsigned> dump_dead_indices;
4407 330081 : auto_vec <unsigned> dump_bad_cond_indices;
4408 1130655 : for (unsigned i = 0; i < len; i++)
4409 : {
4410 800574 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
4411 800574 : if (!dbg_cnt (ipa_sra_params))
4412 : {
4413 0 : desc->locally_unused = false;
4414 0 : desc->split_candidate = false;
4415 0 : continue;
4416 : }
4417 :
4418 800574 : if (desc->split_only_when_retval_removed
4419 6910 : && !ifs->m_return_ignored)
4420 : {
4421 0 : if (dump_file && (dump_flags & TDF_DETAILS)
4422 1526 : && (desc->locally_unused || desc->split_candidate))
4423 0 : dump_bad_cond_indices.safe_push (i);
4424 :
4425 1526 : gcc_checking_assert (!desc->locally_unused
4426 : || desc->remove_only_when_retval_removed);
4427 1526 : desc->locally_unused = false;
4428 1526 : desc->split_candidate = false;
4429 1526 : continue;
4430 : }
4431 799048 : if (desc->remove_only_when_retval_removed
4432 45964 : && !ifs->m_return_ignored)
4433 : {
4434 18 : if (dump_file && (dump_flags & TDF_DETAILS)
4435 36593 : && (desc->locally_unused || desc->split_candidate))
4436 12 : dump_bad_cond_indices.safe_push (i);
4437 :
4438 36581 : desc->locally_unused = false;
4439 : }
4440 799048 : if (check_surviving
4441 836973 : && (i >= surviving_params.length ()
4442 19157 : || !surviving_params[i]))
4443 : {
4444 : /* Even if the parameter was removed by a previous IPA pass, we do
4445 : not clear locally_unused because if it really is unused, this
4446 : information might be useful in callers. */
4447 22726 : desc->split_candidate = false;
4448 :
4449 22726 : if (dump_file && (dump_flags & TDF_DETAILS))
4450 0 : dump_dead_indices.safe_push (i);
4451 : }
4452 :
4453 799048 : if (desc->split_candidate && desc->conditionally_dereferenceable)
4454 : {
4455 886 : gcc_assert (desc->safe_size_set);
4456 1483 : for (param_access *pa : *desc->accesses)
4457 1329 : if ((pa->unit_offset + pa->unit_size) > desc->safe_size)
4458 : {
4459 732 : if (dump_file && (dump_flags & TDF_DETAILS))
4460 1 : dump_bad_cond_indices.safe_push (i);
4461 732 : desc->split_candidate = false;
4462 732 : break;
4463 : }
4464 : }
4465 :
4466 799048 : if (desc->split_candidate)
4467 : {
4468 212799 : if (desc->by_ref && !desc->not_specially_constructed)
4469 : {
4470 32498 : int extra_factor
4471 32498 : = opt_for_fn (node->decl,
4472 : param_ipa_sra_ptrwrap_growth_factor);
4473 32498 : desc->param_size_limit = extra_factor * desc->param_size_limit;
4474 : }
4475 : /* Don't disqualify assume bodies, as they don't make into the final
4476 : binary anyway. */
4477 212799 : if (!node->assume_function
4478 212799 : && size_would_violate_limit_p (desc, desc->size_reached))
4479 3049 : desc->split_candidate = false;
4480 : }
4481 :
4482 : /* Avoid ICEs on size-mismatched VIEW_CONVERT_EXPRs when callers and
4483 : callees don't agree on types in aggregates and we try to do both
4484 : IPA-CP and IPA-SRA. */
4485 799048 : if (ipcp_ts && desc->split_candidate)
4486 : {
4487 22860 : ipa_argagg_value_list avl (ipcp_ts);
4488 58645 : for (const param_access *pa : desc->accesses)
4489 : {
4490 17787 : if (!pa->certain)
4491 1739 : continue;
4492 16048 : tree value = avl.get_value (i, pa->unit_offset);
4493 16048 : if (value
4494 16048 : && ((tree_to_uhwi (TYPE_SIZE (TREE_TYPE (value)))
4495 7220 : / BITS_PER_UNIT)
4496 7220 : != pa->unit_size))
4497 : {
4498 48 : desc->split_candidate = false;
4499 48 : if (dump_file && (dump_flags & TDF_DETAILS))
4500 0 : dump_dead_indices.safe_push (i);
4501 : break;
4502 : }
4503 : }
4504 : }
4505 :
4506 799048 : if (desc->locally_unused || desc->split_candidate)
4507 800574 : ret = false;
4508 : }
4509 :
4510 330081 : dump_list_of_param_indices (node, "are dead on arrival or have a type "
4511 : "mismatch with IPA-CP", dump_dead_indices);
4512 330081 : dump_list_of_param_indices (node, "fail additional requirements ",
4513 : dump_bad_cond_indices);
4514 :
4515 330081 : return ret;
4516 330081 : }
4517 :
4518 :
4519 : /* Run the interprocedural part of IPA-SRA. */
4520 :
4521 : static unsigned int
4522 130649 : ipa_sra_analysis (void)
4523 : {
4524 130649 : if (dump_file)
4525 : {
4526 62 : fprintf (dump_file, "\n========== IPA-SRA IPA stage ==========\n");
4527 62 : ipa_sra_dump_all_summaries (dump_file, false);
4528 : }
4529 :
4530 130649 : gcc_checking_assert (func_sums);
4531 130649 : gcc_checking_assert (call_sums);
4532 130649 : cgraph_node **order = XCNEWVEC (cgraph_node *, symtab->cgraph_count);
4533 130649 : auto_vec <cgraph_node *, 16> stack;
4534 130649 : int node_scc_count = ipa_reduced_postorder (order, true, NULL);
4535 :
4536 : /* One sweep from callers to callees for return value removal. */
4537 1506547 : for (int i = node_scc_count - 1; i >= 0 ; i--)
4538 : {
4539 1375898 : cgraph_node *scc_rep = order[i];
4540 1375898 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (scc_rep);
4541 :
4542 : /* Preliminary IPA function level checks. */
4543 5509567 : for (cgraph_node *v : cycle_nodes)
4544 : {
4545 1381873 : isra_func_summary *ifs = func_sums->get (v);
4546 1381873 : if (!ifs || !ifs->m_candidate)
4547 325165 : continue;
4548 1056708 : if (!ipa_sra_ipa_function_checks (v)
4549 1056708 : || check_all_callers_for_issues (v))
4550 670426 : ifs->zap ();
4551 : }
4552 :
4553 4133669 : for (cgraph_node *v : cycle_nodes)
4554 : {
4555 1381873 : isra_func_summary *ifs = func_sums->get (v);
4556 1381873 : if (!ifs || !ifs->m_candidate)
4557 995591 : continue;
4558 386282 : bool return_needed
4559 386282 : = (ifs->m_returns_value
4560 386282 : && (!dbg_cnt (ipa_sra_retvalues)
4561 191878 : || v->call_for_symbol_and_aliases (retval_used_p,
4562 386282 : NULL, true)));
4563 386282 : ifs->m_return_ignored = !return_needed;
4564 386282 : if (return_needed)
4565 313282 : isra_push_node_to_stack (v, ifs, &stack);
4566 : }
4567 :
4568 1533083 : while (!stack.is_empty ())
4569 : {
4570 157185 : cgraph_node *node = stack.pop ();
4571 157185 : isra_func_summary *ifs = func_sums->get (node);
4572 157185 : gcc_checking_assert (ifs && ifs->m_queued);
4573 157185 : ifs->m_queued = false;
4574 :
4575 873314 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
4576 716129 : if (ipa_edge_within_scc (cs)
4577 716129 : && call_sums->get (cs)->m_return_returned)
4578 : {
4579 1253 : enum availability av;
4580 1253 : cgraph_node *callee = cs->callee->function_symbol (&av);
4581 1253 : isra_func_summary *to_ifs = func_sums->get (callee);
4582 1253 : if (to_ifs && to_ifs->m_return_ignored)
4583 : {
4584 544 : to_ifs->m_return_ignored = false;
4585 1088 : isra_push_node_to_stack (callee, to_ifs, &stack);
4586 : }
4587 : }
4588 : }
4589 :
4590 : /* Parameter hint propagation. */
4591 4133669 : for (cgraph_node *v : cycle_nodes)
4592 : {
4593 1381873 : isra_func_summary *ifs = func_sums->get (v);
4594 1381873 : propagate_hints_to_all_callees (v, ifs, &stack);
4595 : }
4596 :
4597 1378654 : while (!stack.is_empty ())
4598 : {
4599 2756 : cgraph_node *node = stack.pop ();
4600 2756 : isra_func_summary *ifs = func_sums->get (node);
4601 2756 : gcc_checking_assert (ifs && ifs->m_queued);
4602 2756 : ifs->m_queued = false;
4603 2756 : propagate_hints_to_all_callees (node, ifs, &stack);
4604 : }
4605 :
4606 1375898 : cycle_nodes.release ();
4607 : }
4608 :
4609 : /* One sweep from callees to callers for parameter removal and splitting. */
4610 1506547 : for (int i = 0; i < node_scc_count; i++)
4611 : {
4612 1375898 : cgraph_node *scc_rep = order[i];
4613 1375898 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (scc_rep);
4614 :
4615 : /* First step of parameter removal. */
4616 5509567 : for (cgraph_node *v : cycle_nodes)
4617 : {
4618 1381873 : isra_func_summary *ifs = func_sums->get (v);
4619 1381873 : if (!ifs || !ifs->m_candidate)
4620 995591 : continue;
4621 386282 : if (adjust_parameter_descriptions (v, ifs))
4622 198386 : continue;
4623 202053 : for (cgraph_edge *cs = v->indirect_calls; cs; cs = cs->next_callee)
4624 14157 : process_edge_to_unknown_caller (cs);
4625 1100546 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
4626 912650 : if (!ipa_edge_within_scc (cs))
4627 907024 : param_removal_cross_scc_edge (cs);
4628 : }
4629 :
4630 : /* Look at edges within the current SCC and propagate used-ness across
4631 : them, pushing onto the stack all notes which might need to be
4632 : revisited. */
4633 4133669 : for (cgraph_node *v : cycle_nodes)
4634 1381873 : v->call_for_symbol_thunks_and_aliases (propagate_used_to_scc_callers,
4635 : &stack, true);
4636 :
4637 : /* Keep revisiting and pushing until nothing changes. */
4638 1376678 : while (!stack.is_empty ())
4639 : {
4640 780 : cgraph_node *v = stack.pop ();
4641 780 : isra_func_summary *ifs = func_sums->get (v);
4642 780 : gcc_checking_assert (ifs && ifs->m_queued);
4643 780 : ifs->m_queued = false;
4644 :
4645 780 : v->call_for_symbol_thunks_and_aliases (propagate_used_to_scc_callers,
4646 : &stack, true);
4647 : }
4648 :
4649 : /* Parameter splitting. */
4650 1424396 : bool repeat_scc_access_propagation;
4651 1424396 : do
4652 : {
4653 1424396 : repeat_scc_access_propagation = false;
4654 4282105 : for (cgraph_node *v : cycle_nodes)
4655 : {
4656 1433313 : isra_func_summary *ifs = func_sums->get (v);
4657 2485734 : if (!ifs
4658 1111196 : || !ifs->m_candidate
4659 1870406 : || vec_safe_is_empty (ifs->m_parameters))
4660 1052421 : continue;
4661 2085147 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
4662 1704255 : if (param_splitting_across_edge (cs))
4663 52455 : repeat_scc_access_propagation = true;
4664 : }
4665 : }
4666 : while (repeat_scc_access_propagation);
4667 :
4668 1375898 : if (flag_checking)
4669 4133618 : for (cgraph_node *v : cycle_nodes)
4670 1381856 : verify_splitting_accesses (v, true);
4671 :
4672 1375898 : cycle_nodes.release ();
4673 : }
4674 :
4675 130649 : ipa_free_postorder_info ();
4676 130649 : free (order);
4677 :
4678 130649 : if (dump_file)
4679 : {
4680 62 : if (dump_flags & TDF_DETAILS)
4681 : {
4682 17 : fprintf (dump_file, "\n========== IPA-SRA propagation final state "
4683 : " ==========\n");
4684 17 : ipa_sra_dump_all_summaries (dump_file, true);
4685 : }
4686 62 : fprintf (dump_file, "\n========== IPA-SRA decisions ==========\n");
4687 : }
4688 :
4689 130649 : hash_map<const char *, unsigned> *clone_num_suffixes
4690 130649 : = new hash_map<const char *, unsigned>;
4691 :
4692 130649 : cgraph_node *node;
4693 1435097 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
4694 1304448 : process_isra_node_results (node, clone_num_suffixes);
4695 :
4696 130649 : delete clone_num_suffixes;
4697 130649 : ggc_delete (func_sums);
4698 130649 : func_sums = NULL;
4699 130649 : delete call_sums;
4700 130649 : call_sums = NULL;
4701 :
4702 130649 : if (dump_file)
4703 62 : fprintf (dump_file, "\n========== IPA SRA IPA analysis done "
4704 : "==========\n\n");
4705 130649 : return 0;
4706 130649 : }
4707 :
4708 :
4709 : const pass_data pass_data_ipa_sra =
4710 : {
4711 : IPA_PASS, /* type */
4712 : "sra", /* name */
4713 : OPTGROUP_NONE, /* optinfo_flags */
4714 : TV_IPA_SRA, /* tv_id */
4715 : 0, /* properties_required */
4716 : 0, /* properties_provided */
4717 : 0, /* properties_destroyed */
4718 : 0, /* todo_flags_start */
4719 : ( TODO_dump_symtab | TODO_remove_functions ), /* todo_flags_finish */
4720 : };
4721 :
4722 : class pass_ipa_sra : public ipa_opt_pass_d
4723 : {
4724 : public:
4725 297180 : pass_ipa_sra (gcc::context *ctxt)
4726 : : ipa_opt_pass_d (pass_data_ipa_sra, ctxt,
4727 : ipa_sra_generate_summary, /* generate_summary */
4728 : ipa_sra_write_summary, /* write_summary */
4729 : ipa_sra_read_summary, /* read_summary */
4730 : NULL , /* write_optimization_summary */
4731 : NULL, /* read_optimization_summary */
4732 : NULL, /* stmt_fixup */
4733 : 0, /* function_transform_todo_flags_start */
4734 : NULL, /* function_transform */
4735 297180 : NULL) /* variable_transform */
4736 297180 : {}
4737 :
4738 : /* opt_pass methods: */
4739 609367 : bool gate (function *) final override
4740 : {
4741 : /* TODO: We should remove the optimize check after we ensure we never run
4742 : IPA passes when not optimizing. */
4743 609367 : return (flag_ipa_sra && optimize);
4744 : }
4745 :
4746 130649 : unsigned int execute (function *) final override
4747 : {
4748 130649 : return ipa_sra_analysis ();
4749 : }
4750 :
4751 : }; // class pass_ipa_sra
4752 :
4753 : } // anon namespace
4754 :
4755 : /* Intraprocedural part of IPA-SRA analysis. Scan function body of NODE and
4756 : create a summary structure describing IPA-SRA opportunities and constraints
4757 : in it. */
4758 :
4759 : static void
4760 1304084 : ipa_sra_summarize_function (cgraph_node *node)
4761 : {
4762 1304084 : if (dump_file)
4763 226 : fprintf (dump_file, "Creating summary for %s/%i:\n", node->name (),
4764 : node->get_uid ());
4765 1304084 : gcc_obstack_init (&gensum_obstack);
4766 1304084 : loaded_decls = new hash_set<tree>;
4767 :
4768 1304084 : isra_func_summary *ifs = NULL;
4769 1304084 : unsigned count = 0;
4770 1304084 : if (ipa_sra_preliminary_function_checks (node))
4771 : {
4772 1063309 : ifs = func_sums->get_create (node);
4773 1063309 : ifs->m_candidate = true;
4774 1063309 : tree ret = TREE_TYPE (TREE_TYPE (node->decl));
4775 1063309 : ifs->m_returns_value = (TREE_CODE (ret) != VOID_TYPE);
4776 1063309 : for (tree parm = DECL_ARGUMENTS (node->decl);
4777 3510717 : parm;
4778 2447408 : parm = DECL_CHAIN (parm))
4779 2447408 : count++;
4780 : }
4781 1304084 : auto_vec<gensum_param_desc, 16> param_descriptions (count);
4782 :
4783 1304084 : struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
4784 1304084 : bool cfun_pushed = false;
4785 1304084 : if (count > 0)
4786 : {
4787 861037 : decl2desc = new hash_map<tree, gensum_param_desc *>;
4788 861037 : param_descriptions.reserve_exact (count);
4789 861037 : param_descriptions.quick_grow_cleared (count);
4790 :
4791 861037 : if (create_parameter_descriptors (node, ¶m_descriptions))
4792 : {
4793 426790 : push_cfun (fun);
4794 426790 : cfun_pushed = true;
4795 426790 : final_bbs = BITMAP_ALLOC (NULL);
4796 426790 : bb_dereferences = XCNEWVEC (HOST_WIDE_INT,
4797 : unsafe_by_ref_count
4798 : * last_basic_block_for_fn (fun));
4799 426790 : aa_walking_limit = opt_for_fn (node->decl, param_ipa_max_aa_steps);
4800 : }
4801 : }
4802 : /* Scan function is run even when there are no removal or splitting
4803 : candidates so that we can calculate hints on call edges which can be
4804 : useful in callees. */
4805 1304084 : scan_function (node, fun);
4806 :
4807 1304084 : if (count > 0)
4808 : {
4809 861037 : if (dump_file)
4810 : {
4811 172 : dump_gensum_param_descriptors (dump_file, node->decl,
4812 : ¶m_descriptions);
4813 172 : fprintf (dump_file, "----------------------------------------\n");
4814 : }
4815 :
4816 861037 : process_scan_results (node, fun, ifs, ¶m_descriptions);
4817 :
4818 861037 : if (cfun_pushed)
4819 426790 : pop_cfun ();
4820 861037 : if (bb_dereferences)
4821 : {
4822 426790 : free (bb_dereferences);
4823 426790 : bb_dereferences = NULL;
4824 426790 : BITMAP_FREE (final_bbs);
4825 426790 : final_bbs = NULL;
4826 : }
4827 : }
4828 1304084 : isra_analyze_all_outgoing_calls (node);
4829 :
4830 2608168 : delete loaded_decls;
4831 1304084 : loaded_decls = NULL;
4832 1304084 : if (decl2desc)
4833 : {
4834 861037 : delete decl2desc;
4835 861037 : decl2desc = NULL;
4836 : }
4837 1304084 : obstack_free (&gensum_obstack, NULL);
4838 1304084 : if (dump_file)
4839 226 : fprintf (dump_file, "\n\n");
4840 1304084 : if (flag_checking)
4841 1304066 : verify_splitting_accesses (node, false);
4842 1304084 : return;
4843 1304084 : }
4844 :
4845 : ipa_opt_pass_d *
4846 297180 : make_pass_ipa_sra (gcc::context *ctxt)
4847 : {
4848 297180 : return new pass_ipa_sra (ctxt);
4849 : }
4850 :
4851 : /* Reset all state within ipa-sra.cc so that we can rerun the compiler
4852 : within the same process. For use by toplev::finalize. */
4853 :
4854 : void
4855 266934 : ipa_sra_cc_finalize (void)
4856 : {
4857 266934 : if (func_sums)
4858 9390 : ggc_delete (func_sums);
4859 266934 : func_sums = NULL;
4860 266934 : delete call_sums;
4861 266934 : call_sums = NULL;
4862 266934 : }
4863 :
4864 : #include "gt-ipa-sra.h"
|