Branch data Line data Source code
1 : : /* Interprocedural scalar replacement of aggregates
2 : : Copyright (C) 2019-2025 Free Software Foundation, Inc.
3 : : Contributed by Martin Jambor <mjambor@suse.cz>
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : /* IPA-SRA is an interprocedural pass that removes unused function return
22 : : values (turning functions returning a value which is never used into void
23 : : functions) and removes unused function parameters. It can also replace an
24 : : aggregate parameter by a set of other parameters representing part of the
25 : : original, turning those passed by reference into new ones which pass the
26 : : value directly.
27 : :
28 : : The pass is a true IPA one, which means that it works in three stages in
29 : : order to be able to take advantage of LTO. First, summaries about functions
30 : : and each calls are generated. Function summaries (often called call graph
31 : : node summaries) contain mainly information about which parameters are
32 : : potential transformation candidates and which bits of candidates are
33 : : accessed. We differentiate between accesses done as a part of a call
34 : : statement (which might be not necessary if the callee is also transformed)
35 : : and others (which are mandatory). Call summaries (often called call graph
36 : : edge summaries) contain information about which function formal parameters
37 : : feed into which actual call arguments so that if two parameters are only
38 : : used in a sum which is then passed to another function which then however
39 : : does not use this parameter, all three parameters of the two functions can
40 : : be eliminated. Edge summaries also have flags whether the return value is
41 : : used or if it is only returned in the caller too. In LTO mode these
42 : : summaries are then streamed to the object file in the compilation phase and
43 : : streamed back in in the WPA analysis stage.
44 : :
45 : : The interprocedural analysis phase traverses the graph in topological order
46 : : in two sweeps, one in each direction. First, from callees to callers for
47 : : parameter removal and splitting. Each strongly-connected component is
48 : : processed iteratively until the situation in it stabilizes. The pass from
49 : : callers to callees is then carried out to remove unused return values in a
50 : : very similar fashion.
51 : :
52 : : Because parameter manipulation has big implications for call redirection
53 : : which is done only after all call graph nodes materialize, the
54 : : transformation phase is not part of this patch but is carried out by the
55 : : clone materialization and edge redirection itself, see comments in
56 : : ipa-param-manipulation.h for more details. */
57 : :
58 : :
59 : : #include "config.h"
60 : : #include "system.h"
61 : : #include "coretypes.h"
62 : : #include "backend.h"
63 : : #include "tree.h"
64 : : #include "gimple.h"
65 : : #include "predict.h"
66 : : #include "tree-pass.h"
67 : : #include "ssa.h"
68 : : #include "cgraph.h"
69 : : #include "gimple-pretty-print.h"
70 : : #include "alias.h"
71 : : #include "tree-eh.h"
72 : : #include "gimple-iterator.h"
73 : : #include "gimple-walk.h"
74 : : #include "tree-dfa.h"
75 : : #include "tree-sra.h"
76 : : #include "alloc-pool.h"
77 : : #include "symbol-summary.h"
78 : : #include "dbgcnt.h"
79 : : #include "tree-inline.h"
80 : : #include "ipa-utils.h"
81 : : #include "builtins.h"
82 : : #include "cfganal.h"
83 : : #include "tree-streamer.h"
84 : : #include "internal-fn.h"
85 : : #include "symtab-clones.h"
86 : : #include "attribs.h"
87 : : #include "sreal.h"
88 : : #include "ipa-cp.h"
89 : : #include "ipa-prop.h"
90 : :
91 : : static void ipa_sra_summarize_function (cgraph_node *);
92 : :
93 : : /* Bits used to track size of an aggregate in bytes interprocedurally. */
94 : : #define ISRA_ARG_SIZE_LIMIT_BITS 16
95 : : #define ISRA_ARG_SIZE_LIMIT (1 << ISRA_ARG_SIZE_LIMIT_BITS)
96 : : /* How many parameters can feed into a call actual argument and still be
97 : : tracked. */
98 : : #define IPA_SRA_MAX_PARAM_FLOW_LEN 7
99 : :
100 : : /* Structure describing accesses to a specific portion of an aggregate
101 : : parameter, as given by the offset and size. Any smaller accesses that occur
102 : : within a function that fall within another access form a tree. The pass
103 : : cannot analyze parameters with only partially overlapping accesses. */
104 : :
105 : : struct GTY(()) param_access
106 : : {
107 : : /* Type that a potential replacement should have. This field only has
108 : : meaning in the summary building and transformation phases, when it is
109 : : reconstructed from the body. Must not be touched in IPA analysis
110 : : stage. */
111 : : tree type;
112 : :
113 : : /* Alias reference type to be used in MEM_REFs when adjusting caller
114 : : arguments. */
115 : : tree alias_ptr_type;
116 : :
117 : : /* Values returned by get_ref_base_and_extent but converted to bytes and
118 : : stored as unsigned ints. */
119 : : unsigned unit_offset;
120 : : unsigned unit_size : ISRA_ARG_SIZE_LIMIT_BITS;
121 : :
122 : : /* Set once we are sure that the access will really end up in a potentially
123 : : transformed function - initially not set for portions of formal parameters
124 : : that are only used as actual function arguments passed to callees. */
125 : : unsigned certain : 1;
126 : : /* Set if the access has reverse scalar storage order. */
127 : : unsigned reverse : 1;
128 : : };
129 : :
130 : : /* This structure has the same purpose as the one above and additionally it
131 : : contains some fields that are only necessary in the summary generation
132 : : phase. */
133 : :
134 : : struct gensum_param_access
135 : : {
136 : : /* Values returned by get_ref_base_and_extent. */
137 : : HOST_WIDE_INT offset;
138 : : HOST_WIDE_INT size;
139 : :
140 : : /* if this access has any children (in terms of the definition above), this
141 : : points to the first one. */
142 : : struct gensum_param_access *first_child;
143 : : /* In intraprocedural SRA, pointer to the next sibling in the access tree as
144 : : described above. */
145 : : struct gensum_param_access *next_sibling;
146 : :
147 : : /* Type that a potential replacement should have. This field only has
148 : : meaning in the summary building and transformation phases, when it is
149 : : reconstructed from the body. Must not be touched in IPA analysis
150 : : stage. */
151 : : tree type;
152 : : /* Alias reference type to be used in MEM_REFs when adjusting caller
153 : : arguments. */
154 : : tree alias_ptr_type;
155 : :
156 : : /* Cumulative count of all loads. */
157 : : profile_count load_count;
158 : : /* Have there been writes to or reads from this exact location except for as
159 : : arguments to a function call that can be tracked. */
160 : : bool nonarg;
161 : :
162 : : /* Set if the access has reverse scalar storage order. */
163 : : bool reverse;
164 : : };
165 : :
166 : : /* Summary describing a parameter in the IPA stages. */
167 : :
168 : : struct GTY(()) isra_param_desc
169 : : {
170 : : /* List of access representatives to the parameters, sorted according to
171 : : their offset. */
172 : : vec <param_access *, va_gc> *accesses;
173 : :
174 : : /* Unit size limit of total size of all replacements. */
175 : : unsigned param_size_limit : ISRA_ARG_SIZE_LIMIT_BITS;
176 : : /* Sum of unit sizes of all certain replacements. */
177 : : unsigned size_reached : ISRA_ARG_SIZE_LIMIT_BITS;
178 : : /* Minimum offset that is known to be safe to dereference because of callers
179 : : pass pointers to DECLs of at least this size or because of dereferences in
180 : : callers. */
181 : : unsigned safe_size : ISRA_ARG_SIZE_LIMIT_BITS;
182 : :
183 : : /* A parameter that is used only in call arguments and can be removed if all
184 : : concerned actual arguments are removed. */
185 : : unsigned locally_unused : 1;
186 : : /* An aggregate that is a candidate for breaking up or complete removal. */
187 : : unsigned split_candidate : 1;
188 : : /* Is this a parameter passing stuff by reference? */
189 : : unsigned by_ref : 1;
190 : : /* If set, this parameter can only be a candidate for removal if the function
191 : : is going to loose its return value. */
192 : : unsigned remove_only_when_retval_removed : 1;
193 : : /* If set, this parameter can only be a candidate for splitting if the
194 : : function is going to loose its return value. Can only be meaningfully set
195 : : for by_ref parameters. */
196 : : unsigned split_only_when_retval_removed : 1;
197 : : /* Parameter hint set during IPA analysis when there is a caller which does
198 : : not construct the argument just to pass it to calls. Only meaningful for
199 : : by_ref parameters. */
200 : : unsigned not_specially_constructed : 1;
201 : : /* Only meaningful for by_ref parameters. If set, this parameter can only be
202 : : a split candidate if all callers pass pointers that are known to point to
203 : : a chunk of memory large enough to contain all accesses. */
204 : : unsigned conditionally_dereferenceable : 1;
205 : : /* Set when safe_size has been updated from at least one caller. */
206 : : unsigned safe_size_set : 1;
207 : : };
208 : :
209 : : /* Structure used when generating summaries that describes a parameter. */
210 : :
211 : : struct gensum_param_desc
212 : : {
213 : : /* Roots of param_accesses. */
214 : : gensum_param_access *accesses;
215 : : /* Number of accesses in the access tree rooted in field accesses. */
216 : : unsigned access_count;
217 : :
218 : : /* If the below is non-zero, this is the number of uses as actual
219 : : arguments. */
220 : : int call_uses;
221 : : /* Number of times this parameter has been directly passed to. */
222 : : unsigned ptr_pt_count;
223 : :
224 : : /* Size limit of total size of all replacements. */
225 : : unsigned param_size_limit;
226 : : /* Sum of sizes of nonarg accesses. */
227 : : unsigned nonarg_acc_size;
228 : :
229 : : /* A parameter that is used only in call arguments and can be removed if all
230 : : concerned actual arguments are removed. */
231 : : bool locally_unused;
232 : : /* An aggregate that is a candidate for breaking up or a pointer passing data
233 : : by reference that is a candidate for being converted to a set of
234 : : parameters passing those data by value. */
235 : : bool split_candidate;
236 : : /* Is this a parameter passing stuff by reference (either a pointer or a
237 : : source language reference type)? */
238 : : bool by_ref;
239 : : /* If this parameter passes stuff by reference, can it be safely dereferenced
240 : : without performing further checks (for example because it is a
241 : : REFERENCE_TYPE)? */
242 : : bool safe_ref;
243 : : /* If set, this parameter can only be a candidate for removal if the function
244 : : is going to loose its return value. */
245 : : bool remove_only_when_retval_removed;
246 : : /* If set, this parameter can only be a candidate for splitting if the
247 : : function is going to loose its return value. Can only be meaningfully set
248 : : for by_ref parameters. */
249 : : bool split_only_when_retval_removed;
250 : : /* Only meaningful for by_ref parameters. If set, this parameter can only be
251 : : a split candidate if all callers pass pointers that are known to point to
252 : : a chunk of memory large enough to contain all accesses. */
253 : : bool conditionally_dereferenceable;
254 : :
255 : : /* The number of this parameter as they are ordered in function decl. */
256 : : int param_number;
257 : : /* For parameters passing data by reference, this is parameter index to
258 : : compute indices to bb_dereferences. */
259 : : int deref_index;
260 : : };
261 : :
262 : : /* Properly deallocate accesses of DESC. TODO: Since this data structure is
263 : : allocated in GC memory, this is not necessary and we can consider removing
264 : : the function. */
265 : :
266 : : static void
267 : 2772806 : free_param_decl_accesses (isra_param_desc *desc)
268 : : {
269 : 2772806 : unsigned len = vec_safe_length (desc->accesses);
270 : 3391037 : for (unsigned i = 0; i < len; ++i)
271 : 618231 : ggc_free ((*desc->accesses)[i]);
272 : 2772806 : vec_free (desc->accesses);
273 : 2772806 : }
274 : :
275 : : /* Class used to convey information about functions from the
276 : : intra-procedural analysis stage to inter-procedural one. */
277 : :
278 : : class GTY((for_user)) isra_func_summary
279 : : {
280 : : public:
281 : : /* initialize the object. */
282 : :
283 : 1186779 : isra_func_summary ()
284 : 1186779 : : m_parameters (NULL), m_candidate (false), m_returns_value (false),
285 : 1186779 : m_return_ignored (false), m_queued (false)
286 : : {}
287 : :
288 : : /* Destroy m_parameters. */
289 : :
290 : : ~isra_func_summary ();
291 : :
292 : : /* Mark the function as not a candidate for any IPA-SRA transformation.
293 : : Return true if it was a candidate until now. */
294 : :
295 : : bool zap ();
296 : :
297 : : /* Vector of parameter descriptors corresponding to the function being
298 : : analyzed. */
299 : : vec<isra_param_desc, va_gc> *m_parameters;
300 : :
301 : : /* Whether the node is even a candidate for any IPA-SRA transformation at
302 : : all. */
303 : : unsigned m_candidate : 1;
304 : :
305 : : /* Whether the original function returns any value. */
306 : : unsigned m_returns_value : 1;
307 : :
308 : : /* Set to true if all call statements do not actually use the returned
309 : : value. */
310 : :
311 : : unsigned m_return_ignored : 1;
312 : :
313 : : /* Whether the node is already queued in IPA SRA stack during processing of
314 : : call graphs SCCs. */
315 : :
316 : : unsigned m_queued : 1;
317 : : };
318 : :
319 : : /* Deallocate the memory pointed to by isra_func_summary. TODO: Since this
320 : : data structure is allocated in GC memory, this is not necessary and we can
321 : : consider removing the destructor. */
322 : :
323 : 1186778 : isra_func_summary::~isra_func_summary ()
324 : : {
325 : 1186778 : unsigned len = vec_safe_length (m_parameters);
326 : 2471261 : for (unsigned i = 0; i < len; ++i)
327 : 1284483 : free_param_decl_accesses (&(*m_parameters)[i]);
328 : 1186778 : vec_free (m_parameters);
329 : 1186778 : }
330 : :
331 : : /* Mark the function as not a candidate for any IPA-SRA transformation. Return
332 : : true if it was a candidate until now. */
333 : :
334 : : bool
335 : 644122 : isra_func_summary::zap ()
336 : : {
337 : 644122 : bool ret = m_candidate;
338 : 644122 : m_candidate = false;
339 : :
340 : : /* TODO: see the destructor above. */
341 : 644122 : unsigned len = vec_safe_length (m_parameters);
342 : 2132445 : for (unsigned i = 0; i < len; ++i)
343 : 1488323 : free_param_decl_accesses (&(*m_parameters)[i]);
344 : 644122 : vec_free (m_parameters);
345 : :
346 : 644122 : return ret;
347 : : }
348 : :
349 : : /* Structure to describe which formal parameters feed into a particular actual
350 : : argument. */
351 : :
352 : : struct isra_param_flow
353 : : {
354 : : /* Number of elements in array inputs that contain valid data. */
355 : : char length;
356 : : /* Indices of formal parameters that feed into the described actual argument.
357 : : If aggregate_pass_through or pointer_pass_through below are true, it must
358 : : contain exactly one element which is passed through from a formal
359 : : parameter if the given number. Otherwise, the array contains indices of
360 : : callee's formal parameters which are used to calculate value of this
361 : : actual argument. */
362 : : unsigned char inputs[IPA_SRA_MAX_PARAM_FLOW_LEN];
363 : :
364 : : /* Offset within the formal parameter. */
365 : : unsigned unit_offset;
366 : : /* When aggregate_pass_through is set, this is the size of the portion of an
367 : : aggregate formal parameter that is being passed. Otherwise, this is size
368 : : of pointed to memory that is known to be valid be dereferenced. */
369 : : unsigned unit_size : ISRA_ARG_SIZE_LIMIT_BITS;
370 : :
371 : : /* True when the value of this actual argument is a portion of a formal
372 : : parameter. */
373 : : unsigned aggregate_pass_through : 1;
374 : : /* True when the value of this actual copy is a verbatim pass through of an
375 : : obtained pointer. */
376 : : unsigned pointer_pass_through : 1;
377 : : /* True when it is safe to copy access candidates here from the callee, which
378 : : would mean introducing dereferences into callers of the caller. */
379 : : unsigned safe_to_import_accesses : 1;
380 : : /* True when the passed value is an address of a structure that has been
381 : : constructed in the caller just to be passed by reference to functions
382 : : (i.e. is never read). */
383 : : unsigned constructed_for_calls : 1;
384 : : };
385 : :
386 : : /* Structure used to convey information about calls from the intra-procedural
387 : : analysis stage to inter-procedural one. */
388 : :
389 : 5998136 : class isra_call_summary
390 : : {
391 : : public:
392 : 5998136 : isra_call_summary ()
393 : 0 : : m_arg_flow (), m_return_ignored (false), m_return_returned (false),
394 : 0 : m_bit_aligned_arg (false), m_before_any_store (false)
395 : : {}
396 : :
397 : : void init_inputs (unsigned arg_count);
398 : : void dump (FILE *f);
399 : :
400 : : /* Information about what formal parameters of the caller are used to compute
401 : : individual actual arguments of this call. */
402 : : auto_vec <isra_param_flow> m_arg_flow;
403 : :
404 : : /* Set to true if the call statement does not have a LHS. */
405 : : unsigned m_return_ignored : 1;
406 : :
407 : : /* Set to true if the LHS of call statement is only used to construct the
408 : : return value of the caller. */
409 : : unsigned m_return_returned : 1;
410 : :
411 : : /* Set when any of the call arguments are not byte-aligned. */
412 : : unsigned m_bit_aligned_arg : 1;
413 : :
414 : : /* Set to true if the call happend before any (other) store to memory in the
415 : : caller. */
416 : : unsigned m_before_any_store : 1;
417 : : };
418 : :
419 : : /* Class to manage function summaries. */
420 : :
421 : : class GTY((user)) ipa_sra_function_summaries
422 : : : public function_summary <isra_func_summary *>
423 : : {
424 : : public:
425 : 132040 : ipa_sra_function_summaries (symbol_table *table, bool ggc):
426 : 264080 : function_summary<isra_func_summary *> (table, ggc) { }
427 : :
428 : : void duplicate (cgraph_node *, cgraph_node *,
429 : : isra_func_summary *old_sum,
430 : : isra_func_summary *new_sum) final override;
431 : : void insert (cgraph_node *, isra_func_summary *) final override;
432 : : };
433 : :
434 : : /* Hook that is called by summary when a node is duplicated. */
435 : :
436 : : void
437 : 125851 : ipa_sra_function_summaries::duplicate (cgraph_node *, cgraph_node *,
438 : : isra_func_summary *old_sum,
439 : : isra_func_summary *new_sum)
440 : : {
441 : : /* TODO: Somehow stop copying when ISRA is doing the cloning, it is
442 : : useless. */
443 : 125851 : new_sum->m_candidate = old_sum->m_candidate;
444 : 125851 : new_sum->m_returns_value = old_sum->m_returns_value;
445 : 125851 : new_sum->m_return_ignored = old_sum->m_return_ignored;
446 : 125851 : gcc_assert (!old_sum->m_queued);
447 : 125851 : new_sum->m_queued = false;
448 : :
449 : 125851 : unsigned param_count = vec_safe_length (old_sum->m_parameters);
450 : 125135 : if (!param_count)
451 : : return;
452 : 125135 : vec_safe_reserve_exact (new_sum->m_parameters, param_count);
453 : 125135 : new_sum->m_parameters->quick_grow_cleared (param_count);
454 : 482718 : for (unsigned i = 0; i < param_count; i++)
455 : : {
456 : 357583 : isra_param_desc *s = &(*old_sum->m_parameters)[i];
457 : 357583 : isra_param_desc *d = &(*new_sum->m_parameters)[i];
458 : :
459 : 357583 : d->param_size_limit = s->param_size_limit;
460 : 357583 : d->size_reached = s->size_reached;
461 : 357583 : d->safe_size = s->safe_size;
462 : 357583 : d->locally_unused = s->locally_unused;
463 : 357583 : d->split_candidate = s->split_candidate;
464 : 357583 : d->by_ref = s->by_ref;
465 : 357583 : d->remove_only_when_retval_removed = s->remove_only_when_retval_removed;
466 : 357583 : d->split_only_when_retval_removed = s->split_only_when_retval_removed;
467 : 357583 : d->not_specially_constructed = s->not_specially_constructed;
468 : 357583 : d->conditionally_dereferenceable = s->conditionally_dereferenceable;
469 : 357583 : d->safe_size_set = s->safe_size_set;
470 : :
471 : 357583 : unsigned acc_count = vec_safe_length (s->accesses);
472 : 357583 : vec_safe_reserve_exact (d->accesses, acc_count);
473 : 516214 : for (unsigned j = 0; j < acc_count; j++)
474 : : {
475 : 158631 : param_access *from = (*s->accesses)[j];
476 : 158631 : param_access *to = ggc_cleared_alloc<param_access> ();
477 : 158631 : to->type = from->type;
478 : 158631 : to->alias_ptr_type = from->alias_ptr_type;
479 : 158631 : to->unit_offset = from->unit_offset;
480 : 158631 : to->unit_size = from->unit_size;
481 : 158631 : to->certain = from->certain;
482 : 158631 : to->reverse = from->reverse;
483 : 158631 : d->accesses->quick_push (to);
484 : : }
485 : : }
486 : : }
487 : :
488 : : /* Pointer to the pass function summary holder. */
489 : :
490 : : static GTY(()) ipa_sra_function_summaries *func_sums;
491 : :
492 : : /* Hook that is called by summary when new node appears. */
493 : :
494 : : void
495 : 17545 : ipa_sra_function_summaries::insert (cgraph_node *node, isra_func_summary *)
496 : : {
497 : 17545 : if (opt_for_fn (node->decl, flag_ipa_sra))
498 : : {
499 : 17545 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
500 : 17545 : ipa_sra_summarize_function (node);
501 : 17545 : pop_cfun ();
502 : : }
503 : : else
504 : 0 : func_sums->remove (node);
505 : 17545 : }
506 : :
507 : : /* Class to manage call summaries. */
508 : :
509 : : class ipa_sra_call_summaries: public call_summary <isra_call_summary *>
510 : : {
511 : : public:
512 : 132040 : ipa_sra_call_summaries (symbol_table *table):
513 : 264080 : call_summary<isra_call_summary *> (table) { }
514 : :
515 : : /* Duplicate info when an edge is cloned. */
516 : : void duplicate (cgraph_edge *, cgraph_edge *,
517 : : isra_call_summary *old_sum,
518 : : isra_call_summary *new_sum) final override;
519 : : };
520 : :
521 : : static ipa_sra_call_summaries *call_sums;
522 : :
523 : :
524 : : /* Initialize m_arg_flow of a particular instance of isra_call_summary.
525 : : ARG_COUNT is the number of actual arguments passed. */
526 : :
527 : : void
528 : 7079560 : isra_call_summary::init_inputs (unsigned arg_count)
529 : : {
530 : 7079560 : if (arg_count == 0)
531 : : {
532 : 324394 : gcc_checking_assert (m_arg_flow.length () == 0);
533 : : return;
534 : : }
535 : 6755166 : if (m_arg_flow.length () == 0)
536 : : {
537 : 3285362 : m_arg_flow.reserve_exact (arg_count);
538 : 3285362 : m_arg_flow.quick_grow_cleared (arg_count);
539 : : }
540 : : else
541 : 3469804 : gcc_checking_assert (arg_count == m_arg_flow.length ());
542 : : }
543 : :
544 : : /* Dump all information in call summary to F. */
545 : :
546 : : void
547 : 180 : isra_call_summary::dump (FILE *f)
548 : : {
549 : 180 : if (m_return_ignored)
550 : 128 : fprintf (f, " return value ignored\n");
551 : 180 : if (m_return_returned)
552 : 38 : fprintf (f, " return value used only to compute caller return value\n");
553 : 180 : if (m_before_any_store)
554 : 17 : fprintf (f, " happens before any store to memory\n");
555 : 409 : for (unsigned i = 0; i < m_arg_flow.length (); i++)
556 : : {
557 : 229 : fprintf (f, " Parameter %u:\n", i);
558 : 229 : isra_param_flow *ipf = &m_arg_flow[i];
559 : :
560 : 229 : if (ipf->length)
561 : : {
562 : 158 : bool first = true;
563 : 158 : fprintf (f, " Scalar param sources: ");
564 : 319 : for (int j = 0; j < ipf->length; j++)
565 : : {
566 : 161 : if (!first)
567 : 3 : fprintf (f, ", ");
568 : : else
569 : : first = false;
570 : 161 : fprintf (f, "%i", (int) ipf->inputs[j]);
571 : : }
572 : 158 : fprintf (f, "\n");
573 : : }
574 : 229 : if (ipf->aggregate_pass_through)
575 : 12 : fprintf (f, " Aggregate pass through from the param given above, "
576 : : "unit offset: %u , unit size: %u\n",
577 : 12 : ipf->unit_offset, ipf->unit_size);
578 : 217 : else if (ipf->unit_size > 0)
579 : 52 : fprintf (f, " Known dereferenceable size: %u\n", ipf->unit_size);
580 : 229 : if (ipf->pointer_pass_through)
581 : 19 : fprintf (f, " Pointer pass through from the param given above, "
582 : 19 : "safe_to_import_accesses: %u\n", ipf->safe_to_import_accesses);
583 : 229 : if (ipf->constructed_for_calls)
584 : 28 : fprintf (f, " Variable constructed just to be passed to "
585 : : "calls.\n");
586 : : }
587 : 180 : }
588 : :
589 : : /* Duplicate edge summary when an edge is cloned. */
590 : :
591 : : void
592 : 610917 : ipa_sra_call_summaries::duplicate (cgraph_edge *, cgraph_edge *,
593 : : isra_call_summary *old_sum,
594 : : isra_call_summary *new_sum)
595 : : {
596 : 610917 : unsigned arg_count = old_sum->m_arg_flow.length ();
597 : 610917 : new_sum->init_inputs (arg_count);
598 : 1812181 : for (unsigned i = 0; i < arg_count; i++)
599 : 1201264 : new_sum->m_arg_flow[i] = old_sum->m_arg_flow[i];
600 : :
601 : 610917 : new_sum->m_return_ignored = old_sum->m_return_ignored;
602 : 610917 : new_sum->m_return_returned = old_sum->m_return_returned;
603 : 610917 : new_sum->m_bit_aligned_arg = old_sum->m_bit_aligned_arg;
604 : 610917 : new_sum->m_before_any_store = old_sum->m_before_any_store;
605 : 610917 : }
606 : :
607 : :
608 : : /* With all GTY stuff done, we can move to anonymous namespace. */
609 : : namespace {
610 : : /* Quick mapping from a decl to its param descriptor. */
611 : :
612 : : hash_map<tree, gensum_param_desc *> *decl2desc;
613 : :
614 : : /* All local DECLs ever loaded from of and of those that have their address
615 : : assigned to a variable. */
616 : :
617 : : hash_set <tree> *loaded_decls;
618 : :
619 : : /* Countdown of allowed Alias Analysis steps during summary building. */
620 : :
621 : : int aa_walking_limit;
622 : :
623 : : /* This is a table in which for each basic block and parameter there is a
624 : : distance (offset + size) in that parameter which is dereferenced and
625 : : accessed in that BB. */
626 : : HOST_WIDE_INT *bb_dereferences = NULL;
627 : : /* How many by-reference parameters there are in the current function. */
628 : : int unsafe_by_ref_count;
629 : :
630 : : /* Bitmap of BBs that can cause the function to "stop" progressing by
631 : : returning, throwing externally, looping infinitely or calling a function
632 : : which might abort etc.. */
633 : : bitmap final_bbs;
634 : :
635 : : /* Obstack to allocate various small structures required only when generating
636 : : summary for a function. */
637 : : struct obstack gensum_obstack;
638 : :
639 : : /* Return false the function is apparently unsuitable for IPA-SRA based on it's
640 : : attributes, return true otherwise. NODE is the cgraph node of the current
641 : : function. */
642 : :
643 : : static bool
644 : 1258829 : ipa_sra_preliminary_function_checks (cgraph_node *node)
645 : : {
646 : 1258829 : if (!node->can_change_signature)
647 : : {
648 : 34997 : if (dump_file)
649 : 0 : fprintf (dump_file, "Function cannot change signature.\n");
650 : 34997 : return false;
651 : : }
652 : :
653 : 1223832 : if (!tree_versionable_function_p (node->decl))
654 : : {
655 : 116585 : if (dump_file)
656 : 7 : fprintf (dump_file, "Function is not versionable.\n");
657 : 116585 : return false;
658 : : }
659 : :
660 : 1107247 : if (!opt_for_fn (node->decl, optimize)
661 : 1107247 : || !opt_for_fn (node->decl, flag_ipa_sra))
662 : : {
663 : 192 : if (dump_file)
664 : 0 : fprintf (dump_file, "Not optimizing or IPA-SRA turned off for this "
665 : : "function.\n");
666 : 192 : return false;
667 : : }
668 : :
669 : 1107055 : if (DECL_VIRTUAL_P (node->decl))
670 : : {
671 : 33219 : if (dump_file)
672 : 0 : fprintf (dump_file, "Function is a virtual method.\n");
673 : 33219 : return false;
674 : : }
675 : :
676 : 1073836 : struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
677 : 1073836 : if (fun->stdarg)
678 : : {
679 : 123 : if (dump_file)
680 : 0 : fprintf (dump_file, "Function uses stdarg. \n");
681 : 123 : return false;
682 : : }
683 : :
684 : 1073713 : if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
685 : : {
686 : 48747 : if (dump_file)
687 : 0 : fprintf (dump_file, "Always inline function will be inlined "
688 : : "anyway. \n");
689 : 48747 : return false;
690 : : }
691 : :
692 : : return true;
693 : : }
694 : :
695 : : /* Print access tree starting at ACCESS to F. */
696 : :
697 : : static void
698 : 69 : dump_gensum_access (FILE *f, gensum_param_access *access, unsigned indent)
699 : : {
700 : 69 : fprintf (f, " ");
701 : 211 : for (unsigned i = 0; i < indent; i++)
702 : 142 : fprintf (f, " ");
703 : 69 : fprintf (f, " * Access to offset: " HOST_WIDE_INT_PRINT_DEC,
704 : : access->offset);
705 : 69 : fprintf (f, ", size: " HOST_WIDE_INT_PRINT_DEC, access->size);
706 : 69 : fprintf (f, ", type: ");
707 : 69 : print_generic_expr (f, access->type);
708 : 69 : fprintf (f, ", alias_ptr_type: ");
709 : 69 : print_generic_expr (f, access->alias_ptr_type);
710 : 69 : fprintf (f, ", load_count: ");
711 : 69 : access->load_count.dump (f);
712 : 69 : fprintf (f, ", nonarg: %u, reverse: %u\n", access->nonarg, access->reverse);
713 : 69 : for (gensum_param_access *ch = access->first_child;
714 : 71 : ch;
715 : 2 : ch = ch->next_sibling)
716 : 2 : dump_gensum_access (f, ch, indent + 2);
717 : 69 : }
718 : :
719 : :
720 : : /* Print access tree starting at ACCESS to F. */
721 : :
722 : : static void
723 : 152 : dump_isra_access (FILE *f, param_access *access)
724 : : {
725 : 152 : fprintf (f, " * Access to unit offset: %u", access->unit_offset);
726 : 152 : fprintf (f, ", unit size: %u", access->unit_size);
727 : 152 : fprintf (f, ", type: ");
728 : 152 : print_generic_expr (f, access->type);
729 : 152 : fprintf (f, ", alias_ptr_type: ");
730 : 152 : print_generic_expr (f, access->alias_ptr_type);
731 : 152 : if (access->certain)
732 : 130 : fprintf (f, ", certain");
733 : : else
734 : 22 : fprintf (f, ", not certain");
735 : 152 : if (access->reverse)
736 : 0 : fprintf (f, ", reverse");
737 : 152 : fprintf (f, "\n");
738 : 152 : }
739 : :
740 : : /* Dump access tree starting at ACCESS to stderr. */
741 : :
742 : : DEBUG_FUNCTION void
743 : 0 : debug_isra_access (param_access *access)
744 : : {
745 : 0 : dump_isra_access (stderr, access);
746 : 0 : }
747 : :
748 : : /* Dump DESC to F. */
749 : :
750 : : static void
751 : 332 : dump_gensum_param_descriptor (FILE *f, gensum_param_desc *desc)
752 : : {
753 : 332 : if (desc->locally_unused)
754 : 166 : fprintf (f, " unused with %i call_uses%s\n", desc->call_uses,
755 : 166 : desc->remove_only_when_retval_removed ?
756 : : " remove_only_when_retval_removed" : "");
757 : 332 : if (!desc->split_candidate)
758 : : {
759 : 244 : fprintf (f, " not a candidate\n");
760 : 244 : return;
761 : : }
762 : 88 : if (desc->by_ref)
763 : 73 : fprintf (f, " %s%s%s by_ref with %u pass throughs\n",
764 : 73 : desc->safe_ref ? "safe" : "unsafe",
765 : 73 : desc->conditionally_dereferenceable
766 : : ? " conditionally_dereferenceable" : "",
767 : 73 : desc->split_only_when_retval_removed
768 : : ? " split_only_when_retval_removed" : "",
769 : : desc->ptr_pt_count);
770 : :
771 : 155 : for (gensum_param_access *acc = desc->accesses; acc; acc = acc->next_sibling)
772 : 67 : dump_gensum_access (f, acc, 2);
773 : : }
774 : :
775 : : /* Dump all parameter descriptors in IFS, assuming it describes FNDECL, to
776 : : F. */
777 : :
778 : : static void
779 : 135 : dump_gensum_param_descriptors (FILE *f, tree fndecl,
780 : : vec<gensum_param_desc> *param_descriptions)
781 : : {
782 : 135 : tree parm = DECL_ARGUMENTS (fndecl);
783 : 135 : for (unsigned i = 0;
784 : 467 : i < param_descriptions->length ();
785 : 332 : ++i, parm = DECL_CHAIN (parm))
786 : : {
787 : 332 : fprintf (f, " Descriptor for parameter %i ", i);
788 : 332 : print_generic_expr (f, parm, TDF_UID);
789 : 332 : fprintf (f, "\n");
790 : 332 : dump_gensum_param_descriptor (f, &(*param_descriptions)[i]);
791 : : }
792 : 135 : }
793 : :
794 : :
795 : : /* Dump DESC to F. If HINTS is true, also dump IPA-analysis computed
796 : : hints. */
797 : :
798 : : static void
799 : 646 : dump_isra_param_descriptor (FILE *f, isra_param_desc *desc, bool hints)
800 : : {
801 : 646 : if (desc->locally_unused)
802 : : {
803 : 335 : fprintf (f, " (locally) unused\n");
804 : : }
805 : 646 : if (!desc->split_candidate)
806 : : {
807 : 471 : fprintf (f, " not a candidate for splitting");
808 : 471 : if (hints && desc->by_ref && desc->safe_size_set)
809 : 9 : fprintf (f, ", safe_size: %u", (unsigned) desc->safe_size);
810 : 471 : fprintf (f, "\n");
811 : 471 : return;
812 : : }
813 : 350 : fprintf (f, " param_size_limit: %u, size_reached: %u%s",
814 : 175 : desc->param_size_limit, desc->size_reached,
815 : 175 : desc->by_ref ? ", by_ref" : "");
816 : 175 : if (desc->remove_only_when_retval_removed)
817 : 12 : fprintf (f, ", remove_only_when_retval_removed");
818 : 175 : if (desc->split_only_when_retval_removed)
819 : 4 : fprintf (f, ", split_only_when_retval_removed");
820 : 175 : if (desc->by_ref && desc->conditionally_dereferenceable)
821 : 33 : fprintf (f, ", conditionally_dereferenceable");
822 : 175 : if (hints)
823 : : {
824 : 6 : if (desc->by_ref && !desc->not_specially_constructed)
825 : 3 : fprintf (f, ", args_specially_constructed");
826 : 6 : if (desc->by_ref && desc->safe_size_set)
827 : 5 : fprintf (f, ", safe_size: %u", (unsigned) desc->safe_size);
828 : : }
829 : 175 : fprintf (f, "\n");
830 : :
831 : 327 : for (unsigned i = 0; i < vec_safe_length (desc->accesses); ++i)
832 : : {
833 : 152 : param_access *access = (*desc->accesses)[i];
834 : 152 : dump_isra_access (f, access);
835 : : }
836 : : }
837 : :
838 : : /* Dump all parameter descriptors in IFS, assuming it describes FNDECL, to F.
839 : : If HINTS is true, also dump IPA-analysis computed hints. */
840 : :
841 : : static void
842 : 135 : dump_isra_param_descriptors (FILE *f, tree fndecl, isra_func_summary *ifs,
843 : : bool hints)
844 : : {
845 : 135 : tree parm = DECL_ARGUMENTS (fndecl);
846 : 135 : if (!ifs->m_parameters)
847 : : {
848 : 0 : fprintf (f, " parameter descriptors not available\n");
849 : 0 : return;
850 : : }
851 : :
852 : : for (unsigned i = 0;
853 : 467 : i < ifs->m_parameters->length ();
854 : 332 : ++i, parm = DECL_CHAIN (parm))
855 : : {
856 : 332 : fprintf (f, " Descriptor for parameter %i ", i);
857 : 332 : print_generic_expr (f, parm, TDF_UID);
858 : 332 : fprintf (f, "\n");
859 : 332 : dump_isra_param_descriptor (f, &(*ifs->m_parameters)[i], hints);
860 : : }
861 : : }
862 : :
863 : : /* Add SRC to inputs of PARAM_FLOW, unless it would exceed storage. If the
864 : : function fails return false, otherwise return true. SRC must fit into an
865 : : unsigned char. Used for purposes of transitive unused parameter
866 : : removal. */
867 : :
868 : : static bool
869 : 1357124 : add_src_to_param_flow (isra_param_flow *param_flow, int src)
870 : : {
871 : 1357124 : gcc_checking_assert (src >= 0 && src <= UCHAR_MAX);
872 : 1357124 : if (param_flow->length == IPA_SRA_MAX_PARAM_FLOW_LEN)
873 : : return false;
874 : :
875 : 1357121 : param_flow->inputs[(int) param_flow->length] = src;
876 : 1357121 : param_flow->length++;
877 : 1357121 : return true;
878 : : }
879 : :
880 : : /* Add a SRC to the inputs of PARAM_FLOW unless it is already there and assert
881 : : it is the only input. Used for purposes of transitive parameter
882 : : splitting. */
883 : :
884 : : static void
885 : 833868 : set_single_param_flow_source (isra_param_flow *param_flow, int src)
886 : : {
887 : 833868 : gcc_checking_assert (src >= 0 && src <= UCHAR_MAX);
888 : 833868 : if (param_flow->length == 0)
889 : : {
890 : 136941 : param_flow->inputs[0] = src;
891 : 136941 : param_flow->length = 1;
892 : : }
893 : 696927 : else if (param_flow->length == 1)
894 : 696927 : gcc_assert (param_flow->inputs[0] == src);
895 : : else
896 : 0 : gcc_unreachable ();
897 : 833868 : }
898 : :
899 : : /* Assert that there is only a single value in PARAM_FLOW's inputs and return
900 : : it. */
901 : :
902 : : static unsigned
903 : 1416273 : get_single_param_flow_source (const isra_param_flow *param_flow)
904 : : {
905 : 1416273 : gcc_assert (param_flow->length == 1);
906 : 1416273 : return param_flow->inputs[0];
907 : : }
908 : :
909 : : /* Inspect all uses of NAME and simple arithmetic calculations involving NAME
910 : : in FUN represented with NODE and return a negative number if any of them is
911 : : used for something else than either an actual call argument, simple return,
912 : : simple arithmetic operation or debug statement. If there are no such uses,
913 : : return the number of actual arguments that this parameter eventually feeds
914 : : to (or zero if there is none). If there are any simple return uses, set
915 : : DESC->remove_only_when_retval_removed. For any such parameter, mark
916 : : PARM_NUM as one of its sources. ANALYZED is a bitmap that tracks which SSA
917 : : names we have already started investigating. */
918 : :
919 : : static int
920 : 3407404 : isra_track_scalar_value_uses (function *fun, cgraph_node *node, tree name,
921 : : int parm_num, bitmap analyzed,
922 : : gensum_param_desc *desc)
923 : : {
924 : 3407404 : int res = 0;
925 : 3407404 : imm_use_iterator imm_iter;
926 : 3407404 : gimple *stmt;
927 : :
928 : 7012147 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
929 : : {
930 : 5250307 : if (is_gimple_debug (stmt)
931 : 5250307 : || gimple_clobber_p (stmt))
932 : 866554 : continue;
933 : :
934 : : /* TODO: We could handle at least const builtin functions like arithmetic
935 : : operations below. */
936 : 4383753 : if (is_gimple_call (stmt))
937 : : {
938 : 1369401 : int all_uses = 0;
939 : 1369401 : use_operand_p use_p;
940 : 2752773 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
941 : 1383372 : all_uses++;
942 : :
943 : 1369401 : gcall *call = as_a <gcall *> (stmt);
944 : 1369401 : unsigned arg_count;
945 : 1369401 : if (gimple_call_internal_p (call)
946 : 1369401 : || (arg_count = gimple_call_num_args (call)) == 0)
947 : : {
948 : : res = -1;
949 : : break;
950 : : }
951 : :
952 : 1364764 : cgraph_edge *cs = node->get_edge (stmt);
953 : 1364764 : gcc_checking_assert (cs);
954 : 1364764 : isra_call_summary *csum = call_sums->get_create (cs);
955 : 1364764 : csum->init_inputs (arg_count);
956 : :
957 : 1364764 : int simple_uses = 0;
958 : 6254358 : for (unsigned i = 0; i < arg_count; i++)
959 : 4889597 : if (gimple_call_arg (call, i) == name)
960 : : {
961 : 1357124 : if (!add_src_to_param_flow (&csum->m_arg_flow[i], parm_num))
962 : : {
963 : : simple_uses = -1;
964 : : break;
965 : : }
966 : 1357121 : simple_uses++;
967 : : }
968 : :
969 : 1364764 : if (simple_uses < 0
970 : 1364764 : || all_uses != simple_uses)
971 : : {
972 : : res = -1;
973 : : break;
974 : : }
975 : 1343589 : res += all_uses;
976 : : }
977 : 3014352 : else if (!stmt_unremovable_because_of_non_call_eh_p (fun, stmt)
978 : 3014352 : && ((is_gimple_assign (stmt) && !gimple_has_volatile_ops (stmt))
979 : 677229 : || gimple_code (stmt) == GIMPLE_PHI))
980 : : {
981 : 2403037 : tree lhs;
982 : 2403037 : if (gimple_code (stmt) == GIMPLE_PHI)
983 : 173420 : lhs = gimple_phi_result (stmt);
984 : : else
985 : 2229617 : lhs = gimple_assign_lhs (stmt);
986 : :
987 : 2403037 : if (TREE_CODE (lhs) != SSA_NAME)
988 : : {
989 : : res = -1;
990 : : break;
991 : : }
992 : 2014782 : gcc_assert (!gimple_vdef (stmt));
993 : 2014782 : if (bitmap_set_bit (analyzed, SSA_NAME_VERSION (lhs)))
994 : : {
995 : 1922712 : int tmp = isra_track_scalar_value_uses (fun, node, lhs, parm_num,
996 : : analyzed, desc);
997 : 1922712 : if (tmp < 0)
998 : : {
999 : : res = tmp;
1000 : : break;
1001 : : }
1002 : 1139619 : res += tmp;
1003 : : }
1004 : : }
1005 : 611315 : else if (greturn *gr = dyn_cast<greturn *>(stmt))
1006 : : {
1007 : 162911 : tree rv = gimple_return_retval (gr);
1008 : 162911 : if (rv != name)
1009 : : {
1010 : : res = -1;
1011 : : break;
1012 : : }
1013 : 162911 : desc->remove_only_when_retval_removed = true;
1014 : : }
1015 : : else
1016 : : {
1017 : : res = -1;
1018 : : break;
1019 : : }
1020 : 3407404 : }
1021 : 3407404 : return res;
1022 : : }
1023 : :
1024 : : /* Inspect all uses of PARM, which must be a gimple register, in FUN (which is
1025 : : also described by NODE) and simple arithmetic calculations involving PARM
1026 : : and return false if any of them is used for something else than either an
1027 : : actual call argument, simple return, simple arithmetic operation or debug
1028 : : statement. If there are no such uses, return true and store the number of
1029 : : actual arguments that this parameter eventually feeds to (or zero if there
1030 : : is none) to DESC->call_uses and set DESC->remove_only_when_retval_removed if
1031 : : there are any uses in return statemens. For any such parameter, mark
1032 : : PARM_NUM as one of its sources.
1033 : :
1034 : : This function is similar to ptr_parm_has_nonarg_uses but its results are
1035 : : meant for unused parameter removal, as opposed to splitting of parameters
1036 : : passed by reference or converting them to passed by value. */
1037 : :
1038 : : static bool
1039 : 2173493 : isra_track_scalar_param_local_uses (function *fun, cgraph_node *node, tree parm,
1040 : : int parm_num, gensum_param_desc *desc)
1041 : : {
1042 : 2173493 : gcc_checking_assert (is_gimple_reg (parm));
1043 : :
1044 : 2173493 : tree name = ssa_default_def (fun, parm);
1045 : 2173493 : if (!name || has_zero_uses (name))
1046 : : {
1047 : 688801 : desc->call_uses = 0;
1048 : 688801 : return false;
1049 : : }
1050 : :
1051 : : /* Edge summaries can only handle callers with fewer than 256 parameters. */
1052 : 1484692 : if (parm_num > UCHAR_MAX)
1053 : : return true;
1054 : :
1055 : 1484692 : bitmap analyzed = BITMAP_ALLOC (NULL);
1056 : 1484692 : int call_uses = isra_track_scalar_value_uses (fun, node, name, parm_num,
1057 : : analyzed, desc);
1058 : 1484692 : BITMAP_FREE (analyzed);
1059 : 1484692 : if (call_uses < 0)
1060 : : return true;
1061 : 622221 : desc->call_uses = call_uses;
1062 : 622221 : return false;
1063 : : }
1064 : :
1065 : : /* Scan immediate uses of a default definition SSA name of a parameter PARM and
1066 : : examine whether there are any nonarg uses that are not actual arguments or
1067 : : otherwise infeasible uses. If so, return true, otherwise return false.
1068 : : Create pass-through IPA flow records for any direct uses as argument calls
1069 : : and if returning false, store their number into DESC->ptr_pt_count. If
1070 : : removal of return value would still allow splitting, return true but set
1071 : : DESC->split_only_when_retval_removed. NODE and FUN must represent the
1072 : : function that is currently analyzed, PARM_NUM must be the index of the
1073 : : analyzed parameter.
1074 : :
1075 : : This function is similar to isra_track_scalar_param_local_uses but its
1076 : : results are meant for splitting of parameters passed by reference or turning
1077 : : them into bits passed by value, as opposed to generic unused parameter
1078 : : removal. */
1079 : :
1080 : : static bool
1081 : 1158881 : ptr_parm_has_nonarg_uses (cgraph_node *node, function *fun, tree parm,
1082 : : int parm_num, gensum_param_desc *desc)
1083 : : {
1084 : 1158881 : imm_use_iterator ui;
1085 : 1158881 : gimple *stmt;
1086 : 1158881 : tree name = ssa_default_def (fun, parm);
1087 : 1158881 : bool ret = false;
1088 : 1158881 : unsigned pt_count = 0;
1089 : :
1090 : 1158881 : if (!name || has_zero_uses (name))
1091 : : return false;
1092 : :
1093 : : /* Edge summaries can only handle callers with fewer than 256 parameters. */
1094 : 1050391 : if (parm_num > UCHAR_MAX)
1095 : : return true;
1096 : :
1097 : 2932927 : FOR_EACH_IMM_USE_STMT (stmt, ui, name)
1098 : : {
1099 : 2412806 : unsigned uses_ok = 0;
1100 : 2412806 : use_operand_p use_p;
1101 : :
1102 : 2412806 : if (is_gimple_debug (stmt)
1103 : 2412806 : || gimple_clobber_p (stmt))
1104 : 609130 : continue;
1105 : :
1106 : 1803676 : if (gimple_assign_single_p (stmt))
1107 : : {
1108 : 910606 : tree rhs = gimple_assign_rhs1 (stmt);
1109 : 910606 : if (!TREE_THIS_VOLATILE (rhs))
1110 : : {
1111 : 1753195 : while (handled_component_p (rhs))
1112 : 842661 : rhs = TREE_OPERAND (rhs, 0);
1113 : 910534 : if (TREE_CODE (rhs) == MEM_REF
1114 : 640434 : && TREE_OPERAND (rhs, 0) == name
1115 : 636771 : && integer_zerop (TREE_OPERAND (rhs, 1))
1116 : 1501538 : && types_compatible_p (TREE_TYPE (rhs),
1117 : 591004 : TREE_TYPE (TREE_TYPE (name))))
1118 : : uses_ok++;
1119 : : }
1120 : : }
1121 : 893070 : else if (is_gimple_call (stmt))
1122 : : {
1123 : 748277 : gcall *call = as_a <gcall *> (stmt);
1124 : 748277 : unsigned arg_count;
1125 : 748277 : if (gimple_call_internal_p (call)
1126 : 748277 : || (arg_count = gimple_call_num_args (call)) == 0)
1127 : : {
1128 : : ret = true;
1129 : : break;
1130 : : }
1131 : :
1132 : 747535 : cgraph_edge *cs = node->get_edge (stmt);
1133 : 747535 : gcc_checking_assert (cs);
1134 : 747535 : isra_call_summary *csum = call_sums->get_create (cs);
1135 : 747535 : csum->init_inputs (arg_count);
1136 : :
1137 : 2901875 : for (unsigned i = 0; i < arg_count; ++i)
1138 : : {
1139 : 2154340 : tree arg = gimple_call_arg (stmt, i);
1140 : :
1141 : 2154340 : if (arg == name)
1142 : : {
1143 : : /* TODO: Allow &MEM_REF[name + offset] here,
1144 : : ipa_param_body_adjustments::modify_call_stmt has to be
1145 : : adjusted too. */
1146 : 736742 : csum->m_arg_flow[i].pointer_pass_through = true;
1147 : 736742 : set_single_param_flow_source (&csum->m_arg_flow[i], parm_num);
1148 : 736742 : pt_count++;
1149 : 736742 : uses_ok++;
1150 : 736742 : continue;
1151 : : }
1152 : :
1153 : 1417598 : if (!TREE_THIS_VOLATILE (arg))
1154 : : {
1155 : 1440952 : while (handled_component_p (arg))
1156 : 23354 : arg = TREE_OPERAND (arg, 0);
1157 : 1417598 : if (TREE_CODE (arg) == MEM_REF
1158 : 19106 : && TREE_OPERAND (arg, 0) == name
1159 : 10590 : && integer_zerop (TREE_OPERAND (arg, 1))
1160 : 1428188 : && types_compatible_p (TREE_TYPE (arg),
1161 : 10590 : TREE_TYPE (TREE_TYPE (name))))
1162 : 7803 : uses_ok++;
1163 : : }
1164 : : }
1165 : : }
1166 : 144793 : else if (greturn *gr = dyn_cast<greturn *>(stmt))
1167 : : {
1168 : 7949 : tree rv = gimple_return_retval (gr);
1169 : 7949 : if (rv == name)
1170 : : {
1171 : 7949 : uses_ok++;
1172 : : /* Analysis for feasibility of removal must have already reached
1173 : : the conclusion that the flag must be set if it completed. */
1174 : 7949 : gcc_assert (!desc->locally_unused
1175 : : || desc->remove_only_when_retval_removed);
1176 : 7949 : desc->split_only_when_retval_removed = true;
1177 : : }
1178 : : }
1179 : :
1180 : : /* If the number of valid uses does not match the number of
1181 : : uses in this stmt there is an unhandled use. */
1182 : 1802934 : unsigned all_uses = 0;
1183 : 3612624 : FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
1184 : 1809690 : all_uses++;
1185 : :
1186 : 1802934 : gcc_checking_assert (uses_ok <= all_uses);
1187 : 1802934 : if (uses_ok != all_uses)
1188 : : {
1189 : : ret = true;
1190 : : break;
1191 : : }
1192 : 1050391 : }
1193 : :
1194 : 1050391 : desc->ptr_pt_count = pt_count;
1195 : 1050391 : return ret;
1196 : : }
1197 : :
1198 : : /* Initialize vector of parameter descriptors of NODE. Return true if there
1199 : : are any candidates for splitting or unused aggregate parameter removal (the
1200 : : function may return false if there are candidates for removal of register
1201 : : parameters). */
1202 : :
1203 : : static bool
1204 : 833517 : create_parameter_descriptors (cgraph_node *node,
1205 : : vec<gensum_param_desc> *param_descriptions)
1206 : : {
1207 : 833517 : function *fun = DECL_STRUCT_FUNCTION (node->decl);
1208 : 833517 : bool ret = false;
1209 : :
1210 : 833517 : int num = 0;
1211 : 833517 : for (tree parm = DECL_ARGUMENTS (node->decl);
1212 : 3222497 : parm;
1213 : 2388980 : parm = DECL_CHAIN (parm), num++)
1214 : : {
1215 : 2388980 : const char *msg;
1216 : 2388980 : gensum_param_desc *desc = &(*param_descriptions)[num];
1217 : : /* param_descriptions vector is grown cleared in the caller. */
1218 : 2388980 : desc->param_number = num;
1219 : 2388980 : decl2desc->put (parm, desc);
1220 : :
1221 : 2388980 : if (dump_file && (dump_flags & TDF_DETAILS))
1222 : 39 : print_generic_expr (dump_file, parm, TDF_UID);
1223 : :
1224 : 2388980 : tree type = TREE_TYPE (parm);
1225 : 2388980 : if (TREE_THIS_VOLATILE (parm))
1226 : : {
1227 : 19 : if (dump_file && (dump_flags & TDF_DETAILS))
1228 : 0 : fprintf (dump_file, " not a candidate, is volatile\n");
1229 : 1762543 : continue;
1230 : : }
1231 : 2388961 : if (!is_gimple_reg_type (type) && is_va_list_type (type))
1232 : : {
1233 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1234 : 0 : fprintf (dump_file, " not a candidate, is a va_list type\n");
1235 : 0 : continue;
1236 : : }
1237 : 2388961 : if (TREE_ADDRESSABLE (parm))
1238 : : {
1239 : 36041 : if (dump_file && (dump_flags & TDF_DETAILS))
1240 : 0 : fprintf (dump_file, " not a candidate, is addressable\n");
1241 : 36041 : continue;
1242 : : }
1243 : 2352920 : if (TREE_ADDRESSABLE (type))
1244 : : {
1245 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1246 : 0 : fprintf (dump_file, " not a candidate, type cannot be split\n");
1247 : 0 : continue;
1248 : : }
1249 : :
1250 : 2352920 : if (is_gimple_reg (parm)
1251 : 2352920 : && !isra_track_scalar_param_local_uses (fun, node, parm, num, desc))
1252 : : {
1253 : 1311022 : desc->locally_unused = true;
1254 : :
1255 : 1311022 : if (dump_file && (dump_flags & TDF_DETAILS))
1256 : 19 : fprintf (dump_file, " is a scalar with only %i call uses%s\n",
1257 : : desc->call_uses,
1258 : 19 : desc->remove_only_when_retval_removed
1259 : : ? " and return uses" : "");
1260 : : }
1261 : :
1262 : 2352920 : if (POINTER_TYPE_P (type))
1263 : : {
1264 : 1165034 : desc->by_ref = true;
1265 : 1165034 : if (TREE_CODE (type) == REFERENCE_TYPE
1266 : 1165034 : || (num == 0
1267 : 561968 : && TREE_CODE (TREE_TYPE (node->decl)) == METHOD_TYPE))
1268 : 515256 : desc->safe_ref = true;
1269 : : else
1270 : 649778 : desc->safe_ref = false;
1271 : 1165034 : type = TREE_TYPE (type);
1272 : :
1273 : 1169736 : if (TREE_CODE (type) == FUNCTION_TYPE
1274 : 1165034 : || TREE_CODE (type) == METHOD_TYPE)
1275 : : {
1276 : 4702 : if (dump_file && (dump_flags & TDF_DETAILS))
1277 : 0 : fprintf (dump_file, " not a candidate, reference to "
1278 : : "a function\n");
1279 : 4702 : continue;
1280 : : }
1281 : 1160332 : if (TYPE_VOLATILE (type))
1282 : : {
1283 : 1060 : if (dump_file && (dump_flags & TDF_DETAILS))
1284 : 0 : fprintf (dump_file, " not a candidate, reference to "
1285 : : "a volatile type\n");
1286 : 1060 : continue;
1287 : : }
1288 : 1159272 : if (TREE_CODE (type) == ARRAY_TYPE
1289 : 1159272 : && TYPE_NONALIASED_COMPONENT (type))
1290 : : {
1291 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1292 : 0 : fprintf (dump_file, " not a candidate, reference to "
1293 : : "a nonaliased component array\n");
1294 : 0 : continue;
1295 : : }
1296 : 1159272 : if (!is_gimple_reg (parm))
1297 : : {
1298 : 6 : if (dump_file && (dump_flags & TDF_DETAILS))
1299 : 0 : fprintf (dump_file, " not a candidate, a reference which is "
1300 : : "not a gimple register (probably addressable)\n");
1301 : 6 : continue;
1302 : : }
1303 : 1159266 : if (is_va_list_type (type))
1304 : : {
1305 : 385 : if (dump_file && (dump_flags & TDF_DETAILS))
1306 : 0 : fprintf (dump_file, " not a candidate, reference to "
1307 : : "a va list\n");
1308 : 385 : continue;
1309 : : }
1310 : 1158881 : if (ptr_parm_has_nonarg_uses (node, fun, parm, num, desc))
1311 : : {
1312 : 530270 : if (dump_file && (dump_flags & TDF_DETAILS))
1313 : 7 : fprintf (dump_file, " not a candidate, reference has "
1314 : : "nonarg uses\n");
1315 : 530270 : continue;
1316 : : }
1317 : : }
1318 : 1187886 : else if (!AGGREGATE_TYPE_P (type))
1319 : : {
1320 : : /* This is in an else branch because scalars passed by reference are
1321 : : still candidates to be passed by value. */
1322 : 1008659 : if (dump_file && (dump_flags & TDF_DETAILS))
1323 : 17 : fprintf (dump_file, " not a candidate, not an aggregate\n");
1324 : 1008659 : continue;
1325 : : }
1326 : :
1327 : 807838 : if (!COMPLETE_TYPE_P (type))
1328 : : {
1329 : 168350 : if (dump_file && (dump_flags & TDF_DETAILS))
1330 : 0 : fprintf (dump_file, " not a candidate, not a complete type\n");
1331 : 168350 : continue;
1332 : : }
1333 : 639488 : if (!tree_fits_uhwi_p (TYPE_SIZE (type)))
1334 : : {
1335 : 3 : if (dump_file && (dump_flags & TDF_DETAILS))
1336 : 0 : fprintf (dump_file, " not a candidate, size not representable\n");
1337 : 3 : continue;
1338 : : }
1339 : 639485 : unsigned HOST_WIDE_INT type_size
1340 : 639485 : = tree_to_uhwi (TYPE_SIZE (type)) / BITS_PER_UNIT;
1341 : 643526 : if (type_size == 0
1342 : 639485 : || type_size >= ISRA_ARG_SIZE_LIMIT)
1343 : : {
1344 : 4041 : if (dump_file && (dump_flags & TDF_DETAILS))
1345 : 0 : fprintf (dump_file, " not a candidate, has zero or huge size\n");
1346 : 4041 : continue;
1347 : : }
1348 : 635444 : if (type_internals_preclude_sra_p (type, &msg))
1349 : : {
1350 : 9007 : if (dump_file && (dump_flags & TDF_DETAILS))
1351 : 0 : fprintf (dump_file, " not a candidate, %s\n", msg);
1352 : 9007 : continue;
1353 : : }
1354 : :
1355 : 626437 : if (dump_file && (dump_flags & TDF_DETAILS))
1356 : 15 : fprintf (dump_file, " is a candidate\n");
1357 : :
1358 : 626437 : ret = true;
1359 : 626437 : desc->split_candidate = true;
1360 : 626437 : if (desc->by_ref && !desc->safe_ref)
1361 : 188516 : desc->deref_index = unsafe_by_ref_count++;
1362 : : }
1363 : 833517 : return ret;
1364 : : }
1365 : :
1366 : : /* Return pointer to descriptor of parameter DECL or NULL if it cannot be
1367 : : found, which happens if DECL is for a static chain. */
1368 : :
1369 : : static gensum_param_desc *
1370 : 2513151 : get_gensum_param_desc (tree decl)
1371 : : {
1372 : 2513151 : if (!decl2desc)
1373 : : return NULL;
1374 : 2336188 : gcc_checking_assert (TREE_CODE (decl) == PARM_DECL);
1375 : 2336188 : gensum_param_desc **slot = decl2desc->get (decl);
1376 : 2336188 : if (!slot)
1377 : : /* This can happen for static chains which we cannot handle so far. */
1378 : : return NULL;
1379 : 2310852 : gcc_checking_assert (*slot);
1380 : : return *slot;
1381 : : }
1382 : :
1383 : :
1384 : : /* Remove parameter described by DESC from candidates for IPA-SRA splitting and
1385 : : write REASON to the dump file if there is one. */
1386 : :
1387 : : static void
1388 : 176611 : disqualify_split_candidate (gensum_param_desc *desc, const char *reason)
1389 : : {
1390 : 176611 : if (!desc->split_candidate)
1391 : : return;
1392 : :
1393 : 176582 : if (dump_file && (dump_flags & TDF_DETAILS))
1394 : 0 : fprintf (dump_file, "! Disqualifying parameter number %i - %s\n",
1395 : : desc->param_number, reason);
1396 : :
1397 : 176582 : desc->split_candidate = false;
1398 : : }
1399 : :
1400 : : /* Remove DECL from candidates for IPA-SRA and write REASON to the dump file if
1401 : : there is one. */
1402 : :
1403 : : static void
1404 : 177 : disqualify_split_candidate (tree decl, const char *reason)
1405 : : {
1406 : 177 : gensum_param_desc *desc = get_gensum_param_desc (decl);
1407 : 177 : if (desc)
1408 : 29 : disqualify_split_candidate (desc, reason);
1409 : 177 : }
1410 : :
1411 : : /* Allocate a new access to DESC and fill it in with OFFSET and SIZE. But
1412 : : first, check that there are not too many of them already. If so, do not
1413 : : allocate anything and return NULL. */
1414 : :
1415 : : static gensum_param_access *
1416 : 452303 : allocate_access (gensum_param_desc *desc,
1417 : : HOST_WIDE_INT offset, HOST_WIDE_INT size)
1418 : : {
1419 : 452303 : if (desc->access_count
1420 : 452303 : == (unsigned) param_ipa_sra_max_replacements)
1421 : : {
1422 : 0 : disqualify_split_candidate (desc, "Too many replacement candidates");
1423 : 0 : return NULL;
1424 : : }
1425 : :
1426 : 452303 : gensum_param_access *access
1427 : 452303 : = (gensum_param_access *) obstack_alloc (&gensum_obstack,
1428 : : sizeof (gensum_param_access));
1429 : 452303 : memset (access, 0, sizeof (*access));
1430 : 452303 : access->offset = offset;
1431 : 452303 : access->size = size;
1432 : 452303 : access->load_count = profile_count::zero ();
1433 : 452303 : return access;
1434 : : }
1435 : :
1436 : : /* In what context scan_expr_access has been called, whether it deals with a
1437 : : load, a function argument, or a store. Please note that in rare
1438 : : circumstances when it is not clear if the access is a load or store,
1439 : : ISRA_CTX_STORE is used too. */
1440 : :
1441 : : enum isra_scan_context {ISRA_CTX_LOAD, ISRA_CTX_ARG, ISRA_CTX_STORE};
1442 : :
1443 : : /* Return an access describing memory access to the variable described by DESC
1444 : : at OFFSET with SIZE in context CTX, starting at pointer to the linked list
1445 : : at a certain tree level FIRST. Attempt to create it and put into the
1446 : : appropriate place in the access tree if does not exist, but fail and return
1447 : : NULL if there are already too many accesses, if it would create a partially
1448 : : overlapping access or if an access would end up within a pre-existing
1449 : : non-call access. */
1450 : :
1451 : : static gensum_param_access *
1452 : 550882 : get_access_1 (gensum_param_desc *desc, gensum_param_access **first,
1453 : : HOST_WIDE_INT offset, HOST_WIDE_INT size, isra_scan_context ctx)
1454 : : {
1455 : 561089 : gensum_param_access *access = *first, **ptr = first;
1456 : :
1457 : 561089 : if (!access)
1458 : : {
1459 : : /* No pre-existing access at this level, just create it. */
1460 : 299466 : gensum_param_access *a = allocate_access (desc, offset, size);
1461 : 299466 : if (!a)
1462 : : return NULL;
1463 : 299466 : *first = a;
1464 : 299466 : return *first;
1465 : : }
1466 : :
1467 : 261623 : if (access->offset >= offset + size)
1468 : : {
1469 : : /* We want to squeeze it in front of the very first access, just do
1470 : : it. */
1471 : 44841 : gensum_param_access *r = allocate_access (desc, offset, size);
1472 : 44841 : if (!r)
1473 : : return NULL;
1474 : 44841 : r->next_sibling = access;
1475 : 44841 : *first = r;
1476 : 44841 : return r;
1477 : : }
1478 : :
1479 : : /* Skip all accesses that have to come before us until the next sibling is
1480 : : already too far. */
1481 : 309766 : while (offset >= access->offset + access->size
1482 : 190331 : && access->next_sibling
1483 : 413755 : && access->next_sibling->offset < offset + size)
1484 : : {
1485 : 92984 : ptr = &access->next_sibling;
1486 : 92984 : access = access->next_sibling;
1487 : : }
1488 : :
1489 : : /* At this point we know we do not belong before access. */
1490 : 216782 : gcc_assert (access->offset < offset + size);
1491 : :
1492 : 216782 : if (access->offset == offset && access->size == size)
1493 : : /* We found what we were looking for. */
1494 : : return access;
1495 : :
1496 : 124117 : if (access->offset <= offset
1497 : 121138 : && access->offset + access->size >= offset + size)
1498 : : {
1499 : : /* We fit into access which is larger than us. We need to find/create
1500 : : something below access. But we only allow nesting in call
1501 : : arguments. */
1502 : 12565 : if (access->nonarg)
1503 : : return NULL;
1504 : :
1505 : 10207 : return get_access_1 (desc, &access->first_child, offset, size, ctx);
1506 : : }
1507 : :
1508 : 111552 : if (offset <= access->offset
1509 : 14205 : && offset + size >= access->offset + access->size)
1510 : : /* We are actually bigger than access, which fully fits into us, take its
1511 : : place and make all accesses fitting into it its children. */
1512 : : {
1513 : : /* But first, we only allow nesting in call arguments so check if that is
1514 : : what we are trying to represent. */
1515 : 14205 : if (ctx != ISRA_CTX_ARG)
1516 : : return NULL;
1517 : :
1518 : 10649 : gensum_param_access *r = allocate_access (desc, offset, size);
1519 : 10649 : if (!r)
1520 : : return NULL;
1521 : 10649 : r->first_child = access;
1522 : :
1523 : 10649 : while (access->next_sibling
1524 : 18858 : && access->next_sibling->offset < offset + size)
1525 : : access = access->next_sibling;
1526 : 10649 : if (access->offset + access->size > offset + size)
1527 : : {
1528 : : /* This must be a different access, which are sorted, so the
1529 : : following must be true and this signals a partial overlap. */
1530 : 0 : gcc_assert (access->offset > offset);
1531 : : return NULL;
1532 : : }
1533 : :
1534 : 10649 : r->next_sibling = access->next_sibling;
1535 : 10649 : access->next_sibling = NULL;
1536 : 10649 : *ptr = r;
1537 : 10649 : return r;
1538 : : }
1539 : :
1540 : 97347 : if (offset >= access->offset + access->size)
1541 : : {
1542 : : /* We belong after access. */
1543 : 97347 : gensum_param_access *r = allocate_access (desc, offset, size);
1544 : 97347 : if (!r)
1545 : : return NULL;
1546 : 97347 : r->next_sibling = access->next_sibling;
1547 : 97347 : access->next_sibling = r;
1548 : 97347 : return r;
1549 : : }
1550 : :
1551 : 0 : if (offset < access->offset)
1552 : : {
1553 : : /* We know the following, otherwise we would have created a
1554 : : super-access. */
1555 : 0 : gcc_checking_assert (offset + size < access->offset + access->size);
1556 : : return NULL;
1557 : : }
1558 : :
1559 : 0 : if (offset + size > access->offset + access->size)
1560 : : {
1561 : : /* Likewise. */
1562 : 0 : gcc_checking_assert (offset > access->offset);
1563 : : return NULL;
1564 : : }
1565 : :
1566 : 0 : gcc_unreachable ();
1567 : : }
1568 : :
1569 : : /* Return an access describing memory access to the variable described by DESC
1570 : : at OFFSET with SIZE in context CTX, mark it as used in context CTX. Attempt
1571 : : to create if it does not exist, but fail and return NULL if there are
1572 : : already too many accesses, if it would create a partially overlapping access
1573 : : or if an access would end up in a non-call access. */
1574 : :
1575 : : static gensum_param_access *
1576 : 550882 : get_access (gensum_param_desc *desc, HOST_WIDE_INT offset, HOST_WIDE_INT size,
1577 : : isra_scan_context ctx)
1578 : : {
1579 : 550882 : gcc_checking_assert (desc->split_candidate);
1580 : :
1581 : 550882 : gensum_param_access *access = get_access_1 (desc, &desc->accesses, offset,
1582 : : size, ctx);
1583 : 550882 : if (!access)
1584 : : {
1585 : 5914 : disqualify_split_candidate (desc,
1586 : : "Bad access overlap or too many accesses");
1587 : 5914 : return NULL;
1588 : : }
1589 : :
1590 : 544968 : switch (ctx)
1591 : : {
1592 : 20619 : case ISRA_CTX_STORE:
1593 : 20619 : gcc_assert (!desc->by_ref);
1594 : : /* Fall-through */
1595 : 444516 : case ISRA_CTX_LOAD:
1596 : 444516 : access->nonarg = true;
1597 : 444516 : break;
1598 : : case ISRA_CTX_ARG:
1599 : : break;
1600 : : }
1601 : :
1602 : : return access;
1603 : : }
1604 : :
1605 : : /* Verify that parameter access tree starting with ACCESS is in good shape.
1606 : : PARENT_OFFSET and PARENT_SIZE are the corresponding fields of parent of
1607 : : ACCESS or zero if there is none. */
1608 : :
1609 : : static bool
1610 : 939677 : verify_access_tree_1 (gensum_param_access *access, HOST_WIDE_INT parent_offset,
1611 : : HOST_WIDE_INT parent_size)
1612 : : {
1613 : 1338888 : while (access)
1614 : : {
1615 : 399211 : gcc_assert (access->offset >= 0 && access->size >= 0);
1616 : :
1617 : 399211 : if (parent_size != 0)
1618 : : {
1619 : 21638 : if (access->offset < parent_offset)
1620 : : {
1621 : 0 : error ("Access offset before parent offset");
1622 : 0 : return true;
1623 : : }
1624 : 21638 : if (access->size >= parent_size)
1625 : : {
1626 : 0 : error ("Access size greater or equal to its parent size");
1627 : 0 : return true;
1628 : : }
1629 : 21638 : if (access->offset + access->size > parent_offset + parent_size)
1630 : : {
1631 : 0 : error ("Access terminates outside of its parent");
1632 : 0 : return true;
1633 : : }
1634 : : }
1635 : :
1636 : 399211 : if (verify_access_tree_1 (access->first_child, access->offset,
1637 : : access->size))
1638 : : return true;
1639 : :
1640 : 399211 : if (access->next_sibling
1641 : 129442 : && (access->next_sibling->offset < access->offset + access->size))
1642 : : {
1643 : 0 : error ("Access overlaps with its sibling");
1644 : 0 : return true;
1645 : : }
1646 : :
1647 : : access = access->next_sibling;
1648 : : }
1649 : : return false;
1650 : : }
1651 : :
1652 : : /* Verify that parameter access tree starting with ACCESS is in good shape,
1653 : : halt compilation and dump the tree to stderr if not. */
1654 : :
1655 : : DEBUG_FUNCTION void
1656 : 540466 : isra_verify_access_tree (gensum_param_access *access)
1657 : : {
1658 : 540466 : if (verify_access_tree_1 (access, 0, 0))
1659 : : {
1660 : 0 : for (; access; access = access->next_sibling)
1661 : 0 : dump_gensum_access (stderr, access, 2);
1662 : 0 : internal_error ("IPA-SRA access verification failed");
1663 : : }
1664 : 540466 : }
1665 : :
1666 : :
1667 : : /* Callback of walk_stmt_load_store_addr_ops visit_addr used to determine
1668 : : GIMPLE_ASM operands with memory constrains which cannot be scalarized. */
1669 : :
1670 : : static bool
1671 : 6549 : asm_visit_addr (gimple *, tree op, tree, void *)
1672 : : {
1673 : 6549 : op = get_base_address (op);
1674 : 6549 : if (op
1675 : 6549 : && TREE_CODE (op) == PARM_DECL)
1676 : 177 : disqualify_split_candidate (op, "Non-scalarizable GIMPLE_ASM operand.");
1677 : :
1678 : 6549 : return false;
1679 : : }
1680 : :
1681 : : /* Mark a dereference of parameter identified by DESC of distance DIST in a
1682 : : basic block BB, unless the BB has already been marked as a potentially
1683 : : final. */
1684 : :
1685 : : static void
1686 : 255011 : mark_param_dereference (gensum_param_desc *desc, HOST_WIDE_INT dist,
1687 : : basic_block bb)
1688 : : {
1689 : 255011 : gcc_assert (desc->by_ref);
1690 : 255011 : gcc_checking_assert (desc->split_candidate);
1691 : :
1692 : 255011 : if (desc->safe_ref
1693 : 255011 : || bitmap_bit_p (final_bbs, bb->index))
1694 : 230848 : return;
1695 : :
1696 : 24163 : int idx = bb->index * unsafe_by_ref_count + desc->deref_index;
1697 : 24163 : if (bb_dereferences[idx] < dist)
1698 : 22639 : bb_dereferences[idx] = dist;
1699 : : }
1700 : :
1701 : : /* Return true, if any potential replacements should use NEW_TYPE as opposed to
1702 : : previously recorded OLD_TYPE. */
1703 : :
1704 : : static bool
1705 : 83257 : type_prevails_p (tree old_type, tree new_type)
1706 : : {
1707 : 83257 : if (old_type == new_type)
1708 : : return false;
1709 : :
1710 : : /* Non-aggregates are always better. */
1711 : 78 : if (!is_gimple_reg_type (old_type)
1712 : 78 : && is_gimple_reg_type (new_type))
1713 : : return true;
1714 : 78 : if (is_gimple_reg_type (old_type)
1715 : 78 : && !is_gimple_reg_type (new_type))
1716 : : return false;
1717 : :
1718 : : /* Prefer any complex or vector type over any other scalar type. */
1719 : 68 : if (TREE_CODE (old_type) != COMPLEX_TYPE
1720 : 68 : && TREE_CODE (old_type) != VECTOR_TYPE
1721 : 55 : && (TREE_CODE (new_type) == COMPLEX_TYPE
1722 : 55 : || VECTOR_TYPE_P (new_type)))
1723 : : return true;
1724 : 68 : if ((TREE_CODE (old_type) == COMPLEX_TYPE
1725 : : || VECTOR_TYPE_P (old_type))
1726 : 13 : && TREE_CODE (new_type) != COMPLEX_TYPE
1727 : 0 : && TREE_CODE (new_type) != VECTOR_TYPE)
1728 : : return false;
1729 : :
1730 : : /* Use the integral type with the bigger precision. */
1731 : 68 : if (INTEGRAL_TYPE_P (old_type)
1732 : 24 : && INTEGRAL_TYPE_P (new_type))
1733 : 24 : return (TYPE_PRECISION (new_type) > TYPE_PRECISION (old_type));
1734 : :
1735 : : /* Attempt to disregard any integral type with non-full precision. */
1736 : 44 : if (INTEGRAL_TYPE_P (old_type)
1737 : 44 : && (TREE_INT_CST_LOW (TYPE_SIZE (old_type))
1738 : 0 : != TYPE_PRECISION (old_type)))
1739 : : return true;
1740 : 44 : if (INTEGRAL_TYPE_P (new_type)
1741 : 44 : && (TREE_INT_CST_LOW (TYPE_SIZE (new_type))
1742 : 6 : != TYPE_PRECISION (new_type)))
1743 : : return false;
1744 : : /* Stabilize the selection. */
1745 : 44 : return TYPE_UID (old_type) < TYPE_UID (new_type);
1746 : : }
1747 : :
1748 : : /* When scanning an expression which is a call argument, this structure
1749 : : specifies the call and the position of the argument. */
1750 : :
1751 : : struct scan_call_info
1752 : : {
1753 : : /* Call graph edge representing the call. */
1754 : : cgraph_edge *cs;
1755 : : /* Total number of arguments in the call. */
1756 : : unsigned argument_count;
1757 : : /* Number of the actual argument being scanned. */
1758 : : unsigned arg_idx;
1759 : : };
1760 : :
1761 : : /* Record use of ACCESS which belongs to a parameter described by DESC in a
1762 : : call argument described by CALL_INFO. */
1763 : :
1764 : : static void
1765 : 97126 : record_nonregister_call_use (gensum_param_desc *desc,
1766 : : scan_call_info *call_info,
1767 : : unsigned unit_offset, unsigned unit_size)
1768 : : {
1769 : 97126 : isra_call_summary *csum = call_sums->get_create (call_info->cs);
1770 : 97126 : csum->init_inputs (call_info->argument_count);
1771 : :
1772 : 97126 : isra_param_flow *param_flow = &csum->m_arg_flow[call_info->arg_idx];
1773 : 97126 : param_flow->aggregate_pass_through = true;
1774 : 97126 : set_single_param_flow_source (param_flow, desc->param_number);
1775 : 97126 : param_flow->unit_offset = unit_offset;
1776 : 97126 : param_flow->unit_size = unit_size;
1777 : 97126 : desc->call_uses++;
1778 : 97126 : }
1779 : :
1780 : : /* Callback of walk_aliased_vdefs, just mark that there was a possible
1781 : : modification. */
1782 : :
1783 : : static bool
1784 : 71733 : mark_maybe_modified (ao_ref *, tree, void *data)
1785 : : {
1786 : 71733 : bool *maybe_modified = (bool *) data;
1787 : 71733 : *maybe_modified = true;
1788 : 71733 : return true;
1789 : : }
1790 : :
1791 : : /* Analyze expression EXPR from GIMPLE for accesses to parameters. CTX
1792 : : specifies whether EXPR is used in a load, store or as an argument call. BB
1793 : : must be the basic block in which expr resides. If CTX specifies call
1794 : : argument context, CALL_INFO must describe that call and argument position,
1795 : : otherwise it is ignored. */
1796 : :
1797 : : static void
1798 : 35800839 : scan_expr_access (tree expr, gimple *stmt, isra_scan_context ctx,
1799 : : basic_block bb, scan_call_info *call_info = NULL)
1800 : : {
1801 : 35800839 : poly_int64 poffset, psize, pmax_size;
1802 : 35800839 : HOST_WIDE_INT offset, size, max_size;
1803 : 35800839 : tree base;
1804 : 35800839 : bool deref = false;
1805 : 35800839 : bool reverse;
1806 : :
1807 : 35800839 : if (TREE_CODE (expr) == ADDR_EXPR)
1808 : : {
1809 : 4382855 : if (ctx == ISRA_CTX_ARG)
1810 : : return;
1811 : 1188807 : tree t = get_base_address (TREE_OPERAND (expr, 0));
1812 : 1188807 : if (VAR_P (t) && !TREE_STATIC (t))
1813 : 241183 : loaded_decls->add (t);
1814 : 1188807 : return;
1815 : : }
1816 : 31417984 : if (TREE_CODE (expr) == SSA_NAME
1817 : 17486555 : || CONSTANT_CLASS_P (expr))
1818 : : return;
1819 : :
1820 : 12652641 : if (TREE_CODE (expr) == BIT_FIELD_REF
1821 : 12652641 : || TREE_CODE (expr) == IMAGPART_EXPR
1822 : 12542027 : || TREE_CODE (expr) == REALPART_EXPR)
1823 : 162991 : expr = TREE_OPERAND (expr, 0);
1824 : :
1825 : 12652641 : base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size, &reverse);
1826 : :
1827 : 12652641 : if (TREE_CODE (base) == MEM_REF)
1828 : : {
1829 : 4013142 : tree op = TREE_OPERAND (base, 0);
1830 : 4013142 : if (TREE_CODE (op) != SSA_NAME
1831 : 4013142 : || !SSA_NAME_IS_DEFAULT_DEF (op))
1832 : : return;
1833 : 2108090 : base = SSA_NAME_VAR (op);
1834 : 2108090 : if (!base)
1835 : : return;
1836 : : deref = true;
1837 : : }
1838 : 8639499 : else if (VAR_P (base)
1839 : 7475288 : && !TREE_STATIC (base)
1840 : 6527107 : && (ctx == ISRA_CTX_ARG
1841 : : || ctx == ISRA_CTX_LOAD))
1842 : 2403227 : loaded_decls->add (base);
1843 : :
1844 : 10747589 : if (TREE_CODE (base) != PARM_DECL)
1845 : : return;
1846 : :
1847 : 2512974 : gensum_param_desc *desc = get_gensum_param_desc (base);
1848 : 2512974 : if (!desc || !desc->split_candidate)
1849 : : return;
1850 : :
1851 : 621525 : if (storage_order_barrier_p (expr))
1852 : : {
1853 : 0 : disqualify_split_candidate (desc, "Encountered a storage order barrier.");
1854 : 0 : return;
1855 : : }
1856 : :
1857 : 621525 : if (!poffset.is_constant (&offset)
1858 : 621525 : || !psize.is_constant (&size)
1859 : 621525 : || !pmax_size.is_constant (&max_size))
1860 : : {
1861 : : disqualify_split_candidate (desc, "Encountered a polynomial-sized "
1862 : : "access.");
1863 : : return;
1864 : : }
1865 : 621525 : if (size < 0 || size != max_size)
1866 : : {
1867 : 5667 : disqualify_split_candidate (desc, "Encountered a variable sized access.");
1868 : 5667 : return;
1869 : : }
1870 : 615858 : if (TREE_CODE (expr) == COMPONENT_REF
1871 : 615858 : && DECL_BIT_FIELD (TREE_OPERAND (expr, 1)))
1872 : : {
1873 : 1817 : disqualify_split_candidate (desc, "Encountered a bit-field access.");
1874 : 1817 : return;
1875 : : }
1876 : 614041 : if (offset < 0)
1877 : : {
1878 : 4 : disqualify_split_candidate (desc, "Encountered an access at a "
1879 : : "negative offset.");
1880 : 4 : return;
1881 : : }
1882 : 614037 : gcc_assert ((offset % BITS_PER_UNIT) == 0);
1883 : 614037 : gcc_assert ((size % BITS_PER_UNIT) == 0);
1884 : 614037 : if ((offset / BITS_PER_UNIT) >= (UINT_MAX - ISRA_ARG_SIZE_LIMIT)
1885 : 614037 : || (size / BITS_PER_UNIT) >= ISRA_ARG_SIZE_LIMIT)
1886 : : {
1887 : 0 : disqualify_split_candidate (desc, "Encountered an access with too big "
1888 : : "offset or size");
1889 : 0 : return;
1890 : : }
1891 : :
1892 : 614037 : tree type = TREE_TYPE (expr);
1893 : 614037 : unsigned int exp_align = get_object_alignment (expr);
1894 : :
1895 : 614037 : if (exp_align < TYPE_ALIGN (type))
1896 : : {
1897 : 1328 : disqualify_split_candidate (desc, "Underaligned access.");
1898 : 1328 : return;
1899 : : }
1900 : :
1901 : 612709 : if (deref)
1902 : : {
1903 : 316838 : if (!desc->by_ref)
1904 : : {
1905 : 0 : disqualify_split_candidate (desc, "Dereferencing a non-reference.");
1906 : 61827 : return;
1907 : : }
1908 : 316838 : else if (ctx == ISRA_CTX_STORE)
1909 : : {
1910 : 0 : disqualify_split_candidate (desc, "Storing to data passed by "
1911 : : "reference.");
1912 : 0 : return;
1913 : : }
1914 : :
1915 : 316838 : if (!aa_walking_limit)
1916 : : {
1917 : 0 : disqualify_split_candidate (desc, "Out of alias analysis step "
1918 : : "limit.");
1919 : 0 : return;
1920 : : }
1921 : :
1922 : 633676 : gcc_checking_assert (gimple_vuse (stmt));
1923 : 316838 : bool maybe_modified = false;
1924 : 316838 : ao_ref ar;
1925 : :
1926 : 316838 : ao_ref_init (&ar, expr);
1927 : 316838 : bitmap visited = BITMAP_ALLOC (NULL);
1928 : 633676 : int walked = walk_aliased_vdefs (&ar, gimple_vuse (stmt),
1929 : : mark_maybe_modified, &maybe_modified,
1930 : : &visited, NULL, aa_walking_limit);
1931 : 316838 : BITMAP_FREE (visited);
1932 : 316838 : if (walked > 0)
1933 : : {
1934 : 112084 : gcc_assert (aa_walking_limit > walked);
1935 : 112084 : aa_walking_limit = aa_walking_limit - walked;
1936 : : }
1937 : 316838 : if (walked < 0)
1938 : 0 : aa_walking_limit = 0;
1939 : 316838 : if (maybe_modified || walked < 0)
1940 : : {
1941 : 61827 : disqualify_split_candidate (desc, "Data passed by reference possibly "
1942 : : "modified through an alias.");
1943 : 61827 : return;
1944 : : }
1945 : : else
1946 : 255011 : mark_param_dereference (desc, offset + size, bb);
1947 : : }
1948 : : else
1949 : : /* Pointer parameters with direct uses should have been ruled out by
1950 : : analyzing SSA default def when looking at the parameters. */
1951 : 295871 : gcc_assert (!desc->by_ref);
1952 : :
1953 : 550882 : gensum_param_access *access = get_access (desc, offset, size, ctx);
1954 : 550882 : if (!access)
1955 : : return;
1956 : :
1957 : 544968 : if (ctx == ISRA_CTX_ARG)
1958 : : {
1959 : 100452 : gcc_checking_assert (call_info);
1960 : :
1961 : 100452 : if (!deref)
1962 : 97126 : record_nonregister_call_use (desc, call_info, offset / BITS_PER_UNIT,
1963 : 97126 : size / BITS_PER_UNIT);
1964 : : else
1965 : : /* This is not a pass-through of a pointer, this is a use like any
1966 : : other. */
1967 : 3326 : access->nonarg = true;
1968 : : }
1969 : 444516 : else if (ctx == ISRA_CTX_LOAD && bb->count.initialized_p ())
1970 : 423809 : access->load_count += bb->count;
1971 : :
1972 : 544968 : if (!access->type)
1973 : : {
1974 : 452303 : access->type = type;
1975 : 452303 : access->alias_ptr_type = reference_alias_ptr_type (expr);
1976 : 452303 : access->reverse = reverse;
1977 : : }
1978 : : else
1979 : : {
1980 : 92665 : if (exp_align < TYPE_ALIGN (access->type))
1981 : : {
1982 : 0 : disqualify_split_candidate (desc, "Reference has lower alignment "
1983 : : "than a previous one.");
1984 : 0 : return;
1985 : : }
1986 : 92665 : if (access->alias_ptr_type != reference_alias_ptr_type (expr))
1987 : : {
1988 : 3884 : disqualify_split_candidate (desc, "Multiple alias pointer types.");
1989 : 3884 : return;
1990 : : }
1991 : 88781 : if (access->reverse != reverse)
1992 : : {
1993 : 0 : disqualify_split_candidate (desc, "Both normal and reverse "
1994 : : "scalar storage order.");
1995 : 0 : return;
1996 : : }
1997 : 88781 : if (!deref
1998 : 72827 : && (AGGREGATE_TYPE_P (type) || AGGREGATE_TYPE_P (access->type))
1999 : 148220 : && (TYPE_MAIN_VARIANT (access->type) != TYPE_MAIN_VARIANT (type)))
2000 : : {
2001 : : /* We need the same aggregate type on all accesses to be able to
2002 : : distinguish transformation spots from pass-through arguments in
2003 : : the transformation phase. */
2004 : 5524 : disqualify_split_candidate (desc, "We do not support aggregate "
2005 : : "type punning.");
2006 : 5524 : return;
2007 : : }
2008 : :
2009 : 83257 : if (type_prevails_p (access->type, type))
2010 : 25 : access->type = type;
2011 : : }
2012 : : }
2013 : :
2014 : : /* Scan body function described by NODE and FUN and create access trees for
2015 : : parameters. */
2016 : :
2017 : : static void
2018 : 1258829 : scan_function (cgraph_node *node, struct function *fun)
2019 : : {
2020 : 1258829 : basic_block bb;
2021 : :
2022 : 10342219 : FOR_EACH_BB_FN (bb, fun)
2023 : : {
2024 : 9083390 : gimple_stmt_iterator gsi;
2025 : 70974029 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2026 : : {
2027 : 52807249 : gimple *stmt = gsi_stmt (gsi);
2028 : :
2029 : 52807249 : if (final_bbs && stmt_can_throw_external (fun, stmt))
2030 : 1377897 : bitmap_set_bit (final_bbs, bb->index);
2031 : 52807249 : switch (gimple_code (stmt))
2032 : : {
2033 : 1226994 : case GIMPLE_RETURN:
2034 : 1226994 : {
2035 : 1226994 : tree t = gimple_return_retval (as_a <greturn *> (stmt));
2036 : 1226994 : if (t != NULL_TREE)
2037 : 676827 : scan_expr_access (t, stmt, ISRA_CTX_LOAD, bb);
2038 : 1226994 : if (final_bbs)
2039 : 414512 : bitmap_set_bit (final_bbs, bb->index);
2040 : : }
2041 : : break;
2042 : :
2043 : 18338977 : case GIMPLE_ASSIGN:
2044 : 18338977 : if (gimple_assign_single_p (stmt)
2045 : 18338977 : && !gimple_clobber_p (stmt))
2046 : : {
2047 : 11081674 : tree rhs = gimple_assign_rhs1 (stmt);
2048 : 11081674 : scan_expr_access (rhs, stmt, ISRA_CTX_LOAD, bb);
2049 : 11081674 : tree lhs = gimple_assign_lhs (stmt);
2050 : 11081674 : scan_expr_access (lhs, stmt, ISRA_CTX_STORE, bb);
2051 : : }
2052 : : break;
2053 : :
2054 : 5292088 : case GIMPLE_CALL:
2055 : 5292088 : {
2056 : 5292088 : unsigned argument_count = gimple_call_num_args (stmt);
2057 : 5292088 : isra_scan_context ctx = ISRA_CTX_ARG;
2058 : 5292088 : scan_call_info call_info, *call_info_p = &call_info;
2059 : 5292088 : if (gimple_call_internal_p (stmt))
2060 : : {
2061 : 155442 : call_info_p = NULL;
2062 : 155442 : ctx = ISRA_CTX_LOAD;
2063 : 155442 : internal_fn ifn = gimple_call_internal_fn (stmt);
2064 : 155442 : if (internal_store_fn_p (ifn))
2065 : 0 : ctx = ISRA_CTX_STORE;
2066 : : }
2067 : : else
2068 : : {
2069 : 5136646 : call_info.cs = node->get_edge (stmt);
2070 : 5136646 : call_info.argument_count = argument_count;
2071 : : }
2072 : :
2073 : 16011503 : for (unsigned i = 0; i < argument_count; i++)
2074 : : {
2075 : 10719415 : call_info.arg_idx = i;
2076 : 10719415 : scan_expr_access (gimple_call_arg (stmt, i), stmt,
2077 : : ctx, bb, call_info_p);
2078 : : }
2079 : :
2080 : 5292088 : tree lhs = gimple_call_lhs (stmt);
2081 : 5292088 : if (lhs)
2082 : 2158738 : scan_expr_access (lhs, stmt, ISRA_CTX_STORE, bb);
2083 : 5292088 : int flags = gimple_call_flags (stmt);
2084 : 5292088 : if (final_bbs
2085 : 2111940 : && (((flags & (ECF_CONST | ECF_PURE)) == 0)
2086 : 321113 : || (flags & ECF_LOOPING_CONST_OR_PURE)))
2087 : 1846801 : bitmap_set_bit (final_bbs, bb->index);
2088 : : }
2089 : 5292088 : break;
2090 : :
2091 : 47155 : case GIMPLE_ASM:
2092 : 47155 : {
2093 : 47155 : gasm *asm_stmt = as_a <gasm *> (stmt);
2094 : 47155 : walk_stmt_load_store_addr_ops (asm_stmt, NULL, NULL, NULL,
2095 : : asm_visit_addr);
2096 : 47155 : if (final_bbs)
2097 : 1355 : bitmap_set_bit (final_bbs, bb->index);
2098 : :
2099 : 81132 : for (unsigned i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
2100 : : {
2101 : 33977 : tree t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
2102 : 33977 : scan_expr_access (t, stmt, ISRA_CTX_LOAD, bb);
2103 : : }
2104 : 95689 : for (unsigned i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
2105 : : {
2106 : 48534 : tree t = TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
2107 : 48534 : scan_expr_access (t, stmt, ISRA_CTX_STORE, bb);
2108 : : }
2109 : : }
2110 : : break;
2111 : :
2112 : : default:
2113 : : break;
2114 : : }
2115 : : }
2116 : : }
2117 : 1258829 : }
2118 : :
2119 : : /* Return true if SSA_NAME NAME of function described by FUN is only used in
2120 : : return statements, or if results of any operations it is involved in are
2121 : : only used in return statements. ANALYZED is a bitmap that tracks which SSA
2122 : : names we have already started investigating. */
2123 : :
2124 : : static bool
2125 : 2426606 : ssa_name_only_returned_p (function *fun, tree name, bitmap analyzed)
2126 : : {
2127 : 2426606 : bool res = true;
2128 : 2426606 : imm_use_iterator imm_iter;
2129 : 2426606 : gimple *stmt;
2130 : :
2131 : 3069467 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
2132 : : {
2133 : 2654605 : if (is_gimple_debug (stmt))
2134 : 183768 : continue;
2135 : :
2136 : 2470837 : if (gimple_code (stmt) == GIMPLE_RETURN)
2137 : : {
2138 : 247177 : tree t = gimple_return_retval (as_a <greturn *> (stmt));
2139 : 247177 : if (t != name)
2140 : : {
2141 : : res = false;
2142 : : break;
2143 : : }
2144 : : }
2145 : 2223660 : else if (!stmt_unremovable_because_of_non_call_eh_p (fun, stmt)
2146 : 2223660 : && ((is_gimple_assign (stmt) && !gimple_has_volatile_ops (stmt))
2147 : 1154996 : || gimple_code (stmt) == GIMPLE_PHI))
2148 : : {
2149 : : /* TODO: And perhaps for const function calls too? */
2150 : 1191389 : tree lhs;
2151 : 1191389 : if (gimple_code (stmt) == GIMPLE_PHI)
2152 : 218697 : lhs = gimple_phi_result (stmt);
2153 : : else
2154 : 972692 : lhs = gimple_assign_lhs (stmt);
2155 : :
2156 : 1191389 : if (TREE_CODE (lhs) != SSA_NAME)
2157 : : {
2158 : : res = false;
2159 : : break;
2160 : : }
2161 : 840117 : gcc_assert (!gimple_vdef (stmt));
2162 : 840117 : if (bitmap_set_bit (analyzed, SSA_NAME_VERSION (lhs))
2163 : 840117 : && !ssa_name_only_returned_p (fun, lhs, analyzed))
2164 : : {
2165 : : res = false;
2166 : : break;
2167 : : }
2168 : : }
2169 : : else
2170 : : {
2171 : : res = false;
2172 : : break;
2173 : : }
2174 : 2426606 : }
2175 : 2426606 : return res;
2176 : : }
2177 : :
2178 : : /* Inspect the uses of the return value of the call associated with CS, and if
2179 : : it is not used or if it is only used to construct the return value of the
2180 : : caller, mark it as such in call or caller summary. Also check for
2181 : : misaligned arguments. */
2182 : :
2183 : : static void
2184 : 5136646 : isra_analyze_call (cgraph_edge *cs)
2185 : : {
2186 : 5136646 : gcall *call_stmt = cs->call_stmt;
2187 : 5136646 : unsigned count = gimple_call_num_args (call_stmt);
2188 : 5136646 : isra_call_summary *csum = call_sums->get_create (cs);
2189 : :
2190 : 15338498 : for (unsigned i = 0; i < count; i++)
2191 : : {
2192 : 10201852 : tree arg = gimple_call_arg (call_stmt, i);
2193 : 10201852 : if (TREE_CODE (arg) == ADDR_EXPR)
2194 : : {
2195 : 3194048 : poly_int64 poffset, psize, pmax_size;
2196 : 3194048 : bool reverse;
2197 : 3194048 : tree base = get_ref_base_and_extent (TREE_OPERAND (arg, 0), &poffset,
2198 : 3194048 : &psize, &pmax_size, &reverse);
2199 : 3194048 : HOST_WIDE_INT offset;
2200 : 3194048 : unsigned HOST_WIDE_INT ds;
2201 : 3194048 : if (DECL_P (base)
2202 : 2287996 : && (poffset.is_constant (&offset))
2203 : 2287996 : && tree_fits_uhwi_p (DECL_SIZE (base))
2204 : 5357321 : && ((ds = tree_to_uhwi (DECL_SIZE (base)) - offset)
2205 : : < ISRA_ARG_SIZE_LIMIT * BITS_PER_UNIT))
2206 : : {
2207 : 2030061 : csum->init_inputs (count);
2208 : 2030061 : gcc_assert (!csum->m_arg_flow[i].aggregate_pass_through);
2209 : 2030061 : csum->m_arg_flow[i].unit_size = ds / BITS_PER_UNIT;
2210 : : }
2211 : :
2212 : 3194048 : if (TREE_CODE (base) == VAR_DECL
2213 : 2124977 : && !TREE_STATIC (base)
2214 : 4576921 : && !loaded_decls->contains (base))
2215 : : {
2216 : 904615 : csum->init_inputs (count);
2217 : 904615 : csum->m_arg_flow[i].constructed_for_calls = true;
2218 : : }
2219 : : }
2220 : :
2221 : 10201852 : if (is_gimple_reg (arg))
2222 : 4042320 : continue;
2223 : :
2224 : 6159532 : tree offset;
2225 : 6159532 : poly_int64 bitsize, bitpos;
2226 : 6159532 : machine_mode mode;
2227 : 6159532 : int unsignedp, reversep, volatilep = 0;
2228 : 6159532 : get_inner_reference (arg, &bitsize, &bitpos, &offset, &mode,
2229 : : &unsignedp, &reversep, &volatilep);
2230 : 12319064 : if (!multiple_p (bitpos, BITS_PER_UNIT))
2231 : : {
2232 : 0 : csum->m_bit_aligned_arg = true;
2233 : 0 : break;
2234 : : }
2235 : : }
2236 : :
2237 : 5136646 : tree lhs = gimple_call_lhs (call_stmt);
2238 : 5136646 : if (lhs)
2239 : : {
2240 : : /* TODO: Also detect aggregates on a LHS of a call that are only returned
2241 : : from this function (without being read anywhere). */
2242 : 2019745 : if (TREE_CODE (lhs) == SSA_NAME)
2243 : : {
2244 : 1608633 : bitmap analyzed = BITMAP_ALLOC (NULL);
2245 : 1608633 : if (ssa_name_only_returned_p (DECL_STRUCT_FUNCTION (cs->caller->decl),
2246 : : lhs, analyzed))
2247 : 225090 : csum->m_return_returned = true;
2248 : 1608633 : BITMAP_FREE (analyzed);
2249 : : }
2250 : : }
2251 : : /* Don't set m_return_ignored for musttail calls. The tailc/musttail passes
2252 : : compare the returned value against the IPA-VRP return value range if
2253 : : it is a singleton, but if the call is changed to something which doesn't
2254 : : return anything, it will always fail. */
2255 : 3116901 : else if (!gimple_call_must_tail_p (call_stmt))
2256 : 3116850 : csum->m_return_ignored = true;
2257 : 5136646 : }
2258 : :
2259 : : /* Look at all calls going out of NODE, described also by IFS and perform all
2260 : : analyses necessary for IPA-SRA that are not done at body scan time or done
2261 : : even when body is not scanned because the function is not a candidate. */
2262 : :
2263 : : static void
2264 : 1258829 : isra_analyze_all_outgoing_calls (cgraph_node *node)
2265 : : {
2266 : 6258908 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
2267 : 5000079 : isra_analyze_call (cs);
2268 : 1395396 : for (cgraph_edge *cs = node->indirect_calls; cs; cs = cs->next_callee)
2269 : 136567 : isra_analyze_call (cs);
2270 : 1258829 : }
2271 : :
2272 : : /* Dump a dereferences table with heading STR to file F. */
2273 : :
2274 : : static void
2275 : 10 : dump_dereferences_table (FILE *f, struct function *fun, const char *str)
2276 : : {
2277 : 10 : basic_block bb;
2278 : :
2279 : 10 : fprintf (dump_file, "%s", str);
2280 : 52 : FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun),
2281 : : EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
2282 : : {
2283 : 42 : fprintf (f, "%4i %i ", bb->index, bitmap_bit_p (final_bbs, bb->index));
2284 : 42 : if (bb != EXIT_BLOCK_PTR_FOR_FN (fun))
2285 : : {
2286 : : int i;
2287 : 104 : for (i = 0; i < unsafe_by_ref_count; i++)
2288 : : {
2289 : 62 : int idx = bb->index * unsafe_by_ref_count + i;
2290 : 62 : fprintf (f, " %4" HOST_WIDE_INT_PRINT "d", bb_dereferences[idx]);
2291 : : }
2292 : : }
2293 : 42 : fprintf (f, "\n");
2294 : : }
2295 : 10 : fprintf (dump_file, "\n");
2296 : 10 : }
2297 : :
2298 : : /* Propagate distances in bb_dereferences in the opposite direction than the
2299 : : control flow edges, in each step storing the maximum of the current value
2300 : : and the minimum of all successors. These steps are repeated until the table
2301 : : stabilizes. Note that BBs which might terminate the functions (according to
2302 : : final_bbs bitmap) never updated in this way. */
2303 : :
2304 : : static void
2305 : 25625 : propagate_dereference_distances (struct function *fun)
2306 : : {
2307 : 25625 : basic_block bb;
2308 : :
2309 : 25625 : if (dump_file && (dump_flags & TDF_DETAILS))
2310 : 5 : dump_dereferences_table (dump_file, fun,
2311 : : "Dereference table before propagation:\n");
2312 : :
2313 : 25625 : auto_vec<basic_block> queue (last_basic_block_for_fn (fun));
2314 : 25625 : queue.quick_push (ENTRY_BLOCK_PTR_FOR_FN (fun));
2315 : 217214 : FOR_EACH_BB_FN (bb, fun)
2316 : : {
2317 : 191589 : queue.quick_push (bb);
2318 : 191589 : bb->aux = bb;
2319 : : }
2320 : :
2321 : 243655 : while (!queue.is_empty ())
2322 : : {
2323 : 218030 : edge_iterator ei;
2324 : 218030 : edge e;
2325 : 218030 : bool change = false;
2326 : 218030 : int i;
2327 : :
2328 : 218030 : bb = queue.pop ();
2329 : 218030 : bb->aux = NULL;
2330 : :
2331 : 218030 : if (bitmap_bit_p (final_bbs, bb->index))
2332 : 67214 : continue;
2333 : :
2334 : 6127885 : for (i = 0; i < unsafe_by_ref_count; i++)
2335 : : {
2336 : 5977069 : int idx = bb->index * unsafe_by_ref_count + i;
2337 : 5977069 : bool first = true;
2338 : 5977069 : HOST_WIDE_INT inh = 0;
2339 : :
2340 : 14024984 : FOR_EACH_EDGE (e, ei, bb->succs)
2341 : : {
2342 : 8047915 : int succ_idx = e->dest->index * unsafe_by_ref_count + i;
2343 : :
2344 : 8047915 : if (e->dest == EXIT_BLOCK_PTR_FOR_FN (fun))
2345 : 0 : continue;
2346 : :
2347 : 8047915 : if (first)
2348 : : {
2349 : 5977064 : first = false;
2350 : 5977064 : inh = bb_dereferences [succ_idx];
2351 : : }
2352 : 2070851 : else if (bb_dereferences [succ_idx] < inh)
2353 : : inh = bb_dereferences [succ_idx];
2354 : : }
2355 : :
2356 : 5977069 : if (!first && bb_dereferences[idx] < inh)
2357 : : {
2358 : 9756 : bb_dereferences[idx] = inh;
2359 : 9756 : change = true;
2360 : : }
2361 : : }
2362 : :
2363 : 150816 : if (change)
2364 : 10847 : FOR_EACH_EDGE (e, ei, bb->preds)
2365 : : {
2366 : 1671 : if (e->src->aux)
2367 : 855 : continue;
2368 : :
2369 : 816 : e->src->aux = e->src;
2370 : 816 : queue.quick_push (e->src);
2371 : : }
2372 : : }
2373 : :
2374 : 25625 : if (dump_file && (dump_flags & TDF_DETAILS))
2375 : 5 : dump_dereferences_table (dump_file, fun,
2376 : : "Dereference table after propagation:\n");
2377 : 25625 : }
2378 : :
2379 : : /* Return true if the ACCESS loads happen frequently enough in FUN to risk
2380 : : moving them to the caller and only pass the result. */
2381 : :
2382 : : static bool
2383 : 177788 : dereference_probable_p (struct function *fun, gensum_param_access *access)
2384 : : {
2385 : 177788 : int threshold = opt_for_fn (fun->decl, param_ipa_sra_deref_prob_threshold);
2386 : 177788 : return access->load_count
2387 : 177788 : >= ENTRY_BLOCK_PTR_FOR_FN (fun)->count.apply_scale (threshold, 100);
2388 : : }
2389 : :
2390 : : /* Perform basic checks on ACCESS to PARM (of FUN) described by DESC and all
2391 : : its children, return true if the parameter cannot be split, otherwise return
2392 : : false and update *NONARG_ACC_SIZE and *ONLY_CALLS. ENTRY_BB_INDEX must be
2393 : : the index of the entry BB in the function of PARM. */
2394 : :
2395 : : static bool
2396 : 391599 : check_gensum_access (struct function *fun, tree parm, gensum_param_desc *desc,
2397 : : gensum_param_access *access,
2398 : : HOST_WIDE_INT *nonarg_acc_size, bool *only_calls,
2399 : : int entry_bb_index)
2400 : : {
2401 : 391599 : if (access->nonarg)
2402 : : {
2403 : 340240 : *only_calls = false;
2404 : 340240 : *nonarg_acc_size += access->size;
2405 : :
2406 : 340240 : if (access->first_child)
2407 : : {
2408 : 412 : disqualify_split_candidate (desc, "Overlapping non-call uses.");
2409 : 412 : return true;
2410 : : }
2411 : : }
2412 : : /* Do not decompose a non-BLKmode param in a way that would create
2413 : : BLKmode params. Especially for by-reference passing (thus,
2414 : : pointer-type param) this is hardly worthwhile. */
2415 : 391187 : if (DECL_MODE (parm) != BLKmode
2416 : 391187 : && TYPE_MODE (access->type) == BLKmode)
2417 : : {
2418 : 6662 : disqualify_split_candidate (desc, "Would convert a non-BLK to a BLK.");
2419 : 6662 : return true;
2420 : : }
2421 : :
2422 : 384525 : if (desc->by_ref)
2423 : : {
2424 : 187887 : if (desc->safe_ref)
2425 : : {
2426 : 153476 : if (!dereference_probable_p (fun, access))
2427 : : {
2428 : 7241 : disqualify_split_candidate (desc, "Dereferences in callers "
2429 : : "would happen much more frequently.");
2430 : 7241 : return true;
2431 : : }
2432 : : }
2433 : : else
2434 : : {
2435 : 34411 : int idx = (entry_bb_index * unsafe_by_ref_count + desc->deref_index);
2436 : 34411 : if ((access->offset + access->size) > bb_dereferences[idx])
2437 : : {
2438 : 24312 : if (!dereference_probable_p (fun, access))
2439 : : {
2440 : 1807 : disqualify_split_candidate (desc, "Would create a possibly "
2441 : : "illegal dereference in a "
2442 : : "caller.");
2443 : 1807 : return true;
2444 : : }
2445 : 22505 : desc->conditionally_dereferenceable = true;
2446 : : }
2447 : : }
2448 : : }
2449 : :
2450 : 375477 : for (gensum_param_access *ch = access->first_child;
2451 : 396315 : ch;
2452 : 20838 : ch = ch->next_sibling)
2453 : 20840 : if (check_gensum_access (fun, parm, desc, ch, nonarg_acc_size, only_calls,
2454 : : entry_bb_index))
2455 : : return true;
2456 : :
2457 : : return false;
2458 : : }
2459 : :
2460 : : /* Copy data from FROM and all of its children to a vector of accesses in IPA
2461 : : descriptor DESC. */
2462 : :
2463 : : static void
2464 : 452303 : copy_accesses_to_ipa_desc (gensum_param_access *from, isra_param_desc *desc)
2465 : : {
2466 : 452303 : param_access *to = ggc_cleared_alloc<param_access> ();
2467 : 452303 : gcc_checking_assert ((from->offset % BITS_PER_UNIT) == 0);
2468 : 452303 : gcc_checking_assert ((from->size % BITS_PER_UNIT) == 0);
2469 : 452303 : to->unit_offset = from->offset / BITS_PER_UNIT;
2470 : 452303 : to->unit_size = from->size / BITS_PER_UNIT;
2471 : 452303 : to->type = from->type;
2472 : 452303 : to->alias_ptr_type = from->alias_ptr_type;
2473 : 452303 : to->certain = from->nonarg;
2474 : 452303 : to->reverse = from->reverse;
2475 : 452303 : vec_safe_push (desc->accesses, to);
2476 : :
2477 : 452303 : for (gensum_param_access *ch = from->first_child;
2478 : 475292 : ch;
2479 : 22989 : ch = ch->next_sibling)
2480 : 22989 : copy_accesses_to_ipa_desc (ch, desc);
2481 : 452303 : }
2482 : :
2483 : : /* Analyze function body scan results stored in param_accesses and
2484 : : param_accesses, detect possible transformations and store information of
2485 : : those in function summary. NODE, FUN and IFS are all various structures
2486 : : describing the currently analyzed function. */
2487 : :
2488 : : static void
2489 : 833517 : process_scan_results (cgraph_node *node, struct function *fun,
2490 : : isra_func_summary *ifs,
2491 : : vec<gensum_param_desc> *param_descriptions)
2492 : : {
2493 : 833517 : bool check_pass_throughs = false;
2494 : 833517 : bool dereferences_propagated = false;
2495 : 833517 : tree parm = DECL_ARGUMENTS (node->decl);
2496 : 833517 : unsigned param_count = param_descriptions->length();
2497 : :
2498 : 833517 : for (unsigned desc_index = 0;
2499 : 3222497 : desc_index < param_count;
2500 : 2388980 : desc_index++, parm = DECL_CHAIN (parm))
2501 : : {
2502 : 2388980 : gensum_param_desc *desc = &(*param_descriptions)[desc_index];
2503 : 2388980 : if (!desc->split_candidate)
2504 : 1864630 : continue;
2505 : :
2506 : 540472 : if (flag_checking)
2507 : 540466 : isra_verify_access_tree (desc->accesses);
2508 : :
2509 : 540472 : if (!dereferences_propagated
2510 : 535149 : && desc->by_ref
2511 : 374769 : && !desc->safe_ref
2512 : 157950 : && desc->accesses)
2513 : : {
2514 : 25625 : propagate_dereference_distances (fun);
2515 : 25625 : dereferences_propagated = true;
2516 : : }
2517 : :
2518 : 540472 : HOST_WIDE_INT nonarg_acc_size = 0;
2519 : 540472 : bool only_calls = true;
2520 : 540472 : bool check_failed = false;
2521 : :
2522 : 540472 : int entry_bb_index = ENTRY_BLOCK_PTR_FOR_FN (fun)->index;
2523 : 540472 : for (gensum_param_access *acc = desc->accesses;
2524 : 895109 : acc;
2525 : 354637 : acc = acc->next_sibling)
2526 : 370759 : if (check_gensum_access (fun, parm, desc, acc, &nonarg_acc_size,
2527 : : &only_calls, entry_bb_index))
2528 : : {
2529 : : check_failed = true;
2530 : : break;
2531 : : }
2532 : 540472 : if (check_failed)
2533 : 16122 : continue;
2534 : :
2535 : 524350 : if (only_calls)
2536 : 322356 : desc->locally_unused = true;
2537 : :
2538 : 524350 : HOST_WIDE_INT cur_param_size
2539 : 524350 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm)));
2540 : 524350 : HOST_WIDE_INT param_size_limit, optimistic_limit;
2541 : 524350 : if (!desc->by_ref || optimize_function_for_size_p (fun))
2542 : : {
2543 : : param_size_limit = cur_param_size;
2544 : : optimistic_limit = cur_param_size;
2545 : : }
2546 : : else
2547 : : {
2548 : 342136 : param_size_limit
2549 : 342136 : = opt_for_fn (node->decl,
2550 : : param_ipa_sra_ptr_growth_factor) * cur_param_size;
2551 : 342136 : optimistic_limit
2552 : 342136 : = (opt_for_fn (node->decl, param_ipa_sra_ptrwrap_growth_factor)
2553 : : * param_size_limit);
2554 : : }
2555 : :
2556 : 524350 : if (nonarg_acc_size > optimistic_limit
2557 : 522020 : || (!desc->by_ref && nonarg_acc_size == param_size_limit))
2558 : : {
2559 : 74495 : disqualify_split_candidate (desc, "Would result into a too big set "
2560 : : "of replacements even in best "
2561 : : "scenarios.");
2562 : : }
2563 : : else
2564 : : {
2565 : : /* create_parameter_descriptors makes sure unit sizes of all
2566 : : candidate parameters fit unsigned integers restricted to
2567 : : ISRA_ARG_SIZE_LIMIT. */
2568 : 449855 : desc->param_size_limit = param_size_limit / BITS_PER_UNIT;
2569 : 449855 : desc->nonarg_acc_size = nonarg_acc_size / BITS_PER_UNIT;
2570 : 449855 : if (desc->split_candidate && desc->ptr_pt_count)
2571 : : {
2572 : 195294 : gcc_assert (desc->by_ref);
2573 : : check_pass_throughs = true;
2574 : : }
2575 : : }
2576 : : }
2577 : :
2578 : : /* When a pointer parameter is passed-through to a callee, in which it is
2579 : : only used to read only one or a few items, we can attempt to transform it
2580 : : to obtaining and passing through the items instead of the pointer. But we
2581 : : must take extra care that 1) we do not introduce any segfault by moving
2582 : : dereferences above control flow and that 2) the data is not modified
2583 : : through an alias in this function. The IPA analysis must not introduce
2584 : : any accesses candidates unless it can prove both.
2585 : :
2586 : : The current solution is very crude as it consists of ensuring that the
2587 : : call postdominates entry BB and that the definition of VUSE of the call is
2588 : : default definition. TODO: For non-recursive callees in the same
2589 : : compilation unit we could do better by doing analysis in topological order
2590 : : an looking into access candidates of callees, using their alias_ptr_types
2591 : : to attempt real AA. We could also use the maximum known dereferenced
2592 : : offset in this function at IPA level.
2593 : :
2594 : : TODO: Measure the overhead and the effect of just being pessimistic.
2595 : : Maybe this is only -O3 material? */
2596 : :
2597 : 833517 : hash_map<gimple *, bool> analyzed_stmts;
2598 : 833517 : bitmap always_executed_bbs = NULL;
2599 : 833517 : if (check_pass_throughs)
2600 : 1229496 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
2601 : : {
2602 : 1073969 : gcall *call_stmt = cs->call_stmt;
2603 : 1073969 : tree vuse = gimple_vuse (call_stmt);
2604 : :
2605 : : /* If the callee is a const function, we don't get a VUSE. In such
2606 : : case there will be no memory accesses in the called function (or the
2607 : : const attribute is wrong) and then we just don't care. */
2608 : 2118850 : bool uses_memory_as_obtained = vuse && SSA_NAME_IS_DEFAULT_DEF (vuse);
2609 : :
2610 : 1073969 : unsigned count = gimple_call_num_args (call_stmt);
2611 : 1073969 : isra_call_summary *csum = call_sums->get_create (cs);
2612 : 1073969 : csum->init_inputs (count);
2613 : 1073969 : csum->m_before_any_store = uses_memory_as_obtained;
2614 : 3462872 : for (unsigned argidx = 0; argidx < count; argidx++)
2615 : : {
2616 : 2388903 : if (!csum->m_arg_flow[argidx].pointer_pass_through)
2617 : 4325578 : continue;
2618 : 452228 : unsigned pidx
2619 : 452228 : = get_single_param_flow_source (&csum->m_arg_flow[argidx]);
2620 : 452228 : gensum_param_desc *desc = &(*param_descriptions)[pidx];
2621 : 452228 : if (!desc->split_candidate)
2622 : : {
2623 : 43306 : csum->m_arg_flow[argidx].pointer_pass_through = false;
2624 : 43306 : continue;
2625 : : }
2626 : 408922 : if (!uses_memory_as_obtained)
2627 : 158683 : continue;
2628 : :
2629 : 250239 : if (desc->safe_ref)
2630 : : {
2631 : 52889 : csum->m_arg_flow[argidx].safe_to_import_accesses = true;
2632 : 52889 : continue;
2633 : : }
2634 : :
2635 : : /* Walk basic block and see if its execution can terminate earlier.
2636 : : Keep the info for later re-use to avoid quadratic behavoiur here. */
2637 : 197350 : gimple_stmt_iterator gsi = gsi_for_stmt (call_stmt);
2638 : 197350 : bool safe = true;
2639 : 197350 : int n = 0;
2640 : 585450 : for (gsi_prev (&gsi); !gsi_end_p (gsi); gsi_prev (&gsi))
2641 : : {
2642 : 109369 : bool *b = analyzed_stmts.get (gsi_stmt (gsi));
2643 : 109369 : if (b)
2644 : : {
2645 : 9390 : safe = *b;
2646 : 9390 : gsi_next (&gsi);
2647 : 9390 : break;
2648 : : }
2649 : 99979 : n++;
2650 : 99979 : if (stmt_may_terminate_function_p (fun, gsi_stmt (gsi), false))
2651 : : {
2652 : : safe = false;
2653 : : break;
2654 : : }
2655 : : }
2656 : 197350 : if (n)
2657 : : {
2658 : 32080 : if (gsi_end_p (gsi))
2659 : 54952 : gsi = gsi_start_bb (gimple_bb (call_stmt));
2660 : 132059 : for (; gsi_stmt (gsi) != call_stmt; gsi_next (&gsi))
2661 : 99979 : analyzed_stmts.get_or_insert (gsi_stmt (gsi)) = safe;
2662 : : }
2663 : :
2664 : 197350 : if (safe && !always_executed_bbs)
2665 : : {
2666 : 54100 : mark_dfs_back_edges ();
2667 : 54100 : always_executed_bbs = find_always_executed_bbs (fun, false);
2668 : : }
2669 : 197350 : if (safe && bitmap_bit_p (always_executed_bbs, gimple_bb (call_stmt)->index))
2670 : 44470 : csum->m_arg_flow[argidx].safe_to_import_accesses = true;
2671 : : }
2672 : :
2673 : : }
2674 : 833517 : BITMAP_FREE (always_executed_bbs);
2675 : :
2676 : : /* TODO: Add early exit if we disqualified everything. This also requires
2677 : : that we either relax the restriction that
2678 : : ipa_param_adjustments.m_always_copy_start must be the number of PARM_DECLs
2679 : : or store the number of parameters to IPA-SRA function summary and use that
2680 : : when just removing params. */
2681 : :
2682 : 833517 : vec_safe_reserve_exact (ifs->m_parameters, param_count);
2683 : 833517 : ifs->m_parameters->quick_grow_cleared (param_count);
2684 : 3222497 : for (unsigned desc_index = 0; desc_index < param_count; desc_index++)
2685 : : {
2686 : 2388980 : gensum_param_desc *s = &(*param_descriptions)[desc_index];
2687 : 2388980 : isra_param_desc *d = &(*ifs->m_parameters)[desc_index];
2688 : :
2689 : 2388980 : d->param_size_limit = s->param_size_limit;
2690 : 2388980 : d->size_reached = s->nonarg_acc_size;
2691 : 2388980 : d->locally_unused = s->locally_unused;
2692 : 2388980 : d->split_candidate = s->split_candidate;
2693 : 2388980 : d->by_ref = s->by_ref;
2694 : 2388980 : d->remove_only_when_retval_removed = s->remove_only_when_retval_removed;
2695 : 2388980 : d->split_only_when_retval_removed = s->split_only_when_retval_removed;
2696 : 2388980 : d->conditionally_dereferenceable = s->conditionally_dereferenceable;
2697 : :
2698 : 2388980 : for (gensum_param_access *acc = s->accesses;
2699 : 2818294 : acc;
2700 : 429314 : acc = acc->next_sibling)
2701 : 429314 : copy_accesses_to_ipa_desc (acc, d);
2702 : : }
2703 : :
2704 : 833517 : if (dump_file)
2705 : 135 : dump_isra_param_descriptors (dump_file, node->decl, ifs, false);
2706 : 833517 : }
2707 : :
2708 : : /* Return true if there are any overlaps among certain accesses of DESC. If
2709 : : non-NULL, set *CERTAIN_ACCESS_PRESENT_P upon encountering a certain access
2710 : : too. DESC is assumed to be a split candidate that is not locally
2711 : : unused. */
2712 : :
2713 : : static bool
2714 : 179016 : overlapping_certain_accesses_p (isra_param_desc *desc,
2715 : : bool *certain_access_present_p)
2716 : : {
2717 : 179016 : unsigned pclen = vec_safe_length (desc->accesses);
2718 : 462510 : for (unsigned i = 0; i < pclen; i++)
2719 : : {
2720 : 285384 : param_access *a1 = (*desc->accesses)[i];
2721 : :
2722 : 285384 : if (!a1->certain)
2723 : 5621 : continue;
2724 : 279763 : if (certain_access_present_p)
2725 : 262871 : *certain_access_present_p = true;
2726 : 445249 : for (unsigned j = i + 1; j < pclen; j++)
2727 : : {
2728 : 167376 : param_access *a2 = (*desc->accesses)[j];
2729 : 167376 : if (a2->certain
2730 : 167302 : && a1->unit_offset < a2->unit_offset + a2->unit_size
2731 : 166962 : && a1->unit_offset + a1->unit_size > a2->unit_offset)
2732 : : return true;
2733 : : }
2734 : : }
2735 : : return false;
2736 : : }
2737 : :
2738 : : /* Check for any overlaps of certain param accesses among splitting candidates
2739 : : and signal an ICE if there are any. If CERTAIN_MUST_EXIST is set, also
2740 : : check that used splitting candidates have at least one certain access. */
2741 : :
2742 : : static void
2743 : 2591904 : verify_splitting_accesses (cgraph_node *node, bool certain_must_exist)
2744 : : {
2745 : 2591904 : isra_func_summary *ifs = func_sums->get (node);
2746 : 2591904 : if (!ifs || !ifs->m_candidate)
2747 : : return;
2748 : 1400272 : unsigned param_count = vec_safe_length (ifs->m_parameters);
2749 : 4574542 : for (unsigned pidx = 0; pidx < param_count; pidx++)
2750 : : {
2751 : 3174270 : isra_param_desc *desc = &(*ifs->m_parameters)[pidx];
2752 : 3174270 : if (!desc->split_candidate || desc->locally_unused)
2753 : 3012077 : continue;
2754 : :
2755 : 162193 : bool certain_access_present = !certain_must_exist;
2756 : 162193 : if (overlapping_certain_accesses_p (desc, &certain_access_present))
2757 : 0 : internal_error ("function %qs, parameter %u, has IPA-SRA accesses "
2758 : : "which overlap", node->dump_name (), pidx);
2759 : 162193 : if (!certain_access_present)
2760 : 0 : internal_error ("function %qs, parameter %u, is used but does not "
2761 : : "have any certain IPA-SRA access",
2762 : : node->dump_name (), pidx);
2763 : : }
2764 : : }
2765 : :
2766 : : /* Intraprocedural part of IPA-SRA analysis. Scan bodies of all functions in
2767 : : this compilation unit and create summary structures describing IPA-SRA
2768 : : opportunities and constraints in them. */
2769 : :
2770 : : static void
2771 : 122315 : ipa_sra_generate_summary (void)
2772 : : {
2773 : 122315 : struct cgraph_node *node;
2774 : :
2775 : 122315 : gcc_checking_assert (!func_sums);
2776 : 122315 : gcc_checking_assert (!call_sums);
2777 : 122315 : func_sums
2778 : 122315 : = (new (ggc_alloc_no_dtor <ipa_sra_function_summaries> ())
2779 : 122315 : ipa_sra_function_summaries (symtab, true));
2780 : 122315 : call_sums = new ipa_sra_call_summaries (symtab);
2781 : :
2782 : 1363599 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
2783 : 1241284 : ipa_sra_summarize_function (node);
2784 : 122315 : return;
2785 : : }
2786 : :
2787 : : /* Write intraprocedural analysis information about E and all of its outgoing
2788 : : edges into a stream for LTO WPA. */
2789 : :
2790 : : static void
2791 : 277427 : isra_write_edge_summary (output_block *ob, cgraph_edge *e)
2792 : : {
2793 : 277427 : isra_call_summary *csum = call_sums->get (e);
2794 : 277427 : unsigned input_count = csum->m_arg_flow.length ();
2795 : 277427 : streamer_write_uhwi (ob, input_count);
2796 : 748628 : for (unsigned i = 0; i < input_count; i++)
2797 : : {
2798 : 471201 : isra_param_flow *ipf = &csum->m_arg_flow[i];
2799 : 471201 : streamer_write_hwi (ob, ipf->length);
2800 : 471201 : bitpack_d bp = bitpack_create (ob->main_stream);
2801 : 545541 : for (int j = 0; j < ipf->length; j++)
2802 : 74340 : bp_pack_value (&bp, ipf->inputs[j], 8);
2803 : 471201 : bp_pack_value (&bp, ipf->aggregate_pass_through, 1);
2804 : 471201 : bp_pack_value (&bp, ipf->pointer_pass_through, 1);
2805 : 471201 : bp_pack_value (&bp, ipf->safe_to_import_accesses, 1);
2806 : 471201 : bp_pack_value (&bp, ipf->constructed_for_calls, 1);
2807 : 471201 : streamer_write_bitpack (&bp);
2808 : 471201 : streamer_write_uhwi (ob, ipf->unit_offset);
2809 : 471201 : streamer_write_uhwi (ob, ipf->unit_size);
2810 : : }
2811 : 277427 : bitpack_d bp = bitpack_create (ob->main_stream);
2812 : 277427 : bp_pack_value (&bp, csum->m_return_ignored, 1);
2813 : 277427 : bp_pack_value (&bp, csum->m_return_returned, 1);
2814 : 277427 : bp_pack_value (&bp, csum->m_bit_aligned_arg, 1);
2815 : 277427 : bp_pack_value (&bp, csum->m_before_any_store, 1);
2816 : 277427 : streamer_write_bitpack (&bp);
2817 : 277427 : }
2818 : :
2819 : : /* Write intraprocedural analysis information about NODE and all of its outgoing
2820 : : edges into a stream for LTO WPA. */
2821 : :
2822 : : static void
2823 : 46958 : isra_write_node_summary (output_block *ob, cgraph_node *node)
2824 : : {
2825 : 46958 : isra_func_summary *ifs = func_sums->get (node);
2826 : 46958 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
2827 : 46958 : int node_ref = lto_symtab_encoder_encode (encoder, node);
2828 : 46958 : streamer_write_uhwi (ob, node_ref);
2829 : :
2830 : 46958 : unsigned param_desc_count = vec_safe_length (ifs->m_parameters);
2831 : 46958 : streamer_write_uhwi (ob, param_desc_count);
2832 : 294478 : for (unsigned i = 0; i < param_desc_count; i++)
2833 : : {
2834 : 247520 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
2835 : 247520 : unsigned access_count = vec_safe_length (desc->accesses);
2836 : 247520 : streamer_write_uhwi (ob, access_count);
2837 : 250699 : for (unsigned j = 0; j < access_count; j++)
2838 : : {
2839 : 3179 : param_access *acc = (*desc->accesses)[j];
2840 : 3179 : stream_write_tree (ob, acc->type, true);
2841 : 3179 : stream_write_tree (ob, acc->alias_ptr_type, true);
2842 : 3179 : streamer_write_uhwi (ob, acc->unit_offset);
2843 : 3179 : streamer_write_uhwi (ob, acc->unit_size);
2844 : 3179 : bitpack_d bp = bitpack_create (ob->main_stream);
2845 : 3179 : bp_pack_value (&bp, acc->certain, 1);
2846 : 3179 : bp_pack_value (&bp, acc->reverse, 1);
2847 : 3179 : streamer_write_bitpack (&bp);
2848 : : }
2849 : 247520 : streamer_write_uhwi (ob, desc->param_size_limit);
2850 : 247520 : streamer_write_uhwi (ob, desc->size_reached);
2851 : 247520 : gcc_assert (desc->safe_size == 0);
2852 : 247520 : bitpack_d bp = bitpack_create (ob->main_stream);
2853 : 247520 : bp_pack_value (&bp, desc->locally_unused, 1);
2854 : 247520 : bp_pack_value (&bp, desc->split_candidate, 1);
2855 : 247520 : bp_pack_value (&bp, desc->by_ref, 1);
2856 : 247520 : gcc_assert (!desc->not_specially_constructed);
2857 : 247520 : bp_pack_value (&bp, desc->remove_only_when_retval_removed, 1);
2858 : 247520 : bp_pack_value (&bp, desc->split_only_when_retval_removed, 1);
2859 : 247520 : bp_pack_value (&bp, desc->conditionally_dereferenceable, 1);
2860 : 247520 : gcc_assert (!desc->safe_size_set);
2861 : 247520 : streamer_write_bitpack (&bp);
2862 : : }
2863 : 46958 : bitpack_d bp = bitpack_create (ob->main_stream);
2864 : 46958 : bp_pack_value (&bp, ifs->m_candidate, 1);
2865 : 46958 : bp_pack_value (&bp, ifs->m_returns_value, 1);
2866 : 46958 : bp_pack_value (&bp, ifs->m_return_ignored, 1);
2867 : 46958 : gcc_assert (!ifs->m_queued);
2868 : 46958 : streamer_write_bitpack (&bp);
2869 : :
2870 : 322736 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
2871 : 275778 : isra_write_edge_summary (ob, e);
2872 : 48607 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
2873 : 1649 : isra_write_edge_summary (ob, e);
2874 : 46958 : }
2875 : :
2876 : : /* Write intraprocedural analysis information into a stream for LTO WPA. */
2877 : :
2878 : : static void
2879 : 19036 : ipa_sra_write_summary (void)
2880 : : {
2881 : 19036 : if (!func_sums || !call_sums)
2882 : 0 : return;
2883 : :
2884 : 19036 : struct output_block *ob = create_output_block (LTO_section_ipa_sra);
2885 : 19036 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
2886 : 19036 : ob->symbol = NULL;
2887 : :
2888 : 19036 : unsigned int count = 0;
2889 : 19036 : lto_symtab_encoder_iterator lsei;
2890 : 19036 : for (lsei = lsei_start_function_in_partition (encoder);
2891 : 109814 : !lsei_end_p (lsei);
2892 : 90778 : lsei_next_function_in_partition (&lsei))
2893 : : {
2894 : 90778 : cgraph_node *node = lsei_cgraph_node (lsei);
2895 : 90778 : if (node->has_gimple_body_p ()
2896 : 90778 : && func_sums->get (node) != NULL)
2897 : 46958 : count++;
2898 : : }
2899 : 19036 : streamer_write_uhwi (ob, count);
2900 : :
2901 : : /* Process all of the functions. */
2902 : 109814 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
2903 : 90778 : lsei_next_function_in_partition (&lsei))
2904 : : {
2905 : 90778 : cgraph_node *node = lsei_cgraph_node (lsei);
2906 : 90778 : if (node->has_gimple_body_p ()
2907 : 90778 : && func_sums->get (node) != NULL)
2908 : 46958 : isra_write_node_summary (ob, node);
2909 : : }
2910 : 19036 : streamer_write_char_stream (ob->main_stream, 0);
2911 : 19036 : produce_asm (ob);
2912 : 19036 : destroy_output_block (ob);
2913 : : }
2914 : :
2915 : : /* Read intraprocedural analysis information about E and all of its outgoing
2916 : : edges into a stream for LTO WPA. */
2917 : :
2918 : : static void
2919 : 250573 : isra_read_edge_summary (struct lto_input_block *ib, cgraph_edge *cs)
2920 : : {
2921 : 250573 : isra_call_summary *csum = call_sums->get_create (cs);
2922 : 250573 : unsigned input_count = streamer_read_uhwi (ib);
2923 : 250573 : csum->init_inputs (input_count);
2924 : 696829 : for (unsigned i = 0; i < input_count; i++)
2925 : : {
2926 : 446256 : isra_param_flow *ipf = &csum->m_arg_flow[i];
2927 : 446256 : ipf->length = streamer_read_hwi (ib);
2928 : 446256 : bitpack_d bp = streamer_read_bitpack (ib);
2929 : 513373 : for (int j = 0; j < ipf->length; j++)
2930 : 67117 : ipf->inputs[j] = bp_unpack_value (&bp, 8);
2931 : 446256 : ipf->aggregate_pass_through = bp_unpack_value (&bp, 1);
2932 : 446256 : ipf->pointer_pass_through = bp_unpack_value (&bp, 1);
2933 : 446256 : ipf->safe_to_import_accesses = bp_unpack_value (&bp, 1);
2934 : 446256 : ipf->constructed_for_calls = bp_unpack_value (&bp, 1);
2935 : 446256 : ipf->unit_offset = streamer_read_uhwi (ib);
2936 : 446256 : ipf->unit_size = streamer_read_uhwi (ib);
2937 : : }
2938 : 250573 : bitpack_d bp = streamer_read_bitpack (ib);
2939 : 250573 : csum->m_return_ignored = bp_unpack_value (&bp, 1);
2940 : 250573 : csum->m_return_returned = bp_unpack_value (&bp, 1);
2941 : 250573 : csum->m_bit_aligned_arg = bp_unpack_value (&bp, 1);
2942 : 250573 : csum->m_before_any_store = bp_unpack_value (&bp, 1);
2943 : 250573 : }
2944 : :
2945 : : /* Read intraprocedural analysis information about NODE and all of its outgoing
2946 : : edges into a stream for LTO WPA. */
2947 : :
2948 : : static void
2949 : 33011 : isra_read_node_info (struct lto_input_block *ib, cgraph_node *node,
2950 : : struct data_in *data_in)
2951 : : {
2952 : 33011 : isra_func_summary *ifs = func_sums->get_create (node);
2953 : 33011 : unsigned param_desc_count = streamer_read_uhwi (ib);
2954 : 33011 : if (param_desc_count > 0)
2955 : : {
2956 : 16636 : vec_safe_reserve_exact (ifs->m_parameters, param_desc_count);
2957 : 16636 : ifs->m_parameters->quick_grow_cleared (param_desc_count);
2958 : : }
2959 : 59255 : for (unsigned i = 0; i < param_desc_count; i++)
2960 : : {
2961 : 26244 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
2962 : 26244 : unsigned access_count = streamer_read_uhwi (ib);
2963 : 27922 : for (unsigned j = 0; j < access_count; j++)
2964 : : {
2965 : 1678 : param_access *acc = ggc_cleared_alloc<param_access> ();
2966 : 1678 : acc->type = stream_read_tree (ib, data_in);
2967 : 1678 : acc->alias_ptr_type = stream_read_tree (ib, data_in);
2968 : 1678 : acc->unit_offset = streamer_read_uhwi (ib);
2969 : 1678 : acc->unit_size = streamer_read_uhwi (ib);
2970 : 1678 : bitpack_d bp = streamer_read_bitpack (ib);
2971 : 1678 : acc->certain = bp_unpack_value (&bp, 1);
2972 : 1678 : acc->reverse = bp_unpack_value (&bp, 1);
2973 : 1678 : vec_safe_push (desc->accesses, acc);
2974 : : }
2975 : 26244 : desc->param_size_limit = streamer_read_uhwi (ib);
2976 : 26244 : desc->size_reached = streamer_read_uhwi (ib);
2977 : 26244 : desc->safe_size = 0;
2978 : 26244 : bitpack_d bp = streamer_read_bitpack (ib);
2979 : 26244 : desc->locally_unused = bp_unpack_value (&bp, 1);
2980 : 26244 : desc->split_candidate = bp_unpack_value (&bp, 1);
2981 : 26244 : desc->by_ref = bp_unpack_value (&bp, 1);
2982 : 26244 : desc->not_specially_constructed = 0;
2983 : 26244 : desc->remove_only_when_retval_removed = bp_unpack_value (&bp, 1);
2984 : 26244 : desc->split_only_when_retval_removed = bp_unpack_value (&bp, 1);
2985 : 26244 : desc->conditionally_dereferenceable = bp_unpack_value (&bp, 1);
2986 : 26244 : desc->safe_size_set = 0;
2987 : : }
2988 : 33011 : bitpack_d bp = streamer_read_bitpack (ib);
2989 : 33011 : ifs->m_candidate = bp_unpack_value (&bp, 1);
2990 : 33011 : ifs->m_returns_value = bp_unpack_value (&bp, 1);
2991 : 33011 : ifs->m_return_ignored = bp_unpack_value (&bp, 1);
2992 : 33011 : ifs->m_queued = 0;
2993 : :
2994 : 282282 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
2995 : 249271 : isra_read_edge_summary (ib, e);
2996 : 34313 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
2997 : 1302 : isra_read_edge_summary (ib, e);
2998 : 33011 : }
2999 : :
3000 : : /* Read IPA-SRA summaries from a section in file FILE_DATA of length LEN with
3001 : : data DATA. TODO: This function was copied almost verbatim from ipa-prop.cc,
3002 : : it should be possible to unify them somehow. */
3003 : :
3004 : : static void
3005 : 10382 : isra_read_summary_section (struct lto_file_decl_data *file_data,
3006 : : const char *data, size_t len)
3007 : : {
3008 : 10382 : const struct lto_function_header *header =
3009 : : (const struct lto_function_header *) data;
3010 : 10382 : const int cfg_offset = sizeof (struct lto_function_header);
3011 : 10382 : const int main_offset = cfg_offset + header->cfg_size;
3012 : 10382 : const int string_offset = main_offset + header->main_size;
3013 : 10382 : struct data_in *data_in;
3014 : 10382 : unsigned int i;
3015 : 10382 : unsigned int count;
3016 : :
3017 : 10382 : lto_input_block ib_main ((const char *) data + main_offset,
3018 : 10382 : header->main_size, file_data);
3019 : :
3020 : 10382 : data_in =
3021 : 20764 : lto_data_in_create (file_data, (const char *) data + string_offset,
3022 : 10382 : header->string_size, vNULL);
3023 : 10382 : count = streamer_read_uhwi (&ib_main);
3024 : :
3025 : 43393 : for (i = 0; i < count; i++)
3026 : : {
3027 : 33011 : unsigned int index;
3028 : 33011 : struct cgraph_node *node;
3029 : 33011 : lto_symtab_encoder_t encoder;
3030 : :
3031 : 33011 : index = streamer_read_uhwi (&ib_main);
3032 : 33011 : encoder = file_data->symtab_node_encoder;
3033 : 33011 : node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
3034 : : index));
3035 : 33011 : gcc_assert (node->definition);
3036 : 33011 : isra_read_node_info (&ib_main, node, data_in);
3037 : : }
3038 : 10382 : lto_free_section_data (file_data, LTO_section_ipa_sra, NULL, data,
3039 : : len);
3040 : 10382 : lto_data_in_delete (data_in);
3041 : 10382 : }
3042 : :
3043 : : /* Read intraprocedural analysis information into a stream for LTO WPA. */
3044 : :
3045 : : static void
3046 : 9725 : ipa_sra_read_summary (void)
3047 : : {
3048 : 9725 : struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
3049 : 9725 : struct lto_file_decl_data *file_data;
3050 : 9725 : unsigned int j = 0;
3051 : :
3052 : 9725 : gcc_checking_assert (!func_sums);
3053 : 9725 : gcc_checking_assert (!call_sums);
3054 : 9725 : func_sums
3055 : 9725 : = (new (ggc_alloc_no_dtor <ipa_sra_function_summaries> ())
3056 : 9725 : ipa_sra_function_summaries (symtab, true));
3057 : 9725 : call_sums = new ipa_sra_call_summaries (symtab);
3058 : :
3059 : 20118 : while ((file_data = file_data_vec[j++]))
3060 : : {
3061 : 10393 : size_t len;
3062 : 10393 : const char *data
3063 : 10393 : = lto_get_summary_section_data (file_data, LTO_section_ipa_sra, &len);
3064 : 10393 : if (data)
3065 : 10382 : isra_read_summary_section (file_data, data, len);
3066 : : }
3067 : 9725 : }
3068 : :
3069 : : /* Dump all IPA-SRA summary data for all cgraph nodes and edges to file F. If
3070 : : HINTS is true, also dump IPA-analysis computed hints. */
3071 : :
3072 : : static void
3073 : 63 : ipa_sra_dump_all_summaries (FILE *f, bool hints)
3074 : : {
3075 : 63 : cgraph_node *node;
3076 : 258 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
3077 : : {
3078 : 195 : fprintf (f, "\nSummary for node %s:\n", node->dump_name ());
3079 : :
3080 : 195 : isra_func_summary *ifs = func_sums->get (node);
3081 : 195 : if (!ifs)
3082 : 10 : fprintf (f, " Function does not have any associated IPA-SRA "
3083 : : "summary\n");
3084 : 185 : else if (!ifs->m_candidate)
3085 : 12 : fprintf (f, " Not a candidate function\n");
3086 : : else
3087 : : {
3088 : 173 : if (ifs->m_returns_value)
3089 : 95 : fprintf (f, " Returns value\n");
3090 : 173 : if (vec_safe_is_empty (ifs->m_parameters))
3091 : 41 : fprintf (f, " No parameter information. \n");
3092 : : else
3093 : 446 : for (unsigned i = 0; i < ifs->m_parameters->length (); ++i)
3094 : : {
3095 : 314 : fprintf (f, " Descriptor for parameter %i:\n", i);
3096 : 314 : dump_isra_param_descriptor (f, &(*ifs->m_parameters)[i], hints);
3097 : : }
3098 : 173 : fprintf (f, "\n");
3099 : : }
3100 : :
3101 : 195 : struct cgraph_edge *cs;
3102 : 375 : for (cs = node->callees; cs; cs = cs->next_callee)
3103 : : {
3104 : 180 : fprintf (f, " Summary for edge %s->%s:\n", cs->caller->dump_name (),
3105 : 180 : cs->callee->dump_name ());
3106 : 180 : isra_call_summary *csum = call_sums->get (cs);
3107 : 180 : if (csum)
3108 : 180 : csum->dump (f);
3109 : : else
3110 : 0 : fprintf (f, " Call summary is MISSING!\n");
3111 : : }
3112 : :
3113 : : }
3114 : 63 : fprintf (f, "\n\n");
3115 : 63 : }
3116 : :
3117 : : /* Perform function-scope viability tests that can be only made at IPA level
3118 : : and return false if the function is deemed unsuitable for IPA-SRA. */
3119 : :
3120 : : static bool
3121 : 1019446 : ipa_sra_ipa_function_checks (cgraph_node *node)
3122 : : {
3123 : 1019446 : if (!node->can_be_local_p ())
3124 : : {
3125 : 643920 : if (dump_file)
3126 : 87 : fprintf (dump_file, "Function %s disqualified because it cannot be "
3127 : : "made local.\n", node->dump_name ());
3128 : 643920 : return false;
3129 : : }
3130 : 375526 : if (!node->can_change_signature)
3131 : : {
3132 : 0 : if (dump_file)
3133 : 0 : fprintf (dump_file, "Function can not change signature.\n");
3134 : 0 : return false;
3135 : : }
3136 : :
3137 : : return true;
3138 : : }
3139 : :
3140 : : /* Issues found out by check_callers_for_issues. */
3141 : :
3142 : : struct caller_issues
3143 : : {
3144 : : /* The candidate being considered. */
3145 : : cgraph_node *candidate;
3146 : : /* There is a thunk among callers. */
3147 : : bool thunk;
3148 : : /* Set if there is at least one caller that is OK. */
3149 : : bool there_is_one;
3150 : : /* Call site with no available information. */
3151 : : bool unknown_callsite;
3152 : : /* Call from outside the candidate's comdat group. */
3153 : : bool call_from_outside_comdat;
3154 : : /* There is a bit-aligned load into one of non-gimple-typed arguments. */
3155 : : bool bit_aligned_aggregate_argument;
3156 : : };
3157 : :
3158 : : /* Worker for call_for_symbol_and_aliases, set any flags of passed caller_issues
3159 : : that apply. */
3160 : :
3161 : : static bool
3162 : 380751 : check_for_caller_issues (struct cgraph_node *node, void *data)
3163 : : {
3164 : 380751 : struct caller_issues *issues = (struct caller_issues *) data;
3165 : :
3166 : 1472103 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
3167 : : {
3168 : 1091554 : if (cs->caller->thunk)
3169 : : {
3170 : 0 : issues->thunk = true;
3171 : : /* TODO: We should be able to process at least some types of
3172 : : thunks. */
3173 : 0 : return true;
3174 : : }
3175 : 1091554 : if (issues->candidate->calls_comdat_local
3176 : 12 : && issues->candidate->same_comdat_group
3177 : 1091566 : && !issues->candidate->in_same_comdat_group_p (cs->caller))
3178 : : {
3179 : 0 : issues->call_from_outside_comdat = true;
3180 : 0 : return true;
3181 : : }
3182 : :
3183 : 1091554 : isra_call_summary *csum = call_sums->get (cs);
3184 : 1091554 : if (!csum)
3185 : : {
3186 : 202 : issues->unknown_callsite = true;
3187 : 202 : return true;
3188 : : }
3189 : :
3190 : 1091352 : if (csum->m_bit_aligned_arg)
3191 : 0 : issues->bit_aligned_aggregate_argument = true;
3192 : :
3193 : 1091352 : issues->there_is_one = true;
3194 : : }
3195 : : return false;
3196 : : }
3197 : :
3198 : : /* Look at all incoming edges to NODE, including aliases and thunks and look
3199 : : for problems. Return true if NODE type should not be modified at all. */
3200 : :
3201 : : static bool
3202 : 375526 : check_all_callers_for_issues (cgraph_node *node)
3203 : : {
3204 : 375526 : struct caller_issues issues;
3205 : 375526 : memset (&issues, 0, sizeof (issues));
3206 : 375526 : issues.candidate = node;
3207 : :
3208 : 375526 : node->call_for_symbol_and_aliases (check_for_caller_issues, &issues, true);
3209 : 375526 : if (issues.unknown_callsite)
3210 : : {
3211 : 202 : if (dump_file && (dump_flags & TDF_DETAILS))
3212 : 0 : fprintf (dump_file, "A call of %s has not been analyzed. Disabling "
3213 : : "all modifications.\n", node->dump_name ());
3214 : 202 : return true;
3215 : : }
3216 : : /* TODO: We should be able to process at least some types of thunks. */
3217 : 375324 : if (issues.thunk)
3218 : : {
3219 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3220 : 0 : fprintf (dump_file, "A call of %s is through thunk, which are not"
3221 : : " handled yet. Disabling all modifications.\n",
3222 : : node->dump_name ());
3223 : 0 : return true;
3224 : : }
3225 : 375324 : if (issues.call_from_outside_comdat)
3226 : : {
3227 : 0 : if (dump_file)
3228 : 0 : fprintf (dump_file, "Function would become private comdat called "
3229 : : "outside of its comdat group.\n");
3230 : 0 : return true;
3231 : : }
3232 : :
3233 : 375324 : if (issues.bit_aligned_aggregate_argument)
3234 : : {
3235 : : /* Let's only remove parameters/return values from such functions.
3236 : : TODO: We could only prevent splitting the problematic parameters if
3237 : : anybody thinks it is worth it. */
3238 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3239 : 0 : fprintf (dump_file, "A call of %s has bit-aligned aggregate argument,"
3240 : : " disabling parameter splitting.\n", node->dump_name ());
3241 : :
3242 : 0 : isra_func_summary *ifs = func_sums->get (node);
3243 : 0 : gcc_checking_assert (ifs);
3244 : 0 : unsigned param_count = vec_safe_length (ifs->m_parameters);
3245 : 0 : for (unsigned i = 0; i < param_count; i++)
3246 : 0 : (*ifs->m_parameters)[i].split_candidate = false;
3247 : : }
3248 : 375324 : if (!issues.there_is_one)
3249 : : {
3250 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3251 : 0 : fprintf (dump_file, "There is no call to %s that we can modify. "
3252 : : "Disabling all modifications.\n", node->dump_name ());
3253 : 0 : return true;
3254 : : }
3255 : : return false;
3256 : : }
3257 : :
3258 : : /* Find the access with corresponding OFFSET and SIZE among accesses in
3259 : : PARAM_DESC and return it or NULL if such an access is not there. */
3260 : :
3261 : : static param_access *
3262 : 17966 : find_param_access (isra_param_desc *param_desc, unsigned offset, unsigned size)
3263 : : {
3264 : 17966 : unsigned pclen = vec_safe_length (param_desc->accesses);
3265 : :
3266 : : /* The search is linear but the number of stored accesses is bound by
3267 : : PARAM_IPA_SRA_MAX_REPLACEMENTS, so most probably 8. */
3268 : :
3269 : 18042 : for (unsigned i = 0; i < pclen; i++)
3270 : 17997 : if ((*param_desc->accesses)[i]->unit_offset == offset
3271 : 17997 : && (*param_desc->accesses)[i]->unit_size == size)
3272 : : return (*param_desc->accesses)[i];
3273 : :
3274 : : return NULL;
3275 : : }
3276 : :
3277 : : /* Return iff the total size of definite replacement SIZE would violate the
3278 : : limit set for it in PARAM. */
3279 : :
3280 : : static bool
3281 : 234157 : size_would_violate_limit_p (isra_param_desc *desc, unsigned size)
3282 : : {
3283 : 234157 : unsigned limit = desc->param_size_limit;
3284 : 0 : if (size > limit
3285 : 230311 : || (!desc->by_ref && size == limit))
3286 : 0 : return true;
3287 : : return false;
3288 : : }
3289 : :
3290 : : /* Increase reached size of DESC by SIZE or disqualify it if it would violate
3291 : : the set limit. IDX is the parameter number which is dumped when
3292 : : disqualifying. */
3293 : :
3294 : : static void
3295 : 14933 : bump_reached_size (isra_param_desc *desc, unsigned size, unsigned idx)
3296 : : {
3297 : 14933 : unsigned after = desc->size_reached + size;
3298 : 14933 : if (size_would_violate_limit_p (desc, after))
3299 : : {
3300 : 14901 : if (dump_file && (dump_flags & TDF_DETAILS))
3301 : 0 : fprintf (dump_file, " ...size limit reached, disqualifying "
3302 : : "candidate parameter %u\n", idx);
3303 : 14901 : desc->split_candidate = false;
3304 : 14901 : return;
3305 : : }
3306 : 32 : desc->size_reached = after;
3307 : : }
3308 : :
3309 : : /* Take all actions required to deal with an edge CS that represents a call to
3310 : : an unknown or un-analyzed function, for both parameter removal and
3311 : : splitting. */
3312 : :
3313 : : static void
3314 : 507330 : process_edge_to_unknown_caller (cgraph_edge *cs)
3315 : : {
3316 : 507330 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3317 : 507330 : gcc_checking_assert (from_ifs);
3318 : 507330 : isra_call_summary *csum = call_sums->get (cs);
3319 : :
3320 : 507330 : if (dump_file && (dump_flags & TDF_DETAILS))
3321 : 4 : fprintf (dump_file, "Processing an edge to an unknown caller from %s:\n",
3322 : : cs->caller->dump_name ());
3323 : :
3324 : 507330 : unsigned args_count = csum->m_arg_flow.length ();
3325 : 1250137 : for (unsigned i = 0; i < args_count; i++)
3326 : : {
3327 : 742807 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3328 : :
3329 : 742807 : if (ipf->pointer_pass_through)
3330 : : {
3331 : 193312 : isra_param_desc *param_desc
3332 : 193312 : = &(*from_ifs->m_parameters)[get_single_param_flow_source (ipf)];
3333 : 193312 : param_desc->locally_unused = false;
3334 : 193312 : param_desc->split_candidate = false;
3335 : 193312 : continue;
3336 : 193312 : }
3337 : 549495 : if (ipf->aggregate_pass_through)
3338 : : {
3339 : 5474 : unsigned idx = get_single_param_flow_source (ipf);
3340 : 5474 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3341 : :
3342 : 5474 : param_desc->locally_unused = false;
3343 : 5474 : if (!param_desc->split_candidate)
3344 : 1734 : continue;
3345 : 3740 : gcc_assert (!param_desc->by_ref);
3346 : 7480 : param_access *pacc = find_param_access (param_desc, ipf->unit_offset,
3347 : 3740 : ipf->unit_size);
3348 : 3740 : gcc_checking_assert (pacc);
3349 : 3740 : pacc->certain = true;
3350 : 3740 : if (overlapping_certain_accesses_p (param_desc, NULL))
3351 : : {
3352 : 657 : if (dump_file && (dump_flags & TDF_DETAILS))
3353 : 0 : fprintf (dump_file, " ...leading to overlap, "
3354 : : " disqualifying candidate parameter %u\n",
3355 : : idx);
3356 : 657 : param_desc->split_candidate = false;
3357 : : }
3358 : : else
3359 : 3083 : bump_reached_size (param_desc, pacc->unit_size, idx);
3360 : 3740 : ipf->aggregate_pass_through = false;
3361 : 3740 : continue;
3362 : 3740 : }
3363 : :
3364 : 642406 : for (int j = 0; j < ipf->length; j++)
3365 : : {
3366 : 98385 : int input_idx = ipf->inputs[j];
3367 : 98385 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3368 : : }
3369 : : }
3370 : 507330 : }
3371 : :
3372 : : /* Propagate parameter removal information through cross-SCC edge CS,
3373 : : i.e. decrease the use count in the caller parameter descriptor for each use
3374 : : in this call. */
3375 : :
3376 : : static void
3377 : 875415 : param_removal_cross_scc_edge (cgraph_edge *cs)
3378 : : {
3379 : 875415 : enum availability availability;
3380 : 875415 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3381 : 875415 : isra_func_summary *to_ifs = func_sums->get (callee);
3382 : 436443 : if (!to_ifs || !to_ifs->m_candidate
3383 : 394916 : || (availability < AVAIL_AVAILABLE)
3384 : 1270331 : || vec_safe_is_empty (to_ifs->m_parameters))
3385 : : {
3386 : 491083 : process_edge_to_unknown_caller (cs);
3387 : 491083 : return;
3388 : : }
3389 : 384332 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3390 : 384332 : gcc_checking_assert (from_ifs);
3391 : :
3392 : 384332 : isra_call_summary *csum = call_sums->get (cs);
3393 : 384332 : unsigned args_count = csum->m_arg_flow.length ();
3394 : 384332 : unsigned param_count = vec_safe_length (to_ifs->m_parameters);
3395 : :
3396 : 1326571 : for (unsigned i = 0; i < args_count; i++)
3397 : : {
3398 : 942239 : bool unused_in_callee;
3399 : 942239 : if (i < param_count)
3400 : 942235 : unused_in_callee = (*to_ifs->m_parameters)[i].locally_unused;
3401 : : else
3402 : : unused_in_callee = false;
3403 : :
3404 : 942235 : if (!unused_in_callee)
3405 : : {
3406 : 898907 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3407 : 1187555 : for (int j = 0; j < ipf->length; j++)
3408 : : {
3409 : 288648 : int input_idx = ipf->inputs[j];
3410 : 288648 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3411 : : }
3412 : : }
3413 : : }
3414 : : }
3415 : :
3416 : : /* Unless it is already there, push NODE which is also described by IFS to
3417 : : STACK. */
3418 : :
3419 : : static void
3420 : 166053 : isra_push_node_to_stack (cgraph_node *node, isra_func_summary *ifs,
3421 : : vec<cgraph_node *> *stack)
3422 : : {
3423 : 0 : if (!ifs->m_queued)
3424 : : {
3425 : 165274 : ifs->m_queued = true;
3426 : 165274 : stack->safe_push (node);
3427 : : }
3428 : 0 : }
3429 : :
3430 : : /* If parameter with index INPUT_IDX is marked as locally unused, mark it as
3431 : : used and push CALLER on STACK. */
3432 : :
3433 : : static void
3434 : 25346 : isra_mark_caller_param_used (isra_func_summary *from_ifs, int input_idx,
3435 : : cgraph_node *caller, vec<cgraph_node *> *stack)
3436 : : {
3437 : 25346 : if ((*from_ifs->m_parameters)[input_idx].locally_unused)
3438 : : {
3439 : 2980 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3440 : 5228 : isra_push_node_to_stack (caller, from_ifs, stack);
3441 : : }
3442 : 25346 : }
3443 : :
3444 : : /* Combine safe_size of DESC with SIZE and return true if it has changed. */
3445 : :
3446 : : static bool
3447 : 2395105 : update_safe_size (isra_param_desc *desc, unsigned size)
3448 : : {
3449 : 0 : if (!desc->safe_size_set)
3450 : : {
3451 : 775521 : desc->safe_size_set = 1;
3452 : 775521 : desc->safe_size = size;
3453 : 775521 : return true;
3454 : : }
3455 : 1619584 : if (desc->safe_size <= size)
3456 : : return false;
3457 : 21213 : desc->safe_size = size;
3458 : 21213 : return true;
3459 : : }
3460 : :
3461 : : /* Set all param hints in DESC to the pessimistic values. Return true if any
3462 : : hints that need to potentially trigger further propagation have changed. */
3463 : :
3464 : : static bool
3465 : 702805 : flip_all_hints_pessimistic (isra_param_desc *desc)
3466 : : {
3467 : 702805 : desc->not_specially_constructed = true;
3468 : 0 : return update_safe_size (desc, 0);
3469 : : }
3470 : :
3471 : : /* Because we have not analyzed or otherwise problematic caller, go over all
3472 : : parameter int flags of IFS describing a call graph node of a calllee and
3473 : : turn them pessimistic. Return true if any hints that need to potentially
3474 : : trigger further propagation have changed. */
3475 : :
3476 : : static bool
3477 : 420151 : flip_all_param_hints_pessimistic (isra_func_summary *ifs)
3478 : : {
3479 : 420151 : if (!ifs || !ifs->m_candidate)
3480 : : return false;
3481 : :
3482 : 27226 : bool ret = false;
3483 : 27226 : unsigned param_count = vec_safe_length (ifs->m_parameters);
3484 : :
3485 : 76795 : for (unsigned i = 0; i < param_count; i++)
3486 : 99138 : ret |= flip_all_hints_pessimistic (&(*ifs->m_parameters)[i]);
3487 : :
3488 : : return ret;
3489 : : }
3490 : :
3491 : : /* Propagate hints accross edge CS which ultimately leads to a node described
3492 : : by TO_IFS. Return true if any hints of the callee which should potentially
3493 : : trigger further propagation have changed. */
3494 : :
3495 : : static bool
3496 : 4821231 : propagate_param_hints_accross_call (cgraph_edge *cs, isra_func_summary *to_ifs)
3497 : : {
3498 : 4821231 : if (!to_ifs || !to_ifs->m_candidate)
3499 : : return false;
3500 : :
3501 : 1630107 : isra_call_summary *csum = call_sums->get (cs);
3502 : 1630107 : bool ret = false;
3503 : 1630107 : unsigned args_count = csum->m_arg_flow.length ();
3504 : 1630107 : unsigned param_count = vec_safe_length (to_ifs->m_parameters);
3505 : :
3506 : 5229722 : for (unsigned i = 0; i < param_count; i++)
3507 : : {
3508 : 3599615 : isra_param_desc *desc = &(*to_ifs->m_parameters)[i];
3509 : 3599615 : if (i >= args_count)
3510 : : {
3511 : 653236 : ret |= flip_all_hints_pessimistic (desc);
3512 : 653236 : continue;
3513 : : }
3514 : :
3515 : 2946379 : if (desc->by_ref)
3516 : : {
3517 : 1692318 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3518 : :
3519 : 1692318 : if (!ipf->constructed_for_calls)
3520 : 1282187 : desc->not_specially_constructed = true;
3521 : :
3522 : 1692318 : if (ipf->pointer_pass_through)
3523 : : {
3524 : 297828 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3525 : 297828 : int srcidx = get_single_param_flow_source (ipf);
3526 : 297828 : if (vec_safe_length (from_ifs->m_parameters) > (unsigned) srcidx)
3527 : : {
3528 : 193692 : isra_param_desc *src_d = &(*from_ifs->m_parameters)[srcidx];
3529 : 193692 : if (src_d->safe_size_set)
3530 : 387348 : ret |= update_safe_size (desc, src_d->safe_size);
3531 : : }
3532 : : else
3533 : 208272 : ret |= update_safe_size (desc, 0);
3534 : : }
3535 : 1394490 : else if (!ipf->aggregate_pass_through)
3536 : 2788978 : ret |= update_safe_size (desc, ipf->unit_size);
3537 : : else
3538 : : /* LTOing type-mismatched programs can end up here. */
3539 : 2 : ret |= update_safe_size (desc, 0);
3540 : : }
3541 : : }
3542 : : return ret;
3543 : : }
3544 : :
3545 : : /* Propagate hints from NODE described by FROM_IFS to all its (dorect) callees,
3546 : : push those that may need re-visiting onto STACK. */
3547 : :
3548 : : static void
3549 : 1336977 : propagate_hints_to_all_callees (cgraph_node *node, isra_func_summary *from_ifs,
3550 : : vec<cgraph_node *> *stack)
3551 : : {
3552 : 6578359 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
3553 : : {
3554 : 5241382 : enum availability availability;
3555 : 5241382 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3556 : 5241382 : isra_func_summary *to_ifs = func_sums->get (callee);
3557 : 5241382 : if (!from_ifs)
3558 : : {
3559 : 420151 : if (flip_all_param_hints_pessimistic (to_ifs)
3560 : 420151 : && ipa_edge_within_scc (cs))
3561 : 26 : isra_push_node_to_stack (callee, to_ifs, stack);
3562 : : }
3563 : 4821231 : else if (propagate_param_hints_accross_call (cs, to_ifs)
3564 : 4821231 : && ipa_edge_within_scc (cs))
3565 : 7755 : isra_push_node_to_stack (callee, to_ifs, stack);
3566 : : }
3567 : 1336977 : }
3568 : :
3569 : : /* Propagate information that any parameter is not used only locally within a
3570 : : SCC across CS to the caller, which must be in the same SCC as the
3571 : : callee. Push any callers that need to be re-processed to STACK. */
3572 : :
3573 : : static void
3574 : 32839 : propagate_used_across_scc_edge (cgraph_edge *cs, vec<cgraph_node *> *stack)
3575 : : {
3576 : 32839 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3577 : 32839 : if (!from_ifs || vec_safe_is_empty (from_ifs->m_parameters))
3578 : 14435 : return;
3579 : :
3580 : 18404 : isra_call_summary *csum = call_sums->get (cs);
3581 : 18404 : gcc_checking_assert (csum);
3582 : 18404 : unsigned args_count = csum->m_arg_flow.length ();
3583 : 18404 : enum availability availability;
3584 : 18404 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3585 : 18404 : isra_func_summary *to_ifs = func_sums->get (callee);
3586 : :
3587 : 18404 : unsigned param_count
3588 : 18316 : = (to_ifs && (availability >= AVAIL_AVAILABLE))
3589 : 36460 : ? vec_safe_length (to_ifs->m_parameters) : 0;
3590 : 60284 : for (unsigned i = 0; i < args_count; i++)
3591 : : {
3592 : 45503 : if (i < param_count
3593 : 41880 : && (*to_ifs->m_parameters)[i].locally_unused)
3594 : 3623 : continue;
3595 : :
3596 : : /* The argument is needed in the callee it, we must mark the parameter as
3597 : : used also in the caller and its callers within this SCC. */
3598 : 38257 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3599 : 63603 : for (int j = 0; j < ipf->length; j++)
3600 : : {
3601 : 25346 : int input_idx = ipf->inputs[j];
3602 : 25346 : isra_mark_caller_param_used (from_ifs, input_idx, cs->caller, stack);
3603 : : }
3604 : : }
3605 : : }
3606 : :
3607 : : /* Propagate information that any parameter is not used only locally within a
3608 : : SCC (i.e. is used also elsewhere) to all callers of NODE that are in the
3609 : : same SCC. Push any callers that need to be re-processed to STACK. */
3610 : :
3611 : : static bool
3612 : 1410012 : propagate_used_to_scc_callers (cgraph_node *node, void *data)
3613 : : {
3614 : 1410012 : vec<cgraph_node *> *stack = (vec<cgraph_node *> *) data;
3615 : 1410012 : cgraph_edge *cs;
3616 : 3447015 : for (cs = node->callers; cs; cs = cs->next_caller)
3617 : 2037003 : if (ipa_edge_within_scc (cs))
3618 : 32839 : propagate_used_across_scc_edge (cs, stack);
3619 : 1410012 : return false;
3620 : : }
3621 : :
3622 : : /* Return true iff all certain accesses in ARG_DESC are also present as
3623 : : certain accesses in PARAM_DESC. */
3624 : :
3625 : : static bool
3626 : 70 : all_callee_accesses_present_p (isra_param_desc *param_desc,
3627 : : isra_param_desc *arg_desc)
3628 : : {
3629 : 70 : unsigned aclen = vec_safe_length (arg_desc->accesses);
3630 : 74 : for (unsigned j = 0; j < aclen; j++)
3631 : : {
3632 : 50 : param_access *argacc = (*arg_desc->accesses)[j];
3633 : 50 : if (!argacc->certain)
3634 : 0 : continue;
3635 : 100 : param_access *pacc = find_param_access (param_desc, argacc->unit_offset,
3636 : 50 : argacc->unit_size);
3637 : 50 : if (!pacc
3638 : 5 : || !pacc->certain
3639 : 55 : || !types_compatible_p (argacc->type, pacc->type))
3640 : 46 : return false;
3641 : : }
3642 : : return true;
3643 : : }
3644 : :
3645 : : /* Type internal to function pull_accesses_from_callee. Unfortunately gcc 4.8
3646 : : does not allow instantiating an auto_vec with a type defined within a
3647 : : function so it is a global type. */
3648 : : enum acc_prop_kind {ACC_PROP_DONT, ACC_PROP_COPY, ACC_PROP_CERTAIN};
3649 : :
3650 : :
3651 : : /* Attempt to propagate all definite accesses from ARG_DESC to PARAM_DESC,
3652 : : (which belongs to CALLER) if they would not violate some constraint there.
3653 : : CALLER_IPCP_TS describes the caller, PARAM_IDX is the index of the parameter
3654 : : described by PARAM_DESC. If successful, return NULL, otherwise return the
3655 : : string reason for failure (which can be written to the dump file).
3656 : : DELTA_OFFSET is the known offset of the actual argument withing the formal
3657 : : parameter (so of ARG_DESCS within PARAM_DESCS), ARG_SIZE is the size of the
3658 : : actual argument or zero, if not known. In case of success, set *CHANGE_P to
3659 : : true if propagation actually changed anything. */
3660 : :
3661 : : static const char *
3662 : 7303 : pull_accesses_from_callee (cgraph_node *caller,
3663 : : ipcp_transformation *caller_ipcp_ts,
3664 : : int param_idx,
3665 : : isra_param_desc *param_desc,
3666 : : isra_param_desc *arg_desc,
3667 : : unsigned delta_offset, unsigned arg_size,
3668 : : bool *change_p)
3669 : : {
3670 : 7303 : unsigned pclen = vec_safe_length (param_desc->accesses);
3671 : 7303 : unsigned aclen = vec_safe_length (arg_desc->accesses);
3672 : 7303 : unsigned prop_count = 0;
3673 : 7303 : unsigned prop_size = 0;
3674 : 7303 : bool change = false;
3675 : :
3676 : 7303 : auto_vec <enum acc_prop_kind, 8> prop_kinds (aclen);
3677 : 16837 : for (unsigned j = 0; j < aclen; j++)
3678 : : {
3679 : 9695 : param_access *argacc = (*arg_desc->accesses)[j];
3680 : 9695 : prop_kinds.safe_push (ACC_PROP_DONT);
3681 : :
3682 : 9695 : if (arg_size > 0
3683 : 2589 : && argacc->unit_offset + argacc->unit_size > arg_size)
3684 : : return "callee access outsize size boundary";
3685 : :
3686 : 9695 : if (!argacc->certain)
3687 : 425 : continue;
3688 : :
3689 : 9270 : unsigned offset = argacc->unit_offset + delta_offset;
3690 : :
3691 : 9270 : if (caller_ipcp_ts && !AGGREGATE_TYPE_P (argacc->type))
3692 : : {
3693 : 337 : ipa_argagg_value_list avl (caller_ipcp_ts);
3694 : 337 : tree value = avl.get_value (param_idx, offset);
3695 : 337 : if (value && ((tree_to_uhwi (TYPE_SIZE (TREE_TYPE (value)))
3696 : 42 : / BITS_PER_UNIT)
3697 : 42 : != argacc->unit_size))
3698 : 1 : return " propagated access would conflict with an IPA-CP constant";
3699 : : }
3700 : :
3701 : : /* Given that accesses are initially stored according to increasing
3702 : : offset and decreasing size in case of equal offsets, the following
3703 : : searches could be written more efficiently if we kept the ordering
3704 : : when copying. But the number of accesses is capped at
3705 : : PARAM_IPA_SRA_MAX_REPLACEMENTS (so most likely 8) and the code gets
3706 : : messy quickly, so let's improve on that only if necessary. */
3707 : :
3708 : : bool exact_match = false;
3709 : 19433 : for (unsigned i = 0; i < pclen; i++)
3710 : : {
3711 : : /* Check for overlaps. */
3712 : 10324 : param_access *pacc = (*param_desc->accesses)[i];
3713 : 10324 : if (pacc->unit_offset == offset
3714 : 4655 : && pacc->unit_size == argacc->unit_size)
3715 : : {
3716 : 3602 : if (argacc->alias_ptr_type != pacc->alias_ptr_type
3717 : 3445 : || !types_compatible_p (argacc->type, pacc->type)
3718 : 7047 : || argacc->reverse != pacc->reverse)
3719 : 157 : return "propagated access types would not match existing ones";
3720 : :
3721 : 3445 : exact_match = true;
3722 : 3445 : if (!pacc->certain)
3723 : : {
3724 : 0 : prop_kinds[j] = ACC_PROP_CERTAIN;
3725 : 0 : prop_size += argacc->unit_size;
3726 : 0 : change = true;
3727 : : }
3728 : 3445 : continue;
3729 : : }
3730 : :
3731 : 6722 : if (offset < pacc->unit_offset + pacc->unit_size
3732 : 3819 : && offset + argacc->unit_size > pacc->unit_offset)
3733 : : {
3734 : : /* None permissible with load accesses, possible to fit into
3735 : : argument ones. */
3736 : 2167 : if (pacc->certain
3737 : 2164 : || offset < pacc->unit_offset
3738 : 2164 : || (offset + argacc->unit_size
3739 : : > pacc->unit_offset + pacc->unit_size))
3740 : : return "a propagated access would conflict in caller";
3741 : : }
3742 : : }
3743 : :
3744 : 9109 : if (!exact_match)
3745 : : {
3746 : 5664 : prop_kinds[j] = ACC_PROP_COPY;
3747 : 5664 : prop_count++;
3748 : 5664 : prop_size += argacc->unit_size;
3749 : 5664 : change = true;
3750 : : }
3751 : : }
3752 : :
3753 : 7142 : if (!change)
3754 : : return NULL;
3755 : :
3756 : 4579 : if ((prop_count + pclen
3757 : 4579 : > (unsigned) opt_for_fn (caller->decl, param_ipa_sra_max_replacements))
3758 : 11882 : || size_would_violate_limit_p (param_desc,
3759 : 4579 : param_desc->size_reached + prop_size))
3760 : : return "propagating accesses would violate the count or size limit";
3761 : :
3762 : 4552 : *change_p = true;
3763 : 10569 : for (unsigned j = 0; j < aclen; j++)
3764 : : {
3765 : 6017 : if (prop_kinds[j] == ACC_PROP_COPY)
3766 : : {
3767 : 5619 : param_access *argacc = (*arg_desc->accesses)[j];
3768 : :
3769 : 5619 : param_access *copy = ggc_cleared_alloc<param_access> ();
3770 : 5619 : copy->unit_offset = argacc->unit_offset + delta_offset;
3771 : 5619 : copy->unit_size = argacc->unit_size;
3772 : 5619 : copy->type = argacc->type;
3773 : 5619 : copy->alias_ptr_type = argacc->alias_ptr_type;
3774 : 5619 : copy->certain = true;
3775 : 5619 : copy->reverse = argacc->reverse;
3776 : 5619 : vec_safe_push (param_desc->accesses, copy);
3777 : : }
3778 : 398 : else if (prop_kinds[j] == ACC_PROP_CERTAIN)
3779 : : {
3780 : 0 : param_access *argacc = (*arg_desc->accesses)[j];
3781 : 0 : param_access *csp
3782 : 0 : = find_param_access (param_desc, argacc->unit_offset + delta_offset,
3783 : 0 : argacc->unit_size);
3784 : 0 : csp->certain = true;
3785 : : }
3786 : : }
3787 : :
3788 : 4552 : param_desc->size_reached += prop_size;
3789 : :
3790 : 4552 : return NULL;
3791 : 7303 : }
3792 : :
3793 : : /* Propagate parameter splitting information through call graph edge CS.
3794 : : Return true if any changes that might need to be propagated within SCCs have
3795 : : been made. The function also clears the aggregate_pass_through and
3796 : : pointer_pass_through in call summaries which do not need to be processed
3797 : : again if this CS is revisited when iterating while changes are propagated
3798 : : within an SCC. */
3799 : :
3800 : : static bool
3801 : 1665359 : param_splitting_across_edge (cgraph_edge *cs)
3802 : : {
3803 : 1665359 : bool res = false;
3804 : 1665359 : bool cross_scc = !ipa_edge_within_scc (cs);
3805 : 1665359 : enum availability availability;
3806 : 1665359 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3807 : 1665359 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3808 : 1665359 : gcc_checking_assert (from_ifs && from_ifs->m_parameters);
3809 : 1665359 : ipcp_transformation *caller_ipcp_ts
3810 : 1665359 : = ipcp_get_transformation_summary (cs->caller);
3811 : :
3812 : 1665359 : isra_call_summary *csum = call_sums->get (cs);
3813 : 1665359 : gcc_checking_assert (csum);
3814 : 1665359 : unsigned args_count = csum->m_arg_flow.length ();
3815 : 1665359 : isra_func_summary *to_ifs = func_sums->get (callee);
3816 : 1665359 : unsigned param_count
3817 : 923538 : = ((to_ifs && to_ifs->m_candidate && (availability >= AVAIL_AVAILABLE))
3818 : 2500700 : ? vec_safe_length (to_ifs->m_parameters)
3819 : : : 0);
3820 : :
3821 : 1665359 : if (dump_file && (dump_flags & TDF_DETAILS))
3822 : 18 : fprintf (dump_file, "Splitting across %s->%s:\n",
3823 : 9 : cs->caller->dump_name (), callee->dump_name ());
3824 : :
3825 : : unsigned i;
3826 : 3398811 : for (i = 0; (i < args_count) && (i < param_count); i++)
3827 : : {
3828 : 1733452 : isra_param_desc *arg_desc = &(*to_ifs->m_parameters)[i];
3829 : 1733452 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3830 : :
3831 : 1733452 : if (arg_desc->locally_unused)
3832 : : {
3833 : 81983 : if (dump_file && (dump_flags & TDF_DETAILS))
3834 : 0 : fprintf (dump_file, " ->%u: unused in callee\n", i);
3835 : 81983 : ipf->pointer_pass_through = false;
3836 : 81983 : continue;
3837 : : }
3838 : :
3839 : 1651469 : if (ipf->pointer_pass_through)
3840 : : {
3841 : 203772 : int idx = get_single_param_flow_source (ipf);
3842 : 203772 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3843 : 203772 : if (!param_desc->split_candidate)
3844 : 142650 : continue;
3845 : 61122 : gcc_assert (param_desc->by_ref);
3846 : :
3847 : 61122 : if (!arg_desc->split_candidate || !arg_desc->by_ref)
3848 : : {
3849 : 52894 : if (dump_file && (dump_flags & TDF_DETAILS))
3850 : 2 : fprintf (dump_file, " %u->%u: not candidate or not by "
3851 : : "reference in callee\n", idx, i);
3852 : 52894 : param_desc->split_candidate = false;
3853 : 52894 : ipf->pointer_pass_through = false;
3854 : 52894 : res = true;
3855 : : }
3856 : 8228 : else if (!ipf->safe_to_import_accesses)
3857 : : {
3858 : 2179 : if (!csum->m_before_any_store
3859 : 2179 : || !all_callee_accesses_present_p (param_desc, arg_desc))
3860 : : {
3861 : 2155 : if (dump_file && (dump_flags & TDF_DETAILS))
3862 : 0 : fprintf (dump_file, " %u->%u: cannot import accesses.\n",
3863 : : idx, i);
3864 : 2155 : param_desc->split_candidate = false;
3865 : 2155 : ipf->pointer_pass_through = false;
3866 : 2155 : res = true;
3867 : :
3868 : : }
3869 : : else
3870 : : {
3871 : 24 : if (dump_file && (dump_flags & TDF_DETAILS))
3872 : 0 : fprintf (dump_file, " %u->%u: verified callee accesses "
3873 : : "present.\n", idx, i);
3874 : 24 : if (cross_scc)
3875 : 0 : ipf->pointer_pass_through = false;
3876 : : }
3877 : : }
3878 : : else
3879 : : {
3880 : 6049 : const char *pull_failure
3881 : 6049 : = pull_accesses_from_callee (cs->caller, caller_ipcp_ts, idx,
3882 : : param_desc, arg_desc, 0, 0, &res);
3883 : 6049 : if (pull_failure)
3884 : : {
3885 : 27 : if (dump_file && (dump_flags & TDF_DETAILS))
3886 : 0 : fprintf (dump_file, " %u->%u: by_ref access pull "
3887 : : "failed: %s.\n", idx, i, pull_failure);
3888 : 27 : param_desc->split_candidate = false;
3889 : 27 : ipf->pointer_pass_through = false;
3890 : 27 : res = true;
3891 : : }
3892 : : else
3893 : : {
3894 : 6022 : if (dump_file && (dump_flags & TDF_DETAILS))
3895 : 1 : fprintf (dump_file, " %u->%u: by_ref access pull "
3896 : : "succeeded.\n", idx, i);
3897 : 6022 : if (cross_scc)
3898 : 5947 : ipf->pointer_pass_through = false;
3899 : : }
3900 : : }
3901 : : }
3902 : 1447697 : else if (ipf->aggregate_pass_through)
3903 : : {
3904 : 63524 : int idx = get_single_param_flow_source (ipf);
3905 : 63524 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3906 : 63524 : if (!param_desc->split_candidate)
3907 : 49348 : continue;
3908 : 14176 : gcc_assert (!param_desc->by_ref);
3909 : 28352 : param_access *pacc = find_param_access (param_desc, ipf->unit_offset,
3910 : 14176 : ipf->unit_size);
3911 : 14176 : gcc_checking_assert (pacc);
3912 : :
3913 : 14176 : if (pacc->certain)
3914 : : {
3915 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3916 : 0 : fprintf (dump_file, " %u->%u: already certain\n", idx, i);
3917 : 0 : ipf->aggregate_pass_through = false;
3918 : : }
3919 : 14176 : else if (!arg_desc->split_candidate || arg_desc->by_ref)
3920 : : {
3921 : 12922 : if (dump_file && (dump_flags & TDF_DETAILS))
3922 : 0 : fprintf (dump_file, " %u->%u: not candidate or by "
3923 : : "reference in callee\n", idx, i);
3924 : :
3925 : 12922 : pacc->certain = true;
3926 : 12922 : if (overlapping_certain_accesses_p (param_desc, NULL))
3927 : : {
3928 : 1072 : if (dump_file && (dump_flags & TDF_DETAILS))
3929 : 0 : fprintf (dump_file, " ...leading to overlap, "
3930 : : " disqualifying candidate parameter %u\n",
3931 : : idx);
3932 : 1072 : param_desc->split_candidate = false;
3933 : : }
3934 : : else
3935 : 11850 : bump_reached_size (param_desc, pacc->unit_size, idx);
3936 : :
3937 : 12922 : ipf->aggregate_pass_through = false;
3938 : 12922 : res = true;
3939 : : }
3940 : : else
3941 : : {
3942 : 1254 : const char *pull_failure
3943 : 1254 : = pull_accesses_from_callee (cs->caller, caller_ipcp_ts, idx,
3944 : : param_desc, arg_desc,
3945 : : ipf->unit_offset,
3946 : : ipf->unit_size, &res);
3947 : 1254 : if (pull_failure)
3948 : : {
3949 : 161 : if (dump_file && (dump_flags & TDF_DETAILS))
3950 : 0 : fprintf (dump_file, " %u->%u: arg access pull "
3951 : : "failed: %s.\n", idx, i, pull_failure);
3952 : :
3953 : 161 : ipf->aggregate_pass_through = false;
3954 : 161 : pacc->certain = true;
3955 : :
3956 : 161 : if (overlapping_certain_accesses_p (param_desc, NULL))
3957 : : {
3958 : 161 : if (dump_file && (dump_flags & TDF_DETAILS))
3959 : 0 : fprintf (dump_file, " ...leading to overlap, "
3960 : : " disqualifying candidate parameter %u\n",
3961 : : idx);
3962 : 161 : param_desc->split_candidate = false;
3963 : : }
3964 : : else
3965 : 0 : bump_reached_size (param_desc, pacc->unit_size, idx);
3966 : :
3967 : 161 : res = true;
3968 : : }
3969 : : else
3970 : : {
3971 : 1093 : if (dump_file && (dump_flags & TDF_DETAILS))
3972 : 0 : fprintf (dump_file, " %u->%u: arg access pull "
3973 : : "succeeded.\n", idx, i);
3974 : 1093 : if (cross_scc)
3975 : 892 : ipf->aggregate_pass_through = false;
3976 : : }
3977 : : }
3978 : : }
3979 : : }
3980 : :
3981 : : /* Handle argument-parameter count mismatches. */
3982 : 2663075 : for (; (i < args_count); i++)
3983 : : {
3984 : 997716 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3985 : :
3986 : 997716 : if (ipf->pointer_pass_through || ipf->aggregate_pass_through)
3987 : : {
3988 : 200135 : int idx = get_single_param_flow_source (ipf);
3989 : 200135 : ipf->pointer_pass_through = false;
3990 : 200135 : ipf->aggregate_pass_through = false;
3991 : 200135 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3992 : 200135 : if (!param_desc->split_candidate)
3993 : 200054 : continue;
3994 : :
3995 : 81 : if (dump_file && (dump_flags & TDF_DETAILS))
3996 : 0 : fprintf (dump_file, " %u->%u: no corresponding formal parameter\n",
3997 : : idx, i);
3998 : 81 : param_desc->split_candidate = false;
3999 : 81 : res = true;
4000 : : }
4001 : : }
4002 : 1665359 : return res;
4003 : : }
4004 : :
4005 : : /* Worker for call_for_symbol_and_aliases, look at all callers and if all their
4006 : : callers ignore the return value, or come from the same SCC and use the
4007 : : return value only to compute their return value, return false, otherwise
4008 : : return true. */
4009 : :
4010 : : static bool
4011 : 187598 : retval_used_p (cgraph_node *node, void *)
4012 : : {
4013 : 269187 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4014 : : {
4015 : 240399 : isra_call_summary *csum = call_sums->get (cs);
4016 : 240399 : gcc_checking_assert (csum);
4017 : 240399 : if (csum->m_return_ignored)
4018 : 77457 : continue;
4019 : 162942 : if (!csum->m_return_returned)
4020 : : return true;
4021 : :
4022 : 24779 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
4023 : 24779 : if (!from_ifs || !from_ifs->m_candidate)
4024 : : return true;
4025 : :
4026 : 18474 : if (!ipa_edge_within_scc (cs)
4027 : 18474 : && !from_ifs->m_return_ignored)
4028 : : return true;
4029 : : }
4030 : :
4031 : : return false;
4032 : : }
4033 : :
4034 : : /* Push into NEW_PARAMS all required parameter adjustment entries to copy or
4035 : : modify parameter which originally had index BASE_INDEX, in the adjustment
4036 : : vector of parent clone (if any) had PREV_CLONE_INDEX and was described by
4037 : : PREV_ADJUSTMENT. If IPA-CP has created a transformation summary for the
4038 : : original node, it needs to be passed in IPCP_TS, otherwise it should be
4039 : : NULL. If the parent clone is the original function, PREV_ADJUSTMENT is NULL
4040 : : and PREV_CLONE_INDEX is equal to BASE_INDEX. */
4041 : :
4042 : : static void
4043 : 308324 : push_param_adjustments_for_index (isra_func_summary *ifs, unsigned base_index,
4044 : : unsigned prev_clone_index,
4045 : : ipa_adjusted_param *prev_adjustment,
4046 : : ipcp_transformation *ipcp_ts,
4047 : : vec<ipa_adjusted_param, va_gc> **new_params)
4048 : : {
4049 : 308324 : isra_param_desc *desc = &(*ifs->m_parameters)[base_index];
4050 : 308324 : if (desc->locally_unused)
4051 : : {
4052 : 59420 : if (dump_file)
4053 : 27 : fprintf (dump_file, " Will remove parameter %u\n", base_index);
4054 : 59420 : return;
4055 : : }
4056 : :
4057 : 248904 : if (!desc->split_candidate)
4058 : : {
4059 : 196853 : ipa_adjusted_param adj;
4060 : 196853 : if (prev_adjustment)
4061 : : {
4062 : 1515 : adj = *prev_adjustment;
4063 : 1515 : adj.prev_clone_adjustment = true;
4064 : 1515 : adj.prev_clone_index = prev_clone_index;
4065 : : }
4066 : : else
4067 : : {
4068 : 195338 : memset (&adj, 0, sizeof (adj));
4069 : 195338 : adj.op = IPA_PARAM_OP_COPY;
4070 : 195338 : adj.base_index = base_index;
4071 : 195338 : adj.prev_clone_index = prev_clone_index;
4072 : : }
4073 : 196853 : vec_safe_push ((*new_params), adj);
4074 : 196853 : return;
4075 : : }
4076 : :
4077 : 52051 : if (dump_file)
4078 : 39 : fprintf (dump_file, " Will split parameter %u\n", base_index);
4079 : :
4080 : 52051 : gcc_assert (!prev_adjustment || prev_adjustment->op == IPA_PARAM_OP_COPY);
4081 : 52051 : unsigned aclen = vec_safe_length (desc->accesses);
4082 : 140712 : for (unsigned j = 0; j < aclen; j++)
4083 : : {
4084 : 88661 : param_access *pa = (*desc->accesses)[j];
4085 : 88661 : if (!pa->certain)
4086 : 7021 : continue;
4087 : :
4088 : 87862 : if (ipcp_ts)
4089 : : {
4090 : 14031 : ipa_argagg_value_list avl (ipcp_ts);
4091 : 14031 : tree value = avl.get_value (base_index, pa->unit_offset);
4092 : 14031 : if (value && !AGGREGATE_TYPE_P (pa->type))
4093 : : {
4094 : 6222 : if (dump_file)
4095 : 20 : fprintf (dump_file, " - omitting component at byte "
4096 : : "offset %u which is known to have a constant value\n ",
4097 : : pa->unit_offset);
4098 : 6222 : continue;
4099 : : }
4100 : : }
4101 : :
4102 : 81640 : if (dump_file)
4103 : 38 : fprintf (dump_file, " - component at byte offset %u, "
4104 : 38 : "size %u\n", pa->unit_offset, pa->unit_size);
4105 : :
4106 : 81640 : ipa_adjusted_param adj;
4107 : 81640 : memset (&adj, 0, sizeof (adj));
4108 : 81640 : adj.op = IPA_PARAM_OP_SPLIT;
4109 : 81640 : adj.base_index = base_index;
4110 : 81640 : adj.prev_clone_index = prev_clone_index;
4111 : 81640 : adj.param_prefix_index = IPA_PARAM_PREFIX_ISRA;
4112 : 81640 : adj.reverse = pa->reverse;
4113 : 81640 : adj.type = pa->type;
4114 : 81640 : adj.alias_ptr_type = pa->alias_ptr_type;
4115 : 81640 : adj.unit_offset = pa->unit_offset;
4116 : 81640 : vec_safe_push ((*new_params), adj);
4117 : : }
4118 : : }
4119 : :
4120 : : /* Worker for all call_for_symbol_thunks_and_aliases. Set calls_comdat_local
4121 : : flag of all callers of NODE. */
4122 : :
4123 : : static bool
4124 : 1 : mark_callers_calls_comdat_local (struct cgraph_node *node, void *)
4125 : : {
4126 : 3 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4127 : 2 : cs->caller->calls_comdat_local = true;
4128 : 1 : return false;
4129 : : }
4130 : :
4131 : : /* Remove any IPA-CP results stored in TS that are associated with removed
4132 : : parameters as marked in IFS. */
4133 : :
4134 : : static void
4135 : 8088 : zap_useless_ipcp_results (const isra_func_summary *ifs, ipcp_transformation *ts)
4136 : : {
4137 : 8088 : ts->remove_argaggs_if ([ifs](const ipa_argagg_value &v)
4138 : : {
4139 : 14223 : return (*ifs->m_parameters)[v.index].locally_unused;
4140 : : });
4141 : :
4142 : 8088 : bool useful_vr = false;
4143 : 8088 : unsigned count = vec_safe_length (ts->m_vr);
4144 : 27256 : for (unsigned i = 0; i < count; i++)
4145 : 19168 : if ((*ts->m_vr)[i].known_p ())
4146 : : {
4147 : 13635 : const isra_param_desc *desc = &(*ifs->m_parameters)[i];
4148 : 13635 : if (desc->locally_unused)
4149 : 1788 : (*ts->m_vr)[i].set_unknown ();
4150 : : else
4151 : : useful_vr = true;
4152 : : }
4153 : 8088 : if (!useful_vr)
4154 : 1494 : ts->m_vr = NULL;
4155 : 8088 : }
4156 : :
4157 : : /* Do final processing of results of IPA propagation regarding NODE, clone it
4158 : : if appropriate. */
4159 : :
4160 : : static void
4161 : 1259630 : process_isra_node_results (cgraph_node *node,
4162 : : hash_map<const char *, unsigned> *clone_num_suffixes)
4163 : : {
4164 : 1259630 : isra_func_summary *ifs = func_sums->get (node);
4165 : 1259630 : if (!ifs || !ifs->m_candidate)
4166 : 1151171 : return;
4167 : :
4168 : 375324 : auto_vec<bool, 16> surviving_params;
4169 : 375324 : bool check_surviving = false;
4170 : 375324 : clone_info *cinfo = clone_info::get (node);
4171 : 375324 : if (cinfo && cinfo->param_adjustments)
4172 : : {
4173 : 12899 : check_surviving = true;
4174 : 12899 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
4175 : : }
4176 : :
4177 : 375324 : unsigned param_count = vec_safe_length (ifs->m_parameters);
4178 : 375324 : bool will_change_function = false;
4179 : 375324 : if (ifs->m_returns_value && ifs->m_return_ignored)
4180 : : will_change_function = true;
4181 : : else
4182 : 897673 : for (unsigned i = 0; i < param_count; i++)
4183 : : {
4184 : 630808 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
4185 : 581350 : if ((desc->locally_unused || desc->split_candidate)
4186 : : /* Make sure we do not clone just to attempt to remove an already
4187 : : removed unused argument. */
4188 : 669418 : && (!check_surviving
4189 : 553278 : || (i < surviving_params.length ()
4190 : 2511 : && surviving_params[i])))
4191 : : {
4192 : : will_change_function = true;
4193 : : break;
4194 : : }
4195 : : }
4196 : 346906 : if (!will_change_function)
4197 : 266865 : return;
4198 : :
4199 : 108459 : if (dump_file)
4200 : : {
4201 : 61 : fprintf (dump_file, "\nEvaluating analysis results for %s\n",
4202 : : node->dump_name ());
4203 : 61 : if (ifs->m_returns_value && ifs->m_return_ignored)
4204 : 15 : fprintf (dump_file, " Will remove return value.\n");
4205 : : }
4206 : :
4207 : 108459 : ipcp_transformation *ipcp_ts = ipcp_get_transformation_summary (node);
4208 : 108459 : if (ipcp_ts)
4209 : 8088 : zap_useless_ipcp_results (ifs, ipcp_ts);
4210 : 108459 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
4211 : 108459 : if (ipa_param_adjustments *old_adjustments
4212 : 108459 : = cinfo ? cinfo->param_adjustments : NULL)
4213 : : {
4214 : 2211 : unsigned old_adj_len = vec_safe_length (old_adjustments->m_adj_params);
4215 : 5494 : for (unsigned i = 0; i < old_adj_len; i++)
4216 : : {
4217 : 3283 : ipa_adjusted_param *old_adj = &(*old_adjustments->m_adj_params)[i];
4218 : 3283 : push_param_adjustments_for_index (ifs, old_adj->base_index, i,
4219 : : old_adj, ipcp_ts, &new_params);
4220 : : }
4221 : : }
4222 : : else
4223 : 411289 : for (unsigned i = 0; i < param_count; i++)
4224 : 305041 : push_param_adjustments_for_index (ifs, i, i, NULL, ipcp_ts, &new_params);
4225 : :
4226 : 108459 : ipa_param_adjustments *new_adjustments
4227 : 108459 : = (new (ggc_alloc <ipa_param_adjustments> ())
4228 : : ipa_param_adjustments (new_params, param_count,
4229 : 188500 : ifs->m_returns_value && ifs->m_return_ignored));
4230 : :
4231 : 108459 : if (dump_file && (dump_flags & TDF_DETAILS))
4232 : : {
4233 : 9 : fprintf (dump_file, "\n Created adjustments:\n");
4234 : 9 : new_adjustments->dump (dump_file);
4235 : : }
4236 : :
4237 : 325377 : unsigned &suffix_counter = clone_num_suffixes->get_or_insert (
4238 : 108459 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (
4239 : : node->decl)));
4240 : 108459 : auto_vec<cgraph_edge *> callers = node->collect_callers ();
4241 : 108459 : cgraph_node *new_node
4242 : 108459 : = node->create_virtual_clone (callers, NULL, new_adjustments, "isra",
4243 : : suffix_counter);
4244 : 108459 : suffix_counter++;
4245 : 108459 : if (node->calls_comdat_local && node->same_comdat_group)
4246 : : {
4247 : 1 : new_node->add_to_same_comdat_group (node);
4248 : 1 : new_node->call_for_symbol_and_aliases (mark_callers_calls_comdat_local,
4249 : : NULL, true);
4250 : : }
4251 : 108459 : new_node->calls_comdat_local = node->calls_comdat_local;
4252 : :
4253 : 108459 : if (dump_file)
4254 : 61 : fprintf (dump_file, " Created new node %s\n", new_node->dump_name ());
4255 : 108459 : callers.release ();
4256 : 375324 : }
4257 : :
4258 : : /* If INDICES is not empty, dump a combination of NODE's dump_name and MSG
4259 : : followed by the list of numbers in INDICES. */
4260 : :
4261 : : static void
4262 : 645010 : dump_list_of_param_indices (const cgraph_node *node, const char* msg,
4263 : : const vec<unsigned> &indices)
4264 : : {
4265 : 645010 : if (indices.is_empty ())
4266 : : return;
4267 : 1 : fprintf (dump_file, "The following parameters of %s %s:", node->dump_name (),
4268 : : msg);
4269 : 4 : for (unsigned i : indices)
4270 : 1 : fprintf (dump_file, " %u", i);
4271 : 1 : fprintf (dump_file, "\n");
4272 : : }
4273 : :
4274 : : /* Check which parameters of NODE described by IFS have survived until IPA-SRA
4275 : : and disable transformations for those which have not or which should not
4276 : : transformed because the associated debug counter reached its limit. Return
4277 : : true if none survived or if there were no candidates to begin with.
4278 : : Additionally, also adjust parameter descriptions based on debug counters and
4279 : : hints propagated earlier. */
4280 : :
4281 : : static bool
4282 : 375324 : adjust_parameter_descriptions (cgraph_node *node, isra_func_summary *ifs)
4283 : : {
4284 : 375324 : bool ret = true;
4285 : 375324 : unsigned len = vec_safe_length (ifs->m_parameters);
4286 : 322505 : if (!len)
4287 : : return true;
4288 : :
4289 : 322505 : auto_vec<bool, 16> surviving_params;
4290 : 322505 : bool check_surviving = false;
4291 : 322505 : clone_info *cinfo = clone_info::get (node);
4292 : 322505 : if (cinfo && cinfo->param_adjustments)
4293 : : {
4294 : 12899 : check_surviving = true;
4295 : 12899 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
4296 : : }
4297 : 322505 : ipcp_transformation *ipcp_ts = ipcp_get_transformation_summary (node);
4298 : 322505 : auto_vec <unsigned> dump_dead_indices;
4299 : 322505 : auto_vec <unsigned> dump_bad_cond_indices;
4300 : 1107810 : for (unsigned i = 0; i < len; i++)
4301 : : {
4302 : 785305 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
4303 : 785305 : if (!dbg_cnt (ipa_sra_params))
4304 : : {
4305 : 0 : desc->locally_unused = false;
4306 : 0 : desc->split_candidate = false;
4307 : 0 : continue;
4308 : : }
4309 : :
4310 : 785305 : if (desc->split_only_when_retval_removed
4311 : 5554 : && !ifs->m_return_ignored)
4312 : : {
4313 : 0 : if (dump_file && (dump_flags & TDF_DETAILS)
4314 : 1202 : && (desc->locally_unused || desc->split_candidate))
4315 : 0 : dump_bad_cond_indices.safe_push (i);
4316 : :
4317 : 1202 : gcc_checking_assert (!desc->locally_unused
4318 : : || desc->remove_only_when_retval_removed);
4319 : 1202 : desc->locally_unused = false;
4320 : 1202 : desc->split_candidate = false;
4321 : 1202 : continue;
4322 : : }
4323 : 784103 : if (desc->remove_only_when_retval_removed
4324 : 40550 : && !ifs->m_return_ignored)
4325 : : {
4326 : 6 : if (dump_file && (dump_flags & TDF_DETAILS)
4327 : 32592 : && (desc->locally_unused || desc->split_candidate))
4328 : 0 : dump_bad_cond_indices.safe_push (i);
4329 : :
4330 : 32592 : desc->locally_unused = false;
4331 : : }
4332 : 784103 : if (check_surviving
4333 : 820923 : && (i >= surviving_params.length ()
4334 : 17571 : || !surviving_params[i]))
4335 : : {
4336 : : /* Even if the parameter was removed by a previous IPA pass, we do
4337 : : not clear locally_unused because if it really is unused, this
4338 : : information might be useful in callers. */
4339 : 23233 : desc->split_candidate = false;
4340 : :
4341 : 23233 : if (dump_file && (dump_flags & TDF_DETAILS))
4342 : 0 : dump_dead_indices.safe_push (i);
4343 : : }
4344 : :
4345 : 784103 : if (desc->split_candidate && desc->conditionally_dereferenceable)
4346 : : {
4347 : 727 : gcc_assert (desc->safe_size_set);
4348 : 1269 : for (param_access *pa : *desc->accesses)
4349 : 1148 : if ((pa->unit_offset + pa->unit_size) > desc->safe_size)
4350 : : {
4351 : 606 : if (dump_file && (dump_flags & TDF_DETAILS))
4352 : 1 : dump_bad_cond_indices.safe_push (i);
4353 : 606 : desc->split_candidate = false;
4354 : 606 : break;
4355 : : }
4356 : : }
4357 : :
4358 : 784103 : if (desc->split_candidate)
4359 : : {
4360 : 214645 : if (desc->by_ref && !desc->not_specially_constructed)
4361 : : {
4362 : 31037 : int extra_factor
4363 : 31037 : = opt_for_fn (node->decl,
4364 : : param_ipa_sra_ptrwrap_growth_factor);
4365 : 31037 : desc->param_size_limit = extra_factor * desc->param_size_limit;
4366 : : }
4367 : 214645 : if (size_would_violate_limit_p (desc, desc->size_reached))
4368 : 3827 : desc->split_candidate = false;
4369 : : }
4370 : :
4371 : : /* Avoid ICEs on size-mismatched VIEW_CONVERT_EXPRs when callers and
4372 : : callees don't agree on types in aggregates and we try to do both
4373 : : IPA-CP and IPA-SRA. */
4374 : 784103 : if (ipcp_ts && desc->split_candidate)
4375 : : {
4376 : 21891 : ipa_argagg_value_list avl (ipcp_ts);
4377 : 53324 : for (const param_access *pa : desc->accesses)
4378 : : {
4379 : 15581 : if (!pa->certain)
4380 : 897 : continue;
4381 : 14684 : tree value = avl.get_value (i, pa->unit_offset);
4382 : 14684 : if (value
4383 : 14684 : && ((tree_to_uhwi (TYPE_SIZE (TREE_TYPE (value)))
4384 : 6314 : / BITS_PER_UNIT)
4385 : 6314 : != pa->unit_size))
4386 : : {
4387 : 30 : desc->split_candidate = false;
4388 : 30 : if (dump_file && (dump_flags & TDF_DETAILS))
4389 : 0 : dump_dead_indices.safe_push (i);
4390 : : break;
4391 : : }
4392 : : }
4393 : : }
4394 : :
4395 : 784103 : if (desc->locally_unused || desc->split_candidate)
4396 : 785305 : ret = false;
4397 : : }
4398 : :
4399 : 322505 : dump_list_of_param_indices (node, "are dead on arrival or have a type "
4400 : : "mismatch with IPA-CP", dump_dead_indices);
4401 : 322505 : dump_list_of_param_indices (node, "fail additional requirements ",
4402 : : dump_bad_cond_indices);
4403 : :
4404 : 322505 : return ret;
4405 : 322505 : }
4406 : :
4407 : :
4408 : : /* Run the interprocedural part of IPA-SRA. */
4409 : :
4410 : : static unsigned int
4411 : 123303 : ipa_sra_analysis (void)
4412 : : {
4413 : 123303 : if (dump_file)
4414 : : {
4415 : 54 : fprintf (dump_file, "\n========== IPA-SRA IPA stage ==========\n");
4416 : 54 : ipa_sra_dump_all_summaries (dump_file, false);
4417 : : }
4418 : :
4419 : 123303 : gcc_checking_assert (func_sums);
4420 : 123303 : gcc_checking_assert (call_sums);
4421 : 123303 : cgraph_node **order = XCNEWVEC (cgraph_node *, symtab->cgraph_count);
4422 : 123303 : auto_vec <cgraph_node *, 16> stack;
4423 : 123303 : int node_scc_count = ipa_reduced_postorder (order, true, NULL);
4424 : :
4425 : : /* One sweep from callers to callees for return value removal. */
4426 : 1449682 : for (int i = node_scc_count - 1; i >= 0 ; i--)
4427 : : {
4428 : 1326379 : cgraph_node *scc_rep = order[i];
4429 : 1326379 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (scc_rep);
4430 : :
4431 : : /* Preliminary IPA function level checks. */
4432 : 5312247 : for (cgraph_node *v : cycle_nodes)
4433 : : {
4434 : 1333110 : isra_func_summary *ifs = func_sums->get (v);
4435 : 1333110 : if (!ifs || !ifs->m_candidate)
4436 : 313664 : continue;
4437 : 1019446 : if (!ipa_sra_ipa_function_checks (v)
4438 : 1019446 : || check_all_callers_for_issues (v))
4439 : 644122 : ifs->zap ();
4440 : : }
4441 : :
4442 : 3985868 : for (cgraph_node *v : cycle_nodes)
4443 : : {
4444 : 1333110 : isra_func_summary *ifs = func_sums->get (v);
4445 : 1333110 : if (!ifs || !ifs->m_candidate)
4446 : 957786 : continue;
4447 : 375324 : bool return_needed
4448 : 375324 : = (ifs->m_returns_value
4449 : 375324 : && (!dbg_cnt (ipa_sra_retvalues)
4450 : 187577 : || v->call_for_symbol_and_aliases (retval_used_p,
4451 : 375324 : NULL, true)));
4452 : 375324 : ifs->m_return_ignored = !return_needed;
4453 : 375324 : if (return_needed)
4454 : 317620 : isra_push_node_to_stack (v, ifs, &stack);
4455 : : }
4456 : :
4457 : 1485538 : while (!stack.is_empty ())
4458 : : {
4459 : 159159 : cgraph_node *node = stack.pop ();
4460 : 159159 : isra_func_summary *ifs = func_sums->get (node);
4461 : 159159 : gcc_checking_assert (ifs && ifs->m_queued);
4462 : 159159 : ifs->m_queued = false;
4463 : :
4464 : 951861 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
4465 : 792702 : if (ipa_edge_within_scc (cs)
4466 : 792702 : && call_sums->get (cs)->m_return_returned)
4467 : : {
4468 : 871 : enum availability av;
4469 : 871 : cgraph_node *callee = cs->callee->function_symbol (&av);
4470 : 871 : isra_func_summary *to_ifs = func_sums->get (callee);
4471 : 871 : if (to_ifs && to_ifs->m_return_ignored)
4472 : : {
4473 : 349 : to_ifs->m_return_ignored = false;
4474 : 698 : isra_push_node_to_stack (callee, to_ifs, &stack);
4475 : : }
4476 : : }
4477 : : }
4478 : :
4479 : : /* Parameter hint propagation. */
4480 : 3985868 : for (cgraph_node *v : cycle_nodes)
4481 : : {
4482 : 1333110 : isra_func_summary *ifs = func_sums->get (v);
4483 : 1333110 : propagate_hints_to_all_callees (v, ifs, &stack);
4484 : : }
4485 : :
4486 : 1330246 : while (!stack.is_empty ())
4487 : : {
4488 : 3867 : cgraph_node *node = stack.pop ();
4489 : 3867 : isra_func_summary *ifs = func_sums->get (node);
4490 : 3867 : gcc_checking_assert (ifs && ifs->m_queued);
4491 : 3867 : ifs->m_queued = false;
4492 : 3867 : propagate_hints_to_all_callees (node, ifs, &stack);
4493 : : }
4494 : :
4495 : 1326379 : cycle_nodes.release ();
4496 : : }
4497 : :
4498 : : /* One sweep from callees to callers for parameter removal and splitting. */
4499 : 1449682 : for (int i = 0; i < node_scc_count; i++)
4500 : : {
4501 : 1326379 : cgraph_node *scc_rep = order[i];
4502 : 1326379 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (scc_rep);
4503 : :
4504 : : /* First step of parameter removal. */
4505 : 5312247 : for (cgraph_node *v : cycle_nodes)
4506 : : {
4507 : 1333110 : isra_func_summary *ifs = func_sums->get (v);
4508 : 1333110 : if (!ifs || !ifs->m_candidate)
4509 : 957786 : continue;
4510 : 375324 : if (adjust_parameter_descriptions (v, ifs))
4511 : 193001 : continue;
4512 : 198570 : for (cgraph_edge *cs = v->indirect_calls; cs; cs = cs->next_callee)
4513 : 16247 : process_edge_to_unknown_caller (cs);
4514 : 1067094 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
4515 : 884771 : if (!ipa_edge_within_scc (cs))
4516 : 875415 : param_removal_cross_scc_edge (cs);
4517 : : }
4518 : :
4519 : : /* Look at edges within the current SCC and propagate used-ness across
4520 : : them, pushing onto the stack all notes which might need to be
4521 : : revisited. */
4522 : 3985868 : for (cgraph_node *v : cycle_nodes)
4523 : 1333110 : v->call_for_symbol_thunks_and_aliases (propagate_used_to_scc_callers,
4524 : : &stack, true);
4525 : :
4526 : : /* Keep revisiting and pushing until nothing changes. */
4527 : 1328627 : while (!stack.is_empty ())
4528 : : {
4529 : 2248 : cgraph_node *v = stack.pop ();
4530 : 2248 : isra_func_summary *ifs = func_sums->get (v);
4531 : 2248 : gcc_checking_assert (ifs && ifs->m_queued);
4532 : 2248 : ifs->m_queued = false;
4533 : :
4534 : 2248 : v->call_for_symbol_thunks_and_aliases (propagate_used_to_scc_callers,
4535 : : &stack, true);
4536 : : }
4537 : :
4538 : : /* Parameter splitting. */
4539 : 1379080 : bool repeat_scc_access_propagation;
4540 : 1379080 : do
4541 : : {
4542 : 1379080 : repeat_scc_access_propagation = false;
4543 : 4148155 : for (cgraph_node *v : cycle_nodes)
4544 : : {
4545 : 1389995 : isra_func_summary *ifs = func_sums->get (v);
4546 : 2401220 : if (!ifs
4547 : 1079279 : || !ifs->m_candidate
4548 : 1821584 : || vec_safe_is_empty (ifs->m_parameters))
4549 : 1011225 : continue;
4550 : 2044129 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
4551 : 1665359 : if (param_splitting_across_edge (cs))
4552 : 61883 : repeat_scc_access_propagation = true;
4553 : : }
4554 : : }
4555 : : while (repeat_scc_access_propagation);
4556 : :
4557 : 1326379 : if (flag_checking)
4558 : 3985817 : for (cgraph_node *v : cycle_nodes)
4559 : 1333093 : verify_splitting_accesses (v, true);
4560 : :
4561 : 1326379 : cycle_nodes.release ();
4562 : : }
4563 : :
4564 : 123303 : ipa_free_postorder_info ();
4565 : 123303 : free (order);
4566 : :
4567 : 123303 : if (dump_file)
4568 : : {
4569 : 54 : if (dump_flags & TDF_DETAILS)
4570 : : {
4571 : 9 : fprintf (dump_file, "\n========== IPA-SRA propagation final state "
4572 : : " ==========\n");
4573 : 9 : ipa_sra_dump_all_summaries (dump_file, true);
4574 : : }
4575 : 54 : fprintf (dump_file, "\n========== IPA-SRA decisions ==========\n");
4576 : : }
4577 : :
4578 : 123303 : hash_map<const char *, unsigned> *clone_num_suffixes
4579 : 123303 : = new hash_map<const char *, unsigned>;
4580 : :
4581 : 123303 : cgraph_node *node;
4582 : 1382933 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
4583 : 1259630 : process_isra_node_results (node, clone_num_suffixes);
4584 : :
4585 : 123303 : delete clone_num_suffixes;
4586 : 123303 : ggc_delete (func_sums);
4587 : 123303 : func_sums = NULL;
4588 : 123303 : delete call_sums;
4589 : 123303 : call_sums = NULL;
4590 : :
4591 : 123303 : if (dump_file)
4592 : 54 : fprintf (dump_file, "\n========== IPA SRA IPA analysis done "
4593 : : "==========\n\n");
4594 : 123303 : return 0;
4595 : 123303 : }
4596 : :
4597 : :
4598 : : const pass_data pass_data_ipa_sra =
4599 : : {
4600 : : IPA_PASS, /* type */
4601 : : "sra", /* name */
4602 : : OPTGROUP_NONE, /* optinfo_flags */
4603 : : TV_IPA_SRA, /* tv_id */
4604 : : 0, /* properties_required */
4605 : : 0, /* properties_provided */
4606 : : 0, /* properties_destroyed */
4607 : : 0, /* todo_flags_start */
4608 : : ( TODO_dump_symtab | TODO_remove_functions ), /* todo_flags_finish */
4609 : : };
4610 : :
4611 : : class pass_ipa_sra : public ipa_opt_pass_d
4612 : : {
4613 : : public:
4614 : 285081 : pass_ipa_sra (gcc::context *ctxt)
4615 : : : ipa_opt_pass_d (pass_data_ipa_sra, ctxt,
4616 : : ipa_sra_generate_summary, /* generate_summary */
4617 : : ipa_sra_write_summary, /* write_summary */
4618 : : ipa_sra_read_summary, /* read_summary */
4619 : : NULL , /* write_optimization_summary */
4620 : : NULL, /* read_optimization_summary */
4621 : : NULL, /* stmt_fixup */
4622 : : 0, /* function_transform_todo_flags_start */
4623 : : NULL, /* function_transform */
4624 : 285081 : NULL) /* variable_transform */
4625 : 285081 : {}
4626 : :
4627 : : /* opt_pass methods: */
4628 : 585879 : bool gate (function *) final override
4629 : : {
4630 : : /* TODO: We should remove the optimize check after we ensure we never run
4631 : : IPA passes when not optimizing. */
4632 : 585879 : return (flag_ipa_sra && optimize);
4633 : : }
4634 : :
4635 : 123303 : unsigned int execute (function *) final override
4636 : : {
4637 : 123303 : return ipa_sra_analysis ();
4638 : : }
4639 : :
4640 : : }; // class pass_ipa_sra
4641 : :
4642 : : } // anon namespace
4643 : :
4644 : : /* Intraprocedural part of IPA-SRA analysis. Scan function body of NODE and
4645 : : create a summary structure describing IPA-SRA opportunities and constraints
4646 : : in it. */
4647 : :
4648 : : static void
4649 : 1258829 : ipa_sra_summarize_function (cgraph_node *node)
4650 : : {
4651 : 1258829 : if (dump_file)
4652 : 187 : fprintf (dump_file, "Creating summary for %s/%i:\n", node->name (),
4653 : : node->get_uid ());
4654 : 1258829 : gcc_obstack_init (&gensum_obstack);
4655 : 1258829 : loaded_decls = new hash_set<tree>;
4656 : :
4657 : 1258829 : isra_func_summary *ifs = NULL;
4658 : 1258829 : unsigned count = 0;
4659 : 1258829 : if (ipa_sra_preliminary_function_checks (node))
4660 : : {
4661 : 1024966 : ifs = func_sums->get_create (node);
4662 : 1024966 : ifs->m_candidate = true;
4663 : 1024966 : tree ret = TREE_TYPE (TREE_TYPE (node->decl));
4664 : 1024966 : ifs->m_returns_value = (TREE_CODE (ret) != VOID_TYPE);
4665 : 1024966 : for (tree parm = DECL_ARGUMENTS (node->decl);
4666 : 3413946 : parm;
4667 : 2388980 : parm = DECL_CHAIN (parm))
4668 : 2388980 : count++;
4669 : : }
4670 : 1258829 : auto_vec<gensum_param_desc, 16> param_descriptions (count);
4671 : :
4672 : 1258829 : struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
4673 : 1258829 : bool cfun_pushed = false;
4674 : 1258829 : if (count > 0)
4675 : : {
4676 : 833517 : decl2desc = new hash_map<tree, gensum_param_desc *>;
4677 : 833517 : param_descriptions.reserve_exact (count);
4678 : 833517 : param_descriptions.quick_grow_cleared (count);
4679 : :
4680 : 833517 : if (create_parameter_descriptors (node, ¶m_descriptions))
4681 : : {
4682 : 418064 : push_cfun (fun);
4683 : 418064 : cfun_pushed = true;
4684 : 418064 : final_bbs = BITMAP_ALLOC (NULL);
4685 : 418064 : bb_dereferences = XCNEWVEC (HOST_WIDE_INT,
4686 : : unsafe_by_ref_count
4687 : : * last_basic_block_for_fn (fun));
4688 : 418064 : aa_walking_limit = opt_for_fn (node->decl, param_ipa_max_aa_steps);
4689 : : }
4690 : : }
4691 : : /* Scan function is run even when there are no removal or splitting
4692 : : candidates so that we can calculate hints on call edges which can be
4693 : : useful in callees. */
4694 : 1258829 : scan_function (node, fun);
4695 : :
4696 : 1258829 : if (count > 0)
4697 : : {
4698 : 833517 : if (dump_file)
4699 : : {
4700 : 135 : dump_gensum_param_descriptors (dump_file, node->decl,
4701 : : ¶m_descriptions);
4702 : 135 : fprintf (dump_file, "----------------------------------------\n");
4703 : : }
4704 : :
4705 : 833517 : process_scan_results (node, fun, ifs, ¶m_descriptions);
4706 : :
4707 : 833517 : if (cfun_pushed)
4708 : 418064 : pop_cfun ();
4709 : 833517 : if (bb_dereferences)
4710 : : {
4711 : 418064 : free (bb_dereferences);
4712 : 418064 : bb_dereferences = NULL;
4713 : 418064 : BITMAP_FREE (final_bbs);
4714 : 418064 : final_bbs = NULL;
4715 : : }
4716 : : }
4717 : 1258829 : isra_analyze_all_outgoing_calls (node);
4718 : :
4719 : 2517658 : delete loaded_decls;
4720 : 1258829 : loaded_decls = NULL;
4721 : 1258829 : if (decl2desc)
4722 : : {
4723 : 833517 : delete decl2desc;
4724 : 833517 : decl2desc = NULL;
4725 : : }
4726 : 1258829 : obstack_free (&gensum_obstack, NULL);
4727 : 1258829 : if (dump_file)
4728 : 187 : fprintf (dump_file, "\n\n");
4729 : 1258829 : if (flag_checking)
4730 : 1258811 : verify_splitting_accesses (node, false);
4731 : 1258829 : return;
4732 : 1258829 : }
4733 : :
4734 : : ipa_opt_pass_d *
4735 : 285081 : make_pass_ipa_sra (gcc::context *ctxt)
4736 : : {
4737 : 285081 : return new pass_ipa_sra (ctxt);
4738 : : }
4739 : :
4740 : : /* Reset all state within ipa-sra.cc so that we can rerun the compiler
4741 : : within the same process. For use by toplev::finalize. */
4742 : :
4743 : : void
4744 : 256374 : ipa_sra_cc_finalize (void)
4745 : : {
4746 : 256374 : if (func_sums)
4747 : 8736 : ggc_delete (func_sums);
4748 : 256374 : func_sums = NULL;
4749 : 256374 : delete call_sums;
4750 : 256374 : call_sums = NULL;
4751 : 256374 : }
4752 : :
4753 : : #include "gt-ipa-sra.h"
|
