Branch data Line data Source code
1 : : /* Interprocedural scalar replacement of aggregates
2 : : Copyright (C) 2019-2024 Free Software Foundation, Inc.
3 : : Contributed by Martin Jambor <mjambor@suse.cz>
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : /* IPA-SRA is an interprocedural pass that removes unused function return
22 : : values (turning functions returning a value which is never used into void
23 : : functions) and removes unused function parameters. It can also replace an
24 : : aggregate parameter by a set of other parameters representing part of the
25 : : original, turning those passed by reference into new ones which pass the
26 : : value directly.
27 : :
28 : : The pass is a true IPA one, which means that it works in three stages in
29 : : order to be able to take advantage of LTO. First, summaries about functions
30 : : and each calls are generated. Function summaries (often called call graph
31 : : node summaries) contain mainly information about which parameters are
32 : : potential transformation candidates and which bits of candidates are
33 : : accessed. We differentiate between accesses done as a part of a call
34 : : statement (which might be not necessary if the callee is also transformed)
35 : : and others (which are mandatory). Call summaries (often called call graph
36 : : edge summaries) contain information about which function formal parameters
37 : : feed into which actual call arguments so that if two parameters are only
38 : : used in a sum which is then passed to another function which then however
39 : : does not use this parameter, all three parameters of the two functions can
40 : : be eliminated. Edge summaries also have flags whether the return value is
41 : : used or if it is only returned in the caller too. In LTO mode these
42 : : summaries are then streamed to the object file in the compilation phase and
43 : : streamed back in in the WPA analysis stage.
44 : :
45 : : The interprocedural analysis phase traverses the graph in topological order
46 : : in two sweeps, one in each direction. First, from callees to callers for
47 : : parameter removal and splitting. Each strongly-connected component is
48 : : processed iteratively until the situation in it stabilizes. The pass from
49 : : callers to callees is then carried out to remove unused return values in a
50 : : very similar fashion.
51 : :
52 : : Because parameter manipulation has big implications for call redirection
53 : : which is done only after all call graph nodes materialize, the
54 : : transformation phase is not part of this patch but is carried out by the
55 : : clone materialization and edge redirection itself, see comments in
56 : : ipa-param-manipulation.h for more details. */
57 : :
58 : :
59 : : #include "config.h"
60 : : #include "system.h"
61 : : #include "coretypes.h"
62 : : #include "backend.h"
63 : : #include "tree.h"
64 : : #include "gimple.h"
65 : : #include "predict.h"
66 : : #include "tree-pass.h"
67 : : #include "ssa.h"
68 : : #include "cgraph.h"
69 : : #include "gimple-pretty-print.h"
70 : : #include "alias.h"
71 : : #include "tree-eh.h"
72 : : #include "gimple-iterator.h"
73 : : #include "gimple-walk.h"
74 : : #include "tree-dfa.h"
75 : : #include "tree-sra.h"
76 : : #include "alloc-pool.h"
77 : : #include "symbol-summary.h"
78 : : #include "dbgcnt.h"
79 : : #include "tree-inline.h"
80 : : #include "ipa-utils.h"
81 : : #include "builtins.h"
82 : : #include "cfganal.h"
83 : : #include "tree-streamer.h"
84 : : #include "internal-fn.h"
85 : : #include "symtab-clones.h"
86 : : #include "attribs.h"
87 : : #include "sreal.h"
88 : : #include "ipa-cp.h"
89 : : #include "ipa-prop.h"
90 : :
91 : : static void ipa_sra_summarize_function (cgraph_node *);
92 : :
93 : : /* Bits used to track size of an aggregate in bytes interprocedurally. */
94 : : #define ISRA_ARG_SIZE_LIMIT_BITS 16
95 : : #define ISRA_ARG_SIZE_LIMIT (1 << ISRA_ARG_SIZE_LIMIT_BITS)
96 : : /* How many parameters can feed into a call actual argument and still be
97 : : tracked. */
98 : : #define IPA_SRA_MAX_PARAM_FLOW_LEN 7
99 : :
100 : : /* Structure describing accesses to a specific portion of an aggregate
101 : : parameter, as given by the offset and size. Any smaller accesses that occur
102 : : within a function that fall within another access form a tree. The pass
103 : : cannot analyze parameters with only partially overlapping accesses. */
104 : :
105 : : struct GTY(()) param_access
106 : : {
107 : : /* Type that a potential replacement should have. This field only has
108 : : meaning in the summary building and transformation phases, when it is
109 : : reconstructed from the body. Must not be touched in IPA analysis
110 : : stage. */
111 : : tree type;
112 : :
113 : : /* Alias reference type to be used in MEM_REFs when adjusting caller
114 : : arguments. */
115 : : tree alias_ptr_type;
116 : :
117 : : /* Values returned by get_ref_base_and_extent but converted to bytes and
118 : : stored as unsigned ints. */
119 : : unsigned unit_offset;
120 : : unsigned unit_size : ISRA_ARG_SIZE_LIMIT_BITS;
121 : :
122 : : /* Set once we are sure that the access will really end up in a potentially
123 : : transformed function - initially not set for portions of formal parameters
124 : : that are only used as actual function arguments passed to callees. */
125 : : unsigned certain : 1;
126 : : /* Set if the access has reverse scalar storage order. */
127 : : unsigned reverse : 1;
128 : : };
129 : :
130 : : /* This structure has the same purpose as the one above and additionally it
131 : : contains some fields that are only necessary in the summary generation
132 : : phase. */
133 : :
134 : : struct gensum_param_access
135 : : {
136 : : /* Values returned by get_ref_base_and_extent. */
137 : : HOST_WIDE_INT offset;
138 : : HOST_WIDE_INT size;
139 : :
140 : : /* if this access has any children (in terms of the definition above), this
141 : : points to the first one. */
142 : : struct gensum_param_access *first_child;
143 : : /* In intraprocedural SRA, pointer to the next sibling in the access tree as
144 : : described above. */
145 : : struct gensum_param_access *next_sibling;
146 : :
147 : : /* Type that a potential replacement should have. This field only has
148 : : meaning in the summary building and transformation phases, when it is
149 : : reconstructed from the body. Must not be touched in IPA analysis
150 : : stage. */
151 : : tree type;
152 : : /* Alias reference type to be used in MEM_REFs when adjusting caller
153 : : arguments. */
154 : : tree alias_ptr_type;
155 : :
156 : : /* Cumulative count of all loads. */
157 : : profile_count load_count;
158 : : /* Have there been writes to or reads from this exact location except for as
159 : : arguments to a function call that can be tracked. */
160 : : bool nonarg;
161 : :
162 : : /* Set if the access has reverse scalar storage order. */
163 : : bool reverse;
164 : : };
165 : :
166 : : /* Summary describing a parameter in the IPA stages. */
167 : :
168 : : struct GTY(()) isra_param_desc
169 : : {
170 : : /* List of access representatives to the parameters, sorted according to
171 : : their offset. */
172 : : vec <param_access *, va_gc> *accesses;
173 : :
174 : : /* Unit size limit of total size of all replacements. */
175 : : unsigned param_size_limit : ISRA_ARG_SIZE_LIMIT_BITS;
176 : : /* Sum of unit sizes of all certain replacements. */
177 : : unsigned size_reached : ISRA_ARG_SIZE_LIMIT_BITS;
178 : : /* Minimum offset that is known to be safe to dereference because of callers
179 : : pass pointers to DECLs of at least this size or because of dereferences in
180 : : callers. */
181 : : unsigned safe_size : ISRA_ARG_SIZE_LIMIT_BITS;
182 : :
183 : : /* A parameter that is used only in call arguments and can be removed if all
184 : : concerned actual arguments are removed. */
185 : : unsigned locally_unused : 1;
186 : : /* An aggregate that is a candidate for breaking up or complete removal. */
187 : : unsigned split_candidate : 1;
188 : : /* Is this a parameter passing stuff by reference? */
189 : : unsigned by_ref : 1;
190 : : /* If set, this parameter can only be a candidate for removal if the function
191 : : is going to loose its return value. */
192 : : unsigned remove_only_when_retval_removed : 1;
193 : : /* If set, this parameter can only be a candidate for splitting if the
194 : : function is going to loose its return value. Can only be meaningfully set
195 : : for by_ref parameters. */
196 : : unsigned split_only_when_retval_removed : 1;
197 : : /* Parameter hint set during IPA analysis when there is a caller which does
198 : : not construct the argument just to pass it to calls. Only meaningful for
199 : : by_ref parameters. */
200 : : unsigned not_specially_constructed : 1;
201 : : /* Only meaningful for by_ref parameters. If set, this parameter can only be
202 : : a split candidate if all callers pass pointers that are known to point to
203 : : a chunk of memory large enough to contain all accesses. */
204 : : unsigned conditionally_dereferenceable : 1;
205 : : /* Set when safe_size has been updated from at least one caller. */
206 : : unsigned safe_size_set : 1;
207 : : };
208 : :
209 : : /* Structure used when generating summaries that describes a parameter. */
210 : :
211 : : struct gensum_param_desc
212 : : {
213 : : /* Roots of param_accesses. */
214 : : gensum_param_access *accesses;
215 : : /* Number of accesses in the access tree rooted in field accesses. */
216 : : unsigned access_count;
217 : :
218 : : /* If the below is non-zero, this is the number of uses as actual
219 : : arguments. */
220 : : int call_uses;
221 : : /* Number of times this parameter has been directly passed to. */
222 : : unsigned ptr_pt_count;
223 : :
224 : : /* Size limit of total size of all replacements. */
225 : : unsigned param_size_limit;
226 : : /* Sum of sizes of nonarg accesses. */
227 : : unsigned nonarg_acc_size;
228 : :
229 : : /* A parameter that is used only in call arguments and can be removed if all
230 : : concerned actual arguments are removed. */
231 : : bool locally_unused;
232 : : /* An aggregate that is a candidate for breaking up or a pointer passing data
233 : : by reference that is a candidate for being converted to a set of
234 : : parameters passing those data by value. */
235 : : bool split_candidate;
236 : : /* Is this a parameter passing stuff by reference (either a pointer or a
237 : : source language reference type)? */
238 : : bool by_ref;
239 : : /* If this parameter passes stuff by reference, can it be safely dereferenced
240 : : without performing further checks (for example because it is a
241 : : REFERENCE_TYPE)? */
242 : : bool safe_ref;
243 : : /* If set, this parameter can only be a candidate for removal if the function
244 : : is going to loose its return value. */
245 : : bool remove_only_when_retval_removed;
246 : : /* If set, this parameter can only be a candidate for splitting if the
247 : : function is going to loose its return value. Can only be meaningfully set
248 : : for by_ref parameters. */
249 : : bool split_only_when_retval_removed;
250 : : /* Only meaningful for by_ref parameters. If set, this parameter can only be
251 : : a split candidate if all callers pass pointers that are known to point to
252 : : a chunk of memory large enough to contain all accesses. */
253 : : bool conditionally_dereferenceable;
254 : :
255 : : /* The number of this parameter as they are ordered in function decl. */
256 : : int param_number;
257 : : /* For parameters passing data by reference, this is parameter index to
258 : : compute indices to bb_dereferences. */
259 : : int deref_index;
260 : : };
261 : :
262 : : /* Properly deallocate accesses of DESC. TODO: Since this data structure is
263 : : allocated in GC memory, this is not necessary and we can consider removing
264 : : the function. */
265 : :
266 : : static void
267 : 2575758 : free_param_decl_accesses (isra_param_desc *desc)
268 : : {
269 : 2575758 : unsigned len = vec_safe_length (desc->accesses);
270 : 3114830 : for (unsigned i = 0; i < len; ++i)
271 : 539072 : ggc_free ((*desc->accesses)[i]);
272 : 2575758 : vec_free (desc->accesses);
273 : 2575758 : }
274 : :
275 : : /* Class used to convey information about functions from the
276 : : intra-procedural analysis stage to inter-procedural one. */
277 : :
278 : : class GTY((for_user)) isra_func_summary
279 : : {
280 : : public:
281 : : /* initialize the object. */
282 : :
283 : 1112156 : isra_func_summary ()
284 : 1112156 : : m_parameters (NULL), m_candidate (false), m_returns_value (false),
285 : 1112156 : m_return_ignored (false), m_queued (false)
286 : : {}
287 : :
288 : : /* Destroy m_parameters. */
289 : :
290 : : ~isra_func_summary ();
291 : :
292 : : /* Mark the function as not a candidate for any IPA-SRA transformation.
293 : : Return true if it was a candidate until now. */
294 : :
295 : : bool zap ();
296 : :
297 : : /* Vector of parameter descriptors corresponding to the function being
298 : : analyzed. */
299 : : vec<isra_param_desc, va_gc> *m_parameters;
300 : :
301 : : /* Whether the node is even a candidate for any IPA-SRA transformation at
302 : : all. */
303 : : unsigned m_candidate : 1;
304 : :
305 : : /* Whether the original function returns any value. */
306 : : unsigned m_returns_value : 1;
307 : :
308 : : /* Set to true if all call statements do not actually use the returned
309 : : value. */
310 : :
311 : : unsigned m_return_ignored : 1;
312 : :
313 : : /* Whether the node is already queued in IPA SRA stack during processing of
314 : : call graphs SCCs. */
315 : :
316 : : unsigned m_queued : 1;
317 : : };
318 : :
319 : : /* Deallocate the memory pointed to by isra_func_summary. TODO: Since this
320 : : data structure is allocated in GC memory, this is not necessary and we can
321 : : consider removing the destructor. */
322 : :
323 : 1112155 : isra_func_summary::~isra_func_summary ()
324 : : {
325 : 1112155 : unsigned len = vec_safe_length (m_parameters);
326 : 2218030 : for (unsigned i = 0; i < len; ++i)
327 : 1105875 : free_param_decl_accesses (&(*m_parameters)[i]);
328 : 1112155 : vec_free (m_parameters);
329 : 1112155 : }
330 : :
331 : : /* Mark the function as not a candidate for any IPA-SRA transformation. Return
332 : : true if it was a candidate until now. */
333 : :
334 : : bool
335 : 636354 : isra_func_summary::zap ()
336 : : {
337 : 636354 : bool ret = m_candidate;
338 : 636354 : m_candidate = false;
339 : :
340 : : /* TODO: see the destructor above. */
341 : 636354 : unsigned len = vec_safe_length (m_parameters);
342 : 2106237 : for (unsigned i = 0; i < len; ++i)
343 : 1469883 : free_param_decl_accesses (&(*m_parameters)[i]);
344 : 636354 : vec_free (m_parameters);
345 : :
346 : 636354 : return ret;
347 : : }
348 : :
349 : : /* Structure to describe which formal parameters feed into a particular actual
350 : : argument. */
351 : :
352 : : struct isra_param_flow
353 : : {
354 : : /* Number of elements in array inputs that contain valid data. */
355 : : char length;
356 : : /* Indices of formal parameters that feed into the described actual argument.
357 : : If aggregate_pass_through or pointer_pass_through below are true, it must
358 : : contain exactly one element which is passed through from a formal
359 : : parameter if the given number. Otherwise, the array contains indices of
360 : : callee's formal parameters which are used to calculate value of this
361 : : actual argument. */
362 : : unsigned char inputs[IPA_SRA_MAX_PARAM_FLOW_LEN];
363 : :
364 : : /* Offset within the formal parameter. */
365 : : unsigned unit_offset;
366 : : /* When aggregate_pass_through is set, this is the size of the portion of an
367 : : aggregate formal parameter that is being passed. Otherwise, this is size
368 : : of pointed to memory that is known to be valid be dereferenced. */
369 : : unsigned unit_size : ISRA_ARG_SIZE_LIMIT_BITS;
370 : :
371 : : /* True when the value of this actual argument is a portion of a formal
372 : : parameter. */
373 : : unsigned aggregate_pass_through : 1;
374 : : /* True when the value of this actual copy is a verbatim pass through of an
375 : : obtained pointer. */
376 : : unsigned pointer_pass_through : 1;
377 : : /* True when it is safe to copy access candidates here from the callee, which
378 : : would mean introducing dereferences into callers of the caller. */
379 : : unsigned safe_to_import_accesses : 1;
380 : : /* True when the passed value is an address of a structure that has been
381 : : constructed in the caller just to be passed by reference to functions
382 : : (i.e. is never read). */
383 : : unsigned constructed_for_calls : 1;
384 : : };
385 : :
386 : : /* Structure used to convey information about calls from the intra-procedural
387 : : analysis stage to inter-procedural one. */
388 : :
389 : 5644675 : class isra_call_summary
390 : : {
391 : : public:
392 : 5644675 : isra_call_summary ()
393 : 0 : : m_arg_flow (), m_return_ignored (false), m_return_returned (false),
394 : 0 : m_bit_aligned_arg (false), m_before_any_store (false)
395 : : {}
396 : :
397 : : void init_inputs (unsigned arg_count);
398 : : void dump (FILE *f);
399 : :
400 : : /* Information about what formal parameters of the caller are used to compute
401 : : individual actual arguments of this call. */
402 : : auto_vec <isra_param_flow> m_arg_flow;
403 : :
404 : : /* Set to true if the call statement does not have a LHS. */
405 : : unsigned m_return_ignored : 1;
406 : :
407 : : /* Set to true if the LHS of call statement is only used to construct the
408 : : return value of the caller. */
409 : : unsigned m_return_returned : 1;
410 : :
411 : : /* Set when any of the call arguments are not byte-aligned. */
412 : : unsigned m_bit_aligned_arg : 1;
413 : :
414 : : /* Set to true if the call happend before any (other) store to memory in the
415 : : caller. */
416 : : unsigned m_before_any_store : 1;
417 : : };
418 : :
419 : : /* Class to manage function summaries. */
420 : :
421 : : class GTY((user)) ipa_sra_function_summaries
422 : : : public function_summary <isra_func_summary *>
423 : : {
424 : : public:
425 : 130209 : ipa_sra_function_summaries (symbol_table *table, bool ggc):
426 : 260418 : function_summary<isra_func_summary *> (table, ggc) { }
427 : :
428 : : void duplicate (cgraph_node *, cgraph_node *,
429 : : isra_func_summary *old_sum,
430 : : isra_func_summary *new_sum) final override;
431 : : void insert (cgraph_node *, isra_func_summary *) final override;
432 : : };
433 : :
434 : : /* Hook that is called by summary when a node is duplicated. */
435 : :
436 : : void
437 : 109735 : ipa_sra_function_summaries::duplicate (cgraph_node *, cgraph_node *,
438 : : isra_func_summary *old_sum,
439 : : isra_func_summary *new_sum)
440 : : {
441 : : /* TODO: Somehow stop copying when ISRA is doing the cloning, it is
442 : : useless. */
443 : 109735 : new_sum->m_candidate = old_sum->m_candidate;
444 : 109735 : new_sum->m_returns_value = old_sum->m_returns_value;
445 : 109735 : new_sum->m_return_ignored = old_sum->m_return_ignored;
446 : 109735 : gcc_assert (!old_sum->m_queued);
447 : 109735 : new_sum->m_queued = false;
448 : :
449 : 109735 : unsigned param_count = vec_safe_length (old_sum->m_parameters);
450 : 109034 : if (!param_count)
451 : : return;
452 : 109034 : vec_safe_reserve_exact (new_sum->m_parameters, param_count);
453 : 109034 : new_sum->m_parameters->quick_grow_cleared (param_count);
454 : 414149 : for (unsigned i = 0; i < param_count; i++)
455 : : {
456 : 305115 : isra_param_desc *s = &(*old_sum->m_parameters)[i];
457 : 305115 : isra_param_desc *d = &(*new_sum->m_parameters)[i];
458 : :
459 : 305115 : d->param_size_limit = s->param_size_limit;
460 : 305115 : d->size_reached = s->size_reached;
461 : 305115 : d->safe_size = s->safe_size;
462 : 305115 : d->locally_unused = s->locally_unused;
463 : 305115 : d->split_candidate = s->split_candidate;
464 : 305115 : d->by_ref = s->by_ref;
465 : 305115 : d->remove_only_when_retval_removed = s->remove_only_when_retval_removed;
466 : 305115 : d->split_only_when_retval_removed = s->split_only_when_retval_removed;
467 : 305115 : d->not_specially_constructed = s->not_specially_constructed;
468 : 305115 : d->conditionally_dereferenceable = s->conditionally_dereferenceable;
469 : 305115 : d->safe_size_set = s->safe_size_set;
470 : :
471 : 305115 : unsigned acc_count = vec_safe_length (s->accesses);
472 : 305115 : vec_safe_reserve_exact (d->accesses, acc_count);
473 : 431663 : for (unsigned j = 0; j < acc_count; j++)
474 : : {
475 : 126548 : param_access *from = (*s->accesses)[j];
476 : 126548 : param_access *to = ggc_cleared_alloc<param_access> ();
477 : 126548 : to->type = from->type;
478 : 126548 : to->alias_ptr_type = from->alias_ptr_type;
479 : 126548 : to->unit_offset = from->unit_offset;
480 : 126548 : to->unit_size = from->unit_size;
481 : 126548 : to->certain = from->certain;
482 : 126548 : to->reverse = from->reverse;
483 : 126548 : d->accesses->quick_push (to);
484 : : }
485 : : }
486 : : }
487 : :
488 : : /* Pointer to the pass function summary holder. */
489 : :
490 : : static GTY(()) ipa_sra_function_summaries *func_sums;
491 : :
492 : : /* Hook that is called by summary when new node appears. */
493 : :
494 : : void
495 : 17069 : ipa_sra_function_summaries::insert (cgraph_node *node, isra_func_summary *)
496 : : {
497 : 17069 : if (opt_for_fn (node->decl, flag_ipa_sra))
498 : : {
499 : 17069 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
500 : 17069 : ipa_sra_summarize_function (node);
501 : 17069 : pop_cfun ();
502 : : }
503 : : else
504 : 0 : func_sums->remove (node);
505 : 17069 : }
506 : :
507 : : /* Class to manage call summaries. */
508 : :
509 : : class ipa_sra_call_summaries: public call_summary <isra_call_summary *>
510 : : {
511 : : public:
512 : 130209 : ipa_sra_call_summaries (symbol_table *table):
513 : 260418 : call_summary<isra_call_summary *> (table) { }
514 : :
515 : : /* Duplicate info when an edge is cloned. */
516 : : void duplicate (cgraph_edge *, cgraph_edge *,
517 : : isra_call_summary *old_sum,
518 : : isra_call_summary *new_sum) final override;
519 : : };
520 : :
521 : : static ipa_sra_call_summaries *call_sums;
522 : :
523 : :
524 : : /* Initialize m_arg_flow of a particular instance of isra_call_summary.
525 : : ARG_COUNT is the number of actual arguments passed. */
526 : :
527 : : void
528 : 6564359 : isra_call_summary::init_inputs (unsigned arg_count)
529 : : {
530 : 6564359 : if (arg_count == 0)
531 : : {
532 : 304580 : gcc_checking_assert (m_arg_flow.length () == 0);
533 : : return;
534 : : }
535 : 6259779 : if (m_arg_flow.length () == 0)
536 : : {
537 : 3081504 : m_arg_flow.reserve_exact (arg_count);
538 : 3081504 : m_arg_flow.quick_grow_cleared (arg_count);
539 : : }
540 : : else
541 : 3178275 : gcc_checking_assert (arg_count == m_arg_flow.length ());
542 : : }
543 : :
544 : : /* Dump all information in call summary to F. */
545 : :
546 : : void
547 : 180 : isra_call_summary::dump (FILE *f)
548 : : {
549 : 180 : if (m_return_ignored)
550 : 128 : fprintf (f, " return value ignored\n");
551 : 180 : if (m_return_returned)
552 : 38 : fprintf (f, " return value used only to compute caller return value\n");
553 : 180 : if (m_before_any_store)
554 : 17 : fprintf (f, " happens before any store to memory\n");
555 : 409 : for (unsigned i = 0; i < m_arg_flow.length (); i++)
556 : : {
557 : 229 : fprintf (f, " Parameter %u:\n", i);
558 : 229 : isra_param_flow *ipf = &m_arg_flow[i];
559 : :
560 : 229 : if (ipf->length)
561 : : {
562 : 158 : bool first = true;
563 : 158 : fprintf (f, " Scalar param sources: ");
564 : 319 : for (int j = 0; j < ipf->length; j++)
565 : : {
566 : 161 : if (!first)
567 : 3 : fprintf (f, ", ");
568 : : else
569 : : first = false;
570 : 161 : fprintf (f, "%i", (int) ipf->inputs[j]);
571 : : }
572 : 158 : fprintf (f, "\n");
573 : : }
574 : 229 : if (ipf->aggregate_pass_through)
575 : 12 : fprintf (f, " Aggregate pass through from the param given above, "
576 : : "unit offset: %u , unit size: %u\n",
577 : 12 : ipf->unit_offset, ipf->unit_size);
578 : 217 : else if (ipf->unit_size > 0)
579 : 52 : fprintf (f, " Known dereferenceable size: %u\n", ipf->unit_size);
580 : 229 : if (ipf->pointer_pass_through)
581 : 19 : fprintf (f, " Pointer pass through from the param given above, "
582 : 19 : "safe_to_import_accesses: %u\n", ipf->safe_to_import_accesses);
583 : 229 : if (ipf->constructed_for_calls)
584 : 28 : fprintf (f, " Variable constructed just to be passed to "
585 : : "calls.\n");
586 : : }
587 : 180 : }
588 : :
589 : : /* Duplicate edge summary when an edge is cloned. */
590 : :
591 : : void
592 : 567984 : ipa_sra_call_summaries::duplicate (cgraph_edge *, cgraph_edge *,
593 : : isra_call_summary *old_sum,
594 : : isra_call_summary *new_sum)
595 : : {
596 : 567984 : unsigned arg_count = old_sum->m_arg_flow.length ();
597 : 567984 : new_sum->init_inputs (arg_count);
598 : 1701332 : for (unsigned i = 0; i < arg_count; i++)
599 : 1133348 : new_sum->m_arg_flow[i] = old_sum->m_arg_flow[i];
600 : :
601 : 567984 : new_sum->m_return_ignored = old_sum->m_return_ignored;
602 : 567984 : new_sum->m_return_returned = old_sum->m_return_returned;
603 : 567984 : new_sum->m_bit_aligned_arg = old_sum->m_bit_aligned_arg;
604 : 567984 : new_sum->m_before_any_store = old_sum->m_before_any_store;
605 : 567984 : }
606 : :
607 : :
608 : : /* With all GTY stuff done, we can move to anonymous namespace. */
609 : : namespace {
610 : : /* Quick mapping from a decl to its param descriptor. */
611 : :
612 : : hash_map<tree, gensum_param_desc *> *decl2desc;
613 : :
614 : : /* All local DECLs ever loaded from of and of those that have their address
615 : : assigned to a variable. */
616 : :
617 : : hash_set <tree> *loaded_decls;
618 : :
619 : : /* Countdown of allowed Alias Analysis steps during summary building. */
620 : :
621 : : int aa_walking_limit;
622 : :
623 : : /* This is a table in which for each basic block and parameter there is a
624 : : distance (offset + size) in that parameter which is dereferenced and
625 : : accessed in that BB. */
626 : : HOST_WIDE_INT *bb_dereferences = NULL;
627 : : /* How many by-reference parameters there are in the current function. */
628 : : int unsafe_by_ref_count;
629 : :
630 : : /* Bitmap of BBs that can cause the function to "stop" progressing by
631 : : returning, throwing externally, looping infinitely or calling a function
632 : : which might abort etc.. */
633 : : bitmap final_bbs;
634 : :
635 : : /* Obstack to allocate various small structures required only when generating
636 : : summary for a function. */
637 : : struct obstack gensum_obstack;
638 : :
639 : : /* Return false the function is apparently unsuitable for IPA-SRA based on it's
640 : : attributes, return true otherwise. NODE is the cgraph node of the current
641 : : function. */
642 : :
643 : : static bool
644 : 1198159 : ipa_sra_preliminary_function_checks (cgraph_node *node)
645 : : {
646 : 1198159 : if (!node->can_change_signature)
647 : : {
648 : 34707 : if (dump_file)
649 : 0 : fprintf (dump_file, "Function cannot change signature.\n");
650 : 34707 : return false;
651 : : }
652 : :
653 : 1163452 : if (!tree_versionable_function_p (node->decl))
654 : : {
655 : 116024 : if (dump_file)
656 : 7 : fprintf (dump_file, "Function is not versionable.\n");
657 : 116024 : return false;
658 : : }
659 : :
660 : 1047428 : if (!opt_for_fn (node->decl, optimize)
661 : 1047428 : || !opt_for_fn (node->decl, flag_ipa_sra))
662 : : {
663 : 192 : if (dump_file)
664 : 0 : fprintf (dump_file, "Not optimizing or IPA-SRA turned off for this "
665 : : "function.\n");
666 : 192 : return false;
667 : : }
668 : :
669 : 1047236 : if (DECL_VIRTUAL_P (node->decl))
670 : : {
671 : 29493 : if (dump_file)
672 : 0 : fprintf (dump_file, "Function is a virtual method.\n");
673 : 29493 : return false;
674 : : }
675 : :
676 : 1017743 : struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
677 : 1017743 : if (fun->stdarg)
678 : : {
679 : 122 : if (dump_file)
680 : 0 : fprintf (dump_file, "Function uses stdarg. \n");
681 : 122 : return false;
682 : : }
683 : :
684 : 1017621 : if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
685 : : {
686 : 50101 : if (dump_file)
687 : 0 : fprintf (dump_file, "Always inline function will be inlined "
688 : : "anyway. \n");
689 : 50101 : return false;
690 : : }
691 : :
692 : : return true;
693 : : }
694 : :
695 : : /* Print access tree starting at ACCESS to F. */
696 : :
697 : : static void
698 : 69 : dump_gensum_access (FILE *f, gensum_param_access *access, unsigned indent)
699 : : {
700 : 69 : fprintf (f, " ");
701 : 211 : for (unsigned i = 0; i < indent; i++)
702 : 142 : fprintf (f, " ");
703 : 69 : fprintf (f, " * Access to offset: " HOST_WIDE_INT_PRINT_DEC,
704 : : access->offset);
705 : 69 : fprintf (f, ", size: " HOST_WIDE_INT_PRINT_DEC, access->size);
706 : 69 : fprintf (f, ", type: ");
707 : 69 : print_generic_expr (f, access->type);
708 : 69 : fprintf (f, ", alias_ptr_type: ");
709 : 69 : print_generic_expr (f, access->alias_ptr_type);
710 : 69 : fprintf (f, ", load_count: ");
711 : 69 : access->load_count.dump (f);
712 : 69 : fprintf (f, ", nonarg: %u, reverse: %u\n", access->nonarg, access->reverse);
713 : 69 : for (gensum_param_access *ch = access->first_child;
714 : 71 : ch;
715 : 2 : ch = ch->next_sibling)
716 : 2 : dump_gensum_access (f, ch, indent + 2);
717 : 69 : }
718 : :
719 : :
720 : : /* Print access tree starting at ACCESS to F. */
721 : :
722 : : static void
723 : 152 : dump_isra_access (FILE *f, param_access *access)
724 : : {
725 : 152 : fprintf (f, " * Access to unit offset: %u", access->unit_offset);
726 : 152 : fprintf (f, ", unit size: %u", access->unit_size);
727 : 152 : fprintf (f, ", type: ");
728 : 152 : print_generic_expr (f, access->type);
729 : 152 : fprintf (f, ", alias_ptr_type: ");
730 : 152 : print_generic_expr (f, access->alias_ptr_type);
731 : 152 : if (access->certain)
732 : 130 : fprintf (f, ", certain");
733 : : else
734 : 22 : fprintf (f, ", not certain");
735 : 152 : if (access->reverse)
736 : 0 : fprintf (f, ", reverse");
737 : 152 : fprintf (f, "\n");
738 : 152 : }
739 : :
740 : : /* Dump access tree starting at ACCESS to stderr. */
741 : :
742 : : DEBUG_FUNCTION void
743 : 0 : debug_isra_access (param_access *access)
744 : : {
745 : 0 : dump_isra_access (stderr, access);
746 : 0 : }
747 : :
748 : : /* Dump DESC to F. */
749 : :
750 : : static void
751 : 332 : dump_gensum_param_descriptor (FILE *f, gensum_param_desc *desc)
752 : : {
753 : 332 : if (desc->locally_unused)
754 : 166 : fprintf (f, " unused with %i call_uses%s\n", desc->call_uses,
755 : 166 : desc->remove_only_when_retval_removed ?
756 : : " remove_only_when_retval_removed" : "");
757 : 332 : if (!desc->split_candidate)
758 : : {
759 : 244 : fprintf (f, " not a candidate\n");
760 : 244 : return;
761 : : }
762 : 88 : if (desc->by_ref)
763 : 73 : fprintf (f, " %s%s%s by_ref with %u pass throughs\n",
764 : 73 : desc->safe_ref ? "safe" : "unsafe",
765 : 73 : desc->conditionally_dereferenceable
766 : : ? " conditionally_dereferenceable" : "",
767 : 73 : desc->split_only_when_retval_removed
768 : : ? " split_only_when_retval_removed" : "",
769 : : desc->ptr_pt_count);
770 : :
771 : 155 : for (gensum_param_access *acc = desc->accesses; acc; acc = acc->next_sibling)
772 : 67 : dump_gensum_access (f, acc, 2);
773 : : }
774 : :
775 : : /* Dump all parameter descriptors in IFS, assuming it describes FNDECL, to
776 : : F. */
777 : :
778 : : static void
779 : 135 : dump_gensum_param_descriptors (FILE *f, tree fndecl,
780 : : vec<gensum_param_desc> *param_descriptions)
781 : : {
782 : 135 : tree parm = DECL_ARGUMENTS (fndecl);
783 : 135 : for (unsigned i = 0;
784 : 467 : i < param_descriptions->length ();
785 : 332 : ++i, parm = DECL_CHAIN (parm))
786 : : {
787 : 332 : fprintf (f, " Descriptor for parameter %i ", i);
788 : 332 : print_generic_expr (f, parm, TDF_UID);
789 : 332 : fprintf (f, "\n");
790 : 332 : dump_gensum_param_descriptor (f, &(*param_descriptions)[i]);
791 : : }
792 : 135 : }
793 : :
794 : :
795 : : /* Dump DESC to F. If HINTS is true, also dump IPA-analysis computed
796 : : hints. */
797 : :
798 : : static void
799 : 646 : dump_isra_param_descriptor (FILE *f, isra_param_desc *desc, bool hints)
800 : : {
801 : 646 : if (desc->locally_unused)
802 : : {
803 : 335 : fprintf (f, " (locally) unused\n");
804 : : }
805 : 646 : if (!desc->split_candidate)
806 : : {
807 : 471 : fprintf (f, " not a candidate for splitting");
808 : 471 : if (hints && desc->by_ref && desc->safe_size_set)
809 : 9 : fprintf (f, ", safe_size: %u", (unsigned) desc->safe_size);
810 : 471 : fprintf (f, "\n");
811 : 471 : return;
812 : : }
813 : 350 : fprintf (f, " param_size_limit: %u, size_reached: %u%s",
814 : 175 : desc->param_size_limit, desc->size_reached,
815 : 175 : desc->by_ref ? ", by_ref" : "");
816 : 175 : if (desc->remove_only_when_retval_removed)
817 : 12 : fprintf (f, ", remove_only_when_retval_removed");
818 : 175 : if (desc->split_only_when_retval_removed)
819 : 4 : fprintf (f, ", split_only_when_retval_removed");
820 : 175 : if (desc->by_ref && desc->conditionally_dereferenceable)
821 : 33 : fprintf (f, ", conditionally_dereferenceable");
822 : 175 : if (hints)
823 : : {
824 : 6 : if (desc->by_ref && !desc->not_specially_constructed)
825 : 3 : fprintf (f, ", args_specially_constructed");
826 : 6 : if (desc->by_ref && desc->safe_size_set)
827 : 5 : fprintf (f, ", safe_size: %u", (unsigned) desc->safe_size);
828 : : }
829 : 175 : fprintf (f, "\n");
830 : :
831 : 327 : for (unsigned i = 0; i < vec_safe_length (desc->accesses); ++i)
832 : : {
833 : 152 : param_access *access = (*desc->accesses)[i];
834 : 152 : dump_isra_access (f, access);
835 : : }
836 : : }
837 : :
838 : : /* Dump all parameter descriptors in IFS, assuming it describes FNDECL, to F.
839 : : If HINTS is true, also dump IPA-analysis computed hints. */
840 : :
841 : : static void
842 : 135 : dump_isra_param_descriptors (FILE *f, tree fndecl, isra_func_summary *ifs,
843 : : bool hints)
844 : : {
845 : 135 : tree parm = DECL_ARGUMENTS (fndecl);
846 : 135 : if (!ifs->m_parameters)
847 : : {
848 : 0 : fprintf (f, " parameter descriptors not available\n");
849 : 0 : return;
850 : : }
851 : :
852 : : for (unsigned i = 0;
853 : 467 : i < ifs->m_parameters->length ();
854 : 332 : ++i, parm = DECL_CHAIN (parm))
855 : : {
856 : 332 : fprintf (f, " Descriptor for parameter %i ", i);
857 : 332 : print_generic_expr (f, parm, TDF_UID);
858 : 332 : fprintf (f, "\n");
859 : 332 : dump_isra_param_descriptor (f, &(*ifs->m_parameters)[i], hints);
860 : : }
861 : : }
862 : :
863 : : /* Add SRC to inputs of PARAM_FLOW, unless it would exceed storage. If the
864 : : function fails return false, otherwise return true. SRC must fit into an
865 : : unsigned char. Used for purposes of transitive unused parameter
866 : : removal. */
867 : :
868 : : static bool
869 : 1266378 : add_src_to_param_flow (isra_param_flow *param_flow, int src)
870 : : {
871 : 1266378 : gcc_checking_assert (src >= 0 && src <= UCHAR_MAX);
872 : 1266378 : if (param_flow->length == IPA_SRA_MAX_PARAM_FLOW_LEN)
873 : : return false;
874 : :
875 : 1266375 : param_flow->inputs[(int) param_flow->length] = src;
876 : 1266375 : param_flow->length++;
877 : 1266375 : return true;
878 : : }
879 : :
880 : : /* Add a SRC to the inputs of PARAM_FLOW unless it is already there and assert
881 : : it is the only input. Used for purposes of transitive parameter
882 : : splitting. */
883 : :
884 : : static void
885 : 771601 : set_single_param_flow_source (isra_param_flow *param_flow, int src)
886 : : {
887 : 771601 : gcc_checking_assert (src >= 0 && src <= UCHAR_MAX);
888 : 771601 : if (param_flow->length == 0)
889 : : {
890 : 116226 : param_flow->inputs[0] = src;
891 : 116226 : param_flow->length = 1;
892 : : }
893 : 655375 : else if (param_flow->length == 1)
894 : 655375 : gcc_assert (param_flow->inputs[0] == src);
895 : : else
896 : 0 : gcc_unreachable ();
897 : 771601 : }
898 : :
899 : : /* Assert that there is only a single value in PARAM_FLOW's inputs and return
900 : : it. */
901 : :
902 : : static unsigned
903 : 1272123 : get_single_param_flow_source (const isra_param_flow *param_flow)
904 : : {
905 : 1272123 : gcc_assert (param_flow->length == 1);
906 : 1272123 : return param_flow->inputs[0];
907 : : }
908 : :
909 : : /* Inspect all uses of NAME and simple arithmetic calculations involving NAME
910 : : in FUN represented with NODE and return a negative number if any of them is
911 : : used for something else than either an actual call argument, simple return,
912 : : simple arithmetic operation or debug statement. If there are no such uses,
913 : : return the number of actual arguments that this parameter eventually feeds
914 : : to (or zero if there is none). If there are any simple return uses, set
915 : : DESC->remove_only_when_retval_removed. For any such parameter, mark
916 : : PARM_NUM as one of its sources. ANALYZED is a bitmap that tracks which SSA
917 : : names we have already started investigating. */
918 : :
919 : : static int
920 : 3204422 : isra_track_scalar_value_uses (function *fun, cgraph_node *node, tree name,
921 : : int parm_num, bitmap analyzed,
922 : : gensum_param_desc *desc)
923 : : {
924 : 3204422 : int res = 0;
925 : 3204422 : imm_use_iterator imm_iter;
926 : 3204422 : gimple *stmt;
927 : :
928 : 6547225 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
929 : : {
930 : 4845232 : if (is_gimple_debug (stmt)
931 : 4845232 : || gimple_clobber_p (stmt))
932 : 734542 : continue;
933 : :
934 : : /* TODO: We could handle at least const builtin functions like arithmetic
935 : : operations below. */
936 : 4110690 : if (is_gimple_call (stmt))
937 : : {
938 : 1276849 : int all_uses = 0;
939 : 1276849 : use_operand_p use_p;
940 : 2567423 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
941 : 1290574 : all_uses++;
942 : :
943 : 1276849 : gcall *call = as_a <gcall *> (stmt);
944 : 1276849 : unsigned arg_count;
945 : 1276849 : if (gimple_call_internal_p (call)
946 : 1276849 : || (arg_count = gimple_call_num_args (call)) == 0)
947 : : {
948 : : res = -1;
949 : : break;
950 : : }
951 : :
952 : 1272473 : cgraph_edge *cs = node->get_edge (stmt);
953 : 1272473 : gcc_checking_assert (cs);
954 : 1272473 : isra_call_summary *csum = call_sums->get_create (cs);
955 : 1272473 : csum->init_inputs (arg_count);
956 : :
957 : 1272473 : int simple_uses = 0;
958 : 5867026 : for (unsigned i = 0; i < arg_count; i++)
959 : 4594556 : if (gimple_call_arg (call, i) == name)
960 : : {
961 : 1266378 : if (!add_src_to_param_flow (&csum->m_arg_flow[i], parm_num))
962 : : {
963 : : simple_uses = -1;
964 : : break;
965 : : }
966 : 1266375 : simple_uses++;
967 : : }
968 : :
969 : 1272473 : if (simple_uses < 0
970 : 1272473 : || all_uses != simple_uses)
971 : : {
972 : : res = -1;
973 : : break;
974 : : }
975 : 1253029 : res += all_uses;
976 : : }
977 : 2833841 : else if (!stmt_unremovable_because_of_non_call_eh_p (fun, stmt)
978 : 2833841 : && ((is_gimple_assign (stmt) && !gimple_has_volatile_ops (stmt))
979 : 626696 : || gimple_code (stmt) == GIMPLE_PHI))
980 : : {
981 : 2259450 : tree lhs;
982 : 2259450 : if (gimple_code (stmt) == GIMPLE_PHI)
983 : 159383 : lhs = gimple_phi_result (stmt);
984 : : else
985 : 2100067 : lhs = gimple_assign_lhs (stmt);
986 : :
987 : 2259450 : if (TREE_CODE (lhs) != SSA_NAME)
988 : : {
989 : : res = -1;
990 : : break;
991 : : }
992 : 1911841 : gcc_assert (!gimple_vdef (stmt));
993 : 1911841 : if (bitmap_set_bit (analyzed, SSA_NAME_VERSION (lhs)))
994 : : {
995 : 1822083 : int tmp = isra_track_scalar_value_uses (fun, node, lhs, parm_num,
996 : : analyzed, desc);
997 : 1822083 : if (tmp < 0)
998 : : {
999 : : res = tmp;
1000 : : break;
1001 : : }
1002 : 1107344 : res += tmp;
1003 : : }
1004 : : }
1005 : 574391 : else if (greturn *gr = dyn_cast<greturn *>(stmt))
1006 : : {
1007 : 158130 : tree rv = gimple_return_retval (gr);
1008 : 158130 : if (rv != name)
1009 : : {
1010 : : res = -1;
1011 : : break;
1012 : : }
1013 : 158130 : desc->remove_only_when_retval_removed = true;
1014 : : }
1015 : : else
1016 : : {
1017 : : res = -1;
1018 : : break;
1019 : : }
1020 : 3204422 : }
1021 : 3204422 : return res;
1022 : : }
1023 : :
1024 : : /* Inspect all uses of PARM, which must be a gimple register, in FUN (which is
1025 : : also described by NODE) and simple arithmetic calculations involving PARM
1026 : : and return false if any of them is used for something else than either an
1027 : : actual call argument, simple return, simple arithmetic operation or debug
1028 : : statement. If there are no such uses, return true and store the number of
1029 : : actual arguments that this parameter eventually feeds to (or zero if there
1030 : : is none) to DESC->call_uses and set DESC->remove_only_when_retval_removed if
1031 : : there are any uses in return statemens. For any such parameter, mark
1032 : : PARM_NUM as one of its sources.
1033 : :
1034 : : This function is similar to ptr_parm_has_nonarg_uses but its results are
1035 : : meant for unused parameter removal, as opposed to splitting of parameters
1036 : : passed by reference or converting them to passed by value. */
1037 : :
1038 : : static bool
1039 : 2059560 : isra_track_scalar_param_local_uses (function *fun, cgraph_node *node, tree parm,
1040 : : int parm_num, gensum_param_desc *desc)
1041 : : {
1042 : 2059560 : gcc_checking_assert (is_gimple_reg (parm));
1043 : :
1044 : 2059560 : tree name = ssa_default_def (fun, parm);
1045 : 2059560 : if (!name || has_zero_uses (name))
1046 : : {
1047 : 677221 : desc->call_uses = 0;
1048 : 677221 : return false;
1049 : : }
1050 : :
1051 : : /* Edge summaries can only handle callers with fewer than 256 parameters. */
1052 : 1382339 : if (parm_num > UCHAR_MAX)
1053 : : return true;
1054 : :
1055 : 1382339 : bitmap analyzed = BITMAP_ALLOC (NULL);
1056 : 1382339 : int call_uses = isra_track_scalar_value_uses (fun, node, name, parm_num,
1057 : : analyzed, desc);
1058 : 1382339 : BITMAP_FREE (analyzed);
1059 : 1382339 : if (call_uses < 0)
1060 : : return true;
1061 : 594649 : desc->call_uses = call_uses;
1062 : 594649 : return false;
1063 : : }
1064 : :
1065 : : /* Scan immediate uses of a default definition SSA name of a parameter PARM and
1066 : : examine whether there are any nonarg uses that are not actual arguments or
1067 : : otherwise infeasible uses. If so, return true, otherwise return false.
1068 : : Create pass-through IPA flow records for any direct uses as argument calls
1069 : : and if returning false, store their number into DESC->ptr_pt_count. If
1070 : : removal of return value would still allow splitting, return true but set
1071 : : DESC->split_only_when_retval_removed. NODE and FUN must represent the
1072 : : function that is currently analyzed, PARM_NUM must be the index of the
1073 : : analyzed parameter.
1074 : :
1075 : : This function is similar to isra_track_scalar_param_local_uses but its
1076 : : results are meant for splitting of parameters passed by reference or turning
1077 : : them into bits passed by value, as opposed to generic unused parameter
1078 : : removal. */
1079 : :
1080 : : static bool
1081 : 1070956 : ptr_parm_has_nonarg_uses (cgraph_node *node, function *fun, tree parm,
1082 : : int parm_num, gensum_param_desc *desc)
1083 : : {
1084 : 1070956 : imm_use_iterator ui;
1085 : 1070956 : gimple *stmt;
1086 : 1070956 : tree name = ssa_default_def (fun, parm);
1087 : 1070956 : bool ret = false;
1088 : 1070956 : unsigned pt_count = 0;
1089 : :
1090 : 1070956 : if (!name || has_zero_uses (name))
1091 : : return false;
1092 : :
1093 : : /* Edge summaries can only handle callers with fewer than 256 parameters. */
1094 : 972292 : if (parm_num > UCHAR_MAX)
1095 : : return true;
1096 : :
1097 : 2646627 : FOR_EACH_IMM_USE_STMT (stmt, ui, name)
1098 : : {
1099 : 2165748 : unsigned uses_ok = 0;
1100 : 2165748 : use_operand_p use_p;
1101 : :
1102 : 2165748 : if (is_gimple_debug (stmt)
1103 : 2165748 : || gimple_clobber_p (stmt))
1104 : 506551 : continue;
1105 : :
1106 : 1659197 : if (gimple_assign_single_p (stmt))
1107 : : {
1108 : 816087 : tree rhs = gimple_assign_rhs1 (stmt);
1109 : 816087 : if (!TREE_THIS_VOLATILE (rhs))
1110 : : {
1111 : 1572741 : while (handled_component_p (rhs))
1112 : 756726 : rhs = TREE_OPERAND (rhs, 0);
1113 : 816015 : if (TREE_CODE (rhs) == MEM_REF
1114 : 571537 : && TREE_OPERAND (rhs, 0) == name
1115 : 567956 : && integer_zerop (TREE_OPERAND (rhs, 1))
1116 : 1343370 : && types_compatible_p (TREE_TYPE (rhs),
1117 : 527355 : TREE_TYPE (TREE_TYPE (name))))
1118 : : uses_ok++;
1119 : : }
1120 : : }
1121 : 843110 : else if (is_gimple_call (stmt))
1122 : : {
1123 : 702817 : gcall *call = as_a <gcall *> (stmt);
1124 : 702817 : unsigned arg_count;
1125 : 702817 : if (gimple_call_internal_p (call)
1126 : 702817 : || (arg_count = gimple_call_num_args (call)) == 0)
1127 : : {
1128 : : ret = true;
1129 : : break;
1130 : : }
1131 : :
1132 : 702080 : cgraph_edge *cs = node->get_edge (stmt);
1133 : 702080 : gcc_checking_assert (cs);
1134 : 702080 : isra_call_summary *csum = call_sums->get_create (cs);
1135 : 702080 : csum->init_inputs (arg_count);
1136 : :
1137 : 2712309 : for (unsigned i = 0; i < arg_count; ++i)
1138 : : {
1139 : 2010229 : tree arg = gimple_call_arg (stmt, i);
1140 : :
1141 : 2010229 : if (arg == name)
1142 : : {
1143 : : /* TODO: Allow &MEM_REF[name + offset] here,
1144 : : ipa_param_body_adjustments::modify_call_stmt has to be
1145 : : adjusted too. */
1146 : 692555 : csum->m_arg_flow[i].pointer_pass_through = true;
1147 : 692555 : set_single_param_flow_source (&csum->m_arg_flow[i], parm_num);
1148 : 692555 : pt_count++;
1149 : 692555 : uses_ok++;
1150 : 692555 : continue;
1151 : : }
1152 : :
1153 : 1317674 : if (!TREE_THIS_VOLATILE (arg))
1154 : : {
1155 : 1340557 : while (handled_component_p (arg))
1156 : 22883 : arg = TREE_OPERAND (arg, 0);
1157 : 1317674 : if (TREE_CODE (arg) == MEM_REF
1158 : 17622 : && TREE_OPERAND (arg, 0) == name
1159 : 9328 : && integer_zerop (TREE_OPERAND (arg, 1))
1160 : 1327002 : && types_compatible_p (TREE_TYPE (arg),
1161 : 9328 : TREE_TYPE (TREE_TYPE (name))))
1162 : 6525 : uses_ok++;
1163 : : }
1164 : : }
1165 : : }
1166 : 140293 : else if (greturn *gr = dyn_cast<greturn *>(stmt))
1167 : : {
1168 : 7028 : tree rv = gimple_return_retval (gr);
1169 : 7028 : if (rv == name)
1170 : : {
1171 : 7028 : uses_ok++;
1172 : : /* Analysis for feasibility of removal must have already reached
1173 : : the conclusion that the flag must be set if it completed. */
1174 : 7028 : gcc_assert (!desc->locally_unused
1175 : : || desc->remove_only_when_retval_removed);
1176 : 7028 : desc->split_only_when_retval_removed = true;
1177 : : }
1178 : : }
1179 : :
1180 : : /* If the number of valid uses does not match the number of
1181 : : uses in this stmt there is an unhandled use. */
1182 : 1658460 : unsigned all_uses = 0;
1183 : 3324015 : FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
1184 : 1665555 : all_uses++;
1185 : :
1186 : 1658460 : gcc_checking_assert (uses_ok <= all_uses);
1187 : 1658460 : if (uses_ok != all_uses)
1188 : : {
1189 : : ret = true;
1190 : : break;
1191 : : }
1192 : 972292 : }
1193 : :
1194 : 972292 : desc->ptr_pt_count = pt_count;
1195 : 972292 : return ret;
1196 : : }
1197 : :
1198 : : /* Initialize vector of parameter descriptors of NODE. Return true if there
1199 : : are any candidates for splitting or unused aggregate parameter removal (the
1200 : : function may return false if there are candidates for removal of register
1201 : : parameters). */
1202 : :
1203 : : static bool
1204 : 781674 : create_parameter_descriptors (cgraph_node *node,
1205 : : vec<gensum_param_desc> *param_descriptions)
1206 : : {
1207 : 781674 : function *fun = DECL_STRUCT_FUNCTION (node->decl);
1208 : 781674 : bool ret = false;
1209 : :
1210 : 781674 : int num = 0;
1211 : 781674 : for (tree parm = DECL_ARGUMENTS (node->decl);
1212 : 3026606 : parm;
1213 : 2244932 : parm = DECL_CHAIN (parm), num++)
1214 : : {
1215 : 2244932 : const char *msg;
1216 : 2244932 : gensum_param_desc *desc = &(*param_descriptions)[num];
1217 : : /* param_descriptions vector is grown cleared in the caller. */
1218 : 2244932 : desc->param_number = num;
1219 : 2244932 : decl2desc->put (parm, desc);
1220 : :
1221 : 2244932 : if (dump_file && (dump_flags & TDF_DETAILS))
1222 : 39 : print_generic_expr (dump_file, parm, TDF_UID);
1223 : :
1224 : 2244932 : tree type = TREE_TYPE (parm);
1225 : 2244932 : if (TREE_THIS_VOLATILE (parm))
1226 : : {
1227 : 19 : if (dump_file && (dump_flags & TDF_DETAILS))
1228 : 0 : fprintf (dump_file, " not a candidate, is volatile\n");
1229 : 1688302 : continue;
1230 : : }
1231 : 2244913 : if (!is_gimple_reg_type (type) && is_va_list_type (type))
1232 : : {
1233 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1234 : 0 : fprintf (dump_file, " not a candidate, is a va_list type\n");
1235 : 0 : continue;
1236 : : }
1237 : 2244913 : if (TREE_ADDRESSABLE (parm))
1238 : : {
1239 : 30130 : if (dump_file && (dump_flags & TDF_DETAILS))
1240 : 0 : fprintf (dump_file, " not a candidate, is addressable\n");
1241 : 30130 : continue;
1242 : : }
1243 : 2214783 : if (TREE_ADDRESSABLE (type))
1244 : : {
1245 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1246 : 0 : fprintf (dump_file, " not a candidate, type cannot be split\n");
1247 : 0 : continue;
1248 : : }
1249 : :
1250 : 2214783 : if (is_gimple_reg (parm)
1251 : 2214783 : && !isra_track_scalar_param_local_uses (fun, node, parm, num, desc))
1252 : : {
1253 : 1271870 : desc->locally_unused = true;
1254 : :
1255 : 1271870 : if (dump_file && (dump_flags & TDF_DETAILS))
1256 : 19 : fprintf (dump_file, " is a scalar with only %i call uses%s\n",
1257 : : desc->call_uses,
1258 : 19 : desc->remove_only_when_retval_removed
1259 : : ? " and return uses" : "");
1260 : : }
1261 : :
1262 : 2214783 : if (POINTER_TYPE_P (type))
1263 : : {
1264 : 1077047 : desc->by_ref = true;
1265 : 1077047 : if (TREE_CODE (type) == REFERENCE_TYPE
1266 : 1077047 : || (num == 0
1267 : 524182 : && TREE_CODE (TREE_TYPE (node->decl)) == METHOD_TYPE))
1268 : 439322 : desc->safe_ref = true;
1269 : : else
1270 : 637725 : desc->safe_ref = false;
1271 : 1077047 : type = TREE_TYPE (type);
1272 : :
1273 : 1081709 : if (TREE_CODE (type) == FUNCTION_TYPE
1274 : 1077047 : || TREE_CODE (type) == METHOD_TYPE)
1275 : : {
1276 : 4662 : if (dump_file && (dump_flags & TDF_DETAILS))
1277 : 0 : fprintf (dump_file, " not a candidate, reference to "
1278 : : "a function\n");
1279 : 4662 : continue;
1280 : : }
1281 : 1072385 : if (TYPE_VOLATILE (type))
1282 : : {
1283 : 1050 : if (dump_file && (dump_flags & TDF_DETAILS))
1284 : 0 : fprintf (dump_file, " not a candidate, reference to "
1285 : : "a volatile type\n");
1286 : 1050 : continue;
1287 : : }
1288 : 1071335 : if (TREE_CODE (type) == ARRAY_TYPE
1289 : 1071335 : && TYPE_NONALIASED_COMPONENT (type))
1290 : : {
1291 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
1292 : 0 : fprintf (dump_file, " not a candidate, reference to "
1293 : : "a nonaliased component array\n");
1294 : 0 : continue;
1295 : : }
1296 : 1071335 : if (!is_gimple_reg (parm))
1297 : : {
1298 : 6 : if (dump_file && (dump_flags & TDF_DETAILS))
1299 : 0 : fprintf (dump_file, " not a candidate, a reference which is "
1300 : : "not a gimple register (probably addressable)\n");
1301 : 6 : continue;
1302 : : }
1303 : 1071329 : if (is_va_list_type (type))
1304 : : {
1305 : 373 : if (dump_file && (dump_flags & TDF_DETAILS))
1306 : 0 : fprintf (dump_file, " not a candidate, reference to "
1307 : : "a va list\n");
1308 : 373 : continue;
1309 : : }
1310 : 1070956 : if (ptr_parm_has_nonarg_uses (node, fun, parm, num, desc))
1311 : : {
1312 : 491413 : if (dump_file && (dump_flags & TDF_DETAILS))
1313 : 7 : fprintf (dump_file, " not a candidate, reference has "
1314 : : "nonarg uses\n");
1315 : 491413 : continue;
1316 : : }
1317 : : }
1318 : 1137736 : else if (!AGGREGATE_TYPE_P (type))
1319 : : {
1320 : : /* This is in an else branch because scalars passed by reference are
1321 : : still candidates to be passed by value. */
1322 : 982702 : if (dump_file && (dump_flags & TDF_DETAILS))
1323 : 17 : fprintf (dump_file, " not a candidate, not an aggregate\n");
1324 : 982702 : continue;
1325 : : }
1326 : :
1327 : 734577 : if (!COMPLETE_TYPE_P (type))
1328 : : {
1329 : 165706 : if (dump_file && (dump_flags & TDF_DETAILS))
1330 : 0 : fprintf (dump_file, " not a candidate, not a complete type\n");
1331 : 165706 : continue;
1332 : : }
1333 : 568871 : if (!tree_fits_uhwi_p (TYPE_SIZE (type)))
1334 : : {
1335 : 3 : if (dump_file && (dump_flags & TDF_DETAILS))
1336 : 0 : fprintf (dump_file, " not a candidate, size not representable\n");
1337 : 3 : continue;
1338 : : }
1339 : 568868 : unsigned HOST_WIDE_INT type_size
1340 : 568868 : = tree_to_uhwi (TYPE_SIZE (type)) / BITS_PER_UNIT;
1341 : 572129 : if (type_size == 0
1342 : 568868 : || type_size >= ISRA_ARG_SIZE_LIMIT)
1343 : : {
1344 : 3261 : if (dump_file && (dump_flags & TDF_DETAILS))
1345 : 0 : fprintf (dump_file, " not a candidate, has zero or huge size\n");
1346 : 3261 : continue;
1347 : : }
1348 : 565607 : if (type_internals_preclude_sra_p (type, &msg))
1349 : : {
1350 : 8977 : if (dump_file && (dump_flags & TDF_DETAILS))
1351 : 0 : fprintf (dump_file, " not a candidate, %s\n", msg);
1352 : 8977 : continue;
1353 : : }
1354 : :
1355 : 556630 : if (dump_file && (dump_flags & TDF_DETAILS))
1356 : 15 : fprintf (dump_file, " is a candidate\n");
1357 : :
1358 : 556630 : ret = true;
1359 : 556630 : desc->split_candidate = true;
1360 : 556630 : if (desc->by_ref && !desc->safe_ref)
1361 : 185495 : desc->deref_index = unsafe_by_ref_count++;
1362 : : }
1363 : 781674 : return ret;
1364 : : }
1365 : :
1366 : : /* Return pointer to descriptor of parameter DECL or NULL if it cannot be
1367 : : found, which happens if DECL is for a static chain. */
1368 : :
1369 : : static gensum_param_desc *
1370 : 2237899 : get_gensum_param_desc (tree decl)
1371 : : {
1372 : 2237899 : if (!decl2desc)
1373 : : return NULL;
1374 : 2069902 : gcc_checking_assert (TREE_CODE (decl) == PARM_DECL);
1375 : 2069902 : gensum_param_desc **slot = decl2desc->get (decl);
1376 : 2069902 : if (!slot)
1377 : : /* This can happen for static chains which we cannot handle so far. */
1378 : : return NULL;
1379 : 2044641 : gcc_checking_assert (*slot);
1380 : : return *slot;
1381 : : }
1382 : :
1383 : :
1384 : : /* Remove parameter described by DESC from candidates for IPA-SRA splitting and
1385 : : write REASON to the dump file if there is one. */
1386 : :
1387 : : static void
1388 : 145554 : disqualify_split_candidate (gensum_param_desc *desc, const char *reason)
1389 : : {
1390 : 145554 : if (!desc->split_candidate)
1391 : : return;
1392 : :
1393 : 145525 : if (dump_file && (dump_flags & TDF_DETAILS))
1394 : 0 : fprintf (dump_file, "! Disqualifying parameter number %i - %s\n",
1395 : : desc->param_number, reason);
1396 : :
1397 : 145525 : desc->split_candidate = false;
1398 : : }
1399 : :
1400 : : /* Remove DECL from candidates for IPA-SRA and write REASON to the dump file if
1401 : : there is one. */
1402 : :
1403 : : static void
1404 : 177 : disqualify_split_candidate (tree decl, const char *reason)
1405 : : {
1406 : 177 : gensum_param_desc *desc = get_gensum_param_desc (decl);
1407 : 177 : if (desc)
1408 : 29 : disqualify_split_candidate (desc, reason);
1409 : 177 : }
1410 : :
1411 : : /* Allocate a new access to DESC and fill it in with OFFSET and SIZE. But
1412 : : first, check that there are not too many of them already. If so, do not
1413 : : allocate anything and return NULL. */
1414 : :
1415 : : static gensum_param_access *
1416 : 406229 : allocate_access (gensum_param_desc *desc,
1417 : : HOST_WIDE_INT offset, HOST_WIDE_INT size)
1418 : : {
1419 : 406229 : if (desc->access_count
1420 : 406229 : == (unsigned) param_ipa_sra_max_replacements)
1421 : : {
1422 : 0 : disqualify_split_candidate (desc, "Too many replacement candidates");
1423 : 0 : return NULL;
1424 : : }
1425 : :
1426 : 406229 : gensum_param_access *access
1427 : 406229 : = (gensum_param_access *) obstack_alloc (&gensum_obstack,
1428 : : sizeof (gensum_param_access));
1429 : 406229 : memset (access, 0, sizeof (*access));
1430 : 406229 : access->offset = offset;
1431 : 406229 : access->size = size;
1432 : 406229 : access->load_count = profile_count::zero ();
1433 : 406229 : return access;
1434 : : }
1435 : :
1436 : : /* In what context scan_expr_access has been called, whether it deals with a
1437 : : load, a function argument, or a store. Please note that in rare
1438 : : circumstances when it is not clear if the access is a load or store,
1439 : : ISRA_CTX_STORE is used too. */
1440 : :
1441 : : enum isra_scan_context {ISRA_CTX_LOAD, ISRA_CTX_ARG, ISRA_CTX_STORE};
1442 : :
1443 : : /* Return an access describing memory access to the variable described by DESC
1444 : : at OFFSET with SIZE in context CTX, starting at pointer to the linked list
1445 : : at a certain tree level FIRST. Attempt to create it and put into the
1446 : : appropriate place in the access tree if does not exist, but fail and return
1447 : : NULL if there are already too many accesses, if it would create a partially
1448 : : overlapping access or if an access would end up within a pre-existing
1449 : : non-call access. */
1450 : :
1451 : : static gensum_param_access *
1452 : 474010 : get_access_1 (gensum_param_desc *desc, gensum_param_access **first,
1453 : : HOST_WIDE_INT offset, HOST_WIDE_INT size, isra_scan_context ctx)
1454 : : {
1455 : 484596 : gensum_param_access *access = *first, **ptr = first;
1456 : :
1457 : 484596 : if (!access)
1458 : : {
1459 : : /* No pre-existing access at this level, just create it. */
1460 : 263532 : gensum_param_access *a = allocate_access (desc, offset, size);
1461 : 263532 : if (!a)
1462 : : return NULL;
1463 : 263532 : *first = a;
1464 : 263532 : return *first;
1465 : : }
1466 : :
1467 : 221064 : if (access->offset >= offset + size)
1468 : : {
1469 : : /* We want to squeeze it in front of the very first access, just do
1470 : : it. */
1471 : 40365 : gensum_param_access *r = allocate_access (desc, offset, size);
1472 : 40365 : if (!r)
1473 : : return NULL;
1474 : 40365 : r->next_sibling = access;
1475 : 40365 : *first = r;
1476 : 40365 : return r;
1477 : : }
1478 : :
1479 : : /* Skip all accesses that have to come before us until the next sibling is
1480 : : already too far. */
1481 : 264523 : while (offset >= access->offset + access->size
1482 : 175907 : && access->next_sibling
1483 : 358336 : && access->next_sibling->offset < offset + size)
1484 : : {
1485 : 83824 : ptr = &access->next_sibling;
1486 : 83824 : access = access->next_sibling;
1487 : : }
1488 : :
1489 : : /* At this point we know we do not belong before access. */
1490 : 180699 : gcc_assert (access->offset < offset + size);
1491 : :
1492 : 180699 : if (access->offset == offset && access->size == size)
1493 : : /* We found what we were looking for. */
1494 : : return access;
1495 : :
1496 : 117759 : if (access->offset <= offset
1497 : 114804 : && access->offset + access->size >= offset + size)
1498 : : {
1499 : : /* We fit into access which is larger than us. We need to find/create
1500 : : something below access. But we only allow nesting in call
1501 : : arguments. */
1502 : 12219 : if (access->nonarg)
1503 : : return NULL;
1504 : :
1505 : 10586 : return get_access_1 (desc, &access->first_child, offset, size, ctx);
1506 : : }
1507 : :
1508 : 105540 : if (offset <= access->offset
1509 : 13457 : && offset + size >= access->offset + access->size)
1510 : : /* We are actually bigger than access, which fully fits into us, take its
1511 : : place and make all accesses fitting into it its children. */
1512 : : {
1513 : : /* But first, we only allow nesting in call arguments so check if that is
1514 : : what we are trying to represent. */
1515 : 13457 : if (ctx != ISRA_CTX_ARG)
1516 : : return NULL;
1517 : :
1518 : 10249 : gensum_param_access *r = allocate_access (desc, offset, size);
1519 : 10249 : if (!r)
1520 : : return NULL;
1521 : 10249 : r->first_child = access;
1522 : :
1523 : 10249 : while (access->next_sibling
1524 : 18688 : && access->next_sibling->offset < offset + size)
1525 : : access = access->next_sibling;
1526 : 10249 : if (access->offset + access->size > offset + size)
1527 : : {
1528 : : /* This must be a different access, which are sorted, so the
1529 : : following must be true and this signals a partial overlap. */
1530 : 0 : gcc_assert (access->offset > offset);
1531 : : return NULL;
1532 : : }
1533 : :
1534 : 10249 : r->next_sibling = access->next_sibling;
1535 : 10249 : access->next_sibling = NULL;
1536 : 10249 : *ptr = r;
1537 : 10249 : return r;
1538 : : }
1539 : :
1540 : 92083 : if (offset >= access->offset + access->size)
1541 : : {
1542 : : /* We belong after access. */
1543 : 92083 : gensum_param_access *r = allocate_access (desc, offset, size);
1544 : 92083 : if (!r)
1545 : : return NULL;
1546 : 92083 : r->next_sibling = access->next_sibling;
1547 : 92083 : access->next_sibling = r;
1548 : 92083 : return r;
1549 : : }
1550 : :
1551 : 0 : if (offset < access->offset)
1552 : : {
1553 : : /* We know the following, otherwise we would have created a
1554 : : super-access. */
1555 : 0 : gcc_checking_assert (offset + size < access->offset + access->size);
1556 : : return NULL;
1557 : : }
1558 : :
1559 : 0 : if (offset + size > access->offset + access->size)
1560 : : {
1561 : : /* Likewise. */
1562 : 0 : gcc_checking_assert (offset > access->offset);
1563 : : return NULL;
1564 : : }
1565 : :
1566 : 0 : gcc_unreachable ();
1567 : : }
1568 : :
1569 : : /* Return an access describing memory access to the variable described by DESC
1570 : : at OFFSET with SIZE in context CTX, mark it as used in context CTX. Attempt
1571 : : to create if it does not exist, but fail and return NULL if there are
1572 : : already too many accesses, if it would create a partially overlapping access
1573 : : or if an access would end up in a non-call access. */
1574 : :
1575 : : static gensum_param_access *
1576 : 474010 : get_access (gensum_param_desc *desc, HOST_WIDE_INT offset, HOST_WIDE_INT size,
1577 : : isra_scan_context ctx)
1578 : : {
1579 : 474010 : gcc_checking_assert (desc->split_candidate);
1580 : :
1581 : 474010 : gensum_param_access *access = get_access_1 (desc, &desc->accesses, offset,
1582 : : size, ctx);
1583 : 474010 : if (!access)
1584 : : {
1585 : 4841 : disqualify_split_candidate (desc,
1586 : : "Bad access overlap or too many accesses");
1587 : 4841 : return NULL;
1588 : : }
1589 : :
1590 : 469169 : switch (ctx)
1591 : : {
1592 : 13408 : case ISRA_CTX_STORE:
1593 : 13408 : gcc_assert (!desc->by_ref);
1594 : : /* Fall-through */
1595 : 387509 : case ISRA_CTX_LOAD:
1596 : 387509 : access->nonarg = true;
1597 : 387509 : break;
1598 : : case ISRA_CTX_ARG:
1599 : : break;
1600 : : }
1601 : :
1602 : : return access;
1603 : : }
1604 : :
1605 : : /* Verify that parameter access tree starting with ACCESS is in good shape.
1606 : : PARENT_OFFSET and PARENT_SIZE are the corresponding fields of parent of
1607 : : ACCESS or zero if there is none. */
1608 : :
1609 : : static bool
1610 : 849052 : verify_access_tree_1 (gensum_param_access *access, HOST_WIDE_INT parent_offset,
1611 : : HOST_WIDE_INT parent_size)
1612 : : {
1613 : 1211516 : while (access)
1614 : : {
1615 : 362464 : gcc_assert (access->offset >= 0 && access->size >= 0);
1616 : :
1617 : 362464 : if (parent_size != 0)
1618 : : {
1619 : 21560 : if (access->offset < parent_offset)
1620 : : {
1621 : 0 : error ("Access offset before parent offset");
1622 : 0 : return true;
1623 : : }
1624 : 21560 : if (access->size >= parent_size)
1625 : : {
1626 : 0 : error ("Access size greater or equal to its parent size");
1627 : 0 : return true;
1628 : : }
1629 : 21560 : if (access->offset + access->size > parent_offset + parent_size)
1630 : : {
1631 : 0 : error ("Access terminates outside of its parent");
1632 : 0 : return true;
1633 : : }
1634 : : }
1635 : :
1636 : 362464 : if (verify_access_tree_1 (access->first_child, access->offset,
1637 : : access->size))
1638 : : return true;
1639 : :
1640 : 362464 : if (access->next_sibling
1641 : 120023 : && (access->next_sibling->offset < access->offset + access->size))
1642 : : {
1643 : 0 : error ("Access overlaps with its sibling");
1644 : 0 : return true;
1645 : : }
1646 : :
1647 : : access = access->next_sibling;
1648 : : }
1649 : : return false;
1650 : : }
1651 : :
1652 : : /* Verify that parameter access tree starting with ACCESS is in good shape,
1653 : : halt compilation and dump the tree to stderr if not. */
1654 : :
1655 : : DEBUG_FUNCTION void
1656 : 486588 : isra_verify_access_tree (gensum_param_access *access)
1657 : : {
1658 : 486588 : if (verify_access_tree_1 (access, 0, 0))
1659 : : {
1660 : 0 : for (; access; access = access->next_sibling)
1661 : 0 : dump_gensum_access (stderr, access, 2);
1662 : 0 : internal_error ("IPA-SRA access verification failed");
1663 : : }
1664 : 486588 : }
1665 : :
1666 : :
1667 : : /* Callback of walk_stmt_load_store_addr_ops visit_addr used to determine
1668 : : GIMPLE_ASM operands with memory constrains which cannot be scalarized. */
1669 : :
1670 : : static bool
1671 : 6526 : asm_visit_addr (gimple *, tree op, tree, void *)
1672 : : {
1673 : 6526 : op = get_base_address (op);
1674 : 6526 : if (op
1675 : 6526 : && TREE_CODE (op) == PARM_DECL)
1676 : 177 : disqualify_split_candidate (op, "Non-scalarizable GIMPLE_ASM operand.");
1677 : :
1678 : 6526 : return false;
1679 : : }
1680 : :
1681 : : /* Mark a dereference of parameter identified by DESC of distance DIST in a
1682 : : basic block BB, unless the BB has already been marked as a potentially
1683 : : final. */
1684 : :
1685 : : static void
1686 : 225591 : mark_param_dereference (gensum_param_desc *desc, HOST_WIDE_INT dist,
1687 : : basic_block bb)
1688 : : {
1689 : 225591 : gcc_assert (desc->by_ref);
1690 : 225591 : gcc_checking_assert (desc->split_candidate);
1691 : :
1692 : 225591 : if (desc->safe_ref
1693 : 225591 : || bitmap_bit_p (final_bbs, bb->index))
1694 : 201861 : return;
1695 : :
1696 : 23730 : int idx = bb->index * unsafe_by_ref_count + desc->deref_index;
1697 : 23730 : if (bb_dereferences[idx] < dist)
1698 : 22234 : bb_dereferences[idx] = dist;
1699 : : }
1700 : :
1701 : : /* Return true, if any potential replacements should use NEW_TYPE as opposed to
1702 : : previously recorded OLD_TYPE. */
1703 : :
1704 : : static bool
1705 : 57365 : type_prevails_p (tree old_type, tree new_type)
1706 : : {
1707 : 57365 : if (old_type == new_type)
1708 : : return false;
1709 : :
1710 : : /* Non-aggregates are always better. */
1711 : 78 : if (!is_gimple_reg_type (old_type)
1712 : 78 : && is_gimple_reg_type (new_type))
1713 : : return true;
1714 : 78 : if (is_gimple_reg_type (old_type)
1715 : 78 : && !is_gimple_reg_type (new_type))
1716 : : return false;
1717 : :
1718 : : /* Prefer any complex or vector type over any other scalar type. */
1719 : 68 : if (TREE_CODE (old_type) != COMPLEX_TYPE
1720 : 68 : && TREE_CODE (old_type) != VECTOR_TYPE
1721 : 55 : && (TREE_CODE (new_type) == COMPLEX_TYPE
1722 : 55 : || VECTOR_TYPE_P (new_type)))
1723 : : return true;
1724 : 68 : if ((TREE_CODE (old_type) == COMPLEX_TYPE
1725 : : || VECTOR_TYPE_P (old_type))
1726 : 13 : && TREE_CODE (new_type) != COMPLEX_TYPE
1727 : 0 : && TREE_CODE (new_type) != VECTOR_TYPE)
1728 : : return false;
1729 : :
1730 : : /* Use the integral type with the bigger precision. */
1731 : 68 : if (INTEGRAL_TYPE_P (old_type)
1732 : 24 : && INTEGRAL_TYPE_P (new_type))
1733 : 24 : return (TYPE_PRECISION (new_type) > TYPE_PRECISION (old_type));
1734 : :
1735 : : /* Attempt to disregard any integral type with non-full precision. */
1736 : 44 : if (INTEGRAL_TYPE_P (old_type)
1737 : 44 : && (TREE_INT_CST_LOW (TYPE_SIZE (old_type))
1738 : 0 : != TYPE_PRECISION (old_type)))
1739 : : return true;
1740 : 44 : if (INTEGRAL_TYPE_P (new_type)
1741 : 44 : && (TREE_INT_CST_LOW (TYPE_SIZE (new_type))
1742 : 6 : != TYPE_PRECISION (new_type)))
1743 : : return false;
1744 : : /* Stabilize the selection. */
1745 : 44 : return TYPE_UID (old_type) < TYPE_UID (new_type);
1746 : : }
1747 : :
1748 : : /* When scanning an expression which is a call argument, this structure
1749 : : specifies the call and the position of the argument. */
1750 : :
1751 : : struct scan_call_info
1752 : : {
1753 : : /* Call graph edge representing the call. */
1754 : : cgraph_edge *cs;
1755 : : /* Total number of arguments in the call. */
1756 : : unsigned argument_count;
1757 : : /* Number of the actual argument being scanned. */
1758 : : unsigned arg_idx;
1759 : : };
1760 : :
1761 : : /* Record use of ACCESS which belongs to a parameter described by DESC in a
1762 : : call argument described by CALL_INFO. */
1763 : :
1764 : : static void
1765 : 79046 : record_nonregister_call_use (gensum_param_desc *desc,
1766 : : scan_call_info *call_info,
1767 : : unsigned unit_offset, unsigned unit_size)
1768 : : {
1769 : 79046 : isra_call_summary *csum = call_sums->get_create (call_info->cs);
1770 : 79046 : csum->init_inputs (call_info->argument_count);
1771 : :
1772 : 79046 : isra_param_flow *param_flow = &csum->m_arg_flow[call_info->arg_idx];
1773 : 79046 : param_flow->aggregate_pass_through = true;
1774 : 79046 : set_single_param_flow_source (param_flow, desc->param_number);
1775 : 79046 : param_flow->unit_offset = unit_offset;
1776 : 79046 : param_flow->unit_size = unit_size;
1777 : 79046 : desc->call_uses++;
1778 : 79046 : }
1779 : :
1780 : : /* Callback of walk_aliased_vdefs, just mark that there was a possible
1781 : : modification. */
1782 : :
1783 : : static bool
1784 : 60117 : mark_maybe_modified (ao_ref *, tree, void *data)
1785 : : {
1786 : 60117 : bool *maybe_modified = (bool *) data;
1787 : 60117 : *maybe_modified = true;
1788 : 60117 : return true;
1789 : : }
1790 : :
1791 : : /* Analyze expression EXPR from GIMPLE for accesses to parameters. CTX
1792 : : specifies whether EXPR is used in a load, store or as an argument call. BB
1793 : : must be the basic block in which expr resides. If CTX specifies call
1794 : : argument context, CALL_INFO must describe that call and argument position,
1795 : : otherwise it is ignored. */
1796 : :
1797 : : static void
1798 : 33855980 : scan_expr_access (tree expr, gimple *stmt, isra_scan_context ctx,
1799 : : basic_block bb, scan_call_info *call_info = NULL)
1800 : : {
1801 : 33855980 : poly_int64 poffset, psize, pmax_size;
1802 : 33855980 : HOST_WIDE_INT offset, size, max_size;
1803 : 33855980 : tree base;
1804 : 33855980 : bool deref = false;
1805 : 33855980 : bool reverse;
1806 : :
1807 : 33855980 : if (TREE_CODE (expr) == ADDR_EXPR)
1808 : : {
1809 : 4039254 : if (ctx == ISRA_CTX_ARG)
1810 : : return;
1811 : 1093466 : tree t = get_base_address (TREE_OPERAND (expr, 0));
1812 : 1093466 : if (VAR_P (t) && !TREE_STATIC (t))
1813 : 199763 : loaded_decls->add (t);
1814 : 1093466 : return;
1815 : : }
1816 : 29816726 : if (TREE_CODE (expr) == SSA_NAME
1817 : 16451613 : || CONSTANT_CLASS_P (expr))
1818 : : return;
1819 : :
1820 : 11866842 : if (TREE_CODE (expr) == BIT_FIELD_REF
1821 : 11866842 : || TREE_CODE (expr) == IMAGPART_EXPR
1822 : 11764736 : || TREE_CODE (expr) == REALPART_EXPR)
1823 : 154215 : expr = TREE_OPERAND (expr, 0);
1824 : :
1825 : 11866842 : base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size, &reverse);
1826 : :
1827 : 11866842 : if (TREE_CODE (base) == MEM_REF)
1828 : : {
1829 : 3655484 : tree op = TREE_OPERAND (base, 0);
1830 : 3655484 : if (TREE_CODE (op) != SSA_NAME
1831 : 3655484 : || !SSA_NAME_IS_DEFAULT_DEF (op))
1832 : : return;
1833 : 1906440 : base = SSA_NAME_VAR (op);
1834 : 1906440 : if (!base)
1835 : : return;
1836 : : deref = true;
1837 : : }
1838 : 8211358 : else if (VAR_P (base)
1839 : 8211358 : && !TREE_STATIC (base)
1840 : 6176383 : && (ctx == ISRA_CTX_ARG
1841 : : || ctx == ISRA_CTX_LOAD))
1842 : 2356931 : loaded_decls->add (base);
1843 : :
1844 : 10117798 : if (TREE_CODE (base) != PARM_DECL)
1845 : : return;
1846 : :
1847 : 2237722 : gensum_param_desc *desc = get_gensum_param_desc (base);
1848 : 2237722 : if (!desc || !desc->split_candidate)
1849 : : return;
1850 : :
1851 : 533630 : if (!poffset.is_constant (&offset)
1852 : 533630 : || !psize.is_constant (&size)
1853 : 533630 : || !pmax_size.is_constant (&max_size))
1854 : : {
1855 : : disqualify_split_candidate (desc, "Encountered a polynomial-sized "
1856 : : "access.");
1857 : : return;
1858 : : }
1859 : 533630 : if (size < 0 || size != max_size)
1860 : : {
1861 : 5487 : disqualify_split_candidate (desc, "Encountered a variable sized access.");
1862 : 5487 : return;
1863 : : }
1864 : 528143 : if (TREE_CODE (expr) == COMPONENT_REF
1865 : 528143 : && DECL_BIT_FIELD (TREE_OPERAND (expr, 1)))
1866 : : {
1867 : 1362 : disqualify_split_candidate (desc, "Encountered a bit-field access.");
1868 : 1362 : return;
1869 : : }
1870 : 526781 : if (offset < 0)
1871 : : {
1872 : 4 : disqualify_split_candidate (desc, "Encountered an access at a "
1873 : : "negative offset.");
1874 : 4 : return;
1875 : : }
1876 : 526777 : gcc_assert ((offset % BITS_PER_UNIT) == 0);
1877 : 526777 : gcc_assert ((size % BITS_PER_UNIT) == 0);
1878 : 526777 : if ((offset / BITS_PER_UNIT) >= (UINT_MAX - ISRA_ARG_SIZE_LIMIT)
1879 : 526777 : || (size / BITS_PER_UNIT) >= ISRA_ARG_SIZE_LIMIT)
1880 : : {
1881 : 0 : disqualify_split_candidate (desc, "Encountered an access with too big "
1882 : : "offset or size");
1883 : 0 : return;
1884 : : }
1885 : :
1886 : 526777 : tree type = TREE_TYPE (expr);
1887 : 526777 : unsigned int exp_align = get_object_alignment (expr);
1888 : :
1889 : 526777 : if (exp_align < TYPE_ALIGN (type))
1890 : : {
1891 : 1333 : disqualify_split_candidate (desc, "Underaligned access.");
1892 : 1333 : return;
1893 : : }
1894 : :
1895 : 525444 : if (deref)
1896 : : {
1897 : 277025 : if (!desc->by_ref)
1898 : : {
1899 : 0 : disqualify_split_candidate (desc, "Dereferencing a non-reference.");
1900 : 51434 : return;
1901 : : }
1902 : 277025 : else if (ctx == ISRA_CTX_STORE)
1903 : : {
1904 : 0 : disqualify_split_candidate (desc, "Storing to data passed by "
1905 : : "reference.");
1906 : 0 : return;
1907 : : }
1908 : :
1909 : 277025 : if (!aa_walking_limit)
1910 : : {
1911 : 0 : disqualify_split_candidate (desc, "Out of alias analysis step "
1912 : : "limit.");
1913 : 0 : return;
1914 : : }
1915 : :
1916 : 554050 : gcc_checking_assert (gimple_vuse (stmt));
1917 : 277025 : bool maybe_modified = false;
1918 : 277025 : ao_ref ar;
1919 : :
1920 : 277025 : ao_ref_init (&ar, expr);
1921 : 277025 : bitmap visited = BITMAP_ALLOC (NULL);
1922 : 554050 : int walked = walk_aliased_vdefs (&ar, gimple_vuse (stmt),
1923 : : mark_maybe_modified, &maybe_modified,
1924 : : &visited, NULL, aa_walking_limit);
1925 : 277025 : BITMAP_FREE (visited);
1926 : 277025 : if (walked > 0)
1927 : : {
1928 : 89677 : gcc_assert (aa_walking_limit > walked);
1929 : 89677 : aa_walking_limit = aa_walking_limit - walked;
1930 : : }
1931 : 277025 : if (walked < 0)
1932 : 0 : aa_walking_limit = 0;
1933 : 277025 : if (maybe_modified || walked < 0)
1934 : : {
1935 : 51434 : disqualify_split_candidate (desc, "Data passed by reference possibly "
1936 : : "modified through an alias.");
1937 : 51434 : return;
1938 : : }
1939 : : else
1940 : 225591 : mark_param_dereference (desc, offset + size, bb);
1941 : : }
1942 : : else
1943 : : /* Pointer parameters with direct uses should have been ruled out by
1944 : : analyzing SSA default def when looking at the parameters. */
1945 : 248419 : gcc_assert (!desc->by_ref);
1946 : :
1947 : 474010 : gensum_param_access *access = get_access (desc, offset, size, ctx);
1948 : 474010 : if (!access)
1949 : : return;
1950 : :
1951 : 469169 : if (ctx == ISRA_CTX_ARG)
1952 : : {
1953 : 81660 : gcc_checking_assert (call_info);
1954 : :
1955 : 81660 : if (!deref)
1956 : 79046 : record_nonregister_call_use (desc, call_info, offset / BITS_PER_UNIT,
1957 : 79046 : size / BITS_PER_UNIT);
1958 : : else
1959 : : /* This is not a pass-through of a pointer, this is a use like any
1960 : : other. */
1961 : 2614 : access->nonarg = true;
1962 : : }
1963 : 387509 : else if (ctx == ISRA_CTX_LOAD && bb->count.initialized_p ())
1964 : 374007 : access->load_count += bb->count;
1965 : :
1966 : 469169 : if (!access->type)
1967 : : {
1968 : 406229 : access->type = type;
1969 : 406229 : access->alias_ptr_type = reference_alias_ptr_type (expr);
1970 : 406229 : access->reverse = reverse;
1971 : : }
1972 : : else
1973 : : {
1974 : 62940 : if (exp_align < TYPE_ALIGN (access->type))
1975 : : {
1976 : 0 : disqualify_split_candidate (desc, "Reference has lower alignment "
1977 : : "than a previous one.");
1978 : 0 : return;
1979 : : }
1980 : 62940 : if (access->alias_ptr_type != reference_alias_ptr_type (expr))
1981 : : {
1982 : 2771 : disqualify_split_candidate (desc, "Multiple alias pointer types.");
1983 : 2771 : return;
1984 : : }
1985 : 60169 : if (access->reverse != reverse)
1986 : : {
1987 : 0 : disqualify_split_candidate (desc, "Both normal and reverse "
1988 : : "scalar storage order.");
1989 : 0 : return;
1990 : : }
1991 : 60169 : if (!deref
1992 : 46224 : && (AGGREGATE_TYPE_P (type) || AGGREGATE_TYPE_P (access->type))
1993 : 93565 : && (TYPE_MAIN_VARIANT (access->type) != TYPE_MAIN_VARIANT (type)))
1994 : : {
1995 : : /* We need the same aggregate type on all accesses to be able to
1996 : : distinguish transformation spots from pass-through arguments in
1997 : : the transformation phase. */
1998 : 2804 : disqualify_split_candidate (desc, "We do not support aggregate "
1999 : : "type punning.");
2000 : 2804 : return;
2001 : : }
2002 : :
2003 : 57365 : if (type_prevails_p (access->type, type))
2004 : 25 : access->type = type;
2005 : : }
2006 : : }
2007 : :
2008 : : /* Scan body function described by NODE and FUN and create access trees for
2009 : : parameters. */
2010 : :
2011 : : static void
2012 : 1198159 : scan_function (cgraph_node *node, struct function *fun)
2013 : : {
2014 : 1198159 : basic_block bb;
2015 : :
2016 : 9743221 : FOR_EACH_BB_FN (bb, fun)
2017 : : {
2018 : 8545062 : gimple_stmt_iterator gsi;
2019 : 64596716 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2020 : : {
2021 : 47506592 : gimple *stmt = gsi_stmt (gsi);
2022 : :
2023 : 47506592 : if (final_bbs && stmt_can_throw_external (fun, stmt))
2024 : 1244598 : bitmap_set_bit (final_bbs, bb->index);
2025 : 47506592 : switch (gimple_code (stmt))
2026 : : {
2027 : 1168005 : case GIMPLE_RETURN:
2028 : 1168005 : {
2029 : 1168005 : tree t = gimple_return_retval (as_a <greturn *> (stmt));
2030 : 1168005 : if (t != NULL_TREE)
2031 : 646540 : scan_expr_access (t, stmt, ISRA_CTX_LOAD, bb);
2032 : 1168005 : if (final_bbs)
2033 : 380597 : bitmap_set_bit (final_bbs, bb->index);
2034 : : }
2035 : : break;
2036 : :
2037 : 17345866 : case GIMPLE_ASSIGN:
2038 : 17345866 : if (gimple_assign_single_p (stmt)
2039 : 17345866 : && !gimple_clobber_p (stmt))
2040 : : {
2041 : 10518226 : tree rhs = gimple_assign_rhs1 (stmt);
2042 : 10518226 : scan_expr_access (rhs, stmt, ISRA_CTX_LOAD, bb);
2043 : 10518226 : tree lhs = gimple_assign_lhs (stmt);
2044 : 10518226 : scan_expr_access (lhs, stmt, ISRA_CTX_STORE, bb);
2045 : : }
2046 : : break;
2047 : :
2048 : 4979900 : case GIMPLE_CALL:
2049 : 4979900 : {
2050 : 4979900 : unsigned argument_count = gimple_call_num_args (stmt);
2051 : 4979900 : isra_scan_context ctx = ISRA_CTX_ARG;
2052 : 4979900 : scan_call_info call_info, *call_info_p = &call_info;
2053 : 4979900 : if (gimple_call_internal_p (stmt))
2054 : : {
2055 : 152274 : call_info_p = NULL;
2056 : 152274 : ctx = ISRA_CTX_LOAD;
2057 : 152274 : internal_fn ifn = gimple_call_internal_fn (stmt);
2058 : 152274 : if (internal_store_fn_p (ifn))
2059 : 0 : ctx = ISRA_CTX_STORE;
2060 : : }
2061 : : else
2062 : : {
2063 : 4827626 : call_info.cs = node->get_edge (stmt);
2064 : 4827626 : call_info.argument_count = argument_count;
2065 : : }
2066 : :
2067 : 15038957 : for (unsigned i = 0; i < argument_count; i++)
2068 : : {
2069 : 10059057 : call_info.arg_idx = i;
2070 : 10059057 : scan_expr_access (gimple_call_arg (stmt, i), stmt,
2071 : : ctx, bb, call_info_p);
2072 : : }
2073 : :
2074 : 4979900 : tree lhs = gimple_call_lhs (stmt);
2075 : 4979900 : if (lhs)
2076 : 2031614 : scan_expr_access (lhs, stmt, ISRA_CTX_STORE, bb);
2077 : 4979900 : int flags = gimple_call_flags (stmt);
2078 : 4979900 : if (final_bbs
2079 : 1971352 : && (((flags & (ECF_CONST | ECF_PURE)) == 0)
2080 : 294057 : || (flags & ECF_LOOPING_CONST_OR_PURE)))
2081 : 1727100 : bitmap_set_bit (final_bbs, bb->index);
2082 : : }
2083 : 4979900 : break;
2084 : :
2085 : 46941 : case GIMPLE_ASM:
2086 : 46941 : {
2087 : 46941 : gasm *asm_stmt = as_a <gasm *> (stmt);
2088 : 46941 : walk_stmt_load_store_addr_ops (asm_stmt, NULL, NULL, NULL,
2089 : : asm_visit_addr);
2090 : 46941 : if (final_bbs)
2091 : 1351 : bitmap_set_bit (final_bbs, bb->index);
2092 : :
2093 : 80843 : for (unsigned i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
2094 : : {
2095 : 33902 : tree t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
2096 : 33902 : scan_expr_access (t, stmt, ISRA_CTX_LOAD, bb);
2097 : : }
2098 : 95356 : for (unsigned i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
2099 : : {
2100 : 48415 : tree t = TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
2101 : 48415 : scan_expr_access (t, stmt, ISRA_CTX_STORE, bb);
2102 : : }
2103 : : }
2104 : : break;
2105 : :
2106 : : default:
2107 : : break;
2108 : : }
2109 : : }
2110 : : }
2111 : 1198159 : }
2112 : :
2113 : : /* Return true if SSA_NAME NAME of function described by FUN is only used in
2114 : : return statements, or if results of any operations it is involved in are
2115 : : only used in return statements. ANALYZED is a bitmap that tracks which SSA
2116 : : names we have already started investigating. */
2117 : :
2118 : : static bool
2119 : 2325473 : ssa_name_only_returned_p (function *fun, tree name, bitmap analyzed)
2120 : : {
2121 : 2325473 : bool res = true;
2122 : 2325473 : imm_use_iterator imm_iter;
2123 : 2325473 : gimple *stmt;
2124 : :
2125 : 2940788 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
2126 : : {
2127 : 2536382 : if (is_gimple_debug (stmt))
2128 : 169582 : continue;
2129 : :
2130 : 2366800 : if (gimple_code (stmt) == GIMPLE_RETURN)
2131 : : {
2132 : 241064 : tree t = gimple_return_retval (as_a <greturn *> (stmt));
2133 : 241064 : if (t != name)
2134 : : {
2135 : : res = false;
2136 : : break;
2137 : : }
2138 : : }
2139 : 2125736 : else if (!stmt_unremovable_because_of_non_call_eh_p (fun, stmt)
2140 : 2125736 : && ((is_gimple_assign (stmt) && !gimple_has_volatile_ops (stmt))
2141 : 1089133 : || gimple_code (stmt) == GIMPLE_PHI))
2142 : : {
2143 : : /* TODO: And perhaps for const function calls too? */
2144 : 1151405 : tree lhs;
2145 : 1151405 : if (gimple_code (stmt) == GIMPLE_PHI)
2146 : 203040 : lhs = gimple_phi_result (stmt);
2147 : : else
2148 : 948365 : lhs = gimple_assign_lhs (stmt);
2149 : :
2150 : 1151405 : if (TREE_CODE (lhs) != SSA_NAME)
2151 : : {
2152 : : res = false;
2153 : : break;
2154 : : }
2155 : 810059 : gcc_assert (!gimple_vdef (stmt));
2156 : 810059 : if (bitmap_set_bit (analyzed, SSA_NAME_VERSION (lhs))
2157 : 810059 : && !ssa_name_only_returned_p (fun, lhs, analyzed))
2158 : : {
2159 : : res = false;
2160 : : break;
2161 : : }
2162 : : }
2163 : : else
2164 : : {
2165 : : res = false;
2166 : : break;
2167 : : }
2168 : 2325473 : }
2169 : 2325473 : return res;
2170 : : }
2171 : :
2172 : : /* Inspect the uses of the return value of the call associated with CS, and if
2173 : : it is not used or if it is only used to construct the return value of the
2174 : : caller, mark it as such in call or caller summary. Also check for
2175 : : misaligned arguments. */
2176 : :
2177 : : static void
2178 : 4827626 : isra_analyze_call (cgraph_edge *cs)
2179 : : {
2180 : 4827626 : gcall *call_stmt = cs->call_stmt;
2181 : 4827626 : unsigned count = gimple_call_num_args (call_stmt);
2182 : 4827626 : isra_call_summary *csum = call_sums->get_create (cs);
2183 : :
2184 : 14378555 : for (unsigned i = 0; i < count; i++)
2185 : : {
2186 : 9550929 : tree arg = gimple_call_arg (call_stmt, i);
2187 : 9550929 : if (TREE_CODE (arg) == ADDR_EXPR)
2188 : : {
2189 : 2945788 : poly_int64 poffset, psize, pmax_size;
2190 : 2945788 : bool reverse;
2191 : 2945788 : tree base = get_ref_base_and_extent (TREE_OPERAND (arg, 0), &poffset,
2192 : 2945788 : &psize, &pmax_size, &reverse);
2193 : 2945788 : HOST_WIDE_INT offset;
2194 : 2945788 : unsigned HOST_WIDE_INT ds;
2195 : 2945788 : if (DECL_P (base)
2196 : 2141278 : && (poffset.is_constant (&offset))
2197 : 2141278 : && tree_fits_uhwi_p (DECL_SIZE (base))
2198 : 4964763 : && ((ds = tree_to_uhwi (DECL_SIZE (base)) - offset)
2199 : : < ISRA_ARG_SIZE_LIMIT * BITS_PER_UNIT))
2200 : : {
2201 : 1885777 : csum->init_inputs (count);
2202 : 1885777 : gcc_assert (!csum->m_arg_flow[i].aggregate_pass_through);
2203 : 1885777 : csum->m_arg_flow[i].unit_size = ds / BITS_PER_UNIT;
2204 : : }
2205 : :
2206 : 2945788 : if (TREE_CODE (base) == VAR_DECL
2207 : 2945788 : && !TREE_STATIC (base)
2208 : 2945788 : && !loaded_decls->contains (base))
2209 : : {
2210 : 795217 : csum->init_inputs (count);
2211 : 795217 : csum->m_arg_flow[i].constructed_for_calls = true;
2212 : : }
2213 : : }
2214 : :
2215 : 9550929 : if (is_gimple_reg (arg))
2216 : 3836326 : continue;
2217 : :
2218 : 5714603 : tree offset;
2219 : 5714603 : poly_int64 bitsize, bitpos;
2220 : 5714603 : machine_mode mode;
2221 : 5714603 : int unsignedp, reversep, volatilep = 0;
2222 : 5714603 : get_inner_reference (arg, &bitsize, &bitpos, &offset, &mode,
2223 : : &unsignedp, &reversep, &volatilep);
2224 : 11429206 : if (!multiple_p (bitpos, BITS_PER_UNIT))
2225 : : {
2226 : 0 : csum->m_bit_aligned_arg = true;
2227 : 0 : break;
2228 : : }
2229 : : }
2230 : :
2231 : 4827626 : tree lhs = gimple_call_lhs (call_stmt);
2232 : 4827626 : if (lhs)
2233 : : {
2234 : : /* TODO: Also detect aggregates on a LHS of a call that are only returned
2235 : : from this function (without being read anywhere). */
2236 : 1895752 : if (TREE_CODE (lhs) == SSA_NAME)
2237 : : {
2238 : 1536716 : bitmap analyzed = BITMAP_ALLOC (NULL);
2239 : 1536716 : if (ssa_name_only_returned_p (DECL_STRUCT_FUNCTION (cs->caller->decl),
2240 : : lhs, analyzed))
2241 : 221039 : csum->m_return_returned = true;
2242 : 1536716 : BITMAP_FREE (analyzed);
2243 : : }
2244 : : }
2245 : : else
2246 : 2931874 : csum->m_return_ignored = true;
2247 : 4827626 : }
2248 : :
2249 : : /* Look at all calls going out of NODE, described also by IFS and perform all
2250 : : analyses necessary for IPA-SRA that are not done at body scan time or done
2251 : : even when body is not scanned because the function is not a candidate. */
2252 : :
2253 : : static void
2254 : 1198159 : isra_analyze_all_outgoing_calls (cgraph_node *node)
2255 : : {
2256 : 5892915 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
2257 : 4694756 : isra_analyze_call (cs);
2258 : 1331029 : for (cgraph_edge *cs = node->indirect_calls; cs; cs = cs->next_callee)
2259 : 132870 : isra_analyze_call (cs);
2260 : 1198159 : }
2261 : :
2262 : : /* Dump a dereferences table with heading STR to file F. */
2263 : :
2264 : : static void
2265 : 10 : dump_dereferences_table (FILE *f, struct function *fun, const char *str)
2266 : : {
2267 : 10 : basic_block bb;
2268 : :
2269 : 10 : fprintf (dump_file, "%s", str);
2270 : 52 : FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun),
2271 : : EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
2272 : : {
2273 : 42 : fprintf (f, "%4i %i ", bb->index, bitmap_bit_p (final_bbs, bb->index));
2274 : 42 : if (bb != EXIT_BLOCK_PTR_FOR_FN (fun))
2275 : : {
2276 : : int i;
2277 : 104 : for (i = 0; i < unsafe_by_ref_count; i++)
2278 : : {
2279 : 62 : int idx = bb->index * unsafe_by_ref_count + i;
2280 : 62 : fprintf (f, " %4" HOST_WIDE_INT_PRINT "d", bb_dereferences[idx]);
2281 : : }
2282 : : }
2283 : 42 : fprintf (f, "\n");
2284 : : }
2285 : 10 : fprintf (dump_file, "\n");
2286 : 10 : }
2287 : :
2288 : : /* Propagate distances in bb_dereferences in the opposite direction than the
2289 : : control flow edges, in each step storing the maximum of the current value
2290 : : and the minimum of all successors. These steps are repeated until the table
2291 : : stabilizes. Note that BBs which might terminate the functions (according to
2292 : : final_bbs bitmap) never updated in this way. */
2293 : :
2294 : : static void
2295 : 25677 : propagate_dereference_distances (struct function *fun)
2296 : : {
2297 : 25677 : basic_block bb;
2298 : :
2299 : 25677 : if (dump_file && (dump_flags & TDF_DETAILS))
2300 : 5 : dump_dereferences_table (dump_file, fun,
2301 : : "Dereference table before propagation:\n");
2302 : :
2303 : 25677 : auto_vec<basic_block> queue (last_basic_block_for_fn (fun));
2304 : 25677 : queue.quick_push (ENTRY_BLOCK_PTR_FOR_FN (fun));
2305 : 218513 : FOR_EACH_BB_FN (bb, fun)
2306 : : {
2307 : 192836 : queue.quick_push (bb);
2308 : 192836 : bb->aux = bb;
2309 : : }
2310 : :
2311 : 244995 : while (!queue.is_empty ())
2312 : : {
2313 : 219318 : edge_iterator ei;
2314 : 219318 : edge e;
2315 : 219318 : bool change = false;
2316 : 219318 : int i;
2317 : :
2318 : 219318 : bb = queue.pop ();
2319 : 219318 : bb->aux = NULL;
2320 : :
2321 : 219318 : if (bitmap_bit_p (final_bbs, bb->index))
2322 : 68793 : continue;
2323 : :
2324 : 6285117 : for (i = 0; i < unsafe_by_ref_count; i++)
2325 : : {
2326 : 6134592 : int idx = bb->index * unsafe_by_ref_count + i;
2327 : 6134592 : bool first = true;
2328 : 6134592 : HOST_WIDE_INT inh = 0;
2329 : :
2330 : 14399963 : FOR_EACH_EDGE (e, ei, bb->succs)
2331 : : {
2332 : 8265371 : int succ_idx = e->dest->index * unsafe_by_ref_count + i;
2333 : :
2334 : 8265371 : if (e->dest == EXIT_BLOCK_PTR_FOR_FN (fun))
2335 : 0 : continue;
2336 : :
2337 : 8265371 : if (first)
2338 : : {
2339 : 6134587 : first = false;
2340 : 6134587 : inh = bb_dereferences [succ_idx];
2341 : : }
2342 : 2130784 : else if (bb_dereferences [succ_idx] < inh)
2343 : : inh = bb_dereferences [succ_idx];
2344 : : }
2345 : :
2346 : 6134592 : if (!first && bb_dereferences[idx] < inh)
2347 : : {
2348 : 9607 : bb_dereferences[idx] = inh;
2349 : 9607 : change = true;
2350 : : }
2351 : : }
2352 : :
2353 : 150525 : if (change)
2354 : 10686 : FOR_EACH_EDGE (e, ei, bb->preds)
2355 : : {
2356 : 1655 : if (e->src->aux)
2357 : 850 : continue;
2358 : :
2359 : 805 : e->src->aux = e->src;
2360 : 805 : queue.quick_push (e->src);
2361 : : }
2362 : : }
2363 : :
2364 : 25677 : if (dump_file && (dump_flags & TDF_DETAILS))
2365 : 5 : dump_dereferences_table (dump_file, fun,
2366 : : "Dereference table after propagation:\n");
2367 : 25677 : }
2368 : :
2369 : : /* Return true if the ACCESS loads happen frequently enough in FUN to risk
2370 : : moving them to the caller and only pass the result. */
2371 : :
2372 : : static bool
2373 : 159244 : dereference_probable_p (struct function *fun, gensum_param_access *access)
2374 : : {
2375 : 159244 : int threshold = opt_for_fn (fun->decl, param_ipa_sra_deref_prob_threshold);
2376 : 159244 : return access->load_count
2377 : 159244 : >= ENTRY_BLOCK_PTR_FOR_FN (fun)->count.apply_scale (threshold, 100);
2378 : : }
2379 : :
2380 : : /* Perform basic checks on ACCESS to PARM (of FUN) described by DESC and all
2381 : : its children, return true if the parameter cannot be split, otherwise return
2382 : : false and update *NONARG_ACC_SIZE and *ONLY_CALLS. ENTRY_BB_INDEX must be
2383 : : the index of the entry BB in the function of PARM. */
2384 : :
2385 : : static bool
2386 : 355365 : check_gensum_access (struct function *fun, tree parm, gensum_param_desc *desc,
2387 : : gensum_param_access *access,
2388 : : HOST_WIDE_INT *nonarg_acc_size, bool *only_calls,
2389 : : int entry_bb_index)
2390 : : {
2391 : 355365 : if (access->nonarg)
2392 : : {
2393 : 306135 : *only_calls = false;
2394 : 306135 : *nonarg_acc_size += access->size;
2395 : :
2396 : 306135 : if (access->first_child)
2397 : : {
2398 : 399 : disqualify_split_candidate (desc, "Overlapping non-call uses.");
2399 : 399 : return true;
2400 : : }
2401 : : }
2402 : : /* Do not decompose a non-BLKmode param in a way that would create
2403 : : BLKmode params. Especially for by-reference passing (thus,
2404 : : pointer-type param) this is hardly worthwhile. */
2405 : 354966 : if (DECL_MODE (parm) != BLKmode
2406 : 354966 : && TYPE_MODE (access->type) == BLKmode)
2407 : : {
2408 : 4723 : disqualify_split_candidate (desc, "Would convert a non-BLK to a BLK.");
2409 : 4723 : return true;
2410 : : }
2411 : :
2412 : 350243 : if (desc->by_ref)
2413 : : {
2414 : 169105 : if (desc->safe_ref)
2415 : : {
2416 : 135036 : if (!dereference_probable_p (fun, access))
2417 : : {
2418 : 5963 : disqualify_split_candidate (desc, "Dereferences in callers "
2419 : : "would happen much more frequently.");
2420 : 5963 : return true;
2421 : : }
2422 : : }
2423 : : else
2424 : : {
2425 : 34069 : int idx = (entry_bb_index * unsafe_by_ref_count + desc->deref_index);
2426 : 34069 : if ((access->offset + access->size) > bb_dereferences[idx])
2427 : : {
2428 : 24208 : if (!dereference_probable_p (fun, access))
2429 : : {
2430 : 1712 : disqualify_split_candidate (desc, "Would create a possibly "
2431 : : "illegal dereference in a "
2432 : : "caller.");
2433 : 1712 : return true;
2434 : : }
2435 : 22496 : desc->conditionally_dereferenceable = true;
2436 : : }
2437 : : }
2438 : : }
2439 : :
2440 : 342568 : for (gensum_param_access *ch = access->first_child;
2441 : 363312 : ch;
2442 : 20744 : ch = ch->next_sibling)
2443 : 20751 : if (check_gensum_access (fun, parm, desc, ch, nonarg_acc_size, only_calls,
2444 : : entry_bb_index))
2445 : : return true;
2446 : :
2447 : : return false;
2448 : : }
2449 : :
2450 : : /* Copy data from FROM and all of its children to a vector of accesses in IPA
2451 : : descriptor DESC. */
2452 : :
2453 : : static void
2454 : 406229 : copy_accesses_to_ipa_desc (gensum_param_access *from, isra_param_desc *desc)
2455 : : {
2456 : 406229 : param_access *to = ggc_cleared_alloc<param_access> ();
2457 : 406229 : gcc_checking_assert ((from->offset % BITS_PER_UNIT) == 0);
2458 : 406229 : gcc_checking_assert ((from->size % BITS_PER_UNIT) == 0);
2459 : 406229 : to->unit_offset = from->offset / BITS_PER_UNIT;
2460 : 406229 : to->unit_size = from->size / BITS_PER_UNIT;
2461 : 406229 : to->type = from->type;
2462 : 406229 : to->alias_ptr_type = from->alias_ptr_type;
2463 : 406229 : to->certain = from->nonarg;
2464 : 406229 : to->reverse = from->reverse;
2465 : 406229 : vec_safe_push (desc->accesses, to);
2466 : :
2467 : 406229 : for (gensum_param_access *ch = from->first_child;
2468 : 429310 : ch;
2469 : 23081 : ch = ch->next_sibling)
2470 : 23081 : copy_accesses_to_ipa_desc (ch, desc);
2471 : 406229 : }
2472 : :
2473 : : /* Analyze function body scan results stored in param_accesses and
2474 : : param_accesses, detect possible transformations and store information of
2475 : : those in function summary. NODE, FUN and IFS are all various structures
2476 : : describing the currently analyzed function. */
2477 : :
2478 : : static void
2479 : 781674 : process_scan_results (cgraph_node *node, struct function *fun,
2480 : : isra_func_summary *ifs,
2481 : : vec<gensum_param_desc> *param_descriptions)
2482 : : {
2483 : 781674 : bool check_pass_throughs = false;
2484 : 781674 : bool dereferences_propagated = false;
2485 : 781674 : tree parm = DECL_ARGUMENTS (node->decl);
2486 : 781674 : unsigned param_count = param_descriptions->length();
2487 : :
2488 : 781674 : for (unsigned desc_index = 0;
2489 : 3026606 : desc_index < param_count;
2490 : 2244932 : desc_index++, parm = DECL_CHAIN (parm))
2491 : : {
2492 : 2244932 : gensum_param_desc *desc = &(*param_descriptions)[desc_index];
2493 : 2244932 : if (!desc->split_candidate)
2494 : 1771135 : continue;
2495 : :
2496 : 486594 : if (flag_checking)
2497 : 486588 : isra_verify_access_tree (desc->accesses);
2498 : :
2499 : 486594 : if (!dereferences_propagated
2500 : 481049 : && desc->by_ref
2501 : 340897 : && !desc->safe_ref
2502 : 156179 : && desc->accesses)
2503 : : {
2504 : 25677 : propagate_dereference_distances (fun);
2505 : 25677 : dereferences_propagated = true;
2506 : : }
2507 : :
2508 : 486594 : HOST_WIDE_INT nonarg_acc_size = 0;
2509 : 486594 : bool only_calls = true;
2510 : 486594 : bool check_failed = false;
2511 : :
2512 : 486594 : int entry_bb_index = ENTRY_BLOCK_PTR_FOR_FN (fun)->index;
2513 : 486594 : for (gensum_param_access *acc = desc->accesses;
2514 : 808411 : acc;
2515 : 321817 : acc = acc->next_sibling)
2516 : 334614 : if (check_gensum_access (fun, parm, desc, acc, &nonarg_acc_size,
2517 : : &only_calls, entry_bb_index))
2518 : : {
2519 : : check_failed = true;
2520 : : break;
2521 : : }
2522 : 486594 : if (check_failed)
2523 : 12797 : continue;
2524 : :
2525 : 473797 : if (only_calls)
2526 : 293669 : desc->locally_unused = true;
2527 : :
2528 : 473797 : HOST_WIDE_INT cur_param_size
2529 : 473797 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm)));
2530 : 473797 : HOST_WIDE_INT param_size_limit, optimistic_limit;
2531 : 473797 : if (!desc->by_ref || optimize_function_for_size_p (fun))
2532 : : {
2533 : : param_size_limit = cur_param_size;
2534 : : optimistic_limit = cur_param_size;
2535 : : }
2536 : : else
2537 : : {
2538 : 312592 : param_size_limit
2539 : 312592 : = opt_for_fn (node->decl,
2540 : : param_ipa_sra_ptr_growth_factor) * cur_param_size;
2541 : 312592 : optimistic_limit
2542 : 312592 : = (opt_for_fn (node->decl, param_ipa_sra_ptrwrap_growth_factor)
2543 : : * param_size_limit);
2544 : : }
2545 : :
2546 : 473797 : if (nonarg_acc_size > optimistic_limit
2547 : 471619 : || (!desc->by_ref && nonarg_acc_size == param_size_limit))
2548 : : {
2549 : 62692 : disqualify_split_candidate (desc, "Would result into a too big set "
2550 : : "of replacements even in best "
2551 : : "scenarios.");
2552 : : }
2553 : : else
2554 : : {
2555 : : /* create_parameter_descriptors makes sure unit sizes of all
2556 : : candidate parameters fit unsigned integers restricted to
2557 : : ISRA_ARG_SIZE_LIMIT. */
2558 : 411105 : desc->param_size_limit = param_size_limit / BITS_PER_UNIT;
2559 : 411105 : desc->nonarg_acc_size = nonarg_acc_size / BITS_PER_UNIT;
2560 : 411105 : if (desc->split_candidate && desc->ptr_pt_count)
2561 : : {
2562 : 179527 : gcc_assert (desc->by_ref);
2563 : : check_pass_throughs = true;
2564 : : }
2565 : : }
2566 : : }
2567 : :
2568 : : /* When a pointer parameter is passed-through to a callee, in which it is
2569 : : only used to read only one or a few items, we can attempt to transform it
2570 : : to obtaining and passing through the items instead of the pointer. But we
2571 : : must take extra care that 1) we do not introduce any segfault by moving
2572 : : dereferences above control flow and that 2) the data is not modified
2573 : : through an alias in this function. The IPA analysis must not introduce
2574 : : any accesses candidates unless it can prove both.
2575 : :
2576 : : The current solution is very crude as it consists of ensuring that the
2577 : : call postdominates entry BB and that the definition of VUSE of the call is
2578 : : default definition. TODO: For non-recursive callees in the same
2579 : : compilation unit we could do better by doing analysis in topological order
2580 : : an looking into access candidates of callees, using their alias_ptr_types
2581 : : to attempt real AA. We could also use the maximum known dereferenced
2582 : : offset in this function at IPA level.
2583 : :
2584 : : TODO: Measure the overhead and the effect of just being pessimistic.
2585 : : Maybe this is only -O3 material? */
2586 : :
2587 : 781674 : hash_map<gimple *, bool> analyzed_stmts;
2588 : 781674 : bitmap always_executed_bbs = NULL;
2589 : 781674 : if (check_pass_throughs)
2590 : 1156291 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
2591 : : {
2592 : 1012717 : gcall *call_stmt = cs->call_stmt;
2593 : 1012717 : tree vuse = gimple_vuse (call_stmt);
2594 : :
2595 : : /* If the callee is a const function, we don't get a VUSE. In such
2596 : : case there will be no memory accesses in the called function (or the
2597 : : const attribute is wrong) and then we just don't care. */
2598 : 1994647 : bool uses_memory_as_obtained = vuse && SSA_NAME_IS_DEFAULT_DEF (vuse);
2599 : :
2600 : 1012717 : unsigned count = gimple_call_num_args (call_stmt);
2601 : 1012717 : isra_call_summary *csum = call_sums->get_create (cs);
2602 : 1012717 : csum->init_inputs (count);
2603 : 1012717 : csum->m_before_any_store = uses_memory_as_obtained;
2604 : 3255488 : for (unsigned argidx = 0; argidx < count; argidx++)
2605 : : {
2606 : 2242771 : if (!csum->m_arg_flow[argidx].pointer_pass_through)
2607 : 4067231 : continue;
2608 : 418311 : unsigned pidx
2609 : 418311 : = get_single_param_flow_source (&csum->m_arg_flow[argidx]);
2610 : 418311 : gensum_param_desc *desc = &(*param_descriptions)[pidx];
2611 : 418311 : if (!desc->split_candidate)
2612 : : {
2613 : 37633 : csum->m_arg_flow[argidx].pointer_pass_through = false;
2614 : 37633 : continue;
2615 : : }
2616 : 380678 : if (!uses_memory_as_obtained)
2617 : 139841 : continue;
2618 : :
2619 : 240837 : if (desc->safe_ref)
2620 : : {
2621 : 46484 : csum->m_arg_flow[argidx].safe_to_import_accesses = true;
2622 : 46484 : continue;
2623 : : }
2624 : :
2625 : : /* Walk basic block and see if its execution can terminate earlier.
2626 : : Keep the info for later re-use to avoid quadratic behavoiur here. */
2627 : 194353 : gimple_stmt_iterator gsi = gsi_for_stmt (call_stmt);
2628 : 194353 : bool safe = true;
2629 : 194353 : int n = 0;
2630 : 577486 : for (gsi_prev (&gsi); !gsi_end_p (gsi); gsi_prev (&gsi))
2631 : : {
2632 : 106868 : bool *b = analyzed_stmts.get (gsi_stmt (gsi));
2633 : 106868 : if (b)
2634 : : {
2635 : 8584 : safe = *b;
2636 : 8584 : gsi_next (&gsi);
2637 : 8584 : break;
2638 : : }
2639 : 98284 : n++;
2640 : 98284 : if (stmt_may_terminate_function_p (fun, gsi_stmt (gsi), false))
2641 : : {
2642 : : safe = false;
2643 : : break;
2644 : : }
2645 : : }
2646 : 194353 : if (n)
2647 : : {
2648 : 31317 : if (gsi_end_p (gsi))
2649 : 54846 : gsi = gsi_start_bb (gimple_bb (call_stmt));
2650 : 129601 : for (; gsi_stmt (gsi) != call_stmt; gsi_next (&gsi))
2651 : 98284 : analyzed_stmts.get_or_insert (gsi_stmt (gsi)) = safe;
2652 : : }
2653 : :
2654 : 194353 : if (safe && !always_executed_bbs)
2655 : : {
2656 : 53103 : mark_dfs_back_edges ();
2657 : 53103 : always_executed_bbs = find_always_executed_bbs (fun, false);
2658 : : }
2659 : 194353 : if (safe && bitmap_bit_p (always_executed_bbs, gimple_bb (call_stmt)->index))
2660 : 44237 : csum->m_arg_flow[argidx].safe_to_import_accesses = true;
2661 : : }
2662 : :
2663 : : }
2664 : 781674 : BITMAP_FREE (always_executed_bbs);
2665 : :
2666 : : /* TODO: Add early exit if we disqualified everything. This also requires
2667 : : that we either relax the restriction that
2668 : : ipa_param_adjustments.m_always_copy_start must be the number of PARM_DECLs
2669 : : or store the number of parameters to IPA-SRA function summary and use that
2670 : : when just removing params. */
2671 : :
2672 : 781674 : vec_safe_reserve_exact (ifs->m_parameters, param_count);
2673 : 781674 : ifs->m_parameters->quick_grow_cleared (param_count);
2674 : 3026606 : for (unsigned desc_index = 0; desc_index < param_count; desc_index++)
2675 : : {
2676 : 2244932 : gensum_param_desc *s = &(*param_descriptions)[desc_index];
2677 : 2244932 : isra_param_desc *d = &(*ifs->m_parameters)[desc_index];
2678 : :
2679 : 2244932 : d->param_size_limit = s->param_size_limit;
2680 : 2244932 : d->size_reached = s->nonarg_acc_size;
2681 : 2244932 : d->locally_unused = s->locally_unused;
2682 : 2244932 : d->split_candidate = s->split_candidate;
2683 : 2244932 : d->by_ref = s->by_ref;
2684 : 2244932 : d->remove_only_when_retval_removed = s->remove_only_when_retval_removed;
2685 : 2244932 : d->split_only_when_retval_removed = s->split_only_when_retval_removed;
2686 : 2244932 : d->conditionally_dereferenceable = s->conditionally_dereferenceable;
2687 : :
2688 : 2244932 : for (gensum_param_access *acc = s->accesses;
2689 : 2628080 : acc;
2690 : 383148 : acc = acc->next_sibling)
2691 : 383148 : copy_accesses_to_ipa_desc (acc, d);
2692 : : }
2693 : :
2694 : 781674 : if (dump_file)
2695 : 135 : dump_isra_param_descriptors (dump_file, node->decl, ifs, false);
2696 : 781674 : }
2697 : :
2698 : : /* Return true if there are any overlaps among certain accesses of DESC. If
2699 : : non-NULL, set *CERTAIN_ACCESS_PRESENT_P upon encountering a certain access
2700 : : too. DESC is assumed to be a split candidate that is not locally
2701 : : unused. */
2702 : :
2703 : : static bool
2704 : 159106 : overlapping_certain_accesses_p (isra_param_desc *desc,
2705 : : bool *certain_access_present_p)
2706 : : {
2707 : 159106 : unsigned pclen = vec_safe_length (desc->accesses);
2708 : 411941 : for (unsigned i = 0; i < pclen; i++)
2709 : : {
2710 : 254179 : param_access *a1 = (*desc->accesses)[i];
2711 : :
2712 : 254179 : if (!a1->certain)
2713 : 5111 : continue;
2714 : 249068 : if (certain_access_present_p)
2715 : 235119 : *certain_access_present_p = true;
2716 : 397192 : for (unsigned j = i + 1; j < pclen; j++)
2717 : : {
2718 : 149468 : param_access *a2 = (*desc->accesses)[j];
2719 : 149468 : if (a2->certain
2720 : 149388 : && a1->unit_offset < a2->unit_offset + a2->unit_size
2721 : 149079 : && a1->unit_offset + a1->unit_size > a2->unit_offset)
2722 : : return true;
2723 : : }
2724 : : }
2725 : : return false;
2726 : : }
2727 : :
2728 : : /* Check for any overlaps of certain param accesses among splitting candidates
2729 : : and signal an ICE if there are any. If CERTAIN_MUST_EXIST is set, also
2730 : : check that used splitting candidates have at least one certain access. */
2731 : :
2732 : : static void
2733 : 2468495 : verify_splitting_accesses (cgraph_node *node, bool certain_must_exist)
2734 : : {
2735 : 2468495 : isra_func_summary *ifs = func_sums->get (node);
2736 : 2468495 : if (!ifs || !ifs->m_candidate)
2737 : : return;
2738 : 1291693 : unsigned param_count = vec_safe_length (ifs->m_parameters);
2739 : 4193555 : for (unsigned pidx = 0; pidx < param_count; pidx++)
2740 : : {
2741 : 2901862 : isra_param_desc *desc = &(*ifs->m_parameters)[pidx];
2742 : 2901862 : if (!desc->split_candidate || desc->locally_unused)
2743 : 2756636 : continue;
2744 : :
2745 : 145226 : bool certain_access_present = !certain_must_exist;
2746 : 145226 : if (overlapping_certain_accesses_p (desc, &certain_access_present))
2747 : 0 : internal_error ("function %qs, parameter %u, has IPA-SRA accesses "
2748 : : "which overlap", node->dump_name (), pidx);
2749 : 145226 : if (!certain_access_present)
2750 : 0 : internal_error ("function %qs, parameter %u, is used but does not "
2751 : : "have any certain IPA-SRA access",
2752 : : node->dump_name (), pidx);
2753 : : }
2754 : : }
2755 : :
2756 : : /* Intraprocedural part of IPA-SRA analysis. Scan bodies of all functions in
2757 : : this compilation unit and create summary structures describing IPA-SRA
2758 : : opportunities and constraints in them. */
2759 : :
2760 : : static void
2761 : 120640 : ipa_sra_generate_summary (void)
2762 : : {
2763 : 120640 : struct cgraph_node *node;
2764 : :
2765 : 120640 : gcc_checking_assert (!func_sums);
2766 : 120640 : gcc_checking_assert (!call_sums);
2767 : 120640 : func_sums
2768 : 120640 : = (new (ggc_alloc_no_dtor <ipa_sra_function_summaries> ())
2769 : 120640 : ipa_sra_function_summaries (symtab, true));
2770 : 120640 : call_sums = new ipa_sra_call_summaries (symtab);
2771 : :
2772 : 1301730 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
2773 : 1181090 : ipa_sra_summarize_function (node);
2774 : 120640 : return;
2775 : : }
2776 : :
2777 : : /* Write intraprocedural analysis information about E and all of its outgoing
2778 : : edges into a stream for LTO WPA. */
2779 : :
2780 : : static void
2781 : 275733 : isra_write_edge_summary (output_block *ob, cgraph_edge *e)
2782 : : {
2783 : 275733 : isra_call_summary *csum = call_sums->get (e);
2784 : 275733 : unsigned input_count = csum->m_arg_flow.length ();
2785 : 275733 : streamer_write_uhwi (ob, input_count);
2786 : 746205 : for (unsigned i = 0; i < input_count; i++)
2787 : : {
2788 : 470472 : isra_param_flow *ipf = &csum->m_arg_flow[i];
2789 : 470472 : streamer_write_hwi (ob, ipf->length);
2790 : 470472 : bitpack_d bp = bitpack_create (ob->main_stream);
2791 : 544595 : for (int j = 0; j < ipf->length; j++)
2792 : 74123 : bp_pack_value (&bp, ipf->inputs[j], 8);
2793 : 470472 : bp_pack_value (&bp, ipf->aggregate_pass_through, 1);
2794 : 470472 : bp_pack_value (&bp, ipf->pointer_pass_through, 1);
2795 : 470472 : bp_pack_value (&bp, ipf->safe_to_import_accesses, 1);
2796 : 470472 : bp_pack_value (&bp, ipf->constructed_for_calls, 1);
2797 : 470472 : streamer_write_bitpack (&bp);
2798 : 470472 : streamer_write_uhwi (ob, ipf->unit_offset);
2799 : 470472 : streamer_write_uhwi (ob, ipf->unit_size);
2800 : : }
2801 : 275733 : bitpack_d bp = bitpack_create (ob->main_stream);
2802 : 275733 : bp_pack_value (&bp, csum->m_return_ignored, 1);
2803 : 275733 : bp_pack_value (&bp, csum->m_return_returned, 1);
2804 : 275733 : bp_pack_value (&bp, csum->m_bit_aligned_arg, 1);
2805 : 275733 : bp_pack_value (&bp, csum->m_before_any_store, 1);
2806 : 275733 : streamer_write_bitpack (&bp);
2807 : 275733 : }
2808 : :
2809 : : /* Write intraprocedural analysis information about NODE and all of its outgoing
2810 : : edges into a stream for LTO WPA. */
2811 : :
2812 : : static void
2813 : 46166 : isra_write_node_summary (output_block *ob, cgraph_node *node)
2814 : : {
2815 : 46166 : isra_func_summary *ifs = func_sums->get (node);
2816 : 46166 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
2817 : 46166 : int node_ref = lto_symtab_encoder_encode (encoder, node);
2818 : 46166 : streamer_write_uhwi (ob, node_ref);
2819 : :
2820 : 46166 : unsigned param_desc_count = vec_safe_length (ifs->m_parameters);
2821 : 46166 : streamer_write_uhwi (ob, param_desc_count);
2822 : 292965 : for (unsigned i = 0; i < param_desc_count; i++)
2823 : : {
2824 : 246799 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
2825 : 246799 : unsigned access_count = vec_safe_length (desc->accesses);
2826 : 246799 : streamer_write_uhwi (ob, access_count);
2827 : 249892 : for (unsigned j = 0; j < access_count; j++)
2828 : : {
2829 : 3093 : param_access *acc = (*desc->accesses)[j];
2830 : 3093 : stream_write_tree (ob, acc->type, true);
2831 : 3093 : stream_write_tree (ob, acc->alias_ptr_type, true);
2832 : 3093 : streamer_write_uhwi (ob, acc->unit_offset);
2833 : 3093 : streamer_write_uhwi (ob, acc->unit_size);
2834 : 3093 : bitpack_d bp = bitpack_create (ob->main_stream);
2835 : 3093 : bp_pack_value (&bp, acc->certain, 1);
2836 : 3093 : bp_pack_value (&bp, acc->reverse, 1);
2837 : 3093 : streamer_write_bitpack (&bp);
2838 : : }
2839 : 246799 : streamer_write_uhwi (ob, desc->param_size_limit);
2840 : 246799 : streamer_write_uhwi (ob, desc->size_reached);
2841 : 246799 : gcc_assert (desc->safe_size == 0);
2842 : 246799 : bitpack_d bp = bitpack_create (ob->main_stream);
2843 : 246799 : bp_pack_value (&bp, desc->locally_unused, 1);
2844 : 246799 : bp_pack_value (&bp, desc->split_candidate, 1);
2845 : 246799 : bp_pack_value (&bp, desc->by_ref, 1);
2846 : 246799 : gcc_assert (!desc->not_specially_constructed);
2847 : 246799 : bp_pack_value (&bp, desc->remove_only_when_retval_removed, 1);
2848 : 246799 : bp_pack_value (&bp, desc->split_only_when_retval_removed, 1);
2849 : 246799 : bp_pack_value (&bp, desc->conditionally_dereferenceable, 1);
2850 : 246799 : gcc_assert (!desc->safe_size_set);
2851 : 246799 : streamer_write_bitpack (&bp);
2852 : : }
2853 : 46166 : bitpack_d bp = bitpack_create (ob->main_stream);
2854 : 46166 : bp_pack_value (&bp, ifs->m_candidate, 1);
2855 : 46166 : bp_pack_value (&bp, ifs->m_returns_value, 1);
2856 : 46166 : bp_pack_value (&bp, ifs->m_return_ignored, 1);
2857 : 46166 : gcc_assert (!ifs->m_queued);
2858 : 46166 : streamer_write_bitpack (&bp);
2859 : :
2860 : 320302 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
2861 : 274136 : isra_write_edge_summary (ob, e);
2862 : 47763 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
2863 : 1597 : isra_write_edge_summary (ob, e);
2864 : 46166 : }
2865 : :
2866 : : /* Write intraprocedural analysis information into a stream for LTO WPA. */
2867 : :
2868 : : static void
2869 : 18774 : ipa_sra_write_summary (void)
2870 : : {
2871 : 18774 : if (!func_sums || !call_sums)
2872 : 0 : return;
2873 : :
2874 : 18774 : struct output_block *ob = create_output_block (LTO_section_ipa_sra);
2875 : 18774 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
2876 : 18774 : ob->symbol = NULL;
2877 : :
2878 : 18774 : unsigned int count = 0;
2879 : 18774 : lto_symtab_encoder_iterator lsei;
2880 : 18774 : for (lsei = lsei_start_function_in_partition (encoder);
2881 : 108690 : !lsei_end_p (lsei);
2882 : 89916 : lsei_next_function_in_partition (&lsei))
2883 : : {
2884 : 89916 : cgraph_node *node = lsei_cgraph_node (lsei);
2885 : 89916 : if (node->has_gimple_body_p ()
2886 : 89916 : && func_sums->get (node) != NULL)
2887 : 46166 : count++;
2888 : : }
2889 : 18774 : streamer_write_uhwi (ob, count);
2890 : :
2891 : : /* Process all of the functions. */
2892 : 108690 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
2893 : 89916 : lsei_next_function_in_partition (&lsei))
2894 : : {
2895 : 89916 : cgraph_node *node = lsei_cgraph_node (lsei);
2896 : 89916 : if (node->has_gimple_body_p ()
2897 : 89916 : && func_sums->get (node) != NULL)
2898 : 46166 : isra_write_node_summary (ob, node);
2899 : : }
2900 : 18774 : streamer_write_char_stream (ob->main_stream, 0);
2901 : 18774 : produce_asm (ob);
2902 : 18774 : destroy_output_block (ob);
2903 : : }
2904 : :
2905 : : /* Read intraprocedural analysis information about E and all of its outgoing
2906 : : edges into a stream for LTO WPA. */
2907 : :
2908 : : static void
2909 : 249065 : isra_read_edge_summary (struct lto_input_block *ib, cgraph_edge *cs)
2910 : : {
2911 : 249065 : isra_call_summary *csum = call_sums->get_create (cs);
2912 : 249065 : unsigned input_count = streamer_read_uhwi (ib);
2913 : 249065 : csum->init_inputs (input_count);
2914 : 694728 : for (unsigned i = 0; i < input_count; i++)
2915 : : {
2916 : 445663 : isra_param_flow *ipf = &csum->m_arg_flow[i];
2917 : 445663 : ipf->length = streamer_read_hwi (ib);
2918 : 445663 : bitpack_d bp = streamer_read_bitpack (ib);
2919 : 512624 : for (int j = 0; j < ipf->length; j++)
2920 : 66961 : ipf->inputs[j] = bp_unpack_value (&bp, 8);
2921 : 445663 : ipf->aggregate_pass_through = bp_unpack_value (&bp, 1);
2922 : 445663 : ipf->pointer_pass_through = bp_unpack_value (&bp, 1);
2923 : 445663 : ipf->safe_to_import_accesses = bp_unpack_value (&bp, 1);
2924 : 445663 : ipf->constructed_for_calls = bp_unpack_value (&bp, 1);
2925 : 445663 : ipf->unit_offset = streamer_read_uhwi (ib);
2926 : 445663 : ipf->unit_size = streamer_read_uhwi (ib);
2927 : : }
2928 : 249065 : bitpack_d bp = streamer_read_bitpack (ib);
2929 : 249065 : csum->m_return_ignored = bp_unpack_value (&bp, 1);
2930 : 249065 : csum->m_return_returned = bp_unpack_value (&bp, 1);
2931 : 249065 : csum->m_bit_aligned_arg = bp_unpack_value (&bp, 1);
2932 : 249065 : csum->m_before_any_store = bp_unpack_value (&bp, 1);
2933 : 249065 : }
2934 : :
2935 : : /* Read intraprocedural analysis information about NODE and all of its outgoing
2936 : : edges into a stream for LTO WPA. */
2937 : :
2938 : : static void
2939 : 32370 : isra_read_node_info (struct lto_input_block *ib, cgraph_node *node,
2940 : : struct data_in *data_in)
2941 : : {
2942 : 32370 : isra_func_summary *ifs = func_sums->get_create (node);
2943 : 32370 : unsigned param_desc_count = streamer_read_uhwi (ib);
2944 : 32370 : if (param_desc_count > 0)
2945 : : {
2946 : 16281 : vec_safe_reserve_exact (ifs->m_parameters, param_desc_count);
2947 : 16281 : ifs->m_parameters->quick_grow_cleared (param_desc_count);
2948 : : }
2949 : 58082 : for (unsigned i = 0; i < param_desc_count; i++)
2950 : : {
2951 : 25712 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
2952 : 25712 : unsigned access_count = streamer_read_uhwi (ib);
2953 : 27330 : for (unsigned j = 0; j < access_count; j++)
2954 : : {
2955 : 1618 : param_access *acc = ggc_cleared_alloc<param_access> ();
2956 : 1618 : acc->type = stream_read_tree (ib, data_in);
2957 : 1618 : acc->alias_ptr_type = stream_read_tree (ib, data_in);
2958 : 1618 : acc->unit_offset = streamer_read_uhwi (ib);
2959 : 1618 : acc->unit_size = streamer_read_uhwi (ib);
2960 : 1618 : bitpack_d bp = streamer_read_bitpack (ib);
2961 : 1618 : acc->certain = bp_unpack_value (&bp, 1);
2962 : 1618 : acc->reverse = bp_unpack_value (&bp, 1);
2963 : 1618 : vec_safe_push (desc->accesses, acc);
2964 : : }
2965 : 25712 : desc->param_size_limit = streamer_read_uhwi (ib);
2966 : 25712 : desc->size_reached = streamer_read_uhwi (ib);
2967 : 25712 : desc->safe_size = 0;
2968 : 25712 : bitpack_d bp = streamer_read_bitpack (ib);
2969 : 25712 : desc->locally_unused = bp_unpack_value (&bp, 1);
2970 : 25712 : desc->split_candidate = bp_unpack_value (&bp, 1);
2971 : 25712 : desc->by_ref = bp_unpack_value (&bp, 1);
2972 : 25712 : desc->not_specially_constructed = 0;
2973 : 25712 : desc->remove_only_when_retval_removed = bp_unpack_value (&bp, 1);
2974 : 25712 : desc->split_only_when_retval_removed = bp_unpack_value (&bp, 1);
2975 : 25712 : desc->conditionally_dereferenceable = bp_unpack_value (&bp, 1);
2976 : 25712 : desc->safe_size_set = 0;
2977 : : }
2978 : 32370 : bitpack_d bp = streamer_read_bitpack (ib);
2979 : 32370 : ifs->m_candidate = bp_unpack_value (&bp, 1);
2980 : 32370 : ifs->m_returns_value = bp_unpack_value (&bp, 1);
2981 : 32370 : ifs->m_return_ignored = bp_unpack_value (&bp, 1);
2982 : 32370 : ifs->m_queued = 0;
2983 : :
2984 : 280183 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
2985 : 247813 : isra_read_edge_summary (ib, e);
2986 : 33622 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
2987 : 1252 : isra_read_edge_summary (ib, e);
2988 : 32370 : }
2989 : :
2990 : : /* Read IPA-SRA summaries from a section in file FILE_DATA of length LEN with
2991 : : data DATA. TODO: This function was copied almost verbatim from ipa-prop.cc,
2992 : : it should be possible to unify them somehow. */
2993 : :
2994 : : static void
2995 : 10221 : isra_read_summary_section (struct lto_file_decl_data *file_data,
2996 : : const char *data, size_t len)
2997 : : {
2998 : 10221 : const struct lto_function_header *header =
2999 : : (const struct lto_function_header *) data;
3000 : 10221 : const int cfg_offset = sizeof (struct lto_function_header);
3001 : 10221 : const int main_offset = cfg_offset + header->cfg_size;
3002 : 10221 : const int string_offset = main_offset + header->main_size;
3003 : 10221 : struct data_in *data_in;
3004 : 10221 : unsigned int i;
3005 : 10221 : unsigned int count;
3006 : :
3007 : 10221 : lto_input_block ib_main ((const char *) data + main_offset,
3008 : 10221 : header->main_size, file_data);
3009 : :
3010 : 10221 : data_in =
3011 : 20442 : lto_data_in_create (file_data, (const char *) data + string_offset,
3012 : 10221 : header->string_size, vNULL);
3013 : 10221 : count = streamer_read_uhwi (&ib_main);
3014 : :
3015 : 42591 : for (i = 0; i < count; i++)
3016 : : {
3017 : 32370 : unsigned int index;
3018 : 32370 : struct cgraph_node *node;
3019 : 32370 : lto_symtab_encoder_t encoder;
3020 : :
3021 : 32370 : index = streamer_read_uhwi (&ib_main);
3022 : 32370 : encoder = file_data->symtab_node_encoder;
3023 : 32370 : node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
3024 : : index));
3025 : 32370 : gcc_assert (node->definition);
3026 : 32370 : isra_read_node_info (&ib_main, node, data_in);
3027 : : }
3028 : 10221 : lto_free_section_data (file_data, LTO_section_ipa_sra, NULL, data,
3029 : : len);
3030 : 10221 : lto_data_in_delete (data_in);
3031 : 10221 : }
3032 : :
3033 : : /* Read intraprocedural analysis information into a stream for LTO WPA. */
3034 : :
3035 : : static void
3036 : 9569 : ipa_sra_read_summary (void)
3037 : : {
3038 : 9569 : struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
3039 : 9569 : struct lto_file_decl_data *file_data;
3040 : 9569 : unsigned int j = 0;
3041 : :
3042 : 9569 : gcc_checking_assert (!func_sums);
3043 : 9569 : gcc_checking_assert (!call_sums);
3044 : 9569 : func_sums
3045 : 9569 : = (new (ggc_alloc_no_dtor <ipa_sra_function_summaries> ())
3046 : 9569 : ipa_sra_function_summaries (symtab, true));
3047 : 9569 : call_sums = new ipa_sra_call_summaries (symtab);
3048 : :
3049 : 19801 : while ((file_data = file_data_vec[j++]))
3050 : : {
3051 : 10232 : size_t len;
3052 : 10232 : const char *data
3053 : 10232 : = lto_get_summary_section_data (file_data, LTO_section_ipa_sra, &len);
3054 : 10232 : if (data)
3055 : 10221 : isra_read_summary_section (file_data, data, len);
3056 : : }
3057 : 9569 : }
3058 : :
3059 : : /* Dump all IPA-SRA summary data for all cgraph nodes and edges to file F. If
3060 : : HINTS is true, also dump IPA-analysis computed hints. */
3061 : :
3062 : : static void
3063 : 63 : ipa_sra_dump_all_summaries (FILE *f, bool hints)
3064 : : {
3065 : 63 : cgraph_node *node;
3066 : 258 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
3067 : : {
3068 : 195 : fprintf (f, "\nSummary for node %s:\n", node->dump_name ());
3069 : :
3070 : 195 : isra_func_summary *ifs = func_sums->get (node);
3071 : 195 : if (!ifs)
3072 : 10 : fprintf (f, " Function does not have any associated IPA-SRA "
3073 : : "summary\n");
3074 : 185 : else if (!ifs->m_candidate)
3075 : 12 : fprintf (f, " Not a candidate function\n");
3076 : : else
3077 : : {
3078 : 173 : if (ifs->m_returns_value)
3079 : 95 : fprintf (f, " Returns value\n");
3080 : 173 : if (vec_safe_is_empty (ifs->m_parameters))
3081 : 41 : fprintf (f, " No parameter information. \n");
3082 : : else
3083 : 446 : for (unsigned i = 0; i < ifs->m_parameters->length (); ++i)
3084 : : {
3085 : 314 : fprintf (f, " Descriptor for parameter %i:\n", i);
3086 : 314 : dump_isra_param_descriptor (f, &(*ifs->m_parameters)[i], hints);
3087 : : }
3088 : 173 : fprintf (f, "\n");
3089 : : }
3090 : :
3091 : 195 : struct cgraph_edge *cs;
3092 : 375 : for (cs = node->callees; cs; cs = cs->next_callee)
3093 : : {
3094 : 180 : fprintf (f, " Summary for edge %s->%s:\n", cs->caller->dump_name (),
3095 : 180 : cs->callee->dump_name ());
3096 : 180 : isra_call_summary *csum = call_sums->get (cs);
3097 : 180 : if (csum)
3098 : 180 : csum->dump (f);
3099 : : else
3100 : 0 : fprintf (f, " Call summary is MISSING!\n");
3101 : : }
3102 : :
3103 : : }
3104 : 63 : fprintf (f, "\n\n");
3105 : 63 : }
3106 : :
3107 : : /* Perform function-scope viability tests that can be only made at IPA level
3108 : : and return false if the function is deemed unsuitable for IPA-SRA. */
3109 : :
3110 : : static bool
3111 : 960545 : ipa_sra_ipa_function_checks (cgraph_node *node)
3112 : : {
3113 : 960545 : if (!node->can_be_local_p ())
3114 : : {
3115 : 636153 : if (dump_file)
3116 : 87 : fprintf (dump_file, "Function %s disqualified because it cannot be "
3117 : : "made local.\n", node->dump_name ());
3118 : 636153 : return false;
3119 : : }
3120 : 324392 : if (!node->can_change_signature)
3121 : : {
3122 : 0 : if (dump_file)
3123 : 0 : fprintf (dump_file, "Function can not change signature.\n");
3124 : 0 : return false;
3125 : : }
3126 : :
3127 : : return true;
3128 : : }
3129 : :
3130 : : /* Issues found out by check_callers_for_issues. */
3131 : :
3132 : : struct caller_issues
3133 : : {
3134 : : /* The candidate being considered. */
3135 : : cgraph_node *candidate;
3136 : : /* There is a thunk among callers. */
3137 : : bool thunk;
3138 : : /* Set if there is at least one caller that is OK. */
3139 : : bool there_is_one;
3140 : : /* Call site with no available information. */
3141 : : bool unknown_callsite;
3142 : : /* Call from outside the candidate's comdat group. */
3143 : : bool call_from_outside_comdat;
3144 : : /* There is a bit-aligned load into one of non-gimple-typed arguments. */
3145 : : bool bit_aligned_aggregate_argument;
3146 : : };
3147 : :
3148 : : /* Worker for call_for_symbol_and_aliases, set any flags of passed caller_issues
3149 : : that apply. */
3150 : :
3151 : : static bool
3152 : 329750 : check_for_caller_issues (struct cgraph_node *node, void *data)
3153 : : {
3154 : 329750 : struct caller_issues *issues = (struct caller_issues *) data;
3155 : :
3156 : 1254628 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
3157 : : {
3158 : 925079 : if (cs->caller->thunk)
3159 : : {
3160 : 0 : issues->thunk = true;
3161 : : /* TODO: We should be able to process at least some types of
3162 : : thunks. */
3163 : 0 : return true;
3164 : : }
3165 : 925079 : if (issues->candidate->calls_comdat_local
3166 : 12 : && issues->candidate->same_comdat_group
3167 : 925091 : && !issues->candidate->in_same_comdat_group_p (cs->caller))
3168 : : {
3169 : 0 : issues->call_from_outside_comdat = true;
3170 : 0 : return true;
3171 : : }
3172 : :
3173 : 925079 : isra_call_summary *csum = call_sums->get (cs);
3174 : 925079 : if (!csum)
3175 : : {
3176 : 201 : issues->unknown_callsite = true;
3177 : 201 : return true;
3178 : : }
3179 : :
3180 : 924878 : if (csum->m_bit_aligned_arg)
3181 : 0 : issues->bit_aligned_aggregate_argument = true;
3182 : :
3183 : 924878 : issues->there_is_one = true;
3184 : : }
3185 : : return false;
3186 : : }
3187 : :
3188 : : /* Look at all incoming edges to NODE, including aliases and thunks and look
3189 : : for problems. Return true if NODE type should not be modified at all. */
3190 : :
3191 : : static bool
3192 : 324392 : check_all_callers_for_issues (cgraph_node *node)
3193 : : {
3194 : 324392 : struct caller_issues issues;
3195 : 324392 : memset (&issues, 0, sizeof (issues));
3196 : 324392 : issues.candidate = node;
3197 : :
3198 : 324392 : node->call_for_symbol_and_aliases (check_for_caller_issues, &issues, true);
3199 : 324392 : if (issues.unknown_callsite)
3200 : : {
3201 : 201 : if (dump_file && (dump_flags & TDF_DETAILS))
3202 : 0 : fprintf (dump_file, "A call of %s has not been analyzed. Disabling "
3203 : : "all modifications.\n", node->dump_name ());
3204 : 201 : return true;
3205 : : }
3206 : : /* TODO: We should be able to process at least some types of thunks. */
3207 : 324191 : if (issues.thunk)
3208 : : {
3209 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3210 : 0 : fprintf (dump_file, "A call of %s is through thunk, which are not"
3211 : : " handled yet. Disabling all modifications.\n",
3212 : : node->dump_name ());
3213 : 0 : return true;
3214 : : }
3215 : 324191 : if (issues.call_from_outside_comdat)
3216 : : {
3217 : 0 : if (dump_file)
3218 : 0 : fprintf (dump_file, "Function would become private comdat called "
3219 : : "outside of its comdat group.\n");
3220 : 0 : return true;
3221 : : }
3222 : :
3223 : 324191 : if (issues.bit_aligned_aggregate_argument)
3224 : : {
3225 : : /* Let's only remove parameters/return values from such functions.
3226 : : TODO: We could only prevent splitting the problematic parameters if
3227 : : anybody thinks it is worth it. */
3228 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3229 : 0 : fprintf (dump_file, "A call of %s has bit-aligned aggregate argument,"
3230 : : " disabling parameter splitting.\n", node->dump_name ());
3231 : :
3232 : 0 : isra_func_summary *ifs = func_sums->get (node);
3233 : 0 : gcc_checking_assert (ifs);
3234 : 0 : unsigned param_count = vec_safe_length (ifs->m_parameters);
3235 : 0 : for (unsigned i = 0; i < param_count; i++)
3236 : 0 : (*ifs->m_parameters)[i].split_candidate = false;
3237 : : }
3238 : 324191 : if (!issues.there_is_one)
3239 : : {
3240 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3241 : 0 : fprintf (dump_file, "There is no call to %s that we can modify. "
3242 : : "Disabling all modifications.\n", node->dump_name ());
3243 : 0 : return true;
3244 : : }
3245 : : return false;
3246 : : }
3247 : :
3248 : : /* Find the access with corresponding OFFSET and SIZE among accesses in
3249 : : PARAM_DESC and return it or NULL if such an access is not there. */
3250 : :
3251 : : static param_access *
3252 : 14954 : find_param_access (isra_param_desc *param_desc, unsigned offset, unsigned size)
3253 : : {
3254 : 14954 : unsigned pclen = vec_safe_length (param_desc->accesses);
3255 : :
3256 : : /* The search is linear but the number of stored accesses is bound by
3257 : : PARAM_IPA_SRA_MAX_REPLACEMENTS, so most probably 8. */
3258 : :
3259 : 15030 : for (unsigned i = 0; i < pclen; i++)
3260 : 14985 : if ((*param_desc->accesses)[i]->unit_offset == offset
3261 : 14985 : && (*param_desc->accesses)[i]->unit_size == size)
3262 : : return (*param_desc->accesses)[i];
3263 : :
3264 : : return NULL;
3265 : : }
3266 : :
3267 : : /* Return iff the total size of definite replacement SIZE would violate the
3268 : : limit set for it in PARAM. */
3269 : :
3270 : : static bool
3271 : 197423 : size_would_violate_limit_p (isra_param_desc *desc, unsigned size)
3272 : : {
3273 : 197423 : unsigned limit = desc->param_size_limit;
3274 : 0 : if (size > limit
3275 : 194535 : || (!desc->by_ref && size == limit))
3276 : 0 : return true;
3277 : : return false;
3278 : : }
3279 : :
3280 : : /* Increase reached size of DESC by SIZE or disqualify it if it would violate
3281 : : the set limit. IDX is the parameter number which is dumped when
3282 : : disqualifying. */
3283 : :
3284 : : static void
3285 : 12536 : bump_reached_size (isra_param_desc *desc, unsigned size, unsigned idx)
3286 : : {
3287 : 12536 : unsigned after = desc->size_reached + size;
3288 : 12536 : if (size_would_violate_limit_p (desc, after))
3289 : : {
3290 : 12504 : if (dump_file && (dump_flags & TDF_DETAILS))
3291 : 0 : fprintf (dump_file, " ...size limit reached, disqualifying "
3292 : : "candidate parameter %u\n", idx);
3293 : 12504 : desc->split_candidate = false;
3294 : 12504 : return;
3295 : : }
3296 : 32 : desc->size_reached = after;
3297 : : }
3298 : :
3299 : : /* Take all actions required to deal with an edge CS that represents a call to
3300 : : an unknown or un-analyzed function, for both parameter removal and
3301 : : splitting. */
3302 : :
3303 : : static void
3304 : 473837 : process_edge_to_unknown_caller (cgraph_edge *cs)
3305 : : {
3306 : 473837 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3307 : 473837 : gcc_checking_assert (from_ifs);
3308 : 473837 : isra_call_summary *csum = call_sums->get (cs);
3309 : :
3310 : 473837 : if (dump_file && (dump_flags & TDF_DETAILS))
3311 : 4 : fprintf (dump_file, "Processing an edge to an unknown caller from %s:\n",
3312 : : cs->caller->dump_name ());
3313 : :
3314 : 473837 : unsigned args_count = csum->m_arg_flow.length ();
3315 : 1164529 : for (unsigned i = 0; i < args_count; i++)
3316 : : {
3317 : 690692 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3318 : :
3319 : 690692 : if (ipf->pointer_pass_through)
3320 : : {
3321 : 190589 : isra_param_desc *param_desc
3322 : 190589 : = &(*from_ifs->m_parameters)[get_single_param_flow_source (ipf)];
3323 : 190589 : param_desc->locally_unused = false;
3324 : 190589 : param_desc->split_candidate = false;
3325 : 190589 : continue;
3326 : 190589 : }
3327 : 500103 : if (ipf->aggregate_pass_through)
3328 : : {
3329 : 6536 : unsigned idx = get_single_param_flow_source (ipf);
3330 : 6536 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3331 : :
3332 : 6536 : param_desc->locally_unused = false;
3333 : 6536 : if (!param_desc->split_candidate)
3334 : 1767 : continue;
3335 : 4769 : gcc_assert (!param_desc->by_ref);
3336 : 9538 : param_access *pacc = find_param_access (param_desc, ipf->unit_offset,
3337 : 4769 : ipf->unit_size);
3338 : 4769 : gcc_checking_assert (pacc);
3339 : 4769 : pacc->certain = true;
3340 : 4769 : if (overlapping_certain_accesses_p (param_desc, NULL))
3341 : : {
3342 : 566 : if (dump_file && (dump_flags & TDF_DETAILS))
3343 : 0 : fprintf (dump_file, " ...leading to overlap, "
3344 : : " disqualifying candidate parameter %u\n",
3345 : : idx);
3346 : 566 : param_desc->split_candidate = false;
3347 : : }
3348 : : else
3349 : 4203 : bump_reached_size (param_desc, pacc->unit_size, idx);
3350 : 4769 : ipf->aggregate_pass_through = false;
3351 : 4769 : continue;
3352 : 4769 : }
3353 : :
3354 : 580701 : for (int j = 0; j < ipf->length; j++)
3355 : : {
3356 : 87134 : int input_idx = ipf->inputs[j];
3357 : 87134 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3358 : : }
3359 : : }
3360 : 473837 : }
3361 : :
3362 : : /* Propagate parameter removal information through cross-SCC edge CS,
3363 : : i.e. decrease the use count in the caller parameter descriptor for each use
3364 : : in this call. */
3365 : :
3366 : : static void
3367 : 770911 : param_removal_cross_scc_edge (cgraph_edge *cs)
3368 : : {
3369 : 770911 : enum availability availability;
3370 : 770911 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3371 : 770911 : isra_func_summary *to_ifs = func_sums->get (callee);
3372 : 360830 : if (!to_ifs || !to_ifs->m_candidate
3373 : 321185 : || (availability < AVAIL_AVAILABLE)
3374 : 1092096 : || vec_safe_is_empty (to_ifs->m_parameters))
3375 : : {
3376 : 459027 : process_edge_to_unknown_caller (cs);
3377 : 459027 : return;
3378 : : }
3379 : 311884 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3380 : 311884 : gcc_checking_assert (from_ifs);
3381 : :
3382 : 311884 : isra_call_summary *csum = call_sums->get (cs);
3383 : 311884 : unsigned args_count = csum->m_arg_flow.length ();
3384 : 311884 : unsigned param_count = vec_safe_length (to_ifs->m_parameters);
3385 : :
3386 : 1090625 : for (unsigned i = 0; i < args_count; i++)
3387 : : {
3388 : 778741 : bool unused_in_callee;
3389 : 778741 : if (i < param_count)
3390 : 778737 : unused_in_callee = (*to_ifs->m_parameters)[i].locally_unused;
3391 : : else
3392 : : unused_in_callee = false;
3393 : :
3394 : 778737 : if (!unused_in_callee)
3395 : : {
3396 : 745983 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3397 : 960923 : for (int j = 0; j < ipf->length; j++)
3398 : : {
3399 : 214940 : int input_idx = ipf->inputs[j];
3400 : 214940 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3401 : : }
3402 : : }
3403 : : }
3404 : : }
3405 : :
3406 : : /* Unless it is already there, push NODE which is also described by IFS to
3407 : : STACK. */
3408 : :
3409 : : static void
3410 : 140147 : isra_push_node_to_stack (cgraph_node *node, isra_func_summary *ifs,
3411 : : vec<cgraph_node *> *stack)
3412 : : {
3413 : 0 : if (!ifs->m_queued)
3414 : : {
3415 : 139358 : ifs->m_queued = true;
3416 : 139358 : stack->safe_push (node);
3417 : : }
3418 : 0 : }
3419 : :
3420 : : /* If parameter with index INPUT_IDX is marked as locally unused, mark it as
3421 : : used and push CALLER on STACK. */
3422 : :
3423 : : static void
3424 : 25947 : isra_mark_caller_param_used (isra_func_summary *from_ifs, int input_idx,
3425 : : cgraph_node *caller, vec<cgraph_node *> *stack)
3426 : : {
3427 : 25947 : if ((*from_ifs->m_parameters)[input_idx].locally_unused)
3428 : : {
3429 : 3061 : (*from_ifs->m_parameters)[input_idx].locally_unused = false;
3430 : 5380 : isra_push_node_to_stack (caller, from_ifs, stack);
3431 : : }
3432 : 25947 : }
3433 : :
3434 : : /* Combine safe_size of DESC with SIZE and return true if it has changed. */
3435 : :
3436 : : static bool
3437 : 2127567 : update_safe_size (isra_param_desc *desc, unsigned size)
3438 : : {
3439 : 0 : if (!desc->safe_size_set)
3440 : : {
3441 : 685755 : desc->safe_size_set = 1;
3442 : 685755 : desc->safe_size = size;
3443 : 685755 : return true;
3444 : : }
3445 : 1441812 : if (desc->safe_size <= size)
3446 : : return false;
3447 : 18838 : desc->safe_size = size;
3448 : 18838 : return true;
3449 : : }
3450 : :
3451 : : /* Set all param hints in DESC to the pessimistic values. Return true if any
3452 : : hints that need to potentially trigger further propagation have changed. */
3453 : :
3454 : : static bool
3455 : 654564 : flip_all_hints_pessimistic (isra_param_desc *desc)
3456 : : {
3457 : 654564 : desc->not_specially_constructed = true;
3458 : 0 : return update_safe_size (desc, 0);
3459 : : }
3460 : :
3461 : : /* Because we have not analyzed or otherwise problematic caller, go over all
3462 : : parameter int flags of IFS describing a call graph node of a calllee and
3463 : : turn them pessimistic. Return true if any hints that need to potentially
3464 : : trigger further propagation have changed. */
3465 : :
3466 : : static bool
3467 : 412934 : flip_all_param_hints_pessimistic (isra_func_summary *ifs)
3468 : : {
3469 : 412934 : if (!ifs || !ifs->m_candidate)
3470 : : return false;
3471 : :
3472 : 23492 : bool ret = false;
3473 : 23492 : unsigned param_count = vec_safe_length (ifs->m_parameters);
3474 : :
3475 : 66007 : for (unsigned i = 0; i < param_count; i++)
3476 : 85030 : ret |= flip_all_hints_pessimistic (&(*ifs->m_parameters)[i]);
3477 : :
3478 : : return ret;
3479 : : }
3480 : :
3481 : : /* Propagate hints accross edge CS which ultimately leads to a node described
3482 : : by TO_IFS. Return true if any hints of the callee which should potentially
3483 : : trigger further propagation have changed. */
3484 : :
3485 : : static bool
3486 : 4515713 : propagate_param_hints_accross_call (cgraph_edge *cs, isra_func_summary *to_ifs)
3487 : : {
3488 : 4515713 : if (!to_ifs || !to_ifs->m_candidate)
3489 : : return false;
3490 : :
3491 : 1460173 : isra_call_summary *csum = call_sums->get (cs);
3492 : 1460173 : bool ret = false;
3493 : 1460173 : unsigned args_count = csum->m_arg_flow.length ();
3494 : 1460173 : unsigned param_count = vec_safe_length (to_ifs->m_parameters);
3495 : :
3496 : 4684031 : for (unsigned i = 0; i < param_count; i++)
3497 : : {
3498 : 3223858 : isra_param_desc *desc = &(*to_ifs->m_parameters)[i];
3499 : 3223858 : if (i >= args_count)
3500 : : {
3501 : 612049 : ret |= flip_all_hints_pessimistic (desc);
3502 : 612049 : continue;
3503 : : }
3504 : :
3505 : 2611809 : if (desc->by_ref)
3506 : : {
3507 : 1473021 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3508 : :
3509 : 1473021 : if (!ipf->constructed_for_calls)
3510 : 1131287 : desc->not_specially_constructed = true;
3511 : :
3512 : 1473021 : if (ipf->pointer_pass_through)
3513 : : {
3514 : 262578 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3515 : 262578 : int srcidx = get_single_param_flow_source (ipf);
3516 : 262578 : if (vec_safe_length (from_ifs->m_parameters) > (unsigned) srcidx)
3517 : : {
3518 : 162293 : isra_param_desc *src_d = &(*from_ifs->m_parameters)[srcidx];
3519 : 162293 : if (src_d->safe_size_set)
3520 : 324550 : ret |= update_safe_size (desc, src_d->safe_size);
3521 : : }
3522 : : else
3523 : 200570 : ret |= update_safe_size (desc, 0);
3524 : : }
3525 : 1210443 : else if (!ipf->aggregate_pass_through)
3526 : 2420884 : ret |= update_safe_size (desc, ipf->unit_size);
3527 : : else
3528 : : /* LTOing type-mismatched programs can end up here. */
3529 : 2 : ret |= update_safe_size (desc, 0);
3530 : : }
3531 : : }
3532 : : return ret;
3533 : : }
3534 : :
3535 : : /* Propagate hints from NODE described by FROM_IFS to all its (dorect) callees,
3536 : : push those that may need re-visiting onto STACK. */
3537 : :
3538 : : static void
3539 : 1274281 : propagate_hints_to_all_callees (cgraph_node *node, isra_func_summary *from_ifs,
3540 : : vec<cgraph_node *> *stack)
3541 : : {
3542 : 6202928 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
3543 : : {
3544 : 4928647 : enum availability availability;
3545 : 4928647 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3546 : 4928647 : isra_func_summary *to_ifs = func_sums->get (callee);
3547 : 4928647 : if (!from_ifs)
3548 : : {
3549 : 412934 : if (flip_all_param_hints_pessimistic (to_ifs)
3550 : 412934 : && ipa_edge_within_scc (cs))
3551 : 26 : isra_push_node_to_stack (callee, to_ifs, stack);
3552 : : }
3553 : 4515713 : else if (propagate_param_hints_accross_call (cs, to_ifs)
3554 : 4515713 : && ipa_edge_within_scc (cs))
3555 : 7841 : isra_push_node_to_stack (callee, to_ifs, stack);
3556 : : }
3557 : 1274281 : }
3558 : :
3559 : : /* Propagate information that any parameter is not used only locally within a
3560 : : SCC across CS to the caller, which must be in the same SCC as the
3561 : : callee. Push any callers that need to be re-processed to STACK. */
3562 : :
3563 : : static void
3564 : 32743 : propagate_used_across_scc_edge (cgraph_edge *cs, vec<cgraph_node *> *stack)
3565 : : {
3566 : 32743 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3567 : 32743 : if (!from_ifs || vec_safe_is_empty (from_ifs->m_parameters))
3568 : 14468 : return;
3569 : :
3570 : 18275 : isra_call_summary *csum = call_sums->get (cs);
3571 : 18275 : gcc_checking_assert (csum);
3572 : 18275 : unsigned args_count = csum->m_arg_flow.length ();
3573 : 18275 : enum availability availability;
3574 : 18275 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3575 : 18275 : isra_func_summary *to_ifs = func_sums->get (callee);
3576 : :
3577 : 18275 : unsigned param_count
3578 : 18187 : = (to_ifs && (availability >= AVAIL_AVAILABLE))
3579 : 36202 : ? vec_safe_length (to_ifs->m_parameters) : 0;
3580 : 60547 : for (unsigned i = 0; i < args_count; i++)
3581 : : {
3582 : 45593 : if (i < param_count
3583 : 42272 : && (*to_ifs->m_parameters)[i].locally_unused)
3584 : 3321 : continue;
3585 : :
3586 : : /* The argument is needed in the callee it, we must mark the parameter as
3587 : : used also in the caller and its callers within this SCC. */
3588 : 38951 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3589 : 64898 : for (int j = 0; j < ipf->length; j++)
3590 : : {
3591 : 25947 : int input_idx = ipf->inputs[j];
3592 : 25947 : isra_mark_caller_param_used (from_ifs, input_idx, cs->caller, stack);
3593 : : }
3594 : : }
3595 : : }
3596 : :
3597 : : /* Propagate information that any parameter is not used only locally within a
3598 : : SCC (i.e. is used also elsewhere) to all callers of NODE that are in the
3599 : : same SCC. Push any callers that need to be re-processed to STACK. */
3600 : :
3601 : : static bool
3602 : 1346726 : propagate_used_to_scc_callers (cgraph_node *node, void *data)
3603 : : {
3604 : 1346726 : vec<cgraph_node *> *stack = (vec<cgraph_node *> *) data;
3605 : 1346726 : cgraph_edge *cs;
3606 : 3199693 : for (cs = node->callers; cs; cs = cs->next_caller)
3607 : 1852967 : if (ipa_edge_within_scc (cs))
3608 : 32743 : propagate_used_across_scc_edge (cs, stack);
3609 : 1346726 : return false;
3610 : : }
3611 : :
3612 : : /* Return true iff all certain accesses in ARG_DESC are also present as
3613 : : certain accesses in PARAM_DESC. */
3614 : :
3615 : : static bool
3616 : 70 : all_callee_accesses_present_p (isra_param_desc *param_desc,
3617 : : isra_param_desc *arg_desc)
3618 : : {
3619 : 70 : unsigned aclen = vec_safe_length (arg_desc->accesses);
3620 : 74 : for (unsigned j = 0; j < aclen; j++)
3621 : : {
3622 : 50 : param_access *argacc = (*arg_desc->accesses)[j];
3623 : 50 : if (!argacc->certain)
3624 : 0 : continue;
3625 : 100 : param_access *pacc = find_param_access (param_desc, argacc->unit_offset,
3626 : 50 : argacc->unit_size);
3627 : 50 : if (!pacc
3628 : 5 : || !pacc->certain
3629 : 55 : || !types_compatible_p (argacc->type, pacc->type))
3630 : 46 : return false;
3631 : : }
3632 : : return true;
3633 : : }
3634 : :
3635 : : /* Type internal to function pull_accesses_from_callee. Unfortunately gcc 4.8
3636 : : does not allow instantiating an auto_vec with a type defined within a
3637 : : function so it is a global type. */
3638 : : enum acc_prop_kind {ACC_PROP_DONT, ACC_PROP_COPY, ACC_PROP_CERTAIN};
3639 : :
3640 : :
3641 : : /* Attempt to propagate all definite accesses from ARG_DESC to PARAM_DESC,
3642 : : (which belongs to CALLER) if they would not violate some constraint there.
3643 : : If successful, return NULL, otherwise return the string reason for failure
3644 : : (which can be written to the dump file). DELTA_OFFSET is the known offset
3645 : : of the actual argument withing the formal parameter (so of ARG_DESCS within
3646 : : PARAM_DESCS), ARG_SIZE is the size of the actual argument or zero, if not
3647 : : known. In case of success, set *CHANGE_P to true if propagation actually
3648 : : changed anything. */
3649 : :
3650 : : static const char *
3651 : 6146 : pull_accesses_from_callee (cgraph_node *caller, isra_param_desc *param_desc,
3652 : : isra_param_desc *arg_desc,
3653 : : unsigned delta_offset, unsigned arg_size,
3654 : : bool *change_p)
3655 : : {
3656 : 6146 : unsigned pclen = vec_safe_length (param_desc->accesses);
3657 : 6146 : unsigned aclen = vec_safe_length (arg_desc->accesses);
3658 : 6146 : unsigned prop_count = 0;
3659 : 6146 : unsigned prop_size = 0;
3660 : 6146 : bool change = false;
3661 : :
3662 : 6146 : auto_vec <enum acc_prop_kind, 8> prop_kinds (aclen);
3663 : 14400 : for (unsigned j = 0; j < aclen; j++)
3664 : : {
3665 : 8409 : param_access *argacc = (*arg_desc->accesses)[j];
3666 : 8409 : prop_kinds.safe_push (ACC_PROP_DONT);
3667 : :
3668 : 8409 : if (arg_size > 0
3669 : 2467 : && argacc->unit_offset + argacc->unit_size > arg_size)
3670 : : return "callee access outsize size boundary";
3671 : :
3672 : 8409 : if (!argacc->certain)
3673 : 361 : continue;
3674 : :
3675 : 8048 : unsigned offset = argacc->unit_offset + delta_offset;
3676 : : /* Given that accesses are initially stored according to increasing
3677 : : offset and decreasing size in case of equal offsets, the following
3678 : : searches could be written more efficiently if we kept the ordering
3679 : : when copying. But the number of accesses is capped at
3680 : : PARAM_IPA_SRA_MAX_REPLACEMENTS (so most likely 8) and the code gets
3681 : : messy quickly, so let's improve on that only if necessary. */
3682 : :
3683 : 8048 : bool exact_match = false;
3684 : 17629 : for (unsigned i = 0; i < pclen; i++)
3685 : : {
3686 : : /* Check for overlaps. */
3687 : 9736 : param_access *pacc = (*param_desc->accesses)[i];
3688 : 9736 : if (pacc->unit_offset == offset
3689 : 4350 : && pacc->unit_size == argacc->unit_size)
3690 : : {
3691 : 3323 : if (argacc->alias_ptr_type != pacc->alias_ptr_type
3692 : 3171 : || !types_compatible_p (argacc->type, pacc->type)
3693 : 6494 : || argacc->reverse != pacc->reverse)
3694 : 152 : return "propagated access types would not match existing ones";
3695 : :
3696 : 3171 : exact_match = true;
3697 : 3171 : if (!pacc->certain)
3698 : : {
3699 : 0 : prop_kinds[j] = ACC_PROP_CERTAIN;
3700 : 0 : prop_size += argacc->unit_size;
3701 : 0 : change = true;
3702 : : }
3703 : 3171 : continue;
3704 : : }
3705 : :
3706 : 6413 : if (offset < pacc->unit_offset + pacc->unit_size
3707 : 3659 : && offset + argacc->unit_size > pacc->unit_offset)
3708 : : {
3709 : : /* None permissible with load accesses, possible to fit into
3710 : : argument ones. */
3711 : 2109 : if (pacc->certain
3712 : 2106 : || offset < pacc->unit_offset
3713 : 2106 : || (offset + argacc->unit_size
3714 : : > pacc->unit_offset + pacc->unit_size))
3715 : : return "a propagated access would conflict in caller";
3716 : : }
3717 : : }
3718 : :
3719 : 7893 : if (!exact_match)
3720 : : {
3721 : 4722 : prop_kinds[j] = ACC_PROP_COPY;
3722 : 4722 : prop_count++;
3723 : 4722 : prop_size += argacc->unit_size;
3724 : 4722 : change = true;
3725 : : }
3726 : : }
3727 : :
3728 : 5991 : if (!change)
3729 : : return NULL;
3730 : :
3731 : 3688 : if ((prop_count + pclen
3732 : 3688 : > (unsigned) opt_for_fn (caller->decl, param_ipa_sra_max_replacements))
3733 : 9834 : || size_would_violate_limit_p (param_desc,
3734 : 3688 : param_desc->size_reached + prop_size))
3735 : : return "propagating accesses would violate the count or size limit";
3736 : :
3737 : 3661 : *change_p = true;
3738 : 8691 : for (unsigned j = 0; j < aclen; j++)
3739 : : {
3740 : 5030 : if (prop_kinds[j] == ACC_PROP_COPY)
3741 : : {
3742 : 4677 : param_access *argacc = (*arg_desc->accesses)[j];
3743 : :
3744 : 4677 : param_access *copy = ggc_cleared_alloc<param_access> ();
3745 : 4677 : copy->unit_offset = argacc->unit_offset + delta_offset;
3746 : 4677 : copy->unit_size = argacc->unit_size;
3747 : 4677 : copy->type = argacc->type;
3748 : 4677 : copy->alias_ptr_type = argacc->alias_ptr_type;
3749 : 4677 : copy->certain = true;
3750 : 4677 : copy->reverse = argacc->reverse;
3751 : 4677 : vec_safe_push (param_desc->accesses, copy);
3752 : : }
3753 : 353 : else if (prop_kinds[j] == ACC_PROP_CERTAIN)
3754 : : {
3755 : 0 : param_access *argacc = (*arg_desc->accesses)[j];
3756 : 0 : param_access *csp
3757 : 0 : = find_param_access (param_desc, argacc->unit_offset + delta_offset,
3758 : 0 : argacc->unit_size);
3759 : 0 : csp->certain = true;
3760 : : }
3761 : : }
3762 : :
3763 : 3661 : param_desc->size_reached += prop_size;
3764 : :
3765 : 3661 : return NULL;
3766 : 6146 : }
3767 : :
3768 : : /* Propagate parameter splitting information through call graph edge CS.
3769 : : Return true if any changes that might need to be propagated within SCCs have
3770 : : been made. The function also clears the aggregate_pass_through and
3771 : : pointer_pass_through in call summaries which do not need to be processed
3772 : : again if this CS is revisited when iterating while changes are propagated
3773 : : within an SCC. */
3774 : :
3775 : : static bool
3776 : 1421900 : param_splitting_across_edge (cgraph_edge *cs)
3777 : : {
3778 : 1421900 : bool res = false;
3779 : 1421900 : bool cross_scc = !ipa_edge_within_scc (cs);
3780 : 1421900 : enum availability availability;
3781 : 1421900 : cgraph_node *callee = cs->callee->function_symbol (&availability);
3782 : 1421900 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3783 : 1421900 : gcc_checking_assert (from_ifs && from_ifs->m_parameters);
3784 : :
3785 : 1421900 : isra_call_summary *csum = call_sums->get (cs);
3786 : 1421900 : gcc_checking_assert (csum);
3787 : 1421900 : unsigned args_count = csum->m_arg_flow.length ();
3788 : 1421900 : isra_func_summary *to_ifs = func_sums->get (callee);
3789 : 1421900 : unsigned param_count
3790 : 752063 : = ((to_ifs && to_ifs->m_candidate && (availability >= AVAIL_AVAILABLE))
3791 : 2090090 : ? vec_safe_length (to_ifs->m_parameters)
3792 : 1421900 : : 0);
3793 : :
3794 : 1421900 : if (dump_file && (dump_flags & TDF_DETAILS))
3795 : 18 : fprintf (dump_file, "Splitting across %s->%s:\n",
3796 : 9 : cs->caller->dump_name (), callee->dump_name ());
3797 : :
3798 : : unsigned i;
3799 : 2805661 : for (i = 0; (i < args_count) && (i < param_count); i++)
3800 : : {
3801 : 1383761 : isra_param_desc *arg_desc = &(*to_ifs->m_parameters)[i];
3802 : 1383761 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3803 : :
3804 : 1383761 : if (arg_desc->locally_unused)
3805 : : {
3806 : 61255 : if (dump_file && (dump_flags & TDF_DETAILS))
3807 : 0 : fprintf (dump_file, " ->%u: unused in callee\n", i);
3808 : 61255 : ipf->pointer_pass_through = false;
3809 : 61255 : continue;
3810 : : }
3811 : :
3812 : 1322506 : if (ipf->pointer_pass_through)
3813 : : {
3814 : 166425 : int idx = get_single_param_flow_source (ipf);
3815 : 166425 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3816 : 166425 : if (!param_desc->split_candidate)
3817 : 115844 : continue;
3818 : 50581 : gcc_assert (param_desc->by_ref);
3819 : :
3820 : 50581 : if (!arg_desc->split_candidate || !arg_desc->by_ref)
3821 : : {
3822 : 43878 : if (dump_file && (dump_flags & TDF_DETAILS))
3823 : 2 : fprintf (dump_file, " %u->%u: not candidate or not by "
3824 : : "reference in callee\n", idx, i);
3825 : 43878 : param_desc->split_candidate = false;
3826 : 43878 : ipf->pointer_pass_through = false;
3827 : 43878 : res = true;
3828 : : }
3829 : 6703 : else if (!ipf->safe_to_import_accesses)
3830 : : {
3831 : 1741 : if (!csum->m_before_any_store
3832 : 1741 : || !all_callee_accesses_present_p (param_desc, arg_desc))
3833 : : {
3834 : 1717 : if (dump_file && (dump_flags & TDF_DETAILS))
3835 : 0 : fprintf (dump_file, " %u->%u: cannot import accesses.\n",
3836 : : idx, i);
3837 : 1717 : param_desc->split_candidate = false;
3838 : 1717 : ipf->pointer_pass_through = false;
3839 : 1717 : res = true;
3840 : :
3841 : : }
3842 : : else
3843 : : {
3844 : 24 : if (dump_file && (dump_flags & TDF_DETAILS))
3845 : 0 : fprintf (dump_file, " %u->%u: verified callee accesses "
3846 : : "present.\n", idx, i);
3847 : 24 : if (cross_scc)
3848 : 0 : ipf->pointer_pass_through = false;
3849 : : }
3850 : : }
3851 : : else
3852 : : {
3853 : 4962 : const char *pull_failure
3854 : 4962 : = pull_accesses_from_callee (cs->caller, param_desc, arg_desc,
3855 : : 0, 0, &res);
3856 : 4962 : if (pull_failure)
3857 : : {
3858 : 22 : if (dump_file && (dump_flags & TDF_DETAILS))
3859 : 0 : fprintf (dump_file, " %u->%u: by_ref access pull "
3860 : : "failed: %s.\n", idx, i, pull_failure);
3861 : 22 : param_desc->split_candidate = false;
3862 : 22 : ipf->pointer_pass_through = false;
3863 : 22 : res = true;
3864 : : }
3865 : : else
3866 : : {
3867 : 4940 : if (dump_file && (dump_flags & TDF_DETAILS))
3868 : 1 : fprintf (dump_file, " %u->%u: by_ref access pull "
3869 : : "succeeded.\n", idx, i);
3870 : 4940 : if (cross_scc)
3871 : 4923 : ipf->pointer_pass_through = false;
3872 : : }
3873 : : }
3874 : : }
3875 : 1156081 : else if (ipf->aggregate_pass_through)
3876 : : {
3877 : 31171 : int idx = get_single_param_flow_source (ipf);
3878 : 31171 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3879 : 31171 : if (!param_desc->split_candidate)
3880 : 21036 : continue;
3881 : 10135 : gcc_assert (!param_desc->by_ref);
3882 : 20270 : param_access *pacc = find_param_access (param_desc, ipf->unit_offset,
3883 : 10135 : ipf->unit_size);
3884 : 10135 : gcc_checking_assert (pacc);
3885 : :
3886 : 10135 : if (pacc->certain)
3887 : : {
3888 : 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3889 : 0 : fprintf (dump_file, " %u->%u: already certain\n", idx, i);
3890 : 0 : ipf->aggregate_pass_through = false;
3891 : : }
3892 : 10135 : else if (!arg_desc->split_candidate || arg_desc->by_ref)
3893 : : {
3894 : 8951 : if (dump_file && (dump_flags & TDF_DETAILS))
3895 : 0 : fprintf (dump_file, " %u->%u: not candidate or by "
3896 : : "reference in callee\n", idx, i);
3897 : :
3898 : 8951 : pacc->certain = true;
3899 : 8951 : if (overlapping_certain_accesses_p (param_desc, NULL))
3900 : : {
3901 : 618 : if (dump_file && (dump_flags & TDF_DETAILS))
3902 : 0 : fprintf (dump_file, " ...leading to overlap, "
3903 : : " disqualifying candidate parameter %u\n",
3904 : : idx);
3905 : 618 : param_desc->split_candidate = false;
3906 : : }
3907 : : else
3908 : 8333 : bump_reached_size (param_desc, pacc->unit_size, idx);
3909 : :
3910 : 8951 : ipf->aggregate_pass_through = false;
3911 : 8951 : res = true;
3912 : : }
3913 : : else
3914 : : {
3915 : 1184 : const char *pull_failure
3916 : 1184 : = pull_accesses_from_callee (cs->caller, param_desc, arg_desc,
3917 : : ipf->unit_offset,
3918 : : ipf->unit_size, &res);
3919 : 1184 : if (pull_failure)
3920 : : {
3921 : 160 : if (dump_file && (dump_flags & TDF_DETAILS))
3922 : 0 : fprintf (dump_file, " %u->%u: arg access pull "
3923 : : "failed: %s.\n", idx, i, pull_failure);
3924 : :
3925 : 160 : ipf->aggregate_pass_through = false;
3926 : 160 : pacc->certain = true;
3927 : :
3928 : 160 : if (overlapping_certain_accesses_p (param_desc, NULL))
3929 : : {
3930 : 160 : if (dump_file && (dump_flags & TDF_DETAILS))
3931 : 0 : fprintf (dump_file, " ...leading to overlap, "
3932 : : " disqualifying candidate parameter %u\n",
3933 : : idx);
3934 : 160 : param_desc->split_candidate = false;
3935 : : }
3936 : : else
3937 : 0 : bump_reached_size (param_desc, pacc->unit_size, idx);
3938 : :
3939 : 160 : res = true;
3940 : : }
3941 : : else
3942 : : {
3943 : 1024 : if (dump_file && (dump_flags & TDF_DETAILS))
3944 : 0 : fprintf (dump_file, " %u->%u: arg access pull "
3945 : : "succeeded.\n", idx, i);
3946 : 1024 : if (cross_scc)
3947 : 867 : ipf->aggregate_pass_through = false;
3948 : : }
3949 : : }
3950 : : }
3951 : : }
3952 : :
3953 : : /* Handle argument-parameter count mismatches. */
3954 : 2338001 : for (; (i < args_count); i++)
3955 : : {
3956 : 916101 : isra_param_flow *ipf = &csum->m_arg_flow[i];
3957 : :
3958 : 916101 : if (ipf->pointer_pass_through || ipf->aggregate_pass_through)
3959 : : {
3960 : 196513 : int idx = get_single_param_flow_source (ipf);
3961 : 196513 : ipf->pointer_pass_through = false;
3962 : 196513 : ipf->aggregate_pass_through = false;
3963 : 196513 : isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
3964 : 196513 : if (!param_desc->split_candidate)
3965 : 196432 : continue;
3966 : :
3967 : 81 : if (dump_file && (dump_flags & TDF_DETAILS))
3968 : 0 : fprintf (dump_file, " %u->%u: no corresponding formal parameter\n",
3969 : : idx, i);
3970 : 81 : param_desc->split_candidate = false;
3971 : 81 : res = true;
3972 : : }
3973 : : }
3974 : 1421900 : return res;
3975 : : }
3976 : :
3977 : : /* Worker for call_for_symbol_and_aliases, look at all callers and if all their
3978 : : callers ignore the return value, or come from the same SCC and use the
3979 : : return value only to compute their return value, return false, otherwise
3980 : : return true. */
3981 : :
3982 : : static bool
3983 : 158431 : retval_used_p (cgraph_node *node, void *)
3984 : : {
3985 : 233211 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
3986 : : {
3987 : 207597 : isra_call_summary *csum = call_sums->get (cs);
3988 : 207597 : gcc_checking_assert (csum);
3989 : 207597 : if (csum->m_return_ignored)
3990 : 70861 : continue;
3991 : 136736 : if (!csum->m_return_returned)
3992 : : return true;
3993 : :
3994 : 21784 : isra_func_summary *from_ifs = func_sums->get (cs->caller);
3995 : 21784 : if (!from_ifs || !from_ifs->m_candidate)
3996 : : return true;
3997 : :
3998 : 16007 : if (!ipa_edge_within_scc (cs)
3999 : 16007 : && !from_ifs->m_return_ignored)
4000 : : return true;
4001 : : }
4002 : :
4003 : : return false;
4004 : : }
4005 : :
4006 : : /* Push into NEW_PARAMS all required parameter adjustment entries to copy or
4007 : : modify parameter which originally had index BASE_INDEX, in the adjustment
4008 : : vector of parent clone (if any) had PREV_CLONE_INDEX and was described by
4009 : : PREV_ADJUSTMENT. If IPA-CP has created a transformation summary for the
4010 : : original node, it needs to be passed in IPCP_TS, otherwise it should be
4011 : : NULL. If the parent clone is the original function, PREV_ADJUSTMENT is NULL
4012 : : and PREV_CLONE_INDEX is equal to BASE_INDEX. */
4013 : :
4014 : : static void
4015 : 260684 : push_param_adjustments_for_index (isra_func_summary *ifs, unsigned base_index,
4016 : : unsigned prev_clone_index,
4017 : : ipa_adjusted_param *prev_adjustment,
4018 : : ipcp_transformation *ipcp_ts,
4019 : : vec<ipa_adjusted_param, va_gc> **new_params)
4020 : : {
4021 : 260684 : isra_param_desc *desc = &(*ifs->m_parameters)[base_index];
4022 : 260684 : if (desc->locally_unused)
4023 : : {
4024 : 48112 : if (dump_file)
4025 : 27 : fprintf (dump_file, " Will remove parameter %u\n", base_index);
4026 : 48112 : return;
4027 : : }
4028 : :
4029 : 212572 : if (!desc->split_candidate)
4030 : : {
4031 : 168086 : ipa_adjusted_param adj;
4032 : 168086 : if (prev_adjustment)
4033 : : {
4034 : 1296 : adj = *prev_adjustment;
4035 : 1296 : adj.prev_clone_adjustment = true;
4036 : 1296 : adj.prev_clone_index = prev_clone_index;
4037 : : }
4038 : : else
4039 : : {
4040 : 166790 : memset (&adj, 0, sizeof (adj));
4041 : 166790 : adj.op = IPA_PARAM_OP_COPY;
4042 : 166790 : adj.base_index = base_index;
4043 : 166790 : adj.prev_clone_index = prev_clone_index;
4044 : : }
4045 : 168086 : vec_safe_push ((*new_params), adj);
4046 : 168086 : return;
4047 : : }
4048 : :
4049 : 44486 : if (dump_file)
4050 : 39 : fprintf (dump_file, " Will split parameter %u\n", base_index);
4051 : :
4052 : 44486 : gcc_assert (!prev_adjustment || prev_adjustment->op == IPA_PARAM_OP_COPY);
4053 : 44486 : unsigned aclen = vec_safe_length (desc->accesses);
4054 : 120316 : for (unsigned j = 0; j < aclen; j++)
4055 : : {
4056 : 75830 : param_access *pa = (*desc->accesses)[j];
4057 : 75830 : if (!pa->certain)
4058 : 4778 : continue;
4059 : :
4060 : 75046 : if (ipcp_ts)
4061 : : {
4062 : 10956 : ipa_argagg_value_list avl (ipcp_ts);
4063 : 10956 : tree value = avl.get_value (base_index, pa->unit_offset);
4064 : 10956 : if (value && !AGGREGATE_TYPE_P (pa->type))
4065 : : {
4066 : 3994 : if (dump_file)
4067 : 20 : fprintf (dump_file, " - omitting component at byte "
4068 : : "offset %u which is known to have a constant value\n ",
4069 : : pa->unit_offset);
4070 : 3994 : continue;
4071 : : }
4072 : : }
4073 : :
4074 : 71052 : if (dump_file)
4075 : 38 : fprintf (dump_file, " - component at byte offset %u, "
4076 : 38 : "size %u\n", pa->unit_offset, pa->unit_size);
4077 : :
4078 : 71052 : ipa_adjusted_param adj;
4079 : 71052 : memset (&adj, 0, sizeof (adj));
4080 : 71052 : adj.op = IPA_PARAM_OP_SPLIT;
4081 : 71052 : adj.base_index = base_index;
4082 : 71052 : adj.prev_clone_index = prev_clone_index;
4083 : 71052 : adj.param_prefix_index = IPA_PARAM_PREFIX_ISRA;
4084 : 71052 : adj.reverse = pa->reverse;
4085 : 71052 : adj.type = pa->type;
4086 : 71052 : adj.alias_ptr_type = pa->alias_ptr_type;
4087 : 71052 : adj.unit_offset = pa->unit_offset;
4088 : 71052 : vec_safe_push ((*new_params), adj);
4089 : : }
4090 : : }
4091 : :
4092 : : /* Worker for all call_for_symbol_thunks_and_aliases. Set calls_comdat_local
4093 : : flag of all callers of NODE. */
4094 : :
4095 : : static bool
4096 : 1 : mark_callers_calls_comdat_local (struct cgraph_node *node, void *)
4097 : : {
4098 : 3 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4099 : 2 : cs->caller->calls_comdat_local = true;
4100 : 1 : return false;
4101 : : }
4102 : :
4103 : : /* Remove any IPA-CP results stored in TS that are associated with removed
4104 : : parameters as marked in IFS. */
4105 : :
4106 : : static void
4107 : 6822 : zap_useless_ipcp_results (const isra_func_summary *ifs, ipcp_transformation *ts)
4108 : : {
4109 : 6822 : ts->remove_argaggs_if ([ifs](const ipa_argagg_value &v)
4110 : : {
4111 : 9172 : return (*ifs->m_parameters)[v.index].locally_unused;
4112 : : });
4113 : :
4114 : 6822 : bool useful_vr = false;
4115 : 6822 : unsigned count = vec_safe_length (ts->m_vr);
4116 : 23093 : for (unsigned i = 0; i < count; i++)
4117 : 16271 : if ((*ts->m_vr)[i].known_p ())
4118 : : {
4119 : 11638 : const isra_param_desc *desc = &(*ifs->m_parameters)[i];
4120 : 11638 : if (desc->locally_unused)
4121 : 1572 : (*ts->m_vr)[i].set_unknown ();
4122 : : else
4123 : : useful_vr = true;
4124 : : }
4125 : 6822 : if (!useful_vr)
4126 : 1135 : ts->m_vr = NULL;
4127 : 6822 : }
4128 : :
4129 : : /* Do final processing of results of IPA propagation regarding NODE, clone it
4130 : : if appropriate. */
4131 : :
4132 : : static void
4133 : 1197584 : process_isra_node_results (cgraph_node *node,
4134 : : hash_map<const char *, unsigned> *clone_num_suffixes)
4135 : : {
4136 : 1197584 : isra_func_summary *ifs = func_sums->get (node);
4137 : 1197584 : if (!ifs || !ifs->m_candidate)
4138 : 1103628 : return;
4139 : :
4140 : 324191 : auto_vec<bool, 16> surviving_params;
4141 : 324191 : bool check_surviving = false;
4142 : 324191 : clone_info *cinfo = clone_info::get (node);
4143 : 324191 : if (cinfo && cinfo->param_adjustments)
4144 : : {
4145 : 11785 : check_surviving = true;
4146 : 11785 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
4147 : : }
4148 : :
4149 : 324191 : unsigned param_count = vec_safe_length (ifs->m_parameters);
4150 : 324191 : bool will_change_function = false;
4151 : 324191 : if (ifs->m_returns_value && ifs->m_return_ignored)
4152 : : will_change_function = true;
4153 : : else
4154 : 758456 : for (unsigned i = 0; i < param_count; i++)
4155 : : {
4156 : 528221 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
4157 : 528221 : if ((desc->locally_unused || desc->split_candidate)
4158 : : /* Make sure we do not clone just to attempt to remove an already
4159 : : removed unused argument. */
4160 : 528221 : && (!check_surviving
4161 : 461598 : || (i < surviving_params.length ()
4162 : 2042 : && surviving_params[i])))
4163 : : {
4164 : : will_change_function = true;
4165 : : break;
4166 : : }
4167 : : }
4168 : 298900 : if (!will_change_function)
4169 : 230235 : return;
4170 : :
4171 : 93956 : if (dump_file)
4172 : : {
4173 : 61 : fprintf (dump_file, "\nEvaluating analysis results for %s\n",
4174 : : node->dump_name ());
4175 : 61 : if (ifs->m_returns_value && ifs->m_return_ignored)
4176 : 15 : fprintf (dump_file, " Will remove return value.\n");
4177 : : }
4178 : :
4179 : 93956 : ipcp_transformation *ipcp_ts = ipcp_get_transformation_summary (node);
4180 : 93956 : if (ipcp_ts)
4181 : 6822 : zap_useless_ipcp_results (ifs, ipcp_ts);
4182 : 93956 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
4183 : 93956 : if (ipa_param_adjustments *old_adjustments
4184 : 93956 : = cinfo ? cinfo->param_adjustments : NULL)
4185 : : {
4186 : 1761 : unsigned old_adj_len = vec_safe_length (old_adjustments->m_adj_params);
4187 : 4275 : for (unsigned i = 0; i < old_adj_len; i++)
4188 : : {
4189 : 2514 : ipa_adjusted_param *old_adj = &(*old_adjustments->m_adj_params)[i];
4190 : 2514 : push_param_adjustments_for_index (ifs, old_adj->base_index, i,
4191 : : old_adj, ipcp_ts, &new_params);
4192 : : }
4193 : : }
4194 : : else
4195 : 350365 : for (unsigned i = 0; i < param_count; i++)
4196 : 258170 : push_param_adjustments_for_index (ifs, i, i, NULL, ipcp_ts, &new_params);
4197 : :
4198 : 93956 : ipa_param_adjustments *new_adjustments
4199 : 93956 : = (new (ggc_alloc <ipa_param_adjustments> ())
4200 : : ipa_param_adjustments (new_params, param_count,
4201 : 93956 : ifs->m_returns_value && ifs->m_return_ignored));
4202 : :
4203 : 93956 : if (dump_file && (dump_flags & TDF_DETAILS))
4204 : : {
4205 : 9 : fprintf (dump_file, "\n Created adjustments:\n");
4206 : 9 : new_adjustments->dump (dump_file);
4207 : : }
4208 : :
4209 : 281868 : unsigned &suffix_counter = clone_num_suffixes->get_or_insert (
4210 : 93956 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (
4211 : : node->decl)));
4212 : 93956 : auto_vec<cgraph_edge *> callers = node->collect_callers ();
4213 : 93956 : cgraph_node *new_node
4214 : 93956 : = node->create_virtual_clone (callers, NULL, new_adjustments, "isra",
4215 : : suffix_counter);
4216 : 93956 : suffix_counter++;
4217 : 93956 : if (node->calls_comdat_local && node->same_comdat_group)
4218 : : {
4219 : 1 : new_node->add_to_same_comdat_group (node);
4220 : 1 : new_node->call_for_symbol_and_aliases (mark_callers_calls_comdat_local,
4221 : : NULL, true);
4222 : : }
4223 : 93956 : new_node->calls_comdat_local = node->calls_comdat_local;
4224 : :
4225 : 93956 : if (dump_file)
4226 : 61 : fprintf (dump_file, " Created new node %s\n", new_node->dump_name ());
4227 : 93956 : callers.release ();
4228 : 324191 : }
4229 : :
4230 : : /* If INDICES is not empty, dump a combination of NODE's dump_name and MSG
4231 : : followed by the list of numbers in INDICES. */
4232 : :
4233 : : static void
4234 : 550242 : dump_list_of_param_indices (const cgraph_node *node, const char* msg,
4235 : : const vec<unsigned> &indices)
4236 : : {
4237 : 550242 : if (indices.is_empty ())
4238 : : return;
4239 : 1 : fprintf (dump_file, "The following parameters of %s %s:", node->dump_name (),
4240 : : msg);
4241 : 4 : for (unsigned i : indices)
4242 : 1 : fprintf (dump_file, " %u", i);
4243 : 1 : fprintf (dump_file, "\n");
4244 : : }
4245 : :
4246 : : /* Check which parameters of NODE described by IFS have survived until IPA-SRA
4247 : : and disable transformations for those which have not or which should not
4248 : : transformed because the associated debug counter reached its limit. Return
4249 : : true if none survived or if there were no candidates to begin with.
4250 : : Additionally, also adjust parameter descriptions based on debug counters and
4251 : : hints propagated earlier. */
4252 : :
4253 : : static bool
4254 : 324191 : adjust_parameter_descriptions (cgraph_node *node, isra_func_summary *ifs)
4255 : : {
4256 : 324191 : bool ret = true;
4257 : 324191 : unsigned len = vec_safe_length (ifs->m_parameters);
4258 : 275121 : if (!len)
4259 : : return true;
4260 : :
4261 : 275121 : auto_vec<bool, 16> surviving_params;
4262 : 275121 : bool check_surviving = false;
4263 : 275121 : clone_info *cinfo = clone_info::get (node);
4264 : 275121 : if (cinfo && cinfo->param_adjustments)
4265 : : {
4266 : 11785 : check_surviving = true;
4267 : 11785 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
4268 : : }
4269 : 275121 : ipcp_transformation *ipcp_ts = ipcp_get_transformation_summary (node);
4270 : 275121 : auto_vec <unsigned> dump_dead_indices;
4271 : 275121 : auto_vec <unsigned> dump_bad_cond_indices;
4272 : 932066 : for (unsigned i = 0; i < len; i++)
4273 : : {
4274 : 656945 : isra_param_desc *desc = &(*ifs->m_parameters)[i];
4275 : 656945 : if (!dbg_cnt (ipa_sra_params))
4276 : : {
4277 : 0 : desc->locally_unused = false;
4278 : 0 : desc->split_candidate = false;
4279 : 0 : continue;
4280 : : }
4281 : :
4282 : 656945 : if (desc->split_only_when_retval_removed
4283 : 4636 : && !ifs->m_return_ignored)
4284 : : {
4285 : 0 : if (dump_file && (dump_flags & TDF_DETAILS)
4286 : 1244 : && (desc->locally_unused || desc->split_candidate))
4287 : 0 : dump_bad_cond_indices.safe_push (i);
4288 : :
4289 : 1244 : gcc_checking_assert (!desc->locally_unused
4290 : : || desc->remove_only_when_retval_removed);
4291 : 1244 : desc->locally_unused = false;
4292 : 1244 : desc->split_candidate = false;
4293 : 1244 : continue;
4294 : : }
4295 : 655701 : if (desc->remove_only_when_retval_removed
4296 : 34404 : && !ifs->m_return_ignored)
4297 : : {
4298 : 6 : if (dump_file && (dump_flags & TDF_DETAILS)
4299 : 27581 : && (desc->locally_unused || desc->split_candidate))
4300 : 0 : dump_bad_cond_indices.safe_push (i);
4301 : :
4302 : 27581 : desc->locally_unused = false;
4303 : : }
4304 : 655701 : if (check_surviving
4305 : 689322 : && (i >= surviving_params.length ()
4306 : 15802 : || !surviving_params[i]))
4307 : : {
4308 : : /* Even if the parameter was removed by a previous IPA pass, we do
4309 : : not clear locally_unused because if it really is unused, this
4310 : : information might be useful in callers. */
4311 : 21369 : desc->split_candidate = false;
4312 : :
4313 : 21369 : if (dump_file && (dump_flags & TDF_DETAILS))
4314 : 0 : dump_dead_indices.safe_push (i);
4315 : : }
4316 : :
4317 : 655701 : if (desc->split_candidate && desc->conditionally_dereferenceable)
4318 : : {
4319 : 722 : gcc_assert (desc->safe_size_set);
4320 : 1248 : for (param_access *pa : *desc->accesses)
4321 : 1140 : if ((pa->unit_offset + pa->unit_size) > desc->safe_size)
4322 : : {
4323 : 614 : if (dump_file && (dump_flags & TDF_DETAILS))
4324 : 1 : dump_bad_cond_indices.safe_push (i);
4325 : 614 : desc->split_candidate = false;
4326 : 614 : break;
4327 : : }
4328 : : }
4329 : :
4330 : 655701 : if (desc->split_candidate)
4331 : : {
4332 : 181199 : if (desc->by_ref && !desc->not_specially_constructed)
4333 : : {
4334 : 24513 : int extra_factor
4335 : 24513 : = opt_for_fn (node->decl,
4336 : : param_ipa_sra_ptrwrap_growth_factor);
4337 : 24513 : desc->param_size_limit = extra_factor * desc->param_size_limit;
4338 : : }
4339 : 181199 : if (size_would_violate_limit_p (desc, desc->size_reached))
4340 : 2869 : desc->split_candidate = false;
4341 : : }
4342 : :
4343 : : /* Avoid ICEs on size-mismatched VIEW_CONVERT_EXPRs when callers and
4344 : : callees don't agree on types in aggregates and we try to do both
4345 : : IPA-CP and IPA-SRA. */
4346 : 655701 : if (ipcp_ts && desc->split_candidate)
4347 : : {
4348 : 19838 : ipa_argagg_value_list avl (ipcp_ts);
4349 : 45596 : for (const param_access *pa : desc->accesses)
4350 : : {
4351 : 12369 : if (!pa->certain)
4352 : 799 : continue;
4353 : 11570 : tree value = avl.get_value (i, pa->unit_offset);
4354 : 11570 : if (value
4355 : 11570 : && ((tree_to_uhwi (TYPE_SIZE (TREE_TYPE (value)))
4356 : 4059 : / BITS_PER_UNIT)
4357 : 4059 : != pa->unit_size))
4358 : : {
4359 : 21 : desc->split_candidate = false;
4360 : 21 : if (dump_file && (dump_flags & TDF_DETAILS))
4361 : 0 : dump_dead_indices.safe_push (i);
4362 : : break;
4363 : : }
4364 : : }
4365 : : }
4366 : :
4367 : 655701 : if (desc->locally_unused || desc->split_candidate)
4368 : 656945 : ret = false;
4369 : : }
4370 : :
4371 : 275121 : dump_list_of_param_indices (node, "are dead on arrival or have a type "
4372 : : "mismatch with IPA-CP", dump_dead_indices);
4373 : 275121 : dump_list_of_param_indices (node, "fail additional requirements ",
4374 : : dump_bad_cond_indices);
4375 : :
4376 : 275121 : return ret;
4377 : 275121 : }
4378 : :
4379 : :
4380 : : /* Run the interprocedural part of IPA-SRA. */
4381 : :
4382 : : static unsigned int
4383 : 121589 : ipa_sra_analysis (void)
4384 : : {
4385 : 121589 : if (dump_file)
4386 : : {
4387 : 54 : fprintf (dump_file, "\n========== IPA-SRA IPA stage ==========\n");
4388 : 54 : ipa_sra_dump_all_summaries (dump_file, false);
4389 : : }
4390 : :
4391 : 121589 : gcc_checking_assert (func_sums);
4392 : 121589 : gcc_checking_assert (call_sums);
4393 : 121589 : cgraph_node **order = XCNEWVEC (cgraph_node *, symtab->cgraph_count);
4394 : 121589 : auto_vec <cgraph_node *, 16> stack;
4395 : 121589 : int node_scc_count = ipa_reduced_postorder (order, true, NULL);
4396 : :
4397 : : /* One sweep from callers to callees for return value removal. */
4398 : 1385181 : for (int i = node_scc_count - 1; i >= 0 ; i--)
4399 : : {
4400 : 1263592 : cgraph_node *scc_rep = order[i];
4401 : 1263592 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (scc_rep);
4402 : :
4403 : : /* Preliminary IPA function level checks. */
4404 : 5061147 : for (cgraph_node *v : cycle_nodes)
4405 : : {
4406 : 1270371 : isra_func_summary *ifs = func_sums->get (v);
4407 : 1270371 : if (!ifs || !ifs->m_candidate)
4408 : 309826 : continue;
4409 : 960545 : if (!ipa_sra_ipa_function_checks (v)
4410 : 960545 : || check_all_callers_for_issues (v))
4411 : 636354 : ifs->zap ();
4412 : : }
4413 : :
4414 : 3797555 : for (cgraph_node *v : cycle_nodes)
4415 : : {
4416 : 1270371 : isra_func_summary *ifs = func_sums->get (v);
4417 : 1270371 : if (!ifs || !ifs->m_candidate)
4418 : 946180 : continue;
4419 : 324191 : bool return_needed
4420 : 324191 : = (ifs->m_returns_value
4421 : 324191 : && (!dbg_cnt (ipa_sra_retvalues)
4422 : 158420 : || v->call_for_symbol_and_aliases (retval_used_p,
4423 : 324191 : NULL, true)));
4424 : 324191 : ifs->m_return_ignored = !return_needed;
4425 : 324191 : if (return_needed)
4426 : 265634 : isra_push_node_to_stack (v, ifs, &stack);
4427 : : }
4428 : :
4429 : 1396721 : while (!stack.is_empty ())
4430 : : {
4431 : 133129 : cgraph_node *node = stack.pop ();
4432 : 133129 : isra_func_summary *ifs = func_sums->get (node);
4433 : 133129 : gcc_checking_assert (ifs && ifs->m_queued);
4434 : 133129 : ifs->m_queued = false;
4435 : :
4436 : 834981 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
4437 : 701852 : if (ipa_edge_within_scc (cs)
4438 : 701852 : && call_sums->get (cs)->m_return_returned)
4439 : : {
4440 : 838 : enum availability av;
4441 : 838 : cgraph_node *callee = cs->callee->function_symbol (&av);
4442 : 838 : isra_func_summary *to_ifs = func_sums->get (callee);
4443 : 838 : if (to_ifs && to_ifs->m_return_ignored)
4444 : : {
4445 : 312 : to_ifs->m_return_ignored = false;
4446 : 624 : isra_push_node_to_stack (callee, to_ifs, &stack);
4447 : : }
4448 : : }
4449 : : }
4450 : :
4451 : : /* Parameter hint propagation. */
4452 : 3797555 : for (cgraph_node *v : cycle_nodes)
4453 : : {
4454 : 1270371 : isra_func_summary *ifs = func_sums->get (v);
4455 : 1270371 : propagate_hints_to_all_callees (v, ifs, &stack);
4456 : : }
4457 : :
4458 : 1267502 : while (!stack.is_empty ())
4459 : : {
4460 : 3910 : cgraph_node *node = stack.pop ();
4461 : 3910 : isra_func_summary *ifs = func_sums->get (node);
4462 : 3910 : gcc_checking_assert (ifs && ifs->m_queued);
4463 : 3910 : ifs->m_queued = false;
4464 : 3910 : propagate_hints_to_all_callees (node, ifs, &stack);
4465 : : }
4466 : :
4467 : 1263592 : cycle_nodes.release ();
4468 : : }
4469 : :
4470 : : /* One sweep from callees to callers for parameter removal and splitting. */
4471 : 1385181 : for (int i = 0; i < node_scc_count; i++)
4472 : : {
4473 : 1263592 : cgraph_node *scc_rep = order[i];
4474 : 1263592 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (scc_rep);
4475 : :
4476 : : /* First step of parameter removal. */
4477 : 5061147 : for (cgraph_node *v : cycle_nodes)
4478 : : {
4479 : 1270371 : isra_func_summary *ifs = func_sums->get (v);
4480 : 1270371 : if (!ifs || !ifs->m_candidate)
4481 : 946180 : continue;
4482 : 324191 : if (adjust_parameter_descriptions (v, ifs))
4483 : 168380 : continue;
4484 : 170621 : for (cgraph_edge *cs = v->indirect_calls; cs; cs = cs->next_callee)
4485 : 14810 : process_edge_to_unknown_caller (cs);
4486 : 935909 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
4487 : 780098 : if (!ipa_edge_within_scc (cs))
4488 : 770911 : param_removal_cross_scc_edge (cs);
4489 : : }
4490 : :
4491 : : /* Look at edges within the current SCC and propagate used-ness across
4492 : : them, pushing onto the stack all notes which might need to be
4493 : : revisited. */
4494 : 3797555 : for (cgraph_node *v : cycle_nodes)
4495 : 1270371 : v->call_for_symbol_thunks_and_aliases (propagate_used_to_scc_callers,
4496 : : &stack, true);
4497 : :
4498 : : /* Keep revisiting and pushing until nothing changes. */
4499 : 1265911 : while (!stack.is_empty ())
4500 : : {
4501 : 2319 : cgraph_node *v = stack.pop ();
4502 : 2319 : isra_func_summary *ifs = func_sums->get (v);
4503 : 2319 : gcc_checking_assert (ifs && ifs->m_queued);
4504 : 2319 : ifs->m_queued = false;
4505 : :
4506 : 2319 : v->call_for_symbol_thunks_and_aliases (propagate_used_to_scc_callers,
4507 : : &stack, true);
4508 : : }
4509 : :
4510 : : /* Parameter splitting. */
4511 : 1304916 : bool repeat_scc_access_propagation;
4512 : 1304916 : do
4513 : : {
4514 : 1304916 : repeat_scc_access_propagation = false;
4515 : 3925740 : for (cgraph_node *v : cycle_nodes)
4516 : : {
4517 : 1315908 : isra_func_summary *ifs = func_sums->get (v);
4518 : 2311778 : if (!ifs
4519 : 1008610 : || !ifs->m_candidate
4520 : 1685016 : || vec_safe_is_empty (ifs->m_parameters))
4521 : 995870 : continue;
4522 : 1741938 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
4523 : 1421900 : if (param_splitting_across_edge (cs))
4524 : 49512 : repeat_scc_access_propagation = true;
4525 : : }
4526 : : }
4527 : : while (repeat_scc_access_propagation);
4528 : :
4529 : 1263592 : if (flag_checking)
4530 : 3797504 : for (cgraph_node *v : cycle_nodes)
4531 : 1270354 : verify_splitting_accesses (v, true);
4532 : :
4533 : 1263592 : cycle_nodes.release ();
4534 : : }
4535 : :
4536 : 121589 : ipa_free_postorder_info ();
4537 : 121589 : free (order);
4538 : :
4539 : 121589 : if (dump_file)
4540 : : {
4541 : 54 : if (dump_flags & TDF_DETAILS)
4542 : : {
4543 : 9 : fprintf (dump_file, "\n========== IPA-SRA propagation final state "
4544 : : " ==========\n");
4545 : 9 : ipa_sra_dump_all_summaries (dump_file, true);
4546 : : }
4547 : 54 : fprintf (dump_file, "\n========== IPA-SRA decisions ==========\n");
4548 : : }
4549 : :
4550 : 121589 : hash_map<const char *, unsigned> *clone_num_suffixes
4551 : 121589 : = new hash_map<const char *, unsigned>;
4552 : :
4553 : 121589 : cgraph_node *node;
4554 : 1319173 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
4555 : 1197584 : process_isra_node_results (node, clone_num_suffixes);
4556 : :
4557 : 121589 : delete clone_num_suffixes;
4558 : 121589 : ggc_delete (func_sums);
4559 : 121589 : func_sums = NULL;
4560 : 121589 : delete call_sums;
4561 : 121589 : call_sums = NULL;
4562 : :
4563 : 121589 : if (dump_file)
4564 : 54 : fprintf (dump_file, "\n========== IPA SRA IPA analysis done "
4565 : : "==========\n\n");
4566 : 121589 : return 0;
4567 : 121589 : }
4568 : :
4569 : :
4570 : : const pass_data pass_data_ipa_sra =
4571 : : {
4572 : : IPA_PASS, /* type */
4573 : : "sra", /* name */
4574 : : OPTGROUP_NONE, /* optinfo_flags */
4575 : : TV_IPA_SRA, /* tv_id */
4576 : : 0, /* properties_required */
4577 : : 0, /* properties_provided */
4578 : : 0, /* properties_destroyed */
4579 : : 0, /* todo_flags_start */
4580 : : ( TODO_dump_symtab | TODO_remove_functions ), /* todo_flags_finish */
4581 : : };
4582 : :
4583 : : class pass_ipa_sra : public ipa_opt_pass_d
4584 : : {
4585 : : public:
4586 : 280114 : pass_ipa_sra (gcc::context *ctxt)
4587 : : : ipa_opt_pass_d (pass_data_ipa_sra, ctxt,
4588 : : ipa_sra_generate_summary, /* generate_summary */
4589 : : ipa_sra_write_summary, /* write_summary */
4590 : : ipa_sra_read_summary, /* read_summary */
4591 : : NULL , /* write_optimization_summary */
4592 : : NULL, /* read_optimization_summary */
4593 : : NULL, /* stmt_fixup */
4594 : : 0, /* function_transform_todo_flags_start */
4595 : : NULL, /* function_transform */
4596 : 280114 : NULL) /* variable_transform */
4597 : 280114 : {}
4598 : :
4599 : : /* opt_pass methods: */
4600 : 577796 : bool gate (function *) final override
4601 : : {
4602 : : /* TODO: We should remove the optimize check after we ensure we never run
4603 : : IPA passes when not optimizing. */
4604 : 577796 : return (flag_ipa_sra && optimize);
4605 : : }
4606 : :
4607 : 121589 : unsigned int execute (function *) final override
4608 : : {
4609 : 121589 : return ipa_sra_analysis ();
4610 : : }
4611 : :
4612 : : }; // class pass_ipa_sra
4613 : :
4614 : : } // anon namespace
4615 : :
4616 : : /* Intraprocedural part of IPA-SRA analysis. Scan function body of NODE and
4617 : : create a summary structure describing IPA-SRA opportunities and constraints
4618 : : in it. */
4619 : :
4620 : : static void
4621 : 1198159 : ipa_sra_summarize_function (cgraph_node *node)
4622 : : {
4623 : 1198159 : if (dump_file)
4624 : 186 : fprintf (dump_file, "Creating summary for %s/%i:\n", node->name (),
4625 : : node->order);
4626 : 1198159 : gcc_obstack_init (&gensum_obstack);
4627 : 1198159 : loaded_decls = new hash_set<tree>;
4628 : :
4629 : 1198159 : isra_func_summary *ifs = NULL;
4630 : 1198159 : unsigned count = 0;
4631 : 1198159 : if (ipa_sra_preliminary_function_checks (node))
4632 : : {
4633 : 967520 : ifs = func_sums->get_create (node);
4634 : 967520 : ifs->m_candidate = true;
4635 : 967520 : tree ret = TREE_TYPE (TREE_TYPE (node->decl));
4636 : 967520 : ifs->m_returns_value = (TREE_CODE (ret) != VOID_TYPE);
4637 : 967520 : for (tree parm = DECL_ARGUMENTS (node->decl);
4638 : 3212452 : parm;
4639 : 2244932 : parm = DECL_CHAIN (parm))
4640 : 2244932 : count++;
4641 : : }
4642 : 1198159 : auto_vec<gensum_param_desc, 16> param_descriptions (count);
4643 : :
4644 : 1198159 : struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
4645 : 1198159 : bool cfun_pushed = false;
4646 : 1198159 : if (count > 0)
4647 : : {
4648 : 781674 : decl2desc = new hash_map<tree, gensum_param_desc *>;
4649 : 781674 : param_descriptions.reserve_exact (count);
4650 : 781674 : param_descriptions.quick_grow_cleared (count);
4651 : :
4652 : 781674 : if (create_parameter_descriptors (node, ¶m_descriptions))
4653 : : {
4654 : 383962 : push_cfun (fun);
4655 : 383962 : cfun_pushed = true;
4656 : 383962 : final_bbs = BITMAP_ALLOC (NULL);
4657 : 383962 : bb_dereferences = XCNEWVEC (HOST_WIDE_INT,
4658 : : unsafe_by_ref_count
4659 : : * last_basic_block_for_fn (fun));
4660 : 383962 : aa_walking_limit = opt_for_fn (node->decl, param_ipa_max_aa_steps);
4661 : : }
4662 : : }
4663 : : /* Scan function is run even when there are no removal or splitting
4664 : : candidates so that we can calculate hints on call edges which can be
4665 : : useful in callees. */
4666 : 1198159 : scan_function (node, fun);
4667 : :
4668 : 1198159 : if (count > 0)
4669 : : {
4670 : 781674 : if (dump_file)
4671 : : {
4672 : 135 : dump_gensum_param_descriptors (dump_file, node->decl,
4673 : : ¶m_descriptions);
4674 : 135 : fprintf (dump_file, "----------------------------------------\n");
4675 : : }
4676 : :
4677 : 781674 : process_scan_results (node, fun, ifs, ¶m_descriptions);
4678 : :
4679 : 781674 : if (cfun_pushed)
4680 : 383962 : pop_cfun ();
4681 : 781674 : if (bb_dereferences)
4682 : : {
4683 : 383962 : free (bb_dereferences);
4684 : 383962 : bb_dereferences = NULL;
4685 : 383962 : BITMAP_FREE (final_bbs);
4686 : 383962 : final_bbs = NULL;
4687 : : }
4688 : : }
4689 : 1198159 : isra_analyze_all_outgoing_calls (node);
4690 : :
4691 : 2396318 : delete loaded_decls;
4692 : 1198159 : loaded_decls = NULL;
4693 : 1198159 : if (decl2desc)
4694 : : {
4695 : 781674 : delete decl2desc;
4696 : 781674 : decl2desc = NULL;
4697 : : }
4698 : 1198159 : obstack_free (&gensum_obstack, NULL);
4699 : 1198159 : if (dump_file)
4700 : 186 : fprintf (dump_file, "\n\n");
4701 : 1198159 : if (flag_checking)
4702 : 1198141 : verify_splitting_accesses (node, false);
4703 : 1198159 : return;
4704 : 1198159 : }
4705 : :
4706 : : ipa_opt_pass_d *
4707 : 280114 : make_pass_ipa_sra (gcc::context *ctxt)
4708 : : {
4709 : 280114 : return new pass_ipa_sra (ctxt);
4710 : : }
4711 : :
4712 : : /* Reset all state within ipa-sra.cc so that we can rerun the compiler
4713 : : within the same process. For use by toplev::finalize. */
4714 : :
4715 : : void
4716 : 252466 : ipa_sra_cc_finalize (void)
4717 : : {
4718 : 252466 : if (func_sums)
4719 : 8619 : ggc_delete (func_sums);
4720 : 252466 : func_sums = NULL;
4721 : 252466 : delete call_sums;
4722 : 252466 : call_sums = NULL;
4723 : 252466 : }
4724 : :
4725 : : #include "gt-ipa-sra.h"
|
