Line data Source code
1 : /* Interprocedural constant propagation
2 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
3 :
4 : Contributed by Razya Ladelsky <RAZYA@il.ibm.com> and Martin Jambor
5 : <mjambor@suse.cz>
6 :
7 : This file is part of GCC.
8 :
9 : GCC is free software; you can redistribute it and/or modify it under
10 : the terms of the GNU General Public License as published by the Free
11 : Software Foundation; either version 3, or (at your option) any later
12 : version.
13 :
14 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 : for more details.
18 :
19 : You should have received a copy of the GNU General Public License
20 : along with GCC; see the file COPYING3. If not see
21 : <http://www.gnu.org/licenses/>. */
22 :
23 : /* Interprocedural constant propagation (IPA-CP).
24 :
25 : The goal of this transformation is to
26 :
27 : 1) discover functions which are always invoked with some arguments with the
28 : same known constant values and modify the functions so that the
29 : subsequent optimizations can take advantage of the knowledge, and
30 :
31 : 2) partial specialization - create specialized versions of functions
32 : transformed in this way if some parameters are known constants only in
33 : certain contexts but the estimated tradeoff between speedup and cost size
34 : is deemed good.
35 :
36 : The algorithm also propagates types and attempts to perform type based
37 : devirtualization. Types are propagated much like constants.
38 :
39 : The algorithm basically consists of three stages. In the first, functions
40 : are analyzed one at a time and jump functions are constructed for all known
41 : call-sites. In the second phase, the pass propagates information from the
42 : jump functions across the call to reveal what values are available at what
43 : call sites, performs estimations of effects of known values on functions and
44 : their callees, and finally decides what specialized extra versions should be
45 : created. In the third, the special versions materialize and appropriate
46 : calls are redirected.
47 :
48 : The algorithm used is to a certain extent based on "Interprocedural Constant
49 : Propagation", by David Callahan, Keith D Cooper, Ken Kennedy, Linda Torczon,
50 : Comp86, pg 152-161 and "A Methodology for Procedure Cloning" by Keith D
51 : Cooper, Mary W. Hall, and Ken Kennedy.
52 :
53 :
54 : First stage - intraprocedural analysis
55 : =======================================
56 :
57 : This phase computes jump_function and modification flags.
58 :
59 : A jump function for a call-site represents the values passed as an actual
60 : arguments of a given call-site. In principle, there are three types of
61 : values:
62 :
63 : Pass through - the caller's formal parameter is passed as an actual
64 : argument, plus an operation on it can be performed.
65 : Constant - a constant is passed as an actual argument.
66 : Unknown - neither of the above.
67 :
68 : All jump function types are described in detail in ipa-prop.h, together with
69 : the data structures that represent them and methods of accessing them.
70 :
71 : ipcp_generate_summary() is the main function of the first stage.
72 :
73 : Second stage - interprocedural analysis
74 : ========================================
75 :
76 : This stage is itself divided into two phases. In the first, we propagate
77 : known values over the call graph, in the second, we make cloning decisions.
78 : It uses a different algorithm than the original Callahan's paper.
79 :
80 : First, we traverse the functions topologically from callers to callees and,
81 : for each strongly connected component (SCC), we propagate constants
82 : according to previously computed jump functions. We also record what known
83 : values depend on other known values and estimate local effects. Finally, we
84 : propagate cumulative information about these effects from dependent values
85 : to those on which they depend.
86 :
87 : Second, we again traverse the call graph in the same topological order and
88 : make clones for functions which we know are called with the same values in
89 : all contexts and decide about extra specialized clones of functions just for
90 : some contexts - these decisions are based on both local estimates and
91 : cumulative estimates propagated from callees.
92 :
93 : ipcp_propagate_stage() and ipcp_decision_stage() together constitute the
94 : third stage.
95 :
96 : Third phase - materialization of clones, call statement updates.
97 : ============================================
98 :
99 : This stage is currently performed by call graph code (mainly in cgraphunit.cc
100 : and tree-inline.cc) according to instructions inserted to the call graph by
101 : the second stage. */
102 :
103 : #define INCLUDE_ALGORITHM
104 : #include "config.h"
105 : #include "system.h"
106 : #include "coretypes.h"
107 : #include "backend.h"
108 : #include "tree.h"
109 : #include "gimple-expr.h"
110 : #include "gimple.h"
111 : #include "predict.h"
112 : #include "sreal.h"
113 : #include "alloc-pool.h"
114 : #include "tree-pass.h"
115 : #include "cgraph.h"
116 : #include "diagnostic.h"
117 : #include "fold-const.h"
118 : #include "gimple-iterator.h"
119 : #include "gimple-fold.h"
120 : #include "symbol-summary.h"
121 : #include "tree-vrp.h"
122 : #include "ipa-cp.h"
123 : #include "ipa-prop.h"
124 : #include "tree-pretty-print.h"
125 : #include "tree-inline.h"
126 : #include "ipa-fnsummary.h"
127 : #include "ipa-utils.h"
128 : #include "tree-ssa-ccp.h"
129 : #include "stringpool.h"
130 : #include "attribs.h"
131 : #include "dbgcnt.h"
132 : #include "symtab-clones.h"
133 : #include "gimple-range.h"
134 : #include "attr-callback.h"
135 : #include "lto-streamer.h"
136 : #include "callback-info.h"
137 :
138 : /* Allocation pools for values and their sources in ipa-cp. */
139 :
140 : object_allocator<ipcp_value<tree> > ipcp_cst_values_pool
141 : ("IPA-CP constant values");
142 :
143 : object_allocator<ipcp_value<ipa_polymorphic_call_context> >
144 : ipcp_poly_ctx_values_pool ("IPA-CP polymorphic contexts");
145 :
146 : object_allocator<ipcp_value_source<tree> > ipcp_sources_pool
147 : ("IPA-CP value sources");
148 :
149 : object_allocator<ipcp_agg_lattice> ipcp_agg_lattice_pool
150 : ("IPA_CP aggregate lattices");
151 :
152 : /* Original overall size of the program. */
153 :
154 : static long overall_size, orig_overall_size;
155 :
156 : /* Overall run-time of the part of the program or the compilation unit for
157 : which profile information is available, scaled by
158 : param_ipa_cp_min_recursive_probability. */
159 :
160 : static sreal overall_scaled_runtime;
161 :
162 : /* The maximum number of IPA-CP decision sweeps that any node requested in its
163 : param. */
164 : static int max_number_sweeps;
165 :
166 : /* Node name to unique clone suffix number map. */
167 : static hash_map<const char *, unsigned> *clone_num_suffixes;
168 :
169 : /* Return the param lattices structure corresponding to the Ith formal
170 : parameter of the function described by INFO. */
171 : static inline class ipcp_param_lattices *
172 32043888 : ipa_get_parm_lattices (class ipa_node_params *info, int i)
173 : {
174 64087776 : gcc_assert (i >= 0 && i < ipa_get_param_count (info));
175 32043888 : gcc_checking_assert (!info->ipcp_orig_node);
176 32043888 : return &(info->lattices[i]);
177 : }
178 :
179 : /* Return the lattice corresponding to the scalar value of the Ith formal
180 : parameter of the function described by INFO. */
181 : static inline ipcp_lattice<tree> *
182 6133701 : ipa_get_scalar_lat (class ipa_node_params *info, int i)
183 : {
184 6322755 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
185 6133701 : return &plats->itself;
186 : }
187 :
188 : /* Return the lattice corresponding to the scalar value of the Ith formal
189 : parameter of the function described by INFO. */
190 : static inline ipcp_lattice<ipa_polymorphic_call_context> *
191 826393 : ipa_get_poly_ctx_lat (class ipa_node_params *info, int i)
192 : {
193 826393 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
194 826393 : return &plats->ctxlat;
195 : }
196 :
197 : /* Return whether LAT is a lattice with a single constant and without an
198 : undefined value. */
199 :
200 : template <typename valtype>
201 : inline bool
202 10408125 : ipcp_lattice<valtype>::is_single_const ()
203 : {
204 3030781 : if (bottom || contains_variable || values_count != 1)
205 : return false;
206 : else
207 : return true;
208 : }
209 :
210 : /* Return true iff X and Y should be considered equal values by IPA-CP. */
211 :
212 : bool
213 1436039 : values_equal_for_ipcp_p (tree x, tree y)
214 : {
215 1436039 : gcc_checking_assert (x != NULL_TREE && y != NULL_TREE);
216 :
217 1436039 : if (x == y)
218 : return true;
219 :
220 632608 : if (TREE_CODE (x) == ADDR_EXPR
221 228470 : && TREE_CODE (y) == ADDR_EXPR
222 227734 : && (TREE_CODE (TREE_OPERAND (x, 0)) == CONST_DECL
223 179744 : || (TREE_CODE (TREE_OPERAND (x, 0)) == VAR_DECL
224 95657 : && DECL_IN_CONSTANT_POOL (TREE_OPERAND (x, 0))))
225 680598 : && (TREE_CODE (TREE_OPERAND (y, 0)) == CONST_DECL
226 13 : || (TREE_CODE (TREE_OPERAND (y, 0)) == VAR_DECL
227 8 : && DECL_IN_CONSTANT_POOL (TREE_OPERAND (y, 0)))))
228 47977 : return TREE_OPERAND (x, 0) == TREE_OPERAND (y, 0)
229 95698 : || operand_equal_p (DECL_INITIAL (TREE_OPERAND (x, 0)),
230 47721 : DECL_INITIAL (TREE_OPERAND (y, 0)), 0);
231 : else
232 584631 : return operand_equal_p (x, y, 0);
233 : }
234 :
235 : /* Print V which is extracted from a value in a lattice to F. This overloaded
236 : function is used to print tree constants. */
237 :
238 : static void
239 901 : print_ipcp_constant_value (FILE * f, tree v)
240 : {
241 0 : ipa_print_constant_value (f, v);
242 0 : }
243 :
244 : /* Print V which is extracted from a value in a lattice to F. This overloaded
245 : function is used to print constant polymorphic call contexts. */
246 :
247 : static void
248 214 : print_ipcp_constant_value (FILE * f, ipa_polymorphic_call_context v)
249 : {
250 214 : v.dump(f, false);
251 0 : }
252 :
253 : /* Print a lattice LAT to F. */
254 :
255 : template <typename valtype>
256 : void
257 2118 : ipcp_lattice<valtype>::print (FILE * f, bool dump_sources, bool dump_benefits)
258 : {
259 : ipcp_value<valtype> *val;
260 2118 : bool prev = false;
261 :
262 2118 : if (bottom)
263 : {
264 864 : fprintf (f, "BOTTOM\n");
265 864 : return;
266 : }
267 :
268 1254 : if (!values_count && !contains_variable)
269 : {
270 0 : fprintf (f, "TOP\n");
271 0 : return;
272 : }
273 :
274 1254 : if (contains_variable)
275 : {
276 961 : fprintf (f, "VARIABLE");
277 961 : prev = true;
278 961 : if (dump_benefits)
279 961 : fprintf (f, "\n");
280 : }
281 :
282 1917 : for (val = values; val; val = val->next)
283 : {
284 663 : if (dump_benefits && prev)
285 370 : fprintf (f, " ");
286 293 : else if (!dump_benefits && prev)
287 0 : fprintf (f, ", ");
288 : else
289 : prev = true;
290 :
291 663 : print_ipcp_constant_value (f, val->value);
292 :
293 663 : if (dump_sources)
294 : {
295 : ipcp_value_source<valtype> *s;
296 :
297 196 : if (val->self_recursion_generated_p ())
298 27 : fprintf (f, " [self_gen(%i), from:",
299 : val->self_recursion_generated_level);
300 : else
301 169 : fprintf (f, " [scc: %i, from:", val->scc_no);
302 410 : for (s = val->sources; s; s = s->next)
303 214 : fprintf (f, " %i(%f)", s->cs->caller->get_uid (),
304 428 : s->cs->sreal_frequency ().to_double ());
305 196 : fprintf (f, "]");
306 : }
307 :
308 663 : if (dump_benefits)
309 663 : fprintf (f, " [loc_time: %g, loc_size: %i, "
310 : "prop_time: %g, prop_size: %i]\n",
311 : val->local_time_benefit.to_double (), val->local_size_cost,
312 : val->prop_time_benefit.to_double (), val->prop_size_cost);
313 : }
314 1254 : if (!dump_benefits)
315 0 : fprintf (f, "\n");
316 : }
317 :
318 : /* Print VALUE to F in a form which in usual cases does not take thousands of
319 : characters. */
320 :
321 : static void
322 1542 : ipcp_print_widest_int (FILE *f, const widest_int &value)
323 : {
324 1542 : if (value == -1)
325 0 : fprintf (f, "-1");
326 1542 : else if (wi::arshift (value, 128) == -1)
327 : {
328 340 : char buf[35], *p = buf + 2;
329 340 : widest_int v = wi::zext (value, 128);
330 340 : size_t len;
331 340 : print_hex (v, buf);
332 340 : len = strlen (p);
333 340 : if (len == 32)
334 : {
335 340 : fprintf (f, "0xf..f");
336 10097 : while (*p == 'f')
337 9417 : ++p;
338 : }
339 : else
340 0 : fprintf (f, "0xf..f%0*d", (int) (32 - len), 0);
341 340 : fputs (p, f);
342 340 : }
343 : else
344 1202 : print_hex (value, f);
345 1542 : }
346 :
347 : void
348 970 : ipcp_bits_lattice::print (FILE *f)
349 : {
350 970 : if (bottom_p ())
351 : {
352 632 : fprintf (f, " Bits unusable (BOTTOM)\n");
353 632 : return;
354 : }
355 :
356 338 : if (top_p ())
357 0 : fprintf (f, " Bits unknown (TOP)");
358 : else
359 : {
360 338 : fprintf (f, " Bits: value = ");
361 338 : ipcp_print_widest_int (f, get_value ());
362 338 : fprintf (f, ", mask = ");
363 338 : ipcp_print_widest_int (f, get_mask ());
364 : }
365 :
366 338 : if (m_recipient_only)
367 155 : fprintf (f, " (recipient only)");
368 338 : fprintf (f, "\n");
369 : }
370 :
371 : /* Print value range lattice to F. */
372 :
373 : void
374 970 : ipcp_vr_lattice::print (FILE * f)
375 : {
376 970 : if (m_recipient_only)
377 290 : fprintf (f, "(recipient only) ");
378 970 : m_vr.dump (f);
379 970 : }
380 :
381 : /* Print all ipcp_lattices of all functions to F. */
382 :
383 : static void
384 175 : print_all_lattices (FILE * f, bool dump_sources, bool dump_benefits)
385 : {
386 175 : struct cgraph_node *node;
387 175 : int i, count;
388 :
389 175 : fprintf (f, "\nLattices:\n");
390 940 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
391 : {
392 765 : class ipa_node_params *info;
393 :
394 765 : info = ipa_node_params_sum->get (node);
395 : /* Skip unoptimized functions and constprop clones since we don't make
396 : lattices for them. */
397 765 : if (!info || info->ipcp_orig_node)
398 0 : continue;
399 765 : fprintf (f, " Node: %s:\n", node->dump_name ());
400 765 : count = ipa_get_param_count (info);
401 1735 : for (i = 0; i < count; i++)
402 : {
403 970 : struct ipcp_agg_lattice *aglat;
404 970 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
405 970 : fprintf (f, " param [%d]: ", i);
406 970 : plats->itself.print (f, dump_sources, dump_benefits);
407 970 : fprintf (f, " ctxs: ");
408 970 : plats->ctxlat.print (f, dump_sources, dump_benefits);
409 970 : plats->bits_lattice.print (f);
410 970 : fprintf (f, " ");
411 970 : plats->m_value_range.print (f);
412 970 : fprintf (f, "\n");
413 970 : if (plats->virt_call)
414 75 : fprintf (f, " virt_call flag set\n");
415 :
416 970 : if (plats->aggs_bottom)
417 : {
418 452 : fprintf (f, " AGGS BOTTOM\n");
419 452 : continue;
420 : }
421 518 : if (plats->aggs_contain_variable)
422 476 : fprintf (f, " AGGS VARIABLE\n");
423 696 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
424 : {
425 178 : fprintf (f, " %soffset " HOST_WIDE_INT_PRINT_DEC ": ",
426 178 : plats->aggs_by_ref ? "ref " : "", aglat->offset);
427 178 : aglat->print (f, dump_sources, dump_benefits);
428 : }
429 : }
430 : }
431 175 : }
432 :
433 : /* Determine whether it is at all technically possible to create clones of NODE
434 : and store this information in the ipa_node_params structure associated
435 : with NODE. */
436 :
437 : static void
438 1300323 : determine_versionability (struct cgraph_node *node,
439 : class ipa_node_params *info)
440 : {
441 1300323 : const char *reason = NULL;
442 :
443 : /* There are a number of generic reasons functions cannot be versioned. We
444 : also cannot remove parameters if there are type attributes such as fnspec
445 : present. */
446 1300323 : if (node->alias || node->thunk)
447 : reason = "alias or thunk";
448 1300323 : else if (!node->versionable)
449 : reason = "not a tree_versionable_function";
450 1167437 : else if (node->get_availability () <= AVAIL_INTERPOSABLE)
451 : reason = "insufficient body availability";
452 1099948 : else if (!opt_for_fn (node->decl, optimize)
453 1099948 : || !opt_for_fn (node->decl, flag_ipa_cp))
454 : reason = "non-optimized function";
455 1099948 : else if (lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (node->decl)))
456 : {
457 : /* Ideally we should clone the SIMD clones themselves and create
458 : vector copies of them, so IPA-cp and SIMD clones can happily
459 : coexist, but that may not be worth the effort. */
460 : reason = "function has SIMD clones";
461 : }
462 1099583 : else if (lookup_attribute ("target_clones", DECL_ATTRIBUTES (node->decl)))
463 : {
464 : /* Ideally we should clone the target clones themselves and create
465 : copies of them, so IPA-cp and target clones can happily
466 : coexist, but that may not be worth the effort. */
467 : reason = "function target_clones attribute";
468 : }
469 : /* Don't clone decls local to a comdat group; it breaks and for C++
470 : decloned constructors, inlining is always better anyway. */
471 1099583 : else if (node->comdat_local_p ())
472 : reason = "comdat-local function";
473 1097211 : else if (node->calls_comdat_local)
474 : {
475 : /* TODO: call is versionable if we make sure that all
476 : callers are inside of a comdat group. */
477 2476 : reason = "calls comdat-local function";
478 : }
479 :
480 : /* Functions calling BUILT_IN_VA_ARG_PACK and BUILT_IN_VA_ARG_PACK_LEN
481 : work only when inlined. Cloning them may still lead to better code
482 : because ipa-cp will not give up on cloning further. If the function is
483 : external this however leads to wrong code because we may end up producing
484 : offline copy of the function. */
485 1300323 : if (DECL_EXTERNAL (node->decl))
486 168358 : for (cgraph_edge *edge = node->callees; !reason && edge;
487 123526 : edge = edge->next_callee)
488 123526 : if (fndecl_built_in_p (edge->callee->decl, BUILT_IN_NORMAL))
489 : {
490 34680 : if (DECL_FUNCTION_CODE (edge->callee->decl) == BUILT_IN_VA_ARG_PACK)
491 0 : reason = "external function which calls va_arg_pack";
492 34680 : if (DECL_FUNCTION_CODE (edge->callee->decl)
493 : == BUILT_IN_VA_ARG_PACK_LEN)
494 0 : reason = "external function which calls va_arg_pack_len";
495 : }
496 :
497 1300323 : if (reason && dump_file && !node->alias && !node->thunk)
498 53 : fprintf (dump_file, "Function %s is not versionable, reason: %s.\n",
499 : node->dump_name (), reason);
500 :
501 1300323 : info->versionable = (reason == NULL);
502 1300323 : }
503 :
504 : /* Return true if it is at all technically possible to create clones of a
505 : NODE. */
506 :
507 : static bool
508 6220437 : ipcp_versionable_function_p (struct cgraph_node *node)
509 : {
510 6220437 : ipa_node_params *info = ipa_node_params_sum->get (node);
511 6220437 : return info && info->versionable;
512 : }
513 :
514 : /* Structure holding accumulated information about callers of a node. */
515 :
516 1049433 : struct caller_statistics
517 : {
518 : /* If requested (see below), self-recursive call counts are summed into this
519 : field. */
520 : profile_count rec_count_sum;
521 : /* The sum of all ipa counts of all the other (non-recursive) calls. */
522 : profile_count count_sum;
523 : /* Sum of all frequencies for all calls. */
524 : sreal freq_sum;
525 : /* Number of calls and calls considered interesting respectively. */
526 : int n_calls, n_interesting_calls;
527 : /* If itself is set up, also count the number of non-self-recursive
528 : calls. */
529 : int n_nonrec_calls;
530 : /* If non-NULL, this is the node itself and calls from it should have their
531 : counts included in rec_count_sum and not count_sum. */
532 : cgraph_node *itself;
533 : /* True if there is a caller that has no IPA profile. */
534 : bool called_without_ipa_profile;
535 : };
536 :
537 : /* Initialize fields of STAT to zeroes and optionally set it up so that edges
538 : from IGNORED_CALLER are not counted. */
539 :
540 : static inline void
541 289132 : init_caller_stats (caller_statistics *stats, cgraph_node *itself = NULL)
542 : {
543 289132 : stats->rec_count_sum = profile_count::zero ();
544 289132 : stats->count_sum = profile_count::zero ();
545 289132 : stats->n_calls = 0;
546 289132 : stats->n_interesting_calls = 0;
547 289132 : stats->n_nonrec_calls = 0;
548 289132 : stats->freq_sum = 0;
549 289132 : stats->itself = itself;
550 289132 : stats->called_without_ipa_profile = false;
551 289132 : }
552 :
553 : /* We want to propagate across edges that may be executed, however
554 : we do not want to check maybe_hot, since call itself may be cold
555 : while calee contains some heavy loop which makes propagation still
556 : relevant.
557 :
558 : In particular, even edge called once may lead to significant
559 : improvement. */
560 :
561 : static bool
562 988042 : cs_interesting_for_ipcp_p (cgraph_edge *e)
563 : {
564 : /* If profile says the edge is executed, we want to optimize. */
565 988042 : if (e->count.ipa ().nonzero_p ())
566 501 : return true;
567 : /* If local (possibly guseed or adjusted 0 profile) claims edge is
568 : not executed, do not propagate.
569 : Do not trust AFDO since branch needs to be executed multiple
570 : time to count while we want to propagate even call called
571 : once during the train run if callee is important. */
572 987541 : if (e->count.initialized_p () && !e->count.nonzero_p ()
573 1037665 : && e->count.quality () != AFDO)
574 : return false;
575 : /* If we have zero IPA profile, still consider edge for cloning
576 : in case we do partial training. */
577 937417 : if (e->count.ipa ().initialized_p ()
578 937417 : && e->count.ipa ().quality () != AFDO
579 937422 : && !opt_for_fn (e->callee->decl,flag_profile_partial_training))
580 5 : return false;
581 : return true;
582 : }
583 :
584 : /* Worker callback of cgraph_for_node_and_aliases accumulating statistics of
585 : non-thunk incoming edges to NODE. */
586 :
587 : static bool
588 297228 : gather_caller_stats (struct cgraph_node *node, void *data)
589 : {
590 297228 : struct caller_statistics *stats = (struct caller_statistics *) data;
591 297228 : struct cgraph_edge *cs;
592 :
593 1049156 : for (cs = node->callers; cs; cs = cs->next_caller)
594 751928 : if (!cs->caller->thunk)
595 : {
596 751739 : ipa_node_params *info = ipa_node_params_sum->get (cs->caller);
597 751739 : if (info && info->node_dead)
598 150680 : continue;
599 :
600 601059 : if (cs->count.ipa ().initialized_p ())
601 : {
602 3869 : if (stats->itself && stats->itself == cs->caller)
603 0 : stats->rec_count_sum += cs->count.ipa ();
604 : else
605 3869 : stats->count_sum += cs->count.ipa ();
606 : }
607 : else
608 597190 : stats->called_without_ipa_profile = true;
609 601059 : stats->freq_sum += cs->sreal_frequency ();
610 601059 : stats->n_calls++;
611 601059 : if (stats->itself && stats->itself != cs->caller)
612 53 : stats->n_nonrec_calls++;
613 :
614 : /* If profile known to be zero, we do not want to clone for performance.
615 : However if call is cold, the called function may still contain
616 : important hot loops. */
617 601059 : if (cs_interesting_for_ipcp_p (cs))
618 596645 : stats->n_interesting_calls++;
619 : }
620 297228 : return false;
621 :
622 : }
623 :
624 : /* Return true if this NODE is viable candidate for cloning. */
625 :
626 : static bool
627 808710 : ipcp_cloning_candidate_p (struct cgraph_node *node)
628 : {
629 808710 : struct caller_statistics stats;
630 :
631 808710 : gcc_checking_assert (node->has_gimple_body_p ());
632 :
633 808710 : if (!opt_for_fn (node->decl, flag_ipa_cp_clone))
634 : {
635 755865 : if (dump_file)
636 32 : fprintf (dump_file, "Not considering %s for cloning; "
637 : "-fipa-cp-clone disabled.\n",
638 : node->dump_name ());
639 : return false;
640 : }
641 :
642 : /* Do not use profile here since cold wrapper wrap
643 : hot function. */
644 52845 : if (opt_for_fn (node->decl, optimize_size))
645 : {
646 10 : if (dump_file)
647 0 : fprintf (dump_file, "Not considering %s for cloning; "
648 : "optimizing it for size.\n",
649 : node->dump_name ());
650 : return false;
651 : }
652 :
653 52835 : init_caller_stats (&stats);
654 52835 : node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats, false);
655 :
656 52835 : if (ipa_size_summaries->get (node)->self_size < stats.n_calls)
657 : {
658 312 : if (dump_file)
659 0 : fprintf (dump_file, "Considering %s for cloning; code might shrink.\n",
660 : node->dump_name ());
661 : return true;
662 : }
663 52523 : if (!stats.n_interesting_calls)
664 : {
665 40215 : if (dump_file)
666 209 : fprintf (dump_file, "Not considering %s for cloning; "
667 : "no calls considered interesting by profile.\n",
668 : node->dump_name ());
669 : return false;
670 : }
671 12308 : if (dump_file)
672 202 : fprintf (dump_file, "Considering %s for cloning.\n",
673 : node->dump_name ());
674 : return true;
675 : }
676 :
677 : template <typename valtype>
678 : class value_topo_info
679 : {
680 : public:
681 : /* Head of the linked list of topologically sorted values. */
682 : ipcp_value<valtype> *values_topo;
683 : /* Stack for creating SCCs, represented by a linked list too. */
684 : ipcp_value<valtype> *stack;
685 : /* Counter driving the algorithm in add_val_to_toposort. */
686 : int dfs_counter;
687 :
688 132707 : value_topo_info () : values_topo (NULL), stack (NULL), dfs_counter (0)
689 : {}
690 : void add_val (ipcp_value<valtype> *cur_val);
691 : void propagate_effects ();
692 : };
693 :
694 : /* Arrays representing a topological ordering of call graph nodes and a stack
695 : of nodes used during constant propagation and also data required to perform
696 : topological sort of values and propagation of benefits in the determined
697 : order. */
698 :
699 : class ipa_topo_info
700 : {
701 : public:
702 : /* Array with obtained topological order of cgraph nodes. */
703 : struct cgraph_node **order;
704 : /* Stack of cgraph nodes used during propagation within SCC until all values
705 : in the SCC stabilize. */
706 : struct cgraph_node **stack;
707 : int nnodes, stack_top;
708 :
709 : value_topo_info<tree> constants;
710 : value_topo_info<ipa_polymorphic_call_context> contexts;
711 :
712 132707 : ipa_topo_info () : order(NULL), stack(NULL), nnodes(0), stack_top(0),
713 132707 : constants ()
714 : {}
715 : };
716 :
717 : /* Skip edges from and to nodes without ipa_cp enabled.
718 : Ignore not available symbols. */
719 :
720 : static bool
721 5428339 : ignore_edge_p (cgraph_edge *e)
722 : {
723 5428339 : enum availability avail;
724 5428339 : cgraph_node *ultimate_target
725 5428339 : = e->callee->function_or_virtual_thunk_symbol (&avail, e->caller);
726 :
727 5428339 : return (avail <= AVAIL_INTERPOSABLE
728 1942106 : || !opt_for_fn (ultimate_target->decl, optimize)
729 7361626 : || !opt_for_fn (ultimate_target->decl, flag_ipa_cp));
730 : }
731 :
732 : /* Allocate the arrays in TOPO and topologically sort the nodes into order. */
733 :
734 : static void
735 132707 : build_toporder_info (class ipa_topo_info *topo)
736 : {
737 132707 : topo->order = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
738 132707 : topo->stack = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
739 :
740 132707 : gcc_checking_assert (topo->stack_top == 0);
741 132707 : topo->nnodes = ipa_reduced_postorder (topo->order, true,
742 : ignore_edge_p);
743 132707 : }
744 :
745 : /* Free information about strongly connected components and the arrays in
746 : TOPO. */
747 :
748 : static void
749 132707 : free_toporder_info (class ipa_topo_info *topo)
750 : {
751 132707 : ipa_free_postorder_info ();
752 132707 : free (topo->order);
753 132707 : free (topo->stack);
754 132707 : }
755 :
756 : /* Add NODE to the stack in TOPO, unless it is already there. */
757 :
758 : static inline void
759 1304521 : push_node_to_stack (class ipa_topo_info *topo, struct cgraph_node *node)
760 : {
761 1304521 : ipa_node_params *info = ipa_node_params_sum->get (node);
762 1304521 : if (info->node_enqueued)
763 : return;
764 1303538 : info->node_enqueued = 1;
765 1303538 : topo->stack[topo->stack_top++] = node;
766 : }
767 :
768 : /* Pop a node from the stack in TOPO and return it or return NULL if the stack
769 : is empty. */
770 :
771 : static struct cgraph_node *
772 2686571 : pop_node_from_stack (class ipa_topo_info *topo)
773 : {
774 2686571 : if (topo->stack_top)
775 : {
776 1303538 : struct cgraph_node *node;
777 1303538 : topo->stack_top--;
778 1303538 : node = topo->stack[topo->stack_top];
779 1303538 : ipa_node_params_sum->get (node)->node_enqueued = 0;
780 1303538 : return node;
781 : }
782 : else
783 : return NULL;
784 : }
785 :
786 : /* Set lattice LAT to bottom and return true if it previously was not set as
787 : such. */
788 :
789 : template <typename valtype>
790 : inline bool
791 2163623 : ipcp_lattice<valtype>::set_to_bottom ()
792 : {
793 2163623 : bool ret = !bottom;
794 2163623 : bottom = true;
795 : return ret;
796 : }
797 :
798 : /* Mark lattice as containing an unknown value and return true if it previously
799 : was not marked as such. */
800 :
801 : template <typename valtype>
802 : inline bool
803 1578219 : ipcp_lattice<valtype>::set_contains_variable ()
804 : {
805 1578219 : bool ret = !contains_variable;
806 1578219 : contains_variable = true;
807 : return ret;
808 : }
809 :
810 : /* Set all aggregate lattices in PLATS to bottom and return true if they were
811 : not previously set as such. */
812 :
813 : static inline bool
814 2163363 : set_agg_lats_to_bottom (class ipcp_param_lattices *plats)
815 : {
816 2163363 : bool ret = !plats->aggs_bottom;
817 2163363 : plats->aggs_bottom = true;
818 2163363 : return ret;
819 : }
820 :
821 : /* Mark all aggregate lattices in PLATS as containing an unknown value and
822 : return true if they were not previously marked as such. */
823 :
824 : static inline bool
825 1070015 : set_agg_lats_contain_variable (class ipcp_param_lattices *plats)
826 : {
827 1070015 : bool ret = !plats->aggs_contain_variable;
828 1070015 : plats->aggs_contain_variable = true;
829 1070015 : return ret;
830 : }
831 :
832 : bool
833 0 : ipcp_vr_lattice::meet_with (const ipcp_vr_lattice &other)
834 : {
835 0 : return meet_with_1 (other.m_vr);
836 : }
837 :
838 : /* Meet the current value of the lattice with the range described by
839 : P_VR. */
840 :
841 : bool
842 499139 : ipcp_vr_lattice::meet_with (const vrange &p_vr)
843 : {
844 499139 : return meet_with_1 (p_vr);
845 : }
846 :
847 : /* Meet the current value of the lattice with the range described by
848 : OTHER_VR. Return TRUE if anything changed. */
849 :
850 : bool
851 499139 : ipcp_vr_lattice::meet_with_1 (const vrange &other_vr)
852 : {
853 499139 : if (bottom_p ())
854 : return false;
855 :
856 499139 : if (other_vr.varying_p ())
857 0 : return set_to_bottom ();
858 :
859 499139 : bool res;
860 499139 : if (flag_checking)
861 : {
862 499139 : value_range save (m_vr);
863 499139 : res = m_vr.union_ (other_vr);
864 499139 : gcc_assert (res == (m_vr != save));
865 499139 : }
866 : else
867 0 : res = m_vr.union_ (other_vr);
868 : return res;
869 : }
870 :
871 : /* Return true if value range information in the lattice is yet unknown. */
872 :
873 : bool
874 : ipcp_vr_lattice::top_p () const
875 : {
876 173512 : return m_vr.undefined_p ();
877 : }
878 :
879 : /* Return true if value range information in the lattice is known to be
880 : unusable. */
881 :
882 : bool
883 5031360 : ipcp_vr_lattice::bottom_p () const
884 : {
885 499139 : return m_vr.varying_p ();
886 : }
887 :
888 : /* Set value range information in the lattice to bottom. Return true if it
889 : previously was in a different state. */
890 :
891 : bool
892 2453063 : ipcp_vr_lattice::set_to_bottom ()
893 : {
894 2453063 : if (m_vr.varying_p ())
895 : return false;
896 :
897 : /* Setting an unsupported type here forces the temporary to default
898 : to unsupported_range, which can handle VARYING/DEFINED ranges,
899 : but nothing else (union, intersect, etc). This allows us to set
900 : bottoms on any ranges, and is safe as all users of the lattice
901 : check for bottom first. */
902 2300626 : m_vr.set_range_class (void_type_node);
903 2300626 : m_vr.set_varying (void_type_node);
904 :
905 2300626 : return true;
906 : }
907 :
908 : /* Set the flag that this lattice is a recipient only, return true if it was
909 : not set before. */
910 :
911 : bool
912 30423 : ipcp_vr_lattice::set_recipient_only ()
913 : {
914 30423 : if (m_recipient_only)
915 : return false;
916 30423 : m_recipient_only = true;
917 30423 : return true;
918 : }
919 :
920 : /* Set lattice value to bottom, if it already isn't the case. */
921 :
922 : bool
923 2472839 : ipcp_bits_lattice::set_to_bottom ()
924 : {
925 2472839 : if (bottom_p ())
926 : return false;
927 2320921 : m_lattice_val = IPA_BITS_VARYING;
928 2320921 : m_value = 0;
929 2320921 : m_mask = -1;
930 2320921 : return true;
931 : }
932 :
933 : /* Set to constant if it isn't already. Only meant to be called
934 : when switching state from TOP. */
935 :
936 : bool
937 79078 : ipcp_bits_lattice::set_to_constant (const widest_int &value,
938 : const widest_int &mask)
939 : {
940 79078 : gcc_assert (top_p ());
941 79078 : m_lattice_val = IPA_BITS_CONSTANT;
942 79078 : m_value = wi::bit_and (wi::bit_not (mask), value);
943 79078 : m_mask = mask;
944 79078 : return true;
945 : }
946 :
947 : /* Return true if any of the known bits are non-zero. */
948 :
949 : bool
950 506 : ipcp_bits_lattice::known_nonzero_p () const
951 : {
952 506 : if (!constant_p ())
953 : return false;
954 506 : return wi::ne_p (wi::bit_and (wi::bit_not (m_mask), m_value), 0);
955 : }
956 :
957 : /* Set the flag that this lattice is a recipient only, return true if it was not
958 : set before. */
959 :
960 : bool
961 30423 : ipcp_bits_lattice::set_recipient_only ()
962 : {
963 30423 : if (m_recipient_only)
964 : return false;
965 30423 : m_recipient_only = true;
966 30423 : return true;
967 : }
968 :
969 : /* Convert operand to value, mask form. */
970 :
971 : void
972 2061 : ipcp_bits_lattice::get_value_and_mask (tree operand, widest_int *valuep, widest_int *maskp)
973 : {
974 2061 : if (TREE_CODE (operand) == INTEGER_CST)
975 : {
976 2061 : *valuep = wi::to_widest (operand);
977 2061 : *maskp = 0;
978 : }
979 : else
980 : {
981 0 : *valuep = 0;
982 0 : *maskp = -1;
983 : }
984 2061 : }
985 :
986 : /* Meet operation, similar to ccp_lattice_meet, we xor values
987 : if this->value, value have different values at same bit positions, we want
988 : to drop that bit to varying. Return true if mask is changed.
989 : This function assumes that the lattice value is in CONSTANT state. If
990 : DROP_ALL_ONES, mask out any known bits with value one afterwards. */
991 :
992 : bool
993 305804 : ipcp_bits_lattice::meet_with_1 (const widest_int &value,
994 : const widest_int &mask,
995 : unsigned precision, bool drop_all_ones)
996 : {
997 305804 : gcc_assert (constant_p ());
998 :
999 305804 : widest_int old_mask = m_mask;
1000 305804 : m_mask = (m_mask | mask) | (m_value ^ value);
1001 305804 : if (drop_all_ones)
1002 211 : m_mask |= m_value;
1003 :
1004 305804 : widest_int cap_mask = wi::shifted_mask <widest_int> (0, precision, true);
1005 305804 : m_mask |= cap_mask;
1006 305804 : if (wi::sext (m_mask, precision) == -1)
1007 3642 : return set_to_bottom ();
1008 :
1009 302162 : m_value &= ~m_mask;
1010 302162 : return m_mask != old_mask;
1011 305804 : }
1012 :
1013 : /* Meet the bits lattice with operand
1014 : described by <value, mask, sgn, precision. */
1015 :
1016 : bool
1017 418626 : ipcp_bits_lattice::meet_with (const widest_int &value,
1018 : const widest_int &mask,
1019 : unsigned precision)
1020 : {
1021 418626 : if (bottom_p ())
1022 : return false;
1023 :
1024 418626 : if (top_p ())
1025 : {
1026 125541 : if (wi::sext (mask, precision) == -1)
1027 51955 : return set_to_bottom ();
1028 73586 : return set_to_constant (value, mask);
1029 : }
1030 :
1031 293085 : return meet_with_1 (value, mask, precision, false);
1032 : }
1033 :
1034 : /* Meet bits lattice with the result of bit_value_binop (other, operand)
1035 : if code is binary operation or bit_value_unop (other) if code is unary op.
1036 : In the case when code is nop_expr, no adjustment is required. If
1037 : DROP_ALL_ONES, mask out any known bits with value one afterwards. */
1038 :
1039 : bool
1040 21822 : ipcp_bits_lattice::meet_with (ipcp_bits_lattice& other, unsigned precision,
1041 : signop sgn, enum tree_code code, tree operand,
1042 : bool drop_all_ones)
1043 : {
1044 21822 : if (other.bottom_p ())
1045 0 : return set_to_bottom ();
1046 :
1047 21822 : if (bottom_p () || other.top_p ())
1048 : return false;
1049 :
1050 18327 : widest_int adjusted_value, adjusted_mask;
1051 :
1052 18327 : if (TREE_CODE_CLASS (code) == tcc_binary)
1053 : {
1054 2061 : tree type = TREE_TYPE (operand);
1055 2061 : widest_int o_value, o_mask;
1056 2061 : get_value_and_mask (operand, &o_value, &o_mask);
1057 :
1058 2061 : bit_value_binop (code, sgn, precision, &adjusted_value, &adjusted_mask,
1059 4122 : sgn, precision, other.get_value (), other.get_mask (),
1060 2061 : TYPE_SIGN (type), TYPE_PRECISION (type), o_value, o_mask);
1061 :
1062 2061 : if (wi::sext (adjusted_mask, precision) == -1)
1063 87 : return set_to_bottom ();
1064 2061 : }
1065 :
1066 16266 : else if (TREE_CODE_CLASS (code) == tcc_unary)
1067 : {
1068 32482 : bit_value_unop (code, sgn, precision, &adjusted_value,
1069 32482 : &adjusted_mask, sgn, precision, other.get_value (),
1070 16241 : other.get_mask ());
1071 :
1072 16241 : if (wi::sext (adjusted_mask, precision) == -1)
1073 4 : return set_to_bottom ();
1074 : }
1075 :
1076 : else
1077 25 : return set_to_bottom ();
1078 :
1079 18211 : if (top_p ())
1080 : {
1081 5492 : if (drop_all_ones)
1082 : {
1083 295 : adjusted_mask |= adjusted_value;
1084 295 : adjusted_value &= ~adjusted_mask;
1085 : }
1086 5492 : widest_int cap_mask = wi::shifted_mask <widest_int> (0, precision, true);
1087 5492 : adjusted_mask |= cap_mask;
1088 5492 : if (wi::sext (adjusted_mask, precision) == -1)
1089 0 : return set_to_bottom ();
1090 5492 : return set_to_constant (adjusted_value, adjusted_mask);
1091 5492 : }
1092 : else
1093 12719 : return meet_with_1 (adjusted_value, adjusted_mask, precision,
1094 12719 : drop_all_ones);
1095 18327 : }
1096 :
1097 : /* Dump the contents of the list to FILE. */
1098 :
1099 : void
1100 135 : ipa_argagg_value_list::dump (FILE *f)
1101 : {
1102 135 : bool comma = false;
1103 370 : for (const ipa_argagg_value &av : m_elts)
1104 : {
1105 235 : fprintf (f, "%s %i[%u]=", comma ? "," : "",
1106 235 : av.index, av.unit_offset);
1107 235 : print_generic_expr (f, av.value);
1108 235 : if (av.by_ref)
1109 208 : fprintf (f, "(by_ref)");
1110 235 : if (av.killed)
1111 1 : fprintf (f, "(killed)");
1112 235 : comma = true;
1113 : }
1114 135 : fprintf (f, "\n");
1115 135 : }
1116 :
1117 : /* Dump the contents of the list to stderr. */
1118 :
1119 : void
1120 0 : ipa_argagg_value_list::debug ()
1121 : {
1122 0 : dump (stderr);
1123 0 : }
1124 :
1125 : /* Return the item describing a constant stored for INDEX at UNIT_OFFSET or
1126 : NULL if there is no such constant. */
1127 :
1128 : const ipa_argagg_value *
1129 30659962 : ipa_argagg_value_list::get_elt (int index, unsigned unit_offset) const
1130 : {
1131 30659962 : ipa_argagg_value key;
1132 30659962 : key.index = index;
1133 30659962 : key.unit_offset = unit_offset;
1134 30659962 : const ipa_argagg_value *res
1135 30659962 : = std::lower_bound (m_elts.begin (), m_elts.end (), key,
1136 7473534 : [] (const ipa_argagg_value &elt,
1137 : const ipa_argagg_value &val)
1138 : {
1139 7473534 : if (elt.index < val.index)
1140 : return true;
1141 6382485 : if (elt.index > val.index)
1142 : return false;
1143 5085508 : if (elt.unit_offset < val.unit_offset)
1144 : return true;
1145 : return false;
1146 : });
1147 :
1148 30659962 : if (res == m_elts.end ()
1149 3229207 : || res->index != index
1150 33330676 : || res->unit_offset != unit_offset)
1151 : res = nullptr;
1152 :
1153 : /* TODO: perhaps remove the check (that the underlying array is indeed
1154 : sorted) if it turns out it can be too slow? */
1155 30659962 : if (!flag_checking)
1156 : return res;
1157 :
1158 : const ipa_argagg_value *slow_res = NULL;
1159 : int prev_index = -1;
1160 : unsigned prev_unit_offset = 0;
1161 48427711 : for (const ipa_argagg_value &av : m_elts)
1162 : {
1163 17767749 : gcc_assert (prev_index < 0
1164 : || prev_index < av.index
1165 : || prev_unit_offset < av.unit_offset);
1166 17767749 : prev_index = av.index;
1167 17767749 : prev_unit_offset = av.unit_offset;
1168 17767749 : if (av.index == index
1169 8239933 : && av.unit_offset == unit_offset)
1170 17767749 : slow_res = &av;
1171 : }
1172 30659962 : gcc_assert (res == slow_res);
1173 :
1174 : return res;
1175 : }
1176 :
1177 : /* Return the first item describing a constant stored for parameter with INDEX,
1178 : regardless of offset or reference, or NULL if there is no such constant. */
1179 :
1180 : const ipa_argagg_value *
1181 252393 : ipa_argagg_value_list::get_elt_for_index (int index) const
1182 : {
1183 252393 : const ipa_argagg_value *res
1184 252393 : = std::lower_bound (m_elts.begin (), m_elts.end (), index,
1185 19485 : [] (const ipa_argagg_value &elt, unsigned idx)
1186 : {
1187 19485 : return elt.index < idx;
1188 : });
1189 252393 : if (res == m_elts.end ()
1190 252393 : || res->index != index)
1191 : res = nullptr;
1192 252393 : return res;
1193 : }
1194 :
1195 : /* Return the aggregate constant stored for INDEX at UNIT_OFFSET, not
1196 : performing any check of whether value is passed by reference, or NULL_TREE
1197 : if there is no such constant. */
1198 :
1199 : tree
1200 39431 : ipa_argagg_value_list::get_value (int index, unsigned unit_offset) const
1201 : {
1202 39431 : const ipa_argagg_value *av = get_elt (index, unit_offset);
1203 39431 : return av ? av->value : NULL_TREE;
1204 : }
1205 :
1206 : /* Return the aggregate constant stored for INDEX at UNIT_OFFSET, if it is
1207 : passed by reference or not according to BY_REF, or NULL_TREE if there is
1208 : no such constant. */
1209 :
1210 : tree
1211 30610497 : ipa_argagg_value_list::get_value (int index, unsigned unit_offset,
1212 : bool by_ref) const
1213 : {
1214 30610497 : const ipa_argagg_value *av = get_elt (index, unit_offset);
1215 30610497 : if (av && av->by_ref == by_ref)
1216 2157185 : return av->value;
1217 : return NULL_TREE;
1218 : }
1219 :
1220 : /* Return true if all elements present in OTHER are also present in this
1221 : list. */
1222 :
1223 : bool
1224 44 : ipa_argagg_value_list::superset_of_p (const ipa_argagg_value_list &other) const
1225 : {
1226 44 : unsigned j = 0;
1227 202 : for (unsigned i = 0; i < other.m_elts.size (); i++)
1228 : {
1229 175 : unsigned other_index = other.m_elts[i].index;
1230 175 : unsigned other_offset = other.m_elts[i].unit_offset;
1231 :
1232 175 : while (j < m_elts.size ()
1233 334 : && (m_elts[j].index < other_index
1234 320 : || (m_elts[j].index == other_index
1235 320 : && m_elts[j].unit_offset < other_offset)))
1236 159 : j++;
1237 :
1238 175 : if (j >= m_elts.size ()
1239 162 : || m_elts[j].index != other_index
1240 162 : || m_elts[j].unit_offset != other_offset
1241 162 : || m_elts[j].by_ref != other.m_elts[i].by_ref
1242 162 : || !m_elts[j].value
1243 337 : || !values_equal_for_ipcp_p (m_elts[j].value, other.m_elts[i].value))
1244 : return false;
1245 : }
1246 : return true;
1247 : }
1248 :
1249 : /* Push all items in this list that describe parameter SRC_INDEX into RES as
1250 : ones describing DST_INDEX while subtracting UNIT_DELTA from their unit
1251 : offsets but skip those which would end up with a negative offset. */
1252 :
1253 : void
1254 3163 : ipa_argagg_value_list::push_adjusted_values (unsigned src_index,
1255 : unsigned dest_index,
1256 : unsigned unit_delta,
1257 : vec<ipa_argagg_value> *res) const
1258 : {
1259 3163 : const ipa_argagg_value *av = get_elt_for_index (src_index);
1260 3163 : if (!av)
1261 : return;
1262 : unsigned prev_unit_offset = 0;
1263 : bool first = true;
1264 12602 : for (; av < m_elts.end (); ++av)
1265 : {
1266 10147 : if (av->index > src_index)
1267 : return;
1268 9535 : if (av->index == src_index
1269 9535 : && (av->unit_offset >= unit_delta)
1270 9391 : && av->value)
1271 : {
1272 9391 : ipa_argagg_value new_av;
1273 9391 : gcc_checking_assert (av->value);
1274 9391 : new_av.value = av->value;
1275 9391 : new_av.unit_offset = av->unit_offset - unit_delta;
1276 9391 : new_av.index = dest_index;
1277 9391 : new_av.by_ref = av->by_ref;
1278 9391 : gcc_assert (!av->killed);
1279 9391 : new_av.killed = false;
1280 :
1281 : /* Quick check that the offsets we push are indeed increasing. */
1282 9391 : gcc_assert (first
1283 : || new_av.unit_offset > prev_unit_offset);
1284 9391 : prev_unit_offset = new_av.unit_offset;
1285 9391 : first = false;
1286 :
1287 9391 : res->safe_push (new_av);
1288 : }
1289 : }
1290 : }
1291 :
1292 : /* Push to RES information about single lattices describing aggregate values in
1293 : PLATS as those describing parameter DEST_INDEX and the original offset minus
1294 : UNIT_DELTA. Return true if any item has been pushed to RES. */
1295 :
1296 : static bool
1297 2322078 : push_agg_values_from_plats (ipcp_param_lattices *plats, int dest_index,
1298 : unsigned unit_delta,
1299 : vec<ipa_argagg_value> *res)
1300 : {
1301 2322078 : if (plats->aggs_contain_variable)
1302 : return false;
1303 :
1304 1679171 : bool pushed_sth = false;
1305 1679171 : bool first = true;
1306 1679171 : unsigned prev_unit_offset = 0;
1307 1747024 : for (struct ipcp_agg_lattice *aglat = plats->aggs; aglat; aglat = aglat->next)
1308 134498 : if (aglat->is_single_const ()
1309 42415 : && (aglat->offset / BITS_PER_UNIT - unit_delta) >= 0)
1310 : {
1311 42415 : ipa_argagg_value iav;
1312 42415 : iav.value = aglat->values->value;
1313 42415 : iav.unit_offset = aglat->offset / BITS_PER_UNIT - unit_delta;
1314 42415 : iav.index = dest_index;
1315 42415 : iav.by_ref = plats->aggs_by_ref;
1316 42415 : iav.killed = false;
1317 :
1318 42415 : gcc_assert (first
1319 : || iav.unit_offset > prev_unit_offset);
1320 42415 : prev_unit_offset = iav.unit_offset;
1321 42415 : first = false;
1322 :
1323 42415 : pushed_sth = true;
1324 42415 : res->safe_push (iav);
1325 : }
1326 : return pushed_sth;
1327 : }
1328 :
1329 : /* Turn all values in LIST that are not present in OTHER into NULL_TREEs.
1330 : Return the number of remaining valid entries. */
1331 :
1332 : static unsigned
1333 56875 : intersect_argaggs_with (vec<ipa_argagg_value> &elts,
1334 : const vec<ipa_argagg_value> &other)
1335 : {
1336 56875 : unsigned valid_entries = 0;
1337 56875 : unsigned j = 0;
1338 432741 : for (unsigned i = 0; i < elts.length (); i++)
1339 : {
1340 375866 : if (!elts[i].value)
1341 55520 : continue;
1342 :
1343 320346 : unsigned this_index = elts[i].index;
1344 320346 : unsigned this_offset = elts[i].unit_offset;
1345 :
1346 320346 : while (j < other.length ()
1347 1204183 : && (other[j].index < this_index
1348 567128 : || (other[j].index == this_index
1349 563722 : && other[j].unit_offset < this_offset)))
1350 286244 : j++;
1351 :
1352 320346 : if (j >= other.length ())
1353 : {
1354 8997 : elts[i].value = NULL_TREE;
1355 8997 : continue;
1356 : }
1357 :
1358 311349 : if (other[j].index == this_index
1359 307943 : && other[j].unit_offset == this_offset
1360 300225 : && other[j].by_ref == elts[i].by_ref
1361 300225 : && other[j].value
1362 611574 : && values_equal_for_ipcp_p (other[j].value, elts[i].value))
1363 280080 : valid_entries++;
1364 : else
1365 31269 : elts[i].value = NULL_TREE;
1366 : }
1367 56875 : return valid_entries;
1368 : }
1369 :
1370 : /* Mark bot aggregate and scalar lattices as containing an unknown variable,
1371 : return true is any of them has not been marked as such so far. If if
1372 : MAKE_SIMPLE_RECIPIENTS is true, set the lattices that can only hold one
1373 : value to being recipients only, otherwise also set them to bottom. */
1374 :
1375 : static inline bool
1376 182596 : set_all_contains_variable (class ipcp_param_lattices *plats,
1377 : bool make_simple_recipients = false)
1378 : {
1379 182596 : bool ret;
1380 182596 : ret = plats->itself.set_contains_variable ();
1381 182596 : ret |= plats->ctxlat.set_contains_variable ();
1382 182596 : ret |= set_agg_lats_contain_variable (plats);
1383 182596 : if (make_simple_recipients)
1384 : {
1385 30423 : ret |= plats->bits_lattice.set_recipient_only ();
1386 30423 : ret |= plats->m_value_range.set_recipient_only ();
1387 : }
1388 : else
1389 : {
1390 152173 : ret |= plats->bits_lattice.set_to_bottom ();
1391 152173 : ret |= plats->m_value_range.set_to_bottom ();
1392 : }
1393 182596 : return ret;
1394 : }
1395 :
1396 : /* Worker of call_for_symbol_thunks_and_aliases, increment the integer DATA
1397 : points to by the number of callers to NODE. */
1398 :
1399 : static bool
1400 102245 : count_callers (cgraph_node *node, void *data)
1401 : {
1402 102245 : int *caller_count = (int *) data;
1403 :
1404 412368 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
1405 : /* Local thunks can be handled transparently, but if the thunk cannot
1406 : be optimized out, count it as a real use. */
1407 310123 : if (!cs->caller->thunk || !cs->caller->local)
1408 310123 : ++*caller_count;
1409 102245 : return false;
1410 : }
1411 :
1412 : /* Worker of call_for_symbol_thunks_and_aliases, it is supposed to be called on
1413 : the one caller of some other node. Set the caller's corresponding flag. */
1414 :
1415 : static bool
1416 58808 : set_single_call_flag (cgraph_node *node, void *)
1417 : {
1418 58808 : cgraph_edge *cs = node->callers;
1419 : /* Local thunks can be handled transparently, skip them. */
1420 58808 : while (cs && cs->caller->thunk && cs->caller->local)
1421 0 : cs = cs->next_caller;
1422 58808 : if (cs)
1423 58212 : if (ipa_node_params* info = ipa_node_params_sum->get (cs->caller))
1424 : {
1425 58211 : info->node_calling_single_call = true;
1426 58211 : return true;
1427 : }
1428 : return false;
1429 : }
1430 :
1431 : /* Initialize ipcp_lattices. */
1432 :
1433 : static void
1434 1300323 : initialize_node_lattices (struct cgraph_node *node)
1435 : {
1436 1300323 : ipa_node_params *info = ipa_node_params_sum->get (node);
1437 1300323 : struct cgraph_edge *ie;
1438 1300323 : bool disable = false, variable = false;
1439 1300323 : int i;
1440 :
1441 1300323 : gcc_checking_assert (node->has_gimple_body_p ());
1442 :
1443 1300323 : if (!ipa_get_param_count (info))
1444 : disable = true;
1445 1064033 : else if (node->local)
1446 : {
1447 91166 : int caller_count = 0;
1448 91166 : node->call_for_symbol_thunks_and_aliases (count_callers, &caller_count,
1449 : true);
1450 91166 : if (caller_count == 1)
1451 58212 : node->call_for_symbol_thunks_and_aliases (set_single_call_flag,
1452 : NULL, true);
1453 32954 : else if (caller_count == 0)
1454 : {
1455 1 : gcc_checking_assert (!opt_for_fn (node->decl, flag_toplevel_reorder));
1456 : variable = true;
1457 : }
1458 : }
1459 : else
1460 : {
1461 : /* When cloning is allowed, we can assume that externally visible
1462 : functions are not called. We will compensate this by cloning
1463 : later. */
1464 972867 : if (ipcp_versionable_function_p (node)
1465 972867 : && ipcp_cloning_candidate_p (node))
1466 : variable = true;
1467 : else
1468 : disable = true;
1469 : }
1470 :
1471 765 : if (dump_file && (dump_flags & TDF_DETAILS)
1472 1300526 : && !node->alias && !node->thunk)
1473 : {
1474 203 : fprintf (dump_file, "Initializing lattices of %s\n",
1475 : node->dump_name ());
1476 203 : if (disable || variable)
1477 170 : fprintf (dump_file, " Marking all lattices as %s\n",
1478 : disable ? "BOTTOM" : "VARIABLE");
1479 : }
1480 :
1481 1300323 : auto_vec<bool, 16> surviving_params;
1482 1300323 : bool pre_modified = false;
1483 :
1484 1300323 : clone_info *cinfo = clone_info::get (node);
1485 :
1486 1300323 : if (!disable && cinfo && cinfo->param_adjustments)
1487 : {
1488 : /* At the moment all IPA optimizations should use the number of
1489 : parameters of the prevailing decl as the m_always_copy_start.
1490 : Handling any other value would complicate the code below, so for the
1491 : time bing let's only assert it is so. */
1492 0 : gcc_assert ((cinfo->param_adjustments->m_always_copy_start
1493 : == ipa_get_param_count (info))
1494 : || cinfo->param_adjustments->m_always_copy_start < 0);
1495 :
1496 0 : pre_modified = true;
1497 0 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
1498 :
1499 0 : if (dump_file && (dump_flags & TDF_DETAILS)
1500 0 : && !node->alias && !node->thunk)
1501 : {
1502 : bool first = true;
1503 0 : for (int j = 0; j < ipa_get_param_count (info); j++)
1504 : {
1505 0 : if (j < (int) surviving_params.length ()
1506 0 : && surviving_params[j])
1507 0 : continue;
1508 0 : if (first)
1509 : {
1510 0 : fprintf (dump_file,
1511 : " The following parameters are dead on arrival:");
1512 0 : first = false;
1513 : }
1514 0 : fprintf (dump_file, " %u", j);
1515 : }
1516 0 : if (!first)
1517 0 : fprintf (dump_file, "\n");
1518 : }
1519 : }
1520 :
1521 7150628 : for (i = 0; i < ipa_get_param_count (info); i++)
1522 : {
1523 2393136 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
1524 2393136 : tree type = ipa_get_type (info, i);
1525 2393136 : if (disable
1526 230919 : || !ipa_get_type (info, i)
1527 2624055 : || (pre_modified && (surviving_params.length () <= (unsigned) i
1528 0 : || !surviving_params[i])))
1529 : {
1530 2162217 : plats->itself.set_to_bottom ();
1531 2162217 : plats->ctxlat.set_to_bottom ();
1532 2162217 : set_agg_lats_to_bottom (plats);
1533 2162217 : plats->bits_lattice.set_to_bottom ();
1534 2162217 : plats->m_value_range.init (type);
1535 2162217 : plats->m_value_range.set_to_bottom ();
1536 : }
1537 : else
1538 : {
1539 230919 : plats->m_value_range.init (type);
1540 230919 : if (variable)
1541 30423 : set_all_contains_variable (plats, true);
1542 : }
1543 : }
1544 :
1545 1436721 : for (ie = node->indirect_calls; ie; ie = ie->next_callee)
1546 136398 : if (ie->indirect_info->param_index >= 0
1547 145777 : && is_a <cgraph_polymorphic_indirect_info *> (ie->indirect_info))
1548 9379 : ipa_get_parm_lattices (info,
1549 9379 : ie->indirect_info->param_index)->virt_call = 1;
1550 1300323 : }
1551 :
1552 : /* Return VALUE if it is NULL_TREE or if it can be directly safely IPA-CP
1553 : propagated to a parameter of type PARAM_TYPE, or return a fold-converted
1554 : VALUE to PARAM_TYPE if that is possible. Return NULL_TREE otherwise. */
1555 :
1556 : tree
1557 5139891 : ipacp_value_safe_for_type (tree param_type, tree value)
1558 : {
1559 5139891 : if (!value)
1560 : return NULL_TREE;
1561 5139557 : tree val_type = TREE_TYPE (value);
1562 5139557 : if (param_type == val_type
1563 5139557 : || useless_type_conversion_p (param_type, val_type))
1564 : return value;
1565 3384 : if (fold_convertible_p (param_type, value))
1566 3175 : return fold_convert (param_type, value);
1567 : else
1568 : return NULL_TREE;
1569 : }
1570 :
1571 : /* Return the result of a (possibly arithmetic) operation determined by OPCODE
1572 : on the constant value INPUT. OPERAND is 2nd operand for binary operation
1573 : and is required for binary operations. RES_TYPE, required when opcode is
1574 : not NOP_EXPR, is the type in which any operation is to be performed. Return
1575 : NULL_TREE if that cannot be determined or be considered an interprocedural
1576 : invariant. */
1577 :
1578 : static tree
1579 70741 : ipa_get_jf_arith_result (enum tree_code opcode, tree input, tree operand,
1580 : tree res_type)
1581 : {
1582 70741 : tree res;
1583 :
1584 70741 : if (opcode == NOP_EXPR)
1585 : return input;
1586 6861 : if (!is_gimple_ip_invariant (input))
1587 : return NULL_TREE;
1588 :
1589 6861 : if (opcode == ASSERT_EXPR)
1590 : {
1591 3874 : if (values_equal_for_ipcp_p (input, operand))
1592 : return input;
1593 : else
1594 112 : return NULL_TREE;
1595 : }
1596 :
1597 2987 : if (TREE_CODE_CLASS (opcode) == tcc_unary)
1598 103 : res = fold_unary (opcode, res_type, input);
1599 : else
1600 2884 : res = fold_binary (opcode, res_type, input, operand);
1601 :
1602 2987 : if (res && !is_gimple_ip_invariant (res))
1603 0 : return NULL_TREE;
1604 :
1605 : return res;
1606 : }
1607 :
1608 : /* Return the result of an ancestor jump function JFUNC on the constant value
1609 : INPUT. Return NULL_TREE if that cannot be determined. */
1610 :
1611 : static tree
1612 1302 : ipa_get_jf_ancestor_result (struct ipa_jump_func *jfunc, tree input)
1613 : {
1614 1302 : gcc_checking_assert (TREE_CODE (input) != TREE_BINFO);
1615 1302 : if (TREE_CODE (input) == ADDR_EXPR)
1616 : {
1617 1220 : gcc_checking_assert (is_gimple_ip_invariant_address (input));
1618 1220 : poly_int64 off = ipa_get_jf_ancestor_offset (jfunc);
1619 1220 : if (known_eq (off, 0))
1620 : return input;
1621 1098 : poly_int64 byte_offset = exact_div (off, BITS_PER_UNIT);
1622 2196 : return build1 (ADDR_EXPR, TREE_TYPE (input),
1623 1098 : fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (input)), input,
1624 1098 : build_int_cst (ptr_type_node, byte_offset)));
1625 : }
1626 82 : else if (ipa_get_jf_ancestor_keep_null (jfunc)
1627 82 : && zerop (input))
1628 : return input;
1629 : else
1630 78 : return NULL_TREE;
1631 : }
1632 :
1633 : /* Determine whether JFUNC evaluates to a single known constant value and if
1634 : so, return it. Otherwise return NULL. INFO describes the caller node or
1635 : the one it is inlined to, so that pass-through jump functions can be
1636 : evaluated. PARM_TYPE is the type of the parameter to which the result is
1637 : passed. */
1638 :
1639 : tree
1640 18414740 : ipa_value_from_jfunc (class ipa_node_params *info, struct ipa_jump_func *jfunc,
1641 : tree parm_type)
1642 : {
1643 18414740 : if (!parm_type)
1644 : return NULL_TREE;
1645 18171897 : if (jfunc->type == IPA_JF_CONST)
1646 4533734 : return ipacp_value_safe_for_type (parm_type, ipa_get_jf_constant (jfunc));
1647 13638163 : else if (jfunc->type == IPA_JF_PASS_THROUGH
1648 10654597 : || jfunc->type == IPA_JF_ANCESTOR)
1649 : {
1650 3819985 : tree input;
1651 3819985 : int idx;
1652 :
1653 3819985 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1654 2983566 : idx = ipa_get_jf_pass_through_formal_id (jfunc);
1655 : else
1656 836419 : idx = ipa_get_jf_ancestor_formal_id (jfunc);
1657 :
1658 3819985 : if (info->ipcp_orig_node)
1659 49844 : input = info->known_csts[idx];
1660 : else
1661 : {
1662 3770141 : ipcp_lattice<tree> *lat;
1663 :
1664 6744247 : if (info->lattices.is_empty ()
1665 2974106 : || idx >= ipa_get_param_count (info))
1666 : return NULL_TREE;
1667 2974106 : lat = ipa_get_scalar_lat (info, idx);
1668 2974106 : if (!lat->is_single_const ())
1669 : return NULL_TREE;
1670 152 : input = lat->values->value;
1671 : }
1672 :
1673 49996 : if (!input)
1674 : return NULL_TREE;
1675 :
1676 20085 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1677 : {
1678 19101 : enum tree_code opcode = ipa_get_jf_pass_through_operation (jfunc);
1679 19101 : tree op2 = ipa_get_jf_pass_through_operand (jfunc);
1680 19101 : tree op_type
1681 19101 : = (opcode == NOP_EXPR) ? NULL_TREE
1682 961 : : ipa_get_jf_pass_through_op_type (jfunc);
1683 19101 : tree cstval = ipa_get_jf_arith_result (opcode, input, op2, op_type);
1684 19101 : return ipacp_value_safe_for_type (parm_type, cstval);
1685 : }
1686 : else
1687 984 : return ipacp_value_safe_for_type (parm_type,
1688 : ipa_get_jf_ancestor_result (jfunc,
1689 984 : input));
1690 : }
1691 : else
1692 : return NULL_TREE;
1693 : }
1694 :
1695 : /* Determine whether JFUNC evaluates to single known polymorphic context, given
1696 : that INFO describes the caller node or the one it is inlined to, CS is the
1697 : call graph edge corresponding to JFUNC and CSIDX index of the described
1698 : parameter. */
1699 :
1700 : ipa_polymorphic_call_context
1701 942649 : ipa_context_from_jfunc (ipa_node_params *info, cgraph_edge *cs, int csidx,
1702 : ipa_jump_func *jfunc)
1703 : {
1704 942649 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
1705 942649 : ipa_polymorphic_call_context ctx;
1706 942649 : ipa_polymorphic_call_context *edge_ctx
1707 942649 : = cs ? ipa_get_ith_polymorhic_call_context (args, csidx) : NULL;
1708 :
1709 398923 : if (edge_ctx && !edge_ctx->useless_p ())
1710 390902 : ctx = *edge_ctx;
1711 :
1712 942649 : if (jfunc->type == IPA_JF_PASS_THROUGH
1713 842626 : || jfunc->type == IPA_JF_ANCESTOR)
1714 : {
1715 108784 : ipa_polymorphic_call_context srcctx;
1716 108784 : int srcidx;
1717 108784 : bool type_preserved = true;
1718 108784 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1719 : {
1720 100023 : if (ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR)
1721 1787 : return ctx;
1722 98236 : type_preserved = ipa_get_jf_pass_through_type_preserved (jfunc);
1723 98236 : srcidx = ipa_get_jf_pass_through_formal_id (jfunc);
1724 : }
1725 : else
1726 : {
1727 8761 : type_preserved = ipa_get_jf_ancestor_type_preserved (jfunc);
1728 8761 : srcidx = ipa_get_jf_ancestor_formal_id (jfunc);
1729 : }
1730 106997 : if (info->ipcp_orig_node)
1731 : {
1732 12101 : if (info->known_contexts.exists ())
1733 1725 : srcctx = info->known_contexts[srcidx];
1734 : }
1735 : else
1736 : {
1737 184351 : if (info->lattices.is_empty ()
1738 89455 : || srcidx >= ipa_get_param_count (info))
1739 5441 : return ctx;
1740 89455 : ipcp_lattice<ipa_polymorphic_call_context> *lat;
1741 89455 : lat = ipa_get_poly_ctx_lat (info, srcidx);
1742 89455 : if (!lat->is_single_const ())
1743 85446 : return ctx;
1744 4009 : srcctx = lat->values->value;
1745 : }
1746 16110 : if (srcctx.useless_p ())
1747 10829 : return ctx;
1748 5281 : if (jfunc->type == IPA_JF_ANCESTOR)
1749 259 : srcctx.offset_by (ipa_get_jf_ancestor_offset (jfunc));
1750 5281 : if (!type_preserved)
1751 3229 : srcctx.possible_dynamic_type_change (cs->in_polymorphic_cdtor);
1752 5281 : srcctx.combine_with (ctx);
1753 5281 : return srcctx;
1754 : }
1755 :
1756 833865 : return ctx;
1757 : }
1758 :
1759 : /* Emulate effects of unary OPERATION and/or conversion from SRC_TYPE to
1760 : DST_TYPE on value range in SRC_VR and store it to DST_VR. Return true if
1761 : the result is a range that is not VARYING nor UNDEFINED. */
1762 :
1763 : bool
1764 9531934 : ipa_vr_operation_and_type_effects (vrange &dst_vr,
1765 : const vrange &src_vr,
1766 : enum tree_code operation,
1767 : tree dst_type, tree src_type)
1768 : {
1769 17860675 : if (!ipa_vr_supported_type_p (dst_type)
1770 19063868 : || !ipa_vr_supported_type_p (src_type))
1771 : return false;
1772 :
1773 9531934 : range_op_handler handler (operation);
1774 9531934 : if (!handler)
1775 : return false;
1776 :
1777 9531934 : value_range varying (dst_type);
1778 9531934 : varying.set_varying (dst_type);
1779 :
1780 9531934 : return (handler.operand_check_p (dst_type, src_type, dst_type)
1781 9531934 : && handler.fold_range (dst_vr, dst_type, src_vr, varying)
1782 9531932 : && !dst_vr.varying_p ()
1783 19063783 : && !dst_vr.undefined_p ());
1784 9531934 : }
1785 :
1786 : /* Same as above, but the SRC_VR argument is an IPA_VR which must
1787 : first be extracted onto a vrange. */
1788 :
1789 : bool
1790 9524332 : ipa_vr_operation_and_type_effects (vrange &dst_vr,
1791 : const ipa_vr &src_vr,
1792 : enum tree_code operation,
1793 : tree dst_type, tree src_type)
1794 : {
1795 9524332 : value_range tmp;
1796 9524332 : src_vr.get_vrange (tmp);
1797 9524332 : return ipa_vr_operation_and_type_effects (dst_vr, tmp, operation,
1798 9524332 : dst_type, src_type);
1799 9524332 : }
1800 :
1801 : /* Given a PASS_THROUGH jump function JFUNC that takes as its source SRC_VR of
1802 : SRC_TYPE and the result needs to be DST_TYPE, if any value range information
1803 : can be deduced at all, intersect VR with it. CONTEXT_NODE is the call graph
1804 : node representing the function for which optimization flags should be
1805 : evaluated. */
1806 :
1807 : static void
1808 93628 : ipa_vr_intersect_with_arith_jfunc (vrange &vr,
1809 : ipa_jump_func *jfunc,
1810 : cgraph_node *context_node,
1811 : const value_range &src_vr,
1812 : tree src_type,
1813 : tree dst_type)
1814 : {
1815 93628 : if (src_vr.undefined_p () || src_vr.varying_p ())
1816 92402 : return;
1817 :
1818 93144 : enum tree_code operation = ipa_get_jf_pass_through_operation (jfunc);
1819 93144 : if (TREE_CODE_CLASS (operation) == tcc_unary)
1820 : {
1821 91918 : value_range op_res;
1822 91918 : const value_range *inter_vr;
1823 91918 : if (operation != NOP_EXPR)
1824 : {
1825 93 : tree operation_type = ipa_get_jf_pass_through_op_type (jfunc);
1826 93 : op_res.set_varying (operation_type);
1827 93 : if (!ipa_vr_operation_and_type_effects (op_res, src_vr, operation,
1828 : operation_type, src_type))
1829 : return;
1830 : inter_vr = &op_res;
1831 : src_type = operation_type;
1832 : }
1833 : else
1834 : inter_vr = &src_vr;
1835 :
1836 91918 : if (src_type != dst_type)
1837 : {
1838 6283 : value_range tmp_res (dst_type);
1839 6283 : if (!ipa_vr_operation_and_type_effects (tmp_res, *inter_vr, NOP_EXPR,
1840 : dst_type, src_type))
1841 0 : return;
1842 6283 : vr.intersect (tmp_res);
1843 6283 : }
1844 : else
1845 85635 : vr.intersect (*inter_vr);
1846 : return;
1847 91918 : }
1848 :
1849 1226 : tree operand = ipa_get_jf_pass_through_operand (jfunc);
1850 1226 : range_op_handler handler (operation);
1851 1226 : if (!handler)
1852 : return;
1853 1226 : value_range op_vr (TREE_TYPE (operand));
1854 1226 : ipa_get_range_from_ip_invariant (op_vr, operand, context_node);
1855 :
1856 1226 : tree operation_type = ipa_get_jf_pass_through_op_type (jfunc);
1857 1226 : value_range op_res (operation_type);
1858 1666 : if (!ipa_vr_supported_type_p (operation_type)
1859 1226 : || !handler.operand_check_p (operation_type, src_type, op_vr.type ())
1860 1226 : || !handler.fold_range (op_res, operation_type, src_vr, op_vr))
1861 0 : return;
1862 :
1863 1226 : value_range tmp_res (dst_type);
1864 1226 : if (ipa_vr_operation_and_type_effects (tmp_res, op_res, NOP_EXPR, dst_type,
1865 : operation_type))
1866 1178 : vr.intersect (tmp_res);
1867 1226 : }
1868 :
1869 : /* Determine range of JFUNC given that INFO describes the caller node or
1870 : the one it is inlined to, CS is the call graph edge corresponding to JFUNC
1871 : and PARM_TYPE of the parameter. */
1872 :
1873 : void
1874 12311984 : ipa_value_range_from_jfunc (vrange &vr,
1875 : ipa_node_params *info, cgraph_edge *cs,
1876 : ipa_jump_func *jfunc, tree parm_type)
1877 : {
1878 12311984 : vr.set_varying (parm_type);
1879 :
1880 12311984 : if (jfunc->m_vr && jfunc->m_vr->known_p ())
1881 8684574 : ipa_vr_operation_and_type_effects (vr,
1882 : *jfunc->m_vr,
1883 : NOP_EXPR, parm_type,
1884 8684574 : jfunc->m_vr->type ());
1885 12311984 : if (vr.singleton_p ())
1886 : return;
1887 :
1888 12311883 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1889 : {
1890 2394846 : ipcp_transformation *sum
1891 2394846 : = ipcp_get_transformation_summary (cs->caller->inlined_to
1892 : ? cs->caller->inlined_to
1893 : : cs->caller);
1894 2394846 : if (!sum || !sum->m_vr)
1895 2316711 : return;
1896 :
1897 120443 : int idx = ipa_get_jf_pass_through_formal_id (jfunc);
1898 :
1899 120443 : if (!(*sum->m_vr)[idx].known_p ())
1900 : return;
1901 78135 : tree src_type = ipa_get_type (info, idx);
1902 78135 : value_range srcvr;
1903 78135 : (*sum->m_vr)[idx].get_vrange (srcvr);
1904 :
1905 78135 : ipa_vr_intersect_with_arith_jfunc (vr, jfunc, cs->caller, srcvr, src_type,
1906 : parm_type);
1907 78135 : }
1908 : }
1909 :
1910 : /* Determine whether ITEM, jump function for an aggregate part, evaluates to a
1911 : single known constant value and if so, return it. Otherwise return NULL.
1912 : NODE and INFO describes the caller node or the one it is inlined to, and
1913 : its related info. */
1914 :
1915 : tree
1916 3516447 : ipa_agg_value_from_jfunc (ipa_node_params *info, cgraph_node *node,
1917 : const ipa_agg_jf_item *item)
1918 : {
1919 3516447 : tree value = NULL_TREE;
1920 3516447 : int src_idx;
1921 :
1922 3516447 : if (item->offset < 0
1923 3466557 : || item->jftype == IPA_JF_UNKNOWN
1924 3303688 : || item->offset >= (HOST_WIDE_INT) UINT_MAX * BITS_PER_UNIT)
1925 : return NULL_TREE;
1926 :
1927 3303688 : if (item->jftype == IPA_JF_CONST)
1928 2936847 : return item->value.constant;
1929 :
1930 366841 : gcc_checking_assert (item->jftype == IPA_JF_PASS_THROUGH
1931 : || item->jftype == IPA_JF_LOAD_AGG);
1932 :
1933 366841 : src_idx = item->value.pass_through.formal_id;
1934 :
1935 366841 : if (info->ipcp_orig_node)
1936 : {
1937 17024 : if (item->jftype == IPA_JF_PASS_THROUGH)
1938 3877 : value = info->known_csts[src_idx];
1939 13147 : else if (ipcp_transformation *ts = ipcp_get_transformation_summary (node))
1940 : {
1941 13147 : ipa_argagg_value_list avl (ts);
1942 13147 : value = avl.get_value (src_idx,
1943 13147 : item->value.load_agg.offset / BITS_PER_UNIT,
1944 13147 : item->value.load_agg.by_ref);
1945 : }
1946 : }
1947 349817 : else if (!info->lattices.is_empty ())
1948 : {
1949 231541 : class ipcp_param_lattices *src_plats
1950 231541 : = ipa_get_parm_lattices (info, src_idx);
1951 :
1952 231541 : if (item->jftype == IPA_JF_PASS_THROUGH)
1953 : {
1954 136465 : struct ipcp_lattice<tree> *lat = &src_plats->itself;
1955 :
1956 136465 : if (!lat->is_single_const ())
1957 : return NULL_TREE;
1958 :
1959 0 : value = lat->values->value;
1960 : }
1961 95076 : else if (src_plats->aggs
1962 12316 : && !src_plats->aggs_bottom
1963 12316 : && !src_plats->aggs_contain_variable
1964 1503 : && src_plats->aggs_by_ref == item->value.load_agg.by_ref)
1965 : {
1966 : struct ipcp_agg_lattice *aglat;
1967 :
1968 2370 : for (aglat = src_plats->aggs; aglat; aglat = aglat->next)
1969 : {
1970 2370 : if (aglat->offset > item->value.load_agg.offset)
1971 : break;
1972 :
1973 2338 : if (aglat->offset == item->value.load_agg.offset)
1974 : {
1975 1471 : if (aglat->is_single_const ())
1976 7 : value = aglat->values->value;
1977 : break;
1978 : }
1979 : }
1980 : }
1981 : }
1982 :
1983 17063 : if (!value)
1984 : return NULL_TREE;
1985 :
1986 10307 : if (item->jftype == IPA_JF_LOAD_AGG)
1987 : {
1988 7974 : tree load_type = item->value.load_agg.type;
1989 7974 : tree value_type = TREE_TYPE (value);
1990 :
1991 : /* Ensure value type is compatible with load type. */
1992 7974 : if (!useless_type_conversion_p (load_type, value_type))
1993 : return NULL_TREE;
1994 : }
1995 :
1996 20614 : tree cstval = ipa_get_jf_arith_result (item->value.pass_through.operation,
1997 : value,
1998 10307 : item->value.pass_through.operand,
1999 10307 : item->value.pass_through.op_type);
2000 10307 : return ipacp_value_safe_for_type (item->type, cstval);
2001 : }
2002 :
2003 : /* Process all items in AGG_JFUNC relative to caller (or the node the original
2004 : caller is inlined to) NODE which described by INFO and push the results to
2005 : RES as describing values passed in parameter DST_INDEX. */
2006 :
2007 : void
2008 14744211 : ipa_push_agg_values_from_jfunc (ipa_node_params *info, cgraph_node *node,
2009 : ipa_agg_jump_function *agg_jfunc,
2010 : unsigned dst_index,
2011 : vec<ipa_argagg_value> *res)
2012 : {
2013 14744211 : unsigned prev_unit_offset = 0;
2014 14744211 : bool first = true;
2015 :
2016 19815137 : for (const ipa_agg_jf_item &item : agg_jfunc->items)
2017 : {
2018 2469498 : tree value = ipa_agg_value_from_jfunc (info, node, &item);
2019 2469498 : if (!value)
2020 543658 : continue;
2021 :
2022 1925840 : ipa_argagg_value iav;
2023 1925840 : iav.value = value;
2024 1925840 : iav.unit_offset = item.offset / BITS_PER_UNIT;
2025 1925840 : iav.index = dst_index;
2026 1925840 : iav.by_ref = agg_jfunc->by_ref;
2027 1925840 : iav.killed = 0;
2028 :
2029 1925840 : gcc_assert (first
2030 : || iav.unit_offset > prev_unit_offset);
2031 1925840 : prev_unit_offset = iav.unit_offset;
2032 1925840 : first = false;
2033 :
2034 1925840 : res->safe_push (iav);
2035 : }
2036 14744211 : }
2037 :
2038 : /* If checking is enabled, verify that no lattice is in the TOP state, i.e. not
2039 : bottom, not containing a variable component and without any known value at
2040 : the same time. */
2041 :
2042 : DEBUG_FUNCTION void
2043 132699 : ipcp_verify_propagated_values (void)
2044 : {
2045 132699 : struct cgraph_node *node;
2046 :
2047 1442028 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
2048 : {
2049 1309329 : ipa_node_params *info = ipa_node_params_sum->get (node);
2050 1309329 : if (!opt_for_fn (node->decl, flag_ipa_cp)
2051 1309329 : || !opt_for_fn (node->decl, optimize))
2052 9023 : continue;
2053 1300306 : int i, count = ipa_get_param_count (info);
2054 :
2055 3693428 : for (i = 0; i < count; i++)
2056 : {
2057 2393122 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (info, i);
2058 :
2059 2393122 : if (!lat->bottom
2060 229875 : && !lat->contains_variable
2061 32588 : && lat->values_count == 0)
2062 : {
2063 0 : if (dump_file)
2064 : {
2065 0 : symtab->dump (dump_file);
2066 0 : fprintf (dump_file, "\nIPA lattices after constant "
2067 : "propagation, before gcc_unreachable:\n");
2068 0 : print_all_lattices (dump_file, true, false);
2069 : }
2070 :
2071 0 : gcc_unreachable ();
2072 : }
2073 : }
2074 : }
2075 132699 : }
2076 :
2077 : /* Return true iff X and Y should be considered equal contexts by IPA-CP. */
2078 :
2079 : static bool
2080 2887 : values_equal_for_ipcp_p (ipa_polymorphic_call_context x,
2081 : ipa_polymorphic_call_context y)
2082 : {
2083 2339 : return x.equal_to (y);
2084 : }
2085 :
2086 :
2087 : /* Add a new value source to the value represented by THIS, marking that a
2088 : value comes from edge CS and (if the underlying jump function is a
2089 : pass-through or an ancestor one) from a caller value SRC_VAL of a caller
2090 : parameter described by SRC_INDEX. OFFSET is negative if the source was the
2091 : scalar value of the parameter itself or the offset within an aggregate. */
2092 :
2093 : template <typename valtype>
2094 : void
2095 342892 : ipcp_value<valtype>::add_source (cgraph_edge *cs, ipcp_value *src_val,
2096 : int src_idx, HOST_WIDE_INT offset)
2097 : {
2098 : ipcp_value_source<valtype> *src;
2099 :
2100 495851 : src = new (ipcp_sources_pool.allocate ()) ipcp_value_source<valtype>;
2101 495851 : src->offset = offset;
2102 495851 : src->cs = cs;
2103 495851 : src->val = src_val;
2104 495851 : src->index = src_idx;
2105 :
2106 495851 : src->next = sources;
2107 495851 : sources = src;
2108 : }
2109 :
2110 : /* Allocate a new ipcp_value holding a tree constant, initialize its value to
2111 : SOURCE and clear all other fields. */
2112 :
2113 : static ipcp_value<tree> *
2114 145073 : allocate_and_init_ipcp_value (tree cst, unsigned same_lat_gen_level)
2115 : {
2116 145073 : ipcp_value<tree> *val;
2117 :
2118 145073 : val = new (ipcp_cst_values_pool.allocate ()) ipcp_value<tree>();
2119 145073 : val->value = cst;
2120 145073 : val->self_recursion_generated_level = same_lat_gen_level;
2121 145073 : return val;
2122 : }
2123 :
2124 : /* Allocate a new ipcp_value holding a polymorphic context, initialize its
2125 : value to SOURCE and clear all other fields. */
2126 :
2127 : static ipcp_value<ipa_polymorphic_call_context> *
2128 7886 : allocate_and_init_ipcp_value (ipa_polymorphic_call_context ctx,
2129 : unsigned same_lat_gen_level)
2130 : {
2131 7886 : ipcp_value<ipa_polymorphic_call_context> *val;
2132 :
2133 7886 : val = new (ipcp_poly_ctx_values_pool.allocate ())
2134 7886 : ipcp_value<ipa_polymorphic_call_context>();
2135 7886 : val->value = ctx;
2136 7886 : val->self_recursion_generated_level = same_lat_gen_level;
2137 7886 : return val;
2138 : }
2139 :
2140 : /* Try to add NEWVAL to LAT, potentially creating a new ipcp_value for it. CS,
2141 : SRC_VAL SRC_INDEX and OFFSET are meant for add_source and have the same
2142 : meaning. OFFSET -1 means the source is scalar and not a part of an
2143 : aggregate. If non-NULL, VAL_P records address of existing or newly added
2144 : ipcp_value.
2145 :
2146 : If the value is generated for a self-recursive call as a result of an
2147 : arithmetic pass-through jump-function acting on a value in the same lattice,
2148 : SAME_LAT_GEN_LEVEL must be the length of such chain, otherwise it must be
2149 : zero. If it is non-zero, PARAM_IPA_CP_VALUE_LIST_SIZE limit is ignored. */
2150 :
2151 : template <typename valtype>
2152 : bool
2153 508294 : ipcp_lattice<valtype>::add_value (valtype newval, cgraph_edge *cs,
2154 : ipcp_value<valtype> *src_val,
2155 : int src_idx, HOST_WIDE_INT offset,
2156 : ipcp_value<valtype> **val_p,
2157 : unsigned same_lat_gen_level)
2158 : {
2159 508294 : ipcp_value<valtype> *val, *last_val = NULL;
2160 :
2161 508294 : if (val_p)
2162 1257 : *val_p = NULL;
2163 :
2164 508294 : if (bottom)
2165 : return false;
2166 :
2167 982447 : for (val = values; val; last_val = val, val = val->next)
2168 828082 : if (values_equal_for_ipcp_p (val->value, newval))
2169 : {
2170 350565 : if (val_p)
2171 416 : *val_p = val;
2172 :
2173 350565 : if (val->self_recursion_generated_level < same_lat_gen_level)
2174 179 : val->self_recursion_generated_level = same_lat_gen_level;
2175 :
2176 350565 : if (ipa_edge_within_scc (cs))
2177 : {
2178 : ipcp_value_source<valtype> *s;
2179 48906 : for (s = val->sources; s; s = s->next)
2180 44670 : if (s->cs == cs && s->val == src_val)
2181 : break;
2182 11909 : if (s)
2183 : return false;
2184 : }
2185 :
2186 342892 : val->add_source (cs, src_val, src_idx, offset);
2187 342892 : return false;
2188 : }
2189 :
2190 154365 : if (!same_lat_gen_level && values_count >= opt_for_fn (cs->callee->decl,
2191 : param_ipa_cp_value_list_size))
2192 : {
2193 : /* We can only free sources, not the values themselves, because sources
2194 : of other values in this SCC might point to them. */
2195 12636 : for (val = values; val; val = val->next)
2196 : {
2197 41082 : while (val->sources)
2198 : {
2199 29852 : ipcp_value_source<valtype> *src = val->sources;
2200 29852 : val->sources = src->next;
2201 29852 : ipcp_sources_pool.remove ((ipcp_value_source<tree>*)src);
2202 : }
2203 : }
2204 1406 : values = NULL;
2205 1406 : return set_to_bottom ();
2206 : }
2207 :
2208 152959 : values_count++;
2209 152959 : val = allocate_and_init_ipcp_value (newval, same_lat_gen_level);
2210 152959 : val->add_source (cs, src_val, src_idx, offset);
2211 152959 : val->next = NULL;
2212 :
2213 : /* Add the new value to end of value list, which can reduce iterations
2214 : of propagation stage for recursive function. */
2215 152959 : if (last_val)
2216 46811 : last_val->next = val;
2217 : else
2218 106148 : values = val;
2219 :
2220 152959 : if (val_p)
2221 841 : *val_p = val;
2222 :
2223 : return true;
2224 : }
2225 :
2226 : /* A helper function that returns result of operation specified by OPCODE on
2227 : the value of SRC_VAL. If non-NULL, OPND1_TYPE is expected type for the
2228 : value of SRC_VAL. If the operation is binary, OPND2 is a constant value
2229 : acting as its second operand. OP_TYPE is the type in which the operation is
2230 : performed. */
2231 :
2232 : static tree
2233 22156 : get_val_across_arith_op (enum tree_code opcode,
2234 : tree opnd1_type,
2235 : tree opnd2,
2236 : ipcp_value<tree> *src_val,
2237 : tree op_type)
2238 : {
2239 22156 : tree opnd1 = src_val->value;
2240 :
2241 : /* Skip source values that is incompatible with specified type. */
2242 22156 : if (opnd1_type
2243 22156 : && !useless_type_conversion_p (opnd1_type, TREE_TYPE (opnd1)))
2244 : return NULL_TREE;
2245 :
2246 22156 : return ipa_get_jf_arith_result (opcode, opnd1, opnd2, op_type);
2247 : }
2248 :
2249 : /* Propagate values through an arithmetic transformation described by a jump
2250 : function associated with edge CS, taking values from SRC_LAT and putting
2251 : them into DEST_LAT. OPND1_TYPE, if non-NULL, is the expected type for the
2252 : values in SRC_LAT. OPND2 is a constant value if transformation is a binary
2253 : operation. SRC_OFFSET specifies offset in an aggregate if SRC_LAT describes
2254 : lattice of a part of an aggregate, otherwise it should be -1. SRC_IDX is
2255 : the index of the source parameter. OP_TYPE is the type in which the
2256 : operation is performed and can be NULL when OPCODE is NOP_EXPR. RES_TYPE is
2257 : the value type of result being propagated into. Return true if DEST_LAT
2258 : changed. */
2259 :
2260 : static bool
2261 78030 : propagate_vals_across_arith_jfunc (cgraph_edge *cs,
2262 : enum tree_code opcode,
2263 : tree opnd1_type,
2264 : tree opnd2,
2265 : ipcp_lattice<tree> *src_lat,
2266 : ipcp_lattice<tree> *dest_lat,
2267 : HOST_WIDE_INT src_offset,
2268 : int src_idx,
2269 : tree op_type,
2270 : tree res_type)
2271 : {
2272 78030 : ipcp_value<tree> *src_val;
2273 78030 : bool ret = false;
2274 :
2275 : /* Due to circular dependencies, propagating within an SCC through arithmetic
2276 : transformation would create infinite number of values. But for
2277 : self-feeding recursive function, we could allow propagation in a limited
2278 : count, and this can enable a simple kind of recursive function versioning.
2279 : For other scenario, we would just make lattices bottom. */
2280 78030 : if (opcode != NOP_EXPR && ipa_edge_within_scc (cs))
2281 : {
2282 2190 : int i;
2283 :
2284 2190 : int max_recursive_depth = opt_for_fn(cs->caller->decl,
2285 : param_ipa_cp_max_recursive_depth);
2286 2190 : if (src_lat != dest_lat || max_recursive_depth < 1)
2287 1666 : return dest_lat->set_contains_variable ();
2288 :
2289 : /* No benefit if recursive execution is in low probability. */
2290 1306 : if (cs->sreal_frequency () * 100
2291 2612 : <= ((sreal) 1) * opt_for_fn (cs->caller->decl,
2292 : param_ipa_cp_min_recursive_probability))
2293 89 : return dest_lat->set_contains_variable ();
2294 :
2295 1217 : auto_vec<ipcp_value<tree> *, 8> val_seeds;
2296 :
2297 2264 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2298 : {
2299 : /* Now we do not use self-recursively generated value as propagation
2300 : source, this is absolutely conservative, but could avoid explosion
2301 : of lattice's value space, especially when one recursive function
2302 : calls another recursive. */
2303 1740 : if (src_val->self_recursion_generated_p ())
2304 : {
2305 909 : ipcp_value_source<tree> *s;
2306 :
2307 : /* If the lattice has already been propagated for the call site,
2308 : no need to do that again. */
2309 1422 : for (s = src_val->sources; s; s = s->next)
2310 1206 : if (s->cs == cs)
2311 693 : return dest_lat->set_contains_variable ();
2312 : }
2313 : else
2314 831 : val_seeds.safe_push (src_val);
2315 : }
2316 :
2317 1048 : gcc_assert ((int) val_seeds.length () <= param_ipa_cp_value_list_size);
2318 :
2319 : /* Recursively generate lattice values with a limited count. */
2320 1366 : FOR_EACH_VEC_ELT (val_seeds, i, src_val)
2321 : {
2322 1416 : for (int j = 1; j < max_recursive_depth; j++)
2323 : {
2324 1261 : tree cstval = get_val_across_arith_op (opcode, opnd1_type, opnd2,
2325 : src_val, op_type);
2326 1261 : cstval = ipacp_value_safe_for_type (res_type, cstval);
2327 1261 : if (!cstval)
2328 : break;
2329 :
2330 1257 : ret |= dest_lat->add_value (cstval, cs, src_val, src_idx,
2331 : src_offset, &src_val, j);
2332 1257 : gcc_checking_assert (src_val);
2333 : }
2334 : }
2335 524 : ret |= dest_lat->set_contains_variable ();
2336 1217 : }
2337 : else
2338 96860 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2339 : {
2340 : /* Now we do not use self-recursively generated value as propagation
2341 : source, otherwise it is easy to make value space of normal lattice
2342 : overflow. */
2343 21020 : if (src_val->self_recursion_generated_p ())
2344 : {
2345 125 : ret |= dest_lat->set_contains_variable ();
2346 125 : continue;
2347 : }
2348 :
2349 20895 : tree cstval = get_val_across_arith_op (opcode, opnd1_type, opnd2,
2350 : src_val, op_type);
2351 20895 : cstval = ipacp_value_safe_for_type (res_type, cstval);
2352 20895 : if (cstval)
2353 20694 : ret |= dest_lat->add_value (cstval, cs, src_val, src_idx,
2354 : src_offset);
2355 : else
2356 201 : ret |= dest_lat->set_contains_variable ();
2357 : }
2358 :
2359 : return ret;
2360 : }
2361 :
2362 : /* Propagate values through a pass-through jump function JFUNC associated with
2363 : edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
2364 : is the index of the source parameter. PARM_TYPE is the type of the
2365 : parameter to which the result is passed. */
2366 :
2367 : static bool
2368 73218 : propagate_vals_across_pass_through (cgraph_edge *cs, ipa_jump_func *jfunc,
2369 : ipcp_lattice<tree> *src_lat,
2370 : ipcp_lattice<tree> *dest_lat, int src_idx,
2371 : tree parm_type)
2372 : {
2373 73218 : gcc_checking_assert (parm_type);
2374 73218 : enum tree_code opcode = ipa_get_jf_pass_through_operation (jfunc);
2375 73218 : tree op_type = (opcode == NOP_EXPR) ? NULL_TREE
2376 2390 : : ipa_get_jf_pass_through_op_type (jfunc);
2377 73218 : return propagate_vals_across_arith_jfunc (cs, opcode, NULL_TREE,
2378 : ipa_get_jf_pass_through_operand (jfunc),
2379 : src_lat, dest_lat, -1, src_idx, op_type,
2380 73218 : parm_type);
2381 : }
2382 :
2383 : /* Propagate values through an ancestor jump function JFUNC associated with
2384 : edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
2385 : is the index of the source parameter. */
2386 :
2387 : static bool
2388 2302 : propagate_vals_across_ancestor (struct cgraph_edge *cs,
2389 : struct ipa_jump_func *jfunc,
2390 : ipcp_lattice<tree> *src_lat,
2391 : ipcp_lattice<tree> *dest_lat, int src_idx,
2392 : tree param_type)
2393 : {
2394 2302 : ipcp_value<tree> *src_val;
2395 2302 : bool ret = false;
2396 :
2397 2302 : if (ipa_edge_within_scc (cs))
2398 14 : return dest_lat->set_contains_variable ();
2399 :
2400 2606 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2401 : {
2402 318 : tree t = ipa_get_jf_ancestor_result (jfunc, src_val->value);
2403 318 : t = ipacp_value_safe_for_type (param_type, t);
2404 318 : if (t)
2405 260 : ret |= dest_lat->add_value (t, cs, src_val, src_idx);
2406 : else
2407 58 : ret |= dest_lat->set_contains_variable ();
2408 : }
2409 :
2410 : return ret;
2411 : }
2412 :
2413 : /* Propagate scalar values across jump function JFUNC that is associated with
2414 : edge CS and put the values into DEST_LAT. PARM_TYPE is the type of the
2415 : parameter to which the result is passed. */
2416 :
2417 : static bool
2418 4026337 : propagate_scalar_across_jump_function (struct cgraph_edge *cs,
2419 : struct ipa_jump_func *jfunc,
2420 : ipcp_lattice<tree> *dest_lat,
2421 : tree param_type)
2422 : {
2423 4026337 : if (dest_lat->bottom)
2424 : return false;
2425 :
2426 833563 : if (jfunc->type == IPA_JF_CONST)
2427 : {
2428 373626 : tree val = ipa_get_jf_constant (jfunc);
2429 373626 : val = ipacp_value_safe_for_type (param_type, val);
2430 373626 : if (val)
2431 373608 : return dest_lat->add_value (val, cs, NULL, 0);
2432 : else
2433 18 : return dest_lat->set_contains_variable ();
2434 : }
2435 459937 : else if (jfunc->type == IPA_JF_PASS_THROUGH
2436 275517 : || jfunc->type == IPA_JF_ANCESTOR)
2437 : {
2438 189054 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2439 189054 : ipcp_lattice<tree> *src_lat;
2440 189054 : int src_idx;
2441 189054 : bool ret;
2442 :
2443 189054 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2444 184420 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2445 : else
2446 4634 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2447 :
2448 189054 : src_lat = ipa_get_scalar_lat (caller_info, src_idx);
2449 189054 : if (src_lat->bottom)
2450 113387 : return dest_lat->set_contains_variable ();
2451 :
2452 : /* If we would need to clone the caller and cannot, do not propagate. */
2453 75667 : if (!ipcp_versionable_function_p (cs->caller)
2454 75667 : && (src_lat->contains_variable
2455 134 : || (src_lat->values_count > 1)))
2456 147 : return dest_lat->set_contains_variable ();
2457 :
2458 75520 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2459 73218 : ret = propagate_vals_across_pass_through (cs, jfunc, src_lat,
2460 : dest_lat, src_idx,
2461 : param_type);
2462 : else
2463 2302 : ret = propagate_vals_across_ancestor (cs, jfunc, src_lat, dest_lat,
2464 : src_idx, param_type);
2465 :
2466 75520 : if (src_lat->contains_variable)
2467 65922 : ret |= dest_lat->set_contains_variable ();
2468 :
2469 : return ret;
2470 : }
2471 :
2472 : /* TODO: We currently do not handle member method pointers in IPA-CP (we only
2473 : use it for indirect inlining), we should propagate them too. */
2474 270883 : return dest_lat->set_contains_variable ();
2475 : }
2476 :
2477 : /* Propagate scalar values across jump function JFUNC that is associated with
2478 : edge CS and describes argument IDX and put the values into DEST_LAT. */
2479 :
2480 : static bool
2481 4026337 : propagate_context_across_jump_function (cgraph_edge *cs,
2482 : ipa_jump_func *jfunc, int idx,
2483 : ipcp_lattice<ipa_polymorphic_call_context> *dest_lat)
2484 : {
2485 4026337 : if (dest_lat->bottom)
2486 : return false;
2487 930433 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
2488 930433 : bool ret = false;
2489 930433 : bool added_sth = false;
2490 930433 : bool type_preserved = true;
2491 :
2492 930433 : ipa_polymorphic_call_context edge_ctx, *edge_ctx_ptr
2493 945954 : = ipa_get_ith_polymorhic_call_context (args, idx);
2494 :
2495 15521 : if (edge_ctx_ptr)
2496 15521 : edge_ctx = *edge_ctx_ptr;
2497 :
2498 930433 : if (jfunc->type == IPA_JF_PASS_THROUGH
2499 745512 : || jfunc->type == IPA_JF_ANCESTOR)
2500 : {
2501 189651 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2502 189651 : int src_idx;
2503 189651 : ipcp_lattice<ipa_polymorphic_call_context> *src_lat;
2504 :
2505 : /* TODO: Once we figure out how to propagate speculations, it will
2506 : probably be a good idea to switch to speculation if type_preserved is
2507 : not set instead of punting. */
2508 189651 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2509 : {
2510 184921 : if (ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR)
2511 7534 : goto prop_fail;
2512 177387 : type_preserved = ipa_get_jf_pass_through_type_preserved (jfunc);
2513 177387 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2514 : }
2515 : else
2516 : {
2517 4730 : type_preserved = ipa_get_jf_ancestor_type_preserved (jfunc);
2518 4730 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2519 : }
2520 :
2521 182117 : src_lat = ipa_get_poly_ctx_lat (caller_info, src_idx);
2522 : /* If we would need to clone the caller and cannot, do not propagate. */
2523 182117 : if (!ipcp_versionable_function_p (cs->caller)
2524 182117 : && (src_lat->contains_variable
2525 14534 : || (src_lat->values_count > 1)))
2526 2509 : goto prop_fail;
2527 :
2528 179608 : ipcp_value<ipa_polymorphic_call_context> *src_val;
2529 180949 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2530 : {
2531 1341 : ipa_polymorphic_call_context cur = src_val->value;
2532 :
2533 1341 : if (!type_preserved)
2534 900 : cur.possible_dynamic_type_change (cs->in_polymorphic_cdtor);
2535 1341 : if (jfunc->type == IPA_JF_ANCESTOR)
2536 329 : cur.offset_by (ipa_get_jf_ancestor_offset (jfunc));
2537 : /* TODO: In cases we know how the context is going to be used,
2538 : we can improve the result by passing proper OTR_TYPE. */
2539 1341 : cur.combine_with (edge_ctx);
2540 2682 : if (!cur.useless_p ())
2541 : {
2542 870 : if (src_lat->contains_variable
2543 870 : && !edge_ctx.equal_to (cur))
2544 281 : ret |= dest_lat->set_contains_variable ();
2545 870 : ret |= dest_lat->add_value (cur, cs, src_val, src_idx);
2546 870 : added_sth = true;
2547 : }
2548 : }
2549 : }
2550 :
2551 740782 : prop_fail:
2552 189651 : if (!added_sth)
2553 : {
2554 929636 : if (!edge_ctx.useless_p ())
2555 8692 : ret |= dest_lat->add_value (edge_ctx, cs);
2556 : else
2557 920944 : ret |= dest_lat->set_contains_variable ();
2558 : }
2559 :
2560 : return ret;
2561 : }
2562 :
2563 : /* Propagate bits across jfunc that is associated with
2564 : edge cs and update dest_lattice accordingly. */
2565 :
2566 : bool
2567 4026337 : propagate_bits_across_jump_function (cgraph_edge *cs, int idx,
2568 : ipa_jump_func *jfunc,
2569 : ipcp_bits_lattice *dest_lattice)
2570 : {
2571 4026337 : if (dest_lattice->bottom_p ())
2572 : return false;
2573 :
2574 543184 : enum availability availability;
2575 543184 : cgraph_node *callee = cs->callee->function_symbol (&availability);
2576 543184 : ipa_node_params *callee_info = ipa_node_params_sum->get (callee);
2577 543184 : tree parm_type = ipa_get_type (callee_info, idx);
2578 :
2579 : /* For K&R C programs, ipa_get_type() could return NULL_TREE. Avoid the
2580 : transform for these cases. Similarly, we can have bad type mismatches
2581 : with LTO, avoid doing anything with those too. */
2582 543184 : if (!parm_type
2583 543184 : || (!INTEGRAL_TYPE_P (parm_type) && !POINTER_TYPE_P (parm_type)))
2584 : {
2585 30441 : if (dump_file && (dump_flags & TDF_DETAILS))
2586 11 : fprintf (dump_file, "Setting dest_lattice to bottom, because type of "
2587 : "param %i of %s is NULL or unsuitable for bits propagation\n",
2588 11 : idx, cs->callee->dump_name ());
2589 :
2590 30441 : return dest_lattice->set_to_bottom ();
2591 : }
2592 :
2593 512743 : if (jfunc->type == IPA_JF_PASS_THROUGH
2594 411863 : || jfunc->type == IPA_JF_ANCESTOR)
2595 : {
2596 103464 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2597 103464 : tree operand = NULL_TREE;
2598 103464 : tree op_type = NULL_TREE;
2599 103464 : enum tree_code code;
2600 103464 : unsigned src_idx;
2601 103464 : bool keep_null = false;
2602 :
2603 103464 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2604 : {
2605 100880 : code = ipa_get_jf_pass_through_operation (jfunc);
2606 100880 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2607 100880 : if (code != NOP_EXPR)
2608 : {
2609 2041 : operand = ipa_get_jf_pass_through_operand (jfunc);
2610 2041 : op_type = ipa_get_jf_pass_through_op_type (jfunc);
2611 : }
2612 : }
2613 : else
2614 : {
2615 2584 : code = POINTER_PLUS_EXPR;
2616 2584 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2617 2584 : unsigned HOST_WIDE_INT offset
2618 2584 : = ipa_get_jf_ancestor_offset (jfunc) / BITS_PER_UNIT;
2619 2584 : keep_null = (ipa_get_jf_ancestor_keep_null (jfunc) || !offset);
2620 2584 : operand = build_int_cstu (size_type_node, offset);
2621 : }
2622 :
2623 103464 : class ipcp_param_lattices *src_lats
2624 103464 : = ipa_get_parm_lattices (caller_info, src_idx);
2625 :
2626 : /* Try to propagate bits if src_lattice is bottom, but jfunc is known.
2627 : for eg consider:
2628 : int f(int x)
2629 : {
2630 : g (x & 0xff);
2631 : }
2632 : Assume lattice for x is bottom, however we can still propagate
2633 : result of x & 0xff == 0xff, which gets computed during ccp1 pass
2634 : and we store it in jump function during analysis stage. */
2635 :
2636 103464 : if (!src_lats->bits_lattice.bottom_p ()
2637 103464 : && !src_lats->bits_lattice.recipient_only_p ())
2638 : {
2639 21822 : if (!op_type)
2640 20693 : op_type = ipa_get_type (caller_info, src_idx);
2641 :
2642 21822 : unsigned precision = TYPE_PRECISION (op_type);
2643 21822 : signop sgn = TYPE_SIGN (op_type);
2644 21822 : bool drop_all_ones
2645 21822 : = keep_null && !src_lats->bits_lattice.known_nonzero_p ();
2646 :
2647 21822 : return dest_lattice->meet_with (src_lats->bits_lattice, precision,
2648 21822 : sgn, code, operand, drop_all_ones);
2649 : }
2650 : }
2651 :
2652 490921 : value_range vr (parm_type);
2653 490921 : if (jfunc->m_vr)
2654 : {
2655 418627 : jfunc->m_vr->get_vrange (vr);
2656 418627 : if (!vr.undefined_p () && !vr.varying_p ())
2657 : {
2658 418626 : irange_bitmask bm = vr.get_bitmask ();
2659 418626 : widest_int mask
2660 418626 : = widest_int::from (bm.mask (), TYPE_SIGN (parm_type));
2661 418626 : widest_int value
2662 418626 : = widest_int::from (bm.value (), TYPE_SIGN (parm_type));
2663 418626 : return dest_lattice->meet_with (value, mask,
2664 418626 : TYPE_PRECISION (parm_type));
2665 418626 : }
2666 : }
2667 72295 : return dest_lattice->set_to_bottom ();
2668 490921 : }
2669 :
2670 : /* Propagate value range across jump function JFUNC that is associated with
2671 : edge CS with param of callee of PARAM_TYPE and update DEST_PLATS
2672 : accordingly. */
2673 :
2674 : static bool
2675 4025504 : propagate_vr_across_jump_function (cgraph_edge *cs, ipa_jump_func *jfunc,
2676 : class ipcp_param_lattices *dest_plats,
2677 : tree param_type)
2678 : {
2679 4025504 : ipcp_vr_lattice *dest_lat = &dest_plats->m_value_range;
2680 :
2681 4025504 : if (dest_lat->bottom_p ())
2682 : return false;
2683 :
2684 636979 : if (!param_type
2685 636979 : || !ipa_vr_supported_type_p (param_type))
2686 30381 : return dest_lat->set_to_bottom ();
2687 :
2688 606598 : value_range vr (param_type);
2689 606598 : vr.set_varying (param_type);
2690 606598 : if (jfunc->m_vr)
2691 523223 : ipa_vr_operation_and_type_effects (vr, *jfunc->m_vr, NOP_EXPR,
2692 : param_type,
2693 523223 : jfunc->m_vr->type ());
2694 :
2695 606598 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2696 : {
2697 94991 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2698 94991 : int src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2699 94991 : class ipcp_param_lattices *src_lats
2700 94991 : = ipa_get_parm_lattices (caller_info, src_idx);
2701 94991 : tree operand_type = ipa_get_type (caller_info, src_idx);
2702 :
2703 94991 : if (src_lats->m_value_range.bottom_p ()
2704 94991 : || src_lats->m_value_range.recipient_only_p ())
2705 79003 : return dest_lat->set_to_bottom ();
2706 :
2707 15988 : if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR
2708 15988 : || !ipa_edge_within_scc (cs))
2709 15493 : ipa_vr_intersect_with_arith_jfunc (vr, jfunc, cs->caller,
2710 15493 : src_lats->m_value_range.m_vr,
2711 : operand_type, param_type);
2712 : }
2713 :
2714 527595 : if (!vr.undefined_p () && !vr.varying_p ())
2715 499139 : return dest_lat->meet_with (vr);
2716 : else
2717 28456 : return dest_lat->set_to_bottom ();
2718 606598 : }
2719 :
2720 : /* If DEST_PLATS already has aggregate items, check that aggs_by_ref matches
2721 : NEW_AGGS_BY_REF and if not, mark all aggs as bottoms and return true (in all
2722 : other cases, return false). If there are no aggregate items, set
2723 : aggs_by_ref to NEW_AGGS_BY_REF. */
2724 :
2725 : static bool
2726 42969 : set_check_aggs_by_ref (class ipcp_param_lattices *dest_plats,
2727 : bool new_aggs_by_ref)
2728 : {
2729 0 : if (dest_plats->aggs)
2730 : {
2731 22687 : if (dest_plats->aggs_by_ref != new_aggs_by_ref)
2732 : {
2733 0 : set_agg_lats_to_bottom (dest_plats);
2734 0 : return true;
2735 : }
2736 : }
2737 : else
2738 0 : dest_plats->aggs_by_ref = new_aggs_by_ref;
2739 : return false;
2740 : }
2741 :
2742 : /* Walk aggregate lattices in DEST_PLATS from ***AGLAT on, until ***aglat is an
2743 : already existing lattice for the given OFFSET and SIZE, marking all skipped
2744 : lattices as containing variable and checking for overlaps. If there is no
2745 : already existing lattice for the OFFSET and VAL_SIZE, create one, initialize
2746 : it with offset, size and contains_variable to PRE_EXISTING, and return true,
2747 : unless there are too many already. If there are two many, return false. If
2748 : there are overlaps turn whole DEST_PLATS to bottom and return false. If any
2749 : skipped lattices were newly marked as containing variable, set *CHANGE to
2750 : true. MAX_AGG_ITEMS is the maximum number of lattices. */
2751 :
2752 : static bool
2753 118165 : merge_agg_lats_step (class ipcp_param_lattices *dest_plats,
2754 : HOST_WIDE_INT offset, HOST_WIDE_INT val_size,
2755 : struct ipcp_agg_lattice ***aglat,
2756 : bool pre_existing, bool *change, int max_agg_items)
2757 : {
2758 118165 : gcc_checking_assert (offset >= 0);
2759 :
2760 122763 : while (**aglat && (**aglat)->offset < offset)
2761 : {
2762 4598 : if ((**aglat)->offset + (**aglat)->size > offset)
2763 : {
2764 0 : set_agg_lats_to_bottom (dest_plats);
2765 0 : return false;
2766 : }
2767 4598 : *change |= (**aglat)->set_contains_variable ();
2768 4598 : *aglat = &(**aglat)->next;
2769 : }
2770 :
2771 118165 : if (**aglat && (**aglat)->offset == offset)
2772 : {
2773 58422 : if ((**aglat)->size != val_size)
2774 : {
2775 13 : set_agg_lats_to_bottom (dest_plats);
2776 13 : return false;
2777 : }
2778 58409 : gcc_assert (!(**aglat)->next
2779 : || (**aglat)->next->offset >= offset + val_size);
2780 : return true;
2781 : }
2782 : else
2783 : {
2784 59743 : struct ipcp_agg_lattice *new_al;
2785 :
2786 59743 : if (**aglat && (**aglat)->offset < offset + val_size)
2787 : {
2788 3 : set_agg_lats_to_bottom (dest_plats);
2789 3 : return false;
2790 : }
2791 59740 : if (dest_plats->aggs_count == max_agg_items)
2792 : return false;
2793 59701 : dest_plats->aggs_count++;
2794 59701 : new_al = ipcp_agg_lattice_pool.allocate ();
2795 :
2796 59701 : new_al->offset = offset;
2797 59701 : new_al->size = val_size;
2798 59701 : new_al->contains_variable = pre_existing;
2799 :
2800 59701 : new_al->next = **aglat;
2801 59701 : **aglat = new_al;
2802 59701 : return true;
2803 : }
2804 : }
2805 :
2806 : /* Set all AGLAT and all other aggregate lattices reachable by next pointers as
2807 : containing an unknown value. */
2808 :
2809 : static bool
2810 42951 : set_chain_of_aglats_contains_variable (struct ipcp_agg_lattice *aglat)
2811 : {
2812 42951 : bool ret = false;
2813 45556 : while (aglat)
2814 : {
2815 2605 : ret |= aglat->set_contains_variable ();
2816 2605 : aglat = aglat->next;
2817 : }
2818 42951 : return ret;
2819 : }
2820 :
2821 : /* Merge existing aggregate lattices in SRC_PLATS to DEST_PLATS, subtracting
2822 : DELTA_OFFSET. CS is the call graph edge and SRC_IDX the index of the source
2823 : parameter used for lattice value sources. Return true if DEST_PLATS changed
2824 : in any way. */
2825 :
2826 : static bool
2827 3959 : merge_aggregate_lattices (struct cgraph_edge *cs,
2828 : class ipcp_param_lattices *dest_plats,
2829 : class ipcp_param_lattices *src_plats,
2830 : int src_idx, HOST_WIDE_INT offset_delta)
2831 : {
2832 3959 : bool pre_existing = dest_plats->aggs != NULL;
2833 3959 : struct ipcp_agg_lattice **dst_aglat;
2834 3959 : bool ret = false;
2835 :
2836 3959 : if (set_check_aggs_by_ref (dest_plats, src_plats->aggs_by_ref))
2837 0 : return true;
2838 3959 : if (src_plats->aggs_bottom)
2839 2 : return set_agg_lats_contain_variable (dest_plats);
2840 3957 : if (src_plats->aggs_contain_variable)
2841 2335 : ret |= set_agg_lats_contain_variable (dest_plats);
2842 3957 : dst_aglat = &dest_plats->aggs;
2843 :
2844 3957 : int max_agg_items = opt_for_fn (cs->callee->function_symbol ()->decl,
2845 : param_ipa_max_agg_items);
2846 3957 : for (struct ipcp_agg_lattice *src_aglat = src_plats->aggs;
2847 11694 : src_aglat;
2848 7737 : src_aglat = src_aglat->next)
2849 : {
2850 7737 : HOST_WIDE_INT new_offset = src_aglat->offset - offset_delta;
2851 :
2852 7737 : if (new_offset < 0)
2853 51 : continue;
2854 7686 : if (merge_agg_lats_step (dest_plats, new_offset, src_aglat->size,
2855 : &dst_aglat, pre_existing, &ret, max_agg_items))
2856 : {
2857 7682 : struct ipcp_agg_lattice *new_al = *dst_aglat;
2858 :
2859 7682 : dst_aglat = &(*dst_aglat)->next;
2860 7682 : if (src_aglat->bottom)
2861 : {
2862 0 : ret |= new_al->set_contains_variable ();
2863 0 : continue;
2864 : }
2865 7682 : if (src_aglat->contains_variable)
2866 4504 : ret |= new_al->set_contains_variable ();
2867 7682 : for (ipcp_value<tree> *val = src_aglat->values;
2868 11948 : val;
2869 4266 : val = val->next)
2870 4266 : ret |= new_al->add_value (val->value, cs, val, src_idx,
2871 : src_aglat->offset);
2872 : }
2873 4 : else if (dest_plats->aggs_bottom)
2874 : return true;
2875 : }
2876 3957 : ret |= set_chain_of_aglats_contains_variable (*dst_aglat);
2877 3957 : return ret;
2878 : }
2879 :
2880 : /* Determine whether there is anything to propagate FROM SRC_PLATS through a
2881 : pass-through JFUNC and if so, whether it has conform and conforms to the
2882 : rules about propagating values passed by reference. */
2883 :
2884 : static bool
2885 177226 : agg_pass_through_permissible_p (class ipcp_param_lattices *src_plats,
2886 : struct ipa_jump_func *jfunc)
2887 : {
2888 177226 : return src_plats->aggs
2889 177226 : && (!src_plats->aggs_by_ref
2890 5134 : || ipa_get_jf_pass_through_agg_preserved (jfunc));
2891 : }
2892 :
2893 : /* Propagate values through ITEM, jump function for a part of an aggregate,
2894 : into corresponding aggregate lattice AGLAT. CS is the call graph edge
2895 : associated with the jump function. Return true if AGLAT changed in any
2896 : way. */
2897 :
2898 : static bool
2899 110428 : propagate_aggregate_lattice (struct cgraph_edge *cs,
2900 : struct ipa_agg_jf_item *item,
2901 : struct ipcp_agg_lattice *aglat)
2902 : {
2903 110428 : class ipa_node_params *caller_info;
2904 110428 : class ipcp_param_lattices *src_plats;
2905 110428 : struct ipcp_lattice<tree> *src_lat;
2906 110428 : HOST_WIDE_INT src_offset;
2907 110428 : int src_idx;
2908 110428 : tree load_type;
2909 110428 : bool ret;
2910 :
2911 110428 : if (item->jftype == IPA_JF_CONST)
2912 : {
2913 98647 : tree value = item->value.constant;
2914 :
2915 98647 : gcc_checking_assert (is_gimple_ip_invariant (value));
2916 98647 : return aglat->add_value (value, cs, NULL, 0);
2917 : }
2918 :
2919 11781 : gcc_checking_assert (item->jftype == IPA_JF_PASS_THROUGH
2920 : || item->jftype == IPA_JF_LOAD_AGG);
2921 :
2922 11781 : caller_info = ipa_node_params_sum->get (cs->caller);
2923 11781 : src_idx = item->value.pass_through.formal_id;
2924 11781 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
2925 :
2926 11781 : if (item->jftype == IPA_JF_PASS_THROUGH)
2927 : {
2928 3419 : load_type = NULL_TREE;
2929 3419 : src_lat = &src_plats->itself;
2930 3419 : src_offset = -1;
2931 : }
2932 : else
2933 : {
2934 8362 : HOST_WIDE_INT load_offset = item->value.load_agg.offset;
2935 8362 : struct ipcp_agg_lattice *src_aglat;
2936 :
2937 13192 : for (src_aglat = src_plats->aggs; src_aglat; src_aglat = src_aglat->next)
2938 8696 : if (src_aglat->offset >= load_offset)
2939 : break;
2940 :
2941 8362 : load_type = item->value.load_agg.type;
2942 8362 : if (!src_aglat
2943 3866 : || src_aglat->offset > load_offset
2944 3514 : || src_aglat->size != tree_to_shwi (TYPE_SIZE (load_type))
2945 11876 : || src_plats->aggs_by_ref != item->value.load_agg.by_ref)
2946 4848 : return aglat->set_contains_variable ();
2947 :
2948 : src_lat = src_aglat;
2949 : src_offset = load_offset;
2950 : }
2951 :
2952 6933 : if (src_lat->bottom
2953 6933 : || (!ipcp_versionable_function_p (cs->caller)
2954 6933 : && !src_lat->is_single_const ()))
2955 2121 : return aglat->set_contains_variable ();
2956 :
2957 4812 : ret = propagate_vals_across_arith_jfunc (cs,
2958 : item->value.pass_through.operation,
2959 : load_type,
2960 : item->value.pass_through.operand,
2961 : src_lat, aglat,
2962 : src_offset,
2963 : src_idx,
2964 : item->value.pass_through.op_type,
2965 : item->type);
2966 :
2967 4812 : if (src_lat->contains_variable)
2968 2777 : ret |= aglat->set_contains_variable ();
2969 :
2970 : return ret;
2971 : }
2972 :
2973 : /* Propagate scalar values across jump function JFUNC that is associated with
2974 : edge CS and put the values into DEST_LAT. */
2975 :
2976 : static bool
2977 4026337 : propagate_aggs_across_jump_function (struct cgraph_edge *cs,
2978 : struct ipa_jump_func *jfunc,
2979 : class ipcp_param_lattices *dest_plats)
2980 : {
2981 4026337 : bool ret = false;
2982 :
2983 4026337 : if (dest_plats->aggs_bottom)
2984 : return false;
2985 :
2986 929181 : if (jfunc->type == IPA_JF_PASS_THROUGH
2987 929181 : && ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
2988 : {
2989 177226 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2990 177226 : int src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2991 177226 : class ipcp_param_lattices *src_plats;
2992 :
2993 177226 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
2994 177226 : if (agg_pass_through_permissible_p (src_plats, jfunc))
2995 : {
2996 : /* Currently we do not produce clobber aggregate jump
2997 : functions, replace with merging when we do. */
2998 3829 : gcc_assert (!jfunc->agg.items);
2999 3829 : ret |= merge_aggregate_lattices (cs, dest_plats, src_plats,
3000 : src_idx, 0);
3001 3829 : return ret;
3002 : }
3003 : }
3004 751955 : else if (jfunc->type == IPA_JF_ANCESTOR
3005 751955 : && ipa_get_jf_ancestor_agg_preserved (jfunc))
3006 : {
3007 1264 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
3008 1264 : int src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
3009 1264 : class ipcp_param_lattices *src_plats;
3010 :
3011 1264 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
3012 1264 : if (src_plats->aggs && src_plats->aggs_by_ref)
3013 : {
3014 : /* Currently we do not produce clobber aggregate jump
3015 : functions, replace with merging when we do. */
3016 130 : gcc_assert (!jfunc->agg.items);
3017 130 : ret |= merge_aggregate_lattices (cs, dest_plats, src_plats, src_idx,
3018 : ipa_get_jf_ancestor_offset (jfunc));
3019 : }
3020 1134 : else if (!src_plats->aggs_by_ref)
3021 1130 : ret |= set_agg_lats_to_bottom (dest_plats);
3022 : else
3023 4 : ret |= set_agg_lats_contain_variable (dest_plats);
3024 1264 : return ret;
3025 : }
3026 :
3027 924088 : if (jfunc->agg.items)
3028 : {
3029 39010 : bool pre_existing = dest_plats->aggs != NULL;
3030 39010 : struct ipcp_agg_lattice **aglat = &dest_plats->aggs;
3031 39010 : struct ipa_agg_jf_item *item;
3032 39010 : int i;
3033 :
3034 39010 : if (set_check_aggs_by_ref (dest_plats, jfunc->agg.by_ref))
3035 16 : return true;
3036 :
3037 39010 : int max_agg_items = opt_for_fn (cs->callee->function_symbol ()->decl,
3038 : param_ipa_max_agg_items);
3039 149790 : FOR_EACH_VEC_ELT (*jfunc->agg.items, i, item)
3040 : {
3041 110796 : HOST_WIDE_INT val_size;
3042 :
3043 110796 : if (item->offset < 0 || item->jftype == IPA_JF_UNKNOWN)
3044 317 : continue;
3045 110479 : val_size = tree_to_shwi (TYPE_SIZE (item->type));
3046 :
3047 110479 : if (merge_agg_lats_step (dest_plats, item->offset, val_size,
3048 : &aglat, pre_existing, &ret, max_agg_items))
3049 : {
3050 110428 : ret |= propagate_aggregate_lattice (cs, item, *aglat);
3051 110428 : aglat = &(*aglat)->next;
3052 : }
3053 51 : else if (dest_plats->aggs_bottom)
3054 : return true;
3055 : }
3056 :
3057 77988 : ret |= set_chain_of_aglats_contains_variable (*aglat);
3058 : }
3059 : else
3060 885078 : ret |= set_agg_lats_contain_variable (dest_plats);
3061 :
3062 924072 : return ret;
3063 : }
3064 :
3065 : /* Return true if on the way cfrom CS->caller to the final (non-alias and
3066 : non-thunk) destination, the call passes through a thunk. */
3067 :
3068 : static bool
3069 2006664 : call_passes_through_thunk (cgraph_edge *cs)
3070 : {
3071 2006664 : cgraph_node *alias_or_thunk = cs->callee;
3072 2145488 : while (alias_or_thunk->alias)
3073 138824 : alias_or_thunk = alias_or_thunk->get_alias_target ();
3074 2006664 : return alias_or_thunk->thunk;
3075 : }
3076 :
3077 : /* Propagate constants from the caller to the callee of CS. INFO describes the
3078 : caller. */
3079 :
3080 : static bool
3081 5434535 : propagate_constants_across_call (struct cgraph_edge *cs)
3082 : {
3083 5434535 : class ipa_node_params *callee_info;
3084 5434535 : enum availability availability;
3085 5434535 : cgraph_node *callee;
3086 5434535 : class ipa_edge_args *args;
3087 5434535 : bool ret = false;
3088 5434535 : int i, args_count, parms_count;
3089 :
3090 5434535 : callee = cs->callee->function_symbol (&availability);
3091 5434535 : if (!callee->definition)
3092 : return false;
3093 2028249 : gcc_checking_assert (callee->has_gimple_body_p ());
3094 2028249 : callee_info = ipa_node_params_sum->get (callee);
3095 2028249 : if (!callee_info)
3096 : return false;
3097 :
3098 2019727 : args = ipa_edge_args_sum->get (cs);
3099 2019727 : parms_count = ipa_get_param_count (callee_info);
3100 1822019 : if (parms_count == 0)
3101 : return false;
3102 1822019 : if (!args
3103 1821732 : || !opt_for_fn (cs->caller->decl, flag_ipa_cp)
3104 3643751 : || !opt_for_fn (cs->caller->decl, optimize))
3105 : {
3106 864 : for (i = 0; i < parms_count; i++)
3107 577 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info,
3108 : i));
3109 : return ret;
3110 : }
3111 1821732 : args_count = ipa_get_cs_argument_count (args);
3112 :
3113 : /* If this call goes through a thunk we must not propagate to the first (0th)
3114 : parameter. However, we might need to uncover a thunk from below a series
3115 : of aliases first. */
3116 1821732 : if (call_passes_through_thunk (cs))
3117 : {
3118 227 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info,
3119 : 0));
3120 227 : i = 1;
3121 : }
3122 : else
3123 1821732 : i = 0;
3124 :
3125 5999255 : for (; (i < args_count) && (i < parms_count); i++)
3126 : {
3127 4177523 : struct ipa_jump_func *jump_func = ipa_get_ith_jump_func (args, i);
3128 4177523 : class ipcp_param_lattices *dest_plats;
3129 4177523 : tree param_type = ipa_get_type (callee_info, i);
3130 :
3131 4177523 : dest_plats = ipa_get_parm_lattices (callee_info, i);
3132 4177523 : if (availability == AVAIL_INTERPOSABLE)
3133 151186 : ret |= set_all_contains_variable (dest_plats);
3134 : else
3135 : {
3136 4026337 : ret |= propagate_scalar_across_jump_function (cs, jump_func,
3137 : &dest_plats->itself,
3138 : param_type);
3139 4026337 : ret |= propagate_context_across_jump_function (cs, jump_func, i,
3140 : &dest_plats->ctxlat);
3141 4026337 : ret
3142 4026337 : |= propagate_bits_across_jump_function (cs, i, jump_func,
3143 : &dest_plats->bits_lattice);
3144 4026337 : ret |= propagate_aggs_across_jump_function (cs, jump_func,
3145 : dest_plats);
3146 4026337 : if (opt_for_fn (callee->decl, flag_ipa_vrp))
3147 4025504 : ret |= propagate_vr_across_jump_function (cs, jump_func,
3148 : dest_plats, param_type);
3149 : else
3150 833 : ret |= dest_plats->m_value_range.set_to_bottom ();
3151 : }
3152 : }
3153 1821915 : for (; i < parms_count; i++)
3154 183 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info, i));
3155 :
3156 : return ret;
3157 : }
3158 :
3159 : /* If an indirect edge IE can be turned into a direct one based on KNOWN_VALS
3160 : KNOWN_CONTEXTS, and known aggregates either in AVS or KNOWN_AGGS return
3161 : the destination. The latter three can be NULL. If AGG_REPS is not NULL,
3162 : KNOWN_AGGS is ignored. */
3163 :
3164 : static tree
3165 1577384 : ipa_get_indirect_edge_target_1 (struct cgraph_edge *ie,
3166 : const vec<tree> &known_csts,
3167 : const vec<ipa_polymorphic_call_context> &known_contexts,
3168 : const ipa_argagg_value_list &avs,
3169 : bool *speculative)
3170 : {
3171 1577384 : int param_index = ie->indirect_info->param_index;
3172 1577384 : *speculative = false;
3173 :
3174 1577384 : if (param_index == -1)
3175 : return NULL_TREE;
3176 :
3177 619339 : if (cgraph_simple_indirect_info *sii
3178 619339 : = dyn_cast <cgraph_simple_indirect_info *> (ie->indirect_info))
3179 : {
3180 302295 : tree t = NULL;
3181 :
3182 302295 : if (sii->agg_contents)
3183 : {
3184 68868 : t = NULL;
3185 68868 : if ((unsigned) param_index < known_csts.length ()
3186 68868 : && known_csts[param_index])
3187 62798 : t = ipa_find_agg_cst_from_init (known_csts[param_index],
3188 : sii->offset,
3189 : sii->by_ref);
3190 :
3191 68868 : if (!t && sii->guaranteed_unmodified)
3192 61865 : t = avs.get_value (param_index, sii->offset / BITS_PER_UNIT,
3193 : sii->by_ref);
3194 : }
3195 233427 : else if ((unsigned) param_index < known_csts.length ())
3196 233427 : t = known_csts[param_index];
3197 :
3198 302240 : if (t
3199 205989 : && TREE_CODE (t) == ADDR_EXPR
3200 508014 : && TREE_CODE (TREE_OPERAND (t, 0)) == FUNCTION_DECL)
3201 205774 : return TREE_OPERAND (t, 0);
3202 : else
3203 : return NULL_TREE;
3204 : }
3205 :
3206 317044 : if (!opt_for_fn (ie->caller->decl, flag_devirtualize))
3207 : return NULL_TREE;
3208 :
3209 317044 : cgraph_polymorphic_indirect_info *pii
3210 317044 : = as_a <cgraph_polymorphic_indirect_info *> (ie->indirect_info);
3211 317044 : if (!pii->usable_p ())
3212 : return NULL_TREE;
3213 :
3214 317044 : HOST_WIDE_INT anc_offset = pii->offset;
3215 317044 : tree t = NULL;
3216 317044 : tree target = NULL;
3217 317044 : if ((unsigned) param_index < known_csts.length ()
3218 317044 : && known_csts[param_index])
3219 18393 : t = ipa_find_agg_cst_from_init (known_csts[param_index], anc_offset, true);
3220 :
3221 : /* Try to work out value of virtual table pointer value in replacements. */
3222 : /* or known aggregate values. */
3223 18393 : if (!t)
3224 317035 : t = avs.get_value (param_index, anc_offset / BITS_PER_UNIT, true);
3225 :
3226 : /* If we found the virtual table pointer, lookup the target. */
3227 317035 : if (t)
3228 : {
3229 8114 : tree vtable;
3230 8114 : unsigned HOST_WIDE_INT offset;
3231 8114 : if (vtable_pointer_value_to_vtable (t, &vtable, &offset))
3232 : {
3233 8114 : bool can_refer;
3234 8114 : target = gimple_get_virt_method_for_vtable (pii->otr_token, vtable,
3235 : offset, &can_refer);
3236 8114 : if (can_refer)
3237 : {
3238 8051 : if (!target
3239 8051 : || fndecl_built_in_p (target, BUILT_IN_UNREACHABLE)
3240 15982 : || !possible_polymorphic_call_target_p
3241 7931 : (ie, cgraph_node::get (target)))
3242 : {
3243 : /* Do not speculate builtin_unreachable, it is stupid! */
3244 237 : if (pii->vptr_changed)
3245 6574 : return NULL;
3246 237 : target = ipa_impossible_devirt_target (ie, target);
3247 : }
3248 8051 : *speculative = pii->vptr_changed;
3249 8051 : if (!*speculative)
3250 : return target;
3251 : }
3252 : }
3253 : }
3254 :
3255 : /* Do we know the constant value of pointer? */
3256 310470 : if (!t && (unsigned) param_index < known_csts.length ())
3257 45937 : t = known_csts[param_index];
3258 :
3259 310470 : ipa_polymorphic_call_context context;
3260 310470 : if (known_contexts.length () > (unsigned int) param_index)
3261 : {
3262 310098 : context = known_contexts[param_index];
3263 310098 : context.offset_by (anc_offset);
3264 310098 : if (pii->vptr_changed)
3265 57188 : context.possible_dynamic_type_change (ie->in_polymorphic_cdtor,
3266 : pii->otr_type);
3267 310098 : if (t)
3268 : {
3269 12509 : ipa_polymorphic_call_context ctx2
3270 12509 : = ipa_polymorphic_call_context (t, pii->otr_type, anc_offset);
3271 25018 : if (!ctx2.useless_p ())
3272 10974 : context.combine_with (ctx2, pii->otr_type);
3273 : }
3274 : }
3275 372 : else if (t)
3276 : {
3277 21 : context = ipa_polymorphic_call_context (t, pii->otr_type, anc_offset);
3278 21 : if (pii->vptr_changed)
3279 6 : context.possible_dynamic_type_change (ie->in_polymorphic_cdtor,
3280 : pii->otr_type);
3281 : }
3282 : else
3283 : return NULL_TREE;
3284 :
3285 310119 : vec <cgraph_node *>targets;
3286 310119 : bool final;
3287 :
3288 310119 : targets = possible_polymorphic_call_targets (pii->otr_type, pii->otr_token,
3289 : context, &final);
3290 321972 : if (!final || targets.length () > 1)
3291 : {
3292 298940 : struct cgraph_node *node;
3293 298940 : if (*speculative)
3294 : return target;
3295 298911 : if (!opt_for_fn (ie->caller->decl, flag_devirtualize_speculatively)
3296 298911 : || ie->speculative || !ie->maybe_hot_p ())
3297 : return NULL;
3298 91259 : node = try_speculative_devirtualization (pii->otr_type, pii->otr_token,
3299 : context);
3300 91259 : if (node)
3301 : {
3302 697 : *speculative = true;
3303 697 : target = node->decl;
3304 : }
3305 : else
3306 : return NULL;
3307 : }
3308 : else
3309 : {
3310 11179 : *speculative = false;
3311 11179 : if (targets.length () == 1)
3312 11140 : target = targets[0]->decl;
3313 : else
3314 39 : target = ipa_impossible_devirt_target (ie, NULL_TREE);
3315 : }
3316 :
3317 11876 : if (target && !possible_polymorphic_call_target_p (ie,
3318 : cgraph_node::get (target)))
3319 : {
3320 48 : if (*speculative)
3321 : return NULL;
3322 40 : target = ipa_impossible_devirt_target (ie, target);
3323 : }
3324 :
3325 : return target;
3326 : }
3327 :
3328 : /* If an indirect edge IE can be turned into a direct one based on data in
3329 : AVALS, return the destination. Store into *SPECULATIVE a boolean determinig
3330 : whether the discovered target is only speculative guess. */
3331 :
3332 : tree
3333 1500279 : ipa_get_indirect_edge_target (struct cgraph_edge *ie,
3334 : ipa_call_arg_values *avals,
3335 : bool *speculative)
3336 : {
3337 1500279 : ipa_argagg_value_list avl (avals);
3338 1500279 : return ipa_get_indirect_edge_target_1 (ie, avals->m_known_vals,
3339 1500279 : avals->m_known_contexts,
3340 1500279 : avl, speculative);
3341 : }
3342 :
3343 : /* Calculate devirtualization time bonus for NODE, assuming we know information
3344 : about arguments stored in AVALS.
3345 :
3346 : FIXME: This function will also consider devirtualization of calls that are
3347 : known to be dead in the clone. */
3348 :
3349 : static sreal
3350 453444 : devirtualization_time_bonus (struct cgraph_node *node,
3351 : ipa_auto_call_arg_values *avals)
3352 : {
3353 453444 : struct cgraph_edge *ie;
3354 453444 : sreal res = 0;
3355 :
3356 528519 : for (ie = node->indirect_calls; ie; ie = ie->next_callee)
3357 : {
3358 75075 : struct cgraph_node *callee;
3359 75075 : class ipa_fn_summary *isummary;
3360 75075 : enum availability avail;
3361 75075 : tree target;
3362 75075 : bool speculative;
3363 :
3364 75075 : ipa_argagg_value_list avl (avals);
3365 75075 : target = ipa_get_indirect_edge_target_1 (ie, avals->m_known_vals,
3366 : avals->m_known_contexts,
3367 : avl, &speculative);
3368 75075 : if (!target)
3369 74075 : continue;
3370 :
3371 : /* Only bare minimum benefit for clearly un-inlineable targets. */
3372 3288 : res = res + ie->combined_sreal_frequency ();
3373 3288 : callee = cgraph_node::get (target);
3374 3288 : if (!callee || !callee->definition)
3375 636 : continue;
3376 2652 : callee = callee->function_symbol (&avail);
3377 2652 : if (avail < AVAIL_AVAILABLE)
3378 24 : continue;
3379 2628 : isummary = ipa_fn_summaries->get (callee);
3380 2628 : if (!isummary || !isummary->inlinable)
3381 67 : continue;
3382 :
3383 2561 : int savings = 0;
3384 2561 : int size = ipa_size_summaries->get (callee)->size;
3385 : /* FIXME: The values below need re-considering and perhaps also
3386 : integrating into the cost metrics, at lest in some very basic way. */
3387 2561 : int max_inline_insns_auto
3388 2561 : = opt_for_fn (callee->decl, param_max_inline_insns_auto);
3389 2561 : if (size <= max_inline_insns_auto / 4)
3390 412 : savings = 31 / ((int)speculative + 1);
3391 2149 : else if (size <= max_inline_insns_auto / 2)
3392 392 : savings = 15 / ((int)speculative + 1);
3393 3318 : else if (size <= max_inline_insns_auto
3394 1757 : || DECL_DECLARED_INLINE_P (callee->decl))
3395 196 : savings = 7 / ((int)speculative + 1);
3396 : else
3397 1561 : continue;
3398 1000 : res = res + ie->combined_sreal_frequency () * (sreal) savings;
3399 : }
3400 :
3401 453444 : return res;
3402 : }
3403 :
3404 : /* Return time bonus incurred because of hints stored in ESTIMATES. */
3405 :
3406 : static sreal
3407 233515 : hint_time_bonus (cgraph_node *node, const ipa_call_estimates &estimates)
3408 : {
3409 233515 : sreal result = 0;
3410 233515 : ipa_hints hints = estimates.hints;
3411 233515 : if (hints & (INLINE_HINT_loop_iterations | INLINE_HINT_loop_stride))
3412 26710 : result += opt_for_fn (node->decl, param_ipa_cp_loop_hint_bonus);
3413 :
3414 233515 : sreal bonus_for_one = opt_for_fn (node->decl, param_ipa_cp_loop_hint_bonus);
3415 :
3416 233515 : if (hints & INLINE_HINT_loop_iterations)
3417 18337 : result += estimates.loops_with_known_iterations * bonus_for_one;
3418 :
3419 233515 : if (hints & INLINE_HINT_loop_stride)
3420 10887 : result += estimates.loops_with_known_strides * bonus_for_one;
3421 :
3422 233515 : return result;
3423 : }
3424 :
3425 : /* If there is a reason to penalize the function described by INFO in the
3426 : cloning goodness evaluation, do so. */
3427 :
3428 : static inline sreal
3429 104496 : incorporate_penalties (cgraph_node *node, ipa_node_params *info,
3430 : sreal evaluation)
3431 : {
3432 104496 : if (info->node_within_scc && !info->node_is_self_scc)
3433 1678 : evaluation = (evaluation
3434 1678 : * (100 - opt_for_fn (node->decl,
3435 3356 : param_ipa_cp_recursion_penalty))) / 100;
3436 :
3437 104496 : if (info->node_calling_single_call)
3438 7100 : evaluation = (evaluation
3439 7100 : * (100 - opt_for_fn (node->decl,
3440 7100 : param_ipa_cp_single_call_penalty)))
3441 14200 : / 100;
3442 :
3443 104496 : return evaluation;
3444 : }
3445 :
3446 : /* Return true if cloning NODE is a good idea, given the estimated TIME_BENEFIT
3447 : and SIZE_COST and with the sum of frequencies of incoming edges to the
3448 : potential new clone in FREQUENCIES. CUR_SWEEP is the number of the current
3449 : sweep of IPA-CP over the call-graph in the decision stage. */
3450 :
3451 : static bool
3452 308062 : good_cloning_opportunity_p (struct cgraph_node *node, sreal time_benefit,
3453 : sreal freq_sum, profile_count count_sum,
3454 : int size_cost, bool called_without_ipa_profile,
3455 : int cur_sweep)
3456 : {
3457 308062 : gcc_assert (count_sum.ipa () == count_sum);
3458 308062 : if (count_sum.quality () == AFDO)
3459 0 : count_sum = count_sum.force_nonzero ();
3460 308062 : if (!opt_for_fn (node->decl, flag_ipa_cp_clone))
3461 : {
3462 179619 : if (dump_file && (dump_flags & TDF_DETAILS))
3463 30 : fprintf (dump_file,
3464 : " good_cloning_opportunity_p (time: %g, "
3465 : "size: %i): Cloning is prohibited.\n",
3466 : time_benefit.to_double (), size_cost);
3467 : return false;
3468 : }
3469 :
3470 : /* Assume bodies don't make it into the final binary, so they have no real
3471 : size. Special case them, as cloning them is always a good idea. */
3472 128443 : if (node->assume_function)
3473 : {
3474 13 : if (dump_file && (dump_flags & TDF_DETAILS))
3475 12 : fprintf (dump_file, " good_cloning_opportunity_p: Always clone "
3476 : "assume body functions.\n");
3477 : return true;
3478 : }
3479 :
3480 152364 : if (time_benefit == 0
3481 : /* If there is no call which was executed in profiling or where
3482 : profile is missing, we do not want to clone. */
3483 104496 : || (!called_without_ipa_profile && !count_sum.nonzero_p ()))
3484 : {
3485 23934 : if (dump_file && (dump_flags & TDF_DETAILS))
3486 0 : fprintf (dump_file,
3487 : " good_cloning_opportunity_p (time: %g, "
3488 : "size: %i): Definitely not good.\n",
3489 : time_benefit.to_double (), size_cost);
3490 : return false;
3491 : }
3492 :
3493 104496 : gcc_assert (size_cost > 0);
3494 :
3495 104496 : ipa_node_params *info = ipa_node_params_sum->get (node);
3496 104496 : int num_sweeps = opt_for_fn (node->decl, param_ipa_cp_sweeps);
3497 104496 : int eval_threshold = opt_for_fn (node->decl, param_ipa_cp_eval_threshold);
3498 104496 : eval_threshold = (eval_threshold * num_sweeps) / cur_sweep;
3499 : /* If we know the execution IPA execution counts, we can estimate overall
3500 : speedup of the program. */
3501 104496 : if (count_sum.nonzero_p ())
3502 : {
3503 225 : profile_count saved_time = count_sum * time_benefit;
3504 225 : sreal evaluation = (saved_time.to_sreal_scale (profile_count::one ())
3505 450 : / overall_scaled_runtime) / size_cost;
3506 225 : evaluation = incorporate_penalties (node, info, evaluation);
3507 :
3508 225 : if (dump_file && (dump_flags & TDF_DETAILS))
3509 : {
3510 0 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3511 : "size: %i, count_sum: ", time_benefit.to_double (),
3512 : size_cost);
3513 0 : count_sum.dump (dump_file);
3514 0 : fprintf (dump_file, ", overall time saved: ");
3515 0 : saved_time.dump (dump_file);
3516 0 : fprintf (dump_file, "%s%s) -> evaluation: %.2f, threshold: %i\n",
3517 0 : info->node_within_scc
3518 0 : ? (info->node_is_self_scc ? ", self_scc" : ", scc") : "",
3519 0 : info->node_calling_single_call ? ", single_call" : "",
3520 : evaluation.to_double (), eval_threshold);
3521 : }
3522 225 : gcc_checking_assert (saved_time == saved_time.ipa ());
3523 225 : if (!maybe_hot_count_p (NULL, saved_time))
3524 : {
3525 9 : if (dump_file && (dump_flags & TDF_DETAILS))
3526 0 : fprintf (dump_file, " not cloning: time saved is not hot\n");
3527 : }
3528 : /* Evaluation approximately corresponds to time saved per instruction
3529 : introduced. This is likely almost always going to be true, since we
3530 : already checked that time saved is large enough to be considered
3531 : hot. */
3532 216 : else if (evaluation >= (sreal)eval_threshold)
3533 225 : return true;
3534 : /* If all call sites have profile known; we know we do not want t clone.
3535 : If there are calls with unknown profile; try local heuristics. */
3536 162 : if (!called_without_ipa_profile)
3537 : return false;
3538 : }
3539 104271 : sreal evaluation = (time_benefit * freq_sum) / size_cost;
3540 104271 : evaluation = incorporate_penalties (node, info, evaluation);
3541 104271 : evaluation *= 1000;
3542 :
3543 104271 : if (dump_file && (dump_flags & TDF_DETAILS))
3544 358 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3545 : "size: %i, freq_sum: %g%s%s) -> evaluation: %.2f, "
3546 : "threshold: %i\n",
3547 : time_benefit.to_double (), size_cost, freq_sum.to_double (),
3548 179 : info->node_within_scc
3549 26 : ? (info->node_is_self_scc ? ", self_scc" : ", scc") : "",
3550 179 : info->node_calling_single_call ? ", single_call" : "",
3551 : evaluation.to_double (), eval_threshold);
3552 :
3553 104271 : return evaluation >= eval_threshold;
3554 : }
3555 :
3556 : /* Grow vectors in AVALS and fill them with information about values of
3557 : parameters that are known to be independent of the context. INFO describes
3558 : the function. If REMOVABLE_PARAMS_COST is non-NULL, the movement cost of
3559 : all removable parameters will be stored in it.
3560 :
3561 : TODO: Also grow context independent value range vectors. */
3562 :
3563 : static bool
3564 1112393 : gather_context_independent_values (class ipa_node_params *info,
3565 : ipa_auto_call_arg_values *avals,
3566 : int *removable_params_cost)
3567 : {
3568 1112393 : int i, count = ipa_get_param_count (info);
3569 1112393 : bool ret = false;
3570 :
3571 1112393 : avals->m_known_vals.safe_grow_cleared (count, true);
3572 1112393 : avals->m_known_contexts.safe_grow_cleared (count, true);
3573 :
3574 1112393 : if (removable_params_cost)
3575 1112393 : *removable_params_cost = 0;
3576 :
3577 3684318 : for (i = 0; i < count; i++)
3578 : {
3579 2571925 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3580 2571925 : ipcp_lattice<tree> *lat = &plats->itself;
3581 :
3582 2571925 : if (lat->is_single_const ())
3583 : {
3584 35281 : ipcp_value<tree> *val = lat->values;
3585 35281 : gcc_checking_assert (TREE_CODE (val->value) != TREE_BINFO);
3586 35281 : avals->m_known_vals[i] = val->value;
3587 35281 : if (removable_params_cost)
3588 70562 : *removable_params_cost
3589 35281 : += estimate_move_cost (TREE_TYPE (val->value), false);
3590 : ret = true;
3591 : }
3592 2536644 : else if (removable_params_cost
3593 2536644 : && !ipa_is_param_used (info, i))
3594 489346 : *removable_params_cost
3595 244673 : += ipa_get_param_move_cost (info, i);
3596 :
3597 2571925 : if (!ipa_is_param_used (info, i))
3598 249930 : continue;
3599 :
3600 2321995 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3601 : /* Do not account known context as reason for cloning. We can see
3602 : if it permits devirtualization. */
3603 2321995 : if (ctxlat->is_single_const ())
3604 23407 : avals->m_known_contexts[i] = ctxlat->values->value;
3605 :
3606 2321995 : ret |= push_agg_values_from_plats (plats, i, 0, &avals->m_known_aggs);
3607 : }
3608 :
3609 1112393 : return ret;
3610 : }
3611 :
3612 : /* Perform time and size measurement of NODE with the context given in AVALS,
3613 : calculate the benefit compared to the node without specialization and store
3614 : it into VAL. Take into account REMOVABLE_PARAMS_COST of all
3615 : context-independent or unused removable parameters and EST_MOVE_COST, the
3616 : estimated movement of the considered parameter. */
3617 :
3618 : static void
3619 81728 : perform_estimation_of_a_value (cgraph_node *node,
3620 : ipa_auto_call_arg_values *avals,
3621 : int removable_params_cost, int est_move_cost,
3622 : ipcp_value_base *val)
3623 : {
3624 81728 : sreal time_benefit;
3625 81728 : ipa_call_estimates estimates;
3626 :
3627 81728 : estimate_ipcp_clone_size_and_time (node, avals, &estimates);
3628 :
3629 : /* Extern inline functions have no cloning local time benefits because they
3630 : will be inlined anyway. The only reason to clone them is if it enables
3631 : optimization in any of the functions they call. */
3632 81728 : if (DECL_EXTERNAL (node->decl) && DECL_DECLARED_INLINE_P (node->decl))
3633 114 : time_benefit = 0;
3634 : else
3635 81614 : time_benefit = (estimates.nonspecialized_time - estimates.time)
3636 163228 : + hint_time_bonus (node, estimates)
3637 163228 : + (devirtualization_time_bonus (node, avals)
3638 163228 : + removable_params_cost + est_move_cost);
3639 :
3640 81728 : int size = estimates.size;
3641 81728 : gcc_checking_assert (size >=0);
3642 : /* The inliner-heuristics based estimates may think that in certain
3643 : contexts some functions do not have any size at all but we want
3644 : all specializations to have at least a tiny cost, not least not to
3645 : divide by zero. */
3646 81728 : if (size == 0)
3647 0 : size = 1;
3648 :
3649 81728 : val->local_time_benefit = time_benefit;
3650 81728 : val->local_size_cost = size;
3651 81728 : }
3652 :
3653 : /* Get the overall limit of growth based on parameters extracted from NODE. It
3654 : does not really make sense to mix functions with different overall growth
3655 : limits or even number of sweeps but it is possible and if it happens, we do
3656 : not want to select one limit at random, so get the limits from NODE. */
3657 :
3658 : static long
3659 222514 : get_max_overall_size (cgraph_node *node)
3660 : {
3661 222514 : long max_new_size = orig_overall_size;
3662 222514 : long large_unit = opt_for_fn (node->decl, param_ipa_cp_large_unit_insns);
3663 222514 : if (max_new_size < large_unit)
3664 : max_new_size = large_unit;
3665 222514 : int unit_growth = opt_for_fn (node->decl, param_ipa_cp_unit_growth);
3666 222514 : max_new_size += max_new_size * unit_growth / 100 + 1;
3667 :
3668 222514 : return max_new_size;
3669 : }
3670 :
3671 : /* Return true if NODE should be cloned just for a parameter removal, possibly
3672 : dumping a reason if not. */
3673 :
3674 : static bool
3675 8672 : clone_for_param_removal_p (cgraph_node *node)
3676 : {
3677 8672 : if (!node->can_change_signature)
3678 : {
3679 1574 : if (dump_file && (dump_flags & TDF_DETAILS))
3680 0 : fprintf (dump_file, " Not considering cloning to remove parameters, "
3681 : "function cannot change signature.\n");
3682 : return false;
3683 : }
3684 7098 : if (node->can_be_local_p ())
3685 : {
3686 7098 : if (dump_file && (dump_flags & TDF_DETAILS))
3687 0 : fprintf (dump_file, " Not considering cloning to remove parameters, "
3688 : "IPA-SRA can do it potentially better.\n");
3689 : return false;
3690 : }
3691 : return true;
3692 : }
3693 :
3694 : /* Iterate over known values of parameters of NODE and estimate the local
3695 : effects in terms of time and size they have. */
3696 :
3697 : static void
3698 1300323 : estimate_local_effects (struct cgraph_node *node)
3699 : {
3700 1300323 : ipa_node_params *info = ipa_node_params_sum->get (node);
3701 1300323 : int count = ipa_get_param_count (info);
3702 1064033 : int removable_params_cost;
3703 :
3704 1064033 : if (!count || !ipcp_versionable_function_p (node))
3705 407859 : return;
3706 :
3707 892464 : if (dump_file && (dump_flags & TDF_DETAILS))
3708 150 : fprintf (dump_file, "\nEstimating effects for %s.\n", node->dump_name ());
3709 :
3710 892464 : ipa_auto_call_arg_values avals;
3711 892464 : gather_context_independent_values (info, &avals, &removable_params_cost);
3712 :
3713 3874533 : for (int i = 0; i < count; i++)
3714 : {
3715 2089605 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3716 2089605 : ipcp_lattice<tree> *lat = &plats->itself;
3717 2089605 : ipcp_value<tree> *val;
3718 :
3719 4157097 : if (lat->bottom
3720 218755 : || !lat->values
3721 2129440 : || avals.m_known_vals[i])
3722 2067492 : continue;
3723 :
3724 67694 : for (val = lat->values; val; val = val->next)
3725 : {
3726 45581 : gcc_checking_assert (TREE_CODE (val->value) != TREE_BINFO);
3727 45581 : avals.m_known_vals[i] = val->value;
3728 :
3729 45581 : int emc = estimate_move_cost (TREE_TYPE (val->value), true);
3730 45581 : perform_estimation_of_a_value (node, &avals, removable_params_cost,
3731 : emc, val);
3732 :
3733 45581 : if (dump_file && (dump_flags & TDF_DETAILS))
3734 : {
3735 52 : fprintf (dump_file, " - estimates for value ");
3736 52 : print_ipcp_constant_value (dump_file, val->value);
3737 52 : fprintf (dump_file, " for ");
3738 52 : ipa_dump_param (dump_file, info, i);
3739 52 : fprintf (dump_file, ": time_benefit: %g, size: %i\n",
3740 : val->local_time_benefit.to_double (),
3741 : val->local_size_cost);
3742 : }
3743 : }
3744 22113 : avals.m_known_vals[i] = NULL_TREE;
3745 : }
3746 :
3747 2982069 : for (int i = 0; i < count; i++)
3748 : {
3749 2089605 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3750 :
3751 2089605 : if (!plats->virt_call)
3752 2081746 : continue;
3753 :
3754 7859 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3755 7859 : ipcp_value<ipa_polymorphic_call_context> *val;
3756 :
3757 15556 : if (ctxlat->bottom
3758 2823 : || !ctxlat->values
3759 10676 : || !avals.m_known_contexts[i].useless_p ())
3760 7697 : continue;
3761 :
3762 396 : for (val = ctxlat->values; val; val = val->next)
3763 : {
3764 234 : avals.m_known_contexts[i] = val->value;
3765 234 : perform_estimation_of_a_value (node, &avals, removable_params_cost,
3766 : 0, val);
3767 :
3768 234 : if (dump_file && (dump_flags & TDF_DETAILS))
3769 : {
3770 0 : fprintf (dump_file, " - estimates for polymorphic context ");
3771 0 : print_ipcp_constant_value (dump_file, val->value);
3772 0 : fprintf (dump_file, " for ");
3773 0 : ipa_dump_param (dump_file, info, i);
3774 0 : fprintf (dump_file, ": time_benefit: %g, size: %i\n",
3775 : val->local_time_benefit.to_double (),
3776 : val->local_size_cost);
3777 : }
3778 : }
3779 162 : avals.m_known_contexts[i] = ipa_polymorphic_call_context ();
3780 : }
3781 :
3782 892464 : unsigned all_ctx_len = avals.m_known_aggs.length ();
3783 892464 : auto_vec<ipa_argagg_value, 32> all_ctx;
3784 892464 : all_ctx.reserve_exact (all_ctx_len);
3785 892464 : all_ctx.splice (avals.m_known_aggs);
3786 892464 : avals.m_known_aggs.safe_grow_cleared (all_ctx_len + 1);
3787 :
3788 892464 : unsigned j = 0;
3789 3874533 : for (int index = 0; index < count; index++)
3790 : {
3791 2089605 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, index);
3792 :
3793 2089605 : if (plats->aggs_bottom || !plats->aggs)
3794 2069948 : continue;
3795 :
3796 76354 : for (ipcp_agg_lattice *aglat = plats->aggs; aglat; aglat = aglat->next)
3797 : {
3798 56697 : ipcp_value<tree> *val;
3799 56339 : if (aglat->bottom || !aglat->values
3800 : /* If the following is true, the one value is already part of all
3801 : context estimations. */
3802 105588 : || (!plats->aggs_contain_variable
3803 26289 : && aglat->is_single_const ()))
3804 29653 : continue;
3805 :
3806 27044 : unsigned unit_offset = aglat->offset / BITS_PER_UNIT;
3807 27044 : while (j < all_ctx_len
3808 35989 : && (all_ctx[j].index < index
3809 3539 : || (all_ctx[j].index == index
3810 2579 : && all_ctx[j].unit_offset < unit_offset)))
3811 : {
3812 3496 : avals.m_known_aggs[j] = all_ctx[j];
3813 3496 : j++;
3814 : }
3815 :
3816 36335 : for (unsigned k = j; k < all_ctx_len; k++)
3817 9291 : avals.m_known_aggs[k+1] = all_ctx[k];
3818 :
3819 62957 : for (val = aglat->values; val; val = val->next)
3820 : {
3821 35913 : avals.m_known_aggs[j].value = val->value;
3822 35913 : avals.m_known_aggs[j].unit_offset = unit_offset;
3823 35913 : avals.m_known_aggs[j].index = index;
3824 35913 : avals.m_known_aggs[j].by_ref = plats->aggs_by_ref;
3825 35913 : avals.m_known_aggs[j].killed = false;
3826 :
3827 35913 : perform_estimation_of_a_value (node, &avals,
3828 : removable_params_cost, 0, val);
3829 :
3830 35913 : if (dump_file && (dump_flags & TDF_DETAILS))
3831 : {
3832 85 : fprintf (dump_file, " - estimates for value ");
3833 85 : print_ipcp_constant_value (dump_file, val->value);
3834 85 : fprintf (dump_file, " for ");
3835 85 : ipa_dump_param (dump_file, info, index);
3836 170 : fprintf (dump_file, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
3837 : "]: time_benefit: %g, size: %i\n",
3838 85 : plats->aggs_by_ref ? "ref " : "",
3839 : aglat->offset,
3840 : val->local_time_benefit.to_double (),
3841 : val->local_size_cost);
3842 : }
3843 : }
3844 : }
3845 : }
3846 892464 : }
3847 :
3848 :
3849 : /* Add value CUR_VAL and all yet-unsorted values it is dependent on to the
3850 : topological sort of values. */
3851 :
3852 : template <typename valtype>
3853 : void
3854 141882 : value_topo_info<valtype>::add_val (ipcp_value<valtype> *cur_val)
3855 : {
3856 : ipcp_value_source<valtype> *src;
3857 :
3858 141882 : if (cur_val->dfs)
3859 : return;
3860 :
3861 141720 : dfs_counter++;
3862 141720 : cur_val->dfs = dfs_counter;
3863 141720 : cur_val->low_link = dfs_counter;
3864 :
3865 141720 : cur_val->topo_next = stack;
3866 141720 : stack = cur_val;
3867 141720 : cur_val->on_stack = true;
3868 :
3869 607686 : for (src = cur_val->sources; src; src = src->next)
3870 465966 : if (src->val)
3871 : {
3872 21774 : if (src->val->dfs == 0)
3873 : {
3874 186 : add_val (src->val);
3875 186 : if (src->val->low_link < cur_val->low_link)
3876 19 : cur_val->low_link = src->val->low_link;
3877 : }
3878 21588 : else if (src->val->on_stack
3879 1600 : && src->val->dfs < cur_val->low_link)
3880 73 : cur_val->low_link = src->val->dfs;
3881 : }
3882 :
3883 141720 : if (cur_val->dfs == cur_val->low_link)
3884 : {
3885 : ipcp_value<valtype> *v, *scc_list = NULL;
3886 :
3887 : do
3888 : {
3889 141720 : v = stack;
3890 141720 : stack = v->topo_next;
3891 141720 : v->on_stack = false;
3892 141720 : v->scc_no = cur_val->dfs;
3893 :
3894 141720 : v->scc_next = scc_list;
3895 141720 : scc_list = v;
3896 : }
3897 141720 : while (v != cur_val);
3898 :
3899 141632 : cur_val->topo_next = values_topo;
3900 141632 : values_topo = cur_val;
3901 : }
3902 : }
3903 :
3904 : /* Add all values in lattices associated with NODE to the topological sort if
3905 : they are not there yet. */
3906 :
3907 : static void
3908 1300323 : add_all_node_vals_to_toposort (cgraph_node *node, ipa_topo_info *topo)
3909 : {
3910 1300323 : ipa_node_params *info = ipa_node_params_sum->get (node);
3911 1300323 : int i, count = ipa_get_param_count (info);
3912 :
3913 3693459 : for (i = 0; i < count; i++)
3914 : {
3915 2393136 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3916 2393136 : ipcp_lattice<tree> *lat = &plats->itself;
3917 2393136 : struct ipcp_agg_lattice *aglat;
3918 :
3919 2393136 : if (!lat->bottom)
3920 : {
3921 229875 : ipcp_value<tree> *val;
3922 303985 : for (val = lat->values; val; val = val->next)
3923 74110 : topo->constants.add_val (val);
3924 : }
3925 :
3926 2393136 : if (!plats->aggs_bottom)
3927 289439 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
3928 59666 : if (!aglat->bottom)
3929 : {
3930 59308 : ipcp_value<tree> *val;
3931 119040 : for (val = aglat->values; val; val = val->next)
3932 59732 : topo->constants.add_val (val);
3933 : }
3934 :
3935 2393136 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3936 2393136 : if (!ctxlat->bottom)
3937 : {
3938 230915 : ipcp_value<ipa_polymorphic_call_context> *ctxval;
3939 238769 : for (ctxval = ctxlat->values; ctxval; ctxval = ctxval->next)
3940 7854 : topo->contexts.add_val (ctxval);
3941 : }
3942 : }
3943 1300323 : }
3944 :
3945 : /* One pass of constants propagation along the call graph edges, from callers
3946 : to callees (requires topological ordering in TOPO), iterate over strongly
3947 : connected components. */
3948 :
3949 : static void
3950 132707 : propagate_constants_topo (class ipa_topo_info *topo)
3951 : {
3952 132707 : int i;
3953 :
3954 1515740 : for (i = topo->nnodes - 1; i >= 0; i--)
3955 : {
3956 1383033 : unsigned j;
3957 1383033 : struct cgraph_node *v, *node = topo->order[i];
3958 1383033 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (node);
3959 :
3960 : /* First, iteratively propagate within the strongly connected component
3961 : until all lattices stabilize. */
3962 4154173 : FOR_EACH_VEC_ELT (cycle_nodes, j, v)
3963 1388107 : if (v->has_gimple_body_p ())
3964 : {
3965 1309346 : if (opt_for_fn (v->decl, flag_ipa_cp)
3966 1309346 : && opt_for_fn (v->decl, optimize))
3967 1300323 : push_node_to_stack (topo, v);
3968 : /* When V is not optimized, we can not push it to stack, but
3969 : still we need to set all its callees lattices to bottom. */
3970 : else
3971 : {
3972 22099 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
3973 13076 : propagate_constants_across_call (cs);
3974 : }
3975 : }
3976 :
3977 1383033 : v = pop_node_from_stack (topo);
3978 4069604 : while (v)
3979 : {
3980 1303538 : struct cgraph_edge *cs;
3981 1303538 : class ipa_node_params *info = NULL;
3982 1303538 : bool self_scc = true;
3983 :
3984 6751249 : for (cs = v->callees; cs; cs = cs->next_callee)
3985 5447711 : if (ipa_edge_within_scc (cs))
3986 : {
3987 30529 : cgraph_node *callee = cs->callee->function_symbol ();
3988 :
3989 30529 : if (v != callee)
3990 18308 : self_scc = false;
3991 :
3992 30529 : if (!info)
3993 : {
3994 14266 : info = ipa_node_params_sum->get (v);
3995 14266 : info->node_within_scc = true;
3996 : }
3997 :
3998 30529 : if (propagate_constants_across_call (cs))
3999 4198 : push_node_to_stack (topo, callee);
4000 : }
4001 :
4002 1303538 : if (info)
4003 14266 : info->node_is_self_scc = self_scc;
4004 :
4005 1303538 : v = pop_node_from_stack (topo);
4006 : }
4007 :
4008 : /* Afterwards, propagate along edges leading out of the SCC, calculates
4009 : the local effects of the discovered constants and all valid values to
4010 : their topological sort. */
4011 2771140 : FOR_EACH_VEC_ELT (cycle_nodes, j, v)
4012 1388107 : if (v->has_gimple_body_p ()
4013 1309346 : && opt_for_fn (v->decl, flag_ipa_cp)
4014 2688430 : && opt_for_fn (v->decl, optimize))
4015 : {
4016 1300323 : struct cgraph_edge *cs;
4017 :
4018 1300323 : estimate_local_effects (v);
4019 1300323 : add_all_node_vals_to_toposort (v, topo);
4020 6714375 : for (cs = v->callees; cs; cs = cs->next_callee)
4021 5414052 : if (!ipa_edge_within_scc (cs))
4022 5390930 : propagate_constants_across_call (cs);
4023 : }
4024 1383033 : cycle_nodes.release ();
4025 : }
4026 132707 : }
4027 :
4028 : /* Propagate the estimated effects of individual values along the topological
4029 : from the dependent values to those they depend on. */
4030 :
4031 : template <typename valtype>
4032 : void
4033 265414 : value_topo_info<valtype>::propagate_effects ()
4034 : {
4035 : ipcp_value<valtype> *base;
4036 265414 : hash_set<ipcp_value<valtype> *> processed_srcvals;
4037 :
4038 407046 : for (base = values_topo; base; base = base->topo_next)
4039 : {
4040 : ipcp_value_source<valtype> *src;
4041 : ipcp_value<valtype> *val;
4042 141632 : sreal time = 0;
4043 141632 : HOST_WIDE_INT size = 0;
4044 :
4045 283352 : for (val = base; val; val = val->scc_next)
4046 : {
4047 141720 : time = time + val->local_time_benefit + val->prop_time_benefit;
4048 141720 : size = size + val->local_size_cost + val->prop_size_cost;
4049 : }
4050 :
4051 283352 : for (val = base; val; val = val->scc_next)
4052 : {
4053 141720 : processed_srcvals.empty ();
4054 607686 : for (src = val->sources; src; src = src->next)
4055 465966 : if (src->val
4056 465966 : && cs_interesting_for_ipcp_p (src->cs))
4057 : {
4058 21734 : if (!processed_srcvals.add (src->val))
4059 : {
4060 17461 : HOST_WIDE_INT prop_size = size + src->val->prop_size_cost;
4061 17461 : if (prop_size < INT_MAX)
4062 17461 : src->val->prop_size_cost = prop_size;
4063 : else
4064 0 : continue;
4065 : }
4066 :
4067 21734 : int special_factor = 1;
4068 21734 : if (val->same_scc (src->val))
4069 : special_factor
4070 1688 : = opt_for_fn(src->cs->caller->decl,
4071 : param_ipa_cp_recursive_freq_factor);
4072 20046 : else if (val->self_recursion_generated_p ()
4073 20046 : && (src->cs->callee->function_symbol ()
4074 822 : == src->cs->caller))
4075 : {
4076 822 : int max_recur_gen_depth
4077 822 : = opt_for_fn(src->cs->caller->decl,
4078 : param_ipa_cp_max_recursive_depth);
4079 822 : special_factor = max_recur_gen_depth
4080 822 : - val->self_recursion_generated_level + 1;
4081 : }
4082 :
4083 21734 : src->val->prop_time_benefit
4084 43468 : += time * special_factor * src->cs->sreal_frequency ();
4085 : }
4086 :
4087 141720 : if (size < INT_MAX)
4088 : {
4089 141720 : val->prop_time_benefit = time;
4090 141720 : val->prop_size_cost = size;
4091 : }
4092 : else
4093 : {
4094 0 : val->prop_time_benefit = 0;
4095 : val->prop_size_cost = 0;
4096 : }
4097 : }
4098 : }
4099 265414 : }
4100 :
4101 :
4102 : /* Propagate constants, polymorphic contexts and their effects from the
4103 : summaries interprocedurally. */
4104 :
4105 : static void
4106 132707 : ipcp_propagate_stage (class ipa_topo_info *topo)
4107 : {
4108 132707 : struct cgraph_node *node;
4109 132707 : sreal overall_runtime = 0;
4110 132707 : overall_scaled_runtime = 0;
4111 :
4112 132707 : if (dump_file)
4113 175 : fprintf (dump_file, "\n Propagating constants:\n\n");
4114 :
4115 1520818 : FOR_EACH_DEFINED_FUNCTION (node)
4116 : {
4117 1388111 : if (node->has_gimple_body_p ()
4118 1309346 : && opt_for_fn (node->decl, flag_ipa_cp)
4119 2688434 : && opt_for_fn (node->decl, optimize))
4120 : {
4121 1300323 : ipa_node_params *info = ipa_node_params_sum->get (node);
4122 1300323 : determine_versionability (node, info);
4123 :
4124 1300323 : unsigned nlattices = ipa_get_param_count (info);
4125 1300323 : info->lattices.safe_grow_cleared (nlattices, true);
4126 1300323 : initialize_node_lattices (node);
4127 :
4128 1300323 : int num_sweeps = opt_for_fn (node->decl, param_ipa_cp_sweeps);
4129 1300323 : if (max_number_sweeps < num_sweeps)
4130 124786 : max_number_sweeps = num_sweeps;
4131 : }
4132 :
4133 1388111 : if (!node->alias)
4134 : {
4135 1310557 : ipa_size_summary *szs = ipa_size_summaries->get (node);
4136 1310557 : if (szs)
4137 1310272 : overall_size += szs->self_size;
4138 1310557 : ipa_fn_summary *fns = ipa_fn_summaries->get (node);
4139 1310557 : if (fns)
4140 : {
4141 1310272 : int shift = opt_for_fn (node->decl,
4142 : param_ipa_cp_profile_runtime_scaling);
4143 1310272 : sreal frt = fns->time;
4144 1310272 : overall_runtime += frt;
4145 1310272 : overall_scaled_runtime += frt.shift(-shift);
4146 : }
4147 : }
4148 : }
4149 :
4150 132707 : orig_overall_size = overall_size;
4151 :
4152 132707 : if (dump_file)
4153 : {
4154 175 : fprintf (dump_file, "\noverall_size: %li\n", overall_size);
4155 175 : fprintf (dump_file, "overall_runtime: %.2e\n",
4156 : overall_runtime.to_double ());
4157 175 : fprintf (dump_file, "overall_scaled_runtime: %.2e\n",
4158 : overall_scaled_runtime.to_double ());
4159 : }
4160 :
4161 132707 : propagate_constants_topo (topo);
4162 132707 : if (flag_checking)
4163 132699 : ipcp_verify_propagated_values ();
4164 132707 : topo->constants.propagate_effects ();
4165 132707 : topo->contexts.propagate_effects ();
4166 :
4167 132707 : if (dump_file)
4168 : {
4169 175 : fprintf (dump_file, "\nIPA lattices after all propagation:\n");
4170 175 : print_all_lattices (dump_file, (dump_flags & TDF_DETAILS), true);
4171 : }
4172 132707 : }
4173 :
4174 : /* Discover newly direct outgoing edges from NODE which is a new clone with
4175 : known KNOWN_CSTS and make them direct. */
4176 :
4177 : static void
4178 19622 : ipcp_discover_new_direct_edges (struct cgraph_node *node,
4179 : vec<tree> known_csts,
4180 : vec<ipa_polymorphic_call_context>
4181 : known_contexts,
4182 : vec<ipa_argagg_value, va_gc> *aggvals)
4183 : {
4184 19622 : struct cgraph_edge *ie, *next_ie;
4185 19622 : bool found = false;
4186 :
4187 21652 : for (ie = node->indirect_calls; ie; ie = next_ie)
4188 : {
4189 2030 : tree target;
4190 2030 : bool speculative;
4191 :
4192 2030 : next_ie = ie->next_callee;
4193 2030 : ipa_argagg_value_list avs (aggvals);
4194 2030 : target = ipa_get_indirect_edge_target_1 (ie, known_csts, known_contexts,
4195 : avs, &speculative);
4196 2030 : if (target)
4197 : {
4198 592 : cgraph_polymorphic_indirect_info *pii
4199 592 : = dyn_cast <cgraph_polymorphic_indirect_info *> (ie->indirect_info);
4200 592 : cgraph_simple_indirect_info *sii
4201 1112 : = dyn_cast <cgraph_simple_indirect_info *> (ie->indirect_info);
4202 440 : bool agg_contents = sii && sii->agg_contents;
4203 592 : bool polymorphic = !!pii;
4204 592 : int param_index = ie->indirect_info->param_index;
4205 592 : struct cgraph_edge *cs = ipa_make_edge_direct_to_target (ie, target,
4206 : speculative);
4207 592 : found = true;
4208 :
4209 592 : if (cs && !agg_contents && !polymorphic)
4210 : {
4211 368 : ipa_node_params *info = ipa_node_params_sum->get (node);
4212 368 : int c = ipa_get_controlled_uses (info, param_index);
4213 368 : if (c != IPA_UNDESCRIBED_USE
4214 368 : && !ipa_get_param_load_dereferenced (info, param_index))
4215 : {
4216 364 : struct ipa_ref *to_del;
4217 :
4218 364 : c--;
4219 364 : ipa_set_controlled_uses (info, param_index, c);
4220 364 : if (dump_file && (dump_flags & TDF_DETAILS))
4221 3 : fprintf (dump_file, " controlled uses count of param "
4222 : "%i bumped down to %i\n", param_index, c);
4223 364 : if (c == 0
4224 364 : && (to_del = node->find_reference (cs->callee, NULL, 0,
4225 : IPA_REF_ADDR)))
4226 : {
4227 300 : if (dump_file && (dump_flags & TDF_DETAILS))
4228 3 : fprintf (dump_file, " and even removing its "
4229 : "cloning-created reference\n");
4230 300 : to_del->remove_reference ();
4231 : }
4232 : }
4233 : }
4234 : }
4235 : }
4236 : /* Turning calls to direct calls will improve overall summary. */
4237 19622 : if (found)
4238 493 : ipa_update_overall_fn_summary (node);
4239 19622 : }
4240 :
4241 : class edge_clone_summary;
4242 : static call_summary <edge_clone_summary *> *edge_clone_summaries = NULL;
4243 :
4244 : /* Edge clone summary. */
4245 :
4246 : class edge_clone_summary
4247 : {
4248 : public:
4249 : /* Default constructor. */
4250 379291 : edge_clone_summary (): prev_clone (NULL), next_clone (NULL) {}
4251 :
4252 : /* Default destructor. */
4253 379291 : ~edge_clone_summary ()
4254 : {
4255 379291 : if (prev_clone)
4256 35338 : edge_clone_summaries->get (prev_clone)->next_clone = next_clone;
4257 379291 : if (next_clone)
4258 157784 : edge_clone_summaries->get (next_clone)->prev_clone = prev_clone;
4259 379291 : }
4260 :
4261 : cgraph_edge *prev_clone;
4262 : cgraph_edge *next_clone;
4263 : };
4264 :
4265 : class edge_clone_summary_t:
4266 : public call_summary <edge_clone_summary *>
4267 : {
4268 : public:
4269 132707 : edge_clone_summary_t (symbol_table *symtab):
4270 265414 : call_summary <edge_clone_summary *> (symtab)
4271 : {
4272 132707 : m_initialize_when_cloning = true;
4273 : }
4274 :
4275 : void duplicate (cgraph_edge *src_edge, cgraph_edge *dst_edge,
4276 : edge_clone_summary *src_data,
4277 : edge_clone_summary *dst_data) final override;
4278 : };
4279 :
4280 : /* Edge duplication hook. */
4281 :
4282 : void
4283 192450 : edge_clone_summary_t::duplicate (cgraph_edge *src_edge, cgraph_edge *dst_edge,
4284 : edge_clone_summary *src_data,
4285 : edge_clone_summary *dst_data)
4286 : {
4287 192450 : if (src_data->next_clone)
4288 5592 : edge_clone_summaries->get (src_data->next_clone)->prev_clone = dst_edge;
4289 192450 : dst_data->prev_clone = src_edge;
4290 192450 : dst_data->next_clone = src_data->next_clone;
4291 192450 : src_data->next_clone = dst_edge;
4292 192450 : }
4293 :
4294 : /* Return true is CS calls DEST or its clone for all contexts. When
4295 : ALLOW_RECURSION_TO_CLONE is false, also return false for self-recursive
4296 : edges from/to an all-context clone. */
4297 :
4298 : static bool
4299 1852853 : calls_same_node_or_its_all_contexts_clone_p (cgraph_edge *cs, cgraph_node *dest,
4300 : bool allow_recursion_to_clone)
4301 : {
4302 1852853 : enum availability availability;
4303 1852853 : cgraph_node *callee = cs->callee->function_symbol (&availability);
4304 :
4305 1852853 : if (availability <= AVAIL_INTERPOSABLE)
4306 : return false;
4307 1846852 : if (callee == dest)
4308 : return true;
4309 628316 : if (!allow_recursion_to_clone && cs->caller == callee)
4310 : return false;
4311 :
4312 628159 : ipa_node_params *info = ipa_node_params_sum->get (callee);
4313 628159 : return info->is_all_contexts_clone && info->ipcp_orig_node == dest;
4314 : }
4315 :
4316 : /* Return true if edge CS does bring about the value described by SRC to
4317 : DEST_VAL of node DEST or its clone for all contexts. */
4318 :
4319 : static bool
4320 1842093 : cgraph_edge_brings_value_p (cgraph_edge *cs, ipcp_value_source<tree> *src,
4321 : cgraph_node *dest, ipcp_value<tree> *dest_val)
4322 : {
4323 1842093 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
4324 :
4325 1842093 : if (!calls_same_node_or_its_all_contexts_clone_p (cs, dest, !src->val)
4326 1842093 : || caller_info->node_dead)
4327 : return false;
4328 :
4329 767702 : if (!src->val)
4330 : return true;
4331 :
4332 64883 : if (caller_info->ipcp_orig_node)
4333 : {
4334 20370 : tree t = NULL_TREE;
4335 20370 : if (src->offset == -1)
4336 14607 : t = caller_info->known_csts[src->index];
4337 5763 : else if (ipcp_transformation *ts
4338 5763 : = ipcp_get_transformation_summary (cs->caller))
4339 : {
4340 5763 : ipa_argagg_value_list avl (ts);
4341 5763 : t = avl.get_value (src->index, src->offset / BITS_PER_UNIT);
4342 : }
4343 20370 : return (t != NULL_TREE
4344 20370 : && values_equal_for_ipcp_p (src->val->value, t));
4345 : }
4346 : else
4347 : {
4348 44513 : if (src->val == dest_val)
4349 : return true;
4350 :
4351 38723 : struct ipcp_agg_lattice *aglat;
4352 38723 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (caller_info,
4353 : src->index);
4354 38723 : if (src->offset == -1)
4355 28515 : return (plats->itself.is_single_const ()
4356 20 : && values_equal_for_ipcp_p (src->val->value,
4357 20 : plats->itself.values->value));
4358 : else
4359 : {
4360 10208 : if (plats->aggs_bottom || plats->aggs_contain_variable)
4361 : return false;
4362 3882 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
4363 3882 : if (aglat->offset == src->offset)
4364 1748 : return (aglat->is_single_const ()
4365 8 : && values_equal_for_ipcp_p (src->val->value,
4366 8 : aglat->values->value));
4367 : }
4368 : return false;
4369 : }
4370 : }
4371 :
4372 : /* Return true if edge CS does bring about the value described by SRC to
4373 : DST_VAL of node DEST or its clone for all contexts. */
4374 :
4375 : static bool
4376 10760 : cgraph_edge_brings_value_p (cgraph_edge *cs,
4377 : ipcp_value_source<ipa_polymorphic_call_context> *src,
4378 : cgraph_node *dest,
4379 : ipcp_value<ipa_polymorphic_call_context> *)
4380 : {
4381 10760 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
4382 :
4383 10760 : if (!calls_same_node_or_its_all_contexts_clone_p (cs, dest, true)
4384 10760 : || caller_info->node_dead)
4385 : return false;
4386 9650 : if (!src->val)
4387 : return true;
4388 :
4389 1840 : if (caller_info->ipcp_orig_node)
4390 266 : return (caller_info->known_contexts.length () > (unsigned) src->index)
4391 532 : && values_equal_for_ipcp_p (src->val->value,
4392 266 : caller_info->known_contexts[src->index]);
4393 :
4394 1574 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (caller_info,
4395 : src->index);
4396 1574 : return plats->ctxlat.is_single_const ()
4397 282 : && values_equal_for_ipcp_p (src->val->value,
4398 282 : plats->ctxlat.values->value);
4399 : }
4400 :
4401 : /* Get the next clone in the linked list of clones of an edge. */
4402 :
4403 : static inline struct cgraph_edge *
4404 1853118 : get_next_cgraph_edge_clone (struct cgraph_edge *cs)
4405 : {
4406 1853118 : edge_clone_summary *s = edge_clone_summaries->get (cs);
4407 1853118 : return s != NULL ? s->next_clone : NULL;
4408 : }
4409 :
4410 : /* Given VAL that is intended for DEST, iterate over all its sources and if any
4411 : of them is viable and hot, return true. In that case, for those that still
4412 : hold, add their edge frequency and their number and cumulative profile
4413 : counts of self-ecursive and other edges into *FREQUENCY, *CALLER_COUNT,
4414 : REC_COUNT_SUM and NONREC_COUNT_SUM respectively. */
4415 :
4416 : template <typename valtype>
4417 : static bool
4418 222064 : get_info_about_necessary_edges (ipcp_value<valtype> *val, cgraph_node *dest,
4419 : sreal *freq_sum, int *caller_count,
4420 : profile_count *rec_count_sum,
4421 : profile_count *nonrec_count_sum,
4422 : bool *called_without_ipa_profile)
4423 : {
4424 : ipcp_value_source<valtype> *src;
4425 222064 : sreal freq = 0;
4426 222064 : int count = 0;
4427 222064 : profile_count rec_cnt = profile_count::zero ();
4428 222064 : profile_count nonrec_cnt = profile_count::zero ();
4429 222064 : bool interesting = false;
4430 222064 : bool non_self_recursive = false;
4431 222064 : *called_without_ipa_profile = false;
4432 :
4433 998116 : for (src = val->sources; src; src = src->next)
4434 : {
4435 776052 : struct cgraph_edge *cs = src->cs;
4436 1919360 : while (cs)
4437 : {
4438 1143308 : if (cgraph_edge_brings_value_p (cs, src, dest, val))
4439 : {
4440 365209 : count++;
4441 365209 : freq += cs->sreal_frequency ();
4442 365209 : interesting |= cs_interesting_for_ipcp_p (cs);
4443 365209 : if (cs->caller != dest)
4444 : {
4445 358228 : non_self_recursive = true;
4446 358228 : if (cs->count.ipa ().initialized_p ())
4447 974 : rec_cnt += cs->count.ipa ();
4448 : else
4449 357254 : *called_without_ipa_profile = true;
4450 : }
4451 6981 : else if (cs->count.ipa ().initialized_p ())
4452 0 : nonrec_cnt += cs->count.ipa ();
4453 : else
4454 6981 : *called_without_ipa_profile = true;
4455 : }
4456 1143308 : cs = get_next_cgraph_edge_clone (cs);
4457 : }
4458 : }
4459 :
4460 : /* If the only edges bringing a value are self-recursive ones, do not bother
4461 : evaluating it. */
4462 222064 : if (!non_self_recursive)
4463 : return false;
4464 :
4465 156802 : *freq_sum = freq;
4466 156802 : *caller_count = count;
4467 156802 : *rec_count_sum = rec_cnt;
4468 156802 : *nonrec_count_sum = nonrec_cnt;
4469 :
4470 156802 : return interesting;
4471 : }
4472 :
4473 : /* Given a NODE, and a set of its CALLERS, try to adjust order of the callers
4474 : to let a non-self-recursive caller be the first element. Thus, we can
4475 : simplify intersecting operations on values that arrive from all of these
4476 : callers, especially when there exists self-recursive call. Return true if
4477 : this kind of adjustment is possible. */
4478 :
4479 : static bool
4480 57469 : adjust_callers_for_value_intersection (vec<cgraph_edge *> &callers,
4481 : cgraph_node *node)
4482 : {
4483 61697 : for (unsigned i = 0; i < callers.length (); i++)
4484 : {
4485 61630 : cgraph_edge *cs = callers[i];
4486 :
4487 61630 : if (cs->caller != node)
4488 : {
4489 57402 : if (i > 0)
4490 : {
4491 1995 : callers[i] = callers[0];
4492 1995 : callers[0] = cs;
4493 : }
4494 : return true;
4495 : }
4496 : }
4497 : return false;
4498 : }
4499 :
4500 : /* Return a vector of incoming edges that do bring value VAL to node DEST. It
4501 : is assumed their number is known and equal to CALLER_COUNT. */
4502 :
4503 : template <typename valtype>
4504 : static auto_vec<cgraph_edge *>
4505 156161 : gather_edges_for_value (ipcp_value<valtype> *val, cgraph_node *dest,
4506 : int caller_count)
4507 : {
4508 : ipcp_value_source<valtype> *src;
4509 156161 : auto_vec<cgraph_edge *> ret (caller_count);
4510 :
4511 535617 : for (src = val->sources; src; src = src->next)
4512 : {
4513 379456 : struct cgraph_edge *cs = src->cs;
4514 847634 : while (cs)
4515 : {
4516 468178 : if (cgraph_edge_brings_value_p (cs, src, dest, val))
4517 361705 : ret.quick_push (cs);
4518 468178 : cs = get_next_cgraph_edge_clone (cs);
4519 : }
4520 : }
4521 :
4522 156161 : if (caller_count > 1)
4523 42407 : adjust_callers_for_value_intersection (ret, dest);
4524 :
4525 156161 : return ret;
4526 : }
4527 :
4528 : /* Construct a replacement map for a know VALUE for a formal parameter PARAM.
4529 : Return it or NULL if for some reason it cannot be created. FORCE_LOAD_REF
4530 : should be set to true when the reference created for the constant should be
4531 : a load one and not an address one because the corresponding parameter p is
4532 : only used as *p. */
4533 :
4534 : static struct ipa_replace_map *
4535 24972 : get_replacement_map (class ipa_node_params *info, tree value, int parm_num,
4536 : bool force_load_ref)
4537 : {
4538 24972 : struct ipa_replace_map *replace_map;
4539 :
4540 24972 : replace_map = ggc_alloc<ipa_replace_map> ();
4541 24972 : if (dump_file)
4542 : {
4543 183 : fprintf (dump_file, " replacing ");
4544 183 : ipa_dump_param (dump_file, info, parm_num);
4545 :
4546 183 : fprintf (dump_file, " with const ");
4547 183 : print_generic_expr (dump_file, value);
4548 :
4549 183 : if (force_load_ref)
4550 11 : fprintf (dump_file, " - forcing load reference\n");
4551 : else
4552 172 : fprintf (dump_file, "\n");
4553 : }
4554 24972 : replace_map->parm_num = parm_num;
4555 24972 : replace_map->new_tree = value;
4556 24972 : replace_map->force_load_ref = force_load_ref;
4557 24972 : return replace_map;
4558 : }
4559 :
4560 : /* Dump new profiling counts of NODE. SPEC is true when NODE is a specialzied
4561 : one, otherwise it will be referred to as the original node. */
4562 :
4563 : static void
4564 4 : dump_profile_updates (cgraph_node *node, bool spec)
4565 : {
4566 4 : if (spec)
4567 2 : fprintf (dump_file, " setting count of the specialized node %s to ",
4568 : node->dump_name ());
4569 : else
4570 2 : fprintf (dump_file, " setting count of the original node %s to ",
4571 : node->dump_name ());
4572 :
4573 4 : node->count.dump (dump_file);
4574 4 : fprintf (dump_file, "\n");
4575 6 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
4576 : {
4577 2 : fprintf (dump_file, " edge to %s has count ",
4578 2 : cs->callee->dump_name ());
4579 2 : cs->count.dump (dump_file);
4580 2 : fprintf (dump_file, "\n");
4581 : }
4582 4 : }
4583 :
4584 : /* With partial train run we do not want to assume that original's count is
4585 : zero whenever we redurect all executed edges to clone. Simply drop profile
4586 : to local one in this case. In eany case, return the new value. ORIG_NODE
4587 : is the original node and its count has not been updated yet. */
4588 :
4589 : profile_count
4590 65 : lenient_count_portion_handling (profile_count remainder, cgraph_node *orig_node)
4591 : {
4592 130 : if (remainder.ipa_p () && !remainder.ipa ().nonzero_p ()
4593 117 : && orig_node->count.ipa_p () && orig_node->count.ipa ().nonzero_p ()
4594 26 : && opt_for_fn (orig_node->decl, flag_profile_partial_training))
4595 0 : remainder = orig_node->count.guessed_local ();
4596 :
4597 65 : return remainder;
4598 : }
4599 :
4600 : /* Structure to sum counts coming from nodes other than the original node and
4601 : its clones. */
4602 :
4603 : struct gather_other_count_struct
4604 : {
4605 : cgraph_node *orig;
4606 : profile_count other_count;
4607 : };
4608 :
4609 : /* Worker callback of call_for_symbol_thunks_and_aliases summing the number of
4610 : counts that come from non-self-recursive calls.. */
4611 :
4612 : static bool
4613 16 : gather_count_of_non_rec_edges (cgraph_node *node, void *data)
4614 : {
4615 16 : gather_other_count_struct *desc = (gather_other_count_struct *) data;
4616 34 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4617 18 : if (cs->caller != desc->orig && cs->caller->clone_of != desc->orig)
4618 0 : if (cs->count.ipa ().initialized_p ())
4619 0 : desc->other_count += cs->count.ipa ();
4620 16 : return false;
4621 : }
4622 :
4623 : /* Structure to help analyze if we need to boost counts of some clones of some
4624 : non-recursive edges to match the new callee count. */
4625 :
4626 : struct desc_incoming_count_struct
4627 : {
4628 : cgraph_node *orig;
4629 : hash_set <cgraph_edge *> *processed_edges;
4630 : profile_count count;
4631 : unsigned unproc_orig_rec_edges;
4632 : };
4633 :
4634 : /* Go over edges calling NODE and its thunks and gather information about
4635 : incoming counts so that we know if we need to make any adjustments. */
4636 :
4637 : static void
4638 16 : analyze_clone_icoming_counts (cgraph_node *node,
4639 : desc_incoming_count_struct *desc)
4640 : {
4641 34 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4642 18 : if (cs->caller->thunk)
4643 : {
4644 0 : analyze_clone_icoming_counts (cs->caller, desc);
4645 0 : continue;
4646 : }
4647 : else
4648 : {
4649 18 : if (cs->count.initialized_p ())
4650 18 : desc->count += cs->count.ipa ();
4651 18 : if (!desc->processed_edges->contains (cs)
4652 18 : && cs->caller->clone_of == desc->orig)
4653 2 : desc->unproc_orig_rec_edges++;
4654 : }
4655 16 : }
4656 :
4657 : /* If caller edge counts of a clone created for a self-recursive arithmetic
4658 : jump function must be adjusted because it is coming from a the "seed" clone
4659 : for the first value and so has been excessively scaled back as if it was not
4660 : a recursive call, adjust it so that the incoming counts of NODE match its
4661 : count. NODE is the node or its thunk. */
4662 :
4663 : static void
4664 0 : adjust_clone_incoming_counts (cgraph_node *node,
4665 : desc_incoming_count_struct *desc)
4666 : {
4667 0 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4668 0 : if (cs->caller->thunk)
4669 : {
4670 0 : adjust_clone_incoming_counts (cs->caller, desc);
4671 : /* Same rationale as commit 8c6b6adce45a550c52dc35e3df4e0c477f5404fa
4672 : for scaling recursive edges in update_counts_for_self_gen_clones:
4673 : adjusting non-IPA edge counts here does not update matching gimple
4674 : BB frequencies and breaks verify_cgraph_node. */
4675 0 : if (cs->count.ipa_p ())
4676 : {
4677 0 : profile_count sum = profile_count::zero ();
4678 0 : for (cgraph_edge *e = cs->caller->callers; e; e = e->next_caller)
4679 0 : if (e->count.initialized_p ())
4680 0 : sum += e->count.ipa ();
4681 0 : cs->count = cs->count.combine_with_ipa_count (sum);
4682 : }
4683 0 : else if (dump_file)
4684 0 : fprintf (dump_file, " Skipping adjustment of the count of an "
4685 : "incoming edge of a clone %s -> %s\n",
4686 0 : cs->caller->dump_name (), cs->callee->dump_name ());
4687 : }
4688 0 : else if (!desc->processed_edges->contains (cs)
4689 0 : && cs->caller->clone_of == desc->orig
4690 0 : && cs->count.compatible_p (desc->count))
4691 : {
4692 0 : if (cs->count.ipa_p ())
4693 : {
4694 0 : cs->count += desc->count;
4695 0 : if (dump_file)
4696 : {
4697 0 : fprintf (dump_file, " Adjusted count of an incoming edge "
4698 0 : "of a clone %s -> %s to ", cs->caller->dump_name (),
4699 0 : cs->callee->dump_name ());
4700 0 : cs->count.dump (dump_file);
4701 0 : fprintf (dump_file, "\n");
4702 : }
4703 : }
4704 0 : else if (dump_file)
4705 0 : fprintf (dump_file, " Skipping adjustment of the count of an "
4706 : "incoming edge of a clone %s -> %s\n",
4707 0 : cs->caller->dump_name (), cs->callee->dump_name ());
4708 : }
4709 0 : }
4710 :
4711 : /* When ORIG_NODE has been cloned for values which have been generated fora
4712 : self-recursive call as a result of an arithmetic pass-through
4713 : jump-functions, adjust its count together with counts of all such clones in
4714 : SELF_GEN_CLONES which also at this point contains ORIG_NODE itself.
4715 :
4716 : The function sums the counts of the original node and all its clones that
4717 : cannot be attributed to a specific clone because it comes from a
4718 : non-recursive edge. This sum is then evenly divided between the clones and
4719 : on top of that each one gets all the counts which can be attributed directly
4720 : to it. */
4721 :
4722 : static void
4723 31 : update_counts_for_self_gen_clones (cgraph_node *orig_node,
4724 : const vec<cgraph_node *> &self_gen_clones)
4725 : {
4726 31 : profile_count redist_sum = orig_node->count.ipa ();
4727 31 : if (!redist_sum.nonzero_p ())
4728 : return;
4729 :
4730 2 : if (dump_file)
4731 0 : fprintf (dump_file, " Updating profile of self recursive clone "
4732 : "series\n");
4733 :
4734 2 : gather_other_count_struct gocs;
4735 2 : gocs.orig = orig_node;
4736 2 : gocs.other_count = profile_count::zero ();
4737 :
4738 2 : auto_vec <profile_count, 8> other_edges_count;
4739 22 : for (cgraph_node *n : self_gen_clones)
4740 : {
4741 16 : gocs.other_count = profile_count::zero ();
4742 16 : n->call_for_symbol_thunks_and_aliases (gather_count_of_non_rec_edges,
4743 : &gocs, false);
4744 16 : other_edges_count.safe_push (gocs.other_count);
4745 16 : redist_sum -= gocs.other_count;
4746 : }
4747 :
4748 2 : hash_set<cgraph_edge *> processed_edges;
4749 2 : unsigned i = 0;
4750 22 : for (cgraph_node *n : self_gen_clones)
4751 : {
4752 16 : profile_count new_count
4753 32 : = (redist_sum / self_gen_clones.length () + other_edges_count[i]);
4754 16 : new_count = lenient_count_portion_handling (new_count, orig_node);
4755 16 : n->scale_profile_to (new_count);
4756 32 : for (cgraph_edge *cs = n->callees; cs; cs = cs->next_callee)
4757 16 : processed_edges.add (cs);
4758 :
4759 16 : i++;
4760 : }
4761 :
4762 : /* There are still going to be edges to ORIG_NODE that have one or more
4763 : clones coming from another node clone in SELF_GEN_CLONES and which we
4764 : scaled by the same amount, which means that the total incoming sum of
4765 : counts to ORIG_NODE will be too high, scale such edges back. */
4766 4 : for (cgraph_edge *cs = orig_node->callees; cs; cs = cs->next_callee)
4767 : {
4768 2 : if (cs->callee->ultimate_alias_target () == orig_node)
4769 : {
4770 2 : unsigned den = 0;
4771 20 : for (cgraph_edge *e = cs; e; e = get_next_cgraph_edge_clone (e))
4772 18 : if (e->callee->ultimate_alias_target () == orig_node
4773 18 : && processed_edges.contains (e))
4774 4 : den++;
4775 2 : if (den > 0)
4776 20 : for (cgraph_edge *e = cs; e; e = get_next_cgraph_edge_clone (e))
4777 18 : if (e->callee->ultimate_alias_target () == orig_node
4778 4 : && processed_edges.contains (e)
4779 : /* If count is not IPA, this adjustment makes verifier
4780 : unhappy, since we expect bb->count to match e->count.
4781 : We may add a flag to mark edge conts that has been
4782 : modified by IPA code, but so far it does not seem
4783 : to be worth the effort. With local counts the profile
4784 : will not propagate at IPA level. */
4785 26 : && e->count.ipa_p ())
4786 4 : e->count /= den;
4787 : }
4788 : }
4789 :
4790 : /* Edges from the seeds of the values generated for arithmetic jump-functions
4791 : along self-recursive edges are likely to have fairly low count and so
4792 : edges from them to nodes in the self_gen_clones do not correspond to the
4793 : artificially distributed count of the nodes, the total sum of incoming
4794 : edges to some clones might be too low. Detect this situation and correct
4795 : it. */
4796 22 : for (cgraph_node *n : self_gen_clones)
4797 : {
4798 16 : if (!n->count.ipa ().nonzero_p ())
4799 0 : continue;
4800 :
4801 16 : desc_incoming_count_struct desc;
4802 16 : desc.orig = orig_node;
4803 16 : desc.processed_edges = &processed_edges;
4804 16 : desc.count = profile_count::zero ();
4805 16 : desc.unproc_orig_rec_edges = 0;
4806 16 : analyze_clone_icoming_counts (n, &desc);
4807 :
4808 16 : if (n->count.differs_from_p (desc.count))
4809 : {
4810 0 : if (n->count > desc.count
4811 0 : && desc.unproc_orig_rec_edges > 0)
4812 : {
4813 0 : desc.count = n->count - desc.count;
4814 0 : desc.count = desc.count /= desc.unproc_orig_rec_edges;
4815 0 : adjust_clone_incoming_counts (n, &desc);
4816 : }
4817 0 : else if (dump_file)
4818 0 : fprintf (dump_file,
4819 : " Unable to fix up incoming counts for %s.\n",
4820 : n->dump_name ());
4821 : }
4822 : }
4823 :
4824 2 : if (dump_file)
4825 0 : for (cgraph_node *n : self_gen_clones)
4826 0 : dump_profile_updates (n, n != orig_node);
4827 2 : return;
4828 2 : }
4829 :
4830 : /* After a specialized NEW_NODE version of ORIG_NODE has been created, update
4831 : their profile information to reflect this. This function should not be used
4832 : for clones generated for arithmetic pass-through jump functions on a
4833 : self-recursive call graph edge, that situation is handled by
4834 : update_counts_for_self_gen_clones. */
4835 :
4836 : static void
4837 4475 : update_profiling_info (struct cgraph_node *orig_node,
4838 : struct cgraph_node *new_node)
4839 : {
4840 4475 : struct caller_statistics stats;
4841 4475 : profile_count new_sum;
4842 4475 : profile_count remainder, orig_node_count = orig_node->count.ipa ();
4843 :
4844 4475 : if (!orig_node_count.nonzero_p ())
4845 4426 : return;
4846 :
4847 49 : if (dump_file)
4848 : {
4849 2 : fprintf (dump_file, " Updating profile from original count: ");
4850 2 : orig_node_count.dump (dump_file);
4851 2 : fprintf (dump_file, "\n");
4852 : }
4853 :
4854 49 : init_caller_stats (&stats, new_node);
4855 49 : new_node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
4856 : false);
4857 49 : new_sum = stats.count_sum;
4858 :
4859 49 : bool orig_edges_processed = false;
4860 49 : if (new_sum > orig_node_count)
4861 : {
4862 : /* Profile has already gone astray, keep what we have but lower it
4863 : to global0adjusted or to local if we have partial training. */
4864 0 : if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
4865 0 : orig_node->make_profile_local ();
4866 0 : else if (new_sum.quality () == AFDO)
4867 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_AFDO);
4868 : else
4869 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_ADJUSTED);
4870 : orig_edges_processed = true;
4871 : }
4872 49 : else if (stats.rec_count_sum.nonzero_p ())
4873 : {
4874 0 : int new_nonrec_calls = stats.n_nonrec_calls;
4875 : /* There are self-recursive edges which are likely to bring in the
4876 : majority of calls but which we must divide in between the original and
4877 : new node. */
4878 0 : init_caller_stats (&stats, orig_node);
4879 0 : orig_node->call_for_symbol_thunks_and_aliases (gather_caller_stats,
4880 : &stats, false);
4881 0 : int orig_nonrec_calls = stats.n_nonrec_calls;
4882 0 : profile_count orig_nonrec_call_count = stats.count_sum;
4883 :
4884 0 : if (orig_node->local)
4885 : {
4886 0 : if (!orig_nonrec_call_count.nonzero_p ())
4887 : {
4888 0 : if (dump_file)
4889 0 : fprintf (dump_file, " The original is local and the only "
4890 : "incoming edges from non-dead callers with nonzero "
4891 : "counts are self-recursive, assuming it is cold.\n");
4892 : /* The NEW_NODE count and counts of all its outgoing edges
4893 : are still unmodified copies of ORIG_NODE's. Just clear
4894 : the latter and bail out. */
4895 0 : if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
4896 0 : orig_node->make_profile_local ();
4897 0 : else if (orig_nonrec_call_count.quality () == AFDO)
4898 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_AFDO);
4899 : else
4900 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_ADJUSTED);
4901 0 : return;
4902 : }
4903 : }
4904 : else
4905 : {
4906 : /* Let's behave as if there was another caller that accounts for all
4907 : the calls that were either indirect or from other compilation
4908 : units. */
4909 0 : orig_nonrec_calls++;
4910 0 : profile_count pretend_caller_count
4911 0 : = (orig_node_count - new_sum - orig_nonrec_call_count
4912 0 : - stats.rec_count_sum);
4913 0 : orig_nonrec_call_count += pretend_caller_count;
4914 : }
4915 :
4916 : /* Divide all "unexplained" counts roughly proportionally to sums of
4917 : counts of non-recursive calls.
4918 :
4919 : We put rather arbitrary limits on how many counts we claim because the
4920 : number of non-self-recursive incoming count is only a rough guideline
4921 : and there are cases (such as mcf) where using it blindly just takes
4922 : too many. And if lattices are considered in the opposite order we
4923 : could also take too few. */
4924 0 : profile_count unexp = orig_node_count - new_sum - orig_nonrec_call_count;
4925 :
4926 0 : int limit_den = 2 * (orig_nonrec_calls + new_nonrec_calls);
4927 0 : profile_count new_part = unexp.apply_scale (limit_den - 1, limit_den);
4928 0 : profile_count den = new_sum + orig_nonrec_call_count;
4929 0 : if (den.nonzero_p ())
4930 0 : new_part = MIN (unexp.apply_scale (new_sum, den), new_part);
4931 0 : new_part = MAX (new_part,
4932 : unexp.apply_scale (new_nonrec_calls, limit_den));
4933 0 : if (dump_file)
4934 : {
4935 0 : fprintf (dump_file, " Claiming ");
4936 0 : new_part.dump (dump_file);
4937 0 : fprintf (dump_file, " of unexplained ");
4938 0 : unexp.dump (dump_file);
4939 0 : fprintf (dump_file, " counts because of self-recursive "
4940 : "calls\n");
4941 : }
4942 0 : new_sum += new_part;
4943 0 : remainder = lenient_count_portion_handling (orig_node_count - new_sum,
4944 : orig_node);
4945 : }
4946 : else
4947 49 : remainder = lenient_count_portion_handling (orig_node_count - new_sum,
4948 : orig_node);
4949 :
4950 49 : new_node->scale_profile_to (new_sum);
4951 :
4952 49 : if (!orig_edges_processed)
4953 49 : orig_node->scale_profile_to (remainder);
4954 :
4955 49 : if (dump_file)
4956 : {
4957 2 : dump_profile_updates (new_node, true);
4958 2 : dump_profile_updates (orig_node, false);
4959 : }
4960 : }
4961 :
4962 : /* Update the respective profile of specialized NEW_NODE and the original
4963 : ORIG_NODE after additional edges with cumulative count sum REDIRECTED_SUM
4964 : have been redirected to the specialized version. */
4965 :
4966 : static void
4967 0 : update_specialized_profile (struct cgraph_node *new_node,
4968 : struct cgraph_node *orig_node,
4969 : profile_count redirected_sum)
4970 : {
4971 0 : if (dump_file)
4972 : {
4973 0 : fprintf (dump_file, " the sum of counts of redirected edges is ");
4974 0 : redirected_sum.dump (dump_file);
4975 0 : fprintf (dump_file, "\n old ipa count of the original node is ");
4976 0 : orig_node->count.dump (dump_file);
4977 0 : fprintf (dump_file, "\n");
4978 : }
4979 0 : if (!orig_node->count.ipa ().nonzero_p ()
4980 0 : || !redirected_sum.nonzero_p ())
4981 : return;
4982 :
4983 0 : orig_node->scale_profile_to
4984 0 : (lenient_count_portion_handling (orig_node->count.ipa () - redirected_sum,
4985 : orig_node));
4986 :
4987 0 : new_node->scale_profile_to (new_node->count.ipa () + redirected_sum);
4988 :
4989 0 : if (dump_file)
4990 : {
4991 0 : dump_profile_updates (new_node, true);
4992 0 : dump_profile_updates (orig_node, false);
4993 : }
4994 : }
4995 :
4996 : static void adjust_references_in_caller (cgraph_edge *cs,
4997 : symtab_node *symbol, int index);
4998 :
4999 : /* Simple structure to pass a symbol and index (with same meaning as parameters
5000 : of adjust_references_in_caller) through a void* parameter of a
5001 : call_for_symbol_thunks_and_aliases callback. */
5002 : struct symbol_and_index_together
5003 : {
5004 : symtab_node *symbol;
5005 : int index;
5006 : };
5007 :
5008 : /* Worker callback of call_for_symbol_thunks_and_aliases to recursively call
5009 : adjust_references_in_caller on edges up in the call-graph, if necessary. */
5010 : static bool
5011 9 : adjust_refs_in_act_callers (struct cgraph_node *node, void *data)
5012 : {
5013 9 : symbol_and_index_together *pack = (symbol_and_index_together *) data;
5014 40 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
5015 31 : if (!cs->caller->thunk)
5016 31 : adjust_references_in_caller (cs, pack->symbol, pack->index);
5017 9 : return false;
5018 : }
5019 :
5020 : /* At INDEX of a function being called by CS there is an ADDR_EXPR of a
5021 : variable which is only dereferenced and which is represented by SYMBOL. See
5022 : if we can remove ADDR reference in callers associated with the call. */
5023 :
5024 : static void
5025 415 : adjust_references_in_caller (cgraph_edge *cs, symtab_node *symbol, int index)
5026 : {
5027 415 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5028 415 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, index);
5029 415 : if (jfunc->type == IPA_JF_CONST)
5030 : {
5031 396 : ipa_ref *to_del = cs->caller->find_reference (symbol, cs->call_stmt,
5032 : cs->lto_stmt_uid,
5033 : IPA_REF_ADDR);
5034 396 : if (!to_del)
5035 406 : return;
5036 396 : to_del->remove_reference ();
5037 396 : ipa_zap_jf_refdesc (jfunc);
5038 396 : if (dump_file)
5039 22 : fprintf (dump_file, " Removed a reference from %s to %s.\n",
5040 11 : cs->caller->dump_name (), symbol->dump_name ());
5041 : return;
5042 : }
5043 :
5044 19 : if (jfunc->type != IPA_JF_PASS_THROUGH
5045 19 : || ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR
5046 38 : || ipa_get_jf_pass_through_refdesc_decremented (jfunc))
5047 : return;
5048 :
5049 19 : int fidx = ipa_get_jf_pass_through_formal_id (jfunc);
5050 19 : cgraph_node *caller = cs->caller;
5051 19 : ipa_node_params *caller_info = ipa_node_params_sum->get (caller);
5052 : /* TODO: This consistency check may be too big and not really
5053 : that useful. Consider removing it. */
5054 19 : tree cst;
5055 19 : if (caller_info->ipcp_orig_node)
5056 17 : cst = caller_info->known_csts[fidx];
5057 : else
5058 : {
5059 2 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (caller_info, fidx);
5060 2 : gcc_assert (lat->is_single_const ());
5061 2 : cst = lat->values->value;
5062 : }
5063 19 : gcc_assert (TREE_CODE (cst) == ADDR_EXPR
5064 : && (symtab_node::get (get_base_address (TREE_OPERAND (cst, 0)))
5065 : == symbol));
5066 :
5067 19 : int cuses = ipa_get_controlled_uses (caller_info, fidx);
5068 19 : if (cuses == IPA_UNDESCRIBED_USE)
5069 : return;
5070 19 : gcc_assert (cuses > 0);
5071 19 : cuses--;
5072 19 : ipa_set_controlled_uses (caller_info, fidx, cuses);
5073 19 : ipa_set_jf_pass_through_refdesc_decremented (jfunc, true);
5074 19 : if (dump_file && (dump_flags & TDF_DETAILS))
5075 3 : fprintf (dump_file, " Controlled uses of parameter %i of %s dropped "
5076 : "to %i.\n", fidx, caller->dump_name (), cuses);
5077 19 : if (cuses)
5078 : return;
5079 :
5080 9 : if (caller_info->ipcp_orig_node)
5081 : {
5082 : /* Cloning machinery has created a reference here, we need to either
5083 : remove it or change it to a read one. */
5084 7 : ipa_ref *to_del = caller->find_reference (symbol, NULL, 0, IPA_REF_ADDR);
5085 7 : if (to_del)
5086 : {
5087 7 : to_del->remove_reference ();
5088 7 : if (dump_file)
5089 6 : fprintf (dump_file, " Removed a reference from %s to %s.\n",
5090 3 : cs->caller->dump_name (), symbol->dump_name ());
5091 7 : if (ipa_get_param_load_dereferenced (caller_info, fidx))
5092 : {
5093 3 : caller->create_reference (symbol, IPA_REF_LOAD, NULL);
5094 3 : if (dump_file)
5095 2 : fprintf (dump_file,
5096 : " ...and replaced it with LOAD one.\n");
5097 : }
5098 : }
5099 : }
5100 :
5101 9 : symbol_and_index_together pack;
5102 9 : pack.symbol = symbol;
5103 9 : pack.index = fidx;
5104 9 : if (caller->can_change_signature)
5105 9 : caller->call_for_symbol_thunks_and_aliases (adjust_refs_in_act_callers,
5106 : &pack, true);
5107 : }
5108 :
5109 :
5110 : /* Return true if we would like to remove a parameter from NODE when cloning it
5111 : with KNOWN_CSTS scalar constants. */
5112 :
5113 : static bool
5114 18181 : want_remove_some_param_p (cgraph_node *node, vec<tree> known_csts)
5115 : {
5116 18181 : auto_vec<bool, 16> surviving;
5117 18181 : bool filled_vec = false;
5118 18181 : ipa_node_params *info = ipa_node_params_sum->get (node);
5119 18181 : int i, count = ipa_get_param_count (info);
5120 :
5121 37747 : for (i = 0; i < count; i++)
5122 : {
5123 32992 : if (!known_csts[i] && ipa_is_param_used (info, i))
5124 19566 : continue;
5125 :
5126 13426 : if (!filled_vec)
5127 : {
5128 13426 : clone_info *info = clone_info::get (node);
5129 13426 : if (!info || !info->param_adjustments)
5130 : return true;
5131 0 : info->param_adjustments->get_surviving_params (&surviving);
5132 0 : filled_vec = true;
5133 : }
5134 0 : if (surviving.length() < (unsigned) i && surviving[i])
5135 : return true;
5136 : }
5137 : return false;
5138 18181 : }
5139 :
5140 : /* Create a specialized version of NODE with known constants in KNOWN_CSTS,
5141 : known contexts in KNOWN_CONTEXTS and known aggregate values in AGGVALS and
5142 : redirect all edges in CALLERS to it. */
5143 :
5144 : static struct cgraph_node *
5145 19622 : create_specialized_node (struct cgraph_node *node,
5146 : vec<tree> known_csts,
5147 : vec<ipa_polymorphic_call_context> known_contexts,
5148 : vec<ipa_argagg_value, va_gc> *aggvals,
5149 : vec<cgraph_edge *> &callers)
5150 : {
5151 19622 : ipa_node_params *new_info, *info = ipa_node_params_sum->get (node);
5152 19622 : vec<ipa_replace_map *, va_gc> *replace_trees = NULL;
5153 19622 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
5154 19622 : struct cgraph_node *new_node;
5155 19622 : int i, count = ipa_get_param_count (info);
5156 19622 : clone_info *cinfo = clone_info::get (node);
5157 0 : ipa_param_adjustments *old_adjustments = cinfo
5158 19622 : ? cinfo->param_adjustments : NULL;
5159 19622 : ipa_param_adjustments *new_adjustments;
5160 19622 : gcc_assert (!info->ipcp_orig_node);
5161 19622 : gcc_assert (node->can_change_signature
5162 : || !old_adjustments);
5163 :
5164 18181 : if (old_adjustments)
5165 : {
5166 : /* At the moment all IPA optimizations should use the number of
5167 : parameters of the prevailing decl as the m_always_copy_start.
5168 : Handling any other value would complicate the code below, so for the
5169 : time bing let's only assert it is so. */
5170 0 : gcc_assert (old_adjustments->m_always_copy_start == count
5171 : || old_adjustments->m_always_copy_start < 0);
5172 0 : int old_adj_count = vec_safe_length (old_adjustments->m_adj_params);
5173 0 : for (i = 0; i < old_adj_count; i++)
5174 : {
5175 0 : ipa_adjusted_param *old_adj = &(*old_adjustments->m_adj_params)[i];
5176 0 : if (!node->can_change_signature
5177 0 : || old_adj->op != IPA_PARAM_OP_COPY
5178 0 : || (!known_csts[old_adj->base_index]
5179 0 : && ipa_is_param_used (info, old_adj->base_index)))
5180 : {
5181 0 : ipa_adjusted_param new_adj = *old_adj;
5182 :
5183 0 : new_adj.prev_clone_adjustment = true;
5184 0 : new_adj.prev_clone_index = i;
5185 0 : vec_safe_push (new_params, new_adj);
5186 : }
5187 : }
5188 0 : bool skip_return = old_adjustments->m_skip_return;
5189 0 : new_adjustments = (new (ggc_alloc <ipa_param_adjustments> ())
5190 : ipa_param_adjustments (new_params, count,
5191 0 : skip_return));
5192 : }
5193 19622 : else if (node->can_change_signature
5194 19622 : && want_remove_some_param_p (node, known_csts))
5195 : {
5196 13426 : ipa_adjusted_param adj;
5197 13426 : memset (&adj, 0, sizeof (adj));
5198 13426 : adj.op = IPA_PARAM_OP_COPY;
5199 52016 : for (i = 0; i < count; i++)
5200 38590 : if (!known_csts[i] && ipa_is_param_used (info, i))
5201 : {
5202 15498 : adj.base_index = i;
5203 15498 : adj.prev_clone_index = i;
5204 15498 : vec_safe_push (new_params, adj);
5205 : }
5206 13426 : new_adjustments = (new (ggc_alloc <ipa_param_adjustments> ())
5207 13426 : ipa_param_adjustments (new_params, count, false));
5208 : }
5209 : else
5210 : new_adjustments = NULL;
5211 :
5212 19622 : auto_vec<cgraph_edge *, 2> self_recursive_calls;
5213 157427 : for (i = callers.length () - 1; i >= 0; i--)
5214 : {
5215 118183 : cgraph_edge *cs = callers[i];
5216 118183 : if (cs->caller == node)
5217 : {
5218 115 : self_recursive_calls.safe_push (cs);
5219 115 : callers.unordered_remove (i);
5220 : }
5221 : }
5222 19622 : replace_trees = cinfo ? vec_safe_copy (cinfo->tree_map) : NULL;
5223 73971 : for (i = 0; i < count; i++)
5224 : {
5225 54349 : tree t = known_csts[i];
5226 54349 : if (!t)
5227 29377 : continue;
5228 :
5229 24972 : gcc_checking_assert (TREE_CODE (t) != TREE_BINFO);
5230 :
5231 24972 : bool load_ref = false;
5232 24972 : symtab_node *ref_symbol;
5233 24972 : if (TREE_CODE (t) == ADDR_EXPR)
5234 : {
5235 6754 : tree base = get_base_address (TREE_OPERAND (t, 0));
5236 6754 : if (TREE_CODE (base) == VAR_DECL
5237 3254 : && ipa_get_controlled_uses (info, i) == 0
5238 960 : && ipa_get_param_load_dereferenced (info, i)
5239 7153 : && (ref_symbol = symtab_node::get (base)))
5240 : {
5241 399 : load_ref = true;
5242 399 : if (node->can_change_signature)
5243 1455 : for (cgraph_edge *caller : callers)
5244 384 : adjust_references_in_caller (caller, ref_symbol, i);
5245 : }
5246 : }
5247 :
5248 24972 : ipa_replace_map *replace_map = get_replacement_map (info, t, i, load_ref);
5249 24972 : if (replace_map)
5250 24972 : vec_safe_push (replace_trees, replace_map);
5251 : }
5252 :
5253 58866 : unsigned &suffix_counter = clone_num_suffixes->get_or_insert (
5254 19622 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (
5255 : node->decl)));
5256 19622 : new_node = node->create_virtual_clone (callers, replace_trees,
5257 : new_adjustments, "constprop",
5258 : suffix_counter);
5259 19622 : suffix_counter++;
5260 :
5261 19622 : bool have_self_recursive_calls = !self_recursive_calls.is_empty ();
5262 19737 : for (unsigned j = 0; j < self_recursive_calls.length (); j++)
5263 : {
5264 115 : cgraph_edge *cs = get_next_cgraph_edge_clone (self_recursive_calls[j]);
5265 : /* Cloned edges can disappear during cloning as speculation can be
5266 : resolved, check that we have one and that it comes from the last
5267 : cloning. */
5268 115 : if (cs && cs->caller == new_node)
5269 114 : cs->redirect_callee_duplicating_thunks (new_node);
5270 : /* Any future code that would make more than one clone of an outgoing
5271 : edge would confuse this mechanism, so let's check that does not
5272 : happen. */
5273 114 : gcc_checking_assert (!cs
5274 : || !get_next_cgraph_edge_clone (cs)
5275 : || get_next_cgraph_edge_clone (cs)->caller != new_node);
5276 : }
5277 19622 : if (have_self_recursive_calls)
5278 107 : new_node->expand_all_artificial_thunks ();
5279 :
5280 19622 : ipa_set_node_agg_value_chain (new_node, aggvals);
5281 51933 : for (const ipa_argagg_value &av : aggvals)
5282 32311 : new_node->maybe_create_reference (av.value, NULL);
5283 :
5284 19622 : if (dump_file && (dump_flags & TDF_DETAILS))
5285 : {
5286 111 : fprintf (dump_file, " the new node is %s.\n", new_node->dump_name ());
5287 111 : if (known_contexts.exists ())
5288 : {
5289 0 : for (i = 0; i < count; i++)
5290 0 : if (!known_contexts[i].useless_p ())
5291 : {
5292 0 : fprintf (dump_file, " known ctx %i is ", i);
5293 0 : known_contexts[i].dump (dump_file);
5294 : }
5295 : }
5296 111 : if (aggvals)
5297 : {
5298 57 : fprintf (dump_file, " Aggregate replacements:");
5299 57 : ipa_argagg_value_list avs (aggvals);
5300 57 : avs.dump (dump_file);
5301 : }
5302 : }
5303 :
5304 19622 : new_info = ipa_node_params_sum->get (new_node);
5305 19622 : new_info->ipcp_orig_node = node;
5306 19622 : new_node->ipcp_clone = true;
5307 19622 : new_info->known_csts = known_csts;
5308 19622 : new_info->known_contexts = known_contexts;
5309 :
5310 19622 : ipcp_discover_new_direct_edges (new_node, known_csts, known_contexts,
5311 : aggvals);
5312 :
5313 19622 : return new_node;
5314 19622 : }
5315 :
5316 : /* Return true if JFUNC, which describes a i-th parameter of call CS, is a
5317 : pass-through function to itself when the cgraph_node involved is not an
5318 : IPA-CP clone. When SIMPLE is true, further check if JFUNC is a simple
5319 : no-operation pass-through. */
5320 :
5321 : static bool
5322 805354 : self_recursive_pass_through_p (cgraph_edge *cs, ipa_jump_func *jfunc, int i,
5323 : bool simple = true)
5324 : {
5325 805354 : enum availability availability;
5326 805354 : if (jfunc->type == IPA_JF_PASS_THROUGH
5327 79376 : && cs->caller == cs->callee->function_symbol (&availability)
5328 19379 : && availability > AVAIL_INTERPOSABLE
5329 19379 : && (!simple || ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
5330 19379 : && ipa_get_jf_pass_through_formal_id (jfunc) == i
5331 19379 : && ipa_node_params_sum->get (cs->caller)
5332 824733 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5333 19350 : return true;
5334 : return false;
5335 : }
5336 :
5337 : /* Return true if JFUNC, which describes the i-th parameter of call CS, is an
5338 : ancestor function with zero offset to itself when the cgraph_node involved
5339 : is not an IPA-CP clone. */
5340 :
5341 : static bool
5342 786004 : self_recursive_ancestor_p (cgraph_edge *cs, ipa_jump_func *jfunc, int i)
5343 : {
5344 786004 : enum availability availability;
5345 786004 : if (jfunc->type == IPA_JF_ANCESTOR
5346 3255 : && cs->caller == cs->callee->function_symbol (&availability)
5347 1 : && availability > AVAIL_INTERPOSABLE
5348 1 : && ipa_get_jf_ancestor_offset (jfunc) == 0
5349 1 : && ipa_get_jf_ancestor_formal_id (jfunc) == i
5350 1 : && ipa_node_params_sum->get (cs->caller)
5351 786005 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5352 1 : return true;
5353 : return false;
5354 : }
5355 :
5356 : /* Return true if JFUNC, which describes a part of an aggregate represented or
5357 : pointed to by the i-th parameter of call CS, is a pass-through function to
5358 : itself when the cgraph_node involved is not an IPA-CP clone.. When
5359 : SIMPLE is true, further check if JFUNC is a simple no-operation
5360 : pass-through. */
5361 :
5362 : static bool
5363 362883 : self_recursive_agg_pass_through_p (const cgraph_edge *cs,
5364 : const ipa_agg_jf_item *jfunc,
5365 : int i, bool simple = true)
5366 : {
5367 362883 : enum availability availability;
5368 362883 : if (cs->caller == cs->callee->function_symbol (&availability)
5369 3597 : && availability > AVAIL_INTERPOSABLE
5370 3597 : && jfunc->jftype == IPA_JF_LOAD_AGG
5371 487 : && jfunc->offset == jfunc->value.load_agg.offset
5372 487 : && (!simple || jfunc->value.pass_through.operation == NOP_EXPR)
5373 487 : && jfunc->value.pass_through.formal_id == i
5374 481 : && useless_type_conversion_p (jfunc->value.load_agg.type, jfunc->type)
5375 481 : && ipa_node_params_sum->get (cs->caller)
5376 363364 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5377 481 : return true;
5378 : return false;
5379 : }
5380 :
5381 : /* Given a NODE, and a subset of its CALLERS, try to populate blanks slots in
5382 : KNOWN_CSTS with constants that are also known for all of the CALLERS. */
5383 :
5384 : static void
5385 171156 : find_scalar_values_for_callers_subset (vec<tree> &known_csts,
5386 : ipa_node_params *info,
5387 : const vec<cgraph_edge *> &callers)
5388 : {
5389 171156 : int i, count = ipa_get_param_count (info);
5390 :
5391 748573 : for (i = 0; i < count; i++)
5392 : {
5393 577417 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (info, i);
5394 577417 : if (lat->bottom)
5395 10195 : continue;
5396 567222 : if (lat->is_single_const ())
5397 : {
5398 29670 : known_csts[i] = lat->values->value;
5399 29670 : continue;
5400 : }
5401 :
5402 537552 : struct cgraph_edge *cs;
5403 537552 : tree newval = NULL_TREE;
5404 537552 : int j;
5405 537552 : bool first = true;
5406 537552 : tree type = ipa_get_type (info, i);
5407 :
5408 1545948 : FOR_EACH_VEC_ELT (callers, j, cs)
5409 : {
5410 804778 : struct ipa_jump_func *jump_func;
5411 804778 : tree t;
5412 :
5413 804778 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5414 804778 : if (!args
5415 804778 : || i >= ipa_get_cs_argument_count (args)
5416 1609525 : || (i == 0
5417 184932 : && call_passes_through_thunk (cs)))
5418 : {
5419 : newval = NULL_TREE;
5420 : break;
5421 : }
5422 804699 : jump_func = ipa_get_ith_jump_func (args, i);
5423 :
5424 : /* Besides simple pass-through jump function, arithmetic jump
5425 : function could also introduce argument-direct-pass-through for
5426 : self-feeding recursive call. For example,
5427 :
5428 : fn (int i)
5429 : {
5430 : fn (i & 1);
5431 : }
5432 :
5433 : Given that i is 0, recursive propagation via (i & 1) also gets
5434 : 0. */
5435 804699 : if (self_recursive_pass_through_p (cs, jump_func, i, false))
5436 : {
5437 18702 : gcc_assert (newval);
5438 18702 : enum tree_code opcode
5439 18702 : = ipa_get_jf_pass_through_operation (jump_func);
5440 18702 : tree op_type = (opcode == NOP_EXPR) ? NULL_TREE
5441 49 : : ipa_get_jf_pass_through_op_type (jump_func);
5442 18702 : t = ipa_get_jf_arith_result (opcode, newval,
5443 : ipa_get_jf_pass_through_operand (jump_func),
5444 : op_type);
5445 18702 : t = ipacp_value_safe_for_type (type, t);
5446 : }
5447 785997 : else if (self_recursive_ancestor_p (cs, jump_func, i))
5448 0 : continue;
5449 : else
5450 785997 : t = ipa_value_from_jfunc (ipa_node_params_sum->get (cs->caller),
5451 : jump_func, type);
5452 804699 : if (!t
5453 493195 : || (newval
5454 262179 : && !values_equal_for_ipcp_p (t, newval))
5455 1275543 : || (!first && !newval))
5456 : {
5457 : newval = NULL_TREE;
5458 : break;
5459 : }
5460 : else
5461 : newval = t;
5462 : first = false;
5463 : }
5464 :
5465 537552 : if (newval)
5466 203618 : known_csts[i] = newval;
5467 : }
5468 171156 : }
5469 :
5470 : /* Given a NODE and a subset of its CALLERS, try to populate plank slots in
5471 : KNOWN_CONTEXTS with polymorphic contexts that are also known for all of the
5472 : CALLERS. */
5473 :
5474 : static void
5475 171156 : find_contexts_for_caller_subset (vec<ipa_polymorphic_call_context>
5476 : &known_contexts,
5477 : ipa_node_params *info,
5478 : const vec<cgraph_edge *> &callers)
5479 : {
5480 171156 : int i, count = ipa_get_param_count (info);
5481 :
5482 748554 : for (i = 0; i < count; i++)
5483 : {
5484 577411 : if (!ipa_is_param_used (info, i))
5485 23798 : continue;
5486 :
5487 554821 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat
5488 554821 : = ipa_get_poly_ctx_lat (info, i);
5489 554821 : if (ctxlat->bottom)
5490 0 : continue;
5491 554821 : if (ctxlat->is_single_const ())
5492 : {
5493 1208 : if (!ctxlat->values->value.useless_p ())
5494 : {
5495 1208 : if (known_contexts.is_empty ())
5496 1110 : known_contexts.safe_grow_cleared (count, true);
5497 1208 : known_contexts[i] = ctxlat->values->value;
5498 : }
5499 1208 : continue;
5500 : }
5501 :
5502 553613 : cgraph_edge *cs;
5503 553613 : ipa_polymorphic_call_context newval;
5504 553613 : bool first = true;
5505 553613 : int j;
5506 :
5507 559201 : FOR_EACH_VEC_ELT (callers, j, cs)
5508 : {
5509 555223 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5510 555223 : if (!args
5511 1110446 : || i >= ipa_get_cs_argument_count (args))
5512 13 : return;
5513 555210 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
5514 555210 : ipa_polymorphic_call_context ctx;
5515 555210 : ctx = ipa_context_from_jfunc (ipa_node_params_sum->get (cs->caller),
5516 : cs, i, jfunc);
5517 555210 : if (first)
5518 : {
5519 553600 : newval = ctx;
5520 553600 : first = false;
5521 : }
5522 : else
5523 1610 : newval.meet_with (ctx);
5524 1107551 : if (newval.useless_p ())
5525 : break;
5526 : }
5527 :
5528 1107200 : if (!newval.useless_p ())
5529 : {
5530 3978 : if (known_contexts.is_empty ())
5531 3704 : known_contexts.safe_grow_cleared (count, true);
5532 3978 : known_contexts[i] = newval;
5533 : }
5534 :
5535 : }
5536 : }
5537 :
5538 : /* Push all aggregate values coming along edge CS for parameter number INDEX to
5539 : RES. If INTERIM is non-NULL, it contains the current interim state of
5540 : collected aggregate values which can be used to compute values passed over
5541 : self-recursive edges.
5542 :
5543 : This basically one iteration of push_agg_values_from_edge over one
5544 : parameter, which allows for simpler early returns. */
5545 :
5546 : static void
5547 644832 : push_agg_values_for_index_from_edge (struct cgraph_edge *cs, int index,
5548 : vec<ipa_argagg_value> *res,
5549 : const ipa_argagg_value_list *interim)
5550 : {
5551 644832 : bool agg_values_from_caller = false;
5552 644832 : bool agg_jf_preserved = false;
5553 644832 : unsigned unit_delta = UINT_MAX;
5554 644832 : int src_idx = -1;
5555 644832 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (ipa_edge_args_sum->get (cs),
5556 : index);
5557 :
5558 644832 : if (jfunc->type == IPA_JF_PASS_THROUGH
5559 644832 : && ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
5560 : {
5561 58681 : agg_values_from_caller = true;
5562 58681 : agg_jf_preserved = ipa_get_jf_pass_through_agg_preserved (jfunc);
5563 58681 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
5564 58681 : unit_delta = 0;
5565 : }
5566 586151 : else if (jfunc->type == IPA_JF_ANCESTOR
5567 586151 : && ipa_get_jf_ancestor_agg_preserved (jfunc))
5568 : {
5569 407 : agg_values_from_caller = true;
5570 407 : agg_jf_preserved = true;
5571 407 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
5572 407 : unit_delta = ipa_get_jf_ancestor_offset (jfunc) / BITS_PER_UNIT;
5573 : }
5574 :
5575 644832 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
5576 644832 : if (agg_values_from_caller)
5577 : {
5578 59088 : if (caller_info->ipcp_orig_node)
5579 : {
5580 11328 : struct cgraph_node *orig_node = caller_info->ipcp_orig_node;
5581 11328 : ipcp_transformation *ts
5582 11328 : = ipcp_get_transformation_summary (cs->caller);
5583 11328 : ipa_node_params *orig_info = ipa_node_params_sum->get (orig_node);
5584 11328 : ipcp_param_lattices *orig_plats
5585 11328 : = ipa_get_parm_lattices (orig_info, src_idx);
5586 11328 : if (ts
5587 11328 : && orig_plats->aggs
5588 2991 : && (agg_jf_preserved || !orig_plats->aggs_by_ref))
5589 : {
5590 2514 : ipa_argagg_value_list src (ts);
5591 2514 : src.push_adjusted_values (src_idx, index, unit_delta, res);
5592 2514 : return;
5593 : }
5594 : }
5595 : else
5596 : {
5597 47760 : ipcp_param_lattices *src_plats
5598 47760 : = ipa_get_parm_lattices (caller_info, src_idx);
5599 47760 : if (src_plats->aggs
5600 2461 : && !src_plats->aggs_bottom
5601 2461 : && (agg_jf_preserved || !src_plats->aggs_by_ref))
5602 : {
5603 1473 : if (interim && (self_recursive_pass_through_p (cs, jfunc, index)
5604 7 : || self_recursive_ancestor_p (cs, jfunc, index)))
5605 : {
5606 649 : interim->push_adjusted_values (src_idx, index, unit_delta,
5607 : res);
5608 649 : return;
5609 : }
5610 824 : if (!src_plats->aggs_contain_variable)
5611 : {
5612 83 : push_agg_values_from_plats (src_plats, index, unit_delta,
5613 : res);
5614 83 : return;
5615 : }
5616 : }
5617 : }
5618 : }
5619 :
5620 641586 : if (!jfunc->agg.items)
5621 : return;
5622 227556 : bool first = true;
5623 227556 : unsigned prev_unit_offset = 0;
5624 1273814 : for (const ipa_agg_jf_item &agg_jf : *jfunc->agg.items)
5625 : {
5626 1046258 : tree value, srcvalue;
5627 : /* Besides simple pass-through aggregate jump function, arithmetic
5628 : aggregate jump function could also bring same aggregate value as
5629 : parameter passed-in for self-feeding recursive call. For example,
5630 :
5631 : fn (int *i)
5632 : {
5633 : int j = *i & 1;
5634 : fn (&j);
5635 : }
5636 :
5637 : Given that *i is 0, recursive propagation via (*i & 1) also gets 0. */
5638 1046258 : if (interim
5639 362883 : && self_recursive_agg_pass_through_p (cs, &agg_jf, index, false)
5640 1046739 : && (srcvalue = interim->get_value(index,
5641 481 : agg_jf.offset / BITS_PER_UNIT)))
5642 : {
5643 950 : value = ipa_get_jf_arith_result (agg_jf.value.pass_through.operation,
5644 : srcvalue,
5645 475 : agg_jf.value.pass_through.operand,
5646 475 : agg_jf.value.pass_through.op_type);
5647 475 : value = ipacp_value_safe_for_type (agg_jf.type, value);
5648 : }
5649 : else
5650 1045783 : value = ipa_agg_value_from_jfunc (caller_info, cs->caller,
5651 : &agg_jf);
5652 1046258 : if (value)
5653 : {
5654 1020598 : struct ipa_argagg_value iav;
5655 1020598 : iav.value = value;
5656 1020598 : iav.unit_offset = agg_jf.offset / BITS_PER_UNIT;
5657 1020598 : iav.index = index;
5658 1020598 : iav.by_ref = jfunc->agg.by_ref;
5659 1020598 : iav.killed = false;
5660 :
5661 1020598 : gcc_assert (first
5662 : || iav.unit_offset > prev_unit_offset);
5663 1020598 : prev_unit_offset = iav.unit_offset;
5664 1020598 : first = false;
5665 :
5666 1020598 : res->safe_push (iav);
5667 : }
5668 : }
5669 : return;
5670 : }
5671 :
5672 : /* Push all aggregate values coming along edge CS to RES. DEST_INFO is the
5673 : description of ultimate callee of CS or the one it was cloned from (the
5674 : summary where lattices are). If INTERIM is non-NULL, it contains the
5675 : current interim state of collected aggregate values which can be used to
5676 : compute values passed over self-recursive edges (if OPTIMIZE_SELF_RECURSION
5677 : is true) and to skip values which clearly will not be part of intersection
5678 : with INTERIM. */
5679 :
5680 : static void
5681 228075 : push_agg_values_from_edge (struct cgraph_edge *cs,
5682 : ipa_node_params *dest_info,
5683 : vec<ipa_argagg_value> *res,
5684 : const ipa_argagg_value_list *interim,
5685 : bool optimize_self_recursion)
5686 : {
5687 228075 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5688 228075 : if (!args)
5689 : return;
5690 :
5691 456150 : int count = MIN (ipa_get_param_count (dest_info),
5692 : ipa_get_cs_argument_count (args));
5693 :
5694 228075 : unsigned interim_index = 0;
5695 957142 : for (int index = 0; index < count; index++)
5696 : {
5697 729067 : if (interim)
5698 : {
5699 298960 : while (interim_index < interim->m_elts.size ()
5700 273352 : && interim->m_elts[interim_index].value
5701 526263 : && interim->m_elts[interim_index].index < index)
5702 147279 : interim_index++;
5703 209828 : if (interim_index >= interim->m_elts.size ()
5704 151681 : || interim->m_elts[interim_index].index > index)
5705 58147 : continue;
5706 : }
5707 :
5708 670920 : ipcp_param_lattices *plats = ipa_get_parm_lattices (dest_info, index);
5709 670920 : if (!ipa_is_param_used (dest_info, index)
5710 670920 : || plats->aggs_bottom)
5711 26088 : continue;
5712 644877 : push_agg_values_for_index_from_edge (cs, index, res,
5713 : optimize_self_recursion ? interim
5714 : : NULL);
5715 : }
5716 : }
5717 :
5718 :
5719 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5720 : from all of them into INTERIM. Return how many there are. */
5721 :
5722 : static unsigned int
5723 171156 : find_aggregate_values_for_callers_subset_1 (vec<ipa_argagg_value> &interim,
5724 : struct cgraph_node *node,
5725 : const vec<cgraph_edge *> &callers)
5726 : {
5727 171156 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5728 171156 : if (dest_info->ipcp_orig_node)
5729 0 : dest_info = ipa_node_params_sum->get (dest_info->ipcp_orig_node);
5730 :
5731 : /* gather_edges_for_value puts a non-recursive call into the first element of
5732 : callers if it can. */
5733 171156 : push_agg_values_from_edge (callers[0], dest_info, &interim, NULL, true);
5734 :
5735 171156 : unsigned valid_entries = interim.length ();
5736 171156 : if (!valid_entries)
5737 : return 0;
5738 :
5739 87244 : unsigned caller_count = callers.length();
5740 142436 : for (unsigned i = 1; i < caller_count; i++)
5741 : {
5742 56875 : auto_vec<ipa_argagg_value, 32> last;
5743 56875 : ipa_argagg_value_list avs (&interim);
5744 56875 : push_agg_values_from_edge (callers[i], dest_info, &last, &avs, true);
5745 :
5746 56875 : valid_entries = intersect_argaggs_with (interim, last);
5747 56875 : if (!valid_entries)
5748 1683 : return 0;
5749 56875 : }
5750 :
5751 : return valid_entries;
5752 : }
5753 :
5754 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5755 : from all of them and return them in a garbage-collected vector. Return
5756 : nullptr if there are none. */
5757 :
5758 : static void
5759 156161 : find_aggregate_values_for_callers_subset (vec<ipa_argagg_value> &res,
5760 : struct cgraph_node *node,
5761 : const vec<cgraph_edge *> &callers)
5762 : {
5763 156161 : auto_vec<ipa_argagg_value, 32> interim;
5764 156161 : unsigned valid_entries
5765 156161 : = find_aggregate_values_for_callers_subset_1 (interim, node, callers);
5766 156161 : if (!valid_entries)
5767 : return;
5768 :
5769 883973 : for (const ipa_argagg_value &av : interim)
5770 643424 : if (av.value)
5771 608302 : res.safe_push(av);
5772 : return;
5773 156161 : }
5774 :
5775 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5776 : from all of them and return them in a garbage-collected vector. Return
5777 : nullptr if there are none. */
5778 :
5779 : static struct vec<ipa_argagg_value, va_gc> *
5780 14995 : find_aggregate_values_for_callers_subset_gc (struct cgraph_node *node,
5781 : const vec<cgraph_edge *> &callers)
5782 : {
5783 14995 : auto_vec<ipa_argagg_value, 32> interim;
5784 14995 : unsigned valid_entries
5785 14995 : = find_aggregate_values_for_callers_subset_1 (interim, node, callers);
5786 14995 : if (!valid_entries)
5787 : return nullptr;
5788 :
5789 5378 : vec<ipa_argagg_value, va_gc> *res = NULL;
5790 5378 : vec_safe_reserve_exact (res, valid_entries);
5791 38064 : for (const ipa_argagg_value &av : interim)
5792 21930 : if (av.value)
5793 20522 : res->quick_push(av);
5794 5378 : gcc_checking_assert (res->length () == valid_entries);
5795 : return res;
5796 14995 : }
5797 :
5798 : /* Determine whether CS also brings all scalar values that the NODE is
5799 : specialized for. */
5800 :
5801 : static bool
5802 85 : cgraph_edge_brings_all_scalars_for_node (struct cgraph_edge *cs,
5803 : struct cgraph_node *node)
5804 : {
5805 85 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5806 85 : int count = ipa_get_param_count (dest_info);
5807 85 : class ipa_node_params *caller_info;
5808 85 : class ipa_edge_args *args;
5809 85 : int i;
5810 :
5811 85 : caller_info = ipa_node_params_sum->get (cs->caller);
5812 85 : args = ipa_edge_args_sum->get (cs);
5813 202 : for (i = 0; i < count; i++)
5814 : {
5815 139 : struct ipa_jump_func *jump_func;
5816 139 : tree val, t;
5817 :
5818 139 : val = dest_info->known_csts[i];
5819 139 : if (!val)
5820 98 : continue;
5821 :
5822 82 : if (i >= ipa_get_cs_argument_count (args))
5823 : return false;
5824 41 : jump_func = ipa_get_ith_jump_func (args, i);
5825 41 : t = ipa_value_from_jfunc (caller_info, jump_func,
5826 : ipa_get_type (dest_info, i));
5827 41 : if (!t || !values_equal_for_ipcp_p (val, t))
5828 : return false;
5829 : }
5830 : return true;
5831 : }
5832 :
5833 : /* Determine whether CS also brings all aggregate values that NODE is
5834 : specialized for. */
5835 :
5836 : static bool
5837 63 : cgraph_edge_brings_all_agg_vals_for_node (struct cgraph_edge *cs,
5838 : struct cgraph_node *node)
5839 : {
5840 63 : ipcp_transformation *ts = ipcp_get_transformation_summary (node);
5841 63 : if (!ts || vec_safe_is_empty (ts->m_agg_values))
5842 : return true;
5843 :
5844 44 : const ipa_argagg_value_list existing (ts->m_agg_values);
5845 44 : auto_vec<ipa_argagg_value, 32> edge_values;
5846 44 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5847 44 : gcc_checking_assert (dest_info->ipcp_orig_node);
5848 44 : dest_info = ipa_node_params_sum->get (dest_info->ipcp_orig_node);
5849 44 : push_agg_values_from_edge (cs, dest_info, &edge_values, &existing, false);
5850 44 : const ipa_argagg_value_list avl (&edge_values);
5851 44 : return avl.superset_of_p (existing);
5852 44 : }
5853 :
5854 : /* Determine whether CS also brings all polymorphic contexts that NODE is
5855 : specialized for. */
5856 :
5857 : static bool
5858 46 : cgraph_edge_brings_all_contexts_for_node (struct cgraph_edge *cs,
5859 : struct cgraph_node *node)
5860 : {
5861 46 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5862 46 : unsigned count = dest_info->known_contexts.length ();
5863 15 : if (!count)
5864 : return true;
5865 :
5866 15 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
5867 15 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5868 57 : for (unsigned i = 0; i < count; i++)
5869 : {
5870 45 : ipa_polymorphic_call_context ctx = dest_info->known_contexts[i];
5871 45 : if (ctx.useless_p ())
5872 36 : continue;
5873 :
5874 48 : if (!args || i >= (unsigned) ipa_get_cs_argument_count (args))
5875 3 : return false;
5876 24 : ipa_jump_func *jump_func = ipa_get_ith_jump_func (args, i);
5877 24 : ipa_polymorphic_call_context edge_ctx
5878 24 : = ipa_context_from_jfunc (caller_info, cs, i, jump_func);
5879 27 : if (edge_ctx.useless_p ())
5880 : return false;
5881 24 : if (edge_ctx.equal_to (ctx))
5882 15 : continue;
5883 : /* Also accept a context that is more precise than the one NODE has been
5884 : specialized for. */
5885 9 : ctx.combine_with (edge_ctx);
5886 9 : if (!edge_ctx.equal_to (ctx))
5887 : return false;
5888 : }
5889 : return true;
5890 : }
5891 :
5892 : /* Given an original NODE and a VAL for which we have already created a
5893 : specialized clone, look whether there are incoming edges that still lead
5894 : into the old node but now also bring the requested value and also conform to
5895 : all other criteria such that they can be redirected the special node.
5896 : This function can therefore redirect the final edge in a SCC. */
5897 :
5898 : template <typename valtype>
5899 : static void
5900 9455 : perhaps_add_new_callers (cgraph_node *node, ipcp_value<valtype> *val)
5901 : {
5902 : ipcp_value_source<valtype> *src;
5903 9455 : profile_count redirected_sum = profile_count::zero ();
5904 :
5905 124123 : for (src = val->sources; src; src = src->next)
5906 : {
5907 114668 : struct cgraph_edge *cs = src->cs;
5908 356035 : while (cs)
5909 : {
5910 241367 : if (cgraph_edge_brings_value_p (cs, src, node, val)
5911 85 : && cgraph_edge_brings_all_scalars_for_node (cs, val->spec_node)
5912 63 : && cgraph_edge_brings_all_agg_vals_for_node (cs, val->spec_node)
5913 241413 : && cgraph_edge_brings_all_contexts_for_node (cs, val->spec_node))
5914 : {
5915 43 : if (dump_file)
5916 3 : fprintf (dump_file, " - adding an extra caller %s of %s\n",
5917 3 : cs->caller->dump_name (),
5918 3 : val->spec_node->dump_name ());
5919 :
5920 43 : cs->redirect_callee_duplicating_thunks (val->spec_node);
5921 43 : val->spec_node->expand_all_artificial_thunks ();
5922 43 : if (cs->count.ipa ().initialized_p ())
5923 0 : redirected_sum = redirected_sum + cs->count.ipa ();
5924 : }
5925 241367 : cs = get_next_cgraph_edge_clone (cs);
5926 : }
5927 : }
5928 :
5929 9455 : if (redirected_sum.nonzero_p ())
5930 0 : update_specialized_profile (val->spec_node, node, redirected_sum);
5931 9455 : }
5932 :
5933 : /* Return true if KNOWN_CONTEXTS contain at least one useful context. */
5934 :
5935 : static bool
5936 4627 : known_contexts_useful_p (vec<ipa_polymorphic_call_context> known_contexts)
5937 : {
5938 4627 : ipa_polymorphic_call_context *ctx;
5939 4627 : int i;
5940 :
5941 4628 : FOR_EACH_VEC_ELT (known_contexts, i, ctx)
5942 118 : if (!ctx->useless_p ())
5943 : return true;
5944 : return false;
5945 : }
5946 :
5947 : /* Return a copy of KNOWN_CSTS if it is not empty, otherwise return vNULL. */
5948 :
5949 : static vec<ipa_polymorphic_call_context>
5950 4627 : copy_useful_known_contexts (const vec<ipa_polymorphic_call_context> &known_contexts)
5951 : {
5952 4627 : if (known_contexts_useful_p (known_contexts))
5953 116 : return known_contexts.copy ();
5954 : else
5955 4511 : return vNULL;
5956 : }
5957 :
5958 : /* Return true if the VALUE is represented in KNOWN_CSTS at INDEX if OFFSET is
5959 : minus one or in AGGVALS for INDEX and OFFSET otherwise. */
5960 :
5961 : DEBUG_FUNCTION bool
5962 4561 : ipcp_val_replacement_ok_p (vec<tree> &known_csts,
5963 : vec<ipa_polymorphic_call_context> &,
5964 : vec<ipa_argagg_value, va_gc> *aggvals,
5965 : int index, HOST_WIDE_INT offset, tree value)
5966 : {
5967 4561 : tree v;
5968 4561 : if (offset == -1)
5969 3261 : v = known_csts[index];
5970 : else
5971 : {
5972 1300 : const ipa_argagg_value_list avl (aggvals);
5973 1300 : v = avl.get_value (index, offset / BITS_PER_UNIT);
5974 : }
5975 :
5976 4561 : return v && values_equal_for_ipcp_p (v, value);
5977 : }
5978 :
5979 : /* Dump to F all the values in AVALS for which we are re-evaluating the effects
5980 : on the function represented b INFO. */
5981 :
5982 : DEBUG_FUNCTION void
5983 71 : dump_reestimation_message (FILE *f, ipa_node_params *info,
5984 : const ipa_auto_call_arg_values &avals)
5985 : {
5986 71 : fprintf (f, " Re-estimating effects with\n"
5987 : " Scalar constants:");
5988 71 : int param_count = ipa_get_param_count (info);
5989 177 : for (int i = 0; i < param_count; i++)
5990 106 : if (avals.m_known_vals[i])
5991 : {
5992 44 : fprintf (f, " %i:", i);
5993 44 : print_ipcp_constant_value (f, avals.m_known_vals[i]);
5994 : }
5995 71 : fprintf (f, "\n");
5996 71 : if (!avals.m_known_contexts.is_empty ())
5997 : {
5998 0 : fprintf (f, " Pol. contexts:");
5999 0 : for (int i = 0; i < param_count; i++)
6000 0 : if (!avals.m_known_contexts[i].useless_p ())
6001 : {
6002 0 : fprintf (f, " %i:", i);
6003 0 : avals.m_known_contexts[i].dump (f);
6004 : }
6005 0 : fprintf (f, "\n");
6006 : }
6007 71 : if (!avals.m_known_aggs.is_empty ())
6008 : {
6009 27 : fprintf (f, " Aggregate replacements:");
6010 27 : ipa_argagg_value_list avs (&avals);
6011 27 : avs.dump (f);
6012 : }
6013 71 : }
6014 :
6015 : /* Return true if the VALUE is represented in KNOWN_CONTEXTS at INDEX and that
6016 : if OFFSET is is equal to minus one (because source of a polymorphic context
6017 : cannot be an aggregate value). */
6018 :
6019 : DEBUG_FUNCTION bool
6020 66 : ipcp_val_replacement_ok_p (vec<tree> &,
6021 : vec<ipa_polymorphic_call_context> &known_contexts,
6022 : vec<ipa_argagg_value, va_gc> *,
6023 : int index, HOST_WIDE_INT offset,
6024 : ipa_polymorphic_call_context value)
6025 : {
6026 66 : if (offset != -1
6027 66 : || known_contexts.length () <= (unsigned) index
6028 132 : || known_contexts[index].useless_p ())
6029 : return false;
6030 :
6031 66 : if (known_contexts[index].equal_to (value))
6032 : return true;
6033 :
6034 : /* In some corner cases, the final gathering of contexts can figure out that
6035 : the available context is actually more precise than what we wanted to
6036 : clone for. Allow it. */
6037 0 : value.combine_with (known_contexts[index]);
6038 0 : return known_contexts[index].equal_to (value);
6039 : }
6040 :
6041 : /* Decide whether to create a special version of NODE for value VAL of
6042 : parameter at the given INDEX. If OFFSET is -1, the value is for the
6043 : parameter itself, otherwise it is stored at the given OFFSET of the
6044 : parameter. AVALS describes the other already known values. SELF_GEN_CLONES
6045 : is a vector which contains clones created for self-recursive calls with an
6046 : arithmetic pass-through jump function. CUR_SWEEP is the number of the
6047 : current sweep of the call-graph during the decision stage. */
6048 :
6049 : template <typename valtype>
6050 : static bool
6051 231969 : decide_about_value (struct cgraph_node *node, int index, HOST_WIDE_INT offset,
6052 : ipcp_value<valtype> *val,
6053 : vec<cgraph_node *> *self_gen_clones, int cur_sweep)
6054 : {
6055 : int caller_count;
6056 231969 : sreal freq_sum;
6057 : profile_count count_sum, rec_count_sum;
6058 : bool called_without_ipa_profile;
6059 :
6060 231969 : if (val->spec_node)
6061 : {
6062 9455 : perhaps_add_new_callers (node, val);
6063 9455 : return false;
6064 : }
6065 222514 : else if (val->local_size_cost + overall_size > get_max_overall_size (node))
6066 : {
6067 450 : if (dump_file && (dump_flags & TDF_DETAILS))
6068 0 : fprintf (dump_file, " - ignoring candidate value because "
6069 : "maximum unit size would be reached with %li.\n",
6070 : val->local_size_cost + overall_size);
6071 : return false;
6072 : }
6073 222064 : else if (!get_info_about_necessary_edges (val, node, &freq_sum, &caller_count,
6074 : &rec_count_sum, &count_sum,
6075 : &called_without_ipa_profile))
6076 : {
6077 65903 : if (dump_file && (dump_flags & TDF_DETAILS))
6078 : {
6079 121 : fprintf (dump_file, " - skipping candidate value ");
6080 121 : print_ipcp_constant_value (dump_file, val->value);
6081 121 : fprintf (dump_file, " for ");
6082 121 : ipa_dump_param (dump_file, ipa_node_params_sum->get (node), index);
6083 121 : if (offset != -1)
6084 105 : fprintf (dump_file, ", offset: " HOST_WIDE_INT_PRINT_DEC, offset);
6085 121 : fprintf (dump_file, ": no relevant callers\n");
6086 : }
6087 : return false;
6088 : }
6089 :
6090 156161 : if (!dbg_cnt (ipa_cp_values))
6091 : return false;
6092 :
6093 156161 : if (val->self_recursion_generated_p ())
6094 : {
6095 : /* The edge counts in this case might not have been adjusted yet.
6096 : Nevertleless, even if they were it would be only a guesswork which we
6097 : can do now. The recursive part of the counts can be derived from the
6098 : count of the original node anyway. */
6099 293 : if (node->count.ipa ().nonzero_p ())
6100 : {
6101 14 : unsigned dem = self_gen_clones->length () + 1;
6102 14 : rec_count_sum = node->count.ipa () / dem;
6103 : }
6104 : else
6105 265 : rec_count_sum = profile_count::zero ();
6106 : }
6107 :
6108 : /* get_info_about_necessary_edges only sums up ipa counts. */
6109 156161 : count_sum += rec_count_sum;
6110 :
6111 156161 : if (dump_file && (dump_flags & TDF_DETAILS))
6112 : {
6113 150 : fprintf (dump_file, " - considering value ");
6114 150 : print_ipcp_constant_value (dump_file, val->value);
6115 150 : fprintf (dump_file, " for ");
6116 150 : ipa_dump_param (dump_file, ipa_node_params_sum->get (node), index);
6117 150 : if (offset != -1)
6118 69 : fprintf (dump_file, ", offset: " HOST_WIDE_INT_PRINT_DEC, offset);
6119 150 : fprintf (dump_file, " (caller_count: %i)\n", caller_count);
6120 : }
6121 :
6122 156161 : auto_vec<cgraph_edge *> callers
6123 : = gather_edges_for_value (val, node, caller_count);
6124 156161 : ipa_node_params *info = ipa_node_params_sum->get (node);
6125 156161 : ipa_auto_call_arg_values avals;
6126 156161 : avals.m_known_vals.safe_grow_cleared (ipa_get_param_count (info), true);
6127 156161 : find_scalar_values_for_callers_subset (avals.m_known_vals, info, callers);
6128 156161 : find_contexts_for_caller_subset (avals.m_known_contexts, info, callers);
6129 156161 : find_aggregate_values_for_callers_subset (avals.m_known_aggs, node, callers);
6130 :
6131 :
6132 156161 : if (good_cloning_opportunity_p (node, val->prop_time_benefit,
6133 : freq_sum, count_sum, val->prop_size_cost,
6134 : called_without_ipa_profile, cur_sweep))
6135 : ;
6136 : else
6137 : {
6138 : /* Extern inline functions are only meaningful to clione to propagate
6139 : values to their callees. */
6140 154049 : if (DECL_EXTERNAL (node->decl) && DECL_DECLARED_INLINE_P (node->decl))
6141 : {
6142 345 : if (dump_file && (dump_flags & TDF_DETAILS))
6143 0 : fprintf (dump_file, " Skipping extern inline.\n");
6144 151534 : return false;
6145 : }
6146 153704 : if (dump_file && (dump_flags & TDF_DETAILS))
6147 71 : dump_reestimation_message (dump_file, info, avals);
6148 :
6149 153704 : ipa_call_estimates estimates;
6150 153704 : estimate_ipcp_clone_size_and_time (node, &avals, &estimates);
6151 153704 : int removable_params_cost = 0;
6152 991944 : for (tree t : avals.m_known_vals)
6153 530832 : if (t)
6154 213484 : removable_params_cost += estimate_move_cost (TREE_TYPE (t), true);
6155 :
6156 153704 : int size = estimates.size - caller_count * removable_params_cost;
6157 :
6158 153704 : if (size <= 0)
6159 : {
6160 1803 : if (dump_file)
6161 0 : fprintf (dump_file, " Code not going to grow.\n");
6162 : }
6163 : else
6164 : {
6165 151901 : sreal time_benefit
6166 151901 : = ((estimates.nonspecialized_time - estimates.time)
6167 303802 : + hint_time_bonus (node, estimates)
6168 151901 : + (devirtualization_time_bonus (node, &avals)
6169 151901 : + removable_params_cost));
6170 :
6171 151901 : if (!good_cloning_opportunity_p (node, time_benefit, freq_sum,
6172 : count_sum, size,
6173 : called_without_ipa_profile,
6174 : cur_sweep))
6175 151189 : return false;
6176 : }
6177 : }
6178 :
6179 4627 : if (dump_file)
6180 154 : fprintf (dump_file, " Creating a specialized node of %s.\n",
6181 : node->dump_name ());
6182 :
6183 4627 : vec<tree> known_csts = avals.m_known_vals.copy ();
6184 4627 : vec<ipa_polymorphic_call_context> known_contexts
6185 4627 : = copy_useful_known_contexts (avals.m_known_contexts);
6186 :
6187 4627 : vec<ipa_argagg_value, va_gc> *aggvals = NULL;
6188 4627 : vec_safe_reserve_exact (aggvals, avals.m_known_aggs.length ());
6189 25670 : for (const ipa_argagg_value &av : avals.m_known_aggs)
6190 11789 : aggvals->quick_push (av);
6191 4627 : gcc_checking_assert (ipcp_val_replacement_ok_p (known_csts, known_contexts,
6192 : aggvals, index,
6193 : offset, val->value));
6194 4627 : val->spec_node = create_specialized_node (node, known_csts, known_contexts,
6195 : aggvals, callers);
6196 :
6197 4627 : if (val->self_recursion_generated_p ())
6198 152 : self_gen_clones->safe_push (val->spec_node);
6199 : else
6200 4475 : update_profiling_info (node, val->spec_node);
6201 :
6202 4627 : overall_size += val->local_size_cost;
6203 4627 : if (dump_file && (dump_flags & TDF_DETAILS))
6204 80 : fprintf (dump_file, " overall size reached %li\n",
6205 : overall_size);
6206 :
6207 : /* TODO: If for some lattice there is only one other known value
6208 : left, make a special node for it too. */
6209 :
6210 : return true;
6211 156161 : }
6212 :
6213 : /* Like irange::contains_p(), but convert VAL to the range of R if
6214 : necessary. */
6215 :
6216 : static inline bool
6217 49733 : ipa_range_contains_p (const vrange &r, tree val)
6218 : {
6219 49733 : if (r.undefined_p ())
6220 : return false;
6221 :
6222 49733 : tree type = r.type ();
6223 49733 : if (!wi::fits_to_tree_p (wi::to_wide (val), type))
6224 : return false;
6225 :
6226 49733 : val = fold_convert (type, val);
6227 49733 : return r.contains_p (val);
6228 : }
6229 :
6230 : /* Structure holding opportunitties so that they can be pre-sorted. */
6231 :
6232 231969 : struct cloning_opportunity_ranking
6233 : {
6234 : /* A very rough evaluation of likely benefit. */
6235 : sreal eval;
6236 : /* In the case of aggregate constants, a non-negative offset within their
6237 : aggregates. -1 for scalar constants, -2 for polymorphic contexts. */
6238 : HOST_WIDE_INT offset;
6239 : /* The value being considered for evaluation for cloning. */
6240 : ipcp_value_base *val;
6241 : /* Index of the formal parameter the value is coming in. */
6242 : int index;
6243 : };
6244 :
6245 : /* Helper function to qsort a vector of cloning opportunities. */
6246 :
6247 : static int
6248 2231770 : compare_cloning_opportunities (const void *a, const void *b)
6249 : {
6250 2231770 : const cloning_opportunity_ranking *o1
6251 : = (const cloning_opportunity_ranking *) a;
6252 2231770 : const cloning_opportunity_ranking *o2
6253 : = (const cloning_opportunity_ranking *) b;
6254 2231770 : if (o1->eval < o2->eval)
6255 : return 1;
6256 1740560 : if (o1->eval > o2->eval)
6257 574740 : return -1;
6258 : return 0;
6259 : }
6260 :
6261 : /* Use the estimations in VAL to determine how good a candidate it represents
6262 : for the purposes of ordering real evaluation of opportunities (which
6263 : includes information about incoming edges, among other things). */
6264 :
6265 : static sreal
6266 231969 : cloning_opportunity_ranking_evaluation (const ipcp_value_base *val)
6267 : {
6268 231969 : sreal e1 = (val->local_time_benefit * 1000) / MAX (val->local_size_cost, 1);
6269 231969 : sreal e2 = (val->prop_time_benefit * 1000) / MAX (val->prop_size_cost, 1);
6270 231969 : if (e2 > e1)
6271 16033 : return e2;
6272 : else
6273 215936 : return e1;
6274 : }
6275 :
6276 : /* Decide whether and what specialized clones of NODE should be created.
6277 : CUR_SWEEP is the number of the current sweep of the call-graph during the
6278 : decision stage. */
6279 :
6280 : static bool
6281 3302811 : decide_whether_version_node (struct cgraph_node *node, int cur_sweep)
6282 : {
6283 3302811 : ipa_node_params *info = ipa_node_params_sum->get (node);
6284 3302811 : int count = ipa_get_param_count (info);
6285 3302811 : bool ret = false;
6286 :
6287 3302811 : if (info->node_dead || count == 0)
6288 : return false;
6289 :
6290 2664730 : bool clone_for_all_contexts = node->local;
6291 2664730 : if (dump_file && (dump_flags & TDF_DETAILS))
6292 : {
6293 448 : fprintf (dump_file, "\nEvaluating opportunities for %s.",
6294 : node->dump_name ());
6295 448 : if (clone_for_all_contexts)
6296 129 : fprintf (dump_file, " Will try to create a special all-context "
6297 : "clone.\n");
6298 448 : fprintf (dump_file, "\n");
6299 : }
6300 :
6301 2664730 : auto_vec <cloning_opportunity_ranking, 32> opp_ranking;
6302 8900965 : for (int i = 0; i < count;i++)
6303 : {
6304 6236235 : if (!ipa_is_param_used (info, i))
6305 : {
6306 697644 : if (dump_file && (dump_flags & TDF_DETAILS))
6307 20 : fprintf (dump_file, " - ignoring unused parameter %i.\n", i);
6308 697644 : continue;
6309 : }
6310 :
6311 5538591 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6312 5538591 : ipcp_lattice<tree> *lat = &plats->itself;
6313 5538591 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
6314 :
6315 5538591 : if (!lat->bottom
6316 5538591 : && (!clone_for_all_contexts || !lat->is_single_const ()))
6317 : {
6318 553656 : ipcp_value<tree> *val;
6319 674256 : for (val = lat->values; val; val = val->next)
6320 : {
6321 : /* If some values generated for self-recursive calls with
6322 : arithmetic jump functions fall outside of the known
6323 : range for the parameter, we can skip them. */
6324 120662 : if (TREE_CODE (val->value) == INTEGER_CST
6325 73010 : && !plats->m_value_range.bottom_p ()
6326 170333 : && !ipa_range_contains_p (plats->m_value_range.m_vr,
6327 : val->value))
6328 : {
6329 : /* This can happen also if a constant present in the source
6330 : code falls outside of the range of parameter's type, so we
6331 : cannot assert. */
6332 62 : if (dump_file && (dump_flags & TDF_DETAILS))
6333 : {
6334 0 : fprintf (dump_file, " - skipping%s value ",
6335 0 : val->self_recursion_generated_p ()
6336 : ? " self_recursion_generated" : "");
6337 0 : print_ipcp_constant_value (dump_file, val->value);
6338 0 : fprintf (dump_file, " because it is outside known "
6339 : "value range.\n");
6340 : }
6341 62 : continue;
6342 : }
6343 120538 : cloning_opportunity_ranking opp;
6344 120538 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6345 120538 : opp.offset = -1;
6346 120538 : opp.val = val;
6347 120538 : opp.index = i;
6348 120538 : opp_ranking.safe_push (opp);
6349 : }
6350 : }
6351 :
6352 5538591 : if (!plats->aggs_bottom)
6353 : {
6354 583495 : struct ipcp_agg_lattice *aglat;
6355 583495 : ipcp_value<tree> *val;
6356 732122 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
6357 147545 : if (!aglat->bottom && aglat->values
6358 : /* If the following is false, the one value will be considered
6359 : for cloning for all contexts. */
6360 274771 : && (!clone_for_all_contexts
6361 79373 : || plats->aggs_contain_variable
6362 203216 : || !aglat->is_single_const ()))
6363 190000 : for (val = aglat->values; val; val = val->next)
6364 : {
6365 107430 : cloning_opportunity_ranking opp;
6366 107430 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6367 107430 : opp.offset = aglat->offset;
6368 107430 : opp.val = val;
6369 107430 : opp.index = i;
6370 107430 : opp_ranking.safe_push (opp);
6371 : }
6372 : }
6373 :
6374 5538591 : if (!ctxlat->bottom
6375 6822771 : && (!clone_for_all_contexts || !ctxlat->is_single_const ()))
6376 : {
6377 568686 : ipcp_value<ipa_polymorphic_call_context> *val;
6378 572687 : for (val = ctxlat->values; val; val = val->next)
6379 8002 : if (!val->value.useless_p ())
6380 : {
6381 4001 : cloning_opportunity_ranking opp;
6382 4001 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6383 4001 : opp.offset = -2;
6384 4001 : opp.val = val;
6385 4001 : opp.index = i;
6386 4001 : opp_ranking.safe_push (opp);
6387 : }
6388 : }
6389 : }
6390 :
6391 2664730 : if (!opp_ranking.is_empty ())
6392 : {
6393 53370 : opp_ranking.qsort (compare_cloning_opportunities);
6394 53370 : auto_vec <cgraph_node *, 9> self_gen_clones;
6395 392079 : for (const cloning_opportunity_ranking &opp : opp_ranking)
6396 231969 : if (opp.offset == -2)
6397 : {
6398 4001 : ipcp_value<ipa_polymorphic_call_context> *val
6399 : = static_cast <ipcp_value<ipa_polymorphic_call_context> *>
6400 : (opp.val);
6401 4001 : ret |= decide_about_value (node, opp.index, -1, val,
6402 : &self_gen_clones, cur_sweep);
6403 : }
6404 : else
6405 : {
6406 227968 : ipcp_value<tree> *val = static_cast<ipcp_value<tree> *> (opp.val);
6407 227968 : ret |= decide_about_value (node, opp.index, opp.offset, val,
6408 : &self_gen_clones, cur_sweep);
6409 : }
6410 :
6411 106740 : if (!self_gen_clones.is_empty ())
6412 : {
6413 31 : self_gen_clones.safe_push (node);
6414 31 : update_counts_for_self_gen_clones (node, self_gen_clones);
6415 : }
6416 53370 : }
6417 :
6418 2664730 : if (!clone_for_all_contexts)
6419 : return ret;
6420 :
6421 236248 : struct caller_statistics stats;
6422 236248 : init_caller_stats (&stats);
6423 236248 : node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
6424 : false);
6425 236248 : if (!stats.n_calls)
6426 : {
6427 16319 : if (dump_file && (dump_flags & TDF_DETAILS))
6428 49 : fprintf (dump_file, " Not cloning for all contexts because "
6429 : "there are no callers of the original node (any more).\n");
6430 : return ret;
6431 : }
6432 :
6433 219929 : ipa_auto_call_arg_values avals;
6434 219929 : int removable_params_cost;
6435 219929 : bool ctx_independent_const
6436 219929 : = gather_context_independent_values (info, &avals, &removable_params_cost);
6437 219929 : sreal devirt_bonus = devirtualization_time_bonus (node, &avals);
6438 424796 : if (ctx_independent_const || devirt_bonus > 0
6439 424796 : || (removable_params_cost && clone_for_param_removal_p (node)))
6440 : {
6441 15062 : if (!dbg_cnt (ipa_cp_values))
6442 67 : return ret;
6443 :
6444 15062 : auto_vec<cgraph_edge *> callers = node->collect_callers ();
6445 57033 : for (int i = callers.length () - 1; i >= 0; i--)
6446 : {
6447 26909 : cgraph_edge *cs = callers[i];
6448 26909 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
6449 :
6450 26909 : if (caller_info && caller_info->node_dead)
6451 2682 : callers.unordered_remove (i);
6452 : }
6453 :
6454 15062 : if (!adjust_callers_for_value_intersection (callers, node))
6455 : /* If node is not called by anyone, or all its caller edges are
6456 : self-recursive, the node is not really in use, no need to do
6457 : cloning. */
6458 67 : return ret;
6459 :
6460 14995 : if (dump_file)
6461 99 : fprintf (dump_file, " Creating a specialized node of %s "
6462 : "for all known contexts.\n", node->dump_name ());
6463 :
6464 14995 : vec<tree> known_csts = vNULL;
6465 14995 : known_csts.safe_grow_cleared (count, true);
6466 14995 : find_scalar_values_for_callers_subset (known_csts, info, callers);
6467 14995 : vec<ipa_polymorphic_call_context> known_contexts = vNULL;
6468 14995 : find_contexts_for_caller_subset (known_contexts, info, callers);
6469 14995 : vec<ipa_argagg_value, va_gc> *aggvals
6470 14995 : = find_aggregate_values_for_callers_subset_gc (node, callers);
6471 :
6472 14995 : struct cgraph_node *clone = create_specialized_node (node, known_csts,
6473 : known_contexts,
6474 : aggvals, callers);
6475 14995 : ipa_node_params_sum->get (clone)->is_all_contexts_clone = true;
6476 14995 : ret = true;
6477 15062 : }
6478 :
6479 : return ret;
6480 2884659 : }
6481 :
6482 : /* Transitively mark all callees of NODE within the same SCC as not dead. */
6483 :
6484 : static void
6485 2277 : spread_undeadness (struct cgraph_node *node)
6486 : {
6487 2277 : struct cgraph_edge *cs;
6488 :
6489 12311 : for (cs = node->callees; cs; cs = cs->next_callee)
6490 10034 : if (ipa_edge_within_scc (cs))
6491 : {
6492 833 : struct cgraph_node *callee;
6493 833 : class ipa_node_params *info;
6494 :
6495 833 : callee = cs->callee->function_symbol (NULL);
6496 833 : info = ipa_node_params_sum->get (callee);
6497 :
6498 833 : if (info && info->node_dead)
6499 : {
6500 68 : info->node_dead = 0;
6501 68 : spread_undeadness (callee);
6502 : }
6503 : }
6504 2277 : }
6505 :
6506 : /* Return true if NODE has a caller from outside of its SCC that is not
6507 : dead. Worker callback for cgraph_for_node_and_aliases. */
6508 :
6509 : static bool
6510 16402 : has_undead_caller_from_outside_scc_p (struct cgraph_node *node,
6511 : void *data ATTRIBUTE_UNUSED)
6512 : {
6513 16402 : struct cgraph_edge *cs;
6514 :
6515 81743 : for (cs = node->callers; cs; cs = cs->next_caller)
6516 65800 : if (cs->caller->thunk
6517 65800 : && cs->caller->call_for_symbol_thunks_and_aliases
6518 0 : (has_undead_caller_from_outside_scc_p, NULL, true))
6519 : return true;
6520 65800 : else if (!ipa_edge_within_scc (cs))
6521 : {
6522 65560 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
6523 65560 : if (!caller_info /* Unoptimized caller are like dead ones. */
6524 65558 : || !caller_info->node_dead)
6525 : return true;
6526 : }
6527 : return false;
6528 : }
6529 :
6530 :
6531 : /* Identify nodes within the same SCC as NODE which are no longer needed
6532 : because of new clones and will be removed as unreachable. */
6533 :
6534 : static void
6535 17571 : identify_dead_nodes (struct cgraph_node *node)
6536 : {
6537 17571 : struct cgraph_node *v;
6538 35413 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6539 17842 : if (v->local)
6540 : {
6541 16154 : ipa_node_params *info = ipa_node_params_sum->get (v);
6542 16154 : if (info
6543 32308 : && !v->call_for_symbol_thunks_and_aliases
6544 16154 : (has_undead_caller_from_outside_scc_p, NULL, true))
6545 15695 : info->node_dead = 1;
6546 : }
6547 :
6548 35413 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6549 : {
6550 17842 : ipa_node_params *info = ipa_node_params_sum->get (v);
6551 17842 : if (info && !info->node_dead)
6552 2209 : spread_undeadness (v);
6553 : }
6554 :
6555 17571 : if (dump_file && (dump_flags & TDF_DETAILS))
6556 : {
6557 147 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6558 75 : if (ipa_node_params_sum->get (v)
6559 75 : && ipa_node_params_sum->get (v)->node_dead)
6560 40 : fprintf (dump_file, " Marking node as dead: %s.\n",
6561 : v->dump_name ());
6562 : }
6563 17571 : }
6564 :
6565 : /* Removes all useless callback edges from the callgraph. Useless callback
6566 : edges might mess up the callgraph, because they might be impossible to
6567 : redirect and so on, leading to crashes. Their usefulness is evaluated
6568 : through callback_edge_useful_p. */
6569 :
6570 : static void
6571 132707 : purge_useless_callback_edges ()
6572 : {
6573 132707 : if (dump_file)
6574 175 : fprintf (dump_file, "\nPurging useless callback edges:\n");
6575 :
6576 132707 : cgraph_edge *e;
6577 132707 : cgraph_node *node;
6578 1461675 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6579 : {
6580 6946797 : for (e = node->callees; e; e = e->next_callee)
6581 : {
6582 5617829 : if (e->has_callback)
6583 : {
6584 13739 : if (dump_file)
6585 6 : fprintf (dump_file, "\tExamining callbacks of edge %s -> %s:\n",
6586 6 : e->caller->dump_name (), e->callee->dump_name ());
6587 13739 : if (!lookup_attribute ("callback_only",
6588 13739 : DECL_ATTRIBUTES (e->callee->decl))
6589 13739 : && !callback_is_special_cased (e->callee->decl, e->call_stmt))
6590 : {
6591 2 : if (dump_file)
6592 0 : fprintf (
6593 : dump_file,
6594 : "\t\tPurging callbacks, because the callback-dispatching"
6595 : "function no longer has any callback attributes.\n");
6596 2 : e->purge_callback_edges ();
6597 2 : continue;
6598 : }
6599 13737 : cgraph_edge *cbe, *next;
6600 27476 : for (cbe = e->first_callback_edge (); cbe; cbe = next)
6601 : {
6602 13739 : next = cbe->next_callback_edge ();
6603 13739 : if (!callback_edge_useful_p (cbe))
6604 : {
6605 13519 : if (dump_file)
6606 4 : fprintf (dump_file,
6607 : "\t\tCallback edge %s -> %s not deemed "
6608 : "useful, removing.\n",
6609 4 : cbe->caller->dump_name (),
6610 4 : cbe->callee->dump_name ());
6611 13519 : cgraph_edge::remove (cbe);
6612 : }
6613 : else
6614 : {
6615 220 : if (dump_file)
6616 4 : fprintf (dump_file,
6617 : "\t\tKept callback edge %s -> %s "
6618 : "because it looks useful.\n",
6619 4 : cbe->caller->dump_name (),
6620 4 : cbe->callee->dump_name ());
6621 : }
6622 : }
6623 : }
6624 : }
6625 : }
6626 :
6627 132707 : if (dump_file)
6628 175 : fprintf (dump_file, "\n");
6629 132707 : }
6630 :
6631 : /* The decision stage. Iterate over the topological order of call graph nodes
6632 : TOPO and make specialized clones if deemed beneficial. */
6633 :
6634 : static void
6635 132707 : ipcp_decision_stage (class ipa_topo_info *topo)
6636 : {
6637 132707 : int i;
6638 :
6639 132707 : if (dump_file)
6640 175 : fprintf (dump_file, "\nIPA decision stage (%i sweeps):\n",
6641 : max_number_sweeps);
6642 :
6643 509734 : for (int cur_sweep = 1; cur_sweep <= max_number_sweeps; cur_sweep++)
6644 : {
6645 377027 : if (dump_file && (dump_flags & TDF_DETAILS))
6646 183 : fprintf (dump_file, "\nIPA decision sweep number %i (out of %i):\n",
6647 : cur_sweep, max_number_sweeps);
6648 :
6649 4497912 : for (i = topo->nnodes - 1; i >= 0; i--)
6650 : {
6651 4120885 : struct cgraph_node *node = topo->order[i];
6652 4120885 : bool change = false, iterate = true;
6653 :
6654 8259343 : while (iterate)
6655 : {
6656 : struct cgraph_node *v;
6657 : iterate = false;
6658 4153953 : for (v = node;
6659 8292411 : v;
6660 4153953 : v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6661 4153953 : if (v->has_gimple_body_p ()
6662 3920940 : && ipcp_versionable_function_p (v)
6663 4153953 : && (cur_sweep
6664 3302811 : <= opt_for_fn (node->decl, param_ipa_cp_sweeps)))
6665 3302811 : iterate |= decide_whether_version_node (v, cur_sweep);
6666 :
6667 4138458 : change |= iterate;
6668 : }
6669 4120885 : if (change)
6670 17571 : identify_dead_nodes (node);
6671 : }
6672 : }
6673 :
6674 : /* Currently, the primary use of callback edges is constant propagation.
6675 : Constant propagation is now over, so we have to remove unused callback
6676 : edges. */
6677 132707 : purge_useless_callback_edges ();
6678 132707 : }
6679 :
6680 : /* Look up all VR and bits information that we have discovered and copy it
6681 : over to the transformation summary. */
6682 :
6683 : static void
6684 132707 : ipcp_store_vr_results (void)
6685 : {
6686 132707 : cgraph_node *node;
6687 :
6688 1461675 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6689 : {
6690 1328968 : ipa_node_params *info = ipa_node_params_sum->get (node);
6691 1328968 : bool dumped_sth = false;
6692 1328968 : bool found_useful_result = false;
6693 1328968 : bool do_vr = true;
6694 1328968 : bool do_bits = true;
6695 :
6696 : /* If the function is not local, the gathered information is only useful
6697 : for clones. */
6698 1328968 : if (!node->local)
6699 1161035 : continue;
6700 :
6701 167933 : if (!info || !opt_for_fn (node->decl, flag_ipa_vrp))
6702 : {
6703 4809 : if (dump_file)
6704 6 : fprintf (dump_file, "Not considering %s for VR discovery "
6705 : "and propagate; -fipa-ipa-vrp: disabled.\n",
6706 : node->dump_name ());
6707 : do_vr = false;
6708 : }
6709 167933 : if (!info || !opt_for_fn (node->decl, flag_ipa_bit_cp))
6710 : {
6711 4783 : if (dump_file)
6712 2 : fprintf (dump_file, "Not considering %s for ipa bitwise "
6713 : "propagation ; -fipa-bit-cp: disabled.\n",
6714 : node->dump_name ());
6715 : do_bits = false;
6716 : }
6717 4783 : if (!do_bits && !do_vr)
6718 4777 : continue;
6719 :
6720 163156 : if (info->ipcp_orig_node)
6721 19422 : info = ipa_node_params_sum->get (info->ipcp_orig_node);
6722 163156 : if (info->lattices.is_empty ())
6723 : /* Newly expanded artificial thunks do not have lattices. */
6724 52596 : continue;
6725 :
6726 110560 : unsigned count = ipa_get_param_count (info);
6727 223742 : for (unsigned i = 0; i < count; i++)
6728 : {
6729 175184 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6730 175184 : if (do_vr
6731 175161 : && !plats->m_value_range.bottom_p ()
6732 234954 : && !plats->m_value_range.top_p ())
6733 : {
6734 : found_useful_result = true;
6735 : break;
6736 : }
6737 115415 : if (do_bits && plats->bits_lattice.constant_p ())
6738 : {
6739 : found_useful_result = true;
6740 : break;
6741 : }
6742 : }
6743 110560 : if (!found_useful_result)
6744 48558 : continue;
6745 :
6746 62002 : ipcp_transformation_initialize ();
6747 62002 : ipcp_transformation *ts = ipcp_transformation_sum->get_create (node);
6748 62002 : vec_safe_reserve_exact (ts->m_vr, count);
6749 :
6750 287580 : for (unsigned i = 0; i < count; i++)
6751 : {
6752 163576 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6753 163576 : ipcp_bits_lattice *bits = NULL;
6754 :
6755 163576 : if (do_bits
6756 163572 : && plats->bits_lattice.constant_p ()
6757 258044 : && dbg_cnt (ipa_cp_bits))
6758 94468 : bits = &plats->bits_lattice;
6759 :
6760 163576 : if (do_vr
6761 163555 : && !plats->m_value_range.bottom_p ()
6762 113742 : && !plats->m_value_range.top_p ()
6763 277318 : && dbg_cnt (ipa_cp_vr))
6764 : {
6765 113742 : if (bits)
6766 : {
6767 89136 : value_range tmp = plats->m_value_range.m_vr;
6768 89136 : tree type = ipa_get_type (info, i);
6769 178272 : irange_bitmask bm (wide_int::from (bits->get_value (),
6770 89136 : TYPE_PRECISION (type),
6771 89136 : TYPE_SIGN (type)),
6772 178272 : wide_int::from (bits->get_mask (),
6773 89136 : TYPE_PRECISION (type),
6774 178272 : TYPE_SIGN (type)));
6775 89136 : tmp.update_bitmask (bm);
6776 : // Reflecting the bitmask on the ranges can sometime
6777 : // produce an UNDEFINED value if the the bitmask update
6778 : // was previously deferred. See PR 120048.
6779 89136 : if (tmp.undefined_p ())
6780 0 : tmp.set_varying (type);
6781 89136 : ipa_vr vr (tmp);
6782 89136 : ts->m_vr->quick_push (vr);
6783 89136 : }
6784 : else
6785 : {
6786 24606 : ipa_vr vr (plats->m_value_range.m_vr);
6787 24606 : ts->m_vr->quick_push (vr);
6788 : }
6789 : }
6790 49834 : else if (bits)
6791 : {
6792 5332 : tree type = ipa_get_type (info, i);
6793 5332 : value_range tmp;
6794 5332 : tmp.set_varying (type);
6795 10664 : irange_bitmask bm (wide_int::from (bits->get_value (),
6796 5332 : TYPE_PRECISION (type),
6797 5332 : TYPE_SIGN (type)),
6798 10664 : wide_int::from (bits->get_mask (),
6799 5332 : TYPE_PRECISION (type),
6800 10664 : TYPE_SIGN (type)));
6801 5332 : tmp.update_bitmask (bm);
6802 : // Reflecting the bitmask on the ranges can sometime
6803 : // produce an UNDEFINED value if the the bitmask update
6804 : // was previously deferred. See PR 120048.
6805 5332 : if (tmp.undefined_p ())
6806 0 : tmp.set_varying (type);
6807 5332 : ipa_vr vr (tmp);
6808 5332 : ts->m_vr->quick_push (vr);
6809 5332 : }
6810 : else
6811 : {
6812 44502 : ipa_vr vr;
6813 44502 : ts->m_vr->quick_push (vr);
6814 : }
6815 :
6816 163576 : if (!dump_file || !bits)
6817 163143 : continue;
6818 :
6819 433 : if (!dumped_sth)
6820 : {
6821 312 : fprintf (dump_file, "Propagated bits info for function %s:\n",
6822 : node->dump_name ());
6823 312 : dumped_sth = true;
6824 : }
6825 433 : fprintf (dump_file, " param %i: value = ", i);
6826 433 : ipcp_print_widest_int (dump_file, bits->get_value ());
6827 433 : fprintf (dump_file, ", mask = ");
6828 433 : ipcp_print_widest_int (dump_file, bits->get_mask ());
6829 433 : fprintf (dump_file, "\n");
6830 : }
6831 : }
6832 132707 : }
6833 :
6834 : /* The IPCP driver. */
6835 :
6836 : static unsigned int
6837 132707 : ipcp_driver (void)
6838 : {
6839 132707 : class ipa_topo_info topo;
6840 :
6841 132707 : if (edge_clone_summaries == NULL)
6842 132707 : edge_clone_summaries = new edge_clone_summary_t (symtab);
6843 :
6844 132707 : ipa_check_create_node_params ();
6845 132707 : ipa_check_create_edge_args ();
6846 132707 : callback_info_sum_t::check_create_info_sum ();
6847 132707 : clone_num_suffixes = new hash_map<const char *, unsigned>;
6848 :
6849 132707 : if (dump_file)
6850 : {
6851 175 : fprintf (dump_file, "\nIPA structures before propagation:\n");
6852 175 : if (dump_flags & TDF_DETAILS)
6853 61 : ipa_print_all_params (dump_file);
6854 175 : ipa_print_all_jump_functions (dump_file);
6855 : }
6856 :
6857 : /* Topological sort. */
6858 132707 : build_toporder_info (&topo);
6859 : /* Do the interprocedural propagation. */
6860 132707 : ipcp_propagate_stage (&topo);
6861 : /* Decide what constant propagation and cloning should be performed. */
6862 132707 : ipcp_decision_stage (&topo);
6863 : /* Store results of value range and bits propagation. */
6864 132707 : ipcp_store_vr_results ();
6865 :
6866 : /* Free all IPCP structures. */
6867 265414 : delete clone_num_suffixes;
6868 132707 : free_toporder_info (&topo);
6869 132707 : delete edge_clone_summaries;
6870 132707 : edge_clone_summaries = NULL;
6871 132707 : ipa_free_all_structures_after_ipa_cp ();
6872 132707 : if (dump_file)
6873 175 : fprintf (dump_file, "\nIPA constant propagation end\n");
6874 132707 : return 0;
6875 : }
6876 :
6877 : /* Initialization and computation of IPCP data structures. This is the initial
6878 : intraprocedural analysis of functions, which gathers information to be
6879 : propagated later on. */
6880 :
6881 : static void
6882 129334 : ipcp_generate_summary (void)
6883 : {
6884 129334 : struct cgraph_node *node;
6885 :
6886 129334 : if (dump_file)
6887 177 : fprintf (dump_file, "\nIPA constant propagation start:\n");
6888 129334 : ipa_register_cgraph_hooks ();
6889 :
6890 1414445 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6891 1285111 : ipa_analyze_node (node);
6892 :
6893 129334 : varpool_node *vnode;
6894 1813876 : FOR_EACH_STATIC_INITIALIZER (vnode)
6895 1684542 : ipa_analyze_var_static_initializer (vnode);
6896 129334 : }
6897 :
6898 : namespace {
6899 :
6900 : const pass_data pass_data_ipa_cp =
6901 : {
6902 : IPA_PASS, /* type */
6903 : "cp", /* name */
6904 : OPTGROUP_NONE, /* optinfo_flags */
6905 : TV_IPA_CONSTANT_PROP, /* tv_id */
6906 : 0, /* properties_required */
6907 : 0, /* properties_provided */
6908 : 0, /* properties_destroyed */
6909 : 0, /* todo_flags_start */
6910 : ( TODO_dump_symtab | TODO_remove_functions ), /* todo_flags_finish */
6911 : };
6912 :
6913 : class pass_ipa_cp : public ipa_opt_pass_d
6914 : {
6915 : public:
6916 297180 : pass_ipa_cp (gcc::context *ctxt)
6917 : : ipa_opt_pass_d (pass_data_ipa_cp, ctxt,
6918 : ipcp_generate_summary, /* generate_summary */
6919 : NULL, /* write_summary */
6920 : NULL, /* read_summary */
6921 : ipcp_write_transformation_summaries, /*
6922 : write_optimization_summary */
6923 : ipcp_read_transformation_summaries, /*
6924 : read_optimization_summary */
6925 : NULL, /* stmt_fixup */
6926 : 0, /* function_transform_todo_flags_start */
6927 : ipcp_transform_function, /* function_transform */
6928 297180 : NULL) /* variable_transform */
6929 297180 : {}
6930 :
6931 : /* opt_pass methods: */
6932 593962 : bool gate (function *) final override
6933 : {
6934 : /* FIXME: We should remove the optimize check after we ensure we never run
6935 : IPA passes when not optimizing. */
6936 593962 : return (flag_ipa_cp && optimize) || in_lto_p;
6937 : }
6938 :
6939 132707 : unsigned int execute (function *) final override { return ipcp_driver (); }
6940 :
6941 : }; // class pass_ipa_cp
6942 :
6943 : } // anon namespace
6944 :
6945 : ipa_opt_pass_d *
6946 297180 : make_pass_ipa_cp (gcc::context *ctxt)
6947 : {
6948 297180 : return new pass_ipa_cp (ctxt);
6949 : }
6950 :
6951 : /* Reset all state within ipa-cp.cc so that we can rerun the compiler
6952 : within the same process. For use by toplev::finalize. */
6953 :
6954 : void
6955 266934 : ipa_cp_cc_finalize (void)
6956 : {
6957 266934 : overall_size = 0;
6958 266934 : orig_overall_size = 0;
6959 266934 : overall_scaled_runtime = 0;
6960 266934 : ipcp_free_transformation_sum ();
6961 266934 : }
6962 :
6963 : /* Given PARAM which must be a parameter of function FNDECL described by THIS,
6964 : return its index in the DECL_ARGUMENTS chain, using a pre-computed
6965 : DECL_UID-sorted vector if available (which is pre-computed only if there are
6966 : many parameters). Can return -1 if param is static chain not represented
6967 : among DECL_ARGUMENTS. */
6968 :
6969 : int
6970 127248 : ipcp_transformation::get_param_index (const_tree fndecl, const_tree param) const
6971 : {
6972 127248 : gcc_assert (TREE_CODE (param) == PARM_DECL);
6973 127248 : if (m_uid_to_idx)
6974 : {
6975 0 : unsigned puid = DECL_UID (param);
6976 0 : const ipa_uid_to_idx_map_elt *res
6977 0 : = std::lower_bound (m_uid_to_idx->begin(), m_uid_to_idx->end (), puid,
6978 0 : [] (const ipa_uid_to_idx_map_elt &elt, unsigned uid)
6979 : {
6980 0 : return elt.uid < uid;
6981 : });
6982 0 : if (res == m_uid_to_idx->end ()
6983 0 : || res->uid != puid)
6984 : {
6985 0 : gcc_assert (DECL_STATIC_CHAIN (fndecl));
6986 : return -1;
6987 : }
6988 0 : return res->index;
6989 : }
6990 :
6991 127248 : unsigned index = 0;
6992 290847 : for (tree p = DECL_ARGUMENTS (fndecl); p; p = DECL_CHAIN (p), index++)
6993 289354 : if (p == param)
6994 125755 : return (int) index;
6995 :
6996 1493 : gcc_assert (DECL_STATIC_CHAIN (fndecl));
6997 : return -1;
6998 : }
6999 :
7000 : /* Helper function to qsort a vector of ipa_uid_to_idx_map_elt elements
7001 : according to the uid. */
7002 :
7003 : static int
7004 0 : compare_uids (const void *a, const void *b)
7005 : {
7006 0 : const ipa_uid_to_idx_map_elt *e1 = (const ipa_uid_to_idx_map_elt *) a;
7007 0 : const ipa_uid_to_idx_map_elt *e2 = (const ipa_uid_to_idx_map_elt *) b;
7008 0 : if (e1->uid < e2->uid)
7009 : return -1;
7010 0 : if (e1->uid > e2->uid)
7011 : return 1;
7012 0 : gcc_unreachable ();
7013 : }
7014 :
7015 : /* Assuming THIS describes FNDECL and it has sufficiently many parameters to
7016 : justify the overhead, create a DECL_UID-sorted vector to speed up mapping
7017 : from parameters to their indices in DECL_ARGUMENTS chain. */
7018 :
7019 : void
7020 23119 : ipcp_transformation::maybe_create_parm_idx_map (tree fndecl)
7021 : {
7022 23119 : int c = count_formal_params (fndecl);
7023 23119 : if (c < 32)
7024 : return;
7025 :
7026 0 : m_uid_to_idx = NULL;
7027 0 : vec_safe_reserve (m_uid_to_idx, c, true);
7028 0 : unsigned index = 0;
7029 0 : for (tree p = DECL_ARGUMENTS (fndecl); p; p = DECL_CHAIN (p), index++)
7030 : {
7031 0 : ipa_uid_to_idx_map_elt elt;
7032 0 : elt.uid = DECL_UID (p);
7033 0 : elt.index = index;
7034 0 : m_uid_to_idx->quick_push (elt);
7035 : }
7036 0 : m_uid_to_idx->qsort (compare_uids);
7037 : }
|