Line data Source code
1 : /* Interprocedural constant propagation
2 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
3 :
4 : Contributed by Razya Ladelsky <RAZYA@il.ibm.com> and Martin Jambor
5 : <mjambor@suse.cz>
6 :
7 : This file is part of GCC.
8 :
9 : GCC is free software; you can redistribute it and/or modify it under
10 : the terms of the GNU General Public License as published by the Free
11 : Software Foundation; either version 3, or (at your option) any later
12 : version.
13 :
14 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 : for more details.
18 :
19 : You should have received a copy of the GNU General Public License
20 : along with GCC; see the file COPYING3. If not see
21 : <http://www.gnu.org/licenses/>. */
22 :
23 : /* Interprocedural constant propagation (IPA-CP).
24 :
25 : The goal of this transformation is to
26 :
27 : 1) discover functions which are always invoked with some arguments with the
28 : same known constant values and modify the functions so that the
29 : subsequent optimizations can take advantage of the knowledge, and
30 :
31 : 2) partial specialization - create specialized versions of functions
32 : transformed in this way if some parameters are known constants only in
33 : certain contexts but the estimated tradeoff between speedup and cost size
34 : is deemed good.
35 :
36 : The algorithm also propagates types and attempts to perform type based
37 : devirtualization. Types are propagated much like constants.
38 :
39 : The algorithm basically consists of three stages. In the first, functions
40 : are analyzed one at a time and jump functions are constructed for all known
41 : call-sites. In the second phase, the pass propagates information from the
42 : jump functions across the call to reveal what values are available at what
43 : call sites, performs estimations of effects of known values on functions and
44 : their callees, and finally decides what specialized extra versions should be
45 : created. In the third, the special versions materialize and appropriate
46 : calls are redirected.
47 :
48 : The algorithm used is to a certain extent based on "Interprocedural Constant
49 : Propagation", by David Callahan, Keith D Cooper, Ken Kennedy, Linda Torczon,
50 : Comp86, pg 152-161 and "A Methodology for Procedure Cloning" by Keith D
51 : Cooper, Mary W. Hall, and Ken Kennedy.
52 :
53 :
54 : First stage - intraprocedural analysis
55 : =======================================
56 :
57 : This phase computes jump_function and modification flags.
58 :
59 : A jump function for a call-site represents the values passed as an actual
60 : arguments of a given call-site. In principle, there are three types of
61 : values:
62 :
63 : Pass through - the caller's formal parameter is passed as an actual
64 : argument, plus an operation on it can be performed.
65 : Constant - a constant is passed as an actual argument.
66 : Unknown - neither of the above.
67 :
68 : All jump function types are described in detail in ipa-prop.h, together with
69 : the data structures that represent them and methods of accessing them.
70 :
71 : ipcp_generate_summary() is the main function of the first stage.
72 :
73 : Second stage - interprocedural analysis
74 : ========================================
75 :
76 : This stage is itself divided into two phases. In the first, we propagate
77 : known values over the call graph, in the second, we make cloning decisions.
78 : It uses a different algorithm than the original Callahan's paper.
79 :
80 : First, we traverse the functions topologically from callers to callees and,
81 : for each strongly connected component (SCC), we propagate constants
82 : according to previously computed jump functions. We also record what known
83 : values depend on other known values and estimate local effects. Finally, we
84 : propagate cumulative information about these effects from dependent values
85 : to those on which they depend.
86 :
87 : Second, we again traverse the call graph in the same topological order and
88 : make clones for functions which we know are called with the same values in
89 : all contexts and decide about extra specialized clones of functions just for
90 : some contexts - these decisions are based on both local estimates and
91 : cumulative estimates propagated from callees.
92 :
93 : ipcp_propagate_stage() and ipcp_decision_stage() together constitute the
94 : third stage.
95 :
96 : Third phase - materialization of clones, call statement updates.
97 : ============================================
98 :
99 : This stage is currently performed by call graph code (mainly in cgraphunit.cc
100 : and tree-inline.cc) according to instructions inserted to the call graph by
101 : the second stage. */
102 :
103 : #define INCLUDE_ALGORITHM
104 : #include "config.h"
105 : #include "system.h"
106 : #include "coretypes.h"
107 : #include "backend.h"
108 : #include "tree.h"
109 : #include "gimple-expr.h"
110 : #include "gimple.h"
111 : #include "predict.h"
112 : #include "sreal.h"
113 : #include "alloc-pool.h"
114 : #include "tree-pass.h"
115 : #include "cgraph.h"
116 : #include "diagnostic.h"
117 : #include "fold-const.h"
118 : #include "gimple-iterator.h"
119 : #include "gimple-fold.h"
120 : #include "symbol-summary.h"
121 : #include "tree-vrp.h"
122 : #include "ipa-cp.h"
123 : #include "ipa-prop.h"
124 : #include "tree-pretty-print.h"
125 : #include "tree-inline.h"
126 : #include "ipa-fnsummary.h"
127 : #include "ipa-utils.h"
128 : #include "tree-ssa-ccp.h"
129 : #include "stringpool.h"
130 : #include "attribs.h"
131 : #include "dbgcnt.h"
132 : #include "symtab-clones.h"
133 : #include "gimple-range.h"
134 : #include "attr-callback.h"
135 :
136 : /* Allocation pools for values and their sources in ipa-cp. */
137 :
138 : object_allocator<ipcp_value<tree> > ipcp_cst_values_pool
139 : ("IPA-CP constant values");
140 :
141 : object_allocator<ipcp_value<ipa_polymorphic_call_context> >
142 : ipcp_poly_ctx_values_pool ("IPA-CP polymorphic contexts");
143 :
144 : object_allocator<ipcp_value_source<tree> > ipcp_sources_pool
145 : ("IPA-CP value sources");
146 :
147 : object_allocator<ipcp_agg_lattice> ipcp_agg_lattice_pool
148 : ("IPA_CP aggregate lattices");
149 :
150 : /* Original overall size of the program. */
151 :
152 : static long overall_size, orig_overall_size;
153 :
154 : /* The maximum number of IPA-CP decision sweeps that any node requested in its
155 : param. */
156 : static int max_number_sweeps;
157 :
158 : /* Node name to unique clone suffix number map. */
159 : static hash_map<const char *, unsigned> *clone_num_suffixes;
160 :
161 : /* Return the param lattices structure corresponding to the Ith formal
162 : parameter of the function described by INFO. */
163 : static inline class ipcp_param_lattices *
164 31838768 : ipa_get_parm_lattices (class ipa_node_params *info, int i)
165 : {
166 63677536 : gcc_assert (i >= 0 && i < ipa_get_param_count (info));
167 31838768 : gcc_checking_assert (!info->ipcp_orig_node);
168 31838768 : return &(info->lattices[i]);
169 : }
170 :
171 : /* Return the lattice corresponding to the scalar value of the Ith formal
172 : parameter of the function described by INFO. */
173 : static inline ipcp_lattice<tree> *
174 6067289 : ipa_get_scalar_lat (class ipa_node_params *info, int i)
175 : {
176 6255075 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
177 6067289 : return &plats->itself;
178 : }
179 :
180 : /* Return the lattice corresponding to the scalar value of the Ith formal
181 : parameter of the function described by INFO. */
182 : static inline ipcp_lattice<ipa_polymorphic_call_context> *
183 807778 : ipa_get_poly_ctx_lat (class ipa_node_params *info, int i)
184 : {
185 807778 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
186 807778 : return &plats->ctxlat;
187 : }
188 :
189 : /* Return whether LAT is a lattice with a single constant and without an
190 : undefined value. */
191 :
192 : template <typename valtype>
193 : inline bool
194 10295843 : ipcp_lattice<valtype>::is_single_const ()
195 : {
196 2982010 : if (bottom || contains_variable || values_count != 1)
197 : return false;
198 : else
199 : return true;
200 : }
201 :
202 : /* Return true iff X and Y should be considered equal values by IPA-CP. */
203 :
204 : bool
205 1415738 : values_equal_for_ipcp_p (tree x, tree y)
206 : {
207 1415738 : gcc_checking_assert (x != NULL_TREE && y != NULL_TREE);
208 :
209 1415738 : if (x == y)
210 : return true;
211 :
212 622650 : if (TREE_CODE (x) == ADDR_EXPR
213 220068 : && TREE_CODE (y) == ADDR_EXPR
214 219436 : && (TREE_CODE (TREE_OPERAND (x, 0)) == CONST_DECL
215 171987 : || (TREE_CODE (TREE_OPERAND (x, 0)) == VAR_DECL
216 92274 : && DECL_IN_CONSTANT_POOL (TREE_OPERAND (x, 0))))
217 670099 : && (TREE_CODE (TREE_OPERAND (y, 0)) == CONST_DECL
218 13 : || (TREE_CODE (TREE_OPERAND (y, 0)) == VAR_DECL
219 8 : && DECL_IN_CONSTANT_POOL (TREE_OPERAND (y, 0)))))
220 47436 : return TREE_OPERAND (x, 0) == TREE_OPERAND (y, 0)
221 94616 : || operand_equal_p (DECL_INITIAL (TREE_OPERAND (x, 0)),
222 47180 : DECL_INITIAL (TREE_OPERAND (y, 0)), 0);
223 : else
224 575214 : return operand_equal_p (x, y, 0);
225 : }
226 :
227 : /* Print V which is extracted from a value in a lattice to F. This overloaded
228 : function is used to print tree constants. */
229 :
230 : static void
231 840 : print_ipcp_constant_value (FILE * f, tree v)
232 : {
233 0 : ipa_print_constant_value (f, v);
234 44 : }
235 :
236 : /* Print V which is extracted from a value in a lattice to F. This overloaded
237 : function is used to print constant polymorphic call contexts. */
238 :
239 : static void
240 214 : print_ipcp_constant_value (FILE * f, ipa_polymorphic_call_context v)
241 : {
242 214 : v.dump(f, false);
243 0 : }
244 :
245 : /* Print a lattice LAT to F. */
246 :
247 : template <typename valtype>
248 : void
249 1986 : ipcp_lattice<valtype>::print (FILE * f, bool dump_sources, bool dump_benefits)
250 : {
251 : ipcp_value<valtype> *val;
252 1986 : bool prev = false;
253 :
254 1986 : if (bottom)
255 : {
256 842 : fprintf (f, "BOTTOM\n");
257 842 : return;
258 : }
259 :
260 1144 : if (!values_count && !contains_variable)
261 : {
262 0 : fprintf (f, "TOP\n");
263 0 : return;
264 : }
265 :
266 1144 : if (contains_variable)
267 : {
268 864 : fprintf (f, "VARIABLE");
269 864 : prev = true;
270 864 : if (dump_benefits)
271 864 : fprintf (f, "\n");
272 : }
273 :
274 1774 : for (val = values; val; val = val->next)
275 : {
276 630 : if (dump_benefits && prev)
277 350 : fprintf (f, " ");
278 280 : else if (!dump_benefits && prev)
279 0 : fprintf (f, ", ");
280 : else
281 : prev = true;
282 :
283 630 : print_ipcp_constant_value (f, val->value);
284 :
285 630 : if (dump_sources)
286 : {
287 : ipcp_value_source<valtype> *s;
288 :
289 175 : if (val->self_recursion_generated_p ())
290 27 : fprintf (f, " [self_gen(%i), from:",
291 : val->self_recursion_generated_level);
292 : else
293 148 : fprintf (f, " [scc: %i, from:", val->scc_no);
294 368 : for (s = val->sources; s; s = s->next)
295 193 : fprintf (f, " %i(%f)", s->cs->caller->get_uid (),
296 386 : s->cs->sreal_frequency ().to_double ());
297 175 : fprintf (f, "]");
298 : }
299 :
300 630 : if (dump_benefits)
301 630 : fprintf (f, " [loc_time: %g, loc_size: %i, "
302 : "prop_time: %g, prop_size: %i]\n",
303 : val->local_time_benefit.to_double (), val->local_size_cost,
304 : val->prop_time_benefit.to_double (), val->prop_size_cost);
305 : }
306 1144 : if (!dump_benefits)
307 0 : fprintf (f, "\n");
308 : }
309 :
310 : /* Print VALUE to F in a form which in usual cases does not take thousands of
311 : characters. */
312 :
313 : static void
314 1448 : ipcp_print_widest_int (FILE *f, const widest_int &value)
315 : {
316 1448 : if (value == -1)
317 0 : fprintf (f, "-1");
318 1448 : else if (wi::arshift (value, 128) == -1)
319 : {
320 330 : char buf[35], *p = buf + 2;
321 330 : widest_int v = wi::zext (value, 128);
322 330 : size_t len;
323 330 : print_hex (v, buf);
324 330 : len = strlen (p);
325 330 : if (len == 32)
326 : {
327 330 : fprintf (f, "0xf..f");
328 9795 : while (*p == 'f')
329 9135 : ++p;
330 : }
331 : else
332 0 : fprintf (f, "0xf..f%0*d", (int) (32 - len), 0);
333 330 : fputs (p, f);
334 330 : }
335 : else
336 1118 : print_hex (value, f);
337 1448 : }
338 :
339 : void
340 917 : ipcp_bits_lattice::print (FILE *f)
341 : {
342 917 : if (bottom_p ())
343 : {
344 606 : fprintf (f, " Bits unusable (BOTTOM)\n");
345 606 : return;
346 : }
347 :
348 311 : if (top_p ())
349 0 : fprintf (f, " Bits unknown (TOP)");
350 : else
351 : {
352 311 : fprintf (f, " Bits: value = ");
353 311 : ipcp_print_widest_int (f, get_value ());
354 311 : fprintf (f, ", mask = ");
355 311 : ipcp_print_widest_int (f, get_mask ());
356 : }
357 :
358 311 : if (m_recipient_only)
359 137 : fprintf (f, " (recipient only)");
360 311 : fprintf (f, "\n");
361 : }
362 :
363 : /* Print value range lattice to F. */
364 :
365 : void
366 917 : ipcp_vr_lattice::print (FILE * f)
367 : {
368 917 : if (m_recipient_only)
369 264 : fprintf (f, "(recipient only) ");
370 917 : m_vr.dump (f);
371 917 : }
372 :
373 : /* Print all ipcp_lattices of all functions to F. */
374 :
375 : static void
376 161 : print_all_lattices (FILE * f, bool dump_sources, bool dump_benefits)
377 : {
378 161 : struct cgraph_node *node;
379 161 : int i, count;
380 :
381 161 : fprintf (f, "\nLattices:\n");
382 886 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
383 : {
384 725 : class ipa_node_params *info;
385 :
386 725 : info = ipa_node_params_sum->get (node);
387 : /* Skip unoptimized functions and constprop clones since we don't make
388 : lattices for them. */
389 725 : if (!info || info->ipcp_orig_node)
390 0 : continue;
391 725 : fprintf (f, " Node: %s:\n", node->dump_name ());
392 725 : count = ipa_get_param_count (info);
393 1642 : for (i = 0; i < count; i++)
394 : {
395 917 : struct ipcp_agg_lattice *aglat;
396 917 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
397 917 : fprintf (f, " param [%d]: ", i);
398 917 : plats->itself.print (f, dump_sources, dump_benefits);
399 917 : fprintf (f, " ctxs: ");
400 917 : plats->ctxlat.print (f, dump_sources, dump_benefits);
401 917 : plats->bits_lattice.print (f);
402 917 : fprintf (f, " ");
403 917 : plats->m_value_range.print (f);
404 917 : fprintf (f, "\n");
405 917 : if (plats->virt_call)
406 75 : fprintf (f, " virt_call flag set\n");
407 :
408 917 : if (plats->aggs_bottom)
409 : {
410 441 : fprintf (f, " AGGS BOTTOM\n");
411 441 : continue;
412 : }
413 476 : if (plats->aggs_contain_variable)
414 438 : fprintf (f, " AGGS VARIABLE\n");
415 628 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
416 : {
417 152 : fprintf (f, " %soffset " HOST_WIDE_INT_PRINT_DEC ": ",
418 152 : plats->aggs_by_ref ? "ref " : "", aglat->offset);
419 152 : aglat->print (f, dump_sources, dump_benefits);
420 : }
421 : }
422 : }
423 161 : }
424 :
425 : /* Determine whether it is at all technically possible to create clones of NODE
426 : and store this information in the ipa_node_params structure associated
427 : with NODE. */
428 :
429 : static void
430 1293896 : determine_versionability (struct cgraph_node *node,
431 : class ipa_node_params *info)
432 : {
433 1293896 : const char *reason = NULL;
434 :
435 : /* There are a number of generic reasons functions cannot be versioned. We
436 : also cannot remove parameters if there are type attributes such as fnspec
437 : present. */
438 1293896 : if (node->alias || node->thunk)
439 : reason = "alias or thunk";
440 1293896 : else if (!node->versionable)
441 : reason = "not a tree_versionable_function";
442 1164105 : else if (node->get_availability () <= AVAIL_INTERPOSABLE)
443 : reason = "insufficient body availability";
444 1096059 : else if (!opt_for_fn (node->decl, optimize)
445 1096059 : || !opt_for_fn (node->decl, flag_ipa_cp))
446 : reason = "non-optimized function";
447 1096059 : else if (lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (node->decl)))
448 : {
449 : /* Ideally we should clone the SIMD clones themselves and create
450 : vector copies of them, so IPA-cp and SIMD clones can happily
451 : coexist, but that may not be worth the effort. */
452 : reason = "function has SIMD clones";
453 : }
454 1095699 : else if (lookup_attribute ("target_clones", DECL_ATTRIBUTES (node->decl)))
455 : {
456 : /* Ideally we should clone the target clones themselves and create
457 : copies of them, so IPA-cp and target clones can happily
458 : coexist, but that may not be worth the effort. */
459 : reason = "function target_clones attribute";
460 : }
461 : /* Don't clone decls local to a comdat group; it breaks and for C++
462 : decloned constructors, inlining is always better anyway. */
463 1095699 : else if (node->comdat_local_p ())
464 : reason = "comdat-local function";
465 1093439 : else if (node->calls_comdat_local)
466 : {
467 : /* TODO: call is versionable if we make sure that all
468 : callers are inside of a comdat group. */
469 2362 : reason = "calls comdat-local function";
470 : }
471 :
472 : /* Functions calling BUILT_IN_VA_ARG_PACK and BUILT_IN_VA_ARG_PACK_LEN
473 : work only when inlined. Cloning them may still lead to better code
474 : because ipa-cp will not give up on cloning further. If the function is
475 : external this however leads to wrong code because we may end up producing
476 : offline copy of the function. */
477 1293896 : if (DECL_EXTERNAL (node->decl))
478 179911 : for (cgraph_edge *edge = node->callees; !reason && edge;
479 132209 : edge = edge->next_callee)
480 132209 : if (fndecl_built_in_p (edge->callee->decl, BUILT_IN_NORMAL))
481 : {
482 37232 : if (DECL_FUNCTION_CODE (edge->callee->decl) == BUILT_IN_VA_ARG_PACK)
483 0 : reason = "external function which calls va_arg_pack";
484 37232 : if (DECL_FUNCTION_CODE (edge->callee->decl)
485 : == BUILT_IN_VA_ARG_PACK_LEN)
486 0 : reason = "external function which calls va_arg_pack_len";
487 : }
488 :
489 1293896 : if (reason && dump_file && !node->alias && !node->thunk)
490 53 : fprintf (dump_file, "Function %s is not versionable, reason: %s.\n",
491 : node->dump_name (), reason);
492 :
493 1293896 : info->versionable = (reason == NULL);
494 1293896 : }
495 :
496 : /* Return true if it is at all technically possible to create clones of a
497 : NODE. */
498 :
499 : static bool
500 6195070 : ipcp_versionable_function_p (struct cgraph_node *node)
501 : {
502 6195070 : ipa_node_params *info = ipa_node_params_sum->get (node);
503 6195070 : return info && info->versionable;
504 : }
505 :
506 : /* Structure holding accumulated information about callers of a node. */
507 :
508 1048173 : struct caller_statistics
509 : {
510 : /* If requested (see below), self-recursive call counts are summed into this
511 : field. */
512 : profile_count rec_count_sum;
513 : /* The sum of all ipa counts of all the other (non-recursive) calls. */
514 : profile_count count_sum;
515 : /* Sum of all frequencies for all calls. */
516 : sreal freq_sum;
517 : /* Number of calls and calls considered interesting respectively. */
518 : int n_calls, n_interesting_calls;
519 : /* If itself is set up, also count the number of non-self-recursive
520 : calls. */
521 : int n_nonrec_calls;
522 : /* If non-NULL, this is the node itself and calls from it should have their
523 : counts included in rec_count_sum and not count_sum. */
524 : cgraph_node *itself;
525 : /* True if there is a caller that has no IPA profile. */
526 : bool called_without_ipa_profile;
527 : };
528 :
529 : /* Initialize fields of STAT to zeroes and optionally set it up so that edges
530 : from IGNORED_CALLER are not counted. */
531 :
532 : static inline void
533 285427 : init_caller_stats (caller_statistics *stats, cgraph_node *itself = NULL)
534 : {
535 285427 : stats->rec_count_sum = profile_count::zero ();
536 285427 : stats->count_sum = profile_count::zero ();
537 285427 : stats->n_calls = 0;
538 285427 : stats->n_interesting_calls = 0;
539 285427 : stats->n_nonrec_calls = 0;
540 285427 : stats->freq_sum = 0;
541 285427 : stats->itself = itself;
542 285427 : stats->called_without_ipa_profile = false;
543 285427 : }
544 :
545 : /* We want to propagate across edges that may be executed, however
546 : we do not want to check maybe_hot, since call itself may be cold
547 : while calee contains some heavy loop which makes propagation still
548 : relevant.
549 :
550 : In particular, even edge called once may lead to significant
551 : improvement. */
552 :
553 : static bool
554 973026 : cs_interesting_for_ipcp_p (cgraph_edge *e)
555 : {
556 : /* If profile says the edge is executed, we want to optimize. */
557 973026 : if (e->count.ipa ().nonzero_p ())
558 532 : return true;
559 : /* If local (possibly guseed or adjusted 0 profile) claims edge is
560 : not executed, do not propagate.
561 : Do not trust AFDO since branch needs to be executed multiple
562 : time to count while we want to propagate even call called
563 : once during the train run if callee is important. */
564 972494 : if (e->count.initialized_p () && !e->count.nonzero_p ()
565 1021902 : && e->count.quality () != AFDO)
566 : return false;
567 : /* If we have zero IPA profile, still consider edge for cloning
568 : in case we do partial training. */
569 923086 : if (e->count.ipa ().initialized_p ()
570 923086 : && e->count.ipa ().quality () != AFDO
571 923091 : && !opt_for_fn (e->callee->decl,flag_profile_partial_training))
572 5 : return false;
573 : return true;
574 : }
575 :
576 : /* Worker callback of cgraph_for_node_and_aliases accumulating statistics of
577 : non-thunk incoming edges to NODE. */
578 :
579 : static bool
580 293081 : gather_caller_stats (struct cgraph_node *node, void *data)
581 : {
582 293081 : struct caller_statistics *stats = (struct caller_statistics *) data;
583 293081 : struct cgraph_edge *cs;
584 :
585 1039127 : for (cs = node->callers; cs; cs = cs->next_caller)
586 746046 : if (!cs->caller->thunk)
587 : {
588 745873 : ipa_node_params *info = ipa_node_params_sum->get (cs->caller);
589 745873 : if (info && info->node_dead)
590 150102 : continue;
591 :
592 595771 : if (cs->count.ipa ().initialized_p ())
593 : {
594 3568 : if (stats->itself && stats->itself == cs->caller)
595 0 : stats->rec_count_sum += cs->count.ipa ();
596 : else
597 3568 : stats->count_sum += cs->count.ipa ();
598 : }
599 : else
600 592203 : stats->called_without_ipa_profile = true;
601 595771 : stats->freq_sum += cs->sreal_frequency ();
602 595771 : stats->n_calls++;
603 595771 : if (stats->itself && stats->itself != cs->caller)
604 8 : stats->n_nonrec_calls++;
605 :
606 : /* If profile known to be zero, we do not want to clone for performance.
607 : However if call is cold, the called function may still contain
608 : important hot loops. */
609 595771 : if (cs_interesting_for_ipcp_p (cs))
610 591769 : stats->n_interesting_calls++;
611 : }
612 293081 : return false;
613 :
614 : }
615 :
616 : /* Return true if this NODE is viable candidate for cloning. */
617 :
618 : static bool
619 810023 : ipcp_cloning_candidate_p (struct cgraph_node *node)
620 : {
621 810023 : struct caller_statistics stats;
622 :
623 810023 : gcc_checking_assert (node->has_gimple_body_p ());
624 :
625 810023 : if (!opt_for_fn (node->decl, flag_ipa_cp_clone))
626 : {
627 758529 : if (dump_file)
628 31 : fprintf (dump_file, "Not considering %s for cloning; "
629 : "-fipa-cp-clone disabled.\n",
630 : node->dump_name ());
631 758529 : return false;
632 : }
633 :
634 : /* Do not use profile here since cold wrapper wrap
635 : hot function. */
636 51494 : if (opt_for_fn (node->decl, optimize_size))
637 : {
638 10 : if (dump_file)
639 0 : fprintf (dump_file, "Not considering %s for cloning; "
640 : "optimizing it for size.\n",
641 : node->dump_name ());
642 10 : return false;
643 : }
644 :
645 51484 : init_caller_stats (&stats);
646 51484 : node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats, false);
647 :
648 51484 : if (ipa_size_summaries->get (node)->self_size < stats.n_calls)
649 : {
650 303 : if (dump_file)
651 0 : fprintf (dump_file, "Considering %s for cloning; code might shrink.\n",
652 : node->dump_name ());
653 303 : return true;
654 : }
655 51181 : if (!stats.n_interesting_calls)
656 : {
657 39292 : if (dump_file)
658 200 : fprintf (dump_file, "Not considering %s for cloning; "
659 : "no calls considered interesting by profile.\n",
660 : node->dump_name ());
661 39292 : return false;
662 : }
663 11889 : if (dump_file)
664 187 : fprintf (dump_file, "Considering %s for cloning.\n",
665 : node->dump_name ());
666 : return true;
667 : }
668 :
669 : template <typename valtype>
670 : class value_topo_info
671 : {
672 : public:
673 : /* Head of the linked list of topologically sorted values. */
674 : ipcp_value<valtype> *values_topo;
675 : /* Stack for creating SCCs, represented by a linked list too. */
676 : ipcp_value<valtype> *stack;
677 : /* Counter driving the algorithm in add_val_to_toposort. */
678 : int dfs_counter;
679 :
680 129922 : value_topo_info () : values_topo (NULL), stack (NULL), dfs_counter (0)
681 : {}
682 : void add_val (ipcp_value<valtype> *cur_val);
683 : void propagate_effects ();
684 : };
685 :
686 : /* Arrays representing a topological ordering of call graph nodes and a stack
687 : of nodes used during constant propagation and also data required to perform
688 : topological sort of values and propagation of benefits in the determined
689 : order. */
690 :
691 : class ipa_topo_info
692 : {
693 : public:
694 : /* Array with obtained topological order of cgraph nodes. */
695 : struct cgraph_node **order;
696 : /* Stack of cgraph nodes used during propagation within SCC until all values
697 : in the SCC stabilize. */
698 : struct cgraph_node **stack;
699 : int nnodes, stack_top;
700 :
701 : value_topo_info<tree> constants;
702 : value_topo_info<ipa_polymorphic_call_context> contexts;
703 :
704 129922 : ipa_topo_info () : order(NULL), stack(NULL), nnodes(0), stack_top(0),
705 129922 : constants ()
706 : {}
707 : };
708 :
709 : /* Skip edges from and to nodes without ipa_cp enabled.
710 : Ignore not available symbols. */
711 :
712 : static bool
713 5377169 : ignore_edge_p (cgraph_edge *e)
714 : {
715 5377169 : enum availability avail;
716 5377169 : cgraph_node *ultimate_target
717 5377169 : = e->callee->function_or_virtual_thunk_symbol (&avail, e->caller);
718 :
719 5377169 : return (avail <= AVAIL_INTERPOSABLE
720 1923533 : || !opt_for_fn (ultimate_target->decl, optimize)
721 7291958 : || !opt_for_fn (ultimate_target->decl, flag_ipa_cp));
722 : }
723 :
724 : /* Allocate the arrays in TOPO and topologically sort the nodes into order. */
725 :
726 : static void
727 129922 : build_toporder_info (class ipa_topo_info *topo)
728 : {
729 129922 : topo->order = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
730 129922 : topo->stack = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
731 :
732 129922 : gcc_checking_assert (topo->stack_top == 0);
733 129922 : topo->nnodes = ipa_reduced_postorder (topo->order, true,
734 : ignore_edge_p);
735 129922 : }
736 :
737 : /* Free information about strongly connected components and the arrays in
738 : TOPO. */
739 :
740 : static void
741 129922 : free_toporder_info (class ipa_topo_info *topo)
742 : {
743 129922 : ipa_free_postorder_info ();
744 129922 : free (topo->order);
745 129922 : free (topo->stack);
746 129922 : }
747 :
748 : /* Add NODE to the stack in TOPO, unless it is already there. */
749 :
750 : static inline void
751 1298062 : push_node_to_stack (class ipa_topo_info *topo, struct cgraph_node *node)
752 : {
753 1298062 : ipa_node_params *info = ipa_node_params_sum->get (node);
754 1298062 : if (info->node_enqueued)
755 : return;
756 1297093 : info->node_enqueued = 1;
757 1297093 : topo->stack[topo->stack_top++] = node;
758 : }
759 :
760 : /* Pop a node from the stack in TOPO and return it or return NULL if the stack
761 : is empty. */
762 :
763 : static struct cgraph_node *
764 2674433 : pop_node_from_stack (class ipa_topo_info *topo)
765 : {
766 2674433 : if (topo->stack_top)
767 : {
768 1297093 : struct cgraph_node *node;
769 1297093 : topo->stack_top--;
770 1297093 : node = topo->stack[topo->stack_top];
771 1297093 : ipa_node_params_sum->get (node)->node_enqueued = 0;
772 1297093 : return node;
773 : }
774 : else
775 : return NULL;
776 : }
777 :
778 : /* Set lattice LAT to bottom and return true if it previously was not set as
779 : such. */
780 :
781 : template <typename valtype>
782 : inline bool
783 2159664 : ipcp_lattice<valtype>::set_to_bottom ()
784 : {
785 2159664 : bool ret = !bottom;
786 2159664 : bottom = true;
787 : return ret;
788 : }
789 :
790 : /* Mark lattice as containing an unknown value and return true if it previously
791 : was not marked as such. */
792 :
793 : template <typename valtype>
794 : inline bool
795 1556792 : ipcp_lattice<valtype>::set_contains_variable ()
796 : {
797 1556792 : bool ret = !contains_variable;
798 1556792 : contains_variable = true;
799 : return ret;
800 : }
801 :
802 : /* Set all aggregate lattices in PLATS to bottom and return true if they were
803 : not previously set as such. */
804 :
805 : static inline bool
806 2159381 : set_agg_lats_to_bottom (class ipcp_param_lattices *plats)
807 : {
808 2159381 : bool ret = !plats->aggs_bottom;
809 2159381 : plats->aggs_bottom = true;
810 2159381 : return ret;
811 : }
812 :
813 : /* Mark all aggregate lattices in PLATS as containing an unknown value and
814 : return true if they were not previously marked as such. */
815 :
816 : static inline bool
817 1054379 : set_agg_lats_contain_variable (class ipcp_param_lattices *plats)
818 : {
819 1054379 : bool ret = !plats->aggs_contain_variable;
820 1054379 : plats->aggs_contain_variable = true;
821 1054379 : return ret;
822 : }
823 :
824 : bool
825 0 : ipcp_vr_lattice::meet_with (const ipcp_vr_lattice &other)
826 : {
827 0 : return meet_with_1 (other.m_vr);
828 : }
829 :
830 : /* Meet the current value of the lattice with the range described by
831 : P_VR. */
832 :
833 : bool
834 493228 : ipcp_vr_lattice::meet_with (const vrange &p_vr)
835 : {
836 493228 : return meet_with_1 (p_vr);
837 : }
838 :
839 : /* Meet the current value of the lattice with the range described by
840 : OTHER_VR. Return TRUE if anything changed. */
841 :
842 : bool
843 493228 : ipcp_vr_lattice::meet_with_1 (const vrange &other_vr)
844 : {
845 493228 : if (bottom_p ())
846 : return false;
847 :
848 493228 : if (other_vr.varying_p ())
849 0 : return set_to_bottom ();
850 :
851 493228 : bool res;
852 493228 : if (flag_checking)
853 : {
854 493228 : value_range save (m_vr);
855 493228 : res = m_vr.union_ (other_vr);
856 493228 : gcc_assert (res == (m_vr != save));
857 493228 : }
858 : else
859 0 : res = m_vr.union_ (other_vr);
860 : return res;
861 : }
862 :
863 : /* Return true if value range information in the lattice is yet unknown. */
864 :
865 : bool
866 : ipcp_vr_lattice::top_p () const
867 : {
868 168719 : return m_vr.undefined_p ();
869 : }
870 :
871 : /* Return true if value range information in the lattice is known to be
872 : unusable. */
873 :
874 : bool
875 4978396 : ipcp_vr_lattice::bottom_p () const
876 : {
877 493228 : return m_vr.varying_p ();
878 : }
879 :
880 : /* Set value range information in the lattice to bottom. Return true if it
881 : previously was in a different state. */
882 :
883 : bool
884 2439575 : ipcp_vr_lattice::set_to_bottom ()
885 : {
886 2439575 : if (m_vr.varying_p ())
887 : return false;
888 :
889 : /* Setting an unsupported type here forces the temporary to default
890 : to unsupported_range, which can handle VARYING/DEFINED ranges,
891 : but nothing else (union, intersect, etc). This allows us to set
892 : bottoms on any ranges, and is safe as all users of the lattice
893 : check for bottom first. */
894 2294738 : m_vr.set_range_class (void_type_node);
895 2294738 : m_vr.set_varying (void_type_node);
896 :
897 2294738 : return true;
898 : }
899 :
900 : /* Set the flag that this lattice is a recipient only, return true if it was
901 : not set before. */
902 :
903 : bool
904 29349 : ipcp_vr_lattice::set_recipient_only ()
905 : {
906 29349 : if (m_recipient_only)
907 : return false;
908 29349 : m_recipient_only = true;
909 29349 : return true;
910 : }
911 :
912 : /* Set lattice value to bottom, if it already isn't the case. */
913 :
914 : bool
915 2458922 : ipcp_bits_lattice::set_to_bottom ()
916 : {
917 2458922 : if (bottom_p ())
918 : return false;
919 2314613 : m_lattice_val = IPA_BITS_VARYING;
920 2314613 : m_value = 0;
921 2314613 : m_mask = -1;
922 2314613 : return true;
923 : }
924 :
925 : /* Set to constant if it isn't already. Only meant to be called
926 : when switching state from TOP. */
927 :
928 : bool
929 76967 : ipcp_bits_lattice::set_to_constant (widest_int value, widest_int mask)
930 : {
931 76967 : gcc_assert (top_p ());
932 76967 : m_lattice_val = IPA_BITS_CONSTANT;
933 76967 : m_value = wi::bit_and (wi::bit_not (mask), value);
934 76967 : m_mask = mask;
935 76967 : return true;
936 : }
937 :
938 : /* Return true if any of the known bits are non-zero. */
939 :
940 : bool
941 472 : ipcp_bits_lattice::known_nonzero_p () const
942 : {
943 472 : if (!constant_p ())
944 : return false;
945 472 : return wi::ne_p (wi::bit_and (wi::bit_not (m_mask), m_value), 0);
946 : }
947 :
948 : /* Set the flag that this lattice is a recipient only, return true if it was not
949 : set before. */
950 :
951 : bool
952 29349 : ipcp_bits_lattice::set_recipient_only ()
953 : {
954 29349 : if (m_recipient_only)
955 : return false;
956 29349 : m_recipient_only = true;
957 29349 : return true;
958 : }
959 :
960 : /* Convert operand to value, mask form. */
961 :
962 : void
963 2067 : ipcp_bits_lattice::get_value_and_mask (tree operand, widest_int *valuep, widest_int *maskp)
964 : {
965 2067 : wide_int get_nonzero_bits (const_tree);
966 :
967 2067 : if (TREE_CODE (operand) == INTEGER_CST)
968 : {
969 2067 : *valuep = wi::to_widest (operand);
970 2067 : *maskp = 0;
971 : }
972 : else
973 : {
974 0 : *valuep = 0;
975 0 : *maskp = -1;
976 : }
977 2067 : }
978 :
979 : /* Meet operation, similar to ccp_lattice_meet, we xor values
980 : if this->value, value have different values at same bit positions, we want
981 : to drop that bit to varying. Return true if mask is changed.
982 : This function assumes that the lattice value is in CONSTANT state. If
983 : DROP_ALL_ONES, mask out any known bits with value one afterwards. */
984 :
985 : bool
986 302824 : ipcp_bits_lattice::meet_with_1 (widest_int value, widest_int mask,
987 : unsigned precision, bool drop_all_ones)
988 : {
989 302824 : gcc_assert (constant_p ());
990 :
991 302824 : widest_int old_mask = m_mask;
992 302824 : m_mask = (m_mask | mask) | (m_value ^ value);
993 302824 : if (drop_all_ones)
994 199 : m_mask |= m_value;
995 :
996 302824 : widest_int cap_mask = wi::shifted_mask <widest_int> (0, precision, true);
997 302824 : m_mask |= cap_mask;
998 302824 : if (wi::sext (m_mask, precision) == -1)
999 3558 : return set_to_bottom ();
1000 :
1001 299266 : m_value &= ~m_mask;
1002 299266 : return m_mask != old_mask;
1003 302824 : }
1004 :
1005 : /* Meet the bits lattice with operand
1006 : described by <value, mask, sgn, precision. */
1007 :
1008 : bool
1009 413151 : ipcp_bits_lattice::meet_with (widest_int value, widest_int mask,
1010 : unsigned precision)
1011 : {
1012 413151 : if (bottom_p ())
1013 : return false;
1014 :
1015 413151 : if (top_p ())
1016 : {
1017 122972 : if (wi::sext (mask, precision) == -1)
1018 51296 : return set_to_bottom ();
1019 71676 : return set_to_constant (value, mask);
1020 : }
1021 :
1022 290179 : return meet_with_1 (value, mask, precision, false);
1023 : }
1024 :
1025 : /* Meet bits lattice with the result of bit_value_binop (other, operand)
1026 : if code is binary operation or bit_value_unop (other) if code is unary op.
1027 : In the case when code is nop_expr, no adjustment is required. If
1028 : DROP_ALL_ONES, mask out any known bits with value one afterwards. */
1029 :
1030 : bool
1031 21510 : ipcp_bits_lattice::meet_with (ipcp_bits_lattice& other, unsigned precision,
1032 : signop sgn, enum tree_code code, tree operand,
1033 : bool drop_all_ones)
1034 : {
1035 21510 : if (other.bottom_p ())
1036 0 : return set_to_bottom ();
1037 :
1038 21510 : if (bottom_p () || other.top_p ())
1039 : return false;
1040 :
1041 18052 : widest_int adjusted_value, adjusted_mask;
1042 :
1043 18052 : if (TREE_CODE_CLASS (code) == tcc_binary)
1044 : {
1045 2067 : tree type = TREE_TYPE (operand);
1046 2067 : widest_int o_value, o_mask;
1047 2067 : get_value_and_mask (operand, &o_value, &o_mask);
1048 :
1049 2067 : bit_value_binop (code, sgn, precision, &adjusted_value, &adjusted_mask,
1050 4134 : sgn, precision, other.get_value (), other.get_mask (),
1051 2067 : TYPE_SIGN (type), TYPE_PRECISION (type), o_value, o_mask);
1052 :
1053 2067 : if (wi::sext (adjusted_mask, precision) == -1)
1054 87 : return set_to_bottom ();
1055 2067 : }
1056 :
1057 15985 : else if (TREE_CODE_CLASS (code) == tcc_unary)
1058 : {
1059 31920 : bit_value_unop (code, sgn, precision, &adjusted_value,
1060 31920 : &adjusted_mask, sgn, precision, other.get_value (),
1061 15960 : other.get_mask ());
1062 :
1063 15960 : if (wi::sext (adjusted_mask, precision) == -1)
1064 4 : return set_to_bottom ();
1065 : }
1066 :
1067 : else
1068 25 : return set_to_bottom ();
1069 :
1070 17936 : if (top_p ())
1071 : {
1072 5291 : if (drop_all_ones)
1073 : {
1074 273 : adjusted_mask |= adjusted_value;
1075 273 : adjusted_value &= ~adjusted_mask;
1076 : }
1077 5291 : widest_int cap_mask = wi::shifted_mask <widest_int> (0, precision, true);
1078 5291 : adjusted_mask |= cap_mask;
1079 5291 : if (wi::sext (adjusted_mask, precision) == -1)
1080 0 : return set_to_bottom ();
1081 5291 : return set_to_constant (adjusted_value, adjusted_mask);
1082 5291 : }
1083 : else
1084 12645 : return meet_with_1 (adjusted_value, adjusted_mask, precision,
1085 : drop_all_ones);
1086 18052 : }
1087 :
1088 : /* Dump the contents of the list to FILE. */
1089 :
1090 : void
1091 122 : ipa_argagg_value_list::dump (FILE *f)
1092 : {
1093 122 : bool comma = false;
1094 337 : for (const ipa_argagg_value &av : m_elts)
1095 : {
1096 215 : fprintf (f, "%s %i[%u]=", comma ? "," : "",
1097 215 : av.index, av.unit_offset);
1098 215 : print_generic_expr (f, av.value);
1099 215 : if (av.by_ref)
1100 188 : fprintf (f, "(by_ref)");
1101 215 : if (av.killed)
1102 1 : fprintf (f, "(killed)");
1103 215 : comma = true;
1104 : }
1105 122 : fprintf (f, "\n");
1106 122 : }
1107 :
1108 : /* Dump the contents of the list to stderr. */
1109 :
1110 : void
1111 0 : ipa_argagg_value_list::debug ()
1112 : {
1113 0 : dump (stderr);
1114 0 : }
1115 :
1116 : /* Return the item describing a constant stored for INDEX at UNIT_OFFSET or
1117 : NULL if there is no such constant. */
1118 :
1119 : const ipa_argagg_value *
1120 31015740 : ipa_argagg_value_list::get_elt (int index, unsigned unit_offset) const
1121 : {
1122 31015740 : ipa_argagg_value key;
1123 31015740 : key.index = index;
1124 31015740 : key.unit_offset = unit_offset;
1125 31015740 : const ipa_argagg_value *res
1126 31015740 : = std::lower_bound (m_elts.begin (), m_elts.end (), key,
1127 7497282 : [] (const ipa_argagg_value &elt,
1128 : const ipa_argagg_value &val)
1129 : {
1130 7497282 : if (elt.index < val.index)
1131 : return true;
1132 6394729 : if (elt.index > val.index)
1133 : return false;
1134 5089856 : if (elt.unit_offset < val.unit_offset)
1135 : return true;
1136 : return false;
1137 : });
1138 :
1139 31015740 : if (res == m_elts.end ()
1140 3255460 : || res->index != index
1141 33694187 : || res->unit_offset != unit_offset)
1142 : res = nullptr;
1143 :
1144 : /* TODO: perhaps remove the check (that the underlying array is indeed
1145 : sorted) if it turns out it can be too slow? */
1146 31015740 : if (!flag_checking)
1147 : return res;
1148 :
1149 : const ipa_argagg_value *slow_res = NULL;
1150 : int prev_index = -1;
1151 : unsigned prev_unit_offset = 0;
1152 48637380 : for (const ipa_argagg_value &av : m_elts)
1153 : {
1154 17621640 : gcc_assert (prev_index < 0
1155 : || prev_index < av.index
1156 : || prev_unit_offset < av.unit_offset);
1157 17621640 : prev_index = av.index;
1158 17621640 : prev_unit_offset = av.unit_offset;
1159 17621640 : if (av.index == index
1160 8132360 : && av.unit_offset == unit_offset)
1161 17621640 : slow_res = &av;
1162 : }
1163 31015740 : gcc_assert (res == slow_res);
1164 :
1165 : return res;
1166 : }
1167 :
1168 : /* Return the first item describing a constant stored for parameter with INDEX,
1169 : regardless of offset or reference, or NULL if there is no such constant. */
1170 :
1171 : const ipa_argagg_value *
1172 238120 : ipa_argagg_value_list::get_elt_for_index (int index) const
1173 : {
1174 238120 : const ipa_argagg_value *res
1175 238120 : = std::lower_bound (m_elts.begin (), m_elts.end (), index,
1176 18906 : [] (const ipa_argagg_value &elt, unsigned idx)
1177 : {
1178 18906 : return elt.index < idx;
1179 : });
1180 238120 : if (res == m_elts.end ()
1181 238120 : || res->index != index)
1182 : res = nullptr;
1183 238120 : return res;
1184 : }
1185 :
1186 : /* Return the aggregate constant stored for INDEX at UNIT_OFFSET, not
1187 : performing any check of whether value is passed by reference, or NULL_TREE
1188 : if there is no such constant. */
1189 :
1190 : tree
1191 37268 : ipa_argagg_value_list::get_value (int index, unsigned unit_offset) const
1192 : {
1193 37268 : const ipa_argagg_value *av = get_elt (index, unit_offset);
1194 37268 : return av ? av->value : NULL_TREE;
1195 : }
1196 :
1197 : /* Return the aggregate constant stored for INDEX at UNIT_OFFSET, if it is
1198 : passed by reference or not according to BY_REF, or NULL_TREE if there is
1199 : no such constant. */
1200 :
1201 : tree
1202 30968659 : ipa_argagg_value_list::get_value (int index, unsigned unit_offset,
1203 : bool by_ref) const
1204 : {
1205 30968659 : const ipa_argagg_value *av = get_elt (index, unit_offset);
1206 30968659 : if (av && av->by_ref == by_ref)
1207 2175308 : return av->value;
1208 : return NULL_TREE;
1209 : }
1210 :
1211 : /* Return true if all elements present in OTHER are also present in this
1212 : list. */
1213 :
1214 : bool
1215 42 : ipa_argagg_value_list::superset_of_p (const ipa_argagg_value_list &other) const
1216 : {
1217 42 : unsigned j = 0;
1218 194 : for (unsigned i = 0; i < other.m_elts.size (); i++)
1219 : {
1220 169 : unsigned other_index = other.m_elts[i].index;
1221 169 : unsigned other_offset = other.m_elts[i].unit_offset;
1222 :
1223 169 : while (j < m_elts.size ()
1224 324 : && (m_elts[j].index < other_index
1225 308 : || (m_elts[j].index == other_index
1226 308 : && m_elts[j].unit_offset < other_offset)))
1227 155 : j++;
1228 :
1229 169 : if (j >= m_elts.size ()
1230 156 : || m_elts[j].index != other_index
1231 156 : || m_elts[j].unit_offset != other_offset
1232 156 : || m_elts[j].by_ref != other.m_elts[i].by_ref
1233 156 : || !m_elts[j].value
1234 325 : || !values_equal_for_ipcp_p (m_elts[j].value, other.m_elts[i].value))
1235 17 : return false;
1236 : }
1237 : return true;
1238 : }
1239 :
1240 : /* Push all items in this list that describe parameter SRC_INDEX into RES as
1241 : ones describing DST_INDEX while subtracting UNIT_DELTA from their unit
1242 : offsets but skip those which would end up with a negative offset. */
1243 :
1244 : void
1245 3167 : ipa_argagg_value_list::push_adjusted_values (unsigned src_index,
1246 : unsigned dest_index,
1247 : unsigned unit_delta,
1248 : vec<ipa_argagg_value> *res) const
1249 : {
1250 3167 : const ipa_argagg_value *av = get_elt_for_index (src_index);
1251 3167 : if (!av)
1252 : return;
1253 : unsigned prev_unit_offset = 0;
1254 : bool first = true;
1255 12676 : for (; av < m_elts.end (); ++av)
1256 : {
1257 10209 : if (av->index > src_index)
1258 : return;
1259 9599 : if (av->index == src_index
1260 9599 : && (av->unit_offset >= unit_delta)
1261 9455 : && av->value)
1262 : {
1263 9455 : ipa_argagg_value new_av;
1264 9455 : gcc_checking_assert (av->value);
1265 9455 : new_av.value = av->value;
1266 9455 : new_av.unit_offset = av->unit_offset - unit_delta;
1267 9455 : new_av.index = dest_index;
1268 9455 : new_av.by_ref = av->by_ref;
1269 9455 : gcc_assert (!av->killed);
1270 9455 : new_av.killed = false;
1271 :
1272 : /* Quick check that the offsets we push are indeed increasing. */
1273 9455 : gcc_assert (first
1274 : || new_av.unit_offset > prev_unit_offset);
1275 9455 : prev_unit_offset = new_av.unit_offset;
1276 9455 : first = false;
1277 :
1278 9455 : res->safe_push (new_av);
1279 : }
1280 : }
1281 : }
1282 :
1283 : /* Push to RES information about single lattices describing aggregate values in
1284 : PLATS as those describing parameter DEST_INDEX and the original offset minus
1285 : UNIT_DELTA. Return true if any item has been pushed to RES. */
1286 :
1287 : static bool
1288 2312067 : push_agg_values_from_plats (ipcp_param_lattices *plats, int dest_index,
1289 : unsigned unit_delta,
1290 : vec<ipa_argagg_value> *res)
1291 : {
1292 2312067 : if (plats->aggs_contain_variable)
1293 : return false;
1294 :
1295 1678810 : bool pushed_sth = false;
1296 1678810 : bool first = true;
1297 1678810 : unsigned prev_unit_offset = 0;
1298 1744979 : for (struct ipcp_agg_lattice *aglat = plats->aggs; aglat; aglat = aglat->next)
1299 131130 : if (aglat->is_single_const ()
1300 40747 : && (aglat->offset / BITS_PER_UNIT - unit_delta) >= 0)
1301 : {
1302 40747 : ipa_argagg_value iav;
1303 40747 : iav.value = aglat->values->value;
1304 40747 : iav.unit_offset = aglat->offset / BITS_PER_UNIT - unit_delta;
1305 40747 : iav.index = dest_index;
1306 40747 : iav.by_ref = plats->aggs_by_ref;
1307 40747 : iav.killed = false;
1308 :
1309 40747 : gcc_assert (first
1310 : || iav.unit_offset > prev_unit_offset);
1311 40747 : prev_unit_offset = iav.unit_offset;
1312 40747 : first = false;
1313 :
1314 40747 : pushed_sth = true;
1315 40747 : res->safe_push (iav);
1316 : }
1317 : return pushed_sth;
1318 : }
1319 :
1320 : /* Turn all values in LIST that are not present in OTHER into NULL_TREEs.
1321 : Return the number of remaining valid entries. */
1322 :
1323 : static unsigned
1324 56477 : intersect_argaggs_with (vec<ipa_argagg_value> &elts,
1325 : const vec<ipa_argagg_value> &other)
1326 : {
1327 56477 : unsigned valid_entries = 0;
1328 56477 : unsigned j = 0;
1329 424709 : for (unsigned i = 0; i < elts.length (); i++)
1330 : {
1331 368232 : if (!elts[i].value)
1332 52610 : continue;
1333 :
1334 315622 : unsigned this_index = elts[i].index;
1335 315622 : unsigned this_offset = elts[i].unit_offset;
1336 :
1337 315622 : while (j < other.length ()
1338 1187896 : && (other[j].index < this_index
1339 559350 : || (other[j].index == this_index
1340 555950 : && other[j].unit_offset < this_offset)))
1341 282552 : j++;
1342 :
1343 315622 : if (j >= other.length ())
1344 : {
1345 8452 : elts[i].value = NULL_TREE;
1346 8452 : continue;
1347 : }
1348 :
1349 307170 : if (other[j].index == this_index
1350 303770 : && other[j].unit_offset == this_offset
1351 296887 : && other[j].by_ref == elts[i].by_ref
1352 296887 : && other[j].value
1353 604057 : && values_equal_for_ipcp_p (other[j].value, elts[i].value))
1354 276995 : valid_entries++;
1355 : else
1356 30175 : elts[i].value = NULL_TREE;
1357 : }
1358 56477 : return valid_entries;
1359 : }
1360 :
1361 : /* Mark bot aggregate and scalar lattices as containing an unknown variable,
1362 : return true is any of them has not been marked as such so far. If if
1363 : MAKE_SIMPLE_RECIPIENTS is true, set the lattices that can only hold one
1364 : value to being recipients only, otherwise also set them to bottom. */
1365 :
1366 : static inline bool
1367 173913 : set_all_contains_variable (class ipcp_param_lattices *plats,
1368 : bool make_simple_recipients = false)
1369 : {
1370 173913 : bool ret;
1371 173913 : ret = plats->itself.set_contains_variable ();
1372 173913 : ret |= plats->ctxlat.set_contains_variable ();
1373 173913 : ret |= set_agg_lats_contain_variable (plats);
1374 173913 : if (make_simple_recipients)
1375 : {
1376 29349 : ret |= plats->bits_lattice.set_recipient_only ();
1377 29349 : ret |= plats->m_value_range.set_recipient_only ();
1378 : }
1379 : else
1380 : {
1381 144564 : ret |= plats->bits_lattice.set_to_bottom ();
1382 144564 : ret |= plats->m_value_range.set_to_bottom ();
1383 : }
1384 173913 : return ret;
1385 : }
1386 :
1387 : /* Worker of call_for_symbol_thunks_and_aliases, increment the integer DATA
1388 : points to by the number of callers to NODE. */
1389 :
1390 : static bool
1391 101055 : count_callers (cgraph_node *node, void *data)
1392 : {
1393 101055 : int *caller_count = (int *) data;
1394 :
1395 409227 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
1396 : /* Local thunks can be handled transparently, but if the thunk cannot
1397 : be optimized out, count it as a real use. */
1398 308172 : if (!cs->caller->thunk || !cs->caller->local)
1399 308172 : ++*caller_count;
1400 101055 : return false;
1401 : }
1402 :
1403 : /* Worker of call_for_symbol_thunks_and_aliases, it is supposed to be called on
1404 : the one caller of some other node. Set the caller's corresponding flag. */
1405 :
1406 : static bool
1407 58009 : set_single_call_flag (cgraph_node *node, void *)
1408 : {
1409 58009 : cgraph_edge *cs = node->callers;
1410 : /* Local thunks can be handled transparently, skip them. */
1411 58009 : while (cs && cs->caller->thunk && cs->caller->local)
1412 0 : cs = cs->next_caller;
1413 58009 : if (cs)
1414 57441 : if (ipa_node_params* info = ipa_node_params_sum->get (cs->caller))
1415 : {
1416 57440 : info->node_calling_single_call = true;
1417 57440 : return true;
1418 : }
1419 : return false;
1420 : }
1421 :
1422 : /* Initialize ipcp_lattices. */
1423 :
1424 : static void
1425 1293896 : initialize_node_lattices (struct cgraph_node *node)
1426 : {
1427 1293896 : ipa_node_params *info = ipa_node_params_sum->get (node);
1428 1293896 : struct cgraph_edge *ie;
1429 1293896 : bool disable = false, variable = false;
1430 1293896 : int i;
1431 :
1432 1293896 : gcc_checking_assert (node->has_gimple_body_p ());
1433 :
1434 1293896 : if (!ipa_get_param_count (info))
1435 : disable = true;
1436 1062053 : else if (node->local)
1437 : {
1438 90112 : int caller_count = 0;
1439 90112 : node->call_for_symbol_thunks_and_aliases (count_callers, &caller_count,
1440 : true);
1441 90112 : if (caller_count == 1)
1442 57441 : node->call_for_symbol_thunks_and_aliases (set_single_call_flag,
1443 : NULL, true);
1444 32671 : else if (caller_count == 0)
1445 : {
1446 1 : gcc_checking_assert (!opt_for_fn (node->decl, flag_toplevel_reorder));
1447 : variable = true;
1448 : }
1449 : }
1450 : else
1451 : {
1452 : /* When cloning is allowed, we can assume that externally visible
1453 : functions are not called. We will compensate this by cloning
1454 : later. */
1455 971941 : if (ipcp_versionable_function_p (node)
1456 971941 : && ipcp_cloning_candidate_p (node))
1457 : variable = true;
1458 : else
1459 : disable = true;
1460 : }
1461 :
1462 725 : if (dump_file && (dump_flags & TDF_DETAILS)
1463 1294063 : && !node->alias && !node->thunk)
1464 : {
1465 167 : fprintf (dump_file, "Initializing lattices of %s\n",
1466 : node->dump_name ());
1467 167 : if (disable || variable)
1468 133 : fprintf (dump_file, " Marking all lattices as %s\n",
1469 : disable ? "BOTTOM" : "VARIABLE");
1470 : }
1471 :
1472 1293896 : auto_vec<bool, 16> surviving_params;
1473 1293896 : bool pre_modified = false;
1474 :
1475 1293896 : clone_info *cinfo = clone_info::get (node);
1476 :
1477 1293896 : if (!disable && cinfo && cinfo->param_adjustments)
1478 : {
1479 : /* At the moment all IPA optimizations should use the number of
1480 : parameters of the prevailing decl as the m_always_copy_start.
1481 : Handling any other value would complicate the code below, so for the
1482 : time bing let's only assert it is so. */
1483 0 : gcc_assert ((cinfo->param_adjustments->m_always_copy_start
1484 : == ipa_get_param_count (info))
1485 : || cinfo->param_adjustments->m_always_copy_start < 0);
1486 :
1487 0 : pre_modified = true;
1488 0 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
1489 :
1490 0 : if (dump_file && (dump_flags & TDF_DETAILS)
1491 0 : && !node->alias && !node->thunk)
1492 : {
1493 : bool first = true;
1494 0 : for (int j = 0; j < ipa_get_param_count (info); j++)
1495 : {
1496 0 : if (j < (int) surviving_params.length ()
1497 0 : && surviving_params[j])
1498 0 : continue;
1499 0 : if (first)
1500 : {
1501 0 : fprintf (dump_file,
1502 : " The following parameters are dead on arrival:");
1503 0 : first = false;
1504 : }
1505 0 : fprintf (dump_file, " %u", j);
1506 : }
1507 0 : if (!first)
1508 0 : fprintf (dump_file, "\n");
1509 : }
1510 : }
1511 :
1512 7125585 : for (i = 0; i < ipa_get_param_count (info); i++)
1513 : {
1514 2384818 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
1515 2384818 : tree type = ipa_get_type (info, i);
1516 2384818 : if (disable
1517 226522 : || !ipa_get_type (info, i)
1518 2611340 : || (pre_modified && (surviving_params.length () <= (unsigned) i
1519 0 : || !surviving_params[i])))
1520 : {
1521 2158296 : plats->itself.set_to_bottom ();
1522 2158296 : plats->ctxlat.set_to_bottom ();
1523 2158296 : set_agg_lats_to_bottom (plats);
1524 2158296 : plats->bits_lattice.set_to_bottom ();
1525 2158296 : plats->m_value_range.init (type);
1526 2158296 : plats->m_value_range.set_to_bottom ();
1527 : }
1528 : else
1529 : {
1530 226522 : plats->m_value_range.init (type);
1531 226522 : if (variable)
1532 29349 : set_all_contains_variable (plats, true);
1533 : }
1534 : }
1535 :
1536 1428969 : for (ie = node->indirect_calls; ie; ie = ie->next_callee)
1537 135073 : if (ie->indirect_info->param_index >= 0
1538 144371 : && is_a <cgraph_polymorphic_indirect_info *> (ie->indirect_info))
1539 9298 : ipa_get_parm_lattices (info,
1540 9298 : ie->indirect_info->param_index)->virt_call = 1;
1541 1293896 : }
1542 :
1543 : /* Return VALUE if it is NULL_TREE or if it can be directly safely IPA-CP
1544 : propagated to a parameter of type PARAM_TYPE, or return a fold-converted
1545 : VALUE to PARAM_TYPE if that is possible. Return NULL_TREE otherwise. */
1546 :
1547 : tree
1548 5256174 : ipacp_value_safe_for_type (tree param_type, tree value)
1549 : {
1550 5256174 : if (!value)
1551 : return NULL_TREE;
1552 5255840 : tree val_type = TREE_TYPE (value);
1553 5255840 : if (param_type == val_type
1554 5255840 : || useless_type_conversion_p (param_type, val_type))
1555 5252445 : return value;
1556 3395 : if (fold_convertible_p (param_type, value))
1557 3186 : return fold_convert (param_type, value);
1558 : else
1559 : return NULL_TREE;
1560 : }
1561 :
1562 : /* Return the result of a (possibly arithmetic) operation determined by OPCODE
1563 : on the constant value INPUT. OPERAND is 2nd operand for binary operation
1564 : and is required for binary operations. RES_TYPE, required when opcode is
1565 : not NOP_EXPR, is the type in which any operation is to be performed. Return
1566 : NULL_TREE if that cannot be determined or be considered an interprocedural
1567 : invariant. */
1568 :
1569 : static tree
1570 69234 : ipa_get_jf_arith_result (enum tree_code opcode, tree input, tree operand,
1571 : tree res_type)
1572 : {
1573 69234 : tree res;
1574 :
1575 69234 : if (opcode == NOP_EXPR)
1576 : return input;
1577 6837 : if (!is_gimple_ip_invariant (input))
1578 : return NULL_TREE;
1579 :
1580 6837 : if (opcode == ASSERT_EXPR)
1581 : {
1582 3766 : if (values_equal_for_ipcp_p (input, operand))
1583 : return input;
1584 : else
1585 : return NULL_TREE;
1586 : }
1587 :
1588 3071 : if (TREE_CODE_CLASS (opcode) == tcc_unary)
1589 102 : res = fold_unary (opcode, res_type, input);
1590 : else
1591 2969 : res = fold_binary (opcode, res_type, input, operand);
1592 :
1593 3071 : if (res && !is_gimple_ip_invariant (res))
1594 : return NULL_TREE;
1595 :
1596 : return res;
1597 : }
1598 :
1599 : /* Return the result of an ancestor jump function JFUNC on the constant value
1600 : INPUT. Return NULL_TREE if that cannot be determined. */
1601 :
1602 : static tree
1603 1296 : ipa_get_jf_ancestor_result (struct ipa_jump_func *jfunc, tree input)
1604 : {
1605 1296 : gcc_checking_assert (TREE_CODE (input) != TREE_BINFO);
1606 1296 : if (TREE_CODE (input) == ADDR_EXPR)
1607 : {
1608 1214 : gcc_checking_assert (is_gimple_ip_invariant_address (input));
1609 1214 : poly_int64 off = ipa_get_jf_ancestor_offset (jfunc);
1610 1214 : if (known_eq (off, 0))
1611 : return input;
1612 1092 : poly_int64 byte_offset = exact_div (off, BITS_PER_UNIT);
1613 2184 : return build1 (ADDR_EXPR, TREE_TYPE (input),
1614 1092 : fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (input)), input,
1615 1092 : build_int_cst (ptr_type_node, byte_offset)));
1616 : }
1617 82 : else if (ipa_get_jf_ancestor_keep_null (jfunc)
1618 82 : && zerop (input))
1619 : return input;
1620 : else
1621 78 : return NULL_TREE;
1622 : }
1623 :
1624 : /* Determine whether JFUNC evaluates to a single known constant value and if
1625 : so, return it. Otherwise return NULL. INFO describes the caller node or
1626 : the one it is inlined to, so that pass-through jump functions can be
1627 : evaluated. PARM_TYPE is the type of the parameter to which the result is
1628 : passed. */
1629 :
1630 : tree
1631 18653058 : ipa_value_from_jfunc (class ipa_node_params *info, struct ipa_jump_func *jfunc,
1632 : tree parm_type)
1633 : {
1634 18653058 : if (!parm_type)
1635 : return NULL_TREE;
1636 18410262 : if (jfunc->type == IPA_JF_CONST)
1637 4647670 : return ipacp_value_safe_for_type (parm_type, ipa_get_jf_constant (jfunc));
1638 13762592 : else if (jfunc->type == IPA_JF_PASS_THROUGH
1639 10808350 : || jfunc->type == IPA_JF_ANCESTOR)
1640 : {
1641 3774676 : tree input;
1642 3774676 : int idx;
1643 :
1644 3774676 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1645 2954242 : idx = ipa_get_jf_pass_through_formal_id (jfunc);
1646 : else
1647 820434 : idx = ipa_get_jf_ancestor_formal_id (jfunc);
1648 :
1649 3774676 : if (info->ipcp_orig_node)
1650 47793 : input = info->known_csts[idx];
1651 : else
1652 : {
1653 3726883 : ipcp_lattice<tree> *lat;
1654 :
1655 6659000 : if (info->lattices.is_empty ()
1656 2932117 : || idx >= ipa_get_param_count (info))
1657 : return NULL_TREE;
1658 2932117 : lat = ipa_get_scalar_lat (info, idx);
1659 2932117 : if (!lat->is_single_const ())
1660 : return NULL_TREE;
1661 151 : input = lat->values->value;
1662 : }
1663 :
1664 47944 : if (!input)
1665 : return NULL_TREE;
1666 :
1667 19440 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1668 : {
1669 18459 : enum tree_code opcode = ipa_get_jf_pass_through_operation (jfunc);
1670 18459 : tree op2 = ipa_get_jf_pass_through_operand (jfunc);
1671 18459 : tree op_type
1672 18459 : = (opcode == NOP_EXPR) ? NULL_TREE
1673 988 : : ipa_get_jf_pass_through_op_type (jfunc);
1674 18459 : tree cstval = ipa_get_jf_arith_result (opcode, input, op2, op_type);
1675 18459 : return ipacp_value_safe_for_type (parm_type, cstval);
1676 : }
1677 : else
1678 981 : return ipacp_value_safe_for_type (parm_type,
1679 : ipa_get_jf_ancestor_result (jfunc,
1680 981 : input));
1681 : }
1682 : else
1683 : return NULL_TREE;
1684 : }
1685 :
1686 : /* Determine whether JFUNC evaluates to single known polymorphic context, given
1687 : that INFO describes the caller node or the one it is inlined to, CS is the
1688 : call graph edge corresponding to JFUNC and CSIDX index of the described
1689 : parameter. */
1690 :
1691 : ipa_polymorphic_call_context
1692 901609 : ipa_context_from_jfunc (ipa_node_params *info, cgraph_edge *cs, int csidx,
1693 : ipa_jump_func *jfunc)
1694 : {
1695 901609 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
1696 901609 : ipa_polymorphic_call_context ctx;
1697 901609 : ipa_polymorphic_call_context *edge_ctx
1698 901609 : = cs ? ipa_get_ith_polymorhic_call_context (args, csidx) : NULL;
1699 :
1700 369267 : if (edge_ctx && !edge_ctx->useless_p ())
1701 363920 : ctx = *edge_ctx;
1702 :
1703 901609 : if (jfunc->type == IPA_JF_PASS_THROUGH
1704 808257 : || jfunc->type == IPA_JF_ANCESTOR)
1705 : {
1706 101483 : ipa_polymorphic_call_context srcctx;
1707 101483 : int srcidx;
1708 101483 : bool type_preserved = true;
1709 101483 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1710 : {
1711 93352 : if (ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR)
1712 1842 : return ctx;
1713 91510 : type_preserved = ipa_get_jf_pass_through_type_preserved (jfunc);
1714 91510 : srcidx = ipa_get_jf_pass_through_formal_id (jfunc);
1715 : }
1716 : else
1717 : {
1718 8131 : type_preserved = ipa_get_jf_ancestor_type_preserved (jfunc);
1719 8131 : srcidx = ipa_get_jf_ancestor_formal_id (jfunc);
1720 : }
1721 99641 : if (info->ipcp_orig_node)
1722 : {
1723 11467 : if (info->known_contexts.exists ())
1724 1351 : srcctx = info->known_contexts[srcidx];
1725 : }
1726 : else
1727 : {
1728 174709 : if (info->lattices.is_empty ()
1729 86535 : || srcidx >= ipa_get_param_count (info))
1730 1639 : return ctx;
1731 86535 : ipcp_lattice<ipa_polymorphic_call_context> *lat;
1732 86535 : lat = ipa_get_poly_ctx_lat (info, srcidx);
1733 86535 : if (!lat->is_single_const ())
1734 82534 : return ctx;
1735 4001 : srcctx = lat->values->value;
1736 : }
1737 15468 : if (srcctx.useless_p ())
1738 10567 : return ctx;
1739 4901 : if (jfunc->type == IPA_JF_ANCESTOR)
1740 253 : srcctx.offset_by (ipa_get_jf_ancestor_offset (jfunc));
1741 4901 : if (!type_preserved)
1742 2917 : srcctx.possible_dynamic_type_change (cs->in_polymorphic_cdtor);
1743 4901 : srcctx.combine_with (ctx);
1744 4901 : return srcctx;
1745 : }
1746 :
1747 800126 : return ctx;
1748 : }
1749 :
1750 : /* Emulate effects of unary OPERATION and/or conversion from SRC_TYPE to
1751 : DST_TYPE on value range in SRC_VR and store it to DST_VR. Return true if
1752 : the result is a range that is not VARYING nor UNDEFINED. */
1753 :
1754 : bool
1755 9540864 : ipa_vr_operation_and_type_effects (vrange &dst_vr,
1756 : const vrange &src_vr,
1757 : enum tree_code operation,
1758 : tree dst_type, tree src_type)
1759 : {
1760 17896847 : if (!ipa_vr_supported_type_p (dst_type)
1761 0 : || !ipa_vr_supported_type_p (src_type))
1762 : return false;
1763 :
1764 9540864 : range_op_handler handler (operation);
1765 9540864 : if (!handler)
1766 : return false;
1767 :
1768 9540864 : value_range varying (dst_type);
1769 9540864 : varying.set_varying (dst_type);
1770 :
1771 9540864 : return (handler.operand_check_p (dst_type, src_type, dst_type)
1772 9540864 : && handler.fold_range (dst_vr, dst_type, src_vr, varying)
1773 9540862 : && !dst_vr.varying_p ()
1774 19081666 : && !dst_vr.undefined_p ());
1775 9540864 : }
1776 :
1777 : /* Same as above, but the SRC_VR argument is an IPA_VR which must
1778 : first be extracted onto a vrange. */
1779 :
1780 : bool
1781 9533126 : ipa_vr_operation_and_type_effects (vrange &dst_vr,
1782 : const ipa_vr &src_vr,
1783 : enum tree_code operation,
1784 : tree dst_type, tree src_type)
1785 : {
1786 9533126 : value_range tmp;
1787 9533126 : src_vr.get_vrange (tmp);
1788 9533126 : return ipa_vr_operation_and_type_effects (dst_vr, tmp, operation,
1789 9533126 : dst_type, src_type);
1790 9533126 : }
1791 :
1792 : /* Given a PASS_THROUGH jump function JFUNC that takes as its source SRC_VR of
1793 : SRC_TYPE and the result needs to be DST_TYPE, if any value range information
1794 : can be deduced at all, intersect VR with it. CONTEXT_NODE is the call graph
1795 : node representing the function for which optimization flags should be
1796 : evaluated. */
1797 :
1798 : static void
1799 92818 : ipa_vr_intersect_with_arith_jfunc (vrange &vr,
1800 : ipa_jump_func *jfunc,
1801 : cgraph_node *context_node,
1802 : const value_range &src_vr,
1803 : tree src_type,
1804 : tree dst_type)
1805 : {
1806 92818 : if (src_vr.undefined_p () || src_vr.varying_p ())
1807 91542 : return;
1808 :
1809 92359 : enum tree_code operation = ipa_get_jf_pass_through_operation (jfunc);
1810 92359 : if (TREE_CODE_CLASS (operation) == tcc_unary)
1811 : {
1812 91083 : value_range op_res;
1813 91083 : const value_range *inter_vr;
1814 91083 : if (operation != NOP_EXPR)
1815 : {
1816 93 : tree operation_type = ipa_get_jf_pass_through_op_type (jfunc);
1817 93 : op_res.set_varying (operation_type);
1818 93 : if (!ipa_vr_operation_and_type_effects (op_res, src_vr, operation,
1819 : operation_type, src_type))
1820 : return;
1821 : inter_vr = &op_res;
1822 : src_type = operation_type;
1823 : }
1824 : else
1825 : inter_vr = &src_vr;
1826 :
1827 91083 : if (src_type != dst_type)
1828 : {
1829 6369 : value_range tmp_res (dst_type);
1830 6369 : if (!ipa_vr_operation_and_type_effects (tmp_res, *inter_vr, NOP_EXPR,
1831 : dst_type, src_type))
1832 0 : return;
1833 6369 : vr.intersect (tmp_res);
1834 6369 : }
1835 : else
1836 84714 : vr.intersect (*inter_vr);
1837 91083 : return;
1838 91083 : }
1839 :
1840 1276 : tree operand = ipa_get_jf_pass_through_operand (jfunc);
1841 1276 : range_op_handler handler (operation);
1842 1276 : if (!handler)
1843 : return;
1844 1276 : value_range op_vr (TREE_TYPE (operand));
1845 1276 : ipa_get_range_from_ip_invariant (op_vr, operand, context_node);
1846 :
1847 1276 : tree operation_type = ipa_get_jf_pass_through_op_type (jfunc);
1848 1276 : value_range op_res (operation_type);
1849 1716 : if (!ipa_vr_supported_type_p (operation_type)
1850 1276 : || !handler.operand_check_p (operation_type, src_type, op_vr.type ())
1851 1276 : || !handler.fold_range (op_res, operation_type, src_vr, op_vr))
1852 0 : return;
1853 :
1854 1276 : value_range tmp_res (dst_type);
1855 1276 : if (ipa_vr_operation_and_type_effects (tmp_res, op_res, NOP_EXPR, dst_type,
1856 : operation_type))
1857 1228 : vr.intersect (tmp_res);
1858 1276 : }
1859 :
1860 : /* Determine range of JFUNC given that INFO describes the caller node or
1861 : the one it is inlined to, CS is the call graph edge corresponding to JFUNC
1862 : and PARM_TYPE of the parameter. */
1863 :
1864 : void
1865 12454812 : ipa_value_range_from_jfunc (vrange &vr,
1866 : ipa_node_params *info, cgraph_edge *cs,
1867 : ipa_jump_func *jfunc, tree parm_type)
1868 : {
1869 12454812 : vr.set_varying (parm_type);
1870 :
1871 12454812 : if (jfunc->m_vr && jfunc->m_vr->known_p ())
1872 8699245 : ipa_vr_operation_and_type_effects (vr,
1873 : *jfunc->m_vr,
1874 : NOP_EXPR, parm_type,
1875 8699245 : jfunc->m_vr->type ());
1876 12454812 : if (vr.singleton_p ())
1877 : return;
1878 :
1879 12454672 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1880 : {
1881 2371552 : ipcp_transformation *sum
1882 2371552 : = ipcp_get_transformation_summary (cs->caller->inlined_to
1883 : ? cs->caller->inlined_to
1884 : : cs->caller);
1885 2371552 : if (!sum || !sum->m_vr)
1886 2293991 : return;
1887 :
1888 119775 : int idx = ipa_get_jf_pass_through_formal_id (jfunc);
1889 :
1890 119775 : if (!(*sum->m_vr)[idx].known_p ())
1891 : return;
1892 77561 : tree src_type = ipa_get_type (info, idx);
1893 77561 : value_range srcvr;
1894 77561 : (*sum->m_vr)[idx].get_vrange (srcvr);
1895 :
1896 77561 : ipa_vr_intersect_with_arith_jfunc (vr, jfunc, cs->caller, srcvr, src_type,
1897 : parm_type);
1898 77561 : }
1899 : }
1900 :
1901 : /* Determine whether ITEM, jump function for an aggregate part, evaluates to a
1902 : single known constant value and if so, return it. Otherwise return NULL.
1903 : NODE and INFO describes the caller node or the one it is inlined to, and
1904 : its related info. */
1905 :
1906 : tree
1907 3525696 : ipa_agg_value_from_jfunc (ipa_node_params *info, cgraph_node *node,
1908 : const ipa_agg_jf_item *item)
1909 : {
1910 3525696 : tree value = NULL_TREE;
1911 3525696 : int src_idx;
1912 :
1913 3525696 : if (item->offset < 0
1914 3478316 : || item->jftype == IPA_JF_UNKNOWN
1915 3317904 : || item->offset >= (HOST_WIDE_INT) UINT_MAX * BITS_PER_UNIT)
1916 : return NULL_TREE;
1917 :
1918 3317904 : if (item->jftype == IPA_JF_CONST)
1919 2956790 : return item->value.constant;
1920 :
1921 361114 : gcc_checking_assert (item->jftype == IPA_JF_PASS_THROUGH
1922 : || item->jftype == IPA_JF_LOAD_AGG);
1923 :
1924 361114 : src_idx = item->value.pass_through.formal_id;
1925 :
1926 361114 : if (info->ipcp_orig_node)
1927 : {
1928 16570 : if (item->jftype == IPA_JF_PASS_THROUGH)
1929 3795 : value = info->known_csts[src_idx];
1930 12775 : else if (ipcp_transformation *ts = ipcp_get_transformation_summary (node))
1931 : {
1932 12775 : ipa_argagg_value_list avl (ts);
1933 12775 : value = avl.get_value (src_idx,
1934 12775 : item->value.load_agg.offset / BITS_PER_UNIT,
1935 12775 : item->value.load_agg.by_ref);
1936 : }
1937 : }
1938 344544 : else if (!info->lattices.is_empty ())
1939 : {
1940 229009 : class ipcp_param_lattices *src_plats
1941 229009 : = ipa_get_parm_lattices (info, src_idx);
1942 :
1943 229009 : if (item->jftype == IPA_JF_PASS_THROUGH)
1944 : {
1945 137912 : struct ipcp_lattice<tree> *lat = &src_plats->itself;
1946 :
1947 567427 : if (!lat->is_single_const ())
1948 : return NULL_TREE;
1949 :
1950 0 : value = lat->values->value;
1951 : }
1952 91097 : else if (src_plats->aggs
1953 12045 : && !src_plats->aggs_bottom
1954 12045 : && !src_plats->aggs_contain_variable
1955 1503 : && src_plats->aggs_by_ref == item->value.load_agg.by_ref)
1956 : {
1957 : struct ipcp_agg_lattice *aglat;
1958 :
1959 2370 : for (aglat = src_plats->aggs; aglat; aglat = aglat->next)
1960 : {
1961 2370 : if (aglat->offset > item->value.load_agg.offset)
1962 : break;
1963 :
1964 2338 : if (aglat->offset == item->value.load_agg.offset)
1965 : {
1966 1471 : if (aglat->is_single_const ())
1967 7 : value = aglat->values->value;
1968 : break;
1969 : }
1970 : }
1971 : }
1972 : }
1973 :
1974 16609 : if (!value)
1975 213109 : return NULL_TREE;
1976 :
1977 10093 : if (item->jftype == IPA_JF_LOAD_AGG)
1978 : {
1979 7776 : tree load_type = item->value.load_agg.type;
1980 7776 : tree value_type = TREE_TYPE (value);
1981 :
1982 : /* Ensure value type is compatible with load type. */
1983 7776 : if (!useless_type_conversion_p (load_type, value_type))
1984 : return NULL_TREE;
1985 : }
1986 :
1987 20186 : tree cstval = ipa_get_jf_arith_result (item->value.pass_through.operation,
1988 : value,
1989 10093 : item->value.pass_through.operand,
1990 10093 : item->value.pass_through.op_type);
1991 10093 : return ipacp_value_safe_for_type (item->type, cstval);
1992 : }
1993 :
1994 : /* Process all items in AGG_JFUNC relative to caller (or the node the original
1995 : caller is inlined to) NODE which described by INFO and push the results to
1996 : RES as describing values passed in parameter DST_INDEX. */
1997 :
1998 : void
1999 14999159 : ipa_push_agg_values_from_jfunc (ipa_node_params *info, cgraph_node *node,
2000 : ipa_agg_jump_function *agg_jfunc,
2001 : unsigned dst_index,
2002 : vec<ipa_argagg_value> *res)
2003 : {
2004 14999159 : unsigned prev_unit_offset = 0;
2005 14999159 : bool first = true;
2006 :
2007 20134805 : for (const ipa_agg_jf_item &item : agg_jfunc->items)
2008 : {
2009 2497448 : tree value = ipa_agg_value_from_jfunc (info, node, &item);
2010 2497448 : if (!value)
2011 533292 : continue;
2012 :
2013 1964156 : ipa_argagg_value iav;
2014 1964156 : iav.value = value;
2015 1964156 : iav.unit_offset = item.offset / BITS_PER_UNIT;
2016 1964156 : iav.index = dst_index;
2017 1964156 : iav.by_ref = agg_jfunc->by_ref;
2018 1964156 : iav.killed = 0;
2019 :
2020 1964156 : gcc_assert (first
2021 : || iav.unit_offset > prev_unit_offset);
2022 1964156 : prev_unit_offset = iav.unit_offset;
2023 1964156 : first = false;
2024 :
2025 1964156 : res->safe_push (iav);
2026 : }
2027 14999159 : }
2028 :
2029 : /* If checking is enabled, verify that no lattice is in the TOP state, i.e. not
2030 : bottom, not containing a variable component and without any known value at
2031 : the same time. */
2032 :
2033 : DEBUG_FUNCTION void
2034 129914 : ipcp_verify_propagated_values (void)
2035 : {
2036 129914 : struct cgraph_node *node;
2037 :
2038 1432583 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
2039 : {
2040 1302669 : ipa_node_params *info = ipa_node_params_sum->get (node);
2041 1302669 : if (!opt_for_fn (node->decl, flag_ipa_cp)
2042 1302669 : || !opt_for_fn (node->decl, optimize))
2043 8790 : continue;
2044 1293879 : int i, count = ipa_get_param_count (info);
2045 :
2046 3678683 : for (i = 0; i < count; i++)
2047 : {
2048 2384804 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (info, i);
2049 :
2050 2384804 : if (!lat->bottom
2051 225516 : && !lat->contains_variable
2052 31859 : && lat->values_count == 0)
2053 : {
2054 0 : if (dump_file)
2055 : {
2056 0 : symtab->dump (dump_file);
2057 0 : fprintf (dump_file, "\nIPA lattices after constant "
2058 : "propagation, before gcc_unreachable:\n");
2059 0 : print_all_lattices (dump_file, true, false);
2060 : }
2061 :
2062 0 : gcc_unreachable ();
2063 : }
2064 : }
2065 : }
2066 129914 : }
2067 :
2068 : /* Return true iff X and Y should be considered equal contexts by IPA-CP. */
2069 :
2070 : static bool
2071 2704 : values_equal_for_ipcp_p (ipa_polymorphic_call_context x,
2072 : ipa_polymorphic_call_context y)
2073 : {
2074 2204 : return x.equal_to (y);
2075 : }
2076 :
2077 :
2078 : /* Add a new value source to the value represented by THIS, marking that a
2079 : value comes from edge CS and (if the underlying jump function is a
2080 : pass-through or an ancestor one) from a caller value SRC_VAL of a caller
2081 : parameter described by SRC_INDEX. OFFSET is negative if the source was the
2082 : scalar value of the parameter itself or the offset within an aggregate. */
2083 :
2084 : template <typename valtype>
2085 : void
2086 489032 : ipcp_value<valtype>::add_source (cgraph_edge *cs, ipcp_value *src_val,
2087 : int src_idx, HOST_WIDE_INT offset)
2088 : {
2089 : ipcp_value_source<valtype> *src;
2090 :
2091 489032 : src = new (ipcp_sources_pool.allocate ()) ipcp_value_source<valtype>;
2092 489032 : src->offset = offset;
2093 489032 : src->cs = cs;
2094 489032 : src->val = src_val;
2095 489032 : src->index = src_idx;
2096 :
2097 489032 : src->next = sources;
2098 489032 : sources = src;
2099 489032 : }
2100 :
2101 : /* Allocate a new ipcp_value holding a tree constant, initialize its value to
2102 : SOURCE and clear all other fields. */
2103 :
2104 : static ipcp_value<tree> *
2105 140683 : allocate_and_init_ipcp_value (tree cst, unsigned same_lat_gen_level)
2106 : {
2107 140683 : ipcp_value<tree> *val;
2108 :
2109 140683 : val = new (ipcp_cst_values_pool.allocate ()) ipcp_value<tree>();
2110 140683 : val->value = cst;
2111 140683 : val->self_recursion_generated_level = same_lat_gen_level;
2112 140683 : return val;
2113 : }
2114 :
2115 : /* Allocate a new ipcp_value holding a polymorphic context, initialize its
2116 : value to SOURCE and clear all other fields. */
2117 :
2118 : static ipcp_value<ipa_polymorphic_call_context> *
2119 7952 : allocate_and_init_ipcp_value (ipa_polymorphic_call_context ctx,
2120 : unsigned same_lat_gen_level)
2121 : {
2122 7952 : ipcp_value<ipa_polymorphic_call_context> *val;
2123 :
2124 7952 : val = new (ipcp_poly_ctx_values_pool.allocate ())
2125 7952 : ipcp_value<ipa_polymorphic_call_context>();
2126 7952 : val->value = ctx;
2127 7952 : val->self_recursion_generated_level = same_lat_gen_level;
2128 7952 : return val;
2129 : }
2130 :
2131 : /* Try to add NEWVAL to LAT, potentially creating a new ipcp_value for it. CS,
2132 : SRC_VAL SRC_INDEX and OFFSET are meant for add_source and have the same
2133 : meaning. OFFSET -1 means the source is scalar and not a part of an
2134 : aggregate. If non-NULL, VAL_P records address of existing or newly added
2135 : ipcp_value.
2136 :
2137 : If the value is generated for a self-recursive call as a result of an
2138 : arithmetic pass-through jump-function acting on a value in the same lattice,
2139 : SAME_LAT_GEN_LEVEL must be the length of such chain, otherwise it must be
2140 : zero. If it is non-zero, PARAM_IPA_CP_VALUE_LIST_SIZE limit is ignored. */
2141 :
2142 : template <typename valtype>
2143 : bool
2144 501411 : ipcp_lattice<valtype>::add_value (valtype newval, cgraph_edge *cs,
2145 : ipcp_value<valtype> *src_val,
2146 : int src_idx, HOST_WIDE_INT offset,
2147 : ipcp_value<valtype> **val_p,
2148 : unsigned same_lat_gen_level)
2149 : {
2150 501411 : ipcp_value<valtype> *val, *last_val = NULL;
2151 :
2152 501411 : if (val_p)
2153 1257 : *val_p = NULL;
2154 :
2155 501411 : if (bottom)
2156 : return false;
2157 :
2158 971083 : for (val = values; val; last_val = val, val = val->next)
2159 821080 : if (values_equal_for_ipcp_p (val->value, newval))
2160 : {
2161 348044 : if (val_p)
2162 416 : *val_p = val;
2163 :
2164 348044 : if (val->self_recursion_generated_level < same_lat_gen_level)
2165 179 : val->self_recursion_generated_level = same_lat_gen_level;
2166 :
2167 348044 : if (ipa_edge_within_scc (cs))
2168 : {
2169 : ipcp_value_source<valtype> *s;
2170 48522 : for (s = val->sources; s; s = s->next)
2171 44345 : if (s->cs == cs && s->val == src_val)
2172 : break;
2173 11824 : if (s)
2174 : return false;
2175 : }
2176 :
2177 340397 : val->add_source (cs, src_val, src_idx, offset);
2178 340397 : return false;
2179 : }
2180 :
2181 150003 : if (!same_lat_gen_level && values_count >= opt_for_fn (cs->callee->decl,
2182 : param_ipa_cp_value_list_size))
2183 : {
2184 : /* We can only free sources, not the values themselves, because sources
2185 : of other values in this SCC might point to them. */
2186 12294 : for (val = values; val; val = val->next)
2187 : {
2188 40471 : while (val->sources)
2189 : {
2190 29545 : ipcp_value_source<valtype> *src = val->sources;
2191 29545 : val->sources = src->next;
2192 29545 : ipcp_sources_pool.remove ((ipcp_value_source<tree>*)src);
2193 : }
2194 : }
2195 1368 : values = NULL;
2196 1368 : return set_to_bottom ();
2197 : }
2198 :
2199 148635 : values_count++;
2200 148635 : val = allocate_and_init_ipcp_value (newval, same_lat_gen_level);
2201 148635 : val->add_source (cs, src_val, src_idx, offset);
2202 148635 : val->next = NULL;
2203 :
2204 : /* Add the new value to end of value list, which can reduce iterations
2205 : of propagation stage for recursive function. */
2206 148635 : if (last_val)
2207 45589 : last_val->next = val;
2208 : else
2209 103046 : values = val;
2210 :
2211 148635 : if (val_p)
2212 841 : *val_p = val;
2213 :
2214 : return true;
2215 : }
2216 :
2217 : /* A helper function that returns result of operation specified by OPCODE on
2218 : the value of SRC_VAL. If non-NULL, OPND1_TYPE is expected type for the
2219 : value of SRC_VAL. If the operation is binary, OPND2 is a constant value
2220 : acting as its second operand. OP_TYPE is the type in which the operation is
2221 : performed. */
2222 :
2223 : static tree
2224 21505 : get_val_across_arith_op (enum tree_code opcode,
2225 : tree opnd1_type,
2226 : tree opnd2,
2227 : ipcp_value<tree> *src_val,
2228 : tree op_type)
2229 : {
2230 21505 : tree opnd1 = src_val->value;
2231 :
2232 : /* Skip source values that is incompatible with specified type. */
2233 21505 : if (opnd1_type
2234 21505 : && !useless_type_conversion_p (opnd1_type, TREE_TYPE (opnd1)))
2235 : return NULL_TREE;
2236 :
2237 21505 : return ipa_get_jf_arith_result (opcode, opnd1, opnd2, op_type);
2238 : }
2239 :
2240 : /* Propagate values through an arithmetic transformation described by a jump
2241 : function associated with edge CS, taking values from SRC_LAT and putting
2242 : them into DEST_LAT. OPND1_TYPE, if non-NULL, is the expected type for the
2243 : values in SRC_LAT. OPND2 is a constant value if transformation is a binary
2244 : operation. SRC_OFFSET specifies offset in an aggregate if SRC_LAT describes
2245 : lattice of a part of an aggregate, otherwise it should be -1. SRC_IDX is
2246 : the index of the source parameter. OP_TYPE is the type in which the
2247 : operation is performed and can be NULL when OPCODE is NOP_EXPR. RES_TYPE is
2248 : the value type of result being propagated into. Return true if DEST_LAT
2249 : changed. */
2250 :
2251 : static bool
2252 77174 : propagate_vals_across_arith_jfunc (cgraph_edge *cs,
2253 : enum tree_code opcode,
2254 : tree opnd1_type,
2255 : tree opnd2,
2256 : ipcp_lattice<tree> *src_lat,
2257 : ipcp_lattice<tree> *dest_lat,
2258 : HOST_WIDE_INT src_offset,
2259 : int src_idx,
2260 : tree op_type,
2261 : tree res_type)
2262 : {
2263 77174 : ipcp_value<tree> *src_val;
2264 77174 : bool ret = false;
2265 :
2266 : /* Due to circular dependencies, propagating within an SCC through arithmetic
2267 : transformation would create infinite number of values. But for
2268 : self-feeding recursive function, we could allow propagation in a limited
2269 : count, and this can enable a simple kind of recursive function versioning.
2270 : For other scenario, we would just make lattices bottom. */
2271 77174 : if (opcode != NOP_EXPR && ipa_edge_within_scc (cs))
2272 : {
2273 2184 : int i;
2274 :
2275 2184 : int max_recursive_depth = opt_for_fn(cs->caller->decl,
2276 : param_ipa_cp_max_recursive_depth);
2277 2184 : if (src_lat != dest_lat || max_recursive_depth < 1)
2278 1666 : return dest_lat->set_contains_variable ();
2279 :
2280 : /* No benefit if recursive execution is in low probability. */
2281 1300 : if (cs->sreal_frequency () * 100
2282 2600 : <= ((sreal) 1) * opt_for_fn (cs->caller->decl,
2283 : param_ipa_cp_min_recursive_probability))
2284 89 : return dest_lat->set_contains_variable ();
2285 :
2286 1211 : auto_vec<ipcp_value<tree> *, 8> val_seeds;
2287 :
2288 2258 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2289 : {
2290 : /* Now we do not use self-recursively generated value as propagation
2291 : source, this is absolutely conservative, but could avoid explosion
2292 : of lattice's value space, especially when one recursive function
2293 : calls another recursive. */
2294 1740 : if (src_val->self_recursion_generated_p ())
2295 : {
2296 909 : ipcp_value_source<tree> *s;
2297 :
2298 : /* If the lattice has already been propagated for the call site,
2299 : no need to do that again. */
2300 1422 : for (s = src_val->sources; s; s = s->next)
2301 1206 : if (s->cs == cs)
2302 693 : return dest_lat->set_contains_variable ();
2303 : }
2304 : else
2305 831 : val_seeds.safe_push (src_val);
2306 : }
2307 :
2308 1036 : gcc_assert ((int) val_seeds.length () <= param_ipa_cp_value_list_size);
2309 :
2310 : /* Recursively generate lattice values with a limited count. */
2311 836 : FOR_EACH_VEC_ELT (val_seeds, i, src_val)
2312 : {
2313 1416 : for (int j = 1; j < max_recursive_depth; j++)
2314 : {
2315 1261 : tree cstval = get_val_across_arith_op (opcode, opnd1_type, opnd2,
2316 : src_val, op_type);
2317 1261 : cstval = ipacp_value_safe_for_type (res_type, cstval);
2318 1261 : if (!cstval)
2319 : break;
2320 :
2321 1257 : ret |= dest_lat->add_value (cstval, cs, src_val, src_idx,
2322 : src_offset, &src_val, j);
2323 1257 : gcc_checking_assert (src_val);
2324 : }
2325 : }
2326 518 : ret |= dest_lat->set_contains_variable ();
2327 1211 : }
2328 : else
2329 95359 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2330 : {
2331 : /* Now we do not use self-recursively generated value as propagation
2332 : source, otherwise it is easy to make value space of normal lattice
2333 : overflow. */
2334 20369 : if (src_val->self_recursion_generated_p ())
2335 : {
2336 125 : ret |= dest_lat->set_contains_variable ();
2337 125 : continue;
2338 : }
2339 :
2340 20244 : tree cstval = get_val_across_arith_op (opcode, opnd1_type, opnd2,
2341 : src_val, op_type);
2342 20244 : cstval = ipacp_value_safe_for_type (res_type, cstval);
2343 20244 : if (cstval)
2344 20043 : ret |= dest_lat->add_value (cstval, cs, src_val, src_idx,
2345 : src_offset);
2346 : else
2347 201 : ret |= dest_lat->set_contains_variable ();
2348 : }
2349 :
2350 : return ret;
2351 : }
2352 :
2353 : /* Propagate values through a pass-through jump function JFUNC associated with
2354 : edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
2355 : is the index of the source parameter. PARM_TYPE is the type of the
2356 : parameter to which the result is passed. */
2357 :
2358 : static bool
2359 72539 : propagate_vals_across_pass_through (cgraph_edge *cs, ipa_jump_func *jfunc,
2360 : ipcp_lattice<tree> *src_lat,
2361 : ipcp_lattice<tree> *dest_lat, int src_idx,
2362 : tree parm_type)
2363 : {
2364 72539 : gcc_checking_assert (parm_type);
2365 72539 : enum tree_code opcode = ipa_get_jf_pass_through_operation (jfunc);
2366 72539 : tree op_type = (opcode == NOP_EXPR) ? NULL_TREE
2367 2429 : : ipa_get_jf_pass_through_op_type (jfunc);
2368 72539 : return propagate_vals_across_arith_jfunc (cs, opcode, NULL_TREE,
2369 : ipa_get_jf_pass_through_operand (jfunc),
2370 : src_lat, dest_lat, -1, src_idx, op_type,
2371 72539 : parm_type);
2372 : }
2373 :
2374 : /* Propagate values through an ancestor jump function JFUNC associated with
2375 : edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
2376 : is the index of the source parameter. */
2377 :
2378 : static bool
2379 2206 : propagate_vals_across_ancestor (struct cgraph_edge *cs,
2380 : struct ipa_jump_func *jfunc,
2381 : ipcp_lattice<tree> *src_lat,
2382 : ipcp_lattice<tree> *dest_lat, int src_idx,
2383 : tree param_type)
2384 : {
2385 2206 : ipcp_value<tree> *src_val;
2386 2206 : bool ret = false;
2387 :
2388 2206 : if (ipa_edge_within_scc (cs))
2389 14 : return dest_lat->set_contains_variable ();
2390 :
2391 2507 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2392 : {
2393 315 : tree t = ipa_get_jf_ancestor_result (jfunc, src_val->value);
2394 315 : t = ipacp_value_safe_for_type (param_type, t);
2395 315 : if (t)
2396 257 : ret |= dest_lat->add_value (t, cs, src_val, src_idx);
2397 : else
2398 58 : ret |= dest_lat->set_contains_variable ();
2399 : }
2400 :
2401 : return ret;
2402 : }
2403 :
2404 : /* Propagate scalar values across jump function JFUNC that is associated with
2405 : edge CS and put the values into DEST_LAT. PARM_TYPE is the type of the
2406 : parameter to which the result is passed. */
2407 :
2408 : static bool
2409 3988729 : propagate_scalar_across_jump_function (struct cgraph_edge *cs,
2410 : struct ipa_jump_func *jfunc,
2411 : ipcp_lattice<tree> *dest_lat,
2412 : tree param_type)
2413 : {
2414 3988729 : if (dest_lat->bottom)
2415 : return false;
2416 :
2417 826065 : if (jfunc->type == IPA_JF_CONST)
2418 : {
2419 370687 : tree val = ipa_get_jf_constant (jfunc);
2420 370687 : val = ipacp_value_safe_for_type (param_type, val);
2421 370687 : if (val)
2422 370669 : return dest_lat->add_value (val, cs, NULL, 0);
2423 : else
2424 18 : return dest_lat->set_contains_variable ();
2425 : }
2426 455378 : else if (jfunc->type == IPA_JF_PASS_THROUGH
2427 272074 : || jfunc->type == IPA_JF_ANCESTOR)
2428 : {
2429 187786 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2430 187786 : ipcp_lattice<tree> *src_lat;
2431 187786 : int src_idx;
2432 187786 : bool ret;
2433 :
2434 187786 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2435 183304 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2436 : else
2437 4482 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2438 :
2439 187786 : src_lat = ipa_get_scalar_lat (caller_info, src_idx);
2440 187786 : if (src_lat->bottom)
2441 112894 : return dest_lat->set_contains_variable ();
2442 :
2443 : /* If we would need to clone the caller and cannot, do not propagate. */
2444 74892 : if (!ipcp_versionable_function_p (cs->caller)
2445 74892 : && (src_lat->contains_variable
2446 134 : || (src_lat->values_count > 1)))
2447 147 : return dest_lat->set_contains_variable ();
2448 :
2449 74745 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2450 72539 : ret = propagate_vals_across_pass_through (cs, jfunc, src_lat,
2451 : dest_lat, src_idx,
2452 : param_type);
2453 : else
2454 2206 : ret = propagate_vals_across_ancestor (cs, jfunc, src_lat, dest_lat,
2455 : src_idx, param_type);
2456 :
2457 74745 : if (src_lat->contains_variable)
2458 65336 : ret |= dest_lat->set_contains_variable ();
2459 :
2460 74745 : return ret;
2461 : }
2462 :
2463 : /* TODO: We currently do not handle member method pointers in IPA-CP (we only
2464 : use it for indirect inlining), we should propagate them too. */
2465 267592 : return dest_lat->set_contains_variable ();
2466 : }
2467 :
2468 : /* Propagate scalar values across jump function JFUNC that is associated with
2469 : edge CS and describes argument IDX and put the values into DEST_LAT. */
2470 :
2471 : static bool
2472 3988729 : propagate_context_across_jump_function (cgraph_edge *cs,
2473 : ipa_jump_func *jfunc, int idx,
2474 : ipcp_lattice<ipa_polymorphic_call_context> *dest_lat)
2475 : {
2476 3988729 : if (dest_lat->bottom)
2477 : return false;
2478 922543 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
2479 922543 : bool ret = false;
2480 922543 : bool added_sth = false;
2481 922543 : bool type_preserved = true;
2482 :
2483 922543 : ipa_polymorphic_call_context edge_ctx, *edge_ctx_ptr
2484 936826 : = ipa_get_ith_polymorhic_call_context (args, idx);
2485 :
2486 14283 : if (edge_ctx_ptr)
2487 14283 : edge_ctx = *edge_ctx_ptr;
2488 :
2489 922543 : if (jfunc->type == IPA_JF_PASS_THROUGH
2490 738748 : || jfunc->type == IPA_JF_ANCESTOR)
2491 : {
2492 188373 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2493 188373 : int src_idx;
2494 188373 : ipcp_lattice<ipa_polymorphic_call_context> *src_lat;
2495 :
2496 : /* TODO: Once we figure out how to propagate speculations, it will
2497 : probably be a good idea to switch to speculation if type_preserved is
2498 : not set instead of punting. */
2499 188373 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2500 : {
2501 183795 : if (ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR)
2502 7612 : goto prop_fail;
2503 176183 : type_preserved = ipa_get_jf_pass_through_type_preserved (jfunc);
2504 176183 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2505 : }
2506 : else
2507 : {
2508 4578 : type_preserved = ipa_get_jf_ancestor_type_preserved (jfunc);
2509 4578 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2510 : }
2511 :
2512 180761 : src_lat = ipa_get_poly_ctx_lat (caller_info, src_idx);
2513 : /* If we would need to clone the caller and cannot, do not propagate. */
2514 180761 : if (!ipcp_versionable_function_p (cs->caller)
2515 180761 : && (src_lat->contains_variable
2516 14258 : || (src_lat->values_count > 1)))
2517 2486 : goto prop_fail;
2518 :
2519 178275 : ipcp_value<ipa_polymorphic_call_context> *src_val;
2520 179584 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2521 : {
2522 1309 : ipa_polymorphic_call_context cur = src_val->value;
2523 :
2524 1309 : if (!type_preserved)
2525 883 : cur.possible_dynamic_type_change (cs->in_polymorphic_cdtor);
2526 1309 : if (jfunc->type == IPA_JF_ANCESTOR)
2527 325 : cur.offset_by (ipa_get_jf_ancestor_offset (jfunc));
2528 : /* TODO: In cases we know how the context is going to be used,
2529 : we can improve the result by passing proper OTR_TYPE. */
2530 1309 : cur.combine_with (edge_ctx);
2531 2618 : if (!cur.useless_p ())
2532 : {
2533 838 : if (src_lat->contains_variable
2534 838 : && !edge_ctx.equal_to (cur))
2535 260 : ret |= dest_lat->set_contains_variable ();
2536 838 : ret |= dest_lat->add_value (cur, cs, src_val, src_idx);
2537 838 : added_sth = true;
2538 : }
2539 : }
2540 : }
2541 :
2542 734170 : prop_fail:
2543 188373 : if (!added_sth)
2544 : {
2545 921768 : if (!edge_ctx.useless_p ())
2546 8729 : ret |= dest_lat->add_value (edge_ctx, cs);
2547 : else
2548 913039 : ret |= dest_lat->set_contains_variable ();
2549 : }
2550 :
2551 : return ret;
2552 : }
2553 :
2554 : /* Propagate bits across jfunc that is associated with
2555 : edge cs and update dest_lattice accordingly. */
2556 :
2557 : bool
2558 3988729 : propagate_bits_across_jump_function (cgraph_edge *cs, int idx,
2559 : ipa_jump_func *jfunc,
2560 : ipcp_bits_lattice *dest_lattice)
2561 : {
2562 3988729 : if (dest_lattice->bottom_p ())
2563 : return false;
2564 :
2565 535753 : enum availability availability;
2566 535753 : cgraph_node *callee = cs->callee->function_symbol (&availability);
2567 535753 : ipa_node_params *callee_info = ipa_node_params_sum->get (callee);
2568 535753 : tree parm_type = ipa_get_type (callee_info, idx);
2569 :
2570 : /* For K&R C programs, ipa_get_type() could return NULL_TREE. Avoid the
2571 : transform for these cases. Similarly, we can have bad type mismatches
2572 : with LTO, avoid doing anything with those too. */
2573 535753 : if (!parm_type
2574 535753 : || (!INTEGRAL_TYPE_P (parm_type) && !POINTER_TYPE_P (parm_type)))
2575 : {
2576 29212 : if (dump_file && (dump_flags & TDF_DETAILS))
2577 11 : fprintf (dump_file, "Setting dest_lattice to bottom, because type of "
2578 : "param %i of %s is NULL or unsuitable for bits propagation\n",
2579 11 : idx, cs->callee->dump_name ());
2580 :
2581 29212 : return dest_lattice->set_to_bottom ();
2582 : }
2583 :
2584 506541 : if (jfunc->type == IPA_JF_PASS_THROUGH
2585 406500 : || jfunc->type == IPA_JF_ANCESTOR)
2586 : {
2587 102531 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2588 102531 : tree operand = NULL_TREE;
2589 102531 : tree op_type = NULL_TREE;
2590 102531 : enum tree_code code;
2591 102531 : unsigned src_idx;
2592 102531 : bool keep_null = false;
2593 :
2594 102531 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2595 : {
2596 100041 : code = ipa_get_jf_pass_through_operation (jfunc);
2597 100041 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2598 100041 : if (code != NOP_EXPR)
2599 : {
2600 2087 : operand = ipa_get_jf_pass_through_operand (jfunc);
2601 2087 : op_type = ipa_get_jf_pass_through_op_type (jfunc);
2602 : }
2603 : }
2604 : else
2605 : {
2606 2490 : code = POINTER_PLUS_EXPR;
2607 2490 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2608 2490 : unsigned HOST_WIDE_INT offset
2609 2490 : = ipa_get_jf_ancestor_offset (jfunc) / BITS_PER_UNIT;
2610 2490 : keep_null = (ipa_get_jf_ancestor_keep_null (jfunc) || !offset);
2611 2490 : operand = build_int_cstu (size_type_node, offset);
2612 : }
2613 :
2614 102531 : class ipcp_param_lattices *src_lats
2615 102531 : = ipa_get_parm_lattices (caller_info, src_idx);
2616 :
2617 : /* Try to propagate bits if src_lattice is bottom, but jfunc is known.
2618 : for eg consider:
2619 : int f(int x)
2620 : {
2621 : g (x & 0xff);
2622 : }
2623 : Assume lattice for x is bottom, however we can still propagate
2624 : result of x & 0xff == 0xff, which gets computed during ccp1 pass
2625 : and we store it in jump function during analysis stage. */
2626 :
2627 102531 : if (!src_lats->bits_lattice.bottom_p ()
2628 102531 : && !src_lats->bits_lattice.recipient_only_p ())
2629 : {
2630 21510 : if (!op_type)
2631 20335 : op_type = ipa_get_type (caller_info, src_idx);
2632 :
2633 21510 : unsigned precision = TYPE_PRECISION (op_type);
2634 21510 : signop sgn = TYPE_SIGN (op_type);
2635 21510 : bool drop_all_ones
2636 21510 : = keep_null && !src_lats->bits_lattice.known_nonzero_p ();
2637 :
2638 21510 : return dest_lattice->meet_with (src_lats->bits_lattice, precision,
2639 21510 : sgn, code, operand, drop_all_ones);
2640 : }
2641 : }
2642 :
2643 485031 : value_range vr (parm_type);
2644 485031 : if (jfunc->m_vr)
2645 : {
2646 413151 : jfunc->m_vr->get_vrange (vr);
2647 413151 : if (!vr.undefined_p () && !vr.varying_p ())
2648 : {
2649 413151 : irange_bitmask bm = vr.get_bitmask ();
2650 413151 : widest_int mask
2651 413151 : = widest_int::from (bm.mask (), TYPE_SIGN (parm_type));
2652 413151 : widest_int value
2653 413151 : = widest_int::from (bm.value (), TYPE_SIGN (parm_type));
2654 413151 : return dest_lattice->meet_with (value, mask,
2655 413151 : TYPE_PRECISION (parm_type));
2656 413151 : }
2657 : }
2658 71880 : return dest_lattice->set_to_bottom ();
2659 485031 : }
2660 :
2661 : /* Propagate value range across jump function JFUNC that is associated with
2662 : edge CS with param of callee of PARAM_TYPE and update DEST_PLATS
2663 : accordingly. */
2664 :
2665 : static bool
2666 3987884 : propagate_vr_across_jump_function (cgraph_edge *cs, ipa_jump_func *jfunc,
2667 : class ipcp_param_lattices *dest_plats,
2668 : tree param_type)
2669 : {
2670 3987884 : ipcp_vr_lattice *dest_lat = &dest_plats->m_value_range;
2671 :
2672 3987884 : if (dest_lat->bottom_p ())
2673 : return false;
2674 :
2675 629098 : if (!param_type
2676 629098 : || !ipa_vr_supported_type_p (param_type))
2677 29152 : return dest_lat->set_to_bottom ();
2678 :
2679 599946 : value_range vr (param_type);
2680 599946 : vr.set_varying (param_type);
2681 599946 : if (jfunc->m_vr)
2682 517217 : ipa_vr_operation_and_type_effects (vr, *jfunc->m_vr, NOP_EXPR,
2683 : param_type,
2684 517217 : jfunc->m_vr->type ());
2685 :
2686 599946 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2687 : {
2688 94193 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2689 94193 : int src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2690 94193 : class ipcp_param_lattices *src_lats
2691 94193 : = ipa_get_parm_lattices (caller_info, src_idx);
2692 94193 : tree operand_type = ipa_get_type (caller_info, src_idx);
2693 :
2694 94193 : if (src_lats->m_value_range.bottom_p ()
2695 94193 : || src_lats->m_value_range.recipient_only_p ())
2696 78441 : return dest_lat->set_to_bottom ();
2697 :
2698 15752 : if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR
2699 15752 : || !ipa_edge_within_scc (cs))
2700 15257 : ipa_vr_intersect_with_arith_jfunc (vr, jfunc, cs->caller,
2701 15257 : src_lats->m_value_range.m_vr,
2702 : operand_type, param_type);
2703 : }
2704 :
2705 521505 : if (!vr.undefined_p () && !vr.varying_p ())
2706 493228 : return dest_lat->meet_with (vr);
2707 : else
2708 28277 : return dest_lat->set_to_bottom ();
2709 599946 : }
2710 :
2711 : /* If DEST_PLATS already has aggregate items, check that aggs_by_ref matches
2712 : NEW_AGGS_BY_REF and if not, mark all aggs as bottoms and return true (in all
2713 : other cases, return false). If there are no aggregate items, set
2714 : aggs_by_ref to NEW_AGGS_BY_REF. */
2715 :
2716 : static bool
2717 42103 : set_check_aggs_by_ref (class ipcp_param_lattices *dest_plats,
2718 : bool new_aggs_by_ref)
2719 : {
2720 0 : if (dest_plats->aggs)
2721 : {
2722 22376 : if (dest_plats->aggs_by_ref != new_aggs_by_ref)
2723 : {
2724 0 : set_agg_lats_to_bottom (dest_plats);
2725 0 : return true;
2726 : }
2727 : }
2728 : else
2729 19727 : dest_plats->aggs_by_ref = new_aggs_by_ref;
2730 : return false;
2731 : }
2732 :
2733 : /* Walk aggregate lattices in DEST_PLATS from ***AGLAT on, until ***aglat is an
2734 : already existing lattice for the given OFFSET and SIZE, marking all skipped
2735 : lattices as containing variable and checking for overlaps. If there is no
2736 : already existing lattice for the OFFSET and VAL_SIZE, create one, initialize
2737 : it with offset, size and contains_variable to PRE_EXISTING, and return true,
2738 : unless there are too many already. If there are two many, return false. If
2739 : there are overlaps turn whole DEST_PLATS to bottom and return false. If any
2740 : skipped lattices were newly marked as containing variable, set *CHANGE to
2741 : true. MAX_AGG_ITEMS is the maximum number of lattices. */
2742 :
2743 : static bool
2744 114608 : merge_agg_lats_step (class ipcp_param_lattices *dest_plats,
2745 : HOST_WIDE_INT offset, HOST_WIDE_INT val_size,
2746 : struct ipcp_agg_lattice ***aglat,
2747 : bool pre_existing, bool *change, int max_agg_items)
2748 : {
2749 114608 : gcc_checking_assert (offset >= 0);
2750 :
2751 119041 : while (**aglat && (**aglat)->offset < offset)
2752 : {
2753 4433 : if ((**aglat)->offset + (**aglat)->size > offset)
2754 : {
2755 0 : set_agg_lats_to_bottom (dest_plats);
2756 0 : return false;
2757 : }
2758 4433 : *change |= (**aglat)->set_contains_variable ();
2759 4433 : *aglat = &(**aglat)->next;
2760 : }
2761 :
2762 114608 : if (**aglat && (**aglat)->offset == offset)
2763 : {
2764 56999 : if ((**aglat)->size != val_size)
2765 : {
2766 13 : set_agg_lats_to_bottom (dest_plats);
2767 13 : return false;
2768 : }
2769 56986 : gcc_assert (!(**aglat)->next
2770 : || (**aglat)->next->offset >= offset + val_size);
2771 : return true;
2772 : }
2773 : else
2774 : {
2775 57609 : struct ipcp_agg_lattice *new_al;
2776 :
2777 57609 : if (**aglat && (**aglat)->offset < offset + val_size)
2778 : {
2779 3 : set_agg_lats_to_bottom (dest_plats);
2780 3 : return false;
2781 : }
2782 57606 : if (dest_plats->aggs_count == max_agg_items)
2783 : return false;
2784 57567 : dest_plats->aggs_count++;
2785 57567 : new_al = ipcp_agg_lattice_pool.allocate ();
2786 :
2787 57567 : new_al->offset = offset;
2788 57567 : new_al->size = val_size;
2789 57567 : new_al->contains_variable = pre_existing;
2790 :
2791 57567 : new_al->next = **aglat;
2792 57567 : **aglat = new_al;
2793 57567 : return true;
2794 : }
2795 : }
2796 :
2797 : /* Set all AGLAT and all other aggregate lattices reachable by next pointers as
2798 : containing an unknown value. */
2799 :
2800 : static bool
2801 42085 : set_chain_of_aglats_contains_variable (struct ipcp_agg_lattice *aglat)
2802 : {
2803 42085 : bool ret = false;
2804 44664 : while (aglat)
2805 : {
2806 2579 : ret |= aglat->set_contains_variable ();
2807 2579 : aglat = aglat->next;
2808 : }
2809 42085 : return ret;
2810 : }
2811 :
2812 : /* Merge existing aggregate lattices in SRC_PLATS to DEST_PLATS, subtracting
2813 : DELTA_OFFSET. CS is the call graph edge and SRC_IDX the index of the source
2814 : parameter used for lattice value sources. Return true if DEST_PLATS changed
2815 : in any way. */
2816 :
2817 : static bool
2818 3911 : merge_aggregate_lattices (struct cgraph_edge *cs,
2819 : class ipcp_param_lattices *dest_plats,
2820 : class ipcp_param_lattices *src_plats,
2821 : int src_idx, HOST_WIDE_INT offset_delta)
2822 : {
2823 3911 : bool pre_existing = dest_plats->aggs != NULL;
2824 3911 : struct ipcp_agg_lattice **dst_aglat;
2825 3911 : bool ret = false;
2826 :
2827 3911 : if (set_check_aggs_by_ref (dest_plats, src_plats->aggs_by_ref))
2828 0 : return true;
2829 3911 : if (src_plats->aggs_bottom)
2830 2 : return set_agg_lats_contain_variable (dest_plats);
2831 3909 : if (src_plats->aggs_contain_variable)
2832 2308 : ret |= set_agg_lats_contain_variable (dest_plats);
2833 3909 : dst_aglat = &dest_plats->aggs;
2834 :
2835 3909 : int max_agg_items = opt_for_fn (cs->callee->function_symbol ()->decl,
2836 : param_ipa_max_agg_items);
2837 3909 : for (struct ipcp_agg_lattice *src_aglat = src_plats->aggs;
2838 11555 : src_aglat;
2839 7646 : src_aglat = src_aglat->next)
2840 : {
2841 7646 : HOST_WIDE_INT new_offset = src_aglat->offset - offset_delta;
2842 :
2843 7646 : if (new_offset < 0)
2844 51 : continue;
2845 7595 : if (merge_agg_lats_step (dest_plats, new_offset, src_aglat->size,
2846 : &dst_aglat, pre_existing, &ret, max_agg_items))
2847 : {
2848 7591 : struct ipcp_agg_lattice *new_al = *dst_aglat;
2849 :
2850 7591 : dst_aglat = &(*dst_aglat)->next;
2851 7591 : if (src_aglat->bottom)
2852 : {
2853 0 : ret |= new_al->set_contains_variable ();
2854 0 : continue;
2855 : }
2856 7591 : if (src_aglat->contains_variable)
2857 4467 : ret |= new_al->set_contains_variable ();
2858 7591 : for (ipcp_value<tree> *val = src_aglat->values;
2859 11780 : val;
2860 4189 : val = val->next)
2861 4189 : ret |= new_al->add_value (val->value, cs, val, src_idx,
2862 : src_aglat->offset);
2863 : }
2864 4 : else if (dest_plats->aggs_bottom)
2865 : return true;
2866 : }
2867 3909 : ret |= set_chain_of_aglats_contains_variable (*dst_aglat);
2868 3909 : return ret;
2869 : }
2870 :
2871 : /* Determine whether there is anything to propagate FROM SRC_PLATS through a
2872 : pass-through JFUNC and if so, whether it has conform and conforms to the
2873 : rules about propagating values passed by reference. */
2874 :
2875 : static bool
2876 176024 : agg_pass_through_permissible_p (class ipcp_param_lattices *src_plats,
2877 : struct ipa_jump_func *jfunc)
2878 : {
2879 176024 : return src_plats->aggs
2880 176024 : && (!src_plats->aggs_by_ref
2881 5071 : || ipa_get_jf_pass_through_agg_preserved (jfunc));
2882 : }
2883 :
2884 : /* Propagate values through ITEM, jump function for a part of an aggregate,
2885 : into corresponding aggregate lattice AGLAT. CS is the call graph edge
2886 : associated with the jump function. Return true if AGLAT changed in any
2887 : way. */
2888 :
2889 : static bool
2890 106962 : propagate_aggregate_lattice (struct cgraph_edge *cs,
2891 : struct ipa_agg_jf_item *item,
2892 : struct ipcp_agg_lattice *aglat)
2893 : {
2894 106962 : class ipa_node_params *caller_info;
2895 106962 : class ipcp_param_lattices *src_plats;
2896 106962 : struct ipcp_lattice<tree> *src_lat;
2897 106962 : HOST_WIDE_INT src_offset;
2898 106962 : int src_idx;
2899 106962 : tree load_type;
2900 106962 : bool ret;
2901 :
2902 106962 : if (item->jftype == IPA_JF_CONST)
2903 : {
2904 95429 : tree value = item->value.constant;
2905 :
2906 95429 : gcc_checking_assert (is_gimple_ip_invariant (value));
2907 95429 : return aglat->add_value (value, cs, NULL, 0);
2908 : }
2909 :
2910 11533 : gcc_checking_assert (item->jftype == IPA_JF_PASS_THROUGH
2911 : || item->jftype == IPA_JF_LOAD_AGG);
2912 :
2913 11533 : caller_info = ipa_node_params_sum->get (cs->caller);
2914 11533 : src_idx = item->value.pass_through.formal_id;
2915 11533 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
2916 :
2917 11533 : if (item->jftype == IPA_JF_PASS_THROUGH)
2918 : {
2919 3394 : load_type = NULL_TREE;
2920 3394 : src_lat = &src_plats->itself;
2921 3394 : src_offset = -1;
2922 : }
2923 : else
2924 : {
2925 8139 : HOST_WIDE_INT load_offset = item->value.load_agg.offset;
2926 8139 : struct ipcp_agg_lattice *src_aglat;
2927 :
2928 12771 : for (src_aglat = src_plats->aggs; src_aglat; src_aglat = src_aglat->next)
2929 8447 : if (src_aglat->offset >= load_offset)
2930 : break;
2931 :
2932 8139 : load_type = item->value.load_agg.type;
2933 8139 : if (!src_aglat
2934 3815 : || src_aglat->offset > load_offset
2935 3477 : || src_aglat->size != tree_to_shwi (TYPE_SIZE (load_type))
2936 11616 : || src_plats->aggs_by_ref != item->value.load_agg.by_ref)
2937 4662 : return aglat->set_contains_variable ();
2938 :
2939 : src_lat = src_aglat;
2940 : src_offset = load_offset;
2941 : }
2942 :
2943 6871 : if (src_lat->bottom
2944 6871 : || (!ipcp_versionable_function_p (cs->caller)
2945 6871 : && !src_lat->is_single_const ()))
2946 2236 : return aglat->set_contains_variable ();
2947 :
2948 4635 : ret = propagate_vals_across_arith_jfunc (cs,
2949 : item->value.pass_through.operation,
2950 : load_type,
2951 : item->value.pass_through.operand,
2952 : src_lat, aglat,
2953 : src_offset,
2954 : src_idx,
2955 : item->value.pass_through.op_type,
2956 : item->type);
2957 :
2958 4635 : if (src_lat->contains_variable)
2959 2634 : ret |= aglat->set_contains_variable ();
2960 :
2961 : return ret;
2962 : }
2963 :
2964 : /* Propagate scalar values across jump function JFUNC that is associated with
2965 : edge CS and put the values into DEST_LAT. */
2966 :
2967 : static bool
2968 3988729 : propagate_aggs_across_jump_function (struct cgraph_edge *cs,
2969 : struct ipa_jump_func *jfunc,
2970 : class ipcp_param_lattices *dest_plats)
2971 : {
2972 3988729 : bool ret = false;
2973 :
2974 3988729 : if (dest_plats->aggs_bottom)
2975 : return false;
2976 :
2977 921328 : if (jfunc->type == IPA_JF_PASS_THROUGH
2978 921328 : && ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
2979 : {
2980 176024 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2981 176024 : int src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2982 176024 : class ipcp_param_lattices *src_plats;
2983 :
2984 176024 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
2985 176024 : if (agg_pass_through_permissible_p (src_plats, jfunc))
2986 : {
2987 : /* Currently we do not produce clobber aggregate jump
2988 : functions, replace with merging when we do. */
2989 3781 : gcc_assert (!jfunc->agg.items);
2990 3781 : ret |= merge_aggregate_lattices (cs, dest_plats, src_plats,
2991 : src_idx, 0);
2992 3781 : return ret;
2993 : }
2994 : }
2995 745304 : else if (jfunc->type == IPA_JF_ANCESTOR
2996 745304 : && ipa_get_jf_ancestor_agg_preserved (jfunc))
2997 : {
2998 1203 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2999 1203 : int src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
3000 1203 : class ipcp_param_lattices *src_plats;
3001 :
3002 1203 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
3003 1203 : if (src_plats->aggs && src_plats->aggs_by_ref)
3004 : {
3005 : /* Currently we do not produce clobber aggregate jump
3006 : functions, replace with merging when we do. */
3007 130 : gcc_assert (!jfunc->agg.items);
3008 130 : ret |= merge_aggregate_lattices (cs, dest_plats, src_plats, src_idx,
3009 : ipa_get_jf_ancestor_offset (jfunc));
3010 : }
3011 1073 : else if (!src_plats->aggs_by_ref)
3012 1069 : ret |= set_agg_lats_to_bottom (dest_plats);
3013 : else
3014 4 : ret |= set_agg_lats_contain_variable (dest_plats);
3015 1203 : return ret;
3016 : }
3017 :
3018 916344 : if (jfunc->agg.items)
3019 : {
3020 38192 : bool pre_existing = dest_plats->aggs != NULL;
3021 38192 : struct ipcp_agg_lattice **aglat = &dest_plats->aggs;
3022 38192 : struct ipa_agg_jf_item *item;
3023 38192 : int i;
3024 :
3025 38192 : if (set_check_aggs_by_ref (dest_plats, jfunc->agg.by_ref))
3026 16 : return true;
3027 :
3028 38192 : int max_agg_items = opt_for_fn (cs->callee->function_symbol ()->decl,
3029 : param_ipa_max_agg_items);
3030 145510 : FOR_EACH_VEC_ELT (*jfunc->agg.items, i, item)
3031 : {
3032 107334 : HOST_WIDE_INT val_size;
3033 :
3034 107334 : if (item->offset < 0 || item->jftype == IPA_JF_UNKNOWN)
3035 321 : continue;
3036 107013 : val_size = tree_to_shwi (TYPE_SIZE (item->type));
3037 :
3038 107013 : if (merge_agg_lats_step (dest_plats, item->offset, val_size,
3039 : &aglat, pre_existing, &ret, max_agg_items))
3040 : {
3041 106962 : ret |= propagate_aggregate_lattice (cs, item, *aglat);
3042 106962 : aglat = &(*aglat)->next;
3043 : }
3044 51 : else if (dest_plats->aggs_bottom)
3045 : return true;
3046 : }
3047 :
3048 76352 : ret |= set_chain_of_aglats_contains_variable (*aglat);
3049 : }
3050 : else
3051 878152 : ret |= set_agg_lats_contain_variable (dest_plats);
3052 :
3053 916328 : return ret;
3054 : }
3055 :
3056 : /* Return true if on the way cfrom CS->caller to the final (non-alias and
3057 : non-thunk) destination, the call passes through a thunk. */
3058 :
3059 : static bool
3060 1982317 : call_passes_through_thunk (cgraph_edge *cs)
3061 : {
3062 1982317 : cgraph_node *alias_or_thunk = cs->callee;
3063 2125170 : while (alias_or_thunk->alias)
3064 142853 : alias_or_thunk = alias_or_thunk->get_alias_target ();
3065 1982317 : return alias_or_thunk->thunk;
3066 : }
3067 :
3068 : /* Propagate constants from the caller to the callee of CS. INFO describes the
3069 : caller. */
3070 :
3071 : static bool
3072 5383387 : propagate_constants_across_call (struct cgraph_edge *cs)
3073 : {
3074 5383387 : class ipa_node_params *callee_info;
3075 5383387 : enum availability availability;
3076 5383387 : cgraph_node *callee;
3077 5383387 : class ipa_edge_args *args;
3078 5383387 : bool ret = false;
3079 5383387 : int i, args_count, parms_count;
3080 :
3081 5383387 : callee = cs->callee->function_symbol (&availability);
3082 5383387 : if (!callee->definition)
3083 : return false;
3084 2005503 : gcc_checking_assert (callee->has_gimple_body_p ());
3085 2005503 : callee_info = ipa_node_params_sum->get (callee);
3086 2005503 : if (!callee_info)
3087 : return false;
3088 :
3089 1996999 : args = ipa_edge_args_sum->get (cs);
3090 1996999 : parms_count = ipa_get_param_count (callee_info);
3091 1802057 : if (parms_count == 0)
3092 : return false;
3093 1802057 : if (!args
3094 1801771 : || !opt_for_fn (cs->caller->decl, flag_ipa_cp)
3095 3603828 : || !opt_for_fn (cs->caller->decl, optimize))
3096 : {
3097 857 : for (i = 0; i < parms_count; i++)
3098 571 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info,
3099 : i));
3100 : return ret;
3101 : }
3102 1801771 : args_count = ipa_get_cs_argument_count (args);
3103 :
3104 : /* If this call goes through a thunk we must not propagate to the first (0th)
3105 : parameter. However, we might need to uncover a thunk from below a series
3106 : of aliases first. */
3107 1801771 : if (call_passes_through_thunk (cs))
3108 : {
3109 227 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info,
3110 : 0));
3111 227 : i = 1;
3112 : }
3113 : else
3114 : i = 0;
3115 :
3116 5934083 : for (; (i < args_count) && (i < parms_count); i++)
3117 : {
3118 4132312 : struct ipa_jump_func *jump_func = ipa_get_ith_jump_func (args, i);
3119 4132312 : class ipcp_param_lattices *dest_plats;
3120 4132312 : tree param_type = ipa_get_type (callee_info, i);
3121 :
3122 4132312 : dest_plats = ipa_get_parm_lattices (callee_info, i);
3123 4132312 : if (availability == AVAIL_INTERPOSABLE)
3124 143583 : ret |= set_all_contains_variable (dest_plats);
3125 : else
3126 : {
3127 3988729 : ret |= propagate_scalar_across_jump_function (cs, jump_func,
3128 : &dest_plats->itself,
3129 : param_type);
3130 3988729 : ret |= propagate_context_across_jump_function (cs, jump_func, i,
3131 : &dest_plats->ctxlat);
3132 3988729 : ret
3133 3988729 : |= propagate_bits_across_jump_function (cs, i, jump_func,
3134 : &dest_plats->bits_lattice);
3135 3988729 : ret |= propagate_aggs_across_jump_function (cs, jump_func,
3136 : dest_plats);
3137 3988729 : if (opt_for_fn (callee->decl, flag_ipa_vrp))
3138 3987884 : ret |= propagate_vr_across_jump_function (cs, jump_func,
3139 : dest_plats, param_type);
3140 : else
3141 845 : ret |= dest_plats->m_value_range.set_to_bottom ();
3142 : }
3143 : }
3144 1801954 : for (; i < parms_count; i++)
3145 183 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info, i));
3146 :
3147 : return ret;
3148 : }
3149 :
3150 : /* If an indirect edge IE can be turned into a direct one based on KNOWN_VALS
3151 : KNOWN_CONTEXTS, and known aggregates either in AVS or KNOWN_AGGS return
3152 : the destination. The latter three can be NULL. If AGG_REPS is not NULL,
3153 : KNOWN_AGGS is ignored. */
3154 :
3155 : static tree
3156 1524093 : ipa_get_indirect_edge_target_1 (struct cgraph_edge *ie,
3157 : const vec<tree> &known_csts,
3158 : const vec<ipa_polymorphic_call_context> &known_contexts,
3159 : const ipa_argagg_value_list &avs,
3160 : bool *speculative)
3161 : {
3162 1524093 : int param_index = ie->indirect_info->param_index;
3163 1524093 : *speculative = false;
3164 :
3165 1524093 : if (param_index == -1)
3166 : return NULL_TREE;
3167 :
3168 596744 : if (cgraph_simple_indirect_info *sii
3169 596744 : = dyn_cast <cgraph_simple_indirect_info *> (ie->indirect_info))
3170 : {
3171 300780 : tree t = NULL;
3172 :
3173 300780 : if (sii->agg_contents)
3174 : {
3175 68757 : t = NULL;
3176 68757 : if ((unsigned) param_index < known_csts.length ()
3177 68757 : && known_csts[param_index])
3178 62715 : t = ipa_find_agg_cst_from_init (known_csts[param_index],
3179 : sii->offset,
3180 : sii->by_ref);
3181 :
3182 68757 : if (!t && sii->guaranteed_unmodified)
3183 61773 : t = avs.get_value (param_index, sii->offset / BITS_PER_UNIT,
3184 : sii->by_ref);
3185 : }
3186 232023 : else if ((unsigned) param_index < known_csts.length ())
3187 232023 : t = known_csts[param_index];
3188 :
3189 300725 : if (t
3190 204711 : && TREE_CODE (t) == ADDR_EXPR
3191 505221 : && TREE_CODE (TREE_OPERAND (t, 0)) == FUNCTION_DECL)
3192 204496 : return TREE_OPERAND (t, 0);
3193 : else
3194 96284 : return NULL_TREE;
3195 : }
3196 :
3197 295964 : if (!opt_for_fn (ie->caller->decl, flag_devirtualize))
3198 : return NULL_TREE;
3199 :
3200 295964 : cgraph_polymorphic_indirect_info *pii
3201 295964 : = as_a <cgraph_polymorphic_indirect_info *> (ie->indirect_info);
3202 295964 : if (!pii->usable_p ())
3203 : return NULL_TREE;
3204 :
3205 295964 : HOST_WIDE_INT anc_offset = pii->offset;
3206 295964 : tree t = NULL;
3207 295964 : tree target = NULL;
3208 295964 : if ((unsigned) param_index < known_csts.length ()
3209 295964 : && known_csts[param_index])
3210 17876 : t = ipa_find_agg_cst_from_init (known_csts[param_index], anc_offset, true);
3211 :
3212 : /* Try to work out value of virtual table pointer value in replacements. */
3213 : /* or known aggregate values. */
3214 17876 : if (!t)
3215 295955 : t = avs.get_value (param_index, anc_offset / BITS_PER_UNIT, true);
3216 :
3217 : /* If we found the virtual table pointer, lookup the target. */
3218 295955 : if (t)
3219 : {
3220 7817 : tree vtable;
3221 7817 : unsigned HOST_WIDE_INT offset;
3222 7817 : if (vtable_pointer_value_to_vtable (t, &vtable, &offset))
3223 : {
3224 7817 : bool can_refer;
3225 7817 : target = gimple_get_virt_method_for_vtable (pii->otr_token, vtable,
3226 : offset, &can_refer);
3227 7817 : if (can_refer)
3228 : {
3229 7754 : if (!target
3230 7754 : || fndecl_built_in_p (target, BUILT_IN_UNREACHABLE)
3231 15388 : || !possible_polymorphic_call_target_p
3232 7634 : (ie, cgraph_node::get (target)))
3233 : {
3234 : /* Do not speculate builtin_unreachable, it is stupid! */
3235 237 : if (pii->vptr_changed)
3236 6277 : return NULL;
3237 237 : target = ipa_impossible_devirt_target (ie, target);
3238 : }
3239 7754 : *speculative = pii->vptr_changed;
3240 7754 : if (!*speculative)
3241 : return target;
3242 : }
3243 : }
3244 : }
3245 :
3246 : /* Do we know the constant value of pointer? */
3247 289687 : if (!t && (unsigned) param_index < known_csts.length ())
3248 43529 : t = known_csts[param_index];
3249 :
3250 289687 : ipa_polymorphic_call_context context;
3251 289687 : if (known_contexts.length () > (unsigned int) param_index)
3252 : {
3253 289313 : context = known_contexts[param_index];
3254 289313 : context.offset_by (anc_offset);
3255 289313 : if (pii->vptr_changed)
3256 46753 : context.possible_dynamic_type_change (ie->in_polymorphic_cdtor,
3257 : pii->otr_type);
3258 289313 : if (t)
3259 : {
3260 12249 : ipa_polymorphic_call_context ctx2
3261 12249 : = ipa_polymorphic_call_context (t, pii->otr_type, anc_offset);
3262 24498 : if (!ctx2.useless_p ())
3263 10714 : context.combine_with (ctx2, pii->otr_type);
3264 : }
3265 : }
3266 374 : else if (t)
3267 : {
3268 23 : context = ipa_polymorphic_call_context (t, pii->otr_type, anc_offset);
3269 23 : if (pii->vptr_changed)
3270 8 : context.possible_dynamic_type_change (ie->in_polymorphic_cdtor,
3271 : pii->otr_type);
3272 : }
3273 : else
3274 : return NULL_TREE;
3275 :
3276 289336 : vec <cgraph_node *>targets;
3277 289336 : bool final;
3278 :
3279 289336 : targets = possible_polymorphic_call_targets (pii->otr_type, pii->otr_token,
3280 : context, &final);
3281 300933 : if (!final || targets.length () > 1)
3282 : {
3283 278413 : struct cgraph_node *node;
3284 278413 : if (*speculative)
3285 : return target;
3286 278384 : if (!opt_for_fn (ie->caller->decl, flag_devirtualize_speculatively)
3287 278384 : || ie->speculative || !ie->maybe_hot_p ())
3288 193286 : return NULL;
3289 85098 : node = try_speculative_devirtualization (pii->otr_type, pii->otr_token,
3290 : context);
3291 85098 : if (node)
3292 : {
3293 662 : *speculative = true;
3294 662 : target = node->decl;
3295 : }
3296 : else
3297 : return NULL;
3298 : }
3299 : else
3300 : {
3301 10923 : *speculative = false;
3302 10923 : if (targets.length () == 1)
3303 10884 : target = targets[0]->decl;
3304 : else
3305 39 : target = ipa_impossible_devirt_target (ie, NULL_TREE);
3306 : }
3307 :
3308 11585 : if (target && !possible_polymorphic_call_target_p (ie,
3309 : cgraph_node::get (target)))
3310 : {
3311 48 : if (*speculative)
3312 : return NULL;
3313 40 : target = ipa_impossible_devirt_target (ie, target);
3314 : }
3315 :
3316 : return target;
3317 : }
3318 :
3319 : /* If an indirect edge IE can be turned into a direct one based on data in
3320 : AVALS, return the destination. Store into *SPECULATIVE a boolean determinig
3321 : whether the discovered target is only speculative guess. */
3322 :
3323 : tree
3324 1448278 : ipa_get_indirect_edge_target (struct cgraph_edge *ie,
3325 : ipa_call_arg_values *avals,
3326 : bool *speculative)
3327 : {
3328 1448278 : ipa_argagg_value_list avl (avals);
3329 1448278 : return ipa_get_indirect_edge_target_1 (ie, avals->m_known_vals,
3330 1448278 : avals->m_known_contexts,
3331 1448278 : avl, speculative);
3332 : }
3333 :
3334 : /* Calculate devirtualization time bonus for NODE, assuming we know information
3335 : about arguments stored in AVALS.
3336 :
3337 : FIXME: This function will also consider devirtualization of calls that are
3338 : known to be dead in the clone. */
3339 :
3340 : static sreal
3341 444927 : devirtualization_time_bonus (struct cgraph_node *node,
3342 : ipa_auto_call_arg_values *avals)
3343 : {
3344 444927 : struct cgraph_edge *ie;
3345 444927 : sreal res = 0;
3346 :
3347 518822 : for (ie = node->indirect_calls; ie; ie = ie->next_callee)
3348 : {
3349 73895 : struct cgraph_node *callee;
3350 73895 : class ipa_fn_summary *isummary;
3351 73895 : enum availability avail;
3352 73895 : tree target;
3353 73895 : bool speculative;
3354 :
3355 73895 : ipa_argagg_value_list avl (avals);
3356 73895 : target = ipa_get_indirect_edge_target_1 (ie, avals->m_known_vals,
3357 : avals->m_known_contexts,
3358 : avl, &speculative);
3359 73895 : if (!target)
3360 72910 : continue;
3361 :
3362 : /* Only bare minimum benefit for clearly un-inlineable targets. */
3363 3236 : res = res + ie->combined_sreal_frequency ();
3364 3236 : callee = cgraph_node::get (target);
3365 3236 : if (!callee || !callee->definition)
3366 624 : continue;
3367 2612 : callee = callee->function_symbol (&avail);
3368 2612 : if (avail < AVAIL_AVAILABLE)
3369 0 : continue;
3370 2612 : isummary = ipa_fn_summaries->get (callee);
3371 2612 : if (!isummary || !isummary->inlinable)
3372 66 : continue;
3373 :
3374 2546 : int savings = 0;
3375 2546 : int size = ipa_size_summaries->get (callee)->size;
3376 : /* FIXME: The values below need re-considering and perhaps also
3377 : integrating into the cost metrics, at lest in some very basic way. */
3378 2546 : int max_inline_insns_auto
3379 2546 : = opt_for_fn (callee->decl, param_max_inline_insns_auto);
3380 2546 : if (size <= max_inline_insns_auto / 4)
3381 403 : savings = 31 / ((int)speculative + 1);
3382 2143 : else if (size <= max_inline_insns_auto / 2)
3383 392 : savings = 15 / ((int)speculative + 1);
3384 3312 : else if (size <= max_inline_insns_auto
3385 1751 : || DECL_DECLARED_INLINE_P (callee->decl))
3386 190 : savings = 7 / ((int)speculative + 1);
3387 : else
3388 1561 : continue;
3389 985 : res = res + ie->combined_sreal_frequency () * (sreal) savings;
3390 : }
3391 :
3392 444927 : return res;
3393 : }
3394 :
3395 : /* Return time bonus incurred because of hints stored in ESTIMATES. */
3396 :
3397 : static sreal
3398 226792 : hint_time_bonus (cgraph_node *node, const ipa_call_estimates &estimates)
3399 : {
3400 226792 : sreal result = 0;
3401 226792 : ipa_hints hints = estimates.hints;
3402 226792 : if (hints & (INLINE_HINT_loop_iterations | INLINE_HINT_loop_stride))
3403 25534 : result += opt_for_fn (node->decl, param_ipa_cp_loop_hint_bonus);
3404 :
3405 226792 : sreal bonus_for_one = opt_for_fn (node->decl, param_ipa_cp_loop_hint_bonus);
3406 :
3407 226792 : if (hints & INLINE_HINT_loop_iterations)
3408 17346 : result += estimates.loops_with_known_iterations * bonus_for_one;
3409 :
3410 226792 : if (hints & INLINE_HINT_loop_stride)
3411 10675 : result += estimates.loops_with_known_strides * bonus_for_one;
3412 :
3413 226792 : return result;
3414 : }
3415 :
3416 : /* If there is a reason to penalize the function described by INFO in the
3417 : cloning goodness evaluation, do so. */
3418 :
3419 : static inline sreal
3420 101762 : incorporate_penalties (cgraph_node *node, ipa_node_params *info,
3421 : sreal evaluation)
3422 : {
3423 101762 : if (info->node_within_scc && !info->node_is_self_scc)
3424 1710 : evaluation = (evaluation
3425 1710 : * (100 - opt_for_fn (node->decl,
3426 3420 : param_ipa_cp_recursion_penalty))) / 100;
3427 :
3428 101762 : if (info->node_calling_single_call)
3429 6897 : evaluation = (evaluation
3430 6897 : * (100 - opt_for_fn (node->decl,
3431 6897 : param_ipa_cp_single_call_penalty)))
3432 13794 : / 100;
3433 :
3434 101762 : return evaluation;
3435 : }
3436 :
3437 : /* Return true if cloning NODE is a good idea, given the estimated TIME_BENEFIT
3438 : and SIZE_COST and with the sum of frequencies of incoming edges to the
3439 : potential new clone in FREQUENCIES. CUR_SWEEP is the number of the current
3440 : sweep of IPA-CP over the call-graph in the decision stage. */
3441 :
3442 : static bool
3443 299756 : good_cloning_opportunity_p (struct cgraph_node *node, sreal time_benefit,
3444 : sreal freq_sum, profile_count count_sum,
3445 : int size_cost, bool called_without_ipa_profile,
3446 : int cur_sweep)
3447 : {
3448 299756 : gcc_assert (count_sum.ipa () == count_sum);
3449 299756 : if (count_sum.quality () == AFDO)
3450 0 : count_sum = count_sum.force_nonzero ();
3451 299756 : if (time_benefit == 0
3452 246557 : || !opt_for_fn (node->decl, flag_ipa_cp_clone)
3453 : /* If there is no call which was executed in profiling or where
3454 : profile is missing, we do not want to clone. */
3455 101762 : || (!called_without_ipa_profile && !count_sum.nonzero_p ()))
3456 : {
3457 197994 : if (dump_file && (dump_flags & TDF_DETAILS))
3458 24 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3459 : "size: %i): Definitely not good or prohibited.\n",
3460 : time_benefit.to_double (), size_cost);
3461 197994 : return false;
3462 : }
3463 :
3464 101762 : gcc_assert (size_cost > 0);
3465 :
3466 101762 : ipa_node_params *info = ipa_node_params_sum->get (node);
3467 101762 : int num_sweeps = opt_for_fn (node->decl, param_ipa_cp_sweeps);
3468 101762 : int eval_threshold = opt_for_fn (node->decl, param_ipa_cp_eval_threshold);
3469 101762 : eval_threshold = (eval_threshold * num_sweeps) / cur_sweep;
3470 : /* If we know the execution IPA execution counts, we can estimate overall
3471 : speedup of the program. */
3472 101762 : if (count_sum.nonzero_p ())
3473 : {
3474 371 : profile_count saved_time = count_sum * time_benefit;
3475 371 : sreal evaluation = saved_time.to_sreal_scale (profile_count::one ())
3476 742 : / size_cost;
3477 371 : evaluation = incorporate_penalties (node, info, evaluation);
3478 :
3479 371 : if (dump_file && (dump_flags & TDF_DETAILS))
3480 : {
3481 0 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3482 : "size: %i, count_sum: ", time_benefit.to_double (),
3483 : size_cost);
3484 0 : count_sum.dump (dump_file);
3485 0 : fprintf (dump_file, ", overall time saved: ");
3486 0 : saved_time.dump (dump_file);
3487 0 : fprintf (dump_file, "%s%s) -> evaluation: %.2f, threshold: %i\n",
3488 0 : info->node_within_scc
3489 0 : ? (info->node_is_self_scc ? ", self_scc" : ", scc") : "",
3490 0 : info->node_calling_single_call ? ", single_call" : "",
3491 : evaluation.to_double (), eval_threshold);
3492 : }
3493 371 : gcc_checking_assert (saved_time == saved_time.ipa ());
3494 371 : if (!maybe_hot_count_p (NULL, saved_time))
3495 : {
3496 24 : if (dump_file && (dump_flags & TDF_DETAILS))
3497 0 : fprintf (dump_file, " not cloning: time saved is not hot\n");
3498 : }
3499 : /* Evaluation approximately corresponds to time saved per instruction
3500 : introduced. This is likely almost always going to be true, since we
3501 : already checked that time saved is large enough to be considered
3502 : hot. */
3503 347 : else if (evaluation >= (sreal)eval_threshold)
3504 371 : return true;
3505 : /* If all call sites have profile known; we know we do not want t clone.
3506 : If there are calls with unknown profile; try local heuristics. */
3507 359 : if (!called_without_ipa_profile)
3508 : return false;
3509 : }
3510 101391 : sreal evaluation = (time_benefit * freq_sum) / size_cost;
3511 101391 : evaluation = incorporate_penalties (node, info, evaluation);
3512 101391 : evaluation *= 1000;
3513 :
3514 101391 : if (dump_file && (dump_flags & TDF_DETAILS))
3515 358 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3516 : "size: %i, freq_sum: %g%s%s) -> evaluation: %.2f, "
3517 : "threshold: %i\n",
3518 : time_benefit.to_double (), size_cost, freq_sum.to_double (),
3519 179 : info->node_within_scc
3520 26 : ? (info->node_is_self_scc ? ", self_scc" : ", scc") : "",
3521 179 : info->node_calling_single_call ? ", single_call" : "",
3522 : evaluation.to_double (), eval_threshold);
3523 :
3524 101391 : return evaluation >= eval_threshold;
3525 : }
3526 :
3527 : /* Grow vectors in AVALS and fill them with information about values of
3528 : parameters that are known to be independent of the context. INFO describes
3529 : the function. If REMOVABLE_PARAMS_COST is non-NULL, the movement cost of
3530 : all removable parameters will be stored in it.
3531 :
3532 : TODO: Also grow context independent value range vectors. */
3533 :
3534 : static bool
3535 1110994 : gather_context_independent_values (class ipa_node_params *info,
3536 : ipa_auto_call_arg_values *avals,
3537 : int *removable_params_cost)
3538 : {
3539 1110994 : int i, count = ipa_get_param_count (info);
3540 1110994 : bool ret = false;
3541 :
3542 1110994 : avals->m_known_vals.safe_grow_cleared (count, true);
3543 1110994 : avals->m_known_contexts.safe_grow_cleared (count, true);
3544 :
3545 1110994 : if (removable_params_cost)
3546 1110994 : *removable_params_cost = 0;
3547 :
3548 3673428 : for (i = 0; i < count; i++)
3549 : {
3550 2562434 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3551 2562434 : ipcp_lattice<tree> *lat = &plats->itself;
3552 :
3553 2562434 : if (lat->is_single_const ())
3554 : {
3555 34489 : ipcp_value<tree> *val = lat->values;
3556 34489 : gcc_checking_assert (TREE_CODE (val->value) != TREE_BINFO);
3557 34489 : avals->m_known_vals[i] = val->value;
3558 34489 : if (removable_params_cost)
3559 68978 : *removable_params_cost
3560 34489 : += estimate_move_cost (TREE_TYPE (val->value), false);
3561 : ret = true;
3562 : }
3563 2527945 : else if (removable_params_cost
3564 2527945 : && !ipa_is_param_used (info, i))
3565 490546 : *removable_params_cost
3566 245273 : += ipa_get_param_move_cost (info, i);
3567 :
3568 2562434 : if (!ipa_is_param_used (info, i))
3569 250450 : continue;
3570 :
3571 2311984 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3572 : /* Do not account known context as reason for cloning. We can see
3573 : if it permits devirtualization. */
3574 2311984 : if (ctxlat->is_single_const ())
3575 24195 : avals->m_known_contexts[i] = ctxlat->values->value;
3576 :
3577 2311984 : ret |= push_agg_values_from_plats (plats, i, 0, &avals->m_known_aggs);
3578 : }
3579 :
3580 1110994 : return ret;
3581 : }
3582 :
3583 : /* Perform time and size measurement of NODE with the context given in AVALS,
3584 : calculate the benefit compared to the node without specialization and store
3585 : it into VAL. Take into account REMOVABLE_PARAMS_COST of all
3586 : context-independent or unused removable parameters and EST_MOVE_COST, the
3587 : estimated movement of the considered parameter. */
3588 :
3589 : static void
3590 79053 : perform_estimation_of_a_value (cgraph_node *node,
3591 : ipa_auto_call_arg_values *avals,
3592 : int removable_params_cost, int est_move_cost,
3593 : ipcp_value_base *val)
3594 : {
3595 79053 : sreal time_benefit;
3596 79053 : ipa_call_estimates estimates;
3597 :
3598 79053 : estimate_ipcp_clone_size_and_time (node, avals, &estimates);
3599 :
3600 : /* Extern inline functions have no cloning local time benefits because they
3601 : will be inlined anyway. The only reason to clone them is if it enables
3602 : optimization in any of the functions they call. */
3603 79053 : if (DECL_EXTERNAL (node->decl) && DECL_DECLARED_INLINE_P (node->decl))
3604 114 : time_benefit = 0;
3605 : else
3606 78939 : time_benefit = (estimates.nonspecialized_time - estimates.time)
3607 157878 : + hint_time_bonus (node, estimates)
3608 157878 : + (devirtualization_time_bonus (node, avals)
3609 157878 : + removable_params_cost + est_move_cost);
3610 :
3611 79053 : int size = estimates.size;
3612 79053 : gcc_checking_assert (size >=0);
3613 : /* The inliner-heuristics based estimates may think that in certain
3614 : contexts some functions do not have any size at all but we want
3615 : all specializations to have at least a tiny cost, not least not to
3616 : divide by zero. */
3617 79053 : if (size == 0)
3618 0 : size = 1;
3619 :
3620 79053 : val->local_time_benefit = time_benefit;
3621 79053 : val->local_size_cost = size;
3622 79053 : }
3623 :
3624 : /* Get the overall limit of growth based on parameters extracted from NODE. It
3625 : does not really make sense to mix functions with different overall growth
3626 : limits or even number of sweeps but it is possible and if it happens, we do
3627 : not want to select one limit at random, so get the limits from NODE. */
3628 :
3629 : static long
3630 214418 : get_max_overall_size (cgraph_node *node)
3631 : {
3632 214418 : long max_new_size = orig_overall_size;
3633 214418 : long large_unit = opt_for_fn (node->decl, param_ipa_cp_large_unit_insns);
3634 214418 : if (max_new_size < large_unit)
3635 : max_new_size = large_unit;
3636 214418 : int unit_growth = opt_for_fn (node->decl, param_ipa_cp_unit_growth);
3637 214418 : max_new_size += max_new_size * unit_growth / 100 + 1;
3638 :
3639 214418 : return max_new_size;
3640 : }
3641 :
3642 : /* Return true if NODE should be cloned just for a parameter removal, possibly
3643 : dumping a reason if not. */
3644 :
3645 : static bool
3646 8431 : clone_for_param_removal_p (cgraph_node *node)
3647 : {
3648 8431 : if (!node->can_change_signature)
3649 : {
3650 1562 : if (dump_file && (dump_flags & TDF_DETAILS))
3651 0 : fprintf (dump_file, " Not considering cloning to remove parameters, "
3652 : "function cannot change signature.\n");
3653 1562 : return false;
3654 : }
3655 6869 : if (node->can_be_local_p ())
3656 : {
3657 6869 : if (dump_file && (dump_flags & TDF_DETAILS))
3658 0 : fprintf (dump_file, " Not considering cloning to remove parameters, "
3659 : "IPA-SRA can do it potentially better.\n");
3660 6869 : return false;
3661 : }
3662 : return true;
3663 : }
3664 :
3665 : /* Iterate over known values of parameters of NODE and estimate the local
3666 : effects in terms of time and size they have. */
3667 :
3668 : static void
3669 1293896 : estimate_local_effects (struct cgraph_node *node)
3670 : {
3671 1293896 : ipa_node_params *info = ipa_node_params_sum->get (node);
3672 1293896 : int count = ipa_get_param_count (info);
3673 1062053 : int removable_params_cost;
3674 :
3675 1062053 : if (!count || !ipcp_versionable_function_p (node))
3676 401037 : return;
3677 :
3678 892859 : if (dump_file && (dump_flags & TDF_DETAILS))
3679 117 : fprintf (dump_file, "\nEstimating effects for %s.\n", node->dump_name ());
3680 :
3681 892859 : ipa_auto_call_arg_values avals;
3682 892859 : gather_context_independent_values (info, &avals, &removable_params_cost);
3683 :
3684 2979784 : for (int i = 0; i < count; i++)
3685 : {
3686 2086925 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3687 2086925 : ipcp_lattice<tree> *lat = &plats->itself;
3688 2086925 : ipcp_value<tree> *val;
3689 :
3690 4152418 : if (lat->bottom
3691 214658 : || !lat->values
3692 2125678 : || avals.m_known_vals[i])
3693 2065493 : continue;
3694 :
3695 65573 : for (val = lat->values; val; val = val->next)
3696 : {
3697 44141 : gcc_checking_assert (TREE_CODE (val->value) != TREE_BINFO);
3698 44141 : avals.m_known_vals[i] = val->value;
3699 :
3700 44141 : int emc = estimate_move_cost (TREE_TYPE (val->value), true);
3701 44141 : perform_estimation_of_a_value (node, &avals, removable_params_cost,
3702 : emc, val);
3703 :
3704 44141 : if (dump_file && (dump_flags & TDF_DETAILS))
3705 : {
3706 44 : fprintf (dump_file, " - estimates for value ");
3707 44 : print_ipcp_constant_value (dump_file, val->value);
3708 44 : fprintf (dump_file, " for ");
3709 44 : ipa_dump_param (dump_file, info, i);
3710 44 : fprintf (dump_file, ": time_benefit: %g, size: %i\n",
3711 : val->local_time_benefit.to_double (),
3712 : val->local_size_cost);
3713 : }
3714 : }
3715 21432 : avals.m_known_vals[i] = NULL_TREE;
3716 : }
3717 :
3718 2979784 : for (int i = 0; i < count; i++)
3719 : {
3720 2086925 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3721 :
3722 2086925 : if (!plats->virt_call)
3723 2079023 : continue;
3724 :
3725 7902 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3726 7902 : ipcp_value<ipa_polymorphic_call_context> *val;
3727 :
3728 15645 : if (ctxlat->bottom
3729 2922 : || !ctxlat->values
3730 10818 : || !avals.m_known_contexts[i].useless_p ())
3731 7743 : continue;
3732 :
3733 387 : for (val = ctxlat->values; val; val = val->next)
3734 : {
3735 228 : avals.m_known_contexts[i] = val->value;
3736 228 : perform_estimation_of_a_value (node, &avals, removable_params_cost,
3737 : 0, val);
3738 :
3739 228 : if (dump_file && (dump_flags & TDF_DETAILS))
3740 : {
3741 0 : fprintf (dump_file, " - estimates for polymorphic context ");
3742 0 : print_ipcp_constant_value (dump_file, val->value);
3743 0 : fprintf (dump_file, " for ");
3744 0 : ipa_dump_param (dump_file, info, i);
3745 0 : fprintf (dump_file, ": time_benefit: %g, size: %i\n",
3746 : val->local_time_benefit.to_double (),
3747 : val->local_size_cost);
3748 : }
3749 : }
3750 159 : avals.m_known_contexts[i] = ipa_polymorphic_call_context ();
3751 : }
3752 :
3753 892859 : unsigned all_ctx_len = avals.m_known_aggs.length ();
3754 892859 : auto_vec<ipa_argagg_value, 32> all_ctx;
3755 892859 : all_ctx.reserve_exact (all_ctx_len);
3756 892859 : all_ctx.splice (avals.m_known_aggs);
3757 892859 : avals.m_known_aggs.safe_grow_cleared (all_ctx_len + 1);
3758 :
3759 892859 : unsigned j = 0;
3760 2979784 : for (int index = 0; index < count; index++)
3761 : {
3762 2086925 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, index);
3763 :
3764 2086925 : if (plats->aggs_bottom || !plats->aggs)
3765 2067783 : continue;
3766 :
3767 73849 : for (ipcp_agg_lattice *aglat = plats->aggs; aglat; aglat = aglat->next)
3768 : {
3769 54707 : ipcp_value<tree> *val;
3770 54349 : if (aglat->bottom || !aglat->values
3771 : /* If the following is true, the one value is already part of all
3772 : context estimations. */
3773 101621 : || (!plats->aggs_contain_variable
3774 25324 : && aglat->is_single_const ()))
3775 28725 : continue;
3776 :
3777 25982 : unsigned unit_offset = aglat->offset / BITS_PER_UNIT;
3778 25982 : while (j < all_ctx_len
3779 34426 : && (all_ctx[j].index < index
3780 3386 : || (all_ctx[j].index == index
3781 2398 : && all_ctx[j].unit_offset < unit_offset)))
3782 : {
3783 3251 : avals.m_known_aggs[j] = all_ctx[j];
3784 3251 : j++;
3785 : }
3786 :
3787 35222 : for (unsigned k = j; k < all_ctx_len; k++)
3788 9240 : avals.m_known_aggs[k+1] = all_ctx[k];
3789 :
3790 60666 : for (val = aglat->values; val; val = val->next)
3791 : {
3792 34684 : avals.m_known_aggs[j].value = val->value;
3793 34684 : avals.m_known_aggs[j].unit_offset = unit_offset;
3794 34684 : avals.m_known_aggs[j].index = index;
3795 34684 : avals.m_known_aggs[j].by_ref = plats->aggs_by_ref;
3796 34684 : avals.m_known_aggs[j].killed = false;
3797 :
3798 34684 : perform_estimation_of_a_value (node, &avals,
3799 : removable_params_cost, 0, val);
3800 :
3801 34684 : if (dump_file && (dump_flags & TDF_DETAILS))
3802 : {
3803 80 : fprintf (dump_file, " - estimates for value ");
3804 80 : print_ipcp_constant_value (dump_file, val->value);
3805 80 : fprintf (dump_file, " for ");
3806 80 : ipa_dump_param (dump_file, info, index);
3807 160 : fprintf (dump_file, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
3808 : "]: time_benefit: %g, size: %i\n",
3809 80 : plats->aggs_by_ref ? "ref " : "",
3810 : aglat->offset,
3811 : val->local_time_benefit.to_double (),
3812 : val->local_size_cost);
3813 : }
3814 : }
3815 : }
3816 : }
3817 892859 : }
3818 :
3819 :
3820 : /* Add value CUR_VAL and all yet-unsorted values it is dependent on to the
3821 : topological sort of values. */
3822 :
3823 : template <typename valtype>
3824 : void
3825 137862 : value_topo_info<valtype>::add_val (ipcp_value<valtype> *cur_val)
3826 : {
3827 : ipcp_value_source<valtype> *src;
3828 :
3829 137862 : if (cur_val->dfs)
3830 : return;
3831 :
3832 137700 : dfs_counter++;
3833 137700 : cur_val->dfs = dfs_counter;
3834 137700 : cur_val->low_link = dfs_counter;
3835 :
3836 137700 : cur_val->topo_next = stack;
3837 137700 : stack = cur_val;
3838 137700 : cur_val->on_stack = true;
3839 :
3840 597154 : for (src = cur_val->sources; src; src = src->next)
3841 459454 : if (src->val)
3842 : {
3843 21031 : if (src->val->dfs == 0)
3844 : {
3845 186 : add_val (src->val);
3846 186 : if (src->val->low_link < cur_val->low_link)
3847 19 : cur_val->low_link = src->val->low_link;
3848 : }
3849 20845 : else if (src->val->on_stack
3850 1575 : && src->val->dfs < cur_val->low_link)
3851 73 : cur_val->low_link = src->val->dfs;
3852 : }
3853 :
3854 137700 : if (cur_val->dfs == cur_val->low_link)
3855 : {
3856 : ipcp_value<valtype> *v, *scc_list = NULL;
3857 :
3858 : do
3859 : {
3860 137700 : v = stack;
3861 137700 : stack = v->topo_next;
3862 137700 : v->on_stack = false;
3863 137700 : v->scc_no = cur_val->dfs;
3864 :
3865 137700 : v->scc_next = scc_list;
3866 137700 : scc_list = v;
3867 : }
3868 137700 : while (v != cur_val);
3869 :
3870 137612 : cur_val->topo_next = values_topo;
3871 137612 : values_topo = cur_val;
3872 : }
3873 : }
3874 :
3875 : /* Add all values in lattices associated with NODE to the topological sort if
3876 : they are not there yet. */
3877 :
3878 : static void
3879 1293896 : add_all_node_vals_to_toposort (cgraph_node *node, ipa_topo_info *topo)
3880 : {
3881 1293896 : ipa_node_params *info = ipa_node_params_sum->get (node);
3882 1293896 : int i, count = ipa_get_param_count (info);
3883 :
3884 3678714 : for (i = 0; i < count; i++)
3885 : {
3886 2384818 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3887 2384818 : ipcp_lattice<tree> *lat = &plats->itself;
3888 2384818 : struct ipcp_agg_lattice *aglat;
3889 :
3890 2384818 : if (!lat->bottom)
3891 : {
3892 225516 : ipcp_value<tree> *val;
3893 297768 : for (val = lat->values; val; val = val->next)
3894 72252 : topo->constants.add_val (val);
3895 : }
3896 :
3897 2384818 : if (!plats->aggs_bottom)
3898 282969 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
3899 57532 : if (!aglat->bottom)
3900 : {
3901 57174 : ipcp_value<tree> *val;
3902 114678 : for (val = aglat->values; val; val = val->next)
3903 57504 : topo->constants.add_val (val);
3904 : }
3905 :
3906 2384818 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3907 2384818 : if (!ctxlat->bottom)
3908 : {
3909 226518 : ipcp_value<ipa_polymorphic_call_context> *ctxval;
3910 234438 : for (ctxval = ctxlat->values; ctxval; ctxval = ctxval->next)
3911 7920 : topo->contexts.add_val (ctxval);
3912 : }
3913 : }
3914 1293896 : }
3915 :
3916 : /* One pass of constants propagation along the call graph edges, from callers
3917 : to callees (requires topological ordering in TOPO), iterate over strongly
3918 : connected components. */
3919 :
3920 : static void
3921 129922 : propagate_constants_topo (class ipa_topo_info *topo)
3922 : {
3923 129922 : int i;
3924 :
3925 1507262 : for (i = topo->nnodes - 1; i >= 0; i--)
3926 : {
3927 1377340 : unsigned j;
3928 1377340 : struct cgraph_node *v, *node = topo->order[i];
3929 1377340 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (node);
3930 :
3931 : /* First, iteratively propagate within the strongly connected component
3932 : until all lattices stabilize. */
3933 2759590 : FOR_EACH_VEC_ELT (cycle_nodes, j, v)
3934 1382250 : if (v->has_gimple_body_p ())
3935 : {
3936 1302686 : if (opt_for_fn (v->decl, flag_ipa_cp)
3937 1302686 : && opt_for_fn (v->decl, optimize))
3938 1293896 : push_node_to_stack (topo, v);
3939 : /* When V is not optimized, we can not push it to stack, but
3940 : still we need to set all its callees lattices to bottom. */
3941 : else
3942 : {
3943 21839 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
3944 13049 : propagate_constants_across_call (cs);
3945 : }
3946 : }
3947 :
3948 1377340 : v = pop_node_from_stack (topo);
3949 4051773 : while (v)
3950 : {
3951 1297093 : struct cgraph_edge *cs;
3952 1297093 : class ipa_node_params *info = NULL;
3953 1297093 : bool self_scc = true;
3954 :
3955 6693853 : for (cs = v->callees; cs; cs = cs->next_callee)
3956 5396760 : if (ipa_edge_within_scc (cs))
3957 : {
3958 29746 : cgraph_node *callee = cs->callee->function_symbol ();
3959 :
3960 29746 : if (v != callee)
3961 18072 : self_scc = false;
3962 :
3963 29746 : if (!info)
3964 : {
3965 14019 : info = ipa_node_params_sum->get (v);
3966 14019 : info->node_within_scc = true;
3967 : }
3968 :
3969 29746 : if (propagate_constants_across_call (cs))
3970 4166 : push_node_to_stack (topo, callee);
3971 : }
3972 :
3973 1297093 : if (info)
3974 14019 : info->node_is_self_scc = self_scc;
3975 :
3976 1297093 : v = pop_node_from_stack (topo);
3977 : }
3978 :
3979 : /* Afterwards, propagate along edges leading out of the SCC, calculates
3980 : the local effects of the discovered constants and all valid values to
3981 : their topological sort. */
3982 2759590 : FOR_EACH_VEC_ELT (cycle_nodes, j, v)
3983 1382250 : if (v->has_gimple_body_p ()
3984 1302686 : && opt_for_fn (v->decl, flag_ipa_cp)
3985 2676146 : && opt_for_fn (v->decl, optimize))
3986 : {
3987 1293896 : struct cgraph_edge *cs;
3988 :
3989 1293896 : estimate_local_effects (v);
3990 1293896 : add_all_node_vals_to_toposort (v, topo);
3991 6656849 : for (cs = v->callees; cs; cs = cs->next_callee)
3992 5362953 : if (!ipa_edge_within_scc (cs))
3993 5340592 : propagate_constants_across_call (cs);
3994 : }
3995 1377340 : cycle_nodes.release ();
3996 : }
3997 129922 : }
3998 :
3999 : /* Propagate the estimated effects of individual values along the topological
4000 : from the dependent values to those they depend on. */
4001 :
4002 : template <typename valtype>
4003 : void
4004 259844 : value_topo_info<valtype>::propagate_effects ()
4005 : {
4006 : ipcp_value<valtype> *base;
4007 259844 : hash_set<ipcp_value<valtype> *> processed_srcvals;
4008 :
4009 397456 : for (base = values_topo; base; base = base->topo_next)
4010 : {
4011 : ipcp_value_source<valtype> *src;
4012 : ipcp_value<valtype> *val;
4013 137612 : sreal time = 0;
4014 137612 : HOST_WIDE_INT size = 0;
4015 :
4016 275312 : for (val = base; val; val = val->scc_next)
4017 : {
4018 137700 : time = time + val->local_time_benefit + val->prop_time_benefit;
4019 137700 : size = size + val->local_size_cost + val->prop_size_cost;
4020 : }
4021 :
4022 275312 : for (val = base; val; val = val->scc_next)
4023 : {
4024 137700 : processed_srcvals.empty ();
4025 597154 : for (src = val->sources; src; src = src->next)
4026 459454 : if (src->val
4027 459454 : && cs_interesting_for_ipcp_p (src->cs))
4028 : {
4029 20991 : if (!processed_srcvals.add (src->val))
4030 : {
4031 16840 : HOST_WIDE_INT prop_size = size + src->val->prop_size_cost;
4032 16840 : if (prop_size < INT_MAX)
4033 16840 : src->val->prop_size_cost = prop_size;
4034 : else
4035 0 : continue;
4036 : }
4037 :
4038 20991 : int special_factor = 1;
4039 20991 : if (val->same_scc (src->val))
4040 : special_factor
4041 1663 : = opt_for_fn(src->cs->caller->decl,
4042 : param_ipa_cp_recursive_freq_factor);
4043 19328 : else if (val->self_recursion_generated_p ()
4044 19328 : && (src->cs->callee->function_symbol ()
4045 822 : == src->cs->caller))
4046 : {
4047 822 : int max_recur_gen_depth
4048 822 : = opt_for_fn(src->cs->caller->decl,
4049 : param_ipa_cp_max_recursive_depth);
4050 822 : special_factor = max_recur_gen_depth
4051 822 : - val->self_recursion_generated_level + 1;
4052 : }
4053 :
4054 20991 : src->val->prop_time_benefit
4055 41982 : += time * special_factor * src->cs->sreal_frequency ();
4056 : }
4057 :
4058 137700 : if (size < INT_MAX)
4059 : {
4060 137700 : val->prop_time_benefit = time;
4061 137700 : val->prop_size_cost = size;
4062 : }
4063 : else
4064 : {
4065 0 : val->prop_time_benefit = 0;
4066 0 : val->prop_size_cost = 0;
4067 : }
4068 : }
4069 : }
4070 259844 : }
4071 :
4072 :
4073 : /* Propagate constants, polymorphic contexts and their effects from the
4074 : summaries interprocedurally. */
4075 :
4076 : static void
4077 129922 : ipcp_propagate_stage (class ipa_topo_info *topo)
4078 : {
4079 129922 : struct cgraph_node *node;
4080 :
4081 129922 : if (dump_file)
4082 161 : fprintf (dump_file, "\n Propagating constants:\n\n");
4083 :
4084 1512176 : FOR_EACH_DEFINED_FUNCTION (node)
4085 : {
4086 1382254 : if (node->has_gimple_body_p ()
4087 1302686 : && opt_for_fn (node->decl, flag_ipa_cp)
4088 2676150 : && opt_for_fn (node->decl, optimize))
4089 : {
4090 1293896 : ipa_node_params *info = ipa_node_params_sum->get (node);
4091 1293896 : determine_versionability (node, info);
4092 :
4093 1293896 : unsigned nlattices = ipa_get_param_count (info);
4094 1293896 : info->lattices.safe_grow_cleared (nlattices, true);
4095 1293896 : initialize_node_lattices (node);
4096 :
4097 1293896 : int num_sweeps = opt_for_fn (node->decl, param_ipa_cp_sweeps);
4098 1293896 : if (max_number_sweeps < num_sweeps)
4099 122134 : max_number_sweeps = num_sweeps;
4100 : }
4101 1382254 : ipa_size_summary *s = ipa_size_summaries->get (node);
4102 1382254 : if (node->definition && !node->alias && s != NULL)
4103 1303639 : overall_size += s->self_size;
4104 : }
4105 :
4106 129922 : orig_overall_size = overall_size;
4107 :
4108 129922 : if (dump_file)
4109 161 : fprintf (dump_file, "\noverall_size: %li\n", overall_size);
4110 :
4111 129922 : propagate_constants_topo (topo);
4112 129922 : if (flag_checking)
4113 129914 : ipcp_verify_propagated_values ();
4114 129922 : topo->constants.propagate_effects ();
4115 129922 : topo->contexts.propagate_effects ();
4116 :
4117 129922 : if (dump_file)
4118 : {
4119 161 : fprintf (dump_file, "\nIPA lattices after all propagation:\n");
4120 161 : print_all_lattices (dump_file, (dump_flags & TDF_DETAILS), true);
4121 : }
4122 129922 : }
4123 :
4124 : /* Discover newly direct outgoing edges from NODE which is a new clone with
4125 : known KNOWN_CSTS and make them direct. */
4126 :
4127 : static void
4128 18931 : ipcp_discover_new_direct_edges (struct cgraph_node *node,
4129 : vec<tree> known_csts,
4130 : vec<ipa_polymorphic_call_context>
4131 : known_contexts,
4132 : vec<ipa_argagg_value, va_gc> *aggvals)
4133 : {
4134 18931 : struct cgraph_edge *ie, *next_ie;
4135 18931 : bool found = false;
4136 :
4137 20851 : for (ie = node->indirect_calls; ie; ie = next_ie)
4138 : {
4139 1920 : tree target;
4140 1920 : bool speculative;
4141 :
4142 1920 : next_ie = ie->next_callee;
4143 1920 : ipa_argagg_value_list avs (aggvals);
4144 1920 : target = ipa_get_indirect_edge_target_1 (ie, known_csts, known_contexts,
4145 : avs, &speculative);
4146 1920 : if (target)
4147 : {
4148 565 : cgraph_polymorphic_indirect_info *pii
4149 565 : = dyn_cast <cgraph_polymorphic_indirect_info *> (ie->indirect_info);
4150 565 : cgraph_simple_indirect_info *sii
4151 1060 : = dyn_cast <cgraph_simple_indirect_info *> (ie->indirect_info);
4152 420 : bool agg_contents = sii && sii->agg_contents;
4153 565 : bool polymorphic = !!pii;
4154 565 : int param_index = ie->indirect_info->param_index;
4155 565 : struct cgraph_edge *cs = ipa_make_edge_direct_to_target (ie, target,
4156 : speculative);
4157 565 : found = true;
4158 :
4159 565 : if (cs && !agg_contents && !polymorphic)
4160 : {
4161 350 : ipa_node_params *info = ipa_node_params_sum->get (node);
4162 350 : int c = ipa_get_controlled_uses (info, param_index);
4163 350 : if (c != IPA_UNDESCRIBED_USE
4164 350 : && !ipa_get_param_load_dereferenced (info, param_index))
4165 : {
4166 346 : struct ipa_ref *to_del;
4167 :
4168 346 : c--;
4169 346 : ipa_set_controlled_uses (info, param_index, c);
4170 346 : if (dump_file && (dump_flags & TDF_DETAILS))
4171 3 : fprintf (dump_file, " controlled uses count of param "
4172 : "%i bumped down to %i\n", param_index, c);
4173 346 : if (c == 0
4174 346 : && (to_del = node->find_reference (cs->callee, NULL, 0,
4175 : IPA_REF_ADDR)))
4176 : {
4177 282 : if (dump_file && (dump_flags & TDF_DETAILS))
4178 3 : fprintf (dump_file, " and even removing its "
4179 : "cloning-created reference\n");
4180 282 : to_del->remove_reference ();
4181 : }
4182 : }
4183 : }
4184 : }
4185 : }
4186 : /* Turning calls to direct calls will improve overall summary. */
4187 18931 : if (found)
4188 468 : ipa_update_overall_fn_summary (node);
4189 18931 : }
4190 :
4191 : class edge_clone_summary;
4192 : static call_summary <edge_clone_summary *> *edge_clone_summaries = NULL;
4193 :
4194 : /* Edge clone summary. */
4195 :
4196 : class edge_clone_summary
4197 : {
4198 : public:
4199 : /* Default constructor. */
4200 375087 : edge_clone_summary (): prev_clone (NULL), next_clone (NULL) {}
4201 :
4202 : /* Default destructor. */
4203 375087 : ~edge_clone_summary ()
4204 : {
4205 375087 : if (prev_clone)
4206 32807 : edge_clone_summaries->get (prev_clone)->next_clone = next_clone;
4207 375087 : if (next_clone)
4208 157992 : edge_clone_summaries->get (next_clone)->prev_clone = prev_clone;
4209 375087 : }
4210 :
4211 : cgraph_edge *prev_clone;
4212 : cgraph_edge *next_clone;
4213 : };
4214 :
4215 : class edge_clone_summary_t:
4216 : public call_summary <edge_clone_summary *>
4217 : {
4218 : public:
4219 129922 : edge_clone_summary_t (symbol_table *symtab):
4220 259844 : call_summary <edge_clone_summary *> (symtab)
4221 : {
4222 129922 : m_initialize_when_cloning = true;
4223 : }
4224 :
4225 : void duplicate (cgraph_edge *src_edge, cgraph_edge *dst_edge,
4226 : edge_clone_summary *src_data,
4227 : edge_clone_summary *dst_data) final override;
4228 : };
4229 :
4230 : /* Edge duplication hook. */
4231 :
4232 : void
4233 190213 : edge_clone_summary_t::duplicate (cgraph_edge *src_edge, cgraph_edge *dst_edge,
4234 : edge_clone_summary *src_data,
4235 : edge_clone_summary *dst_data)
4236 : {
4237 190213 : if (src_data->next_clone)
4238 5330 : edge_clone_summaries->get (src_data->next_clone)->prev_clone = dst_edge;
4239 190213 : dst_data->prev_clone = src_edge;
4240 190213 : dst_data->next_clone = src_data->next_clone;
4241 190213 : src_data->next_clone = dst_edge;
4242 190213 : }
4243 :
4244 : /* Return true is CS calls DEST or its clone for all contexts. When
4245 : ALLOW_RECURSION_TO_CLONE is false, also return false for self-recursive
4246 : edges from/to an all-context clone. */
4247 :
4248 : static bool
4249 1825718 : calls_same_node_or_its_all_contexts_clone_p (cgraph_edge *cs, cgraph_node *dest,
4250 : bool allow_recursion_to_clone)
4251 : {
4252 1825718 : enum availability availability;
4253 1825718 : cgraph_node *callee = cs->callee->function_symbol (&availability);
4254 :
4255 1825718 : if (availability <= AVAIL_INTERPOSABLE)
4256 : return false;
4257 1819641 : if (callee == dest)
4258 : return true;
4259 621668 : if (!allow_recursion_to_clone && cs->caller == callee)
4260 : return false;
4261 :
4262 621511 : ipa_node_params *info = ipa_node_params_sum->get (callee);
4263 621511 : return info->is_all_contexts_clone && info->ipcp_orig_node == dest;
4264 : }
4265 :
4266 : /* Return true if edge CS does bring about the value described by SRC to
4267 : DEST_VAL of node DEST or its clone for all contexts. */
4268 :
4269 : static bool
4270 1815676 : cgraph_edge_brings_value_p (cgraph_edge *cs, ipcp_value_source<tree> *src,
4271 : cgraph_node *dest, ipcp_value<tree> *dest_val)
4272 : {
4273 1815676 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
4274 :
4275 1815676 : if (!calls_same_node_or_its_all_contexts_clone_p (cs, dest, !src->val)
4276 1815676 : || caller_info->node_dead)
4277 : return false;
4278 :
4279 748317 : if (!src->val)
4280 : return true;
4281 :
4282 62429 : if (caller_info->ipcp_orig_node)
4283 : {
4284 19706 : tree t = NULL_TREE;
4285 19706 : if (src->offset == -1)
4286 13626 : t = caller_info->known_csts[src->index];
4287 6080 : else if (ipcp_transformation *ts
4288 6080 : = ipcp_get_transformation_summary (cs->caller))
4289 : {
4290 6080 : ipa_argagg_value_list avl (ts);
4291 6080 : t = avl.get_value (src->index, src->offset / BITS_PER_UNIT);
4292 : }
4293 19706 : return (t != NULL_TREE
4294 19706 : && values_equal_for_ipcp_p (src->val->value, t));
4295 : }
4296 : else
4297 : {
4298 42723 : if (src->val == dest_val)
4299 : return true;
4300 :
4301 37005 : struct ipcp_agg_lattice *aglat;
4302 37005 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (caller_info,
4303 : src->index);
4304 37005 : if (src->offset == -1)
4305 26970 : return (plats->itself.is_single_const ()
4306 20 : && values_equal_for_ipcp_p (src->val->value,
4307 20 : plats->itself.values->value));
4308 : else
4309 : {
4310 10035 : if (plats->aggs_bottom || plats->aggs_contain_variable)
4311 : return false;
4312 3882 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
4313 3882 : if (aglat->offset == src->offset)
4314 1748 : return (aglat->is_single_const ()
4315 8 : && values_equal_for_ipcp_p (src->val->value,
4316 8 : aglat->values->value));
4317 : }
4318 : return false;
4319 : }
4320 : }
4321 :
4322 : /* Return true if edge CS does bring about the value described by SRC to
4323 : DST_VAL of node DEST or its clone for all contexts. */
4324 :
4325 : static bool
4326 10042 : cgraph_edge_brings_value_p (cgraph_edge *cs,
4327 : ipcp_value_source<ipa_polymorphic_call_context> *src,
4328 : cgraph_node *dest,
4329 : ipcp_value<ipa_polymorphic_call_context> *)
4330 : {
4331 10042 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
4332 :
4333 10042 : if (!calls_same_node_or_its_all_contexts_clone_p (cs, dest, true)
4334 10042 : || caller_info->node_dead)
4335 : return false;
4336 9099 : if (!src->val)
4337 : return true;
4338 :
4339 1678 : if (caller_info->ipcp_orig_node)
4340 2369 : return (caller_info->known_contexts.length () > (unsigned) src->index)
4341 460 : && values_equal_for_ipcp_p (src->val->value,
4342 230 : caller_info->known_contexts[src->index]);
4343 :
4344 1448 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (caller_info,
4345 : src->index);
4346 1448 : return plats->ctxlat.is_single_const ()
4347 270 : && values_equal_for_ipcp_p (src->val->value,
4348 270 : plats->ctxlat.values->value);
4349 : }
4350 :
4351 : /* Get the next clone in the linked list of clones of an edge. */
4352 :
4353 : static inline struct cgraph_edge *
4354 1825971 : get_next_cgraph_edge_clone (struct cgraph_edge *cs)
4355 : {
4356 1825971 : edge_clone_summary *s = edge_clone_summaries->get (cs);
4357 1825971 : return s != NULL ? s->next_clone : NULL;
4358 : }
4359 :
4360 : /* Given VAL that is intended for DEST, iterate over all its sources and if any
4361 : of them is viable and hot, return true. In that case, for those that still
4362 : hold, add their edge frequency and their number and cumulative profile
4363 : counts of self-ecursive and other edges into *FREQUENCY, *CALLER_COUNT,
4364 : REC_COUNT_SUM and NONREC_COUNT_SUM respectively. */
4365 :
4366 : template <typename valtype>
4367 : static bool
4368 213968 : get_info_about_necessary_edges (ipcp_value<valtype> *val, cgraph_node *dest,
4369 : sreal *freq_sum, int *caller_count,
4370 : profile_count *rec_count_sum,
4371 : profile_count *nonrec_count_sum,
4372 : bool *called_without_ipa_profile)
4373 : {
4374 : ipcp_value_source<valtype> *src;
4375 213968 : sreal freq = 0;
4376 213968 : int count = 0;
4377 213968 : profile_count rec_cnt = profile_count::zero ();
4378 213968 : profile_count nonrec_cnt = profile_count::zero ();
4379 213968 : bool interesting = false;
4380 213968 : bool non_self_recursive = false;
4381 213968 : *called_without_ipa_profile = false;
4382 :
4383 975837 : for (src = val->sources; src; src = src->next)
4384 : {
4385 761869 : struct cgraph_edge *cs = src->cs;
4386 1889840 : while (cs)
4387 : {
4388 1127971 : if (cgraph_edge_brings_value_p (cs, src, dest, val))
4389 : {
4390 356224 : count++;
4391 356224 : freq += cs->sreal_frequency ();
4392 356224 : interesting |= cs_interesting_for_ipcp_p (cs);
4393 356224 : if (cs->caller != dest)
4394 : {
4395 349405 : non_self_recursive = true;
4396 349405 : if (cs->count.ipa ().initialized_p ())
4397 972 : rec_cnt += cs->count.ipa ();
4398 : else
4399 348433 : *called_without_ipa_profile = true;
4400 : }
4401 6819 : else if (cs->count.ipa ().initialized_p ())
4402 0 : nonrec_cnt += cs->count.ipa ();
4403 : else
4404 6819 : *called_without_ipa_profile = true;
4405 : }
4406 1127971 : cs = get_next_cgraph_edge_clone (cs);
4407 : }
4408 : }
4409 :
4410 : /* If the only edges bringing a value are self-recursive ones, do not bother
4411 : evaluating it. */
4412 213968 : if (!non_self_recursive)
4413 : return false;
4414 :
4415 152265 : *freq_sum = freq;
4416 152265 : *caller_count = count;
4417 152265 : *rec_count_sum = rec_cnt;
4418 152265 : *nonrec_count_sum = nonrec_cnt;
4419 :
4420 152265 : return interesting;
4421 : }
4422 :
4423 : /* Given a NODE, and a set of its CALLERS, try to adjust order of the callers
4424 : to let a non-self-recursive caller be the first element. Thus, we can
4425 : simplify intersecting operations on values that arrive from all of these
4426 : callers, especially when there exists self-recursive call. Return true if
4427 : this kind of adjustment is possible. */
4428 :
4429 : static bool
4430 55875 : adjust_callers_for_value_intersection (vec<cgraph_edge *> &callers,
4431 : cgraph_node *node)
4432 : {
4433 60053 : for (unsigned i = 0; i < callers.length (); i++)
4434 : {
4435 59986 : cgraph_edge *cs = callers[i];
4436 :
4437 59986 : if (cs->caller != node)
4438 : {
4439 55808 : if (i > 0)
4440 : {
4441 1965 : callers[i] = callers[0];
4442 1965 : callers[0] = cs;
4443 : }
4444 55808 : return true;
4445 : }
4446 : }
4447 : return false;
4448 : }
4449 :
4450 : /* Return a vector of incoming edges that do bring value VAL to node DEST. It
4451 : is assumed their number is known and equal to CALLER_COUNT. */
4452 :
4453 : template <typename valtype>
4454 : static auto_vec<cgraph_edge *>
4455 151903 : gather_edges_for_value (ipcp_value<valtype> *val, cgraph_node *dest,
4456 : int caller_count)
4457 : {
4458 : ipcp_value_source<valtype> *src;
4459 151903 : auto_vec<cgraph_edge *> ret (caller_count);
4460 :
4461 521147 : for (src = val->sources; src; src = src->next)
4462 : {
4463 369244 : struct cgraph_edge *cs = src->cs;
4464 826754 : while (cs)
4465 : {
4466 457510 : if (cgraph_edge_brings_value_p (cs, src, dest, val))
4467 353126 : ret.quick_push (cs);
4468 457510 : cs = get_next_cgraph_edge_clone (cs);
4469 : }
4470 : }
4471 :
4472 151903 : if (caller_count > 1)
4473 41234 : adjust_callers_for_value_intersection (ret, dest);
4474 :
4475 151903 : return ret;
4476 : }
4477 :
4478 : /* Construct a replacement map for a know VALUE for a formal parameter PARAM.
4479 : Return it or NULL if for some reason it cannot be created. FORCE_LOAD_REF
4480 : should be set to true when the reference created for the constant should be
4481 : a load one and not an address one because the corresponding parameter p is
4482 : only used as *p. */
4483 :
4484 : static struct ipa_replace_map *
4485 24340 : get_replacement_map (class ipa_node_params *info, tree value, int parm_num,
4486 : bool force_load_ref)
4487 : {
4488 24340 : struct ipa_replace_map *replace_map;
4489 :
4490 24340 : replace_map = ggc_alloc<ipa_replace_map> ();
4491 24340 : if (dump_file)
4492 : {
4493 168 : fprintf (dump_file, " replacing ");
4494 168 : ipa_dump_param (dump_file, info, parm_num);
4495 :
4496 168 : fprintf (dump_file, " with const ");
4497 168 : print_generic_expr (dump_file, value);
4498 :
4499 168 : if (force_load_ref)
4500 11 : fprintf (dump_file, " - forcing load reference\n");
4501 : else
4502 157 : fprintf (dump_file, "\n");
4503 : }
4504 24340 : replace_map->parm_num = parm_num;
4505 24340 : replace_map->new_tree = value;
4506 24340 : replace_map->force_load_ref = force_load_ref;
4507 24340 : return replace_map;
4508 : }
4509 :
4510 : /* Dump new profiling counts of NODE. SPEC is true when NODE is a specialzied
4511 : one, otherwise it will be referred to as the original node. */
4512 :
4513 : static void
4514 4 : dump_profile_updates (cgraph_node *node, bool spec)
4515 : {
4516 4 : if (spec)
4517 2 : fprintf (dump_file, " setting count of the specialized node %s to ",
4518 : node->dump_name ());
4519 : else
4520 2 : fprintf (dump_file, " setting count of the original node %s to ",
4521 : node->dump_name ());
4522 :
4523 4 : node->count.dump (dump_file);
4524 4 : fprintf (dump_file, "\n");
4525 6 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
4526 : {
4527 2 : fprintf (dump_file, " edge to %s has count ",
4528 2 : cs->callee->dump_name ());
4529 2 : cs->count.dump (dump_file);
4530 2 : fprintf (dump_file, "\n");
4531 : }
4532 4 : }
4533 :
4534 : /* With partial train run we do not want to assume that original's count is
4535 : zero whenever we redurect all executed edges to clone. Simply drop profile
4536 : to local one in this case. In eany case, return the new value. ORIG_NODE
4537 : is the original node and its count has not been updated yet. */
4538 :
4539 : profile_count
4540 16 : lenient_count_portion_handling (profile_count remainder, cgraph_node *orig_node)
4541 : {
4542 32 : if (remainder.ipa_p () && !remainder.ipa ().nonzero_p ()
4543 24 : && orig_node->count.ipa_p () && orig_node->count.ipa ().nonzero_p ()
4544 4 : && opt_for_fn (orig_node->decl, flag_profile_partial_training))
4545 0 : remainder = orig_node->count.guessed_local ();
4546 :
4547 16 : return remainder;
4548 : }
4549 :
4550 : /* Structure to sum counts coming from nodes other than the original node and
4551 : its clones. */
4552 :
4553 : struct gather_other_count_struct
4554 : {
4555 : cgraph_node *orig;
4556 : profile_count other_count;
4557 : };
4558 :
4559 : /* Worker callback of call_for_symbol_thunks_and_aliases summing the number of
4560 : counts that come from non-self-recursive calls.. */
4561 :
4562 : static bool
4563 8 : gather_count_of_non_rec_edges (cgraph_node *node, void *data)
4564 : {
4565 8 : gather_other_count_struct *desc = (gather_other_count_struct *) data;
4566 20 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4567 12 : if (cs->caller != desc->orig && cs->caller->clone_of != desc->orig)
4568 0 : if (cs->count.ipa ().initialized_p ())
4569 0 : desc->other_count += cs->count.ipa ();
4570 8 : return false;
4571 : }
4572 :
4573 : /* Structure to help analyze if we need to boost counts of some clones of some
4574 : non-recursive edges to match the new callee count. */
4575 :
4576 : struct desc_incoming_count_struct
4577 : {
4578 : cgraph_node *orig;
4579 : hash_set <cgraph_edge *> *processed_edges;
4580 : profile_count count;
4581 : unsigned unproc_orig_rec_edges;
4582 : };
4583 :
4584 : /* Go over edges calling NODE and its thunks and gather information about
4585 : incoming counts so that we know if we need to make any adjustments. */
4586 :
4587 : static void
4588 8 : analyze_clone_icoming_counts (cgraph_node *node,
4589 : desc_incoming_count_struct *desc)
4590 : {
4591 20 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4592 12 : if (cs->caller->thunk)
4593 : {
4594 0 : analyze_clone_icoming_counts (cs->caller, desc);
4595 0 : continue;
4596 : }
4597 : else
4598 : {
4599 12 : if (cs->count.initialized_p ())
4600 12 : desc->count += cs->count.ipa ();
4601 12 : if (!desc->processed_edges->contains (cs)
4602 12 : && cs->caller->clone_of == desc->orig)
4603 4 : desc->unproc_orig_rec_edges++;
4604 : }
4605 8 : }
4606 :
4607 : /* If caller edge counts of a clone created for a self-recursive arithmetic
4608 : jump function must be adjusted because it is coming from a the "seed" clone
4609 : for the first value and so has been excessively scaled back as if it was not
4610 : a recursive call, adjust it so that the incoming counts of NODE match its
4611 : count. NODE is the node or its thunk. */
4612 :
4613 : static void
4614 0 : adjust_clone_incoming_counts (cgraph_node *node,
4615 : desc_incoming_count_struct *desc)
4616 : {
4617 0 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4618 0 : if (cs->caller->thunk)
4619 : {
4620 0 : adjust_clone_incoming_counts (cs->caller, desc);
4621 0 : profile_count sum = profile_count::zero ();
4622 0 : for (cgraph_edge *e = cs->caller->callers; e; e = e->next_caller)
4623 0 : if (e->count.initialized_p ())
4624 0 : sum += e->count.ipa ();
4625 0 : cs->count = cs->count.combine_with_ipa_count (sum);
4626 : }
4627 0 : else if (!desc->processed_edges->contains (cs)
4628 0 : && cs->caller->clone_of == desc->orig
4629 0 : && cs->count.compatible_p (desc->count))
4630 : {
4631 0 : cs->count += desc->count;
4632 0 : if (dump_file)
4633 : {
4634 0 : fprintf (dump_file, " Adjusted count of an incoming edge of "
4635 0 : "a clone %s -> %s to ", cs->caller->dump_name (),
4636 0 : cs->callee->dump_name ());
4637 0 : cs->count.dump (dump_file);
4638 0 : fprintf (dump_file, "\n");
4639 : }
4640 : }
4641 0 : }
4642 :
4643 : /* When ORIG_NODE has been cloned for values which have been generated fora
4644 : self-recursive call as a result of an arithmetic pass-through
4645 : jump-functions, adjust its count together with counts of all such clones in
4646 : SELF_GEN_CLONES which also at this point contains ORIG_NODE itself.
4647 :
4648 : The function sums the counts of the original node and all its clones that
4649 : cannot be attributed to a specific clone because it comes from a
4650 : non-recursive edge. This sum is then evenly divided between the clones and
4651 : on top of that each one gets all the counts which can be attributed directly
4652 : to it. */
4653 :
4654 : static void
4655 33 : update_counts_for_self_gen_clones (cgraph_node *orig_node,
4656 : const vec<cgraph_node *> &self_gen_clones)
4657 : {
4658 33 : profile_count redist_sum = orig_node->count.ipa ();
4659 33 : if (!redist_sum.nonzero_p ())
4660 : return;
4661 :
4662 4 : if (dump_file)
4663 0 : fprintf (dump_file, " Updating profile of self recursive clone "
4664 : "series\n");
4665 :
4666 4 : gather_other_count_struct gocs;
4667 4 : gocs.orig = orig_node;
4668 4 : gocs.other_count = profile_count::zero ();
4669 :
4670 4 : auto_vec <profile_count, 8> other_edges_count;
4671 20 : for (cgraph_node *n : self_gen_clones)
4672 : {
4673 8 : gocs.other_count = profile_count::zero ();
4674 8 : n->call_for_symbol_thunks_and_aliases (gather_count_of_non_rec_edges,
4675 : &gocs, false);
4676 8 : other_edges_count.safe_push (gocs.other_count);
4677 8 : redist_sum -= gocs.other_count;
4678 : }
4679 :
4680 4 : hash_set<cgraph_edge *> processed_edges;
4681 4 : unsigned i = 0;
4682 20 : for (cgraph_node *n : self_gen_clones)
4683 : {
4684 8 : profile_count new_count
4685 16 : = (redist_sum / self_gen_clones.length () + other_edges_count[i]);
4686 8 : new_count = lenient_count_portion_handling (new_count, orig_node);
4687 8 : n->scale_profile_to (new_count);
4688 16 : for (cgraph_edge *cs = n->callees; cs; cs = cs->next_callee)
4689 8 : processed_edges.add (cs);
4690 :
4691 8 : i++;
4692 : }
4693 :
4694 : /* There are still going to be edges to ORIG_NODE that have one or more
4695 : clones coming from another node clone in SELF_GEN_CLONES and which we
4696 : scaled by the same amount, which means that the total incoming sum of
4697 : counts to ORIG_NODE will be too high, scale such edges back. */
4698 8 : for (cgraph_edge *cs = orig_node->callees; cs; cs = cs->next_callee)
4699 : {
4700 4 : if (cs->callee->ultimate_alias_target () == orig_node)
4701 : {
4702 4 : unsigned den = 0;
4703 18 : for (cgraph_edge *e = cs; e; e = get_next_cgraph_edge_clone (e))
4704 14 : if (e->callee->ultimate_alias_target () == orig_node
4705 14 : && processed_edges.contains (e))
4706 8 : den++;
4707 4 : if (den > 0)
4708 18 : for (cgraph_edge *e = cs; e; e = get_next_cgraph_edge_clone (e))
4709 14 : if (e->callee->ultimate_alias_target () == orig_node
4710 8 : && processed_edges.contains (e)
4711 : /* If count is not IPA, this adjustment makes verifier
4712 : unhappy, since we expect bb->count to match e->count.
4713 : We may add a flag to mark edge conts that has been
4714 : modified by IPA code, but so far it does not seem
4715 : to be worth the effort. With local counts the profile
4716 : will not propagate at IPA level. */
4717 30 : && e->count.ipa_p ())
4718 8 : e->count /= den;
4719 : }
4720 : }
4721 :
4722 : /* Edges from the seeds of the values generated for arithmetic jump-functions
4723 : along self-recursive edges are likely to have fairly low count and so
4724 : edges from them to nodes in the self_gen_clones do not correspond to the
4725 : artificially distributed count of the nodes, the total sum of incoming
4726 : edges to some clones might be too low. Detect this situation and correct
4727 : it. */
4728 20 : for (cgraph_node *n : self_gen_clones)
4729 : {
4730 8 : if (!n->count.ipa ().nonzero_p ())
4731 0 : continue;
4732 :
4733 8 : desc_incoming_count_struct desc;
4734 8 : desc.orig = orig_node;
4735 8 : desc.processed_edges = &processed_edges;
4736 8 : desc.count = profile_count::zero ();
4737 8 : desc.unproc_orig_rec_edges = 0;
4738 8 : analyze_clone_icoming_counts (n, &desc);
4739 :
4740 8 : if (n->count.differs_from_p (desc.count))
4741 : {
4742 0 : if (n->count > desc.count
4743 0 : && desc.unproc_orig_rec_edges > 0)
4744 : {
4745 0 : desc.count = n->count - desc.count;
4746 0 : desc.count = desc.count /= desc.unproc_orig_rec_edges;
4747 0 : adjust_clone_incoming_counts (n, &desc);
4748 : }
4749 0 : else if (dump_file)
4750 0 : fprintf (dump_file,
4751 : " Unable to fix up incoming counts for %s.\n",
4752 : n->dump_name ());
4753 : }
4754 : }
4755 :
4756 4 : if (dump_file)
4757 0 : for (cgraph_node *n : self_gen_clones)
4758 0 : dump_profile_updates (n, n != orig_node);
4759 4 : return;
4760 4 : }
4761 :
4762 : /* After a specialized NEW_NODE version of ORIG_NODE has been created, update
4763 : their profile information to reflect this. This function should not be used
4764 : for clones generated for arithmetic pass-through jump functions on a
4765 : self-recursive call graph edge, that situation is handled by
4766 : update_counts_for_self_gen_clones. */
4767 :
4768 : static void
4769 4215 : update_profiling_info (struct cgraph_node *orig_node,
4770 : struct cgraph_node *new_node)
4771 : {
4772 4215 : struct caller_statistics stats;
4773 4215 : profile_count new_sum;
4774 4215 : profile_count remainder, orig_node_count = orig_node->count.ipa ();
4775 :
4776 4215 : if (!orig_node_count.nonzero_p ())
4777 4207 : return;
4778 :
4779 8 : if (dump_file)
4780 : {
4781 2 : fprintf (dump_file, " Updating profile from original count: ");
4782 2 : orig_node_count.dump (dump_file);
4783 2 : fprintf (dump_file, "\n");
4784 : }
4785 :
4786 8 : init_caller_stats (&stats, new_node);
4787 8 : new_node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
4788 : false);
4789 8 : new_sum = stats.count_sum;
4790 :
4791 8 : bool orig_edges_processed = false;
4792 8 : if (new_sum > orig_node_count)
4793 : {
4794 : /* Profile has already gone astray, keep what we have but lower it
4795 : to global0adjusted or to local if we have partial training. */
4796 0 : if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
4797 0 : orig_node->make_profile_local ();
4798 0 : if (new_sum.quality () == AFDO)
4799 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_AFDO);
4800 : else
4801 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_ADJUSTED);
4802 : orig_edges_processed = true;
4803 : }
4804 8 : else if (stats.rec_count_sum.nonzero_p ())
4805 : {
4806 0 : int new_nonrec_calls = stats.n_nonrec_calls;
4807 : /* There are self-recursive edges which are likely to bring in the
4808 : majority of calls but which we must divide in between the original and
4809 : new node. */
4810 0 : init_caller_stats (&stats, orig_node);
4811 0 : orig_node->call_for_symbol_thunks_and_aliases (gather_caller_stats,
4812 : &stats, false);
4813 0 : int orig_nonrec_calls = stats.n_nonrec_calls;
4814 0 : profile_count orig_nonrec_call_count = stats.count_sum;
4815 :
4816 0 : if (orig_node->local)
4817 : {
4818 0 : if (!orig_nonrec_call_count.nonzero_p ())
4819 : {
4820 0 : if (dump_file)
4821 0 : fprintf (dump_file, " The original is local and the only "
4822 : "incoming edges from non-dead callers with nonzero "
4823 : "counts are self-recursive, assuming it is cold.\n");
4824 : /* The NEW_NODE count and counts of all its outgoing edges
4825 : are still unmodified copies of ORIG_NODE's. Just clear
4826 : the latter and bail out. */
4827 0 : if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
4828 0 : orig_node->make_profile_local ();
4829 0 : else if (orig_nonrec_call_count.quality () == AFDO)
4830 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_AFDO);
4831 : else
4832 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_ADJUSTED);
4833 0 : return;
4834 : }
4835 : }
4836 : else
4837 : {
4838 : /* Let's behave as if there was another caller that accounts for all
4839 : the calls that were either indirect or from other compilation
4840 : units. */
4841 0 : orig_nonrec_calls++;
4842 0 : profile_count pretend_caller_count
4843 0 : = (orig_node_count - new_sum - orig_nonrec_call_count
4844 0 : - stats.rec_count_sum);
4845 0 : orig_nonrec_call_count += pretend_caller_count;
4846 : }
4847 :
4848 : /* Divide all "unexplained" counts roughly proportionally to sums of
4849 : counts of non-recursive calls.
4850 :
4851 : We put rather arbitrary limits on how many counts we claim because the
4852 : number of non-self-recursive incoming count is only a rough guideline
4853 : and there are cases (such as mcf) where using it blindly just takes
4854 : too many. And if lattices are considered in the opposite order we
4855 : could also take too few. */
4856 0 : profile_count unexp = orig_node_count - new_sum - orig_nonrec_call_count;
4857 :
4858 0 : int limit_den = 2 * (orig_nonrec_calls + new_nonrec_calls);
4859 0 : profile_count new_part = unexp.apply_scale (limit_den - 1, limit_den);
4860 0 : profile_count den = new_sum + orig_nonrec_call_count;
4861 0 : if (den.nonzero_p ())
4862 0 : new_part = MIN (unexp.apply_scale (new_sum, den), new_part);
4863 0 : new_part = MAX (new_part,
4864 : unexp.apply_scale (new_nonrec_calls, limit_den));
4865 0 : if (dump_file)
4866 : {
4867 0 : fprintf (dump_file, " Claiming ");
4868 0 : new_part.dump (dump_file);
4869 0 : fprintf (dump_file, " of unexplained ");
4870 0 : unexp.dump (dump_file);
4871 0 : fprintf (dump_file, " counts because of self-recursive "
4872 : "calls\n");
4873 : }
4874 0 : new_sum += new_part;
4875 0 : remainder = lenient_count_portion_handling (orig_node_count - new_sum,
4876 : orig_node);
4877 : }
4878 : else
4879 8 : remainder = lenient_count_portion_handling (orig_node_count - new_sum,
4880 : orig_node);
4881 :
4882 8 : new_node->scale_profile_to (new_sum);
4883 :
4884 8 : if (!orig_edges_processed)
4885 8 : orig_node->scale_profile_to (remainder);
4886 :
4887 8 : if (dump_file)
4888 : {
4889 2 : dump_profile_updates (new_node, true);
4890 2 : dump_profile_updates (orig_node, false);
4891 : }
4892 : }
4893 :
4894 : /* Update the respective profile of specialized NEW_NODE and the original
4895 : ORIG_NODE after additional edges with cumulative count sum REDIRECTED_SUM
4896 : have been redirected to the specialized version. */
4897 :
4898 : static void
4899 0 : update_specialized_profile (struct cgraph_node *new_node,
4900 : struct cgraph_node *orig_node,
4901 : profile_count redirected_sum)
4902 : {
4903 0 : if (dump_file)
4904 : {
4905 0 : fprintf (dump_file, " the sum of counts of redirected edges is ");
4906 0 : redirected_sum.dump (dump_file);
4907 0 : fprintf (dump_file, "\n old ipa count of the original node is ");
4908 0 : orig_node->count.dump (dump_file);
4909 0 : fprintf (dump_file, "\n");
4910 : }
4911 0 : if (!orig_node->count.ipa ().nonzero_p ()
4912 0 : || !redirected_sum.nonzero_p ())
4913 0 : return;
4914 :
4915 0 : orig_node->scale_profile_to
4916 0 : (lenient_count_portion_handling (orig_node->count.ipa () - redirected_sum,
4917 : orig_node));
4918 :
4919 0 : new_node->scale_profile_to (new_node->count.ipa () + redirected_sum);
4920 :
4921 0 : if (dump_file)
4922 : {
4923 0 : dump_profile_updates (new_node, true);
4924 0 : dump_profile_updates (orig_node, false);
4925 : }
4926 : }
4927 :
4928 : static void adjust_references_in_caller (cgraph_edge *cs,
4929 : symtab_node *symbol, int index);
4930 :
4931 : /* Simple structure to pass a symbol and index (with same meaning as parameters
4932 : of adjust_references_in_caller) through a void* parameter of a
4933 : call_for_symbol_thunks_and_aliases callback. */
4934 : struct symbol_and_index_together
4935 : {
4936 : symtab_node *symbol;
4937 : int index;
4938 : };
4939 :
4940 : /* Worker callback of call_for_symbol_thunks_and_aliases to recursively call
4941 : adjust_references_in_caller on edges up in the call-graph, if necessary. */
4942 : static bool
4943 9 : adjust_refs_in_act_callers (struct cgraph_node *node, void *data)
4944 : {
4945 9 : symbol_and_index_together *pack = (symbol_and_index_together *) data;
4946 40 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4947 31 : if (!cs->caller->thunk)
4948 31 : adjust_references_in_caller (cs, pack->symbol, pack->index);
4949 9 : return false;
4950 : }
4951 :
4952 : /* At INDEX of a function being called by CS there is an ADDR_EXPR of a
4953 : variable which is only dereferenced and which is represented by SYMBOL. See
4954 : if we can remove ADDR reference in callers associated with the call. */
4955 :
4956 : static void
4957 402 : adjust_references_in_caller (cgraph_edge *cs, symtab_node *symbol, int index)
4958 : {
4959 402 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
4960 402 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, index);
4961 402 : if (jfunc->type == IPA_JF_CONST)
4962 : {
4963 383 : ipa_ref *to_del = cs->caller->find_reference (symbol, cs->call_stmt,
4964 : cs->lto_stmt_uid,
4965 : IPA_REF_ADDR);
4966 383 : if (!to_del)
4967 393 : return;
4968 383 : to_del->remove_reference ();
4969 383 : ipa_zap_jf_refdesc (jfunc);
4970 383 : if (dump_file)
4971 22 : fprintf (dump_file, " Removed a reference from %s to %s.\n",
4972 11 : cs->caller->dump_name (), symbol->dump_name ());
4973 383 : return;
4974 : }
4975 :
4976 19 : if (jfunc->type != IPA_JF_PASS_THROUGH
4977 19 : || ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR
4978 38 : || ipa_get_jf_pass_through_refdesc_decremented (jfunc))
4979 : return;
4980 :
4981 19 : int fidx = ipa_get_jf_pass_through_formal_id (jfunc);
4982 19 : cgraph_node *caller = cs->caller;
4983 19 : ipa_node_params *caller_info = ipa_node_params_sum->get (caller);
4984 : /* TODO: This consistency check may be too big and not really
4985 : that useful. Consider removing it. */
4986 19 : tree cst;
4987 19 : if (caller_info->ipcp_orig_node)
4988 17 : cst = caller_info->known_csts[fidx];
4989 : else
4990 : {
4991 2 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (caller_info, fidx);
4992 2 : gcc_assert (lat->is_single_const ());
4993 2 : cst = lat->values->value;
4994 : }
4995 19 : gcc_assert (TREE_CODE (cst) == ADDR_EXPR
4996 : && (symtab_node::get (get_base_address (TREE_OPERAND (cst, 0)))
4997 : == symbol));
4998 :
4999 19 : int cuses = ipa_get_controlled_uses (caller_info, fidx);
5000 19 : if (cuses == IPA_UNDESCRIBED_USE)
5001 : return;
5002 19 : gcc_assert (cuses > 0);
5003 19 : cuses--;
5004 19 : ipa_set_controlled_uses (caller_info, fidx, cuses);
5005 19 : ipa_set_jf_pass_through_refdesc_decremented (jfunc, true);
5006 19 : if (dump_file && (dump_flags & TDF_DETAILS))
5007 3 : fprintf (dump_file, " Controlled uses of parameter %i of %s dropped "
5008 : "to %i.\n", fidx, caller->dump_name (), cuses);
5009 19 : if (cuses)
5010 : return;
5011 :
5012 9 : if (caller_info->ipcp_orig_node)
5013 : {
5014 : /* Cloning machinery has created a reference here, we need to either
5015 : remove it or change it to a read one. */
5016 7 : ipa_ref *to_del = caller->find_reference (symbol, NULL, 0, IPA_REF_ADDR);
5017 7 : if (to_del)
5018 : {
5019 7 : to_del->remove_reference ();
5020 7 : if (dump_file)
5021 6 : fprintf (dump_file, " Removed a reference from %s to %s.\n",
5022 3 : cs->caller->dump_name (), symbol->dump_name ());
5023 7 : if (ipa_get_param_load_dereferenced (caller_info, fidx))
5024 : {
5025 3 : caller->create_reference (symbol, IPA_REF_LOAD, NULL);
5026 3 : if (dump_file)
5027 2 : fprintf (dump_file,
5028 : " ...and replaced it with LOAD one.\n");
5029 : }
5030 : }
5031 : }
5032 :
5033 9 : symbol_and_index_together pack;
5034 9 : pack.symbol = symbol;
5035 9 : pack.index = fidx;
5036 9 : if (caller->can_change_signature)
5037 9 : caller->call_for_symbol_thunks_and_aliases (adjust_refs_in_act_callers,
5038 : &pack, true);
5039 : }
5040 :
5041 :
5042 : /* Return true if we would like to remove a parameter from NODE when cloning it
5043 : with KNOWN_CSTS scalar constants. */
5044 :
5045 : static bool
5046 17516 : want_remove_some_param_p (cgraph_node *node, vec<tree> known_csts)
5047 : {
5048 17516 : auto_vec<bool, 16> surviving;
5049 17516 : bool filled_vec = false;
5050 17516 : ipa_node_params *info = ipa_node_params_sum->get (node);
5051 17516 : int i, count = ipa_get_param_count (info);
5052 :
5053 36332 : for (i = 0; i < count; i++)
5054 : {
5055 31779 : if (!known_csts[i] && ipa_is_param_used (info, i))
5056 18816 : continue;
5057 :
5058 12963 : if (!filled_vec)
5059 : {
5060 12963 : clone_info *info = clone_info::get (node);
5061 12963 : if (!info || !info->param_adjustments)
5062 : return true;
5063 0 : info->param_adjustments->get_surviving_params (&surviving);
5064 0 : filled_vec = true;
5065 : }
5066 0 : if (surviving.length() < (unsigned) i && surviving[i])
5067 : return true;
5068 : }
5069 : return false;
5070 17516 : }
5071 :
5072 : /* Create a specialized version of NODE with known constants in KNOWN_CSTS,
5073 : known contexts in KNOWN_CONTEXTS and known aggregate values in AGGVALS and
5074 : redirect all edges in CALLERS to it. */
5075 :
5076 : static struct cgraph_node *
5077 18931 : create_specialized_node (struct cgraph_node *node,
5078 : vec<tree> known_csts,
5079 : vec<ipa_polymorphic_call_context> known_contexts,
5080 : vec<ipa_argagg_value, va_gc> *aggvals,
5081 : vec<cgraph_edge *> &callers)
5082 : {
5083 18931 : ipa_node_params *new_info, *info = ipa_node_params_sum->get (node);
5084 18931 : vec<ipa_replace_map *, va_gc> *replace_trees = NULL;
5085 18931 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
5086 18931 : struct cgraph_node *new_node;
5087 18931 : int i, count = ipa_get_param_count (info);
5088 18931 : clone_info *cinfo = clone_info::get (node);
5089 0 : ipa_param_adjustments *old_adjustments = cinfo
5090 18931 : ? cinfo->param_adjustments : NULL;
5091 18931 : ipa_param_adjustments *new_adjustments;
5092 18931 : gcc_assert (!info->ipcp_orig_node);
5093 18931 : gcc_assert (node->can_change_signature
5094 : || !old_adjustments);
5095 :
5096 17516 : if (old_adjustments)
5097 : {
5098 : /* At the moment all IPA optimizations should use the number of
5099 : parameters of the prevailing decl as the m_always_copy_start.
5100 : Handling any other value would complicate the code below, so for the
5101 : time bing let's only assert it is so. */
5102 0 : gcc_assert (old_adjustments->m_always_copy_start == count
5103 : || old_adjustments->m_always_copy_start < 0);
5104 0 : int old_adj_count = vec_safe_length (old_adjustments->m_adj_params);
5105 0 : for (i = 0; i < old_adj_count; i++)
5106 : {
5107 0 : ipa_adjusted_param *old_adj = &(*old_adjustments->m_adj_params)[i];
5108 0 : if (!node->can_change_signature
5109 0 : || old_adj->op != IPA_PARAM_OP_COPY
5110 0 : || (!known_csts[old_adj->base_index]
5111 0 : && ipa_is_param_used (info, old_adj->base_index)))
5112 : {
5113 0 : ipa_adjusted_param new_adj = *old_adj;
5114 :
5115 0 : new_adj.prev_clone_adjustment = true;
5116 0 : new_adj.prev_clone_index = i;
5117 0 : vec_safe_push (new_params, new_adj);
5118 : }
5119 : }
5120 0 : bool skip_return = old_adjustments->m_skip_return;
5121 0 : new_adjustments = (new (ggc_alloc <ipa_param_adjustments> ())
5122 : ipa_param_adjustments (new_params, count,
5123 0 : skip_return));
5124 : }
5125 18931 : else if (node->can_change_signature
5126 18931 : && want_remove_some_param_p (node, known_csts))
5127 : {
5128 12963 : ipa_adjusted_param adj;
5129 12963 : memset (&adj, 0, sizeof (adj));
5130 12963 : adj.op = IPA_PARAM_OP_COPY;
5131 50325 : for (i = 0; i < count; i++)
5132 37362 : if (!known_csts[i] && ipa_is_param_used (info, i))
5133 : {
5134 14930 : adj.base_index = i;
5135 14930 : adj.prev_clone_index = i;
5136 14930 : vec_safe_push (new_params, adj);
5137 : }
5138 12963 : new_adjustments = (new (ggc_alloc <ipa_param_adjustments> ())
5139 12963 : ipa_param_adjustments (new_params, count, false));
5140 : }
5141 : else
5142 : new_adjustments = NULL;
5143 :
5144 18931 : auto_vec<cgraph_edge *, 2> self_recursive_calls;
5145 155274 : for (i = callers.length () - 1; i >= 0; i--)
5146 : {
5147 117412 : cgraph_edge *cs = callers[i];
5148 117412 : if (cs->caller == node)
5149 : {
5150 113 : self_recursive_calls.safe_push (cs);
5151 113 : callers.unordered_remove (i);
5152 : }
5153 : }
5154 18931 : replace_trees = cinfo ? vec_safe_copy (cinfo->tree_map) : NULL;
5155 71598 : for (i = 0; i < count; i++)
5156 : {
5157 52667 : tree t = known_csts[i];
5158 52667 : if (!t)
5159 28327 : continue;
5160 :
5161 24340 : gcc_checking_assert (TREE_CODE (t) != TREE_BINFO);
5162 :
5163 24340 : bool load_ref = false;
5164 24340 : symtab_node *ref_symbol;
5165 24340 : if (TREE_CODE (t) == ADDR_EXPR)
5166 : {
5167 6539 : tree base = get_base_address (TREE_OPERAND (t, 0));
5168 6539 : if (TREE_CODE (base) == VAR_DECL
5169 3119 : && ipa_get_controlled_uses (info, i) == 0
5170 917 : && ipa_get_param_load_dereferenced (info, i)
5171 6925 : && (ref_symbol = symtab_node::get (base)))
5172 : {
5173 386 : load_ref = true;
5174 386 : if (node->can_change_signature)
5175 1403 : for (cgraph_edge *caller : callers)
5176 371 : adjust_references_in_caller (caller, ref_symbol, i);
5177 : }
5178 : }
5179 :
5180 24340 : ipa_replace_map *replace_map = get_replacement_map (info, t, i, load_ref);
5181 24340 : if (replace_map)
5182 24340 : vec_safe_push (replace_trees, replace_map);
5183 : }
5184 :
5185 56793 : unsigned &suffix_counter = clone_num_suffixes->get_or_insert (
5186 18931 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (
5187 : node->decl)));
5188 18931 : new_node = node->create_virtual_clone (callers, replace_trees,
5189 : new_adjustments, "constprop",
5190 : suffix_counter);
5191 18931 : suffix_counter++;
5192 :
5193 18931 : bool have_self_recursive_calls = !self_recursive_calls.is_empty ();
5194 19044 : for (unsigned j = 0; j < self_recursive_calls.length (); j++)
5195 : {
5196 113 : cgraph_edge *cs = get_next_cgraph_edge_clone (self_recursive_calls[j]);
5197 : /* Cloned edges can disappear during cloning as speculation can be
5198 : resolved, check that we have one and that it comes from the last
5199 : cloning. */
5200 113 : if (cs && cs->caller == new_node)
5201 112 : cs->redirect_callee_duplicating_thunks (new_node);
5202 : /* Any future code that would make more than one clone of an outgoing
5203 : edge would confuse this mechanism, so let's check that does not
5204 : happen. */
5205 112 : gcc_checking_assert (!cs
5206 : || !get_next_cgraph_edge_clone (cs)
5207 : || get_next_cgraph_edge_clone (cs)->caller != new_node);
5208 : }
5209 18931 : if (have_self_recursive_calls)
5210 107 : new_node->expand_all_artificial_thunks ();
5211 :
5212 18931 : ipa_set_node_agg_value_chain (new_node, aggvals);
5213 49189 : for (const ipa_argagg_value &av : aggvals)
5214 30258 : new_node->maybe_create_reference (av.value, NULL);
5215 :
5216 18931 : if (dump_file && (dump_flags & TDF_DETAILS))
5217 : {
5218 91 : fprintf (dump_file, " the new node is %s.\n", new_node->dump_name ());
5219 91 : if (known_contexts.exists ())
5220 : {
5221 0 : for (i = 0; i < count; i++)
5222 0 : if (!known_contexts[i].useless_p ())
5223 : {
5224 0 : fprintf (dump_file, " known ctx %i is ", i);
5225 0 : known_contexts[i].dump (dump_file);
5226 : }
5227 : }
5228 91 : if (aggvals)
5229 : {
5230 49 : fprintf (dump_file, " Aggregate replacements:");
5231 49 : ipa_argagg_value_list avs (aggvals);
5232 49 : avs.dump (dump_file);
5233 : }
5234 : }
5235 :
5236 18931 : new_info = ipa_node_params_sum->get (new_node);
5237 18931 : new_info->ipcp_orig_node = node;
5238 18931 : new_node->ipcp_clone = true;
5239 18931 : new_info->known_csts = known_csts;
5240 18931 : new_info->known_contexts = known_contexts;
5241 :
5242 18931 : ipcp_discover_new_direct_edges (new_node, known_csts, known_contexts,
5243 : aggvals);
5244 :
5245 18931 : return new_node;
5246 18931 : }
5247 :
5248 : /* Return true if JFUNC, which describes a i-th parameter of call CS, is a
5249 : pass-through function to itself when the cgraph_node involved is not an
5250 : IPA-CP clone. When SIMPLE is true, further check if JFUNC is a simple
5251 : no-operation pass-through. */
5252 :
5253 : static bool
5254 783824 : self_recursive_pass_through_p (cgraph_edge *cs, ipa_jump_func *jfunc, int i,
5255 : bool simple = true)
5256 : {
5257 783824 : enum availability availability;
5258 783824 : if (jfunc->type == IPA_JF_PASS_THROUGH
5259 78631 : && cs->caller == cs->callee->function_symbol (&availability)
5260 19355 : && availability > AVAIL_INTERPOSABLE
5261 19355 : && (!simple || ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
5262 19355 : && ipa_get_jf_pass_through_formal_id (jfunc) == i
5263 19355 : && ipa_node_params_sum->get (cs->caller)
5264 803179 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5265 : return true;
5266 : return false;
5267 : }
5268 :
5269 : /* Return true if JFUNC, which describes the i-th parameter of call CS, is an
5270 : ancestor function with zero offset to itself when the cgraph_node involved
5271 : is not an IPA-CP clone. */
5272 :
5273 : static bool
5274 764498 : self_recursive_ancestor_p (cgraph_edge *cs, ipa_jump_func *jfunc, int i)
5275 : {
5276 764498 : enum availability availability;
5277 764498 : if (jfunc->type == IPA_JF_ANCESTOR
5278 3230 : && cs->caller == cs->callee->function_symbol (&availability)
5279 1 : && availability > AVAIL_INTERPOSABLE
5280 1 : && ipa_get_jf_ancestor_offset (jfunc) == 0
5281 1 : && ipa_get_jf_ancestor_formal_id (jfunc) == i
5282 1 : && ipa_node_params_sum->get (cs->caller)
5283 764499 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5284 : return true;
5285 : return false;
5286 : }
5287 :
5288 : /* Return true if JFUNC, which describes a part of an aggregate represented or
5289 : pointed to by the i-th parameter of call CS, is a pass-through function to
5290 : itself when the cgraph_node involved is not an IPA-CP clone.. When
5291 : SIMPLE is true, further check if JFUNC is a simple no-operation
5292 : pass-through. */
5293 :
5294 : static bool
5295 358673 : self_recursive_agg_pass_through_p (const cgraph_edge *cs,
5296 : const ipa_agg_jf_item *jfunc,
5297 : int i, bool simple = true)
5298 : {
5299 358673 : enum availability availability;
5300 358673 : if (cs->caller == cs->callee->function_symbol (&availability)
5301 3687 : && availability > AVAIL_INTERPOSABLE
5302 3687 : && jfunc->jftype == IPA_JF_LOAD_AGG
5303 487 : && jfunc->offset == jfunc->value.load_agg.offset
5304 487 : && (!simple || jfunc->value.pass_through.operation == NOP_EXPR)
5305 487 : && jfunc->value.pass_through.formal_id == i
5306 481 : && useless_type_conversion_p (jfunc->value.load_agg.type, jfunc->type)
5307 481 : && ipa_node_params_sum->get (cs->caller)
5308 359154 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5309 : return true;
5310 : return false;
5311 : }
5312 :
5313 : /* Given a NODE, and a subset of its CALLERS, try to populate blanks slots in
5314 : KNOWN_CSTS with constants that are also known for all of the CALLERS. */
5315 :
5316 : static void
5317 166477 : find_scalar_values_for_callers_subset (vec<tree> &known_csts,
5318 : ipa_node_params *info,
5319 : const vec<cgraph_edge *> &callers)
5320 : {
5321 166477 : int i, count = ipa_get_param_count (info);
5322 :
5323 729057 : for (i = 0; i < count; i++)
5324 : {
5325 562580 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (info, i);
5326 562580 : if (lat->bottom)
5327 562580 : continue;
5328 553658 : if (lat->is_single_const ())
5329 : {
5330 29353 : known_csts[i] = lat->values->value;
5331 29353 : continue;
5332 : }
5333 :
5334 524305 : struct cgraph_edge *cs;
5335 524305 : tree newval = NULL_TREE;
5336 524305 : int j;
5337 524305 : bool first = true;
5338 524305 : tree type = ipa_get_type (info, i);
5339 :
5340 980348 : FOR_EACH_VEC_ELT (callers, j, cs)
5341 : {
5342 783272 : struct ipa_jump_func *jump_func;
5343 783272 : tree t;
5344 :
5345 783272 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5346 783272 : if (!args
5347 783272 : || i >= ipa_get_cs_argument_count (args)
5348 1566513 : || (i == 0
5349 180546 : && call_passes_through_thunk (cs)))
5350 : {
5351 : newval = NULL_TREE;
5352 : break;
5353 : }
5354 783193 : jump_func = ipa_get_ith_jump_func (args, i);
5355 :
5356 : /* Besides simple pass-through jump function, arithmetic jump
5357 : function could also introduce argument-direct-pass-through for
5358 : self-feeding recursive call. For example,
5359 :
5360 : fn (int i)
5361 : {
5362 : fn (i & 1);
5363 : }
5364 :
5365 : Given that i is 0, recursive propagation via (i & 1) also gets
5366 : 0. */
5367 783193 : if (self_recursive_pass_through_p (cs, jump_func, i, false))
5368 : {
5369 18702 : gcc_assert (newval);
5370 18702 : enum tree_code opcode
5371 18702 : = ipa_get_jf_pass_through_operation (jump_func);
5372 18702 : tree op_type = (opcode == NOP_EXPR) ? NULL_TREE
5373 49 : : ipa_get_jf_pass_through_op_type (jump_func);
5374 18702 : t = ipa_get_jf_arith_result (opcode, newval,
5375 : ipa_get_jf_pass_through_operand (jump_func),
5376 : op_type);
5377 18702 : t = ipacp_value_safe_for_type (type, t);
5378 : }
5379 764491 : else if (self_recursive_ancestor_p (cs, jump_func, i))
5380 0 : continue;
5381 : else
5382 764491 : t = ipa_value_from_jfunc (ipa_node_params_sum->get (cs->caller),
5383 : jump_func, type);
5384 783193 : if (!t
5385 477670 : || (newval
5386 254219 : && !values_equal_for_ipcp_p (t, newval))
5387 1239236 : || (!first && !newval))
5388 : {
5389 : newval = NULL_TREE;
5390 : break;
5391 : }
5392 : else
5393 : newval = t;
5394 : first = false;
5395 : }
5396 :
5397 524305 : if (newval)
5398 197076 : known_csts[i] = newval;
5399 : }
5400 166477 : }
5401 :
5402 : /* Given a NODE and a subset of its CALLERS, try to populate plank slots in
5403 : KNOWN_CONTEXTS with polymorphic contexts that are also known for all of the
5404 : CALLERS. */
5405 :
5406 : static void
5407 166477 : find_contexts_for_caller_subset (vec<ipa_polymorphic_call_context>
5408 : &known_contexts,
5409 : ipa_node_params *info,
5410 : const vec<cgraph_edge *> &callers)
5411 : {
5412 166477 : int i, count = ipa_get_param_count (info);
5413 :
5414 729038 : for (i = 0; i < count; i++)
5415 : {
5416 562574 : if (!ipa_is_param_used (info, i))
5417 23222 : continue;
5418 :
5419 540482 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat
5420 540482 : = ipa_get_poly_ctx_lat (info, i);
5421 540482 : if (ctxlat->bottom)
5422 0 : continue;
5423 540482 : if (ctxlat->is_single_const ())
5424 : {
5425 1130 : if (!ctxlat->values->value.useless_p ())
5426 : {
5427 1130 : if (known_contexts.is_empty ())
5428 1069 : known_contexts.safe_grow_cleared (count, true);
5429 1130 : known_contexts[i] = ctxlat->values->value;
5430 : }
5431 1130 : continue;
5432 : }
5433 :
5434 539352 : cgraph_edge *cs;
5435 539352 : ipa_polymorphic_call_context newval;
5436 539352 : bool first = true;
5437 539352 : int j;
5438 :
5439 544662 : FOR_EACH_VEC_ELT (callers, j, cs)
5440 : {
5441 540885 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5442 540885 : if (!args
5443 1081770 : || i >= ipa_get_cs_argument_count (args))
5444 13 : return;
5445 540872 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
5446 540872 : ipa_polymorphic_call_context ctx;
5447 540872 : ctx = ipa_context_from_jfunc (ipa_node_params_sum->get (cs->caller),
5448 : cs, i, jfunc);
5449 540872 : if (first)
5450 : {
5451 539339 : newval = ctx;
5452 539339 : first = false;
5453 : }
5454 : else
5455 1533 : newval.meet_with (ctx);
5456 1079003 : if (newval.useless_p ())
5457 : break;
5458 : }
5459 :
5460 1078678 : if (!newval.useless_p ())
5461 : {
5462 3777 : if (known_contexts.is_empty ())
5463 3554 : known_contexts.safe_grow_cleared (count, true);
5464 3777 : known_contexts[i] = newval;
5465 : }
5466 :
5467 : }
5468 : }
5469 :
5470 : /* Push all aggregate values coming along edge CS for parameter number INDEX to
5471 : RES. If INTERIM is non-NULL, it contains the current interim state of
5472 : collected aggregate values which can be used to compute values passed over
5473 : self-recursive edges.
5474 :
5475 : This basically one iteration of push_agg_values_from_edge over one
5476 : parameter, which allows for simpler early returns. */
5477 :
5478 : static void
5479 629753 : push_agg_values_for_index_from_edge (struct cgraph_edge *cs, int index,
5480 : vec<ipa_argagg_value> *res,
5481 : const ipa_argagg_value_list *interim)
5482 : {
5483 629753 : bool agg_values_from_caller = false;
5484 629753 : bool agg_jf_preserved = false;
5485 629753 : unsigned unit_delta = UINT_MAX;
5486 629753 : int src_idx = -1;
5487 629753 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (ipa_edge_args_sum->get (cs),
5488 : index);
5489 :
5490 629753 : if (jfunc->type == IPA_JF_PASS_THROUGH
5491 629753 : && ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
5492 : {
5493 57885 : agg_values_from_caller = true;
5494 57885 : agg_jf_preserved = ipa_get_jf_pass_through_agg_preserved (jfunc);
5495 57885 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
5496 57885 : unit_delta = 0;
5497 : }
5498 571868 : else if (jfunc->type == IPA_JF_ANCESTOR
5499 571868 : && ipa_get_jf_ancestor_agg_preserved (jfunc))
5500 : {
5501 407 : agg_values_from_caller = true;
5502 407 : agg_jf_preserved = true;
5503 407 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
5504 407 : unit_delta = ipa_get_jf_ancestor_offset (jfunc) / BITS_PER_UNIT;
5505 : }
5506 :
5507 629753 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
5508 629753 : if (agg_values_from_caller)
5509 : {
5510 58292 : if (caller_info->ipcp_orig_node)
5511 : {
5512 11085 : struct cgraph_node *orig_node = caller_info->ipcp_orig_node;
5513 11085 : ipcp_transformation *ts
5514 11085 : = ipcp_get_transformation_summary (cs->caller);
5515 11085 : ipa_node_params *orig_info = ipa_node_params_sum->get (orig_node);
5516 11085 : ipcp_param_lattices *orig_plats
5517 11085 : = ipa_get_parm_lattices (orig_info, src_idx);
5518 11085 : if (ts
5519 11085 : && orig_plats->aggs
5520 3013 : && (agg_jf_preserved || !orig_plats->aggs_by_ref))
5521 : {
5522 2542 : ipa_argagg_value_list src (ts);
5523 2542 : src.push_adjusted_values (src_idx, index, unit_delta, res);
5524 2542 : return;
5525 : }
5526 : }
5527 : else
5528 : {
5529 47207 : ipcp_param_lattices *src_plats
5530 47207 : = ipa_get_parm_lattices (caller_info, src_idx);
5531 47207 : if (src_plats->aggs
5532 2433 : && !src_plats->aggs_bottom
5533 2433 : && (agg_jf_preserved || !src_plats->aggs_by_ref))
5534 : {
5535 1449 : if (interim && (self_recursive_pass_through_p (cs, jfunc, index)
5536 7 : || self_recursive_ancestor_p (cs, jfunc, index)))
5537 : {
5538 625 : interim->push_adjusted_values (src_idx, index, unit_delta,
5539 : res);
5540 625 : return;
5541 : }
5542 824 : if (!src_plats->aggs_contain_variable)
5543 : {
5544 83 : push_agg_values_from_plats (src_plats, index, unit_delta,
5545 : res);
5546 83 : return;
5547 : }
5548 : }
5549 : }
5550 : }
5551 :
5552 626503 : if (!jfunc->agg.items)
5553 : return;
5554 224906 : bool first = true;
5555 224906 : unsigned prev_unit_offset = 0;
5556 1252485 : for (const ipa_agg_jf_item &agg_jf : *jfunc->agg.items)
5557 : {
5558 1027579 : tree value, srcvalue;
5559 : /* Besides simple pass-through aggregate jump function, arithmetic
5560 : aggregate jump function could also bring same aggregate value as
5561 : parameter passed-in for self-feeding recursive call. For example,
5562 :
5563 : fn (int *i)
5564 : {
5565 : int j = *i & 1;
5566 : fn (&j);
5567 : }
5568 :
5569 : Given that *i is 0, recursive propagation via (*i & 1) also gets 0. */
5570 1027579 : if (interim
5571 358673 : && self_recursive_agg_pass_through_p (cs, &agg_jf, index, false)
5572 1028060 : && (srcvalue = interim->get_value(index,
5573 481 : agg_jf.offset / BITS_PER_UNIT)))
5574 : {
5575 950 : value = ipa_get_jf_arith_result (agg_jf.value.pass_through.operation,
5576 : srcvalue,
5577 475 : agg_jf.value.pass_through.operand,
5578 475 : agg_jf.value.pass_through.op_type);
5579 475 : value = ipacp_value_safe_for_type (agg_jf.type, value);
5580 : }
5581 : else
5582 1027104 : value = ipa_agg_value_from_jfunc (caller_info, cs->caller,
5583 : &agg_jf);
5584 1027579 : if (value)
5585 : {
5586 1002033 : struct ipa_argagg_value iav;
5587 1002033 : iav.value = value;
5588 1002033 : iav.unit_offset = agg_jf.offset / BITS_PER_UNIT;
5589 1002033 : iav.index = index;
5590 1002033 : iav.by_ref = jfunc->agg.by_ref;
5591 1002033 : iav.killed = false;
5592 :
5593 1002033 : gcc_assert (first
5594 : || iav.unit_offset > prev_unit_offset);
5595 1002033 : prev_unit_offset = iav.unit_offset;
5596 1002033 : first = false;
5597 :
5598 1002033 : res->safe_push (iav);
5599 : }
5600 : }
5601 : return;
5602 : }
5603 :
5604 : /* Push all aggregate values coming along edge CS to RES. DEST_INFO is the
5605 : description of ultimate callee of CS or the one it was cloned from (the
5606 : summary where lattices are). If INTERIM is non-NULL, it contains the
5607 : current interim state of collected aggregate values which can be used to
5608 : compute values passed over self-recursive edges (if OPTIMIZE_SELF_RECURSION
5609 : is true) and to skip values which clearly will not be part of intersection
5610 : with INTERIM. */
5611 :
5612 : static void
5613 222996 : push_agg_values_from_edge (struct cgraph_edge *cs,
5614 : ipa_node_params *dest_info,
5615 : vec<ipa_argagg_value> *res,
5616 : const ipa_argagg_value_list *interim,
5617 : bool optimize_self_recursion)
5618 : {
5619 222996 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5620 222996 : if (!args)
5621 : return;
5622 :
5623 445992 : int count = MIN (ipa_get_param_count (dest_info),
5624 : ipa_get_cs_argument_count (args));
5625 :
5626 222996 : unsigned interim_index = 0;
5627 935908 : for (int index = 0; index < count; index++)
5628 : {
5629 712912 : if (interim)
5630 : {
5631 295311 : while (interim_index < interim->m_elts.size ()
5632 269740 : && interim->m_elts[interim_index].value
5633 519767 : && interim->m_elts[interim_index].index < index)
5634 144948 : interim_index++;
5635 208330 : if (interim_index >= interim->m_elts.size ()
5636 150363 : || interim->m_elts[interim_index].index > index)
5637 57967 : continue;
5638 : }
5639 :
5640 654945 : ipcp_param_lattices *plats = ipa_get_parm_lattices (dest_info, index);
5641 654945 : if (!ipa_is_param_used (dest_info, index)
5642 654945 : || plats->aggs_bottom)
5643 25192 : continue;
5644 629798 : push_agg_values_for_index_from_edge (cs, index, res,
5645 : optimize_self_recursion ? interim
5646 : : NULL);
5647 : }
5648 : }
5649 :
5650 :
5651 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5652 : from all of them into INTERIM. Return how many there are. */
5653 :
5654 : static unsigned int
5655 166477 : find_aggregate_values_for_callers_subset_1 (vec<ipa_argagg_value> &interim,
5656 : struct cgraph_node *node,
5657 : const vec<cgraph_edge *> &callers)
5658 : {
5659 166477 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5660 166477 : if (dest_info->ipcp_orig_node)
5661 0 : dest_info = ipa_node_params_sum->get (dest_info->ipcp_orig_node);
5662 :
5663 : /* gather_edges_for_value puts a non-recursive call into the first element of
5664 : callers if it can. */
5665 166477 : push_agg_values_from_edge (callers[0], dest_info, &interim, NULL, true);
5666 :
5667 249345 : unsigned valid_entries = interim.length ();
5668 166477 : if (!valid_entries)
5669 : return 0;
5670 :
5671 85281 : unsigned caller_count = callers.length();
5672 140086 : for (unsigned i = 1; i < caller_count; i++)
5673 : {
5674 56477 : auto_vec<ipa_argagg_value, 32> last;
5675 56477 : ipa_argagg_value_list avs (&interim);
5676 56477 : push_agg_values_from_edge (callers[i], dest_info, &last, &avs, true);
5677 :
5678 56477 : valid_entries = intersect_argaggs_with (interim, last);
5679 56477 : if (!valid_entries)
5680 1672 : return 0;
5681 56477 : }
5682 :
5683 : return valid_entries;
5684 : }
5685 :
5686 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5687 : from all of them and return them in a garbage-collected vector. Return
5688 : nullptr if there are none. */
5689 :
5690 : static void
5691 151903 : find_aggregate_values_for_callers_subset (vec<ipa_argagg_value> &res,
5692 : struct cgraph_node *node,
5693 : const vec<cgraph_edge *> &callers)
5694 : {
5695 151903 : auto_vec<ipa_argagg_value, 32> interim;
5696 151903 : unsigned valid_entries
5697 151903 : = find_aggregate_values_for_callers_subset_1 (interim, node, callers);
5698 151903 : if (!valid_entries)
5699 : return;
5700 :
5701 864985 : for (const ipa_argagg_value &av : interim)
5702 629833 : if (av.value)
5703 596248 : res.safe_push(av);
5704 : return;
5705 151903 : }
5706 :
5707 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5708 : from all of them and return them in a garbage-collected vector. Return
5709 : nullptr if there are none. */
5710 :
5711 : static struct vec<ipa_argagg_value, va_gc> *
5712 14574 : find_aggregate_values_for_callers_subset_gc (struct cgraph_node *node,
5713 : const vec<cgraph_edge *> &callers)
5714 : {
5715 14574 : auto_vec<ipa_argagg_value, 32> interim;
5716 14574 : unsigned valid_entries
5717 14574 : = find_aggregate_values_for_callers_subset_1 (interim, node, callers);
5718 14574 : if (!valid_entries)
5719 : return nullptr;
5720 :
5721 5225 : vec<ipa_argagg_value, va_gc> *res = NULL;
5722 5225 : vec_safe_reserve_exact (res, valid_entries);
5723 36861 : for (const ipa_argagg_value &av : interim)
5724 21186 : if (av.value)
5725 19773 : res->quick_push(av);
5726 5225 : gcc_checking_assert (res->length () == valid_entries);
5727 : return res;
5728 14574 : }
5729 :
5730 : /* Determine whether CS also brings all scalar values that the NODE is
5731 : specialized for. */
5732 :
5733 : static bool
5734 74 : cgraph_edge_brings_all_scalars_for_node (struct cgraph_edge *cs,
5735 : struct cgraph_node *node)
5736 : {
5737 74 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5738 74 : int count = ipa_get_param_count (dest_info);
5739 74 : class ipa_node_params *caller_info;
5740 74 : class ipa_edge_args *args;
5741 74 : int i;
5742 :
5743 74 : caller_info = ipa_node_params_sum->get (cs->caller);
5744 74 : args = ipa_edge_args_sum->get (cs);
5745 162 : for (i = 0; i < count; i++)
5746 : {
5747 110 : struct ipa_jump_func *jump_func;
5748 110 : tree val, t;
5749 :
5750 110 : val = dest_info->known_csts[i];
5751 110 : if (!val)
5752 69 : continue;
5753 :
5754 82 : if (i >= ipa_get_cs_argument_count (args))
5755 : return false;
5756 41 : jump_func = ipa_get_ith_jump_func (args, i);
5757 41 : t = ipa_value_from_jfunc (caller_info, jump_func,
5758 : ipa_get_type (dest_info, i));
5759 41 : if (!t || !values_equal_for_ipcp_p (val, t))
5760 22 : return false;
5761 : }
5762 : return true;
5763 : }
5764 :
5765 : /* Determine whether CS also brings all aggregate values that NODE is
5766 : specialized for. */
5767 :
5768 : static bool
5769 52 : cgraph_edge_brings_all_agg_vals_for_node (struct cgraph_edge *cs,
5770 : struct cgraph_node *node)
5771 : {
5772 52 : ipcp_transformation *ts = ipcp_get_transformation_summary (node);
5773 52 : if (!ts || vec_safe_is_empty (ts->m_agg_values))
5774 : return true;
5775 :
5776 42 : const ipa_argagg_value_list existing (ts->m_agg_values);
5777 42 : auto_vec<ipa_argagg_value, 32> edge_values;
5778 42 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5779 42 : gcc_checking_assert (dest_info->ipcp_orig_node);
5780 42 : dest_info = ipa_node_params_sum->get (dest_info->ipcp_orig_node);
5781 42 : push_agg_values_from_edge (cs, dest_info, &edge_values, &existing, false);
5782 42 : const ipa_argagg_value_list avl (&edge_values);
5783 42 : return avl.superset_of_p (existing);
5784 42 : }
5785 :
5786 : /* Given an original NODE and a VAL for which we have already created a
5787 : specialized clone, look whether there are incoming edges that still lead
5788 : into the old node but now also bring the requested value and also conform to
5789 : all other criteria such that they can be redirected the special node.
5790 : This function can therefore redirect the final edge in a SCC. */
5791 :
5792 : template <typename valtype>
5793 : static void
5794 8852 : perhaps_add_new_callers (cgraph_node *node, ipcp_value<valtype> *val)
5795 : {
5796 : ipcp_value_source<valtype> *src;
5797 8852 : profile_count redirected_sum = profile_count::zero ();
5798 :
5799 123158 : for (src = val->sources; src; src = src->next)
5800 : {
5801 114306 : struct cgraph_edge *cs = src->cs;
5802 354543 : while (cs)
5803 : {
5804 240237 : if (cgraph_edge_brings_value_p (cs, src, node, val)
5805 74 : && cgraph_edge_brings_all_scalars_for_node (cs, val->spec_node)
5806 240289 : && cgraph_edge_brings_all_agg_vals_for_node (cs, val->spec_node))
5807 : {
5808 35 : if (dump_file)
5809 3 : fprintf (dump_file, " - adding an extra caller %s of %s\n",
5810 3 : cs->caller->dump_name (),
5811 3 : val->spec_node->dump_name ());
5812 :
5813 35 : cs->redirect_callee_duplicating_thunks (val->spec_node);
5814 35 : val->spec_node->expand_all_artificial_thunks ();
5815 35 : if (cs->count.ipa ().initialized_p ())
5816 0 : redirected_sum = redirected_sum + cs->count.ipa ();
5817 : }
5818 240237 : cs = get_next_cgraph_edge_clone (cs);
5819 : }
5820 : }
5821 :
5822 8852 : if (redirected_sum.nonzero_p ())
5823 0 : update_specialized_profile (val->spec_node, node, redirected_sum);
5824 8852 : }
5825 :
5826 : /* Return true if KNOWN_CONTEXTS contain at least one useful context. */
5827 :
5828 : static bool
5829 4357 : known_contexts_useful_p (vec<ipa_polymorphic_call_context> known_contexts)
5830 : {
5831 4357 : ipa_polymorphic_call_context *ctx;
5832 4357 : int i;
5833 :
5834 4357 : FOR_EACH_VEC_ELT (known_contexts, i, ctx)
5835 99 : if (!ctx->useless_p ())
5836 : return true;
5837 : return false;
5838 : }
5839 :
5840 : /* Return a copy of KNOWN_CSTS if it is not empty, otherwise return vNULL. */
5841 :
5842 : static vec<ipa_polymorphic_call_context>
5843 4357 : copy_useful_known_contexts (const vec<ipa_polymorphic_call_context> &known_contexts)
5844 : {
5845 4357 : if (known_contexts_useful_p (known_contexts))
5846 99 : return known_contexts.copy ();
5847 : else
5848 4258 : return vNULL;
5849 : }
5850 :
5851 : /* Return true if the VALUE is represented in KNOWN_CSTS at INDEX if OFFSET is
5852 : minus one or in AGGVALS for INDEX and OFFSET otherwise. */
5853 :
5854 : DEBUG_FUNCTION bool
5855 4306 : ipcp_val_replacement_ok_p (vec<tree> &known_csts,
5856 : vec<ipa_polymorphic_call_context> &,
5857 : vec<ipa_argagg_value, va_gc> *aggvals,
5858 : int index, HOST_WIDE_INT offset, tree value)
5859 : {
5860 4306 : tree v;
5861 4306 : if (offset == -1)
5862 3134 : v = known_csts[index];
5863 : else
5864 : {
5865 1172 : const ipa_argagg_value_list avl (aggvals);
5866 1172 : v = avl.get_value (index, offset / BITS_PER_UNIT);
5867 : }
5868 :
5869 4306 : return v && values_equal_for_ipcp_p (v, value);
5870 : }
5871 :
5872 : /* Dump to F all the values in AVALS for which we are re-evaluating the effects
5873 : on the function represented b INFO. */
5874 :
5875 : DEBUG_FUNCTION void
5876 68 : dump_reestimation_message (FILE *f, ipa_node_params *info,
5877 : const ipa_auto_call_arg_values &avals)
5878 : {
5879 68 : fprintf (f, " Re-estimating effects with\n"
5880 : " Scalar constants:");
5881 68 : int param_count = ipa_get_param_count (info);
5882 168 : for (int i = 0; i < param_count; i++)
5883 100 : if (avals.m_known_vals[i])
5884 : {
5885 44 : fprintf (f, " %i:", i);
5886 44 : print_ipcp_constant_value (f, avals.m_known_vals[i]);
5887 : }
5888 68 : fprintf (f, "\n");
5889 68 : if (!avals.m_known_contexts.is_empty ())
5890 : {
5891 0 : fprintf (f, " Pol. contexts:");
5892 0 : for (int i = 0; i < param_count; i++)
5893 0 : if (!avals.m_known_contexts[i].useless_p ())
5894 : {
5895 0 : fprintf (f, " %i:", i);
5896 0 : avals.m_known_contexts[i].dump (f);
5897 : }
5898 0 : fprintf (f, "\n");
5899 : }
5900 68 : if (!avals.m_known_aggs.is_empty ())
5901 : {
5902 24 : fprintf (f, " Aggregate replacements:");
5903 24 : ipa_argagg_value_list avs (&avals);
5904 24 : avs.dump (f);
5905 : }
5906 68 : }
5907 :
5908 : /* Return true if the VALUE is represented in KNOWN_CONTEXTS at INDEX and that
5909 : if OFFSET is is equal to minus one (because source of a polymorphic context
5910 : cannot be an aggregate value). */
5911 :
5912 : DEBUG_FUNCTION bool
5913 51 : ipcp_val_replacement_ok_p (vec<tree> &,
5914 : vec<ipa_polymorphic_call_context> &known_contexts,
5915 : vec<ipa_argagg_value, va_gc> *,
5916 : int index, HOST_WIDE_INT offset,
5917 : ipa_polymorphic_call_context value)
5918 : {
5919 51 : if (offset != -1
5920 51 : || known_contexts.length () <= (unsigned) index
5921 102 : || known_contexts[index].useless_p ())
5922 : return false;
5923 :
5924 51 : if (known_contexts[index].equal_to (value))
5925 : return true;
5926 :
5927 : /* In some corner cases, the final gathering of contexts can figure out that
5928 : the available context is actually more precise than what we wanted to
5929 : clone for. Allow it. */
5930 0 : value.combine_with (known_contexts[index]);
5931 0 : return known_contexts[index].equal_to (value);
5932 : }
5933 :
5934 : /* Decide whether to create a special version of NODE for value VAL of
5935 : parameter at the given INDEX. If OFFSET is -1, the value is for the
5936 : parameter itself, otherwise it is stored at the given OFFSET of the
5937 : parameter. AVALS describes the other already known values. SELF_GEN_CLONES
5938 : is a vector which contains clones created for self-recursive calls with an
5939 : arithmetic pass-through jump function. CUR_SWEEP is the number of the
5940 : current sweep of the call-graph during the decision stage. */
5941 :
5942 : template <typename valtype>
5943 : static bool
5944 223270 : decide_about_value (struct cgraph_node *node, int index, HOST_WIDE_INT offset,
5945 : ipcp_value<valtype> *val,
5946 : vec<cgraph_node *> *self_gen_clones, int cur_sweep)
5947 : {
5948 : int caller_count;
5949 223270 : sreal freq_sum;
5950 : profile_count count_sum, rec_count_sum;
5951 : bool called_without_ipa_profile;
5952 :
5953 223270 : if (val->spec_node)
5954 : {
5955 8852 : perhaps_add_new_callers (node, val);
5956 8852 : return false;
5957 : }
5958 214418 : else if (val->local_size_cost + overall_size > get_max_overall_size (node))
5959 : {
5960 450 : if (dump_file && (dump_flags & TDF_DETAILS))
5961 0 : fprintf (dump_file, " - ignoring candidate value because "
5962 : "maximum unit size would be reached with %li.\n",
5963 : val->local_size_cost + overall_size);
5964 450 : return false;
5965 : }
5966 213968 : else if (!get_info_about_necessary_edges (val, node, &freq_sum, &caller_count,
5967 : &rec_count_sum, &count_sum,
5968 : &called_without_ipa_profile))
5969 : {
5970 62065 : if (dump_file && (dump_flags & TDF_DETAILS))
5971 : {
5972 121 : fprintf (dump_file, " - skipping candidate value ");
5973 121 : print_ipcp_constant_value (dump_file, val->value);
5974 121 : fprintf (dump_file, " for ");
5975 121 : ipa_dump_param (dump_file, ipa_node_params_sum->get (node), index);
5976 121 : if (offset != -1)
5977 105 : fprintf (dump_file, ", offset: " HOST_WIDE_INT_PRINT_DEC, offset);
5978 121 : fprintf (dump_file, ": no relevant callers\n");
5979 : }
5980 62065 : return false;
5981 : }
5982 :
5983 151903 : if (!dbg_cnt (ipa_cp_values))
5984 : return false;
5985 :
5986 151903 : if (val->self_recursion_generated_p ())
5987 : {
5988 : /* The edge counts in this case might not have been adjusted yet.
5989 : Nevertleless, even if they were it would be only a guesswork which we
5990 : can do now. The recursive part of the counts can be derived from the
5991 : count of the original node anyway. */
5992 293 : if (node->count.ipa ().nonzero_p ())
5993 : {
5994 14 : unsigned dem = self_gen_clones->length () + 1;
5995 14 : rec_count_sum = node->count.ipa () / dem;
5996 : }
5997 : else
5998 265 : rec_count_sum = profile_count::zero ();
5999 : }
6000 :
6001 : /* get_info_about_necessary_edges only sums up ipa counts. */
6002 151903 : count_sum += rec_count_sum;
6003 :
6004 151903 : if (dump_file && (dump_flags & TDF_DETAILS))
6005 : {
6006 135 : fprintf (dump_file, " - considering value ");
6007 135 : print_ipcp_constant_value (dump_file, val->value);
6008 135 : fprintf (dump_file, " for ");
6009 135 : ipa_dump_param (dump_file, ipa_node_params_sum->get (node), index);
6010 135 : if (offset != -1)
6011 62 : fprintf (dump_file, ", offset: " HOST_WIDE_INT_PRINT_DEC, offset);
6012 135 : fprintf (dump_file, " (caller_count: %i)\n", caller_count);
6013 : }
6014 :
6015 151903 : auto_vec<cgraph_edge *> callers
6016 : = gather_edges_for_value (val, node, caller_count);
6017 151903 : ipa_node_params *info = ipa_node_params_sum->get (node);
6018 151903 : ipa_auto_call_arg_values avals;
6019 151903 : avals.m_known_vals.safe_grow_cleared (ipa_get_param_count (info), true);
6020 151903 : find_scalar_values_for_callers_subset (avals.m_known_vals, info, callers);
6021 151903 : find_contexts_for_caller_subset (avals.m_known_contexts, info, callers);
6022 151903 : find_aggregate_values_for_callers_subset (avals.m_known_aggs, node, callers);
6023 :
6024 :
6025 151903 : if (good_cloning_opportunity_p (node, val->prop_time_benefit,
6026 : freq_sum, count_sum, val->prop_size_cost,
6027 : called_without_ipa_profile, cur_sweep))
6028 : ;
6029 : else
6030 : {
6031 : /* Extern inline functions are only meaningful to clione to propagate
6032 : values to their callees. */
6033 150000 : if (DECL_EXTERNAL (node->decl) && DECL_DECLARED_INLINE_P (node->decl))
6034 : {
6035 345 : if (dump_file && (dump_flags & TDF_DETAILS))
6036 0 : fprintf (dump_file, " Skipping extern inline.\n");
6037 147546 : return false;
6038 : }
6039 149655 : if (dump_file && (dump_flags & TDF_DETAILS))
6040 68 : dump_reestimation_message (dump_file, info, avals);
6041 :
6042 149655 : ipa_call_estimates estimates;
6043 149655 : estimate_ipcp_clone_size_and_time (node, &avals, &estimates);
6044 149655 : int removable_params_cost = 0;
6045 966404 : for (tree t : avals.m_known_vals)
6046 517439 : if (t)
6047 207189 : removable_params_cost += estimate_move_cost (TREE_TYPE (t), true);
6048 :
6049 149655 : int size = estimates.size - caller_count * removable_params_cost;
6050 :
6051 149655 : if (size <= 0)
6052 : {
6053 1802 : if (dump_file)
6054 0 : fprintf (dump_file, " Code not going to grow.\n");
6055 : }
6056 : else
6057 : {
6058 : sreal time_benefit
6059 147853 : = ((estimates.nonspecialized_time - estimates.time)
6060 295706 : + hint_time_bonus (node, estimates)
6061 147853 : + (devirtualization_time_bonus (node, &avals)
6062 147853 : + removable_params_cost));
6063 :
6064 147853 : if (!good_cloning_opportunity_p (node, time_benefit, freq_sum,
6065 : count_sum, size,
6066 : called_without_ipa_profile,
6067 : cur_sweep))
6068 147201 : return false;
6069 : }
6070 : }
6071 :
6072 4357 : if (dump_file)
6073 139 : fprintf (dump_file, " Creating a specialized node of %s.\n",
6074 : node->dump_name ());
6075 :
6076 4357 : vec<tree> known_csts = avals.m_known_vals.copy ();
6077 : vec<ipa_polymorphic_call_context> known_contexts
6078 4357 : = copy_useful_known_contexts (avals.m_known_contexts);
6079 :
6080 4357 : vec<ipa_argagg_value, va_gc> *aggvals = NULL;
6081 4357 : vec_safe_reserve_exact (aggvals, avals.m_known_aggs.length ());
6082 23556 : for (const ipa_argagg_value &av : avals.m_known_aggs)
6083 10485 : aggvals->quick_push (av);
6084 4357 : gcc_checking_assert (ipcp_val_replacement_ok_p (known_csts, known_contexts,
6085 : aggvals, index,
6086 : offset, val->value));
6087 4357 : val->spec_node = create_specialized_node (node, known_csts, known_contexts,
6088 : aggvals, callers);
6089 :
6090 4357 : if (val->self_recursion_generated_p ())
6091 142 : self_gen_clones->safe_push (val->spec_node);
6092 : else
6093 4215 : update_profiling_info (node, val->spec_node);
6094 :
6095 4357 : overall_size += val->local_size_cost;
6096 4357 : if (dump_file && (dump_flags & TDF_DETAILS))
6097 68 : fprintf (dump_file, " overall size reached %li\n",
6098 : overall_size);
6099 :
6100 : /* TODO: If for some lattice there is only one other known value
6101 : left, make a special node for it too. */
6102 :
6103 : return true;
6104 151903 : }
6105 :
6106 : /* Like irange::contains_p(), but convert VAL to the range of R if
6107 : necessary. */
6108 :
6109 : static inline bool
6110 47855 : ipa_range_contains_p (const vrange &r, tree val)
6111 : {
6112 47855 : if (r.undefined_p ())
6113 : return false;
6114 :
6115 47855 : tree type = r.type ();
6116 47855 : if (!wi::fits_to_tree_p (wi::to_wide (val), type))
6117 : return false;
6118 :
6119 47855 : val = fold_convert (type, val);
6120 47855 : return r.contains_p (val);
6121 : }
6122 :
6123 : /* Structure holding opportunitties so that they can be pre-sorted. */
6124 :
6125 223270 : struct cloning_opportunity_ranking
6126 : {
6127 : /* A very rough evaluation of likely benefit. */
6128 : sreal eval;
6129 : /* In the case of aggregate constants, a non-negative offset within their
6130 : aggregates. -1 for scalar constants, -2 for polymorphic contexts. */
6131 : HOST_WIDE_INT offset;
6132 : /* The value being considered for evaluation for cloning. */
6133 : ipcp_value_base *val;
6134 : /* Index of the formal parameter the value is coming in. */
6135 : int index;
6136 : };
6137 :
6138 : /* Helper function to qsort a vector of cloning opportunities. */
6139 :
6140 : static int
6141 2152861 : compare_cloning_opportunities (const void *a, const void *b)
6142 : {
6143 2152861 : const cloning_opportunity_ranking *o1
6144 : = (const cloning_opportunity_ranking *) a;
6145 2152861 : const cloning_opportunity_ranking *o2
6146 : = (const cloning_opportunity_ranking *) b;
6147 2152861 : if (o1->eval < o2->eval)
6148 : return 1;
6149 1680378 : if (o1->eval > o2->eval)
6150 551676 : return -1;
6151 : return 0;
6152 : }
6153 :
6154 : /* Use the estimations in VAL to determine how good a candidate it represents
6155 : for the purposes of ordering real evaluation of opportunities (which
6156 : includes information about incoming edges, among other things). */
6157 :
6158 : static sreal
6159 223270 : cloning_opportunity_ranking_evaluation (const ipcp_value_base *val)
6160 : {
6161 223270 : sreal e1 = (val->local_time_benefit * 1000) / MAX (val->local_size_cost, 1);
6162 223270 : sreal e2 = (val->prop_time_benefit * 1000) / MAX (val->prop_size_cost, 1);
6163 223270 : if (e2 > e1)
6164 15434 : return e2;
6165 : else
6166 207836 : return e1;
6167 : }
6168 :
6169 : /* Decide whether and what specialized clones of NODE should be created.
6170 : CUR_SWEEP is the number of the current sweep of the call-graph during the
6171 : decision stage. */
6172 :
6173 : static bool
6174 3291238 : decide_whether_version_node (struct cgraph_node *node, int cur_sweep)
6175 : {
6176 3291238 : ipa_node_params *info = ipa_node_params_sum->get (node);
6177 3291238 : int count = ipa_get_param_count (info);
6178 3291238 : bool ret = false;
6179 :
6180 3291238 : if (info->node_dead || count == 0)
6181 : return false;
6182 :
6183 2666217 : bool clone_for_all_contexts = node->local;
6184 2666217 : if (dump_file && (dump_flags & TDF_DETAILS))
6185 : {
6186 345 : fprintf (dump_file, "\nEvaluating opportunities for %s.",
6187 : node->dump_name ());
6188 345 : if (clone_for_all_contexts)
6189 104 : fprintf (dump_file, " Will try to create a special all-context "
6190 : "clone.\n");
6191 345 : fprintf (dump_file, "\n");
6192 : }
6193 :
6194 2666217 : auto_vec <cloning_opportunity_ranking, 32> opp_ranking;
6195 8895220 : for (int i = 0; i < count;i++)
6196 : {
6197 6229003 : if (!ipa_is_param_used (info, i))
6198 : {
6199 700359 : if (dump_file && (dump_flags & TDF_DETAILS))
6200 20 : fprintf (dump_file, " - ignoring unused parameter %i.\n", i);
6201 700359 : continue;
6202 : }
6203 :
6204 5528644 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6205 5528644 : ipcp_lattice<tree> *lat = &plats->itself;
6206 5528644 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
6207 :
6208 5528644 : if (!lat->bottom
6209 5528644 : && (!clone_for_all_contexts || !lat->is_single_const ()))
6210 : {
6211 543703 : ipcp_value<tree> *val;
6212 660126 : for (val = lat->values; val; val = val->next)
6213 : {
6214 : /* If some values generated for self-recursive calls with
6215 : arithmetic jump functions fall outside of the known
6216 : range for the parameter, we can skip them. */
6217 116485 : if (TREE_CODE (val->value) == INTEGER_CST
6218 70620 : && !plats->m_value_range.bottom_p ()
6219 164278 : && !ipa_range_contains_p (plats->m_value_range.m_vr,
6220 : val->value))
6221 : {
6222 : /* This can happen also if a constant present in the source
6223 : code falls outside of the range of parameter's type, so we
6224 : cannot assert. */
6225 62 : if (dump_file && (dump_flags & TDF_DETAILS))
6226 : {
6227 0 : fprintf (dump_file, " - skipping%s value ",
6228 0 : val->self_recursion_generated_p ()
6229 : ? " self_recursion_generated" : "");
6230 0 : print_ipcp_constant_value (dump_file, val->value);
6231 0 : fprintf (dump_file, " because it is outside known "
6232 : "value range.\n");
6233 : }
6234 62 : continue;
6235 : }
6236 116361 : cloning_opportunity_ranking opp;
6237 116361 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6238 116361 : opp.offset = -1;
6239 116361 : opp.val = val;
6240 116361 : opp.index = i;
6241 116361 : opp_ranking.safe_push (opp);
6242 : }
6243 : }
6244 :
6245 5528644 : if (!plats->aggs_bottom)
6246 : {
6247 572834 : struct ipcp_agg_lattice *aglat;
6248 572834 : ipcp_value<tree> *val;
6249 715644 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
6250 141728 : if (!aglat->bottom && aglat->values
6251 : /* If the following is false, the one value will be considered
6252 : for cloning for all contexts. */
6253 263323 : && (!clone_for_all_contexts
6254 77042 : || plats->aggs_contain_variable
6255 195474 : || !aglat->is_single_const ()))
6256 181883 : for (val = aglat->values; val; val = val->next)
6257 : {
6258 103114 : cloning_opportunity_ranking opp;
6259 103114 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6260 103114 : opp.offset = aglat->offset;
6261 103114 : opp.val = val;
6262 103114 : opp.index = i;
6263 103114 : opp_ranking.safe_push (opp);
6264 : }
6265 : }
6266 :
6267 5528644 : if (!ctxlat->bottom
6268 6804749 : && (!clone_for_all_contexts || !ctxlat->is_single_const ()))
6269 : {
6270 557344 : ipcp_value<ipa_polymorphic_call_context> *val;
6271 561139 : for (val = ctxlat->values; val; val = val->next)
6272 7590 : if (!val->value.useless_p ())
6273 : {
6274 3795 : cloning_opportunity_ranking opp;
6275 3795 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6276 3795 : opp.offset = -2;
6277 3795 : opp.val = val;
6278 3795 : opp.index = i;
6279 3795 : opp_ranking.safe_push (opp);
6280 : }
6281 : }
6282 : }
6283 :
6284 2666217 : if (!opp_ranking.is_empty ())
6285 : {
6286 51636 : opp_ranking.qsort (compare_cloning_opportunities);
6287 51636 : auto_vec <cgraph_node *, 9> self_gen_clones;
6288 378178 : for (const cloning_opportunity_ranking &opp : opp_ranking)
6289 223270 : if (opp.offset == -2)
6290 : {
6291 3795 : ipcp_value<ipa_polymorphic_call_context> *val
6292 : = static_cast <ipcp_value<ipa_polymorphic_call_context> *>
6293 : (opp.val);
6294 3795 : ret |= decide_about_value (node, opp.index, -1, val,
6295 : &self_gen_clones, cur_sweep);
6296 : }
6297 : else
6298 : {
6299 219475 : ipcp_value<tree> *val = static_cast<ipcp_value<tree> *> (opp.val);
6300 219475 : ret |= decide_about_value (node, opp.index, opp.offset, val,
6301 : &self_gen_clones, cur_sweep);
6302 : }
6303 :
6304 103272 : if (!self_gen_clones.is_empty ())
6305 : {
6306 33 : self_gen_clones.safe_push (node);
6307 33 : update_counts_for_self_gen_clones (node, self_gen_clones);
6308 : }
6309 51636 : }
6310 :
6311 2666217 : if (!clone_for_all_contexts)
6312 : return ret;
6313 :
6314 233935 : struct caller_statistics stats;
6315 233935 : init_caller_stats (&stats);
6316 233935 : node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
6317 : false);
6318 233935 : if (!stats.n_calls)
6319 : {
6320 15800 : if (dump_file && (dump_flags & TDF_DETAILS))
6321 41 : fprintf (dump_file, " Not cloning for all contexts because "
6322 : "there are no callers of the original node (any more).\n");
6323 15800 : return ret;
6324 : }
6325 :
6326 218135 : ipa_auto_call_arg_values avals;
6327 218135 : int removable_params_cost;
6328 218135 : bool ctx_independent_const
6329 218135 : = gather_context_independent_values (info, &avals, &removable_params_cost);
6330 218135 : sreal devirt_bonus = devirtualization_time_bonus (node, &avals);
6331 406994 : if (ctx_independent_const || devirt_bonus > 0
6332 421629 : || (removable_params_cost && clone_for_param_removal_p (node)))
6333 : {
6334 14641 : if (!dbg_cnt (ipa_cp_values))
6335 67 : return ret;
6336 :
6337 14641 : auto_vec<cgraph_edge *> callers = node->collect_callers ();
6338 55614 : for (int i = callers.length () - 1; i >= 0; i--)
6339 : {
6340 26332 : cgraph_edge *cs = callers[i];
6341 26332 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
6342 :
6343 26332 : if (caller_info && caller_info->node_dead)
6344 2586 : callers.unordered_remove (i);
6345 : }
6346 :
6347 14641 : if (!adjust_callers_for_value_intersection (callers, node))
6348 : /* If node is not called by anyone, or all its caller edges are
6349 : self-recursive, the node is not really in use, no need to do
6350 : cloning. */
6351 67 : return ret;
6352 :
6353 14574 : if (dump_file)
6354 91 : fprintf (dump_file, " Creating a specialized node of %s "
6355 : "for all known contexts.\n", node->dump_name ());
6356 :
6357 14574 : vec<tree> known_csts = vNULL;
6358 14574 : known_csts.safe_grow_cleared (count, true);
6359 14574 : find_scalar_values_for_callers_subset (known_csts, info, callers);
6360 14574 : vec<ipa_polymorphic_call_context> known_contexts = vNULL;
6361 14574 : find_contexts_for_caller_subset (known_contexts, info, callers);
6362 14574 : vec<ipa_argagg_value, va_gc> *aggvals
6363 14574 : = find_aggregate_values_for_callers_subset_gc (node, callers);
6364 :
6365 14574 : struct cgraph_node *clone = create_specialized_node (node, known_csts,
6366 : known_contexts,
6367 : aggvals, callers);
6368 14574 : ipa_node_params_sum->get (clone)->is_all_contexts_clone = true;
6369 14574 : ret = true;
6370 14641 : }
6371 :
6372 : return ret;
6373 2884352 : }
6374 :
6375 : /* Transitively mark all callees of NODE within the same SCC as not dead. */
6376 :
6377 : static void
6378 2128 : spread_undeadness (struct cgraph_node *node)
6379 : {
6380 2128 : struct cgraph_edge *cs;
6381 :
6382 11749 : for (cs = node->callees; cs; cs = cs->next_callee)
6383 9621 : if (ipa_edge_within_scc (cs))
6384 : {
6385 823 : struct cgraph_node *callee;
6386 823 : class ipa_node_params *info;
6387 :
6388 823 : callee = cs->callee->function_symbol (NULL);
6389 823 : info = ipa_node_params_sum->get (callee);
6390 :
6391 823 : if (info && info->node_dead)
6392 : {
6393 68 : info->node_dead = 0;
6394 68 : spread_undeadness (callee);
6395 : }
6396 : }
6397 2128 : }
6398 :
6399 : /* Return true if NODE has a caller from outside of its SCC that is not
6400 : dead. Worker callback for cgraph_for_node_and_aliases. */
6401 :
6402 : static bool
6403 15932 : has_undead_caller_from_outside_scc_p (struct cgraph_node *node,
6404 : void *data ATTRIBUTE_UNUSED)
6405 : {
6406 15932 : struct cgraph_edge *cs;
6407 :
6408 81157 : for (cs = node->callers; cs; cs = cs->next_caller)
6409 65674 : if (cs->caller->thunk
6410 65674 : && cs->caller->call_for_symbol_thunks_and_aliases
6411 0 : (has_undead_caller_from_outside_scc_p, NULL, true))
6412 : return true;
6413 65674 : else if (!ipa_edge_within_scc (cs))
6414 : {
6415 65434 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
6416 65434 : if (!caller_info /* Unoptimized caller are like dead ones. */
6417 65432 : || !caller_info->node_dead)
6418 : return true;
6419 : }
6420 : return false;
6421 : }
6422 :
6423 :
6424 : /* Identify nodes within the same SCC as NODE which are no longer needed
6425 : because of new clones and will be removed as unreachable. */
6426 :
6427 : static void
6428 16973 : identify_dead_nodes (struct cgraph_node *node)
6429 : {
6430 16973 : struct cgraph_node *v;
6431 34214 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6432 17241 : if (v->local)
6433 : {
6434 15692 : ipa_node_params *info = ipa_node_params_sum->get (v);
6435 15692 : if (info
6436 31384 : && !v->call_for_symbol_thunks_and_aliases
6437 15692 : (has_undead_caller_from_outside_scc_p, NULL, true))
6438 15243 : info->node_dead = 1;
6439 : }
6440 :
6441 34214 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6442 : {
6443 17241 : ipa_node_params *info = ipa_node_params_sum->get (v);
6444 17241 : if (info && !info->node_dead)
6445 2060 : spread_undeadness (v);
6446 : }
6447 :
6448 16973 : if (dump_file && (dump_flags & TDF_DETAILS))
6449 : {
6450 107 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6451 55 : if (ipa_node_params_sum->get (v)
6452 55 : && ipa_node_params_sum->get (v)->node_dead)
6453 32 : fprintf (dump_file, " Marking node as dead: %s.\n",
6454 : v->dump_name ());
6455 : }
6456 16973 : }
6457 :
6458 : /* Removes all useless callback edges from the callgraph. Useless callback
6459 : edges might mess up the callgraph, because they might be impossible to
6460 : redirect and so on, leading to crashes. Their usefulness is evaluated
6461 : through callback_edge_useful_p. */
6462 :
6463 : static void
6464 129922 : purge_useless_callback_edges ()
6465 : {
6466 129922 : if (dump_file)
6467 161 : fprintf (dump_file, "\nPurging useless callback edges:\n");
6468 :
6469 129922 : cgraph_edge *e;
6470 129922 : cgraph_node *node;
6471 1451539 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6472 : {
6473 6886184 : for (e = node->callees; e; e = e->next_callee)
6474 : {
6475 5564567 : if (e->has_callback)
6476 : {
6477 13641 : if (dump_file)
6478 4 : fprintf (dump_file, "\tExamining callbacks of edge %s -> %s:\n",
6479 4 : e->caller->dump_name (), e->callee->dump_name ());
6480 13641 : if (!lookup_attribute (CALLBACK_ATTR_IDENT,
6481 13641 : DECL_ATTRIBUTES (e->callee->decl))
6482 13641 : && !callback_is_special_cased (e->callee->decl, e->call_stmt))
6483 : {
6484 1 : if (dump_file)
6485 0 : fprintf (
6486 : dump_file,
6487 : "\t\tPurging callbacks, because the callback-dispatching"
6488 : "function no longer has any callback attributes.\n");
6489 1 : e->purge_callback_edges ();
6490 1 : continue;
6491 : }
6492 13640 : cgraph_edge *cbe, *next;
6493 27280 : for (cbe = e->first_callback_edge (); cbe; cbe = next)
6494 : {
6495 13640 : next = cbe->next_callback_edge ();
6496 13640 : if (!callback_edge_useful_p (cbe))
6497 : {
6498 13280 : if (dump_file)
6499 2 : fprintf (dump_file,
6500 : "\t\tCallback edge %s -> %s not deemed "
6501 : "useful, removing.\n",
6502 2 : cbe->caller->dump_name (),
6503 2 : cbe->callee->dump_name ());
6504 13280 : cgraph_edge::remove (cbe);
6505 : }
6506 : else
6507 : {
6508 360 : if (dump_file)
6509 2 : fprintf (dump_file,
6510 : "\t\tKept callback edge %s -> %s "
6511 : "because it looks useful.\n",
6512 2 : cbe->caller->dump_name (),
6513 2 : cbe->callee->dump_name ());
6514 : }
6515 : }
6516 : }
6517 : }
6518 : }
6519 :
6520 129922 : if (dump_file)
6521 161 : fprintf (dump_file, "\n");
6522 129922 : }
6523 :
6524 : /* The decision stage. Iterate over the topological order of call graph nodes
6525 : TOPO and make specialized clones if deemed beneficial. */
6526 :
6527 : static void
6528 129922 : ipcp_decision_stage (class ipa_topo_info *topo)
6529 : {
6530 129922 : int i;
6531 :
6532 129922 : if (dump_file)
6533 161 : fprintf (dump_file, "\nIPA decision stage (%i sweeps):\n",
6534 : max_number_sweeps);
6535 :
6536 498993 : for (int cur_sweep = 1; cur_sweep <= max_number_sweeps; cur_sweep++)
6537 : {
6538 369071 : if (dump_file && (dump_flags & TDF_DETAILS))
6539 144 : fprintf (dump_file, "\nIPA decision sweep number %i (out of %i):\n",
6540 : cur_sweep, max_number_sweeps);
6541 :
6542 4473321 : for (i = topo->nnodes - 1; i >= 0; i--)
6543 : {
6544 4104250 : struct cgraph_node *node = topo->order[i];
6545 4104250 : bool change = false, iterate = true;
6546 :
6547 8225477 : while (iterate)
6548 : {
6549 : struct cgraph_node *v;
6550 : iterate = false;
6551 4136227 : for (v = node;
6552 8257454 : v;
6553 4136227 : v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6554 4136227 : if (v->has_gimple_body_p ()
6555 3900787 : && ipcp_versionable_function_p (v)
6556 4136227 : && (cur_sweep
6557 3291238 : <= opt_for_fn (node->decl, param_ipa_cp_sweeps)))
6558 3291238 : iterate |= decide_whether_version_node (v, cur_sweep);
6559 :
6560 4121227 : change |= iterate;
6561 : }
6562 4104250 : if (change)
6563 16973 : identify_dead_nodes (node);
6564 : }
6565 : }
6566 :
6567 : /* Currently, the primary use of callback edges is constant propagation.
6568 : Constant propagation is now over, so we have to remove unused callback
6569 : edges. */
6570 129922 : purge_useless_callback_edges ();
6571 129922 : }
6572 :
6573 : /* Look up all VR and bits information that we have discovered and copy it
6574 : over to the transformation summary. */
6575 :
6576 : static void
6577 129922 : ipcp_store_vr_results (void)
6578 : {
6579 129922 : cgraph_node *node;
6580 :
6581 1451539 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6582 : {
6583 1321617 : ipa_node_params *info = ipa_node_params_sum->get (node);
6584 1321617 : bool dumped_sth = false;
6585 1321617 : bool found_useful_result = false;
6586 1321617 : bool do_vr = true;
6587 1321617 : bool do_bits = true;
6588 :
6589 : /* If the function is not local, the gathered information is only useful
6590 : for clones. */
6591 1321617 : if (!node->local)
6592 1156331 : continue;
6593 :
6594 165286 : if (!info || !opt_for_fn (node->decl, flag_ipa_vrp))
6595 : {
6596 4813 : if (dump_file)
6597 6 : fprintf (dump_file, "Not considering %s for VR discovery "
6598 : "and propagate; -fipa-ipa-vrp: disabled.\n",
6599 : node->dump_name ());
6600 : do_vr = false;
6601 : }
6602 165286 : if (!info || !opt_for_fn (node->decl, flag_ipa_bit_cp))
6603 : {
6604 4784 : if (dump_file)
6605 2 : fprintf (dump_file, "Not considering %s for ipa bitwise "
6606 : "propagation ; -fipa-bit-cp: disabled.\n",
6607 : node->dump_name ());
6608 4784 : do_bits = false;
6609 : }
6610 4784 : if (!do_bits && !do_vr)
6611 4778 : continue;
6612 :
6613 160508 : if (info->ipcp_orig_node)
6614 18746 : info = ipa_node_params_sum->get (info->ipcp_orig_node);
6615 160508 : if (info->lattices.is_empty ())
6616 : /* Newly expanded artificial thunks do not have lattices. */
6617 51678 : continue;
6618 :
6619 108830 : unsigned count = ipa_get_param_count (info);
6620 221640 : for (unsigned i = 0; i < count; i++)
6621 : {
6622 173275 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6623 173275 : if (do_vr
6624 173249 : && !plats->m_value_range.bottom_p ()
6625 231545 : && !plats->m_value_range.top_p ())
6626 : {
6627 : found_useful_result = true;
6628 : break;
6629 : }
6630 115006 : if (do_bits && plats->bits_lattice.constant_p ())
6631 : {
6632 : found_useful_result = true;
6633 : break;
6634 : }
6635 : }
6636 108830 : if (!found_useful_result)
6637 48365 : continue;
6638 :
6639 60465 : ipcp_transformation_initialize ();
6640 60465 : ipcp_transformation *ts = ipcp_transformation_sum->get_create (node);
6641 60465 : vec_safe_reserve_exact (ts->m_vr, count);
6642 :
6643 219711 : for (unsigned i = 0; i < count; i++)
6644 : {
6645 159246 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6646 159246 : ipcp_bits_lattice *bits = NULL;
6647 :
6648 159246 : if (do_bits
6649 159242 : && plats->bits_lattice.constant_p ()
6650 251018 : && dbg_cnt (ipa_cp_bits))
6651 91772 : bits = &plats->bits_lattice;
6652 :
6653 159246 : if (do_vr
6654 159222 : && !plats->m_value_range.bottom_p ()
6655 110449 : && !plats->m_value_range.top_p ()
6656 269695 : && dbg_cnt (ipa_cp_vr))
6657 : {
6658 110449 : if (bits)
6659 : {
6660 86541 : value_range tmp = plats->m_value_range.m_vr;
6661 86541 : tree type = ipa_get_type (info, i);
6662 173082 : irange_bitmask bm (wide_int::from (bits->get_value (),
6663 86541 : TYPE_PRECISION (type),
6664 86541 : TYPE_SIGN (type)),
6665 173082 : wide_int::from (bits->get_mask (),
6666 86541 : TYPE_PRECISION (type),
6667 173082 : TYPE_SIGN (type)));
6668 86541 : tmp.update_bitmask (bm);
6669 : // Reflecting the bitmask on the ranges can sometime
6670 : // produce an UNDEFINED value if the the bitmask update
6671 : // was previously deferred. See PR 120048.
6672 86541 : if (tmp.undefined_p ())
6673 0 : tmp.set_varying (type);
6674 86541 : ipa_vr vr (tmp);
6675 86541 : ts->m_vr->quick_push (vr);
6676 86541 : }
6677 : else
6678 : {
6679 23908 : ipa_vr vr (plats->m_value_range.m_vr);
6680 23908 : ts->m_vr->quick_push (vr);
6681 : }
6682 : }
6683 48797 : else if (bits)
6684 : {
6685 5231 : tree type = ipa_get_type (info, i);
6686 5231 : value_range tmp;
6687 5231 : tmp.set_varying (type);
6688 10462 : irange_bitmask bm (wide_int::from (bits->get_value (),
6689 5231 : TYPE_PRECISION (type),
6690 5231 : TYPE_SIGN (type)),
6691 10462 : wide_int::from (bits->get_mask (),
6692 5231 : TYPE_PRECISION (type),
6693 10462 : TYPE_SIGN (type)));
6694 5231 : tmp.update_bitmask (bm);
6695 : // Reflecting the bitmask on the ranges can sometime
6696 : // produce an UNDEFINED value if the the bitmask update
6697 : // was previously deferred. See PR 120048.
6698 5231 : if (tmp.undefined_p ())
6699 0 : tmp.set_varying (type);
6700 5231 : ipa_vr vr (tmp);
6701 5231 : ts->m_vr->quick_push (vr);
6702 5231 : }
6703 : else
6704 : {
6705 43566 : ipa_vr vr;
6706 43566 : ts->m_vr->quick_push (vr);
6707 : }
6708 :
6709 159246 : if (!dump_file || !bits)
6710 158833 : continue;
6711 :
6712 413 : if (!dumped_sth)
6713 : {
6714 294 : fprintf (dump_file, "Propagated bits info for function %s:\n",
6715 : node->dump_name ());
6716 294 : dumped_sth = true;
6717 : }
6718 413 : fprintf (dump_file, " param %i: value = ", i);
6719 413 : ipcp_print_widest_int (dump_file, bits->get_value ());
6720 413 : fprintf (dump_file, ", mask = ");
6721 413 : ipcp_print_widest_int (dump_file, bits->get_mask ());
6722 413 : fprintf (dump_file, "\n");
6723 : }
6724 : }
6725 129922 : }
6726 :
6727 : /* The IPCP driver. */
6728 :
6729 : static unsigned int
6730 129922 : ipcp_driver (void)
6731 : {
6732 129922 : class ipa_topo_info topo;
6733 :
6734 129922 : if (edge_clone_summaries == NULL)
6735 129922 : edge_clone_summaries = new edge_clone_summary_t (symtab);
6736 :
6737 129922 : ipa_check_create_node_params ();
6738 129922 : ipa_check_create_edge_args ();
6739 129922 : clone_num_suffixes = new hash_map<const char *, unsigned>;
6740 :
6741 129922 : if (dump_file)
6742 : {
6743 161 : fprintf (dump_file, "\nIPA structures before propagation:\n");
6744 161 : if (dump_flags & TDF_DETAILS)
6745 48 : ipa_print_all_params (dump_file);
6746 161 : ipa_print_all_jump_functions (dump_file);
6747 : }
6748 :
6749 : /* Topological sort. */
6750 129922 : build_toporder_info (&topo);
6751 : /* Do the interprocedural propagation. */
6752 129922 : ipcp_propagate_stage (&topo);
6753 : /* Decide what constant propagation and cloning should be performed. */
6754 129922 : ipcp_decision_stage (&topo);
6755 : /* Store results of value range and bits propagation. */
6756 129922 : ipcp_store_vr_results ();
6757 :
6758 : /* Free all IPCP structures. */
6759 259844 : delete clone_num_suffixes;
6760 129922 : free_toporder_info (&topo);
6761 129922 : delete edge_clone_summaries;
6762 129922 : edge_clone_summaries = NULL;
6763 129922 : ipa_free_all_structures_after_ipa_cp ();
6764 129922 : if (dump_file)
6765 161 : fprintf (dump_file, "\nIPA constant propagation end\n");
6766 129922 : return 0;
6767 : }
6768 :
6769 : /* Initialization and computation of IPCP data structures. This is the initial
6770 : intraprocedural analysis of functions, which gathers information to be
6771 : propagated later on. */
6772 :
6773 : static void
6774 126871 : ipcp_generate_summary (void)
6775 : {
6776 126871 : struct cgraph_node *node;
6777 :
6778 126871 : if (dump_file)
6779 163 : fprintf (dump_file, "\nIPA constant propagation start:\n");
6780 126871 : ipa_register_cgraph_hooks ();
6781 :
6782 1406256 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6783 1279385 : ipa_analyze_node (node);
6784 :
6785 126871 : varpool_node *vnode;
6786 1807570 : FOR_EACH_STATIC_INITIALIZER (vnode)
6787 1680699 : ipa_analyze_var_static_initializer (vnode);
6788 126871 : }
6789 :
6790 : namespace {
6791 :
6792 : const pass_data pass_data_ipa_cp =
6793 : {
6794 : IPA_PASS, /* type */
6795 : "cp", /* name */
6796 : OPTGROUP_NONE, /* optinfo_flags */
6797 : TV_IPA_CONSTANT_PROP, /* tv_id */
6798 : 0, /* properties_required */
6799 : 0, /* properties_provided */
6800 : 0, /* properties_destroyed */
6801 : 0, /* todo_flags_start */
6802 : ( TODO_dump_symtab | TODO_remove_functions ), /* todo_flags_finish */
6803 : };
6804 :
6805 : class pass_ipa_cp : public ipa_opt_pass_d
6806 : {
6807 : public:
6808 292371 : pass_ipa_cp (gcc::context *ctxt)
6809 : : ipa_opt_pass_d (pass_data_ipa_cp, ctxt,
6810 : ipcp_generate_summary, /* generate_summary */
6811 : NULL, /* write_summary */
6812 : NULL, /* read_summary */
6813 : ipcp_write_transformation_summaries, /*
6814 : write_optimization_summary */
6815 : ipcp_read_transformation_summaries, /*
6816 : read_optimization_summary */
6817 : NULL, /* stmt_fixup */
6818 : 0, /* function_transform_todo_flags_start */
6819 : ipcp_transform_function, /* function_transform */
6820 292371 : NULL) /* variable_transform */
6821 292371 : {}
6822 :
6823 : /* opt_pass methods: */
6824 583567 : bool gate (function *) final override
6825 : {
6826 : /* FIXME: We should remove the optimize check after we ensure we never run
6827 : IPA passes when not optimizing. */
6828 583567 : return (flag_ipa_cp && optimize) || in_lto_p;
6829 : }
6830 :
6831 129922 : unsigned int execute (function *) final override { return ipcp_driver (); }
6832 :
6833 : }; // class pass_ipa_cp
6834 :
6835 : } // anon namespace
6836 :
6837 : ipa_opt_pass_d *
6838 292371 : make_pass_ipa_cp (gcc::context *ctxt)
6839 : {
6840 292371 : return new pass_ipa_cp (ctxt);
6841 : }
6842 :
6843 : /* Reset all state within ipa-cp.cc so that we can rerun the compiler
6844 : within the same process. For use by toplev::finalize. */
6845 :
6846 : void
6847 262609 : ipa_cp_cc_finalize (void)
6848 : {
6849 262609 : overall_size = 0;
6850 262609 : orig_overall_size = 0;
6851 262609 : ipcp_free_transformation_sum ();
6852 262609 : }
6853 :
6854 : /* Given PARAM which must be a parameter of function FNDECL described by THIS,
6855 : return its index in the DECL_ARGUMENTS chain, using a pre-computed
6856 : DECL_UID-sorted vector if available (which is pre-computed only if there are
6857 : many parameters). Can return -1 if param is static chain not represented
6858 : among DECL_ARGUMENTS. */
6859 :
6860 : int
6861 125380 : ipcp_transformation::get_param_index (const_tree fndecl, const_tree param) const
6862 : {
6863 125380 : gcc_assert (TREE_CODE (param) == PARM_DECL);
6864 125380 : if (m_uid_to_idx)
6865 : {
6866 0 : unsigned puid = DECL_UID (param);
6867 0 : const ipa_uid_to_idx_map_elt *res
6868 0 : = std::lower_bound (m_uid_to_idx->begin(), m_uid_to_idx->end (), puid,
6869 0 : [] (const ipa_uid_to_idx_map_elt &elt, unsigned uid)
6870 : {
6871 0 : return elt.uid < uid;
6872 : });
6873 0 : if (res == m_uid_to_idx->end ()
6874 0 : || res->uid != puid)
6875 : {
6876 0 : gcc_assert (DECL_STATIC_CHAIN (fndecl));
6877 : return -1;
6878 : }
6879 0 : return res->index;
6880 : }
6881 :
6882 125380 : unsigned index = 0;
6883 286519 : for (tree p = DECL_ARGUMENTS (fndecl); p; p = DECL_CHAIN (p), index++)
6884 285029 : if (p == param)
6885 123890 : return (int) index;
6886 :
6887 1490 : gcc_assert (DECL_STATIC_CHAIN (fndecl));
6888 : return -1;
6889 : }
6890 :
6891 : /* Helper function to qsort a vector of ipa_uid_to_idx_map_elt elements
6892 : according to the uid. */
6893 :
6894 : static int
6895 0 : compare_uids (const void *a, const void *b)
6896 : {
6897 0 : const ipa_uid_to_idx_map_elt *e1 = (const ipa_uid_to_idx_map_elt *) a;
6898 0 : const ipa_uid_to_idx_map_elt *e2 = (const ipa_uid_to_idx_map_elt *) b;
6899 0 : if (e1->uid < e2->uid)
6900 : return -1;
6901 0 : if (e1->uid > e2->uid)
6902 : return 1;
6903 0 : gcc_unreachable ();
6904 : }
6905 :
6906 : /* Assuming THIS describes FNDECL and it has sufficiently many parameters to
6907 : justify the overhead, create a DECL_UID-sorted vector to speed up mapping
6908 : from parameters to their indices in DECL_ARGUMENTS chain. */
6909 :
6910 : void
6911 22906 : ipcp_transformation::maybe_create_parm_idx_map (tree fndecl)
6912 : {
6913 22906 : int c = count_formal_params (fndecl);
6914 22906 : if (c < 32)
6915 : return;
6916 :
6917 0 : m_uid_to_idx = NULL;
6918 0 : vec_safe_reserve (m_uid_to_idx, c, true);
6919 0 : unsigned index = 0;
6920 0 : for (tree p = DECL_ARGUMENTS (fndecl); p; p = DECL_CHAIN (p), index++)
6921 : {
6922 0 : ipa_uid_to_idx_map_elt elt;
6923 0 : elt.uid = DECL_UID (p);
6924 0 : elt.index = index;
6925 0 : m_uid_to_idx->quick_push (elt);
6926 : }
6927 0 : m_uid_to_idx->qsort (compare_uids);
6928 : }
|
