Line data Source code
1 : /* Interprocedural constant propagation
2 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
3 :
4 : Contributed by Razya Ladelsky <RAZYA@il.ibm.com> and Martin Jambor
5 : <mjambor@suse.cz>
6 :
7 : This file is part of GCC.
8 :
9 : GCC is free software; you can redistribute it and/or modify it under
10 : the terms of the GNU General Public License as published by the Free
11 : Software Foundation; either version 3, or (at your option) any later
12 : version.
13 :
14 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 : for more details.
18 :
19 : You should have received a copy of the GNU General Public License
20 : along with GCC; see the file COPYING3. If not see
21 : <http://www.gnu.org/licenses/>. */
22 :
23 : /* Interprocedural constant propagation (IPA-CP).
24 :
25 : The goal of this transformation is to
26 :
27 : 1) discover functions which are always invoked with some arguments with the
28 : same known constant values and modify the functions so that the
29 : subsequent optimizations can take advantage of the knowledge, and
30 :
31 : 2) partial specialization - create specialized versions of functions
32 : transformed in this way if some parameters are known constants only in
33 : certain contexts but the estimated tradeoff between speedup and cost size
34 : is deemed good.
35 :
36 : The algorithm also propagates types and attempts to perform type based
37 : devirtualization. Types are propagated much like constants.
38 :
39 : The algorithm basically consists of three stages. In the first, functions
40 : are analyzed one at a time and jump functions are constructed for all known
41 : call-sites. In the second phase, the pass propagates information from the
42 : jump functions across the call to reveal what values are available at what
43 : call sites, performs estimations of effects of known values on functions and
44 : their callees, and finally decides what specialized extra versions should be
45 : created. In the third, the special versions materialize and appropriate
46 : calls are redirected.
47 :
48 : The algorithm used is to a certain extent based on "Interprocedural Constant
49 : Propagation", by David Callahan, Keith D Cooper, Ken Kennedy, Linda Torczon,
50 : Comp86, pg 152-161 and "A Methodology for Procedure Cloning" by Keith D
51 : Cooper, Mary W. Hall, and Ken Kennedy.
52 :
53 :
54 : First stage - intraprocedural analysis
55 : =======================================
56 :
57 : This phase computes jump_function and modification flags.
58 :
59 : A jump function for a call-site represents the values passed as an actual
60 : arguments of a given call-site. In principle, there are three types of
61 : values:
62 :
63 : Pass through - the caller's formal parameter is passed as an actual
64 : argument, plus an operation on it can be performed.
65 : Constant - a constant is passed as an actual argument.
66 : Unknown - neither of the above.
67 :
68 : All jump function types are described in detail in ipa-prop.h, together with
69 : the data structures that represent them and methods of accessing them.
70 :
71 : ipcp_generate_summary() is the main function of the first stage.
72 :
73 : Second stage - interprocedural analysis
74 : ========================================
75 :
76 : This stage is itself divided into two phases. In the first, we propagate
77 : known values over the call graph, in the second, we make cloning decisions.
78 : It uses a different algorithm than the original Callahan's paper.
79 :
80 : First, we traverse the functions topologically from callers to callees and,
81 : for each strongly connected component (SCC), we propagate constants
82 : according to previously computed jump functions. We also record what known
83 : values depend on other known values and estimate local effects. Finally, we
84 : propagate cumulative information about these effects from dependent values
85 : to those on which they depend.
86 :
87 : Second, we again traverse the call graph in the same topological order and
88 : make clones for functions which we know are called with the same values in
89 : all contexts and decide about extra specialized clones of functions just for
90 : some contexts - these decisions are based on both local estimates and
91 : cumulative estimates propagated from callees.
92 :
93 : ipcp_propagate_stage() and ipcp_decision_stage() together constitute the
94 : third stage.
95 :
96 : Third phase - materialization of clones, call statement updates.
97 : ============================================
98 :
99 : This stage is currently performed by call graph code (mainly in cgraphunit.cc
100 : and tree-inline.cc) according to instructions inserted to the call graph by
101 : the second stage. */
102 :
103 : #define INCLUDE_ALGORITHM
104 : #include "config.h"
105 : #include "system.h"
106 : #include "coretypes.h"
107 : #include "backend.h"
108 : #include "tree.h"
109 : #include "gimple-expr.h"
110 : #include "gimple.h"
111 : #include "predict.h"
112 : #include "sreal.h"
113 : #include "alloc-pool.h"
114 : #include "tree-pass.h"
115 : #include "cgraph.h"
116 : #include "diagnostic.h"
117 : #include "fold-const.h"
118 : #include "gimple-iterator.h"
119 : #include "gimple-fold.h"
120 : #include "symbol-summary.h"
121 : #include "tree-vrp.h"
122 : #include "ipa-cp.h"
123 : #include "ipa-prop.h"
124 : #include "tree-pretty-print.h"
125 : #include "tree-inline.h"
126 : #include "ipa-fnsummary.h"
127 : #include "ipa-utils.h"
128 : #include "tree-ssa-ccp.h"
129 : #include "stringpool.h"
130 : #include "attribs.h"
131 : #include "dbgcnt.h"
132 : #include "symtab-clones.h"
133 : #include "gimple-range.h"
134 : #include "attr-callback.h"
135 : #include "lto-streamer.h"
136 : #include "callback-info.h"
137 :
138 : /* Allocation pools for values and their sources in ipa-cp. */
139 :
140 : object_allocator<ipcp_value<tree> > ipcp_cst_values_pool
141 : ("IPA-CP constant values");
142 :
143 : object_allocator<ipcp_value<ipa_polymorphic_call_context> >
144 : ipcp_poly_ctx_values_pool ("IPA-CP polymorphic contexts");
145 :
146 : object_allocator<ipcp_value_source<tree> > ipcp_sources_pool
147 : ("IPA-CP value sources");
148 :
149 : object_allocator<ipcp_agg_lattice> ipcp_agg_lattice_pool
150 : ("IPA_CP aggregate lattices");
151 :
152 : /* Original overall size of the program. */
153 :
154 : static long overall_size, orig_overall_size;
155 :
156 : /* The maximum number of IPA-CP decision sweeps that any node requested in its
157 : param. */
158 : static int max_number_sweeps;
159 :
160 : /* Node name to unique clone suffix number map. */
161 : static hash_map<const char *, unsigned> *clone_num_suffixes;
162 :
163 : /* Return the param lattices structure corresponding to the Ith formal
164 : parameter of the function described by INFO. */
165 : static inline class ipcp_param_lattices *
166 32110804 : ipa_get_parm_lattices (class ipa_node_params *info, int i)
167 : {
168 64221608 : gcc_assert (i >= 0 && i < ipa_get_param_count (info));
169 32110804 : gcc_checking_assert (!info->ipcp_orig_node);
170 32110804 : return &(info->lattices[i]);
171 : }
172 :
173 : /* Return the lattice corresponding to the scalar value of the Ith formal
174 : parameter of the function described by INFO. */
175 : static inline ipcp_lattice<tree> *
176 6121575 : ipa_get_scalar_lat (class ipa_node_params *info, int i)
177 : {
178 6311120 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
179 6121575 : return &plats->itself;
180 : }
181 :
182 : /* Return the lattice corresponding to the scalar value of the Ith formal
183 : parameter of the function described by INFO. */
184 : static inline ipcp_lattice<ipa_polymorphic_call_context> *
185 815176 : ipa_get_poly_ctx_lat (class ipa_node_params *info, int i)
186 : {
187 815176 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
188 815176 : return &plats->ctxlat;
189 : }
190 :
191 : /* Return whether LAT is a lattice with a single constant and without an
192 : undefined value. */
193 :
194 : template <typename valtype>
195 : inline bool
196 10395866 : ipcp_lattice<valtype>::is_single_const ()
197 : {
198 3017193 : if (bottom || contains_variable || values_count != 1)
199 : return false;
200 : else
201 : return true;
202 : }
203 :
204 : /* Return true iff X and Y should be considered equal values by IPA-CP. */
205 :
206 : bool
207 1419960 : values_equal_for_ipcp_p (tree x, tree y)
208 : {
209 1419960 : gcc_checking_assert (x != NULL_TREE && y != NULL_TREE);
210 :
211 1419960 : if (x == y)
212 : return true;
213 :
214 626501 : if (TREE_CODE (x) == ADDR_EXPR
215 223854 : && TREE_CODE (y) == ADDR_EXPR
216 223126 : && (TREE_CODE (TREE_OPERAND (x, 0)) == CONST_DECL
217 175421 : || (TREE_CODE (TREE_OPERAND (x, 0)) == VAR_DECL
218 92327 : && DECL_IN_CONSTANT_POOL (TREE_OPERAND (x, 0))))
219 674206 : && (TREE_CODE (TREE_OPERAND (y, 0)) == CONST_DECL
220 13 : || (TREE_CODE (TREE_OPERAND (y, 0)) == VAR_DECL
221 8 : && DECL_IN_CONSTANT_POOL (TREE_OPERAND (y, 0)))))
222 47692 : return TREE_OPERAND (x, 0) == TREE_OPERAND (y, 0)
223 95128 : || operand_equal_p (DECL_INITIAL (TREE_OPERAND (x, 0)),
224 47436 : DECL_INITIAL (TREE_OPERAND (y, 0)), 0);
225 : else
226 578809 : return operand_equal_p (x, y, 0);
227 : }
228 :
229 : /* Print V which is extracted from a value in a lattice to F. This overloaded
230 : function is used to print tree constants. */
231 :
232 : static void
233 852 : print_ipcp_constant_value (FILE * f, tree v)
234 : {
235 0 : ipa_print_constant_value (f, v);
236 0 : }
237 :
238 : /* Print V which is extracted from a value in a lattice to F. This overloaded
239 : function is used to print constant polymorphic call contexts. */
240 :
241 : static void
242 214 : print_ipcp_constant_value (FILE * f, ipa_polymorphic_call_context v)
243 : {
244 214 : v.dump(f, false);
245 0 : }
246 :
247 : /* Print a lattice LAT to F. */
248 :
249 : template <typename valtype>
250 : void
251 2007 : ipcp_lattice<valtype>::print (FILE * f, bool dump_sources, bool dump_benefits)
252 : {
253 : ipcp_value<valtype> *val;
254 2007 : bool prev = false;
255 :
256 2007 : if (bottom)
257 : {
258 842 : fprintf (f, "BOTTOM\n");
259 842 : return;
260 : }
261 :
262 1165 : if (!values_count && !contains_variable)
263 : {
264 0 : fprintf (f, "TOP\n");
265 0 : return;
266 : }
267 :
268 1165 : if (contains_variable)
269 : {
270 885 : fprintf (f, "VARIABLE");
271 885 : prev = true;
272 885 : if (dump_benefits)
273 885 : fprintf (f, "\n");
274 : }
275 :
276 1807 : for (val = values; val; val = val->next)
277 : {
278 642 : if (dump_benefits && prev)
279 362 : fprintf (f, " ");
280 280 : else if (!dump_benefits && prev)
281 0 : fprintf (f, ", ");
282 : else
283 : prev = true;
284 :
285 642 : print_ipcp_constant_value (f, val->value);
286 :
287 642 : if (dump_sources)
288 : {
289 : ipcp_value_source<valtype> *s;
290 :
291 175 : if (val->self_recursion_generated_p ())
292 27 : fprintf (f, " [self_gen(%i), from:",
293 : val->self_recursion_generated_level);
294 : else
295 148 : fprintf (f, " [scc: %i, from:", val->scc_no);
296 368 : for (s = val->sources; s; s = s->next)
297 193 : fprintf (f, " %i(%f)", s->cs->caller->get_uid (),
298 386 : s->cs->sreal_frequency ().to_double ());
299 175 : fprintf (f, "]");
300 : }
301 :
302 642 : if (dump_benefits)
303 642 : fprintf (f, " [loc_time: %g, loc_size: %i, "
304 : "prop_time: %g, prop_size: %i]\n",
305 : val->local_time_benefit.to_double (), val->local_size_cost,
306 : val->prop_time_benefit.to_double (), val->prop_size_cost);
307 : }
308 1165 : if (!dump_benefits)
309 0 : fprintf (f, "\n");
310 : }
311 :
312 : /* Print VALUE to F in a form which in usual cases does not take thousands of
313 : characters. */
314 :
315 : static void
316 1466 : ipcp_print_widest_int (FILE *f, const widest_int &value)
317 : {
318 1466 : if (value == -1)
319 0 : fprintf (f, "-1");
320 1466 : else if (wi::arshift (value, 128) == -1)
321 : {
322 330 : char buf[35], *p = buf + 2;
323 330 : widest_int v = wi::zext (value, 128);
324 330 : size_t len;
325 330 : print_hex (v, buf);
326 330 : len = strlen (p);
327 330 : if (len == 32)
328 : {
329 330 : fprintf (f, "0xf..f");
330 9795 : while (*p == 'f')
331 9135 : ++p;
332 : }
333 : else
334 0 : fprintf (f, "0xf..f%0*d", (int) (32 - len), 0);
335 330 : fputs (p, f);
336 330 : }
337 : else
338 1136 : print_hex (value, f);
339 1466 : }
340 :
341 : void
342 923 : ipcp_bits_lattice::print (FILE *f)
343 : {
344 923 : if (bottom_p ())
345 : {
346 606 : fprintf (f, " Bits unusable (BOTTOM)\n");
347 606 : return;
348 : }
349 :
350 317 : if (top_p ())
351 0 : fprintf (f, " Bits unknown (TOP)");
352 : else
353 : {
354 317 : fprintf (f, " Bits: value = ");
355 317 : ipcp_print_widest_int (f, get_value ());
356 317 : fprintf (f, ", mask = ");
357 317 : ipcp_print_widest_int (f, get_mask ());
358 : }
359 :
360 317 : if (m_recipient_only)
361 143 : fprintf (f, " (recipient only)");
362 317 : fprintf (f, "\n");
363 : }
364 :
365 : /* Print value range lattice to F. */
366 :
367 : void
368 923 : ipcp_vr_lattice::print (FILE * f)
369 : {
370 923 : if (m_recipient_only)
371 270 : fprintf (f, "(recipient only) ");
372 923 : m_vr.dump (f);
373 923 : }
374 :
375 : /* Print all ipcp_lattices of all functions to F. */
376 :
377 : static void
378 162 : print_all_lattices (FILE * f, bool dump_sources, bool dump_benefits)
379 : {
380 162 : struct cgraph_node *node;
381 162 : int i, count;
382 :
383 162 : fprintf (f, "\nLattices:\n");
384 891 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
385 : {
386 729 : class ipa_node_params *info;
387 :
388 729 : info = ipa_node_params_sum->get (node);
389 : /* Skip unoptimized functions and constprop clones since we don't make
390 : lattices for them. */
391 729 : if (!info || info->ipcp_orig_node)
392 0 : continue;
393 729 : fprintf (f, " Node: %s:\n", node->dump_name ());
394 729 : count = ipa_get_param_count (info);
395 1652 : for (i = 0; i < count; i++)
396 : {
397 923 : struct ipcp_agg_lattice *aglat;
398 923 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
399 923 : fprintf (f, " param [%d]: ", i);
400 923 : plats->itself.print (f, dump_sources, dump_benefits);
401 923 : fprintf (f, " ctxs: ");
402 923 : plats->ctxlat.print (f, dump_sources, dump_benefits);
403 923 : plats->bits_lattice.print (f);
404 923 : fprintf (f, " ");
405 923 : plats->m_value_range.print (f);
406 923 : fprintf (f, "\n");
407 923 : if (plats->virt_call)
408 75 : fprintf (f, " virt_call flag set\n");
409 :
410 923 : if (plats->aggs_bottom)
411 : {
412 441 : fprintf (f, " AGGS BOTTOM\n");
413 441 : continue;
414 : }
415 482 : if (plats->aggs_contain_variable)
416 444 : fprintf (f, " AGGS VARIABLE\n");
417 643 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
418 : {
419 161 : fprintf (f, " %soffset " HOST_WIDE_INT_PRINT_DEC ": ",
420 161 : plats->aggs_by_ref ? "ref " : "", aglat->offset);
421 161 : aglat->print (f, dump_sources, dump_benefits);
422 : }
423 : }
424 : }
425 162 : }
426 :
427 : /* Determine whether it is at all technically possible to create clones of NODE
428 : and store this information in the ipa_node_params structure associated
429 : with NODE. */
430 :
431 : static void
432 1303950 : determine_versionability (struct cgraph_node *node,
433 : class ipa_node_params *info)
434 : {
435 1303950 : const char *reason = NULL;
436 :
437 : /* There are a number of generic reasons functions cannot be versioned. We
438 : also cannot remove parameters if there are type attributes such as fnspec
439 : present. */
440 1303950 : if (node->alias || node->thunk)
441 : reason = "alias or thunk";
442 1303950 : else if (!node->versionable)
443 : reason = "not a tree_versionable_function";
444 1173139 : else if (node->get_availability () <= AVAIL_INTERPOSABLE)
445 : reason = "insufficient body availability";
446 1105133 : else if (!opt_for_fn (node->decl, optimize)
447 1105133 : || !opt_for_fn (node->decl, flag_ipa_cp))
448 : reason = "non-optimized function";
449 1105133 : else if (lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (node->decl)))
450 : {
451 : /* Ideally we should clone the SIMD clones themselves and create
452 : vector copies of them, so IPA-cp and SIMD clones can happily
453 : coexist, but that may not be worth the effort. */
454 : reason = "function has SIMD clones";
455 : }
456 1104772 : else if (lookup_attribute ("target_clones", DECL_ATTRIBUTES (node->decl)))
457 : {
458 : /* Ideally we should clone the target clones themselves and create
459 : copies of them, so IPA-cp and target clones can happily
460 : coexist, but that may not be worth the effort. */
461 : reason = "function target_clones attribute";
462 : }
463 : /* Don't clone decls local to a comdat group; it breaks and for C++
464 : decloned constructors, inlining is always better anyway. */
465 1104772 : else if (node->comdat_local_p ())
466 : reason = "comdat-local function";
467 1102486 : else if (node->calls_comdat_local)
468 : {
469 : /* TODO: call is versionable if we make sure that all
470 : callers are inside of a comdat group. */
471 2388 : reason = "calls comdat-local function";
472 : }
473 :
474 : /* Functions calling BUILT_IN_VA_ARG_PACK and BUILT_IN_VA_ARG_PACK_LEN
475 : work only when inlined. Cloning them may still lead to better code
476 : because ipa-cp will not give up on cloning further. If the function is
477 : external this however leads to wrong code because we may end up producing
478 : offline copy of the function. */
479 1303950 : if (DECL_EXTERNAL (node->decl))
480 179879 : for (cgraph_edge *edge = node->callees; !reason && edge;
481 132228 : edge = edge->next_callee)
482 132228 : if (fndecl_built_in_p (edge->callee->decl, BUILT_IN_NORMAL))
483 : {
484 37277 : if (DECL_FUNCTION_CODE (edge->callee->decl) == BUILT_IN_VA_ARG_PACK)
485 0 : reason = "external function which calls va_arg_pack";
486 37277 : if (DECL_FUNCTION_CODE (edge->callee->decl)
487 : == BUILT_IN_VA_ARG_PACK_LEN)
488 0 : reason = "external function which calls va_arg_pack_len";
489 : }
490 :
491 1303950 : if (reason && dump_file && !node->alias && !node->thunk)
492 53 : fprintf (dump_file, "Function %s is not versionable, reason: %s.\n",
493 : node->dump_name (), reason);
494 :
495 1303950 : info->versionable = (reason == NULL);
496 1303950 : }
497 :
498 : /* Return true if it is at all technically possible to create clones of a
499 : NODE. */
500 :
501 : static bool
502 6244053 : ipcp_versionable_function_p (struct cgraph_node *node)
503 : {
504 6244053 : ipa_node_params *info = ipa_node_params_sum->get (node);
505 6244053 : return info && info->versionable;
506 : }
507 :
508 : /* Structure holding accumulated information about callers of a node. */
509 :
510 1057887 : struct caller_statistics
511 : {
512 : /* If requested (see below), self-recursive call counts are summed into this
513 : field. */
514 : profile_count rec_count_sum;
515 : /* The sum of all ipa counts of all the other (non-recursive) calls. */
516 : profile_count count_sum;
517 : /* Sum of all frequencies for all calls. */
518 : sreal freq_sum;
519 : /* Number of calls and calls considered interesting respectively. */
520 : int n_calls, n_interesting_calls;
521 : /* If itself is set up, also count the number of non-self-recursive
522 : calls. */
523 : int n_nonrec_calls;
524 : /* If non-NULL, this is the node itself and calls from it should have their
525 : counts included in rec_count_sum and not count_sum. */
526 : cgraph_node *itself;
527 : /* True if there is a caller that has no IPA profile. */
528 : bool called_without_ipa_profile;
529 : };
530 :
531 : /* Initialize fields of STAT to zeroes and optionally set it up so that edges
532 : from IGNORED_CALLER are not counted. */
533 :
534 : static inline void
535 289315 : init_caller_stats (caller_statistics *stats, cgraph_node *itself = NULL)
536 : {
537 289315 : stats->rec_count_sum = profile_count::zero ();
538 289315 : stats->count_sum = profile_count::zero ();
539 289315 : stats->n_calls = 0;
540 289315 : stats->n_interesting_calls = 0;
541 289315 : stats->n_nonrec_calls = 0;
542 289315 : stats->freq_sum = 0;
543 289315 : stats->itself = itself;
544 289315 : stats->called_without_ipa_profile = false;
545 289315 : }
546 :
547 : /* We want to propagate across edges that may be executed, however
548 : we do not want to check maybe_hot, since call itself may be cold
549 : while calee contains some heavy loop which makes propagation still
550 : relevant.
551 :
552 : In particular, even edge called once may lead to significant
553 : improvement. */
554 :
555 : static bool
556 982926 : cs_interesting_for_ipcp_p (cgraph_edge *e)
557 : {
558 : /* If profile says the edge is executed, we want to optimize. */
559 982926 : if (e->count.ipa ().nonzero_p ())
560 532 : return true;
561 : /* If local (possibly guseed or adjusted 0 profile) claims edge is
562 : not executed, do not propagate.
563 : Do not trust AFDO since branch needs to be executed multiple
564 : time to count while we want to propagate even call called
565 : once during the train run if callee is important. */
566 982394 : if (e->count.initialized_p () && !e->count.nonzero_p ()
567 1032102 : && e->count.quality () != AFDO)
568 : return false;
569 : /* If we have zero IPA profile, still consider edge for cloning
570 : in case we do partial training. */
571 932686 : if (e->count.ipa ().initialized_p ()
572 932686 : && e->count.ipa ().quality () != AFDO
573 932691 : && !opt_for_fn (e->callee->decl,flag_profile_partial_training))
574 5 : return false;
575 : return true;
576 : }
577 :
578 : /* Worker callback of cgraph_for_node_and_aliases accumulating statistics of
579 : non-thunk incoming edges to NODE. */
580 :
581 : static bool
582 296953 : gather_caller_stats (struct cgraph_node *node, void *data)
583 : {
584 296953 : struct caller_statistics *stats = (struct caller_statistics *) data;
585 296953 : struct cgraph_edge *cs;
586 :
587 1049117 : for (cs = node->callers; cs; cs = cs->next_caller)
588 752164 : if (!cs->caller->thunk)
589 : {
590 751991 : ipa_node_params *info = ipa_node_params_sum->get (cs->caller);
591 751991 : if (info && info->node_dead)
592 150541 : continue;
593 :
594 601450 : if (cs->count.ipa ().initialized_p ())
595 : {
596 3817 : if (stats->itself && stats->itself == cs->caller)
597 0 : stats->rec_count_sum += cs->count.ipa ();
598 : else
599 3817 : stats->count_sum += cs->count.ipa ();
600 : }
601 : else
602 597633 : stats->called_without_ipa_profile = true;
603 601450 : stats->freq_sum += cs->sreal_frequency ();
604 601450 : stats->n_calls++;
605 601450 : if (stats->itself && stats->itself != cs->caller)
606 8 : stats->n_nonrec_calls++;
607 :
608 : /* If profile known to be zero, we do not want to clone for performance.
609 : However if call is cold, the called function may still contain
610 : important hot loops. */
611 601450 : if (cs_interesting_for_ipcp_p (cs))
612 597199 : stats->n_interesting_calls++;
613 : }
614 296953 : return false;
615 :
616 : }
617 :
618 : /* Return true if this NODE is viable candidate for cloning. */
619 :
620 : static bool
621 816454 : ipcp_cloning_candidate_p (struct cgraph_node *node)
622 : {
623 816454 : struct caller_statistics stats;
624 :
625 816454 : gcc_checking_assert (node->has_gimple_body_p ());
626 :
627 816454 : if (!opt_for_fn (node->decl, flag_ipa_cp_clone))
628 : {
629 764287 : if (dump_file)
630 31 : fprintf (dump_file, "Not considering %s for cloning; "
631 : "-fipa-cp-clone disabled.\n",
632 : node->dump_name ());
633 : return false;
634 : }
635 :
636 : /* Do not use profile here since cold wrapper wrap
637 : hot function. */
638 52167 : if (opt_for_fn (node->decl, optimize_size))
639 : {
640 10 : if (dump_file)
641 0 : fprintf (dump_file, "Not considering %s for cloning; "
642 : "optimizing it for size.\n",
643 : node->dump_name ());
644 : return false;
645 : }
646 :
647 52157 : init_caller_stats (&stats);
648 52157 : node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats, false);
649 :
650 52157 : if (ipa_size_summaries->get (node)->self_size < stats.n_calls)
651 : {
652 311 : if (dump_file)
653 0 : fprintf (dump_file, "Considering %s for cloning; code might shrink.\n",
654 : node->dump_name ());
655 : return true;
656 : }
657 51846 : if (!stats.n_interesting_calls)
658 : {
659 39750 : if (dump_file)
660 200 : fprintf (dump_file, "Not considering %s for cloning; "
661 : "no calls considered interesting by profile.\n",
662 : node->dump_name ());
663 : return false;
664 : }
665 12096 : if (dump_file)
666 190 : fprintf (dump_file, "Considering %s for cloning.\n",
667 : node->dump_name ());
668 : return true;
669 : }
670 :
671 : template <typename valtype>
672 : class value_topo_info
673 : {
674 : public:
675 : /* Head of the linked list of topologically sorted values. */
676 : ipcp_value<valtype> *values_topo;
677 : /* Stack for creating SCCs, represented by a linked list too. */
678 : ipcp_value<valtype> *stack;
679 : /* Counter driving the algorithm in add_val_to_toposort. */
680 : int dfs_counter;
681 :
682 131565 : value_topo_info () : values_topo (NULL), stack (NULL), dfs_counter (0)
683 : {}
684 : void add_val (ipcp_value<valtype> *cur_val);
685 : void propagate_effects ();
686 : };
687 :
688 : /* Arrays representing a topological ordering of call graph nodes and a stack
689 : of nodes used during constant propagation and also data required to perform
690 : topological sort of values and propagation of benefits in the determined
691 : order. */
692 :
693 : class ipa_topo_info
694 : {
695 : public:
696 : /* Array with obtained topological order of cgraph nodes. */
697 : struct cgraph_node **order;
698 : /* Stack of cgraph nodes used during propagation within SCC until all values
699 : in the SCC stabilize. */
700 : struct cgraph_node **stack;
701 : int nnodes, stack_top;
702 :
703 : value_topo_info<tree> constants;
704 : value_topo_info<ipa_polymorphic_call_context> contexts;
705 :
706 131565 : ipa_topo_info () : order(NULL), stack(NULL), nnodes(0), stack_top(0),
707 131565 : constants ()
708 : {}
709 : };
710 :
711 : /* Skip edges from and to nodes without ipa_cp enabled.
712 : Ignore not available symbols. */
713 :
714 : static bool
715 5404503 : ignore_edge_p (cgraph_edge *e)
716 : {
717 5404503 : enum availability avail;
718 5404503 : cgraph_node *ultimate_target
719 5404503 : = e->callee->function_or_virtual_thunk_symbol (&avail, e->caller);
720 :
721 5404503 : return (avail <= AVAIL_INTERPOSABLE
722 1934612 : || !opt_for_fn (ultimate_target->decl, optimize)
723 7330363 : || !opt_for_fn (ultimate_target->decl, flag_ipa_cp));
724 : }
725 :
726 : /* Allocate the arrays in TOPO and topologically sort the nodes into order. */
727 :
728 : static void
729 131565 : build_toporder_info (class ipa_topo_info *topo)
730 : {
731 131565 : topo->order = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
732 131565 : topo->stack = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
733 :
734 131565 : gcc_checking_assert (topo->stack_top == 0);
735 131565 : topo->nnodes = ipa_reduced_postorder (topo->order, true,
736 : ignore_edge_p);
737 131565 : }
738 :
739 : /* Free information about strongly connected components and the arrays in
740 : TOPO. */
741 :
742 : static void
743 131565 : free_toporder_info (class ipa_topo_info *topo)
744 : {
745 131565 : ipa_free_postorder_info ();
746 131565 : free (topo->order);
747 131565 : free (topo->stack);
748 131565 : }
749 :
750 : /* Add NODE to the stack in TOPO, unless it is already there. */
751 :
752 : static inline void
753 1308135 : push_node_to_stack (class ipa_topo_info *topo, struct cgraph_node *node)
754 : {
755 1308135 : ipa_node_params *info = ipa_node_params_sum->get (node);
756 1308135 : if (info->node_enqueued)
757 : return;
758 1307160 : info->node_enqueued = 1;
759 1307160 : topo->stack[topo->stack_top++] = node;
760 : }
761 :
762 : /* Pop a node from the stack in TOPO and return it or return NULL if the stack
763 : is empty. */
764 :
765 : static struct cgraph_node *
766 2695106 : pop_node_from_stack (class ipa_topo_info *topo)
767 : {
768 2695106 : if (topo->stack_top)
769 : {
770 1307160 : struct cgraph_node *node;
771 1307160 : topo->stack_top--;
772 1307160 : node = topo->stack[topo->stack_top];
773 1307160 : ipa_node_params_sum->get (node)->node_enqueued = 0;
774 1307160 : return node;
775 : }
776 : else
777 : return NULL;
778 : }
779 :
780 : /* Set lattice LAT to bottom and return true if it previously was not set as
781 : such. */
782 :
783 : template <typename valtype>
784 : inline bool
785 2176441 : ipcp_lattice<valtype>::set_to_bottom ()
786 : {
787 2176441 : bool ret = !bottom;
788 2176441 : bottom = true;
789 : return ret;
790 : }
791 :
792 : /* Mark lattice as containing an unknown value and return true if it previously
793 : was not marked as such. */
794 :
795 : template <typename valtype>
796 : inline bool
797 1568487 : ipcp_lattice<valtype>::set_contains_variable ()
798 : {
799 1568487 : bool ret = !contains_variable;
800 1568487 : contains_variable = true;
801 : return ret;
802 : }
803 :
804 : /* Set all aggregate lattices in PLATS to bottom and return true if they were
805 : not previously set as such. */
806 :
807 : static inline bool
808 2176215 : set_agg_lats_to_bottom (class ipcp_param_lattices *plats)
809 : {
810 2176215 : bool ret = !plats->aggs_bottom;
811 2176215 : plats->aggs_bottom = true;
812 2176215 : return ret;
813 : }
814 :
815 : /* Mark all aggregate lattices in PLATS as containing an unknown value and
816 : return true if they were not previously marked as such. */
817 :
818 : static inline bool
819 1061233 : set_agg_lats_contain_variable (class ipcp_param_lattices *plats)
820 : {
821 1061233 : bool ret = !plats->aggs_contain_variable;
822 1061233 : plats->aggs_contain_variable = true;
823 1061233 : return ret;
824 : }
825 :
826 : bool
827 0 : ipcp_vr_lattice::meet_with (const ipcp_vr_lattice &other)
828 : {
829 0 : return meet_with_1 (other.m_vr);
830 : }
831 :
832 : /* Meet the current value of the lattice with the range described by
833 : P_VR. */
834 :
835 : bool
836 495568 : ipcp_vr_lattice::meet_with (const vrange &p_vr)
837 : {
838 495568 : return meet_with_1 (p_vr);
839 : }
840 :
841 : /* Meet the current value of the lattice with the range described by
842 : OTHER_VR. Return TRUE if anything changed. */
843 :
844 : bool
845 495568 : ipcp_vr_lattice::meet_with_1 (const vrange &other_vr)
846 : {
847 495568 : if (bottom_p ())
848 : return false;
849 :
850 495568 : if (other_vr.varying_p ())
851 0 : return set_to_bottom ();
852 :
853 495568 : bool res;
854 495568 : if (flag_checking)
855 : {
856 495568 : value_range save (m_vr);
857 495568 : res = m_vr.union_ (other_vr);
858 495568 : gcc_assert (res == (m_vr != save));
859 495568 : }
860 : else
861 0 : res = m_vr.union_ (other_vr);
862 : return res;
863 : }
864 :
865 : /* Return true if value range information in the lattice is yet unknown. */
866 :
867 : bool
868 : ipcp_vr_lattice::top_p () const
869 : {
870 171302 : return m_vr.undefined_p ();
871 : }
872 :
873 : /* Return true if value range information in the lattice is known to be
874 : unusable. */
875 :
876 : bool
877 5011954 : ipcp_vr_lattice::bottom_p () const
878 : {
879 495568 : return m_vr.varying_p ();
880 : }
881 :
882 : /* Set value range information in the lattice to bottom. Return true if it
883 : previously was in a different state. */
884 :
885 : bool
886 2460004 : ipcp_vr_lattice::set_to_bottom ()
887 : {
888 2460004 : if (m_vr.varying_p ())
889 : return false;
890 :
891 : /* Setting an unsupported type here forces the temporary to default
892 : to unsupported_range, which can handle VARYING/DEFINED ranges,
893 : but nothing else (union, intersect, etc). This allows us to set
894 : bottoms on any ranges, and is safe as all users of the lattice
895 : check for bottom first. */
896 2313134 : m_vr.set_range_class (void_type_node);
897 2313134 : m_vr.set_varying (void_type_node);
898 :
899 2313134 : return true;
900 : }
901 :
902 : /* Set the flag that this lattice is a recipient only, return true if it was
903 : not set before. */
904 :
905 : bool
906 29818 : ipcp_vr_lattice::set_recipient_only ()
907 : {
908 29818 : if (m_recipient_only)
909 : return false;
910 29818 : m_recipient_only = true;
911 29818 : return true;
912 : }
913 :
914 : /* Set lattice value to bottom, if it already isn't the case. */
915 :
916 : bool
917 2479715 : ipcp_bits_lattice::set_to_bottom ()
918 : {
919 2479715 : if (bottom_p ())
920 : return false;
921 2333364 : m_lattice_val = IPA_BITS_VARYING;
922 2333364 : m_value = 0;
923 2333364 : m_mask = -1;
924 2333364 : return true;
925 : }
926 :
927 : /* Set to constant if it isn't already. Only meant to be called
928 : when switching state from TOP. */
929 :
930 : bool
931 78047 : ipcp_bits_lattice::set_to_constant (widest_int value, widest_int mask)
932 : {
933 78047 : gcc_assert (top_p ());
934 78047 : m_lattice_val = IPA_BITS_CONSTANT;
935 78047 : m_value = wi::bit_and (wi::bit_not (mask), value);
936 78047 : m_mask = mask;
937 78047 : return true;
938 : }
939 :
940 : /* Return true if any of the known bits are non-zero. */
941 :
942 : bool
943 507 : ipcp_bits_lattice::known_nonzero_p () const
944 : {
945 507 : if (!constant_p ())
946 : return false;
947 507 : return wi::ne_p (wi::bit_and (wi::bit_not (m_mask), m_value), 0);
948 : }
949 :
950 : /* Set the flag that this lattice is a recipient only, return true if it was not
951 : set before. */
952 :
953 : bool
954 29818 : ipcp_bits_lattice::set_recipient_only ()
955 : {
956 29818 : if (m_recipient_only)
957 : return false;
958 29818 : m_recipient_only = true;
959 29818 : return true;
960 : }
961 :
962 : /* Convert operand to value, mask form. */
963 :
964 : void
965 2054 : ipcp_bits_lattice::get_value_and_mask (tree operand, widest_int *valuep, widest_int *maskp)
966 : {
967 2054 : wide_int get_nonzero_bits (const_tree);
968 :
969 2054 : if (TREE_CODE (operand) == INTEGER_CST)
970 : {
971 2054 : *valuep = wi::to_widest (operand);
972 2054 : *maskp = 0;
973 : }
974 : else
975 : {
976 0 : *valuep = 0;
977 0 : *maskp = -1;
978 : }
979 2054 : }
980 :
981 : /* Meet operation, similar to ccp_lattice_meet, we xor values
982 : if this->value, value have different values at same bit positions, we want
983 : to drop that bit to varying. Return true if mask is changed.
984 : This function assumes that the lattice value is in CONSTANT state. If
985 : DROP_ALL_ONES, mask out any known bits with value one afterwards. */
986 :
987 : bool
988 303437 : ipcp_bits_lattice::meet_with_1 (widest_int value, widest_int mask,
989 : unsigned precision, bool drop_all_ones)
990 : {
991 303437 : gcc_assert (constant_p ());
992 :
993 303437 : widest_int old_mask = m_mask;
994 303437 : m_mask = (m_mask | mask) | (m_value ^ value);
995 303437 : if (drop_all_ones)
996 211 : m_mask |= m_value;
997 :
998 303437 : widest_int cap_mask = wi::shifted_mask <widest_int> (0, precision, true);
999 303437 : m_mask |= cap_mask;
1000 303437 : if (wi::sext (m_mask, precision) == -1)
1001 3605 : return set_to_bottom ();
1002 :
1003 299832 : m_value &= ~m_mask;
1004 299832 : return m_mask != old_mask;
1005 303437 : }
1006 :
1007 : /* Meet the bits lattice with operand
1008 : described by <value, mask, sgn, precision. */
1009 :
1010 : bool
1011 415515 : ipcp_bits_lattice::meet_with (widest_int value, widest_int mask,
1012 : unsigned precision)
1013 : {
1014 415515 : if (bottom_p ())
1015 : return false;
1016 :
1017 415515 : if (top_p ())
1018 : {
1019 124768 : if (wi::sext (mask, precision) == -1)
1020 52164 : return set_to_bottom ();
1021 72604 : return set_to_constant (value, mask);
1022 : }
1023 :
1024 290747 : return meet_with_1 (value, mask, precision, false);
1025 : }
1026 :
1027 : /* Meet bits lattice with the result of bit_value_binop (other, operand)
1028 : if code is binary operation or bit_value_unop (other) if code is unary op.
1029 : In the case when code is nop_expr, no adjustment is required. If
1030 : DROP_ALL_ONES, mask out any known bits with value one afterwards. */
1031 :
1032 : bool
1033 21744 : ipcp_bits_lattice::meet_with (ipcp_bits_lattice& other, unsigned precision,
1034 : signop sgn, enum tree_code code, tree operand,
1035 : bool drop_all_ones)
1036 : {
1037 21744 : if (other.bottom_p ())
1038 0 : return set_to_bottom ();
1039 :
1040 21744 : if (bottom_p () || other.top_p ())
1041 : return false;
1042 :
1043 18249 : widest_int adjusted_value, adjusted_mask;
1044 :
1045 18249 : if (TREE_CODE_CLASS (code) == tcc_binary)
1046 : {
1047 2054 : tree type = TREE_TYPE (operand);
1048 2054 : widest_int o_value, o_mask;
1049 2054 : get_value_and_mask (operand, &o_value, &o_mask);
1050 :
1051 2054 : bit_value_binop (code, sgn, precision, &adjusted_value, &adjusted_mask,
1052 4108 : sgn, precision, other.get_value (), other.get_mask (),
1053 2054 : TYPE_SIGN (type), TYPE_PRECISION (type), o_value, o_mask);
1054 :
1055 2054 : if (wi::sext (adjusted_mask, precision) == -1)
1056 87 : return set_to_bottom ();
1057 2054 : }
1058 :
1059 16195 : else if (TREE_CODE_CLASS (code) == tcc_unary)
1060 : {
1061 32340 : bit_value_unop (code, sgn, precision, &adjusted_value,
1062 32340 : &adjusted_mask, sgn, precision, other.get_value (),
1063 16170 : other.get_mask ());
1064 :
1065 16170 : if (wi::sext (adjusted_mask, precision) == -1)
1066 4 : return set_to_bottom ();
1067 : }
1068 :
1069 : else
1070 25 : return set_to_bottom ();
1071 :
1072 18133 : if (top_p ())
1073 : {
1074 5443 : if (drop_all_ones)
1075 : {
1076 296 : adjusted_mask |= adjusted_value;
1077 296 : adjusted_value &= ~adjusted_mask;
1078 : }
1079 5443 : widest_int cap_mask = wi::shifted_mask <widest_int> (0, precision, true);
1080 5443 : adjusted_mask |= cap_mask;
1081 5443 : if (wi::sext (adjusted_mask, precision) == -1)
1082 0 : return set_to_bottom ();
1083 5443 : return set_to_constant (adjusted_value, adjusted_mask);
1084 5443 : }
1085 : else
1086 12690 : return meet_with_1 (adjusted_value, adjusted_mask, precision,
1087 : drop_all_ones);
1088 18249 : }
1089 :
1090 : /* Dump the contents of the list to FILE. */
1091 :
1092 : void
1093 124 : ipa_argagg_value_list::dump (FILE *f)
1094 : {
1095 124 : bool comma = false;
1096 348 : for (const ipa_argagg_value &av : m_elts)
1097 : {
1098 224 : fprintf (f, "%s %i[%u]=", comma ? "," : "",
1099 224 : av.index, av.unit_offset);
1100 224 : print_generic_expr (f, av.value);
1101 224 : if (av.by_ref)
1102 197 : fprintf (f, "(by_ref)");
1103 224 : if (av.killed)
1104 1 : fprintf (f, "(killed)");
1105 224 : comma = true;
1106 : }
1107 124 : fprintf (f, "\n");
1108 124 : }
1109 :
1110 : /* Dump the contents of the list to stderr. */
1111 :
1112 : void
1113 0 : ipa_argagg_value_list::debug ()
1114 : {
1115 0 : dump (stderr);
1116 0 : }
1117 :
1118 : /* Return the item describing a constant stored for INDEX at UNIT_OFFSET or
1119 : NULL if there is no such constant. */
1120 :
1121 : const ipa_argagg_value *
1122 31295292 : ipa_argagg_value_list::get_elt (int index, unsigned unit_offset) const
1123 : {
1124 31295292 : ipa_argagg_value key;
1125 31295292 : key.index = index;
1126 31295292 : key.unit_offset = unit_offset;
1127 31295292 : const ipa_argagg_value *res
1128 31295292 : = std::lower_bound (m_elts.begin (), m_elts.end (), key,
1129 7580634 : [] (const ipa_argagg_value &elt,
1130 : const ipa_argagg_value &val)
1131 : {
1132 7580634 : if (elt.index < val.index)
1133 : return true;
1134 6473853 : if (elt.index > val.index)
1135 : return false;
1136 5156417 : if (elt.unit_offset < val.unit_offset)
1137 : return true;
1138 : return false;
1139 : });
1140 :
1141 31295292 : if (res == m_elts.end ()
1142 3285375 : || res->index != index
1143 34000405 : || res->unit_offset != unit_offset)
1144 : res = nullptr;
1145 :
1146 : /* TODO: perhaps remove the check (that the underlying array is indeed
1147 : sorted) if it turns out it can be too slow? */
1148 31295292 : if (!flag_checking)
1149 : return res;
1150 :
1151 : const ipa_argagg_value *slow_res = NULL;
1152 : int prev_index = -1;
1153 : unsigned prev_unit_offset = 0;
1154 49162297 : for (const ipa_argagg_value &av : m_elts)
1155 : {
1156 17867005 : gcc_assert (prev_index < 0
1157 : || prev_index < av.index
1158 : || prev_unit_offset < av.unit_offset);
1159 17867005 : prev_index = av.index;
1160 17867005 : prev_unit_offset = av.unit_offset;
1161 17867005 : if (av.index == index
1162 8278084 : && av.unit_offset == unit_offset)
1163 17867005 : slow_res = &av;
1164 : }
1165 31295292 : gcc_assert (res == slow_res);
1166 :
1167 : return res;
1168 : }
1169 :
1170 : /* Return the first item describing a constant stored for parameter with INDEX,
1171 : regardless of offset or reference, or NULL if there is no such constant. */
1172 :
1173 : const ipa_argagg_value *
1174 242391 : ipa_argagg_value_list::get_elt_for_index (int index) const
1175 : {
1176 242391 : const ipa_argagg_value *res
1177 242391 : = std::lower_bound (m_elts.begin (), m_elts.end (), index,
1178 19440 : [] (const ipa_argagg_value &elt, unsigned idx)
1179 : {
1180 19440 : return elt.index < idx;
1181 : });
1182 242391 : if (res == m_elts.end ()
1183 242391 : || res->index != index)
1184 : res = nullptr;
1185 242391 : return res;
1186 : }
1187 :
1188 : /* Return the aggregate constant stored for INDEX at UNIT_OFFSET, not
1189 : performing any check of whether value is passed by reference, or NULL_TREE
1190 : if there is no such constant. */
1191 :
1192 : tree
1193 38623 : ipa_argagg_value_list::get_value (int index, unsigned unit_offset) const
1194 : {
1195 38623 : const ipa_argagg_value *av = get_elt (index, unit_offset);
1196 38623 : return av ? av->value : NULL_TREE;
1197 : }
1198 :
1199 : /* Return the aggregate constant stored for INDEX at UNIT_OFFSET, if it is
1200 : passed by reference or not according to BY_REF, or NULL_TREE if there is
1201 : no such constant. */
1202 :
1203 : tree
1204 31246933 : ipa_argagg_value_list::get_value (int index, unsigned unit_offset,
1205 : bool by_ref) const
1206 : {
1207 31246933 : const ipa_argagg_value *av = get_elt (index, unit_offset);
1208 31246933 : if (av && av->by_ref == by_ref)
1209 2193284 : return av->value;
1210 : return NULL_TREE;
1211 : }
1212 :
1213 : /* Return true if all elements present in OTHER are also present in this
1214 : list. */
1215 :
1216 : bool
1217 46 : ipa_argagg_value_list::superset_of_p (const ipa_argagg_value_list &other) const
1218 : {
1219 46 : unsigned j = 0;
1220 222 : for (unsigned i = 0; i < other.m_elts.size (); i++)
1221 : {
1222 193 : unsigned other_index = other.m_elts[i].index;
1223 193 : unsigned other_offset = other.m_elts[i].unit_offset;
1224 :
1225 193 : while (j < m_elts.size ()
1226 368 : && (m_elts[j].index < other_index
1227 352 : || (m_elts[j].index == other_index
1228 352 : && m_elts[j].unit_offset < other_offset)))
1229 175 : j++;
1230 :
1231 193 : if (j >= m_elts.size ()
1232 180 : || m_elts[j].index != other_index
1233 180 : || m_elts[j].unit_offset != other_offset
1234 180 : || m_elts[j].by_ref != other.m_elts[i].by_ref
1235 180 : || !m_elts[j].value
1236 373 : || !values_equal_for_ipcp_p (m_elts[j].value, other.m_elts[i].value))
1237 : return false;
1238 : }
1239 : return true;
1240 : }
1241 :
1242 : /* Push all items in this list that describe parameter SRC_INDEX into RES as
1243 : ones describing DST_INDEX while subtracting UNIT_DELTA from their unit
1244 : offsets but skip those which would end up with a negative offset. */
1245 :
1246 : void
1247 3171 : ipa_argagg_value_list::push_adjusted_values (unsigned src_index,
1248 : unsigned dest_index,
1249 : unsigned unit_delta,
1250 : vec<ipa_argagg_value> *res) const
1251 : {
1252 3171 : const ipa_argagg_value *av = get_elt_for_index (src_index);
1253 3171 : if (!av)
1254 : return;
1255 : unsigned prev_unit_offset = 0;
1256 : bool first = true;
1257 12679 : for (; av < m_elts.end (); ++av)
1258 : {
1259 10211 : if (av->index > src_index)
1260 : return;
1261 9601 : if (av->index == src_index
1262 9601 : && (av->unit_offset >= unit_delta)
1263 9457 : && av->value)
1264 : {
1265 9457 : ipa_argagg_value new_av;
1266 9457 : gcc_checking_assert (av->value);
1267 9457 : new_av.value = av->value;
1268 9457 : new_av.unit_offset = av->unit_offset - unit_delta;
1269 9457 : new_av.index = dest_index;
1270 9457 : new_av.by_ref = av->by_ref;
1271 9457 : gcc_assert (!av->killed);
1272 9457 : new_av.killed = false;
1273 :
1274 : /* Quick check that the offsets we push are indeed increasing. */
1275 9457 : gcc_assert (first
1276 : || new_av.unit_offset > prev_unit_offset);
1277 9457 : prev_unit_offset = new_av.unit_offset;
1278 9457 : first = false;
1279 :
1280 9457 : res->safe_push (new_av);
1281 : }
1282 : }
1283 : }
1284 :
1285 : /* Push to RES information about single lattices describing aggregate values in
1286 : PLATS as those describing parameter DEST_INDEX and the original offset minus
1287 : UNIT_DELTA. Return true if any item has been pushed to RES. */
1288 :
1289 : static bool
1290 2334261 : push_agg_values_from_plats (ipcp_param_lattices *plats, int dest_index,
1291 : unsigned unit_delta,
1292 : vec<ipa_argagg_value> *res)
1293 : {
1294 2334261 : if (plats->aggs_contain_variable)
1295 : return false;
1296 :
1297 1693015 : bool pushed_sth = false;
1298 1693015 : bool first = true;
1299 1693015 : unsigned prev_unit_offset = 0;
1300 1760004 : for (struct ipcp_agg_lattice *aglat = plats->aggs; aglat; aglat = aglat->next)
1301 132770 : if (aglat->is_single_const ()
1302 41407 : && (aglat->offset / BITS_PER_UNIT - unit_delta) >= 0)
1303 : {
1304 41407 : ipa_argagg_value iav;
1305 41407 : iav.value = aglat->values->value;
1306 41407 : iav.unit_offset = aglat->offset / BITS_PER_UNIT - unit_delta;
1307 41407 : iav.index = dest_index;
1308 41407 : iav.by_ref = plats->aggs_by_ref;
1309 41407 : iav.killed = false;
1310 :
1311 41407 : gcc_assert (first
1312 : || iav.unit_offset > prev_unit_offset);
1313 41407 : prev_unit_offset = iav.unit_offset;
1314 41407 : first = false;
1315 :
1316 41407 : pushed_sth = true;
1317 41407 : res->safe_push (iav);
1318 : }
1319 : return pushed_sth;
1320 : }
1321 :
1322 : /* Turn all values in LIST that are not present in OTHER into NULL_TREEs.
1323 : Return the number of remaining valid entries. */
1324 :
1325 : static unsigned
1326 56486 : intersect_argaggs_with (vec<ipa_argagg_value> &elts,
1327 : const vec<ipa_argagg_value> &other)
1328 : {
1329 56486 : unsigned valid_entries = 0;
1330 56486 : unsigned j = 0;
1331 428621 : for (unsigned i = 0; i < elts.length (); i++)
1332 : {
1333 372135 : if (!elts[i].value)
1334 54929 : continue;
1335 :
1336 317206 : unsigned this_index = elts[i].index;
1337 317206 : unsigned this_offset = elts[i].unit_offset;
1338 :
1339 317206 : while (j < other.length ()
1340 1192678 : && (other[j].index < this_index
1341 561436 : || (other[j].index == this_index
1342 558028 : && other[j].unit_offset < this_offset)))
1343 283560 : j++;
1344 :
1345 317206 : if (j >= other.length ())
1346 : {
1347 8854 : elts[i].value = NULL_TREE;
1348 8854 : continue;
1349 : }
1350 :
1351 308352 : if (other[j].index == this_index
1352 304944 : && other[j].unit_offset == this_offset
1353 297431 : && other[j].by_ref == elts[i].by_ref
1354 297431 : && other[j].value
1355 605783 : && values_equal_for_ipcp_p (other[j].value, elts[i].value))
1356 277474 : valid_entries++;
1357 : else
1358 30878 : elts[i].value = NULL_TREE;
1359 : }
1360 56486 : return valid_entries;
1361 : }
1362 :
1363 : /* Mark bot aggregate and scalar lattices as containing an unknown variable,
1364 : return true is any of them has not been marked as such so far. If if
1365 : MAKE_SIMPLE_RECIPIENTS is true, set the lattices that can only hold one
1366 : value to being recipients only, otherwise also set them to bottom. */
1367 :
1368 : static inline bool
1369 176424 : set_all_contains_variable (class ipcp_param_lattices *plats,
1370 : bool make_simple_recipients = false)
1371 : {
1372 176424 : bool ret;
1373 176424 : ret = plats->itself.set_contains_variable ();
1374 176424 : ret |= plats->ctxlat.set_contains_variable ();
1375 176424 : ret |= set_agg_lats_contain_variable (plats);
1376 176424 : if (make_simple_recipients)
1377 : {
1378 29818 : ret |= plats->bits_lattice.set_recipient_only ();
1379 29818 : ret |= plats->m_value_range.set_recipient_only ();
1380 : }
1381 : else
1382 : {
1383 146606 : ret |= plats->bits_lattice.set_to_bottom ();
1384 146606 : ret |= plats->m_value_range.set_to_bottom ();
1385 : }
1386 176424 : return ret;
1387 : }
1388 :
1389 : /* Worker of call_for_symbol_thunks_and_aliases, increment the integer DATA
1390 : points to by the number of callers to NODE. */
1391 :
1392 : static bool
1393 102237 : count_callers (cgraph_node *node, void *data)
1394 : {
1395 102237 : int *caller_count = (int *) data;
1396 :
1397 412191 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
1398 : /* Local thunks can be handled transparently, but if the thunk cannot
1399 : be optimized out, count it as a real use. */
1400 309954 : if (!cs->caller->thunk || !cs->caller->local)
1401 309954 : ++*caller_count;
1402 102237 : return false;
1403 : }
1404 :
1405 : /* Worker of call_for_symbol_thunks_and_aliases, it is supposed to be called on
1406 : the one caller of some other node. Set the caller's corresponding flag. */
1407 :
1408 : static bool
1409 58944 : set_single_call_flag (cgraph_node *node, void *)
1410 : {
1411 58944 : cgraph_edge *cs = node->callers;
1412 : /* Local thunks can be handled transparently, skip them. */
1413 58944 : while (cs && cs->caller->thunk && cs->caller->local)
1414 0 : cs = cs->next_caller;
1415 58944 : if (cs)
1416 58358 : if (ipa_node_params* info = ipa_node_params_sum->get (cs->caller))
1417 : {
1418 58357 : info->node_calling_single_call = true;
1419 58357 : return true;
1420 : }
1421 : return false;
1422 : }
1423 :
1424 : /* Initialize ipcp_lattices. */
1425 :
1426 : static void
1427 1303950 : initialize_node_lattices (struct cgraph_node *node)
1428 : {
1429 1303950 : ipa_node_params *info = ipa_node_params_sum->get (node);
1430 1303950 : struct cgraph_edge *ie;
1431 1303950 : bool disable = false, variable = false;
1432 1303950 : int i;
1433 :
1434 1303950 : gcc_checking_assert (node->has_gimple_body_p ());
1435 :
1436 1303950 : if (!ipa_get_param_count (info))
1437 : disable = true;
1438 1070583 : else if (node->local)
1439 : {
1440 91299 : int caller_count = 0;
1441 91299 : node->call_for_symbol_thunks_and_aliases (count_callers, &caller_count,
1442 : true);
1443 91299 : if (caller_count == 1)
1444 58358 : node->call_for_symbol_thunks_and_aliases (set_single_call_flag,
1445 : NULL, true);
1446 32941 : else if (caller_count == 0)
1447 : {
1448 1 : gcc_checking_assert (!opt_for_fn (node->decl, flag_toplevel_reorder));
1449 : variable = true;
1450 : }
1451 : }
1452 : else
1453 : {
1454 : /* When cloning is allowed, we can assume that externally visible
1455 : functions are not called. We will compensate this by cloning
1456 : later. */
1457 979284 : if (ipcp_versionable_function_p (node)
1458 979284 : && ipcp_cloning_candidate_p (node))
1459 : variable = true;
1460 : else
1461 : disable = true;
1462 : }
1463 :
1464 729 : if (dump_file && (dump_flags & TDF_DETAILS)
1465 1304117 : && !node->alias && !node->thunk)
1466 : {
1467 167 : fprintf (dump_file, "Initializing lattices of %s\n",
1468 : node->dump_name ());
1469 167 : if (disable || variable)
1470 133 : fprintf (dump_file, " Marking all lattices as %s\n",
1471 : disable ? "BOTTOM" : "VARIABLE");
1472 : }
1473 :
1474 1303950 : auto_vec<bool, 16> surviving_params;
1475 1303950 : bool pre_modified = false;
1476 :
1477 1303950 : clone_info *cinfo = clone_info::get (node);
1478 :
1479 1303950 : if (!disable && cinfo && cinfo->param_adjustments)
1480 : {
1481 : /* At the moment all IPA optimizations should use the number of
1482 : parameters of the prevailing decl as the m_always_copy_start.
1483 : Handling any other value would complicate the code below, so for the
1484 : time bing let's only assert it is so. */
1485 0 : gcc_assert ((cinfo->param_adjustments->m_always_copy_start
1486 : == ipa_get_param_count (info))
1487 : || cinfo->param_adjustments->m_always_copy_start < 0);
1488 :
1489 0 : pre_modified = true;
1490 0 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
1491 :
1492 0 : if (dump_file && (dump_flags & TDF_DETAILS)
1493 0 : && !node->alias && !node->thunk)
1494 : {
1495 : bool first = true;
1496 0 : for (int j = 0; j < ipa_get_param_count (info); j++)
1497 : {
1498 0 : if (j < (int) surviving_params.length ()
1499 0 : && surviving_params[j])
1500 0 : continue;
1501 0 : if (first)
1502 : {
1503 0 : fprintf (dump_file,
1504 : " The following parameters are dead on arrival:");
1505 0 : first = false;
1506 : }
1507 0 : fprintf (dump_file, " %u", j);
1508 : }
1509 0 : if (!first)
1510 0 : fprintf (dump_file, "\n");
1511 : }
1512 : }
1513 :
1514 7183647 : for (i = 0; i < ipa_get_param_count (info); i++)
1515 : {
1516 2404557 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
1517 2404557 : tree type = ipa_get_type (info, i);
1518 2404557 : if (disable
1519 229485 : || !ipa_get_type (info, i)
1520 2634042 : || (pre_modified && (surviving_params.length () <= (unsigned) i
1521 0 : || !surviving_params[i])))
1522 : {
1523 2175072 : plats->itself.set_to_bottom ();
1524 2175072 : plats->ctxlat.set_to_bottom ();
1525 2175072 : set_agg_lats_to_bottom (plats);
1526 2175072 : plats->bits_lattice.set_to_bottom ();
1527 2175072 : plats->m_value_range.init (type);
1528 2175072 : plats->m_value_range.set_to_bottom ();
1529 : }
1530 : else
1531 : {
1532 229485 : plats->m_value_range.init (type);
1533 229485 : if (variable)
1534 29818 : set_all_contains_variable (plats, true);
1535 : }
1536 : }
1537 :
1538 1439568 : for (ie = node->indirect_calls; ie; ie = ie->next_callee)
1539 135618 : if (ie->indirect_info->param_index >= 0
1540 144941 : && is_a <cgraph_polymorphic_indirect_info *> (ie->indirect_info))
1541 9323 : ipa_get_parm_lattices (info,
1542 9323 : ie->indirect_info->param_index)->virt_call = 1;
1543 1303950 : }
1544 :
1545 : /* Return VALUE if it is NULL_TREE or if it can be directly safely IPA-CP
1546 : propagated to a parameter of type PARAM_TYPE, or return a fold-converted
1547 : VALUE to PARAM_TYPE if that is possible. Return NULL_TREE otherwise. */
1548 :
1549 : tree
1550 5200065 : ipacp_value_safe_for_type (tree param_type, tree value)
1551 : {
1552 5200065 : if (!value)
1553 : return NULL_TREE;
1554 5199731 : tree val_type = TREE_TYPE (value);
1555 5199731 : if (param_type == val_type
1556 5199731 : || useless_type_conversion_p (param_type, val_type))
1557 : return value;
1558 3393 : if (fold_convertible_p (param_type, value))
1559 3184 : return fold_convert (param_type, value);
1560 : else
1561 : return NULL_TREE;
1562 : }
1563 :
1564 : /* Return the result of a (possibly arithmetic) operation determined by OPCODE
1565 : on the constant value INPUT. OPERAND is 2nd operand for binary operation
1566 : and is required for binary operations. RES_TYPE, required when opcode is
1567 : not NOP_EXPR, is the type in which any operation is to be performed. Return
1568 : NULL_TREE if that cannot be determined or be considered an interprocedural
1569 : invariant. */
1570 :
1571 : static tree
1572 70234 : ipa_get_jf_arith_result (enum tree_code opcode, tree input, tree operand,
1573 : tree res_type)
1574 : {
1575 70234 : tree res;
1576 :
1577 70234 : if (opcode == NOP_EXPR)
1578 : return input;
1579 6768 : if (!is_gimple_ip_invariant (input))
1580 : return NULL_TREE;
1581 :
1582 6768 : if (opcode == ASSERT_EXPR)
1583 : {
1584 3802 : if (values_equal_for_ipcp_p (input, operand))
1585 : return input;
1586 : else
1587 112 : return NULL_TREE;
1588 : }
1589 :
1590 2966 : if (TREE_CODE_CLASS (opcode) == tcc_unary)
1591 102 : res = fold_unary (opcode, res_type, input);
1592 : else
1593 2864 : res = fold_binary (opcode, res_type, input, operand);
1594 :
1595 2966 : if (res && !is_gimple_ip_invariant (res))
1596 0 : return NULL_TREE;
1597 :
1598 : return res;
1599 : }
1600 :
1601 : /* Return the result of an ancestor jump function JFUNC on the constant value
1602 : INPUT. Return NULL_TREE if that cannot be determined. */
1603 :
1604 : static tree
1605 1302 : ipa_get_jf_ancestor_result (struct ipa_jump_func *jfunc, tree input)
1606 : {
1607 1302 : gcc_checking_assert (TREE_CODE (input) != TREE_BINFO);
1608 1302 : if (TREE_CODE (input) == ADDR_EXPR)
1609 : {
1610 1220 : gcc_checking_assert (is_gimple_ip_invariant_address (input));
1611 1220 : poly_int64 off = ipa_get_jf_ancestor_offset (jfunc);
1612 1220 : if (known_eq (off, 0))
1613 : return input;
1614 1098 : poly_int64 byte_offset = exact_div (off, BITS_PER_UNIT);
1615 2196 : return build1 (ADDR_EXPR, TREE_TYPE (input),
1616 1098 : fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (input)), input,
1617 1098 : build_int_cst (ptr_type_node, byte_offset)));
1618 : }
1619 82 : else if (ipa_get_jf_ancestor_keep_null (jfunc)
1620 82 : && zerop (input))
1621 : return input;
1622 : else
1623 78 : return NULL_TREE;
1624 : }
1625 :
1626 : /* Determine whether JFUNC evaluates to a single known constant value and if
1627 : so, return it. Otherwise return NULL. INFO describes the caller node or
1628 : the one it is inlined to, so that pass-through jump functions can be
1629 : evaluated. PARM_TYPE is the type of the parameter to which the result is
1630 : passed. */
1631 :
1632 : tree
1633 18597862 : ipa_value_from_jfunc (class ipa_node_params *info, struct ipa_jump_func *jfunc,
1634 : tree parm_type)
1635 : {
1636 18597862 : if (!parm_type)
1637 : return NULL_TREE;
1638 18355066 : if (jfunc->type == IPA_JF_CONST)
1639 4588674 : return ipacp_value_safe_for_type (parm_type, ipa_get_jf_constant (jfunc));
1640 13766392 : else if (jfunc->type == IPA_JF_PASS_THROUGH
1641 10785110 : || jfunc->type == IPA_JF_ANCESTOR)
1642 : {
1643 3809409 : tree input;
1644 3809409 : int idx;
1645 :
1646 3809409 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1647 2981282 : idx = ipa_get_jf_pass_through_formal_id (jfunc);
1648 : else
1649 828127 : idx = ipa_get_jf_ancestor_formal_id (jfunc);
1650 :
1651 3809409 : if (info->ipcp_orig_node)
1652 48922 : input = info->known_csts[idx];
1653 : else
1654 : {
1655 3760487 : ipcp_lattice<tree> *lat;
1656 :
1657 6719824 : if (info->lattices.is_empty ()
1658 2959337 : || idx >= ipa_get_param_count (info))
1659 : return NULL_TREE;
1660 2959337 : lat = ipa_get_scalar_lat (info, idx);
1661 2959337 : if (!lat->is_single_const ())
1662 : return NULL_TREE;
1663 152 : input = lat->values->value;
1664 : }
1665 :
1666 49074 : if (!input)
1667 : return NULL_TREE;
1668 :
1669 19859 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1670 : {
1671 18875 : enum tree_code opcode = ipa_get_jf_pass_through_operation (jfunc);
1672 18875 : tree op2 = ipa_get_jf_pass_through_operand (jfunc);
1673 18875 : tree op_type
1674 18875 : = (opcode == NOP_EXPR) ? NULL_TREE
1675 941 : : ipa_get_jf_pass_through_op_type (jfunc);
1676 18875 : tree cstval = ipa_get_jf_arith_result (opcode, input, op2, op_type);
1677 18875 : return ipacp_value_safe_for_type (parm_type, cstval);
1678 : }
1679 : else
1680 984 : return ipacp_value_safe_for_type (parm_type,
1681 : ipa_get_jf_ancestor_result (jfunc,
1682 984 : input));
1683 : }
1684 : else
1685 : return NULL_TREE;
1686 : }
1687 :
1688 : /* Determine whether JFUNC evaluates to single known polymorphic context, given
1689 : that INFO describes the caller node or the one it is inlined to, CS is the
1690 : call graph edge corresponding to JFUNC and CSIDX index of the described
1691 : parameter. */
1692 :
1693 : ipa_polymorphic_call_context
1694 915125 : ipa_context_from_jfunc (ipa_node_params *info, cgraph_edge *cs, int csidx,
1695 : ipa_jump_func *jfunc)
1696 : {
1697 915125 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
1698 915125 : ipa_polymorphic_call_context ctx;
1699 915125 : ipa_polymorphic_call_context *edge_ctx
1700 915125 : = cs ? ipa_get_ith_polymorhic_call_context (args, csidx) : NULL;
1701 :
1702 380209 : if (edge_ctx && !edge_ctx->useless_p ())
1703 372201 : ctx = *edge_ctx;
1704 :
1705 915125 : if (jfunc->type == IPA_JF_PASS_THROUGH
1706 821385 : || jfunc->type == IPA_JF_ANCESTOR)
1707 : {
1708 101900 : ipa_polymorphic_call_context srcctx;
1709 101900 : int srcidx;
1710 101900 : bool type_preserved = true;
1711 101900 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1712 : {
1713 93740 : if (ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR)
1714 1784 : return ctx;
1715 91956 : type_preserved = ipa_get_jf_pass_through_type_preserved (jfunc);
1716 91956 : srcidx = ipa_get_jf_pass_through_formal_id (jfunc);
1717 : }
1718 : else
1719 : {
1720 8160 : type_preserved = ipa_get_jf_ancestor_type_preserved (jfunc);
1721 8160 : srcidx = ipa_get_jf_ancestor_formal_id (jfunc);
1722 : }
1723 100116 : if (info->ipcp_orig_node)
1724 : {
1725 11638 : if (info->known_contexts.exists ())
1726 1359 : srcctx = info->known_contexts[srcidx];
1727 : }
1728 : else
1729 : {
1730 175317 : if (info->lattices.is_empty ()
1731 86839 : || srcidx >= ipa_get_param_count (info))
1732 1639 : return ctx;
1733 86839 : ipcp_lattice<ipa_polymorphic_call_context> *lat;
1734 86839 : lat = ipa_get_poly_ctx_lat (info, srcidx);
1735 86839 : if (!lat->is_single_const ())
1736 82836 : return ctx;
1737 4003 : srcctx = lat->values->value;
1738 : }
1739 15641 : if (srcctx.useless_p ())
1740 10732 : return ctx;
1741 4909 : if (jfunc->type == IPA_JF_ANCESTOR)
1742 253 : srcctx.offset_by (ipa_get_jf_ancestor_offset (jfunc));
1743 4909 : if (!type_preserved)
1744 2917 : srcctx.possible_dynamic_type_change (cs->in_polymorphic_cdtor);
1745 4909 : srcctx.combine_with (ctx);
1746 4909 : return srcctx;
1747 : }
1748 :
1749 813225 : return ctx;
1750 : }
1751 :
1752 : /* Emulate effects of unary OPERATION and/or conversion from SRC_TYPE to
1753 : DST_TYPE on value range in SRC_VR and store it to DST_VR. Return true if
1754 : the result is a range that is not VARYING nor UNDEFINED. */
1755 :
1756 : bool
1757 9644072 : ipa_vr_operation_and_type_effects (vrange &dst_vr,
1758 : const vrange &src_vr,
1759 : enum tree_code operation,
1760 : tree dst_type, tree src_type)
1761 : {
1762 18086620 : if (!ipa_vr_supported_type_p (dst_type)
1763 19288144 : || !ipa_vr_supported_type_p (src_type))
1764 : return false;
1765 :
1766 9644072 : range_op_handler handler (operation);
1767 9644072 : if (!handler)
1768 : return false;
1769 :
1770 9644072 : value_range varying (dst_type);
1771 9644072 : varying.set_varying (dst_type);
1772 :
1773 9644072 : return (handler.operand_check_p (dst_type, src_type, dst_type)
1774 9644072 : && handler.fold_range (dst_vr, dst_type, src_vr, varying)
1775 9644070 : && !dst_vr.varying_p ()
1776 19288082 : && !dst_vr.undefined_p ());
1777 9644072 : }
1778 :
1779 : /* Same as above, but the SRC_VR argument is an IPA_VR which must
1780 : first be extracted onto a vrange. */
1781 :
1782 : bool
1783 9636275 : ipa_vr_operation_and_type_effects (vrange &dst_vr,
1784 : const ipa_vr &src_vr,
1785 : enum tree_code operation,
1786 : tree dst_type, tree src_type)
1787 : {
1788 9636275 : value_range tmp;
1789 9636275 : src_vr.get_vrange (tmp);
1790 9636275 : return ipa_vr_operation_and_type_effects (dst_vr, tmp, operation,
1791 9636275 : dst_type, src_type);
1792 9636275 : }
1793 :
1794 : /* Given a PASS_THROUGH jump function JFUNC that takes as its source SRC_VR of
1795 : SRC_TYPE and the result needs to be DST_TYPE, if any value range information
1796 : can be deduced at all, intersect VR with it. CONTEXT_NODE is the call graph
1797 : node representing the function for which optimization flags should be
1798 : evaluated. */
1799 :
1800 : static void
1801 93435 : ipa_vr_intersect_with_arith_jfunc (vrange &vr,
1802 : ipa_jump_func *jfunc,
1803 : cgraph_node *context_node,
1804 : const value_range &src_vr,
1805 : tree src_type,
1806 : tree dst_type)
1807 : {
1808 93435 : if (src_vr.undefined_p () || src_vr.varying_p ())
1809 92223 : return;
1810 :
1811 92951 : enum tree_code operation = ipa_get_jf_pass_through_operation (jfunc);
1812 92951 : if (TREE_CODE_CLASS (operation) == tcc_unary)
1813 : {
1814 91739 : value_range op_res;
1815 91739 : const value_range *inter_vr;
1816 91739 : if (operation != NOP_EXPR)
1817 : {
1818 93 : tree operation_type = ipa_get_jf_pass_through_op_type (jfunc);
1819 93 : op_res.set_varying (operation_type);
1820 93 : if (!ipa_vr_operation_and_type_effects (op_res, src_vr, operation,
1821 : operation_type, src_type))
1822 : return;
1823 : inter_vr = &op_res;
1824 : src_type = operation_type;
1825 : }
1826 : else
1827 : inter_vr = &src_vr;
1828 :
1829 91739 : if (src_type != dst_type)
1830 : {
1831 6492 : value_range tmp_res (dst_type);
1832 6492 : if (!ipa_vr_operation_and_type_effects (tmp_res, *inter_vr, NOP_EXPR,
1833 : dst_type, src_type))
1834 0 : return;
1835 6492 : vr.intersect (tmp_res);
1836 6492 : }
1837 : else
1838 85247 : vr.intersect (*inter_vr);
1839 : return;
1840 91739 : }
1841 :
1842 1212 : tree operand = ipa_get_jf_pass_through_operand (jfunc);
1843 1212 : range_op_handler handler (operation);
1844 1212 : if (!handler)
1845 : return;
1846 1212 : value_range op_vr (TREE_TYPE (operand));
1847 1212 : ipa_get_range_from_ip_invariant (op_vr, operand, context_node);
1848 :
1849 1212 : tree operation_type = ipa_get_jf_pass_through_op_type (jfunc);
1850 1212 : value_range op_res (operation_type);
1851 1652 : if (!ipa_vr_supported_type_p (operation_type)
1852 1212 : || !handler.operand_check_p (operation_type, src_type, op_vr.type ())
1853 1212 : || !handler.fold_range (op_res, operation_type, src_vr, op_vr))
1854 0 : return;
1855 :
1856 1212 : value_range tmp_res (dst_type);
1857 1212 : if (ipa_vr_operation_and_type_effects (tmp_res, op_res, NOP_EXPR, dst_type,
1858 : operation_type))
1859 1164 : vr.intersect (tmp_res);
1860 1212 : }
1861 :
1862 : /* Determine range of JFUNC given that INFO describes the caller node or
1863 : the one it is inlined to, CS is the call graph edge corresponding to JFUNC
1864 : and PARM_TYPE of the parameter. */
1865 :
1866 : void
1867 12447268 : ipa_value_range_from_jfunc (vrange &vr,
1868 : ipa_node_params *info, cgraph_edge *cs,
1869 : ipa_jump_func *jfunc, tree parm_type)
1870 : {
1871 12447268 : vr.set_varying (parm_type);
1872 :
1873 12447268 : if (jfunc->m_vr && jfunc->m_vr->known_p ())
1874 8797497 : ipa_vr_operation_and_type_effects (vr,
1875 : *jfunc->m_vr,
1876 : NOP_EXPR, parm_type,
1877 8797497 : jfunc->m_vr->type ());
1878 12447268 : if (vr.singleton_p ())
1879 : return;
1880 :
1881 12447183 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1882 : {
1883 2392340 : ipcp_transformation *sum
1884 2392340 : = ipcp_get_transformation_summary (cs->caller->inlined_to
1885 : ? cs->caller->inlined_to
1886 : : cs->caller);
1887 2392340 : if (!sum || !sum->m_vr)
1888 2314366 : return;
1889 :
1890 120742 : int idx = ipa_get_jf_pass_through_formal_id (jfunc);
1891 :
1892 120742 : if (!(*sum->m_vr)[idx].known_p ())
1893 : return;
1894 77974 : tree src_type = ipa_get_type (info, idx);
1895 77974 : value_range srcvr;
1896 77974 : (*sum->m_vr)[idx].get_vrange (srcvr);
1897 :
1898 77974 : ipa_vr_intersect_with_arith_jfunc (vr, jfunc, cs->caller, srcvr, src_type,
1899 : parm_type);
1900 77974 : }
1901 : }
1902 :
1903 : /* Determine whether ITEM, jump function for an aggregate part, evaluates to a
1904 : single known constant value and if so, return it. Otherwise return NULL.
1905 : NODE and INFO describes the caller node or the one it is inlined to, and
1906 : its related info. */
1907 :
1908 : tree
1909 3552101 : ipa_agg_value_from_jfunc (ipa_node_params *info, cgraph_node *node,
1910 : const ipa_agg_jf_item *item)
1911 : {
1912 3552101 : tree value = NULL_TREE;
1913 3552101 : int src_idx;
1914 :
1915 3552101 : if (item->offset < 0
1916 3502790 : || item->jftype == IPA_JF_UNKNOWN
1917 3341151 : || item->offset >= (HOST_WIDE_INT) UINT_MAX * BITS_PER_UNIT)
1918 : return NULL_TREE;
1919 :
1920 3341151 : if (item->jftype == IPA_JF_CONST)
1921 2978511 : return item->value.constant;
1922 :
1923 362640 : gcc_checking_assert (item->jftype == IPA_JF_PASS_THROUGH
1924 : || item->jftype == IPA_JF_LOAD_AGG);
1925 :
1926 362640 : src_idx = item->value.pass_through.formal_id;
1927 :
1928 362640 : if (info->ipcp_orig_node)
1929 : {
1930 16915 : if (item->jftype == IPA_JF_PASS_THROUGH)
1931 3882 : value = info->known_csts[src_idx];
1932 13033 : else if (ipcp_transformation *ts = ipcp_get_transformation_summary (node))
1933 : {
1934 13033 : ipa_argagg_value_list avl (ts);
1935 13033 : value = avl.get_value (src_idx,
1936 13033 : item->value.load_agg.offset / BITS_PER_UNIT,
1937 13033 : item->value.load_agg.by_ref);
1938 : }
1939 : }
1940 345725 : else if (!info->lattices.is_empty ())
1941 : {
1942 230011 : class ipcp_param_lattices *src_plats
1943 230011 : = ipa_get_parm_lattices (info, src_idx);
1944 :
1945 230011 : if (item->jftype == IPA_JF_PASS_THROUGH)
1946 : {
1947 138435 : struct ipcp_lattice<tree> *lat = &src_plats->itself;
1948 :
1949 138435 : if (!lat->is_single_const ())
1950 : return NULL_TREE;
1951 :
1952 0 : value = lat->values->value;
1953 : }
1954 91576 : else if (src_plats->aggs
1955 12316 : && !src_plats->aggs_bottom
1956 12316 : && !src_plats->aggs_contain_variable
1957 1503 : && src_plats->aggs_by_ref == item->value.load_agg.by_ref)
1958 : {
1959 : struct ipcp_agg_lattice *aglat;
1960 :
1961 2370 : for (aglat = src_plats->aggs; aglat; aglat = aglat->next)
1962 : {
1963 2370 : if (aglat->offset > item->value.load_agg.offset)
1964 : break;
1965 :
1966 2338 : if (aglat->offset == item->value.load_agg.offset)
1967 : {
1968 1471 : if (aglat->is_single_const ())
1969 7 : value = aglat->values->value;
1970 : break;
1971 : }
1972 : }
1973 : }
1974 : }
1975 :
1976 16954 : if (!value)
1977 : return NULL_TREE;
1978 :
1979 10190 : if (item->jftype == IPA_JF_LOAD_AGG)
1980 : {
1981 7854 : tree load_type = item->value.load_agg.type;
1982 7854 : tree value_type = TREE_TYPE (value);
1983 :
1984 : /* Ensure value type is compatible with load type. */
1985 7854 : if (!useless_type_conversion_p (load_type, value_type))
1986 : return NULL_TREE;
1987 : }
1988 :
1989 20380 : tree cstval = ipa_get_jf_arith_result (item->value.pass_through.operation,
1990 : value,
1991 10190 : item->value.pass_through.operand,
1992 10190 : item->value.pass_through.op_type);
1993 10190 : return ipacp_value_safe_for_type (item->type, cstval);
1994 : }
1995 :
1996 : /* Process all items in AGG_JFUNC relative to caller (or the node the original
1997 : caller is inlined to) NODE which described by INFO and push the results to
1998 : RES as describing values passed in parameter DST_INDEX. */
1999 :
2000 : void
2001 14920518 : ipa_push_agg_values_from_jfunc (ipa_node_params *info, cgraph_node *node,
2002 : ipa_agg_jump_function *agg_jfunc,
2003 : unsigned dst_index,
2004 : vec<ipa_argagg_value> *res)
2005 : {
2006 14920518 : unsigned prev_unit_offset = 0;
2007 14920518 : bool first = true;
2008 :
2009 20084884 : for (const ipa_agg_jf_item &item : agg_jfunc->items)
2010 : {
2011 2511046 : tree value = ipa_agg_value_from_jfunc (info, node, &item);
2012 2511046 : if (!value)
2013 537528 : continue;
2014 :
2015 1973518 : ipa_argagg_value iav;
2016 1973518 : iav.value = value;
2017 1973518 : iav.unit_offset = item.offset / BITS_PER_UNIT;
2018 1973518 : iav.index = dst_index;
2019 1973518 : iav.by_ref = agg_jfunc->by_ref;
2020 1973518 : iav.killed = 0;
2021 :
2022 1973518 : gcc_assert (first
2023 : || iav.unit_offset > prev_unit_offset);
2024 1973518 : prev_unit_offset = iav.unit_offset;
2025 1973518 : first = false;
2026 :
2027 1973518 : res->safe_push (iav);
2028 : }
2029 14920518 : }
2030 :
2031 : /* If checking is enabled, verify that no lattice is in the TOP state, i.e. not
2032 : bottom, not containing a variable component and without any known value at
2033 : the same time. */
2034 :
2035 : DEBUG_FUNCTION void
2036 131557 : ipcp_verify_propagated_values (void)
2037 : {
2038 131557 : struct cgraph_node *node;
2039 :
2040 1444366 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
2041 : {
2042 1312809 : ipa_node_params *info = ipa_node_params_sum->get (node);
2043 1312809 : if (!opt_for_fn (node->decl, flag_ipa_cp)
2044 1312809 : || !opt_for_fn (node->decl, optimize))
2045 8876 : continue;
2046 1303933 : int i, count = ipa_get_param_count (info);
2047 :
2048 3708476 : for (i = 0; i < count; i++)
2049 : {
2050 2404543 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (info, i);
2051 :
2052 2404543 : if (!lat->bottom
2053 228478 : && !lat->contains_variable
2054 32153 : && lat->values_count == 0)
2055 : {
2056 0 : if (dump_file)
2057 : {
2058 0 : symtab->dump (dump_file);
2059 0 : fprintf (dump_file, "\nIPA lattices after constant "
2060 : "propagation, before gcc_unreachable:\n");
2061 0 : print_all_lattices (dump_file, true, false);
2062 : }
2063 :
2064 0 : gcc_unreachable ();
2065 : }
2066 : }
2067 : }
2068 131557 : }
2069 :
2070 : /* Return true iff X and Y should be considered equal contexts by IPA-CP. */
2071 :
2072 : static bool
2073 2762 : values_equal_for_ipcp_p (ipa_polymorphic_call_context x,
2074 : ipa_polymorphic_call_context y)
2075 : {
2076 2262 : return x.equal_to (y);
2077 : }
2078 :
2079 :
2080 : /* Add a new value source to the value represented by THIS, marking that a
2081 : value comes from edge CS and (if the underlying jump function is a
2082 : pass-through or an ancestor one) from a caller value SRC_VAL of a caller
2083 : parameter described by SRC_INDEX. OFFSET is negative if the source was the
2084 : scalar value of the parameter itself or the offset within an aggregate. */
2085 :
2086 : template <typename valtype>
2087 : void
2088 341299 : ipcp_value<valtype>::add_source (cgraph_edge *cs, ipcp_value *src_val,
2089 : int src_idx, HOST_WIDE_INT offset)
2090 : {
2091 : ipcp_value_source<valtype> *src;
2092 :
2093 491845 : src = new (ipcp_sources_pool.allocate ()) ipcp_value_source<valtype>;
2094 491845 : src->offset = offset;
2095 491845 : src->cs = cs;
2096 491845 : src->val = src_val;
2097 491845 : src->index = src_idx;
2098 :
2099 491845 : src->next = sources;
2100 491845 : sources = src;
2101 : }
2102 :
2103 : /* Allocate a new ipcp_value holding a tree constant, initialize its value to
2104 : SOURCE and clear all other fields. */
2105 :
2106 : static ipcp_value<tree> *
2107 142535 : allocate_and_init_ipcp_value (tree cst, unsigned same_lat_gen_level)
2108 : {
2109 142535 : ipcp_value<tree> *val;
2110 :
2111 142535 : val = new (ipcp_cst_values_pool.allocate ()) ipcp_value<tree>();
2112 142535 : val->value = cst;
2113 142535 : val->self_recursion_generated_level = same_lat_gen_level;
2114 142535 : return val;
2115 : }
2116 :
2117 : /* Allocate a new ipcp_value holding a polymorphic context, initialize its
2118 : value to SOURCE and clear all other fields. */
2119 :
2120 : static ipcp_value<ipa_polymorphic_call_context> *
2121 8011 : allocate_and_init_ipcp_value (ipa_polymorphic_call_context ctx,
2122 : unsigned same_lat_gen_level)
2123 : {
2124 8011 : ipcp_value<ipa_polymorphic_call_context> *val;
2125 :
2126 8011 : val = new (ipcp_poly_ctx_values_pool.allocate ())
2127 8011 : ipcp_value<ipa_polymorphic_call_context>();
2128 8011 : val->value = ctx;
2129 8011 : val->self_recursion_generated_level = same_lat_gen_level;
2130 8011 : return val;
2131 : }
2132 :
2133 : /* Try to add NEWVAL to LAT, potentially creating a new ipcp_value for it. CS,
2134 : SRC_VAL SRC_INDEX and OFFSET are meant for add_source and have the same
2135 : meaning. OFFSET -1 means the source is scalar and not a part of an
2136 : aggregate. If non-NULL, VAL_P records address of existing or newly added
2137 : ipcp_value.
2138 :
2139 : If the value is generated for a self-recursive call as a result of an
2140 : arithmetic pass-through jump-function acting on a value in the same lattice,
2141 : SAME_LAT_GEN_LEVEL must be the length of such chain, otherwise it must be
2142 : zero. If it is non-zero, PARAM_IPA_CP_VALUE_LIST_SIZE limit is ignored. */
2143 :
2144 : template <typename valtype>
2145 : bool
2146 504253 : ipcp_lattice<valtype>::add_value (valtype newval, cgraph_edge *cs,
2147 : ipcp_value<valtype> *src_val,
2148 : int src_idx, HOST_WIDE_INT offset,
2149 : ipcp_value<valtype> **val_p,
2150 : unsigned same_lat_gen_level)
2151 : {
2152 504253 : ipcp_value<valtype> *val, *last_val = NULL;
2153 :
2154 504253 : if (val_p)
2155 1257 : *val_p = NULL;
2156 :
2157 504253 : if (bottom)
2158 : return false;
2159 :
2160 975093 : for (val = values; val; last_val = val, val = val->next)
2161 823178 : if (values_equal_for_ipcp_p (val->value, newval))
2162 : {
2163 348974 : if (val_p)
2164 416 : *val_p = val;
2165 :
2166 348974 : if (val->self_recursion_generated_level < same_lat_gen_level)
2167 179 : val->self_recursion_generated_level = same_lat_gen_level;
2168 :
2169 348974 : if (ipa_edge_within_scc (cs))
2170 : {
2171 : ipcp_value_source<valtype> *s;
2172 48894 : for (s = val->sources; s; s = s->next)
2173 44665 : if (s->cs == cs && s->val == src_val)
2174 : break;
2175 11904 : if (s)
2176 : return false;
2177 : }
2178 :
2179 341299 : val->add_source (cs, src_val, src_idx, offset);
2180 341299 : return false;
2181 : }
2182 :
2183 151915 : if (!same_lat_gen_level && values_count >= opt_for_fn (cs->callee->decl,
2184 : param_ipa_cp_value_list_size))
2185 : {
2186 : /* We can only free sources, not the values themselves, because sources
2187 : of other values in this SCC might point to them. */
2188 12303 : for (val = values; val; val = val->next)
2189 : {
2190 40489 : while (val->sources)
2191 : {
2192 29555 : ipcp_value_source<valtype> *src = val->sources;
2193 29555 : val->sources = src->next;
2194 29555 : ipcp_sources_pool.remove ((ipcp_value_source<tree>*)src);
2195 : }
2196 : }
2197 1369 : values = NULL;
2198 1369 : return set_to_bottom ();
2199 : }
2200 :
2201 150546 : values_count++;
2202 150546 : val = allocate_and_init_ipcp_value (newval, same_lat_gen_level);
2203 150546 : val->add_source (cs, src_val, src_idx, offset);
2204 150546 : val->next = NULL;
2205 :
2206 : /* Add the new value to end of value list, which can reduce iterations
2207 : of propagation stage for recursive function. */
2208 150546 : if (last_val)
2209 45875 : last_val->next = val;
2210 : else
2211 104671 : values = val;
2212 :
2213 150546 : if (val_p)
2214 841 : *val_p = val;
2215 :
2216 : return true;
2217 : }
2218 :
2219 : /* A helper function that returns result of operation specified by OPCODE on
2220 : the value of SRC_VAL. If non-NULL, OPND1_TYPE is expected type for the
2221 : value of SRC_VAL. If the operation is binary, OPND2 is a constant value
2222 : acting as its second operand. OP_TYPE is the type in which the operation is
2223 : performed. */
2224 :
2225 : static tree
2226 21992 : get_val_across_arith_op (enum tree_code opcode,
2227 : tree opnd1_type,
2228 : tree opnd2,
2229 : ipcp_value<tree> *src_val,
2230 : tree op_type)
2231 : {
2232 21992 : tree opnd1 = src_val->value;
2233 :
2234 : /* Skip source values that is incompatible with specified type. */
2235 21992 : if (opnd1_type
2236 21992 : && !useless_type_conversion_p (opnd1_type, TREE_TYPE (opnd1)))
2237 : return NULL_TREE;
2238 :
2239 21992 : return ipa_get_jf_arith_result (opcode, opnd1, opnd2, op_type);
2240 : }
2241 :
2242 : /* Propagate values through an arithmetic transformation described by a jump
2243 : function associated with edge CS, taking values from SRC_LAT and putting
2244 : them into DEST_LAT. OPND1_TYPE, if non-NULL, is the expected type for the
2245 : values in SRC_LAT. OPND2 is a constant value if transformation is a binary
2246 : operation. SRC_OFFSET specifies offset in an aggregate if SRC_LAT describes
2247 : lattice of a part of an aggregate, otherwise it should be -1. SRC_IDX is
2248 : the index of the source parameter. OP_TYPE is the type in which the
2249 : operation is performed and can be NULL when OPCODE is NOP_EXPR. RES_TYPE is
2250 : the value type of result being propagated into. Return true if DEST_LAT
2251 : changed. */
2252 :
2253 : static bool
2254 77942 : propagate_vals_across_arith_jfunc (cgraph_edge *cs,
2255 : enum tree_code opcode,
2256 : tree opnd1_type,
2257 : tree opnd2,
2258 : ipcp_lattice<tree> *src_lat,
2259 : ipcp_lattice<tree> *dest_lat,
2260 : HOST_WIDE_INT src_offset,
2261 : int src_idx,
2262 : tree op_type,
2263 : tree res_type)
2264 : {
2265 77942 : ipcp_value<tree> *src_val;
2266 77942 : bool ret = false;
2267 :
2268 : /* Due to circular dependencies, propagating within an SCC through arithmetic
2269 : transformation would create infinite number of values. But for
2270 : self-feeding recursive function, we could allow propagation in a limited
2271 : count, and this can enable a simple kind of recursive function versioning.
2272 : For other scenario, we would just make lattices bottom. */
2273 77942 : if (opcode != NOP_EXPR && ipa_edge_within_scc (cs))
2274 : {
2275 2184 : int i;
2276 :
2277 2184 : int max_recursive_depth = opt_for_fn(cs->caller->decl,
2278 : param_ipa_cp_max_recursive_depth);
2279 2184 : if (src_lat != dest_lat || max_recursive_depth < 1)
2280 1666 : return dest_lat->set_contains_variable ();
2281 :
2282 : /* No benefit if recursive execution is in low probability. */
2283 1300 : if (cs->sreal_frequency () * 100
2284 2600 : <= ((sreal) 1) * opt_for_fn (cs->caller->decl,
2285 : param_ipa_cp_min_recursive_probability))
2286 89 : return dest_lat->set_contains_variable ();
2287 :
2288 1211 : auto_vec<ipcp_value<tree> *, 8> val_seeds;
2289 :
2290 2258 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2291 : {
2292 : /* Now we do not use self-recursively generated value as propagation
2293 : source, this is absolutely conservative, but could avoid explosion
2294 : of lattice's value space, especially when one recursive function
2295 : calls another recursive. */
2296 1740 : if (src_val->self_recursion_generated_p ())
2297 : {
2298 909 : ipcp_value_source<tree> *s;
2299 :
2300 : /* If the lattice has already been propagated for the call site,
2301 : no need to do that again. */
2302 1422 : for (s = src_val->sources; s; s = s->next)
2303 1206 : if (s->cs == cs)
2304 693 : return dest_lat->set_contains_variable ();
2305 : }
2306 : else
2307 831 : val_seeds.safe_push (src_val);
2308 : }
2309 :
2310 1036 : gcc_assert ((int) val_seeds.length () <= param_ipa_cp_value_list_size);
2311 :
2312 : /* Recursively generate lattice values with a limited count. */
2313 1354 : FOR_EACH_VEC_ELT (val_seeds, i, src_val)
2314 : {
2315 1416 : for (int j = 1; j < max_recursive_depth; j++)
2316 : {
2317 1261 : tree cstval = get_val_across_arith_op (opcode, opnd1_type, opnd2,
2318 : src_val, op_type);
2319 1261 : cstval = ipacp_value_safe_for_type (res_type, cstval);
2320 1261 : if (!cstval)
2321 : break;
2322 :
2323 1257 : ret |= dest_lat->add_value (cstval, cs, src_val, src_idx,
2324 : src_offset, &src_val, j);
2325 1257 : gcc_checking_assert (src_val);
2326 : }
2327 : }
2328 518 : ret |= dest_lat->set_contains_variable ();
2329 1211 : }
2330 : else
2331 96614 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2332 : {
2333 : /* Now we do not use self-recursively generated value as propagation
2334 : source, otherwise it is easy to make value space of normal lattice
2335 : overflow. */
2336 20856 : if (src_val->self_recursion_generated_p ())
2337 : {
2338 125 : ret |= dest_lat->set_contains_variable ();
2339 125 : continue;
2340 : }
2341 :
2342 20731 : tree cstval = get_val_across_arith_op (opcode, opnd1_type, opnd2,
2343 : src_val, op_type);
2344 20731 : cstval = ipacp_value_safe_for_type (res_type, cstval);
2345 20731 : if (cstval)
2346 20530 : ret |= dest_lat->add_value (cstval, cs, src_val, src_idx,
2347 : src_offset);
2348 : else
2349 201 : ret |= dest_lat->set_contains_variable ();
2350 : }
2351 :
2352 : return ret;
2353 : }
2354 :
2355 : /* Propagate values through a pass-through jump function JFUNC associated with
2356 : edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
2357 : is the index of the source parameter. PARM_TYPE is the type of the
2358 : parameter to which the result is passed. */
2359 :
2360 : static bool
2361 73157 : propagate_vals_across_pass_through (cgraph_edge *cs, ipa_jump_func *jfunc,
2362 : ipcp_lattice<tree> *src_lat,
2363 : ipcp_lattice<tree> *dest_lat, int src_idx,
2364 : tree parm_type)
2365 : {
2366 73157 : gcc_checking_assert (parm_type);
2367 73157 : enum tree_code opcode = ipa_get_jf_pass_through_operation (jfunc);
2368 73157 : tree op_type = (opcode == NOP_EXPR) ? NULL_TREE
2369 2375 : : ipa_get_jf_pass_through_op_type (jfunc);
2370 73157 : return propagate_vals_across_arith_jfunc (cs, opcode, NULL_TREE,
2371 : ipa_get_jf_pass_through_operand (jfunc),
2372 : src_lat, dest_lat, -1, src_idx, op_type,
2373 73157 : parm_type);
2374 : }
2375 :
2376 : /* Propagate values through an ancestor jump function JFUNC associated with
2377 : edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
2378 : is the index of the source parameter. */
2379 :
2380 : static bool
2381 2286 : propagate_vals_across_ancestor (struct cgraph_edge *cs,
2382 : struct ipa_jump_func *jfunc,
2383 : ipcp_lattice<tree> *src_lat,
2384 : ipcp_lattice<tree> *dest_lat, int src_idx,
2385 : tree param_type)
2386 : {
2387 2286 : ipcp_value<tree> *src_val;
2388 2286 : bool ret = false;
2389 :
2390 2286 : if (ipa_edge_within_scc (cs))
2391 14 : return dest_lat->set_contains_variable ();
2392 :
2393 2590 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2394 : {
2395 318 : tree t = ipa_get_jf_ancestor_result (jfunc, src_val->value);
2396 318 : t = ipacp_value_safe_for_type (param_type, t);
2397 318 : if (t)
2398 260 : ret |= dest_lat->add_value (t, cs, src_val, src_idx);
2399 : else
2400 58 : ret |= dest_lat->set_contains_variable ();
2401 : }
2402 :
2403 : return ret;
2404 : }
2405 :
2406 : /* Propagate scalar values across jump function JFUNC that is associated with
2407 : edge CS and put the values into DEST_LAT. PARM_TYPE is the type of the
2408 : parameter to which the result is passed. */
2409 :
2410 : static bool
2411 4013832 : propagate_scalar_across_jump_function (struct cgraph_edge *cs,
2412 : struct ipa_jump_func *jfunc,
2413 : ipcp_lattice<tree> *dest_lat,
2414 : tree param_type)
2415 : {
2416 4013832 : if (dest_lat->bottom)
2417 : return false;
2418 :
2419 830949 : if (jfunc->type == IPA_JF_CONST)
2420 : {
2421 371420 : tree val = ipa_get_jf_constant (jfunc);
2422 371420 : val = ipacp_value_safe_for_type (param_type, val);
2423 371420 : if (val)
2424 371402 : return dest_lat->add_value (val, cs, NULL, 0);
2425 : else
2426 18 : return dest_lat->set_contains_variable ();
2427 : }
2428 459529 : else if (jfunc->type == IPA_JF_PASS_THROUGH
2429 274586 : || jfunc->type == IPA_JF_ANCESTOR)
2430 : {
2431 189545 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2432 189545 : ipcp_lattice<tree> *src_lat;
2433 189545 : int src_idx;
2434 189545 : bool ret;
2435 :
2436 189545 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2437 184943 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2438 : else
2439 4602 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2440 :
2441 189545 : src_lat = ipa_get_scalar_lat (caller_info, src_idx);
2442 189545 : if (src_lat->bottom)
2443 113955 : return dest_lat->set_contains_variable ();
2444 :
2445 : /* If we would need to clone the caller and cannot, do not propagate. */
2446 75590 : if (!ipcp_versionable_function_p (cs->caller)
2447 75590 : && (src_lat->contains_variable
2448 134 : || (src_lat->values_count > 1)))
2449 147 : return dest_lat->set_contains_variable ();
2450 :
2451 75443 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2452 73157 : ret = propagate_vals_across_pass_through (cs, jfunc, src_lat,
2453 : dest_lat, src_idx,
2454 : param_type);
2455 : else
2456 2286 : ret = propagate_vals_across_ancestor (cs, jfunc, src_lat, dest_lat,
2457 : src_idx, param_type);
2458 :
2459 75443 : if (src_lat->contains_variable)
2460 65893 : ret |= dest_lat->set_contains_variable ();
2461 :
2462 : return ret;
2463 : }
2464 :
2465 : /* TODO: We currently do not handle member method pointers in IPA-CP (we only
2466 : use it for indirect inlining), we should propagate them too. */
2467 269984 : return dest_lat->set_contains_variable ();
2468 : }
2469 :
2470 : /* Propagate scalar values across jump function JFUNC that is associated with
2471 : edge CS and describes argument IDX and put the values into DEST_LAT. */
2472 :
2473 : static bool
2474 4013832 : propagate_context_across_jump_function (cgraph_edge *cs,
2475 : ipa_jump_func *jfunc, int idx,
2476 : ipcp_lattice<ipa_polymorphic_call_context> *dest_lat)
2477 : {
2478 4013832 : if (dest_lat->bottom)
2479 : return false;
2480 927444 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
2481 927444 : bool ret = false;
2482 927444 : bool added_sth = false;
2483 927444 : bool type_preserved = true;
2484 :
2485 927444 : ipa_polymorphic_call_context edge_ctx, *edge_ctx_ptr
2486 943160 : = ipa_get_ith_polymorhic_call_context (args, idx);
2487 :
2488 15716 : if (edge_ctx_ptr)
2489 15716 : edge_ctx = *edge_ctx_ptr;
2490 :
2491 927444 : if (jfunc->type == IPA_JF_PASS_THROUGH
2492 742010 : || jfunc->type == IPA_JF_ANCESTOR)
2493 : {
2494 190132 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2495 190132 : int src_idx;
2496 190132 : ipcp_lattice<ipa_polymorphic_call_context> *src_lat;
2497 :
2498 : /* TODO: Once we figure out how to propagate speculations, it will
2499 : probably be a good idea to switch to speculation if type_preserved is
2500 : not set instead of punting. */
2501 190132 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2502 : {
2503 185434 : if (ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR)
2504 7554 : goto prop_fail;
2505 177880 : type_preserved = ipa_get_jf_pass_through_type_preserved (jfunc);
2506 177880 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2507 : }
2508 : else
2509 : {
2510 4698 : type_preserved = ipa_get_jf_ancestor_type_preserved (jfunc);
2511 4698 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2512 : }
2513 :
2514 182578 : src_lat = ipa_get_poly_ctx_lat (caller_info, src_idx);
2515 : /* If we would need to clone the caller and cannot, do not propagate. */
2516 182578 : if (!ipcp_versionable_function_p (cs->caller)
2517 182578 : && (src_lat->contains_variable
2518 14320 : || (src_lat->values_count > 1)))
2519 2486 : goto prop_fail;
2520 :
2521 180092 : ipcp_value<ipa_polymorphic_call_context> *src_val;
2522 181407 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2523 : {
2524 1315 : ipa_polymorphic_call_context cur = src_val->value;
2525 :
2526 1315 : if (!type_preserved)
2527 885 : cur.possible_dynamic_type_change (cs->in_polymorphic_cdtor);
2528 1315 : if (jfunc->type == IPA_JF_ANCESTOR)
2529 329 : cur.offset_by (ipa_get_jf_ancestor_offset (jfunc));
2530 : /* TODO: In cases we know how the context is going to be used,
2531 : we can improve the result by passing proper OTR_TYPE. */
2532 1315 : cur.combine_with (edge_ctx);
2533 2630 : if (!cur.useless_p ())
2534 : {
2535 844 : if (src_lat->contains_variable
2536 844 : && !edge_ctx.equal_to (cur))
2537 263 : ret |= dest_lat->set_contains_variable ();
2538 844 : ret |= dest_lat->add_value (cur, cs, src_val, src_idx);
2539 844 : added_sth = true;
2540 : }
2541 : }
2542 : }
2543 :
2544 737312 : prop_fail:
2545 190132 : if (!added_sth)
2546 : {
2547 926664 : if (!edge_ctx.useless_p ())
2548 8812 : ret |= dest_lat->add_value (edge_ctx, cs);
2549 : else
2550 917852 : ret |= dest_lat->set_contains_variable ();
2551 : }
2552 :
2553 : return ret;
2554 : }
2555 :
2556 : /* Propagate bits across jfunc that is associated with
2557 : edge cs and update dest_lattice accordingly. */
2558 :
2559 : bool
2560 4013832 : propagate_bits_across_jump_function (cgraph_edge *cs, int idx,
2561 : ipa_jump_func *jfunc,
2562 : ipcp_bits_lattice *dest_lattice)
2563 : {
2564 4013832 : if (dest_lattice->bottom_p ())
2565 : return false;
2566 :
2567 539411 : enum availability availability;
2568 539411 : cgraph_node *callee = cs->callee->function_symbol (&availability);
2569 539411 : ipa_node_params *callee_info = ipa_node_params_sum->get (callee);
2570 539411 : tree parm_type = ipa_get_type (callee_info, idx);
2571 :
2572 : /* For K&R C programs, ipa_get_type() could return NULL_TREE. Avoid the
2573 : transform for these cases. Similarly, we can have bad type mismatches
2574 : with LTO, avoid doing anything with those too. */
2575 539411 : if (!parm_type
2576 539411 : || (!INTEGRAL_TYPE_P (parm_type) && !POINTER_TYPE_P (parm_type)))
2577 : {
2578 29563 : if (dump_file && (dump_flags & TDF_DETAILS))
2579 11 : fprintf (dump_file, "Setting dest_lattice to bottom, because type of "
2580 : "param %i of %s is NULL or unsuitable for bits propagation\n",
2581 11 : idx, cs->callee->dump_name ());
2582 :
2583 29563 : return dest_lattice->set_to_bottom ();
2584 : }
2585 :
2586 509848 : if (jfunc->type == IPA_JF_PASS_THROUGH
2587 408628 : || jfunc->type == IPA_JF_ANCESTOR)
2588 : {
2589 103796 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2590 103796 : tree operand = NULL_TREE;
2591 103796 : tree op_type = NULL_TREE;
2592 103796 : enum tree_code code;
2593 103796 : unsigned src_idx;
2594 103796 : bool keep_null = false;
2595 :
2596 103796 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2597 : {
2598 101220 : code = ipa_get_jf_pass_through_operation (jfunc);
2599 101220 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2600 101220 : if (code != NOP_EXPR)
2601 : {
2602 2025 : operand = ipa_get_jf_pass_through_operand (jfunc);
2603 2025 : op_type = ipa_get_jf_pass_through_op_type (jfunc);
2604 : }
2605 : }
2606 : else
2607 : {
2608 2576 : code = POINTER_PLUS_EXPR;
2609 2576 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2610 2576 : unsigned HOST_WIDE_INT offset
2611 2576 : = ipa_get_jf_ancestor_offset (jfunc) / BITS_PER_UNIT;
2612 2576 : keep_null = (ipa_get_jf_ancestor_keep_null (jfunc) || !offset);
2613 2576 : operand = build_int_cstu (size_type_node, offset);
2614 : }
2615 :
2616 103796 : class ipcp_param_lattices *src_lats
2617 103796 : = ipa_get_parm_lattices (caller_info, src_idx);
2618 :
2619 : /* Try to propagate bits if src_lattice is bottom, but jfunc is known.
2620 : for eg consider:
2621 : int f(int x)
2622 : {
2623 : g (x & 0xff);
2624 : }
2625 : Assume lattice for x is bottom, however we can still propagate
2626 : result of x & 0xff == 0xff, which gets computed during ccp1 pass
2627 : and we store it in jump function during analysis stage. */
2628 :
2629 103796 : if (!src_lats->bits_lattice.bottom_p ()
2630 103796 : && !src_lats->bits_lattice.recipient_only_p ())
2631 : {
2632 21744 : if (!op_type)
2633 20623 : op_type = ipa_get_type (caller_info, src_idx);
2634 :
2635 21744 : unsigned precision = TYPE_PRECISION (op_type);
2636 21744 : signop sgn = TYPE_SIGN (op_type);
2637 21744 : bool drop_all_ones
2638 21744 : = keep_null && !src_lats->bits_lattice.known_nonzero_p ();
2639 :
2640 21744 : return dest_lattice->meet_with (src_lats->bits_lattice, precision,
2641 21744 : sgn, code, operand, drop_all_ones);
2642 : }
2643 : }
2644 :
2645 488104 : value_range vr (parm_type);
2646 488104 : if (jfunc->m_vr)
2647 : {
2648 415515 : jfunc->m_vr->get_vrange (vr);
2649 415515 : if (!vr.undefined_p () && !vr.varying_p ())
2650 : {
2651 415515 : irange_bitmask bm = vr.get_bitmask ();
2652 415515 : widest_int mask
2653 415515 : = widest_int::from (bm.mask (), TYPE_SIGN (parm_type));
2654 415515 : widest_int value
2655 415515 : = widest_int::from (bm.value (), TYPE_SIGN (parm_type));
2656 415515 : return dest_lattice->meet_with (value, mask,
2657 415515 : TYPE_PRECISION (parm_type));
2658 415515 : }
2659 : }
2660 72589 : return dest_lattice->set_to_bottom ();
2661 488104 : }
2662 :
2663 : /* Propagate value range across jump function JFUNC that is associated with
2664 : edge CS with param of callee of PARAM_TYPE and update DEST_PLATS
2665 : accordingly. */
2666 :
2667 : static bool
2668 4012999 : propagate_vr_across_jump_function (cgraph_edge *cs, ipa_jump_func *jfunc,
2669 : class ipcp_param_lattices *dest_plats,
2670 : tree param_type)
2671 : {
2672 4012999 : ipcp_vr_lattice *dest_lat = &dest_plats->m_value_range;
2673 :
2674 4012999 : if (dest_lat->bottom_p ())
2675 : return false;
2676 :
2677 633061 : if (!param_type
2678 633061 : || !ipa_vr_supported_type_p (param_type))
2679 29503 : return dest_lat->set_to_bottom ();
2680 :
2681 603558 : value_range vr (param_type);
2682 603558 : vr.set_varying (param_type);
2683 603558 : if (jfunc->m_vr)
2684 519960 : ipa_vr_operation_and_type_effects (vr, *jfunc->m_vr, NOP_EXPR,
2685 : param_type,
2686 519960 : jfunc->m_vr->type ());
2687 :
2688 603558 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2689 : {
2690 95354 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2691 95354 : int src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2692 95354 : class ipcp_param_lattices *src_lats
2693 95354 : = ipa_get_parm_lattices (caller_info, src_idx);
2694 95354 : tree operand_type = ipa_get_type (caller_info, src_idx);
2695 :
2696 95354 : if (src_lats->m_value_range.bottom_p ()
2697 95354 : || src_lats->m_value_range.recipient_only_p ())
2698 79398 : return dest_lat->set_to_bottom ();
2699 :
2700 15956 : if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR
2701 15956 : || !ipa_edge_within_scc (cs))
2702 15461 : ipa_vr_intersect_with_arith_jfunc (vr, jfunc, cs->caller,
2703 15461 : src_lats->m_value_range.m_vr,
2704 : operand_type, param_type);
2705 : }
2706 :
2707 524160 : if (!vr.undefined_p () && !vr.varying_p ())
2708 495568 : return dest_lat->meet_with (vr);
2709 : else
2710 28592 : return dest_lat->set_to_bottom ();
2711 603558 : }
2712 :
2713 : /* If DEST_PLATS already has aggregate items, check that aggs_by_ref matches
2714 : NEW_AGGS_BY_REF and if not, mark all aggs as bottoms and return true (in all
2715 : other cases, return false). If there are no aggregate items, set
2716 : aggs_by_ref to NEW_AGGS_BY_REF. */
2717 :
2718 : static bool
2719 42591 : set_check_aggs_by_ref (class ipcp_param_lattices *dest_plats,
2720 : bool new_aggs_by_ref)
2721 : {
2722 0 : if (dest_plats->aggs)
2723 : {
2724 22558 : if (dest_plats->aggs_by_ref != new_aggs_by_ref)
2725 : {
2726 0 : set_agg_lats_to_bottom (dest_plats);
2727 0 : return true;
2728 : }
2729 : }
2730 : else
2731 0 : dest_plats->aggs_by_ref = new_aggs_by_ref;
2732 : return false;
2733 : }
2734 :
2735 : /* Walk aggregate lattices in DEST_PLATS from ***AGLAT on, until ***aglat is an
2736 : already existing lattice for the given OFFSET and SIZE, marking all skipped
2737 : lattices as containing variable and checking for overlaps. If there is no
2738 : already existing lattice for the OFFSET and VAL_SIZE, create one, initialize
2739 : it with offset, size and contains_variable to PRE_EXISTING, and return true,
2740 : unless there are too many already. If there are two many, return false. If
2741 : there are overlaps turn whole DEST_PLATS to bottom and return false. If any
2742 : skipped lattices were newly marked as containing variable, set *CHANGE to
2743 : true. MAX_AGG_ITEMS is the maximum number of lattices. */
2744 :
2745 : static bool
2746 116454 : merge_agg_lats_step (class ipcp_param_lattices *dest_plats,
2747 : HOST_WIDE_INT offset, HOST_WIDE_INT val_size,
2748 : struct ipcp_agg_lattice ***aglat,
2749 : bool pre_existing, bool *change, int max_agg_items)
2750 : {
2751 116454 : gcc_checking_assert (offset >= 0);
2752 :
2753 120955 : while (**aglat && (**aglat)->offset < offset)
2754 : {
2755 4501 : if ((**aglat)->offset + (**aglat)->size > offset)
2756 : {
2757 0 : set_agg_lats_to_bottom (dest_plats);
2758 0 : return false;
2759 : }
2760 4501 : *change |= (**aglat)->set_contains_variable ();
2761 4501 : *aglat = &(**aglat)->next;
2762 : }
2763 :
2764 116454 : if (**aglat && (**aglat)->offset == offset)
2765 : {
2766 57641 : if ((**aglat)->size != val_size)
2767 : {
2768 13 : set_agg_lats_to_bottom (dest_plats);
2769 13 : return false;
2770 : }
2771 57628 : gcc_assert (!(**aglat)->next
2772 : || (**aglat)->next->offset >= offset + val_size);
2773 : return true;
2774 : }
2775 : else
2776 : {
2777 58813 : struct ipcp_agg_lattice *new_al;
2778 :
2779 58813 : if (**aglat && (**aglat)->offset < offset + val_size)
2780 : {
2781 3 : set_agg_lats_to_bottom (dest_plats);
2782 3 : return false;
2783 : }
2784 58810 : if (dest_plats->aggs_count == max_agg_items)
2785 : return false;
2786 58771 : dest_plats->aggs_count++;
2787 58771 : new_al = ipcp_agg_lattice_pool.allocate ();
2788 :
2789 58771 : new_al->offset = offset;
2790 58771 : new_al->size = val_size;
2791 58771 : new_al->contains_variable = pre_existing;
2792 :
2793 58771 : new_al->next = **aglat;
2794 58771 : **aglat = new_al;
2795 58771 : return true;
2796 : }
2797 : }
2798 :
2799 : /* Set all AGLAT and all other aggregate lattices reachable by next pointers as
2800 : containing an unknown value. */
2801 :
2802 : static bool
2803 42573 : set_chain_of_aglats_contains_variable (struct ipcp_agg_lattice *aglat)
2804 : {
2805 42573 : bool ret = false;
2806 45137 : while (aglat)
2807 : {
2808 2564 : ret |= aglat->set_contains_variable ();
2809 2564 : aglat = aglat->next;
2810 : }
2811 42573 : return ret;
2812 : }
2813 :
2814 : /* Merge existing aggregate lattices in SRC_PLATS to DEST_PLATS, subtracting
2815 : DELTA_OFFSET. CS is the call graph edge and SRC_IDX the index of the source
2816 : parameter used for lattice value sources. Return true if DEST_PLATS changed
2817 : in any way. */
2818 :
2819 : static bool
2820 3932 : merge_aggregate_lattices (struct cgraph_edge *cs,
2821 : class ipcp_param_lattices *dest_plats,
2822 : class ipcp_param_lattices *src_plats,
2823 : int src_idx, HOST_WIDE_INT offset_delta)
2824 : {
2825 3932 : bool pre_existing = dest_plats->aggs != NULL;
2826 3932 : struct ipcp_agg_lattice **dst_aglat;
2827 3932 : bool ret = false;
2828 :
2829 3932 : if (set_check_aggs_by_ref (dest_plats, src_plats->aggs_by_ref))
2830 0 : return true;
2831 3932 : if (src_plats->aggs_bottom)
2832 2 : return set_agg_lats_contain_variable (dest_plats);
2833 3930 : if (src_plats->aggs_contain_variable)
2834 2313 : ret |= set_agg_lats_contain_variable (dest_plats);
2835 3930 : dst_aglat = &dest_plats->aggs;
2836 :
2837 3930 : int max_agg_items = opt_for_fn (cs->callee->function_symbol ()->decl,
2838 : param_ipa_max_agg_items);
2839 3930 : for (struct ipcp_agg_lattice *src_aglat = src_plats->aggs;
2840 11598 : src_aglat;
2841 7668 : src_aglat = src_aglat->next)
2842 : {
2843 7668 : HOST_WIDE_INT new_offset = src_aglat->offset - offset_delta;
2844 :
2845 7668 : if (new_offset < 0)
2846 51 : continue;
2847 7617 : if (merge_agg_lats_step (dest_plats, new_offset, src_aglat->size,
2848 : &dst_aglat, pre_existing, &ret, max_agg_items))
2849 : {
2850 7613 : struct ipcp_agg_lattice *new_al = *dst_aglat;
2851 :
2852 7613 : dst_aglat = &(*dst_aglat)->next;
2853 7613 : if (src_aglat->bottom)
2854 : {
2855 0 : ret |= new_al->set_contains_variable ();
2856 0 : continue;
2857 : }
2858 7613 : if (src_aglat->contains_variable)
2859 4486 : ret |= new_al->set_contains_variable ();
2860 7613 : for (ipcp_value<tree> *val = src_aglat->values;
2861 11805 : val;
2862 4192 : val = val->next)
2863 4192 : ret |= new_al->add_value (val->value, cs, val, src_idx,
2864 : src_aglat->offset);
2865 : }
2866 4 : else if (dest_plats->aggs_bottom)
2867 : return true;
2868 : }
2869 3930 : ret |= set_chain_of_aglats_contains_variable (*dst_aglat);
2870 3930 : return ret;
2871 : }
2872 :
2873 : /* Determine whether there is anything to propagate FROM SRC_PLATS through a
2874 : pass-through JFUNC and if so, whether it has conform and conforms to the
2875 : rules about propagating values passed by reference. */
2876 :
2877 : static bool
2878 177719 : agg_pass_through_permissible_p (class ipcp_param_lattices *src_plats,
2879 : struct ipa_jump_func *jfunc)
2880 : {
2881 177719 : return src_plats->aggs
2882 177719 : && (!src_plats->aggs_by_ref
2883 5098 : || ipa_get_jf_pass_through_agg_preserved (jfunc));
2884 : }
2885 :
2886 : /* Propagate values through ITEM, jump function for a part of an aggregate,
2887 : into corresponding aggregate lattice AGLAT. CS is the call graph edge
2888 : associated with the jump function. Return true if AGLAT changed in any
2889 : way. */
2890 :
2891 : static bool
2892 108786 : propagate_aggregate_lattice (struct cgraph_edge *cs,
2893 : struct ipa_agg_jf_item *item,
2894 : struct ipcp_agg_lattice *aglat)
2895 : {
2896 108786 : class ipa_node_params *caller_info;
2897 108786 : class ipcp_param_lattices *src_plats;
2898 108786 : struct ipcp_lattice<tree> *src_lat;
2899 108786 : HOST_WIDE_INT src_offset;
2900 108786 : int src_idx;
2901 108786 : tree load_type;
2902 108786 : bool ret;
2903 :
2904 108786 : if (item->jftype == IPA_JF_CONST)
2905 : {
2906 96956 : tree value = item->value.constant;
2907 :
2908 96956 : gcc_checking_assert (is_gimple_ip_invariant (value));
2909 96956 : return aglat->add_value (value, cs, NULL, 0);
2910 : }
2911 :
2912 11830 : gcc_checking_assert (item->jftype == IPA_JF_PASS_THROUGH
2913 : || item->jftype == IPA_JF_LOAD_AGG);
2914 :
2915 11830 : caller_info = ipa_node_params_sum->get (cs->caller);
2916 11830 : src_idx = item->value.pass_through.formal_id;
2917 11830 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
2918 :
2919 11830 : if (item->jftype == IPA_JF_PASS_THROUGH)
2920 : {
2921 3563 : load_type = NULL_TREE;
2922 3563 : src_lat = &src_plats->itself;
2923 3563 : src_offset = -1;
2924 : }
2925 : else
2926 : {
2927 8267 : HOST_WIDE_INT load_offset = item->value.load_agg.offset;
2928 8267 : struct ipcp_agg_lattice *src_aglat;
2929 :
2930 12983 : for (src_aglat = src_plats->aggs; src_aglat; src_aglat = src_aglat->next)
2931 8558 : if (src_aglat->offset >= load_offset)
2932 : break;
2933 :
2934 8267 : load_type = item->value.load_agg.type;
2935 8267 : if (!src_aglat
2936 3842 : || src_aglat->offset > load_offset
2937 3490 : || src_aglat->size != tree_to_shwi (TYPE_SIZE (load_type))
2938 11757 : || src_plats->aggs_by_ref != item->value.load_agg.by_ref)
2939 4777 : return aglat->set_contains_variable ();
2940 :
2941 : src_lat = src_aglat;
2942 : src_offset = load_offset;
2943 : }
2944 :
2945 7053 : if (src_lat->bottom
2946 7053 : || (!ipcp_versionable_function_p (cs->caller)
2947 7053 : && !src_lat->is_single_const ()))
2948 2268 : return aglat->set_contains_variable ();
2949 :
2950 4785 : ret = propagate_vals_across_arith_jfunc (cs,
2951 : item->value.pass_through.operation,
2952 : load_type,
2953 : item->value.pass_through.operand,
2954 : src_lat, aglat,
2955 : src_offset,
2956 : src_idx,
2957 : item->value.pass_through.op_type,
2958 : item->type);
2959 :
2960 4785 : if (src_lat->contains_variable)
2961 2773 : ret |= aglat->set_contains_variable ();
2962 :
2963 : return ret;
2964 : }
2965 :
2966 : /* Propagate scalar values across jump function JFUNC that is associated with
2967 : edge CS and put the values into DEST_LAT. */
2968 :
2969 : static bool
2970 4013832 : propagate_aggs_across_jump_function (struct cgraph_edge *cs,
2971 : struct ipa_jump_func *jfunc,
2972 : class ipcp_param_lattices *dest_plats)
2973 : {
2974 4013832 : bool ret = false;
2975 :
2976 4013832 : if (dest_plats->aggs_bottom)
2977 : return false;
2978 :
2979 926212 : if (jfunc->type == IPA_JF_PASS_THROUGH
2980 926212 : && ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
2981 : {
2982 177719 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2983 177719 : int src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2984 177719 : class ipcp_param_lattices *src_plats;
2985 :
2986 177719 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
2987 177719 : if (agg_pass_through_permissible_p (src_plats, jfunc))
2988 : {
2989 : /* Currently we do not produce clobber aggregate jump
2990 : functions, replace with merging when we do. */
2991 3802 : gcc_assert (!jfunc->agg.items);
2992 3802 : ret |= merge_aggregate_lattices (cs, dest_plats, src_plats,
2993 : src_idx, 0);
2994 3802 : return ret;
2995 : }
2996 : }
2997 748493 : else if (jfunc->type == IPA_JF_ANCESTOR
2998 748493 : && ipa_get_jf_ancestor_agg_preserved (jfunc))
2999 : {
3000 1261 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
3001 1261 : int src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
3002 1261 : class ipcp_param_lattices *src_plats;
3003 :
3004 1261 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
3005 1261 : if (src_plats->aggs && src_plats->aggs_by_ref)
3006 : {
3007 : /* Currently we do not produce clobber aggregate jump
3008 : functions, replace with merging when we do. */
3009 130 : gcc_assert (!jfunc->agg.items);
3010 130 : ret |= merge_aggregate_lattices (cs, dest_plats, src_plats, src_idx,
3011 : ipa_get_jf_ancestor_offset (jfunc));
3012 : }
3013 1131 : else if (!src_plats->aggs_by_ref)
3014 1127 : ret |= set_agg_lats_to_bottom (dest_plats);
3015 : else
3016 4 : ret |= set_agg_lats_contain_variable (dest_plats);
3017 1261 : return ret;
3018 : }
3019 :
3020 921149 : if (jfunc->agg.items)
3021 : {
3022 38659 : bool pre_existing = dest_plats->aggs != NULL;
3023 38659 : struct ipcp_agg_lattice **aglat = &dest_plats->aggs;
3024 38659 : struct ipa_agg_jf_item *item;
3025 38659 : int i;
3026 :
3027 38659 : if (set_check_aggs_by_ref (dest_plats, jfunc->agg.by_ref))
3028 16 : return true;
3029 :
3030 38659 : int max_agg_items = opt_for_fn (cs->callee->function_symbol ()->decl,
3031 : param_ipa_max_agg_items);
3032 147801 : FOR_EACH_VEC_ELT (*jfunc->agg.items, i, item)
3033 : {
3034 109158 : HOST_WIDE_INT val_size;
3035 :
3036 109158 : if (item->offset < 0 || item->jftype == IPA_JF_UNKNOWN)
3037 321 : continue;
3038 108837 : val_size = tree_to_shwi (TYPE_SIZE (item->type));
3039 :
3040 108837 : if (merge_agg_lats_step (dest_plats, item->offset, val_size,
3041 : &aglat, pre_existing, &ret, max_agg_items))
3042 : {
3043 108786 : ret |= propagate_aggregate_lattice (cs, item, *aglat);
3044 108786 : aglat = &(*aglat)->next;
3045 : }
3046 51 : else if (dest_plats->aggs_bottom)
3047 : return true;
3048 : }
3049 :
3050 77286 : ret |= set_chain_of_aglats_contains_variable (*aglat);
3051 : }
3052 : else
3053 882490 : ret |= set_agg_lats_contain_variable (dest_plats);
3054 :
3055 921133 : return ret;
3056 : }
3057 :
3058 : /* Return true if on the way cfrom CS->caller to the final (non-alias and
3059 : non-thunk) destination, the call passes through a thunk. */
3060 :
3061 : static bool
3062 1995771 : call_passes_through_thunk (cgraph_edge *cs)
3063 : {
3064 1995771 : cgraph_node *alias_or_thunk = cs->callee;
3065 2139725 : while (alias_or_thunk->alias)
3066 143954 : alias_or_thunk = alias_or_thunk->get_alias_target ();
3067 1995771 : return alias_or_thunk->thunk;
3068 : }
3069 :
3070 : /* Propagate constants from the caller to the callee of CS. INFO describes the
3071 : caller. */
3072 :
3073 : static bool
3074 5410734 : propagate_constants_across_call (struct cgraph_edge *cs)
3075 : {
3076 5410734 : class ipa_node_params *callee_info;
3077 5410734 : enum availability availability;
3078 5410734 : cgraph_node *callee;
3079 5410734 : class ipa_edge_args *args;
3080 5410734 : bool ret = false;
3081 5410734 : int i, args_count, parms_count;
3082 :
3083 5410734 : callee = cs->callee->function_symbol (&availability);
3084 5410734 : if (!callee->definition)
3085 : return false;
3086 2017684 : gcc_checking_assert (callee->has_gimple_body_p ());
3087 2017684 : callee_info = ipa_node_params_sum->get (callee);
3088 2017684 : if (!callee_info)
3089 : return false;
3090 :
3091 2009172 : args = ipa_edge_args_sum->get (cs);
3092 2009172 : parms_count = ipa_get_param_count (callee_info);
3093 1813771 : if (parms_count == 0)
3094 : return false;
3095 1813771 : if (!args
3096 1813484 : || !opt_for_fn (cs->caller->decl, flag_ipa_cp)
3097 3627255 : || !opt_for_fn (cs->caller->decl, optimize))
3098 : {
3099 864 : for (i = 0; i < parms_count; i++)
3100 577 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info,
3101 : i));
3102 : return ret;
3103 : }
3104 1813484 : args_count = ipa_get_cs_argument_count (args);
3105 :
3106 : /* If this call goes through a thunk we must not propagate to the first (0th)
3107 : parameter. However, we might need to uncover a thunk from below a series
3108 : of aliases first. */
3109 1813484 : if (call_passes_through_thunk (cs))
3110 : {
3111 227 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info,
3112 : 0));
3113 227 : i = 1;
3114 : }
3115 : else
3116 1813484 : i = 0;
3117 :
3118 5972935 : for (; (i < args_count) && (i < parms_count); i++)
3119 : {
3120 4159451 : struct ipa_jump_func *jump_func = ipa_get_ith_jump_func (args, i);
3121 4159451 : class ipcp_param_lattices *dest_plats;
3122 4159451 : tree param_type = ipa_get_type (callee_info, i);
3123 :
3124 4159451 : dest_plats = ipa_get_parm_lattices (callee_info, i);
3125 4159451 : if (availability == AVAIL_INTERPOSABLE)
3126 145619 : ret |= set_all_contains_variable (dest_plats);
3127 : else
3128 : {
3129 4013832 : ret |= propagate_scalar_across_jump_function (cs, jump_func,
3130 : &dest_plats->itself,
3131 : param_type);
3132 4013832 : ret |= propagate_context_across_jump_function (cs, jump_func, i,
3133 : &dest_plats->ctxlat);
3134 4013832 : ret
3135 4013832 : |= propagate_bits_across_jump_function (cs, i, jump_func,
3136 : &dest_plats->bits_lattice);
3137 4013832 : ret |= propagate_aggs_across_jump_function (cs, jump_func,
3138 : dest_plats);
3139 4013832 : if (opt_for_fn (callee->decl, flag_ipa_vrp))
3140 4012999 : ret |= propagate_vr_across_jump_function (cs, jump_func,
3141 : dest_plats, param_type);
3142 : else
3143 833 : ret |= dest_plats->m_value_range.set_to_bottom ();
3144 : }
3145 : }
3146 1813667 : for (; i < parms_count; i++)
3147 183 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info, i));
3148 :
3149 : return ret;
3150 : }
3151 :
3152 : /* If an indirect edge IE can be turned into a direct one based on KNOWN_VALS
3153 : KNOWN_CONTEXTS, and known aggregates either in AVS or KNOWN_AGGS return
3154 : the destination. The latter three can be NULL. If AGG_REPS is not NULL,
3155 : KNOWN_AGGS is ignored. */
3156 :
3157 : static tree
3158 1549948 : ipa_get_indirect_edge_target_1 (struct cgraph_edge *ie,
3159 : const vec<tree> &known_csts,
3160 : const vec<ipa_polymorphic_call_context> &known_contexts,
3161 : const ipa_argagg_value_list &avs,
3162 : bool *speculative)
3163 : {
3164 1549948 : int param_index = ie->indirect_info->param_index;
3165 1549948 : *speculative = false;
3166 :
3167 1549948 : if (param_index == -1)
3168 : return NULL_TREE;
3169 :
3170 611337 : if (cgraph_simple_indirect_info *sii
3171 611337 : = dyn_cast <cgraph_simple_indirect_info *> (ie->indirect_info))
3172 : {
3173 302198 : tree t = NULL;
3174 :
3175 302198 : if (sii->agg_contents)
3176 : {
3177 68887 : t = NULL;
3178 68887 : if ((unsigned) param_index < known_csts.length ()
3179 68887 : && known_csts[param_index])
3180 62798 : t = ipa_find_agg_cst_from_init (known_csts[param_index],
3181 : sii->offset,
3182 : sii->by_ref);
3183 :
3184 68887 : if (!t && sii->guaranteed_unmodified)
3185 61883 : t = avs.get_value (param_index, sii->offset / BITS_PER_UNIT,
3186 : sii->by_ref);
3187 : }
3188 233311 : else if ((unsigned) param_index < known_csts.length ())
3189 233311 : t = known_csts[param_index];
3190 :
3191 302143 : if (t
3192 205929 : && TREE_CODE (t) == ADDR_EXPR
3193 507857 : && TREE_CODE (TREE_OPERAND (t, 0)) == FUNCTION_DECL)
3194 205714 : return TREE_OPERAND (t, 0);
3195 : else
3196 : return NULL_TREE;
3197 : }
3198 :
3199 309139 : if (!opt_for_fn (ie->caller->decl, flag_devirtualize))
3200 : return NULL_TREE;
3201 :
3202 309139 : cgraph_polymorphic_indirect_info *pii
3203 309139 : = as_a <cgraph_polymorphic_indirect_info *> (ie->indirect_info);
3204 309139 : if (!pii->usable_p ())
3205 : return NULL_TREE;
3206 :
3207 309139 : HOST_WIDE_INT anc_offset = pii->offset;
3208 309139 : tree t = NULL;
3209 309139 : tree target = NULL;
3210 309139 : if ((unsigned) param_index < known_csts.length ()
3211 309139 : && known_csts[param_index])
3212 17876 : t = ipa_find_agg_cst_from_init (known_csts[param_index], anc_offset, true);
3213 :
3214 : /* Try to work out value of virtual table pointer value in replacements. */
3215 : /* or known aggregate values. */
3216 17876 : if (!t)
3217 309130 : t = avs.get_value (param_index, anc_offset / BITS_PER_UNIT, true);
3218 :
3219 : /* If we found the virtual table pointer, lookup the target. */
3220 309130 : if (t)
3221 : {
3222 7817 : tree vtable;
3223 7817 : unsigned HOST_WIDE_INT offset;
3224 7817 : if (vtable_pointer_value_to_vtable (t, &vtable, &offset))
3225 : {
3226 7817 : bool can_refer;
3227 7817 : target = gimple_get_virt_method_for_vtable (pii->otr_token, vtable,
3228 : offset, &can_refer);
3229 7817 : if (can_refer)
3230 : {
3231 7754 : if (!target
3232 7754 : || fndecl_built_in_p (target, BUILT_IN_UNREACHABLE)
3233 15388 : || !possible_polymorphic_call_target_p
3234 7634 : (ie, cgraph_node::get (target)))
3235 : {
3236 : /* Do not speculate builtin_unreachable, it is stupid! */
3237 237 : if (pii->vptr_changed)
3238 6277 : return NULL;
3239 237 : target = ipa_impossible_devirt_target (ie, target);
3240 : }
3241 7754 : *speculative = pii->vptr_changed;
3242 7754 : if (!*speculative)
3243 : return target;
3244 : }
3245 : }
3246 : }
3247 :
3248 : /* Do we know the constant value of pointer? */
3249 302862 : if (!t && (unsigned) param_index < known_csts.length ())
3250 44142 : t = known_csts[param_index];
3251 :
3252 302862 : ipa_polymorphic_call_context context;
3253 302862 : if (known_contexts.length () > (unsigned int) param_index)
3254 : {
3255 302488 : context = known_contexts[param_index];
3256 302488 : context.offset_by (anc_offset);
3257 302488 : if (pii->vptr_changed)
3258 47575 : context.possible_dynamic_type_change (ie->in_polymorphic_cdtor,
3259 : pii->otr_type);
3260 302488 : if (t)
3261 : {
3262 12249 : ipa_polymorphic_call_context ctx2
3263 12249 : = ipa_polymorphic_call_context (t, pii->otr_type, anc_offset);
3264 24498 : if (!ctx2.useless_p ())
3265 10714 : context.combine_with (ctx2, pii->otr_type);
3266 : }
3267 : }
3268 374 : else if (t)
3269 : {
3270 23 : context = ipa_polymorphic_call_context (t, pii->otr_type, anc_offset);
3271 23 : if (pii->vptr_changed)
3272 8 : context.possible_dynamic_type_change (ie->in_polymorphic_cdtor,
3273 : pii->otr_type);
3274 : }
3275 : else
3276 : return NULL_TREE;
3277 :
3278 302511 : vec <cgraph_node *>targets;
3279 302511 : bool final;
3280 :
3281 302511 : targets = possible_polymorphic_call_targets (pii->otr_type, pii->otr_token,
3282 : context, &final);
3283 314112 : if (!final || targets.length () > 1)
3284 : {
3285 291584 : struct cgraph_node *node;
3286 291584 : if (*speculative)
3287 : return target;
3288 291555 : if (!opt_for_fn (ie->caller->decl, flag_devirtualize_speculatively)
3289 291555 : || ie->speculative || !ie->maybe_hot_p ())
3290 : return NULL;
3291 87388 : node = try_speculative_devirtualization (pii->otr_type, pii->otr_token,
3292 : context);
3293 87388 : if (node)
3294 : {
3295 665 : *speculative = true;
3296 665 : target = node->decl;
3297 : }
3298 : else
3299 : return NULL;
3300 : }
3301 : else
3302 : {
3303 10927 : *speculative = false;
3304 10927 : if (targets.length () == 1)
3305 10888 : target = targets[0]->decl;
3306 : else
3307 39 : target = ipa_impossible_devirt_target (ie, NULL_TREE);
3308 : }
3309 :
3310 11592 : if (target && !possible_polymorphic_call_target_p (ie,
3311 : cgraph_node::get (target)))
3312 : {
3313 48 : if (*speculative)
3314 : return NULL;
3315 40 : target = ipa_impossible_devirt_target (ie, target);
3316 : }
3317 :
3318 : return target;
3319 : }
3320 :
3321 : /* If an indirect edge IE can be turned into a direct one based on data in
3322 : AVALS, return the destination. Store into *SPECULATIVE a boolean determinig
3323 : whether the discovered target is only speculative guess. */
3324 :
3325 : tree
3326 1472818 : ipa_get_indirect_edge_target (struct cgraph_edge *ie,
3327 : ipa_call_arg_values *avals,
3328 : bool *speculative)
3329 : {
3330 1472818 : ipa_argagg_value_list avl (avals);
3331 1472818 : return ipa_get_indirect_edge_target_1 (ie, avals->m_known_vals,
3332 1472818 : avals->m_known_contexts,
3333 1472818 : avl, speculative);
3334 : }
3335 :
3336 : /* Calculate devirtualization time bonus for NODE, assuming we know information
3337 : about arguments stored in AVALS.
3338 :
3339 : FIXME: This function will also consider devirtualization of calls that are
3340 : known to be dead in the clone. */
3341 :
3342 : static sreal
3343 450688 : devirtualization_time_bonus (struct cgraph_node *node,
3344 : ipa_auto_call_arg_values *avals)
3345 : {
3346 450688 : struct cgraph_edge *ie;
3347 450688 : sreal res = 0;
3348 :
3349 525849 : for (ie = node->indirect_calls; ie; ie = ie->next_callee)
3350 : {
3351 75161 : struct cgraph_node *callee;
3352 75161 : class ipa_fn_summary *isummary;
3353 75161 : enum availability avail;
3354 75161 : tree target;
3355 75161 : bool speculative;
3356 :
3357 75161 : ipa_argagg_value_list avl (avals);
3358 75161 : target = ipa_get_indirect_edge_target_1 (ie, avals->m_known_vals,
3359 : avals->m_known_contexts,
3360 : avl, &speculative);
3361 75161 : if (!target)
3362 74175 : continue;
3363 :
3364 : /* Only bare minimum benefit for clearly un-inlineable targets. */
3365 3237 : res = res + ie->combined_sreal_frequency ();
3366 3237 : callee = cgraph_node::get (target);
3367 3237 : if (!callee || !callee->definition)
3368 624 : continue;
3369 2613 : callee = callee->function_symbol (&avail);
3370 2613 : if (avail < AVAIL_AVAILABLE)
3371 0 : continue;
3372 2613 : isummary = ipa_fn_summaries->get (callee);
3373 2613 : if (!isummary || !isummary->inlinable)
3374 66 : continue;
3375 :
3376 2547 : int savings = 0;
3377 2547 : int size = ipa_size_summaries->get (callee)->size;
3378 : /* FIXME: The values below need re-considering and perhaps also
3379 : integrating into the cost metrics, at lest in some very basic way. */
3380 2547 : int max_inline_insns_auto
3381 2547 : = opt_for_fn (callee->decl, param_max_inline_insns_auto);
3382 2547 : if (size <= max_inline_insns_auto / 4)
3383 403 : savings = 31 / ((int)speculative + 1);
3384 2144 : else if (size <= max_inline_insns_auto / 2)
3385 392 : savings = 15 / ((int)speculative + 1);
3386 3313 : else if (size <= max_inline_insns_auto
3387 1752 : || DECL_DECLARED_INLINE_P (callee->decl))
3388 191 : savings = 7 / ((int)speculative + 1);
3389 : else
3390 1561 : continue;
3391 986 : res = res + ie->combined_sreal_frequency () * (sreal) savings;
3392 : }
3393 :
3394 450688 : return res;
3395 : }
3396 :
3397 : /* Return time bonus incurred because of hints stored in ESTIMATES. */
3398 :
3399 : static sreal
3400 229606 : hint_time_bonus (cgraph_node *node, const ipa_call_estimates &estimates)
3401 : {
3402 229606 : sreal result = 0;
3403 229606 : ipa_hints hints = estimates.hints;
3404 229606 : if (hints & (INLINE_HINT_loop_iterations | INLINE_HINT_loop_stride))
3405 25621 : result += opt_for_fn (node->decl, param_ipa_cp_loop_hint_bonus);
3406 :
3407 229606 : sreal bonus_for_one = opt_for_fn (node->decl, param_ipa_cp_loop_hint_bonus);
3408 :
3409 229606 : if (hints & INLINE_HINT_loop_iterations)
3410 17334 : result += estimates.loops_with_known_iterations * bonus_for_one;
3411 :
3412 229606 : if (hints & INLINE_HINT_loop_stride)
3413 10734 : result += estimates.loops_with_known_strides * bonus_for_one;
3414 :
3415 229606 : return result;
3416 : }
3417 :
3418 : /* If there is a reason to penalize the function described by INFO in the
3419 : cloning goodness evaluation, do so. */
3420 :
3421 : static inline sreal
3422 103791 : incorporate_penalties (cgraph_node *node, ipa_node_params *info,
3423 : sreal evaluation)
3424 : {
3425 103791 : if (info->node_within_scc && !info->node_is_self_scc)
3426 1710 : evaluation = (evaluation
3427 1710 : * (100 - opt_for_fn (node->decl,
3428 3420 : param_ipa_cp_recursion_penalty))) / 100;
3429 :
3430 103791 : if (info->node_calling_single_call)
3431 7067 : evaluation = (evaluation
3432 7067 : * (100 - opt_for_fn (node->decl,
3433 7067 : param_ipa_cp_single_call_penalty)))
3434 14134 : / 100;
3435 :
3436 103791 : return evaluation;
3437 : }
3438 :
3439 : /* Return true if cloning NODE is a good idea, given the estimated TIME_BENEFIT
3440 : and SIZE_COST and with the sum of frequencies of incoming edges to the
3441 : potential new clone in FREQUENCIES. CUR_SWEEP is the number of the current
3442 : sweep of IPA-CP over the call-graph in the decision stage. */
3443 :
3444 : static bool
3445 303049 : good_cloning_opportunity_p (struct cgraph_node *node, sreal time_benefit,
3446 : sreal freq_sum, profile_count count_sum,
3447 : int size_cost, bool called_without_ipa_profile,
3448 : int cur_sweep)
3449 : {
3450 303049 : gcc_assert (count_sum.ipa () == count_sum);
3451 303049 : if (count_sum.quality () == AFDO)
3452 0 : count_sum = count_sum.force_nonzero ();
3453 502307 : if (time_benefit == 0
3454 249146 : || !opt_for_fn (node->decl, flag_ipa_cp_clone)
3455 : /* If there is no call which was executed in profiling or where
3456 : profile is missing, we do not want to clone. */
3457 103791 : || (!called_without_ipa_profile && !count_sum.nonzero_p ()))
3458 : {
3459 199258 : if (dump_file && (dump_flags & TDF_DETAILS))
3460 24 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3461 : "size: %i): Definitely not good or prohibited.\n",
3462 : time_benefit.to_double (), size_cost);
3463 : return false;
3464 : }
3465 :
3466 103791 : gcc_assert (size_cost > 0);
3467 :
3468 103791 : ipa_node_params *info = ipa_node_params_sum->get (node);
3469 103791 : int num_sweeps = opt_for_fn (node->decl, param_ipa_cp_sweeps);
3470 103791 : int eval_threshold = opt_for_fn (node->decl, param_ipa_cp_eval_threshold);
3471 103791 : eval_threshold = (eval_threshold * num_sweeps) / cur_sweep;
3472 : /* If we know the execution IPA execution counts, we can estimate overall
3473 : speedup of the program. */
3474 103791 : if (count_sum.nonzero_p ())
3475 : {
3476 371 : profile_count saved_time = count_sum * time_benefit;
3477 371 : sreal evaluation = saved_time.to_sreal_scale (profile_count::one ())
3478 742 : / size_cost;
3479 371 : evaluation = incorporate_penalties (node, info, evaluation);
3480 :
3481 371 : if (dump_file && (dump_flags & TDF_DETAILS))
3482 : {
3483 0 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3484 : "size: %i, count_sum: ", time_benefit.to_double (),
3485 : size_cost);
3486 0 : count_sum.dump (dump_file);
3487 0 : fprintf (dump_file, ", overall time saved: ");
3488 0 : saved_time.dump (dump_file);
3489 0 : fprintf (dump_file, "%s%s) -> evaluation: %.2f, threshold: %i\n",
3490 0 : info->node_within_scc
3491 0 : ? (info->node_is_self_scc ? ", self_scc" : ", scc") : "",
3492 0 : info->node_calling_single_call ? ", single_call" : "",
3493 : evaluation.to_double (), eval_threshold);
3494 : }
3495 371 : gcc_checking_assert (saved_time == saved_time.ipa ());
3496 371 : if (!maybe_hot_count_p (NULL, saved_time))
3497 : {
3498 24 : if (dump_file && (dump_flags & TDF_DETAILS))
3499 0 : fprintf (dump_file, " not cloning: time saved is not hot\n");
3500 : }
3501 : /* Evaluation approximately corresponds to time saved per instruction
3502 : introduced. This is likely almost always going to be true, since we
3503 : already checked that time saved is large enough to be considered
3504 : hot. */
3505 347 : else if (evaluation >= (sreal)eval_threshold)
3506 371 : return true;
3507 : /* If all call sites have profile known; we know we do not want t clone.
3508 : If there are calls with unknown profile; try local heuristics. */
3509 359 : if (!called_without_ipa_profile)
3510 : return false;
3511 : }
3512 103420 : sreal evaluation = (time_benefit * freq_sum) / size_cost;
3513 103420 : evaluation = incorporate_penalties (node, info, evaluation);
3514 103420 : evaluation *= 1000;
3515 :
3516 103420 : if (dump_file && (dump_flags & TDF_DETAILS))
3517 358 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3518 : "size: %i, freq_sum: %g%s%s) -> evaluation: %.2f, "
3519 : "threshold: %i\n",
3520 : time_benefit.to_double (), size_cost, freq_sum.to_double (),
3521 179 : info->node_within_scc
3522 26 : ? (info->node_is_self_scc ? ", self_scc" : ", scc") : "",
3523 179 : info->node_calling_single_call ? ", single_call" : "",
3524 : evaluation.to_double (), eval_threshold);
3525 :
3526 103420 : return evaluation >= eval_threshold;
3527 : }
3528 :
3529 : /* Grow vectors in AVALS and fill them with information about values of
3530 : parameters that are known to be independent of the context. INFO describes
3531 : the function. If REMOVABLE_PARAMS_COST is non-NULL, the movement cost of
3532 : all removable parameters will be stored in it.
3533 :
3534 : TODO: Also grow context independent value range vectors. */
3535 :
3536 : static bool
3537 1121521 : gather_context_independent_values (class ipa_node_params *info,
3538 : ipa_auto_call_arg_values *avals,
3539 : int *removable_params_cost)
3540 : {
3541 1121521 : int i, count = ipa_get_param_count (info);
3542 1121521 : bool ret = false;
3543 :
3544 1121521 : avals->m_known_vals.safe_grow_cleared (count, true);
3545 1121521 : avals->m_known_contexts.safe_grow_cleared (count, true);
3546 :
3547 1121521 : if (removable_params_cost)
3548 1121521 : *removable_params_cost = 0;
3549 :
3550 3707785 : for (i = 0; i < count; i++)
3551 : {
3552 2586264 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3553 2586264 : ipcp_lattice<tree> *lat = &plats->itself;
3554 :
3555 2586264 : if (lat->is_single_const ())
3556 : {
3557 34969 : ipcp_value<tree> *val = lat->values;
3558 34969 : gcc_checking_assert (TREE_CODE (val->value) != TREE_BINFO);
3559 34969 : avals->m_known_vals[i] = val->value;
3560 34969 : if (removable_params_cost)
3561 69938 : *removable_params_cost
3562 34969 : += estimate_move_cost (TREE_TYPE (val->value), false);
3563 : ret = true;
3564 : }
3565 2551295 : else if (removable_params_cost
3566 2551295 : && !ipa_is_param_used (info, i))
3567 493638 : *removable_params_cost
3568 246819 : += ipa_get_param_move_cost (info, i);
3569 :
3570 2586264 : if (!ipa_is_param_used (info, i))
3571 252086 : continue;
3572 :
3573 2334178 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3574 : /* Do not account known context as reason for cloning. We can see
3575 : if it permits devirtualization. */
3576 2334178 : if (ctxlat->is_single_const ())
3577 24261 : avals->m_known_contexts[i] = ctxlat->values->value;
3578 :
3579 2334178 : ret |= push_agg_values_from_plats (plats, i, 0, &avals->m_known_aggs);
3580 : }
3581 :
3582 1121521 : return ret;
3583 : }
3584 :
3585 : /* Perform time and size measurement of NODE with the context given in AVALS,
3586 : calculate the benefit compared to the node without specialization and store
3587 : it into VAL. Take into account REMOVABLE_PARAMS_COST of all
3588 : context-independent or unused removable parameters and EST_MOVE_COST, the
3589 : estimated movement of the considered parameter. */
3590 :
3591 : static void
3592 80239 : perform_estimation_of_a_value (cgraph_node *node,
3593 : ipa_auto_call_arg_values *avals,
3594 : int removable_params_cost, int est_move_cost,
3595 : ipcp_value_base *val)
3596 : {
3597 80239 : sreal time_benefit;
3598 80239 : ipa_call_estimates estimates;
3599 :
3600 80239 : estimate_ipcp_clone_size_and_time (node, avals, &estimates);
3601 :
3602 : /* Extern inline functions have no cloning local time benefits because they
3603 : will be inlined anyway. The only reason to clone them is if it enables
3604 : optimization in any of the functions they call. */
3605 80239 : if (DECL_EXTERNAL (node->decl) && DECL_DECLARED_INLINE_P (node->decl))
3606 114 : time_benefit = 0;
3607 : else
3608 80125 : time_benefit = (estimates.nonspecialized_time - estimates.time)
3609 160250 : + hint_time_bonus (node, estimates)
3610 160250 : + (devirtualization_time_bonus (node, avals)
3611 160250 : + removable_params_cost + est_move_cost);
3612 :
3613 80239 : int size = estimates.size;
3614 80239 : gcc_checking_assert (size >=0);
3615 : /* The inliner-heuristics based estimates may think that in certain
3616 : contexts some functions do not have any size at all but we want
3617 : all specializations to have at least a tiny cost, not least not to
3618 : divide by zero. */
3619 80239 : if (size == 0)
3620 0 : size = 1;
3621 :
3622 80239 : val->local_time_benefit = time_benefit;
3623 80239 : val->local_size_cost = size;
3624 80239 : }
3625 :
3626 : /* Get the overall limit of growth based on parameters extracted from NODE. It
3627 : does not really make sense to mix functions with different overall growth
3628 : limits or even number of sweeps but it is possible and if it happens, we do
3629 : not want to select one limit at random, so get the limits from NODE. */
3630 :
3631 : static long
3632 218219 : get_max_overall_size (cgraph_node *node)
3633 : {
3634 218219 : long max_new_size = orig_overall_size;
3635 218219 : long large_unit = opt_for_fn (node->decl, param_ipa_cp_large_unit_insns);
3636 218219 : if (max_new_size < large_unit)
3637 : max_new_size = large_unit;
3638 218219 : int unit_growth = opt_for_fn (node->decl, param_ipa_cp_unit_growth);
3639 218219 : max_new_size += max_new_size * unit_growth / 100 + 1;
3640 :
3641 218219 : return max_new_size;
3642 : }
3643 :
3644 : /* Return true if NODE should be cloned just for a parameter removal, possibly
3645 : dumping a reason if not. */
3646 :
3647 : static bool
3648 8639 : clone_for_param_removal_p (cgraph_node *node)
3649 : {
3650 8639 : if (!node->can_change_signature)
3651 : {
3652 1574 : if (dump_file && (dump_flags & TDF_DETAILS))
3653 0 : fprintf (dump_file, " Not considering cloning to remove parameters, "
3654 : "function cannot change signature.\n");
3655 : return false;
3656 : }
3657 7065 : if (node->can_be_local_p ())
3658 : {
3659 7065 : if (dump_file && (dump_flags & TDF_DETAILS))
3660 0 : fprintf (dump_file, " Not considering cloning to remove parameters, "
3661 : "IPA-SRA can do it potentially better.\n");
3662 : return false;
3663 : }
3664 : return true;
3665 : }
3666 :
3667 : /* Iterate over known values of parameters of NODE and estimate the local
3668 : effects in terms of time and size they have. */
3669 :
3670 : static void
3671 1303950 : estimate_local_effects (struct cgraph_node *node)
3672 : {
3673 1303950 : ipa_node_params *info = ipa_node_params_sum->get (node);
3674 1303950 : int count = ipa_get_param_count (info);
3675 1070583 : int removable_params_cost;
3676 :
3677 1070583 : if (!count || !ipcp_versionable_function_p (node))
3678 403511 : return;
3679 :
3680 900439 : if (dump_file && (dump_flags & TDF_DETAILS))
3681 117 : fprintf (dump_file, "\nEstimating effects for %s.\n", node->dump_name ());
3682 :
3683 900439 : ipa_auto_call_arg_values avals;
3684 900439 : gather_context_independent_values (info, &avals, &removable_params_cost);
3685 :
3686 3905590 : for (int i = 0; i < count; i++)
3687 : {
3688 2104712 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3689 2104712 : ipcp_lattice<tree> *lat = &plats->itself;
3690 2104712 : ipcp_value<tree> *val;
3691 :
3692 4187724 : if (lat->bottom
3693 217536 : || !lat->values
3694 2143978 : || avals.m_known_vals[i])
3695 2083012 : continue;
3696 :
3697 66320 : for (val = lat->values; val; val = val->next)
3698 : {
3699 44620 : gcc_checking_assert (TREE_CODE (val->value) != TREE_BINFO);
3700 44620 : avals.m_known_vals[i] = val->value;
3701 :
3702 44620 : int emc = estimate_move_cost (TREE_TYPE (val->value), true);
3703 44620 : perform_estimation_of_a_value (node, &avals, removable_params_cost,
3704 : emc, val);
3705 :
3706 44620 : if (dump_file && (dump_flags & TDF_DETAILS))
3707 : {
3708 44 : fprintf (dump_file, " - estimates for value ");
3709 44 : print_ipcp_constant_value (dump_file, val->value);
3710 44 : fprintf (dump_file, " for ");
3711 44 : ipa_dump_param (dump_file, info, i);
3712 44 : fprintf (dump_file, ": time_benefit: %g, size: %i\n",
3713 : val->local_time_benefit.to_double (),
3714 : val->local_size_cost);
3715 : }
3716 : }
3717 21700 : avals.m_known_vals[i] = NULL_TREE;
3718 : }
3719 :
3720 3005151 : for (int i = 0; i < count; i++)
3721 : {
3722 2104712 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3723 :
3724 2104712 : if (!plats->virt_call)
3725 2096787 : continue;
3726 :
3727 7925 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3728 7925 : ipcp_value<ipa_polymorphic_call_context> *val;
3729 :
3730 15691 : if (ctxlat->bottom
3731 2931 : || !ctxlat->values
3732 10850 : || !avals.m_known_contexts[i].useless_p ())
3733 7766 : continue;
3734 :
3735 387 : for (val = ctxlat->values; val; val = val->next)
3736 : {
3737 228 : avals.m_known_contexts[i] = val->value;
3738 228 : perform_estimation_of_a_value (node, &avals, removable_params_cost,
3739 : 0, val);
3740 :
3741 228 : if (dump_file && (dump_flags & TDF_DETAILS))
3742 : {
3743 0 : fprintf (dump_file, " - estimates for polymorphic context ");
3744 0 : print_ipcp_constant_value (dump_file, val->value);
3745 0 : fprintf (dump_file, " for ");
3746 0 : ipa_dump_param (dump_file, info, i);
3747 0 : fprintf (dump_file, ": time_benefit: %g, size: %i\n",
3748 : val->local_time_benefit.to_double (),
3749 : val->local_size_cost);
3750 : }
3751 : }
3752 159 : avals.m_known_contexts[i] = ipa_polymorphic_call_context ();
3753 : }
3754 :
3755 900439 : unsigned all_ctx_len = avals.m_known_aggs.length ();
3756 900439 : auto_vec<ipa_argagg_value, 32> all_ctx;
3757 900439 : all_ctx.reserve_exact (all_ctx_len);
3758 900439 : all_ctx.splice (avals.m_known_aggs);
3759 900439 : avals.m_known_aggs.safe_grow_cleared (all_ctx_len + 1);
3760 :
3761 900439 : unsigned j = 0;
3762 3905590 : for (int index = 0; index < count; index++)
3763 : {
3764 2104712 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, index);
3765 :
3766 2104712 : if (plats->aggs_bottom || !plats->aggs)
3767 2085300 : continue;
3768 :
3769 75223 : for (ipcp_agg_lattice *aglat = plats->aggs; aglat; aglat = aglat->next)
3770 : {
3771 55811 : ipcp_value<tree> *val;
3772 55453 : if (aglat->bottom || !aglat->values
3773 : /* If the following is true, the one value is already part of all
3774 : context estimations. */
3775 103704 : || (!plats->aggs_contain_variable
3776 25624 : && aglat->is_single_const ()))
3777 29181 : continue;
3778 :
3779 26630 : unsigned unit_offset = aglat->offset / BITS_PER_UNIT;
3780 26630 : while (j < all_ctx_len
3781 35172 : && (all_ctx[j].index < index
3782 3428 : || (all_ctx[j].index == index
3783 2440 : && all_ctx[j].unit_offset < unit_offset)))
3784 : {
3785 3306 : avals.m_known_aggs[j] = all_ctx[j];
3786 3306 : j++;
3787 : }
3788 :
3789 35883 : for (unsigned k = j; k < all_ctx_len; k++)
3790 9253 : avals.m_known_aggs[k+1] = all_ctx[k];
3791 :
3792 62021 : for (val = aglat->values; val; val = val->next)
3793 : {
3794 35391 : avals.m_known_aggs[j].value = val->value;
3795 35391 : avals.m_known_aggs[j].unit_offset = unit_offset;
3796 35391 : avals.m_known_aggs[j].index = index;
3797 35391 : avals.m_known_aggs[j].by_ref = plats->aggs_by_ref;
3798 35391 : avals.m_known_aggs[j].killed = false;
3799 :
3800 35391 : perform_estimation_of_a_value (node, &avals,
3801 : removable_params_cost, 0, val);
3802 :
3803 35391 : if (dump_file && (dump_flags & TDF_DETAILS))
3804 : {
3805 80 : fprintf (dump_file, " - estimates for value ");
3806 80 : print_ipcp_constant_value (dump_file, val->value);
3807 80 : fprintf (dump_file, " for ");
3808 80 : ipa_dump_param (dump_file, info, index);
3809 160 : fprintf (dump_file, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
3810 : "]: time_benefit: %g, size: %i\n",
3811 80 : plats->aggs_by_ref ? "ref " : "",
3812 : aglat->offset,
3813 : val->local_time_benefit.to_double (),
3814 : val->local_size_cost);
3815 : }
3816 : }
3817 : }
3818 : }
3819 900439 : }
3820 :
3821 :
3822 : /* Add value CUR_VAL and all yet-unsorted values it is dependent on to the
3823 : topological sort of values. */
3824 :
3825 : template <typename valtype>
3826 : void
3827 139765 : value_topo_info<valtype>::add_val (ipcp_value<valtype> *cur_val)
3828 : {
3829 : ipcp_value_source<valtype> *src;
3830 :
3831 139765 : if (cur_val->dfs)
3832 : return;
3833 :
3834 139603 : dfs_counter++;
3835 139603 : cur_val->dfs = dfs_counter;
3836 139603 : cur_val->low_link = dfs_counter;
3837 :
3838 139603 : cur_val->topo_next = stack;
3839 139603 : stack = cur_val;
3840 139603 : cur_val->on_stack = true;
3841 :
3842 601860 : for (src = cur_val->sources; src; src = src->next)
3843 462257 : if (src->val)
3844 : {
3845 21530 : if (src->val->dfs == 0)
3846 : {
3847 186 : add_val (src->val);
3848 186 : if (src->val->low_link < cur_val->low_link)
3849 19 : cur_val->low_link = src->val->low_link;
3850 : }
3851 21344 : else if (src->val->on_stack
3852 1575 : && src->val->dfs < cur_val->low_link)
3853 73 : cur_val->low_link = src->val->dfs;
3854 : }
3855 :
3856 139603 : if (cur_val->dfs == cur_val->low_link)
3857 : {
3858 : ipcp_value<valtype> *v, *scc_list = NULL;
3859 :
3860 : do
3861 : {
3862 139603 : v = stack;
3863 139603 : stack = v->topo_next;
3864 139603 : v->on_stack = false;
3865 139603 : v->scc_no = cur_val->dfs;
3866 :
3867 139603 : v->scc_next = scc_list;
3868 139603 : scc_list = v;
3869 : }
3870 139603 : while (v != cur_val);
3871 :
3872 139515 : cur_val->topo_next = values_topo;
3873 139515 : values_topo = cur_val;
3874 : }
3875 : }
3876 :
3877 : /* Add all values in lattices associated with NODE to the topological sort if
3878 : they are not there yet. */
3879 :
3880 : static void
3881 1303950 : add_all_node_vals_to_toposort (cgraph_node *node, ipa_topo_info *topo)
3882 : {
3883 1303950 : ipa_node_params *info = ipa_node_params_sum->get (node);
3884 1303950 : int i, count = ipa_get_param_count (info);
3885 :
3886 3708507 : for (i = 0; i < count; i++)
3887 : {
3888 2404557 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3889 2404557 : ipcp_lattice<tree> *lat = &plats->itself;
3890 2404557 : struct ipcp_agg_lattice *aglat;
3891 :
3892 2404557 : if (!lat->bottom)
3893 : {
3894 228478 : ipcp_value<tree> *val;
3895 301468 : for (val = lat->values; val; val = val->next)
3896 72990 : topo->constants.add_val (val);
3897 : }
3898 :
3899 2404557 : if (!plats->aggs_bottom)
3900 287078 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
3901 58736 : if (!aglat->bottom)
3902 : {
3903 58378 : ipcp_value<tree> *val;
3904 116988 : for (val = aglat->values; val; val = val->next)
3905 58610 : topo->constants.add_val (val);
3906 : }
3907 :
3908 2404557 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3909 2404557 : if (!ctxlat->bottom)
3910 : {
3911 229481 : ipcp_value<ipa_polymorphic_call_context> *ctxval;
3912 237460 : for (ctxval = ctxlat->values; ctxval; ctxval = ctxval->next)
3913 7979 : topo->contexts.add_val (ctxval);
3914 : }
3915 : }
3916 1303950 : }
3917 :
3918 : /* One pass of constants propagation along the call graph edges, from callers
3919 : to callees (requires topological ordering in TOPO), iterate over strongly
3920 : connected components. */
3921 :
3922 : static void
3923 131565 : propagate_constants_topo (class ipa_topo_info *topo)
3924 : {
3925 131565 : int i;
3926 :
3927 1519511 : for (i = topo->nnodes - 1; i >= 0; i--)
3928 : {
3929 1387946 : unsigned j;
3930 1387946 : struct cgraph_node *v, *node = topo->order[i];
3931 1387946 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (node);
3932 :
3933 : /* First, iteratively propagate within the strongly connected component
3934 : until all lattices stabilize. */
3935 4168742 : FOR_EACH_VEC_ELT (cycle_nodes, j, v)
3936 1392850 : if (v->has_gimple_body_p ())
3937 : {
3938 1312826 : if (opt_for_fn (v->decl, flag_ipa_cp)
3939 1312826 : && opt_for_fn (v->decl, optimize))
3940 1303950 : push_node_to_stack (topo, v);
3941 : /* When V is not optimized, we can not push it to stack, but
3942 : still we need to set all its callees lattices to bottom. */
3943 : else
3944 : {
3945 21937 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
3946 13061 : propagate_constants_across_call (cs);
3947 : }
3948 : }
3949 :
3950 1387946 : v = pop_node_from_stack (topo);
3951 4083052 : while (v)
3952 : {
3953 1307160 : struct cgraph_edge *cs;
3954 1307160 : class ipa_node_params *info = NULL;
3955 1307160 : bool self_scc = true;
3956 :
3957 6731306 : for (cs = v->callees; cs; cs = cs->next_callee)
3958 5424146 : if (ipa_edge_within_scc (cs))
3959 : {
3960 29775 : cgraph_node *callee = cs->callee->function_symbol ();
3961 :
3962 29775 : if (v != callee)
3963 18063 : self_scc = false;
3964 :
3965 29775 : if (!info)
3966 : {
3967 14023 : info = ipa_node_params_sum->get (v);
3968 14023 : info->node_within_scc = true;
3969 : }
3970 :
3971 29775 : if (propagate_constants_across_call (cs))
3972 4185 : push_node_to_stack (topo, callee);
3973 : }
3974 :
3975 1307160 : if (info)
3976 14023 : info->node_is_self_scc = self_scc;
3977 :
3978 1307160 : v = pop_node_from_stack (topo);
3979 : }
3980 :
3981 : /* Afterwards, propagate along edges leading out of the SCC, calculates
3982 : the local effects of the discovered constants and all valid values to
3983 : their topological sort. */
3984 2780796 : FOR_EACH_VEC_ELT (cycle_nodes, j, v)
3985 1392850 : if (v->has_gimple_body_p ()
3986 1312826 : && opt_for_fn (v->decl, flag_ipa_cp)
3987 2696800 : && opt_for_fn (v->decl, optimize))
3988 : {
3989 1303950 : struct cgraph_edge *cs;
3990 :
3991 1303950 : estimate_local_effects (v);
3992 1303950 : add_all_node_vals_to_toposort (v, topo);
3993 6694217 : for (cs = v->callees; cs; cs = cs->next_callee)
3994 5390267 : if (!ipa_edge_within_scc (cs))
3995 5367898 : propagate_constants_across_call (cs);
3996 : }
3997 1387946 : cycle_nodes.release ();
3998 : }
3999 131565 : }
4000 :
4001 : /* Propagate the estimated effects of individual values along the topological
4002 : from the dependent values to those they depend on. */
4003 :
4004 : template <typename valtype>
4005 : void
4006 263130 : value_topo_info<valtype>::propagate_effects ()
4007 : {
4008 : ipcp_value<valtype> *base;
4009 263130 : hash_set<ipcp_value<valtype> *> processed_srcvals;
4010 :
4011 402645 : for (base = values_topo; base; base = base->topo_next)
4012 : {
4013 : ipcp_value_source<valtype> *src;
4014 : ipcp_value<valtype> *val;
4015 139515 : sreal time = 0;
4016 139515 : HOST_WIDE_INT size = 0;
4017 :
4018 279118 : for (val = base; val; val = val->scc_next)
4019 : {
4020 139603 : time = time + val->local_time_benefit + val->prop_time_benefit;
4021 139603 : size = size + val->local_size_cost + val->prop_size_cost;
4022 : }
4023 :
4024 279118 : for (val = base; val; val = val->scc_next)
4025 : {
4026 139603 : processed_srcvals.empty ();
4027 601860 : for (src = val->sources; src; src = src->next)
4028 462257 : if (src->val
4029 462257 : && cs_interesting_for_ipcp_p (src->cs))
4030 : {
4031 21490 : if (!processed_srcvals.add (src->val))
4032 : {
4033 17221 : HOST_WIDE_INT prop_size = size + src->val->prop_size_cost;
4034 17221 : if (prop_size < INT_MAX)
4035 17221 : src->val->prop_size_cost = prop_size;
4036 : else
4037 0 : continue;
4038 : }
4039 :
4040 21490 : int special_factor = 1;
4041 21490 : if (val->same_scc (src->val))
4042 : special_factor
4043 1663 : = opt_for_fn(src->cs->caller->decl,
4044 : param_ipa_cp_recursive_freq_factor);
4045 19827 : else if (val->self_recursion_generated_p ()
4046 19827 : && (src->cs->callee->function_symbol ()
4047 822 : == src->cs->caller))
4048 : {
4049 822 : int max_recur_gen_depth
4050 822 : = opt_for_fn(src->cs->caller->decl,
4051 : param_ipa_cp_max_recursive_depth);
4052 822 : special_factor = max_recur_gen_depth
4053 822 : - val->self_recursion_generated_level + 1;
4054 : }
4055 :
4056 21490 : src->val->prop_time_benefit
4057 42980 : += time * special_factor * src->cs->sreal_frequency ();
4058 : }
4059 :
4060 139603 : if (size < INT_MAX)
4061 : {
4062 139603 : val->prop_time_benefit = time;
4063 139603 : val->prop_size_cost = size;
4064 : }
4065 : else
4066 : {
4067 0 : val->prop_time_benefit = 0;
4068 : val->prop_size_cost = 0;
4069 : }
4070 : }
4071 : }
4072 263130 : }
4073 :
4074 :
4075 : /* Propagate constants, polymorphic contexts and their effects from the
4076 : summaries interprocedurally. */
4077 :
4078 : static void
4079 131565 : ipcp_propagate_stage (class ipa_topo_info *topo)
4080 : {
4081 131565 : struct cgraph_node *node;
4082 :
4083 131565 : if (dump_file)
4084 162 : fprintf (dump_file, "\n Propagating constants:\n\n");
4085 :
4086 1524419 : FOR_EACH_DEFINED_FUNCTION (node)
4087 : {
4088 1392854 : if (node->has_gimple_body_p ()
4089 1312826 : && opt_for_fn (node->decl, flag_ipa_cp)
4090 2696804 : && opt_for_fn (node->decl, optimize))
4091 : {
4092 1303950 : ipa_node_params *info = ipa_node_params_sum->get (node);
4093 1303950 : determine_versionability (node, info);
4094 :
4095 1303950 : unsigned nlattices = ipa_get_param_count (info);
4096 1303950 : info->lattices.safe_grow_cleared (nlattices, true);
4097 1303950 : initialize_node_lattices (node);
4098 :
4099 1303950 : int num_sweeps = opt_for_fn (node->decl, param_ipa_cp_sweeps);
4100 1303950 : if (max_number_sweeps < num_sweeps)
4101 123693 : max_number_sweeps = num_sweeps;
4102 : }
4103 1392854 : ipa_size_summary *s = ipa_size_summaries->get (node);
4104 1392854 : if (node->definition && !node->alias && s != NULL)
4105 1313786 : overall_size += s->self_size;
4106 : }
4107 :
4108 131565 : orig_overall_size = overall_size;
4109 :
4110 131565 : if (dump_file)
4111 162 : fprintf (dump_file, "\noverall_size: %li\n", overall_size);
4112 :
4113 131565 : propagate_constants_topo (topo);
4114 131565 : if (flag_checking)
4115 131557 : ipcp_verify_propagated_values ();
4116 131565 : topo->constants.propagate_effects ();
4117 131565 : topo->contexts.propagate_effects ();
4118 :
4119 131565 : if (dump_file)
4120 : {
4121 162 : fprintf (dump_file, "\nIPA lattices after all propagation:\n");
4122 162 : print_all_lattices (dump_file, (dump_flags & TDF_DETAILS), true);
4123 : }
4124 131565 : }
4125 :
4126 : /* Discover newly direct outgoing edges from NODE which is a new clone with
4127 : known KNOWN_CSTS and make them direct. */
4128 :
4129 : static void
4130 19233 : ipcp_discover_new_direct_edges (struct cgraph_node *node,
4131 : vec<tree> known_csts,
4132 : vec<ipa_polymorphic_call_context>
4133 : known_contexts,
4134 : vec<ipa_argagg_value, va_gc> *aggvals)
4135 : {
4136 19233 : struct cgraph_edge *ie, *next_ie;
4137 19233 : bool found = false;
4138 :
4139 21202 : for (ie = node->indirect_calls; ie; ie = next_ie)
4140 : {
4141 1969 : tree target;
4142 1969 : bool speculative;
4143 :
4144 1969 : next_ie = ie->next_callee;
4145 1969 : ipa_argagg_value_list avs (aggvals);
4146 1969 : target = ipa_get_indirect_edge_target_1 (ie, known_csts, known_contexts,
4147 : avs, &speculative);
4148 1969 : if (target)
4149 : {
4150 566 : cgraph_polymorphic_indirect_info *pii
4151 566 : = dyn_cast <cgraph_polymorphic_indirect_info *> (ie->indirect_info);
4152 566 : cgraph_simple_indirect_info *sii
4153 1061 : = dyn_cast <cgraph_simple_indirect_info *> (ie->indirect_info);
4154 421 : bool agg_contents = sii && sii->agg_contents;
4155 566 : bool polymorphic = !!pii;
4156 566 : int param_index = ie->indirect_info->param_index;
4157 566 : struct cgraph_edge *cs = ipa_make_edge_direct_to_target (ie, target,
4158 : speculative);
4159 566 : found = true;
4160 :
4161 566 : if (cs && !agg_contents && !polymorphic)
4162 : {
4163 350 : ipa_node_params *info = ipa_node_params_sum->get (node);
4164 350 : int c = ipa_get_controlled_uses (info, param_index);
4165 350 : if (c != IPA_UNDESCRIBED_USE
4166 350 : && !ipa_get_param_load_dereferenced (info, param_index))
4167 : {
4168 346 : struct ipa_ref *to_del;
4169 :
4170 346 : c--;
4171 346 : ipa_set_controlled_uses (info, param_index, c);
4172 346 : if (dump_file && (dump_flags & TDF_DETAILS))
4173 3 : fprintf (dump_file, " controlled uses count of param "
4174 : "%i bumped down to %i\n", param_index, c);
4175 346 : if (c == 0
4176 346 : && (to_del = node->find_reference (cs->callee, NULL, 0,
4177 : IPA_REF_ADDR)))
4178 : {
4179 282 : if (dump_file && (dump_flags & TDF_DETAILS))
4180 3 : fprintf (dump_file, " and even removing its "
4181 : "cloning-created reference\n");
4182 282 : to_del->remove_reference ();
4183 : }
4184 : }
4185 : }
4186 : }
4187 : }
4188 : /* Turning calls to direct calls will improve overall summary. */
4189 19233 : if (found)
4190 469 : ipa_update_overall_fn_summary (node);
4191 19233 : }
4192 :
4193 : class edge_clone_summary;
4194 : static call_summary <edge_clone_summary *> *edge_clone_summaries = NULL;
4195 :
4196 : /* Edge clone summary. */
4197 :
4198 : class edge_clone_summary
4199 : {
4200 : public:
4201 : /* Default constructor. */
4202 377478 : edge_clone_summary (): prev_clone (NULL), next_clone (NULL) {}
4203 :
4204 : /* Default destructor. */
4205 377478 : ~edge_clone_summary ()
4206 : {
4207 377478 : if (prev_clone)
4208 34768 : edge_clone_summaries->get (prev_clone)->next_clone = next_clone;
4209 377478 : if (next_clone)
4210 157292 : edge_clone_summaries->get (next_clone)->prev_clone = prev_clone;
4211 377478 : }
4212 :
4213 : cgraph_edge *prev_clone;
4214 : cgraph_edge *next_clone;
4215 : };
4216 :
4217 : class edge_clone_summary_t:
4218 : public call_summary <edge_clone_summary *>
4219 : {
4220 : public:
4221 131565 : edge_clone_summary_t (symbol_table *symtab):
4222 263130 : call_summary <edge_clone_summary *> (symtab)
4223 : {
4224 131565 : m_initialize_when_cloning = true;
4225 : }
4226 :
4227 : void duplicate (cgraph_edge *src_edge, cgraph_edge *dst_edge,
4228 : edge_clone_summary *src_data,
4229 : edge_clone_summary *dst_data) final override;
4230 : };
4231 :
4232 : /* Edge duplication hook. */
4233 :
4234 : void
4235 191448 : edge_clone_summary_t::duplicate (cgraph_edge *src_edge, cgraph_edge *dst_edge,
4236 : edge_clone_summary *src_data,
4237 : edge_clone_summary *dst_data)
4238 : {
4239 191448 : if (src_data->next_clone)
4240 5409 : edge_clone_summaries->get (src_data->next_clone)->prev_clone = dst_edge;
4241 191448 : dst_data->prev_clone = src_edge;
4242 191448 : dst_data->next_clone = src_data->next_clone;
4243 191448 : src_data->next_clone = dst_edge;
4244 191448 : }
4245 :
4246 : /* Return true is CS calls DEST or its clone for all contexts. When
4247 : ALLOW_RECURSION_TO_CLONE is false, also return false for self-recursive
4248 : edges from/to an all-context clone. */
4249 :
4250 : static bool
4251 1838332 : calls_same_node_or_its_all_contexts_clone_p (cgraph_edge *cs, cgraph_node *dest,
4252 : bool allow_recursion_to_clone)
4253 : {
4254 1838332 : enum availability availability;
4255 1838332 : cgraph_node *callee = cs->callee->function_symbol (&availability);
4256 :
4257 1838332 : if (availability <= AVAIL_INTERPOSABLE)
4258 : return false;
4259 1832265 : if (callee == dest)
4260 : return true;
4261 624813 : if (!allow_recursion_to_clone && cs->caller == callee)
4262 : return false;
4263 :
4264 624656 : ipa_node_params *info = ipa_node_params_sum->get (callee);
4265 624656 : return info->is_all_contexts_clone && info->ipcp_orig_node == dest;
4266 : }
4267 :
4268 : /* Return true if edge CS does bring about the value described by SRC to
4269 : DEST_VAL of node DEST or its clone for all contexts. */
4270 :
4271 : static bool
4272 1828101 : cgraph_edge_brings_value_p (cgraph_edge *cs, ipcp_value_source<tree> *src,
4273 : cgraph_node *dest, ipcp_value<tree> *dest_val)
4274 : {
4275 1828101 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
4276 :
4277 1828101 : if (!calls_same_node_or_its_all_contexts_clone_p (cs, dest, !src->val)
4278 1828101 : || caller_info->node_dead)
4279 : return false;
4280 :
4281 757419 : if (!src->val)
4282 : return true;
4283 :
4284 64496 : if (caller_info->ipcp_orig_node)
4285 : {
4286 20488 : tree t = NULL_TREE;
4287 20488 : if (src->offset == -1)
4288 14356 : t = caller_info->known_csts[src->index];
4289 6132 : else if (ipcp_transformation *ts
4290 6132 : = ipcp_get_transformation_summary (cs->caller))
4291 : {
4292 6132 : ipa_argagg_value_list avl (ts);
4293 6132 : t = avl.get_value (src->index, src->offset / BITS_PER_UNIT);
4294 : }
4295 20488 : return (t != NULL_TREE
4296 20488 : && values_equal_for_ipcp_p (src->val->value, t));
4297 : }
4298 : else
4299 : {
4300 44008 : if (src->val == dest_val)
4301 : return true;
4302 :
4303 38290 : struct ipcp_agg_lattice *aglat;
4304 38290 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (caller_info,
4305 : src->index);
4306 38290 : if (src->offset == -1)
4307 28200 : return (plats->itself.is_single_const ()
4308 20 : && values_equal_for_ipcp_p (src->val->value,
4309 20 : plats->itself.values->value));
4310 : else
4311 : {
4312 10090 : if (plats->aggs_bottom || plats->aggs_contain_variable)
4313 : return false;
4314 3882 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
4315 3882 : if (aglat->offset == src->offset)
4316 1748 : return (aglat->is_single_const ()
4317 8 : && values_equal_for_ipcp_p (src->val->value,
4318 8 : aglat->values->value));
4319 : }
4320 : return false;
4321 : }
4322 : }
4323 :
4324 : /* Return true if edge CS does bring about the value described by SRC to
4325 : DST_VAL of node DEST or its clone for all contexts. */
4326 :
4327 : static bool
4328 10231 : cgraph_edge_brings_value_p (cgraph_edge *cs,
4329 : ipcp_value_source<ipa_polymorphic_call_context> *src,
4330 : cgraph_node *dest,
4331 : ipcp_value<ipa_polymorphic_call_context> *)
4332 : {
4333 10231 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
4334 :
4335 10231 : if (!calls_same_node_or_its_all_contexts_clone_p (cs, dest, true)
4336 10231 : || caller_info->node_dead)
4337 : return false;
4338 9288 : if (!src->val)
4339 : return true;
4340 :
4341 1684 : if (caller_info->ipcp_orig_node)
4342 230 : return (caller_info->known_contexts.length () > (unsigned) src->index)
4343 460 : && values_equal_for_ipcp_p (src->val->value,
4344 230 : caller_info->known_contexts[src->index]);
4345 :
4346 1454 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (caller_info,
4347 : src->index);
4348 1454 : return plats->ctxlat.is_single_const ()
4349 270 : && values_equal_for_ipcp_p (src->val->value,
4350 270 : plats->ctxlat.values->value);
4351 : }
4352 :
4353 : /* Get the next clone in the linked list of clones of an edge. */
4354 :
4355 : static inline struct cgraph_edge *
4356 1838593 : get_next_cgraph_edge_clone (struct cgraph_edge *cs)
4357 : {
4358 1838593 : edge_clone_summary *s = edge_clone_summaries->get (cs);
4359 1838593 : return s != NULL ? s->next_clone : NULL;
4360 : }
4361 :
4362 : /* Given VAL that is intended for DEST, iterate over all its sources and if any
4363 : of them is viable and hot, return true. In that case, for those that still
4364 : hold, add their edge frequency and their number and cumulative profile
4365 : counts of self-ecursive and other edges into *FREQUENCY, *CALLER_COUNT,
4366 : REC_COUNT_SUM and NONREC_COUNT_SUM respectively. */
4367 :
4368 : template <typename valtype>
4369 : static bool
4370 217769 : get_info_about_necessary_edges (ipcp_value<valtype> *val, cgraph_node *dest,
4371 : sreal *freq_sum, int *caller_count,
4372 : profile_count *rec_count_sum,
4373 : profile_count *nonrec_count_sum,
4374 : bool *called_without_ipa_profile)
4375 : {
4376 : ipcp_value_source<valtype> *src;
4377 217769 : sreal freq = 0;
4378 217769 : int count = 0;
4379 217769 : profile_count rec_cnt = profile_count::zero ();
4380 217769 : profile_count nonrec_cnt = profile_count::zero ();
4381 217769 : bool interesting = false;
4382 217769 : bool non_self_recursive = false;
4383 217769 : *called_without_ipa_profile = false;
4384 :
4385 986255 : for (src = val->sources; src; src = src->next)
4386 : {
4387 768486 : struct cgraph_edge *cs = src->cs;
4388 1904047 : while (cs)
4389 : {
4390 1135561 : if (cgraph_edge_brings_value_p (cs, src, dest, val))
4391 : {
4392 359946 : count++;
4393 359946 : freq += cs->sreal_frequency ();
4394 359946 : interesting |= cs_interesting_for_ipcp_p (cs);
4395 359946 : if (cs->caller != dest)
4396 : {
4397 352983 : non_self_recursive = true;
4398 352983 : if (cs->count.ipa ().initialized_p ())
4399 1023 : rec_cnt += cs->count.ipa ();
4400 : else
4401 351960 : *called_without_ipa_profile = true;
4402 : }
4403 6963 : else if (cs->count.ipa ().initialized_p ())
4404 0 : nonrec_cnt += cs->count.ipa ();
4405 : else
4406 6963 : *called_without_ipa_profile = true;
4407 : }
4408 1135561 : cs = get_next_cgraph_edge_clone (cs);
4409 : }
4410 : }
4411 :
4412 : /* If the only edges bringing a value are self-recursive ones, do not bother
4413 : evaluating it. */
4414 217769 : if (!non_self_recursive)
4415 : return false;
4416 :
4417 153939 : *freq_sum = freq;
4418 153939 : *caller_count = count;
4419 153939 : *rec_count_sum = rec_cnt;
4420 153939 : *nonrec_count_sum = nonrec_cnt;
4421 :
4422 153939 : return interesting;
4423 : }
4424 :
4425 : /* Given a NODE, and a set of its CALLERS, try to adjust order of the callers
4426 : to let a non-self-recursive caller be the first element. Thus, we can
4427 : simplify intersecting operations on values that arrive from all of these
4428 : callers, especially when there exists self-recursive call. Return true if
4429 : this kind of adjustment is possible. */
4430 :
4431 : static bool
4432 56347 : adjust_callers_for_value_intersection (vec<cgraph_edge *> &callers,
4433 : cgraph_node *node)
4434 : {
4435 60565 : for (unsigned i = 0; i < callers.length (); i++)
4436 : {
4437 60498 : cgraph_edge *cs = callers[i];
4438 :
4439 60498 : if (cs->caller != node)
4440 : {
4441 56280 : if (i > 0)
4442 : {
4443 1985 : callers[i] = callers[0];
4444 1985 : callers[0] = cs;
4445 : }
4446 : return true;
4447 : }
4448 : }
4449 : return false;
4450 : }
4451 :
4452 : /* Return a vector of incoming edges that do bring value VAL to node DEST. It
4453 : is assumed their number is known and equal to CALLER_COUNT. */
4454 :
4455 : template <typename valtype>
4456 : static auto_vec<cgraph_edge *>
4457 153568 : gather_edges_for_value (ipcp_value<valtype> *val, cgraph_node *dest,
4458 : int caller_count)
4459 : {
4460 : ipcp_value_source<valtype> *src;
4461 153568 : auto_vec<cgraph_edge *> ret (caller_count);
4462 :
4463 527754 : for (src = val->sources; src; src = src->next)
4464 : {
4465 374186 : struct cgraph_edge *cs = src->cs;
4466 836861 : while (cs)
4467 : {
4468 462675 : if (cgraph_edge_brings_value_p (cs, src, dest, val))
4469 356720 : ret.quick_push (cs);
4470 462675 : cs = get_next_cgraph_edge_clone (cs);
4471 : }
4472 : }
4473 :
4474 153568 : if (caller_count > 1)
4475 41472 : adjust_callers_for_value_intersection (ret, dest);
4476 :
4477 153568 : return ret;
4478 : }
4479 :
4480 : /* Construct a replacement map for a know VALUE for a formal parameter PARAM.
4481 : Return it or NULL if for some reason it cannot be created. FORCE_LOAD_REF
4482 : should be set to true when the reference created for the constant should be
4483 : a load one and not an address one because the corresponding parameter p is
4484 : only used as *p. */
4485 :
4486 : static struct ipa_replace_map *
4487 24623 : get_replacement_map (class ipa_node_params *info, tree value, int parm_num,
4488 : bool force_load_ref)
4489 : {
4490 24623 : struct ipa_replace_map *replace_map;
4491 :
4492 24623 : replace_map = ggc_alloc<ipa_replace_map> ();
4493 24623 : if (dump_file)
4494 : {
4495 171 : fprintf (dump_file, " replacing ");
4496 171 : ipa_dump_param (dump_file, info, parm_num);
4497 :
4498 171 : fprintf (dump_file, " with const ");
4499 171 : print_generic_expr (dump_file, value);
4500 :
4501 171 : if (force_load_ref)
4502 11 : fprintf (dump_file, " - forcing load reference\n");
4503 : else
4504 160 : fprintf (dump_file, "\n");
4505 : }
4506 24623 : replace_map->parm_num = parm_num;
4507 24623 : replace_map->new_tree = value;
4508 24623 : replace_map->force_load_ref = force_load_ref;
4509 24623 : return replace_map;
4510 : }
4511 :
4512 : /* Dump new profiling counts of NODE. SPEC is true when NODE is a specialzied
4513 : one, otherwise it will be referred to as the original node. */
4514 :
4515 : static void
4516 4 : dump_profile_updates (cgraph_node *node, bool spec)
4517 : {
4518 4 : if (spec)
4519 2 : fprintf (dump_file, " setting count of the specialized node %s to ",
4520 : node->dump_name ());
4521 : else
4522 2 : fprintf (dump_file, " setting count of the original node %s to ",
4523 : node->dump_name ());
4524 :
4525 4 : node->count.dump (dump_file);
4526 4 : fprintf (dump_file, "\n");
4527 6 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
4528 : {
4529 2 : fprintf (dump_file, " edge to %s has count ",
4530 2 : cs->callee->dump_name ());
4531 2 : cs->count.dump (dump_file);
4532 2 : fprintf (dump_file, "\n");
4533 : }
4534 4 : }
4535 :
4536 : /* With partial train run we do not want to assume that original's count is
4537 : zero whenever we redurect all executed edges to clone. Simply drop profile
4538 : to local one in this case. In eany case, return the new value. ORIG_NODE
4539 : is the original node and its count has not been updated yet. */
4540 :
4541 : profile_count
4542 16 : lenient_count_portion_handling (profile_count remainder, cgraph_node *orig_node)
4543 : {
4544 32 : if (remainder.ipa_p () && !remainder.ipa ().nonzero_p ()
4545 26 : && orig_node->count.ipa_p () && orig_node->count.ipa ().nonzero_p ()
4546 5 : && opt_for_fn (orig_node->decl, flag_profile_partial_training))
4547 0 : remainder = orig_node->count.guessed_local ();
4548 :
4549 16 : return remainder;
4550 : }
4551 :
4552 : /* Structure to sum counts coming from nodes other than the original node and
4553 : its clones. */
4554 :
4555 : struct gather_other_count_struct
4556 : {
4557 : cgraph_node *orig;
4558 : profile_count other_count;
4559 : };
4560 :
4561 : /* Worker callback of call_for_symbol_thunks_and_aliases summing the number of
4562 : counts that come from non-self-recursive calls.. */
4563 :
4564 : static bool
4565 8 : gather_count_of_non_rec_edges (cgraph_node *node, void *data)
4566 : {
4567 8 : gather_other_count_struct *desc = (gather_other_count_struct *) data;
4568 20 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4569 12 : if (cs->caller != desc->orig && cs->caller->clone_of != desc->orig)
4570 0 : if (cs->count.ipa ().initialized_p ())
4571 0 : desc->other_count += cs->count.ipa ();
4572 8 : return false;
4573 : }
4574 :
4575 : /* Structure to help analyze if we need to boost counts of some clones of some
4576 : non-recursive edges to match the new callee count. */
4577 :
4578 : struct desc_incoming_count_struct
4579 : {
4580 : cgraph_node *orig;
4581 : hash_set <cgraph_edge *> *processed_edges;
4582 : profile_count count;
4583 : unsigned unproc_orig_rec_edges;
4584 : };
4585 :
4586 : /* Go over edges calling NODE and its thunks and gather information about
4587 : incoming counts so that we know if we need to make any adjustments. */
4588 :
4589 : static void
4590 8 : analyze_clone_icoming_counts (cgraph_node *node,
4591 : desc_incoming_count_struct *desc)
4592 : {
4593 20 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4594 12 : if (cs->caller->thunk)
4595 : {
4596 0 : analyze_clone_icoming_counts (cs->caller, desc);
4597 0 : continue;
4598 : }
4599 : else
4600 : {
4601 12 : if (cs->count.initialized_p ())
4602 12 : desc->count += cs->count.ipa ();
4603 12 : if (!desc->processed_edges->contains (cs)
4604 12 : && cs->caller->clone_of == desc->orig)
4605 4 : desc->unproc_orig_rec_edges++;
4606 : }
4607 8 : }
4608 :
4609 : /* If caller edge counts of a clone created for a self-recursive arithmetic
4610 : jump function must be adjusted because it is coming from a the "seed" clone
4611 : for the first value and so has been excessively scaled back as if it was not
4612 : a recursive call, adjust it so that the incoming counts of NODE match its
4613 : count. NODE is the node or its thunk. */
4614 :
4615 : static void
4616 0 : adjust_clone_incoming_counts (cgraph_node *node,
4617 : desc_incoming_count_struct *desc)
4618 : {
4619 0 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4620 0 : if (cs->caller->thunk)
4621 : {
4622 0 : adjust_clone_incoming_counts (cs->caller, desc);
4623 : /* Same rationale as commit 8c6b6adce45a550c52dc35e3df4e0c477f5404fa
4624 : for scaling recursive edges in update_counts_for_self_gen_clones:
4625 : adjusting non-IPA edge counts here does not update matching gimple
4626 : BB frequencies and breaks verify_cgraph_node. */
4627 0 : if (cs->count.ipa_p ())
4628 : {
4629 0 : profile_count sum = profile_count::zero ();
4630 0 : for (cgraph_edge *e = cs->caller->callers; e; e = e->next_caller)
4631 0 : if (e->count.initialized_p ())
4632 0 : sum += e->count.ipa ();
4633 0 : cs->count = cs->count.combine_with_ipa_count (sum);
4634 : }
4635 0 : else if (dump_file)
4636 0 : fprintf (dump_file, " Skipping adjustment of the count of an "
4637 : "incoming edge of a clone %s -> %s\n",
4638 0 : cs->caller->dump_name (), cs->callee->dump_name ());
4639 : }
4640 0 : else if (!desc->processed_edges->contains (cs)
4641 0 : && cs->caller->clone_of == desc->orig
4642 0 : && cs->count.compatible_p (desc->count))
4643 : {
4644 0 : if (cs->count.ipa_p ())
4645 : {
4646 0 : cs->count += desc->count;
4647 0 : if (dump_file)
4648 : {
4649 0 : fprintf (dump_file, " Adjusted count of an incoming edge "
4650 0 : "of a clone %s -> %s to ", cs->caller->dump_name (),
4651 0 : cs->callee->dump_name ());
4652 0 : cs->count.dump (dump_file);
4653 0 : fprintf (dump_file, "\n");
4654 : }
4655 : }
4656 0 : else if (dump_file)
4657 0 : fprintf (dump_file, " Skipping adjustment of the count of an "
4658 : "incoming edge of a clone %s -> %s\n",
4659 0 : cs->caller->dump_name (), cs->callee->dump_name ());
4660 : }
4661 0 : }
4662 :
4663 : /* When ORIG_NODE has been cloned for values which have been generated fora
4664 : self-recursive call as a result of an arithmetic pass-through
4665 : jump-functions, adjust its count together with counts of all such clones in
4666 : SELF_GEN_CLONES which also at this point contains ORIG_NODE itself.
4667 :
4668 : The function sums the counts of the original node and all its clones that
4669 : cannot be attributed to a specific clone because it comes from a
4670 : non-recursive edge. This sum is then evenly divided between the clones and
4671 : on top of that each one gets all the counts which can be attributed directly
4672 : to it. */
4673 :
4674 : static void
4675 33 : update_counts_for_self_gen_clones (cgraph_node *orig_node,
4676 : const vec<cgraph_node *> &self_gen_clones)
4677 : {
4678 33 : profile_count redist_sum = orig_node->count.ipa ();
4679 33 : if (!redist_sum.nonzero_p ())
4680 : return;
4681 :
4682 4 : if (dump_file)
4683 0 : fprintf (dump_file, " Updating profile of self recursive clone "
4684 : "series\n");
4685 :
4686 4 : gather_other_count_struct gocs;
4687 4 : gocs.orig = orig_node;
4688 4 : gocs.other_count = profile_count::zero ();
4689 :
4690 4 : auto_vec <profile_count, 8> other_edges_count;
4691 20 : for (cgraph_node *n : self_gen_clones)
4692 : {
4693 8 : gocs.other_count = profile_count::zero ();
4694 8 : n->call_for_symbol_thunks_and_aliases (gather_count_of_non_rec_edges,
4695 : &gocs, false);
4696 8 : other_edges_count.safe_push (gocs.other_count);
4697 8 : redist_sum -= gocs.other_count;
4698 : }
4699 :
4700 4 : hash_set<cgraph_edge *> processed_edges;
4701 4 : unsigned i = 0;
4702 20 : for (cgraph_node *n : self_gen_clones)
4703 : {
4704 8 : profile_count new_count
4705 16 : = (redist_sum / self_gen_clones.length () + other_edges_count[i]);
4706 8 : new_count = lenient_count_portion_handling (new_count, orig_node);
4707 8 : n->scale_profile_to (new_count);
4708 16 : for (cgraph_edge *cs = n->callees; cs; cs = cs->next_callee)
4709 8 : processed_edges.add (cs);
4710 :
4711 8 : i++;
4712 : }
4713 :
4714 : /* There are still going to be edges to ORIG_NODE that have one or more
4715 : clones coming from another node clone in SELF_GEN_CLONES and which we
4716 : scaled by the same amount, which means that the total incoming sum of
4717 : counts to ORIG_NODE will be too high, scale such edges back. */
4718 8 : for (cgraph_edge *cs = orig_node->callees; cs; cs = cs->next_callee)
4719 : {
4720 4 : if (cs->callee->ultimate_alias_target () == orig_node)
4721 : {
4722 4 : unsigned den = 0;
4723 18 : for (cgraph_edge *e = cs; e; e = get_next_cgraph_edge_clone (e))
4724 14 : if (e->callee->ultimate_alias_target () == orig_node
4725 14 : && processed_edges.contains (e))
4726 8 : den++;
4727 4 : if (den > 0)
4728 18 : for (cgraph_edge *e = cs; e; e = get_next_cgraph_edge_clone (e))
4729 14 : if (e->callee->ultimate_alias_target () == orig_node
4730 8 : && processed_edges.contains (e)
4731 : /* If count is not IPA, this adjustment makes verifier
4732 : unhappy, since we expect bb->count to match e->count.
4733 : We may add a flag to mark edge conts that has been
4734 : modified by IPA code, but so far it does not seem
4735 : to be worth the effort. With local counts the profile
4736 : will not propagate at IPA level. */
4737 30 : && e->count.ipa_p ())
4738 8 : e->count /= den;
4739 : }
4740 : }
4741 :
4742 : /* Edges from the seeds of the values generated for arithmetic jump-functions
4743 : along self-recursive edges are likely to have fairly low count and so
4744 : edges from them to nodes in the self_gen_clones do not correspond to the
4745 : artificially distributed count of the nodes, the total sum of incoming
4746 : edges to some clones might be too low. Detect this situation and correct
4747 : it. */
4748 20 : for (cgraph_node *n : self_gen_clones)
4749 : {
4750 8 : if (!n->count.ipa ().nonzero_p ())
4751 0 : continue;
4752 :
4753 8 : desc_incoming_count_struct desc;
4754 8 : desc.orig = orig_node;
4755 8 : desc.processed_edges = &processed_edges;
4756 8 : desc.count = profile_count::zero ();
4757 8 : desc.unproc_orig_rec_edges = 0;
4758 8 : analyze_clone_icoming_counts (n, &desc);
4759 :
4760 8 : if (n->count.differs_from_p (desc.count))
4761 : {
4762 0 : if (n->count > desc.count
4763 0 : && desc.unproc_orig_rec_edges > 0)
4764 : {
4765 0 : desc.count = n->count - desc.count;
4766 0 : desc.count = desc.count /= desc.unproc_orig_rec_edges;
4767 0 : adjust_clone_incoming_counts (n, &desc);
4768 : }
4769 0 : else if (dump_file)
4770 0 : fprintf (dump_file,
4771 : " Unable to fix up incoming counts for %s.\n",
4772 : n->dump_name ());
4773 : }
4774 : }
4775 :
4776 4 : if (dump_file)
4777 0 : for (cgraph_node *n : self_gen_clones)
4778 0 : dump_profile_updates (n, n != orig_node);
4779 4 : return;
4780 4 : }
4781 :
4782 : /* After a specialized NEW_NODE version of ORIG_NODE has been created, update
4783 : their profile information to reflect this. This function should not be used
4784 : for clones generated for arithmetic pass-through jump functions on a
4785 : self-recursive call graph edge, that situation is handled by
4786 : update_counts_for_self_gen_clones. */
4787 :
4788 : static void
4789 4283 : update_profiling_info (struct cgraph_node *orig_node,
4790 : struct cgraph_node *new_node)
4791 : {
4792 4283 : struct caller_statistics stats;
4793 4283 : profile_count new_sum;
4794 4283 : profile_count remainder, orig_node_count = orig_node->count.ipa ();
4795 :
4796 4283 : if (!orig_node_count.nonzero_p ())
4797 4275 : return;
4798 :
4799 8 : if (dump_file)
4800 : {
4801 2 : fprintf (dump_file, " Updating profile from original count: ");
4802 2 : orig_node_count.dump (dump_file);
4803 2 : fprintf (dump_file, "\n");
4804 : }
4805 :
4806 8 : init_caller_stats (&stats, new_node);
4807 8 : new_node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
4808 : false);
4809 8 : new_sum = stats.count_sum;
4810 :
4811 8 : bool orig_edges_processed = false;
4812 8 : if (new_sum > orig_node_count)
4813 : {
4814 : /* Profile has already gone astray, keep what we have but lower it
4815 : to global0adjusted or to local if we have partial training. */
4816 0 : if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
4817 0 : orig_node->make_profile_local ();
4818 0 : if (new_sum.quality () == AFDO)
4819 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_AFDO);
4820 : else
4821 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_ADJUSTED);
4822 : orig_edges_processed = true;
4823 : }
4824 8 : else if (stats.rec_count_sum.nonzero_p ())
4825 : {
4826 0 : int new_nonrec_calls = stats.n_nonrec_calls;
4827 : /* There are self-recursive edges which are likely to bring in the
4828 : majority of calls but which we must divide in between the original and
4829 : new node. */
4830 0 : init_caller_stats (&stats, orig_node);
4831 0 : orig_node->call_for_symbol_thunks_and_aliases (gather_caller_stats,
4832 : &stats, false);
4833 0 : int orig_nonrec_calls = stats.n_nonrec_calls;
4834 0 : profile_count orig_nonrec_call_count = stats.count_sum;
4835 :
4836 0 : if (orig_node->local)
4837 : {
4838 0 : if (!orig_nonrec_call_count.nonzero_p ())
4839 : {
4840 0 : if (dump_file)
4841 0 : fprintf (dump_file, " The original is local and the only "
4842 : "incoming edges from non-dead callers with nonzero "
4843 : "counts are self-recursive, assuming it is cold.\n");
4844 : /* The NEW_NODE count and counts of all its outgoing edges
4845 : are still unmodified copies of ORIG_NODE's. Just clear
4846 : the latter and bail out. */
4847 0 : if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
4848 0 : orig_node->make_profile_local ();
4849 0 : else if (orig_nonrec_call_count.quality () == AFDO)
4850 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_AFDO);
4851 : else
4852 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_ADJUSTED);
4853 0 : return;
4854 : }
4855 : }
4856 : else
4857 : {
4858 : /* Let's behave as if there was another caller that accounts for all
4859 : the calls that were either indirect or from other compilation
4860 : units. */
4861 0 : orig_nonrec_calls++;
4862 0 : profile_count pretend_caller_count
4863 0 : = (orig_node_count - new_sum - orig_nonrec_call_count
4864 0 : - stats.rec_count_sum);
4865 0 : orig_nonrec_call_count += pretend_caller_count;
4866 : }
4867 :
4868 : /* Divide all "unexplained" counts roughly proportionally to sums of
4869 : counts of non-recursive calls.
4870 :
4871 : We put rather arbitrary limits on how many counts we claim because the
4872 : number of non-self-recursive incoming count is only a rough guideline
4873 : and there are cases (such as mcf) where using it blindly just takes
4874 : too many. And if lattices are considered in the opposite order we
4875 : could also take too few. */
4876 0 : profile_count unexp = orig_node_count - new_sum - orig_nonrec_call_count;
4877 :
4878 0 : int limit_den = 2 * (orig_nonrec_calls + new_nonrec_calls);
4879 0 : profile_count new_part = unexp.apply_scale (limit_den - 1, limit_den);
4880 0 : profile_count den = new_sum + orig_nonrec_call_count;
4881 0 : if (den.nonzero_p ())
4882 0 : new_part = MIN (unexp.apply_scale (new_sum, den), new_part);
4883 0 : new_part = MAX (new_part,
4884 : unexp.apply_scale (new_nonrec_calls, limit_den));
4885 0 : if (dump_file)
4886 : {
4887 0 : fprintf (dump_file, " Claiming ");
4888 0 : new_part.dump (dump_file);
4889 0 : fprintf (dump_file, " of unexplained ");
4890 0 : unexp.dump (dump_file);
4891 0 : fprintf (dump_file, " counts because of self-recursive "
4892 : "calls\n");
4893 : }
4894 0 : new_sum += new_part;
4895 0 : remainder = lenient_count_portion_handling (orig_node_count - new_sum,
4896 : orig_node);
4897 : }
4898 : else
4899 8 : remainder = lenient_count_portion_handling (orig_node_count - new_sum,
4900 : orig_node);
4901 :
4902 8 : new_node->scale_profile_to (new_sum);
4903 :
4904 8 : if (!orig_edges_processed)
4905 8 : orig_node->scale_profile_to (remainder);
4906 :
4907 8 : if (dump_file)
4908 : {
4909 2 : dump_profile_updates (new_node, true);
4910 2 : dump_profile_updates (orig_node, false);
4911 : }
4912 : }
4913 :
4914 : /* Update the respective profile of specialized NEW_NODE and the original
4915 : ORIG_NODE after additional edges with cumulative count sum REDIRECTED_SUM
4916 : have been redirected to the specialized version. */
4917 :
4918 : static void
4919 0 : update_specialized_profile (struct cgraph_node *new_node,
4920 : struct cgraph_node *orig_node,
4921 : profile_count redirected_sum)
4922 : {
4923 0 : if (dump_file)
4924 : {
4925 0 : fprintf (dump_file, " the sum of counts of redirected edges is ");
4926 0 : redirected_sum.dump (dump_file);
4927 0 : fprintf (dump_file, "\n old ipa count of the original node is ");
4928 0 : orig_node->count.dump (dump_file);
4929 0 : fprintf (dump_file, "\n");
4930 : }
4931 0 : if (!orig_node->count.ipa ().nonzero_p ()
4932 0 : || !redirected_sum.nonzero_p ())
4933 : return;
4934 :
4935 0 : orig_node->scale_profile_to
4936 0 : (lenient_count_portion_handling (orig_node->count.ipa () - redirected_sum,
4937 : orig_node));
4938 :
4939 0 : new_node->scale_profile_to (new_node->count.ipa () + redirected_sum);
4940 :
4941 0 : if (dump_file)
4942 : {
4943 0 : dump_profile_updates (new_node, true);
4944 0 : dump_profile_updates (orig_node, false);
4945 : }
4946 : }
4947 :
4948 : static void adjust_references_in_caller (cgraph_edge *cs,
4949 : symtab_node *symbol, int index);
4950 :
4951 : /* Simple structure to pass a symbol and index (with same meaning as parameters
4952 : of adjust_references_in_caller) through a void* parameter of a
4953 : call_for_symbol_thunks_and_aliases callback. */
4954 : struct symbol_and_index_together
4955 : {
4956 : symtab_node *symbol;
4957 : int index;
4958 : };
4959 :
4960 : /* Worker callback of call_for_symbol_thunks_and_aliases to recursively call
4961 : adjust_references_in_caller on edges up in the call-graph, if necessary. */
4962 : static bool
4963 9 : adjust_refs_in_act_callers (struct cgraph_node *node, void *data)
4964 : {
4965 9 : symbol_and_index_together *pack = (symbol_and_index_together *) data;
4966 40 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4967 31 : if (!cs->caller->thunk)
4968 31 : adjust_references_in_caller (cs, pack->symbol, pack->index);
4969 9 : return false;
4970 : }
4971 :
4972 : /* At INDEX of a function being called by CS there is an ADDR_EXPR of a
4973 : variable which is only dereferenced and which is represented by SYMBOL. See
4974 : if we can remove ADDR reference in callers associated with the call. */
4975 :
4976 : static void
4977 417 : adjust_references_in_caller (cgraph_edge *cs, symtab_node *symbol, int index)
4978 : {
4979 417 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
4980 417 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, index);
4981 417 : if (jfunc->type == IPA_JF_CONST)
4982 : {
4983 398 : ipa_ref *to_del = cs->caller->find_reference (symbol, cs->call_stmt,
4984 : cs->lto_stmt_uid,
4985 : IPA_REF_ADDR);
4986 398 : if (!to_del)
4987 408 : return;
4988 398 : to_del->remove_reference ();
4989 398 : ipa_zap_jf_refdesc (jfunc);
4990 398 : if (dump_file)
4991 22 : fprintf (dump_file, " Removed a reference from %s to %s.\n",
4992 11 : cs->caller->dump_name (), symbol->dump_name ());
4993 : return;
4994 : }
4995 :
4996 19 : if (jfunc->type != IPA_JF_PASS_THROUGH
4997 19 : || ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR
4998 38 : || ipa_get_jf_pass_through_refdesc_decremented (jfunc))
4999 : return;
5000 :
5001 19 : int fidx = ipa_get_jf_pass_through_formal_id (jfunc);
5002 19 : cgraph_node *caller = cs->caller;
5003 19 : ipa_node_params *caller_info = ipa_node_params_sum->get (caller);
5004 : /* TODO: This consistency check may be too big and not really
5005 : that useful. Consider removing it. */
5006 19 : tree cst;
5007 19 : if (caller_info->ipcp_orig_node)
5008 17 : cst = caller_info->known_csts[fidx];
5009 : else
5010 : {
5011 2 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (caller_info, fidx);
5012 2 : gcc_assert (lat->is_single_const ());
5013 2 : cst = lat->values->value;
5014 : }
5015 19 : gcc_assert (TREE_CODE (cst) == ADDR_EXPR
5016 : && (symtab_node::get (get_base_address (TREE_OPERAND (cst, 0)))
5017 : == symbol));
5018 :
5019 19 : int cuses = ipa_get_controlled_uses (caller_info, fidx);
5020 19 : if (cuses == IPA_UNDESCRIBED_USE)
5021 : return;
5022 19 : gcc_assert (cuses > 0);
5023 19 : cuses--;
5024 19 : ipa_set_controlled_uses (caller_info, fidx, cuses);
5025 19 : ipa_set_jf_pass_through_refdesc_decremented (jfunc, true);
5026 19 : if (dump_file && (dump_flags & TDF_DETAILS))
5027 3 : fprintf (dump_file, " Controlled uses of parameter %i of %s dropped "
5028 : "to %i.\n", fidx, caller->dump_name (), cuses);
5029 19 : if (cuses)
5030 : return;
5031 :
5032 9 : if (caller_info->ipcp_orig_node)
5033 : {
5034 : /* Cloning machinery has created a reference here, we need to either
5035 : remove it or change it to a read one. */
5036 7 : ipa_ref *to_del = caller->find_reference (symbol, NULL, 0, IPA_REF_ADDR);
5037 7 : if (to_del)
5038 : {
5039 7 : to_del->remove_reference ();
5040 7 : if (dump_file)
5041 6 : fprintf (dump_file, " Removed a reference from %s to %s.\n",
5042 3 : cs->caller->dump_name (), symbol->dump_name ());
5043 7 : if (ipa_get_param_load_dereferenced (caller_info, fidx))
5044 : {
5045 3 : caller->create_reference (symbol, IPA_REF_LOAD, NULL);
5046 3 : if (dump_file)
5047 2 : fprintf (dump_file,
5048 : " ...and replaced it with LOAD one.\n");
5049 : }
5050 : }
5051 : }
5052 :
5053 9 : symbol_and_index_together pack;
5054 9 : pack.symbol = symbol;
5055 9 : pack.index = fidx;
5056 9 : if (caller->can_change_signature)
5057 9 : caller->call_for_symbol_thunks_and_aliases (adjust_refs_in_act_callers,
5058 : &pack, true);
5059 : }
5060 :
5061 :
5062 : /* Return true if we would like to remove a parameter from NODE when cloning it
5063 : with KNOWN_CSTS scalar constants. */
5064 :
5065 : static bool
5066 17793 : want_remove_some_param_p (cgraph_node *node, vec<tree> known_csts)
5067 : {
5068 17793 : auto_vec<bool, 16> surviving;
5069 17793 : bool filled_vec = false;
5070 17793 : ipa_node_params *info = ipa_node_params_sum->get (node);
5071 17793 : int i, count = ipa_get_param_count (info);
5072 :
5073 36825 : for (i = 0; i < count; i++)
5074 : {
5075 32172 : if (!known_csts[i] && ipa_is_param_used (info, i))
5076 19032 : continue;
5077 :
5078 13140 : if (!filled_vec)
5079 : {
5080 13140 : clone_info *info = clone_info::get (node);
5081 13140 : if (!info || !info->param_adjustments)
5082 : return true;
5083 0 : info->param_adjustments->get_surviving_params (&surviving);
5084 0 : filled_vec = true;
5085 : }
5086 0 : if (surviving.length() < (unsigned) i && surviving[i])
5087 : return true;
5088 : }
5089 : return false;
5090 17793 : }
5091 :
5092 : /* Create a specialized version of NODE with known constants in KNOWN_CSTS,
5093 : known contexts in KNOWN_CONTEXTS and known aggregate values in AGGVALS and
5094 : redirect all edges in CALLERS to it. */
5095 :
5096 : static struct cgraph_node *
5097 19233 : create_specialized_node (struct cgraph_node *node,
5098 : vec<tree> known_csts,
5099 : vec<ipa_polymorphic_call_context> known_contexts,
5100 : vec<ipa_argagg_value, va_gc> *aggvals,
5101 : vec<cgraph_edge *> &callers)
5102 : {
5103 19233 : ipa_node_params *new_info, *info = ipa_node_params_sum->get (node);
5104 19233 : vec<ipa_replace_map *, va_gc> *replace_trees = NULL;
5105 19233 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
5106 19233 : struct cgraph_node *new_node;
5107 19233 : int i, count = ipa_get_param_count (info);
5108 19233 : clone_info *cinfo = clone_info::get (node);
5109 0 : ipa_param_adjustments *old_adjustments = cinfo
5110 19233 : ? cinfo->param_adjustments : NULL;
5111 19233 : ipa_param_adjustments *new_adjustments;
5112 19233 : gcc_assert (!info->ipcp_orig_node);
5113 19233 : gcc_assert (node->can_change_signature
5114 : || !old_adjustments);
5115 :
5116 17793 : if (old_adjustments)
5117 : {
5118 : /* At the moment all IPA optimizations should use the number of
5119 : parameters of the prevailing decl as the m_always_copy_start.
5120 : Handling any other value would complicate the code below, so for the
5121 : time bing let's only assert it is so. */
5122 0 : gcc_assert (old_adjustments->m_always_copy_start == count
5123 : || old_adjustments->m_always_copy_start < 0);
5124 0 : int old_adj_count = vec_safe_length (old_adjustments->m_adj_params);
5125 0 : for (i = 0; i < old_adj_count; i++)
5126 : {
5127 0 : ipa_adjusted_param *old_adj = &(*old_adjustments->m_adj_params)[i];
5128 0 : if (!node->can_change_signature
5129 0 : || old_adj->op != IPA_PARAM_OP_COPY
5130 0 : || (!known_csts[old_adj->base_index]
5131 0 : && ipa_is_param_used (info, old_adj->base_index)))
5132 : {
5133 0 : ipa_adjusted_param new_adj = *old_adj;
5134 :
5135 0 : new_adj.prev_clone_adjustment = true;
5136 0 : new_adj.prev_clone_index = i;
5137 0 : vec_safe_push (new_params, new_adj);
5138 : }
5139 : }
5140 0 : bool skip_return = old_adjustments->m_skip_return;
5141 0 : new_adjustments = (new (ggc_alloc <ipa_param_adjustments> ())
5142 : ipa_param_adjustments (new_params, count,
5143 0 : skip_return));
5144 : }
5145 19233 : else if (node->can_change_signature
5146 19233 : && want_remove_some_param_p (node, known_csts))
5147 : {
5148 13140 : ipa_adjusted_param adj;
5149 13140 : memset (&adj, 0, sizeof (adj));
5150 13140 : adj.op = IPA_PARAM_OP_COPY;
5151 51025 : for (i = 0; i < count; i++)
5152 37885 : if (!known_csts[i] && ipa_is_param_used (info, i))
5153 : {
5154 15149 : adj.base_index = i;
5155 15149 : adj.prev_clone_index = i;
5156 15149 : vec_safe_push (new_params, adj);
5157 : }
5158 13140 : new_adjustments = (new (ggc_alloc <ipa_param_adjustments> ())
5159 13140 : ipa_param_adjustments (new_params, count, false));
5160 : }
5161 : else
5162 : new_adjustments = NULL;
5163 :
5164 19233 : auto_vec<cgraph_edge *, 2> self_recursive_calls;
5165 156148 : for (i = callers.length () - 1; i >= 0; i--)
5166 : {
5167 117682 : cgraph_edge *cs = callers[i];
5168 117682 : if (cs->caller == node)
5169 : {
5170 117 : self_recursive_calls.safe_push (cs);
5171 117 : callers.unordered_remove (i);
5172 : }
5173 : }
5174 19233 : replace_trees = cinfo ? vec_safe_copy (cinfo->tree_map) : NULL;
5175 72599 : for (i = 0; i < count; i++)
5176 : {
5177 53366 : tree t = known_csts[i];
5178 53366 : if (!t)
5179 28743 : continue;
5180 :
5181 24623 : gcc_checking_assert (TREE_CODE (t) != TREE_BINFO);
5182 :
5183 24623 : bool load_ref = false;
5184 24623 : symtab_node *ref_symbol;
5185 24623 : if (TREE_CODE (t) == ADDR_EXPR)
5186 : {
5187 6681 : tree base = get_base_address (TREE_OPERAND (t, 0));
5188 6681 : if (TREE_CODE (base) == VAR_DECL
5189 3226 : && ipa_get_controlled_uses (info, i) == 0
5190 962 : && ipa_get_param_load_dereferenced (info, i)
5191 7082 : && (ref_symbol = symtab_node::get (base)))
5192 : {
5193 401 : load_ref = true;
5194 401 : if (node->can_change_signature)
5195 1463 : for (cgraph_edge *caller : callers)
5196 386 : adjust_references_in_caller (caller, ref_symbol, i);
5197 : }
5198 : }
5199 :
5200 24623 : ipa_replace_map *replace_map = get_replacement_map (info, t, i, load_ref);
5201 24623 : if (replace_map)
5202 24623 : vec_safe_push (replace_trees, replace_map);
5203 : }
5204 :
5205 57699 : unsigned &suffix_counter = clone_num_suffixes->get_or_insert (
5206 19233 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (
5207 : node->decl)));
5208 19233 : new_node = node->create_virtual_clone (callers, replace_trees,
5209 : new_adjustments, "constprop",
5210 : suffix_counter);
5211 19233 : suffix_counter++;
5212 :
5213 19233 : bool have_self_recursive_calls = !self_recursive_calls.is_empty ();
5214 19350 : for (unsigned j = 0; j < self_recursive_calls.length (); j++)
5215 : {
5216 117 : cgraph_edge *cs = get_next_cgraph_edge_clone (self_recursive_calls[j]);
5217 : /* Cloned edges can disappear during cloning as speculation can be
5218 : resolved, check that we have one and that it comes from the last
5219 : cloning. */
5220 117 : if (cs && cs->caller == new_node)
5221 116 : cs->redirect_callee_duplicating_thunks (new_node);
5222 : /* Any future code that would make more than one clone of an outgoing
5223 : edge would confuse this mechanism, so let's check that does not
5224 : happen. */
5225 116 : gcc_checking_assert (!cs
5226 : || !get_next_cgraph_edge_clone (cs)
5227 : || get_next_cgraph_edge_clone (cs)->caller != new_node);
5228 : }
5229 19233 : if (have_self_recursive_calls)
5230 109 : new_node->expand_all_artificial_thunks ();
5231 :
5232 19233 : ipa_set_node_agg_value_chain (new_node, aggvals);
5233 50443 : for (const ipa_argagg_value &av : aggvals)
5234 31210 : new_node->maybe_create_reference (av.value, NULL);
5235 :
5236 19233 : if (dump_file && (dump_flags & TDF_DETAILS))
5237 : {
5238 91 : fprintf (dump_file, " the new node is %s.\n", new_node->dump_name ());
5239 91 : if (known_contexts.exists ())
5240 : {
5241 0 : for (i = 0; i < count; i++)
5242 0 : if (!known_contexts[i].useless_p ())
5243 : {
5244 0 : fprintf (dump_file, " known ctx %i is ", i);
5245 0 : known_contexts[i].dump (dump_file);
5246 : }
5247 : }
5248 91 : if (aggvals)
5249 : {
5250 49 : fprintf (dump_file, " Aggregate replacements:");
5251 49 : ipa_argagg_value_list avs (aggvals);
5252 49 : avs.dump (dump_file);
5253 : }
5254 : }
5255 :
5256 19233 : new_info = ipa_node_params_sum->get (new_node);
5257 19233 : new_info->ipcp_orig_node = node;
5258 19233 : new_node->ipcp_clone = true;
5259 19233 : new_info->known_csts = known_csts;
5260 19233 : new_info->known_contexts = known_contexts;
5261 :
5262 19233 : ipcp_discover_new_direct_edges (new_node, known_csts, known_contexts,
5263 : aggvals);
5264 :
5265 19233 : return new_node;
5266 19233 : }
5267 :
5268 : /* Return true if JFUNC, which describes a i-th parameter of call CS, is a
5269 : pass-through function to itself when the cgraph_node involved is not an
5270 : IPA-CP clone. When SIMPLE is true, further check if JFUNC is a simple
5271 : no-operation pass-through. */
5272 :
5273 : static bool
5274 792482 : self_recursive_pass_through_p (cgraph_edge *cs, ipa_jump_func *jfunc, int i,
5275 : bool simple = true)
5276 : {
5277 792482 : enum availability availability;
5278 792482 : if (jfunc->type == IPA_JF_PASS_THROUGH
5279 78903 : && cs->caller == cs->callee->function_symbol (&availability)
5280 19355 : && availability > AVAIL_INTERPOSABLE
5281 19355 : && (!simple || ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
5282 19355 : && ipa_get_jf_pass_through_formal_id (jfunc) == i
5283 19355 : && ipa_node_params_sum->get (cs->caller)
5284 811837 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5285 19326 : return true;
5286 : return false;
5287 : }
5288 :
5289 : /* Return true if JFUNC, which describes the i-th parameter of call CS, is an
5290 : ancestor function with zero offset to itself when the cgraph_node involved
5291 : is not an IPA-CP clone. */
5292 :
5293 : static bool
5294 773156 : self_recursive_ancestor_p (cgraph_edge *cs, ipa_jump_func *jfunc, int i)
5295 : {
5296 773156 : enum availability availability;
5297 773156 : if (jfunc->type == IPA_JF_ANCESTOR
5298 3255 : && cs->caller == cs->callee->function_symbol (&availability)
5299 1 : && availability > AVAIL_INTERPOSABLE
5300 1 : && ipa_get_jf_ancestor_offset (jfunc) == 0
5301 1 : && ipa_get_jf_ancestor_formal_id (jfunc) == i
5302 1 : && ipa_node_params_sum->get (cs->caller)
5303 773157 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5304 1 : return true;
5305 : return false;
5306 : }
5307 :
5308 : /* Return true if JFUNC, which describes a part of an aggregate represented or
5309 : pointed to by the i-th parameter of call CS, is a pass-through function to
5310 : itself when the cgraph_node involved is not an IPA-CP clone.. When
5311 : SIMPLE is true, further check if JFUNC is a simple no-operation
5312 : pass-through. */
5313 :
5314 : static bool
5315 359718 : self_recursive_agg_pass_through_p (const cgraph_edge *cs,
5316 : const ipa_agg_jf_item *jfunc,
5317 : int i, bool simple = true)
5318 : {
5319 359718 : enum availability availability;
5320 359718 : if (cs->caller == cs->callee->function_symbol (&availability)
5321 3819 : && availability > AVAIL_INTERPOSABLE
5322 3819 : && jfunc->jftype == IPA_JF_LOAD_AGG
5323 487 : && jfunc->offset == jfunc->value.load_agg.offset
5324 487 : && (!simple || jfunc->value.pass_through.operation == NOP_EXPR)
5325 487 : && jfunc->value.pass_through.formal_id == i
5326 481 : && useless_type_conversion_p (jfunc->value.load_agg.type, jfunc->type)
5327 481 : && ipa_node_params_sum->get (cs->caller)
5328 360199 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5329 481 : return true;
5330 : return false;
5331 : }
5332 :
5333 : /* Given a NODE, and a subset of its CALLERS, try to populate blanks slots in
5334 : KNOWN_CSTS with constants that are also known for all of the CALLERS. */
5335 :
5336 : static void
5337 168376 : find_scalar_values_for_callers_subset (vec<tree> &known_csts,
5338 : ipa_node_params *info,
5339 : const vec<cgraph_edge *> &callers)
5340 : {
5341 168376 : int i, count = ipa_get_param_count (info);
5342 :
5343 736524 : for (i = 0; i < count; i++)
5344 : {
5345 568148 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (info, i);
5346 568148 : if (lat->bottom)
5347 9231 : continue;
5348 558917 : if (lat->is_single_const ())
5349 : {
5350 29572 : known_csts[i] = lat->values->value;
5351 29572 : continue;
5352 : }
5353 :
5354 529345 : struct cgraph_edge *cs;
5355 529345 : tree newval = NULL_TREE;
5356 529345 : int j;
5357 529345 : bool first = true;
5358 529345 : tree type = ipa_get_type (info, i);
5359 :
5360 1520624 : FOR_EACH_VEC_ELT (callers, j, cs)
5361 : {
5362 791930 : struct ipa_jump_func *jump_func;
5363 791930 : tree t;
5364 :
5365 791930 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5366 791930 : if (!args
5367 791930 : || i >= ipa_get_cs_argument_count (args)
5368 1583829 : || (i == 0
5369 182287 : && call_passes_through_thunk (cs)))
5370 : {
5371 : newval = NULL_TREE;
5372 : break;
5373 : }
5374 791851 : jump_func = ipa_get_ith_jump_func (args, i);
5375 :
5376 : /* Besides simple pass-through jump function, arithmetic jump
5377 : function could also introduce argument-direct-pass-through for
5378 : self-feeding recursive call. For example,
5379 :
5380 : fn (int i)
5381 : {
5382 : fn (i & 1);
5383 : }
5384 :
5385 : Given that i is 0, recursive propagation via (i & 1) also gets
5386 : 0. */
5387 791851 : if (self_recursive_pass_through_p (cs, jump_func, i, false))
5388 : {
5389 18702 : gcc_assert (newval);
5390 18702 : enum tree_code opcode
5391 18702 : = ipa_get_jf_pass_through_operation (jump_func);
5392 18702 : tree op_type = (opcode == NOP_EXPR) ? NULL_TREE
5393 49 : : ipa_get_jf_pass_through_op_type (jump_func);
5394 18702 : t = ipa_get_jf_arith_result (opcode, newval,
5395 : ipa_get_jf_pass_through_operand (jump_func),
5396 : op_type);
5397 18702 : t = ipacp_value_safe_for_type (type, t);
5398 : }
5399 773149 : else if (self_recursive_ancestor_p (cs, jump_func, i))
5400 0 : continue;
5401 : else
5402 773149 : t = ipa_value_from_jfunc (ipa_node_params_sum->get (cs->caller),
5403 : jump_func, type);
5404 791851 : if (!t
5405 483677 : || (newval
5406 257673 : && !values_equal_for_ipcp_p (t, newval))
5407 1253785 : || (!first && !newval))
5408 : {
5409 : newval = NULL_TREE;
5410 : break;
5411 : }
5412 : else
5413 : newval = t;
5414 : first = false;
5415 : }
5416 :
5417 529345 : if (newval)
5418 199349 : known_csts[i] = newval;
5419 : }
5420 168376 : }
5421 :
5422 : /* Given a NODE and a subset of its CALLERS, try to populate plank slots in
5423 : KNOWN_CONTEXTS with polymorphic contexts that are also known for all of the
5424 : CALLERS. */
5425 :
5426 : static void
5427 168376 : find_contexts_for_caller_subset (vec<ipa_polymorphic_call_context>
5428 : &known_contexts,
5429 : ipa_node_params *info,
5430 : const vec<cgraph_edge *> &callers)
5431 : {
5432 168376 : int i, count = ipa_get_param_count (info);
5433 :
5434 736505 : for (i = 0; i < count; i++)
5435 : {
5436 568142 : if (!ipa_is_param_used (info, i))
5437 23516 : continue;
5438 :
5439 545759 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat
5440 545759 : = ipa_get_poly_ctx_lat (info, i);
5441 545759 : if (ctxlat->bottom)
5442 0 : continue;
5443 545759 : if (ctxlat->is_single_const ())
5444 : {
5445 1133 : if (!ctxlat->values->value.useless_p ())
5446 : {
5447 1133 : if (known_contexts.is_empty ())
5448 1072 : known_contexts.safe_grow_cleared (count, true);
5449 1133 : known_contexts[i] = ctxlat->values->value;
5450 : }
5451 1133 : continue;
5452 : }
5453 :
5454 544626 : cgraph_edge *cs;
5455 544626 : ipa_polymorphic_call_context newval;
5456 544626 : bool first = true;
5457 544626 : int j;
5458 :
5459 550044 : FOR_EACH_VEC_ELT (callers, j, cs)
5460 : {
5461 546228 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5462 546228 : if (!args
5463 1092456 : || i >= ipa_get_cs_argument_count (args))
5464 13 : return;
5465 546215 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
5466 546215 : ipa_polymorphic_call_context ctx;
5467 546215 : ctx = ipa_context_from_jfunc (ipa_node_params_sum->get (cs->caller),
5468 : cs, i, jfunc);
5469 546215 : if (first)
5470 : {
5471 544613 : newval = ctx;
5472 544613 : first = false;
5473 : }
5474 : else
5475 1602 : newval.meet_with (ctx);
5476 1089632 : if (newval.useless_p ())
5477 : break;
5478 : }
5479 :
5480 1089226 : if (!newval.useless_p ())
5481 : {
5482 3816 : if (known_contexts.is_empty ())
5483 3593 : known_contexts.safe_grow_cleared (count, true);
5484 3816 : known_contexts[i] = newval;
5485 : }
5486 :
5487 : }
5488 : }
5489 :
5490 : /* Push all aggregate values coming along edge CS for parameter number INDEX to
5491 : RES. If INTERIM is non-NULL, it contains the current interim state of
5492 : collected aggregate values which can be used to compute values passed over
5493 : self-recursive edges.
5494 :
5495 : This basically one iteration of push_agg_values_from_edge over one
5496 : parameter, which allows for simpler early returns. */
5497 :
5498 : static void
5499 635125 : push_agg_values_for_index_from_edge (struct cgraph_edge *cs, int index,
5500 : vec<ipa_argagg_value> *res,
5501 : const ipa_argagg_value_list *interim)
5502 : {
5503 635125 : bool agg_values_from_caller = false;
5504 635125 : bool agg_jf_preserved = false;
5505 635125 : unsigned unit_delta = UINT_MAX;
5506 635125 : int src_idx = -1;
5507 635125 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (ipa_edge_args_sum->get (cs),
5508 : index);
5509 :
5510 635125 : if (jfunc->type == IPA_JF_PASS_THROUGH
5511 635125 : && ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
5512 : {
5513 58260 : agg_values_from_caller = true;
5514 58260 : agg_jf_preserved = ipa_get_jf_pass_through_agg_preserved (jfunc);
5515 58260 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
5516 58260 : unit_delta = 0;
5517 : }
5518 576865 : else if (jfunc->type == IPA_JF_ANCESTOR
5519 576865 : && ipa_get_jf_ancestor_agg_preserved (jfunc))
5520 : {
5521 407 : agg_values_from_caller = true;
5522 407 : agg_jf_preserved = true;
5523 407 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
5524 407 : unit_delta = ipa_get_jf_ancestor_offset (jfunc) / BITS_PER_UNIT;
5525 : }
5526 :
5527 635125 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
5528 635125 : if (agg_values_from_caller)
5529 : {
5530 58667 : if (caller_info->ipcp_orig_node)
5531 : {
5532 11231 : struct cgraph_node *orig_node = caller_info->ipcp_orig_node;
5533 11231 : ipcp_transformation *ts
5534 11231 : = ipcp_get_transformation_summary (cs->caller);
5535 11231 : ipa_node_params *orig_info = ipa_node_params_sum->get (orig_node);
5536 11231 : ipcp_param_lattices *orig_plats
5537 11231 : = ipa_get_parm_lattices (orig_info, src_idx);
5538 11231 : if (ts
5539 11231 : && orig_plats->aggs
5540 3017 : && (agg_jf_preserved || !orig_plats->aggs_by_ref))
5541 : {
5542 2546 : ipa_argagg_value_list src (ts);
5543 2546 : src.push_adjusted_values (src_idx, index, unit_delta, res);
5544 2546 : return;
5545 : }
5546 : }
5547 : else
5548 : {
5549 47436 : ipcp_param_lattices *src_plats
5550 47436 : = ipa_get_parm_lattices (caller_info, src_idx);
5551 47436 : if (src_plats->aggs
5552 2437 : && !src_plats->aggs_bottom
5553 2437 : && (agg_jf_preserved || !src_plats->aggs_by_ref))
5554 : {
5555 1449 : if (interim && (self_recursive_pass_through_p (cs, jfunc, index)
5556 7 : || self_recursive_ancestor_p (cs, jfunc, index)))
5557 : {
5558 625 : interim->push_adjusted_values (src_idx, index, unit_delta,
5559 : res);
5560 625 : return;
5561 : }
5562 824 : if (!src_plats->aggs_contain_variable)
5563 : {
5564 83 : push_agg_values_from_plats (src_plats, index, unit_delta,
5565 : res);
5566 83 : return;
5567 : }
5568 : }
5569 : }
5570 : }
5571 :
5572 631871 : if (!jfunc->agg.items)
5573 : return;
5574 226414 : bool first = true;
5575 226414 : unsigned prev_unit_offset = 0;
5576 1266800 : for (const ipa_agg_jf_item &agg_jf : *jfunc->agg.items)
5577 : {
5578 1040386 : tree value, srcvalue;
5579 : /* Besides simple pass-through aggregate jump function, arithmetic
5580 : aggregate jump function could also bring same aggregate value as
5581 : parameter passed-in for self-feeding recursive call. For example,
5582 :
5583 : fn (int *i)
5584 : {
5585 : int j = *i & 1;
5586 : fn (&j);
5587 : }
5588 :
5589 : Given that *i is 0, recursive propagation via (*i & 1) also gets 0. */
5590 1040386 : if (interim
5591 359718 : && self_recursive_agg_pass_through_p (cs, &agg_jf, index, false)
5592 1040867 : && (srcvalue = interim->get_value(index,
5593 481 : agg_jf.offset / BITS_PER_UNIT)))
5594 : {
5595 950 : value = ipa_get_jf_arith_result (agg_jf.value.pass_through.operation,
5596 : srcvalue,
5597 475 : agg_jf.value.pass_through.operand,
5598 475 : agg_jf.value.pass_through.op_type);
5599 475 : value = ipacp_value_safe_for_type (agg_jf.type, value);
5600 : }
5601 : else
5602 1039911 : value = ipa_agg_value_from_jfunc (caller_info, cs->caller,
5603 : &agg_jf);
5604 1040386 : if (value)
5605 : {
5606 1014489 : struct ipa_argagg_value iav;
5607 1014489 : iav.value = value;
5608 1014489 : iav.unit_offset = agg_jf.offset / BITS_PER_UNIT;
5609 1014489 : iav.index = index;
5610 1014489 : iav.by_ref = jfunc->agg.by_ref;
5611 1014489 : iav.killed = false;
5612 :
5613 1014489 : gcc_assert (first
5614 : || iav.unit_offset > prev_unit_offset);
5615 1014489 : prev_unit_offset = iav.unit_offset;
5616 1014489 : first = false;
5617 :
5618 1014489 : res->safe_push (iav);
5619 : }
5620 : }
5621 : return;
5622 : }
5623 :
5624 : /* Push all aggregate values coming along edge CS to RES. DEST_INFO is the
5625 : description of ultimate callee of CS or the one it was cloned from (the
5626 : summary where lattices are). If INTERIM is non-NULL, it contains the
5627 : current interim state of collected aggregate values which can be used to
5628 : compute values passed over self-recursive edges (if OPTIMIZE_SELF_RECURSION
5629 : is true) and to skip values which clearly will not be part of intersection
5630 : with INTERIM. */
5631 :
5632 : static void
5633 224908 : push_agg_values_from_edge (struct cgraph_edge *cs,
5634 : ipa_node_params *dest_info,
5635 : vec<ipa_argagg_value> *res,
5636 : const ipa_argagg_value_list *interim,
5637 : bool optimize_self_recursion)
5638 : {
5639 224908 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5640 224908 : if (!args)
5641 : return;
5642 :
5643 449816 : int count = MIN (ipa_get_param_count (dest_info),
5644 : ipa_get_cs_argument_count (args));
5645 :
5646 224908 : unsigned interim_index = 0;
5647 943684 : for (int index = 0; index < count; index++)
5648 : {
5649 718776 : if (interim)
5650 : {
5651 295512 : while (interim_index < interim->m_elts.size ()
5652 270131 : && interim->m_elts[interim_index].value
5653 519894 : && interim->m_elts[interim_index].index < index)
5654 144853 : interim_index++;
5655 208428 : if (interim_index >= interim->m_elts.size ()
5656 150659 : || interim->m_elts[interim_index].index > index)
5657 57769 : continue;
5658 : }
5659 :
5660 661007 : ipcp_param_lattices *plats = ipa_get_parm_lattices (dest_info, index);
5661 661007 : if (!ipa_is_param_used (dest_info, index)
5662 661007 : || plats->aggs_bottom)
5663 25882 : continue;
5664 635174 : push_agg_values_for_index_from_edge (cs, index, res,
5665 : optimize_self_recursion ? interim
5666 : : NULL);
5667 : }
5668 : }
5669 :
5670 :
5671 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5672 : from all of them into INTERIM. Return how many there are. */
5673 :
5674 : static unsigned int
5675 168376 : find_aggregate_values_for_callers_subset_1 (vec<ipa_argagg_value> &interim,
5676 : struct cgraph_node *node,
5677 : const vec<cgraph_edge *> &callers)
5678 : {
5679 168376 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5680 168376 : if (dest_info->ipcp_orig_node)
5681 0 : dest_info = ipa_node_params_sum->get (dest_info->ipcp_orig_node);
5682 :
5683 : /* gather_edges_for_value puts a non-recursive call into the first element of
5684 : callers if it can. */
5685 168376 : push_agg_values_from_edge (callers[0], dest_info, &interim, NULL, true);
5686 :
5687 168376 : unsigned valid_entries = interim.length ();
5688 168376 : if (!valid_entries)
5689 : return 0;
5690 :
5691 86421 : unsigned caller_count = callers.length();
5692 141235 : for (unsigned i = 1; i < caller_count; i++)
5693 : {
5694 56486 : auto_vec<ipa_argagg_value, 32> last;
5695 56486 : ipa_argagg_value_list avs (&interim);
5696 56486 : push_agg_values_from_edge (callers[i], dest_info, &last, &avs, true);
5697 :
5698 56486 : valid_entries = intersect_argaggs_with (interim, last);
5699 56486 : if (!valid_entries)
5700 1672 : return 0;
5701 56486 : }
5702 :
5703 : return valid_entries;
5704 : }
5705 :
5706 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5707 : from all of them and return them in a garbage-collected vector. Return
5708 : nullptr if there are none. */
5709 :
5710 : static void
5711 153568 : find_aggregate_values_for_callers_subset (vec<ipa_argagg_value> &res,
5712 : struct cgraph_node *node,
5713 : const vec<cgraph_edge *> &callers)
5714 : {
5715 153568 : auto_vec<ipa_argagg_value, 32> interim;
5716 153568 : unsigned valid_entries
5717 153568 : = find_aggregate_values_for_callers_subset_1 (interim, node, callers);
5718 153568 : if (!valid_entries)
5719 : return;
5720 :
5721 879395 : for (const ipa_argagg_value &av : interim)
5722 641036 : if (av.value)
5723 606316 : res.safe_push(av);
5724 : return;
5725 153568 : }
5726 :
5727 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5728 : from all of them and return them in a garbage-collected vector. Return
5729 : nullptr if there are none. */
5730 :
5731 : static struct vec<ipa_argagg_value, va_gc> *
5732 14808 : find_aggregate_values_for_callers_subset_gc (struct cgraph_node *node,
5733 : const vec<cgraph_edge *> &callers)
5734 : {
5735 14808 : auto_vec<ipa_argagg_value, 32> interim;
5736 14808 : unsigned valid_entries
5737 14808 : = find_aggregate_values_for_callers_subset_1 (interim, node, callers);
5738 14808 : if (!valid_entries)
5739 : return nullptr;
5740 :
5741 5296 : vec<ipa_argagg_value, va_gc> *res = NULL;
5742 5296 : vec_safe_reserve_exact (res, valid_entries);
5743 37369 : for (const ipa_argagg_value &av : interim)
5744 21481 : if (av.value)
5745 20098 : res->quick_push(av);
5746 5296 : gcc_checking_assert (res->length () == valid_entries);
5747 : return res;
5748 14808 : }
5749 :
5750 : /* Determine whether CS also brings all scalar values that the NODE is
5751 : specialized for. */
5752 :
5753 : static bool
5754 78 : cgraph_edge_brings_all_scalars_for_node (struct cgraph_edge *cs,
5755 : struct cgraph_node *node)
5756 : {
5757 78 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5758 78 : int count = ipa_get_param_count (dest_info);
5759 78 : class ipa_node_params *caller_info;
5760 78 : class ipa_edge_args *args;
5761 78 : int i;
5762 :
5763 78 : caller_info = ipa_node_params_sum->get (cs->caller);
5764 78 : args = ipa_edge_args_sum->get (cs);
5765 170 : for (i = 0; i < count; i++)
5766 : {
5767 114 : struct ipa_jump_func *jump_func;
5768 114 : tree val, t;
5769 :
5770 114 : val = dest_info->known_csts[i];
5771 114 : if (!val)
5772 73 : continue;
5773 :
5774 82 : if (i >= ipa_get_cs_argument_count (args))
5775 : return false;
5776 41 : jump_func = ipa_get_ith_jump_func (args, i);
5777 41 : t = ipa_value_from_jfunc (caller_info, jump_func,
5778 : ipa_get_type (dest_info, i));
5779 41 : if (!t || !values_equal_for_ipcp_p (val, t))
5780 : return false;
5781 : }
5782 : return true;
5783 : }
5784 :
5785 : /* Determine whether CS also brings all aggregate values that NODE is
5786 : specialized for. */
5787 :
5788 : static bool
5789 56 : cgraph_edge_brings_all_agg_vals_for_node (struct cgraph_edge *cs,
5790 : struct cgraph_node *node)
5791 : {
5792 56 : ipcp_transformation *ts = ipcp_get_transformation_summary (node);
5793 56 : if (!ts || vec_safe_is_empty (ts->m_agg_values))
5794 : return true;
5795 :
5796 46 : const ipa_argagg_value_list existing (ts->m_agg_values);
5797 46 : auto_vec<ipa_argagg_value, 32> edge_values;
5798 46 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5799 46 : gcc_checking_assert (dest_info->ipcp_orig_node);
5800 46 : dest_info = ipa_node_params_sum->get (dest_info->ipcp_orig_node);
5801 46 : push_agg_values_from_edge (cs, dest_info, &edge_values, &existing, false);
5802 46 : const ipa_argagg_value_list avl (&edge_values);
5803 46 : return avl.superset_of_p (existing);
5804 46 : }
5805 :
5806 : /* Given an original NODE and a VAL for which we have already created a
5807 : specialized clone, look whether there are incoming edges that still lead
5808 : into the old node but now also bring the requested value and also conform to
5809 : all other criteria such that they can be redirected the special node.
5810 : This function can therefore redirect the final edge in a SCC. */
5811 :
5812 : template <typename valtype>
5813 : static void
5814 9051 : perhaps_add_new_callers (cgraph_node *node, ipcp_value<valtype> *val)
5815 : {
5816 : ipcp_value_source<valtype> *src;
5817 9051 : profile_count redirected_sum = profile_count::zero ();
5818 :
5819 123194 : for (src = val->sources; src; src = src->next)
5820 : {
5821 114143 : struct cgraph_edge *cs = src->cs;
5822 354239 : while (cs)
5823 : {
5824 240096 : if (cgraph_edge_brings_value_p (cs, src, node, val)
5825 78 : && cgraph_edge_brings_all_scalars_for_node (cs, val->spec_node)
5826 240152 : && cgraph_edge_brings_all_agg_vals_for_node (cs, val->spec_node))
5827 : {
5828 39 : if (dump_file)
5829 3 : fprintf (dump_file, " - adding an extra caller %s of %s\n",
5830 3 : cs->caller->dump_name (),
5831 3 : val->spec_node->dump_name ());
5832 :
5833 39 : cs->redirect_callee_duplicating_thunks (val->spec_node);
5834 39 : val->spec_node->expand_all_artificial_thunks ();
5835 39 : if (cs->count.ipa ().initialized_p ())
5836 0 : redirected_sum = redirected_sum + cs->count.ipa ();
5837 : }
5838 240096 : cs = get_next_cgraph_edge_clone (cs);
5839 : }
5840 : }
5841 :
5842 9051 : if (redirected_sum.nonzero_p ())
5843 0 : update_specialized_profile (val->spec_node, node, redirected_sum);
5844 9051 : }
5845 :
5846 : /* Return true if KNOWN_CONTEXTS contain at least one useful context. */
5847 :
5848 : static bool
5849 4425 : known_contexts_useful_p (vec<ipa_polymorphic_call_context> known_contexts)
5850 : {
5851 4425 : ipa_polymorphic_call_context *ctx;
5852 4425 : int i;
5853 :
5854 4425 : FOR_EACH_VEC_ELT (known_contexts, i, ctx)
5855 99 : if (!ctx->useless_p ())
5856 : return true;
5857 : return false;
5858 : }
5859 :
5860 : /* Return a copy of KNOWN_CSTS if it is not empty, otherwise return vNULL. */
5861 :
5862 : static vec<ipa_polymorphic_call_context>
5863 4425 : copy_useful_known_contexts (const vec<ipa_polymorphic_call_context> &known_contexts)
5864 : {
5865 4425 : if (known_contexts_useful_p (known_contexts))
5866 99 : return known_contexts.copy ();
5867 : else
5868 4326 : return vNULL;
5869 : }
5870 :
5871 : /* Return true if the VALUE is represented in KNOWN_CSTS at INDEX if OFFSET is
5872 : minus one or in AGGVALS for INDEX and OFFSET otherwise. */
5873 :
5874 : DEBUG_FUNCTION bool
5875 4374 : ipcp_val_replacement_ok_p (vec<tree> &known_csts,
5876 : vec<ipa_polymorphic_call_context> &,
5877 : vec<ipa_argagg_value, va_gc> *aggvals,
5878 : int index, HOST_WIDE_INT offset, tree value)
5879 : {
5880 4374 : tree v;
5881 4374 : if (offset == -1)
5882 3147 : v = known_csts[index];
5883 : else
5884 : {
5885 1227 : const ipa_argagg_value_list avl (aggvals);
5886 1227 : v = avl.get_value (index, offset / BITS_PER_UNIT);
5887 : }
5888 :
5889 4374 : return v && values_equal_for_ipcp_p (v, value);
5890 : }
5891 :
5892 : /* Dump to F all the values in AVALS for which we are re-evaluating the effects
5893 : on the function represented b INFO. */
5894 :
5895 : DEBUG_FUNCTION void
5896 68 : dump_reestimation_message (FILE *f, ipa_node_params *info,
5897 : const ipa_auto_call_arg_values &avals)
5898 : {
5899 68 : fprintf (f, " Re-estimating effects with\n"
5900 : " Scalar constants:");
5901 68 : int param_count = ipa_get_param_count (info);
5902 168 : for (int i = 0; i < param_count; i++)
5903 100 : if (avals.m_known_vals[i])
5904 : {
5905 44 : fprintf (f, " %i:", i);
5906 44 : print_ipcp_constant_value (f, avals.m_known_vals[i]);
5907 : }
5908 68 : fprintf (f, "\n");
5909 68 : if (!avals.m_known_contexts.is_empty ())
5910 : {
5911 0 : fprintf (f, " Pol. contexts:");
5912 0 : for (int i = 0; i < param_count; i++)
5913 0 : if (!avals.m_known_contexts[i].useless_p ())
5914 : {
5915 0 : fprintf (f, " %i:", i);
5916 0 : avals.m_known_contexts[i].dump (f);
5917 : }
5918 0 : fprintf (f, "\n");
5919 : }
5920 68 : if (!avals.m_known_aggs.is_empty ())
5921 : {
5922 24 : fprintf (f, " Aggregate replacements:");
5923 24 : ipa_argagg_value_list avs (&avals);
5924 24 : avs.dump (f);
5925 : }
5926 68 : }
5927 :
5928 : /* Return true if the VALUE is represented in KNOWN_CONTEXTS at INDEX and that
5929 : if OFFSET is is equal to minus one (because source of a polymorphic context
5930 : cannot be an aggregate value). */
5931 :
5932 : DEBUG_FUNCTION bool
5933 51 : ipcp_val_replacement_ok_p (vec<tree> &,
5934 : vec<ipa_polymorphic_call_context> &known_contexts,
5935 : vec<ipa_argagg_value, va_gc> *,
5936 : int index, HOST_WIDE_INT offset,
5937 : ipa_polymorphic_call_context value)
5938 : {
5939 51 : if (offset != -1
5940 51 : || known_contexts.length () <= (unsigned) index
5941 102 : || known_contexts[index].useless_p ())
5942 : return false;
5943 :
5944 51 : if (known_contexts[index].equal_to (value))
5945 : return true;
5946 :
5947 : /* In some corner cases, the final gathering of contexts can figure out that
5948 : the available context is actually more precise than what we wanted to
5949 : clone for. Allow it. */
5950 0 : value.combine_with (known_contexts[index]);
5951 0 : return known_contexts[index].equal_to (value);
5952 : }
5953 :
5954 : /* Decide whether to create a special version of NODE for value VAL of
5955 : parameter at the given INDEX. If OFFSET is -1, the value is for the
5956 : parameter itself, otherwise it is stored at the given OFFSET of the
5957 : parameter. AVALS describes the other already known values. SELF_GEN_CLONES
5958 : is a vector which contains clones created for self-recursive calls with an
5959 : arithmetic pass-through jump function. CUR_SWEEP is the number of the
5960 : current sweep of the call-graph during the decision stage. */
5961 :
5962 : template <typename valtype>
5963 : static bool
5964 227270 : decide_about_value (struct cgraph_node *node, int index, HOST_WIDE_INT offset,
5965 : ipcp_value<valtype> *val,
5966 : vec<cgraph_node *> *self_gen_clones, int cur_sweep)
5967 : {
5968 : int caller_count;
5969 227270 : sreal freq_sum;
5970 : profile_count count_sum, rec_count_sum;
5971 : bool called_without_ipa_profile;
5972 :
5973 227270 : if (val->spec_node)
5974 : {
5975 9051 : perhaps_add_new_callers (node, val);
5976 9051 : return false;
5977 : }
5978 218219 : else if (val->local_size_cost + overall_size > get_max_overall_size (node))
5979 : {
5980 450 : if (dump_file && (dump_flags & TDF_DETAILS))
5981 0 : fprintf (dump_file, " - ignoring candidate value because "
5982 : "maximum unit size would be reached with %li.\n",
5983 : val->local_size_cost + overall_size);
5984 : return false;
5985 : }
5986 217769 : else if (!get_info_about_necessary_edges (val, node, &freq_sum, &caller_count,
5987 : &rec_count_sum, &count_sum,
5988 : &called_without_ipa_profile))
5989 : {
5990 64201 : if (dump_file && (dump_flags & TDF_DETAILS))
5991 : {
5992 121 : fprintf (dump_file, " - skipping candidate value ");
5993 121 : print_ipcp_constant_value (dump_file, val->value);
5994 121 : fprintf (dump_file, " for ");
5995 121 : ipa_dump_param (dump_file, ipa_node_params_sum->get (node), index);
5996 121 : if (offset != -1)
5997 105 : fprintf (dump_file, ", offset: " HOST_WIDE_INT_PRINT_DEC, offset);
5998 121 : fprintf (dump_file, ": no relevant callers\n");
5999 : }
6000 : return false;
6001 : }
6002 :
6003 153568 : if (!dbg_cnt (ipa_cp_values))
6004 : return false;
6005 :
6006 153568 : if (val->self_recursion_generated_p ())
6007 : {
6008 : /* The edge counts in this case might not have been adjusted yet.
6009 : Nevertleless, even if they were it would be only a guesswork which we
6010 : can do now. The recursive part of the counts can be derived from the
6011 : count of the original node anyway. */
6012 293 : if (node->count.ipa ().nonzero_p ())
6013 : {
6014 14 : unsigned dem = self_gen_clones->length () + 1;
6015 14 : rec_count_sum = node->count.ipa () / dem;
6016 : }
6017 : else
6018 265 : rec_count_sum = profile_count::zero ();
6019 : }
6020 :
6021 : /* get_info_about_necessary_edges only sums up ipa counts. */
6022 153568 : count_sum += rec_count_sum;
6023 :
6024 153568 : if (dump_file && (dump_flags & TDF_DETAILS))
6025 : {
6026 135 : fprintf (dump_file, " - considering value ");
6027 135 : print_ipcp_constant_value (dump_file, val->value);
6028 135 : fprintf (dump_file, " for ");
6029 135 : ipa_dump_param (dump_file, ipa_node_params_sum->get (node), index);
6030 135 : if (offset != -1)
6031 62 : fprintf (dump_file, ", offset: " HOST_WIDE_INT_PRINT_DEC, offset);
6032 135 : fprintf (dump_file, " (caller_count: %i)\n", caller_count);
6033 : }
6034 :
6035 153568 : auto_vec<cgraph_edge *> callers
6036 : = gather_edges_for_value (val, node, caller_count);
6037 153568 : ipa_node_params *info = ipa_node_params_sum->get (node);
6038 153568 : ipa_auto_call_arg_values avals;
6039 153568 : avals.m_known_vals.safe_grow_cleared (ipa_get_param_count (info), true);
6040 153568 : find_scalar_values_for_callers_subset (avals.m_known_vals, info, callers);
6041 153568 : find_contexts_for_caller_subset (avals.m_known_contexts, info, callers);
6042 153568 : find_aggregate_values_for_callers_subset (avals.m_known_aggs, node, callers);
6043 :
6044 :
6045 153568 : if (good_cloning_opportunity_p (node, val->prop_time_benefit,
6046 : freq_sum, count_sum, val->prop_size_cost,
6047 : called_without_ipa_profile, cur_sweep))
6048 : ;
6049 : else
6050 : {
6051 : /* Extern inline functions are only meaningful to clione to propagate
6052 : values to their callees. */
6053 151629 : if (DECL_EXTERNAL (node->decl) && DECL_DECLARED_INLINE_P (node->decl))
6054 : {
6055 345 : if (dump_file && (dump_flags & TDF_DETAILS))
6056 0 : fprintf (dump_file, " Skipping extern inline.\n");
6057 149143 : return false;
6058 : }
6059 151284 : if (dump_file && (dump_flags & TDF_DETAILS))
6060 68 : dump_reestimation_message (dump_file, info, avals);
6061 :
6062 151284 : ipa_call_estimates estimates;
6063 151284 : estimate_ipcp_clone_size_and_time (node, &avals, &estimates);
6064 151284 : int removable_params_cost = 0;
6065 976309 : for (tree t : avals.m_known_vals)
6066 522457 : if (t)
6067 209420 : removable_params_cost += estimate_move_cost (TREE_TYPE (t), true);
6068 :
6069 151284 : int size = estimates.size - caller_count * removable_params_cost;
6070 :
6071 151284 : if (size <= 0)
6072 : {
6073 1803 : if (dump_file)
6074 0 : fprintf (dump_file, " Code not going to grow.\n");
6075 : }
6076 : else
6077 : {
6078 149481 : sreal time_benefit
6079 149481 : = ((estimates.nonspecialized_time - estimates.time)
6080 298962 : + hint_time_bonus (node, estimates)
6081 149481 : + (devirtualization_time_bonus (node, &avals)
6082 149481 : + removable_params_cost));
6083 :
6084 149481 : if (!good_cloning_opportunity_p (node, time_benefit, freq_sum,
6085 : count_sum, size,
6086 : called_without_ipa_profile,
6087 : cur_sweep))
6088 148798 : return false;
6089 : }
6090 : }
6091 :
6092 4425 : if (dump_file)
6093 142 : fprintf (dump_file, " Creating a specialized node of %s.\n",
6094 : node->dump_name ());
6095 :
6096 4425 : vec<tree> known_csts = avals.m_known_vals.copy ();
6097 4425 : vec<ipa_polymorphic_call_context> known_contexts
6098 4425 : = copy_useful_known_contexts (avals.m_known_contexts);
6099 :
6100 4425 : vec<ipa_argagg_value, va_gc> *aggvals = NULL;
6101 4425 : vec_safe_reserve_exact (aggvals, avals.m_known_aggs.length ());
6102 24387 : for (const ipa_argagg_value &av : avals.m_known_aggs)
6103 11112 : aggvals->quick_push (av);
6104 4425 : gcc_checking_assert (ipcp_val_replacement_ok_p (known_csts, known_contexts,
6105 : aggvals, index,
6106 : offset, val->value));
6107 4425 : val->spec_node = create_specialized_node (node, known_csts, known_contexts,
6108 : aggvals, callers);
6109 :
6110 4425 : if (val->self_recursion_generated_p ())
6111 142 : self_gen_clones->safe_push (val->spec_node);
6112 : else
6113 4283 : update_profiling_info (node, val->spec_node);
6114 :
6115 4425 : overall_size += val->local_size_cost;
6116 4425 : if (dump_file && (dump_flags & TDF_DETAILS))
6117 68 : fprintf (dump_file, " overall size reached %li\n",
6118 : overall_size);
6119 :
6120 : /* TODO: If for some lattice there is only one other known value
6121 : left, make a special node for it too. */
6122 :
6123 : return true;
6124 153568 : }
6125 :
6126 : /* Like irange::contains_p(), but convert VAL to the range of R if
6127 : necessary. */
6128 :
6129 : static inline bool
6130 48174 : ipa_range_contains_p (const vrange &r, tree val)
6131 : {
6132 48174 : if (r.undefined_p ())
6133 : return false;
6134 :
6135 48174 : tree type = r.type ();
6136 48174 : if (!wi::fits_to_tree_p (wi::to_wide (val), type))
6137 : return false;
6138 :
6139 48174 : val = fold_convert (type, val);
6140 48174 : return r.contains_p (val);
6141 : }
6142 :
6143 : /* Structure holding opportunitties so that they can be pre-sorted. */
6144 :
6145 227270 : struct cloning_opportunity_ranking
6146 : {
6147 : /* A very rough evaluation of likely benefit. */
6148 : sreal eval;
6149 : /* In the case of aggregate constants, a non-negative offset within their
6150 : aggregates. -1 for scalar constants, -2 for polymorphic contexts. */
6151 : HOST_WIDE_INT offset;
6152 : /* The value being considered for evaluation for cloning. */
6153 : ipcp_value_base *val;
6154 : /* Index of the formal parameter the value is coming in. */
6155 : int index;
6156 : };
6157 :
6158 : /* Helper function to qsort a vector of cloning opportunities. */
6159 :
6160 : static int
6161 2190167 : compare_cloning_opportunities (const void *a, const void *b)
6162 : {
6163 2190167 : const cloning_opportunity_ranking *o1
6164 : = (const cloning_opportunity_ranking *) a;
6165 2190167 : const cloning_opportunity_ranking *o2
6166 : = (const cloning_opportunity_ranking *) b;
6167 2190167 : if (o1->eval < o2->eval)
6168 : return 1;
6169 1707687 : if (o1->eval > o2->eval)
6170 563608 : return -1;
6171 : return 0;
6172 : }
6173 :
6174 : /* Use the estimations in VAL to determine how good a candidate it represents
6175 : for the purposes of ordering real evaluation of opportunities (which
6176 : includes information about incoming edges, among other things). */
6177 :
6178 : static sreal
6179 227270 : cloning_opportunity_ranking_evaluation (const ipcp_value_base *val)
6180 : {
6181 227270 : sreal e1 = (val->local_time_benefit * 1000) / MAX (val->local_size_cost, 1);
6182 227270 : sreal e2 = (val->prop_time_benefit * 1000) / MAX (val->prop_size_cost, 1);
6183 227270 : if (e2 > e1)
6184 15707 : return e2;
6185 : else
6186 211563 : return e1;
6187 : }
6188 :
6189 : /* Decide whether and what specialized clones of NODE should be created.
6190 : CUR_SWEEP is the number of the current sweep of the call-graph during the
6191 : decision stage. */
6192 :
6193 : static bool
6194 3318581 : decide_whether_version_node (struct cgraph_node *node, int cur_sweep)
6195 : {
6196 3318581 : ipa_node_params *info = ipa_node_params_sum->get (node);
6197 3318581 : int count = ipa_get_param_count (info);
6198 3318581 : bool ret = false;
6199 :
6200 3318581 : if (info->node_dead || count == 0)
6201 : return false;
6202 :
6203 2688770 : bool clone_for_all_contexts = node->local;
6204 2688770 : if (dump_file && (dump_flags & TDF_DETAILS))
6205 : {
6206 345 : fprintf (dump_file, "\nEvaluating opportunities for %s.",
6207 : node->dump_name ());
6208 345 : if (clone_for_all_contexts)
6209 104 : fprintf (dump_file, " Will try to create a special all-context "
6210 : "clone.\n");
6211 345 : fprintf (dump_file, "\n");
6212 : }
6213 :
6214 2688770 : auto_vec <cloning_opportunity_ranking, 32> opp_ranking;
6215 8970673 : for (int i = 0; i < count;i++)
6216 : {
6217 6281903 : if (!ipa_is_param_used (info, i))
6218 : {
6219 704226 : if (dump_file && (dump_flags & TDF_DETAILS))
6220 20 : fprintf (dump_file, " - ignoring unused parameter %i.\n", i);
6221 704226 : continue;
6222 : }
6223 :
6224 5577677 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6225 5577677 : ipcp_lattice<tree> *lat = &plats->itself;
6226 5577677 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
6227 :
6228 5577677 : if (!lat->bottom
6229 5577677 : && (!clone_for_all_contexts || !lat->is_single_const ()))
6230 : {
6231 550870 : ipcp_value<tree> *val;
6232 668703 : for (val = lat->values; val; val = val->next)
6233 : {
6234 : /* If some values generated for self-recursive calls with
6235 : arithmetic jump functions fall outside of the known
6236 : range for the parameter, we can skip them. */
6237 117895 : if (TREE_CODE (val->value) == INTEGER_CST
6238 71288 : && !plats->m_value_range.bottom_p ()
6239 166007 : && !ipa_range_contains_p (plats->m_value_range.m_vr,
6240 : val->value))
6241 : {
6242 : /* This can happen also if a constant present in the source
6243 : code falls outside of the range of parameter's type, so we
6244 : cannot assert. */
6245 62 : if (dump_file && (dump_flags & TDF_DETAILS))
6246 : {
6247 0 : fprintf (dump_file, " - skipping%s value ",
6248 0 : val->self_recursion_generated_p ()
6249 : ? " self_recursion_generated" : "");
6250 0 : print_ipcp_constant_value (dump_file, val->value);
6251 0 : fprintf (dump_file, " because it is outside known "
6252 : "value range.\n");
6253 : }
6254 62 : continue;
6255 : }
6256 117771 : cloning_opportunity_ranking opp;
6257 117771 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6258 117771 : opp.offset = -1;
6259 117771 : opp.val = val;
6260 117771 : opp.index = i;
6261 117771 : opp_ranking.safe_push (opp);
6262 : }
6263 : }
6264 :
6265 5577677 : if (!plats->aggs_bottom)
6266 : {
6267 580291 : struct ipcp_agg_lattice *aglat;
6268 580291 : ipcp_value<tree> *val;
6269 726458 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
6270 145085 : if (!aglat->bottom && aglat->values
6271 : /* If the following is false, the one value will be considered
6272 : for cloning for all contexts. */
6273 269668 : && (!clone_for_all_contexts
6274 77816 : || plats->aggs_contain_variable
6275 199410 : || !aglat->is_single_const ()))
6276 186757 : for (val = aglat->values; val; val = val->next)
6277 : {
6278 105662 : cloning_opportunity_ranking opp;
6279 105662 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6280 105662 : opp.offset = aglat->offset;
6281 105662 : opp.val = val;
6282 105662 : opp.index = i;
6283 105662 : opp_ranking.safe_push (opp);
6284 : }
6285 : }
6286 :
6287 5577677 : if (!ctxlat->bottom
6288 6865225 : && (!clone_for_all_contexts || !ctxlat->is_single_const ()))
6289 : {
6290 564869 : ipcp_value<ipa_polymorphic_call_context> *val;
6291 568706 : for (val = ctxlat->values; val; val = val->next)
6292 7674 : if (!val->value.useless_p ())
6293 : {
6294 3837 : cloning_opportunity_ranking opp;
6295 3837 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6296 3837 : opp.offset = -2;
6297 3837 : opp.val = val;
6298 3837 : opp.index = i;
6299 3837 : opp_ranking.safe_push (opp);
6300 : }
6301 : }
6302 : }
6303 :
6304 2688770 : if (!opp_ranking.is_empty ())
6305 : {
6306 52398 : opp_ranking.qsort (compare_cloning_opportunities);
6307 52398 : auto_vec <cgraph_node *, 9> self_gen_clones;
6308 384464 : for (const cloning_opportunity_ranking &opp : opp_ranking)
6309 227270 : if (opp.offset == -2)
6310 : {
6311 3837 : ipcp_value<ipa_polymorphic_call_context> *val
6312 : = static_cast <ipcp_value<ipa_polymorphic_call_context> *>
6313 : (opp.val);
6314 3837 : ret |= decide_about_value (node, opp.index, -1, val,
6315 : &self_gen_clones, cur_sweep);
6316 : }
6317 : else
6318 : {
6319 223433 : ipcp_value<tree> *val = static_cast<ipcp_value<tree> *> (opp.val);
6320 223433 : ret |= decide_about_value (node, opp.index, opp.offset, val,
6321 : &self_gen_clones, cur_sweep);
6322 : }
6323 :
6324 104796 : if (!self_gen_clones.is_empty ())
6325 : {
6326 33 : self_gen_clones.safe_push (node);
6327 33 : update_counts_for_self_gen_clones (node, self_gen_clones);
6328 : }
6329 52398 : }
6330 :
6331 2688770 : if (!clone_for_all_contexts)
6332 : return ret;
6333 :
6334 237150 : struct caller_statistics stats;
6335 237150 : init_caller_stats (&stats);
6336 237150 : node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
6337 : false);
6338 237150 : if (!stats.n_calls)
6339 : {
6340 16068 : if (dump_file && (dump_flags & TDF_DETAILS))
6341 41 : fprintf (dump_file, " Not cloning for all contexts because "
6342 : "there are no callers of the original node (any more).\n");
6343 : return ret;
6344 : }
6345 :
6346 221082 : ipa_auto_call_arg_values avals;
6347 221082 : int removable_params_cost;
6348 221082 : bool ctx_independent_const
6349 221082 : = gather_context_independent_values (info, &avals, &removable_params_cost);
6350 221082 : sreal devirt_bonus = devirtualization_time_bonus (node, &avals);
6351 427289 : if (ctx_independent_const || devirt_bonus > 0
6352 427289 : || (removable_params_cost && clone_for_param_removal_p (node)))
6353 : {
6354 14875 : if (!dbg_cnt (ipa_cp_values))
6355 67 : return ret;
6356 :
6357 14875 : auto_vec<cgraph_edge *> callers = node->collect_callers ();
6358 56427 : for (int i = callers.length () - 1; i >= 0; i--)
6359 : {
6360 26677 : cgraph_edge *cs = callers[i];
6361 26677 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
6362 :
6363 26677 : if (caller_info && caller_info->node_dead)
6364 2656 : callers.unordered_remove (i);
6365 : }
6366 :
6367 14875 : if (!adjust_callers_for_value_intersection (callers, node))
6368 : /* If node is not called by anyone, or all its caller edges are
6369 : self-recursive, the node is not really in use, no need to do
6370 : cloning. */
6371 67 : return ret;
6372 :
6373 14808 : if (dump_file)
6374 91 : fprintf (dump_file, " Creating a specialized node of %s "
6375 : "for all known contexts.\n", node->dump_name ());
6376 :
6377 14808 : vec<tree> known_csts = vNULL;
6378 14808 : known_csts.safe_grow_cleared (count, true);
6379 14808 : find_scalar_values_for_callers_subset (known_csts, info, callers);
6380 14808 : vec<ipa_polymorphic_call_context> known_contexts = vNULL;
6381 14808 : find_contexts_for_caller_subset (known_contexts, info, callers);
6382 14808 : vec<ipa_argagg_value, va_gc> *aggvals
6383 14808 : = find_aggregate_values_for_callers_subset_gc (node, callers);
6384 :
6385 14808 : struct cgraph_node *clone = create_specialized_node (node, known_csts,
6386 : known_contexts,
6387 : aggvals, callers);
6388 14808 : ipa_node_params_sum->get (clone)->is_all_contexts_clone = true;
6389 14808 : ret = true;
6390 14875 : }
6391 :
6392 : return ret;
6393 2909852 : }
6394 :
6395 : /* Transitively mark all callees of NODE within the same SCC as not dead. */
6396 :
6397 : static void
6398 2177 : spread_undeadness (struct cgraph_node *node)
6399 : {
6400 2177 : struct cgraph_edge *cs;
6401 :
6402 11962 : for (cs = node->callees; cs; cs = cs->next_callee)
6403 9785 : if (ipa_edge_within_scc (cs))
6404 : {
6405 825 : struct cgraph_node *callee;
6406 825 : class ipa_node_params *info;
6407 :
6408 825 : callee = cs->callee->function_symbol (NULL);
6409 825 : info = ipa_node_params_sum->get (callee);
6410 :
6411 825 : if (info && info->node_dead)
6412 : {
6413 68 : info->node_dead = 0;
6414 68 : spread_undeadness (callee);
6415 : }
6416 : }
6417 2177 : }
6418 :
6419 : /* Return true if NODE has a caller from outside of its SCC that is not
6420 : dead. Worker callback for cgraph_for_node_and_aliases. */
6421 :
6422 : static bool
6423 16174 : has_undead_caller_from_outside_scc_p (struct cgraph_node *node,
6424 : void *data ATTRIBUTE_UNUSED)
6425 : {
6426 16174 : struct cgraph_edge *cs;
6427 :
6428 81469 : for (cs = node->callers; cs; cs = cs->next_caller)
6429 65748 : if (cs->caller->thunk
6430 65748 : && cs->caller->call_for_symbol_thunks_and_aliases
6431 0 : (has_undead_caller_from_outside_scc_p, NULL, true))
6432 : return true;
6433 65748 : else if (!ipa_edge_within_scc (cs))
6434 : {
6435 65508 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
6436 65508 : if (!caller_info /* Unoptimized caller are like dead ones. */
6437 65506 : || !caller_info->node_dead)
6438 : return true;
6439 : }
6440 : return false;
6441 : }
6442 :
6443 :
6444 : /* Identify nodes within the same SCC as NODE which are no longer needed
6445 : because of new clones and will be removed as unreachable. */
6446 :
6447 : static void
6448 17255 : identify_dead_nodes (struct cgraph_node *node)
6449 : {
6450 17255 : struct cgraph_node *v;
6451 34778 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6452 17523 : if (v->local)
6453 : {
6454 15929 : ipa_node_params *info = ipa_node_params_sum->get (v);
6455 15929 : if (info
6456 31858 : && !v->call_for_symbol_thunks_and_aliases
6457 15929 : (has_undead_caller_from_outside_scc_p, NULL, true))
6458 15476 : info->node_dead = 1;
6459 : }
6460 :
6461 34778 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6462 : {
6463 17523 : ipa_node_params *info = ipa_node_params_sum->get (v);
6464 17523 : if (info && !info->node_dead)
6465 2109 : spread_undeadness (v);
6466 : }
6467 :
6468 17255 : if (dump_file && (dump_flags & TDF_DETAILS))
6469 : {
6470 107 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6471 55 : if (ipa_node_params_sum->get (v)
6472 55 : && ipa_node_params_sum->get (v)->node_dead)
6473 32 : fprintf (dump_file, " Marking node as dead: %s.\n",
6474 : v->dump_name ());
6475 : }
6476 17255 : }
6477 :
6478 : /* Removes all useless callback edges from the callgraph. Useless callback
6479 : edges might mess up the callgraph, because they might be impossible to
6480 : redirect and so on, leading to crashes. Their usefulness is evaluated
6481 : through callback_edge_useful_p. */
6482 :
6483 : static void
6484 131565 : purge_useless_callback_edges ()
6485 : {
6486 131565 : if (dump_file)
6487 162 : fprintf (dump_file, "\nPurging useless callback edges:\n");
6488 :
6489 131565 : cgraph_edge *e;
6490 131565 : cgraph_node *node;
6491 1463624 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6492 : {
6493 6925138 : for (e = node->callees; e; e = e->next_callee)
6494 : {
6495 5593079 : if (e->has_callback)
6496 : {
6497 13673 : if (dump_file)
6498 6 : fprintf (dump_file, "\tExamining callbacks of edge %s -> %s:\n",
6499 6 : e->caller->dump_name (), e->callee->dump_name ());
6500 13673 : if (!lookup_attribute ("callback_only",
6501 13673 : DECL_ATTRIBUTES (e->callee->decl))
6502 13673 : && !callback_is_special_cased (e->callee->decl, e->call_stmt))
6503 : {
6504 2 : if (dump_file)
6505 0 : fprintf (
6506 : dump_file,
6507 : "\t\tPurging callbacks, because the callback-dispatching"
6508 : "function no longer has any callback attributes.\n");
6509 2 : e->purge_callback_edges ();
6510 2 : continue;
6511 : }
6512 13671 : cgraph_edge *cbe, *next;
6513 27344 : for (cbe = e->first_callback_edge (); cbe; cbe = next)
6514 : {
6515 13673 : next = cbe->next_callback_edge ();
6516 13673 : if (!callback_edge_useful_p (cbe))
6517 : {
6518 13457 : if (dump_file)
6519 4 : fprintf (dump_file,
6520 : "\t\tCallback edge %s -> %s not deemed "
6521 : "useful, removing.\n",
6522 4 : cbe->caller->dump_name (),
6523 4 : cbe->callee->dump_name ());
6524 13457 : cgraph_edge::remove (cbe);
6525 : }
6526 : else
6527 : {
6528 216 : if (dump_file)
6529 4 : fprintf (dump_file,
6530 : "\t\tKept callback edge %s -> %s "
6531 : "because it looks useful.\n",
6532 4 : cbe->caller->dump_name (),
6533 4 : cbe->callee->dump_name ());
6534 : }
6535 : }
6536 : }
6537 : }
6538 : }
6539 :
6540 131565 : if (dump_file)
6541 162 : fprintf (dump_file, "\n");
6542 131565 : }
6543 :
6544 : /* The decision stage. Iterate over the topological order of call graph nodes
6545 : TOPO and make specialized clones if deemed beneficial. */
6546 :
6547 : static void
6548 131565 : ipcp_decision_stage (class ipa_topo_info *topo)
6549 : {
6550 131565 : int i;
6551 :
6552 131565 : if (dump_file)
6553 162 : fprintf (dump_file, "\nIPA decision stage (%i sweeps):\n",
6554 : max_number_sweeps);
6555 :
6556 505313 : for (int cur_sweep = 1; cur_sweep <= max_number_sweeps; cur_sweep++)
6557 : {
6558 373748 : if (dump_file && (dump_flags & TDF_DETAILS))
6559 144 : fprintf (dump_file, "\nIPA decision sweep number %i (out of %i):\n",
6560 : cur_sweep, max_number_sweeps);
6561 :
6562 4509555 : for (i = topo->nnodes - 1; i >= 0; i--)
6563 : {
6564 4135807 : struct cgraph_node *node = topo->order[i];
6565 4135807 : bool change = false, iterate = true;
6566 :
6567 8288871 : while (iterate)
6568 : {
6569 : struct cgraph_node *v;
6570 : iterate = false;
6571 4168046 : for (v = node;
6572 8321110 : v;
6573 4168046 : v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6574 4168046 : if (v->has_gimple_body_p ()
6575 3931232 : && ipcp_versionable_function_p (v)
6576 4168046 : && (cur_sweep
6577 3318581 : <= opt_for_fn (node->decl, param_ipa_cp_sweeps)))
6578 3318581 : iterate |= decide_whether_version_node (v, cur_sweep);
6579 :
6580 4153064 : change |= iterate;
6581 : }
6582 4135807 : if (change)
6583 17255 : identify_dead_nodes (node);
6584 : }
6585 : }
6586 :
6587 : /* Currently, the primary use of callback edges is constant propagation.
6588 : Constant propagation is now over, so we have to remove unused callback
6589 : edges. */
6590 131565 : purge_useless_callback_edges ();
6591 131565 : }
6592 :
6593 : /* Look up all VR and bits information that we have discovered and copy it
6594 : over to the transformation summary. */
6595 :
6596 : static void
6597 131565 : ipcp_store_vr_results (void)
6598 : {
6599 131565 : cgraph_node *node;
6600 :
6601 1463624 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6602 : {
6603 1332059 : ipa_node_params *info = ipa_node_params_sum->get (node);
6604 1332059 : bool dumped_sth = false;
6605 1332059 : bool found_useful_result = false;
6606 1332059 : bool do_vr = true;
6607 1332059 : bool do_bits = true;
6608 :
6609 : /* If the function is not local, the gathered information is only useful
6610 : for clones. */
6611 1332059 : if (!node->local)
6612 1165045 : continue;
6613 :
6614 167014 : if (!info || !opt_for_fn (node->decl, flag_ipa_vrp))
6615 : {
6616 4812 : if (dump_file)
6617 6 : fprintf (dump_file, "Not considering %s for VR discovery "
6618 : "and propagate; -fipa-ipa-vrp: disabled.\n",
6619 : node->dump_name ());
6620 : do_vr = false;
6621 : }
6622 167014 : if (!info || !opt_for_fn (node->decl, flag_ipa_bit_cp))
6623 : {
6624 4786 : if (dump_file)
6625 2 : fprintf (dump_file, "Not considering %s for ipa bitwise "
6626 : "propagation ; -fipa-bit-cp: disabled.\n",
6627 : node->dump_name ());
6628 : do_bits = false;
6629 : }
6630 4786 : if (!do_bits && !do_vr)
6631 4780 : continue;
6632 :
6633 162234 : if (info->ipcp_orig_node)
6634 19046 : info = ipa_node_params_sum->get (info->ipcp_orig_node);
6635 162234 : if (info->lattices.is_empty ())
6636 : /* Newly expanded artificial thunks do not have lattices. */
6637 51917 : continue;
6638 :
6639 110317 : unsigned count = ipa_get_param_count (info);
6640 224313 : for (unsigned i = 0; i < count; i++)
6641 : {
6642 175390 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6643 175390 : if (do_vr
6644 175367 : && !plats->m_value_range.bottom_p ()
6645 234598 : && !plats->m_value_range.top_p ())
6646 : {
6647 : found_useful_result = true;
6648 : break;
6649 : }
6650 116183 : if (do_bits && plats->bits_lattice.constant_p ())
6651 : {
6652 : found_useful_result = true;
6653 : break;
6654 : }
6655 : }
6656 110317 : if (!found_useful_result)
6657 48923 : continue;
6658 :
6659 61394 : ipcp_transformation_initialize ();
6660 61394 : ipcp_transformation *ts = ipcp_transformation_sum->get_create (node);
6661 61394 : vec_safe_reserve_exact (ts->m_vr, count);
6662 :
6663 284187 : for (unsigned i = 0; i < count; i++)
6664 : {
6665 161399 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6666 161399 : ipcp_bits_lattice *bits = NULL;
6667 :
6668 161399 : if (do_bits
6669 161395 : && plats->bits_lattice.constant_p ()
6670 254367 : && dbg_cnt (ipa_cp_bits))
6671 92968 : bits = &plats->bits_lattice;
6672 :
6673 161399 : if (do_vr
6674 161378 : && !plats->m_value_range.bottom_p ()
6675 112094 : && !plats->m_value_range.top_p ()
6676 273493 : && dbg_cnt (ipa_cp_vr))
6677 : {
6678 112094 : if (bits)
6679 : {
6680 87746 : value_range tmp = plats->m_value_range.m_vr;
6681 87746 : tree type = ipa_get_type (info, i);
6682 175492 : irange_bitmask bm (wide_int::from (bits->get_value (),
6683 87746 : TYPE_PRECISION (type),
6684 87746 : TYPE_SIGN (type)),
6685 175492 : wide_int::from (bits->get_mask (),
6686 87746 : TYPE_PRECISION (type),
6687 175492 : TYPE_SIGN (type)));
6688 87746 : tmp.update_bitmask (bm);
6689 : // Reflecting the bitmask on the ranges can sometime
6690 : // produce an UNDEFINED value if the the bitmask update
6691 : // was previously deferred. See PR 120048.
6692 87746 : if (tmp.undefined_p ())
6693 0 : tmp.set_varying (type);
6694 87746 : ipa_vr vr (tmp);
6695 87746 : ts->m_vr->quick_push (vr);
6696 87746 : }
6697 : else
6698 : {
6699 24348 : ipa_vr vr (plats->m_value_range.m_vr);
6700 24348 : ts->m_vr->quick_push (vr);
6701 : }
6702 : }
6703 49305 : else if (bits)
6704 : {
6705 5222 : tree type = ipa_get_type (info, i);
6706 5222 : value_range tmp;
6707 5222 : tmp.set_varying (type);
6708 10444 : irange_bitmask bm (wide_int::from (bits->get_value (),
6709 5222 : TYPE_PRECISION (type),
6710 5222 : TYPE_SIGN (type)),
6711 10444 : wide_int::from (bits->get_mask (),
6712 5222 : TYPE_PRECISION (type),
6713 10444 : TYPE_SIGN (type)));
6714 5222 : tmp.update_bitmask (bm);
6715 : // Reflecting the bitmask on the ranges can sometime
6716 : // produce an UNDEFINED value if the the bitmask update
6717 : // was previously deferred. See PR 120048.
6718 5222 : if (tmp.undefined_p ())
6719 0 : tmp.set_varying (type);
6720 5222 : ipa_vr vr (tmp);
6721 5222 : ts->m_vr->quick_push (vr);
6722 5222 : }
6723 : else
6724 : {
6725 44083 : ipa_vr vr;
6726 44083 : ts->m_vr->quick_push (vr);
6727 : }
6728 :
6729 161399 : if (!dump_file || !bits)
6730 160983 : continue;
6731 :
6732 416 : if (!dumped_sth)
6733 : {
6734 295 : fprintf (dump_file, "Propagated bits info for function %s:\n",
6735 : node->dump_name ());
6736 295 : dumped_sth = true;
6737 : }
6738 416 : fprintf (dump_file, " param %i: value = ", i);
6739 416 : ipcp_print_widest_int (dump_file, bits->get_value ());
6740 416 : fprintf (dump_file, ", mask = ");
6741 416 : ipcp_print_widest_int (dump_file, bits->get_mask ());
6742 416 : fprintf (dump_file, "\n");
6743 : }
6744 : }
6745 131565 : }
6746 :
6747 : /* The IPCP driver. */
6748 :
6749 : static unsigned int
6750 131565 : ipcp_driver (void)
6751 : {
6752 131565 : class ipa_topo_info topo;
6753 :
6754 131565 : if (edge_clone_summaries == NULL)
6755 131565 : edge_clone_summaries = new edge_clone_summary_t (symtab);
6756 :
6757 131565 : ipa_check_create_node_params ();
6758 131565 : ipa_check_create_edge_args ();
6759 131565 : callback_info_sum_t::check_create_info_sum ();
6760 131565 : clone_num_suffixes = new hash_map<const char *, unsigned>;
6761 :
6762 131565 : if (dump_file)
6763 : {
6764 162 : fprintf (dump_file, "\nIPA structures before propagation:\n");
6765 162 : if (dump_flags & TDF_DETAILS)
6766 48 : ipa_print_all_params (dump_file);
6767 162 : ipa_print_all_jump_functions (dump_file);
6768 : }
6769 :
6770 : /* Topological sort. */
6771 131565 : build_toporder_info (&topo);
6772 : /* Do the interprocedural propagation. */
6773 131565 : ipcp_propagate_stage (&topo);
6774 : /* Decide what constant propagation and cloning should be performed. */
6775 131565 : ipcp_decision_stage (&topo);
6776 : /* Store results of value range and bits propagation. */
6777 131565 : ipcp_store_vr_results ();
6778 :
6779 : /* Free all IPCP structures. */
6780 263130 : delete clone_num_suffixes;
6781 131565 : free_toporder_info (&topo);
6782 131565 : delete edge_clone_summaries;
6783 131565 : edge_clone_summaries = NULL;
6784 131565 : ipa_free_all_structures_after_ipa_cp ();
6785 131565 : if (dump_file)
6786 162 : fprintf (dump_file, "\nIPA constant propagation end\n");
6787 131565 : return 0;
6788 : }
6789 :
6790 : /* Initialization and computation of IPCP data structures. This is the initial
6791 : intraprocedural analysis of functions, which gathers information to be
6792 : propagated later on. */
6793 :
6794 : static void
6795 128382 : ipcp_generate_summary (void)
6796 : {
6797 128382 : struct cgraph_node *node;
6798 :
6799 128382 : if (dump_file)
6800 164 : fprintf (dump_file, "\nIPA constant propagation start:\n");
6801 128382 : ipa_register_cgraph_hooks ();
6802 :
6803 1417438 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6804 1289056 : ipa_analyze_node (node);
6805 :
6806 128382 : varpool_node *vnode;
6807 1810720 : FOR_EACH_STATIC_INITIALIZER (vnode)
6808 1682338 : ipa_analyze_var_static_initializer (vnode);
6809 128382 : }
6810 :
6811 : namespace {
6812 :
6813 : const pass_data pass_data_ipa_cp =
6814 : {
6815 : IPA_PASS, /* type */
6816 : "cp", /* name */
6817 : OPTGROUP_NONE, /* optinfo_flags */
6818 : TV_IPA_CONSTANT_PROP, /* tv_id */
6819 : 0, /* properties_required */
6820 : 0, /* properties_provided */
6821 : 0, /* properties_destroyed */
6822 : 0, /* todo_flags_start */
6823 : ( TODO_dump_symtab | TODO_remove_functions ), /* todo_flags_finish */
6824 : };
6825 :
6826 : class pass_ipa_cp : public ipa_opt_pass_d
6827 : {
6828 : public:
6829 294587 : pass_ipa_cp (gcc::context *ctxt)
6830 : : ipa_opt_pass_d (pass_data_ipa_cp, ctxt,
6831 : ipcp_generate_summary, /* generate_summary */
6832 : NULL, /* write_summary */
6833 : NULL, /* read_summary */
6834 : ipcp_write_transformation_summaries, /*
6835 : write_optimization_summary */
6836 : ipcp_read_transformation_summaries, /*
6837 : read_optimization_summary */
6838 : NULL, /* stmt_fixup */
6839 : 0, /* function_transform_todo_flags_start */
6840 : ipcp_transform_function, /* function_transform */
6841 294587 : NULL) /* variable_transform */
6842 294587 : {}
6843 :
6844 : /* opt_pass methods: */
6845 587982 : bool gate (function *) final override
6846 : {
6847 : /* FIXME: We should remove the optimize check after we ensure we never run
6848 : IPA passes when not optimizing. */
6849 587982 : return (flag_ipa_cp && optimize) || in_lto_p;
6850 : }
6851 :
6852 131565 : unsigned int execute (function *) final override { return ipcp_driver (); }
6853 :
6854 : }; // class pass_ipa_cp
6855 :
6856 : } // anon namespace
6857 :
6858 : ipa_opt_pass_d *
6859 294587 : make_pass_ipa_cp (gcc::context *ctxt)
6860 : {
6861 294587 : return new pass_ipa_cp (ctxt);
6862 : }
6863 :
6864 : /* Reset all state within ipa-cp.cc so that we can rerun the compiler
6865 : within the same process. For use by toplev::finalize. */
6866 :
6867 : void
6868 264541 : ipa_cp_cc_finalize (void)
6869 : {
6870 264541 : overall_size = 0;
6871 264541 : orig_overall_size = 0;
6872 264541 : ipcp_free_transformation_sum ();
6873 264541 : }
6874 :
6875 : /* Given PARAM which must be a parameter of function FNDECL described by THIS,
6876 : return its index in the DECL_ARGUMENTS chain, using a pre-computed
6877 : DECL_UID-sorted vector if available (which is pre-computed only if there are
6878 : many parameters). Can return -1 if param is static chain not represented
6879 : among DECL_ARGUMENTS. */
6880 :
6881 : int
6882 125507 : ipcp_transformation::get_param_index (const_tree fndecl, const_tree param) const
6883 : {
6884 125507 : gcc_assert (TREE_CODE (param) == PARM_DECL);
6885 125507 : if (m_uid_to_idx)
6886 : {
6887 0 : unsigned puid = DECL_UID (param);
6888 0 : const ipa_uid_to_idx_map_elt *res
6889 0 : = std::lower_bound (m_uid_to_idx->begin(), m_uid_to_idx->end (), puid,
6890 0 : [] (const ipa_uid_to_idx_map_elt &elt, unsigned uid)
6891 : {
6892 0 : return elt.uid < uid;
6893 : });
6894 0 : if (res == m_uid_to_idx->end ()
6895 0 : || res->uid != puid)
6896 : {
6897 0 : gcc_assert (DECL_STATIC_CHAIN (fndecl));
6898 : return -1;
6899 : }
6900 0 : return res->index;
6901 : }
6902 :
6903 125507 : unsigned index = 0;
6904 287074 : for (tree p = DECL_ARGUMENTS (fndecl); p; p = DECL_CHAIN (p), index++)
6905 285590 : if (p == param)
6906 124023 : return (int) index;
6907 :
6908 1484 : gcc_assert (DECL_STATIC_CHAIN (fndecl));
6909 : return -1;
6910 : }
6911 :
6912 : /* Helper function to qsort a vector of ipa_uid_to_idx_map_elt elements
6913 : according to the uid. */
6914 :
6915 : static int
6916 0 : compare_uids (const void *a, const void *b)
6917 : {
6918 0 : const ipa_uid_to_idx_map_elt *e1 = (const ipa_uid_to_idx_map_elt *) a;
6919 0 : const ipa_uid_to_idx_map_elt *e2 = (const ipa_uid_to_idx_map_elt *) b;
6920 0 : if (e1->uid < e2->uid)
6921 : return -1;
6922 0 : if (e1->uid > e2->uid)
6923 : return 1;
6924 0 : gcc_unreachable ();
6925 : }
6926 :
6927 : /* Assuming THIS describes FNDECL and it has sufficiently many parameters to
6928 : justify the overhead, create a DECL_UID-sorted vector to speed up mapping
6929 : from parameters to their indices in DECL_ARGUMENTS chain. */
6930 :
6931 : void
6932 22938 : ipcp_transformation::maybe_create_parm_idx_map (tree fndecl)
6933 : {
6934 22938 : int c = count_formal_params (fndecl);
6935 22938 : if (c < 32)
6936 : return;
6937 :
6938 0 : m_uid_to_idx = NULL;
6939 0 : vec_safe_reserve (m_uid_to_idx, c, true);
6940 0 : unsigned index = 0;
6941 0 : for (tree p = DECL_ARGUMENTS (fndecl); p; p = DECL_CHAIN (p), index++)
6942 : {
6943 0 : ipa_uid_to_idx_map_elt elt;
6944 0 : elt.uid = DECL_UID (p);
6945 0 : elt.index = index;
6946 0 : m_uid_to_idx->quick_push (elt);
6947 : }
6948 0 : m_uid_to_idx->qsort (compare_uids);
6949 : }
|
