Line data Source code
1 : /* Interprocedural constant propagation
2 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
3 :
4 : Contributed by Razya Ladelsky <RAZYA@il.ibm.com> and Martin Jambor
5 : <mjambor@suse.cz>
6 :
7 : This file is part of GCC.
8 :
9 : GCC is free software; you can redistribute it and/or modify it under
10 : the terms of the GNU General Public License as published by the Free
11 : Software Foundation; either version 3, or (at your option) any later
12 : version.
13 :
14 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 : for more details.
18 :
19 : You should have received a copy of the GNU General Public License
20 : along with GCC; see the file COPYING3. If not see
21 : <http://www.gnu.org/licenses/>. */
22 :
23 : /* Interprocedural constant propagation (IPA-CP).
24 :
25 : The goal of this transformation is to
26 :
27 : 1) discover functions which are always invoked with some arguments with the
28 : same known constant values and modify the functions so that the
29 : subsequent optimizations can take advantage of the knowledge, and
30 :
31 : 2) partial specialization - create specialized versions of functions
32 : transformed in this way if some parameters are known constants only in
33 : certain contexts but the estimated tradeoff between speedup and cost size
34 : is deemed good.
35 :
36 : The algorithm also propagates types and attempts to perform type based
37 : devirtualization. Types are propagated much like constants.
38 :
39 : The algorithm basically consists of three stages. In the first, functions
40 : are analyzed one at a time and jump functions are constructed for all known
41 : call-sites. In the second phase, the pass propagates information from the
42 : jump functions across the call to reveal what values are available at what
43 : call sites, performs estimations of effects of known values on functions and
44 : their callees, and finally decides what specialized extra versions should be
45 : created. In the third, the special versions materialize and appropriate
46 : calls are redirected.
47 :
48 : The algorithm used is to a certain extent based on "Interprocedural Constant
49 : Propagation", by David Callahan, Keith D Cooper, Ken Kennedy, Linda Torczon,
50 : Comp86, pg 152-161 and "A Methodology for Procedure Cloning" by Keith D
51 : Cooper, Mary W. Hall, and Ken Kennedy.
52 :
53 :
54 : First stage - intraprocedural analysis
55 : =======================================
56 :
57 : This phase computes jump_function and modification flags.
58 :
59 : A jump function for a call-site represents the values passed as an actual
60 : arguments of a given call-site. In principle, there are three types of
61 : values:
62 :
63 : Pass through - the caller's formal parameter is passed as an actual
64 : argument, plus an operation on it can be performed.
65 : Constant - a constant is passed as an actual argument.
66 : Unknown - neither of the above.
67 :
68 : All jump function types are described in detail in ipa-prop.h, together with
69 : the data structures that represent them and methods of accessing them.
70 :
71 : ipcp_generate_summary() is the main function of the first stage.
72 :
73 : Second stage - interprocedural analysis
74 : ========================================
75 :
76 : This stage is itself divided into two phases. In the first, we propagate
77 : known values over the call graph, in the second, we make cloning decisions.
78 : It uses a different algorithm than the original Callahan's paper.
79 :
80 : First, we traverse the functions topologically from callers to callees and,
81 : for each strongly connected component (SCC), we propagate constants
82 : according to previously computed jump functions. We also record what known
83 : values depend on other known values and estimate local effects. Finally, we
84 : propagate cumulative information about these effects from dependent values
85 : to those on which they depend.
86 :
87 : Second, we again traverse the call graph in the same topological order and
88 : make clones for functions which we know are called with the same values in
89 : all contexts and decide about extra specialized clones of functions just for
90 : some contexts - these decisions are based on both local estimates and
91 : cumulative estimates propagated from callees.
92 :
93 : ipcp_propagate_stage() and ipcp_decision_stage() together constitute the
94 : third stage.
95 :
96 : Third phase - materialization of clones, call statement updates.
97 : ============================================
98 :
99 : This stage is currently performed by call graph code (mainly in cgraphunit.cc
100 : and tree-inline.cc) according to instructions inserted to the call graph by
101 : the second stage. */
102 :
103 : #define INCLUDE_ALGORITHM
104 : #include "config.h"
105 : #include "system.h"
106 : #include "coretypes.h"
107 : #include "backend.h"
108 : #include "tree.h"
109 : #include "gimple-expr.h"
110 : #include "gimple.h"
111 : #include "predict.h"
112 : #include "sreal.h"
113 : #include "alloc-pool.h"
114 : #include "tree-pass.h"
115 : #include "cgraph.h"
116 : #include "diagnostic.h"
117 : #include "fold-const.h"
118 : #include "gimple-iterator.h"
119 : #include "gimple-fold.h"
120 : #include "symbol-summary.h"
121 : #include "tree-vrp.h"
122 : #include "ipa-cp.h"
123 : #include "ipa-prop.h"
124 : #include "tree-pretty-print.h"
125 : #include "tree-inline.h"
126 : #include "ipa-fnsummary.h"
127 : #include "ipa-utils.h"
128 : #include "tree-ssa-ccp.h"
129 : #include "stringpool.h"
130 : #include "attribs.h"
131 : #include "dbgcnt.h"
132 : #include "symtab-clones.h"
133 : #include "gimple-range.h"
134 : #include "attr-callback.h"
135 : #include "lto-streamer.h"
136 : #include "callback-info.h"
137 :
138 : /* Allocation pools for values and their sources in ipa-cp. */
139 :
140 : object_allocator<ipcp_value<tree> > ipcp_cst_values_pool
141 : ("IPA-CP constant values");
142 :
143 : object_allocator<ipcp_value<ipa_polymorphic_call_context> >
144 : ipcp_poly_ctx_values_pool ("IPA-CP polymorphic contexts");
145 :
146 : object_allocator<ipcp_value_source<tree> > ipcp_sources_pool
147 : ("IPA-CP value sources");
148 :
149 : object_allocator<ipcp_agg_lattice> ipcp_agg_lattice_pool
150 : ("IPA_CP aggregate lattices");
151 :
152 : /* Original overall size of the program. */
153 :
154 : static long overall_size, orig_overall_size;
155 :
156 : /* Overall run-time of the part of the program or the compilation unit for
157 : which profile information is available, scaled by
158 : param_ipa_cp_min_recursive_probability. */
159 :
160 : static sreal overall_scaled_runtime;
161 :
162 : /* The maximum number of IPA-CP decision sweeps that any node requested in its
163 : param. */
164 : static int max_number_sweeps;
165 :
166 : /* Node name to unique clone suffix number map. */
167 : static hash_map<const char *, unsigned> *clone_num_suffixes;
168 :
169 : /* Return the param lattices structure corresponding to the Ith formal
170 : parameter of the function described by INFO. */
171 : static inline class ipcp_param_lattices *
172 32480406 : ipa_get_parm_lattices (class ipa_node_params *info, int i)
173 : {
174 64960812 : gcc_assert (i >= 0 && i < ipa_get_param_count (info));
175 32480406 : gcc_checking_assert (!info->ipcp_orig_node);
176 32480406 : return &(info->lattices[i]);
177 : }
178 :
179 : /* Return the lattice corresponding to the scalar value of the Ith formal
180 : parameter of the function described by INFO. */
181 : static inline ipcp_lattice<tree> *
182 6226207 : ipa_get_scalar_lat (class ipa_node_params *info, int i)
183 : {
184 6416844 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
185 6226207 : return &plats->itself;
186 : }
187 :
188 : /* Return the lattice corresponding to the scalar value of the Ith formal
189 : parameter of the function described by INFO. */
190 : static inline ipcp_lattice<ipa_polymorphic_call_context> *
191 821234 : ipa_get_poly_ctx_lat (class ipa_node_params *info, int i)
192 : {
193 821234 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
194 821234 : return &plats->ctxlat;
195 : }
196 :
197 : /* Return whether LAT is a lattice with a single constant and without an
198 : undefined value. */
199 :
200 : template <typename valtype>
201 : inline bool
202 10530045 : ipcp_lattice<valtype>::is_single_const ()
203 : {
204 3032946 : if (bottom || contains_variable || values_count != 1)
205 : return false;
206 : else
207 : return true;
208 : }
209 :
210 : /* Return true iff X and Y should be considered equal values by IPA-CP. */
211 :
212 : bool
213 1423151 : values_equal_for_ipcp_p (tree x, tree y)
214 : {
215 1423151 : gcc_checking_assert (x != NULL_TREE && y != NULL_TREE);
216 :
217 1423151 : if (x == y)
218 : return true;
219 :
220 627159 : if (TREE_CODE (x) == ADDR_EXPR
221 224360 : && TREE_CODE (y) == ADDR_EXPR
222 223632 : && (TREE_CODE (TREE_OPERAND (x, 0)) == CONST_DECL
223 175765 : || (TREE_CODE (TREE_OPERAND (x, 0)) == VAR_DECL
224 92469 : && DECL_IN_CONSTANT_POOL (TREE_OPERAND (x, 0))))
225 675026 : && (TREE_CODE (TREE_OPERAND (y, 0)) == CONST_DECL
226 13 : || (TREE_CODE (TREE_OPERAND (y, 0)) == VAR_DECL
227 8 : && DECL_IN_CONSTANT_POOL (TREE_OPERAND (y, 0)))))
228 47854 : return TREE_OPERAND (x, 0) == TREE_OPERAND (y, 0)
229 95452 : || operand_equal_p (DECL_INITIAL (TREE_OPERAND (x, 0)),
230 47598 : DECL_INITIAL (TREE_OPERAND (y, 0)), 0);
231 : else
232 579305 : return operand_equal_p (x, y, 0);
233 : }
234 :
235 : /* Print V which is extracted from a value in a lattice to F. This overloaded
236 : function is used to print tree constants. */
237 :
238 : static void
239 852 : print_ipcp_constant_value (FILE * f, tree v)
240 : {
241 0 : ipa_print_constant_value (f, v);
242 0 : }
243 :
244 : /* Print V which is extracted from a value in a lattice to F. This overloaded
245 : function is used to print constant polymorphic call contexts. */
246 :
247 : static void
248 214 : print_ipcp_constant_value (FILE * f, ipa_polymorphic_call_context v)
249 : {
250 214 : v.dump(f, false);
251 0 : }
252 :
253 : /* Print a lattice LAT to F. */
254 :
255 : template <typename valtype>
256 : void
257 2007 : ipcp_lattice<valtype>::print (FILE * f, bool dump_sources, bool dump_benefits)
258 : {
259 : ipcp_value<valtype> *val;
260 2007 : bool prev = false;
261 :
262 2007 : if (bottom)
263 : {
264 842 : fprintf (f, "BOTTOM\n");
265 842 : return;
266 : }
267 :
268 1165 : if (!values_count && !contains_variable)
269 : {
270 0 : fprintf (f, "TOP\n");
271 0 : return;
272 : }
273 :
274 1165 : if (contains_variable)
275 : {
276 885 : fprintf (f, "VARIABLE");
277 885 : prev = true;
278 885 : if (dump_benefits)
279 885 : fprintf (f, "\n");
280 : }
281 :
282 1807 : for (val = values; val; val = val->next)
283 : {
284 642 : if (dump_benefits && prev)
285 362 : fprintf (f, " ");
286 280 : else if (!dump_benefits && prev)
287 0 : fprintf (f, ", ");
288 : else
289 : prev = true;
290 :
291 642 : print_ipcp_constant_value (f, val->value);
292 :
293 642 : if (dump_sources)
294 : {
295 : ipcp_value_source<valtype> *s;
296 :
297 175 : if (val->self_recursion_generated_p ())
298 27 : fprintf (f, " [self_gen(%i), from:",
299 : val->self_recursion_generated_level);
300 : else
301 148 : fprintf (f, " [scc: %i, from:", val->scc_no);
302 368 : for (s = val->sources; s; s = s->next)
303 193 : fprintf (f, " %i(%f)", s->cs->caller->get_uid (),
304 386 : s->cs->sreal_frequency ().to_double ());
305 175 : fprintf (f, "]");
306 : }
307 :
308 642 : if (dump_benefits)
309 642 : fprintf (f, " [loc_time: %g, loc_size: %i, "
310 : "prop_time: %g, prop_size: %i]\n",
311 : val->local_time_benefit.to_double (), val->local_size_cost,
312 : val->prop_time_benefit.to_double (), val->prop_size_cost);
313 : }
314 1165 : if (!dump_benefits)
315 0 : fprintf (f, "\n");
316 : }
317 :
318 : /* Print VALUE to F in a form which in usual cases does not take thousands of
319 : characters. */
320 :
321 : static void
322 1466 : ipcp_print_widest_int (FILE *f, const widest_int &value)
323 : {
324 1466 : if (value == -1)
325 0 : fprintf (f, "-1");
326 1466 : else if (wi::arshift (value, 128) == -1)
327 : {
328 330 : char buf[35], *p = buf + 2;
329 330 : widest_int v = wi::zext (value, 128);
330 330 : size_t len;
331 330 : print_hex (v, buf);
332 330 : len = strlen (p);
333 330 : if (len == 32)
334 : {
335 330 : fprintf (f, "0xf..f");
336 9795 : while (*p == 'f')
337 9135 : ++p;
338 : }
339 : else
340 0 : fprintf (f, "0xf..f%0*d", (int) (32 - len), 0);
341 330 : fputs (p, f);
342 330 : }
343 : else
344 1136 : print_hex (value, f);
345 1466 : }
346 :
347 : void
348 923 : ipcp_bits_lattice::print (FILE *f)
349 : {
350 923 : if (bottom_p ())
351 : {
352 606 : fprintf (f, " Bits unusable (BOTTOM)\n");
353 606 : return;
354 : }
355 :
356 317 : if (top_p ())
357 0 : fprintf (f, " Bits unknown (TOP)");
358 : else
359 : {
360 317 : fprintf (f, " Bits: value = ");
361 317 : ipcp_print_widest_int (f, get_value ());
362 317 : fprintf (f, ", mask = ");
363 317 : ipcp_print_widest_int (f, get_mask ());
364 : }
365 :
366 317 : if (m_recipient_only)
367 143 : fprintf (f, " (recipient only)");
368 317 : fprintf (f, "\n");
369 : }
370 :
371 : /* Print value range lattice to F. */
372 :
373 : void
374 923 : ipcp_vr_lattice::print (FILE * f)
375 : {
376 923 : if (m_recipient_only)
377 270 : fprintf (f, "(recipient only) ");
378 923 : m_vr.dump (f);
379 923 : }
380 :
381 : /* Print all ipcp_lattices of all functions to F. */
382 :
383 : static void
384 162 : print_all_lattices (FILE * f, bool dump_sources, bool dump_benefits)
385 : {
386 162 : struct cgraph_node *node;
387 162 : int i, count;
388 :
389 162 : fprintf (f, "\nLattices:\n");
390 891 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
391 : {
392 729 : class ipa_node_params *info;
393 :
394 729 : info = ipa_node_params_sum->get (node);
395 : /* Skip unoptimized functions and constprop clones since we don't make
396 : lattices for them. */
397 729 : if (!info || info->ipcp_orig_node)
398 0 : continue;
399 729 : fprintf (f, " Node: %s:\n", node->dump_name ());
400 729 : count = ipa_get_param_count (info);
401 1652 : for (i = 0; i < count; i++)
402 : {
403 923 : struct ipcp_agg_lattice *aglat;
404 923 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
405 923 : fprintf (f, " param [%d]: ", i);
406 923 : plats->itself.print (f, dump_sources, dump_benefits);
407 923 : fprintf (f, " ctxs: ");
408 923 : plats->ctxlat.print (f, dump_sources, dump_benefits);
409 923 : plats->bits_lattice.print (f);
410 923 : fprintf (f, " ");
411 923 : plats->m_value_range.print (f);
412 923 : fprintf (f, "\n");
413 923 : if (plats->virt_call)
414 75 : fprintf (f, " virt_call flag set\n");
415 :
416 923 : if (plats->aggs_bottom)
417 : {
418 441 : fprintf (f, " AGGS BOTTOM\n");
419 441 : continue;
420 : }
421 482 : if (plats->aggs_contain_variable)
422 444 : fprintf (f, " AGGS VARIABLE\n");
423 643 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
424 : {
425 161 : fprintf (f, " %soffset " HOST_WIDE_INT_PRINT_DEC ": ",
426 161 : plats->aggs_by_ref ? "ref " : "", aglat->offset);
427 161 : aglat->print (f, dump_sources, dump_benefits);
428 : }
429 : }
430 : }
431 162 : }
432 :
433 : /* Determine whether it is at all technically possible to create clones of NODE
434 : and store this information in the ipa_node_params structure associated
435 : with NODE. */
436 :
437 : static void
438 1313343 : determine_versionability (struct cgraph_node *node,
439 : class ipa_node_params *info)
440 : {
441 1313343 : const char *reason = NULL;
442 :
443 : /* There are a number of generic reasons functions cannot be versioned. We
444 : also cannot remove parameters if there are type attributes such as fnspec
445 : present. */
446 1313343 : if (node->alias || node->thunk)
447 : reason = "alias or thunk";
448 1313343 : else if (!node->versionable)
449 : reason = "not a tree_versionable_function";
450 1181987 : else if (node->get_availability () <= AVAIL_INTERPOSABLE)
451 : reason = "insufficient body availability";
452 1113905 : else if (!opt_for_fn (node->decl, optimize)
453 1113905 : || !opt_for_fn (node->decl, flag_ipa_cp))
454 : reason = "non-optimized function";
455 1113905 : else if (lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (node->decl)))
456 : {
457 : /* Ideally we should clone the SIMD clones themselves and create
458 : vector copies of them, so IPA-cp and SIMD clones can happily
459 : coexist, but that may not be worth the effort. */
460 : reason = "function has SIMD clones";
461 : }
462 1113541 : else if (lookup_attribute ("target_clones", DECL_ATTRIBUTES (node->decl)))
463 : {
464 : /* Ideally we should clone the target clones themselves and create
465 : copies of them, so IPA-cp and target clones can happily
466 : coexist, but that may not be worth the effort. */
467 : reason = "function target_clones attribute";
468 : }
469 : /* Don't clone decls local to a comdat group; it breaks and for C++
470 : decloned constructors, inlining is always better anyway. */
471 1113541 : else if (node->comdat_local_p ())
472 : reason = "comdat-local function";
473 1111165 : else if (node->calls_comdat_local)
474 : {
475 : /* TODO: call is versionable if we make sure that all
476 : callers are inside of a comdat group. */
477 2480 : reason = "calls comdat-local function";
478 : }
479 :
480 : /* Functions calling BUILT_IN_VA_ARG_PACK and BUILT_IN_VA_ARG_PACK_LEN
481 : work only when inlined. Cloning them may still lead to better code
482 : because ipa-cp will not give up on cloning further. If the function is
483 : external this however leads to wrong code because we may end up producing
484 : offline copy of the function. */
485 1313343 : if (DECL_EXTERNAL (node->decl))
486 181830 : for (cgraph_edge *edge = node->callees; !reason && edge;
487 133712 : edge = edge->next_callee)
488 133712 : if (fndecl_built_in_p (edge->callee->decl, BUILT_IN_NORMAL))
489 : {
490 37761 : if (DECL_FUNCTION_CODE (edge->callee->decl) == BUILT_IN_VA_ARG_PACK)
491 0 : reason = "external function which calls va_arg_pack";
492 37761 : if (DECL_FUNCTION_CODE (edge->callee->decl)
493 : == BUILT_IN_VA_ARG_PACK_LEN)
494 0 : reason = "external function which calls va_arg_pack_len";
495 : }
496 :
497 1313343 : if (reason && dump_file && !node->alias && !node->thunk)
498 53 : fprintf (dump_file, "Function %s is not versionable, reason: %s.\n",
499 : node->dump_name (), reason);
500 :
501 1313343 : info->versionable = (reason == NULL);
502 1313343 : }
503 :
504 : /* Return true if it is at all technically possible to create clones of a
505 : NODE. */
506 :
507 : static bool
508 6289358 : ipcp_versionable_function_p (struct cgraph_node *node)
509 : {
510 6289358 : ipa_node_params *info = ipa_node_params_sum->get (node);
511 6289358 : return info && info->versionable;
512 : }
513 :
514 : /* Structure holding accumulated information about callers of a node. */
515 :
516 1066496 : struct caller_statistics
517 : {
518 : /* If requested (see below), self-recursive call counts are summed into this
519 : field. */
520 : profile_count rec_count_sum;
521 : /* The sum of all ipa counts of all the other (non-recursive) calls. */
522 : profile_count count_sum;
523 : /* Sum of all frequencies for all calls. */
524 : sreal freq_sum;
525 : /* Number of calls and calls considered interesting respectively. */
526 : int n_calls, n_interesting_calls;
527 : /* If itself is set up, also count the number of non-self-recursive
528 : calls. */
529 : int n_nonrec_calls;
530 : /* If non-NULL, this is the node itself and calls from it should have their
531 : counts included in rec_count_sum and not count_sum. */
532 : cgraph_node *itself;
533 : /* True if there is a caller that has no IPA profile. */
534 : bool called_without_ipa_profile;
535 : };
536 :
537 : /* Initialize fields of STAT to zeroes and optionally set it up so that edges
538 : from IGNORED_CALLER are not counted. */
539 :
540 : static inline void
541 291101 : init_caller_stats (caller_statistics *stats, cgraph_node *itself = NULL)
542 : {
543 291101 : stats->rec_count_sum = profile_count::zero ();
544 291101 : stats->count_sum = profile_count::zero ();
545 291101 : stats->n_calls = 0;
546 291101 : stats->n_interesting_calls = 0;
547 291101 : stats->n_nonrec_calls = 0;
548 291101 : stats->freq_sum = 0;
549 291101 : stats->itself = itself;
550 291101 : stats->called_without_ipa_profile = false;
551 291101 : }
552 :
553 : /* We want to propagate across edges that may be executed, however
554 : we do not want to check maybe_hot, since call itself may be cold
555 : while calee contains some heavy loop which makes propagation still
556 : relevant.
557 :
558 : In particular, even edge called once may lead to significant
559 : improvement. */
560 :
561 : static bool
562 986725 : cs_interesting_for_ipcp_p (cgraph_edge *e)
563 : {
564 : /* If profile says the edge is executed, we want to optimize. */
565 986725 : if (e->count.ipa ().nonzero_p ())
566 501 : return true;
567 : /* If local (possibly guseed or adjusted 0 profile) claims edge is
568 : not executed, do not propagate.
569 : Do not trust AFDO since branch needs to be executed multiple
570 : time to count while we want to propagate even call called
571 : once during the train run if callee is important. */
572 986224 : if (e->count.initialized_p () && !e->count.nonzero_p ()
573 1036000 : && e->count.quality () != AFDO)
574 : return false;
575 : /* If we have zero IPA profile, still consider edge for cloning
576 : in case we do partial training. */
577 936448 : if (e->count.ipa ().initialized_p ()
578 936448 : && e->count.ipa ().quality () != AFDO
579 936453 : && !opt_for_fn (e->callee->decl,flag_profile_partial_training))
580 5 : return false;
581 : return true;
582 : }
583 :
584 : /* Worker callback of cgraph_for_node_and_aliases accumulating statistics of
585 : non-thunk incoming edges to NODE. */
586 :
587 : static bool
588 298876 : gather_caller_stats (struct cgraph_node *node, void *data)
589 : {
590 298876 : struct caller_statistics *stats = (struct caller_statistics *) data;
591 298876 : struct cgraph_edge *cs;
592 :
593 1053808 : for (cs = node->callers; cs; cs = cs->next_caller)
594 754932 : if (!cs->caller->thunk)
595 : {
596 754743 : ipa_node_params *info = ipa_node_params_sum->get (cs->caller);
597 754743 : if (info && info->node_dead)
598 150548 : continue;
599 :
600 604195 : if (cs->count.ipa ().initialized_p ())
601 : {
602 3884 : if (stats->itself && stats->itself == cs->caller)
603 0 : stats->rec_count_sum += cs->count.ipa ();
604 : else
605 3884 : stats->count_sum += cs->count.ipa ();
606 : }
607 : else
608 600311 : stats->called_without_ipa_profile = true;
609 604195 : stats->freq_sum += cs->sreal_frequency ();
610 604195 : stats->n_calls++;
611 604195 : if (stats->itself && stats->itself != cs->caller)
612 53 : stats->n_nonrec_calls++;
613 :
614 : /* If profile known to be zero, we do not want to clone for performance.
615 : However if call is cold, the called function may still contain
616 : important hot loops. */
617 604195 : if (cs_interesting_for_ipcp_p (cs))
618 599908 : stats->n_interesting_calls++;
619 : }
620 298876 : return false;
621 :
622 : }
623 :
624 : /* Return true if this NODE is viable candidate for cloning. */
625 :
626 : static bool
627 823440 : ipcp_cloning_candidate_p (struct cgraph_node *node)
628 : {
629 823440 : struct caller_statistics stats;
630 :
631 823440 : gcc_checking_assert (node->has_gimple_body_p ());
632 :
633 823440 : if (!opt_for_fn (node->decl, flag_ipa_cp_clone))
634 : {
635 771071 : if (dump_file)
636 31 : fprintf (dump_file, "Not considering %s for cloning; "
637 : "-fipa-cp-clone disabled.\n",
638 : node->dump_name ());
639 : return false;
640 : }
641 :
642 : /* Do not use profile here since cold wrapper wrap
643 : hot function. */
644 52369 : if (opt_for_fn (node->decl, optimize_size))
645 : {
646 10 : if (dump_file)
647 0 : fprintf (dump_file, "Not considering %s for cloning; "
648 : "optimizing it for size.\n",
649 : node->dump_name ());
650 : return false;
651 : }
652 :
653 52359 : init_caller_stats (&stats);
654 52359 : node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats, false);
655 :
656 52359 : if (ipa_size_summaries->get (node)->self_size < stats.n_calls)
657 : {
658 312 : if (dump_file)
659 0 : fprintf (dump_file, "Considering %s for cloning; code might shrink.\n",
660 : node->dump_name ());
661 : return true;
662 : }
663 52047 : if (!stats.n_interesting_calls)
664 : {
665 39899 : if (dump_file)
666 200 : fprintf (dump_file, "Not considering %s for cloning; "
667 : "no calls considered interesting by profile.\n",
668 : node->dump_name ());
669 : return false;
670 : }
671 12148 : if (dump_file)
672 190 : fprintf (dump_file, "Considering %s for cloning.\n",
673 : node->dump_name ());
674 : return true;
675 : }
676 :
677 : template <typename valtype>
678 : class value_topo_info
679 : {
680 : public:
681 : /* Head of the linked list of topologically sorted values. */
682 : ipcp_value<valtype> *values_topo;
683 : /* Stack for creating SCCs, represented by a linked list too. */
684 : ipcp_value<valtype> *stack;
685 : /* Counter driving the algorithm in add_val_to_toposort. */
686 : int dfs_counter;
687 :
688 131920 : value_topo_info () : values_topo (NULL), stack (NULL), dfs_counter (0)
689 : {}
690 : void add_val (ipcp_value<valtype> *cur_val);
691 : void propagate_effects ();
692 : };
693 :
694 : /* Arrays representing a topological ordering of call graph nodes and a stack
695 : of nodes used during constant propagation and also data required to perform
696 : topological sort of values and propagation of benefits in the determined
697 : order. */
698 :
699 : class ipa_topo_info
700 : {
701 : public:
702 : /* Array with obtained topological order of cgraph nodes. */
703 : struct cgraph_node **order;
704 : /* Stack of cgraph nodes used during propagation within SCC until all values
705 : in the SCC stabilize. */
706 : struct cgraph_node **stack;
707 : int nnodes, stack_top;
708 :
709 : value_topo_info<tree> constants;
710 : value_topo_info<ipa_polymorphic_call_context> contexts;
711 :
712 131920 : ipa_topo_info () : order(NULL), stack(NULL), nnodes(0), stack_top(0),
713 131920 : constants ()
714 : {}
715 : };
716 :
717 : /* Skip edges from and to nodes without ipa_cp enabled.
718 : Ignore not available symbols. */
719 :
720 : static bool
721 5482946 : ignore_edge_p (cgraph_edge *e)
722 : {
723 5482946 : enum availability avail;
724 5482946 : cgraph_node *ultimate_target
725 5482946 : = e->callee->function_or_virtual_thunk_symbol (&avail, e->caller);
726 :
727 5482946 : return (avail <= AVAIL_INTERPOSABLE
728 1990878 : || !opt_for_fn (ultimate_target->decl, optimize)
729 7465068 : || !opt_for_fn (ultimate_target->decl, flag_ipa_cp));
730 : }
731 :
732 : /* Allocate the arrays in TOPO and topologically sort the nodes into order. */
733 :
734 : static void
735 131920 : build_toporder_info (class ipa_topo_info *topo)
736 : {
737 131920 : topo->order = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
738 131920 : topo->stack = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
739 :
740 131920 : gcc_checking_assert (topo->stack_top == 0);
741 131920 : topo->nnodes = ipa_reduced_postorder (topo->order, true,
742 : ignore_edge_p);
743 131920 : }
744 :
745 : /* Free information about strongly connected components and the arrays in
746 : TOPO. */
747 :
748 : static void
749 131920 : free_toporder_info (class ipa_topo_info *topo)
750 : {
751 131920 : ipa_free_postorder_info ();
752 131920 : free (topo->order);
753 131920 : free (topo->stack);
754 131920 : }
755 :
756 : /* Add NODE to the stack in TOPO, unless it is already there. */
757 :
758 : static inline void
759 1317538 : push_node_to_stack (class ipa_topo_info *topo, struct cgraph_node *node)
760 : {
761 1317538 : ipa_node_params *info = ipa_node_params_sum->get (node);
762 1317538 : if (info->node_enqueued)
763 : return;
764 1316555 : info->node_enqueued = 1;
765 1316555 : topo->stack[topo->stack_top++] = node;
766 : }
767 :
768 : /* Pop a node from the stack in TOPO and return it or return NULL if the stack
769 : is empty. */
770 :
771 : static struct cgraph_node *
772 2714992 : pop_node_from_stack (class ipa_topo_info *topo)
773 : {
774 2714992 : if (topo->stack_top)
775 : {
776 1316555 : struct cgraph_node *node;
777 1316555 : topo->stack_top--;
778 1316555 : node = topo->stack[topo->stack_top];
779 1316555 : ipa_node_params_sum->get (node)->node_enqueued = 0;
780 1316555 : return node;
781 : }
782 : else
783 : return NULL;
784 : }
785 :
786 : /* Set lattice LAT to bottom and return true if it previously was not set as
787 : such. */
788 :
789 : template <typename valtype>
790 : inline bool
791 2193170 : ipcp_lattice<valtype>::set_to_bottom ()
792 : {
793 2193170 : bool ret = !bottom;
794 2193170 : bottom = true;
795 : return ret;
796 : }
797 :
798 : /* Mark lattice as containing an unknown value and return true if it previously
799 : was not marked as such. */
800 :
801 : template <typename valtype>
802 : inline bool
803 1574546 : ipcp_lattice<valtype>::set_contains_variable ()
804 : {
805 1574546 : bool ret = !contains_variable;
806 1574546 : contains_variable = true;
807 : return ret;
808 : }
809 :
810 : /* Set all aggregate lattices in PLATS to bottom and return true if they were
811 : not previously set as such. */
812 :
813 : static inline bool
814 2192940 : set_agg_lats_to_bottom (class ipcp_param_lattices *plats)
815 : {
816 2192940 : bool ret = !plats->aggs_bottom;
817 2192940 : plats->aggs_bottom = true;
818 2192940 : return ret;
819 : }
820 :
821 : /* Mark all aggregate lattices in PLATS as containing an unknown value and
822 : return true if they were not previously marked as such. */
823 :
824 : static inline bool
825 1065083 : set_agg_lats_contain_variable (class ipcp_param_lattices *plats)
826 : {
827 1065083 : bool ret = !plats->aggs_contain_variable;
828 1065083 : plats->aggs_contain_variable = true;
829 1065083 : return ret;
830 : }
831 :
832 : bool
833 0 : ipcp_vr_lattice::meet_with (const ipcp_vr_lattice &other)
834 : {
835 0 : return meet_with_1 (other.m_vr);
836 : }
837 :
838 : /* Meet the current value of the lattice with the range described by
839 : P_VR. */
840 :
841 : bool
842 496196 : ipcp_vr_lattice::meet_with (const vrange &p_vr)
843 : {
844 496196 : return meet_with_1 (p_vr);
845 : }
846 :
847 : /* Meet the current value of the lattice with the range described by
848 : OTHER_VR. Return TRUE if anything changed. */
849 :
850 : bool
851 496196 : ipcp_vr_lattice::meet_with_1 (const vrange &other_vr)
852 : {
853 496196 : if (bottom_p ())
854 : return false;
855 :
856 496196 : if (other_vr.varying_p ())
857 0 : return set_to_bottom ();
858 :
859 496196 : bool res;
860 496196 : if (flag_checking)
861 : {
862 496196 : value_range save (m_vr);
863 496196 : res = m_vr.union_ (other_vr);
864 496196 : gcc_assert (res == (m_vr != save));
865 496196 : }
866 : else
867 0 : res = m_vr.union_ (other_vr);
868 : return res;
869 : }
870 :
871 : /* Return true if value range information in the lattice is yet unknown. */
872 :
873 : bool
874 : ipcp_vr_lattice::top_p () const
875 : {
876 172047 : return m_vr.undefined_p ();
877 : }
878 :
879 : /* Return true if value range information in the lattice is known to be
880 : unusable. */
881 :
882 : bool
883 5114774 : ipcp_vr_lattice::bottom_p () const
884 : {
885 496196 : return m_vr.varying_p ();
886 : }
887 :
888 : /* Set value range information in the lattice to bottom. Return true if it
889 : previously was in a different state. */
890 :
891 : bool
892 2479085 : ipcp_vr_lattice::set_to_bottom ()
893 : {
894 2479085 : if (m_vr.varying_p ())
895 : return false;
896 :
897 : /* Setting an unsupported type here forces the temporary to default
898 : to unsupported_range, which can handle VARYING/DEFINED ranges,
899 : but nothing else (union, intersect, etc). This allows us to set
900 : bottoms on any ranges, and is safe as all users of the lattice
901 : check for bottom first. */
902 2330857 : m_vr.set_range_class (void_type_node);
903 2330857 : m_vr.set_varying (void_type_node);
904 :
905 2330857 : return true;
906 : }
907 :
908 : /* Set the flag that this lattice is a recipient only, return true if it was
909 : not set before. */
910 :
911 : bool
912 29969 : ipcp_vr_lattice::set_recipient_only ()
913 : {
914 29969 : if (m_recipient_only)
915 : return false;
916 29969 : m_recipient_only = true;
917 29969 : return true;
918 : }
919 :
920 : /* Set lattice value to bottom, if it already isn't the case. */
921 :
922 : bool
923 2498968 : ipcp_bits_lattice::set_to_bottom ()
924 : {
925 2498968 : if (bottom_p ())
926 : return false;
927 2351259 : m_lattice_val = IPA_BITS_VARYING;
928 2351259 : m_value = 0;
929 2351259 : m_mask = -1;
930 2351259 : return true;
931 : }
932 :
933 : /* Set to constant if it isn't already. Only meant to be called
934 : when switching state from TOP. */
935 :
936 : bool
937 78245 : ipcp_bits_lattice::set_to_constant (const widest_int &value,
938 : const widest_int &mask)
939 : {
940 78245 : gcc_assert (top_p ());
941 78245 : m_lattice_val = IPA_BITS_CONSTANT;
942 78245 : m_value = wi::bit_and (wi::bit_not (mask), value);
943 78245 : m_mask = mask;
944 78245 : return true;
945 : }
946 :
947 : /* Return true if any of the known bits are non-zero. */
948 :
949 : bool
950 507 : ipcp_bits_lattice::known_nonzero_p () const
951 : {
952 507 : if (!constant_p ())
953 : return false;
954 507 : return wi::ne_p (wi::bit_and (wi::bit_not (m_mask), m_value), 0);
955 : }
956 :
957 : /* Set the flag that this lattice is a recipient only, return true if it was not
958 : set before. */
959 :
960 : bool
961 29969 : ipcp_bits_lattice::set_recipient_only ()
962 : {
963 29969 : if (m_recipient_only)
964 : return false;
965 29969 : m_recipient_only = true;
966 29969 : return true;
967 : }
968 :
969 : /* Convert operand to value, mask form. */
970 :
971 : void
972 2054 : ipcp_bits_lattice::get_value_and_mask (tree operand, widest_int *valuep, widest_int *maskp)
973 : {
974 2054 : wide_int get_nonzero_bits (const_tree);
975 :
976 2054 : if (TREE_CODE (operand) == INTEGER_CST)
977 : {
978 2054 : *valuep = wi::to_widest (operand);
979 2054 : *maskp = 0;
980 : }
981 : else
982 : {
983 0 : *valuep = 0;
984 0 : *maskp = -1;
985 : }
986 2054 : }
987 :
988 : /* Meet operation, similar to ccp_lattice_meet, we xor values
989 : if this->value, value have different values at same bit positions, we want
990 : to drop that bit to varying. Return true if mask is changed.
991 : This function assumes that the lattice value is in CONSTANT state. If
992 : DROP_ALL_ONES, mask out any known bits with value one afterwards. */
993 :
994 : bool
995 303659 : ipcp_bits_lattice::meet_with_1 (const widest_int &value,
996 : const widest_int &mask,
997 : unsigned precision, bool drop_all_ones)
998 : {
999 303659 : gcc_assert (constant_p ());
1000 :
1001 303659 : widest_int old_mask = m_mask;
1002 303659 : m_mask = (m_mask | mask) | (m_value ^ value);
1003 303659 : if (drop_all_ones)
1004 211 : m_mask |= m_value;
1005 :
1006 303659 : widest_int cap_mask = wi::shifted_mask <widest_int> (0, precision, true);
1007 303659 : m_mask |= cap_mask;
1008 303659 : if (wi::sext (m_mask, precision) == -1)
1009 3618 : return set_to_bottom ();
1010 :
1011 300041 : m_value &= ~m_mask;
1012 300041 : return m_mask != old_mask;
1013 303659 : }
1014 :
1015 : /* Meet the bits lattice with operand
1016 : described by <value, mask, sgn, precision. */
1017 :
1018 : bool
1019 416599 : ipcp_bits_lattice::meet_with (const widest_int &value,
1020 : const widest_int &mask,
1021 : unsigned precision)
1022 : {
1023 416599 : if (bottom_p ())
1024 : return false;
1025 :
1026 416599 : if (top_p ())
1027 : {
1028 125630 : if (wi::sext (mask, precision) == -1)
1029 52828 : return set_to_bottom ();
1030 72802 : return set_to_constant (value, mask);
1031 : }
1032 :
1033 290969 : return meet_with_1 (value, mask, precision, false);
1034 : }
1035 :
1036 : /* Meet bits lattice with the result of bit_value_binop (other, operand)
1037 : if code is binary operation or bit_value_unop (other) if code is unary op.
1038 : In the case when code is nop_expr, no adjustment is required. If
1039 : DROP_ALL_ONES, mask out any known bits with value one afterwards. */
1040 :
1041 : bool
1042 21744 : ipcp_bits_lattice::meet_with (ipcp_bits_lattice& other, unsigned precision,
1043 : signop sgn, enum tree_code code, tree operand,
1044 : bool drop_all_ones)
1045 : {
1046 21744 : if (other.bottom_p ())
1047 0 : return set_to_bottom ();
1048 :
1049 21744 : if (bottom_p () || other.top_p ())
1050 : return false;
1051 :
1052 18249 : widest_int adjusted_value, adjusted_mask;
1053 :
1054 18249 : if (TREE_CODE_CLASS (code) == tcc_binary)
1055 : {
1056 2054 : tree type = TREE_TYPE (operand);
1057 2054 : widest_int o_value, o_mask;
1058 2054 : get_value_and_mask (operand, &o_value, &o_mask);
1059 :
1060 2054 : bit_value_binop (code, sgn, precision, &adjusted_value, &adjusted_mask,
1061 4108 : sgn, precision, other.get_value (), other.get_mask (),
1062 2054 : TYPE_SIGN (type), TYPE_PRECISION (type), o_value, o_mask);
1063 :
1064 2054 : if (wi::sext (adjusted_mask, precision) == -1)
1065 87 : return set_to_bottom ();
1066 2054 : }
1067 :
1068 16195 : else if (TREE_CODE_CLASS (code) == tcc_unary)
1069 : {
1070 32340 : bit_value_unop (code, sgn, precision, &adjusted_value,
1071 32340 : &adjusted_mask, sgn, precision, other.get_value (),
1072 16170 : other.get_mask ());
1073 :
1074 16170 : if (wi::sext (adjusted_mask, precision) == -1)
1075 4 : return set_to_bottom ();
1076 : }
1077 :
1078 : else
1079 25 : return set_to_bottom ();
1080 :
1081 18133 : if (top_p ())
1082 : {
1083 5443 : if (drop_all_ones)
1084 : {
1085 296 : adjusted_mask |= adjusted_value;
1086 296 : adjusted_value &= ~adjusted_mask;
1087 : }
1088 5443 : widest_int cap_mask = wi::shifted_mask <widest_int> (0, precision, true);
1089 5443 : adjusted_mask |= cap_mask;
1090 5443 : if (wi::sext (adjusted_mask, precision) == -1)
1091 0 : return set_to_bottom ();
1092 5443 : return set_to_constant (adjusted_value, adjusted_mask);
1093 5443 : }
1094 : else
1095 12690 : return meet_with_1 (adjusted_value, adjusted_mask, precision,
1096 12690 : drop_all_ones);
1097 18249 : }
1098 :
1099 : /* Dump the contents of the list to FILE. */
1100 :
1101 : void
1102 124 : ipa_argagg_value_list::dump (FILE *f)
1103 : {
1104 124 : bool comma = false;
1105 348 : for (const ipa_argagg_value &av : m_elts)
1106 : {
1107 224 : fprintf (f, "%s %i[%u]=", comma ? "," : "",
1108 224 : av.index, av.unit_offset);
1109 224 : print_generic_expr (f, av.value);
1110 224 : if (av.by_ref)
1111 197 : fprintf (f, "(by_ref)");
1112 224 : if (av.killed)
1113 1 : fprintf (f, "(killed)");
1114 224 : comma = true;
1115 : }
1116 124 : fprintf (f, "\n");
1117 124 : }
1118 :
1119 : /* Dump the contents of the list to stderr. */
1120 :
1121 : void
1122 0 : ipa_argagg_value_list::debug ()
1123 : {
1124 0 : dump (stderr);
1125 0 : }
1126 :
1127 : /* Return the item describing a constant stored for INDEX at UNIT_OFFSET or
1128 : NULL if there is no such constant. */
1129 :
1130 : const ipa_argagg_value *
1131 32100336 : ipa_argagg_value_list::get_elt (int index, unsigned unit_offset) const
1132 : {
1133 32100336 : ipa_argagg_value key;
1134 32100336 : key.index = index;
1135 32100336 : key.unit_offset = unit_offset;
1136 32100336 : const ipa_argagg_value *res
1137 32100336 : = std::lower_bound (m_elts.begin (), m_elts.end (), key,
1138 7607830 : [] (const ipa_argagg_value &elt,
1139 : const ipa_argagg_value &val)
1140 : {
1141 7607830 : if (elt.index < val.index)
1142 : return true;
1143 6497559 : if (elt.index > val.index)
1144 : return false;
1145 5198354 : if (elt.unit_offset < val.unit_offset)
1146 : return true;
1147 : return false;
1148 : });
1149 :
1150 32100336 : if (res == m_elts.end ()
1151 3291495 : || res->index != index
1152 34831434 : || res->unit_offset != unit_offset)
1153 : res = nullptr;
1154 :
1155 : /* TODO: perhaps remove the check (that the underlying array is indeed
1156 : sorted) if it turns out it can be too slow? */
1157 32100336 : if (!flag_checking)
1158 : return res;
1159 :
1160 : const ipa_argagg_value *slow_res = NULL;
1161 : int prev_index = -1;
1162 : unsigned prev_unit_offset = 0;
1163 50012808 : for (const ipa_argagg_value &av : m_elts)
1164 : {
1165 17912472 : gcc_assert (prev_index < 0
1166 : || prev_index < av.index
1167 : || prev_unit_offset < av.unit_offset);
1168 17912472 : prev_index = av.index;
1169 17912472 : prev_unit_offset = av.unit_offset;
1170 17912472 : if (av.index == index
1171 8328857 : && av.unit_offset == unit_offset)
1172 17912472 : slow_res = &av;
1173 : }
1174 32100336 : gcc_assert (res == slow_res);
1175 :
1176 : return res;
1177 : }
1178 :
1179 : /* Return the first item describing a constant stored for parameter with INDEX,
1180 : regardless of offset or reference, or NULL if there is no such constant. */
1181 :
1182 : const ipa_argagg_value *
1183 253498 : ipa_argagg_value_list::get_elt_for_index (int index) const
1184 : {
1185 253498 : const ipa_argagg_value *res
1186 253498 : = std::lower_bound (m_elts.begin (), m_elts.end (), index,
1187 19412 : [] (const ipa_argagg_value &elt, unsigned idx)
1188 : {
1189 19412 : return elt.index < idx;
1190 : });
1191 253498 : if (res == m_elts.end ()
1192 253498 : || res->index != index)
1193 : res = nullptr;
1194 253498 : return res;
1195 : }
1196 :
1197 : /* Return the aggregate constant stored for INDEX at UNIT_OFFSET, not
1198 : performing any check of whether value is passed by reference, or NULL_TREE
1199 : if there is no such constant. */
1200 :
1201 : tree
1202 38426 : ipa_argagg_value_list::get_value (int index, unsigned unit_offset) const
1203 : {
1204 38426 : const ipa_argagg_value *av = get_elt (index, unit_offset);
1205 38426 : return av ? av->value : NULL_TREE;
1206 : }
1207 :
1208 : /* Return the aggregate constant stored for INDEX at UNIT_OFFSET, if it is
1209 : passed by reference or not according to BY_REF, or NULL_TREE if there is
1210 : no such constant. */
1211 :
1212 : tree
1213 32051995 : ipa_argagg_value_list::get_value (int index, unsigned unit_offset,
1214 : bool by_ref) const
1215 : {
1216 32051995 : const ipa_argagg_value *av = get_elt (index, unit_offset);
1217 32051995 : if (av && av->by_ref == by_ref)
1218 2211823 : return av->value;
1219 : return NULL_TREE;
1220 : }
1221 :
1222 : /* Return true if all elements present in OTHER are also present in this
1223 : list. */
1224 :
1225 : bool
1226 46 : ipa_argagg_value_list::superset_of_p (const ipa_argagg_value_list &other) const
1227 : {
1228 46 : unsigned j = 0;
1229 222 : for (unsigned i = 0; i < other.m_elts.size (); i++)
1230 : {
1231 193 : unsigned other_index = other.m_elts[i].index;
1232 193 : unsigned other_offset = other.m_elts[i].unit_offset;
1233 :
1234 193 : while (j < m_elts.size ()
1235 368 : && (m_elts[j].index < other_index
1236 352 : || (m_elts[j].index == other_index
1237 352 : && m_elts[j].unit_offset < other_offset)))
1238 175 : j++;
1239 :
1240 193 : if (j >= m_elts.size ()
1241 180 : || m_elts[j].index != other_index
1242 180 : || m_elts[j].unit_offset != other_offset
1243 180 : || m_elts[j].by_ref != other.m_elts[i].by_ref
1244 180 : || !m_elts[j].value
1245 373 : || !values_equal_for_ipcp_p (m_elts[j].value, other.m_elts[i].value))
1246 : return false;
1247 : }
1248 : return true;
1249 : }
1250 :
1251 : /* Push all items in this list that describe parameter SRC_INDEX into RES as
1252 : ones describing DST_INDEX while subtracting UNIT_DELTA from their unit
1253 : offsets but skip those which would end up with a negative offset. */
1254 :
1255 : void
1256 3131 : ipa_argagg_value_list::push_adjusted_values (unsigned src_index,
1257 : unsigned dest_index,
1258 : unsigned unit_delta,
1259 : vec<ipa_argagg_value> *res) const
1260 : {
1261 3131 : const ipa_argagg_value *av = get_elt_for_index (src_index);
1262 3131 : if (!av)
1263 : return;
1264 : unsigned prev_unit_offset = 0;
1265 : bool first = true;
1266 12498 : for (; av < m_elts.end (); ++av)
1267 : {
1268 10073 : if (av->index > src_index)
1269 : return;
1270 9463 : if (av->index == src_index
1271 9463 : && (av->unit_offset >= unit_delta)
1272 9319 : && av->value)
1273 : {
1274 9319 : ipa_argagg_value new_av;
1275 9319 : gcc_checking_assert (av->value);
1276 9319 : new_av.value = av->value;
1277 9319 : new_av.unit_offset = av->unit_offset - unit_delta;
1278 9319 : new_av.index = dest_index;
1279 9319 : new_av.by_ref = av->by_ref;
1280 9319 : gcc_assert (!av->killed);
1281 9319 : new_av.killed = false;
1282 :
1283 : /* Quick check that the offsets we push are indeed increasing. */
1284 9319 : gcc_assert (first
1285 : || new_av.unit_offset > prev_unit_offset);
1286 9319 : prev_unit_offset = new_av.unit_offset;
1287 9319 : first = false;
1288 :
1289 9319 : res->safe_push (new_av);
1290 : }
1291 : }
1292 : }
1293 :
1294 : /* Push to RES information about single lattices describing aggregate values in
1295 : PLATS as those describing parameter DEST_INDEX and the original offset minus
1296 : UNIT_DELTA. Return true if any item has been pushed to RES. */
1297 :
1298 : static bool
1299 2352181 : push_agg_values_from_plats (ipcp_param_lattices *plats, int dest_index,
1300 : unsigned unit_delta,
1301 : vec<ipa_argagg_value> *res)
1302 : {
1303 2352181 : if (plats->aggs_contain_variable)
1304 : return false;
1305 :
1306 1707212 : bool pushed_sth = false;
1307 1707212 : bool first = true;
1308 1707212 : unsigned prev_unit_offset = 0;
1309 1774449 : for (struct ipcp_agg_lattice *aglat = plats->aggs; aglat; aglat = aglat->next)
1310 133266 : if (aglat->is_single_const ()
1311 41653 : && (aglat->offset / BITS_PER_UNIT - unit_delta) >= 0)
1312 : {
1313 41653 : ipa_argagg_value iav;
1314 41653 : iav.value = aglat->values->value;
1315 41653 : iav.unit_offset = aglat->offset / BITS_PER_UNIT - unit_delta;
1316 41653 : iav.index = dest_index;
1317 41653 : iav.by_ref = plats->aggs_by_ref;
1318 41653 : iav.killed = false;
1319 :
1320 41653 : gcc_assert (first
1321 : || iav.unit_offset > prev_unit_offset);
1322 41653 : prev_unit_offset = iav.unit_offset;
1323 41653 : first = false;
1324 :
1325 41653 : pushed_sth = true;
1326 41653 : res->safe_push (iav);
1327 : }
1328 : return pushed_sth;
1329 : }
1330 :
1331 : /* Turn all values in LIST that are not present in OTHER into NULL_TREEs.
1332 : Return the number of remaining valid entries. */
1333 :
1334 : static unsigned
1335 56778 : intersect_argaggs_with (vec<ipa_argagg_value> &elts,
1336 : const vec<ipa_argagg_value> &other)
1337 : {
1338 56778 : unsigned valid_entries = 0;
1339 56778 : unsigned j = 0;
1340 431532 : for (unsigned i = 0; i < elts.length (); i++)
1341 : {
1342 374754 : if (!elts[i].value)
1343 55426 : continue;
1344 :
1345 319328 : unsigned this_index = elts[i].index;
1346 319328 : unsigned this_offset = elts[i].unit_offset;
1347 :
1348 319328 : while (j < other.length ()
1349 1200422 : && (other[j].index < this_index
1350 565281 : || (other[j].index == this_index
1351 561873 : && other[j].unit_offset < this_offset)))
1352 285331 : j++;
1353 :
1354 319328 : if (j >= other.length ())
1355 : {
1356 8896 : elts[i].value = NULL_TREE;
1357 8896 : continue;
1358 : }
1359 :
1360 310432 : if (other[j].index == this_index
1361 307024 : && other[j].unit_offset == this_offset
1362 299296 : && other[j].by_ref == elts[i].by_ref
1363 299296 : && other[j].value
1364 609728 : && values_equal_for_ipcp_p (other[j].value, elts[i].value))
1365 279305 : valid_entries++;
1366 : else
1367 31127 : elts[i].value = NULL_TREE;
1368 : }
1369 56778 : return valid_entries;
1370 : }
1371 :
1372 : /* Mark bot aggregate and scalar lattices as containing an unknown variable,
1373 : return true is any of them has not been marked as such so far. If if
1374 : MAKE_SIMPLE_RECIPIENTS is true, set the lattices that can only hold one
1375 : value to being recipients only, otherwise also set them to bottom. */
1376 :
1377 : static inline bool
1378 177933 : set_all_contains_variable (class ipcp_param_lattices *plats,
1379 : bool make_simple_recipients = false)
1380 : {
1381 177933 : bool ret;
1382 177933 : ret = plats->itself.set_contains_variable ();
1383 177933 : ret |= plats->ctxlat.set_contains_variable ();
1384 177933 : ret |= set_agg_lats_contain_variable (plats);
1385 177933 : if (make_simple_recipients)
1386 : {
1387 29969 : ret |= plats->bits_lattice.set_recipient_only ();
1388 29969 : ret |= plats->m_value_range.set_recipient_only ();
1389 : }
1390 : else
1391 : {
1392 147964 : ret |= plats->bits_lattice.set_to_bottom ();
1393 147964 : ret |= plats->m_value_range.set_to_bottom ();
1394 : }
1395 177933 : return ret;
1396 : }
1397 :
1398 : /* Worker of call_for_symbol_thunks_and_aliases, increment the integer DATA
1399 : points to by the number of callers to NODE. */
1400 :
1401 : static bool
1402 102899 : count_callers (cgraph_node *node, void *data)
1403 : {
1404 102899 : int *caller_count = (int *) data;
1405 :
1406 413858 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
1407 : /* Local thunks can be handled transparently, but if the thunk cannot
1408 : be optimized out, count it as a real use. */
1409 310959 : if (!cs->caller->thunk || !cs->caller->local)
1410 310959 : ++*caller_count;
1411 102899 : return false;
1412 : }
1413 :
1414 : /* Worker of call_for_symbol_thunks_and_aliases, it is supposed to be called on
1415 : the one caller of some other node. Set the caller's corresponding flag. */
1416 :
1417 : static bool
1418 59471 : set_single_call_flag (cgraph_node *node, void *)
1419 : {
1420 59471 : cgraph_edge *cs = node->callers;
1421 : /* Local thunks can be handled transparently, skip them. */
1422 59471 : while (cs && cs->caller->thunk && cs->caller->local)
1423 0 : cs = cs->next_caller;
1424 59471 : if (cs)
1425 58878 : if (ipa_node_params* info = ipa_node_params_sum->get (cs->caller))
1426 : {
1427 58877 : info->node_calling_single_call = true;
1428 58877 : return true;
1429 : }
1430 : return false;
1431 : }
1432 :
1433 : /* Initialize ipcp_lattices. */
1434 :
1435 : static void
1436 1313343 : initialize_node_lattices (struct cgraph_node *node)
1437 : {
1438 1313343 : ipa_node_params *info = ipa_node_params_sum->get (node);
1439 1313343 : struct cgraph_edge *ie;
1440 1313343 : bool disable = false, variable = false;
1441 1313343 : int i;
1442 :
1443 1313343 : gcc_checking_assert (node->has_gimple_body_p ());
1444 :
1445 1313343 : if (!ipa_get_param_count (info))
1446 : disable = true;
1447 1078781 : else if (node->local)
1448 : {
1449 91926 : int caller_count = 0;
1450 91926 : node->call_for_symbol_thunks_and_aliases (count_callers, &caller_count,
1451 : true);
1452 91926 : if (caller_count == 1)
1453 58878 : node->call_for_symbol_thunks_and_aliases (set_single_call_flag,
1454 : NULL, true);
1455 33048 : else if (caller_count == 0)
1456 : {
1457 1 : gcc_checking_assert (!opt_for_fn (node->decl, flag_toplevel_reorder));
1458 : variable = true;
1459 : }
1460 : }
1461 : else
1462 : {
1463 : /* When cloning is allowed, we can assume that externally visible
1464 : functions are not called. We will compensate this by cloning
1465 : later. */
1466 986855 : if (ipcp_versionable_function_p (node)
1467 986855 : && ipcp_cloning_candidate_p (node))
1468 : variable = true;
1469 : else
1470 : disable = true;
1471 : }
1472 :
1473 729 : if (dump_file && (dump_flags & TDF_DETAILS)
1474 1313510 : && !node->alias && !node->thunk)
1475 : {
1476 167 : fprintf (dump_file, "Initializing lattices of %s\n",
1477 : node->dump_name ());
1478 167 : if (disable || variable)
1479 133 : fprintf (dump_file, " Marking all lattices as %s\n",
1480 : disable ? "BOTTOM" : "VARIABLE");
1481 : }
1482 :
1483 1313343 : auto_vec<bool, 16> surviving_params;
1484 1313343 : bool pre_modified = false;
1485 :
1486 1313343 : clone_info *cinfo = clone_info::get (node);
1487 :
1488 1313343 : if (!disable && cinfo && cinfo->param_adjustments)
1489 : {
1490 : /* At the moment all IPA optimizations should use the number of
1491 : parameters of the prevailing decl as the m_always_copy_start.
1492 : Handling any other value would complicate the code below, so for the
1493 : time bing let's only assert it is so. */
1494 0 : gcc_assert ((cinfo->param_adjustments->m_always_copy_start
1495 : == ipa_get_param_count (info))
1496 : || cinfo->param_adjustments->m_always_copy_start < 0);
1497 :
1498 0 : pre_modified = true;
1499 0 : cinfo->param_adjustments->get_surviving_params (&surviving_params);
1500 :
1501 0 : if (dump_file && (dump_flags & TDF_DETAILS)
1502 0 : && !node->alias && !node->thunk)
1503 : {
1504 : bool first = true;
1505 0 : for (int j = 0; j < ipa_get_param_count (info); j++)
1506 : {
1507 0 : if (j < (int) surviving_params.length ()
1508 0 : && surviving_params[j])
1509 0 : continue;
1510 0 : if (first)
1511 : {
1512 0 : fprintf (dump_file,
1513 : " The following parameters are dead on arrival:");
1514 0 : first = false;
1515 : }
1516 0 : fprintf (dump_file, " %u", j);
1517 : }
1518 0 : if (!first)
1519 0 : fprintf (dump_file, "\n");
1520 : }
1521 : }
1522 :
1523 7237376 : for (i = 0; i < ipa_get_param_count (info); i++)
1524 : {
1525 2422626 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
1526 2422626 : tree type = ipa_get_type (info, i);
1527 2422626 : if (disable
1528 230832 : || !ipa_get_type (info, i)
1529 2653458 : || (pre_modified && (surviving_params.length () <= (unsigned) i
1530 0 : || !surviving_params[i])))
1531 : {
1532 2191794 : plats->itself.set_to_bottom ();
1533 2191794 : plats->ctxlat.set_to_bottom ();
1534 2191794 : set_agg_lats_to_bottom (plats);
1535 2191794 : plats->bits_lattice.set_to_bottom ();
1536 2191794 : plats->m_value_range.init (type);
1537 2191794 : plats->m_value_range.set_to_bottom ();
1538 : }
1539 : else
1540 : {
1541 230832 : plats->m_value_range.init (type);
1542 230832 : if (variable)
1543 29969 : set_all_contains_variable (plats, true);
1544 : }
1545 : }
1546 :
1547 1449786 : for (ie = node->indirect_calls; ie; ie = ie->next_callee)
1548 136443 : if (ie->indirect_info->param_index >= 0
1549 146086 : && is_a <cgraph_polymorphic_indirect_info *> (ie->indirect_info))
1550 9643 : ipa_get_parm_lattices (info,
1551 9643 : ie->indirect_info->param_index)->virt_call = 1;
1552 1313343 : }
1553 :
1554 : /* Return VALUE if it is NULL_TREE or if it can be directly safely IPA-CP
1555 : propagated to a parameter of type PARAM_TYPE, or return a fold-converted
1556 : VALUE to PARAM_TYPE if that is possible. Return NULL_TREE otherwise. */
1557 :
1558 : tree
1559 5285666 : ipacp_value_safe_for_type (tree param_type, tree value)
1560 : {
1561 5285666 : if (!value)
1562 : return NULL_TREE;
1563 5285332 : tree val_type = TREE_TYPE (value);
1564 5285332 : if (param_type == val_type
1565 5285332 : || useless_type_conversion_p (param_type, val_type))
1566 : return value;
1567 3396 : if (fold_convertible_p (param_type, value))
1568 3187 : return fold_convert (param_type, value);
1569 : else
1570 : return NULL_TREE;
1571 : }
1572 :
1573 : /* Return the result of a (possibly arithmetic) operation determined by OPCODE
1574 : on the constant value INPUT. OPERAND is 2nd operand for binary operation
1575 : and is required for binary operations. RES_TYPE, required when opcode is
1576 : not NOP_EXPR, is the type in which any operation is to be performed. Return
1577 : NULL_TREE if that cannot be determined or be considered an interprocedural
1578 : invariant. */
1579 :
1580 : static tree
1581 70317 : ipa_get_jf_arith_result (enum tree_code opcode, tree input, tree operand,
1582 : tree res_type)
1583 : {
1584 70317 : tree res;
1585 :
1586 70317 : if (opcode == NOP_EXPR)
1587 : return input;
1588 6789 : if (!is_gimple_ip_invariant (input))
1589 : return NULL_TREE;
1590 :
1591 6789 : if (opcode == ASSERT_EXPR)
1592 : {
1593 3802 : if (values_equal_for_ipcp_p (input, operand))
1594 : return input;
1595 : else
1596 112 : return NULL_TREE;
1597 : }
1598 :
1599 2987 : if (TREE_CODE_CLASS (opcode) == tcc_unary)
1600 103 : res = fold_unary (opcode, res_type, input);
1601 : else
1602 2884 : res = fold_binary (opcode, res_type, input, operand);
1603 :
1604 2987 : if (res && !is_gimple_ip_invariant (res))
1605 0 : return NULL_TREE;
1606 :
1607 : return res;
1608 : }
1609 :
1610 : /* Return the result of an ancestor jump function JFUNC on the constant value
1611 : INPUT. Return NULL_TREE if that cannot be determined. */
1612 :
1613 : static tree
1614 1302 : ipa_get_jf_ancestor_result (struct ipa_jump_func *jfunc, tree input)
1615 : {
1616 1302 : gcc_checking_assert (TREE_CODE (input) != TREE_BINFO);
1617 1302 : if (TREE_CODE (input) == ADDR_EXPR)
1618 : {
1619 1220 : gcc_checking_assert (is_gimple_ip_invariant_address (input));
1620 1220 : poly_int64 off = ipa_get_jf_ancestor_offset (jfunc);
1621 1220 : if (known_eq (off, 0))
1622 : return input;
1623 1098 : poly_int64 byte_offset = exact_div (off, BITS_PER_UNIT);
1624 2196 : return build1 (ADDR_EXPR, TREE_TYPE (input),
1625 1098 : fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (input)), input,
1626 1098 : build_int_cst (ptr_type_node, byte_offset)));
1627 : }
1628 82 : else if (ipa_get_jf_ancestor_keep_null (jfunc)
1629 82 : && zerop (input))
1630 : return input;
1631 : else
1632 78 : return NULL_TREE;
1633 : }
1634 :
1635 : /* Determine whether JFUNC evaluates to a single known constant value and if
1636 : so, return it. Otherwise return NULL. INFO describes the caller node or
1637 : the one it is inlined to, so that pass-through jump functions can be
1638 : evaluated. PARM_TYPE is the type of the parameter to which the result is
1639 : passed. */
1640 :
1641 : tree
1642 19093019 : ipa_value_from_jfunc (class ipa_node_params *info, struct ipa_jump_func *jfunc,
1643 : tree parm_type)
1644 : {
1645 19093019 : if (!parm_type)
1646 : return NULL_TREE;
1647 18850223 : if (jfunc->type == IPA_JF_CONST)
1648 4670511 : return ipacp_value_safe_for_type (parm_type, ipa_get_jf_constant (jfunc));
1649 14179712 : else if (jfunc->type == IPA_JF_PASS_THROUGH
1650 11129327 : || jfunc->type == IPA_JF_ANCESTOR)
1651 : {
1652 3910893 : tree input;
1653 3910893 : int idx;
1654 :
1655 3910893 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1656 3050385 : idx = ipa_get_jf_pass_through_formal_id (jfunc);
1657 : else
1658 860508 : idx = ipa_get_jf_ancestor_formal_id (jfunc);
1659 :
1660 3910893 : if (info->ipcp_orig_node)
1661 49569 : input = info->known_csts[idx];
1662 : else
1663 : {
1664 3861324 : ipcp_lattice<tree> *lat;
1665 :
1666 6904588 : if (info->lattices.is_empty ()
1667 3043264 : || idx >= ipa_get_param_count (info))
1668 : return NULL_TREE;
1669 3043264 : lat = ipa_get_scalar_lat (info, idx);
1670 3043264 : if (!lat->is_single_const ())
1671 : return NULL_TREE;
1672 152 : input = lat->values->value;
1673 : }
1674 :
1675 49721 : if (!input)
1676 : return NULL_TREE;
1677 :
1678 19938 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1679 : {
1680 18954 : enum tree_code opcode = ipa_get_jf_pass_through_operation (jfunc);
1681 18954 : tree op2 = ipa_get_jf_pass_through_operand (jfunc);
1682 18954 : tree op_type
1683 18954 : = (opcode == NOP_EXPR) ? NULL_TREE
1684 961 : : ipa_get_jf_pass_through_op_type (jfunc);
1685 18954 : tree cstval = ipa_get_jf_arith_result (opcode, input, op2, op_type);
1686 18954 : return ipacp_value_safe_for_type (parm_type, cstval);
1687 : }
1688 : else
1689 984 : return ipacp_value_safe_for_type (parm_type,
1690 : ipa_get_jf_ancestor_result (jfunc,
1691 984 : input));
1692 : }
1693 : else
1694 : return NULL_TREE;
1695 : }
1696 :
1697 : /* Determine whether JFUNC evaluates to single known polymorphic context, given
1698 : that INFO describes the caller node or the one it is inlined to, CS is the
1699 : call graph edge corresponding to JFUNC and CSIDX index of the described
1700 : parameter. */
1701 :
1702 : ipa_polymorphic_call_context
1703 937042 : ipa_context_from_jfunc (ipa_node_params *info, cgraph_edge *cs, int csidx,
1704 : ipa_jump_func *jfunc)
1705 : {
1706 937042 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
1707 937042 : ipa_polymorphic_call_context ctx;
1708 937042 : ipa_polymorphic_call_context *edge_ctx
1709 937042 : = cs ? ipa_get_ith_polymorhic_call_context (args, csidx) : NULL;
1710 :
1711 400824 : if (edge_ctx && !edge_ctx->useless_p ())
1712 392803 : ctx = *edge_ctx;
1713 :
1714 937042 : if (jfunc->type == IPA_JF_PASS_THROUGH
1715 836303 : || jfunc->type == IPA_JF_ANCESTOR)
1716 : {
1717 109501 : ipa_polymorphic_call_context srcctx;
1718 109501 : int srcidx;
1719 109501 : bool type_preserved = true;
1720 109501 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1721 : {
1722 100739 : if (ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR)
1723 1787 : return ctx;
1724 98952 : type_preserved = ipa_get_jf_pass_through_type_preserved (jfunc);
1725 98952 : srcidx = ipa_get_jf_pass_through_formal_id (jfunc);
1726 : }
1727 : else
1728 : {
1729 8762 : type_preserved = ipa_get_jf_ancestor_type_preserved (jfunc);
1730 8762 : srcidx = ipa_get_jf_ancestor_formal_id (jfunc);
1731 : }
1732 107714 : if (info->ipcp_orig_node)
1733 : {
1734 12009 : if (info->known_contexts.exists ())
1735 1725 : srcctx = info->known_contexts[srcidx];
1736 : }
1737 : else
1738 : {
1739 185983 : if (info->lattices.is_empty ()
1740 90278 : || srcidx >= ipa_get_param_count (info))
1741 5427 : return ctx;
1742 90278 : ipcp_lattice<ipa_polymorphic_call_context> *lat;
1743 90278 : lat = ipa_get_poly_ctx_lat (info, srcidx);
1744 90278 : if (!lat->is_single_const ())
1745 86269 : return ctx;
1746 4009 : srcctx = lat->values->value;
1747 : }
1748 16018 : if (srcctx.useless_p ())
1749 10737 : return ctx;
1750 5281 : if (jfunc->type == IPA_JF_ANCESTOR)
1751 259 : srcctx.offset_by (ipa_get_jf_ancestor_offset (jfunc));
1752 5281 : if (!type_preserved)
1753 3229 : srcctx.possible_dynamic_type_change (cs->in_polymorphic_cdtor);
1754 5281 : srcctx.combine_with (ctx);
1755 5281 : return srcctx;
1756 : }
1757 :
1758 827541 : return ctx;
1759 : }
1760 :
1761 : /* Emulate effects of unary OPERATION and/or conversion from SRC_TYPE to
1762 : DST_TYPE on value range in SRC_VR and store it to DST_VR. Return true if
1763 : the result is a range that is not VARYING nor UNDEFINED. */
1764 :
1765 : bool
1766 10020879 : ipa_vr_operation_and_type_effects (vrange &dst_vr,
1767 : const vrange &src_vr,
1768 : enum tree_code operation,
1769 : tree dst_type, tree src_type)
1770 : {
1771 18807430 : if (!ipa_vr_supported_type_p (dst_type)
1772 20041758 : || !ipa_vr_supported_type_p (src_type))
1773 : return false;
1774 :
1775 10020879 : range_op_handler handler (operation);
1776 10020879 : if (!handler)
1777 : return false;
1778 :
1779 10020879 : value_range varying (dst_type);
1780 10020879 : varying.set_varying (dst_type);
1781 :
1782 10020879 : return (handler.operand_check_p (dst_type, src_type, dst_type)
1783 10020879 : && handler.fold_range (dst_vr, dst_type, src_vr, varying)
1784 10020877 : && !dst_vr.varying_p ()
1785 20041696 : && !dst_vr.undefined_p ());
1786 10020879 : }
1787 :
1788 : /* Same as above, but the SRC_VR argument is an IPA_VR which must
1789 : first be extracted onto a vrange. */
1790 :
1791 : bool
1792 10013088 : ipa_vr_operation_and_type_effects (vrange &dst_vr,
1793 : const ipa_vr &src_vr,
1794 : enum tree_code operation,
1795 : tree dst_type, tree src_type)
1796 : {
1797 10013088 : value_range tmp;
1798 10013088 : src_vr.get_vrange (tmp);
1799 10013088 : return ipa_vr_operation_and_type_effects (dst_vr, tmp, operation,
1800 10013088 : dst_type, src_type);
1801 10013088 : }
1802 :
1803 : /* Given a PASS_THROUGH jump function JFUNC that takes as its source SRC_VR of
1804 : SRC_TYPE and the result needs to be DST_TYPE, if any value range information
1805 : can be deduced at all, intersect VR with it. CONTEXT_NODE is the call graph
1806 : node representing the function for which optimization flags should be
1807 : evaluated. */
1808 :
1809 : static void
1810 93461 : ipa_vr_intersect_with_arith_jfunc (vrange &vr,
1811 : ipa_jump_func *jfunc,
1812 : cgraph_node *context_node,
1813 : const value_range &src_vr,
1814 : tree src_type,
1815 : tree dst_type)
1816 : {
1817 93461 : if (src_vr.undefined_p () || src_vr.varying_p ())
1818 92251 : return;
1819 :
1820 92977 : enum tree_code operation = ipa_get_jf_pass_through_operation (jfunc);
1821 92977 : if (TREE_CODE_CLASS (operation) == tcc_unary)
1822 : {
1823 91767 : value_range op_res;
1824 91767 : const value_range *inter_vr;
1825 91767 : if (operation != NOP_EXPR)
1826 : {
1827 93 : tree operation_type = ipa_get_jf_pass_through_op_type (jfunc);
1828 93 : op_res.set_varying (operation_type);
1829 93 : if (!ipa_vr_operation_and_type_effects (op_res, src_vr, operation,
1830 : operation_type, src_type))
1831 : return;
1832 : inter_vr = &op_res;
1833 : src_type = operation_type;
1834 : }
1835 : else
1836 : inter_vr = &src_vr;
1837 :
1838 91767 : if (src_type != dst_type)
1839 : {
1840 6488 : value_range tmp_res (dst_type);
1841 6488 : if (!ipa_vr_operation_and_type_effects (tmp_res, *inter_vr, NOP_EXPR,
1842 : dst_type, src_type))
1843 0 : return;
1844 6488 : vr.intersect (tmp_res);
1845 6488 : }
1846 : else
1847 85279 : vr.intersect (*inter_vr);
1848 : return;
1849 91767 : }
1850 :
1851 1210 : tree operand = ipa_get_jf_pass_through_operand (jfunc);
1852 1210 : range_op_handler handler (operation);
1853 1210 : if (!handler)
1854 : return;
1855 1210 : value_range op_vr (TREE_TYPE (operand));
1856 1210 : ipa_get_range_from_ip_invariant (op_vr, operand, context_node);
1857 :
1858 1210 : tree operation_type = ipa_get_jf_pass_through_op_type (jfunc);
1859 1210 : value_range op_res (operation_type);
1860 1650 : if (!ipa_vr_supported_type_p (operation_type)
1861 1210 : || !handler.operand_check_p (operation_type, src_type, op_vr.type ())
1862 1210 : || !handler.fold_range (op_res, operation_type, src_vr, op_vr))
1863 0 : return;
1864 :
1865 1210 : value_range tmp_res (dst_type);
1866 1210 : if (ipa_vr_operation_and_type_effects (tmp_res, op_res, NOP_EXPR, dst_type,
1867 : operation_type))
1868 1162 : vr.intersect (tmp_res);
1869 1210 : }
1870 :
1871 : /* Determine range of JFUNC given that INFO describes the caller node or
1872 : the one it is inlined to, CS is the call graph edge corresponding to JFUNC
1873 : and PARM_TYPE of the parameter. */
1874 :
1875 : void
1876 12883365 : ipa_value_range_from_jfunc (vrange &vr,
1877 : ipa_node_params *info, cgraph_edge *cs,
1878 : ipa_jump_func *jfunc, tree parm_type)
1879 : {
1880 12883365 : vr.set_varying (parm_type);
1881 :
1882 12883365 : if (jfunc->m_vr && jfunc->m_vr->known_p ())
1883 9161799 : ipa_vr_operation_and_type_effects (vr,
1884 : *jfunc->m_vr,
1885 : NOP_EXPR, parm_type,
1886 9161799 : jfunc->m_vr->type ());
1887 12883365 : if (vr.singleton_p ())
1888 : return;
1889 :
1890 12883280 : if (jfunc->type == IPA_JF_PASS_THROUGH)
1891 : {
1892 2450130 : ipcp_transformation *sum
1893 2450130 : = ipcp_get_transformation_summary (cs->caller->inlined_to
1894 : ? cs->caller->inlined_to
1895 : : cs->caller);
1896 2450130 : if (!sum || !sum->m_vr)
1897 2372134 : return;
1898 :
1899 120789 : int idx = ipa_get_jf_pass_through_formal_id (jfunc);
1900 :
1901 120789 : if (!(*sum->m_vr)[idx].known_p ())
1902 : return;
1903 77996 : tree src_type = ipa_get_type (info, idx);
1904 77996 : value_range srcvr;
1905 77996 : (*sum->m_vr)[idx].get_vrange (srcvr);
1906 :
1907 77996 : ipa_vr_intersect_with_arith_jfunc (vr, jfunc, cs->caller, srcvr, src_type,
1908 : parm_type);
1909 77996 : }
1910 : }
1911 :
1912 : /* Determine whether ITEM, jump function for an aggregate part, evaluates to a
1913 : single known constant value and if so, return it. Otherwise return NULL.
1914 : NODE and INFO describes the caller node or the one it is inlined to, and
1915 : its related info. */
1916 :
1917 : tree
1918 3571706 : ipa_agg_value_from_jfunc (ipa_node_params *info, cgraph_node *node,
1919 : const ipa_agg_jf_item *item)
1920 : {
1921 3571706 : tree value = NULL_TREE;
1922 3571706 : int src_idx;
1923 :
1924 3571706 : if (item->offset < 0
1925 3521761 : || item->jftype == IPA_JF_UNKNOWN
1926 3358962 : || item->offset >= (HOST_WIDE_INT) UINT_MAX * BITS_PER_UNIT)
1927 : return NULL_TREE;
1928 :
1929 3358962 : if (item->jftype == IPA_JF_CONST)
1930 2993981 : return item->value.constant;
1931 :
1932 364981 : gcc_checking_assert (item->jftype == IPA_JF_PASS_THROUGH
1933 : || item->jftype == IPA_JF_LOAD_AGG);
1934 :
1935 364981 : src_idx = item->value.pass_through.formal_id;
1936 :
1937 364981 : if (info->ipcp_orig_node)
1938 : {
1939 16915 : if (item->jftype == IPA_JF_PASS_THROUGH)
1940 3882 : value = info->known_csts[src_idx];
1941 13033 : else if (ipcp_transformation *ts = ipcp_get_transformation_summary (node))
1942 : {
1943 13033 : ipa_argagg_value_list avl (ts);
1944 13033 : value = avl.get_value (src_idx,
1945 13033 : item->value.load_agg.offset / BITS_PER_UNIT,
1946 13033 : item->value.load_agg.by_ref);
1947 : }
1948 : }
1949 348066 : else if (!info->lattices.is_empty ())
1950 : {
1951 230797 : class ipcp_param_lattices *src_plats
1952 230797 : = ipa_get_parm_lattices (info, src_idx);
1953 :
1954 230797 : if (item->jftype == IPA_JF_PASS_THROUGH)
1955 : {
1956 138633 : struct ipcp_lattice<tree> *lat = &src_plats->itself;
1957 :
1958 138633 : if (!lat->is_single_const ())
1959 : return NULL_TREE;
1960 :
1961 0 : value = lat->values->value;
1962 : }
1963 92164 : else if (src_plats->aggs
1964 12316 : && !src_plats->aggs_bottom
1965 12316 : && !src_plats->aggs_contain_variable
1966 1503 : && src_plats->aggs_by_ref == item->value.load_agg.by_ref)
1967 : {
1968 : struct ipcp_agg_lattice *aglat;
1969 :
1970 2370 : for (aglat = src_plats->aggs; aglat; aglat = aglat->next)
1971 : {
1972 2370 : if (aglat->offset > item->value.load_agg.offset)
1973 : break;
1974 :
1975 2338 : if (aglat->offset == item->value.load_agg.offset)
1976 : {
1977 1471 : if (aglat->is_single_const ())
1978 7 : value = aglat->values->value;
1979 : break;
1980 : }
1981 : }
1982 : }
1983 : }
1984 :
1985 16954 : if (!value)
1986 : return NULL_TREE;
1987 :
1988 10190 : if (item->jftype == IPA_JF_LOAD_AGG)
1989 : {
1990 7854 : tree load_type = item->value.load_agg.type;
1991 7854 : tree value_type = TREE_TYPE (value);
1992 :
1993 : /* Ensure value type is compatible with load type. */
1994 7854 : if (!useless_type_conversion_p (load_type, value_type))
1995 : return NULL_TREE;
1996 : }
1997 :
1998 20380 : tree cstval = ipa_get_jf_arith_result (item->value.pass_through.operation,
1999 : value,
2000 10190 : item->value.pass_through.operand,
2001 10190 : item->value.pass_through.op_type);
2002 10190 : return ipacp_value_safe_for_type (item->type, cstval);
2003 : }
2004 :
2005 : /* Process all items in AGG_JFUNC relative to caller (or the node the original
2006 : caller is inlined to) NODE which described by INFO and push the results to
2007 : RES as describing values passed in parameter DST_INDEX. */
2008 :
2009 : void
2010 15372413 : ipa_push_agg_values_from_jfunc (ipa_node_params *info, cgraph_node *node,
2011 : ipa_agg_jump_function *agg_jfunc,
2012 : unsigned dst_index,
2013 : vec<ipa_argagg_value> *res)
2014 : {
2015 15372413 : unsigned prev_unit_offset = 0;
2016 15372413 : bool first = true;
2017 :
2018 20565311 : for (const ipa_agg_jf_item &item : agg_jfunc->items)
2019 : {
2020 2527344 : tree value = ipa_agg_value_from_jfunc (info, node, &item);
2021 2527344 : if (!value)
2022 541789 : continue;
2023 :
2024 1985555 : ipa_argagg_value iav;
2025 1985555 : iav.value = value;
2026 1985555 : iav.unit_offset = item.offset / BITS_PER_UNIT;
2027 1985555 : iav.index = dst_index;
2028 1985555 : iav.by_ref = agg_jfunc->by_ref;
2029 1985555 : iav.killed = 0;
2030 :
2031 1985555 : gcc_assert (first
2032 : || iav.unit_offset > prev_unit_offset);
2033 1985555 : prev_unit_offset = iav.unit_offset;
2034 1985555 : first = false;
2035 :
2036 1985555 : res->safe_push (iav);
2037 : }
2038 15372413 : }
2039 :
2040 : /* If checking is enabled, verify that no lattice is in the TOP state, i.e. not
2041 : bottom, not containing a variable component and without any known value at
2042 : the same time. */
2043 :
2044 : DEBUG_FUNCTION void
2045 131912 : ipcp_verify_propagated_values (void)
2046 : {
2047 131912 : struct cgraph_node *node;
2048 :
2049 1454022 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
2050 : {
2051 1322110 : ipa_node_params *info = ipa_node_params_sum->get (node);
2052 1322110 : if (!opt_for_fn (node->decl, flag_ipa_cp)
2053 1322110 : || !opt_for_fn (node->decl, optimize))
2054 8784 : continue;
2055 1313326 : int i, count = ipa_get_param_count (info);
2056 :
2057 3735938 : for (i = 0; i < count; i++)
2058 : {
2059 2422612 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (info, i);
2060 :
2061 2422612 : if (!lat->bottom
2062 229818 : && !lat->contains_variable
2063 32206 : && lat->values_count == 0)
2064 : {
2065 0 : if (dump_file)
2066 : {
2067 0 : symtab->dump (dump_file);
2068 0 : fprintf (dump_file, "\nIPA lattices after constant "
2069 : "propagation, before gcc_unreachable:\n");
2070 0 : print_all_lattices (dump_file, true, false);
2071 : }
2072 :
2073 0 : gcc_unreachable ();
2074 : }
2075 : }
2076 : }
2077 131912 : }
2078 :
2079 : /* Return true iff X and Y should be considered equal contexts by IPA-CP. */
2080 :
2081 : static bool
2082 2886 : values_equal_for_ipcp_p (ipa_polymorphic_call_context x,
2083 : ipa_polymorphic_call_context y)
2084 : {
2085 2338 : return x.equal_to (y);
2086 : }
2087 :
2088 :
2089 : /* Add a new value source to the value represented by THIS, marking that a
2090 : value comes from edge CS and (if the underlying jump function is a
2091 : pass-through or an ancestor one) from a caller value SRC_VAL of a caller
2092 : parameter described by SRC_INDEX. OFFSET is negative if the source was the
2093 : scalar value of the parameter itself or the offset within an aggregate. */
2094 :
2095 : template <typename valtype>
2096 : void
2097 341672 : ipcp_value<valtype>::add_source (cgraph_edge *cs, ipcp_value *src_val,
2098 : int src_idx, HOST_WIDE_INT offset)
2099 : {
2100 : ipcp_value_source<valtype> *src;
2101 :
2102 492951 : src = new (ipcp_sources_pool.allocate ()) ipcp_value_source<valtype>;
2103 492951 : src->offset = offset;
2104 492951 : src->cs = cs;
2105 492951 : src->val = src_val;
2106 492951 : src->index = src_idx;
2107 :
2108 492951 : src->next = sources;
2109 492951 : sources = src;
2110 : }
2111 :
2112 : /* Allocate a new ipcp_value holding a tree constant, initialize its value to
2113 : SOURCE and clear all other fields. */
2114 :
2115 : static ipcp_value<tree> *
2116 143007 : allocate_and_init_ipcp_value (tree cst, unsigned same_lat_gen_level)
2117 : {
2118 143007 : ipcp_value<tree> *val;
2119 :
2120 143007 : val = new (ipcp_cst_values_pool.allocate ()) ipcp_value<tree>();
2121 143007 : val->value = cst;
2122 143007 : val->self_recursion_generated_level = same_lat_gen_level;
2123 143007 : return val;
2124 : }
2125 :
2126 : /* Allocate a new ipcp_value holding a polymorphic context, initialize its
2127 : value to SOURCE and clear all other fields. */
2128 :
2129 : static ipcp_value<ipa_polymorphic_call_context> *
2130 8272 : allocate_and_init_ipcp_value (ipa_polymorphic_call_context ctx,
2131 : unsigned same_lat_gen_level)
2132 : {
2133 8272 : ipcp_value<ipa_polymorphic_call_context> *val;
2134 :
2135 8272 : val = new (ipcp_poly_ctx_values_pool.allocate ())
2136 8272 : ipcp_value<ipa_polymorphic_call_context>();
2137 8272 : val->value = ctx;
2138 8272 : val->self_recursion_generated_level = same_lat_gen_level;
2139 8272 : return val;
2140 : }
2141 :
2142 : /* Try to add NEWVAL to LAT, potentially creating a new ipcp_value for it. CS,
2143 : SRC_VAL SRC_INDEX and OFFSET are meant for add_source and have the same
2144 : meaning. OFFSET -1 means the source is scalar and not a part of an
2145 : aggregate. If non-NULL, VAL_P records address of existing or newly added
2146 : ipcp_value.
2147 :
2148 : If the value is generated for a self-recursive call as a result of an
2149 : arithmetic pass-through jump-function acting on a value in the same lattice,
2150 : SAME_LAT_GEN_LEVEL must be the length of such chain, otherwise it must be
2151 : zero. If it is non-zero, PARAM_IPA_CP_VALUE_LIST_SIZE limit is ignored. */
2152 :
2153 : template <typename valtype>
2154 : bool
2155 505366 : ipcp_lattice<valtype>::add_value (valtype newval, cgraph_edge *cs,
2156 : ipcp_value<valtype> *src_val,
2157 : int src_idx, HOST_WIDE_INT offset,
2158 : ipcp_value<valtype> **val_p,
2159 : unsigned same_lat_gen_level)
2160 : {
2161 505366 : ipcp_value<valtype> *val, *last_val = NULL;
2162 :
2163 505366 : if (val_p)
2164 1257 : *val_p = NULL;
2165 :
2166 505366 : if (bottom)
2167 : return false;
2168 :
2169 976657 : for (val = values; val; last_val = val, val = val->next)
2170 824002 : if (values_equal_for_ipcp_p (val->value, newval))
2171 : {
2172 349347 : if (val_p)
2173 416 : *val_p = val;
2174 :
2175 349347 : if (val->self_recursion_generated_level < same_lat_gen_level)
2176 179 : val->self_recursion_generated_level = same_lat_gen_level;
2177 :
2178 349347 : if (ipa_edge_within_scc (cs))
2179 : {
2180 : ipcp_value_source<valtype> *s;
2181 48912 : for (s = val->sources; s; s = s->next)
2182 44674 : if (s->cs == cs && s->val == src_val)
2183 : break;
2184 11913 : if (s)
2185 : return false;
2186 : }
2187 :
2188 341672 : val->add_source (cs, src_val, src_idx, offset);
2189 341672 : return false;
2190 : }
2191 :
2192 152655 : if (!same_lat_gen_level && values_count >= opt_for_fn (cs->callee->decl,
2193 : param_ipa_cp_value_list_size))
2194 : {
2195 : /* We can only free sources, not the values themselves, because sources
2196 : of other values in this SCC might point to them. */
2197 12366 : for (val = values; val; val = val->next)
2198 : {
2199 40602 : while (val->sources)
2200 : {
2201 29612 : ipcp_value_source<valtype> *src = val->sources;
2202 29612 : val->sources = src->next;
2203 29612 : ipcp_sources_pool.remove ((ipcp_value_source<tree>*)src);
2204 : }
2205 : }
2206 1376 : values = NULL;
2207 1376 : return set_to_bottom ();
2208 : }
2209 :
2210 151279 : values_count++;
2211 151279 : val = allocate_and_init_ipcp_value (newval, same_lat_gen_level);
2212 151279 : val->add_source (cs, src_val, src_idx, offset);
2213 151279 : val->next = NULL;
2214 :
2215 : /* Add the new value to end of value list, which can reduce iterations
2216 : of propagation stage for recursive function. */
2217 151279 : if (last_val)
2218 46020 : last_val->next = val;
2219 : else
2220 105259 : values = val;
2221 :
2222 151279 : if (val_p)
2223 841 : *val_p = val;
2224 :
2225 : return true;
2226 : }
2227 :
2228 : /* A helper function that returns result of operation specified by OPCODE on
2229 : the value of SRC_VAL. If non-NULL, OPND1_TYPE is expected type for the
2230 : value of SRC_VAL. If the operation is binary, OPND2 is a constant value
2231 : acting as its second operand. OP_TYPE is the type in which the operation is
2232 : performed. */
2233 :
2234 : static tree
2235 21996 : get_val_across_arith_op (enum tree_code opcode,
2236 : tree opnd1_type,
2237 : tree opnd2,
2238 : ipcp_value<tree> *src_val,
2239 : tree op_type)
2240 : {
2241 21996 : tree opnd1 = src_val->value;
2242 :
2243 : /* Skip source values that is incompatible with specified type. */
2244 21996 : if (opnd1_type
2245 21996 : && !useless_type_conversion_p (opnd1_type, TREE_TYPE (opnd1)))
2246 : return NULL_TREE;
2247 :
2248 21996 : return ipa_get_jf_arith_result (opcode, opnd1, opnd2, op_type);
2249 : }
2250 :
2251 : /* Propagate values through an arithmetic transformation described by a jump
2252 : function associated with edge CS, taking values from SRC_LAT and putting
2253 : them into DEST_LAT. OPND1_TYPE, if non-NULL, is the expected type for the
2254 : values in SRC_LAT. OPND2 is a constant value if transformation is a binary
2255 : operation. SRC_OFFSET specifies offset in an aggregate if SRC_LAT describes
2256 : lattice of a part of an aggregate, otherwise it should be -1. SRC_IDX is
2257 : the index of the source parameter. OP_TYPE is the type in which the
2258 : operation is performed and can be NULL when OPCODE is NOP_EXPR. RES_TYPE is
2259 : the value type of result being propagated into. Return true if DEST_LAT
2260 : changed. */
2261 :
2262 : static bool
2263 77995 : propagate_vals_across_arith_jfunc (cgraph_edge *cs,
2264 : enum tree_code opcode,
2265 : tree opnd1_type,
2266 : tree opnd2,
2267 : ipcp_lattice<tree> *src_lat,
2268 : ipcp_lattice<tree> *dest_lat,
2269 : HOST_WIDE_INT src_offset,
2270 : int src_idx,
2271 : tree op_type,
2272 : tree res_type)
2273 : {
2274 77995 : ipcp_value<tree> *src_val;
2275 77995 : bool ret = false;
2276 :
2277 : /* Due to circular dependencies, propagating within an SCC through arithmetic
2278 : transformation would create infinite number of values. But for
2279 : self-feeding recursive function, we could allow propagation in a limited
2280 : count, and this can enable a simple kind of recursive function versioning.
2281 : For other scenario, we would just make lattices bottom. */
2282 77995 : if (opcode != NOP_EXPR && ipa_edge_within_scc (cs))
2283 : {
2284 2184 : int i;
2285 :
2286 2184 : int max_recursive_depth = opt_for_fn(cs->caller->decl,
2287 : param_ipa_cp_max_recursive_depth);
2288 2184 : if (src_lat != dest_lat || max_recursive_depth < 1)
2289 1666 : return dest_lat->set_contains_variable ();
2290 :
2291 : /* No benefit if recursive execution is in low probability. */
2292 1300 : if (cs->sreal_frequency () * 100
2293 2600 : <= ((sreal) 1) * opt_for_fn (cs->caller->decl,
2294 : param_ipa_cp_min_recursive_probability))
2295 89 : return dest_lat->set_contains_variable ();
2296 :
2297 1211 : auto_vec<ipcp_value<tree> *, 8> val_seeds;
2298 :
2299 2258 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2300 : {
2301 : /* Now we do not use self-recursively generated value as propagation
2302 : source, this is absolutely conservative, but could avoid explosion
2303 : of lattice's value space, especially when one recursive function
2304 : calls another recursive. */
2305 1740 : if (src_val->self_recursion_generated_p ())
2306 : {
2307 909 : ipcp_value_source<tree> *s;
2308 :
2309 : /* If the lattice has already been propagated for the call site,
2310 : no need to do that again. */
2311 1422 : for (s = src_val->sources; s; s = s->next)
2312 1206 : if (s->cs == cs)
2313 693 : return dest_lat->set_contains_variable ();
2314 : }
2315 : else
2316 831 : val_seeds.safe_push (src_val);
2317 : }
2318 :
2319 1036 : gcc_assert ((int) val_seeds.length () <= param_ipa_cp_value_list_size);
2320 :
2321 : /* Recursively generate lattice values with a limited count. */
2322 1354 : FOR_EACH_VEC_ELT (val_seeds, i, src_val)
2323 : {
2324 1416 : for (int j = 1; j < max_recursive_depth; j++)
2325 : {
2326 1261 : tree cstval = get_val_across_arith_op (opcode, opnd1_type, opnd2,
2327 : src_val, op_type);
2328 1261 : cstval = ipacp_value_safe_for_type (res_type, cstval);
2329 1261 : if (!cstval)
2330 : break;
2331 :
2332 1257 : ret |= dest_lat->add_value (cstval, cs, src_val, src_idx,
2333 : src_offset, &src_val, j);
2334 1257 : gcc_checking_assert (src_val);
2335 : }
2336 : }
2337 518 : ret |= dest_lat->set_contains_variable ();
2338 1211 : }
2339 : else
2340 96671 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2341 : {
2342 : /* Now we do not use self-recursively generated value as propagation
2343 : source, otherwise it is easy to make value space of normal lattice
2344 : overflow. */
2345 20860 : if (src_val->self_recursion_generated_p ())
2346 : {
2347 125 : ret |= dest_lat->set_contains_variable ();
2348 125 : continue;
2349 : }
2350 :
2351 20735 : tree cstval = get_val_across_arith_op (opcode, opnd1_type, opnd2,
2352 : src_val, op_type);
2353 20735 : cstval = ipacp_value_safe_for_type (res_type, cstval);
2354 20735 : if (cstval)
2355 20534 : ret |= dest_lat->add_value (cstval, cs, src_val, src_idx,
2356 : src_offset);
2357 : else
2358 201 : ret |= dest_lat->set_contains_variable ();
2359 : }
2360 :
2361 : return ret;
2362 : }
2363 :
2364 : /* Propagate values through a pass-through jump function JFUNC associated with
2365 : edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
2366 : is the index of the source parameter. PARM_TYPE is the type of the
2367 : parameter to which the result is passed. */
2368 :
2369 : static bool
2370 73210 : propagate_vals_across_pass_through (cgraph_edge *cs, ipa_jump_func *jfunc,
2371 : ipcp_lattice<tree> *src_lat,
2372 : ipcp_lattice<tree> *dest_lat, int src_idx,
2373 : tree parm_type)
2374 : {
2375 73210 : gcc_checking_assert (parm_type);
2376 73210 : enum tree_code opcode = ipa_get_jf_pass_through_operation (jfunc);
2377 73210 : tree op_type = (opcode == NOP_EXPR) ? NULL_TREE
2378 2376 : : ipa_get_jf_pass_through_op_type (jfunc);
2379 73210 : return propagate_vals_across_arith_jfunc (cs, opcode, NULL_TREE,
2380 : ipa_get_jf_pass_through_operand (jfunc),
2381 : src_lat, dest_lat, -1, src_idx, op_type,
2382 73210 : parm_type);
2383 : }
2384 :
2385 : /* Propagate values through an ancestor jump function JFUNC associated with
2386 : edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
2387 : is the index of the source parameter. */
2388 :
2389 : static bool
2390 2287 : propagate_vals_across_ancestor (struct cgraph_edge *cs,
2391 : struct ipa_jump_func *jfunc,
2392 : ipcp_lattice<tree> *src_lat,
2393 : ipcp_lattice<tree> *dest_lat, int src_idx,
2394 : tree param_type)
2395 : {
2396 2287 : ipcp_value<tree> *src_val;
2397 2287 : bool ret = false;
2398 :
2399 2287 : if (ipa_edge_within_scc (cs))
2400 14 : return dest_lat->set_contains_variable ();
2401 :
2402 2591 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2403 : {
2404 318 : tree t = ipa_get_jf_ancestor_result (jfunc, src_val->value);
2405 318 : t = ipacp_value_safe_for_type (param_type, t);
2406 318 : if (t)
2407 260 : ret |= dest_lat->add_value (t, cs, src_val, src_idx);
2408 : else
2409 58 : ret |= dest_lat->set_contains_variable ();
2410 : }
2411 :
2412 : return ret;
2413 : }
2414 :
2415 : /* Propagate scalar values across jump function JFUNC that is associated with
2416 : edge CS and put the values into DEST_LAT. PARM_TYPE is the type of the
2417 : parameter to which the result is passed. */
2418 :
2419 : static bool
2420 4113058 : propagate_scalar_across_jump_function (struct cgraph_edge *cs,
2421 : struct ipa_jump_func *jfunc,
2422 : ipcp_lattice<tree> *dest_lat,
2423 : tree param_type)
2424 : {
2425 4113058 : if (dest_lat->bottom)
2426 : return false;
2427 :
2428 833366 : if (jfunc->type == IPA_JF_CONST)
2429 : {
2430 371676 : tree val = ipa_get_jf_constant (jfunc);
2431 371676 : val = ipacp_value_safe_for_type (param_type, val);
2432 371676 : if (val)
2433 371658 : return dest_lat->add_value (val, cs, NULL, 0);
2434 : else
2435 18 : return dest_lat->set_contains_variable ();
2436 : }
2437 461690 : else if (jfunc->type == IPA_JF_PASS_THROUGH
2438 275659 : || jfunc->type == IPA_JF_ANCESTOR)
2439 : {
2440 190637 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2441 190637 : ipcp_lattice<tree> *src_lat;
2442 190637 : int src_idx;
2443 190637 : bool ret;
2444 :
2445 190637 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2446 186031 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2447 : else
2448 4606 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2449 :
2450 190637 : src_lat = ipa_get_scalar_lat (caller_info, src_idx);
2451 190637 : if (src_lat->bottom)
2452 114993 : return dest_lat->set_contains_variable ();
2453 :
2454 : /* If we would need to clone the caller and cannot, do not propagate. */
2455 75644 : if (!ipcp_versionable_function_p (cs->caller)
2456 75644 : && (src_lat->contains_variable
2457 134 : || (src_lat->values_count > 1)))
2458 147 : return dest_lat->set_contains_variable ();
2459 :
2460 75497 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2461 73210 : ret = propagate_vals_across_pass_through (cs, jfunc, src_lat,
2462 : dest_lat, src_idx,
2463 : param_type);
2464 : else
2465 2287 : ret = propagate_vals_across_ancestor (cs, jfunc, src_lat, dest_lat,
2466 : src_idx, param_type);
2467 :
2468 75497 : if (src_lat->contains_variable)
2469 65947 : ret |= dest_lat->set_contains_variable ();
2470 :
2471 : return ret;
2472 : }
2473 :
2474 : /* TODO: We currently do not handle member method pointers in IPA-CP (we only
2475 : use it for indirect inlining), we should propagate them too. */
2476 271053 : return dest_lat->set_contains_variable ();
2477 : }
2478 :
2479 : /* Propagate scalar values across jump function JFUNC that is associated with
2480 : edge CS and describes argument IDX and put the values into DEST_LAT. */
2481 :
2482 : static bool
2483 4113058 : propagate_context_across_jump_function (cgraph_edge *cs,
2484 : ipa_jump_func *jfunc, int idx,
2485 : ipcp_lattice<ipa_polymorphic_call_context> *dest_lat)
2486 : {
2487 4113058 : if (dest_lat->bottom)
2488 : return false;
2489 929944 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
2490 929944 : bool ret = false;
2491 929944 : bool added_sth = false;
2492 929944 : bool type_preserved = true;
2493 :
2494 929944 : ipa_polymorphic_call_context edge_ctx, *edge_ctx_ptr
2495 946012 : = ipa_get_ith_polymorhic_call_context (args, idx);
2496 :
2497 16068 : if (edge_ctx_ptr)
2498 16068 : edge_ctx = *edge_ctx_ptr;
2499 :
2500 929944 : if (jfunc->type == IPA_JF_PASS_THROUGH
2501 743414 : || jfunc->type == IPA_JF_ANCESTOR)
2502 : {
2503 191232 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2504 191232 : int src_idx;
2505 191232 : ipcp_lattice<ipa_polymorphic_call_context> *src_lat;
2506 :
2507 : /* TODO: Once we figure out how to propagate speculations, it will
2508 : probably be a good idea to switch to speculation if type_preserved is
2509 : not set instead of punting. */
2510 191232 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2511 : {
2512 186530 : if (ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR)
2513 7576 : goto prop_fail;
2514 178954 : type_preserved = ipa_get_jf_pass_through_type_preserved (jfunc);
2515 178954 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2516 : }
2517 : else
2518 : {
2519 4702 : type_preserved = ipa_get_jf_ancestor_type_preserved (jfunc);
2520 4702 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2521 : }
2522 :
2523 183656 : src_lat = ipa_get_poly_ctx_lat (caller_info, src_idx);
2524 : /* If we would need to clone the caller and cannot, do not propagate. */
2525 183656 : if (!ipcp_versionable_function_p (cs->caller)
2526 183656 : && (src_lat->contains_variable
2527 14517 : || (src_lat->values_count > 1)))
2528 2495 : goto prop_fail;
2529 :
2530 181161 : ipcp_value<ipa_polymorphic_call_context> *src_val;
2531 182502 : for (src_val = src_lat->values; src_val; src_val = src_val->next)
2532 : {
2533 1341 : ipa_polymorphic_call_context cur = src_val->value;
2534 :
2535 1341 : if (!type_preserved)
2536 900 : cur.possible_dynamic_type_change (cs->in_polymorphic_cdtor);
2537 1341 : if (jfunc->type == IPA_JF_ANCESTOR)
2538 329 : cur.offset_by (ipa_get_jf_ancestor_offset (jfunc));
2539 : /* TODO: In cases we know how the context is going to be used,
2540 : we can improve the result by passing proper OTR_TYPE. */
2541 1341 : cur.combine_with (edge_ctx);
2542 2682 : if (!cur.useless_p ())
2543 : {
2544 870 : if (src_lat->contains_variable
2545 870 : && !edge_ctx.equal_to (cur))
2546 281 : ret |= dest_lat->set_contains_variable ();
2547 870 : ret |= dest_lat->add_value (cur, cs, src_val, src_idx);
2548 870 : added_sth = true;
2549 : }
2550 : }
2551 : }
2552 :
2553 738712 : prop_fail:
2554 191232 : if (!added_sth)
2555 : {
2556 929147 : if (!edge_ctx.useless_p ())
2557 9078 : ret |= dest_lat->add_value (edge_ctx, cs);
2558 : else
2559 920069 : ret |= dest_lat->set_contains_variable ();
2560 : }
2561 :
2562 : return ret;
2563 : }
2564 :
2565 : /* Propagate bits across jfunc that is associated with
2566 : edge cs and update dest_lattice accordingly. */
2567 :
2568 : bool
2569 4113058 : propagate_bits_across_jump_function (cgraph_edge *cs, int idx,
2570 : ipa_jump_func *jfunc,
2571 : ipcp_bits_lattice *dest_lattice)
2572 : {
2573 4113058 : if (dest_lattice->bottom_p ())
2574 : return false;
2575 :
2576 540991 : enum availability availability;
2577 540991 : cgraph_node *callee = cs->callee->function_symbol (&availability);
2578 540991 : ipa_node_params *callee_info = ipa_node_params_sum->get (callee);
2579 540991 : tree parm_type = ipa_get_type (callee_info, idx);
2580 :
2581 : /* For K&R C programs, ipa_get_type() could return NULL_TREE. Avoid the
2582 : transform for these cases. Similarly, we can have bad type mismatches
2583 : with LTO, avoid doing anything with those too. */
2584 540991 : if (!parm_type
2585 540991 : || (!INTEGRAL_TYPE_P (parm_type) && !POINTER_TYPE_P (parm_type)))
2586 : {
2587 29637 : if (dump_file && (dump_flags & TDF_DETAILS))
2588 11 : fprintf (dump_file, "Setting dest_lattice to bottom, because type of "
2589 : "param %i of %s is NULL or unsuitable for bits propagation\n",
2590 11 : idx, cs->callee->dump_name ());
2591 :
2592 29637 : return dest_lattice->set_to_bottom ();
2593 : }
2594 :
2595 511354 : if (jfunc->type == IPA_JF_PASS_THROUGH
2596 409387 : || jfunc->type == IPA_JF_ANCESTOR)
2597 : {
2598 104547 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2599 104547 : tree operand = NULL_TREE;
2600 104547 : tree op_type = NULL_TREE;
2601 104547 : enum tree_code code;
2602 104547 : unsigned src_idx;
2603 104547 : bool keep_null = false;
2604 :
2605 104547 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2606 : {
2607 101967 : code = ipa_get_jf_pass_through_operation (jfunc);
2608 101967 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2609 101967 : if (code != NOP_EXPR)
2610 : {
2611 2033 : operand = ipa_get_jf_pass_through_operand (jfunc);
2612 2033 : op_type = ipa_get_jf_pass_through_op_type (jfunc);
2613 : }
2614 : }
2615 : else
2616 : {
2617 2580 : code = POINTER_PLUS_EXPR;
2618 2580 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
2619 2580 : unsigned HOST_WIDE_INT offset
2620 2580 : = ipa_get_jf_ancestor_offset (jfunc) / BITS_PER_UNIT;
2621 2580 : keep_null = (ipa_get_jf_ancestor_keep_null (jfunc) || !offset);
2622 2580 : operand = build_int_cstu (size_type_node, offset);
2623 : }
2624 :
2625 104547 : class ipcp_param_lattices *src_lats
2626 104547 : = ipa_get_parm_lattices (caller_info, src_idx);
2627 :
2628 : /* Try to propagate bits if src_lattice is bottom, but jfunc is known.
2629 : for eg consider:
2630 : int f(int x)
2631 : {
2632 : g (x & 0xff);
2633 : }
2634 : Assume lattice for x is bottom, however we can still propagate
2635 : result of x & 0xff == 0xff, which gets computed during ccp1 pass
2636 : and we store it in jump function during analysis stage. */
2637 :
2638 104547 : if (!src_lats->bits_lattice.bottom_p ()
2639 104547 : && !src_lats->bits_lattice.recipient_only_p ())
2640 : {
2641 21744 : if (!op_type)
2642 20623 : op_type = ipa_get_type (caller_info, src_idx);
2643 :
2644 21744 : unsigned precision = TYPE_PRECISION (op_type);
2645 21744 : signop sgn = TYPE_SIGN (op_type);
2646 21744 : bool drop_all_ones
2647 21744 : = keep_null && !src_lats->bits_lattice.known_nonzero_p ();
2648 :
2649 21744 : return dest_lattice->meet_with (src_lats->bits_lattice, precision,
2650 21744 : sgn, code, operand, drop_all_ones);
2651 : }
2652 : }
2653 :
2654 489610 : value_range vr (parm_type);
2655 489610 : if (jfunc->m_vr)
2656 : {
2657 416599 : jfunc->m_vr->get_vrange (vr);
2658 416599 : if (!vr.undefined_p () && !vr.varying_p ())
2659 : {
2660 416599 : irange_bitmask bm = vr.get_bitmask ();
2661 416599 : widest_int mask
2662 416599 : = widest_int::from (bm.mask (), TYPE_SIGN (parm_type));
2663 416599 : widest_int value
2664 416599 : = widest_int::from (bm.value (), TYPE_SIGN (parm_type));
2665 416599 : return dest_lattice->meet_with (value, mask,
2666 416599 : TYPE_PRECISION (parm_type));
2667 416599 : }
2668 : }
2669 73011 : return dest_lattice->set_to_bottom ();
2670 489610 : }
2671 :
2672 : /* Propagate value range across jump function JFUNC that is associated with
2673 : edge CS with param of callee of PARAM_TYPE and update DEST_PLATS
2674 : accordingly. */
2675 :
2676 : static bool
2677 4112225 : propagate_vr_across_jump_function (cgraph_edge *cs, ipa_jump_func *jfunc,
2678 : class ipcp_param_lattices *dest_plats,
2679 : tree param_type)
2680 : {
2681 4112225 : ipcp_vr_lattice *dest_lat = &dest_plats->m_value_range;
2682 :
2683 4112225 : if (dest_lat->bottom_p ())
2684 : return false;
2685 :
2686 634690 : if (!param_type
2687 634690 : || !ipa_vr_supported_type_p (param_type))
2688 29577 : return dest_lat->set_to_bottom ();
2689 :
2690 605113 : value_range vr (param_type);
2691 605113 : vr.set_varying (param_type);
2692 605113 : if (jfunc->m_vr)
2693 521088 : ipa_vr_operation_and_type_effects (vr, *jfunc->m_vr, NOP_EXPR,
2694 : param_type,
2695 521088 : jfunc->m_vr->type ());
2696 :
2697 605113 : if (jfunc->type == IPA_JF_PASS_THROUGH)
2698 : {
2699 96118 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2700 96118 : int src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2701 96118 : class ipcp_param_lattices *src_lats
2702 96118 : = ipa_get_parm_lattices (caller_info, src_idx);
2703 96118 : tree operand_type = ipa_get_type (caller_info, src_idx);
2704 :
2705 96118 : if (src_lats->m_value_range.bottom_p ()
2706 96118 : || src_lats->m_value_range.recipient_only_p ())
2707 80158 : return dest_lat->set_to_bottom ();
2708 :
2709 15960 : if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR
2710 15960 : || !ipa_edge_within_scc (cs))
2711 15465 : ipa_vr_intersect_with_arith_jfunc (vr, jfunc, cs->caller,
2712 15465 : src_lats->m_value_range.m_vr,
2713 : operand_type, param_type);
2714 : }
2715 :
2716 524955 : if (!vr.undefined_p () && !vr.varying_p ())
2717 496196 : return dest_lat->meet_with (vr);
2718 : else
2719 28759 : return dest_lat->set_to_bottom ();
2720 605113 : }
2721 :
2722 : /* If DEST_PLATS already has aggregate items, check that aggs_by_ref matches
2723 : NEW_AGGS_BY_REF and if not, mark all aggs as bottoms and return true (in all
2724 : other cases, return false). If there are no aggregate items, set
2725 : aggs_by_ref to NEW_AGGS_BY_REF. */
2726 :
2727 : static bool
2728 42750 : set_check_aggs_by_ref (class ipcp_param_lattices *dest_plats,
2729 : bool new_aggs_by_ref)
2730 : {
2731 0 : if (dest_plats->aggs)
2732 : {
2733 22619 : if (dest_plats->aggs_by_ref != new_aggs_by_ref)
2734 : {
2735 0 : set_agg_lats_to_bottom (dest_plats);
2736 0 : return true;
2737 : }
2738 : }
2739 : else
2740 0 : dest_plats->aggs_by_ref = new_aggs_by_ref;
2741 : return false;
2742 : }
2743 :
2744 : /* Walk aggregate lattices in DEST_PLATS from ***AGLAT on, until ***aglat is an
2745 : already existing lattice for the given OFFSET and SIZE, marking all skipped
2746 : lattices as containing variable and checking for overlaps. If there is no
2747 : already existing lattice for the OFFSET and VAL_SIZE, create one, initialize
2748 : it with offset, size and contains_variable to PRE_EXISTING, and return true,
2749 : unless there are too many already. If there are two many, return false. If
2750 : there are overlaps turn whole DEST_PLATS to bottom and return false. If any
2751 : skipped lattices were newly marked as containing variable, set *CHANGE to
2752 : true. MAX_AGG_ITEMS is the maximum number of lattices. */
2753 :
2754 : static bool
2755 117050 : merge_agg_lats_step (class ipcp_param_lattices *dest_plats,
2756 : HOST_WIDE_INT offset, HOST_WIDE_INT val_size,
2757 : struct ipcp_agg_lattice ***aglat,
2758 : bool pre_existing, bool *change, int max_agg_items)
2759 : {
2760 117050 : gcc_checking_assert (offset >= 0);
2761 :
2762 121651 : while (**aglat && (**aglat)->offset < offset)
2763 : {
2764 4601 : if ((**aglat)->offset + (**aglat)->size > offset)
2765 : {
2766 0 : set_agg_lats_to_bottom (dest_plats);
2767 0 : return false;
2768 : }
2769 4601 : *change |= (**aglat)->set_contains_variable ();
2770 4601 : *aglat = &(**aglat)->next;
2771 : }
2772 :
2773 117050 : if (**aglat && (**aglat)->offset == offset)
2774 : {
2775 57954 : if ((**aglat)->size != val_size)
2776 : {
2777 13 : set_agg_lats_to_bottom (dest_plats);
2778 13 : return false;
2779 : }
2780 57941 : gcc_assert (!(**aglat)->next
2781 : || (**aglat)->next->offset >= offset + val_size);
2782 : return true;
2783 : }
2784 : else
2785 : {
2786 59096 : struct ipcp_agg_lattice *new_al;
2787 :
2788 59096 : if (**aglat && (**aglat)->offset < offset + val_size)
2789 : {
2790 3 : set_agg_lats_to_bottom (dest_plats);
2791 3 : return false;
2792 : }
2793 59093 : if (dest_plats->aggs_count == max_agg_items)
2794 : return false;
2795 59054 : dest_plats->aggs_count++;
2796 59054 : new_al = ipcp_agg_lattice_pool.allocate ();
2797 :
2798 59054 : new_al->offset = offset;
2799 59054 : new_al->size = val_size;
2800 59054 : new_al->contains_variable = pre_existing;
2801 :
2802 59054 : new_al->next = **aglat;
2803 59054 : **aglat = new_al;
2804 59054 : return true;
2805 : }
2806 : }
2807 :
2808 : /* Set all AGLAT and all other aggregate lattices reachable by next pointers as
2809 : containing an unknown value. */
2810 :
2811 : static bool
2812 42732 : set_chain_of_aglats_contains_variable (struct ipcp_agg_lattice *aglat)
2813 : {
2814 42732 : bool ret = false;
2815 45312 : while (aglat)
2816 : {
2817 2580 : ret |= aglat->set_contains_variable ();
2818 2580 : aglat = aglat->next;
2819 : }
2820 42732 : return ret;
2821 : }
2822 :
2823 : /* Merge existing aggregate lattices in SRC_PLATS to DEST_PLATS, subtracting
2824 : DELTA_OFFSET. CS is the call graph edge and SRC_IDX the index of the source
2825 : parameter used for lattice value sources. Return true if DEST_PLATS changed
2826 : in any way. */
2827 :
2828 : static bool
2829 3937 : merge_aggregate_lattices (struct cgraph_edge *cs,
2830 : class ipcp_param_lattices *dest_plats,
2831 : class ipcp_param_lattices *src_plats,
2832 : int src_idx, HOST_WIDE_INT offset_delta)
2833 : {
2834 3937 : bool pre_existing = dest_plats->aggs != NULL;
2835 3937 : struct ipcp_agg_lattice **dst_aglat;
2836 3937 : bool ret = false;
2837 :
2838 3937 : if (set_check_aggs_by_ref (dest_plats, src_plats->aggs_by_ref))
2839 0 : return true;
2840 3937 : if (src_plats->aggs_bottom)
2841 2 : return set_agg_lats_contain_variable (dest_plats);
2842 3935 : if (src_plats->aggs_contain_variable)
2843 2316 : ret |= set_agg_lats_contain_variable (dest_plats);
2844 3935 : dst_aglat = &dest_plats->aggs;
2845 :
2846 3935 : int max_agg_items = opt_for_fn (cs->callee->function_symbol ()->decl,
2847 : param_ipa_max_agg_items);
2848 3935 : for (struct ipcp_agg_lattice *src_aglat = src_plats->aggs;
2849 11609 : src_aglat;
2850 7674 : src_aglat = src_aglat->next)
2851 : {
2852 7674 : HOST_WIDE_INT new_offset = src_aglat->offset - offset_delta;
2853 :
2854 7674 : if (new_offset < 0)
2855 51 : continue;
2856 7623 : if (merge_agg_lats_step (dest_plats, new_offset, src_aglat->size,
2857 : &dst_aglat, pre_existing, &ret, max_agg_items))
2858 : {
2859 7619 : struct ipcp_agg_lattice *new_al = *dst_aglat;
2860 :
2861 7619 : dst_aglat = &(*dst_aglat)->next;
2862 7619 : if (src_aglat->bottom)
2863 : {
2864 0 : ret |= new_al->set_contains_variable ();
2865 0 : continue;
2866 : }
2867 7619 : if (src_aglat->contains_variable)
2868 4486 : ret |= new_al->set_contains_variable ();
2869 7619 : for (ipcp_value<tree> *val = src_aglat->values;
2870 11820 : val;
2871 4201 : val = val->next)
2872 4201 : ret |= new_al->add_value (val->value, cs, val, src_idx,
2873 : src_aglat->offset);
2874 : }
2875 4 : else if (dest_plats->aggs_bottom)
2876 : return true;
2877 : }
2878 3935 : ret |= set_chain_of_aglats_contains_variable (*dst_aglat);
2879 3935 : return ret;
2880 : }
2881 :
2882 : /* Determine whether there is anything to propagate FROM SRC_PLATS through a
2883 : pass-through JFUNC and if so, whether it has conform and conforms to the
2884 : rules about propagating values passed by reference. */
2885 :
2886 : static bool
2887 178793 : agg_pass_through_permissible_p (class ipcp_param_lattices *src_plats,
2888 : struct ipa_jump_func *jfunc)
2889 : {
2890 178793 : return src_plats->aggs
2891 178793 : && (!src_plats->aggs_by_ref
2892 5107 : || ipa_get_jf_pass_through_agg_preserved (jfunc));
2893 : }
2894 :
2895 : /* Propagate values through ITEM, jump function for a part of an aggregate,
2896 : into corresponding aggregate lattice AGLAT. CS is the call graph edge
2897 : associated with the jump function. Return true if AGLAT changed in any
2898 : way. */
2899 :
2900 : static bool
2901 109376 : propagate_aggregate_lattice (struct cgraph_edge *cs,
2902 : struct ipa_agg_jf_item *item,
2903 : struct ipcp_agg_lattice *aglat)
2904 : {
2905 109376 : class ipa_node_params *caller_info;
2906 109376 : class ipcp_param_lattices *src_plats;
2907 109376 : struct ipcp_lattice<tree> *src_lat;
2908 109376 : HOST_WIDE_INT src_offset;
2909 109376 : int src_idx;
2910 109376 : tree load_type;
2911 109376 : bool ret;
2912 :
2913 109376 : if (item->jftype == IPA_JF_CONST)
2914 : {
2915 97508 : tree value = item->value.constant;
2916 :
2917 97508 : gcc_checking_assert (is_gimple_ip_invariant (value));
2918 97508 : return aglat->add_value (value, cs, NULL, 0);
2919 : }
2920 :
2921 11868 : gcc_checking_assert (item->jftype == IPA_JF_PASS_THROUGH
2922 : || item->jftype == IPA_JF_LOAD_AGG);
2923 :
2924 11868 : caller_info = ipa_node_params_sum->get (cs->caller);
2925 11868 : src_idx = item->value.pass_through.formal_id;
2926 11868 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
2927 :
2928 11868 : if (item->jftype == IPA_JF_PASS_THROUGH)
2929 : {
2930 3581 : load_type = NULL_TREE;
2931 3581 : src_lat = &src_plats->itself;
2932 3581 : src_offset = -1;
2933 : }
2934 : else
2935 : {
2936 8287 : HOST_WIDE_INT load_offset = item->value.load_agg.offset;
2937 8287 : struct ipcp_agg_lattice *src_aglat;
2938 :
2939 13003 : for (src_aglat = src_plats->aggs; src_aglat; src_aglat = src_aglat->next)
2940 8558 : if (src_aglat->offset >= load_offset)
2941 : break;
2942 :
2943 8287 : load_type = item->value.load_agg.type;
2944 8287 : if (!src_aglat
2945 3842 : || src_aglat->offset > load_offset
2946 3490 : || src_aglat->size != tree_to_shwi (TYPE_SIZE (load_type))
2947 11777 : || src_plats->aggs_by_ref != item->value.load_agg.by_ref)
2948 4797 : return aglat->set_contains_variable ();
2949 :
2950 : src_lat = src_aglat;
2951 : src_offset = load_offset;
2952 : }
2953 :
2954 7071 : if (src_lat->bottom
2955 7071 : || (!ipcp_versionable_function_p (cs->caller)
2956 7071 : && !src_lat->is_single_const ()))
2957 2286 : return aglat->set_contains_variable ();
2958 :
2959 4785 : ret = propagate_vals_across_arith_jfunc (cs,
2960 : item->value.pass_through.operation,
2961 : load_type,
2962 : item->value.pass_through.operand,
2963 : src_lat, aglat,
2964 : src_offset,
2965 : src_idx,
2966 : item->value.pass_through.op_type,
2967 : item->type);
2968 :
2969 4785 : if (src_lat->contains_variable)
2970 2773 : ret |= aglat->set_contains_variable ();
2971 :
2972 : return ret;
2973 : }
2974 :
2975 : /* Propagate scalar values across jump function JFUNC that is associated with
2976 : edge CS and put the values into DEST_LAT. */
2977 :
2978 : static bool
2979 4113058 : propagate_aggs_across_jump_function (struct cgraph_edge *cs,
2980 : struct ipa_jump_func *jfunc,
2981 : class ipcp_param_lattices *dest_plats)
2982 : {
2983 4113058 : bool ret = false;
2984 :
2985 4113058 : if (dest_plats->aggs_bottom)
2986 : return false;
2987 :
2988 928712 : if (jfunc->type == IPA_JF_PASS_THROUGH
2989 928712 : && ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
2990 : {
2991 178793 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
2992 178793 : int src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
2993 178793 : class ipcp_param_lattices *src_plats;
2994 :
2995 178793 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
2996 178793 : if (agg_pass_through_permissible_p (src_plats, jfunc))
2997 : {
2998 : /* Currently we do not produce clobber aggregate jump
2999 : functions, replace with merging when we do. */
3000 3807 : gcc_assert (!jfunc->agg.items);
3001 3807 : ret |= merge_aggregate_lattices (cs, dest_plats, src_plats,
3002 : src_idx, 0);
3003 3807 : return ret;
3004 : }
3005 : }
3006 749919 : else if (jfunc->type == IPA_JF_ANCESTOR
3007 749919 : && ipa_get_jf_ancestor_agg_preserved (jfunc))
3008 : {
3009 1264 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
3010 1264 : int src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
3011 1264 : class ipcp_param_lattices *src_plats;
3012 :
3013 1264 : src_plats = ipa_get_parm_lattices (caller_info, src_idx);
3014 1264 : if (src_plats->aggs && src_plats->aggs_by_ref)
3015 : {
3016 : /* Currently we do not produce clobber aggregate jump
3017 : functions, replace with merging when we do. */
3018 130 : gcc_assert (!jfunc->agg.items);
3019 130 : ret |= merge_aggregate_lattices (cs, dest_plats, src_plats, src_idx,
3020 : ipa_get_jf_ancestor_offset (jfunc));
3021 : }
3022 1134 : else if (!src_plats->aggs_by_ref)
3023 1130 : ret |= set_agg_lats_to_bottom (dest_plats);
3024 : else
3025 4 : ret |= set_agg_lats_contain_variable (dest_plats);
3026 1264 : return ret;
3027 : }
3028 :
3029 923641 : if (jfunc->agg.items)
3030 : {
3031 38813 : bool pre_existing = dest_plats->aggs != NULL;
3032 38813 : struct ipcp_agg_lattice **aglat = &dest_plats->aggs;
3033 38813 : struct ipa_agg_jf_item *item;
3034 38813 : int i;
3035 :
3036 38813 : if (set_check_aggs_by_ref (dest_plats, jfunc->agg.by_ref))
3037 16 : return true;
3038 :
3039 38813 : int max_agg_items = opt_for_fn (cs->callee->function_symbol ()->decl,
3040 : param_ipa_max_agg_items);
3041 148545 : FOR_EACH_VEC_ELT (*jfunc->agg.items, i, item)
3042 : {
3043 109748 : HOST_WIDE_INT val_size;
3044 :
3045 109748 : if (item->offset < 0 || item->jftype == IPA_JF_UNKNOWN)
3046 321 : continue;
3047 109427 : val_size = tree_to_shwi (TYPE_SIZE (item->type));
3048 :
3049 109427 : if (merge_agg_lats_step (dest_plats, item->offset, val_size,
3050 : &aglat, pre_existing, &ret, max_agg_items))
3051 : {
3052 109376 : ret |= propagate_aggregate_lattice (cs, item, *aglat);
3053 109376 : aglat = &(*aglat)->next;
3054 : }
3055 51 : else if (dest_plats->aggs_bottom)
3056 : return true;
3057 : }
3058 :
3059 77594 : ret |= set_chain_of_aglats_contains_variable (*aglat);
3060 : }
3061 : else
3062 884828 : ret |= set_agg_lats_contain_variable (dest_plats);
3063 :
3064 923625 : return ret;
3065 : }
3066 :
3067 : /* Return true if on the way cfrom CS->caller to the final (non-alias and
3068 : non-thunk) destination, the call passes through a thunk. */
3069 :
3070 : static bool
3071 2051590 : call_passes_through_thunk (cgraph_edge *cs)
3072 : {
3073 2051590 : cgraph_node *alias_or_thunk = cs->callee;
3074 2201024 : while (alias_or_thunk->alias)
3075 149434 : alias_or_thunk = alias_or_thunk->get_alias_target ();
3076 2051590 : return alias_or_thunk->thunk;
3077 : }
3078 :
3079 : /* Propagate constants from the caller to the callee of CS. INFO describes the
3080 : caller. */
3081 :
3082 : static bool
3083 5489143 : propagate_constants_across_call (struct cgraph_edge *cs)
3084 : {
3085 5489143 : class ipa_node_params *callee_info;
3086 5489143 : enum availability availability;
3087 5489143 : cgraph_node *callee;
3088 5489143 : class ipa_edge_args *args;
3089 5489143 : bool ret = false;
3090 5489143 : int i, args_count, parms_count;
3091 :
3092 5489143 : callee = cs->callee->function_symbol (&availability);
3093 5489143 : if (!callee->definition)
3094 : return false;
3095 2074639 : gcc_checking_assert (callee->has_gimple_body_p ());
3096 2074639 : callee_info = ipa_node_params_sum->get (callee);
3097 2074639 : if (!callee_info)
3098 : return false;
3099 :
3100 2066123 : args = ipa_edge_args_sum->get (cs);
3101 2066123 : parms_count = ipa_get_param_count (callee_info);
3102 1869075 : if (parms_count == 0)
3103 : return false;
3104 1869075 : if (!args
3105 1868788 : || !opt_for_fn (cs->caller->decl, flag_ipa_cp)
3106 3737863 : || !opt_for_fn (cs->caller->decl, optimize))
3107 : {
3108 864 : for (i = 0; i < parms_count; i++)
3109 577 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info,
3110 : i));
3111 : return ret;
3112 : }
3113 1868788 : args_count = ipa_get_cs_argument_count (args);
3114 :
3115 : /* If this call goes through a thunk we must not propagate to the first (0th)
3116 : parameter. However, we might need to uncover a thunk from below a series
3117 : of aliases first. */
3118 1868788 : if (call_passes_through_thunk (cs))
3119 : {
3120 227 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info,
3121 : 0));
3122 227 : i = 1;
3123 : }
3124 : else
3125 1868788 : i = 0;
3126 :
3127 6128823 : for (; (i < args_count) && (i < parms_count); i++)
3128 : {
3129 4260035 : struct ipa_jump_func *jump_func = ipa_get_ith_jump_func (args, i);
3130 4260035 : class ipcp_param_lattices *dest_plats;
3131 4260035 : tree param_type = ipa_get_type (callee_info, i);
3132 :
3133 4260035 : dest_plats = ipa_get_parm_lattices (callee_info, i);
3134 4260035 : if (availability == AVAIL_INTERPOSABLE)
3135 146977 : ret |= set_all_contains_variable (dest_plats);
3136 : else
3137 : {
3138 4113058 : ret |= propagate_scalar_across_jump_function (cs, jump_func,
3139 : &dest_plats->itself,
3140 : param_type);
3141 4113058 : ret |= propagate_context_across_jump_function (cs, jump_func, i,
3142 : &dest_plats->ctxlat);
3143 4113058 : ret
3144 4113058 : |= propagate_bits_across_jump_function (cs, i, jump_func,
3145 : &dest_plats->bits_lattice);
3146 4113058 : ret |= propagate_aggs_across_jump_function (cs, jump_func,
3147 : dest_plats);
3148 4113058 : if (opt_for_fn (callee->decl, flag_ipa_vrp))
3149 4112225 : ret |= propagate_vr_across_jump_function (cs, jump_func,
3150 : dest_plats, param_type);
3151 : else
3152 833 : ret |= dest_plats->m_value_range.set_to_bottom ();
3153 : }
3154 : }
3155 1868971 : for (; i < parms_count; i++)
3156 183 : ret |= set_all_contains_variable (ipa_get_parm_lattices (callee_info, i));
3157 :
3158 : return ret;
3159 : }
3160 :
3161 : /* If an indirect edge IE can be turned into a direct one based on KNOWN_VALS
3162 : KNOWN_CONTEXTS, and known aggregates either in AVS or KNOWN_AGGS return
3163 : the destination. The latter three can be NULL. If AGG_REPS is not NULL,
3164 : KNOWN_AGGS is ignored. */
3165 :
3166 : static tree
3167 1584452 : ipa_get_indirect_edge_target_1 (struct cgraph_edge *ie,
3168 : const vec<tree> &known_csts,
3169 : const vec<ipa_polymorphic_call_context> &known_contexts,
3170 : const ipa_argagg_value_list &avs,
3171 : bool *speculative)
3172 : {
3173 1584452 : int param_index = ie->indirect_info->param_index;
3174 1584452 : *speculative = false;
3175 :
3176 1584452 : if (param_index == -1)
3177 : return NULL_TREE;
3178 :
3179 620834 : if (cgraph_simple_indirect_info *sii
3180 620834 : = dyn_cast <cgraph_simple_indirect_info *> (ie->indirect_info))
3181 : {
3182 302271 : tree t = NULL;
3183 :
3184 302271 : if (sii->agg_contents)
3185 : {
3186 68880 : t = NULL;
3187 68880 : if ((unsigned) param_index < known_csts.length ()
3188 68880 : && known_csts[param_index])
3189 62791 : t = ipa_find_agg_cst_from_init (known_csts[param_index],
3190 : sii->offset,
3191 : sii->by_ref);
3192 :
3193 68880 : if (!t && sii->guaranteed_unmodified)
3194 61876 : t = avs.get_value (param_index, sii->offset / BITS_PER_UNIT,
3195 : sii->by_ref);
3196 : }
3197 233391 : else if ((unsigned) param_index < known_csts.length ())
3198 233391 : t = known_csts[param_index];
3199 :
3200 302216 : if (t
3201 205961 : && TREE_CODE (t) == ADDR_EXPR
3202 507962 : && TREE_CODE (TREE_OPERAND (t, 0)) == FUNCTION_DECL)
3203 205746 : return TREE_OPERAND (t, 0);
3204 : else
3205 : return NULL_TREE;
3206 : }
3207 :
3208 318563 : if (!opt_for_fn (ie->caller->decl, flag_devirtualize))
3209 : return NULL_TREE;
3210 :
3211 318563 : cgraph_polymorphic_indirect_info *pii
3212 318563 : = as_a <cgraph_polymorphic_indirect_info *> (ie->indirect_info);
3213 318563 : if (!pii->usable_p ())
3214 : return NULL_TREE;
3215 :
3216 318563 : HOST_WIDE_INT anc_offset = pii->offset;
3217 318563 : tree t = NULL;
3218 318563 : tree target = NULL;
3219 318563 : if ((unsigned) param_index < known_csts.length ()
3220 318563 : && known_csts[param_index])
3221 18306 : t = ipa_find_agg_cst_from_init (known_csts[param_index], anc_offset, true);
3222 :
3223 : /* Try to work out value of virtual table pointer value in replacements. */
3224 : /* or known aggregate values. */
3225 18306 : if (!t)
3226 318554 : t = avs.get_value (param_index, anc_offset / BITS_PER_UNIT, true);
3227 :
3228 : /* If we found the virtual table pointer, lookup the target. */
3229 318554 : if (t)
3230 : {
3231 8037 : tree vtable;
3232 8037 : unsigned HOST_WIDE_INT offset;
3233 8037 : if (vtable_pointer_value_to_vtable (t, &vtable, &offset))
3234 : {
3235 8037 : bool can_refer;
3236 8037 : target = gimple_get_virt_method_for_vtable (pii->otr_token, vtable,
3237 : offset, &can_refer);
3238 8037 : if (can_refer)
3239 : {
3240 7974 : if (!target
3241 7974 : || fndecl_built_in_p (target, BUILT_IN_UNREACHABLE)
3242 15828 : || !possible_polymorphic_call_target_p
3243 7854 : (ie, cgraph_node::get (target)))
3244 : {
3245 : /* Do not speculate builtin_unreachable, it is stupid! */
3246 237 : if (pii->vptr_changed)
3247 6497 : return NULL;
3248 237 : target = ipa_impossible_devirt_target (ie, target);
3249 : }
3250 7974 : *speculative = pii->vptr_changed;
3251 7974 : if (!*speculative)
3252 : return target;
3253 : }
3254 : }
3255 : }
3256 :
3257 : /* Do we know the constant value of pointer? */
3258 312066 : if (!t && (unsigned) param_index < known_csts.length ())
3259 46377 : t = known_csts[param_index];
3260 :
3261 312066 : ipa_polymorphic_call_context context;
3262 312066 : if (known_contexts.length () > (unsigned int) param_index)
3263 : {
3264 311692 : context = known_contexts[param_index];
3265 311692 : context.offset_by (anc_offset);
3266 311692 : if (pii->vptr_changed)
3267 57165 : context.possible_dynamic_type_change (ie->in_polymorphic_cdtor,
3268 : pii->otr_type);
3269 311692 : if (t)
3270 : {
3271 12479 : ipa_polymorphic_call_context ctx2
3272 12479 : = ipa_polymorphic_call_context (t, pii->otr_type, anc_offset);
3273 24958 : if (!ctx2.useless_p ())
3274 10944 : context.combine_with (ctx2, pii->otr_type);
3275 : }
3276 : }
3277 374 : else if (t)
3278 : {
3279 23 : context = ipa_polymorphic_call_context (t, pii->otr_type, anc_offset);
3280 23 : if (pii->vptr_changed)
3281 8 : context.possible_dynamic_type_change (ie->in_polymorphic_cdtor,
3282 : pii->otr_type);
3283 : }
3284 : else
3285 : return NULL_TREE;
3286 :
3287 311715 : vec <cgraph_node *>targets;
3288 311715 : bool final;
3289 :
3290 311715 : targets = possible_polymorphic_call_targets (pii->otr_type, pii->otr_token,
3291 : context, &final);
3292 323582 : if (!final || targets.length () > 1)
3293 : {
3294 300522 : struct cgraph_node *node;
3295 300522 : if (*speculative)
3296 : return target;
3297 300493 : if (!opt_for_fn (ie->caller->decl, flag_devirtualize_speculatively)
3298 300493 : || ie->speculative || !ie->maybe_hot_p ())
3299 : return NULL;
3300 91165 : node = try_speculative_devirtualization (pii->otr_type, pii->otr_token,
3301 : context);
3302 91165 : if (node)
3303 : {
3304 697 : *speculative = true;
3305 697 : target = node->decl;
3306 : }
3307 : else
3308 : return NULL;
3309 : }
3310 : else
3311 : {
3312 11193 : *speculative = false;
3313 11193 : if (targets.length () == 1)
3314 11154 : target = targets[0]->decl;
3315 : else
3316 39 : target = ipa_impossible_devirt_target (ie, NULL_TREE);
3317 : }
3318 :
3319 11890 : if (target && !possible_polymorphic_call_target_p (ie,
3320 : cgraph_node::get (target)))
3321 : {
3322 48 : if (*speculative)
3323 : return NULL;
3324 40 : target = ipa_impossible_devirt_target (ie, target);
3325 : }
3326 :
3327 : return target;
3328 : }
3329 :
3330 : /* If an indirect edge IE can be turned into a direct one based on data in
3331 : AVALS, return the destination. Store into *SPECULATIVE a boolean determinig
3332 : whether the discovered target is only speculative guess. */
3333 :
3334 : tree
3335 1507107 : ipa_get_indirect_edge_target (struct cgraph_edge *ie,
3336 : ipa_call_arg_values *avals,
3337 : bool *speculative)
3338 : {
3339 1507107 : ipa_argagg_value_list avl (avals);
3340 1507107 : return ipa_get_indirect_edge_target_1 (ie, avals->m_known_vals,
3341 1507107 : avals->m_known_contexts,
3342 1507107 : avl, speculative);
3343 : }
3344 :
3345 : /* Calculate devirtualization time bonus for NODE, assuming we know information
3346 : about arguments stored in AVALS.
3347 :
3348 : FIXME: This function will also consider devirtualization of calls that are
3349 : known to be dead in the clone. */
3350 :
3351 : static sreal
3352 452851 : devirtualization_time_bonus (struct cgraph_node *node,
3353 : ipa_auto_call_arg_values *avals)
3354 : {
3355 452851 : struct cgraph_edge *ie;
3356 452851 : sreal res = 0;
3357 :
3358 528199 : for (ie = node->indirect_calls; ie; ie = ie->next_callee)
3359 : {
3360 75348 : struct cgraph_node *callee;
3361 75348 : class ipa_fn_summary *isummary;
3362 75348 : enum availability avail;
3363 75348 : tree target;
3364 75348 : bool speculative;
3365 :
3366 75348 : ipa_argagg_value_list avl (avals);
3367 75348 : target = ipa_get_indirect_edge_target_1 (ie, avals->m_known_vals,
3368 : avals->m_known_contexts,
3369 : avl, &speculative);
3370 75348 : if (!target)
3371 74359 : continue;
3372 :
3373 : /* Only bare minimum benefit for clearly un-inlineable targets. */
3374 3264 : res = res + ie->combined_sreal_frequency ();
3375 3264 : callee = cgraph_node::get (target);
3376 3264 : if (!callee || !callee->definition)
3377 624 : continue;
3378 2640 : callee = callee->function_symbol (&avail);
3379 2640 : if (avail < AVAIL_AVAILABLE)
3380 24 : continue;
3381 2616 : isummary = ipa_fn_summaries->get (callee);
3382 2616 : if (!isummary || !isummary->inlinable)
3383 66 : continue;
3384 :
3385 2550 : int savings = 0;
3386 2550 : int size = ipa_size_summaries->get (callee)->size;
3387 : /* FIXME: The values below need re-considering and perhaps also
3388 : integrating into the cost metrics, at lest in some very basic way. */
3389 2550 : int max_inline_insns_auto
3390 2550 : = opt_for_fn (callee->decl, param_max_inline_insns_auto);
3391 2550 : if (size <= max_inline_insns_auto / 4)
3392 412 : savings = 31 / ((int)speculative + 1);
3393 2138 : else if (size <= max_inline_insns_auto / 2)
3394 392 : savings = 15 / ((int)speculative + 1);
3395 3307 : else if (size <= max_inline_insns_auto
3396 1746 : || DECL_DECLARED_INLINE_P (callee->decl))
3397 185 : savings = 7 / ((int)speculative + 1);
3398 : else
3399 1561 : continue;
3400 989 : res = res + ie->combined_sreal_frequency () * (sreal) savings;
3401 : }
3402 :
3403 452851 : return res;
3404 : }
3405 :
3406 : /* Return time bonus incurred because of hints stored in ESTIMATES. */
3407 :
3408 : static sreal
3409 230311 : hint_time_bonus (cgraph_node *node, const ipa_call_estimates &estimates)
3410 : {
3411 230311 : sreal result = 0;
3412 230311 : ipa_hints hints = estimates.hints;
3413 230311 : if (hints & (INLINE_HINT_loop_iterations | INLINE_HINT_loop_stride))
3414 25719 : result += opt_for_fn (node->decl, param_ipa_cp_loop_hint_bonus);
3415 :
3416 230311 : sreal bonus_for_one = opt_for_fn (node->decl, param_ipa_cp_loop_hint_bonus);
3417 :
3418 230311 : if (hints & INLINE_HINT_loop_iterations)
3419 17402 : result += estimates.loops_with_known_iterations * bonus_for_one;
3420 :
3421 230311 : if (hints & INLINE_HINT_loop_stride)
3422 10803 : result += estimates.loops_with_known_strides * bonus_for_one;
3423 :
3424 230311 : return result;
3425 : }
3426 :
3427 : /* If there is a reason to penalize the function described by INFO in the
3428 : cloning goodness evaluation, do so. */
3429 :
3430 : static inline sreal
3431 104023 : incorporate_penalties (cgraph_node *node, ipa_node_params *info,
3432 : sreal evaluation)
3433 : {
3434 104023 : if (info->node_within_scc && !info->node_is_self_scc)
3435 1678 : evaluation = (evaluation
3436 1678 : * (100 - opt_for_fn (node->decl,
3437 3356 : param_ipa_cp_recursion_penalty))) / 100;
3438 :
3439 104023 : if (info->node_calling_single_call)
3440 7061 : evaluation = (evaluation
3441 7061 : * (100 - opt_for_fn (node->decl,
3442 7061 : param_ipa_cp_single_call_penalty)))
3443 14122 : / 100;
3444 :
3445 104023 : return evaluation;
3446 : }
3447 :
3448 : /* Return true if cloning NODE is a good idea, given the estimated TIME_BENEFIT
3449 : and SIZE_COST and with the sum of frequencies of incoming edges to the
3450 : potential new clone in FREQUENCIES. CUR_SWEEP is the number of the current
3451 : sweep of IPA-CP over the call-graph in the decision stage. */
3452 :
3453 : static bool
3454 304047 : good_cloning_opportunity_p (struct cgraph_node *node, sreal time_benefit,
3455 : sreal freq_sum, profile_count count_sum,
3456 : int size_cost, bool called_without_ipa_profile,
3457 : int cur_sweep)
3458 : {
3459 304047 : gcc_assert (count_sum.ipa () == count_sum);
3460 304047 : if (count_sum.quality () == AFDO)
3461 0 : count_sum = count_sum.force_nonzero ();
3462 504071 : if (time_benefit == 0
3463 249874 : || !opt_for_fn (node->decl, flag_ipa_cp_clone)
3464 : /* If there is no call which was executed in profiling or where
3465 : profile is missing, we do not want to clone. */
3466 104023 : || (!called_without_ipa_profile && !count_sum.nonzero_p ()))
3467 : {
3468 200024 : if (dump_file && (dump_flags & TDF_DETAILS))
3469 24 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3470 : "size: %i): Definitely not good or prohibited.\n",
3471 : time_benefit.to_double (), size_cost);
3472 : return false;
3473 : }
3474 :
3475 104023 : gcc_assert (size_cost > 0);
3476 :
3477 104023 : ipa_node_params *info = ipa_node_params_sum->get (node);
3478 104023 : int num_sweeps = opt_for_fn (node->decl, param_ipa_cp_sweeps);
3479 104023 : int eval_threshold = opt_for_fn (node->decl, param_ipa_cp_eval_threshold);
3480 104023 : eval_threshold = (eval_threshold * num_sweeps) / cur_sweep;
3481 : /* If we know the execution IPA execution counts, we can estimate overall
3482 : speedup of the program. */
3483 104023 : if (count_sum.nonzero_p ())
3484 : {
3485 225 : profile_count saved_time = count_sum * time_benefit;
3486 225 : sreal evaluation = (saved_time.to_sreal_scale (profile_count::one ())
3487 450 : / overall_scaled_runtime) / size_cost;
3488 225 : evaluation = incorporate_penalties (node, info, evaluation);
3489 :
3490 225 : if (dump_file && (dump_flags & TDF_DETAILS))
3491 : {
3492 0 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3493 : "size: %i, count_sum: ", time_benefit.to_double (),
3494 : size_cost);
3495 0 : count_sum.dump (dump_file);
3496 0 : fprintf (dump_file, ", overall time saved: ");
3497 0 : saved_time.dump (dump_file);
3498 0 : fprintf (dump_file, "%s%s) -> evaluation: %.2f, threshold: %i\n",
3499 0 : info->node_within_scc
3500 0 : ? (info->node_is_self_scc ? ", self_scc" : ", scc") : "",
3501 0 : info->node_calling_single_call ? ", single_call" : "",
3502 : evaluation.to_double (), eval_threshold);
3503 : }
3504 225 : gcc_checking_assert (saved_time == saved_time.ipa ());
3505 225 : if (!maybe_hot_count_p (NULL, saved_time))
3506 : {
3507 9 : if (dump_file && (dump_flags & TDF_DETAILS))
3508 0 : fprintf (dump_file, " not cloning: time saved is not hot\n");
3509 : }
3510 : /* Evaluation approximately corresponds to time saved per instruction
3511 : introduced. This is likely almost always going to be true, since we
3512 : already checked that time saved is large enough to be considered
3513 : hot. */
3514 216 : else if (evaluation >= (sreal)eval_threshold)
3515 225 : return true;
3516 : /* If all call sites have profile known; we know we do not want t clone.
3517 : If there are calls with unknown profile; try local heuristics. */
3518 162 : if (!called_without_ipa_profile)
3519 : return false;
3520 : }
3521 103798 : sreal evaluation = (time_benefit * freq_sum) / size_cost;
3522 103798 : evaluation = incorporate_penalties (node, info, evaluation);
3523 103798 : evaluation *= 1000;
3524 :
3525 103798 : if (dump_file && (dump_flags & TDF_DETAILS))
3526 358 : fprintf (dump_file, " good_cloning_opportunity_p (time: %g, "
3527 : "size: %i, freq_sum: %g%s%s) -> evaluation: %.2f, "
3528 : "threshold: %i\n",
3529 : time_benefit.to_double (), size_cost, freq_sum.to_double (),
3530 179 : info->node_within_scc
3531 26 : ? (info->node_is_self_scc ? ", self_scc" : ", scc") : "",
3532 179 : info->node_calling_single_call ? ", single_call" : "",
3533 : evaluation.to_double (), eval_threshold);
3534 :
3535 103798 : return evaluation >= eval_threshold;
3536 : }
3537 :
3538 : /* Grow vectors in AVALS and fill them with information about values of
3539 : parameters that are known to be independent of the context. INFO describes
3540 : the function. If REMOVABLE_PARAMS_COST is non-NULL, the movement cost of
3541 : all removable parameters will be stored in it.
3542 :
3543 : TODO: Also grow context independent value range vectors. */
3544 :
3545 : static bool
3546 1130505 : gather_context_independent_values (class ipa_node_params *info,
3547 : ipa_auto_call_arg_values *avals,
3548 : int *removable_params_cost)
3549 : {
3550 1130505 : int i, count = ipa_get_param_count (info);
3551 1130505 : bool ret = false;
3552 :
3553 1130505 : avals->m_known_vals.safe_grow_cleared (count, true);
3554 1130505 : avals->m_known_contexts.safe_grow_cleared (count, true);
3555 :
3556 1130505 : if (removable_params_cost)
3557 1130505 : *removable_params_cost = 0;
3558 :
3559 3735981 : for (i = 0; i < count; i++)
3560 : {
3561 2605476 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3562 2605476 : ipcp_lattice<tree> *lat = &plats->itself;
3563 :
3564 2605476 : if (lat->is_single_const ())
3565 : {
3566 35011 : ipcp_value<tree> *val = lat->values;
3567 35011 : gcc_checking_assert (TREE_CODE (val->value) != TREE_BINFO);
3568 35011 : avals->m_known_vals[i] = val->value;
3569 35011 : if (removable_params_cost)
3570 70022 : *removable_params_cost
3571 35011 : += estimate_move_cost (TREE_TYPE (val->value), false);
3572 : ret = true;
3573 : }
3574 2570465 : else if (removable_params_cost
3575 2570465 : && !ipa_is_param_used (info, i))
3576 496222 : *removable_params_cost
3577 248111 : += ipa_get_param_move_cost (info, i);
3578 :
3579 2605476 : if (!ipa_is_param_used (info, i))
3580 253378 : continue;
3581 :
3582 2352098 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3583 : /* Do not account known context as reason for cloning. We can see
3584 : if it permits devirtualization. */
3585 2352098 : if (ctxlat->is_single_const ())
3586 24955 : avals->m_known_contexts[i] = ctxlat->values->value;
3587 :
3588 2352098 : ret |= push_agg_values_from_plats (plats, i, 0, &avals->m_known_aggs);
3589 : }
3590 :
3591 1130505 : return ret;
3592 : }
3593 :
3594 : /* Perform time and size measurement of NODE with the context given in AVALS,
3595 : calculate the benefit compared to the node without specialization and store
3596 : it into VAL. Take into account REMOVABLE_PARAMS_COST of all
3597 : context-independent or unused removable parameters and EST_MOVE_COST, the
3598 : estimated movement of the considered parameter. */
3599 :
3600 : static void
3601 80479 : perform_estimation_of_a_value (cgraph_node *node,
3602 : ipa_auto_call_arg_values *avals,
3603 : int removable_params_cost, int est_move_cost,
3604 : ipcp_value_base *val)
3605 : {
3606 80479 : sreal time_benefit;
3607 80479 : ipa_call_estimates estimates;
3608 :
3609 80479 : estimate_ipcp_clone_size_and_time (node, avals, &estimates);
3610 :
3611 : /* Extern inline functions have no cloning local time benefits because they
3612 : will be inlined anyway. The only reason to clone them is if it enables
3613 : optimization in any of the functions they call. */
3614 80479 : if (DECL_EXTERNAL (node->decl) && DECL_DECLARED_INLINE_P (node->decl))
3615 114 : time_benefit = 0;
3616 : else
3617 80365 : time_benefit = (estimates.nonspecialized_time - estimates.time)
3618 160730 : + hint_time_bonus (node, estimates)
3619 160730 : + (devirtualization_time_bonus (node, avals)
3620 160730 : + removable_params_cost + est_move_cost);
3621 :
3622 80479 : int size = estimates.size;
3623 80479 : gcc_checking_assert (size >=0);
3624 : /* The inliner-heuristics based estimates may think that in certain
3625 : contexts some functions do not have any size at all but we want
3626 : all specializations to have at least a tiny cost, not least not to
3627 : divide by zero. */
3628 80479 : if (size == 0)
3629 0 : size = 1;
3630 :
3631 80479 : val->local_time_benefit = time_benefit;
3632 80479 : val->local_size_cost = size;
3633 80479 : }
3634 :
3635 : /* Get the overall limit of growth based on parameters extracted from NODE. It
3636 : does not really make sense to mix functions with different overall growth
3637 : limits or even number of sweeps but it is possible and if it happens, we do
3638 : not want to select one limit at random, so get the limits from NODE. */
3639 :
3640 : static long
3641 219037 : get_max_overall_size (cgraph_node *node)
3642 : {
3643 219037 : long max_new_size = orig_overall_size;
3644 219037 : long large_unit = opt_for_fn (node->decl, param_ipa_cp_large_unit_insns);
3645 219037 : if (max_new_size < large_unit)
3646 : max_new_size = large_unit;
3647 219037 : int unit_growth = opt_for_fn (node->decl, param_ipa_cp_unit_growth);
3648 219037 : max_new_size += max_new_size * unit_growth / 100 + 1;
3649 :
3650 219037 : return max_new_size;
3651 : }
3652 :
3653 : /* Return true if NODE should be cloned just for a parameter removal, possibly
3654 : dumping a reason if not. */
3655 :
3656 : static bool
3657 8648 : clone_for_param_removal_p (cgraph_node *node)
3658 : {
3659 8648 : if (!node->can_change_signature)
3660 : {
3661 1574 : if (dump_file && (dump_flags & TDF_DETAILS))
3662 0 : fprintf (dump_file, " Not considering cloning to remove parameters, "
3663 : "function cannot change signature.\n");
3664 : return false;
3665 : }
3666 7074 : if (node->can_be_local_p ())
3667 : {
3668 7074 : if (dump_file && (dump_flags & TDF_DETAILS))
3669 0 : fprintf (dump_file, " Not considering cloning to remove parameters, "
3670 : "IPA-SRA can do it potentially better.\n");
3671 : return false;
3672 : }
3673 : return true;
3674 : }
3675 :
3676 : /* Iterate over known values of parameters of NODE and estimate the local
3677 : effects in terms of time and size they have. */
3678 :
3679 : static void
3680 1313343 : estimate_local_effects (struct cgraph_node *node)
3681 : {
3682 1313343 : ipa_node_params *info = ipa_node_params_sum->get (node);
3683 1313343 : int count = ipa_get_param_count (info);
3684 1078781 : int removable_params_cost;
3685 :
3686 1078781 : if (!count || !ipcp_versionable_function_p (node))
3687 405378 : return;
3688 :
3689 907965 : if (dump_file && (dump_flags & TDF_DETAILS))
3690 117 : fprintf (dump_file, "\nEstimating effects for %s.\n", node->dump_name ());
3691 :
3692 907965 : ipa_auto_call_arg_values avals;
3693 907965 : gather_context_independent_values (info, &avals, &removable_params_cost);
3694 :
3695 3937108 : for (int i = 0; i < count; i++)
3696 : {
3697 2121178 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3698 2121178 : ipcp_lattice<tree> *lat = &plats->itself;
3699 2121178 : ipcp_value<tree> *val;
3700 :
3701 4220587 : if (lat->bottom
3702 218713 : || !lat->values
3703 2160534 : || avals.m_known_vals[i])
3704 2099409 : continue;
3705 :
3706 66513 : for (val = lat->values; val; val = val->next)
3707 : {
3708 44744 : gcc_checking_assert (TREE_CODE (val->value) != TREE_BINFO);
3709 44744 : avals.m_known_vals[i] = val->value;
3710 :
3711 44744 : int emc = estimate_move_cost (TREE_TYPE (val->value), true);
3712 44744 : perform_estimation_of_a_value (node, &avals, removable_params_cost,
3713 : emc, val);
3714 :
3715 44744 : if (dump_file && (dump_flags & TDF_DETAILS))
3716 : {
3717 44 : fprintf (dump_file, " - estimates for value ");
3718 44 : print_ipcp_constant_value (dump_file, val->value);
3719 44 : fprintf (dump_file, " for ");
3720 44 : ipa_dump_param (dump_file, info, i);
3721 44 : fprintf (dump_file, ": time_benefit: %g, size: %i\n",
3722 : val->local_time_benefit.to_double (),
3723 : val->local_size_cost);
3724 : }
3725 : }
3726 21769 : avals.m_known_vals[i] = NULL_TREE;
3727 : }
3728 :
3729 3029143 : for (int i = 0; i < count; i++)
3730 : {
3731 2121178 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3732 :
3733 2121178 : if (!plats->virt_call)
3734 2113055 : continue;
3735 :
3736 8123 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3737 8123 : ipcp_value<ipa_polymorphic_call_context> *val;
3738 :
3739 16084 : if (ctxlat->bottom
3740 2979 : || !ctxlat->values
3741 11096 : || !avals.m_known_contexts[i].useless_p ())
3742 7961 : continue;
3743 :
3744 396 : for (val = ctxlat->values; val; val = val->next)
3745 : {
3746 234 : avals.m_known_contexts[i] = val->value;
3747 234 : perform_estimation_of_a_value (node, &avals, removable_params_cost,
3748 : 0, val);
3749 :
3750 234 : if (dump_file && (dump_flags & TDF_DETAILS))
3751 : {
3752 0 : fprintf (dump_file, " - estimates for polymorphic context ");
3753 0 : print_ipcp_constant_value (dump_file, val->value);
3754 0 : fprintf (dump_file, " for ");
3755 0 : ipa_dump_param (dump_file, info, i);
3756 0 : fprintf (dump_file, ": time_benefit: %g, size: %i\n",
3757 : val->local_time_benefit.to_double (),
3758 : val->local_size_cost);
3759 : }
3760 : }
3761 162 : avals.m_known_contexts[i] = ipa_polymorphic_call_context ();
3762 : }
3763 :
3764 907965 : unsigned all_ctx_len = avals.m_known_aggs.length ();
3765 907965 : auto_vec<ipa_argagg_value, 32> all_ctx;
3766 907965 : all_ctx.reserve_exact (all_ctx_len);
3767 907965 : all_ctx.splice (avals.m_known_aggs);
3768 907965 : avals.m_known_aggs.safe_grow_cleared (all_ctx_len + 1);
3769 :
3770 907965 : unsigned j = 0;
3771 3937108 : for (int index = 0; index < count; index++)
3772 : {
3773 2121178 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, index);
3774 :
3775 2121178 : if (plats->aggs_bottom || !plats->aggs)
3776 2101676 : continue;
3777 :
3778 75548 : for (ipcp_agg_lattice *aglat = plats->aggs; aglat; aglat = aglat->next)
3779 : {
3780 56046 : ipcp_value<tree> *val;
3781 55688 : if (aglat->bottom || !aglat->values
3782 : /* If the following is true, the one value is already part of all
3783 : context estimations. */
3784 104166 : || (!plats->aggs_contain_variable
3785 25754 : && aglat->is_single_const ()))
3786 29327 : continue;
3787 :
3788 26719 : unsigned unit_offset = aglat->offset / BITS_PER_UNIT;
3789 26719 : while (j < all_ctx_len
3790 35309 : && (all_ctx[j].index < index
3791 3422 : || (all_ctx[j].index == index
3792 2434 : && all_ctx[j].unit_offset < unit_offset)))
3793 : {
3794 3332 : avals.m_known_aggs[j] = all_ctx[j];
3795 3332 : j++;
3796 : }
3797 :
3798 35996 : for (unsigned k = j; k < all_ctx_len; k++)
3799 9277 : avals.m_known_aggs[k+1] = all_ctx[k];
3800 :
3801 62220 : for (val = aglat->values; val; val = val->next)
3802 : {
3803 35501 : avals.m_known_aggs[j].value = val->value;
3804 35501 : avals.m_known_aggs[j].unit_offset = unit_offset;
3805 35501 : avals.m_known_aggs[j].index = index;
3806 35501 : avals.m_known_aggs[j].by_ref = plats->aggs_by_ref;
3807 35501 : avals.m_known_aggs[j].killed = false;
3808 :
3809 35501 : perform_estimation_of_a_value (node, &avals,
3810 : removable_params_cost, 0, val);
3811 :
3812 35501 : if (dump_file && (dump_flags & TDF_DETAILS))
3813 : {
3814 80 : fprintf (dump_file, " - estimates for value ");
3815 80 : print_ipcp_constant_value (dump_file, val->value);
3816 80 : fprintf (dump_file, " for ");
3817 80 : ipa_dump_param (dump_file, info, index);
3818 160 : fprintf (dump_file, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
3819 : "]: time_benefit: %g, size: %i\n",
3820 80 : plats->aggs_by_ref ? "ref " : "",
3821 : aglat->offset,
3822 : val->local_time_benefit.to_double (),
3823 : val->local_size_cost);
3824 : }
3825 : }
3826 : }
3827 : }
3828 907965 : }
3829 :
3830 :
3831 : /* Add value CUR_VAL and all yet-unsorted values it is dependent on to the
3832 : topological sort of values. */
3833 :
3834 : template <typename valtype>
3835 : void
3836 140442 : value_topo_info<valtype>::add_val (ipcp_value<valtype> *cur_val)
3837 : {
3838 : ipcp_value_source<valtype> *src;
3839 :
3840 140442 : if (cur_val->dfs)
3841 : return;
3842 :
3843 140280 : dfs_counter++;
3844 140280 : cur_val->dfs = dfs_counter;
3845 140280 : cur_val->low_link = dfs_counter;
3846 :
3847 140280 : cur_val->topo_next = stack;
3848 140280 : stack = cur_val;
3849 140280 : cur_val->on_stack = true;
3850 :
3851 603586 : for (src = cur_val->sources; src; src = src->next)
3852 463306 : if (src->val)
3853 : {
3854 21565 : if (src->val->dfs == 0)
3855 : {
3856 186 : add_val (src->val);
3857 186 : if (src->val->low_link < cur_val->low_link)
3858 19 : cur_val->low_link = src->val->low_link;
3859 : }
3860 21379 : else if (src->val->on_stack
3861 1584 : && src->val->dfs < cur_val->low_link)
3862 73 : cur_val->low_link = src->val->dfs;
3863 : }
3864 :
3865 140280 : if (cur_val->dfs == cur_val->low_link)
3866 : {
3867 : ipcp_value<valtype> *v, *scc_list = NULL;
3868 :
3869 : do
3870 : {
3871 140280 : v = stack;
3872 140280 : stack = v->topo_next;
3873 140280 : v->on_stack = false;
3874 140280 : v->scc_no = cur_val->dfs;
3875 :
3876 140280 : v->scc_next = scc_list;
3877 140280 : scc_list = v;
3878 : }
3879 140280 : while (v != cur_val);
3880 :
3881 140192 : cur_val->topo_next = values_topo;
3882 140192 : values_topo = cur_val;
3883 : }
3884 : }
3885 :
3886 : /* Add all values in lattices associated with NODE to the topological sort if
3887 : they are not there yet. */
3888 :
3889 : static void
3890 1313343 : add_all_node_vals_to_toposort (cgraph_node *node, ipa_topo_info *topo)
3891 : {
3892 1313343 : ipa_node_params *info = ipa_node_params_sum->get (node);
3893 1313343 : int i, count = ipa_get_param_count (info);
3894 :
3895 3735969 : for (i = 0; i < count; i++)
3896 : {
3897 2422626 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
3898 2422626 : ipcp_lattice<tree> *lat = &plats->itself;
3899 2422626 : struct ipcp_agg_lattice *aglat;
3900 :
3901 2422626 : if (!lat->bottom)
3902 : {
3903 229818 : ipcp_value<tree> *val;
3904 302956 : for (val = lat->values; val; val = val->next)
3905 73138 : topo->constants.add_val (val);
3906 : }
3907 :
3908 2422626 : if (!plats->aggs_bottom)
3909 288705 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
3910 59019 : if (!aglat->bottom)
3911 : {
3912 58661 : ipcp_value<tree> *val;
3913 117539 : for (val = aglat->values; val; val = val->next)
3914 58878 : topo->constants.add_val (val);
3915 : }
3916 :
3917 2422626 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
3918 2422626 : if (!ctxlat->bottom)
3919 : {
3920 230828 : ipcp_value<ipa_polymorphic_call_context> *ctxval;
3921 239068 : for (ctxval = ctxlat->values; ctxval; ctxval = ctxval->next)
3922 8240 : topo->contexts.add_val (ctxval);
3923 : }
3924 : }
3925 1313343 : }
3926 :
3927 : /* One pass of constants propagation along the call graph edges, from callers
3928 : to callees (requires topological ordering in TOPO), iterate over strongly
3929 : connected components. */
3930 :
3931 : static void
3932 131920 : propagate_constants_topo (class ipa_topo_info *topo)
3933 : {
3934 131920 : int i;
3935 :
3936 1530357 : for (i = topo->nnodes - 1; i >= 0; i--)
3937 : {
3938 1398437 : unsigned j;
3939 1398437 : struct cgraph_node *v, *node = topo->order[i];
3940 1398437 : vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (node);
3941 :
3942 : /* First, iteratively propagate within the strongly connected component
3943 : until all lattices stabilize. */
3944 4200379 : FOR_EACH_VEC_ELT (cycle_nodes, j, v)
3945 1403505 : if (v->has_gimple_body_p ())
3946 : {
3947 1322127 : if (opt_for_fn (v->decl, flag_ipa_cp)
3948 1322127 : && opt_for_fn (v->decl, optimize))
3949 1313343 : push_node_to_stack (topo, v);
3950 : /* When V is not optimized, we can not push it to stack, but
3951 : still we need to set all its callees lattices to bottom. */
3952 : else
3953 : {
3954 21849 : for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
3955 13065 : propagate_constants_across_call (cs);
3956 : }
3957 : }
3958 :
3959 1398437 : v = pop_node_from_stack (topo);
3960 4113429 : while (v)
3961 : {
3962 1316555 : struct cgraph_edge *cs;
3963 1316555 : class ipa_node_params *info = NULL;
3964 1316555 : bool self_scc = true;
3965 :
3966 6819114 : for (cs = v->callees; cs; cs = cs->next_callee)
3967 5502559 : if (ipa_edge_within_scc (cs))
3968 : {
3969 30627 : cgraph_node *callee = cs->callee->function_symbol ();
3970 :
3971 30627 : if (v != callee)
3972 18291 : self_scc = false;
3973 :
3974 30627 : if (!info)
3975 : {
3976 14362 : info = ipa_node_params_sum->get (v);
3977 14362 : info->node_within_scc = true;
3978 : }
3979 :
3980 30627 : if (propagate_constants_across_call (cs))
3981 4195 : push_node_to_stack (topo, callee);
3982 : }
3983 :
3984 1316555 : if (info)
3985 14362 : info->node_is_self_scc = self_scc;
3986 :
3987 1316555 : v = pop_node_from_stack (topo);
3988 : }
3989 :
3990 : /* Afterwards, propagate along edges leading out of the SCC, calculates
3991 : the local effects of the discovered constants and all valid values to
3992 : their topological sort. */
3993 2801942 : FOR_EACH_VEC_ELT (cycle_nodes, j, v)
3994 1403505 : if (v->has_gimple_body_p ()
3995 1322127 : && opt_for_fn (v->decl, flag_ipa_cp)
3996 2716848 : && opt_for_fn (v->decl, optimize))
3997 : {
3998 1313343 : struct cgraph_edge *cs;
3999 :
4000 1313343 : estimate_local_effects (v);
4001 1313343 : add_all_node_vals_to_toposort (v, topo);
4002 6782013 : for (cs = v->callees; cs; cs = cs->next_callee)
4003 5468670 : if (!ipa_edge_within_scc (cs))
4004 5445451 : propagate_constants_across_call (cs);
4005 : }
4006 1398437 : cycle_nodes.release ();
4007 : }
4008 131920 : }
4009 :
4010 : /* Propagate the estimated effects of individual values along the topological
4011 : from the dependent values to those they depend on. */
4012 :
4013 : template <typename valtype>
4014 : void
4015 263840 : value_topo_info<valtype>::propagate_effects ()
4016 : {
4017 : ipcp_value<valtype> *base;
4018 263840 : hash_set<ipcp_value<valtype> *> processed_srcvals;
4019 :
4020 404032 : for (base = values_topo; base; base = base->topo_next)
4021 : {
4022 : ipcp_value_source<valtype> *src;
4023 : ipcp_value<valtype> *val;
4024 140192 : sreal time = 0;
4025 140192 : HOST_WIDE_INT size = 0;
4026 :
4027 280472 : for (val = base; val; val = val->scc_next)
4028 : {
4029 140280 : time = time + val->local_time_benefit + val->prop_time_benefit;
4030 140280 : size = size + val->local_size_cost + val->prop_size_cost;
4031 : }
4032 :
4033 280472 : for (val = base; val; val = val->scc_next)
4034 : {
4035 140280 : processed_srcvals.empty ();
4036 603586 : for (src = val->sources; src; src = src->next)
4037 463306 : if (src->val
4038 463306 : && cs_interesting_for_ipcp_p (src->cs))
4039 : {
4040 21525 : if (!processed_srcvals.add (src->val))
4041 : {
4042 17256 : HOST_WIDE_INT prop_size = size + src->val->prop_size_cost;
4043 17256 : if (prop_size < INT_MAX)
4044 17256 : src->val->prop_size_cost = prop_size;
4045 : else
4046 0 : continue;
4047 : }
4048 :
4049 21525 : int special_factor = 1;
4050 21525 : if (val->same_scc (src->val))
4051 : special_factor
4052 1672 : = opt_for_fn(src->cs->caller->decl,
4053 : param_ipa_cp_recursive_freq_factor);
4054 19853 : else if (val->self_recursion_generated_p ()
4055 19853 : && (src->cs->callee->function_symbol ()
4056 822 : == src->cs->caller))
4057 : {
4058 822 : int max_recur_gen_depth
4059 822 : = opt_for_fn(src->cs->caller->decl,
4060 : param_ipa_cp_max_recursive_depth);
4061 822 : special_factor = max_recur_gen_depth
4062 822 : - val->self_recursion_generated_level + 1;
4063 : }
4064 :
4065 21525 : src->val->prop_time_benefit
4066 43050 : += time * special_factor * src->cs->sreal_frequency ();
4067 : }
4068 :
4069 140280 : if (size < INT_MAX)
4070 : {
4071 140280 : val->prop_time_benefit = time;
4072 140280 : val->prop_size_cost = size;
4073 : }
4074 : else
4075 : {
4076 0 : val->prop_time_benefit = 0;
4077 : val->prop_size_cost = 0;
4078 : }
4079 : }
4080 : }
4081 263840 : }
4082 :
4083 :
4084 : /* Propagate constants, polymorphic contexts and their effects from the
4085 : summaries interprocedurally. */
4086 :
4087 : static void
4088 131920 : ipcp_propagate_stage (class ipa_topo_info *topo)
4089 : {
4090 131920 : struct cgraph_node *node;
4091 131920 : sreal overall_runtime = 0;
4092 131920 : overall_scaled_runtime = 0;
4093 :
4094 131920 : if (dump_file)
4095 162 : fprintf (dump_file, "\n Propagating constants:\n\n");
4096 :
4097 1535429 : FOR_EACH_DEFINED_FUNCTION (node)
4098 : {
4099 1403509 : if (node->has_gimple_body_p ()
4100 1322127 : && opt_for_fn (node->decl, flag_ipa_cp)
4101 2716852 : && opt_for_fn (node->decl, optimize))
4102 : {
4103 1313343 : ipa_node_params *info = ipa_node_params_sum->get (node);
4104 1313343 : determine_versionability (node, info);
4105 :
4106 1313343 : unsigned nlattices = ipa_get_param_count (info);
4107 1313343 : info->lattices.safe_grow_cleared (nlattices, true);
4108 1313343 : initialize_node_lattices (node);
4109 :
4110 1313343 : int num_sweeps = opt_for_fn (node->decl, param_ipa_cp_sweeps);
4111 1313343 : if (max_number_sweeps < num_sweeps)
4112 124180 : max_number_sweeps = num_sweeps;
4113 : }
4114 :
4115 1403509 : if (!node->alias)
4116 : {
4117 1323338 : ipa_size_summary *szs = ipa_size_summaries->get (node);
4118 1323338 : if (szs)
4119 1323113 : overall_size += szs->self_size;
4120 1323338 : ipa_fn_summary *fns = ipa_fn_summaries->get (node);
4121 1323338 : if (fns)
4122 : {
4123 1323113 : int shift = opt_for_fn (node->decl,
4124 : param_ipa_cp_profile_runtime_scaling);
4125 1323113 : sreal frt = fns->time;
4126 1323113 : overall_runtime += frt;
4127 1323113 : overall_scaled_runtime += frt.shift(-shift);
4128 : }
4129 : }
4130 : }
4131 :
4132 131920 : orig_overall_size = overall_size;
4133 :
4134 131920 : if (dump_file)
4135 : {
4136 162 : fprintf (dump_file, "\noverall_size: %li\n", overall_size);
4137 162 : fprintf (dump_file, "overall_runtime: %.2e\n",
4138 : overall_runtime.to_double ());
4139 162 : fprintf (dump_file, "overall_scaled_runtime: %.2e\n",
4140 : overall_scaled_runtime.to_double ());
4141 : }
4142 :
4143 131920 : propagate_constants_topo (topo);
4144 131920 : if (flag_checking)
4145 131912 : ipcp_verify_propagated_values ();
4146 131920 : topo->constants.propagate_effects ();
4147 131920 : topo->contexts.propagate_effects ();
4148 :
4149 131920 : if (dump_file)
4150 : {
4151 162 : fprintf (dump_file, "\nIPA lattices after all propagation:\n");
4152 162 : print_all_lattices (dump_file, (dump_flags & TDF_DETAILS), true);
4153 : }
4154 131920 : }
4155 :
4156 : /* Discover newly direct outgoing edges from NODE which is a new clone with
4157 : known KNOWN_CSTS and make them direct. */
4158 :
4159 : static void
4160 19394 : ipcp_discover_new_direct_edges (struct cgraph_node *node,
4161 : vec<tree> known_csts,
4162 : vec<ipa_polymorphic_call_context>
4163 : known_contexts,
4164 : vec<ipa_argagg_value, va_gc> *aggvals)
4165 : {
4166 19394 : struct cgraph_edge *ie, *next_ie;
4167 19394 : bool found = false;
4168 :
4169 21391 : for (ie = node->indirect_calls; ie; ie = next_ie)
4170 : {
4171 1997 : tree target;
4172 1997 : bool speculative;
4173 :
4174 1997 : next_ie = ie->next_callee;
4175 1997 : ipa_argagg_value_list avs (aggvals);
4176 1997 : target = ipa_get_indirect_edge_target_1 (ie, known_csts, known_contexts,
4177 : avs, &speculative);
4178 1997 : if (target)
4179 : {
4180 593 : cgraph_polymorphic_indirect_info *pii
4181 593 : = dyn_cast <cgraph_polymorphic_indirect_info *> (ie->indirect_info);
4182 593 : cgraph_simple_indirect_info *sii
4183 1115 : = dyn_cast <cgraph_simple_indirect_info *> (ie->indirect_info);
4184 435 : bool agg_contents = sii && sii->agg_contents;
4185 593 : bool polymorphic = !!pii;
4186 593 : int param_index = ie->indirect_info->param_index;
4187 593 : struct cgraph_edge *cs = ipa_make_edge_direct_to_target (ie, target,
4188 : speculative);
4189 593 : found = true;
4190 :
4191 593 : if (cs && !agg_contents && !polymorphic)
4192 : {
4193 364 : ipa_node_params *info = ipa_node_params_sum->get (node);
4194 364 : int c = ipa_get_controlled_uses (info, param_index);
4195 364 : if (c != IPA_UNDESCRIBED_USE
4196 364 : && !ipa_get_param_load_dereferenced (info, param_index))
4197 : {
4198 360 : struct ipa_ref *to_del;
4199 :
4200 360 : c--;
4201 360 : ipa_set_controlled_uses (info, param_index, c);
4202 360 : if (dump_file && (dump_flags & TDF_DETAILS))
4203 3 : fprintf (dump_file, " controlled uses count of param "
4204 : "%i bumped down to %i\n", param_index, c);
4205 360 : if (c == 0
4206 360 : && (to_del = node->find_reference (cs->callee, NULL, 0,
4207 : IPA_REF_ADDR)))
4208 : {
4209 296 : if (dump_file && (dump_flags & TDF_DETAILS))
4210 3 : fprintf (dump_file, " and even removing its "
4211 : "cloning-created reference\n");
4212 296 : to_del->remove_reference ();
4213 : }
4214 : }
4215 : }
4216 : }
4217 : }
4218 : /* Turning calls to direct calls will improve overall summary. */
4219 19394 : if (found)
4220 490 : ipa_update_overall_fn_summary (node);
4221 19394 : }
4222 :
4223 : class edge_clone_summary;
4224 : static call_summary <edge_clone_summary *> *edge_clone_summaries = NULL;
4225 :
4226 : /* Edge clone summary. */
4227 :
4228 : class edge_clone_summary
4229 : {
4230 : public:
4231 : /* Default constructor. */
4232 378112 : edge_clone_summary (): prev_clone (NULL), next_clone (NULL) {}
4233 :
4234 : /* Default destructor. */
4235 378112 : ~edge_clone_summary ()
4236 : {
4237 378112 : if (prev_clone)
4238 34937 : edge_clone_summaries->get (prev_clone)->next_clone = next_clone;
4239 378112 : if (next_clone)
4240 157460 : edge_clone_summaries->get (next_clone)->prev_clone = prev_clone;
4241 378112 : }
4242 :
4243 : cgraph_edge *prev_clone;
4244 : cgraph_edge *next_clone;
4245 : };
4246 :
4247 : class edge_clone_summary_t:
4248 : public call_summary <edge_clone_summary *>
4249 : {
4250 : public:
4251 131920 : edge_clone_summary_t (symbol_table *symtab):
4252 263840 : call_summary <edge_clone_summary *> (symtab)
4253 : {
4254 131920 : m_initialize_when_cloning = true;
4255 : }
4256 :
4257 : void duplicate (cgraph_edge *src_edge, cgraph_edge *dst_edge,
4258 : edge_clone_summary *src_data,
4259 : edge_clone_summary *dst_data) final override;
4260 : };
4261 :
4262 : /* Edge duplication hook. */
4263 :
4264 : void
4265 191798 : edge_clone_summary_t::duplicate (cgraph_edge *src_edge, cgraph_edge *dst_edge,
4266 : edge_clone_summary *src_data,
4267 : edge_clone_summary *dst_data)
4268 : {
4269 191798 : if (src_data->next_clone)
4270 5467 : edge_clone_summaries->get (src_data->next_clone)->prev_clone = dst_edge;
4271 191798 : dst_data->prev_clone = src_edge;
4272 191798 : dst_data->next_clone = src_data->next_clone;
4273 191798 : src_data->next_clone = dst_edge;
4274 191798 : }
4275 :
4276 : /* Return true is CS calls DEST or its clone for all contexts. When
4277 : ALLOW_RECURSION_TO_CLONE is false, also return false for self-recursive
4278 : edges from/to an all-context clone. */
4279 :
4280 : static bool
4281 1841940 : calls_same_node_or_its_all_contexts_clone_p (cgraph_edge *cs, cgraph_node *dest,
4282 : bool allow_recursion_to_clone)
4283 : {
4284 1841940 : enum availability availability;
4285 1841940 : cgraph_node *callee = cs->callee->function_symbol (&availability);
4286 :
4287 1841940 : if (availability <= AVAIL_INTERPOSABLE)
4288 : return false;
4289 1835939 : if (callee == dest)
4290 : return true;
4291 626424 : if (!allow_recursion_to_clone && cs->caller == callee)
4292 : return false;
4293 :
4294 626267 : ipa_node_params *info = ipa_node_params_sum->get (callee);
4295 626267 : return info->is_all_contexts_clone && info->ipcp_orig_node == dest;
4296 : }
4297 :
4298 : /* Return true if edge CS does bring about the value described by SRC to
4299 : DEST_VAL of node DEST or its clone for all contexts. */
4300 :
4301 : static bool
4302 1831126 : cgraph_edge_brings_value_p (cgraph_edge *cs, ipcp_value_source<tree> *src,
4303 : cgraph_node *dest, ipcp_value<tree> *dest_val)
4304 : {
4305 1831126 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
4306 :
4307 1831126 : if (!calls_same_node_or_its_all_contexts_clone_p (cs, dest, !src->val)
4308 1831126 : || caller_info->node_dead)
4309 : return false;
4310 :
4311 758918 : if (!src->val)
4312 : return true;
4313 :
4314 64211 : if (caller_info->ipcp_orig_node)
4315 : {
4316 20156 : tree t = NULL_TREE;
4317 20156 : if (src->offset == -1)
4318 14405 : t = caller_info->known_csts[src->index];
4319 5751 : else if (ipcp_transformation *ts
4320 5751 : = ipcp_get_transformation_summary (cs->caller))
4321 : {
4322 5751 : ipa_argagg_value_list avl (ts);
4323 5751 : t = avl.get_value (src->index, src->offset / BITS_PER_UNIT);
4324 : }
4325 20156 : return (t != NULL_TREE
4326 20156 : && values_equal_for_ipcp_p (src->val->value, t));
4327 : }
4328 : else
4329 : {
4330 44055 : if (src->val == dest_val)
4331 : return true;
4332 :
4333 38337 : struct ipcp_agg_lattice *aglat;
4334 38337 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (caller_info,
4335 : src->index);
4336 38337 : if (src->offset == -1)
4337 28238 : return (plats->itself.is_single_const ()
4338 20 : && values_equal_for_ipcp_p (src->val->value,
4339 20 : plats->itself.values->value));
4340 : else
4341 : {
4342 10099 : if (plats->aggs_bottom || plats->aggs_contain_variable)
4343 : return false;
4344 3882 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
4345 3882 : if (aglat->offset == src->offset)
4346 1748 : return (aglat->is_single_const ()
4347 8 : && values_equal_for_ipcp_p (src->val->value,
4348 8 : aglat->values->value));
4349 : }
4350 : return false;
4351 : }
4352 : }
4353 :
4354 : /* Return true if edge CS does bring about the value described by SRC to
4355 : DST_VAL of node DEST or its clone for all contexts. */
4356 :
4357 : static bool
4358 10814 : cgraph_edge_brings_value_p (cgraph_edge *cs,
4359 : ipcp_value_source<ipa_polymorphic_call_context> *src,
4360 : cgraph_node *dest,
4361 : ipcp_value<ipa_polymorphic_call_context> *)
4362 : {
4363 10814 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
4364 :
4365 10814 : if (!calls_same_node_or_its_all_contexts_clone_p (cs, dest, true)
4366 10814 : || caller_info->node_dead)
4367 : return false;
4368 9672 : if (!src->val)
4369 : return true;
4370 :
4371 1840 : if (caller_info->ipcp_orig_node)
4372 266 : return (caller_info->known_contexts.length () > (unsigned) src->index)
4373 532 : && values_equal_for_ipcp_p (src->val->value,
4374 266 : caller_info->known_contexts[src->index]);
4375 :
4376 1574 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (caller_info,
4377 : src->index);
4378 1574 : return plats->ctxlat.is_single_const ()
4379 282 : && values_equal_for_ipcp_p (src->val->value,
4380 282 : plats->ctxlat.values->value);
4381 : }
4382 :
4383 : /* Get the next clone in the linked list of clones of an edge. */
4384 :
4385 : static inline struct cgraph_edge *
4386 1842209 : get_next_cgraph_edge_clone (struct cgraph_edge *cs)
4387 : {
4388 1842209 : edge_clone_summary *s = edge_clone_summaries->get (cs);
4389 1842209 : return s != NULL ? s->next_clone : NULL;
4390 : }
4391 :
4392 : /* Given VAL that is intended for DEST, iterate over all its sources and if any
4393 : of them is viable and hot, return true. In that case, for those that still
4394 : hold, add their edge frequency and their number and cumulative profile
4395 : counts of self-ecursive and other edges into *FREQUENCY, *CALLER_COUNT,
4396 : REC_COUNT_SUM and NONREC_COUNT_SUM respectively. */
4397 :
4398 : template <typename valtype>
4399 : static bool
4400 218587 : get_info_about_necessary_edges (ipcp_value<valtype> *val, cgraph_node *dest,
4401 : sreal *freq_sum, int *caller_count,
4402 : profile_count *rec_count_sum,
4403 : profile_count *nonrec_count_sum,
4404 : bool *called_without_ipa_profile)
4405 : {
4406 : ipcp_value_source<valtype> *src;
4407 218587 : sreal freq = 0;
4408 218587 : int count = 0;
4409 218587 : profile_count rec_cnt = profile_count::zero ();
4410 218587 : profile_count nonrec_cnt = profile_count::zero ();
4411 218587 : bool interesting = false;
4412 218587 : bool non_self_recursive = false;
4413 218587 : *called_without_ipa_profile = false;
4414 :
4415 988988 : for (src = val->sources; src; src = src->next)
4416 : {
4417 770401 : struct cgraph_edge *cs = src->cs;
4418 1907652 : while (cs)
4419 : {
4420 1137251 : if (cgraph_edge_brings_value_p (cs, src, dest, val))
4421 : {
4422 360965 : count++;
4423 360965 : freq += cs->sreal_frequency ();
4424 360965 : interesting |= cs_interesting_for_ipcp_p (cs);
4425 360965 : if (cs->caller != dest)
4426 : {
4427 354002 : non_self_recursive = true;
4428 354002 : if (cs->count.ipa ().initialized_p ())
4429 993 : rec_cnt += cs->count.ipa ();
4430 : else
4431 353009 : *called_without_ipa_profile = true;
4432 : }
4433 6963 : else if (cs->count.ipa ().initialized_p ())
4434 0 : nonrec_cnt += cs->count.ipa ();
4435 : else
4436 6963 : *called_without_ipa_profile = true;
4437 : }
4438 1137251 : cs = get_next_cgraph_edge_clone (cs);
4439 : }
4440 : }
4441 :
4442 : /* If the only edges bringing a value are self-recursive ones, do not bother
4443 : evaluating it. */
4444 218587 : if (!non_self_recursive)
4445 : return false;
4446 :
4447 154502 : *freq_sum = freq;
4448 154502 : *caller_count = count;
4449 154502 : *rec_count_sum = rec_cnt;
4450 154502 : *nonrec_count_sum = nonrec_cnt;
4451 :
4452 154502 : return interesting;
4453 : }
4454 :
4455 : /* Given a NODE, and a set of its CALLERS, try to adjust order of the callers
4456 : to let a non-self-recursive caller be the first element. Thus, we can
4457 : simplify intersecting operations on values that arrive from all of these
4458 : callers, especially when there exists self-recursive call. Return true if
4459 : this kind of adjustment is possible. */
4460 :
4461 : static bool
4462 56580 : adjust_callers_for_value_intersection (vec<cgraph_edge *> &callers,
4463 : cgraph_node *node)
4464 : {
4465 60798 : for (unsigned i = 0; i < callers.length (); i++)
4466 : {
4467 60731 : cgraph_edge *cs = callers[i];
4468 :
4469 60731 : if (cs->caller != node)
4470 : {
4471 56513 : if (i > 0)
4472 : {
4473 1985 : callers[i] = callers[0];
4474 1985 : callers[0] = cs;
4475 : }
4476 : return true;
4477 : }
4478 : }
4479 : return false;
4480 : }
4481 :
4482 : /* Return a vector of incoming edges that do bring value VAL to node DEST. It
4483 : is assumed their number is known and equal to CALLER_COUNT. */
4484 :
4485 : template <typename valtype>
4486 : static auto_vec<cgraph_edge *>
4487 154101 : gather_edges_for_value (ipcp_value<valtype> *val, cgraph_node *dest,
4488 : int caller_count)
4489 : {
4490 : ipcp_value_source<valtype> *src;
4491 154101 : auto_vec<cgraph_edge *> ret (caller_count);
4492 :
4493 529616 : for (src = val->sources; src; src = src->next)
4494 : {
4495 375515 : struct cgraph_edge *cs = src->cs;
4496 839476 : while (cs)
4497 : {
4498 463961 : if (cgraph_edge_brings_value_p (cs, src, dest, val))
4499 357709 : ret.quick_push (cs);
4500 463961 : cs = get_next_cgraph_edge_clone (cs);
4501 : }
4502 : }
4503 :
4504 154101 : if (caller_count > 1)
4505 41634 : adjust_callers_for_value_intersection (ret, dest);
4506 :
4507 154101 : return ret;
4508 : }
4509 :
4510 : /* Construct a replacement map for a know VALUE for a formal parameter PARAM.
4511 : Return it or NULL if for some reason it cannot be created. FORCE_LOAD_REF
4512 : should be set to true when the reference created for the constant should be
4513 : a load one and not an address one because the corresponding parameter p is
4514 : only used as *p. */
4515 :
4516 : static struct ipa_replace_map *
4517 24729 : get_replacement_map (class ipa_node_params *info, tree value, int parm_num,
4518 : bool force_load_ref)
4519 : {
4520 24729 : struct ipa_replace_map *replace_map;
4521 :
4522 24729 : replace_map = ggc_alloc<ipa_replace_map> ();
4523 24729 : if (dump_file)
4524 : {
4525 171 : fprintf (dump_file, " replacing ");
4526 171 : ipa_dump_param (dump_file, info, parm_num);
4527 :
4528 171 : fprintf (dump_file, " with const ");
4529 171 : print_generic_expr (dump_file, value);
4530 :
4531 171 : if (force_load_ref)
4532 11 : fprintf (dump_file, " - forcing load reference\n");
4533 : else
4534 160 : fprintf (dump_file, "\n");
4535 : }
4536 24729 : replace_map->parm_num = parm_num;
4537 24729 : replace_map->new_tree = value;
4538 24729 : replace_map->force_load_ref = force_load_ref;
4539 24729 : return replace_map;
4540 : }
4541 :
4542 : /* Dump new profiling counts of NODE. SPEC is true when NODE is a specialzied
4543 : one, otherwise it will be referred to as the original node. */
4544 :
4545 : static void
4546 4 : dump_profile_updates (cgraph_node *node, bool spec)
4547 : {
4548 4 : if (spec)
4549 2 : fprintf (dump_file, " setting count of the specialized node %s to ",
4550 : node->dump_name ());
4551 : else
4552 2 : fprintf (dump_file, " setting count of the original node %s to ",
4553 : node->dump_name ());
4554 :
4555 4 : node->count.dump (dump_file);
4556 4 : fprintf (dump_file, "\n");
4557 6 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
4558 : {
4559 2 : fprintf (dump_file, " edge to %s has count ",
4560 2 : cs->callee->dump_name ());
4561 2 : cs->count.dump (dump_file);
4562 2 : fprintf (dump_file, "\n");
4563 : }
4564 4 : }
4565 :
4566 : /* With partial train run we do not want to assume that original's count is
4567 : zero whenever we redurect all executed edges to clone. Simply drop profile
4568 : to local one in this case. In eany case, return the new value. ORIG_NODE
4569 : is the original node and its count has not been updated yet. */
4570 :
4571 : profile_count
4572 65 : lenient_count_portion_handling (profile_count remainder, cgraph_node *orig_node)
4573 : {
4574 130 : if (remainder.ipa_p () && !remainder.ipa ().nonzero_p ()
4575 117 : && orig_node->count.ipa_p () && orig_node->count.ipa ().nonzero_p ()
4576 26 : && opt_for_fn (orig_node->decl, flag_profile_partial_training))
4577 0 : remainder = orig_node->count.guessed_local ();
4578 :
4579 65 : return remainder;
4580 : }
4581 :
4582 : /* Structure to sum counts coming from nodes other than the original node and
4583 : its clones. */
4584 :
4585 : struct gather_other_count_struct
4586 : {
4587 : cgraph_node *orig;
4588 : profile_count other_count;
4589 : };
4590 :
4591 : /* Worker callback of call_for_symbol_thunks_and_aliases summing the number of
4592 : counts that come from non-self-recursive calls.. */
4593 :
4594 : static bool
4595 16 : gather_count_of_non_rec_edges (cgraph_node *node, void *data)
4596 : {
4597 16 : gather_other_count_struct *desc = (gather_other_count_struct *) data;
4598 34 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4599 18 : if (cs->caller != desc->orig && cs->caller->clone_of != desc->orig)
4600 0 : if (cs->count.ipa ().initialized_p ())
4601 0 : desc->other_count += cs->count.ipa ();
4602 16 : return false;
4603 : }
4604 :
4605 : /* Structure to help analyze if we need to boost counts of some clones of some
4606 : non-recursive edges to match the new callee count. */
4607 :
4608 : struct desc_incoming_count_struct
4609 : {
4610 : cgraph_node *orig;
4611 : hash_set <cgraph_edge *> *processed_edges;
4612 : profile_count count;
4613 : unsigned unproc_orig_rec_edges;
4614 : };
4615 :
4616 : /* Go over edges calling NODE and its thunks and gather information about
4617 : incoming counts so that we know if we need to make any adjustments. */
4618 :
4619 : static void
4620 16 : analyze_clone_icoming_counts (cgraph_node *node,
4621 : desc_incoming_count_struct *desc)
4622 : {
4623 34 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4624 18 : if (cs->caller->thunk)
4625 : {
4626 0 : analyze_clone_icoming_counts (cs->caller, desc);
4627 0 : continue;
4628 : }
4629 : else
4630 : {
4631 18 : if (cs->count.initialized_p ())
4632 18 : desc->count += cs->count.ipa ();
4633 18 : if (!desc->processed_edges->contains (cs)
4634 18 : && cs->caller->clone_of == desc->orig)
4635 2 : desc->unproc_orig_rec_edges++;
4636 : }
4637 16 : }
4638 :
4639 : /* If caller edge counts of a clone created for a self-recursive arithmetic
4640 : jump function must be adjusted because it is coming from a the "seed" clone
4641 : for the first value and so has been excessively scaled back as if it was not
4642 : a recursive call, adjust it so that the incoming counts of NODE match its
4643 : count. NODE is the node or its thunk. */
4644 :
4645 : static void
4646 0 : adjust_clone_incoming_counts (cgraph_node *node,
4647 : desc_incoming_count_struct *desc)
4648 : {
4649 0 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4650 0 : if (cs->caller->thunk)
4651 : {
4652 0 : adjust_clone_incoming_counts (cs->caller, desc);
4653 : /* Same rationale as commit 8c6b6adce45a550c52dc35e3df4e0c477f5404fa
4654 : for scaling recursive edges in update_counts_for_self_gen_clones:
4655 : adjusting non-IPA edge counts here does not update matching gimple
4656 : BB frequencies and breaks verify_cgraph_node. */
4657 0 : if (cs->count.ipa_p ())
4658 : {
4659 0 : profile_count sum = profile_count::zero ();
4660 0 : for (cgraph_edge *e = cs->caller->callers; e; e = e->next_caller)
4661 0 : if (e->count.initialized_p ())
4662 0 : sum += e->count.ipa ();
4663 0 : cs->count = cs->count.combine_with_ipa_count (sum);
4664 : }
4665 0 : else if (dump_file)
4666 0 : fprintf (dump_file, " Skipping adjustment of the count of an "
4667 : "incoming edge of a clone %s -> %s\n",
4668 0 : cs->caller->dump_name (), cs->callee->dump_name ());
4669 : }
4670 0 : else if (!desc->processed_edges->contains (cs)
4671 0 : && cs->caller->clone_of == desc->orig
4672 0 : && cs->count.compatible_p (desc->count))
4673 : {
4674 0 : if (cs->count.ipa_p ())
4675 : {
4676 0 : cs->count += desc->count;
4677 0 : if (dump_file)
4678 : {
4679 0 : fprintf (dump_file, " Adjusted count of an incoming edge "
4680 0 : "of a clone %s -> %s to ", cs->caller->dump_name (),
4681 0 : cs->callee->dump_name ());
4682 0 : cs->count.dump (dump_file);
4683 0 : fprintf (dump_file, "\n");
4684 : }
4685 : }
4686 0 : else if (dump_file)
4687 0 : fprintf (dump_file, " Skipping adjustment of the count of an "
4688 : "incoming edge of a clone %s -> %s\n",
4689 0 : cs->caller->dump_name (), cs->callee->dump_name ());
4690 : }
4691 0 : }
4692 :
4693 : /* When ORIG_NODE has been cloned for values which have been generated fora
4694 : self-recursive call as a result of an arithmetic pass-through
4695 : jump-functions, adjust its count together with counts of all such clones in
4696 : SELF_GEN_CLONES which also at this point contains ORIG_NODE itself.
4697 :
4698 : The function sums the counts of the original node and all its clones that
4699 : cannot be attributed to a specific clone because it comes from a
4700 : non-recursive edge. This sum is then evenly divided between the clones and
4701 : on top of that each one gets all the counts which can be attributed directly
4702 : to it. */
4703 :
4704 : static void
4705 31 : update_counts_for_self_gen_clones (cgraph_node *orig_node,
4706 : const vec<cgraph_node *> &self_gen_clones)
4707 : {
4708 31 : profile_count redist_sum = orig_node->count.ipa ();
4709 31 : if (!redist_sum.nonzero_p ())
4710 : return;
4711 :
4712 2 : if (dump_file)
4713 0 : fprintf (dump_file, " Updating profile of self recursive clone "
4714 : "series\n");
4715 :
4716 2 : gather_other_count_struct gocs;
4717 2 : gocs.orig = orig_node;
4718 2 : gocs.other_count = profile_count::zero ();
4719 :
4720 2 : auto_vec <profile_count, 8> other_edges_count;
4721 22 : for (cgraph_node *n : self_gen_clones)
4722 : {
4723 16 : gocs.other_count = profile_count::zero ();
4724 16 : n->call_for_symbol_thunks_and_aliases (gather_count_of_non_rec_edges,
4725 : &gocs, false);
4726 16 : other_edges_count.safe_push (gocs.other_count);
4727 16 : redist_sum -= gocs.other_count;
4728 : }
4729 :
4730 2 : hash_set<cgraph_edge *> processed_edges;
4731 2 : unsigned i = 0;
4732 22 : for (cgraph_node *n : self_gen_clones)
4733 : {
4734 16 : profile_count new_count
4735 32 : = (redist_sum / self_gen_clones.length () + other_edges_count[i]);
4736 16 : new_count = lenient_count_portion_handling (new_count, orig_node);
4737 16 : n->scale_profile_to (new_count);
4738 32 : for (cgraph_edge *cs = n->callees; cs; cs = cs->next_callee)
4739 16 : processed_edges.add (cs);
4740 :
4741 16 : i++;
4742 : }
4743 :
4744 : /* There are still going to be edges to ORIG_NODE that have one or more
4745 : clones coming from another node clone in SELF_GEN_CLONES and which we
4746 : scaled by the same amount, which means that the total incoming sum of
4747 : counts to ORIG_NODE will be too high, scale such edges back. */
4748 4 : for (cgraph_edge *cs = orig_node->callees; cs; cs = cs->next_callee)
4749 : {
4750 2 : if (cs->callee->ultimate_alias_target () == orig_node)
4751 : {
4752 2 : unsigned den = 0;
4753 20 : for (cgraph_edge *e = cs; e; e = get_next_cgraph_edge_clone (e))
4754 18 : if (e->callee->ultimate_alias_target () == orig_node
4755 18 : && processed_edges.contains (e))
4756 4 : den++;
4757 2 : if (den > 0)
4758 20 : for (cgraph_edge *e = cs; e; e = get_next_cgraph_edge_clone (e))
4759 18 : if (e->callee->ultimate_alias_target () == orig_node
4760 4 : && processed_edges.contains (e)
4761 : /* If count is not IPA, this adjustment makes verifier
4762 : unhappy, since we expect bb->count to match e->count.
4763 : We may add a flag to mark edge conts that has been
4764 : modified by IPA code, but so far it does not seem
4765 : to be worth the effort. With local counts the profile
4766 : will not propagate at IPA level. */
4767 26 : && e->count.ipa_p ())
4768 4 : e->count /= den;
4769 : }
4770 : }
4771 :
4772 : /* Edges from the seeds of the values generated for arithmetic jump-functions
4773 : along self-recursive edges are likely to have fairly low count and so
4774 : edges from them to nodes in the self_gen_clones do not correspond to the
4775 : artificially distributed count of the nodes, the total sum of incoming
4776 : edges to some clones might be too low. Detect this situation and correct
4777 : it. */
4778 22 : for (cgraph_node *n : self_gen_clones)
4779 : {
4780 16 : if (!n->count.ipa ().nonzero_p ())
4781 0 : continue;
4782 :
4783 16 : desc_incoming_count_struct desc;
4784 16 : desc.orig = orig_node;
4785 16 : desc.processed_edges = &processed_edges;
4786 16 : desc.count = profile_count::zero ();
4787 16 : desc.unproc_orig_rec_edges = 0;
4788 16 : analyze_clone_icoming_counts (n, &desc);
4789 :
4790 16 : if (n->count.differs_from_p (desc.count))
4791 : {
4792 0 : if (n->count > desc.count
4793 0 : && desc.unproc_orig_rec_edges > 0)
4794 : {
4795 0 : desc.count = n->count - desc.count;
4796 0 : desc.count = desc.count /= desc.unproc_orig_rec_edges;
4797 0 : adjust_clone_incoming_counts (n, &desc);
4798 : }
4799 0 : else if (dump_file)
4800 0 : fprintf (dump_file,
4801 : " Unable to fix up incoming counts for %s.\n",
4802 : n->dump_name ());
4803 : }
4804 : }
4805 :
4806 2 : if (dump_file)
4807 0 : for (cgraph_node *n : self_gen_clones)
4808 0 : dump_profile_updates (n, n != orig_node);
4809 2 : return;
4810 2 : }
4811 :
4812 : /* After a specialized NEW_NODE version of ORIG_NODE has been created, update
4813 : their profile information to reflect this. This function should not be used
4814 : for clones generated for arithmetic pass-through jump functions on a
4815 : self-recursive call graph edge, that situation is handled by
4816 : update_counts_for_self_gen_clones. */
4817 :
4818 : static void
4819 4363 : update_profiling_info (struct cgraph_node *orig_node,
4820 : struct cgraph_node *new_node)
4821 : {
4822 4363 : struct caller_statistics stats;
4823 4363 : profile_count new_sum;
4824 4363 : profile_count remainder, orig_node_count = orig_node->count.ipa ();
4825 :
4826 4363 : if (!orig_node_count.nonzero_p ())
4827 4314 : return;
4828 :
4829 49 : if (dump_file)
4830 : {
4831 2 : fprintf (dump_file, " Updating profile from original count: ");
4832 2 : orig_node_count.dump (dump_file);
4833 2 : fprintf (dump_file, "\n");
4834 : }
4835 :
4836 49 : init_caller_stats (&stats, new_node);
4837 49 : new_node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
4838 : false);
4839 49 : new_sum = stats.count_sum;
4840 :
4841 49 : bool orig_edges_processed = false;
4842 49 : if (new_sum > orig_node_count)
4843 : {
4844 : /* Profile has already gone astray, keep what we have but lower it
4845 : to global0adjusted or to local if we have partial training. */
4846 0 : if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
4847 0 : orig_node->make_profile_local ();
4848 0 : else if (new_sum.quality () == AFDO)
4849 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_AFDO);
4850 : else
4851 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_ADJUSTED);
4852 : orig_edges_processed = true;
4853 : }
4854 49 : else if (stats.rec_count_sum.nonzero_p ())
4855 : {
4856 0 : int new_nonrec_calls = stats.n_nonrec_calls;
4857 : /* There are self-recursive edges which are likely to bring in the
4858 : majority of calls but which we must divide in between the original and
4859 : new node. */
4860 0 : init_caller_stats (&stats, orig_node);
4861 0 : orig_node->call_for_symbol_thunks_and_aliases (gather_caller_stats,
4862 : &stats, false);
4863 0 : int orig_nonrec_calls = stats.n_nonrec_calls;
4864 0 : profile_count orig_nonrec_call_count = stats.count_sum;
4865 :
4866 0 : if (orig_node->local)
4867 : {
4868 0 : if (!orig_nonrec_call_count.nonzero_p ())
4869 : {
4870 0 : if (dump_file)
4871 0 : fprintf (dump_file, " The original is local and the only "
4872 : "incoming edges from non-dead callers with nonzero "
4873 : "counts are self-recursive, assuming it is cold.\n");
4874 : /* The NEW_NODE count and counts of all its outgoing edges
4875 : are still unmodified copies of ORIG_NODE's. Just clear
4876 : the latter and bail out. */
4877 0 : if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
4878 0 : orig_node->make_profile_local ();
4879 0 : else if (orig_nonrec_call_count.quality () == AFDO)
4880 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_AFDO);
4881 : else
4882 0 : orig_node->make_profile_global0 (GUESSED_GLOBAL0_ADJUSTED);
4883 0 : return;
4884 : }
4885 : }
4886 : else
4887 : {
4888 : /* Let's behave as if there was another caller that accounts for all
4889 : the calls that were either indirect or from other compilation
4890 : units. */
4891 0 : orig_nonrec_calls++;
4892 0 : profile_count pretend_caller_count
4893 0 : = (orig_node_count - new_sum - orig_nonrec_call_count
4894 0 : - stats.rec_count_sum);
4895 0 : orig_nonrec_call_count += pretend_caller_count;
4896 : }
4897 :
4898 : /* Divide all "unexplained" counts roughly proportionally to sums of
4899 : counts of non-recursive calls.
4900 :
4901 : We put rather arbitrary limits on how many counts we claim because the
4902 : number of non-self-recursive incoming count is only a rough guideline
4903 : and there are cases (such as mcf) where using it blindly just takes
4904 : too many. And if lattices are considered in the opposite order we
4905 : could also take too few. */
4906 0 : profile_count unexp = orig_node_count - new_sum - orig_nonrec_call_count;
4907 :
4908 0 : int limit_den = 2 * (orig_nonrec_calls + new_nonrec_calls);
4909 0 : profile_count new_part = unexp.apply_scale (limit_den - 1, limit_den);
4910 0 : profile_count den = new_sum + orig_nonrec_call_count;
4911 0 : if (den.nonzero_p ())
4912 0 : new_part = MIN (unexp.apply_scale (new_sum, den), new_part);
4913 0 : new_part = MAX (new_part,
4914 : unexp.apply_scale (new_nonrec_calls, limit_den));
4915 0 : if (dump_file)
4916 : {
4917 0 : fprintf (dump_file, " Claiming ");
4918 0 : new_part.dump (dump_file);
4919 0 : fprintf (dump_file, " of unexplained ");
4920 0 : unexp.dump (dump_file);
4921 0 : fprintf (dump_file, " counts because of self-recursive "
4922 : "calls\n");
4923 : }
4924 0 : new_sum += new_part;
4925 0 : remainder = lenient_count_portion_handling (orig_node_count - new_sum,
4926 : orig_node);
4927 : }
4928 : else
4929 49 : remainder = lenient_count_portion_handling (orig_node_count - new_sum,
4930 : orig_node);
4931 :
4932 49 : new_node->scale_profile_to (new_sum);
4933 :
4934 49 : if (!orig_edges_processed)
4935 49 : orig_node->scale_profile_to (remainder);
4936 :
4937 49 : if (dump_file)
4938 : {
4939 2 : dump_profile_updates (new_node, true);
4940 2 : dump_profile_updates (orig_node, false);
4941 : }
4942 : }
4943 :
4944 : /* Update the respective profile of specialized NEW_NODE and the original
4945 : ORIG_NODE after additional edges with cumulative count sum REDIRECTED_SUM
4946 : have been redirected to the specialized version. */
4947 :
4948 : static void
4949 0 : update_specialized_profile (struct cgraph_node *new_node,
4950 : struct cgraph_node *orig_node,
4951 : profile_count redirected_sum)
4952 : {
4953 0 : if (dump_file)
4954 : {
4955 0 : fprintf (dump_file, " the sum of counts of redirected edges is ");
4956 0 : redirected_sum.dump (dump_file);
4957 0 : fprintf (dump_file, "\n old ipa count of the original node is ");
4958 0 : orig_node->count.dump (dump_file);
4959 0 : fprintf (dump_file, "\n");
4960 : }
4961 0 : if (!orig_node->count.ipa ().nonzero_p ()
4962 0 : || !redirected_sum.nonzero_p ())
4963 : return;
4964 :
4965 0 : orig_node->scale_profile_to
4966 0 : (lenient_count_portion_handling (orig_node->count.ipa () - redirected_sum,
4967 : orig_node));
4968 :
4969 0 : new_node->scale_profile_to (new_node->count.ipa () + redirected_sum);
4970 :
4971 0 : if (dump_file)
4972 : {
4973 0 : dump_profile_updates (new_node, true);
4974 0 : dump_profile_updates (orig_node, false);
4975 : }
4976 : }
4977 :
4978 : static void adjust_references_in_caller (cgraph_edge *cs,
4979 : symtab_node *symbol, int index);
4980 :
4981 : /* Simple structure to pass a symbol and index (with same meaning as parameters
4982 : of adjust_references_in_caller) through a void* parameter of a
4983 : call_for_symbol_thunks_and_aliases callback. */
4984 : struct symbol_and_index_together
4985 : {
4986 : symtab_node *symbol;
4987 : int index;
4988 : };
4989 :
4990 : /* Worker callback of call_for_symbol_thunks_and_aliases to recursively call
4991 : adjust_references_in_caller on edges up in the call-graph, if necessary. */
4992 : static bool
4993 9 : adjust_refs_in_act_callers (struct cgraph_node *node, void *data)
4994 : {
4995 9 : symbol_and_index_together *pack = (symbol_and_index_together *) data;
4996 40 : for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
4997 31 : if (!cs->caller->thunk)
4998 31 : adjust_references_in_caller (cs, pack->symbol, pack->index);
4999 9 : return false;
5000 : }
5001 :
5002 : /* At INDEX of a function being called by CS there is an ADDR_EXPR of a
5003 : variable which is only dereferenced and which is represented by SYMBOL. See
5004 : if we can remove ADDR reference in callers associated with the call. */
5005 :
5006 : static void
5007 417 : adjust_references_in_caller (cgraph_edge *cs, symtab_node *symbol, int index)
5008 : {
5009 417 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5010 417 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, index);
5011 417 : if (jfunc->type == IPA_JF_CONST)
5012 : {
5013 398 : ipa_ref *to_del = cs->caller->find_reference (symbol, cs->call_stmt,
5014 : cs->lto_stmt_uid,
5015 : IPA_REF_ADDR);
5016 398 : if (!to_del)
5017 408 : return;
5018 398 : to_del->remove_reference ();
5019 398 : ipa_zap_jf_refdesc (jfunc);
5020 398 : if (dump_file)
5021 22 : fprintf (dump_file, " Removed a reference from %s to %s.\n",
5022 11 : cs->caller->dump_name (), symbol->dump_name ());
5023 : return;
5024 : }
5025 :
5026 19 : if (jfunc->type != IPA_JF_PASS_THROUGH
5027 19 : || ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR
5028 38 : || ipa_get_jf_pass_through_refdesc_decremented (jfunc))
5029 : return;
5030 :
5031 19 : int fidx = ipa_get_jf_pass_through_formal_id (jfunc);
5032 19 : cgraph_node *caller = cs->caller;
5033 19 : ipa_node_params *caller_info = ipa_node_params_sum->get (caller);
5034 : /* TODO: This consistency check may be too big and not really
5035 : that useful. Consider removing it. */
5036 19 : tree cst;
5037 19 : if (caller_info->ipcp_orig_node)
5038 17 : cst = caller_info->known_csts[fidx];
5039 : else
5040 : {
5041 2 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (caller_info, fidx);
5042 2 : gcc_assert (lat->is_single_const ());
5043 2 : cst = lat->values->value;
5044 : }
5045 19 : gcc_assert (TREE_CODE (cst) == ADDR_EXPR
5046 : && (symtab_node::get (get_base_address (TREE_OPERAND (cst, 0)))
5047 : == symbol));
5048 :
5049 19 : int cuses = ipa_get_controlled_uses (caller_info, fidx);
5050 19 : if (cuses == IPA_UNDESCRIBED_USE)
5051 : return;
5052 19 : gcc_assert (cuses > 0);
5053 19 : cuses--;
5054 19 : ipa_set_controlled_uses (caller_info, fidx, cuses);
5055 19 : ipa_set_jf_pass_through_refdesc_decremented (jfunc, true);
5056 19 : if (dump_file && (dump_flags & TDF_DETAILS))
5057 3 : fprintf (dump_file, " Controlled uses of parameter %i of %s dropped "
5058 : "to %i.\n", fidx, caller->dump_name (), cuses);
5059 19 : if (cuses)
5060 : return;
5061 :
5062 9 : if (caller_info->ipcp_orig_node)
5063 : {
5064 : /* Cloning machinery has created a reference here, we need to either
5065 : remove it or change it to a read one. */
5066 7 : ipa_ref *to_del = caller->find_reference (symbol, NULL, 0, IPA_REF_ADDR);
5067 7 : if (to_del)
5068 : {
5069 7 : to_del->remove_reference ();
5070 7 : if (dump_file)
5071 6 : fprintf (dump_file, " Removed a reference from %s to %s.\n",
5072 3 : cs->caller->dump_name (), symbol->dump_name ());
5073 7 : if (ipa_get_param_load_dereferenced (caller_info, fidx))
5074 : {
5075 3 : caller->create_reference (symbol, IPA_REF_LOAD, NULL);
5076 3 : if (dump_file)
5077 2 : fprintf (dump_file,
5078 : " ...and replaced it with LOAD one.\n");
5079 : }
5080 : }
5081 : }
5082 :
5083 9 : symbol_and_index_together pack;
5084 9 : pack.symbol = symbol;
5085 9 : pack.index = fidx;
5086 9 : if (caller->can_change_signature)
5087 9 : caller->call_for_symbol_thunks_and_aliases (adjust_refs_in_act_callers,
5088 : &pack, true);
5089 : }
5090 :
5091 :
5092 : /* Return true if we would like to remove a parameter from NODE when cloning it
5093 : with KNOWN_CSTS scalar constants. */
5094 :
5095 : static bool
5096 17953 : want_remove_some_param_p (cgraph_node *node, vec<tree> known_csts)
5097 : {
5098 17953 : auto_vec<bool, 16> surviving;
5099 17953 : bool filled_vec = false;
5100 17953 : ipa_node_params *info = ipa_node_params_sum->get (node);
5101 17953 : int i, count = ipa_get_param_count (info);
5102 :
5103 37195 : for (i = 0; i < count; i++)
5104 : {
5105 32457 : if (!known_csts[i] && ipa_is_param_used (info, i))
5106 19242 : continue;
5107 :
5108 13215 : if (!filled_vec)
5109 : {
5110 13215 : clone_info *info = clone_info::get (node);
5111 13215 : if (!info || !info->param_adjustments)
5112 : return true;
5113 0 : info->param_adjustments->get_surviving_params (&surviving);
5114 0 : filled_vec = true;
5115 : }
5116 0 : if (surviving.length() < (unsigned) i && surviving[i])
5117 : return true;
5118 : }
5119 : return false;
5120 17953 : }
5121 :
5122 : /* Create a specialized version of NODE with known constants in KNOWN_CSTS,
5123 : known contexts in KNOWN_CONTEXTS and known aggregate values in AGGVALS and
5124 : redirect all edges in CALLERS to it. */
5125 :
5126 : static struct cgraph_node *
5127 19394 : create_specialized_node (struct cgraph_node *node,
5128 : vec<tree> known_csts,
5129 : vec<ipa_polymorphic_call_context> known_contexts,
5130 : vec<ipa_argagg_value, va_gc> *aggvals,
5131 : vec<cgraph_edge *> &callers)
5132 : {
5133 19394 : ipa_node_params *new_info, *info = ipa_node_params_sum->get (node);
5134 19394 : vec<ipa_replace_map *, va_gc> *replace_trees = NULL;
5135 19394 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
5136 19394 : struct cgraph_node *new_node;
5137 19394 : int i, count = ipa_get_param_count (info);
5138 19394 : clone_info *cinfo = clone_info::get (node);
5139 0 : ipa_param_adjustments *old_adjustments = cinfo
5140 19394 : ? cinfo->param_adjustments : NULL;
5141 19394 : ipa_param_adjustments *new_adjustments;
5142 19394 : gcc_assert (!info->ipcp_orig_node);
5143 19394 : gcc_assert (node->can_change_signature
5144 : || !old_adjustments);
5145 :
5146 17953 : if (old_adjustments)
5147 : {
5148 : /* At the moment all IPA optimizations should use the number of
5149 : parameters of the prevailing decl as the m_always_copy_start.
5150 : Handling any other value would complicate the code below, so for the
5151 : time bing let's only assert it is so. */
5152 0 : gcc_assert (old_adjustments->m_always_copy_start == count
5153 : || old_adjustments->m_always_copy_start < 0);
5154 0 : int old_adj_count = vec_safe_length (old_adjustments->m_adj_params);
5155 0 : for (i = 0; i < old_adj_count; i++)
5156 : {
5157 0 : ipa_adjusted_param *old_adj = &(*old_adjustments->m_adj_params)[i];
5158 0 : if (!node->can_change_signature
5159 0 : || old_adj->op != IPA_PARAM_OP_COPY
5160 0 : || (!known_csts[old_adj->base_index]
5161 0 : && ipa_is_param_used (info, old_adj->base_index)))
5162 : {
5163 0 : ipa_adjusted_param new_adj = *old_adj;
5164 :
5165 0 : new_adj.prev_clone_adjustment = true;
5166 0 : new_adj.prev_clone_index = i;
5167 0 : vec_safe_push (new_params, new_adj);
5168 : }
5169 : }
5170 0 : bool skip_return = old_adjustments->m_skip_return;
5171 0 : new_adjustments = (new (ggc_alloc <ipa_param_adjustments> ())
5172 : ipa_param_adjustments (new_params, count,
5173 0 : skip_return));
5174 : }
5175 19394 : else if (node->can_change_signature
5176 19394 : && want_remove_some_param_p (node, known_csts))
5177 : {
5178 13215 : ipa_adjusted_param adj;
5179 13215 : memset (&adj, 0, sizeof (adj));
5180 13215 : adj.op = IPA_PARAM_OP_COPY;
5181 51244 : for (i = 0; i < count; i++)
5182 38029 : if (!known_csts[i] && ipa_is_param_used (info, i))
5183 : {
5184 15185 : adj.base_index = i;
5185 15185 : adj.prev_clone_index = i;
5186 15185 : vec_safe_push (new_params, adj);
5187 : }
5188 13215 : new_adjustments = (new (ggc_alloc <ipa_param_adjustments> ())
5189 13215 : ipa_param_adjustments (new_params, count, false));
5190 : }
5191 : else
5192 : new_adjustments = NULL;
5193 :
5194 19394 : auto_vec<cgraph_edge *, 2> self_recursive_calls;
5195 156634 : for (i = callers.length () - 1; i >= 0; i--)
5196 : {
5197 117846 : cgraph_edge *cs = callers[i];
5198 117846 : if (cs->caller == node)
5199 : {
5200 117 : self_recursive_calls.safe_push (cs);
5201 117 : callers.unordered_remove (i);
5202 : }
5203 : }
5204 19394 : replace_trees = cinfo ? vec_safe_copy (cinfo->tree_map) : NULL;
5205 73085 : for (i = 0; i < count; i++)
5206 : {
5207 53691 : tree t = known_csts[i];
5208 53691 : if (!t)
5209 28962 : continue;
5210 :
5211 24729 : gcc_checking_assert (TREE_CODE (t) != TREE_BINFO);
5212 :
5213 24729 : bool load_ref = false;
5214 24729 : symtab_node *ref_symbol;
5215 24729 : if (TREE_CODE (t) == ADDR_EXPR)
5216 : {
5217 6696 : tree base = get_base_address (TREE_OPERAND (t, 0));
5218 6696 : if (TREE_CODE (base) == VAR_DECL
5219 3228 : && ipa_get_controlled_uses (info, i) == 0
5220 962 : && ipa_get_param_load_dereferenced (info, i)
5221 7097 : && (ref_symbol = symtab_node::get (base)))
5222 : {
5223 401 : load_ref = true;
5224 401 : if (node->can_change_signature)
5225 1463 : for (cgraph_edge *caller : callers)
5226 386 : adjust_references_in_caller (caller, ref_symbol, i);
5227 : }
5228 : }
5229 :
5230 24729 : ipa_replace_map *replace_map = get_replacement_map (info, t, i, load_ref);
5231 24729 : if (replace_map)
5232 24729 : vec_safe_push (replace_trees, replace_map);
5233 : }
5234 :
5235 58182 : unsigned &suffix_counter = clone_num_suffixes->get_or_insert (
5236 19394 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (
5237 : node->decl)));
5238 19394 : new_node = node->create_virtual_clone (callers, replace_trees,
5239 : new_adjustments, "constprop",
5240 : suffix_counter);
5241 19394 : suffix_counter++;
5242 :
5243 19394 : bool have_self_recursive_calls = !self_recursive_calls.is_empty ();
5244 19511 : for (unsigned j = 0; j < self_recursive_calls.length (); j++)
5245 : {
5246 117 : cgraph_edge *cs = get_next_cgraph_edge_clone (self_recursive_calls[j]);
5247 : /* Cloned edges can disappear during cloning as speculation can be
5248 : resolved, check that we have one and that it comes from the last
5249 : cloning. */
5250 117 : if (cs && cs->caller == new_node)
5251 116 : cs->redirect_callee_duplicating_thunks (new_node);
5252 : /* Any future code that would make more than one clone of an outgoing
5253 : edge would confuse this mechanism, so let's check that does not
5254 : happen. */
5255 116 : gcc_checking_assert (!cs
5256 : || !get_next_cgraph_edge_clone (cs)
5257 : || get_next_cgraph_edge_clone (cs)->caller != new_node);
5258 : }
5259 19394 : if (have_self_recursive_calls)
5260 109 : new_node->expand_all_artificial_thunks ();
5261 :
5262 19394 : ipa_set_node_agg_value_chain (new_node, aggvals);
5263 50843 : for (const ipa_argagg_value &av : aggvals)
5264 31449 : new_node->maybe_create_reference (av.value, NULL);
5265 :
5266 19394 : if (dump_file && (dump_flags & TDF_DETAILS))
5267 : {
5268 91 : fprintf (dump_file, " the new node is %s.\n", new_node->dump_name ());
5269 91 : if (known_contexts.exists ())
5270 : {
5271 0 : for (i = 0; i < count; i++)
5272 0 : if (!known_contexts[i].useless_p ())
5273 : {
5274 0 : fprintf (dump_file, " known ctx %i is ", i);
5275 0 : known_contexts[i].dump (dump_file);
5276 : }
5277 : }
5278 91 : if (aggvals)
5279 : {
5280 49 : fprintf (dump_file, " Aggregate replacements:");
5281 49 : ipa_argagg_value_list avs (aggvals);
5282 49 : avs.dump (dump_file);
5283 : }
5284 : }
5285 :
5286 19394 : new_info = ipa_node_params_sum->get (new_node);
5287 19394 : new_info->ipcp_orig_node = node;
5288 19394 : new_node->ipcp_clone = true;
5289 19394 : new_info->known_csts = known_csts;
5290 19394 : new_info->known_contexts = known_contexts;
5291 :
5292 19394 : ipcp_discover_new_direct_edges (new_node, known_csts, known_contexts,
5293 : aggvals);
5294 :
5295 19394 : return new_node;
5296 19394 : }
5297 :
5298 : /* Return true if JFUNC, which describes a i-th parameter of call CS, is a
5299 : pass-through function to itself when the cgraph_node involved is not an
5300 : IPA-CP clone. When SIMPLE is true, further check if JFUNC is a simple
5301 : no-operation pass-through. */
5302 :
5303 : static bool
5304 794114 : self_recursive_pass_through_p (cgraph_edge *cs, ipa_jump_func *jfunc, int i,
5305 : bool simple = true)
5306 : {
5307 794114 : enum availability availability;
5308 794114 : if (jfunc->type == IPA_JF_PASS_THROUGH
5309 79121 : && cs->caller == cs->callee->function_symbol (&availability)
5310 19355 : && availability > AVAIL_INTERPOSABLE
5311 19355 : && (!simple || ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
5312 19355 : && ipa_get_jf_pass_through_formal_id (jfunc) == i
5313 19355 : && ipa_node_params_sum->get (cs->caller)
5314 813469 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5315 19326 : return true;
5316 : return false;
5317 : }
5318 :
5319 : /* Return true if JFUNC, which describes the i-th parameter of call CS, is an
5320 : ancestor function with zero offset to itself when the cgraph_node involved
5321 : is not an IPA-CP clone. */
5322 :
5323 : static bool
5324 774788 : self_recursive_ancestor_p (cgraph_edge *cs, ipa_jump_func *jfunc, int i)
5325 : {
5326 774788 : enum availability availability;
5327 774788 : if (jfunc->type == IPA_JF_ANCESTOR
5328 3256 : && cs->caller == cs->callee->function_symbol (&availability)
5329 1 : && availability > AVAIL_INTERPOSABLE
5330 1 : && ipa_get_jf_ancestor_offset (jfunc) == 0
5331 1 : && ipa_get_jf_ancestor_formal_id (jfunc) == i
5332 1 : && ipa_node_params_sum->get (cs->caller)
5333 774789 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5334 1 : return true;
5335 : return false;
5336 : }
5337 :
5338 : /* Return true if JFUNC, which describes a part of an aggregate represented or
5339 : pointed to by the i-th parameter of call CS, is a pass-through function to
5340 : itself when the cgraph_node involved is not an IPA-CP clone.. When
5341 : SIMPLE is true, further check if JFUNC is a simple no-operation
5342 : pass-through. */
5343 :
5344 : static bool
5345 361951 : self_recursive_agg_pass_through_p (const cgraph_edge *cs,
5346 : const ipa_agg_jf_item *jfunc,
5347 : int i, bool simple = true)
5348 : {
5349 361951 : enum availability availability;
5350 361951 : if (cs->caller == cs->callee->function_symbol (&availability)
5351 3819 : && availability > AVAIL_INTERPOSABLE
5352 3819 : && jfunc->jftype == IPA_JF_LOAD_AGG
5353 487 : && jfunc->offset == jfunc->value.load_agg.offset
5354 487 : && (!simple || jfunc->value.pass_through.operation == NOP_EXPR)
5355 487 : && jfunc->value.pass_through.formal_id == i
5356 481 : && useless_type_conversion_p (jfunc->value.load_agg.type, jfunc->type)
5357 481 : && ipa_node_params_sum->get (cs->caller)
5358 362432 : && !ipa_node_params_sum->get (cs->caller)->ipcp_orig_node)
5359 481 : return true;
5360 : return false;
5361 : }
5362 :
5363 : /* Given a NODE, and a subset of its CALLERS, try to populate blanks slots in
5364 : KNOWN_CSTS with constants that are also known for all of the CALLERS. */
5365 :
5366 : static void
5367 168980 : find_scalar_values_for_callers_subset (vec<tree> &known_csts,
5368 : ipa_node_params *info,
5369 : const vec<cgraph_edge *> &callers)
5370 : {
5371 168980 : int i, count = ipa_get_param_count (info);
5372 :
5373 738672 : for (i = 0; i < count; i++)
5374 : {
5375 569692 : ipcp_lattice<tree> *lat = ipa_get_scalar_lat (info, i);
5376 569692 : if (lat->bottom)
5377 9426 : continue;
5378 560266 : if (lat->is_single_const ())
5379 : {
5380 29593 : known_csts[i] = lat->values->value;
5381 29593 : continue;
5382 : }
5383 :
5384 530673 : struct cgraph_edge *cs;
5385 530673 : tree newval = NULL_TREE;
5386 530673 : int j;
5387 530673 : bool first = true;
5388 530673 : tree type = ipa_get_type (info, i);
5389 :
5390 1524200 : FOR_EACH_VEC_ELT (callers, j, cs)
5391 : {
5392 793562 : struct ipa_jump_func *jump_func;
5393 793562 : tree t;
5394 :
5395 793562 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5396 793562 : if (!args
5397 793562 : || i >= ipa_get_cs_argument_count (args)
5398 1587093 : || (i == 0
5399 182802 : && call_passes_through_thunk (cs)))
5400 : {
5401 : newval = NULL_TREE;
5402 : break;
5403 : }
5404 793483 : jump_func = ipa_get_ith_jump_func (args, i);
5405 :
5406 : /* Besides simple pass-through jump function, arithmetic jump
5407 : function could also introduce argument-direct-pass-through for
5408 : self-feeding recursive call. For example,
5409 :
5410 : fn (int i)
5411 : {
5412 : fn (i & 1);
5413 : }
5414 :
5415 : Given that i is 0, recursive propagation via (i & 1) also gets
5416 : 0. */
5417 793483 : if (self_recursive_pass_through_p (cs, jump_func, i, false))
5418 : {
5419 18702 : gcc_assert (newval);
5420 18702 : enum tree_code opcode
5421 18702 : = ipa_get_jf_pass_through_operation (jump_func);
5422 18702 : tree op_type = (opcode == NOP_EXPR) ? NULL_TREE
5423 49 : : ipa_get_jf_pass_through_op_type (jump_func);
5424 18702 : t = ipa_get_jf_arith_result (opcode, newval,
5425 : ipa_get_jf_pass_through_operand (jump_func),
5426 : op_type);
5427 18702 : t = ipacp_value_safe_for_type (type, t);
5428 : }
5429 774781 : else if (self_recursive_ancestor_p (cs, jump_func, i))
5430 0 : continue;
5431 : else
5432 774781 : t = ipa_value_from_jfunc (ipa_node_params_sum->get (cs->caller),
5433 : jump_func, type);
5434 793483 : if (!t
5435 484619 : || (newval
5436 257950 : && !values_equal_for_ipcp_p (t, newval))
5437 1256337 : || (!first && !newval))
5438 : {
5439 : newval = NULL_TREE;
5440 : break;
5441 : }
5442 : else
5443 : newval = t;
5444 : first = false;
5445 : }
5446 :
5447 530673 : if (newval)
5448 199965 : known_csts[i] = newval;
5449 : }
5450 168980 : }
5451 :
5452 : /* Given a NODE and a subset of its CALLERS, try to populate plank slots in
5453 : KNOWN_CONTEXTS with polymorphic contexts that are also known for all of the
5454 : CALLERS. */
5455 :
5456 : static void
5457 168980 : find_contexts_for_caller_subset (vec<ipa_polymorphic_call_context>
5458 : &known_contexts,
5459 : ipa_node_params *info,
5460 : const vec<cgraph_edge *> &callers)
5461 : {
5462 168980 : int i, count = ipa_get_param_count (info);
5463 :
5464 738653 : for (i = 0; i < count; i++)
5465 : {
5466 569686 : if (!ipa_is_param_used (info, i))
5467 23594 : continue;
5468 :
5469 547300 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat
5470 547300 : = ipa_get_poly_ctx_lat (info, i);
5471 547300 : if (ctxlat->bottom)
5472 0 : continue;
5473 547300 : if (ctxlat->is_single_const ())
5474 : {
5475 1208 : if (!ctxlat->values->value.useless_p ())
5476 : {
5477 1208 : if (known_contexts.is_empty ())
5478 1110 : known_contexts.safe_grow_cleared (count, true);
5479 1208 : known_contexts[i] = ctxlat->values->value;
5480 : }
5481 1208 : continue;
5482 : }
5483 :
5484 546092 : cgraph_edge *cs;
5485 546092 : ipa_polymorphic_call_context newval;
5486 546092 : bool first = true;
5487 546092 : int j;
5488 :
5489 551709 : FOR_EACH_VEC_ELT (callers, j, cs)
5490 : {
5491 547742 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5492 547742 : if (!args
5493 1095484 : || i >= ipa_get_cs_argument_count (args))
5494 13 : return;
5495 547729 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
5496 547729 : ipa_polymorphic_call_context ctx;
5497 547729 : ctx = ipa_context_from_jfunc (ipa_node_params_sum->get (cs->caller),
5498 : cs, i, jfunc);
5499 547729 : if (first)
5500 : {
5501 546079 : newval = ctx;
5502 546079 : first = false;
5503 : }
5504 : else
5505 1650 : newval.meet_with (ctx);
5506 1092560 : if (newval.useless_p ())
5507 : break;
5508 : }
5509 :
5510 1092158 : if (!newval.useless_p ())
5511 : {
5512 3967 : if (known_contexts.is_empty ())
5513 3693 : known_contexts.safe_grow_cleared (count, true);
5514 3967 : known_contexts[i] = newval;
5515 : }
5516 :
5517 : }
5518 : }
5519 :
5520 : /* Push all aggregate values coming along edge CS for parameter number INDEX to
5521 : RES. If INTERIM is non-NULL, it contains the current interim state of
5522 : collected aggregate values which can be used to compute values passed over
5523 : self-recursive edges.
5524 :
5525 : This basically one iteration of push_agg_values_from_edge over one
5526 : parameter, which allows for simpler early returns. */
5527 :
5528 : static void
5529 637184 : push_agg_values_for_index_from_edge (struct cgraph_edge *cs, int index,
5530 : vec<ipa_argagg_value> *res,
5531 : const ipa_argagg_value_list *interim)
5532 : {
5533 637184 : bool agg_values_from_caller = false;
5534 637184 : bool agg_jf_preserved = false;
5535 637184 : unsigned unit_delta = UINT_MAX;
5536 637184 : int src_idx = -1;
5537 637184 : ipa_jump_func *jfunc = ipa_get_ith_jump_func (ipa_edge_args_sum->get (cs),
5538 : index);
5539 :
5540 637184 : if (jfunc->type == IPA_JF_PASS_THROUGH
5541 637184 : && ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
5542 : {
5543 58452 : agg_values_from_caller = true;
5544 58452 : agg_jf_preserved = ipa_get_jf_pass_through_agg_preserved (jfunc);
5545 58452 : src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
5546 58452 : unit_delta = 0;
5547 : }
5548 578732 : else if (jfunc->type == IPA_JF_ANCESTOR
5549 578732 : && ipa_get_jf_ancestor_agg_preserved (jfunc))
5550 : {
5551 407 : agg_values_from_caller = true;
5552 407 : agg_jf_preserved = true;
5553 407 : src_idx = ipa_get_jf_ancestor_formal_id (jfunc);
5554 407 : unit_delta = ipa_get_jf_ancestor_offset (jfunc) / BITS_PER_UNIT;
5555 : }
5556 :
5557 637184 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
5558 637184 : if (agg_values_from_caller)
5559 : {
5560 58859 : if (caller_info->ipcp_orig_node)
5561 : {
5562 11235 : struct cgraph_node *orig_node = caller_info->ipcp_orig_node;
5563 11235 : ipcp_transformation *ts
5564 11235 : = ipcp_get_transformation_summary (cs->caller);
5565 11235 : ipa_node_params *orig_info = ipa_node_params_sum->get (orig_node);
5566 11235 : ipcp_param_lattices *orig_plats
5567 11235 : = ipa_get_parm_lattices (orig_info, src_idx);
5568 11235 : if (ts
5569 11235 : && orig_plats->aggs
5570 2983 : && (agg_jf_preserved || !orig_plats->aggs_by_ref))
5571 : {
5572 2506 : ipa_argagg_value_list src (ts);
5573 2506 : src.push_adjusted_values (src_idx, index, unit_delta, res);
5574 2506 : return;
5575 : }
5576 : }
5577 : else
5578 : {
5579 47624 : ipcp_param_lattices *src_plats
5580 47624 : = ipa_get_parm_lattices (caller_info, src_idx);
5581 47624 : if (src_plats->aggs
5582 2437 : && !src_plats->aggs_bottom
5583 2437 : && (agg_jf_preserved || !src_plats->aggs_by_ref))
5584 : {
5585 1449 : if (interim && (self_recursive_pass_through_p (cs, jfunc, index)
5586 7 : || self_recursive_ancestor_p (cs, jfunc, index)))
5587 : {
5588 625 : interim->push_adjusted_values (src_idx, index, unit_delta,
5589 : res);
5590 625 : return;
5591 : }
5592 824 : if (!src_plats->aggs_contain_variable)
5593 : {
5594 83 : push_agg_values_from_plats (src_plats, index, unit_delta,
5595 : res);
5596 83 : return;
5597 : }
5598 : }
5599 : }
5600 : }
5601 :
5602 633970 : if (!jfunc->agg.items)
5603 : return;
5604 227010 : bool first = true;
5605 227010 : unsigned prev_unit_offset = 0;
5606 1270693 : for (const ipa_agg_jf_item &agg_jf : *jfunc->agg.items)
5607 : {
5608 1043683 : tree value, srcvalue;
5609 : /* Besides simple pass-through aggregate jump function, arithmetic
5610 : aggregate jump function could also bring same aggregate value as
5611 : parameter passed-in for self-feeding recursive call. For example,
5612 :
5613 : fn (int *i)
5614 : {
5615 : int j = *i & 1;
5616 : fn (&j);
5617 : }
5618 :
5619 : Given that *i is 0, recursive propagation via (*i & 1) also gets 0. */
5620 1043683 : if (interim
5621 361951 : && self_recursive_agg_pass_through_p (cs, &agg_jf, index, false)
5622 1044164 : && (srcvalue = interim->get_value(index,
5623 481 : agg_jf.offset / BITS_PER_UNIT)))
5624 : {
5625 950 : value = ipa_get_jf_arith_result (agg_jf.value.pass_through.operation,
5626 : srcvalue,
5627 475 : agg_jf.value.pass_through.operand,
5628 475 : agg_jf.value.pass_through.op_type);
5629 475 : value = ipacp_value_safe_for_type (agg_jf.type, value);
5630 : }
5631 : else
5632 1043208 : value = ipa_agg_value_from_jfunc (caller_info, cs->caller,
5633 : &agg_jf);
5634 1043683 : if (value)
5635 : {
5636 1017912 : struct ipa_argagg_value iav;
5637 1017912 : iav.value = value;
5638 1017912 : iav.unit_offset = agg_jf.offset / BITS_PER_UNIT;
5639 1017912 : iav.index = index;
5640 1017912 : iav.by_ref = jfunc->agg.by_ref;
5641 1017912 : iav.killed = false;
5642 :
5643 1017912 : gcc_assert (first
5644 : || iav.unit_offset > prev_unit_offset);
5645 1017912 : prev_unit_offset = iav.unit_offset;
5646 1017912 : first = false;
5647 :
5648 1017912 : res->safe_push (iav);
5649 : }
5650 : }
5651 : return;
5652 : }
5653 :
5654 : /* Push all aggregate values coming along edge CS to RES. DEST_INFO is the
5655 : description of ultimate callee of CS or the one it was cloned from (the
5656 : summary where lattices are). If INTERIM is non-NULL, it contains the
5657 : current interim state of collected aggregate values which can be used to
5658 : compute values passed over self-recursive edges (if OPTIMIZE_SELF_RECURSION
5659 : is true) and to skip values which clearly will not be part of intersection
5660 : with INTERIM. */
5661 :
5662 : static void
5663 225804 : push_agg_values_from_edge (struct cgraph_edge *cs,
5664 : ipa_node_params *dest_info,
5665 : vec<ipa_argagg_value> *res,
5666 : const ipa_argagg_value_list *interim,
5667 : bool optimize_self_recursion)
5668 : {
5669 225804 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5670 225804 : if (!args)
5671 : return;
5672 :
5673 451608 : int count = MIN (ipa_get_param_count (dest_info),
5674 : ipa_get_cs_argument_count (args));
5675 :
5676 225804 : unsigned interim_index = 0;
5677 946744 : for (int index = 0; index < count; index++)
5678 : {
5679 720940 : if (interim)
5680 : {
5681 297670 : while (interim_index < interim->m_elts.size ()
5682 272173 : && interim->m_elts[interim_index].value
5683 523847 : && interim->m_elts[interim_index].index < index)
5684 146391 : interim_index++;
5685 209150 : if (interim_index >= interim->m_elts.size ()
5686 151279 : || interim->m_elts[interim_index].index > index)
5687 57871 : continue;
5688 : }
5689 :
5690 663069 : ipcp_param_lattices *plats = ipa_get_parm_lattices (dest_info, index);
5691 663069 : if (!ipa_is_param_used (dest_info, index)
5692 663069 : || plats->aggs_bottom)
5693 25885 : continue;
5694 637233 : push_agg_values_for_index_from_edge (cs, index, res,
5695 : optimize_self_recursion ? interim
5696 : : NULL);
5697 : }
5698 : }
5699 :
5700 :
5701 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5702 : from all of them into INTERIM. Return how many there are. */
5703 :
5704 : static unsigned int
5705 168980 : find_aggregate_values_for_callers_subset_1 (vec<ipa_argagg_value> &interim,
5706 : struct cgraph_node *node,
5707 : const vec<cgraph_edge *> &callers)
5708 : {
5709 168980 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5710 168980 : if (dest_info->ipcp_orig_node)
5711 0 : dest_info = ipa_node_params_sum->get (dest_info->ipcp_orig_node);
5712 :
5713 : /* gather_edges_for_value puts a non-recursive call into the first element of
5714 : callers if it can. */
5715 168980 : push_agg_values_from_edge (callers[0], dest_info, &interim, NULL, true);
5716 :
5717 168980 : unsigned valid_entries = interim.length ();
5718 168980 : if (!valid_entries)
5719 : return 0;
5720 :
5721 86668 : unsigned caller_count = callers.length();
5722 141768 : for (unsigned i = 1; i < caller_count; i++)
5723 : {
5724 56778 : auto_vec<ipa_argagg_value, 32> last;
5725 56778 : ipa_argagg_value_list avs (&interim);
5726 56778 : push_agg_values_from_edge (callers[i], dest_info, &last, &avs, true);
5727 :
5728 56778 : valid_entries = intersect_argaggs_with (interim, last);
5729 56778 : if (!valid_entries)
5730 1678 : return 0;
5731 56778 : }
5732 :
5733 : return valid_entries;
5734 : }
5735 :
5736 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5737 : from all of them and return them in a garbage-collected vector. Return
5738 : nullptr if there are none. */
5739 :
5740 : static void
5741 154101 : find_aggregate_values_for_callers_subset (vec<ipa_argagg_value> &res,
5742 : struct cgraph_node *node,
5743 : const vec<cgraph_edge *> &callers)
5744 : {
5745 154101 : auto_vec<ipa_argagg_value, 32> interim;
5746 154101 : unsigned valid_entries
5747 154101 : = find_aggregate_values_for_callers_subset_1 (interim, node, callers);
5748 154101 : if (!valid_entries)
5749 : return;
5750 :
5751 880944 : for (const ipa_argagg_value &av : interim)
5752 642042 : if (av.value)
5753 607077 : res.safe_push(av);
5754 : return;
5755 154101 : }
5756 :
5757 : /* Look at edges in CALLERS and collect all known aggregate values that arrive
5758 : from all of them and return them in a garbage-collected vector. Return
5759 : nullptr if there are none. */
5760 :
5761 : static struct vec<ipa_argagg_value, va_gc> *
5762 14879 : find_aggregate_values_for_callers_subset_gc (struct cgraph_node *node,
5763 : const vec<cgraph_edge *> &callers)
5764 : {
5765 14879 : auto_vec<ipa_argagg_value, 32> interim;
5766 14879 : unsigned valid_entries
5767 14879 : = find_aggregate_values_for_callers_subset_1 (interim, node, callers);
5768 14879 : if (!valid_entries)
5769 : return nullptr;
5770 :
5771 5356 : vec<ipa_argagg_value, va_gc> *res = NULL;
5772 5356 : vec_safe_reserve_exact (res, valid_entries);
5773 37655 : for (const ipa_argagg_value &av : interim)
5774 21587 : if (av.value)
5775 20206 : res->quick_push(av);
5776 5356 : gcc_checking_assert (res->length () == valid_entries);
5777 : return res;
5778 14879 : }
5779 :
5780 : /* Determine whether CS also brings all scalar values that the NODE is
5781 : specialized for. */
5782 :
5783 : static bool
5784 87 : cgraph_edge_brings_all_scalars_for_node (struct cgraph_edge *cs,
5785 : struct cgraph_node *node)
5786 : {
5787 87 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5788 87 : int count = ipa_get_param_count (dest_info);
5789 87 : class ipa_node_params *caller_info;
5790 87 : class ipa_edge_args *args;
5791 87 : int i;
5792 :
5793 87 : caller_info = ipa_node_params_sum->get (cs->caller);
5794 87 : args = ipa_edge_args_sum->get (cs);
5795 206 : for (i = 0; i < count; i++)
5796 : {
5797 141 : struct ipa_jump_func *jump_func;
5798 141 : tree val, t;
5799 :
5800 141 : val = dest_info->known_csts[i];
5801 141 : if (!val)
5802 100 : continue;
5803 :
5804 82 : if (i >= ipa_get_cs_argument_count (args))
5805 : return false;
5806 41 : jump_func = ipa_get_ith_jump_func (args, i);
5807 41 : t = ipa_value_from_jfunc (caller_info, jump_func,
5808 : ipa_get_type (dest_info, i));
5809 41 : if (!t || !values_equal_for_ipcp_p (val, t))
5810 : return false;
5811 : }
5812 : return true;
5813 : }
5814 :
5815 : /* Determine whether CS also brings all aggregate values that NODE is
5816 : specialized for. */
5817 :
5818 : static bool
5819 65 : cgraph_edge_brings_all_agg_vals_for_node (struct cgraph_edge *cs,
5820 : struct cgraph_node *node)
5821 : {
5822 65 : ipcp_transformation *ts = ipcp_get_transformation_summary (node);
5823 65 : if (!ts || vec_safe_is_empty (ts->m_agg_values))
5824 : return true;
5825 :
5826 46 : const ipa_argagg_value_list existing (ts->m_agg_values);
5827 46 : auto_vec<ipa_argagg_value, 32> edge_values;
5828 46 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5829 46 : gcc_checking_assert (dest_info->ipcp_orig_node);
5830 46 : dest_info = ipa_node_params_sum->get (dest_info->ipcp_orig_node);
5831 46 : push_agg_values_from_edge (cs, dest_info, &edge_values, &existing, false);
5832 46 : const ipa_argagg_value_list avl (&edge_values);
5833 46 : return avl.superset_of_p (existing);
5834 46 : }
5835 :
5836 : /* Determine whether CS also brings all polymorphic contexts that NODE is
5837 : specialized for. */
5838 :
5839 : static bool
5840 48 : cgraph_edge_brings_all_contexts_for_node (struct cgraph_edge *cs,
5841 : struct cgraph_node *node)
5842 : {
5843 48 : ipa_node_params *dest_info = ipa_node_params_sum->get (node);
5844 48 : unsigned count = dest_info->known_contexts.length ();
5845 15 : if (!count)
5846 : return true;
5847 :
5848 15 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
5849 15 : ipa_edge_args *args = ipa_edge_args_sum->get (cs);
5850 57 : for (unsigned i = 0; i < count; i++)
5851 : {
5852 45 : ipa_polymorphic_call_context ctx = dest_info->known_contexts[i];
5853 45 : if (ctx.useless_p ())
5854 36 : continue;
5855 :
5856 48 : if (!args || i >= (unsigned) ipa_get_cs_argument_count (args))
5857 3 : return false;
5858 24 : ipa_jump_func *jump_func = ipa_get_ith_jump_func (args, i);
5859 24 : ipa_polymorphic_call_context edge_ctx
5860 24 : = ipa_context_from_jfunc (caller_info, cs, i, jump_func);
5861 27 : if (edge_ctx.useless_p ())
5862 : return false;
5863 24 : if (edge_ctx.equal_to (ctx))
5864 15 : continue;
5865 : /* Also accept a context that is more precise than the one NODE has been
5866 : specialized for. */
5867 9 : ctx.combine_with (edge_ctx);
5868 9 : if (!edge_ctx.equal_to (ctx))
5869 : return false;
5870 : }
5871 : return true;
5872 : }
5873 :
5874 : /* Given an original NODE and a VAL for which we have already created a
5875 : specialized clone, look whether there are incoming edges that still lead
5876 : into the old node but now also bring the requested value and also conform to
5877 : all other criteria such that they can be redirected the special node.
5878 : This function can therefore redirect the final edge in a SCC. */
5879 :
5880 : template <typename valtype>
5881 : static void
5882 9239 : perhaps_add_new_callers (cgraph_node *node, ipcp_value<valtype> *val)
5883 : {
5884 : ipcp_value_source<valtype> *src;
5885 9239 : profile_count redirected_sum = profile_count::zero ();
5886 :
5887 123593 : for (src = val->sources; src; src = src->next)
5888 : {
5889 114354 : struct cgraph_edge *cs = src->cs;
5890 355082 : while (cs)
5891 : {
5892 240728 : if (cgraph_edge_brings_value_p (cs, src, node, val)
5893 87 : && cgraph_edge_brings_all_scalars_for_node (cs, val->spec_node)
5894 65 : && cgraph_edge_brings_all_agg_vals_for_node (cs, val->spec_node)
5895 240776 : && cgraph_edge_brings_all_contexts_for_node (cs, val->spec_node))
5896 : {
5897 45 : if (dump_file)
5898 3 : fprintf (dump_file, " - adding an extra caller %s of %s\n",
5899 3 : cs->caller->dump_name (),
5900 3 : val->spec_node->dump_name ());
5901 :
5902 45 : cs->redirect_callee_duplicating_thunks (val->spec_node);
5903 45 : val->spec_node->expand_all_artificial_thunks ();
5904 45 : if (cs->count.ipa ().initialized_p ())
5905 0 : redirected_sum = redirected_sum + cs->count.ipa ();
5906 : }
5907 240728 : cs = get_next_cgraph_edge_clone (cs);
5908 : }
5909 : }
5910 :
5911 9239 : if (redirected_sum.nonzero_p ())
5912 0 : update_specialized_profile (val->spec_node, node, redirected_sum);
5913 9239 : }
5914 :
5915 : /* Return true if KNOWN_CONTEXTS contain at least one useful context. */
5916 :
5917 : static bool
5918 4515 : known_contexts_useful_p (vec<ipa_polymorphic_call_context> known_contexts)
5919 : {
5920 4515 : ipa_polymorphic_call_context *ctx;
5921 4515 : int i;
5922 :
5923 4516 : FOR_EACH_VEC_ELT (known_contexts, i, ctx)
5924 120 : if (!ctx->useless_p ())
5925 : return true;
5926 : return false;
5927 : }
5928 :
5929 : /* Return a copy of KNOWN_CSTS if it is not empty, otherwise return vNULL. */
5930 :
5931 : static vec<ipa_polymorphic_call_context>
5932 4515 : copy_useful_known_contexts (const vec<ipa_polymorphic_call_context> &known_contexts)
5933 : {
5934 4515 : if (known_contexts_useful_p (known_contexts))
5935 118 : return known_contexts.copy ();
5936 : else
5937 4397 : return vNULL;
5938 : }
5939 :
5940 : /* Return true if the VALUE is represented in KNOWN_CSTS at INDEX if OFFSET is
5941 : minus one or in AGGVALS for INDEX and OFFSET otherwise. */
5942 :
5943 : DEBUG_FUNCTION bool
5944 4449 : ipcp_val_replacement_ok_p (vec<tree> &known_csts,
5945 : vec<ipa_polymorphic_call_context> &,
5946 : vec<ipa_argagg_value, va_gc> *aggvals,
5947 : int index, HOST_WIDE_INT offset, tree value)
5948 : {
5949 4449 : tree v;
5950 4449 : if (offset == -1)
5951 3201 : v = known_csts[index];
5952 : else
5953 : {
5954 1248 : const ipa_argagg_value_list avl (aggvals);
5955 1248 : v = avl.get_value (index, offset / BITS_PER_UNIT);
5956 : }
5957 :
5958 4449 : return v && values_equal_for_ipcp_p (v, value);
5959 : }
5960 :
5961 : /* Dump to F all the values in AVALS for which we are re-evaluating the effects
5962 : on the function represented b INFO. */
5963 :
5964 : DEBUG_FUNCTION void
5965 68 : dump_reestimation_message (FILE *f, ipa_node_params *info,
5966 : const ipa_auto_call_arg_values &avals)
5967 : {
5968 68 : fprintf (f, " Re-estimating effects with\n"
5969 : " Scalar constants:");
5970 68 : int param_count = ipa_get_param_count (info);
5971 168 : for (int i = 0; i < param_count; i++)
5972 100 : if (avals.m_known_vals[i])
5973 : {
5974 44 : fprintf (f, " %i:", i);
5975 44 : print_ipcp_constant_value (f, avals.m_known_vals[i]);
5976 : }
5977 68 : fprintf (f, "\n");
5978 68 : if (!avals.m_known_contexts.is_empty ())
5979 : {
5980 0 : fprintf (f, " Pol. contexts:");
5981 0 : for (int i = 0; i < param_count; i++)
5982 0 : if (!avals.m_known_contexts[i].useless_p ())
5983 : {
5984 0 : fprintf (f, " %i:", i);
5985 0 : avals.m_known_contexts[i].dump (f);
5986 : }
5987 0 : fprintf (f, "\n");
5988 : }
5989 68 : if (!avals.m_known_aggs.is_empty ())
5990 : {
5991 24 : fprintf (f, " Aggregate replacements:");
5992 24 : ipa_argagg_value_list avs (&avals);
5993 24 : avs.dump (f);
5994 : }
5995 68 : }
5996 :
5997 : /* Return true if the VALUE is represented in KNOWN_CONTEXTS at INDEX and that
5998 : if OFFSET is is equal to minus one (because source of a polymorphic context
5999 : cannot be an aggregate value). */
6000 :
6001 : DEBUG_FUNCTION bool
6002 66 : ipcp_val_replacement_ok_p (vec<tree> &,
6003 : vec<ipa_polymorphic_call_context> &known_contexts,
6004 : vec<ipa_argagg_value, va_gc> *,
6005 : int index, HOST_WIDE_INT offset,
6006 : ipa_polymorphic_call_context value)
6007 : {
6008 66 : if (offset != -1
6009 66 : || known_contexts.length () <= (unsigned) index
6010 132 : || known_contexts[index].useless_p ())
6011 : return false;
6012 :
6013 66 : if (known_contexts[index].equal_to (value))
6014 : return true;
6015 :
6016 : /* In some corner cases, the final gathering of contexts can figure out that
6017 : the available context is actually more precise than what we wanted to
6018 : clone for. Allow it. */
6019 0 : value.combine_with (known_contexts[index]);
6020 0 : return known_contexts[index].equal_to (value);
6021 : }
6022 :
6023 : /* Decide whether to create a special version of NODE for value VAL of
6024 : parameter at the given INDEX. If OFFSET is -1, the value is for the
6025 : parameter itself, otherwise it is stored at the given OFFSET of the
6026 : parameter. AVALS describes the other already known values. SELF_GEN_CLONES
6027 : is a vector which contains clones created for self-recursive calls with an
6028 : arithmetic pass-through jump function. CUR_SWEEP is the number of the
6029 : current sweep of the call-graph during the decision stage. */
6030 :
6031 : template <typename valtype>
6032 : static bool
6033 228276 : decide_about_value (struct cgraph_node *node, int index, HOST_WIDE_INT offset,
6034 : ipcp_value<valtype> *val,
6035 : vec<cgraph_node *> *self_gen_clones, int cur_sweep)
6036 : {
6037 : int caller_count;
6038 228276 : sreal freq_sum;
6039 : profile_count count_sum, rec_count_sum;
6040 : bool called_without_ipa_profile;
6041 :
6042 228276 : if (val->spec_node)
6043 : {
6044 9239 : perhaps_add_new_callers (node, val);
6045 9239 : return false;
6046 : }
6047 219037 : else if (val->local_size_cost + overall_size > get_max_overall_size (node))
6048 : {
6049 450 : if (dump_file && (dump_flags & TDF_DETAILS))
6050 0 : fprintf (dump_file, " - ignoring candidate value because "
6051 : "maximum unit size would be reached with %li.\n",
6052 : val->local_size_cost + overall_size);
6053 : return false;
6054 : }
6055 218587 : else if (!get_info_about_necessary_edges (val, node, &freq_sum, &caller_count,
6056 : &rec_count_sum, &count_sum,
6057 : &called_without_ipa_profile))
6058 : {
6059 64486 : if (dump_file && (dump_flags & TDF_DETAILS))
6060 : {
6061 121 : fprintf (dump_file, " - skipping candidate value ");
6062 121 : print_ipcp_constant_value (dump_file, val->value);
6063 121 : fprintf (dump_file, " for ");
6064 121 : ipa_dump_param (dump_file, ipa_node_params_sum->get (node), index);
6065 121 : if (offset != -1)
6066 105 : fprintf (dump_file, ", offset: " HOST_WIDE_INT_PRINT_DEC, offset);
6067 121 : fprintf (dump_file, ": no relevant callers\n");
6068 : }
6069 : return false;
6070 : }
6071 :
6072 154101 : if (!dbg_cnt (ipa_cp_values))
6073 : return false;
6074 :
6075 154101 : if (val->self_recursion_generated_p ())
6076 : {
6077 : /* The edge counts in this case might not have been adjusted yet.
6078 : Nevertleless, even if they were it would be only a guesswork which we
6079 : can do now. The recursive part of the counts can be derived from the
6080 : count of the original node anyway. */
6081 293 : if (node->count.ipa ().nonzero_p ())
6082 : {
6083 14 : unsigned dem = self_gen_clones->length () + 1;
6084 14 : rec_count_sum = node->count.ipa () / dem;
6085 : }
6086 : else
6087 265 : rec_count_sum = profile_count::zero ();
6088 : }
6089 :
6090 : /* get_info_about_necessary_edges only sums up ipa counts. */
6091 154101 : count_sum += rec_count_sum;
6092 :
6093 154101 : if (dump_file && (dump_flags & TDF_DETAILS))
6094 : {
6095 135 : fprintf (dump_file, " - considering value ");
6096 135 : print_ipcp_constant_value (dump_file, val->value);
6097 135 : fprintf (dump_file, " for ");
6098 135 : ipa_dump_param (dump_file, ipa_node_params_sum->get (node), index);
6099 135 : if (offset != -1)
6100 62 : fprintf (dump_file, ", offset: " HOST_WIDE_INT_PRINT_DEC, offset);
6101 135 : fprintf (dump_file, " (caller_count: %i)\n", caller_count);
6102 : }
6103 :
6104 154101 : auto_vec<cgraph_edge *> callers
6105 : = gather_edges_for_value (val, node, caller_count);
6106 154101 : ipa_node_params *info = ipa_node_params_sum->get (node);
6107 154101 : ipa_auto_call_arg_values avals;
6108 154101 : avals.m_known_vals.safe_grow_cleared (ipa_get_param_count (info), true);
6109 154101 : find_scalar_values_for_callers_subset (avals.m_known_vals, info, callers);
6110 154101 : find_contexts_for_caller_subset (avals.m_known_contexts, info, callers);
6111 154101 : find_aggregate_values_for_callers_subset (avals.m_known_aggs, node, callers);
6112 :
6113 :
6114 154101 : if (good_cloning_opportunity_p (node, val->prop_time_benefit,
6115 : freq_sum, count_sum, val->prop_size_cost,
6116 : called_without_ipa_profile, cur_sweep))
6117 : ;
6118 : else
6119 : {
6120 : /* Extern inline functions are only meaningful to clione to propagate
6121 : values to their callees. */
6122 152094 : if (DECL_EXTERNAL (node->decl) && DECL_DECLARED_INLINE_P (node->decl))
6123 : {
6124 345 : if (dump_file && (dump_flags & TDF_DETAILS))
6125 0 : fprintf (dump_file, " Skipping extern inline.\n");
6126 149586 : return false;
6127 : }
6128 151749 : if (dump_file && (dump_flags & TDF_DETAILS))
6129 68 : dump_reestimation_message (dump_file, info, avals);
6130 :
6131 151749 : ipa_call_estimates estimates;
6132 151749 : estimate_ipcp_clone_size_and_time (node, &avals, &estimates);
6133 151749 : int removable_params_cost = 0;
6134 978984 : for (tree t : avals.m_known_vals)
6135 523737 : if (t)
6136 209983 : removable_params_cost += estimate_move_cost (TREE_TYPE (t), true);
6137 :
6138 151749 : int size = estimates.size - caller_count * removable_params_cost;
6139 :
6140 151749 : if (size <= 0)
6141 : {
6142 1803 : if (dump_file)
6143 0 : fprintf (dump_file, " Code not going to grow.\n");
6144 : }
6145 : else
6146 : {
6147 149946 : sreal time_benefit
6148 149946 : = ((estimates.nonspecialized_time - estimates.time)
6149 299892 : + hint_time_bonus (node, estimates)
6150 149946 : + (devirtualization_time_bonus (node, &avals)
6151 149946 : + removable_params_cost));
6152 :
6153 149946 : if (!good_cloning_opportunity_p (node, time_benefit, freq_sum,
6154 : count_sum, size,
6155 : called_without_ipa_profile,
6156 : cur_sweep))
6157 149241 : return false;
6158 : }
6159 : }
6160 :
6161 4515 : if (dump_file)
6162 142 : fprintf (dump_file, " Creating a specialized node of %s.\n",
6163 : node->dump_name ());
6164 :
6165 4515 : vec<tree> known_csts = avals.m_known_vals.copy ();
6166 4515 : vec<ipa_polymorphic_call_context> known_contexts
6167 4515 : = copy_useful_known_contexts (avals.m_known_contexts);
6168 :
6169 4515 : vec<ipa_argagg_value, va_gc> *aggvals = NULL;
6170 4515 : vec_safe_reserve_exact (aggvals, avals.m_known_aggs.length ());
6171 24788 : for (const ipa_argagg_value &av : avals.m_known_aggs)
6172 11243 : aggvals->quick_push (av);
6173 4515 : gcc_checking_assert (ipcp_val_replacement_ok_p (known_csts, known_contexts,
6174 : aggvals, index,
6175 : offset, val->value));
6176 4515 : val->spec_node = create_specialized_node (node, known_csts, known_contexts,
6177 : aggvals, callers);
6178 :
6179 4515 : if (val->self_recursion_generated_p ())
6180 152 : self_gen_clones->safe_push (val->spec_node);
6181 : else
6182 4363 : update_profiling_info (node, val->spec_node);
6183 :
6184 4515 : overall_size += val->local_size_cost;
6185 4515 : if (dump_file && (dump_flags & TDF_DETAILS))
6186 68 : fprintf (dump_file, " overall size reached %li\n",
6187 : overall_size);
6188 :
6189 : /* TODO: If for some lattice there is only one other known value
6190 : left, make a special node for it too. */
6191 :
6192 : return true;
6193 154101 : }
6194 :
6195 : /* Like irange::contains_p(), but convert VAL to the range of R if
6196 : necessary. */
6197 :
6198 : static inline bool
6199 48372 : ipa_range_contains_p (const vrange &r, tree val)
6200 : {
6201 48372 : if (r.undefined_p ())
6202 : return false;
6203 :
6204 48372 : tree type = r.type ();
6205 48372 : if (!wi::fits_to_tree_p (wi::to_wide (val), type))
6206 : return false;
6207 :
6208 48372 : val = fold_convert (type, val);
6209 48372 : return r.contains_p (val);
6210 : }
6211 :
6212 : /* Structure holding opportunitties so that they can be pre-sorted. */
6213 :
6214 228276 : struct cloning_opportunity_ranking
6215 : {
6216 : /* A very rough evaluation of likely benefit. */
6217 : sreal eval;
6218 : /* In the case of aggregate constants, a non-negative offset within their
6219 : aggregates. -1 for scalar constants, -2 for polymorphic contexts. */
6220 : HOST_WIDE_INT offset;
6221 : /* The value being considered for evaluation for cloning. */
6222 : ipcp_value_base *val;
6223 : /* Index of the formal parameter the value is coming in. */
6224 : int index;
6225 : };
6226 :
6227 : /* Helper function to qsort a vector of cloning opportunities. */
6228 :
6229 : static int
6230 2198275 : compare_cloning_opportunities (const void *a, const void *b)
6231 : {
6232 2198275 : const cloning_opportunity_ranking *o1
6233 : = (const cloning_opportunity_ranking *) a;
6234 2198275 : const cloning_opportunity_ranking *o2
6235 : = (const cloning_opportunity_ranking *) b;
6236 2198275 : if (o1->eval < o2->eval)
6237 : return 1;
6238 1714036 : if (o1->eval > o2->eval)
6239 566000 : return -1;
6240 : return 0;
6241 : }
6242 :
6243 : /* Use the estimations in VAL to determine how good a candidate it represents
6244 : for the purposes of ordering real evaluation of opportunities (which
6245 : includes information about incoming edges, among other things). */
6246 :
6247 : static sreal
6248 228276 : cloning_opportunity_ranking_evaluation (const ipcp_value_base *val)
6249 : {
6250 228276 : sreal e1 = (val->local_time_benefit * 1000) / MAX (val->local_size_cost, 1);
6251 228276 : sreal e2 = (val->prop_time_benefit * 1000) / MAX (val->prop_size_cost, 1);
6252 228276 : if (e2 > e1)
6253 15766 : return e2;
6254 : else
6255 212510 : return e1;
6256 : }
6257 :
6258 : /* Decide whether and what specialized clones of NODE should be created.
6259 : CUR_SWEEP is the number of the current sweep of the call-graph during the
6260 : decision stage. */
6261 :
6262 : static bool
6263 3344477 : decide_whether_version_node (struct cgraph_node *node, int cur_sweep)
6264 : {
6265 3344477 : ipa_node_params *info = ipa_node_params_sum->get (node);
6266 3344477 : int count = ipa_get_param_count (info);
6267 3344477 : bool ret = false;
6268 :
6269 3344477 : if (info->node_dead || count == 0)
6270 : return false;
6271 :
6272 2711328 : bool clone_for_all_contexts = node->local;
6273 2711328 : if (dump_file && (dump_flags & TDF_DETAILS))
6274 : {
6275 345 : fprintf (dump_file, "\nEvaluating opportunities for %s.",
6276 : node->dump_name ());
6277 345 : if (clone_for_all_contexts)
6278 104 : fprintf (dump_file, " Will try to create a special all-context "
6279 : "clone.\n");
6280 345 : fprintf (dump_file, "\n");
6281 : }
6282 :
6283 2711328 : auto_vec <cloning_opportunity_ranking, 32> opp_ranking;
6284 9042564 : for (int i = 0; i < count;i++)
6285 : {
6286 6331236 : if (!ipa_is_param_used (info, i))
6287 : {
6288 708028 : if (dump_file && (dump_flags & TDF_DETAILS))
6289 20 : fprintf (dump_file, " - ignoring unused parameter %i.\n", i);
6290 708028 : continue;
6291 : }
6292 :
6293 5623208 : class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6294 5623208 : ipcp_lattice<tree> *lat = &plats->itself;
6295 5623208 : ipcp_lattice<ipa_polymorphic_call_context> *ctxlat = &plats->ctxlat;
6296 :
6297 5623208 : if (!lat->bottom
6298 5623208 : && (!clone_for_all_contexts || !lat->is_single_const ()))
6299 : {
6300 554188 : ipcp_value<tree> *val;
6301 672427 : for (val = lat->values; val; val = val->next)
6302 : {
6303 : /* If some values generated for self-recursive calls with
6304 : arithmetic jump functions fall outside of the known
6305 : range for the parameter, we can skip them. */
6306 118301 : if (TREE_CODE (val->value) == INTEGER_CST
6307 71598 : && !plats->m_value_range.bottom_p ()
6308 166611 : && !ipa_range_contains_p (plats->m_value_range.m_vr,
6309 : val->value))
6310 : {
6311 : /* This can happen also if a constant present in the source
6312 : code falls outside of the range of parameter's type, so we
6313 : cannot assert. */
6314 62 : if (dump_file && (dump_flags & TDF_DETAILS))
6315 : {
6316 0 : fprintf (dump_file, " - skipping%s value ",
6317 0 : val->self_recursion_generated_p ()
6318 : ? " self_recursion_generated" : "");
6319 0 : print_ipcp_constant_value (dump_file, val->value);
6320 0 : fprintf (dump_file, " because it is outside known "
6321 : "value range.\n");
6322 : }
6323 62 : continue;
6324 : }
6325 118177 : cloning_opportunity_ranking opp;
6326 118177 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6327 118177 : opp.offset = -1;
6328 118177 : opp.val = val;
6329 118177 : opp.index = i;
6330 118177 : opp_ranking.safe_push (opp);
6331 : }
6332 : }
6333 :
6334 5623208 : if (!plats->aggs_bottom)
6335 : {
6336 583666 : struct ipcp_agg_lattice *aglat;
6337 583666 : ipcp_value<tree> *val;
6338 730479 : for (aglat = plats->aggs; aglat; aglat = aglat->next)
6339 145731 : if (!aglat->bottom && aglat->values
6340 : /* If the following is false, the one value will be considered
6341 : for cloning for all contexts. */
6342 270944 : && (!clone_for_all_contexts
6343 78283 : || plats->aggs_contain_variable
6344 200318 : || !aglat->is_single_const ()))
6345 187548 : for (val = aglat->values; val; val = val->next)
6346 : {
6347 106099 : cloning_opportunity_ranking opp;
6348 106099 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6349 106099 : opp.offset = aglat->offset;
6350 106099 : opp.val = val;
6351 106099 : opp.index = i;
6352 106099 : opp_ranking.safe_push (opp);
6353 : }
6354 : }
6355 :
6356 5623208 : if (!ctxlat->bottom
6357 6917942 : && (!clone_for_all_contexts || !ctxlat->is_single_const ()))
6358 : {
6359 567695 : ipcp_value<ipa_polymorphic_call_context> *val;
6360 571695 : for (val = ctxlat->values; val; val = val->next)
6361 8000 : if (!val->value.useless_p ())
6362 : {
6363 4000 : cloning_opportunity_ranking opp;
6364 4000 : opp.eval = cloning_opportunity_ranking_evaluation (val);
6365 4000 : opp.offset = -2;
6366 4000 : opp.val = val;
6367 4000 : opp.index = i;
6368 4000 : opp_ranking.safe_push (opp);
6369 : }
6370 : }
6371 : }
6372 :
6373 2711328 : if (!opp_ranking.is_empty ())
6374 : {
6375 52662 : opp_ranking.qsort (compare_cloning_opportunities);
6376 52662 : auto_vec <cgraph_node *, 9> self_gen_clones;
6377 386262 : for (const cloning_opportunity_ranking &opp : opp_ranking)
6378 228276 : if (opp.offset == -2)
6379 : {
6380 4000 : ipcp_value<ipa_polymorphic_call_context> *val
6381 : = static_cast <ipcp_value<ipa_polymorphic_call_context> *>
6382 : (opp.val);
6383 4000 : ret |= decide_about_value (node, opp.index, -1, val,
6384 : &self_gen_clones, cur_sweep);
6385 : }
6386 : else
6387 : {
6388 224276 : ipcp_value<tree> *val = static_cast<ipcp_value<tree> *> (opp.val);
6389 224276 : ret |= decide_about_value (node, opp.index, opp.offset, val,
6390 : &self_gen_clones, cur_sweep);
6391 : }
6392 :
6393 105324 : if (!self_gen_clones.is_empty ())
6394 : {
6395 31 : self_gen_clones.safe_push (node);
6396 31 : update_counts_for_self_gen_clones (node, self_gen_clones);
6397 : }
6398 52662 : }
6399 :
6400 2711328 : if (!clone_for_all_contexts)
6401 : return ret;
6402 :
6403 238693 : struct caller_statistics stats;
6404 238693 : init_caller_stats (&stats);
6405 238693 : node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
6406 : false);
6407 238693 : if (!stats.n_calls)
6408 : {
6409 16153 : if (dump_file && (dump_flags & TDF_DETAILS))
6410 41 : fprintf (dump_file, " Not cloning for all contexts because "
6411 : "there are no callers of the original node (any more).\n");
6412 : return ret;
6413 : }
6414 :
6415 222540 : ipa_auto_call_arg_values avals;
6416 222540 : int removable_params_cost;
6417 222540 : bool ctx_independent_const
6418 222540 : = gather_context_independent_values (info, &avals, &removable_params_cost);
6419 222540 : sreal devirt_bonus = devirtualization_time_bonus (node, &avals);
6420 430134 : if (ctx_independent_const || devirt_bonus > 0
6421 430134 : || (removable_params_cost && clone_for_param_removal_p (node)))
6422 : {
6423 14946 : if (!dbg_cnt (ipa_cp_values))
6424 67 : return ret;
6425 :
6426 14946 : auto_vec<cgraph_edge *> callers = node->collect_callers ();
6427 56636 : for (int i = callers.length () - 1; i >= 0; i--)
6428 : {
6429 26744 : cgraph_edge *cs = callers[i];
6430 26744 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
6431 :
6432 26744 : if (caller_info && caller_info->node_dead)
6433 2658 : callers.unordered_remove (i);
6434 : }
6435 :
6436 14946 : if (!adjust_callers_for_value_intersection (callers, node))
6437 : /* If node is not called by anyone, or all its caller edges are
6438 : self-recursive, the node is not really in use, no need to do
6439 : cloning. */
6440 67 : return ret;
6441 :
6442 14879 : if (dump_file)
6443 91 : fprintf (dump_file, " Creating a specialized node of %s "
6444 : "for all known contexts.\n", node->dump_name ());
6445 :
6446 14879 : vec<tree> known_csts = vNULL;
6447 14879 : known_csts.safe_grow_cleared (count, true);
6448 14879 : find_scalar_values_for_callers_subset (known_csts, info, callers);
6449 14879 : vec<ipa_polymorphic_call_context> known_contexts = vNULL;
6450 14879 : find_contexts_for_caller_subset (known_contexts, info, callers);
6451 14879 : vec<ipa_argagg_value, va_gc> *aggvals
6452 14879 : = find_aggregate_values_for_callers_subset_gc (node, callers);
6453 :
6454 14879 : struct cgraph_node *clone = create_specialized_node (node, known_csts,
6455 : known_contexts,
6456 : aggvals, callers);
6457 14879 : ipa_node_params_sum->get (clone)->is_all_contexts_clone = true;
6458 14879 : ret = true;
6459 14946 : }
6460 :
6461 : return ret;
6462 2933868 : }
6463 :
6464 : /* Transitively mark all callees of NODE within the same SCC as not dead. */
6465 :
6466 : static void
6467 2234 : spread_undeadness (struct cgraph_node *node)
6468 : {
6469 2234 : struct cgraph_edge *cs;
6470 :
6471 12147 : for (cs = node->callees; cs; cs = cs->next_callee)
6472 9913 : if (ipa_edge_within_scc (cs))
6473 : {
6474 830 : struct cgraph_node *callee;
6475 830 : class ipa_node_params *info;
6476 :
6477 830 : callee = cs->callee->function_symbol (NULL);
6478 830 : info = ipa_node_params_sum->get (callee);
6479 :
6480 830 : if (info && info->node_dead)
6481 : {
6482 68 : info->node_dead = 0;
6483 68 : spread_undeadness (callee);
6484 : }
6485 : }
6486 2234 : }
6487 :
6488 : /* Return true if NODE has a caller from outside of its SCC that is not
6489 : dead. Worker callback for cgraph_for_node_and_aliases. */
6490 :
6491 : static bool
6492 16254 : has_undead_caller_from_outside_scc_p (struct cgraph_node *node,
6493 : void *data ATTRIBUTE_UNUSED)
6494 : {
6495 16254 : struct cgraph_edge *cs;
6496 :
6497 81551 : for (cs = node->callers; cs; cs = cs->next_caller)
6498 65750 : if (cs->caller->thunk
6499 65750 : && cs->caller->call_for_symbol_thunks_and_aliases
6500 0 : (has_undead_caller_from_outside_scc_p, NULL, true))
6501 : return true;
6502 65750 : else if (!ipa_edge_within_scc (cs))
6503 : {
6504 65510 : ipa_node_params *caller_info = ipa_node_params_sum->get (cs->caller);
6505 65510 : if (!caller_info /* Unoptimized caller are like dead ones. */
6506 65508 : || !caller_info->node_dead)
6507 : return true;
6508 : }
6509 : return false;
6510 : }
6511 :
6512 :
6513 : /* Identify nodes within the same SCC as NODE which are no longer needed
6514 : because of new clones and will be removed as unreachable. */
6515 :
6516 : static void
6517 17387 : identify_dead_nodes (struct cgraph_node *node)
6518 : {
6519 17387 : struct cgraph_node *v;
6520 35045 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6521 17658 : if (v->local)
6522 : {
6523 16007 : ipa_node_params *info = ipa_node_params_sum->get (v);
6524 16007 : if (info
6525 32014 : && !v->call_for_symbol_thunks_and_aliases
6526 16007 : (has_undead_caller_from_outside_scc_p, NULL, true))
6527 15554 : info->node_dead = 1;
6528 : }
6529 :
6530 35045 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6531 : {
6532 17658 : ipa_node_params *info = ipa_node_params_sum->get (v);
6533 17658 : if (info && !info->node_dead)
6534 2166 : spread_undeadness (v);
6535 : }
6536 :
6537 17387 : if (dump_file && (dump_flags & TDF_DETAILS))
6538 : {
6539 107 : for (v = node; v; v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6540 55 : if (ipa_node_params_sum->get (v)
6541 55 : && ipa_node_params_sum->get (v)->node_dead)
6542 32 : fprintf (dump_file, " Marking node as dead: %s.\n",
6543 : v->dump_name ());
6544 : }
6545 17387 : }
6546 :
6547 : /* Removes all useless callback edges from the callgraph. Useless callback
6548 : edges might mess up the callgraph, because they might be impossible to
6549 : redirect and so on, leading to crashes. Their usefulness is evaluated
6550 : through callback_edge_useful_p. */
6551 :
6552 : static void
6553 131920 : purge_useless_callback_edges ()
6554 : {
6555 131920 : if (dump_file)
6556 162 : fprintf (dump_file, "\nPurging useless callback edges:\n");
6557 :
6558 131920 : cgraph_edge *e;
6559 131920 : cgraph_node *node;
6560 1473441 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6561 : {
6562 7013339 : for (e = node->callees; e; e = e->next_callee)
6563 : {
6564 5671818 : if (e->has_callback)
6565 : {
6566 13673 : if (dump_file)
6567 6 : fprintf (dump_file, "\tExamining callbacks of edge %s -> %s:\n",
6568 6 : e->caller->dump_name (), e->callee->dump_name ());
6569 13673 : if (!lookup_attribute ("callback_only",
6570 13673 : DECL_ATTRIBUTES (e->callee->decl))
6571 13673 : && !callback_is_special_cased (e->callee->decl, e->call_stmt))
6572 : {
6573 2 : if (dump_file)
6574 0 : fprintf (
6575 : dump_file,
6576 : "\t\tPurging callbacks, because the callback-dispatching"
6577 : "function no longer has any callback attributes.\n");
6578 2 : e->purge_callback_edges ();
6579 2 : continue;
6580 : }
6581 13671 : cgraph_edge *cbe, *next;
6582 27344 : for (cbe = e->first_callback_edge (); cbe; cbe = next)
6583 : {
6584 13673 : next = cbe->next_callback_edge ();
6585 13673 : if (!callback_edge_useful_p (cbe))
6586 : {
6587 13457 : if (dump_file)
6588 4 : fprintf (dump_file,
6589 : "\t\tCallback edge %s -> %s not deemed "
6590 : "useful, removing.\n",
6591 4 : cbe->caller->dump_name (),
6592 4 : cbe->callee->dump_name ());
6593 13457 : cgraph_edge::remove (cbe);
6594 : }
6595 : else
6596 : {
6597 216 : if (dump_file)
6598 4 : fprintf (dump_file,
6599 : "\t\tKept callback edge %s -> %s "
6600 : "because it looks useful.\n",
6601 4 : cbe->caller->dump_name (),
6602 4 : cbe->callee->dump_name ());
6603 : }
6604 : }
6605 : }
6606 : }
6607 : }
6608 :
6609 131920 : if (dump_file)
6610 162 : fprintf (dump_file, "\n");
6611 131920 : }
6612 :
6613 : /* The decision stage. Iterate over the topological order of call graph nodes
6614 : TOPO and make specialized clones if deemed beneficial. */
6615 :
6616 : static void
6617 131920 : ipcp_decision_stage (class ipa_topo_info *topo)
6618 : {
6619 131920 : int i;
6620 :
6621 131920 : if (dump_file)
6622 162 : fprintf (dump_file, "\nIPA decision stage (%i sweeps):\n",
6623 : max_number_sweeps);
6624 :
6625 507129 : for (int cur_sweep = 1; cur_sweep <= max_number_sweeps; cur_sweep++)
6626 : {
6627 375209 : if (dump_file && (dump_flags & TDF_DETAILS))
6628 144 : fprintf (dump_file, "\nIPA decision sweep number %i (out of %i):\n",
6629 : cur_sweep, max_number_sweeps);
6630 :
6631 4542843 : for (i = topo->nnodes - 1; i >= 0; i--)
6632 : {
6633 4167634 : struct cgraph_node *node = topo->order[i];
6634 4167634 : bool change = false, iterate = true;
6635 :
6636 8352657 : while (iterate)
6637 : {
6638 : struct cgraph_node *v;
6639 : iterate = false;
6640 4200500 : for (v = node;
6641 8385523 : v;
6642 4200500 : v = ((struct ipa_dfs_info *) v->aux)->next_cycle)
6643 4200500 : if (v->has_gimple_body_p ()
6644 3959636 : && ipcp_versionable_function_p (v)
6645 4200500 : && (cur_sweep
6646 3344477 : <= opt_for_fn (node->decl, param_ipa_cp_sweeps)))
6647 3344477 : iterate |= decide_whether_version_node (v, cur_sweep);
6648 :
6649 4185023 : change |= iterate;
6650 : }
6651 4167634 : if (change)
6652 17387 : identify_dead_nodes (node);
6653 : }
6654 : }
6655 :
6656 : /* Currently, the primary use of callback edges is constant propagation.
6657 : Constant propagation is now over, so we have to remove unused callback
6658 : edges. */
6659 131920 : purge_useless_callback_edges ();
6660 131920 : }
6661 :
6662 : /* Look up all VR and bits information that we have discovered and copy it
6663 : over to the transformation summary. */
6664 :
6665 : static void
6666 131920 : ipcp_store_vr_results (void)
6667 : {
6668 131920 : cgraph_node *node;
6669 :
6670 1473441 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6671 : {
6672 1341521 : ipa_node_params *info = ipa_node_params_sum->get (node);
6673 1341521 : bool dumped_sth = false;
6674 1341521 : bool found_useful_result = false;
6675 1341521 : bool do_vr = true;
6676 1341521 : bool do_bits = true;
6677 :
6678 : /* If the function is not local, the gathered information is only useful
6679 : for clones. */
6680 1341521 : if (!node->local)
6681 1173388 : continue;
6682 :
6683 168133 : if (!info || !opt_for_fn (node->decl, flag_ipa_vrp))
6684 : {
6685 4812 : if (dump_file)
6686 6 : fprintf (dump_file, "Not considering %s for VR discovery "
6687 : "and propagate; -fipa-ipa-vrp: disabled.\n",
6688 : node->dump_name ());
6689 : do_vr = false;
6690 : }
6691 168133 : if (!info || !opt_for_fn (node->decl, flag_ipa_bit_cp))
6692 : {
6693 4786 : if (dump_file)
6694 2 : fprintf (dump_file, "Not considering %s for ipa bitwise "
6695 : "propagation ; -fipa-bit-cp: disabled.\n",
6696 : node->dump_name ());
6697 : do_bits = false;
6698 : }
6699 4786 : if (!do_bits && !do_vr)
6700 4780 : continue;
6701 :
6702 163353 : if (info->ipcp_orig_node)
6703 19207 : info = ipa_node_params_sum->get (info->ipcp_orig_node);
6704 163353 : if (info->lattices.is_empty ())
6705 : /* Newly expanded artificial thunks do not have lattices. */
6706 52248 : continue;
6707 :
6708 111105 : unsigned count = ipa_get_param_count (info);
6709 226048 : for (unsigned i = 0; i < count; i++)
6710 : {
6711 176618 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6712 176618 : if (do_vr
6713 176595 : && !plats->m_value_range.bottom_p ()
6714 236101 : && !plats->m_value_range.top_p ())
6715 : {
6716 : found_useful_result = true;
6717 : break;
6718 : }
6719 117136 : if (do_bits && plats->bits_lattice.constant_p ())
6720 : {
6721 : found_useful_result = true;
6722 : break;
6723 : }
6724 : }
6725 111105 : if (!found_useful_result)
6726 49430 : continue;
6727 :
6728 61675 : ipcp_transformation_initialize ();
6729 61675 : ipcp_transformation *ts = ipcp_transformation_sum->get_create (node);
6730 61675 : vec_safe_reserve_exact (ts->m_vr, count);
6731 :
6732 285413 : for (unsigned i = 0; i < count; i++)
6733 : {
6734 162063 : ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
6735 162063 : ipcp_bits_lattice *bits = NULL;
6736 :
6737 162063 : if (do_bits
6738 162059 : && plats->bits_lattice.constant_p ()
6739 255270 : && dbg_cnt (ipa_cp_bits))
6740 93207 : bits = &plats->bits_lattice;
6741 :
6742 162063 : if (do_vr
6743 162042 : && !plats->m_value_range.bottom_p ()
6744 112564 : && !plats->m_value_range.top_p ()
6745 274627 : && dbg_cnt (ipa_cp_vr))
6746 : {
6747 112564 : if (bits)
6748 : {
6749 87975 : value_range tmp = plats->m_value_range.m_vr;
6750 87975 : tree type = ipa_get_type (info, i);
6751 175950 : irange_bitmask bm (wide_int::from (bits->get_value (),
6752 87975 : TYPE_PRECISION (type),
6753 87975 : TYPE_SIGN (type)),
6754 175950 : wide_int::from (bits->get_mask (),
6755 87975 : TYPE_PRECISION (type),
6756 175950 : TYPE_SIGN (type)));
6757 87975 : tmp.update_bitmask (bm);
6758 : // Reflecting the bitmask on the ranges can sometime
6759 : // produce an UNDEFINED value if the the bitmask update
6760 : // was previously deferred. See PR 120048.
6761 87975 : if (tmp.undefined_p ())
6762 0 : tmp.set_varying (type);
6763 87975 : ipa_vr vr (tmp);
6764 87975 : ts->m_vr->quick_push (vr);
6765 87975 : }
6766 : else
6767 : {
6768 24589 : ipa_vr vr (plats->m_value_range.m_vr);
6769 24589 : ts->m_vr->quick_push (vr);
6770 : }
6771 : }
6772 49499 : else if (bits)
6773 : {
6774 5232 : tree type = ipa_get_type (info, i);
6775 5232 : value_range tmp;
6776 5232 : tmp.set_varying (type);
6777 10464 : irange_bitmask bm (wide_int::from (bits->get_value (),
6778 5232 : TYPE_PRECISION (type),
6779 5232 : TYPE_SIGN (type)),
6780 10464 : wide_int::from (bits->get_mask (),
6781 5232 : TYPE_PRECISION (type),
6782 10464 : TYPE_SIGN (type)));
6783 5232 : tmp.update_bitmask (bm);
6784 : // Reflecting the bitmask on the ranges can sometime
6785 : // produce an UNDEFINED value if the the bitmask update
6786 : // was previously deferred. See PR 120048.
6787 5232 : if (tmp.undefined_p ())
6788 0 : tmp.set_varying (type);
6789 5232 : ipa_vr vr (tmp);
6790 5232 : ts->m_vr->quick_push (vr);
6791 5232 : }
6792 : else
6793 : {
6794 44267 : ipa_vr vr;
6795 44267 : ts->m_vr->quick_push (vr);
6796 : }
6797 :
6798 162063 : if (!dump_file || !bits)
6799 161647 : continue;
6800 :
6801 416 : if (!dumped_sth)
6802 : {
6803 295 : fprintf (dump_file, "Propagated bits info for function %s:\n",
6804 : node->dump_name ());
6805 295 : dumped_sth = true;
6806 : }
6807 416 : fprintf (dump_file, " param %i: value = ", i);
6808 416 : ipcp_print_widest_int (dump_file, bits->get_value ());
6809 416 : fprintf (dump_file, ", mask = ");
6810 416 : ipcp_print_widest_int (dump_file, bits->get_mask ());
6811 416 : fprintf (dump_file, "\n");
6812 : }
6813 : }
6814 131920 : }
6815 :
6816 : /* The IPCP driver. */
6817 :
6818 : static unsigned int
6819 131920 : ipcp_driver (void)
6820 : {
6821 131920 : class ipa_topo_info topo;
6822 :
6823 131920 : if (edge_clone_summaries == NULL)
6824 131920 : edge_clone_summaries = new edge_clone_summary_t (symtab);
6825 :
6826 131920 : ipa_check_create_node_params ();
6827 131920 : ipa_check_create_edge_args ();
6828 131920 : callback_info_sum_t::check_create_info_sum ();
6829 131920 : clone_num_suffixes = new hash_map<const char *, unsigned>;
6830 :
6831 131920 : if (dump_file)
6832 : {
6833 162 : fprintf (dump_file, "\nIPA structures before propagation:\n");
6834 162 : if (dump_flags & TDF_DETAILS)
6835 48 : ipa_print_all_params (dump_file);
6836 162 : ipa_print_all_jump_functions (dump_file);
6837 : }
6838 :
6839 : /* Topological sort. */
6840 131920 : build_toporder_info (&topo);
6841 : /* Do the interprocedural propagation. */
6842 131920 : ipcp_propagate_stage (&topo);
6843 : /* Decide what constant propagation and cloning should be performed. */
6844 131920 : ipcp_decision_stage (&topo);
6845 : /* Store results of value range and bits propagation. */
6846 131920 : ipcp_store_vr_results ();
6847 :
6848 : /* Free all IPCP structures. */
6849 263840 : delete clone_num_suffixes;
6850 131920 : free_toporder_info (&topo);
6851 131920 : delete edge_clone_summaries;
6852 131920 : edge_clone_summaries = NULL;
6853 131920 : ipa_free_all_structures_after_ipa_cp ();
6854 131920 : if (dump_file)
6855 162 : fprintf (dump_file, "\nIPA constant propagation end\n");
6856 131920 : return 0;
6857 : }
6858 :
6859 : /* Initialization and computation of IPCP data structures. This is the initial
6860 : intraprocedural analysis of functions, which gathers information to be
6861 : propagated later on. */
6862 :
6863 : static void
6864 128826 : ipcp_generate_summary (void)
6865 : {
6866 128826 : struct cgraph_node *node;
6867 :
6868 128826 : if (dump_file)
6869 164 : fprintf (dump_file, "\nIPA constant propagation start:\n");
6870 128826 : ipa_register_cgraph_hooks ();
6871 :
6872 1427224 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
6873 1298398 : ipa_analyze_node (node);
6874 :
6875 128826 : varpool_node *vnode;
6876 1813656 : FOR_EACH_STATIC_INITIALIZER (vnode)
6877 1684830 : ipa_analyze_var_static_initializer (vnode);
6878 128826 : }
6879 :
6880 : namespace {
6881 :
6882 : const pass_data pass_data_ipa_cp =
6883 : {
6884 : IPA_PASS, /* type */
6885 : "cp", /* name */
6886 : OPTGROUP_NONE, /* optinfo_flags */
6887 : TV_IPA_CONSTANT_PROP, /* tv_id */
6888 : 0, /* properties_required */
6889 : 0, /* properties_provided */
6890 : 0, /* properties_destroyed */
6891 : 0, /* todo_flags_start */
6892 : ( TODO_dump_symtab | TODO_remove_functions ), /* todo_flags_finish */
6893 : };
6894 :
6895 : class pass_ipa_cp : public ipa_opt_pass_d
6896 : {
6897 : public:
6898 294536 : pass_ipa_cp (gcc::context *ctxt)
6899 : : ipa_opt_pass_d (pass_data_ipa_cp, ctxt,
6900 : ipcp_generate_summary, /* generate_summary */
6901 : NULL, /* write_summary */
6902 : NULL, /* read_summary */
6903 : ipcp_write_transformation_summaries, /*
6904 : write_optimization_summary */
6905 : ipcp_read_transformation_summaries, /*
6906 : read_optimization_summary */
6907 : NULL, /* stmt_fixup */
6908 : 0, /* function_transform_todo_flags_start */
6909 : ipcp_transform_function, /* function_transform */
6910 294536 : NULL) /* variable_transform */
6911 294536 : {}
6912 :
6913 : /* opt_pass methods: */
6914 588179 : bool gate (function *) final override
6915 : {
6916 : /* FIXME: We should remove the optimize check after we ensure we never run
6917 : IPA passes when not optimizing. */
6918 588179 : return (flag_ipa_cp && optimize) || in_lto_p;
6919 : }
6920 :
6921 131920 : unsigned int execute (function *) final override { return ipcp_driver (); }
6922 :
6923 : }; // class pass_ipa_cp
6924 :
6925 : } // anon namespace
6926 :
6927 : ipa_opt_pass_d *
6928 294536 : make_pass_ipa_cp (gcc::context *ctxt)
6929 : {
6930 294536 : return new pass_ipa_cp (ctxt);
6931 : }
6932 :
6933 : /* Reset all state within ipa-cp.cc so that we can rerun the compiler
6934 : within the same process. For use by toplev::finalize. */
6935 :
6936 : void
6937 264460 : ipa_cp_cc_finalize (void)
6938 : {
6939 264460 : overall_size = 0;
6940 264460 : orig_overall_size = 0;
6941 264460 : overall_scaled_runtime = 0;
6942 264460 : ipcp_free_transformation_sum ();
6943 264460 : }
6944 :
6945 : /* Given PARAM which must be a parameter of function FNDECL described by THIS,
6946 : return its index in the DECL_ARGUMENTS chain, using a pre-computed
6947 : DECL_UID-sorted vector if available (which is pre-computed only if there are
6948 : many parameters). Can return -1 if param is static chain not represented
6949 : among DECL_ARGUMENTS. */
6950 :
6951 : int
6952 125848 : ipcp_transformation::get_param_index (const_tree fndecl, const_tree param) const
6953 : {
6954 125848 : gcc_assert (TREE_CODE (param) == PARM_DECL);
6955 125848 : if (m_uid_to_idx)
6956 : {
6957 0 : unsigned puid = DECL_UID (param);
6958 0 : const ipa_uid_to_idx_map_elt *res
6959 0 : = std::lower_bound (m_uid_to_idx->begin(), m_uid_to_idx->end (), puid,
6960 0 : [] (const ipa_uid_to_idx_map_elt &elt, unsigned uid)
6961 : {
6962 0 : return elt.uid < uid;
6963 : });
6964 0 : if (res == m_uid_to_idx->end ()
6965 0 : || res->uid != puid)
6966 : {
6967 0 : gcc_assert (DECL_STATIC_CHAIN (fndecl));
6968 : return -1;
6969 : }
6970 0 : return res->index;
6971 : }
6972 :
6973 125848 : unsigned index = 0;
6974 287522 : for (tree p = DECL_ARGUMENTS (fndecl); p; p = DECL_CHAIN (p), index++)
6975 286038 : if (p == param)
6976 124364 : return (int) index;
6977 :
6978 1484 : gcc_assert (DECL_STATIC_CHAIN (fndecl));
6979 : return -1;
6980 : }
6981 :
6982 : /* Helper function to qsort a vector of ipa_uid_to_idx_map_elt elements
6983 : according to the uid. */
6984 :
6985 : static int
6986 0 : compare_uids (const void *a, const void *b)
6987 : {
6988 0 : const ipa_uid_to_idx_map_elt *e1 = (const ipa_uid_to_idx_map_elt *) a;
6989 0 : const ipa_uid_to_idx_map_elt *e2 = (const ipa_uid_to_idx_map_elt *) b;
6990 0 : if (e1->uid < e2->uid)
6991 : return -1;
6992 0 : if (e1->uid > e2->uid)
6993 : return 1;
6994 0 : gcc_unreachable ();
6995 : }
6996 :
6997 : /* Assuming THIS describes FNDECL and it has sufficiently many parameters to
6998 : justify the overhead, create a DECL_UID-sorted vector to speed up mapping
6999 : from parameters to their indices in DECL_ARGUMENTS chain. */
7000 :
7001 : void
7002 22971 : ipcp_transformation::maybe_create_parm_idx_map (tree fndecl)
7003 : {
7004 22971 : int c = count_formal_params (fndecl);
7005 22971 : if (c < 32)
7006 : return;
7007 :
7008 0 : m_uid_to_idx = NULL;
7009 0 : vec_safe_reserve (m_uid_to_idx, c, true);
7010 0 : unsigned index = 0;
7011 0 : for (tree p = DECL_ARGUMENTS (fndecl); p; p = DECL_CHAIN (p), index++)
7012 : {
7013 0 : ipa_uid_to_idx_map_elt elt;
7014 0 : elt.uid = DECL_UID (p);
7015 0 : elt.index = index;
7016 0 : m_uid_to_idx->quick_push (elt);
7017 : }
7018 0 : m_uid_to_idx->qsort (compare_uids);
7019 : }
|
