Line data Source code
1 : /* Function summary pass.
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* Analysis of function bodies used by inter-procedural passes
22 :
23 : We estimate for each function
24 : - function body size and size after specializing into given context
25 : - average function execution time in a given context
26 : - function frame size
27 : For each call
28 : - call statement size, time and how often the parameters change
29 :
30 : ipa_fn_summary data structures store above information locally (i.e.
31 : parameters of the function itself) and globally (i.e. parameters of
32 : the function created by applying all the inline decisions already
33 : present in the callgraph).
34 :
35 : We provide access to the ipa_fn_summary data structure and
36 : basic logic updating the parameters when inlining is performed.
37 :
38 : The summaries are context sensitive. Context means
39 : 1) partial assignment of known constant values of operands
40 : 2) whether function is inlined into the call or not.
41 : It is easy to add more variants. To represent function size and time
42 : that depends on context (i.e. it is known to be optimized away when
43 : context is known either by inlining or from IP-CP and cloning),
44 : we use predicates.
45 :
46 : estimate_edge_size_and_time can be used to query
47 : function size/time in the given context. ipa_merge_fn_summary_after_inlining merges
48 : properties of caller and callee after inlining.
49 :
50 : Finally pass_inline_parameters is exported. This is used to drive
51 : computation of function parameters used by the early inliner. IPA
52 : inlined performs analysis via its analyze_function method. */
53 :
54 : #include "config.h"
55 : #define INCLUDE_VECTOR
56 : #include "system.h"
57 : #include "coretypes.h"
58 : #include "backend.h"
59 : #include "target.h"
60 : #include "tree.h"
61 : #include "gimple.h"
62 : #include "alloc-pool.h"
63 : #include "tree-pass.h"
64 : #include "ssa.h"
65 : #include "tree-streamer.h"
66 : #include "cgraph.h"
67 : #include "diagnostic.h"
68 : #include "fold-const.h"
69 : #include "print-tree.h"
70 : #include "tree-inline.h"
71 : #include "gimple-pretty-print.h"
72 : #include "cfganal.h"
73 : #include "gimple-iterator.h"
74 : #include "tree-cfg.h"
75 : #include "tree-ssa-loop-niter.h"
76 : #include "tree-ssa-loop.h"
77 : #include "symbol-summary.h"
78 : #include "sreal.h"
79 : #include "ipa-cp.h"
80 : #include "ipa-prop.h"
81 : #include "ipa-fnsummary.h"
82 : #include "cfgloop.h"
83 : #include "tree-scalar-evolution.h"
84 : #include "ipa-utils.h"
85 : #include "cfgexpand.h"
86 : #include "gimplify.h"
87 : #include "stringpool.h"
88 : #include "attribs.h"
89 : #include "tree-into-ssa.h"
90 : #include "symtab-clones.h"
91 : #include "gimple-range.h"
92 : #include "tree-dfa.h"
93 :
94 : /* Summaries. */
95 : fast_function_summary <ipa_fn_summary *, va_gc> *ipa_fn_summaries;
96 : fast_function_summary <ipa_size_summary *, va_heap> *ipa_size_summaries;
97 : fast_call_summary <ipa_call_summary *, va_heap> *ipa_call_summaries;
98 :
99 : /* Edge predicates goes here. */
100 : static object_allocator<ipa_predicate> edge_predicate_pool ("edge predicates");
101 :
102 :
103 : /* Dump IPA hints. */
104 : void
105 185 : ipa_dump_hints (FILE *f, ipa_hints hints)
106 : {
107 185 : if (!hints)
108 : return;
109 148 : fprintf (f, "IPA hints:");
110 148 : if (hints & INLINE_HINT_indirect_call)
111 : {
112 23 : hints &= ~INLINE_HINT_indirect_call;
113 23 : fprintf (f, " indirect_call");
114 : }
115 148 : if (hints & INLINE_HINT_loop_iterations)
116 : {
117 3 : hints &= ~INLINE_HINT_loop_iterations;
118 3 : fprintf (f, " loop_iterations");
119 : }
120 148 : if (hints & INLINE_HINT_loop_stride)
121 : {
122 3 : hints &= ~INLINE_HINT_loop_stride;
123 3 : fprintf (f, " loop_stride");
124 : }
125 148 : if (hints & INLINE_HINT_same_scc)
126 : {
127 4 : hints &= ~INLINE_HINT_same_scc;
128 4 : fprintf (f, " same_scc");
129 : }
130 148 : if (hints & INLINE_HINT_in_scc)
131 : {
132 11 : hints &= ~INLINE_HINT_in_scc;
133 11 : fprintf (f, " in_scc");
134 : }
135 148 : if (hints & INLINE_HINT_cross_module)
136 : {
137 2 : hints &= ~INLINE_HINT_cross_module;
138 2 : fprintf (f, " cross_module");
139 : }
140 148 : if (hints & INLINE_HINT_declared_inline)
141 : {
142 122 : hints &= ~INLINE_HINT_declared_inline;
143 122 : fprintf (f, " declared_inline");
144 : }
145 148 : if (hints & INLINE_HINT_known_hot)
146 : {
147 1 : hints &= ~INLINE_HINT_known_hot;
148 1 : fprintf (f, " known_hot");
149 : }
150 148 : if (hints & INLINE_HINT_builtin_constant_p)
151 : {
152 4 : hints &= ~INLINE_HINT_builtin_constant_p;
153 4 : fprintf (f, " builtin_constant_p");
154 : }
155 148 : gcc_assert (!hints);
156 : }
157 :
158 :
159 : /* Record SIZE and TIME to SUMMARY.
160 : The accounted code will be executed when EXEC_PRED is true.
161 : When NONCONST_PRED is false the code will evaluate to constant and
162 : will get optimized out in specialized clones of the function.
163 : If CALL is true account to call_size_time_table rather than
164 : size_time_table. */
165 :
166 : void
167 134997740 : ipa_fn_summary::account_size_time (int size, sreal time,
168 : const ipa_predicate &exec_pred,
169 : const ipa_predicate &nonconst_pred_in,
170 : bool call)
171 : {
172 134997740 : size_time_entry *e;
173 134997740 : bool found = false;
174 134997740 : int i;
175 134997740 : ipa_predicate nonconst_pred;
176 134997740 : vec<size_time_entry> *table = call ? &call_size_time_table : &size_time_table;
177 :
178 134997740 : if (exec_pred == false)
179 4788460 : return;
180 :
181 134714935 : nonconst_pred = nonconst_pred_in & exec_pred;
182 :
183 134714935 : if (nonconst_pred == false)
184 : return;
185 :
186 : /* We need to create initial empty unconditional clause, but otherwise
187 : we don't need to account empty times and sizes. */
188 157763454 : if (!size && time == 0 && table->length ())
189 3793330 : return;
190 :
191 : /* Only for calls we are unaccounting what we previously recorded. */
192 130209280 : gcc_checking_assert (time >= 0 || call);
193 :
194 469492965 : for (i = 0; table->iterate (i, &e); i++)
195 440275986 : if (e->exec_predicate == exec_pred
196 440275986 : && e->nonconst_predicate == nonconst_pred)
197 : {
198 : found = true;
199 : break;
200 : }
201 130209280 : if (i == max_size_time_table_size)
202 : {
203 0 : i = 0;
204 0 : found = true;
205 0 : e = &(*table)[0];
206 0 : if (dump_file && (dump_flags & TDF_DETAILS))
207 0 : fprintf (dump_file,
208 : "\t\tReached limit on number of entries, "
209 : "ignoring the predicate.");
210 : }
211 130213500 : if (dump_file && (dump_flags & TDF_DETAILS) && (time != 0 || size))
212 : {
213 3955 : fprintf (dump_file,
214 : "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
215 3724 : ((double) size) / ipa_fn_summary::size_scale,
216 : (time.to_double ()), found ? "" : "new ");
217 3724 : exec_pred.dump (dump_file, conds, 0);
218 3724 : if (exec_pred != nonconst_pred)
219 : {
220 84 : fprintf (dump_file, " nonconst:");
221 84 : nonconst_pred.dump (dump_file, conds);
222 : }
223 : else
224 3640 : fprintf (dump_file, "\n");
225 : }
226 130209280 : if (!found)
227 : {
228 29216979 : class size_time_entry new_entry;
229 29216979 : new_entry.size = size;
230 29216979 : new_entry.time = time;
231 29216979 : new_entry.exec_predicate = exec_pred;
232 29216979 : new_entry.nonconst_predicate = nonconst_pred;
233 29216979 : if (call)
234 1339910 : call_size_time_table.safe_push (new_entry);
235 : else
236 27877069 : size_time_table.safe_push (new_entry);
237 : }
238 : else
239 : {
240 100992301 : e->size += size;
241 100992301 : e->time += time;
242 : /* FIXME: PR bootstrap/92653 gcc_checking_assert (e->time >= -1); */
243 : /* Tolerate small roundoff issues. */
244 100992301 : if (e->time < 0)
245 213 : e->time = 0;
246 : }
247 : }
248 :
249 : /* We proved E to be unreachable, redirect it to __builtin_unreachable. */
250 :
251 : static struct cgraph_edge *
252 327060 : redirect_to_unreachable (struct cgraph_edge *e)
253 : {
254 327060 : struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
255 327060 : struct cgraph_node *target
256 327060 : = cgraph_node::get_create (builtin_decl_unreachable ());
257 :
258 327060 : gcc_checking_assert (lookup_attribute ("cold",
259 : DECL_ATTRIBUTES (target->decl)));
260 :
261 327060 : if (e->speculative)
262 311 : e = cgraph_edge::resolve_speculation (e, target->decl);
263 326749 : else if (!e->callee)
264 756 : e = cgraph_edge::make_direct (e, target);
265 : else
266 325993 : e->redirect_callee (target);
267 327060 : class ipa_call_summary *es = ipa_call_summaries->get (e);
268 327060 : e->inline_failed = CIF_UNREACHABLE;
269 327060 : e->count = profile_count::zero ();
270 327060 : es->call_stmt_size = 0;
271 327060 : es->call_stmt_time = 0;
272 327060 : if (callee)
273 0 : callee->remove_symbol_and_inline_clones ();
274 327060 : if (e->has_callback)
275 4 : for (cgraph_edge *cbe = e->first_callback_edge (); cbe;
276 1 : cbe = cbe->next_callback_edge ())
277 : /* If the carrying edge is unreachable, so are the callback calls. */
278 1 : redirect_to_unreachable (cbe);
279 327060 : return e;
280 : }
281 :
282 : /* Set predicate for edge E. */
283 :
284 : static void
285 30798945 : edge_set_predicate (struct cgraph_edge *e, ipa_predicate *predicate)
286 : {
287 : /* If the edge is determined to be never executed, redirect it
288 : to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
289 : be optimized out. */
290 28091414 : if (predicate && *predicate == false
291 : /* When handling speculative edges, we need to do the redirection
292 : just once. Do it always on the direct edge, so we do not
293 : attempt to resolve speculation while duplicating the edge. */
294 31126080 : && (!e->speculative || e->callee))
295 327059 : e = redirect_to_unreachable (e);
296 :
297 30798945 : class ipa_call_summary *es = ipa_call_summaries->get (e);
298 58890359 : if (predicate && *predicate != true)
299 : {
300 4589549 : if (!es->predicate)
301 4091832 : es->predicate = edge_predicate_pool.allocate ();
302 4589549 : *es->predicate = *predicate;
303 : }
304 : else
305 : {
306 26209396 : if (es->predicate)
307 810898 : edge_predicate_pool.remove (es->predicate);
308 26209396 : es->predicate = NULL;
309 : }
310 30798945 : }
311 :
312 : /* Set predicate for hint *P. */
313 :
314 : static void
315 148264 : set_hint_predicate (ipa_predicate **p, ipa_predicate new_predicate)
316 : {
317 148264 : if (new_predicate == false || new_predicate == true)
318 : {
319 2943 : if (*p)
320 0 : edge_predicate_pool.remove (*p);
321 2943 : *p = NULL;
322 : }
323 : else
324 : {
325 145321 : if (!*p)
326 145321 : *p = edge_predicate_pool.allocate ();
327 145321 : **p = new_predicate;
328 : }
329 148264 : }
330 :
331 : /* Find if NEW_PREDICATE is already in V and if so, increment its freq.
332 : Otherwise add a new item to the vector with this predicate and frerq equal
333 : to add_freq, unless the number of predicates would exceed MAX_NUM_PREDICATES
334 : in which case the function does nothing. */
335 :
336 : static void
337 1110363 : add_freqcounting_predicate (vec<ipa_freqcounting_predicate, va_gc> **v,
338 : const ipa_predicate &new_predicate, sreal add_freq,
339 : unsigned max_num_predicates)
340 : {
341 1110363 : if (new_predicate == false || new_predicate == true)
342 : return;
343 : ipa_freqcounting_predicate *f;
344 140091 : for (int i = 0; vec_safe_iterate (*v, i, &f); i++)
345 71647 : if (new_predicate == f->predicate)
346 : {
347 0 : f->freq += add_freq;
348 0 : return;
349 : }
350 94345 : if (vec_safe_length (*v) >= max_num_predicates)
351 : /* Too many different predicates to account for. */
352 : return;
353 :
354 68172 : ipa_freqcounting_predicate fcp;
355 68172 : fcp.predicate = NULL;
356 68172 : set_hint_predicate (&fcp.predicate, new_predicate);
357 68172 : fcp.freq = add_freq;
358 68172 : vec_safe_push (*v, fcp);
359 68172 : return;
360 : }
361 :
362 : /* Compute what conditions may or may not hold given information about
363 : parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
364 : while RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
365 : copy when called in a given context. It is a bitmask of conditions. Bit
366 : 0 means that condition is known to be false, while bit 1 means that condition
367 : may or may not be true. These differs - for example NOT_INLINED condition
368 : is always false in the second and also builtin_constant_p tests cannot use
369 : the fact that parameter is indeed a constant.
370 :
371 : When INLINE_P is true, assume that we are inlining. AVAL contains known
372 : information about argument values. The function does not modify its content
373 : and so AVALs could also be of type ipa_call_arg_values but so far all
374 : callers work with the auto version and so we avoid the conversion for
375 : convenience.
376 :
377 : ERROR_MARK value of an argument means compile time invariant. */
378 :
379 : static void
380 22114402 : evaluate_conditions_for_known_args (struct cgraph_node *node,
381 : bool inline_p,
382 : ipa_auto_call_arg_values *avals,
383 : clause_t *ret_clause,
384 : clause_t *ret_nonspec_clause,
385 : ipa_call_summary *es)
386 : {
387 22114402 : clause_t clause = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
388 22114402 : clause_t nonspec_clause = 1 << ipa_predicate::not_inlined_condition;
389 22114402 : class ipa_fn_summary *info = ipa_fn_summaries->get (node);
390 22114402 : int i;
391 22114402 : struct condition *c;
392 :
393 92070233 : for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
394 : {
395 69955831 : tree val = NULL;
396 69955831 : tree res;
397 69955831 : int j;
398 69955831 : struct expr_eval_op *op;
399 :
400 69955831 : if (c->code == ipa_predicate::not_sra_candidate)
401 : {
402 16345679 : if (!inline_p
403 16345679 : || !es
404 16460585 : || (int)es->param.length () <= c->operand_num
405 24575950 : || !es->param[c->operand_num].points_to_possible_sra_candidate)
406 12175731 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
407 16345679 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
408 69955831 : continue;
409 : }
410 :
411 53610152 : if (c->agg_contents)
412 : {
413 28737584 : if (c->code == ipa_predicate::changed
414 18465182 : && !c->by_ref
415 31258731 : && (avals->safe_sval_at(c->operand_num) == error_mark_node))
416 3558 : continue;
417 :
418 28730468 : if (tree sval = avals->safe_sval_at (c->operand_num))
419 13267358 : val = ipa_find_agg_cst_from_init (sval, c->offset, c->by_ref);
420 13267358 : if (!val)
421 : {
422 28720067 : ipa_argagg_value_list avs (avals);
423 28720067 : val = avs.get_value (c->operand_num, c->offset / BITS_PER_UNIT,
424 28720067 : c->by_ref);
425 : }
426 : }
427 : else
428 : {
429 24876126 : val = avals->safe_sval_at (c->operand_num);
430 24876126 : if (val && val == error_mark_node
431 2357613 : && c->code != ipa_predicate::changed)
432 : val = NULL_TREE;
433 : }
434 :
435 40535848 : if (!val
436 39929526 : && (c->code == ipa_predicate::changed
437 : || c->code == ipa_predicate::is_not_constant))
438 : {
439 25354914 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
440 25354914 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
441 25354914 : continue;
442 : }
443 28251680 : if (c->code == ipa_predicate::changed)
444 : {
445 7737151 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
446 7737151 : continue;
447 : }
448 :
449 20514529 : if (c->code == ipa_predicate::is_not_constant)
450 : {
451 5537 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
452 5537 : continue;
453 : }
454 :
455 20508992 : if (val && TYPE_SIZE (c->type) == TYPE_SIZE (TREE_TYPE (val)))
456 : {
457 5861020 : if (c->type != TREE_TYPE (val))
458 1299485 : val = fold_unary (VIEW_CONVERT_EXPR, c->type, val);
459 6256007 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
460 : {
461 395431 : if (!val)
462 : break;
463 394987 : if (!op->val[0])
464 164677 : val = fold_unary (op->code, op->type, val);
465 230310 : else if (!op->val[1])
466 460620 : val = fold_binary (op->code, op->type,
467 : op->index ? op->val[0] : val,
468 : op->index ? val : op->val[0]);
469 0 : else if (op->index == 0)
470 0 : val = fold_ternary (op->code, op->type,
471 : val, op->val[0], op->val[1]);
472 0 : else if (op->index == 1)
473 0 : val = fold_ternary (op->code, op->type,
474 : op->val[0], val, op->val[1]);
475 0 : else if (op->index == 2)
476 0 : val = fold_ternary (op->code, op->type,
477 : op->val[0], op->val[1], val);
478 : else
479 : val = NULL_TREE;
480 : }
481 :
482 5861020 : res = val
483 5861020 : ? fold_binary_to_constant (c->code, boolean_type_node, val, c->val)
484 : : NULL;
485 :
486 5860556 : if (res && integer_zerop (res))
487 3000790 : continue;
488 2860230 : if (res && integer_onep (res))
489 : {
490 2837541 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
491 2837541 : nonspec_clause
492 2837541 : |= 1 << (i + ipa_predicate::first_dynamic_condition);
493 2837541 : continue;
494 : }
495 : }
496 14670661 : if (c->operand_num < (int) avals->m_known_value_ranges.length ()
497 8512007 : && !c->agg_contents
498 16879905 : && (!val || TREE_CODE (val) != INTEGER_CST))
499 : {
500 2209173 : value_range vr (avals->m_known_value_ranges[c->operand_num]);
501 2209173 : if (!vr.undefined_p ()
502 1465000 : && !vr.varying_p ()
503 3674173 : && (TYPE_SIZE (c->type) == TYPE_SIZE (vr.type ())))
504 : {
505 1464554 : if (!useless_type_conversion_p (c->type, vr.type ()))
506 427 : range_cast (vr, c->type);
507 :
508 1871263 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
509 : {
510 504655 : if (vr.varying_p () || vr.undefined_p ())
511 : break;
512 :
513 406709 : value_range res (op->type);
514 406709 : if (!op->val[0])
515 : {
516 147358 : value_range varying (op->type);
517 147358 : varying.set_varying (op->type);
518 147358 : range_op_handler handler (op->code);
519 147358 : if (!handler
520 147358 : || !res.supports_type_p (op->type)
521 294716 : || !handler.fold_range (res, op->type, vr, varying))
522 0 : res.set_varying (op->type);
523 147358 : }
524 259351 : else if (!op->val[1])
525 : {
526 259179 : value_range op0 (TREE_TYPE (op->val[0]));
527 259179 : range_op_handler handler (op->code);
528 :
529 259179 : ipa_get_range_from_ip_invariant (op0, op->val[0], node);
530 :
531 259179 : if (!handler
532 259179 : || !res.supports_type_p (op->type)
533 518358 : || !handler.fold_range (res, op->type,
534 259179 : op->index ? op0 : vr,
535 517770 : op->index ? vr : op0))
536 0 : res.set_varying (op->type);
537 259179 : }
538 : else
539 172 : res.set_varying (op->type);
540 406709 : vr = res;
541 406709 : }
542 1464554 : if (!vr.varying_p () && !vr.undefined_p ())
543 : {
544 1361135 : int_range<2> res;
545 1361135 : value_range val_vr (TREE_TYPE (c->val));
546 1361135 : range_op_handler handler (c->code);
547 :
548 1361135 : ipa_get_range_from_ip_invariant (val_vr, c->val, node);
549 :
550 1361135 : if (!handler
551 1361135 : || !val_vr.supports_type_p (TREE_TYPE (c->val))
552 2722270 : || !handler.fold_range (res, boolean_type_node, vr, val_vr))
553 0 : res.set_varying (boolean_type_node);
554 :
555 1361135 : if (res.zero_p ())
556 234677 : continue;
557 1361135 : }
558 : }
559 2209173 : }
560 :
561 14435984 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
562 14435984 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
563 : }
564 22114402 : *ret_clause = clause;
565 22114402 : if (ret_nonspec_clause)
566 19215862 : *ret_nonspec_clause = nonspec_clause;
567 22114402 : }
568 :
569 : /* Return true if VRP will be exectued on the function.
570 : We do not want to anticipate optimizations that will not happen.
571 :
572 : FIXME: This can be confused with -fdisable and debug counters and thus
573 : it should not be used for correctness (only to make heuristics work).
574 : This means that inliner should do its own optimizations of expressions
575 : that it predicts to be constant so wrong code can not be triggered by
576 : builtin_constant_p. */
577 :
578 : static bool
579 12308528 : vrp_will_run_p (struct cgraph_node *node)
580 : {
581 12308528 : return (opt_for_fn (node->decl, optimize)
582 12308528 : && !opt_for_fn (node->decl, optimize_debug)
583 24616332 : && opt_for_fn (node->decl, flag_tree_vrp));
584 : }
585 :
586 : /* Similarly about FRE. */
587 :
588 : static bool
589 14593005 : fre_will_run_p (struct cgraph_node *node)
590 : {
591 14593005 : return (opt_for_fn (node->decl, optimize)
592 14593005 : && !opt_for_fn (node->decl, optimize_debug)
593 29184416 : && opt_for_fn (node->decl, flag_tree_fre));
594 : }
595 :
596 : /* Work out what conditions might be true at invocation of E.
597 : Compute costs for inlined edge if INLINE_P is true.
598 :
599 : Return in CLAUSE_PTR the evaluated conditions and in NONSPEC_CLAUSE_PTR
600 : (if non-NULL) conditions evaluated for nonspecialized clone called
601 : in a given context.
602 :
603 : Vectors in AVALS will be populated with useful known information about
604 : argument values - information not known to have any uses will be omitted -
605 : except for m_known_contexts which will only be calculated if
606 : COMPUTE_CONTEXTS is true. */
607 :
608 : void
609 21810562 : evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p,
610 : clause_t *clause_ptr,
611 : clause_t *nonspec_clause_ptr,
612 : ipa_auto_call_arg_values *avals,
613 : bool compute_contexts)
614 : {
615 21810562 : struct cgraph_node *callee = e->callee->ultimate_alias_target ();
616 21810562 : class ipa_fn_summary *info = ipa_fn_summaries->get (callee);
617 21810562 : class ipa_edge_args *args;
618 21810562 : class ipa_call_summary *es = NULL;
619 :
620 21810562 : if (clause_ptr)
621 21810562 : *clause_ptr = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
622 :
623 21810562 : if (ipa_node_params_sum
624 10125299 : && !e->call_stmt_cannot_inline_p
625 10125299 : && (info->conds || compute_contexts)
626 31935861 : && (args = ipa_edge_args_sum->get (e)) != NULL)
627 : {
628 10070808 : struct cgraph_node *caller;
629 10070808 : class ipa_node_params *caller_parms_info, *callee_pi = NULL;
630 10070808 : int i, count = ipa_get_cs_argument_count (args);
631 10070808 : es = ipa_call_summaries->get (e);
632 :
633 10070808 : if (count)
634 : {
635 9374372 : if (e->caller->inlined_to)
636 : caller = e->caller->inlined_to;
637 : else
638 7541810 : caller = e->caller;
639 9374372 : caller_parms_info = ipa_node_params_sum->get (caller);
640 9374372 : callee_pi = ipa_node_params_sum->get (callee);
641 :
642 : /* Watch for thunks. */
643 9374372 : if (callee_pi)
644 : /* Watch for variadic functions. */
645 9374007 : count = MIN (count, ipa_get_param_count (callee_pi));
646 : }
647 :
648 9374007 : if (callee_pi)
649 30739672 : for (i = 0; i < count; i++)
650 : {
651 21365665 : struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i);
652 :
653 21365665 : if (ipa_is_param_used_by_indirect_call (callee_pi, i)
654 21365665 : || ipa_is_param_used_by_ipa_predicates (callee_pi, i))
655 : {
656 : /* Determine if we know constant value of the parameter. */
657 14593005 : tree type = ipa_get_type (callee_pi, i);
658 14593005 : tree cst = ipa_value_from_jfunc (caller_parms_info, jf, type);
659 :
660 11411611 : if (!cst && e->call_stmt
661 25953514 : && i < (int)gimple_call_num_args (e->call_stmt))
662 : {
663 11360509 : cst = gimple_call_arg (e->call_stmt, i);
664 11360509 : if (!is_gimple_min_invariant (cst))
665 : cst = NULL;
666 : }
667 3409660 : if (cst)
668 : {
669 3358558 : gcc_checking_assert (TREE_CODE (cst) != TREE_BINFO);
670 3358558 : if (!avals->m_known_vals.length ())
671 1839132 : avals->m_known_vals.safe_grow_cleared (count, true);
672 3358558 : avals->m_known_vals[i] = cst;
673 : }
674 11234447 : else if (inline_p && !es->param[i].change_prob)
675 : {
676 4210059 : if (!avals->m_known_vals.length ())
677 3493333 : avals->m_known_vals.safe_grow_cleared (count, true);
678 4210059 : avals->m_known_vals[i] = error_mark_node;
679 : }
680 :
681 : /* If we failed to get simple constant, try value range. */
682 3358558 : if ((!cst || TREE_CODE (cst) != INTEGER_CST)
683 12308528 : && vrp_will_run_p (caller)
684 26738796 : && ipa_is_param_used_by_ipa_predicates (callee_pi, i))
685 : {
686 12105647 : value_range vr (type);
687 :
688 12105647 : ipa_value_range_from_jfunc (vr, caller_parms_info, e, jf, type);
689 12105647 : if (!vr.undefined_p () && !vr.varying_p ())
690 : {
691 8668005 : if (!avals->m_known_value_ranges.length ())
692 : {
693 6221197 : avals->m_known_value_ranges.safe_grow_cleared (count,
694 : true);
695 19780844 : for (int i = 0; i < count; ++i)
696 13559647 : avals->m_known_value_ranges[i].set_range_class
697 13559647 : (void_type_node);
698 : }
699 8668005 : avals->m_known_value_ranges[i] = vr;
700 : }
701 12105647 : }
702 :
703 : /* Determine known aggregate values. */
704 14593005 : if (fre_will_run_p (caller))
705 14591007 : ipa_push_agg_values_from_jfunc (caller_parms_info,
706 : caller, &jf->agg, i,
707 : &avals->m_known_aggs);
708 : }
709 :
710 : /* For calls used in polymorphic calls we further determine
711 : polymorphic call context. */
712 21365665 : if (compute_contexts
713 21365665 : && ipa_is_param_used_by_polymorphic_call (callee_pi, i))
714 : {
715 163681 : ipa_polymorphic_call_context
716 163681 : ctx = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
717 327362 : if (!ctx.useless_p ())
718 : {
719 161723 : if (!avals->m_known_contexts.length ())
720 161586 : avals->m_known_contexts.safe_grow_cleared (count, true);
721 161723 : avals->m_known_contexts[i]
722 323446 : = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
723 : }
724 : }
725 : }
726 : else
727 696801 : gcc_assert (!count || callee->thunk);
728 : }
729 11739754 : else if (e->call_stmt && !e->call_stmt_cannot_inline_p && info->conds)
730 : {
731 9215395 : int i, count = (int)gimple_call_num_args (e->call_stmt);
732 :
733 26223089 : for (i = 0; i < count; i++)
734 : {
735 17007694 : tree cst = gimple_call_arg (e->call_stmt, i);
736 17007694 : if (!is_gimple_min_invariant (cst))
737 : cst = NULL;
738 6034371 : if (cst)
739 : {
740 6034371 : if (!avals->m_known_vals.length ())
741 4161733 : avals->m_known_vals.safe_grow_cleared (count, true);
742 6034371 : avals->m_known_vals[i] = cst;
743 : }
744 : }
745 : }
746 :
747 21810562 : evaluate_conditions_for_known_args (callee, inline_p, avals, clause_ptr,
748 : nonspec_clause_ptr, es);
749 21810562 : }
750 :
751 :
752 : /* Allocate the function summary. */
753 :
754 : static void
755 459168 : ipa_fn_summary_alloc (void)
756 : {
757 459168 : gcc_checking_assert (!ipa_fn_summaries);
758 459168 : ipa_size_summaries = new ipa_size_summary_t (symtab);
759 459168 : ipa_fn_summaries = ipa_fn_summary_t::create_ggc (symtab);
760 459168 : ipa_call_summaries = new ipa_call_summary_t (symtab);
761 459168 : }
762 :
763 27052522 : ipa_call_summary::~ipa_call_summary ()
764 : {
765 27052522 : if (predicate)
766 3280934 : edge_predicate_pool.remove (predicate);
767 :
768 27052522 : param.release ();
769 27052522 : }
770 :
771 9946177 : ipa_fn_summary::~ipa_fn_summary ()
772 : {
773 9946177 : unsigned len = vec_safe_length (loop_iterations);
774 10058000 : for (unsigned i = 0; i < len; i++)
775 111823 : edge_predicate_pool.remove ((*loop_iterations)[i].predicate);
776 9946177 : len = vec_safe_length (loop_strides);
777 9979675 : for (unsigned i = 0; i < len; i++)
778 33498 : edge_predicate_pool.remove ((*loop_strides)[i].predicate);
779 9946177 : vec_free (conds);
780 9946177 : call_size_time_table.release ();
781 9946177 : vec_free (loop_iterations);
782 9946177 : vec_free (loop_strides);
783 9946177 : builtin_constant_p_parms.release ();
784 9946177 : }
785 :
786 : void
787 7200751 : ipa_fn_summary_t::remove_callees (cgraph_node *node)
788 : {
789 7200751 : cgraph_edge *e;
790 29597845 : for (e = node->callees; e; e = e->next_callee)
791 22397094 : ipa_call_summaries->remove (e);
792 7677306 : for (e = node->indirect_calls; e; e = e->next_callee)
793 476555 : ipa_call_summaries->remove (e);
794 7200751 : }
795 :
796 : /* Duplicate predicates in loop hint vector, allocating memory for them and
797 : remove and deallocate any uninteresting (true or false) ones. Return the
798 : result. */
799 :
800 : static vec<ipa_freqcounting_predicate, va_gc> *
801 28064 : remap_freqcounting_preds_after_dup (vec<ipa_freqcounting_predicate, va_gc> *v,
802 : clause_t possible_truths)
803 : {
804 28064 : if (vec_safe_length (v) == 0)
805 : return NULL;
806 :
807 4618 : vec<ipa_freqcounting_predicate, va_gc> *res = v->copy ();
808 4618 : int len = res->length();
809 10665 : for (int i = len - 1; i >= 0; i--)
810 : {
811 6047 : ipa_predicate new_predicate
812 6047 : = (*res)[i].predicate->remap_after_duplication (possible_truths);
813 : /* We do not want to free previous predicate; it is used by node
814 : origin. */
815 6047 : (*res)[i].predicate = NULL;
816 6047 : set_hint_predicate (&(*res)[i].predicate, new_predicate);
817 :
818 6047 : if (!(*res)[i].predicate)
819 2943 : res->unordered_remove (i);
820 : }
821 :
822 : return res;
823 : }
824 :
825 :
826 : /* Hook that is called by cgraph.cc when a node is duplicated. */
827 : void
828 2658237 : ipa_fn_summary_t::duplicate (cgraph_node *src,
829 : cgraph_node *dst,
830 : ipa_fn_summary *src_info,
831 : ipa_fn_summary *info)
832 : {
833 2658237 : new (info) ipa_fn_summary (*src_info);
834 : /* TODO: as an optimization, we may avoid copying conditions
835 : that are known to be false or true. */
836 2658237 : info->conds = vec_safe_copy (info->conds);
837 :
838 2658237 : clone_info *cinfo = clone_info::get (dst);
839 : /* When there are any replacements in the function body, see if we can figure
840 : out that something was optimized out. */
841 2658237 : if (ipa_node_params_sum && cinfo && cinfo->tree_map)
842 : {
843 : /* Use SRC parm info since it may not be copied yet. */
844 14032 : ipa_node_params *parms_info = ipa_node_params_sum->get (src);
845 14032 : ipa_auto_call_arg_values avals;
846 14032 : int count = ipa_get_param_count (parms_info);
847 14032 : int i, j;
848 14032 : clause_t possible_truths;
849 14032 : ipa_predicate true_pred = true;
850 14032 : size_time_entry *e;
851 14032 : int optimized_out_size = 0;
852 14032 : bool inlined_to_p = false;
853 14032 : struct cgraph_edge *edge, *next;
854 :
855 14032 : info->size_time_table.release ();
856 14032 : avals.m_known_vals.safe_grow_cleared (count, true);
857 56483 : for (i = 0; i < count; i++)
858 : {
859 : struct ipa_replace_map *r;
860 :
861 138920 : for (j = 0; vec_safe_iterate (cinfo->tree_map, j, &r); j++)
862 : {
863 78311 : if (r->parm_num == i)
864 : {
865 24293 : avals.m_known_vals[i] = r->new_tree;
866 24293 : break;
867 : }
868 : }
869 : }
870 14032 : evaluate_conditions_for_known_args (dst, false,
871 : &avals,
872 : &possible_truths,
873 : /* We are going to specialize,
874 : so ignore nonspec truths. */
875 : NULL,
876 : NULL);
877 :
878 14032 : info->account_size_time (0, 0, true_pred, true_pred);
879 :
880 : /* Remap size_time vectors.
881 : Simplify the predicate by pruning out alternatives that are known
882 : to be false.
883 : TODO: as on optimization, we can also eliminate conditions known
884 : to be true. */
885 96517 : for (i = 0; src_info->size_time_table.iterate (i, &e); i++)
886 : {
887 82485 : ipa_predicate new_exec_pred;
888 82485 : ipa_predicate new_nonconst_pred;
889 82485 : new_exec_pred = e->exec_predicate.remap_after_duplication
890 82485 : (possible_truths);
891 82485 : new_nonconst_pred = e->nonconst_predicate.remap_after_duplication
892 82485 : (possible_truths);
893 82485 : if (new_exec_pred == false || new_nonconst_pred == false)
894 11388 : optimized_out_size += e->size;
895 : else
896 71097 : info->account_size_time (e->size, e->time, new_exec_pred,
897 : new_nonconst_pred);
898 : }
899 :
900 : /* Remap edge predicates with the same simplification as above.
901 : Also copy constantness arrays. */
902 182419 : for (edge = dst->callees; edge; edge = next)
903 : {
904 168387 : ipa_predicate new_predicate;
905 168387 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
906 168387 : next = edge->next_callee;
907 :
908 168387 : if (!edge->inline_failed)
909 0 : inlined_to_p = true;
910 168387 : if (!es->predicate)
911 156546 : continue;
912 11841 : new_predicate = es->predicate->remap_after_duplication
913 11841 : (possible_truths);
914 11841 : if (new_predicate == false && *es->predicate != false)
915 3075 : optimized_out_size += es->call_stmt_size * ipa_fn_summary::size_scale;
916 11841 : edge_set_predicate (edge, &new_predicate);
917 : }
918 :
919 : /* Remap indirect edge predicates with the same simplification as above.
920 : Also copy constantness arrays. */
921 14911 : for (edge = dst->indirect_calls; edge; edge = next)
922 : {
923 879 : ipa_predicate new_predicate;
924 879 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
925 879 : next = edge->next_callee;
926 :
927 879 : gcc_checking_assert (edge->inline_failed);
928 879 : if (!es->predicate)
929 787 : continue;
930 92 : new_predicate = es->predicate->remap_after_duplication
931 92 : (possible_truths);
932 92 : if (new_predicate == false && *es->predicate != false)
933 21 : optimized_out_size
934 21 : += es->call_stmt_size * ipa_fn_summary::size_scale;
935 92 : edge_set_predicate (edge, &new_predicate);
936 : }
937 14032 : info->loop_iterations
938 14032 : = remap_freqcounting_preds_after_dup (info->loop_iterations,
939 : possible_truths);
940 14032 : info->loop_strides
941 14032 : = remap_freqcounting_preds_after_dup (info->loop_strides,
942 : possible_truths);
943 14032 : if (info->builtin_constant_p_parms.length())
944 : {
945 22 : vec <int, va_heap, vl_ptr> parms = info->builtin_constant_p_parms;
946 22 : int ip;
947 22 : info->builtin_constant_p_parms = vNULL;
948 44 : for (i = 0; parms.iterate (i, &ip); i++)
949 22 : if (!avals.m_known_vals[ip])
950 4 : info->builtin_constant_p_parms.safe_push (ip);
951 : }
952 :
953 : /* If inliner or someone after inliner will ever start producing
954 : non-trivial clones, we will get trouble with lack of information
955 : about updating self sizes, because size vectors already contains
956 : sizes of the callees. */
957 14032 : gcc_assert (!inlined_to_p || !optimized_out_size);
958 14032 : }
959 : else
960 : {
961 2644205 : info->size_time_table = src_info->size_time_table.copy ();
962 2644205 : info->loop_iterations = vec_safe_copy (src_info->loop_iterations);
963 2644205 : info->loop_strides = vec_safe_copy (info->loop_strides);
964 :
965 2644205 : info->builtin_constant_p_parms
966 2644205 : = info->builtin_constant_p_parms.copy ();
967 :
968 2644205 : ipa_freqcounting_predicate *f;
969 2692651 : for (int i = 0; vec_safe_iterate (info->loop_iterations, i, &f); i++)
970 : {
971 48446 : ipa_predicate p = *f->predicate;
972 48446 : f->predicate = NULL;
973 48446 : set_hint_predicate (&f->predicate, p);
974 : }
975 2668168 : for (int i = 0; vec_safe_iterate (info->loop_strides, i, &f); i++)
976 : {
977 23963 : ipa_predicate p = *f->predicate;
978 23963 : f->predicate = NULL;
979 23963 : set_hint_predicate (&f->predicate, p);
980 : }
981 : }
982 2658237 : if (!dst->inlined_to)
983 141000 : ipa_update_overall_fn_summary (dst);
984 2658237 : }
985 :
986 :
987 : /* Hook that is called by cgraph.cc when a node is duplicated. */
988 :
989 : void
990 3831528 : ipa_call_summary_t::duplicate (struct cgraph_edge *src,
991 : struct cgraph_edge *dst,
992 : class ipa_call_summary *srcinfo,
993 : class ipa_call_summary *info)
994 : {
995 3831528 : new (info) ipa_call_summary (*srcinfo);
996 3831528 : info->predicate = NULL;
997 3831528 : edge_set_predicate (dst, srcinfo->predicate);
998 3831528 : info->param = srcinfo->param.copy ();
999 3831528 : if (!dst->indirect_unknown_callee && src->indirect_unknown_callee
1000 : /* Don't subtract the size when dealing with callback pairs, since the
1001 : edge has no real size. */
1002 21962 : && !src->has_callback && !dst->callback)
1003 : {
1004 21962 : info->call_stmt_size -= (eni_size_weights.indirect_call_cost
1005 21962 : - eni_size_weights.call_cost);
1006 21962 : info->call_stmt_time -= (eni_time_weights.indirect_call_cost
1007 21962 : - eni_time_weights.call_cost);
1008 : }
1009 3831528 : }
1010 :
1011 : /* Dump edge summaries associated to NODE and recursively to all clones.
1012 : Indent by INDENT. */
1013 :
1014 : static void
1015 1982 : dump_ipa_call_summary (FILE *f, int indent, struct cgraph_node *node,
1016 : class ipa_fn_summary *info)
1017 : {
1018 1982 : struct cgraph_edge *edge;
1019 9804 : for (edge = node->callees; edge; edge = edge->next_callee)
1020 : {
1021 7822 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
1022 7822 : struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
1023 7822 : int i;
1024 :
1025 15644 : fprintf (f,
1026 : "%*s%s %s\n%*s freq:%4.2f",
1027 : indent, "", callee->dump_name (),
1028 7822 : !edge->inline_failed
1029 7269 : ? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
1030 7822 : indent, "", edge->sreal_frequency ().to_double ());
1031 :
1032 7822 : if (cross_module_call_p (edge))
1033 4 : fprintf (f, " cross module");
1034 :
1035 7822 : if (es)
1036 7269 : fprintf (f, " loop depth:%2i size:%2i time: %2i",
1037 : es->loop_depth, es->call_stmt_size, es->call_stmt_time);
1038 :
1039 7822 : ipa_fn_summary *s = ipa_fn_summaries->get (callee);
1040 7822 : ipa_size_summary *ss = ipa_size_summaries->get (callee);
1041 7822 : if (s != NULL)
1042 624 : fprintf (f, " callee size:%2i stack:%2i",
1043 624 : (int) (ss->size / ipa_fn_summary::size_scale),
1044 624 : (int) s->estimated_stack_size);
1045 :
1046 7822 : if (es && es->predicate)
1047 : {
1048 4627 : fprintf (f, " predicate: ");
1049 4627 : es->predicate->dump (f, info->conds);
1050 : }
1051 : else
1052 3195 : fprintf (f, "\n");
1053 7822 : if (es && es->param.exists ())
1054 5340 : for (i = 0; i < (int) es->param.length (); i++)
1055 : {
1056 1580 : int prob = es->param[i].change_prob;
1057 :
1058 1580 : if (!prob)
1059 612 : fprintf (f, "%*s op%i is compile time invariant\n",
1060 : indent + 2, "", i);
1061 968 : else if (prob != REG_BR_PROB_BASE)
1062 35 : fprintf (f, "%*s op%i change %f%% of time\n", indent + 2, "", i,
1063 35 : prob * 100.0 / REG_BR_PROB_BASE);
1064 1580 : if (es->param[i].points_to_local_or_readonly_memory)
1065 400 : fprintf (f, "%*s op%i points to local or readonly memory\n",
1066 : indent + 2, "", i);
1067 1580 : if (es->param[i].points_to_possible_sra_candidate)
1068 227 : fprintf (f, "%*s op%i points to possible sra candidate\n",
1069 : indent + 2, "", i);
1070 : }
1071 7822 : if (!edge->inline_failed)
1072 : {
1073 553 : ipa_size_summary *ss = ipa_size_summaries->get (callee);
1074 553 : fprintf (f, "%*sStack frame offset %i, callee self size %i\n",
1075 : indent + 2, "",
1076 553 : (int) ipa_get_stack_frame_offset (callee),
1077 553 : (int) ss->estimated_self_stack_size);
1078 553 : dump_ipa_call_summary (f, indent + 2, callee, info);
1079 : }
1080 : }
1081 2280 : for (edge = node->indirect_calls; edge; edge = edge->next_callee)
1082 : {
1083 298 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
1084 298 : fprintf (f, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
1085 : " time: %2i",
1086 : indent, "",
1087 : es->loop_depth,
1088 298 : edge->sreal_frequency ().to_double (), es->call_stmt_size,
1089 : es->call_stmt_time);
1090 298 : if (es->predicate)
1091 : {
1092 6 : fprintf (f, "predicate: ");
1093 6 : es->predicate->dump (f, info->conds);
1094 : }
1095 : else
1096 292 : fprintf (f, "\n");
1097 : }
1098 1982 : }
1099 :
1100 :
1101 : void
1102 1641 : ipa_dump_fn_summary (FILE *f, struct cgraph_node *node)
1103 : {
1104 1641 : if (node->definition)
1105 : {
1106 1641 : class ipa_fn_summary *s = ipa_fn_summaries->get (node);
1107 1641 : class ipa_size_summary *ss = ipa_size_summaries->get (node);
1108 1641 : if (s != NULL)
1109 : {
1110 1429 : size_time_entry *e;
1111 1429 : int i;
1112 1429 : fprintf (f, "IPA function summary for %s", node->dump_name ());
1113 1429 : if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
1114 0 : fprintf (f, " always_inline");
1115 1429 : if (s->inlinable)
1116 1242 : fprintf (f, " inlinable");
1117 1429 : if (s->fp_expressions)
1118 44 : fprintf (f, " fp_expression");
1119 1429 : if (s->builtin_constant_p_parms.length ())
1120 : {
1121 2 : fprintf (f, " builtin_constant_p_parms");
1122 4 : for (unsigned int i = 0;
1123 4 : i < s->builtin_constant_p_parms.length (); i++)
1124 2 : fprintf (f, " %i", s->builtin_constant_p_parms[i]);
1125 : }
1126 1429 : fprintf (f, "\n global time: %f\n", s->time.to_double ());
1127 1429 : fprintf (f, " self size: %i\n", ss->self_size);
1128 1429 : fprintf (f, " global size: %i\n", ss->size);
1129 1429 : fprintf (f, " min size: %i\n", s->min_size);
1130 1429 : fprintf (f, " self stack: %i\n",
1131 1429 : (int) ss->estimated_self_stack_size);
1132 1429 : fprintf (f, " global stack: %i\n", (int) s->estimated_stack_size);
1133 1429 : if (s->growth)
1134 162 : fprintf (f, " estimated growth:%i\n", (int) s->growth);
1135 1429 : if (s->scc_no)
1136 5 : fprintf (f, " In SCC: %i\n", (int) s->scc_no);
1137 5947 : for (i = 0; s->size_time_table.iterate (i, &e); i++)
1138 : {
1139 4518 : fprintf (f, " size:%f, time:%f",
1140 4518 : (double) e->size / ipa_fn_summary::size_scale,
1141 : e->time.to_double ());
1142 4518 : if (e->exec_predicate != true)
1143 : {
1144 2636 : fprintf (f, ", executed if:");
1145 2636 : e->exec_predicate.dump (f, s->conds, 0);
1146 : }
1147 4518 : if (e->exec_predicate != e->nonconst_predicate)
1148 : {
1149 1167 : fprintf (f, ", nonconst if:");
1150 1167 : e->nonconst_predicate.dump (f, s->conds, 0);
1151 : }
1152 4518 : fprintf (f, "\n");
1153 : }
1154 : ipa_freqcounting_predicate *fcp;
1155 : bool first_fcp = true;
1156 1535 : for (int i = 0; vec_safe_iterate (s->loop_iterations, i, &fcp); i++)
1157 : {
1158 106 : if (first_fcp)
1159 : {
1160 78 : fprintf (f, " loop iterations:");
1161 78 : first_fcp = false;
1162 : }
1163 106 : fprintf (f, " %3.2f for ", fcp->freq.to_double ());
1164 106 : fcp->predicate->dump (f, s->conds);
1165 : }
1166 : first_fcp = true;
1167 1447 : for (int i = 0; vec_safe_iterate (s->loop_strides, i, &fcp); i++)
1168 : {
1169 18 : if (first_fcp)
1170 : {
1171 10 : fprintf (f, " loop strides:");
1172 10 : first_fcp = false;
1173 : }
1174 18 : fprintf (f, " %3.2f for :", fcp->freq.to_double ());
1175 18 : fcp->predicate->dump (f, s->conds);
1176 : }
1177 1429 : fprintf (f, " calls:\n");
1178 1429 : dump_ipa_call_summary (f, 4, node, s);
1179 1429 : fprintf (f, "\n");
1180 1429 : if (s->target_info)
1181 0 : fprintf (f, " target_info: %x\n", s->target_info);
1182 : }
1183 : else
1184 212 : fprintf (f, "IPA summary for %s is missing.\n", node->dump_name ());
1185 : }
1186 1641 : }
1187 :
1188 : DEBUG_FUNCTION void
1189 0 : ipa_debug_fn_summary (struct cgraph_node *node)
1190 : {
1191 0 : ipa_dump_fn_summary (stderr, node);
1192 0 : }
1193 :
1194 : void
1195 356 : ipa_dump_fn_summaries (FILE *f)
1196 : {
1197 356 : struct cgraph_node *node;
1198 :
1199 2303 : FOR_EACH_DEFINED_FUNCTION (node)
1200 1947 : if (!node->inlined_to)
1201 1394 : ipa_dump_fn_summary (f, node);
1202 356 : }
1203 :
1204 : /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1205 : boolean variable pointed to by DATA. */
1206 :
1207 : static bool
1208 746981 : mark_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED,
1209 : void *data)
1210 : {
1211 746981 : bool *b = (bool *) data;
1212 746981 : *b = true;
1213 746981 : return true;
1214 : }
1215 :
1216 : /* If OP refers to value of function parameter, return the corresponding
1217 : parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
1218 : PARM_DECL) will be stored to *SIZE_P in that case too. */
1219 :
1220 : static tree
1221 183800205 : unmodified_parm_1 (ipa_func_body_info *fbi, gimple *stmt, tree op,
1222 : poly_int64 *size_p)
1223 : {
1224 : /* SSA_NAME referring to parm default def? */
1225 183800205 : if (TREE_CODE (op) == SSA_NAME
1226 107624876 : && SSA_NAME_IS_DEFAULT_DEF (op)
1227 209213063 : && TREE_CODE (SSA_NAME_VAR (op)) == PARM_DECL)
1228 : {
1229 25205271 : if (size_p)
1230 0 : *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
1231 25205271 : return SSA_NAME_VAR (op);
1232 : }
1233 : /* Non-SSA parm reference? */
1234 158594934 : if (TREE_CODE (op) == PARM_DECL
1235 1362409 : && fbi->aa_walk_budget > 0)
1236 : {
1237 1359863 : bool modified = false;
1238 :
1239 1359863 : ao_ref refd;
1240 1359863 : ao_ref_init (&refd, op);
1241 2719726 : int walked = walk_aliased_vdefs (&refd, gimple_vuse (stmt),
1242 : mark_modified, &modified, NULL, NULL,
1243 : fbi->aa_walk_budget);
1244 1359863 : if (walked < 0)
1245 : {
1246 8 : fbi->aa_walk_budget = 0;
1247 882773 : return NULL_TREE;
1248 : }
1249 1359855 : fbi->aa_walk_budget -= walked;
1250 1359855 : if (!modified)
1251 : {
1252 882765 : if (size_p)
1253 0 : *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
1254 882765 : return op;
1255 : }
1256 : }
1257 : return NULL_TREE;
1258 : }
1259 :
1260 : /* If OP refers to value of function parameter, return the corresponding
1261 : parameter. Also traverse chains of SSA register assignments. If non-NULL,
1262 : the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
1263 : stored to *SIZE_P in that case too. */
1264 :
1265 : static tree
1266 120107069 : unmodified_parm (ipa_func_body_info *fbi, gimple *stmt, tree op,
1267 : poly_int64 *size_p)
1268 : {
1269 149425668 : tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
1270 149425668 : if (res)
1271 : return res;
1272 :
1273 124165580 : if (TREE_CODE (op) == SSA_NAME
1274 70493655 : && !SSA_NAME_IS_DEFAULT_DEF (op)
1275 194456262 : && gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1276 29318599 : return unmodified_parm (fbi, SSA_NAME_DEF_STMT (op),
1277 29318599 : gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op)),
1278 29318599 : size_p);
1279 : return NULL_TREE;
1280 : }
1281 :
1282 : /* If OP refers to a value of a function parameter or value loaded from an
1283 : aggregate passed to a parameter (either by value or reference), return TRUE
1284 : and store the number of the parameter to *INDEX_P, the access size into
1285 : *SIZE_P, and information whether and how it has been loaded from an
1286 : aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1287 : statement in which OP is used or loaded. */
1288 :
1289 : static bool
1290 32936856 : unmodified_parm_or_parm_agg_item (struct ipa_func_body_info *fbi,
1291 : gimple *stmt, tree op, int *index_p,
1292 : poly_int64 *size_p,
1293 : struct agg_position_info *aggpos)
1294 : {
1295 34374537 : tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
1296 :
1297 34374537 : gcc_checking_assert (aggpos);
1298 34374537 : if (res)
1299 : {
1300 827948 : *index_p = ipa_get_param_decl_index (fbi->info, res);
1301 827948 : if (*index_p < 0)
1302 : return false;
1303 827818 : aggpos->agg_contents = false;
1304 827818 : aggpos->by_ref = false;
1305 827818 : return true;
1306 : }
1307 :
1308 33546589 : if (TREE_CODE (op) == SSA_NAME)
1309 : {
1310 11925950 : if (SSA_NAME_IS_DEFAULT_DEF (op)
1311 11925950 : || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1312 : return false;
1313 4777316 : stmt = SSA_NAME_DEF_STMT (op);
1314 4777316 : op = gimple_assign_rhs1 (stmt);
1315 4777316 : if (!REFERENCE_CLASS_P (op))
1316 : return unmodified_parm_or_parm_agg_item (fbi, stmt, op, index_p, size_p,
1317 : aggpos);
1318 : }
1319 :
1320 24960274 : aggpos->agg_contents = true;
1321 24960274 : return ipa_load_from_parm_agg (fbi, fbi->info->descriptors,
1322 : stmt, op, index_p, &aggpos->offset,
1323 24960274 : size_p, &aggpos->by_ref);
1324 : }
1325 :
1326 : /* If stmt is simple load or store of value pointed to by a function parmaeter,
1327 : return its index. */
1328 :
1329 : static int
1330 111325356 : load_or_store_of_ptr_parameter (ipa_func_body_info *fbi, gimple *stmt)
1331 : {
1332 111325356 : if (!optimize)
1333 : return -1;
1334 159650380 : gassign *assign = dyn_cast <gassign *> (stmt);
1335 55496176 : if (!assign)
1336 : return -1;
1337 55496176 : tree param;
1338 55496176 : if (gimple_assign_load_p (stmt))
1339 20378901 : param = gimple_assign_rhs1 (stmt);
1340 35117275 : else if (gimple_store_p (stmt))
1341 18623759 : param = gimple_assign_lhs (stmt);
1342 : else
1343 : return -1;
1344 39002660 : tree base = get_base_address (param);
1345 39002660 : if (TREE_CODE (base) != MEM_REF
1346 13473599 : || TREE_CODE (TREE_OPERAND (base, 0)) != SSA_NAME
1347 52472137 : || !SSA_NAME_IS_DEFAULT_DEF (TREE_OPERAND (base, 0)))
1348 : return -1;
1349 7259880 : tree p = SSA_NAME_VAR (TREE_OPERAND (base, 0));
1350 7259880 : if (TREE_CODE (p) != PARM_DECL)
1351 : return -1;
1352 7171152 : return ipa_get_param_decl_index (fbi->info, p);
1353 : }
1354 :
1355 : /* See if statement might disappear after inlining.
1356 : 0 - means not eliminated
1357 : 1 - half of statements goes away
1358 : 2 - for sure it is eliminated.
1359 : We are not terribly sophisticated, basically looking for simple abstraction
1360 : penalty wrappers. */
1361 :
1362 : static int
1363 111325356 : eliminated_by_inlining_prob (ipa_func_body_info *fbi, gimple *stmt)
1364 : {
1365 111325356 : enum gimple_code code = gimple_code (stmt);
1366 111325356 : enum tree_code rhs_code;
1367 :
1368 111325356 : if (!optimize)
1369 : return 0;
1370 :
1371 93168682 : switch (code)
1372 : {
1373 : case GIMPLE_RETURN:
1374 : return 2;
1375 55496176 : case GIMPLE_ASSIGN:
1376 55496176 : if (gimple_num_ops (stmt) != 2)
1377 : return 0;
1378 :
1379 39807799 : rhs_code = gimple_assign_rhs_code (stmt);
1380 :
1381 : /* Casts of parameters, loads from parameters passed by reference
1382 : and stores to return value or parameters are often free after
1383 : inlining due to SRA and further combining.
1384 : Assume that half of statements goes away. */
1385 39807799 : if (CONVERT_EXPR_CODE_P (rhs_code)
1386 : || rhs_code == VIEW_CONVERT_EXPR
1387 : || rhs_code == ADDR_EXPR
1388 37018308 : || gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS)
1389 : {
1390 39002660 : tree rhs = gimple_assign_rhs1 (stmt);
1391 39002660 : tree lhs = gimple_assign_lhs (stmt);
1392 39002660 : tree inner_rhs = get_base_address (rhs);
1393 39002660 : tree inner_lhs = get_base_address (lhs);
1394 39002660 : bool rhs_free = false;
1395 39002660 : bool lhs_free = false;
1396 :
1397 39002660 : if (!inner_rhs)
1398 0 : inner_rhs = rhs;
1399 39002660 : if (!inner_lhs)
1400 0 : inner_lhs = lhs;
1401 :
1402 : /* Reads of parameter are expected to be free. */
1403 39002660 : if (unmodified_parm (fbi, stmt, inner_rhs, NULL))
1404 : rhs_free = true;
1405 : /* Match expressions of form &this->field. Those will most likely
1406 : combine with something upstream after inlining. */
1407 37732029 : else if (TREE_CODE (inner_rhs) == ADDR_EXPR)
1408 : {
1409 2491021 : tree op = get_base_address (TREE_OPERAND (inner_rhs, 0));
1410 2491021 : if (TREE_CODE (op) == PARM_DECL)
1411 : rhs_free = true;
1412 2452920 : else if (TREE_CODE (op) == MEM_REF
1413 2452920 : && unmodified_parm (fbi, stmt, TREE_OPERAND (op, 0),
1414 : NULL))
1415 : rhs_free = true;
1416 : }
1417 :
1418 : /* When parameter is not SSA register because its address is taken
1419 : and it is just copied into one, the statement will be completely
1420 : free after inlining (we will copy propagate backward). */
1421 1308732 : if (rhs_free && is_gimple_reg (lhs))
1422 : return 2;
1423 :
1424 : /* Reads of parameters passed by reference
1425 : expected to be free (i.e. optimized out after inlining). */
1426 38337326 : if (TREE_CODE (inner_rhs) == MEM_REF
1427 38337326 : && unmodified_parm (fbi, stmt, TREE_OPERAND (inner_rhs, 0), NULL))
1428 : rhs_free = true;
1429 :
1430 : /* Copying parameter passed by reference into gimple register is
1431 : probably also going to copy propagate, but we can't be quite
1432 : sure. */
1433 38337326 : if (rhs_free && is_gimple_reg (lhs))
1434 : lhs_free = true;
1435 :
1436 : /* Writes to parameters, parameters passed by value and return value
1437 : (either directly or passed via invisible reference) are free.
1438 :
1439 : TODO: We ought to handle testcase like
1440 : struct a {int a,b;};
1441 : struct a
1442 : returnstruct (void)
1443 : {
1444 : struct a a ={1,2};
1445 : return a;
1446 : }
1447 :
1448 : This translate into:
1449 :
1450 : returnstruct ()
1451 : {
1452 : int a$b;
1453 : int a$a;
1454 : struct a a;
1455 : struct a D.2739;
1456 :
1457 : <bb 2>:
1458 : D.2739.a = 1;
1459 : D.2739.b = 2;
1460 : return D.2739;
1461 :
1462 : }
1463 : For that we either need to copy ipa-split logic detecting writes
1464 : to return value. */
1465 38337326 : if (TREE_CODE (inner_lhs) == PARM_DECL
1466 38249683 : || TREE_CODE (inner_lhs) == RESULT_DECL
1467 75749003 : || (TREE_CODE (inner_lhs) == MEM_REF
1468 5035384 : && (unmodified_parm (fbi, stmt, TREE_OPERAND (inner_lhs, 0),
1469 : NULL)
1470 2603612 : || (TREE_CODE (TREE_OPERAND (inner_lhs, 0)) == SSA_NAME
1471 2602426 : && SSA_NAME_VAR (TREE_OPERAND (inner_lhs, 0))
1472 368058 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1473 : (inner_lhs,
1474 : 0))) == RESULT_DECL))))
1475 : lhs_free = true;
1476 34891497 : if (lhs_free
1477 38337326 : && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs)))
1478 : rhs_free = true;
1479 38337326 : if (lhs_free && rhs_free)
1480 : return 1;
1481 : }
1482 : return 0;
1483 : default:
1484 : return 0;
1485 : }
1486 : }
1487 :
1488 : /* Analyze EXPR if it represents a series of simple operations performed on
1489 : a function parameter and return true if so. FBI, STMT, EXPR, INDEX_P and
1490 : AGGPOS have the same meaning like in unmodified_parm_or_parm_agg_item.
1491 : Type of the parameter or load from an aggregate via the parameter is
1492 : stored in *TYPE_P. Operations on the parameter are recorded to
1493 : PARAM_OPS_P if it is not NULL. */
1494 :
1495 : static bool
1496 27104404 : decompose_param_expr (struct ipa_func_body_info *fbi,
1497 : gimple *stmt, tree expr,
1498 : int *index_p, tree *type_p,
1499 : struct agg_position_info *aggpos,
1500 : expr_eval_ops *param_ops_p = NULL)
1501 : {
1502 27104404 : int op_limit = opt_for_fn (fbi->node->decl, param_ipa_max_param_expr_ops);
1503 27104404 : int op_count = 0;
1504 :
1505 27104404 : if (param_ops_p)
1506 9265450 : *param_ops_p = NULL;
1507 :
1508 32936856 : while (true)
1509 : {
1510 32936856 : expr_eval_op eval_op;
1511 32936856 : unsigned rhs_count;
1512 32936856 : unsigned cst_count = 0;
1513 :
1514 32936856 : if (unmodified_parm_or_parm_agg_item (fbi, stmt, expr, index_p, NULL,
1515 : aggpos))
1516 : {
1517 4574645 : tree type = TREE_TYPE (expr);
1518 :
1519 4574645 : if (aggpos->agg_contents)
1520 : {
1521 : /* Stop if containing bit-field. */
1522 3746827 : if (TREE_CODE (expr) == BIT_FIELD_REF
1523 3746827 : || contains_bitfld_component_ref_p (expr))
1524 : break;
1525 : }
1526 :
1527 4551786 : *type_p = type;
1528 4551786 : return true;
1529 : }
1530 :
1531 28362211 : if (TREE_CODE (expr) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (expr))
1532 : break;
1533 10602242 : stmt = SSA_NAME_DEF_STMT (expr);
1534 :
1535 10602242 : if (gcall *call = dyn_cast <gcall *> (stmt))
1536 : {
1537 2510424 : int flags = gimple_call_return_flags (call);
1538 2510424 : if (!(flags & ERF_RETURNS_ARG))
1539 3085921 : goto fail;
1540 225596 : int arg = flags & ERF_RETURN_ARG_MASK;
1541 225596 : if (arg >= (int)gimple_call_num_args (call))
1542 0 : goto fail;
1543 225596 : expr = gimple_call_arg (stmt, arg);
1544 4022301 : continue;
1545 225596 : }
1546 :
1547 8091818 : if (!is_gimple_assign (stmt = SSA_NAME_DEF_STMT (expr)))
1548 : break;
1549 :
1550 6408025 : switch (gimple_assign_rhs_class (stmt))
1551 : {
1552 3796705 : case GIMPLE_SINGLE_RHS:
1553 3796705 : expr = gimple_assign_rhs1 (stmt);
1554 3796705 : continue;
1555 :
1556 : case GIMPLE_UNARY_RHS:
1557 : rhs_count = 1;
1558 : break;
1559 :
1560 1562964 : case GIMPLE_BINARY_RHS:
1561 1562964 : rhs_count = 2;
1562 1562964 : break;
1563 :
1564 642 : case GIMPLE_TERNARY_RHS:
1565 642 : rhs_count = 3;
1566 642 : break;
1567 :
1568 0 : default:
1569 0 : goto fail;
1570 : }
1571 :
1572 : /* Stop if expression is too complex. */
1573 2611320 : if (op_count++ == op_limit)
1574 : break;
1575 :
1576 2611244 : if (param_ops_p)
1577 : {
1578 2594592 : eval_op.code = gimple_assign_rhs_code (stmt);
1579 2594592 : eval_op.type = TREE_TYPE (gimple_assign_lhs (stmt));
1580 2594592 : eval_op.val[0] = NULL_TREE;
1581 2594592 : eval_op.val[1] = NULL_TREE;
1582 : }
1583 :
1584 : expr = NULL_TREE;
1585 5985014 : for (unsigned i = 0; i < rhs_count; i++)
1586 : {
1587 4174863 : tree op = gimple_op (stmt, i + 1);
1588 :
1589 4174863 : gcc_assert (op && !TYPE_P (op));
1590 4174863 : if (is_gimple_ip_invariant (op))
1591 : {
1592 764808 : if (++cst_count == rhs_count)
1593 1141 : goto fail;
1594 :
1595 763667 : eval_op.val[cst_count - 1] = op;
1596 : }
1597 3410055 : else if (!expr)
1598 : {
1599 : /* Found a non-constant operand, and record its index in rhs
1600 : operands. */
1601 2610103 : eval_op.index = i;
1602 2610103 : expr = op;
1603 : }
1604 : else
1605 : {
1606 : /* Found more than one non-constant operands. */
1607 799952 : goto fail;
1608 : }
1609 : }
1610 :
1611 1810151 : if (param_ops_p)
1612 1793944 : vec_safe_insert (*param_ops_p, 0, eval_op);
1613 : }
1614 :
1615 : /* Failed to decompose, free resource and return. */
1616 22552618 : fail:
1617 22552618 : if (param_ops_p)
1618 9050353 : vec_free (*param_ops_p);
1619 :
1620 : return false;
1621 : }
1622 :
1623 : /* Record to SUMMARY that PARM is used by builtin_constant_p. */
1624 :
1625 : static void
1626 10685 : add_builtin_constant_p_parm (class ipa_fn_summary *summary, int parm)
1627 : {
1628 10685 : int ip;
1629 :
1630 : /* Avoid duplicates. */
1631 10723 : for (unsigned int i = 0;
1632 10723 : summary->builtin_constant_p_parms.iterate (i, &ip); i++)
1633 5151 : if (ip == parm)
1634 10685 : return;
1635 5572 : summary->builtin_constant_p_parms.safe_push (parm);
1636 : }
1637 :
1638 : /* If BB ends by a conditional we can turn into predicates, attach corresponding
1639 : predicates to the CFG edges. */
1640 :
1641 : static void
1642 35029171 : set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
1643 : class ipa_fn_summary *summary,
1644 : class ipa_node_params *params_summary,
1645 : basic_block bb)
1646 : {
1647 35029171 : tree op, op2;
1648 35029171 : int index;
1649 35029171 : struct agg_position_info aggpos;
1650 35029171 : enum tree_code code, inverted_code;
1651 35029171 : edge e;
1652 35029171 : edge_iterator ei;
1653 35029171 : gimple *set_stmt;
1654 35029171 : tree param_type;
1655 35029171 : expr_eval_ops param_ops;
1656 :
1657 70058342 : gcond *last = safe_dyn_cast <gcond *> (*gsi_last_bb (bb));
1658 11606156 : if (!last)
1659 35018034 : return;
1660 11606156 : if (!is_gimple_ip_invariant (gimple_cond_rhs (last)))
1661 : return;
1662 9186907 : op = gimple_cond_lhs (last);
1663 :
1664 9186907 : if (decompose_param_expr (fbi, last, op, &index, ¶m_type, &aggpos,
1665 : ¶m_ops))
1666 : {
1667 1317391 : code = gimple_cond_code (last);
1668 1317391 : inverted_code = invert_tree_comparison (code, HONOR_NANS (op));
1669 :
1670 3952173 : FOR_EACH_EDGE (e, ei, bb->succs)
1671 : {
1672 1317391 : enum tree_code this_code = (e->flags & EDGE_TRUE_VALUE
1673 2634782 : ? code : inverted_code);
1674 : /* invert_tree_comparison will return ERROR_MARK on FP
1675 : comparisons that are not EQ/NE instead of returning proper
1676 : unordered one. Be sure it is not confused with NON_CONSTANT.
1677 :
1678 : And if the edge's target is the final block of diamond CFG graph
1679 : of this conditional statement, we do not need to compute
1680 : predicate for the edge because the final block's predicate must
1681 : be at least as that of the first block of the statement. */
1682 2634782 : if (this_code != ERROR_MARK
1683 2634782 : && !dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1684 : {
1685 2234252 : ipa_predicate p
1686 2234252 : = add_condition (summary, params_summary, index,
1687 : param_type, &aggpos,
1688 : this_code, gimple_cond_rhs (last), param_ops);
1689 2234252 : e->aux = edge_predicate_pool.allocate ();
1690 2234252 : *(ipa_predicate *) e->aux = p;
1691 : }
1692 : }
1693 1317391 : vec_free (param_ops);
1694 1317391 : return;
1695 : }
1696 :
1697 7869516 : if (TREE_CODE (op) != SSA_NAME)
1698 : return;
1699 : /* Special case
1700 : if (builtin_constant_p (op))
1701 : constant_code
1702 : else
1703 : nonconstant_code.
1704 : Here we can predicate nonconstant_code. We can't
1705 : really handle constant_code since we have no predicate
1706 : for this and also the constant code is not known to be
1707 : optimized away when inliner doesn't see operand is constant.
1708 : Other optimizers might think otherwise. */
1709 7868433 : if (gimple_cond_code (last) != NE_EXPR
1710 7868433 : || !integer_zerop (gimple_cond_rhs (last)))
1711 4464286 : return;
1712 3404147 : set_stmt = SSA_NAME_DEF_STMT (op);
1713 3404147 : if (!gimple_call_builtin_p (set_stmt, BUILT_IN_CONSTANT_P)
1714 3404147 : || gimple_call_num_args (set_stmt) != 1)
1715 : return;
1716 27646 : op2 = gimple_call_arg (set_stmt, 0);
1717 27646 : if (!decompose_param_expr (fbi, set_stmt, op2, &index, ¶m_type, &aggpos))
1718 : return;
1719 11137 : if (!aggpos.by_ref)
1720 10646 : add_builtin_constant_p_parm (summary, index);
1721 33411 : FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
1722 : {
1723 11137 : ipa_predicate p = add_condition (summary, params_summary, index,
1724 : param_type, &aggpos,
1725 : ipa_predicate::is_not_constant, NULL_TREE);
1726 11137 : e->aux = edge_predicate_pool.allocate ();
1727 11137 : *(ipa_predicate *) e->aux = p;
1728 : }
1729 : }
1730 :
1731 :
1732 : /* If BB ends by a switch we can turn into predicates, attach corresponding
1733 : predicates to the CFG edges. */
1734 :
1735 : static void
1736 35029171 : set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
1737 : class ipa_fn_summary *summary,
1738 : class ipa_node_params *params_summary,
1739 : basic_block bb)
1740 : {
1741 35029171 : tree op;
1742 35029171 : int index;
1743 35029171 : struct agg_position_info aggpos;
1744 35029171 : edge e;
1745 35029171 : edge_iterator ei;
1746 35029171 : size_t n;
1747 35029171 : size_t case_idx;
1748 35029171 : tree param_type;
1749 35029171 : expr_eval_ops param_ops;
1750 :
1751 70058342 : gswitch *last = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb));
1752 78543 : if (!last)
1753 35000081 : return;
1754 78543 : op = gimple_switch_index (last);
1755 78543 : if (!decompose_param_expr (fbi, last, op, &index, ¶m_type, &aggpos,
1756 : ¶m_ops))
1757 : return;
1758 :
1759 43999 : auto_vec<std::pair<tree, tree> > ranges;
1760 43999 : tree type = TREE_TYPE (op);
1761 43999 : int bound_limit = opt_for_fn (fbi->node->decl,
1762 : param_ipa_max_switch_predicate_bounds);
1763 43999 : int bound_count = 0;
1764 : // This can safely be an integer range, as switches can only hold
1765 : // integers.
1766 43999 : int_range<2> vr;
1767 :
1768 87998 : get_range_query (cfun)->range_of_expr (vr, op);
1769 43999 : if (vr.undefined_p ())
1770 0 : vr.set_varying (TREE_TYPE (op));
1771 43999 : tree vr_min, vr_max;
1772 : // TODO: This entire function could use a rewrite to use the irange
1773 : // API, instead of trying to recreate its intersection/union logic.
1774 : // Any use of get_legacy_range() is a serious code smell.
1775 43999 : value_range_kind vr_type = get_legacy_range (vr, vr_min, vr_max);
1776 43999 : wide_int vr_wmin = wi::to_wide (vr_min);
1777 43999 : wide_int vr_wmax = wi::to_wide (vr_max);
1778 :
1779 306281 : FOR_EACH_EDGE (e, ei, bb->succs)
1780 : {
1781 262282 : e->aux = edge_predicate_pool.allocate ();
1782 262282 : *(ipa_predicate *) e->aux = false;
1783 : }
1784 :
1785 43999 : e = gimple_switch_edge (cfun, last, 0);
1786 : /* Set BOUND_COUNT to maximum count to bypass computing predicate for
1787 : default case if its target basic block is in convergence point of all
1788 : switch cases, which can be determined by checking whether it
1789 : post-dominates the switch statement. */
1790 43999 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1791 12537 : bound_count = INT_MAX;
1792 :
1793 43999 : n = gimple_switch_num_labels (last);
1794 292723 : for (case_idx = 1; case_idx < n; ++case_idx)
1795 : {
1796 248724 : tree cl = gimple_switch_label (last, case_idx);
1797 248724 : tree min = CASE_LOW (cl);
1798 248724 : tree max = CASE_HIGH (cl);
1799 248724 : ipa_predicate p;
1800 :
1801 248724 : e = gimple_switch_edge (cfun, last, case_idx);
1802 :
1803 : /* The case value might not have same type as switch expression,
1804 : extend the value based on the expression type. */
1805 248724 : if (TREE_TYPE (min) != type)
1806 46423 : min = wide_int_to_tree (type, wi::to_wide (min));
1807 :
1808 248724 : if (!max)
1809 : max = min;
1810 11951 : else if (TREE_TYPE (max) != type)
1811 3298 : max = wide_int_to_tree (type, wi::to_wide (max));
1812 :
1813 : /* The case's target basic block is in convergence point of all switch
1814 : cases, its predicate should be at least as that of the switch
1815 : statement. */
1816 248724 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1817 4624 : p = true;
1818 244100 : else if (min == max)
1819 233685 : p = add_condition (summary, params_summary, index, param_type,
1820 : &aggpos, EQ_EXPR, min, param_ops);
1821 : else
1822 : {
1823 10415 : ipa_predicate p1, p2;
1824 10415 : p1 = add_condition (summary, params_summary, index, param_type,
1825 : &aggpos, GE_EXPR, min, param_ops);
1826 10415 : p2 = add_condition (summary, params_summary,index, param_type,
1827 : &aggpos, LE_EXPR, max, param_ops);
1828 10415 : p = p1 & p2;
1829 : }
1830 248724 : *(ipa_predicate *) e->aux
1831 248724 : = p.or_with (summary->conds, *(ipa_predicate *) e->aux);
1832 :
1833 : /* If there are too many disjoint case ranges, predicate for default
1834 : case might become too complicated. So add a limit here. */
1835 248724 : if (bound_count > bound_limit)
1836 96141 : continue;
1837 :
1838 152583 : bool new_range = true;
1839 :
1840 152583 : if (!ranges.is_empty ())
1841 : {
1842 121121 : wide_int curr_wmin = wi::to_wide (min);
1843 121121 : wide_int last_wmax = wi::to_wide (ranges.last ().second);
1844 :
1845 : /* Merge case ranges if they are continuous. */
1846 121121 : if (curr_wmin == last_wmax + 1)
1847 : new_range = false;
1848 40808 : else if (vr_type == VR_ANTI_RANGE)
1849 : {
1850 : /* If two disjoint case ranges can be connected by anti-range
1851 : of switch index, combine them to one range. */
1852 135 : if (wi::lt_p (vr_wmax, curr_wmin - 1, TYPE_SIGN (type)))
1853 : vr_type = VR_UNDEFINED;
1854 0 : else if (wi::le_p (vr_wmin, last_wmax + 1, TYPE_SIGN (type)))
1855 0 : new_range = false;
1856 : }
1857 121121 : }
1858 :
1859 : /* Create/extend a case range. And we count endpoints of range set,
1860 : this number nearly equals to number of conditions that we will create
1861 : for predicate of default case. */
1862 121121 : if (new_range)
1863 : {
1864 72270 : bound_count += (min == max) ? 1 : 2;
1865 72270 : ranges.safe_push (std::make_pair (min, max));
1866 : }
1867 : else
1868 : {
1869 80313 : bound_count += (ranges.last ().first == ranges.last ().second);
1870 80313 : ranges.last ().second = max;
1871 : }
1872 : }
1873 :
1874 43999 : e = gimple_switch_edge (cfun, last, 0);
1875 43999 : if (bound_count > bound_limit)
1876 : {
1877 14909 : *(ipa_predicate *) e->aux = true;
1878 14909 : vec_free (param_ops);
1879 14909 : return;
1880 : }
1881 :
1882 29090 : ipa_predicate p_seg = true;
1883 29090 : ipa_predicate p_all = false;
1884 :
1885 29090 : if (vr_type != VR_RANGE)
1886 : {
1887 28513 : vr_wmin = wi::to_wide (TYPE_MIN_VALUE (type));
1888 28513 : vr_wmax = wi::to_wide (TYPE_MAX_VALUE (type));
1889 : }
1890 :
1891 : /* Construct predicate to represent default range set that is negation of
1892 : all case ranges. Case range is classified as containing single/non-single
1893 : values. Suppose a piece of case ranges in the following.
1894 :
1895 : [D1...D2] [S1] ... [Sn] [D3...D4]
1896 :
1897 : To represent default case's range sets between two non-single value
1898 : case ranges (From D2 to D3), we construct predicate as:
1899 :
1900 : D2 < x < D3 && x != S1 && ... && x != Sn
1901 : */
1902 89602 : for (size_t i = 0; i < ranges.length (); i++)
1903 : {
1904 60607 : tree min = ranges[i].first;
1905 60607 : tree max = ranges[i].second;
1906 :
1907 60607 : if (min == max)
1908 74936 : p_seg &= add_condition (summary, params_summary, index,
1909 : param_type, &aggpos, NE_EXPR,
1910 37468 : min, param_ops);
1911 : else
1912 : {
1913 : /* Do not create sub-predicate for range that is beyond low bound
1914 : of switch index. */
1915 23139 : if (wi::lt_p (vr_wmin, wi::to_wide (min), TYPE_SIGN (type)))
1916 : {
1917 16137 : p_seg &= add_condition (summary, params_summary, index,
1918 : param_type, &aggpos,
1919 16137 : LT_EXPR, min, param_ops);
1920 16137 : p_all = p_all.or_with (summary->conds, p_seg);
1921 : }
1922 :
1923 : /* Do not create sub-predicate for range that is beyond up bound
1924 : of switch index. */
1925 23139 : if (wi::le_p (vr_wmax, wi::to_wide (max), TYPE_SIGN (type)))
1926 : {
1927 95 : p_seg = false;
1928 95 : break;
1929 : }
1930 :
1931 23044 : p_seg = add_condition (summary, params_summary, index,
1932 : param_type, &aggpos, GT_EXPR,
1933 : max, param_ops);
1934 : }
1935 : }
1936 :
1937 29090 : p_all = p_all.or_with (summary->conds, p_seg);
1938 29090 : *(ipa_predicate *) e->aux
1939 29090 : = p_all.or_with (summary->conds, *(ipa_predicate *) e->aux);
1940 :
1941 35602 : vec_free (param_ops);
1942 43999 : }
1943 :
1944 :
1945 : /* For each BB in NODE attach to its AUX pointer predicate under
1946 : which it is executable. */
1947 :
1948 : static void
1949 6302282 : compute_bb_predicates (struct ipa_func_body_info *fbi,
1950 : struct cgraph_node *node,
1951 : class ipa_fn_summary *summary,
1952 : class ipa_node_params *params_summary)
1953 : {
1954 6302282 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
1955 6302282 : bool done = false;
1956 6302282 : basic_block bb;
1957 :
1958 41331453 : FOR_EACH_BB_FN (bb, my_function)
1959 : {
1960 35029171 : set_cond_stmt_execution_predicate (fbi, summary, params_summary, bb);
1961 35029171 : set_switch_stmt_execution_predicate (fbi, summary, params_summary, bb);
1962 : }
1963 :
1964 : /* Entry block is always executable. */
1965 6302282 : ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
1966 6302282 : = edge_predicate_pool.allocate ();
1967 6302282 : *(ipa_predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux = true;
1968 :
1969 : /* A simple dataflow propagation of predicates forward in the CFG.
1970 : TODO: work in reverse postorder. */
1971 19026316 : while (!done)
1972 : {
1973 12724034 : done = true;
1974 87890478 : FOR_EACH_BB_FN (bb, my_function)
1975 : {
1976 75166444 : ipa_predicate p = false;
1977 75166444 : edge e;
1978 75166444 : edge_iterator ei;
1979 93146365 : FOR_EACH_EDGE (e, ei, bb->preds)
1980 : {
1981 79833968 : if (e->src->aux)
1982 : {
1983 78086401 : ipa_predicate this_bb_predicate
1984 : = *(ipa_predicate *) e->src->aux;
1985 78086401 : if (e->aux)
1986 5222557 : this_bb_predicate &= (*(ipa_predicate *) e->aux);
1987 78086401 : p = p.or_with (summary->conds, this_bb_predicate);
1988 78086401 : if (p == true)
1989 : break;
1990 : }
1991 : }
1992 75166444 : if (p != false)
1993 : {
1994 74025072 : basic_block pdom_bb;
1995 :
1996 74025072 : if (!bb->aux)
1997 : {
1998 27442057 : done = false;
1999 27442057 : bb->aux = edge_predicate_pool.allocate ();
2000 27442057 : *((ipa_predicate *) bb->aux) = p;
2001 : }
2002 46583015 : else if (p != *(ipa_predicate *) bb->aux)
2003 : {
2004 : /* This OR operation is needed to ensure monotonous data flow
2005 : in the case we hit the limit on number of clauses and the
2006 : and/or operations above give approximate answers. */
2007 126449 : p = p.or_with (summary->conds, *(ipa_predicate *)bb->aux);
2008 126449 : if (p != *(ipa_predicate *)bb->aux)
2009 : {
2010 103808 : done = false;
2011 103808 : *((ipa_predicate *)bb->aux) = p;
2012 : }
2013 : }
2014 :
2015 : /* For switch/if statement, we can OR-combine predicates of all
2016 : its cases/branches to get predicate for basic block in their
2017 : convergence point, but sometimes this will generate very
2018 : complicated predicate. Actually, we can get simplified
2019 : predicate in another way by using the fact that predicate
2020 : for a basic block must also hold true for its post dominators.
2021 : To be specific, basic block in convergence point of
2022 : conditional statement should include predicate of the
2023 : statement. */
2024 74025072 : pdom_bb = get_immediate_dominator (CDI_POST_DOMINATORS, bb);
2025 74025072 : if (pdom_bb == EXIT_BLOCK_PTR_FOR_FN (my_function) || !pdom_bb)
2026 : ;
2027 38034976 : else if (!pdom_bb->aux)
2028 : {
2029 7587100 : done = false;
2030 7587100 : pdom_bb->aux = edge_predicate_pool.allocate ();
2031 7587100 : *((ipa_predicate *)pdom_bb->aux) = p;
2032 : }
2033 30447876 : else if (p != *(ipa_predicate *)pdom_bb->aux)
2034 : {
2035 3315200 : p = p.or_with (summary->conds,
2036 : *(ipa_predicate *)pdom_bb->aux);
2037 3315200 : if (p != *(ipa_predicate *)pdom_bb->aux)
2038 : {
2039 143924 : done = false;
2040 143924 : *((ipa_predicate *)pdom_bb->aux) = p;
2041 : }
2042 : }
2043 : }
2044 : }
2045 : }
2046 6302282 : }
2047 :
2048 :
2049 : /* Return predicate specifying when the STMT might have result that is not
2050 : a compile time constant. */
2051 :
2052 : static ipa_predicate
2053 4245212 : will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
2054 : class ipa_fn_summary *summary,
2055 : class ipa_node_params *params_summary,
2056 : tree expr,
2057 : vec<ipa_predicate> nonconstant_names)
2058 : {
2059 4245212 : tree parm;
2060 4245212 : int index;
2061 :
2062 4504890 : while (UNARY_CLASS_P (expr))
2063 259678 : expr = TREE_OPERAND (expr, 0);
2064 :
2065 4245212 : parm = unmodified_parm (fbi, NULL, expr, NULL);
2066 4245212 : if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
2067 221917 : return add_condition (summary, params_summary, index, TREE_TYPE (parm), NULL,
2068 221917 : ipa_predicate::changed, NULL_TREE);
2069 4023295 : if (is_gimple_min_invariant (expr))
2070 62729 : return false;
2071 3960566 : if (TREE_CODE (expr) == SSA_NAME)
2072 3779062 : return nonconstant_names[SSA_NAME_VERSION (expr)];
2073 181504 : if (BINARY_CLASS_P (expr) || COMPARISON_CLASS_P (expr))
2074 : {
2075 180878 : ipa_predicate p1
2076 180878 : = will_be_nonconstant_expr_predicate (fbi, summary,
2077 : params_summary,
2078 180878 : TREE_OPERAND (expr, 0),
2079 : nonconstant_names);
2080 180878 : if (p1 == true)
2081 105433 : return p1;
2082 :
2083 75445 : ipa_predicate p2
2084 75445 : = will_be_nonconstant_expr_predicate (fbi, summary,
2085 : params_summary,
2086 75445 : TREE_OPERAND (expr, 1),
2087 : nonconstant_names);
2088 75445 : return p1.or_with (summary->conds, p2);
2089 : }
2090 626 : else if (TREE_CODE (expr) == COND_EXPR)
2091 : {
2092 184 : ipa_predicate p1
2093 184 : = will_be_nonconstant_expr_predicate (fbi, summary,
2094 : params_summary,
2095 184 : TREE_OPERAND (expr, 0),
2096 : nonconstant_names);
2097 184 : if (p1 == true)
2098 33 : return p1;
2099 :
2100 151 : ipa_predicate p2
2101 151 : = will_be_nonconstant_expr_predicate (fbi, summary,
2102 : params_summary,
2103 151 : TREE_OPERAND (expr, 1),
2104 : nonconstant_names);
2105 151 : if (p2 == true)
2106 151 : return p2;
2107 0 : p1 = p1.or_with (summary->conds, p2);
2108 0 : p2 = will_be_nonconstant_expr_predicate (fbi, summary,
2109 : params_summary,
2110 0 : TREE_OPERAND (expr, 2),
2111 : nonconstant_names);
2112 0 : return p2.or_with (summary->conds, p1);
2113 : }
2114 442 : else if (TREE_CODE (expr) == CALL_EXPR)
2115 442 : return true;
2116 : else
2117 : {
2118 0 : debug_tree (expr);
2119 0 : gcc_unreachable ();
2120 : }
2121 : }
2122 :
2123 :
2124 : /* Return predicate specifying when the STMT might have result that is not
2125 : a compile time constant. */
2126 :
2127 : static ipa_predicate
2128 113736361 : will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
2129 : class ipa_fn_summary *summary,
2130 : class ipa_node_params *params_summary,
2131 : gimple *stmt,
2132 : vec<ipa_predicate> nonconstant_names)
2133 : {
2134 113736361 : ipa_predicate p = true;
2135 113736361 : ssa_op_iter iter;
2136 113736361 : tree use;
2137 113736361 : tree param_type = NULL_TREE;
2138 113736361 : ipa_predicate op_non_const;
2139 113736361 : bool is_load;
2140 113736361 : int base_index;
2141 113736361 : struct agg_position_info aggpos;
2142 :
2143 : /* What statements might be optimized away
2144 : when their arguments are constant. */
2145 113736361 : if (gimple_code (stmt) != GIMPLE_ASSIGN
2146 39083413 : && gimple_code (stmt) != GIMPLE_COND
2147 27731290 : && gimple_code (stmt) != GIMPLE_SWITCH
2148 141389108 : && (gimple_code (stmt) != GIMPLE_CALL
2149 20751415 : || !(gimple_call_flags (stmt) & ECF_CONST)))
2150 25986297 : return p;
2151 :
2152 : /* Stores will stay anyway. */
2153 87750064 : if (gimple_store_p (stmt))
2154 25751762 : return p;
2155 :
2156 61998302 : is_load = gimple_assign_load_p (stmt);
2157 :
2158 : /* Loads can be optimized when the value is known. */
2159 61998302 : if (is_load)
2160 : {
2161 17811308 : tree op = gimple_assign_rhs1 (stmt);
2162 17811308 : if (!decompose_param_expr (fbi, stmt, op, &base_index, ¶m_type,
2163 : &aggpos))
2164 14632049 : return p;
2165 : }
2166 : else
2167 44186994 : base_index = -1;
2168 :
2169 : /* See if we understand all operands before we start
2170 : adding conditionals. */
2171 64172696 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2172 : {
2173 47071302 : tree parm = unmodified_parm (fbi, stmt, use, NULL);
2174 : /* For arguments we can build a condition. */
2175 47071302 : if (parm && ipa_get_param_decl_index (fbi->info, parm) >= 0)
2176 8403813 : continue;
2177 38667489 : if (TREE_CODE (use) != SSA_NAME)
2178 0 : return p;
2179 : /* If we know when operand is constant,
2180 : we still can say something useful. */
2181 38667489 : if (nonconstant_names[SSA_NAME_VERSION (use)] != true)
2182 8402630 : continue;
2183 30264859 : return p;
2184 : }
2185 :
2186 17101394 : if (is_load)
2187 3179178 : op_non_const =
2188 3179178 : add_condition (summary, params_summary,
2189 : base_index, param_type, &aggpos,
2190 : ipa_predicate::changed, NULL_TREE);
2191 : else
2192 13922216 : op_non_const = false;
2193 32881031 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2194 : {
2195 15779637 : tree parm = unmodified_parm (fbi, stmt, use, NULL);
2196 15779637 : int index;
2197 :
2198 15779637 : if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
2199 : {
2200 7915017 : if (index != base_index)
2201 5422816 : p = add_condition (summary, params_summary, index,
2202 5422816 : TREE_TYPE (parm), NULL,
2203 : ipa_predicate::changed, NULL_TREE);
2204 : else
2205 2492201 : continue;
2206 : }
2207 : else
2208 7864620 : p = nonconstant_names[SSA_NAME_VERSION (use)];
2209 13287436 : op_non_const = p.or_with (summary->conds, op_non_const);
2210 : }
2211 17101394 : if ((gimple_code (stmt) == GIMPLE_ASSIGN || gimple_code (stmt) == GIMPLE_CALL)
2212 14884941 : && gimple_op (stmt, 0)
2213 31694854 : && TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
2214 14593460 : nonconstant_names[SSA_NAME_VERSION (gimple_op (stmt, 0))]
2215 14593460 : = op_non_const;
2216 17101394 : return op_non_const;
2217 : }
2218 :
2219 : struct record_modified_bb_info
2220 : {
2221 : tree op;
2222 : bitmap bb_set;
2223 : gimple *stmt;
2224 : };
2225 :
2226 : /* Value is initialized in INIT_BB and used in USE_BB. We want to compute
2227 : probability how often it changes between USE_BB.
2228 : INIT_BB->count/USE_BB->count is an estimate, but if INIT_BB
2229 : is in different loop nest, we can do better.
2230 : This is all just estimate. In theory we look for minimal cut separating
2231 : INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
2232 : anyway. */
2233 :
2234 : static basic_block
2235 10379297 : get_minimal_bb (basic_block init_bb, basic_block use_bb)
2236 : {
2237 10379297 : class loop *l = find_common_loop (init_bb->loop_father, use_bb->loop_father);
2238 10379297 : if (l && l->header->count < init_bb->count)
2239 366959 : return l->header;
2240 : return init_bb;
2241 : }
2242 :
2243 : /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2244 : set except for info->stmt. */
2245 :
2246 : static bool
2247 4800439 : record_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef, void *data)
2248 : {
2249 4800439 : struct record_modified_bb_info *info =
2250 : (struct record_modified_bb_info *) data;
2251 4800439 : if (SSA_NAME_DEF_STMT (vdef) == info->stmt)
2252 : return false;
2253 4772487 : if (gimple_clobber_p (SSA_NAME_DEF_STMT (vdef)))
2254 : return false;
2255 4725913 : bitmap_set_bit (info->bb_set,
2256 4725913 : SSA_NAME_IS_DEFAULT_DEF (vdef)
2257 0 : ? ENTRY_BLOCK_PTR_FOR_FN (cfun)->index
2258 : : get_minimal_bb
2259 4725913 : (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2260 4725913 : gimple_bb (info->stmt))->index);
2261 4725913 : if (dump_file)
2262 : {
2263 0 : fprintf (dump_file, " Param ");
2264 0 : print_generic_expr (dump_file, info->op, TDF_SLIM);
2265 0 : fprintf (dump_file, " changed at bb %i, minimal: %i stmt: ",
2266 0 : gimple_bb (SSA_NAME_DEF_STMT (vdef))->index,
2267 : get_minimal_bb
2268 0 : (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2269 0 : gimple_bb (info->stmt))->index);
2270 0 : print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (vdef), 0);
2271 : }
2272 : return false;
2273 : }
2274 :
2275 : /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2276 : will change since last invocation of STMT.
2277 :
2278 : Value 0 is reserved for compile time invariants.
2279 : For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2280 : ought to be REG_BR_PROB_BASE / estimated_iters. */
2281 :
2282 : static int
2283 38356360 : param_change_prob (ipa_func_body_info *fbi, gimple *stmt, int i)
2284 : {
2285 38356360 : tree op = gimple_call_arg (stmt, i);
2286 38356360 : basic_block bb = gimple_bb (stmt);
2287 :
2288 38356360 : if (TREE_CODE (op) == WITH_SIZE_EXPR)
2289 299 : op = TREE_OPERAND (op, 0);
2290 :
2291 38356360 : tree base = get_base_address (op);
2292 :
2293 : /* Global invariants never change. */
2294 38356360 : if (is_gimple_min_invariant (base))
2295 : return 0;
2296 :
2297 : /* We would have to do non-trivial analysis to really work out what
2298 : is the probability of value to change (i.e. when init statement
2299 : is in a sibling loop of the call).
2300 :
2301 : We do an conservative estimate: when call is executed N times more often
2302 : than the statement defining value, we take the frequency 1/N. */
2303 19233762 : if (TREE_CODE (base) == SSA_NAME)
2304 : {
2305 16935912 : profile_count init_count;
2306 :
2307 25779204 : if (!bb->count.nonzero_p ())
2308 : return REG_BR_PROB_BASE;
2309 :
2310 8684195 : if (SSA_NAME_IS_DEFAULT_DEF (base))
2311 3030811 : init_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2312 : else
2313 5653384 : init_count = get_minimal_bb
2314 5653384 : (gimple_bb (SSA_NAME_DEF_STMT (base)),
2315 : gimple_bb (stmt))->count;
2316 :
2317 8684195 : if (init_count < bb->count)
2318 449231 : return MAX ((init_count.to_sreal_scale (bb->count)
2319 : * REG_BR_PROB_BASE).to_int (), 1);
2320 : return REG_BR_PROB_BASE;
2321 : }
2322 : else
2323 : {
2324 2297850 : ao_ref refd;
2325 2297850 : profile_count max = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2326 2297850 : struct record_modified_bb_info info;
2327 2297850 : tree init = ctor_for_folding (base);
2328 :
2329 2297850 : if (init != error_mark_node)
2330 : return 0;
2331 5101127 : if (!bb->count.nonzero_p () || fbi->aa_walk_budget == 0)
2332 : return REG_BR_PROB_BASE;
2333 1414436 : if (dump_file)
2334 : {
2335 0 : fprintf (dump_file, " Analyzing param change probability of ");
2336 0 : print_generic_expr (dump_file, op, TDF_SLIM);
2337 0 : fprintf (dump_file, "\n");
2338 : }
2339 1414436 : ao_ref_init (&refd, op);
2340 1414436 : info.op = op;
2341 1414436 : info.stmt = stmt;
2342 1414436 : info.bb_set = BITMAP_ALLOC (NULL);
2343 1414436 : int walked
2344 2828872 : = walk_aliased_vdefs (&refd, gimple_vuse (stmt), record_modified, &info,
2345 : NULL, NULL, fbi->aa_walk_budget);
2346 1414436 : if (walked > 0)
2347 1285673 : fbi->aa_walk_budget -= walked;
2348 1414436 : if (walked < 0 || bitmap_bit_p (info.bb_set, bb->index))
2349 : {
2350 868918 : if (walked < 0)
2351 206 : fbi->aa_walk_budget = 0;
2352 868918 : if (dump_file)
2353 : {
2354 0 : if (walked < 0)
2355 0 : fprintf (dump_file, " Ran out of AA walking budget.\n");
2356 : else
2357 0 : fprintf (dump_file, " Set in same BB as used.\n");
2358 : }
2359 868918 : BITMAP_FREE (info.bb_set);
2360 868918 : return REG_BR_PROB_BASE;
2361 : }
2362 :
2363 545518 : bitmap_iterator bi;
2364 545518 : unsigned index;
2365 : /* Lookup the most frequent update of the value and believe that
2366 : it dominates all the other; precise analysis here is difficult. */
2367 1526474 : EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi)
2368 980956 : max = profile_count::max_prefer_initialized
2369 980956 : (max, BASIC_BLOCK_FOR_FN (cfun, index)->count);
2370 545518 : if (dump_file)
2371 : {
2372 0 : fprintf (dump_file, " Set with count ");
2373 0 : max.dump (dump_file);
2374 0 : fprintf (dump_file, " and used with count ");
2375 0 : bb->count.dump (dump_file);
2376 0 : fprintf (dump_file, " freq %f\n",
2377 0 : max.to_sreal_scale (bb->count).to_double ());
2378 : }
2379 :
2380 545518 : BITMAP_FREE (info.bb_set);
2381 545518 : if (max < bb->count)
2382 91494 : return MAX ((max.to_sreal_scale (bb->count)
2383 : * REG_BR_PROB_BASE).to_int (), 1);
2384 : return REG_BR_PROB_BASE;
2385 : }
2386 : }
2387 :
2388 : /* Find whether a basic block BB is the final block of a (half) diamond CFG
2389 : sub-graph and if the predicate the condition depends on is known. If so,
2390 : return true and store the pointer the predicate in *P. */
2391 :
2392 : static bool
2393 6320554 : phi_result_unknown_predicate (ipa_func_body_info *fbi,
2394 : ipa_fn_summary *summary,
2395 : class ipa_node_params *params_summary,
2396 : basic_block bb,
2397 : ipa_predicate *p,
2398 : vec<ipa_predicate> nonconstant_names)
2399 : {
2400 6320554 : edge e;
2401 6320554 : edge_iterator ei;
2402 6320554 : basic_block first_bb = NULL;
2403 :
2404 6320554 : if (single_pred_p (bb))
2405 : {
2406 19910 : *p = false;
2407 19910 : return true;
2408 : }
2409 :
2410 14534534 : FOR_EACH_EDGE (e, ei, bb->preds)
2411 : {
2412 12405165 : if (single_succ_p (e->src))
2413 : {
2414 13015522 : if (!single_pred_p (e->src))
2415 : return false;
2416 7312472 : if (!first_bb)
2417 3198644 : first_bb = single_pred (e->src);
2418 4113828 : else if (single_pred (e->src) != first_bb)
2419 : return false;
2420 : }
2421 : else
2422 : {
2423 4106954 : if (!first_bb)
2424 : first_bb = e->src;
2425 1417765 : else if (e->src != first_bb)
2426 : return false;
2427 : }
2428 : }
2429 :
2430 2129369 : if (!first_bb)
2431 : return false;
2432 :
2433 4258738 : gcond *stmt = safe_dyn_cast <gcond *> (*gsi_last_bb (first_bb));
2434 2087225 : if (!stmt
2435 2087225 : || !is_gimple_ip_invariant (gimple_cond_rhs (stmt)))
2436 311608 : return false;
2437 :
2438 1817761 : *p = will_be_nonconstant_expr_predicate (fbi, summary, params_summary,
2439 : gimple_cond_lhs (stmt),
2440 : nonconstant_names);
2441 1817761 : if (*p == true)
2442 : return false;
2443 : else
2444 : return true;
2445 : }
2446 :
2447 : /* Given a PHI statement in a function described by inline properties SUMMARY
2448 : and *P being the predicate describing whether the selected PHI argument is
2449 : known, store a predicate for the result of the PHI statement into
2450 : NONCONSTANT_NAMES, if possible. */
2451 :
2452 : static void
2453 701343 : predicate_for_phi_result (class ipa_fn_summary *summary, gphi *phi,
2454 : ipa_predicate *p,
2455 : vec<ipa_predicate> nonconstant_names)
2456 : {
2457 701343 : unsigned i;
2458 :
2459 984041 : for (i = 0; i < gimple_phi_num_args (phi); i++)
2460 : {
2461 865704 : tree arg = gimple_phi_arg (phi, i)->def;
2462 865704 : if (!is_gimple_min_invariant (arg))
2463 : {
2464 753204 : gcc_assert (TREE_CODE (arg) == SSA_NAME);
2465 1506408 : *p = p->or_with (summary->conds,
2466 753204 : nonconstant_names[SSA_NAME_VERSION (arg)]);
2467 753204 : if (*p == true)
2468 : return;
2469 : }
2470 : }
2471 :
2472 118337 : if (dump_file && (dump_flags & TDF_DETAILS))
2473 : {
2474 3 : fprintf (dump_file, "\t\tphi predicate: ");
2475 3 : p->dump (dump_file, summary->conds);
2476 : }
2477 118337 : nonconstant_names[SSA_NAME_VERSION (gimple_phi_result (phi))] = *p;
2478 : }
2479 :
2480 : /* For a typical usage of __builtin_expect (a<b, 1), we
2481 : may introduce an extra relation stmt:
2482 : With the builtin, we have
2483 : t1 = a <= b;
2484 : t2 = (long int) t1;
2485 : t3 = __builtin_expect (t2, 1);
2486 : if (t3 != 0)
2487 : goto ...
2488 : Without the builtin, we have
2489 : if (a<=b)
2490 : goto...
2491 : This affects the size/time estimation and may have
2492 : an impact on the earlier inlining.
2493 : Here find this pattern and fix it up later. */
2494 :
2495 : static gimple *
2496 40935480 : find_foldable_builtin_expect (basic_block bb)
2497 : {
2498 40935480 : gimple_stmt_iterator bsi;
2499 :
2500 283219877 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
2501 : {
2502 201526979 : gimple *stmt = gsi_stmt (bsi);
2503 201526979 : if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)
2504 201298484 : || gimple_call_builtin_p (stmt, BUILT_IN_EXPECT_WITH_PROBABILITY)
2505 402825398 : || gimple_call_internal_p (stmt, IFN_BUILTIN_EXPECT))
2506 : {
2507 359162 : tree var = gimple_call_lhs (stmt);
2508 359162 : tree arg = gimple_call_arg (stmt, 0);
2509 359162 : use_operand_p use_p;
2510 359162 : gimple *use_stmt;
2511 359162 : bool match = false;
2512 359162 : bool done = false;
2513 :
2514 359162 : if (!var || !arg)
2515 3 : continue;
2516 359159 : gcc_assert (TREE_CODE (var) == SSA_NAME);
2517 :
2518 717024 : while (TREE_CODE (arg) == SSA_NAME)
2519 : {
2520 717024 : gimple *stmt_tmp = SSA_NAME_DEF_STMT (arg);
2521 717024 : if (!is_gimple_assign (stmt_tmp))
2522 : break;
2523 709854 : switch (gimple_assign_rhs_code (stmt_tmp))
2524 : {
2525 : case LT_EXPR:
2526 : case LE_EXPR:
2527 : case GT_EXPR:
2528 : case GE_EXPR:
2529 : case EQ_EXPR:
2530 : case NE_EXPR:
2531 : match = true;
2532 : done = true;
2533 : break;
2534 : CASE_CONVERT:
2535 : break;
2536 : default:
2537 : done = true;
2538 : break;
2539 : }
2540 357865 : if (done)
2541 : break;
2542 357865 : arg = gimple_assign_rhs1 (stmt_tmp);
2543 : }
2544 :
2545 323255 : if (match && single_imm_use (var, &use_p, &use_stmt)
2546 681994 : && gimple_code (use_stmt) == GIMPLE_COND)
2547 178062 : return use_stmt;
2548 : }
2549 : }
2550 : return NULL;
2551 : }
2552 :
2553 : /* Return true when the basic blocks contains only clobbers followed by RESX.
2554 : Such BBs are kept around to make removal of dead stores possible with
2555 : presence of EH and will be optimized out by optimize_clobbers later in the
2556 : game.
2557 :
2558 : NEED_EH is used to recurse in case the clobber has non-EH predecessors
2559 : that can be clobber only, too.. When it is false, the RESX is not necessary
2560 : on the end of basic block. */
2561 :
2562 : static bool
2563 41546057 : clobber_only_eh_bb_p (basic_block bb, bool need_eh = true)
2564 : {
2565 41546057 : gimple_stmt_iterator gsi = gsi_last_bb (bb);
2566 41546057 : edge_iterator ei;
2567 41546057 : edge e;
2568 :
2569 41546057 : if (need_eh)
2570 : {
2571 41477782 : if (gsi_end_p (gsi))
2572 : return false;
2573 40351043 : if (gimple_code (gsi_stmt (gsi)) != GIMPLE_RESX)
2574 : return false;
2575 1038943 : gsi_prev (&gsi);
2576 : }
2577 41004293 : else if (!single_succ_p (bb))
2578 : return false;
2579 :
2580 4603583 : for (; !gsi_end_p (gsi); gsi_prev (&gsi))
2581 : {
2582 2760739 : gimple *stmt = gsi_stmt (gsi);
2583 2760739 : if (is_gimple_debug (stmt))
2584 904417 : continue;
2585 1856322 : if (gimple_clobber_p (stmt))
2586 876903 : continue;
2587 979419 : if (gimple_code (stmt) == GIMPLE_LABEL)
2588 : break;
2589 : return false;
2590 : }
2591 :
2592 : /* See if all predecessors are either throws or clobber only BBs. */
2593 3020778 : FOR_EACH_EDGE (e, ei, bb->preds)
2594 2477014 : if (!(e->flags & EDGE_EH)
2595 2477014 : && !clobber_only_eh_bb_p (e->src, false))
2596 : return false;
2597 :
2598 : return true;
2599 : }
2600 :
2601 : /* Return true if STMT compute a floating point expression that may be affected
2602 : by -ffast-math and similar flags. */
2603 :
2604 : static bool
2605 94129694 : fp_expression_p (gimple *stmt)
2606 : {
2607 94129694 : ssa_op_iter i;
2608 94129694 : tree op;
2609 :
2610 211134987 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_DEF|SSA_OP_USE)
2611 117600601 : if (FLOAT_TYPE_P (TREE_TYPE (op)))
2612 : return true;
2613 : return false;
2614 : }
2615 :
2616 : /* Return true if T references memory location that is local
2617 : for the function (that means, dead after return) or read-only. */
2618 :
2619 : bool
2620 58758663 : refs_local_or_readonly_memory_p (tree t)
2621 : {
2622 : /* Non-escaping memory is fine. */
2623 58758663 : t = get_base_address (t);
2624 58758663 : if ((TREE_CODE (t) == MEM_REF
2625 58758663 : || TREE_CODE (t) == TARGET_MEM_REF))
2626 24761808 : return points_to_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
2627 :
2628 : /* Automatic variables are fine. */
2629 33996855 : if (DECL_P (t)
2630 33996855 : && auto_var_in_fn_p (t, current_function_decl))
2631 : return true;
2632 :
2633 : /* Read-only variables are fine. */
2634 11079678 : if (DECL_P (t) && TREE_READONLY (t))
2635 : return true;
2636 :
2637 : return false;
2638 : }
2639 :
2640 : /* Return true if T is a pointer pointing to memory location that is local
2641 : for the function (that means, dead after return) or read-only. */
2642 :
2643 : bool
2644 71647446 : points_to_local_or_readonly_memory_p (tree t)
2645 : {
2646 : /* See if memory location is clearly invalid. */
2647 71647446 : if (integer_zerop (t))
2648 2427180 : return flag_delete_null_pointer_checks;
2649 69220266 : if (TREE_CODE (t) == SSA_NAME)
2650 : {
2651 : /* For IPA passes we can consinder accesses to return slot local
2652 : even if it is not local in the sense that memory is dead by
2653 : the end of founction.
2654 : The outer function will see a store in the call assignment
2655 : and thus this will do right thing for all uses of this
2656 : function in the current IPA passes (modref, pure/const discovery
2657 : and inlining heuristics). */
2658 43799722 : if (DECL_RESULT (current_function_decl)
2659 43799722 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
2660 44897242 : && t == ssa_default_def (cfun, DECL_RESULT (current_function_decl)))
2661 : return true;
2662 43476330 : return !ptr_deref_may_alias_global_p (t, false);
2663 : }
2664 25420544 : if (TREE_CODE (t) == ADDR_EXPR
2665 25420544 : && (TREE_CODE (TREE_OPERAND (t, 0)) != TARGET_MEM_REF
2666 4690 : || TREE_CODE (TREE_OPERAND (TREE_OPERAND (t, 0), 0)) != INTEGER_CST))
2667 13092554 : return refs_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
2668 : return false;
2669 : }
2670 :
2671 : /* Return true if T is a pointer pointing to memory location that is possible
2672 : sra candidate if all functions it is passed to are inlined. */
2673 :
2674 : static bool
2675 38356360 : points_to_possible_sra_candidate_p (tree t)
2676 : {
2677 38356360 : if (TREE_CODE (t) != ADDR_EXPR)
2678 : return false;
2679 :
2680 11400514 : t = get_base_address (TREE_OPERAND (t, 0));
2681 :
2682 : /* Automatic variables are fine. */
2683 11400514 : if (DECL_P (t)
2684 11400514 : && auto_var_in_fn_p (t, current_function_decl))
2685 : return true;
2686 : return false;
2687 : }
2688 :
2689 : /* Return true if BB only calls builtin_unreachable.
2690 : We skip empty basic blocks, debug statements, clobbers and predicts.
2691 : CACHE is used to memoize already analyzed blocks. */
2692 :
2693 : static bool
2694 27198380 : builtin_unreachable_bb_p (basic_block bb, vec<unsigned char> &cache)
2695 : {
2696 27198380 : if (cache[bb->index])
2697 2552774 : return cache[bb->index] - 1;
2698 24645606 : gimple_stmt_iterator si;
2699 24645606 : auto_vec <basic_block, 4> visited_bbs;
2700 24645606 : bool ret = false;
2701 25442535 : while (true)
2702 : {
2703 25442535 : bool empty_bb = true;
2704 25442535 : visited_bbs.safe_push (bb);
2705 25442535 : cache[bb->index] = 3;
2706 25442535 : for (si = gsi_start_nondebug_bb (bb);
2707 26929304 : !gsi_end_p (si) && empty_bb;
2708 1486769 : gsi_next_nondebug (&si))
2709 : {
2710 25573004 : if (gimple_code (gsi_stmt (si)) != GIMPLE_PREDICT
2711 25204305 : && !gimple_clobber_p (gsi_stmt (si))
2712 49692543 : && !gimple_nop_p (gsi_stmt (si)))
2713 : {
2714 : empty_bb = false;
2715 : break;
2716 : }
2717 : }
2718 25442535 : if (!empty_bb)
2719 : break;
2720 : else
2721 1356300 : bb = single_succ_edge (bb)->dest;
2722 1356300 : if (cache[bb->index])
2723 : {
2724 559371 : ret = cache[bb->index] == 3 ? false : cache[bb->index] - 1;
2725 559371 : goto done;
2726 : }
2727 : }
2728 24086235 : if (gimple_call_builtin_p (gsi_stmt (si), BUILT_IN_UNREACHABLE)
2729 24086235 : || gimple_call_builtin_p (gsi_stmt (si), BUILT_IN_UNREACHABLE_TRAP))
2730 : ret = true;
2731 24645606 : done:
2732 99379353 : for (basic_block vbb:visited_bbs)
2733 25442535 : cache[vbb->index] = (unsigned char)ret + 1;
2734 24645606 : return ret;
2735 24645606 : }
2736 :
2737 : static bool
2738 13719972 : guards_builtin_unreachable (basic_block bb, vec<unsigned char> &cache)
2739 : {
2740 13719972 : edge_iterator ei;
2741 13719972 : edge e;
2742 40662505 : FOR_EACH_EDGE (e, ei, bb->succs)
2743 27198380 : if (builtin_unreachable_bb_p (e->dest, cache))
2744 : {
2745 255847 : if (dump_file && (dump_flags & TDF_DETAILS))
2746 1 : fprintf (dump_file,
2747 : "BB %i ends with conditional guarding __builtin_unreachable;"
2748 : " conditinal is unnecesary\n", bb->index);
2749 255847 : return true;
2750 : }
2751 : return false;
2752 : }
2753 :
2754 : #define STMT_NECESSARY GF_PLF_1
2755 :
2756 : /* If STMT is not already marked necessary, mark it, and add it to the
2757 : worklist if ADD_TO_WORKLIST is true. */
2758 :
2759 : static inline void
2760 171303225 : mark_stmt_necessary (gimple *stmt, auto_vec<gimple *> &worklist)
2761 : {
2762 171303225 : gcc_assert (stmt);
2763 :
2764 171303225 : if (gimple_plf (stmt, STMT_NECESSARY))
2765 : return;
2766 :
2767 142188084 : if (dump_file && (dump_flags & TDF_DETAILS))
2768 : {
2769 207 : fprintf (dump_file, "Marking useful stmt: ");
2770 207 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
2771 207 : fprintf (dump_file, "\n");
2772 : }
2773 :
2774 142188084 : gimple_set_plf (stmt, STMT_NECESSARY, true);
2775 142188084 : worklist.safe_push (stmt);
2776 : }
2777 :
2778 : /* Mark the statement defining operand OP as necessary. */
2779 :
2780 : static inline void
2781 120587660 : mark_operand_necessary (tree op, auto_vec<gimple *> &worklist)
2782 : {
2783 120587660 : gimple *stmt = SSA_NAME_DEF_STMT (op);
2784 120587660 : if (gimple_nop_p (stmt))
2785 : return;
2786 97782405 : mark_stmt_necessary (stmt, worklist);
2787 : }
2788 :
2789 : /* Mark all statements that will remain in the body after optimizing out
2790 : conditionals guarding __builtin_unreachable which we keep to preserve
2791 : value ranges. */
2792 :
2793 : static void
2794 7199463 : find_necessary_statements (struct cgraph_node *node)
2795 : {
2796 7199463 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
2797 7199463 : auto_vec<unsigned char, 10> cache;
2798 7199463 : basic_block bb;
2799 7199463 : auto_vec<gimple *> worklist;
2800 :
2801 7199463 : cache.safe_grow_cleared (last_basic_block_for_fn (cfun));
2802 : /* Mark all obviously necessary statements. */
2803 48677245 : FOR_EACH_BB_FN (bb, my_function)
2804 : {
2805 41477782 : for (gimple_stmt_iterator gsi = gsi_start_phis (bb);
2806 53010917 : !gsi_end_p (gsi); gsi_next (&gsi))
2807 11533135 : gimple_set_plf (gsi_stmt (gsi), STMT_NECESSARY, false);
2808 :
2809 232117509 : for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
2810 149161945 : gsi_next_nondebug (&bsi))
2811 : {
2812 149161945 : gimple *stmt = gsi_stmt (bsi);
2813 :
2814 149161945 : gimple_set_plf (stmt, STMT_NECESSARY, false);
2815 149161945 : if (gimple_has_side_effects (stmt)
2816 122159787 : || (is_ctrl_stmt (stmt)
2817 21948180 : && (gimple_code (stmt) != GIMPLE_COND
2818 13719972 : || !guards_builtin_unreachable (bb, cache)))
2819 100467454 : || gimple_store_p (stmt)
2820 224824750 : || gimple_code (stmt) == GIMPLE_ASM)
2821 73520820 : mark_stmt_necessary (stmt, worklist);
2822 : }
2823 : }
2824 149387547 : while (worklist.length () > 0)
2825 : {
2826 142188084 : gimple *stmt = worklist.pop ();
2827 :
2828 142188084 : if (dump_file && (dump_flags & TDF_DETAILS))
2829 : {
2830 207 : fprintf (dump_file, "processing: ");
2831 207 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
2832 207 : fprintf (dump_file, "\n");
2833 : }
2834 142188084 : if (gimple_code (stmt) == GIMPLE_PHI)
2835 18244883 : for (unsigned int k = 0; k < gimple_phi_num_args (stmt); k++)
2836 : {
2837 12916366 : tree arg = PHI_ARG_DEF (stmt, k);
2838 :
2839 12916366 : if (TREE_CODE (arg) == SSA_NAME)
2840 9924178 : mark_operand_necessary (arg, worklist);
2841 : }
2842 : else
2843 : {
2844 136859567 : ssa_op_iter iter;
2845 136859567 : tree use;
2846 :
2847 247523049 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2848 110663482 : mark_operand_necessary (use, worklist);
2849 : }
2850 : }
2851 7199463 : }
2852 :
2853 : /* Analyze function body for NODE.
2854 : EARLY indicates run from early optimization pipeline. */
2855 :
2856 : static void
2857 7199463 : analyze_function_body (struct cgraph_node *node, bool early)
2858 : {
2859 7199463 : sreal time = opt_for_fn (node->decl, param_uninlined_function_time);
2860 : /* Estimate static overhead for function prologue/epilogue and alignment. */
2861 7199463 : int size = opt_for_fn (node->decl, param_uninlined_function_insns);
2862 : /* Benefits are scaled by probability of elimination that is in range
2863 : <0,2>. */
2864 7199463 : basic_block bb;
2865 7199463 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
2866 7199463 : sreal freq;
2867 7199463 : class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
2868 7199463 : ipa_node_params *params_summary
2869 7199463 : = early ? NULL : ipa_node_params_sum->get (node);
2870 7199463 : ipa_predicate bb_predicate;
2871 7199463 : struct ipa_func_body_info fbi;
2872 7199463 : vec<ipa_predicate> nonconstant_names = vNULL;
2873 7199463 : int nblocks, n;
2874 7199463 : int *order;
2875 7199463 : gimple *fix_builtin_expect_stmt;
2876 :
2877 7199463 : gcc_assert (my_function && my_function->cfg);
2878 7199463 : gcc_assert (cfun == my_function);
2879 :
2880 7199463 : memset(&fbi, 0, sizeof(fbi));
2881 7199463 : vec_free (info->conds);
2882 7199463 : info->conds = NULL;
2883 7199463 : info->size_time_table.release ();
2884 7199463 : info->call_size_time_table.release ();
2885 :
2886 : /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2887 : so we can produce proper inline hints.
2888 :
2889 : When optimizing and analyzing for early inliner, initialize node params
2890 : so we can produce correct BB predicates. */
2891 :
2892 7199463 : if (opt_for_fn (node->decl, optimize))
2893 : {
2894 6302282 : calculate_dominance_info (CDI_DOMINATORS);
2895 6302282 : calculate_dominance_info (CDI_POST_DOMINATORS);
2896 6302282 : if (!early)
2897 1381691 : loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
2898 : else
2899 : {
2900 4920591 : ipa_check_create_node_params ();
2901 4920591 : ipa_initialize_node_params (node);
2902 : }
2903 :
2904 6302282 : if (ipa_node_params_sum)
2905 : {
2906 6302282 : fbi.node = node;
2907 6302282 : fbi.info = ipa_node_params_sum->get (node);
2908 6302282 : fbi.bb_infos = vNULL;
2909 6302282 : fbi.bb_infos.safe_grow_cleared (last_basic_block_for_fn (cfun), true);
2910 6302282 : fbi.param_count = count_formal_params (node->decl);
2911 6302282 : fbi.aa_walk_budget = opt_for_fn (node->decl, param_ipa_max_aa_steps);
2912 :
2913 6302282 : nonconstant_names.safe_grow_cleared
2914 6302282 : (SSANAMES (my_function)->length (), true);
2915 : }
2916 : }
2917 :
2918 7199463 : if (dump_file)
2919 247 : fprintf (dump_file, "\nAnalyzing function body size: %s\n",
2920 : node->dump_name ());
2921 :
2922 : /* When we run into maximal number of entries, we assign everything to the
2923 : constant truth case. Be sure to have it in list. */
2924 7199463 : bb_predicate = true;
2925 7199463 : info->account_size_time (0, 0, bb_predicate, bb_predicate);
2926 :
2927 7199463 : bb_predicate = ipa_predicate::not_inlined ();
2928 7199463 : info->account_size_time (opt_for_fn (node->decl,
2929 : param_uninlined_function_insns)
2930 : * ipa_fn_summary::size_scale,
2931 7199463 : opt_for_fn (node->decl,
2932 : param_uninlined_function_time),
2933 : bb_predicate,
2934 : bb_predicate);
2935 :
2936 : /* Only look for target information for inlinable functions. */
2937 7199463 : bool scan_for_target_info =
2938 7199463 : info->inlinable
2939 12923365 : && targetm.target_option.need_ipa_fn_target_info (node->decl,
2940 5723902 : info->target_info);
2941 :
2942 7199463 : if (fbi.info)
2943 6302282 : compute_bb_predicates (&fbi, node, info, params_summary);
2944 7199463 : find_necessary_statements (node);
2945 7199463 : const profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2946 7199463 : order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
2947 7199463 : nblocks = pre_and_rev_post_order_compute (NULL, order, false);
2948 48677245 : for (n = 0; n < nblocks; n++)
2949 : {
2950 41477782 : bb = BASIC_BLOCK_FOR_FN (cfun, order[n]);
2951 41477782 : freq = bb->count.to_sreal_scale (entry_count);
2952 41477782 : if (clobber_only_eh_bb_p (bb))
2953 : {
2954 542302 : if (dump_file && (dump_flags & TDF_DETAILS))
2955 0 : fprintf (dump_file, "\n Ignoring BB %i;"
2956 : " it will be optimized away by cleanup_clobbers\n",
2957 : bb->index);
2958 542302 : continue;
2959 : }
2960 :
2961 : /* TODO: Obviously predicates can be propagated down across CFG. */
2962 40935480 : if (fbi.info)
2963 : {
2964 34549223 : if (bb->aux)
2965 34549209 : bb_predicate = *(ipa_predicate *)bb->aux;
2966 : else
2967 14 : bb_predicate = false;
2968 : }
2969 : else
2970 6386257 : bb_predicate = true;
2971 :
2972 40935480 : if (dump_file && (dump_flags & TDF_DETAILS))
2973 : {
2974 67 : fprintf (dump_file, "\n BB %i predicate:", bb->index);
2975 67 : bb_predicate.dump (dump_file, info->conds);
2976 : }
2977 :
2978 40935480 : if (fbi.info && nonconstant_names.exists ())
2979 : {
2980 34549223 : ipa_predicate phi_predicate;
2981 34549223 : bool first_phi = true;
2982 :
2983 35250566 : for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
2984 701343 : gsi_next (&bsi))
2985 : {
2986 6404393 : if (first_phi
2987 6404393 : && !phi_result_unknown_predicate (&fbi, info,
2988 : params_summary,
2989 : bb,
2990 : &phi_predicate,
2991 : nonconstant_names))
2992 : break;
2993 701343 : first_phi = false;
2994 701343 : if (dump_file && (dump_flags & TDF_DETAILS))
2995 : {
2996 3 : fprintf (dump_file, " ");
2997 3 : print_gimple_stmt (dump_file, gsi_stmt (bsi), 0);
2998 : }
2999 701343 : predicate_for_phi_result (info, bsi.phi (), &phi_predicate,
3000 : nonconstant_names);
3001 : }
3002 : }
3003 :
3004 40935480 : fix_builtin_expect_stmt = find_foldable_builtin_expect (bb);
3005 :
3006 40935480 : for (gimple_stmt_iterator bsi = gsi_start_nondebug_bb (bb);
3007 182035781 : !gsi_end_p (bsi); gsi_next_nondebug (&bsi))
3008 : {
3009 141100301 : gimple *stmt = gsi_stmt (bsi);
3010 141100301 : if (!gimple_plf (stmt, STMT_NECESSARY))
3011 : {
3012 5520379 : if (dump_file && (dump_flags & TDF_DETAILS))
3013 : {
3014 19 : fprintf (dump_file, " skipping unnecesary stmt ");
3015 19 : print_gimple_stmt (dump_file, stmt, 0);
3016 : }
3017 : /* TODO: const calls used only to produce values for
3018 : builtion_unreachable guards should not be accounted. However
3019 : we still want to inline them and this does does not work well
3020 : with the cost model. For now account them as usual. */
3021 5520379 : if (!is_gimple_call (stmt)
3022 5520379 : || gimple_call_internal_p (stmt))
3023 5244909 : continue;
3024 : }
3025 135855392 : int this_size = estimate_num_insns (stmt, &eni_size_weights);
3026 135855392 : int this_time = estimate_num_insns (stmt, &eni_time_weights);
3027 135855392 : int prob;
3028 135855392 : ipa_predicate will_be_nonconstant;
3029 :
3030 : /* This relation stmt should be folded after we remove
3031 : __builtin_expect call. Adjust the cost here. */
3032 135855392 : if (stmt == fix_builtin_expect_stmt)
3033 : {
3034 177624 : this_size--;
3035 177624 : this_time--;
3036 : }
3037 :
3038 135855392 : if (dump_file && (dump_flags & TDF_DETAILS))
3039 : {
3040 201 : fprintf (dump_file, " ");
3041 201 : print_gimple_stmt (dump_file, stmt, 0);
3042 201 : fprintf (dump_file, "\t\tfreq:%3.2f size:%3i time:%3i\n",
3043 : freq.to_double (), this_size,
3044 : this_time);
3045 : }
3046 :
3047 135855392 : if (is_gimple_call (stmt)
3048 135855392 : && !gimple_call_internal_p (stmt))
3049 : {
3050 22857280 : struct cgraph_edge *edge = node->get_edge (stmt);
3051 22857280 : ipa_call_summary *es = ipa_call_summaries->get_create (edge);
3052 :
3053 : /* Special case: results of BUILT_IN_CONSTANT_P will be always
3054 : resolved as constant. We however don't want to optimize
3055 : out the cgraph edges. */
3056 22857280 : if (nonconstant_names.exists ()
3057 19975328 : && gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P)
3058 28305 : && gimple_call_lhs (stmt)
3059 22885585 : && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
3060 : {
3061 28305 : ipa_predicate false_p = false;
3062 28305 : nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
3063 28305 : = false_p;
3064 : }
3065 22857280 : if (ipa_node_params_sum)
3066 : {
3067 19988099 : int count = gimple_call_num_args (stmt);
3068 19988099 : int i;
3069 :
3070 19988099 : if (count)
3071 17065999 : es->param.safe_grow_cleared (count, true);
3072 58344459 : for (i = 0; i < count; i++)
3073 : {
3074 38356360 : int prob = param_change_prob (&fbi, stmt, i);
3075 38356360 : gcc_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
3076 38356360 : es->param[i].change_prob = prob;
3077 38356360 : es->param[i].points_to_local_or_readonly_memory
3078 38356360 : = points_to_local_or_readonly_memory_p
3079 38356360 : (gimple_call_arg (stmt, i));
3080 38356360 : es->param[i].points_to_possible_sra_candidate
3081 38356360 : = points_to_possible_sra_candidate_p
3082 38356360 : (gimple_call_arg (stmt, i));
3083 : }
3084 : }
3085 : /* We cannot setup VLA parameters during inlining. */
3086 67045366 : for (unsigned int i = 0; i < gimple_call_num_args (stmt); ++i)
3087 44188482 : if (TREE_CODE (gimple_call_arg (stmt, i)) == WITH_SIZE_EXPR)
3088 : {
3089 396 : edge->inline_failed = CIF_FUNCTION_NOT_INLINABLE;
3090 396 : break;
3091 : }
3092 22857280 : es->call_stmt_size = this_size;
3093 22857280 : es->call_stmt_time = this_time;
3094 22857280 : es->loop_depth = bb_loop_depth (bb);
3095 22857280 : edge_set_predicate (edge, &bb_predicate);
3096 22857280 : if (edge->speculative)
3097 : {
3098 0 : cgraph_edge *indirect
3099 0 : = edge->speculative_call_indirect_edge ();
3100 0 : ipa_call_summary *es2
3101 0 : = ipa_call_summaries->get_create (indirect);
3102 0 : ipa_call_summaries->duplicate (edge, indirect,
3103 : es, es2);
3104 :
3105 : /* Edge is the first direct call.
3106 : create and duplicate call summaries for multiple
3107 : speculative call targets. */
3108 0 : for (cgraph_edge *direct
3109 0 : = edge->next_speculative_call_target ();
3110 0 : direct;
3111 0 : direct = direct->next_speculative_call_target ())
3112 : {
3113 0 : ipa_call_summary *es3
3114 0 : = ipa_call_summaries->get_create (direct);
3115 0 : ipa_call_summaries->duplicate (edge, direct,
3116 : es, es3);
3117 : }
3118 : }
3119 :
3120 : /* If dealing with a carrying edge, copy its summary over to its
3121 : attached edges as well. */
3122 22857280 : if (edge->has_callback)
3123 : {
3124 15081 : cgraph_edge *cbe;
3125 30162 : for (cbe = edge->first_callback_edge (); cbe;
3126 15081 : cbe = cbe->next_callback_edge ())
3127 : {
3128 15081 : ipa_call_summary *es2 = ipa_call_summaries->get_create (cbe);
3129 15081 : ipa_call_summaries->duplicate (edge, cbe, es, es2);
3130 : /* Unlike speculative edges, callback edges have no real
3131 : size or time; the call doesn't exist. Reflect that in
3132 : their summaries. */
3133 15081 : es2->call_stmt_size = 0;
3134 15081 : es2->call_stmt_time = 0;
3135 : }
3136 : }
3137 : }
3138 :
3139 : /* TODO: When conditional jump or switch is known to be constant, but
3140 : we did not translate it into the predicates, we really can account
3141 : just maximum of the possible paths. */
3142 135855392 : if (fbi.info)
3143 113736361 : will_be_nonconstant
3144 113736361 : = will_be_nonconstant_predicate (&fbi, info, params_summary,
3145 : stmt, nonconstant_names);
3146 : else
3147 22119031 : will_be_nonconstant = true;
3148 135855392 : if (this_time || this_size)
3149 : {
3150 111325356 : sreal final_time = (sreal)this_time * freq;
3151 111325356 : prob = eliminated_by_inlining_prob (&fbi, stmt);
3152 111325356 : if (prob == 1 && dump_file && (dump_flags & TDF_DETAILS))
3153 11 : fprintf (dump_file,
3154 : "\t\t50%% will be eliminated by inlining\n");
3155 111325356 : if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
3156 18 : fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
3157 :
3158 111325356 : ipa_predicate p = bb_predicate & will_be_nonconstant;
3159 111325356 : int parm = load_or_store_of_ptr_parameter (&fbi, stmt);
3160 111325356 : ipa_predicate sra_predicate = true;
3161 111325356 : if (parm != -1)
3162 14028946 : sra_predicate &= add_condition (info, params_summary, parm,
3163 : ptr_type_node, NULL,
3164 7014473 : ipa_predicate::not_sra_candidate, NULL, 0);
3165 :
3166 : /* We can ignore statement when we proved it is never going
3167 : to happen, but we cannot do that for call statements
3168 : because edges are accounted specially. */
3169 :
3170 222650712 : if (*(is_gimple_call (stmt) ? &bb_predicate : &p) != false)
3171 : {
3172 110613031 : time += final_time;
3173 110613031 : size += this_size;
3174 : }
3175 :
3176 : /* We account everything but the calls. Calls have their own
3177 : size/time info attached to cgraph edges. This is necessary
3178 : in order to make the cost disappear after inlining. */
3179 111325356 : if (!is_gimple_call (stmt))
3180 : {
3181 89152845 : if (prob)
3182 : {
3183 14803649 : ipa_predicate ip
3184 14803649 : = bb_predicate & ipa_predicate::not_inlined () & sra_predicate;
3185 29607298 : info->account_size_time (this_size * prob,
3186 14803649 : (final_time * prob) / 2, ip,
3187 : p);
3188 : }
3189 14803649 : if (prob != 2)
3190 164579722 : info->account_size_time (this_size * (2 - prob),
3191 82289861 : (final_time * (2 - prob) / 2),
3192 164579722 : bb_predicate & sra_predicate,
3193 : p);
3194 : }
3195 :
3196 111325356 : if (!info->fp_expressions && fp_expression_p (stmt))
3197 : {
3198 595308 : info->fp_expressions = true;
3199 595308 : if (dump_file)
3200 9 : fprintf (dump_file, " fp_expression set\n");
3201 : }
3202 : }
3203 :
3204 : /* For target specific information, we want to scan all statements
3205 : rather than those statements with non-zero weights, to avoid
3206 : missing to scan something interesting for target information,
3207 : such as: internal function calls. */
3208 135855392 : if (scan_for_target_info)
3209 0 : scan_for_target_info =
3210 0 : targetm.target_option.update_ipa_fn_target_info
3211 0 : (info->target_info, stmt);
3212 :
3213 : /* Account cost of address calculations in the statements. */
3214 525259074 : for (unsigned int i = 0; i < gimple_num_ops (stmt); i++)
3215 : {
3216 389403682 : for (tree op = gimple_op (stmt, i);
3217 749705538 : op && handled_component_p (op);
3218 47038843 : op = TREE_OPERAND (op, 0))
3219 47038843 : if ((TREE_CODE (op) == ARRAY_REF
3220 47038843 : || TREE_CODE (op) == ARRAY_RANGE_REF)
3221 47038843 : && TREE_CODE (TREE_OPERAND (op, 1)) == SSA_NAME)
3222 : {
3223 2383627 : ipa_predicate p = bb_predicate;
3224 2383627 : if (fbi.info)
3225 1864748 : p = p & will_be_nonconstant_expr_predicate
3226 1864748 : (&fbi, info, params_summary,
3227 1864748 : TREE_OPERAND (op, 1),
3228 1864748 : nonconstant_names);
3229 2383627 : if (p != false)
3230 : {
3231 2377915 : time += freq;
3232 2377915 : size += 1;
3233 2377915 : if (dump_file)
3234 24 : fprintf (dump_file,
3235 : "\t\tAccounting address calculation.\n");
3236 2377915 : info->account_size_time (ipa_fn_summary::size_scale,
3237 : freq,
3238 : bb_predicate,
3239 : p);
3240 : }
3241 : }
3242 : }
3243 :
3244 : }
3245 : }
3246 7199463 : free (order);
3247 :
3248 7199463 : if (nonconstant_names.exists () && !early)
3249 : {
3250 1381691 : ipa_fn_summary *s = ipa_fn_summaries->get (node);
3251 1381691 : unsigned max_loop_predicates = opt_for_fn (node->decl,
3252 : param_ipa_max_loop_predicates);
3253 :
3254 1381691 : if (dump_file && (dump_flags & TDF_DETAILS))
3255 14 : flow_loops_dump (dump_file, NULL, 0);
3256 1381691 : scev_initialize ();
3257 4766985 : for (auto loop : loops_list (cfun, 0))
3258 : {
3259 621912 : ipa_predicate loop_iterations = true;
3260 621912 : sreal header_freq;
3261 621912 : edge ex;
3262 621912 : unsigned int j;
3263 621912 : class tree_niter_desc niter_desc;
3264 621912 : if (!loop->header->aux)
3265 0 : continue;
3266 :
3267 621912 : profile_count hdr_count = loop->header->count;
3268 621912 : sreal hdr_freq = hdr_count.to_sreal_scale (entry_count);
3269 :
3270 621912 : bb_predicate = *(ipa_predicate *)loop->header->aux;
3271 621912 : auto_vec<edge> exits = get_loop_exit_edges (loop);
3272 1624512 : FOR_EACH_VEC_ELT (exits, j, ex)
3273 1002600 : if (number_of_iterations_exit (loop, ex, &niter_desc, false)
3274 1002600 : && !is_gimple_min_invariant (niter_desc.niter))
3275 : {
3276 153216 : ipa_predicate will_be_nonconstant
3277 153216 : = will_be_nonconstant_expr_predicate (&fbi, info,
3278 : params_summary,
3279 : niter_desc.niter,
3280 : nonconstant_names);
3281 153216 : if (will_be_nonconstant != true)
3282 61521 : will_be_nonconstant = bb_predicate & will_be_nonconstant;
3283 153216 : if (will_be_nonconstant != true
3284 214737 : && will_be_nonconstant != false)
3285 60961 : loop_iterations &= will_be_nonconstant;
3286 : }
3287 621912 : add_freqcounting_predicate (&s->loop_iterations, loop_iterations,
3288 : hdr_freq, max_loop_predicates);
3289 621914 : }
3290 :
3291 : /* To avoid quadratic behavior we analyze stride predicates only
3292 : with respect to the containing loop. Thus we simply iterate
3293 : over all defs in the outermost loop body. */
3294 1381691 : for (class loop *loop = loops_for_fn (cfun)->tree_root->inner;
3295 1870142 : loop != NULL; loop = loop->next)
3296 : {
3297 488451 : ipa_predicate loop_stride = true;
3298 488451 : basic_block *body = get_loop_body (loop);
3299 488451 : profile_count hdr_count = loop->header->count;
3300 488451 : sreal hdr_freq = hdr_count.to_sreal_scale (entry_count);
3301 2879257 : for (unsigned i = 0; i < loop->num_nodes; i++)
3302 : {
3303 2390806 : gimple_stmt_iterator gsi;
3304 2390806 : if (!body[i]->aux)
3305 7 : continue;
3306 :
3307 2390799 : bb_predicate = *(ipa_predicate *)body[i]->aux;
3308 15978028 : for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
3309 11196430 : gsi_next (&gsi))
3310 : {
3311 11196430 : gimple *stmt = gsi_stmt (gsi);
3312 :
3313 11196430 : if (!is_gimple_assign (stmt))
3314 11043601 : continue;
3315 :
3316 5378766 : tree def = gimple_assign_lhs (stmt);
3317 5378766 : if (TREE_CODE (def) != SSA_NAME)
3318 1025325 : continue;
3319 :
3320 4353441 : affine_iv iv;
3321 8706882 : if (!simple_iv (loop_containing_stmt (stmt),
3322 : loop_containing_stmt (stmt),
3323 : def, &iv, true)
3324 4353441 : || is_gimple_min_invariant (iv.step))
3325 4200612 : continue;
3326 :
3327 152829 : ipa_predicate will_be_nonconstant
3328 152829 : = will_be_nonconstant_expr_predicate (&fbi, info,
3329 : params_summary,
3330 : iv.step,
3331 : nonconstant_names);
3332 152829 : if (will_be_nonconstant != true)
3333 54259 : will_be_nonconstant = bb_predicate & will_be_nonconstant;
3334 152829 : if (will_be_nonconstant != true
3335 207088 : && will_be_nonconstant != false)
3336 35381 : loop_stride = loop_stride & will_be_nonconstant;
3337 : }
3338 : }
3339 488451 : add_freqcounting_predicate (&s->loop_strides, loop_stride,
3340 : hdr_freq, max_loop_predicates);
3341 488451 : free (body);
3342 : }
3343 1381691 : scev_finalize ();
3344 : }
3345 63076171 : FOR_ALL_BB_FN (bb, my_function)
3346 : {
3347 55876708 : edge e;
3348 55876708 : edge_iterator ei;
3349 :
3350 55876708 : if (bb->aux)
3351 41331439 : edge_predicate_pool.remove ((ipa_predicate *)bb->aux);
3352 55876708 : bb->aux = NULL;
3353 118822565 : FOR_EACH_EDGE (e, ei, bb->succs)
3354 : {
3355 62945857 : if (e->aux)
3356 2507671 : edge_predicate_pool.remove ((ipa_predicate *)e->aux);
3357 62945857 : e->aux = NULL;
3358 : }
3359 : }
3360 7199463 : ipa_fn_summary *s = ipa_fn_summaries->get (node);
3361 7199463 : ipa_size_summary *ss = ipa_size_summaries->get (node);
3362 7199463 : s->time = time;
3363 7199463 : ss->self_size = size;
3364 7199463 : nonconstant_names.release ();
3365 7199463 : ipa_release_body_info (&fbi);
3366 7199463 : if (opt_for_fn (node->decl, optimize))
3367 : {
3368 6302282 : if (!early)
3369 1381691 : loop_optimizer_finalize ();
3370 4920591 : else if (!ipa_edge_args_sum)
3371 4920577 : ipa_free_all_node_params ();
3372 6302282 : free_dominance_info (CDI_DOMINATORS);
3373 6302282 : free_dominance_info (CDI_POST_DOMINATORS);
3374 : }
3375 7199463 : if (dump_file)
3376 : {
3377 247 : fprintf (dump_file, "\n");
3378 247 : ipa_dump_fn_summary (dump_file, node);
3379 : }
3380 7199463 : }
3381 :
3382 :
3383 : /* Compute function summary.
3384 : EARLY is true when we compute parameters during early opts. */
3385 :
3386 : void
3387 7200751 : compute_fn_summary (struct cgraph_node *node, bool early)
3388 : {
3389 7200751 : HOST_WIDE_INT self_stack_size;
3390 7200751 : struct cgraph_edge *e;
3391 :
3392 7200751 : gcc_assert (!node->inlined_to);
3393 :
3394 7200751 : if (!ipa_fn_summaries)
3395 214623 : ipa_fn_summary_alloc ();
3396 :
3397 : /* Create a new ipa_fn_summary. */
3398 7200751 : ((ipa_fn_summary_t *)ipa_fn_summaries)->remove_callees (node);
3399 7200751 : ipa_fn_summaries->remove (node);
3400 7200751 : class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
3401 7200751 : class ipa_size_summary *size_info = ipa_size_summaries->get_create (node);
3402 :
3403 : /* Estimate the stack size for the function if we're optimizing. */
3404 12605836 : self_stack_size = optimize && !node->thunk
3405 13503033 : ? estimated_stack_frame_size (node) : 0;
3406 7200751 : size_info->estimated_self_stack_size = self_stack_size;
3407 7200751 : info->estimated_stack_size = self_stack_size;
3408 :
3409 7200751 : if (node->thunk)
3410 : {
3411 1288 : ipa_call_summary *es = ipa_call_summaries->get_create (node->callees);
3412 1288 : ipa_predicate t = true;
3413 :
3414 1288 : node->can_change_signature = false;
3415 1288 : es->call_stmt_size = eni_size_weights.call_cost;
3416 1288 : es->call_stmt_time = eni_time_weights.call_cost;
3417 3864 : info->account_size_time (ipa_fn_summary::size_scale
3418 1288 : * opt_for_fn (node->decl,
3419 : param_uninlined_function_thunk_insns),
3420 1288 : opt_for_fn (node->decl,
3421 : param_uninlined_function_thunk_time), t, t);
3422 1288 : t = ipa_predicate::not_inlined ();
3423 1288 : info->account_size_time (2 * ipa_fn_summary::size_scale, 0, t, t);
3424 1288 : ipa_update_overall_fn_summary (node);
3425 1288 : size_info->self_size = size_info->size;
3426 1288 : if (stdarg_p (TREE_TYPE (node->decl)))
3427 : {
3428 9 : info->inlinable = false;
3429 9 : node->callees->inline_failed = CIF_VARIADIC_THUNK;
3430 : }
3431 : else
3432 1279 : info->inlinable = true;
3433 : }
3434 : else
3435 : {
3436 : /* Even is_gimple_min_invariant rely on current_function_decl. */
3437 7199463 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
3438 :
3439 : /* During IPA profile merging we may be called w/o virtual SSA form
3440 : built. */
3441 7199463 : update_ssa (TODO_update_ssa_only_virtuals);
3442 :
3443 : /* Can this function be inlined at all? */
3444 7199463 : if (!opt_for_fn (node->decl, optimize)
3445 8096644 : && !lookup_attribute ("always_inline",
3446 897181 : DECL_ATTRIBUTES (node->decl)))
3447 829165 : info->inlinable = false;
3448 : else
3449 6370298 : info->inlinable = tree_inlinable_function_p (node->decl);
3450 :
3451 7199463 : bool no_signature = false;
3452 :
3453 : /* Don't allow signature changes for functions which have
3454 : [[gnu::musttail]] or [[clang::musttail]] calls. Sometimes
3455 : (more often on targets which pass everything on the stack)
3456 : signature changes can result in tail calls being impossible
3457 : even when without the signature changes they would be ok.
3458 : See PR121023. */
3459 7199463 : if (cfun->has_musttail)
3460 : {
3461 1426 : if (dump_file)
3462 0 : fprintf (dump_file, "No signature change:"
3463 : " function has calls with musttail attribute.\n");
3464 : no_signature = true;
3465 : }
3466 :
3467 : /* Type attributes can use parameter indices to describe them.
3468 : Special case fn spec since we can safely preserve them in
3469 : modref summaries. */
3470 7199463 : for (tree list = TYPE_ATTRIBUTES (TREE_TYPE (node->decl));
3471 7591431 : list && !no_signature; list = TREE_CHAIN (list))
3472 391968 : if (!ipa_param_adjustments::type_attribute_allowed_p
3473 391968 : (get_attribute_name (list)))
3474 : {
3475 157815 : if (dump_file)
3476 : {
3477 0 : fprintf (dump_file, "No signature change:"
3478 : " function type has unhandled attribute %s.\n",
3479 0 : IDENTIFIER_POINTER (get_attribute_name (list)));
3480 : }
3481 : no_signature = true;
3482 : }
3483 7199463 : for (tree parm = DECL_ARGUMENTS (node->decl);
3484 21503451 : parm && !no_signature; parm = DECL_CHAIN (parm))
3485 14303988 : if (variably_modified_type_p (TREE_TYPE (parm), node->decl))
3486 : {
3487 15710 : if (dump_file)
3488 : {
3489 0 : fprintf (dump_file, "No signature change:"
3490 : " has parameter with variably modified type.\n");
3491 : }
3492 : no_signature = true;
3493 : }
3494 :
3495 : /* Likewise for #pragma omp declare simd functions or functions
3496 : with simd attribute. */
3497 7199463 : if (no_signature
3498 14223975 : || lookup_attribute ("omp declare simd",
3499 7024512 : DECL_ATTRIBUTES (node->decl)))
3500 176639 : node->can_change_signature = false;
3501 : else
3502 : {
3503 : /* Otherwise, inlinable functions always can change signature. */
3504 7022824 : if (info->inlinable)
3505 5644795 : node->can_change_signature = true;
3506 : else
3507 : {
3508 : /* Functions calling builtin_apply cannot change signature. */
3509 5400948 : for (e = node->callees; e; e = e->next_callee)
3510 : {
3511 4055463 : tree cdecl = e->callee->decl;
3512 4055463 : if (fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS,
3513 : BUILT_IN_VA_START))
3514 : break;
3515 : }
3516 1378029 : node->can_change_signature = !e;
3517 : }
3518 : }
3519 7199463 : analyze_function_body (node, early);
3520 7199463 : pop_cfun ();
3521 : }
3522 :
3523 : /* Inlining characteristics are maintained by the cgraph_mark_inline. */
3524 7200751 : size_info->size = size_info->self_size;
3525 7200751 : info->estimated_stack_size = size_info->estimated_self_stack_size;
3526 :
3527 : /* Code above should compute exactly the same result as
3528 : ipa_update_overall_fn_summary except for case when speculative
3529 : edges are present since these are accounted to size but not
3530 : self_size. Do not compare time since different order the roundoff
3531 : errors result in slight changes. */
3532 7200751 : ipa_update_overall_fn_summary (node);
3533 7200751 : if (flag_checking)
3534 : {
3535 7677224 : for (e = node->indirect_calls; e; e = e->next_callee)
3536 476555 : if (e->speculative)
3537 : break;
3538 7200669 : gcc_assert (e || size_info->size == size_info->self_size);
3539 : }
3540 7200751 : }
3541 :
3542 :
3543 : /* Compute parameters of functions used by inliner using
3544 : current_function_decl. */
3545 :
3546 : static unsigned int
3547 5758205 : compute_fn_summary_for_current (void)
3548 : {
3549 5758205 : compute_fn_summary (cgraph_node::get (current_function_decl), true);
3550 5758205 : return 0;
3551 : }
3552 :
3553 : /* Estimate benefit devirtualizing indirect edge IE and return true if it can
3554 : be devirtualized and inlined, provided m_known_vals, m_known_contexts and
3555 : m_known_aggs in AVALS. Return false straight away if AVALS is NULL. */
3556 :
3557 : static bool
3558 4705630 : estimate_edge_devirt_benefit (struct cgraph_edge *ie,
3559 : int *size, int *time,
3560 : ipa_call_arg_values *avals)
3561 : {
3562 4705630 : tree target;
3563 4705630 : struct cgraph_node *callee;
3564 4705630 : class ipa_fn_summary *isummary;
3565 4705630 : enum availability avail;
3566 4705630 : bool speculative;
3567 :
3568 4705630 : if (!avals
3569 5911818 : || (!avals->m_known_vals.length() && !avals->m_known_contexts.length ()))
3570 : return false;
3571 1402972 : if (!opt_for_fn (ie->caller->decl, flag_indirect_inlining))
3572 : return false;
3573 :
3574 1396545 : target = ipa_get_indirect_edge_target
3575 1396545 : (ie->callee ? ie->speculative_call_indirect_edge () : ie,
3576 : avals, &speculative);
3577 1396545 : if (!target || speculative)
3578 : return false;
3579 :
3580 : /* If this is speculative call, turn its cost into 0; we will account
3581 : the call when processing the indirect call. */
3582 217510 : if (ie->callee)
3583 : {
3584 4106 : gcc_checking_assert (ie->speculative && *size > 0);
3585 4106 : *size = 0;
3586 4106 : *time = 0;
3587 : }
3588 : else
3589 : {
3590 : /* Account for difference in cost between indirect and direct calls. */
3591 213404 : *size -= (eni_size_weights.indirect_call_cost - eni_size_weights.call_cost);
3592 213404 : *time -= (eni_time_weights.indirect_call_cost - eni_time_weights.call_cost);
3593 : }
3594 217510 : gcc_checking_assert (*time >= 0);
3595 217510 : gcc_checking_assert (*size >= 0);
3596 :
3597 217510 : callee = cgraph_node::get (target);
3598 217510 : if (!callee || !callee->definition)
3599 : return false;
3600 196824 : callee = callee->function_symbol (&avail);
3601 196824 : if (avail < AVAIL_AVAILABLE)
3602 : return false;
3603 196798 : isummary = ipa_fn_summaries->get (callee);
3604 196798 : if (isummary == NULL)
3605 : return false;
3606 :
3607 196784 : return isummary->inlinable;
3608 : }
3609 :
3610 : /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
3611 : handle edge E with probability PROB. Set HINTS accordingly if edge may be
3612 : devirtualized. AVALS, if non-NULL, describes the context of the call site
3613 : as far as values of parameters are concerened. */
3614 :
3615 : static inline void
3616 210829593 : estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *min_size,
3617 : sreal *time, ipa_call_arg_values *avals,
3618 : ipa_hints *hints)
3619 : {
3620 210829593 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3621 210829593 : int call_size = es->call_stmt_size;
3622 210829593 : int call_time = es->call_stmt_time;
3623 210829593 : int cur_size;
3624 :
3625 206685057 : if ((!e->callee || e->speculative)
3626 211390687 : && estimate_edge_devirt_benefit (e, &call_size, &call_time, avals))
3627 : {
3628 196442 : if (hints && e->maybe_hot_p ())
3629 187198 : *hints |= INLINE_HINT_indirect_call;
3630 : }
3631 210829593 : cur_size = call_size * ipa_fn_summary::size_scale;
3632 210829593 : *size += cur_size;
3633 210829593 : if (min_size)
3634 27995435 : *min_size += cur_size;
3635 210829593 : if (time)
3636 201378268 : *time += ((sreal)call_time) * e->sreal_frequency ();
3637 210829593 : }
3638 :
3639 :
3640 : /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3641 : calls in NODE. POSSIBLE_TRUTHS and AVALS describe the context of the call
3642 : site.
3643 :
3644 : Helper for estimate_calls_size_and_time which does the same but
3645 : (in most cases) faster. */
3646 :
3647 : static void
3648 77217421 : estimate_calls_size_and_time_1 (struct cgraph_node *node, int *size,
3649 : int *min_size, sreal *time,
3650 : ipa_hints *hints,
3651 : clause_t possible_truths,
3652 : ipa_call_arg_values *avals)
3653 : {
3654 77217421 : struct cgraph_edge *e;
3655 364040285 : for (e = node->callees; e; e = e->next_callee)
3656 : {
3657 286822864 : if (!e->inline_failed)
3658 : {
3659 57145390 : gcc_checking_assert (!ipa_call_summaries->get (e));
3660 57145390 : estimate_calls_size_and_time_1 (e->callee, size, min_size, time,
3661 : hints, possible_truths, avals);
3662 :
3663 57145390 : continue;
3664 : }
3665 229677474 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3666 :
3667 : /* Do not care about zero sized builtins. */
3668 229677474 : if (!es->call_stmt_size)
3669 : {
3670 31415049 : gcc_checking_assert (!es->call_stmt_time);
3671 31415049 : continue;
3672 : }
3673 198262425 : if (!es->predicate
3674 198262425 : || es->predicate->evaluate (possible_truths))
3675 : {
3676 : /* Predicates of calls shall not use NOT_CHANGED codes,
3677 : so we do not need to compute probabilities. */
3678 195665993 : estimate_edge_size_and_time (e, size,
3679 195665993 : es->predicate ? NULL : min_size,
3680 : time, avals, hints);
3681 : }
3682 : }
3683 81083195 : for (e = node->indirect_calls; e; e = e->next_callee)
3684 : {
3685 3865774 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3686 3865774 : if (!es->predicate
3687 3865774 : || es->predicate->evaluate (possible_truths))
3688 3835304 : estimate_edge_size_and_time (e, size,
3689 3835304 : es->predicate ? NULL : min_size,
3690 : time, avals, hints);
3691 : }
3692 77217421 : }
3693 :
3694 : /* Populate sum->call_size_time_table for edges from NODE. */
3695 :
3696 : static void
3697 3181168 : summarize_calls_size_and_time (struct cgraph_node *node,
3698 : ipa_fn_summary *sum)
3699 : {
3700 3181168 : struct cgraph_edge *e;
3701 14785299 : for (e = node->callees; e; e = e->next_callee)
3702 : {
3703 11604131 : if (!e->inline_failed)
3704 : {
3705 1446319 : gcc_checking_assert (!ipa_call_summaries->get (e));
3706 1446319 : summarize_calls_size_and_time (e->callee, sum);
3707 1446319 : continue;
3708 : }
3709 10157812 : int size = 0;
3710 10157812 : sreal time = 0;
3711 :
3712 10157812 : estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
3713 :
3714 10157812 : ipa_predicate pred = true;
3715 10157812 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3716 :
3717 10157812 : if (es->predicate)
3718 1860756 : pred = *es->predicate;
3719 10157812 : sum->account_size_time (size, time, pred, pred, true);
3720 : }
3721 3490400 : for (e = node->indirect_calls; e; e = e->next_callee)
3722 : {
3723 309232 : int size = 0;
3724 309232 : sreal time = 0;
3725 :
3726 309232 : estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
3727 309232 : ipa_predicate pred = true;
3728 309232 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3729 :
3730 309232 : if (es->predicate)
3731 79026 : pred = *es->predicate;
3732 309232 : sum->account_size_time (size, time, pred, pred, true);
3733 : }
3734 3181168 : }
3735 :
3736 : /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3737 : calls in NODE. POSSIBLE_TRUTHS and AVALS (the latter if non-NULL) describe
3738 : context of the call site. */
3739 :
3740 : static void
3741 20072052 : estimate_calls_size_and_time (struct cgraph_node *node, int *size,
3742 : int *min_size, sreal *time,
3743 : ipa_hints *hints,
3744 : clause_t possible_truths,
3745 : ipa_call_arg_values *avals)
3746 : {
3747 20072052 : class ipa_fn_summary *sum = ipa_fn_summaries->get (node);
3748 20072052 : bool use_table = true;
3749 :
3750 20072052 : gcc_assert (node->callees || node->indirect_calls);
3751 :
3752 : /* During early inlining we do not calculate info for very
3753 : large functions and thus there is no need for producing
3754 : summaries. */
3755 20072052 : if (!ipa_node_params_sum)
3756 : use_table = false;
3757 : /* Do not calculate summaries for simple wrappers; it is waste
3758 : of memory. */
3759 10979649 : else if (node->callees && !node->indirect_calls
3760 10243775 : && node->callees->inline_failed && !node->callees->next_callee)
3761 : use_table = false;
3762 : /* If there is an indirect edge that may be optimized, we need
3763 : to go the slow way. */
3764 8831431 : else if (avals
3765 8831431 : && (avals->m_known_vals.length ()
3766 3224193 : || avals->m_known_contexts.length ()
3767 3088930 : || avals->m_known_aggs.length ()))
3768 : {
3769 4219542 : ipa_node_params *params_summary = ipa_node_params_sum->get (node);
3770 4219542 : unsigned int nargs = params_summary
3771 4219542 : ? ipa_get_param_count (params_summary) : 0;
3772 :
3773 14173370 : for (unsigned int i = 0; i < nargs && use_table; i++)
3774 : {
3775 9953828 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3776 9953828 : && (avals->safe_sval_at (i)
3777 257480 : || (ipa_argagg_value_list (avals).value_for_index_p (i))))
3778 : use_table = false;
3779 9858342 : else if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3780 9858342 : && (avals->m_known_contexts.length () > i
3781 10107679 : && !avals->m_known_contexts[i].useless_p ()))
3782 : use_table = false;
3783 : }
3784 : }
3785 :
3786 : /* Fast path is via the call size time table. */
3787 4219542 : if (use_table)
3788 : {
3789 : /* Build summary if it is absent. */
3790 8582267 : if (!sum->call_size_time_table.length ())
3791 : {
3792 873597 : ipa_predicate true_pred = true;
3793 873597 : sum->account_size_time (0, 0, true_pred, true_pred, true);
3794 873597 : summarize_calls_size_and_time (node, sum);
3795 : }
3796 :
3797 8582267 : int old_size = *size;
3798 8582267 : sreal old_time = time ? *time : 0;
3799 :
3800 8582267 : if (min_size)
3801 8582267 : *min_size += sum->call_size_time_table[0].size;
3802 :
3803 : unsigned int i;
3804 : size_time_entry *e;
3805 :
3806 : /* Walk the table and account sizes and times. */
3807 22999277 : for (i = 0; sum->call_size_time_table.iterate (i, &e);
3808 : i++)
3809 14417010 : if (e->exec_predicate.evaluate (possible_truths))
3810 : {
3811 13291255 : *size += e->size;
3812 13291255 : if (time)
3813 11214664 : *time += e->time;
3814 : }
3815 :
3816 : /* Be careful and see if both methods agree. */
3817 21 : if ((flag_checking || dump_file)
3818 : /* Do not try to sanity check when we know we lost some
3819 : precision. */
3820 8582267 : && sum->call_size_time_table.length ()
3821 : < ipa_fn_summary::max_size_time_table_size)
3822 : {
3823 8582246 : estimate_calls_size_and_time_1 (node, &old_size, NULL, &old_time, NULL,
3824 : possible_truths, avals);
3825 8582246 : gcc_assert (*size == old_size);
3826 15799080 : if (time && (*time - old_time > 1 || *time - old_time < -1)
3827 8582275 : && dump_file)
3828 0 : fprintf (dump_file, "Time mismatch in call summary %f!=%f\n",
3829 : old_time.to_double (),
3830 : time->to_double ());
3831 : }
3832 : }
3833 : /* Slow path by walking all edges. */
3834 : else
3835 11489785 : estimate_calls_size_and_time_1 (node, size, min_size, time, hints,
3836 : possible_truths, avals);
3837 20072052 : }
3838 :
3839 : /* Main constructor for ipa call context. Memory allocation of ARG_VALUES
3840 : is owned by the caller. INLINE_PARAM_SUMMARY is also owned by the
3841 : caller. */
3842 :
3843 19215862 : ipa_call_context::ipa_call_context (cgraph_node *node, clause_t possible_truths,
3844 : clause_t nonspec_possible_truths,
3845 : vec<inline_param_summary>
3846 : inline_param_summary,
3847 19215862 : ipa_auto_call_arg_values *arg_values)
3848 19215862 : : m_node (node), m_possible_truths (possible_truths),
3849 19215862 : m_nonspec_possible_truths (nonspec_possible_truths),
3850 19215862 : m_inline_param_summary (inline_param_summary),
3851 19215862 : m_avals (arg_values)
3852 : {
3853 19215862 : }
3854 :
3855 : /* Set THIS to be a duplicate of CTX. Copy all relevant info. */
3856 :
3857 : void
3858 1914561 : ipa_cached_call_context::duplicate_from (const ipa_call_context &ctx)
3859 : {
3860 1914561 : m_node = ctx.m_node;
3861 1914561 : m_possible_truths = ctx.m_possible_truths;
3862 1914561 : m_nonspec_possible_truths = ctx.m_nonspec_possible_truths;
3863 1914561 : ipa_node_params *params_summary = ipa_node_params_sum->get (m_node);
3864 1914561 : unsigned int nargs = params_summary
3865 1914561 : ? ipa_get_param_count (params_summary) : 0;
3866 :
3867 1914561 : m_inline_param_summary = vNULL;
3868 : /* Copy the info only if there is at least one useful entry. */
3869 1914561 : if (ctx.m_inline_param_summary.exists ())
3870 : {
3871 1702043 : unsigned int n = MIN (ctx.m_inline_param_summary.length (), nargs);
3872 :
3873 3877552 : for (unsigned int i = 0; i < n; i++)
3874 3170716 : if (ipa_is_param_used_by_ipa_predicates (params_summary, i)
3875 4519056 : && !ctx.m_inline_param_summary[i].useless_p ())
3876 : {
3877 995207 : m_inline_param_summary
3878 995207 : = ctx.m_inline_param_summary.copy ();
3879 995207 : break;
3880 : }
3881 : }
3882 1914561 : m_avals.m_known_vals = vNULL;
3883 1914561 : if (ctx.m_avals.m_known_vals.exists ())
3884 : {
3885 1914561 : unsigned int n = MIN (ctx.m_avals.m_known_vals.length (), nargs);
3886 :
3887 4431997 : for (unsigned int i = 0; i < n; i++)
3888 2543608 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3889 2543608 : && ctx.m_avals.m_known_vals[i])
3890 : {
3891 26172 : m_avals.m_known_vals = ctx.m_avals.m_known_vals.copy ();
3892 26172 : break;
3893 : }
3894 : }
3895 :
3896 1914561 : m_avals.m_known_contexts = vNULL;
3897 1914561 : if (ctx.m_avals.m_known_contexts.exists ())
3898 : {
3899 1914561 : unsigned int n = MIN (ctx.m_avals.m_known_contexts.length (), nargs);
3900 :
3901 1916530 : for (unsigned int i = 0; i < n; i++)
3902 26392 : if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3903 26392 : && !ctx.m_avals.m_known_contexts[i].useless_p ())
3904 : {
3905 24423 : m_avals.m_known_contexts = ctx.m_avals.m_known_contexts.copy ();
3906 24423 : break;
3907 : }
3908 : }
3909 :
3910 1914561 : m_avals.m_known_aggs = vNULL;
3911 1914561 : if (ctx.m_avals.m_known_aggs.exists ())
3912 : {
3913 1914561 : const ipa_argagg_value_list avl (&ctx.m_avals);
3914 6340331 : for (unsigned int i = 0; i < nargs; i++)
3915 4428615 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3916 4428615 : && avl.value_for_index_p (i))
3917 : {
3918 2845 : m_avals.m_known_aggs = ctx.m_avals.m_known_aggs.copy ();
3919 2845 : break;
3920 : }
3921 : }
3922 :
3923 1914561 : m_avals.m_known_value_ranges = vNULL;
3924 1914561 : }
3925 :
3926 : /* Release memory used by known_vals/contexts/aggs vectors. and
3927 : inline_param_summary. */
3928 :
3929 : void
3930 3110929 : ipa_cached_call_context::release ()
3931 : {
3932 : /* See if context is initialized at first place. */
3933 3110929 : if (!m_node)
3934 : return;
3935 1914561 : m_avals.m_known_aggs.release ();
3936 1914561 : m_avals.m_known_vals.release ();
3937 1914561 : m_avals.m_known_contexts.release ();
3938 1914561 : m_inline_param_summary.release ();
3939 : }
3940 :
3941 : /* Return true if CTX describes the same call context as THIS. */
3942 :
3943 : bool
3944 6972153 : ipa_call_context::equal_to (const ipa_call_context &ctx)
3945 : {
3946 6972153 : if (m_node != ctx.m_node
3947 5775785 : || m_possible_truths != ctx.m_possible_truths
3948 5189092 : || m_nonspec_possible_truths != ctx.m_nonspec_possible_truths)
3949 : return false;
3950 :
3951 5189092 : ipa_node_params *params_summary = ipa_node_params_sum->get (m_node);
3952 5189092 : unsigned int nargs = params_summary
3953 5189092 : ? ipa_get_param_count (params_summary) : 0;
3954 :
3955 5189092 : if (m_inline_param_summary.exists () || ctx.m_inline_param_summary.exists ())
3956 : {
3957 15593333 : for (unsigned int i = 0; i < nargs; i++)
3958 : {
3959 10733375 : if (!ipa_is_param_used_by_ipa_predicates (params_summary, i))
3960 3358235 : continue;
3961 7375140 : if (i >= m_inline_param_summary.length ()
3962 5317092 : || m_inline_param_summary[i].useless_p ())
3963 : {
3964 3032328 : if (i < ctx.m_inline_param_summary.length ()
3965 3032328 : && !ctx.m_inline_param_summary[i].useless_p ())
3966 : return false;
3967 2986385 : continue;
3968 : }
3969 4342812 : if (i >= ctx.m_inline_param_summary.length ()
3970 4342809 : || ctx.m_inline_param_summary[i].useless_p ())
3971 : {
3972 46258 : if (i < m_inline_param_summary.length ()
3973 46258 : && !m_inline_param_summary[i].useless_p ())
3974 : return false;
3975 0 : continue;
3976 : }
3977 4296554 : if (!m_inline_param_summary[i].equal_to
3978 4296554 : (ctx.m_inline_param_summary[i]))
3979 : return false;
3980 : }
3981 : }
3982 5070815 : if (m_avals.m_known_vals.exists () || ctx.m_avals.m_known_vals.exists ())
3983 : {
3984 15600311 : for (unsigned int i = 0; i < nargs; i++)
3985 : {
3986 10539962 : if (!ipa_is_param_used_by_indirect_call (params_summary, i))
3987 10342341 : continue;
3988 197621 : if (i >= m_avals.m_known_vals.length () || !m_avals.m_known_vals[i])
3989 : {
3990 141144 : if (i < ctx.m_avals.m_known_vals.length ()
3991 141144 : && ctx.m_avals.m_known_vals[i])
3992 : return false;
3993 141055 : continue;
3994 : }
3995 56477 : if (i >= ctx.m_avals.m_known_vals.length ()
3996 56477 : || !ctx.m_avals.m_known_vals[i])
3997 : {
3998 : if (i < m_avals.m_known_vals.length () && m_avals.m_known_vals[i])
3999 : return false;
4000 : continue;
4001 : }
4002 56387 : if (m_avals.m_known_vals[i] != ctx.m_avals.m_known_vals[i])
4003 : return false;
4004 : }
4005 : }
4006 5060349 : if (m_avals.m_known_contexts.exists ()
4007 5060349 : || ctx.m_avals.m_known_contexts.exists ())
4008 : {
4009 15585352 : for (unsigned int i = 0; i < nargs; i++)
4010 : {
4011 10526395 : if (!ipa_is_param_used_by_polymorphic_call (params_summary, i))
4012 10400847 : continue;
4013 125548 : if (i >= m_avals.m_known_contexts.length ()
4014 124365 : || m_avals.m_known_contexts[i].useless_p ())
4015 : {
4016 1183 : if (i < ctx.m_avals.m_known_contexts.length ()
4017 1183 : && !ctx.m_avals.m_known_contexts[i].useless_p ())
4018 : return false;
4019 1174 : continue;
4020 : }
4021 124365 : if (i >= ctx.m_avals.m_known_contexts.length ()
4022 124365 : || ctx.m_avals.m_known_contexts[i].useless_p ())
4023 : {
4024 8 : if (i < m_avals.m_known_contexts.length ()
4025 1914569 : && !m_avals.m_known_contexts[i].useless_p ())
4026 : return false;
4027 0 : continue;
4028 : }
4029 124357 : if (!m_avals.m_known_contexts[i].equal_to
4030 124357 : (ctx.m_avals.m_known_contexts[i]))
4031 : return false;
4032 : }
4033 : }
4034 5058957 : if (m_avals.m_known_aggs.exists () || ctx.m_avals.m_known_aggs.exists ())
4035 : {
4036 : unsigned i = 0, j = 0;
4037 11649352 : while (i < m_avals.m_known_aggs.length ()
4038 5823983 : || j < ctx.m_avals.m_known_aggs.length ())
4039 : {
4040 766391 : if (i >= m_avals.m_known_aggs.length ())
4041 : {
4042 761984 : int idx2 = ctx.m_avals.m_known_aggs[j].index;
4043 761984 : if (ipa_is_param_used_by_indirect_call (params_summary, idx2))
4044 : return false;
4045 761445 : j++;
4046 761445 : continue;
4047 761445 : }
4048 4407 : if (j >= ctx.m_avals.m_known_aggs.length ())
4049 : {
4050 585 : int idx1 = m_avals.m_known_aggs[i].index;
4051 585 : if (ipa_is_param_used_by_indirect_call (params_summary, idx1))
4052 : return false;
4053 8 : i++;
4054 8 : continue;
4055 8 : }
4056 :
4057 3822 : int idx1 = m_avals.m_known_aggs[i].index;
4058 3822 : int idx2 = ctx.m_avals.m_known_aggs[j].index;
4059 3822 : if (idx1 < idx2)
4060 : {
4061 0 : if (ipa_is_param_used_by_indirect_call (params_summary, idx1))
4062 : return false;
4063 0 : i++;
4064 0 : continue;
4065 : }
4066 3822 : if (idx1 > idx2)
4067 : {
4068 0 : if (ipa_is_param_used_by_indirect_call (params_summary, idx2))
4069 : return false;
4070 0 : j++;
4071 0 : continue;
4072 : }
4073 3822 : if (!ipa_is_param_used_by_indirect_call (params_summary, idx1))
4074 : {
4075 476 : i++;
4076 476 : j++;
4077 476 : continue;
4078 : }
4079 :
4080 3346 : if ((m_avals.m_known_aggs[i].unit_offset
4081 3346 : != ctx.m_avals.m_known_aggs[j].unit_offset)
4082 3338 : || (m_avals.m_known_aggs[i].by_ref
4083 3338 : != ctx.m_avals.m_known_aggs[j].by_ref)
4084 6684 : || !operand_equal_p (m_avals.m_known_aggs[i].value,
4085 3338 : ctx.m_avals.m_known_aggs[j].value))
4086 249 : return false;
4087 3097 : i++;
4088 3097 : j++;
4089 : }
4090 : }
4091 : return true;
4092 : }
4093 :
4094 : /* Fill in the selected fields in ESTIMATES with value estimated for call in
4095 : this context. Always compute size and min_size. Only compute time and
4096 : nonspecialized_time if EST_TIMES is true. Only compute hints if EST_HINTS
4097 : is true. */
4098 :
4099 : void
4100 19187183 : ipa_call_context::estimate_size_and_time (ipa_call_estimates *estimates,
4101 : bool est_times, bool est_hints)
4102 : {
4103 19187183 : class ipa_fn_summary *info = ipa_fn_summaries->get (m_node);
4104 19187183 : size_time_entry *e;
4105 19187183 : int size = 0;
4106 19187183 : sreal time = 0;
4107 19187183 : int min_size = 0;
4108 19187183 : ipa_hints hints = 0;
4109 19187183 : sreal loops_with_known_iterations = 0;
4110 19187183 : sreal loops_with_known_strides = 0;
4111 19187183 : int i;
4112 :
4113 19187183 : if (dump_file && (dump_flags & TDF_DETAILS))
4114 : {
4115 1340 : bool found = false;
4116 1340 : fprintf (dump_file, " Estimating body: %s\n"
4117 : " Known to be false: ", m_node->dump_name ());
4118 :
4119 4356 : for (i = ipa_predicate::not_inlined_condition;
4120 8712 : i < (ipa_predicate::first_dynamic_condition
4121 7142 : + (int) vec_safe_length (info->conds)); i++)
4122 3016 : if (!(m_possible_truths & (1 << i)))
4123 : {
4124 1806 : if (found)
4125 532 : fprintf (dump_file, ", ");
4126 1806 : found = true;
4127 1806 : dump_condition (dump_file, info->conds, i);
4128 : }
4129 : }
4130 :
4131 19187183 : if (m_node->callees || m_node->indirect_calls)
4132 25168650 : estimate_calls_size_and_time (m_node, &size, &min_size,
4133 : est_times ? &time : NULL,
4134 : est_hints ? &hints : NULL, m_possible_truths,
4135 : &m_avals);
4136 :
4137 19187183 : sreal nonspecialized_time = time;
4138 :
4139 19187183 : min_size += info->size_time_table[0].size;
4140 132202732 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
4141 : {
4142 113015549 : bool exec = e->exec_predicate.evaluate (m_nonspec_possible_truths);
4143 :
4144 : /* Because predicates are conservative, it can happen that nonconst is 1
4145 : but exec is 0. */
4146 113015549 : if (exec)
4147 : {
4148 110614198 : bool nonconst = e->nonconst_predicate.evaluate (m_possible_truths);
4149 :
4150 110614198 : gcc_checking_assert (e->time >= 0);
4151 110614198 : gcc_checking_assert (time >= 0);
4152 :
4153 : /* We compute specialized size only because size of nonspecialized
4154 : copy is context independent.
4155 :
4156 : The difference between nonspecialized execution and specialized is
4157 : that nonspecialized is not going to have optimized out computations
4158 : known to be constant in a specialized setting. */
4159 110614198 : if (nonconst)
4160 55783891 : size += e->size;
4161 110614198 : if (!est_times)
4162 56025701 : continue;
4163 54588497 : nonspecialized_time += e->time;
4164 54588497 : if (!nonconst)
4165 : ;
4166 26353208 : else if (!m_inline_param_summary.exists ())
4167 : {
4168 2631210 : if (nonconst)
4169 2631210 : time += e->time;
4170 : }
4171 : else
4172 : {
4173 23721998 : int prob = e->nonconst_predicate.probability
4174 23721998 : (info->conds, m_possible_truths,
4175 : m_inline_param_summary);
4176 23721998 : gcc_checking_assert (prob >= 0);
4177 23721998 : gcc_checking_assert (prob <= REG_BR_PROB_BASE);
4178 23721998 : if (prob == REG_BR_PROB_BASE)
4179 19969897 : time += e->time;
4180 : else
4181 3752101 : time += e->time * prob / REG_BR_PROB_BASE;
4182 : }
4183 54588497 : gcc_checking_assert (time >= 0);
4184 : }
4185 : }
4186 19187183 : gcc_checking_assert (info->size_time_table[0].exec_predicate == true);
4187 19187183 : gcc_checking_assert (info->size_time_table[0].nonconst_predicate == true);
4188 19187183 : gcc_checking_assert (min_size >= 0);
4189 19187183 : gcc_checking_assert (size >= 0);
4190 19187183 : gcc_checking_assert (time >= 0);
4191 : /* nonspecialized_time should be always bigger than specialized time.
4192 : Roundoff issues however may get into the way. */
4193 19187183 : gcc_checking_assert ((nonspecialized_time - time * 99 / 100) >= -1);
4194 :
4195 : /* Roundoff issues may make specialized time bigger than nonspecialized
4196 : time. We do not really want that to happen because some heuristics
4197 : may get confused by seeing negative speedups. */
4198 19187183 : if (time > nonspecialized_time)
4199 0 : time = nonspecialized_time;
4200 :
4201 19187183 : if (est_hints)
4202 : {
4203 7237083 : if (info->scc_no)
4204 227030 : hints |= INLINE_HINT_in_scc;
4205 7237083 : if (DECL_DECLARED_INLINE_P (m_node->decl))
4206 4581903 : hints |= INLINE_HINT_declared_inline;
4207 7237083 : if (info->builtin_constant_p_parms.length ()
4208 8940 : && DECL_DECLARED_INLINE_P (m_node->decl))
4209 8882 : hints |= INLINE_HINT_builtin_constant_p;
4210 :
4211 : ipa_freqcounting_predicate *fcp;
4212 7787643 : for (i = 0; vec_safe_iterate (info->loop_iterations, i, &fcp); i++)
4213 550560 : if (!fcp->predicate->evaluate (m_possible_truths))
4214 : {
4215 370632 : hints |= INLINE_HINT_loop_iterations;
4216 370632 : loops_with_known_iterations += fcp->freq;
4217 : }
4218 7237083 : estimates->loops_with_known_iterations = loops_with_known_iterations;
4219 :
4220 7528121 : for (i = 0; vec_safe_iterate (info->loop_strides, i, &fcp); i++)
4221 291038 : if (!fcp->predicate->evaluate (m_possible_truths))
4222 : {
4223 262265 : hints |= INLINE_HINT_loop_stride;
4224 262265 : loops_with_known_strides += fcp->freq;
4225 : }
4226 7237083 : estimates->loops_with_known_strides = loops_with_known_strides;
4227 : }
4228 :
4229 19187183 : size = RDIV (size, ipa_fn_summary::size_scale);
4230 19187183 : min_size = RDIV (min_size, ipa_fn_summary::size_scale);
4231 :
4232 19187183 : if (dump_file && (dump_flags & TDF_DETAILS))
4233 : {
4234 1340 : fprintf (dump_file, "\n size:%i", (int) size);
4235 1340 : if (est_times)
4236 1103 : fprintf (dump_file, " time:%f nonspec time:%f",
4237 : time.to_double (), nonspecialized_time.to_double ());
4238 1340 : if (est_hints)
4239 1103 : fprintf (dump_file, " loops with known iterations:%f "
4240 : "known strides:%f", loops_with_known_iterations.to_double (),
4241 : loops_with_known_strides.to_double ());
4242 1340 : fprintf (dump_file, "\n");
4243 : }
4244 19187183 : if (est_times)
4245 : {
4246 7237083 : estimates->time = time;
4247 7237083 : estimates->nonspecialized_time = nonspecialized_time;
4248 : }
4249 19187183 : estimates->size = size;
4250 19187183 : estimates->min_size = min_size;
4251 19187183 : if (est_hints)
4252 7237083 : estimates->hints = hints;
4253 19187183 : return;
4254 : }
4255 :
4256 :
4257 : /* Estimate size and time needed to execute callee of EDGE assuming that
4258 : parameters known to be constant at caller of EDGE are propagated.
4259 : KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
4260 : and types for parameters. */
4261 :
4262 : void
4263 289808 : estimate_ipcp_clone_size_and_time (struct cgraph_node *node,
4264 : ipa_auto_call_arg_values *avals,
4265 : ipa_call_estimates *estimates)
4266 : {
4267 289808 : clause_t clause, nonspec_clause;
4268 :
4269 289808 : evaluate_conditions_for_known_args (node, false, avals, &clause,
4270 : &nonspec_clause, NULL);
4271 289808 : ipa_call_context ctx (node, clause, nonspec_clause, vNULL, avals);
4272 289808 : ctx.estimate_size_and_time (estimates);
4273 289808 : }
4274 :
4275 : /* Return stack frame offset where frame of NODE is supposed to start inside
4276 : of the function it is inlined to.
4277 : Return 0 for functions that are not inlined. */
4278 :
4279 : HOST_WIDE_INT
4280 4951780 : ipa_get_stack_frame_offset (struct cgraph_node *node)
4281 : {
4282 4951780 : HOST_WIDE_INT offset = 0;
4283 4951780 : if (!node->inlined_to)
4284 : return 0;
4285 3945832 : node = node->callers->caller;
4286 5125600 : while (true)
4287 : {
4288 5125600 : offset += ipa_size_summaries->get (node)->estimated_self_stack_size;
4289 4535716 : if (!node->inlined_to)
4290 : return offset;
4291 589884 : node = node->callers->caller;
4292 : }
4293 : }
4294 :
4295 :
4296 : /* Update summary information of inline clones after inlining.
4297 : Compute peak stack usage. */
4298 :
4299 : static void
4300 4674151 : inline_update_callee_summaries (struct cgraph_node *node, int depth)
4301 : {
4302 4674151 : struct cgraph_edge *e;
4303 :
4304 4674151 : ipa_propagate_frequency (node);
4305 9052909 : for (e = node->callees; e; e = e->next_callee)
4306 : {
4307 4378758 : if (!e->inline_failed)
4308 728872 : inline_update_callee_summaries (e->callee, depth);
4309 : else
4310 3649886 : ipa_call_summaries->get (e)->loop_depth += depth;
4311 : }
4312 4774901 : for (e = node->indirect_calls; e; e = e->next_callee)
4313 100750 : ipa_call_summaries->get (e)->loop_depth += depth;
4314 4674151 : }
4315 :
4316 : /* Update change_prob and points_to_local_or_readonly_memory of EDGE after
4317 : INLINED_EDGE has been inlined.
4318 :
4319 : When function A is inlined in B and A calls C with parameter that
4320 : changes with probability PROB1 and C is known to be passthrough
4321 : of argument if B that change with probability PROB2, the probability
4322 : of change is now PROB1*PROB2. */
4323 :
4324 : static void
4325 3750946 : remap_edge_params (struct cgraph_edge *inlined_edge,
4326 : struct cgraph_edge *edge)
4327 : {
4328 3750946 : if (ipa_node_params_sum)
4329 : {
4330 2681263 : int i;
4331 2681263 : ipa_edge_args *args = ipa_edge_args_sum->get (edge);
4332 2681263 : if (!args)
4333 : return;
4334 1525883 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
4335 1525883 : class ipa_call_summary *inlined_es
4336 1525883 : = ipa_call_summaries->get (inlined_edge);
4337 :
4338 1525883 : if (es->param.length () == 0)
4339 : return;
4340 :
4341 8887232 : for (i = 0; i < ipa_get_cs_argument_count (args); i++)
4342 : {
4343 2993931 : struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
4344 2993931 : if (jfunc->type == IPA_JF_PASS_THROUGH
4345 2189047 : || jfunc->type == IPA_JF_ANCESTOR)
4346 : {
4347 941337 : int id = jfunc->type == IPA_JF_PASS_THROUGH
4348 941337 : ? ipa_get_jf_pass_through_formal_id (jfunc)
4349 136453 : : ipa_get_jf_ancestor_formal_id (jfunc);
4350 1882582 : if (id < (int) inlined_es->param.length ())
4351 : {
4352 941237 : int prob1 = es->param[i].change_prob;
4353 941237 : int prob2 = inlined_es->param[id].change_prob;
4354 941237 : int prob = combine_probabilities (prob1, prob2);
4355 :
4356 941237 : if (prob1 && prob2 && !prob)
4357 941237 : prob = 1;
4358 :
4359 941237 : es->param[i].change_prob = prob;
4360 :
4361 1882474 : if (inlined_es
4362 941237 : ->param[id].points_to_local_or_readonly_memory)
4363 206985 : es->param[i].points_to_local_or_readonly_memory = true;
4364 1882474 : if (inlined_es
4365 941237 : ->param[id].points_to_possible_sra_candidate)
4366 166585 : es->param[i].points_to_possible_sra_candidate = true;
4367 : }
4368 941337 : if (!es->param[i].points_to_local_or_readonly_memory
4369 : && jfunc->type == IPA_JF_CONST
4370 : && points_to_local_or_readonly_memory_p
4371 : (ipa_get_jf_constant (jfunc)))
4372 : es->param[i].points_to_local_or_readonly_memory = true;
4373 : }
4374 : }
4375 : }
4376 : }
4377 :
4378 : /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
4379 :
4380 : Remap predicates of callees of NODE. Rest of arguments match
4381 : remap_predicate.
4382 :
4383 : Also update change probabilities. */
4384 :
4385 : static void
4386 4674151 : remap_edge_summaries (struct cgraph_edge *inlined_edge,
4387 : struct cgraph_node *node,
4388 : class ipa_fn_summary *info,
4389 : class ipa_node_params *params_summary,
4390 : class ipa_fn_summary *callee_info,
4391 : const vec<int> &operand_map,
4392 : const vec<HOST_WIDE_INT> &offset_map,
4393 : clause_t possible_truths,
4394 : ipa_predicate *toplev_predicate)
4395 : {
4396 4674151 : struct cgraph_edge *e, *next;
4397 9052485 : for (e = node->callees; e; e = next)
4398 : {
4399 4378334 : ipa_predicate p;
4400 4378334 : next = e->next_callee;
4401 :
4402 4378334 : if (e->inline_failed)
4403 : {
4404 3649462 : class ipa_call_summary *es = ipa_call_summaries->get (e);
4405 3649462 : remap_edge_params (inlined_edge, e);
4406 :
4407 3649462 : if (es->predicate)
4408 : {
4409 1270581 : p = es->predicate->remap_after_inlining
4410 1270581 : (info, params_summary,
4411 : callee_info, operand_map,
4412 : offset_map, possible_truths,
4413 : *toplev_predicate);
4414 1270581 : edge_set_predicate (e, &p);
4415 : }
4416 : else
4417 2378881 : edge_set_predicate (e, toplev_predicate);
4418 : }
4419 : else
4420 728872 : remap_edge_summaries (inlined_edge, e->callee, info,
4421 : params_summary, callee_info,
4422 : operand_map, offset_map, possible_truths,
4423 : toplev_predicate);
4424 : }
4425 4775635 : for (e = node->indirect_calls; e; e = next)
4426 : {
4427 101484 : class ipa_call_summary *es = ipa_call_summaries->get (e);
4428 101484 : ipa_predicate p;
4429 101484 : next = e->next_callee;
4430 :
4431 101484 : remap_edge_params (inlined_edge, e);
4432 101484 : if (es->predicate)
4433 : {
4434 26101 : p = es->predicate->remap_after_inlining
4435 26101 : (info, params_summary,
4436 : callee_info, operand_map, offset_map,
4437 : possible_truths, *toplev_predicate);
4438 26101 : edge_set_predicate (e, &p);
4439 : }
4440 : else
4441 75383 : edge_set_predicate (e, toplev_predicate);
4442 : }
4443 4674151 : }
4444 :
4445 : /* Run remap_after_inlining on each predicate in V. */
4446 :
4447 : static void
4448 7890558 : remap_freqcounting_predicate (class ipa_fn_summary *info,
4449 : class ipa_node_params *params_summary,
4450 : class ipa_fn_summary *callee_info,
4451 : vec<ipa_freqcounting_predicate, va_gc> *v,
4452 : const vec<int> &operand_map,
4453 : const vec<HOST_WIDE_INT> &offset_map,
4454 : clause_t possible_truths,
4455 : ipa_predicate *toplev_predicate)
4456 :
4457 : {
4458 7890558 : ipa_freqcounting_predicate *fcp;
4459 7916651 : for (int i = 0; vec_safe_iterate (v, i, &fcp); i++)
4460 : {
4461 26093 : ipa_predicate p
4462 26093 : = fcp->predicate->remap_after_inlining (info, params_summary,
4463 : callee_info, operand_map,
4464 : offset_map, possible_truths,
4465 : *toplev_predicate);
4466 36262 : if (p != false && p != true)
4467 2520 : *fcp->predicate &= p;
4468 : }
4469 7890558 : }
4470 :
4471 : /* We inlined EDGE. Update summary of the function we inlined into. */
4472 :
4473 : void
4474 3945279 : ipa_merge_fn_summary_after_inlining (struct cgraph_edge *edge)
4475 : {
4476 3945279 : ipa_fn_summary *callee_info = ipa_fn_summaries->get (edge->callee);
4477 3667059 : struct cgraph_node *to = (edge->caller->inlined_to
4478 3945279 : ? edge->caller->inlined_to : edge->caller);
4479 3945279 : class ipa_fn_summary *info = ipa_fn_summaries->get (to);
4480 3945279 : clause_t clause = 0; /* not_inline is known to be false. */
4481 3945279 : size_time_entry *e;
4482 3945279 : auto_vec<int, 8> operand_map;
4483 3945279 : auto_vec<HOST_WIDE_INT, 8> offset_map;
4484 3945279 : int i;
4485 3945279 : ipa_predicate toplev_predicate;
4486 3945279 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
4487 3945279 : ipa_node_params *params_summary = (ipa_node_params_sum
4488 3945279 : ? ipa_node_params_sum->get (to) : NULL);
4489 :
4490 3945279 : if (es->predicate)
4491 343571 : toplev_predicate = *es->predicate;
4492 : else
4493 3601708 : toplev_predicate = true;
4494 :
4495 3945279 : info->fp_expressions |= callee_info->fp_expressions;
4496 3945279 : info->target_info |= callee_info->target_info;
4497 :
4498 3945279 : if (callee_info->conds)
4499 : {
4500 2884508 : ipa_auto_call_arg_values avals;
4501 2884508 : evaluate_properties_for_edge (edge, true, &clause, NULL, &avals, false);
4502 2884508 : }
4503 3945279 : if (ipa_node_params_sum && callee_info->conds)
4504 : {
4505 801595 : ipa_edge_args *args = ipa_edge_args_sum->get (edge);
4506 801595 : int count = args ? ipa_get_cs_argument_count (args) : 0;
4507 801447 : int i;
4508 :
4509 801447 : if (count)
4510 : {
4511 801447 : operand_map.safe_grow_cleared (count, true);
4512 801447 : offset_map.safe_grow_cleared (count, true);
4513 : }
4514 2570524 : for (i = 0; i < count; i++)
4515 : {
4516 1768929 : struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
4517 1768929 : int map = -1;
4518 :
4519 : /* TODO: handle non-NOPs when merging. */
4520 1768929 : if (jfunc->type == IPA_JF_PASS_THROUGH)
4521 : {
4522 286098 : if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
4523 282692 : map = ipa_get_jf_pass_through_formal_id (jfunc);
4524 286098 : if (!ipa_get_jf_pass_through_agg_preserved (jfunc))
4525 192429 : offset_map[i] = -1;
4526 : }
4527 1482831 : else if (jfunc->type == IPA_JF_ANCESTOR)
4528 : {
4529 91629 : HOST_WIDE_INT offset = ipa_get_jf_ancestor_offset (jfunc);
4530 91629 : if (offset >= 0 && offset < INT_MAX)
4531 : {
4532 91629 : map = ipa_get_jf_ancestor_formal_id (jfunc);
4533 91629 : if (!ipa_get_jf_ancestor_agg_preserved (jfunc))
4534 53740 : offset = -1;
4535 91629 : offset_map[i] = offset;
4536 : }
4537 : }
4538 1768929 : operand_map[i] = map;
4539 3196008 : gcc_assert (map < ipa_get_param_count (params_summary));
4540 : }
4541 :
4542 : int ip;
4543 804189 : for (i = 0; callee_info->builtin_constant_p_parms.iterate (i, &ip); i++)
4544 2594 : if (ip < count && operand_map[ip] >= 0)
4545 39 : add_builtin_constant_p_parm (info, operand_map[ip]);
4546 : }
4547 3945279 : sreal freq = edge->sreal_frequency ();
4548 21778469 : for (i = 0; callee_info->size_time_table.iterate (i, &e); i++)
4549 : {
4550 17833190 : ipa_predicate p;
4551 17833190 : p = e->exec_predicate.remap_after_inlining
4552 17833190 : (info, params_summary,
4553 : callee_info, operand_map,
4554 : offset_map, clause,
4555 : toplev_predicate);
4556 17833190 : ipa_predicate nonconstp;
4557 17833190 : nonconstp = e->nonconst_predicate.remap_after_inlining
4558 17833190 : (info, params_summary,
4559 : callee_info, operand_map,
4560 : offset_map, clause,
4561 : toplev_predicate);
4562 26983555 : if (p != false && nonconstp != false)
4563 : {
4564 8837791 : sreal add_time = ((sreal)e->time * freq);
4565 8837791 : int prob = e->nonconst_predicate.probability (callee_info->conds,
4566 : clause, es->param);
4567 8837791 : if (prob != REG_BR_PROB_BASE)
4568 1059843 : add_time = add_time * prob / REG_BR_PROB_BASE;
4569 1059843 : if (prob != REG_BR_PROB_BASE
4570 1059843 : && dump_file && (dump_flags & TDF_DETAILS))
4571 : {
4572 3 : fprintf (dump_file, "\t\tScaling time by probability:%f\n",
4573 3 : (double) prob / REG_BR_PROB_BASE);
4574 : }
4575 8837791 : info->account_size_time (e->size, add_time, p, nonconstp);
4576 : }
4577 : }
4578 3945279 : remap_edge_summaries (edge, edge->callee, info, params_summary,
4579 : callee_info, operand_map,
4580 : offset_map, clause, &toplev_predicate);
4581 3945279 : remap_freqcounting_predicate (info, params_summary, callee_info,
4582 : info->loop_iterations, operand_map,
4583 : offset_map, clause, &toplev_predicate);
4584 3945279 : remap_freqcounting_predicate (info, params_summary, callee_info,
4585 : info->loop_strides, operand_map,
4586 : offset_map, clause, &toplev_predicate);
4587 :
4588 3945279 : HOST_WIDE_INT stack_frame_offset = ipa_get_stack_frame_offset (edge->callee);
4589 3945279 : HOST_WIDE_INT peak = stack_frame_offset + callee_info->estimated_stack_size;
4590 :
4591 3945279 : if (info->estimated_stack_size < peak)
4592 115758 : info->estimated_stack_size = peak;
4593 :
4594 3945279 : inline_update_callee_summaries (edge->callee, es->loop_depth);
4595 3945279 : if (info->call_size_time_table.length ())
4596 : {
4597 861252 : int edge_size = 0;
4598 861252 : sreal edge_time = 0;
4599 :
4600 861252 : estimate_edge_size_and_time (edge, &edge_size, NULL, &edge_time, NULL, 0);
4601 : /* Unaccount size and time of the optimized out call. */
4602 861252 : info->account_size_time (-edge_size, -edge_time,
4603 861252 : es->predicate ? *es->predicate : true,
4604 861252 : es->predicate ? *es->predicate : true,
4605 : true);
4606 : /* Account new calls. */
4607 861252 : summarize_calls_size_and_time (edge->callee, info);
4608 : }
4609 :
4610 : /* Free summaries that are not maintained for inline clones/edges. */
4611 3945279 : ipa_call_summaries->remove (edge);
4612 3945279 : ipa_fn_summaries->remove (edge->callee);
4613 3945279 : ipa_remove_from_growth_caches (edge);
4614 3945279 : }
4615 :
4616 : /* For performance reasons ipa_merge_fn_summary_after_inlining is not updating
4617 : overall size and time. Recompute it.
4618 : If RESET is true also recompute call_time_size_table. */
4619 :
4620 : void
4621 9421489 : ipa_update_overall_fn_summary (struct cgraph_node *node, bool reset)
4622 : {
4623 9421489 : class ipa_fn_summary *info = ipa_fn_summaries->get (node);
4624 9421489 : class ipa_size_summary *size_info = ipa_size_summaries->get (node);
4625 9421489 : size_time_entry *e;
4626 9421489 : int i;
4627 :
4628 9421489 : size_info->size = 0;
4629 9421489 : info->time = 0;
4630 47805305 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
4631 : {
4632 38383816 : size_info->size += e->size;
4633 38383816 : info->time += e->time;
4634 : }
4635 9421489 : info->min_size = info->size_time_table[0].size;
4636 9421489 : if (reset)
4637 8518603 : info->call_size_time_table.release ();
4638 9421489 : if (node->callees || node->indirect_calls)
4639 7195140 : estimate_calls_size_and_time (node, &size_info->size, &info->min_size,
4640 : &info->time, NULL,
4641 : ~(clause_t) (1 << ipa_predicate::false_condition),
4642 : NULL);
4643 9421489 : size_info->size = RDIV (size_info->size, ipa_fn_summary::size_scale);
4644 9421489 : info->min_size = RDIV (info->min_size, ipa_fn_summary::size_scale);
4645 9421489 : }
4646 :
4647 :
4648 : /* This function performs intraprocedural analysis in NODE that is required to
4649 : inline indirect calls. */
4650 :
4651 : static void
4652 1381687 : inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
4653 : {
4654 1381687 : ipa_analyze_node (node);
4655 1381687 : if (dump_file && (dump_flags & TDF_DETAILS))
4656 : {
4657 14 : ipa_print_node_params (dump_file, node);
4658 14 : ipa_print_node_jump_functions (dump_file, node);
4659 : }
4660 1381687 : }
4661 :
4662 :
4663 : /* Note function body size. */
4664 :
4665 : void
4666 1392325 : inline_analyze_function (struct cgraph_node *node)
4667 : {
4668 1392325 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
4669 :
4670 1392325 : if (dump_file)
4671 94 : fprintf (dump_file, "\nAnalyzing function: %s\n", node->dump_name ());
4672 1392325 : if (opt_for_fn (node->decl, optimize) && !node->thunk)
4673 1381687 : inline_indirect_intraprocedural_analysis (node);
4674 1392325 : compute_fn_summary (node, false);
4675 1392325 : if (!optimize)
4676 : {
4677 9366 : struct cgraph_edge *e;
4678 22078 : for (e = node->callees; e; e = e->next_callee)
4679 12712 : e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4680 9437 : for (e = node->indirect_calls; e; e = e->next_callee)
4681 71 : e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4682 : }
4683 :
4684 1392325 : pop_cfun ();
4685 1392325 : }
4686 :
4687 :
4688 : /* Called when new function is inserted to callgraph late. */
4689 :
4690 : void
4691 19217 : ipa_fn_summary_t::insert (struct cgraph_node *node, ipa_fn_summary *)
4692 : {
4693 19217 : inline_analyze_function (node);
4694 19217 : }
4695 :
4696 : /* Note function body size. */
4697 :
4698 : static void
4699 232298 : ipa_fn_summary_generate (void)
4700 : {
4701 232298 : struct cgraph_node *node;
4702 :
4703 2101906 : FOR_EACH_DEFINED_FUNCTION (node)
4704 1869608 : if (DECL_STRUCT_FUNCTION (node->decl))
4705 1856404 : node->versionable = tree_versionable_function_p (node->decl);
4706 :
4707 232298 : ipa_fn_summary_alloc ();
4708 :
4709 232298 : ipa_fn_summaries->enable_insertion_hook ();
4710 :
4711 232298 : ipa_register_cgraph_hooks ();
4712 :
4713 2101906 : FOR_EACH_DEFINED_FUNCTION (node)
4714 1869608 : if (!node->alias
4715 1869608 : && (flag_generate_lto || flag_generate_offload|| flag_wpa
4716 1678623 : || opt_for_fn (node->decl, optimize)))
4717 1355025 : inline_analyze_function (node);
4718 232298 : }
4719 :
4720 :
4721 : /* Write inline summary for edge E to OB. */
4722 :
4723 : static void
4724 347307 : read_ipa_call_summary (class lto_input_block *ib, struct cgraph_edge *e,
4725 : bool prevails)
4726 : {
4727 347307 : class ipa_call_summary *es = prevails
4728 347307 : ? ipa_call_summaries->get_create (e) : NULL;
4729 347307 : ipa_predicate p;
4730 347307 : int length, i;
4731 :
4732 347307 : int size = streamer_read_uhwi (ib);
4733 347307 : int time = streamer_read_uhwi (ib);
4734 347307 : int depth = streamer_read_uhwi (ib);
4735 :
4736 347307 : if (es)
4737 : {
4738 347258 : es->call_stmt_size = size;
4739 347258 : es->call_stmt_time = time;
4740 347258 : es->loop_depth = depth;
4741 : }
4742 :
4743 347307 : bitpack_d bp = streamer_read_bitpack (ib);
4744 347307 : if (es)
4745 347258 : es->is_return_callee_uncaptured = bp_unpack_value (&bp, 1);
4746 : else
4747 49 : bp_unpack_value (&bp, 1);
4748 :
4749 347307 : p.stream_in (ib);
4750 347307 : if (es)
4751 347258 : edge_set_predicate (e, &p);
4752 347307 : length = streamer_read_uhwi (ib);
4753 347307 : if (length && es
4754 347307 : && (e->possibly_call_in_translation_unit_p ()
4755 : /* Also stream in jump functions to builtins in hope that they
4756 : will get fnspecs. */
4757 118756 : || fndecl_built_in_p (e->callee->decl, BUILT_IN_NORMAL)))
4758 : {
4759 232371 : es->param.safe_grow_cleared (length, true);
4760 799805 : for (i = 0; i < length; i++)
4761 : {
4762 567434 : es->param[i].change_prob = streamer_read_uhwi (ib);
4763 567434 : bitpack_d bp = streamer_read_bitpack (ib);
4764 1702302 : es->param[i].points_to_local_or_readonly_memory
4765 567434 : = bp_unpack_value (&bp, 1);
4766 1702302 : es->param[i].points_to_possible_sra_candidate
4767 567434 : = bp_unpack_value (&bp, 1);
4768 : }
4769 : }
4770 : else
4771 : {
4772 137716 : for (i = 0; i < length; i++)
4773 : {
4774 22780 : streamer_read_uhwi (ib);
4775 22780 : streamer_read_uhwi (ib);
4776 : }
4777 : }
4778 347307 : }
4779 :
4780 :
4781 : /* Stream in inline summaries from the section. */
4782 :
4783 : static void
4784 13302 : inline_read_section (struct lto_file_decl_data *file_data, const char *data,
4785 : size_t len)
4786 : {
4787 13302 : const struct lto_function_header *header =
4788 : (const struct lto_function_header *) data;
4789 13302 : const int cfg_offset = sizeof (struct lto_function_header);
4790 13302 : const int main_offset = cfg_offset + header->cfg_size;
4791 13302 : const int string_offset = main_offset + header->main_size;
4792 13302 : class data_in *data_in;
4793 13302 : unsigned int i, count2, j;
4794 13302 : unsigned int f_count;
4795 :
4796 13302 : lto_input_block ib ((const char *) data + main_offset, header->main_size,
4797 13302 : file_data);
4798 :
4799 13302 : data_in =
4800 26604 : lto_data_in_create (file_data, (const char *) data + string_offset,
4801 13302 : header->string_size, vNULL);
4802 13302 : f_count = streamer_read_uhwi (&ib);
4803 99424 : for (i = 0; i < f_count; i++)
4804 : {
4805 86122 : unsigned int index;
4806 86122 : struct cgraph_node *node;
4807 86122 : class ipa_fn_summary *info;
4808 86122 : class ipa_node_params *params_summary;
4809 86122 : class ipa_size_summary *size_info;
4810 86122 : lto_symtab_encoder_t encoder;
4811 86122 : struct bitpack_d bp;
4812 86122 : struct cgraph_edge *e;
4813 86122 : ipa_predicate p;
4814 :
4815 86122 : index = streamer_read_uhwi (&ib);
4816 86122 : encoder = file_data->symtab_node_encoder;
4817 86122 : node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
4818 : index));
4819 86122 : info = node->prevailing_p () ? ipa_fn_summaries->get_create (node) : NULL;
4820 86122 : params_summary = node->prevailing_p ()
4821 86122 : ? ipa_node_params_sum->get (node) : NULL;
4822 86122 : size_info = node->prevailing_p ()
4823 86122 : ? ipa_size_summaries->get_create (node) : NULL;
4824 :
4825 86122 : int stack_size = streamer_read_uhwi (&ib);
4826 86122 : int size = streamer_read_uhwi (&ib);
4827 86122 : sreal time = sreal::stream_in (&ib);
4828 :
4829 86122 : if (info)
4830 : {
4831 86064 : info->estimated_stack_size
4832 86064 : = size_info->estimated_self_stack_size = stack_size;
4833 86064 : size_info->size = size_info->self_size = size;
4834 86064 : info->time = time;
4835 : }
4836 :
4837 86122 : bp = streamer_read_bitpack (&ib);
4838 86122 : if (info)
4839 : {
4840 86064 : info->inlinable = bp_unpack_value (&bp, 1);
4841 86064 : info->fp_expressions = bp_unpack_value (&bp, 1);
4842 86064 : if (!lto_stream_offload_p)
4843 86064 : info->target_info = streamer_read_uhwi (&ib);
4844 : }
4845 : else
4846 : {
4847 58 : bp_unpack_value (&bp, 1);
4848 58 : bp_unpack_value (&bp, 1);
4849 58 : if (!lto_stream_offload_p)
4850 58 : streamer_read_uhwi (&ib);
4851 : }
4852 :
4853 86122 : count2 = streamer_read_uhwi (&ib);
4854 86122 : gcc_assert (!info || !info->conds);
4855 86064 : if (info)
4856 86064 : vec_safe_reserve_exact (info->conds, count2);
4857 163097 : for (j = 0; j < count2; j++)
4858 : {
4859 76975 : struct condition c;
4860 76975 : unsigned int k, count3;
4861 76975 : c.operand_num = streamer_read_uhwi (&ib);
4862 76975 : c.code = (enum tree_code) streamer_read_uhwi (&ib);
4863 76975 : c.type = stream_read_tree (&ib, data_in);
4864 76975 : c.val = stream_read_tree (&ib, data_in);
4865 76975 : bp = streamer_read_bitpack (&ib);
4866 76975 : c.agg_contents = bp_unpack_value (&bp, 1);
4867 76975 : c.by_ref = bp_unpack_value (&bp, 1);
4868 76975 : if (c.agg_contents)
4869 12405 : c.offset = streamer_read_uhwi (&ib);
4870 76975 : count3 = streamer_read_uhwi (&ib);
4871 76975 : c.param_ops = NULL;
4872 76975 : if (info)
4873 76975 : vec_safe_reserve_exact (c.param_ops, count3);
4874 76975 : if (params_summary)
4875 76975 : ipa_set_param_used_by_ipa_predicates
4876 76975 : (params_summary, c.operand_num, true);
4877 80602 : for (k = 0; k < count3; k++)
4878 : {
4879 3627 : struct expr_eval_op op;
4880 3627 : enum gimple_rhs_class rhs_class;
4881 3627 : op.code = (enum tree_code) streamer_read_uhwi (&ib);
4882 3627 : op.type = stream_read_tree (&ib, data_in);
4883 3627 : switch (rhs_class = get_gimple_rhs_class (op.code))
4884 : {
4885 1499 : case GIMPLE_UNARY_RHS:
4886 1499 : op.index = 0;
4887 1499 : op.val[0] = NULL_TREE;
4888 1499 : op.val[1] = NULL_TREE;
4889 1499 : break;
4890 :
4891 2128 : case GIMPLE_BINARY_RHS:
4892 2128 : case GIMPLE_TERNARY_RHS:
4893 2128 : bp = streamer_read_bitpack (&ib);
4894 2128 : op.index = bp_unpack_value (&bp, 2);
4895 2128 : op.val[0] = stream_read_tree (&ib, data_in);
4896 2128 : if (rhs_class == GIMPLE_BINARY_RHS)
4897 2128 : op.val[1] = NULL_TREE;
4898 : else
4899 0 : op.val[1] = stream_read_tree (&ib, data_in);
4900 : break;
4901 :
4902 0 : default:
4903 0 : fatal_error (UNKNOWN_LOCATION,
4904 : "invalid fnsummary in LTO stream");
4905 : }
4906 3627 : if (info)
4907 3627 : c.param_ops->quick_push (op);
4908 : }
4909 76975 : if (info)
4910 76975 : info->conds->quick_push (c);
4911 : }
4912 86122 : count2 = streamer_read_uhwi (&ib);
4913 86122 : gcc_assert (!info || !info->size_time_table.length ());
4914 86122 : if (info && count2)
4915 86064 : info->size_time_table.reserve_exact (count2);
4916 327137 : for (j = 0; j < count2; j++)
4917 : {
4918 241015 : class size_time_entry e;
4919 :
4920 241015 : e.size = streamer_read_uhwi (&ib);
4921 241015 : e.time = sreal::stream_in (&ib);
4922 241015 : e.exec_predicate.stream_in (&ib);
4923 241015 : e.nonconst_predicate.stream_in (&ib);
4924 :
4925 241015 : if (info)
4926 240899 : info->size_time_table.quick_push (e);
4927 : }
4928 :
4929 86122 : count2 = streamer_read_uhwi (&ib);
4930 87471 : for (j = 0; j < count2; j++)
4931 : {
4932 1349 : p.stream_in (&ib);
4933 1349 : sreal fcp_freq = sreal::stream_in (&ib);
4934 1349 : if (info)
4935 : {
4936 1349 : ipa_freqcounting_predicate fcp;
4937 1349 : fcp.predicate = NULL;
4938 1349 : set_hint_predicate (&fcp.predicate, p);
4939 1349 : fcp.freq = fcp_freq;
4940 1349 : vec_safe_push (info->loop_iterations, fcp);
4941 : }
4942 : }
4943 86122 : count2 = streamer_read_uhwi (&ib);
4944 86409 : for (j = 0; j < count2; j++)
4945 : {
4946 287 : p.stream_in (&ib);
4947 287 : sreal fcp_freq = sreal::stream_in (&ib);
4948 287 : if (info)
4949 : {
4950 287 : ipa_freqcounting_predicate fcp;
4951 287 : fcp.predicate = NULL;
4952 287 : set_hint_predicate (&fcp.predicate, p);
4953 287 : fcp.freq = fcp_freq;
4954 287 : vec_safe_push (info->loop_strides, fcp);
4955 : }
4956 : }
4957 86122 : count2 = streamer_read_uhwi (&ib);
4958 86122 : if (info && count2)
4959 6 : info->builtin_constant_p_parms.reserve_exact (count2);
4960 86128 : for (j = 0; j < count2; j++)
4961 : {
4962 6 : int parm = streamer_read_uhwi (&ib);
4963 6 : if (info)
4964 6 : info->builtin_constant_p_parms.quick_push (parm);
4965 : }
4966 431949 : for (e = node->callees; e; e = e->next_callee)
4967 345827 : read_ipa_call_summary (&ib, e, info != NULL);
4968 87602 : for (e = node->indirect_calls; e; e = e->next_callee)
4969 1480 : read_ipa_call_summary (&ib, e, info != NULL);
4970 : }
4971 :
4972 13302 : lto_free_section_data (file_data, LTO_section_ipa_fn_summary, NULL, data,
4973 : len);
4974 13302 : lto_data_in_delete (data_in);
4975 13302 : }
4976 :
4977 :
4978 : /* Read inline summary. Jump functions are shared among ipa-cp
4979 : and inliner, so when ipa-cp is active, we don't need to write them
4980 : twice. */
4981 :
4982 : static void
4983 12247 : ipa_fn_summary_read (void)
4984 : {
4985 12247 : struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
4986 12247 : struct lto_file_decl_data *file_data;
4987 12247 : unsigned int j = 0;
4988 :
4989 12247 : ipa_prop_read_jump_functions ();
4990 12247 : ipa_fn_summary_alloc ();
4991 :
4992 37796 : while ((file_data = file_data_vec[j++]))
4993 : {
4994 13302 : size_t len;
4995 13302 : const char *data
4996 13302 : = lto_get_summary_section_data (file_data, LTO_section_ipa_fn_summary,
4997 : &len);
4998 13302 : if (data)
4999 13302 : inline_read_section (file_data, data, len);
5000 : else
5001 : /* Fatal error here. We do not want to support compiling ltrans units
5002 : with different version of compiler or different flags than the WPA
5003 : unit, so this should never happen. */
5004 0 : fatal_error (input_location,
5005 : "ipa inline summary is missing in input file");
5006 : }
5007 12247 : ipa_register_cgraph_hooks ();
5008 :
5009 12247 : gcc_assert (ipa_fn_summaries);
5010 12247 : ipa_fn_summaries->enable_insertion_hook ();
5011 12247 : }
5012 :
5013 :
5014 : /* Write inline summary for edge E to OB. */
5015 :
5016 : static void
5017 378873 : write_ipa_call_summary (struct output_block *ob, struct cgraph_edge *e)
5018 : {
5019 378873 : class ipa_call_summary *es = ipa_call_summaries->get (e);
5020 378873 : int i;
5021 :
5022 378873 : streamer_write_uhwi (ob, es->call_stmt_size);
5023 378873 : streamer_write_uhwi (ob, es->call_stmt_time);
5024 378873 : streamer_write_uhwi (ob, es->loop_depth);
5025 :
5026 378873 : bitpack_d bp = bitpack_create (ob->main_stream);
5027 378873 : bp_pack_value (&bp, es->is_return_callee_uncaptured, 1);
5028 378873 : streamer_write_bitpack (&bp);
5029 :
5030 378873 : if (es->predicate)
5031 10649 : es->predicate->stream_out (ob);
5032 : else
5033 368224 : streamer_write_uhwi (ob, 0);
5034 378873 : streamer_write_uhwi (ob, es->param.length ());
5035 2287133 : for (i = 0; i < (int) es->param.length (); i++)
5036 : {
5037 630715 : streamer_write_uhwi (ob, es->param[i].change_prob);
5038 630715 : bp = bitpack_create (ob->main_stream);
5039 630715 : bp_pack_value (&bp, es->param[i].points_to_local_or_readonly_memory, 1);
5040 630715 : bp_pack_value (&bp, es->param[i].points_to_possible_sra_candidate, 1);
5041 630715 : streamer_write_bitpack (&bp);
5042 : }
5043 378873 : }
5044 :
5045 :
5046 : /* Write inline summary for node in SET.
5047 : Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
5048 : active, we don't need to write them twice. */
5049 :
5050 : static void
5051 23172 : ipa_fn_summary_write (void)
5052 : {
5053 23172 : struct output_block *ob = create_output_block (LTO_section_ipa_fn_summary);
5054 23172 : lto_symtab_encoder_iterator lsei;
5055 23172 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
5056 23172 : unsigned int count = 0;
5057 :
5058 128203 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
5059 105031 : lsei_next_function_in_partition (&lsei))
5060 : {
5061 105031 : cgraph_node *cnode = lsei_cgraph_node (lsei);
5062 105031 : if (cnode->definition && !cnode->alias)
5063 102586 : count++;
5064 : }
5065 23172 : streamer_write_uhwi (ob, count);
5066 :
5067 128203 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
5068 105031 : lsei_next_function_in_partition (&lsei))
5069 : {
5070 105031 : cgraph_node *cnode = lsei_cgraph_node (lsei);
5071 105031 : if (cnode->definition && !cnode->alias)
5072 : {
5073 102586 : class ipa_fn_summary *info = ipa_fn_summaries->get (cnode);
5074 102586 : class ipa_size_summary *size_info = ipa_size_summaries->get (cnode);
5075 102586 : struct bitpack_d bp;
5076 102586 : struct cgraph_edge *edge;
5077 102586 : int i;
5078 102586 : size_time_entry *e;
5079 102586 : struct condition *c;
5080 :
5081 102586 : streamer_write_uhwi (ob, lto_symtab_encoder_encode (encoder, cnode));
5082 102586 : streamer_write_hwi (ob, size_info->estimated_self_stack_size);
5083 102586 : streamer_write_hwi (ob, size_info->self_size);
5084 102586 : info->time.stream_out (ob);
5085 102586 : bp = bitpack_create (ob->main_stream);
5086 102586 : bp_pack_value (&bp, info->inlinable, 1);
5087 102586 : bp_pack_value (&bp, info->fp_expressions, 1);
5088 102586 : streamer_write_bitpack (&bp);
5089 102586 : if (!lto_stream_offload_p)
5090 102586 : streamer_write_uhwi (ob, info->target_info);
5091 102586 : streamer_write_uhwi (ob, vec_safe_length (info->conds));
5092 195393 : for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
5093 : {
5094 92807 : int j;
5095 92807 : struct expr_eval_op *op;
5096 :
5097 92807 : streamer_write_uhwi (ob, c->operand_num);
5098 92807 : streamer_write_uhwi (ob, c->code);
5099 92807 : stream_write_tree (ob, c->type, true);
5100 92807 : stream_write_tree (ob, c->val, true);
5101 92807 : bp = bitpack_create (ob->main_stream);
5102 92807 : bp_pack_value (&bp, c->agg_contents, 1);
5103 92807 : bp_pack_value (&bp, c->by_ref, 1);
5104 92807 : streamer_write_bitpack (&bp);
5105 92807 : if (c->agg_contents)
5106 15579 : streamer_write_uhwi (ob, c->offset);
5107 92807 : streamer_write_uhwi (ob, vec_safe_length (c->param_ops));
5108 192875 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
5109 : {
5110 4282 : streamer_write_uhwi (ob, op->code);
5111 4282 : stream_write_tree (ob, op->type, true);
5112 4282 : if (op->val[0])
5113 : {
5114 2517 : bp = bitpack_create (ob->main_stream);
5115 2517 : bp_pack_value (&bp, op->index, 2);
5116 2517 : streamer_write_bitpack (&bp);
5117 2517 : stream_write_tree (ob, op->val[0], true);
5118 2517 : if (op->val[1])
5119 4 : stream_write_tree (ob, op->val[1], true);
5120 : }
5121 : }
5122 : }
5123 102586 : streamer_write_uhwi (ob, info->size_time_table.length ());
5124 495853 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
5125 : {
5126 290681 : streamer_write_uhwi (ob, e->size);
5127 290681 : e->time.stream_out (ob);
5128 290681 : e->exec_predicate.stream_out (ob);
5129 290681 : e->nonconst_predicate.stream_out (ob);
5130 : }
5131 102586 : ipa_freqcounting_predicate *fcp;
5132 102586 : streamer_write_uhwi (ob, vec_safe_length (info->loop_iterations));
5133 207148 : for (i = 0; vec_safe_iterate (info->loop_iterations, i, &fcp); i++)
5134 : {
5135 1976 : fcp->predicate->stream_out (ob);
5136 1976 : fcp->freq.stream_out (ob);
5137 : }
5138 102586 : streamer_write_uhwi (ob, vec_safe_length (info->loop_strides));
5139 205514 : for (i = 0; vec_safe_iterate (info->loop_strides, i, &fcp); i++)
5140 : {
5141 342 : fcp->predicate->stream_out (ob);
5142 342 : fcp->freq.stream_out (ob);
5143 : }
5144 102586 : streamer_write_uhwi (ob, info->builtin_constant_p_parms.length ());
5145 102586 : int ip;
5146 205186 : for (i = 0; info->builtin_constant_p_parms.iterate (i, &ip);
5147 : i++)
5148 14 : streamer_write_uhwi (ob, ip);
5149 478775 : for (edge = cnode->callees; edge; edge = edge->next_callee)
5150 376189 : write_ipa_call_summary (ob, edge);
5151 105270 : for (edge = cnode->indirect_calls; edge; edge = edge->next_callee)
5152 2684 : write_ipa_call_summary (ob, edge);
5153 : }
5154 : }
5155 23172 : streamer_write_char_stream (ob->main_stream, 0);
5156 23172 : produce_asm (ob);
5157 23172 : destroy_output_block (ob);
5158 :
5159 23172 : ipa_prop_write_jump_functions ();
5160 23172 : }
5161 :
5162 :
5163 : /* Release function summary. */
5164 :
5165 : void
5166 725302 : ipa_free_fn_summary (void)
5167 : {
5168 725302 : if (!ipa_call_summaries)
5169 : return;
5170 459163 : ggc_delete (ipa_fn_summaries);
5171 459163 : ipa_fn_summaries = NULL;
5172 459163 : delete ipa_call_summaries;
5173 459163 : ipa_call_summaries = NULL;
5174 459163 : edge_predicate_pool.release ();
5175 : /* During IPA this is one of largest datastructures to release. */
5176 459163 : if (flag_wpa)
5177 7822 : ggc_trim ();
5178 : }
5179 :
5180 : /* Release function summary. */
5181 :
5182 : void
5183 725302 : ipa_free_size_summary (void)
5184 : {
5185 725302 : if (!ipa_size_summaries)
5186 : return;
5187 459163 : delete ipa_size_summaries;
5188 459163 : ipa_size_summaries = NULL;
5189 : }
5190 :
5191 : namespace {
5192 :
5193 : const pass_data pass_data_local_fn_summary =
5194 : {
5195 : GIMPLE_PASS, /* type */
5196 : "local-fnsummary", /* name */
5197 : OPTGROUP_INLINE, /* optinfo_flags */
5198 : TV_INLINE_PARAMETERS, /* tv_id */
5199 : 0, /* properties_required */
5200 : 0, /* properties_provided */
5201 : 0, /* properties_destroyed */
5202 : 0, /* todo_flags_start */
5203 : 0, /* todo_flags_finish */
5204 : };
5205 :
5206 : class pass_local_fn_summary : public gimple_opt_pass
5207 : {
5208 : public:
5209 577534 : pass_local_fn_summary (gcc::context *ctxt)
5210 1155068 : : gimple_opt_pass (pass_data_local_fn_summary, ctxt)
5211 : {}
5212 :
5213 : /* opt_pass methods: */
5214 288767 : opt_pass * clone () final override
5215 : {
5216 288767 : return new pass_local_fn_summary (m_ctxt);
5217 : }
5218 5758205 : unsigned int execute (function *) final override
5219 : {
5220 5758205 : return compute_fn_summary_for_current ();
5221 : }
5222 :
5223 : }; // class pass_local_fn_summary
5224 :
5225 : } // anon namespace
5226 :
5227 : gimple_opt_pass *
5228 288767 : make_pass_local_fn_summary (gcc::context *ctxt)
5229 : {
5230 288767 : return new pass_local_fn_summary (ctxt);
5231 : }
5232 :
5233 :
5234 : /* Free inline summary. */
5235 :
5236 : namespace {
5237 :
5238 : const pass_data pass_data_ipa_free_fn_summary =
5239 : {
5240 : SIMPLE_IPA_PASS, /* type */
5241 : "free-fnsummary", /* name */
5242 : OPTGROUP_NONE, /* optinfo_flags */
5243 : TV_IPA_FREE_INLINE_SUMMARY, /* tv_id */
5244 : 0, /* properties_required */
5245 : 0, /* properties_provided */
5246 : 0, /* properties_destroyed */
5247 : 0, /* todo_flags_start */
5248 : 0, /* todo_flags_finish */
5249 : };
5250 :
5251 : class pass_ipa_free_fn_summary : public simple_ipa_opt_pass
5252 : {
5253 : public:
5254 577534 : pass_ipa_free_fn_summary (gcc::context *ctxt)
5255 577534 : : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary, ctxt),
5256 1155068 : small_p (false)
5257 : {}
5258 :
5259 : /* opt_pass methods: */
5260 288767 : opt_pass *clone () final override
5261 : {
5262 288767 : return new pass_ipa_free_fn_summary (m_ctxt);
5263 : }
5264 577534 : void set_pass_param (unsigned int n, bool param) final override
5265 : {
5266 577534 : gcc_assert (n == 0);
5267 577534 : small_p = param;
5268 577534 : }
5269 485459 : bool gate (function *) final override { return true; }
5270 465005 : unsigned int execute (function *) final override
5271 : {
5272 465005 : ipa_free_fn_summary ();
5273 : /* Free ipa-prop structures if they are no longer needed. */
5274 465005 : ipa_free_all_structures_after_iinln ();
5275 465005 : if (!flag_wpa)
5276 457183 : ipa_free_size_summary ();
5277 465005 : return 0;
5278 : }
5279 :
5280 : private:
5281 : bool small_p;
5282 : }; // class pass_ipa_free_fn_summary
5283 :
5284 : } // anon namespace
5285 :
5286 : simple_ipa_opt_pass *
5287 288767 : make_pass_ipa_free_fn_summary (gcc::context *ctxt)
5288 : {
5289 288767 : return new pass_ipa_free_fn_summary (ctxt);
5290 : }
5291 :
5292 : namespace {
5293 :
5294 : const pass_data pass_data_ipa_fn_summary =
5295 : {
5296 : IPA_PASS, /* type */
5297 : "fnsummary", /* name */
5298 : OPTGROUP_INLINE, /* optinfo_flags */
5299 : TV_IPA_FNSUMMARY, /* tv_id */
5300 : 0, /* properties_required */
5301 : 0, /* properties_provided */
5302 : 0, /* properties_destroyed */
5303 : 0, /* todo_flags_start */
5304 : ( TODO_dump_symtab ), /* todo_flags_finish */
5305 : };
5306 :
5307 : class pass_ipa_fn_summary : public ipa_opt_pass_d
5308 : {
5309 : public:
5310 288767 : pass_ipa_fn_summary (gcc::context *ctxt)
5311 : : ipa_opt_pass_d (pass_data_ipa_fn_summary, ctxt,
5312 : ipa_fn_summary_generate, /* generate_summary */
5313 : ipa_fn_summary_write, /* write_summary */
5314 : ipa_fn_summary_read, /* read_summary */
5315 : NULL, /* write_optimization_summary */
5316 : NULL, /* read_optimization_summary */
5317 : NULL, /* stmt_fixup */
5318 : 0, /* function_transform_todo_flags_start */
5319 : NULL, /* function_transform */
5320 288767 : NULL) /* variable_transform */
5321 288767 : {}
5322 :
5323 : /* opt_pass methods: */
5324 232558 : unsigned int execute (function *) final override { return 0; }
5325 :
5326 : }; // class pass_ipa_fn_summary
5327 :
5328 : } // anon namespace
5329 :
5330 : ipa_opt_pass_d *
5331 288767 : make_pass_ipa_fn_summary (gcc::context *ctxt)
5332 : {
5333 288767 : return new pass_ipa_fn_summary (ctxt);
5334 : }
5335 :
5336 : /* Reset all state within ipa-fnsummary.cc so that we can rerun the compiler
5337 : within the same process. For use by toplev::finalize. */
5338 :
5339 : void
5340 259439 : ipa_fnsummary_cc_finalize (void)
5341 : {
5342 259439 : ipa_free_fn_summary ();
5343 259439 : ipa_free_size_summary ();
5344 259439 : }
|
