Line data Source code
1 : /* Function summary pass.
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* Analysis of function bodies used by inter-procedural passes
22 :
23 : We estimate for each function
24 : - function body size and size after specializing into given context
25 : - average function execution time in a given context
26 : - function frame size
27 : For each call
28 : - call statement size, time and how often the parameters change
29 :
30 : ipa_fn_summary data structures store above information locally (i.e.
31 : parameters of the function itself) and globally (i.e. parameters of
32 : the function created by applying all the inline decisions already
33 : present in the callgraph).
34 :
35 : We provide access to the ipa_fn_summary data structure and
36 : basic logic updating the parameters when inlining is performed.
37 :
38 : The summaries are context sensitive. Context means
39 : 1) partial assignment of known constant values of operands
40 : 2) whether function is inlined into the call or not.
41 : It is easy to add more variants. To represent function size and time
42 : that depends on context (i.e. it is known to be optimized away when
43 : context is known either by inlining or from IP-CP and cloning),
44 : we use predicates.
45 :
46 : estimate_edge_size_and_time can be used to query
47 : function size/time in the given context. ipa_merge_fn_summary_after_inlining merges
48 : properties of caller and callee after inlining.
49 :
50 : Finally pass_inline_parameters is exported. This is used to drive
51 : computation of function parameters used by the early inliner. IPA
52 : inlined performs analysis via its analyze_function method. */
53 :
54 : #include "config.h"
55 : #define INCLUDE_VECTOR
56 : #include "system.h"
57 : #include "coretypes.h"
58 : #include "backend.h"
59 : #include "target.h"
60 : #include "tree.h"
61 : #include "gimple.h"
62 : #include "alloc-pool.h"
63 : #include "tree-pass.h"
64 : #include "ssa.h"
65 : #include "tree-streamer.h"
66 : #include "cgraph.h"
67 : #include "diagnostic.h"
68 : #include "fold-const.h"
69 : #include "print-tree.h"
70 : #include "tree-inline.h"
71 : #include "gimple-pretty-print.h"
72 : #include "cfganal.h"
73 : #include "gimple-iterator.h"
74 : #include "tree-cfg.h"
75 : #include "tree-ssa-loop-niter.h"
76 : #include "tree-ssa-loop.h"
77 : #include "symbol-summary.h"
78 : #include "sreal.h"
79 : #include "ipa-cp.h"
80 : #include "ipa-prop.h"
81 : #include "ipa-fnsummary.h"
82 : #include "cfgloop.h"
83 : #include "tree-scalar-evolution.h"
84 : #include "ipa-utils.h"
85 : #include "cfgexpand.h"
86 : #include "gimplify.h"
87 : #include "stringpool.h"
88 : #include "attribs.h"
89 : #include "tree-into-ssa.h"
90 : #include "symtab-clones.h"
91 : #include "gimple-range.h"
92 : #include "tree-dfa.h"
93 :
94 : /* Summaries. */
95 : fast_function_summary <ipa_fn_summary *, va_gc> *ipa_fn_summaries;
96 : fast_function_summary <ipa_size_summary *, va_heap> *ipa_size_summaries;
97 : fast_call_summary <ipa_call_summary *, va_heap> *ipa_call_summaries;
98 :
99 : /* Edge predicates goes here. */
100 : static object_allocator<ipa_predicate> edge_predicate_pool ("edge predicates");
101 :
102 :
103 : /* Dump IPA hints. */
104 : void
105 185 : ipa_dump_hints (FILE *f, ipa_hints hints)
106 : {
107 185 : if (!hints)
108 : return;
109 148 : fprintf (f, "IPA hints:");
110 148 : if (hints & INLINE_HINT_indirect_call)
111 : {
112 23 : hints &= ~INLINE_HINT_indirect_call;
113 23 : fprintf (f, " indirect_call");
114 : }
115 148 : if (hints & INLINE_HINT_loop_iterations)
116 : {
117 3 : hints &= ~INLINE_HINT_loop_iterations;
118 3 : fprintf (f, " loop_iterations");
119 : }
120 148 : if (hints & INLINE_HINT_loop_stride)
121 : {
122 3 : hints &= ~INLINE_HINT_loop_stride;
123 3 : fprintf (f, " loop_stride");
124 : }
125 148 : if (hints & INLINE_HINT_same_scc)
126 : {
127 4 : hints &= ~INLINE_HINT_same_scc;
128 4 : fprintf (f, " same_scc");
129 : }
130 148 : if (hints & INLINE_HINT_in_scc)
131 : {
132 11 : hints &= ~INLINE_HINT_in_scc;
133 11 : fprintf (f, " in_scc");
134 : }
135 148 : if (hints & INLINE_HINT_cross_module)
136 : {
137 2 : hints &= ~INLINE_HINT_cross_module;
138 2 : fprintf (f, " cross_module");
139 : }
140 148 : if (hints & INLINE_HINT_declared_inline)
141 : {
142 122 : hints &= ~INLINE_HINT_declared_inline;
143 122 : fprintf (f, " declared_inline");
144 : }
145 148 : if (hints & INLINE_HINT_known_hot)
146 : {
147 1 : hints &= ~INLINE_HINT_known_hot;
148 1 : fprintf (f, " known_hot");
149 : }
150 148 : if (hints & INLINE_HINT_builtin_constant_p)
151 : {
152 4 : hints &= ~INLINE_HINT_builtin_constant_p;
153 4 : fprintf (f, " builtin_constant_p");
154 : }
155 148 : gcc_assert (!hints);
156 : }
157 :
158 :
159 : /* Record SIZE and TIME to SUMMARY.
160 : The accounted code will be executed when EXEC_PRED is true.
161 : When NONCONST_PRED is false the code will evaluate to constant and
162 : will get optimized out in specialized clones of the function.
163 : If CALL is true account to call_size_time_table rather than
164 : size_time_table. */
165 :
166 : void
167 134062665 : ipa_fn_summary::account_size_time (int size, sreal time,
168 : const ipa_predicate &exec_pred,
169 : const ipa_predicate &nonconst_pred_in,
170 : bool call)
171 : {
172 134062665 : size_time_entry *e;
173 134062665 : bool found = false;
174 134062665 : int i;
175 134062665 : ipa_predicate nonconst_pred;
176 134062665 : vec<size_time_entry> *table = call ? &call_size_time_table : &size_time_table;
177 :
178 134062665 : if (exec_pred == false)
179 4690601 : return;
180 :
181 133806169 : nonconst_pred = nonconst_pred_in & exec_pred;
182 :
183 133806169 : if (nonconst_pred == false)
184 : return;
185 :
186 : /* We need to create initial empty unconditional clause, but otherwise
187 : we don't need to account empty times and sizes. */
188 156558024 : if (!size && time == 0 && table->length ())
189 3723090 : return;
190 :
191 : /* Only for calls we are unaccounting what we previously recorded. */
192 129372064 : gcc_checking_assert (time >= 0 || call);
193 :
194 467266318 : for (i = 0; table->iterate (i, &e); i++)
195 438357086 : if (e->exec_predicate == exec_pred
196 438357086 : && e->nonconst_predicate == nonconst_pred)
197 : {
198 : found = true;
199 : break;
200 : }
201 129372064 : if (i == max_size_time_table_size)
202 : {
203 0 : i = 0;
204 0 : found = true;
205 0 : e = &(*table)[0];
206 0 : if (dump_file && (dump_flags & TDF_DETAILS))
207 0 : fprintf (dump_file,
208 : "\t\tReached limit on number of entries, "
209 : "ignoring the predicate.");
210 : }
211 129376338 : if (dump_file && (dump_flags & TDF_DETAILS) && (time != 0 || size))
212 : {
213 3997 : fprintf (dump_file,
214 : "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
215 3766 : ((double) size) / ipa_fn_summary::size_scale,
216 : (time.to_double ()), found ? "" : "new ");
217 3766 : exec_pred.dump (dump_file, conds, 0);
218 3766 : if (exec_pred != nonconst_pred)
219 : {
220 84 : fprintf (dump_file, " nonconst:");
221 84 : nonconst_pred.dump (dump_file, conds);
222 : }
223 : else
224 3682 : fprintf (dump_file, "\n");
225 : }
226 129372064 : if (!found)
227 : {
228 28909232 : class size_time_entry new_entry;
229 28909232 : new_entry.size = size;
230 28909232 : new_entry.time = time;
231 28909232 : new_entry.exec_predicate = exec_pred;
232 28909232 : new_entry.nonconst_predicate = nonconst_pred;
233 28909232 : if (call)
234 1303663 : call_size_time_table.safe_push (new_entry);
235 : else
236 27605569 : size_time_table.safe_push (new_entry);
237 : }
238 : else
239 : {
240 100462832 : e->size += size;
241 100462832 : e->time += time;
242 : /* FIXME: PR bootstrap/92653 gcc_checking_assert (e->time >= -1); */
243 : /* Tolerate small roundoff issues. */
244 100462832 : if (e->time < 0)
245 210 : e->time = 0;
246 : }
247 : }
248 :
249 : /* We proved E to be unreachable, redirect it to __builtin_unreachable. */
250 :
251 : static struct cgraph_edge *
252 270172 : redirect_to_unreachable (struct cgraph_edge *e)
253 : {
254 270172 : struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
255 270172 : struct cgraph_node *target
256 270172 : = cgraph_node::get_create (builtin_decl_unreachable ());
257 :
258 270172 : gcc_checking_assert (lookup_attribute ("cold",
259 : DECL_ATTRIBUTES (target->decl)));
260 :
261 270172 : if (e->speculative)
262 314 : e = cgraph_edge::resolve_speculation (e, target->decl);
263 269858 : else if (!e->callee)
264 763 : e = cgraph_edge::make_direct (e, target);
265 : else
266 269095 : e->redirect_callee (target);
267 270172 : class ipa_call_summary *es = ipa_call_summaries->get (e);
268 270172 : e->inline_failed = CIF_UNREACHABLE;
269 270172 : e->count = profile_count::zero ();
270 270172 : es->call_stmt_size = 0;
271 270172 : es->call_stmt_time = 0;
272 270172 : if (callee)
273 0 : callee->remove_symbol_and_inline_clones ();
274 270172 : if (e->has_callback)
275 4 : for (cgraph_edge *cbe = e->first_callback_edge (); cbe;
276 1 : cbe = cbe->next_callback_edge ())
277 : /* If the carrying edge is unreachable, so are the callback calls. */
278 1 : redirect_to_unreachable (cbe);
279 270172 : return e;
280 : }
281 :
282 : /* Set predicate for edge E. */
283 :
284 : static void
285 30455335 : edge_set_predicate (struct cgraph_edge *e, ipa_predicate *predicate)
286 : {
287 : /* If the edge is determined to be never executed, redirect it
288 : to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
289 : be optimized out. */
290 27807036 : if (predicate && *predicate == false
291 : /* When handling speculative edges, we need to do the redirection
292 : just once. Do it always on the direct edge, so we do not
293 : attempt to resolve speculation while duplicating the edge. */
294 30725582 : && (!e->speculative || e->callee))
295 270171 : e = redirect_to_unreachable (e);
296 :
297 30455335 : class ipa_call_summary *es = ipa_call_summaries->get (e);
298 58262371 : if (predicate && *predicate != true)
299 : {
300 4491508 : if (!es->predicate)
301 4027676 : es->predicate = edge_predicate_pool.allocate ();
302 4491508 : *es->predicate = *predicate;
303 : }
304 : else
305 : {
306 25963827 : if (es->predicate)
307 814443 : edge_predicate_pool.remove (es->predicate);
308 25963827 : es->predicate = NULL;
309 : }
310 30455335 : }
311 :
312 : /* Set predicate for hint *P. */
313 :
314 : static void
315 148115 : set_hint_predicate (ipa_predicate **p, ipa_predicate new_predicate)
316 : {
317 148115 : if (new_predicate == false || new_predicate == true)
318 : {
319 3015 : if (*p)
320 0 : edge_predicate_pool.remove (*p);
321 3015 : *p = NULL;
322 : }
323 : else
324 : {
325 145100 : if (!*p)
326 145100 : *p = edge_predicate_pool.allocate ();
327 145100 : **p = new_predicate;
328 : }
329 148115 : }
330 :
331 : /* Find if NEW_PREDICATE is already in V and if so, increment its freq.
332 : Otherwise add a new item to the vector with this predicate and frerq equal
333 : to add_freq, unless the number of predicates would exceed MAX_NUM_PREDICATES
334 : in which case the function does nothing. */
335 :
336 : static void
337 1106683 : add_freqcounting_predicate (vec<ipa_freqcounting_predicate, va_gc> **v,
338 : const ipa_predicate &new_predicate, sreal add_freq,
339 : unsigned max_num_predicates)
340 : {
341 1106683 : if (new_predicate == false || new_predicate == true)
342 : return;
343 : ipa_freqcounting_predicate *f;
344 139724 : for (int i = 0; vec_safe_iterate (*v, i, &f); i++)
345 71500 : if (new_predicate == f->predicate)
346 : {
347 0 : f->freq += add_freq;
348 0 : return;
349 : }
350 94139 : if (vec_safe_length (*v) >= max_num_predicates)
351 : /* Too many different predicates to account for. */
352 : return;
353 :
354 67952 : ipa_freqcounting_predicate fcp;
355 67952 : fcp.predicate = NULL;
356 67952 : set_hint_predicate (&fcp.predicate, new_predicate);
357 67952 : fcp.freq = add_freq;
358 67952 : vec_safe_push (*v, fcp);
359 67952 : return;
360 : }
361 :
362 : /* Compute what conditions may or may not hold given information about
363 : parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
364 : while RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
365 : copy when called in a given context. It is a bitmask of conditions. Bit
366 : 0 means that condition is known to be false, while bit 1 means that condition
367 : may or may not be true. These differs - for example NOT_INLINED condition
368 : is always false in the second and also builtin_constant_p tests cannot use
369 : the fact that parameter is indeed a constant.
370 :
371 : When INLINE_P is true, assume that we are inlining. AVAL contains known
372 : information about argument values. The function does not modify its content
373 : and so AVALs could also be of type ipa_call_arg_values but so far all
374 : callers work with the auto version and so we avoid the conversion for
375 : convenience.
376 :
377 : ERROR_MARK value of an argument means compile time invariant. */
378 :
379 : static void
380 21755602 : evaluate_conditions_for_known_args (struct cgraph_node *node,
381 : bool inline_p,
382 : ipa_auto_call_arg_values *avals,
383 : clause_t *ret_clause,
384 : clause_t *ret_nonspec_clause,
385 : ipa_call_summary *es)
386 : {
387 21755602 : clause_t clause = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
388 21755602 : clause_t nonspec_clause = 1 << ipa_predicate::not_inlined_condition;
389 21755602 : class ipa_fn_summary *info = ipa_fn_summaries->get (node);
390 21755602 : int i;
391 21755602 : struct condition *c;
392 :
393 90117594 : for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
394 : {
395 68361992 : tree val = NULL;
396 68361992 : tree res;
397 68361992 : int j;
398 68361992 : struct expr_eval_op *op;
399 :
400 68361992 : if (c->code == ipa_predicate::not_sra_candidate)
401 : {
402 16070261 : if (!inline_p
403 16070261 : || !es
404 16127583 : || (int)es->param.length () <= c->operand_num
405 24134021 : || !es->param[c->operand_num].points_to_possible_sra_candidate)
406 11970133 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
407 16070261 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
408 68361992 : continue;
409 : }
410 :
411 52291731 : if (c->agg_contents)
412 : {
413 28338823 : if (c->code == ipa_predicate::changed
414 18157342 : && !c->by_ref
415 30844292 : && (avals->safe_sval_at(c->operand_num) == error_mark_node))
416 5208 : continue;
417 :
418 28328407 : if (tree sval = avals->safe_sval_at (c->operand_num))
419 13197190 : val = ipa_find_agg_cst_from_init (sval, c->offset, c->by_ref);
420 13197190 : if (!val)
421 : {
422 28318084 : ipa_argagg_value_list avs (avals);
423 28318084 : val = avs.get_value (c->operand_num, c->offset / BITS_PER_UNIT,
424 28318084 : c->by_ref);
425 : }
426 : }
427 : else
428 : {
429 23958116 : val = avals->safe_sval_at (c->operand_num);
430 23958116 : if (val && val == error_mark_node
431 2257803 : && c->code != ipa_predicate::changed)
432 : val = NULL_TREE;
433 : }
434 :
435 39924929 : if (!val
436 38876158 : && (c->code == ipa_predicate::changed
437 : || c->code == ipa_predicate::is_not_constant))
438 : {
439 24753207 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
440 24753207 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
441 24753207 : continue;
442 : }
443 27533316 : if (c->code == ipa_predicate::changed)
444 : {
445 7558570 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
446 7558570 : continue;
447 : }
448 :
449 19974746 : if (c->code == ipa_predicate::is_not_constant)
450 : {
451 5539 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
452 5539 : continue;
453 : }
454 :
455 19969207 : if (val && TYPE_SIZE (c->type) == TYPE_SIZE (TREE_TYPE (val)))
456 : {
457 5777375 : if (c->type != TREE_TYPE (val))
458 1230836 : val = fold_unary (VIEW_CONVERT_EXPR, c->type, val);
459 6192012 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
460 : {
461 415081 : if (!val)
462 : break;
463 414637 : if (!op->val[0])
464 155540 : val = fold_unary (op->code, op->type, val);
465 259097 : else if (!op->val[1])
466 518194 : val = fold_binary (op->code, op->type,
467 : op->index ? op->val[0] : val,
468 : op->index ? val : op->val[0]);
469 0 : else if (op->index == 0)
470 0 : val = fold_ternary (op->code, op->type,
471 : val, op->val[0], op->val[1]);
472 0 : else if (op->index == 1)
473 0 : val = fold_ternary (op->code, op->type,
474 : op->val[0], val, op->val[1]);
475 0 : else if (op->index == 2)
476 0 : val = fold_ternary (op->code, op->type,
477 : op->val[0], op->val[1], val);
478 : else
479 : val = NULL_TREE;
480 : }
481 :
482 5777375 : res = val
483 5777375 : ? fold_binary_to_constant (c->code, boolean_type_node, val, c->val)
484 : : NULL;
485 :
486 5776911 : if (res && integer_zerop (res))
487 2975139 : continue;
488 2802236 : if (res && integer_onep (res))
489 : {
490 2779594 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
491 2779594 : nonspec_clause
492 2779594 : |= 1 << (i + ipa_predicate::first_dynamic_condition);
493 2779594 : continue;
494 : }
495 : }
496 14214474 : if (c->operand_num < (int) avals->m_known_value_ranges.length ()
497 8182308 : && !c->agg_contents
498 16120833 : && (!val || TREE_CODE (val) != INTEGER_CST))
499 : {
500 1906270 : value_range vr (avals->m_known_value_ranges[c->operand_num]);
501 1906270 : if (!vr.undefined_p ()
502 1243629 : && !vr.varying_p ()
503 3149899 : && (TYPE_SIZE (c->type) == TYPE_SIZE (vr.type ())))
504 : {
505 1243192 : if (!useless_type_conversion_p (c->type, vr.type ()))
506 634 : range_cast (vr, c->type);
507 :
508 1558116 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
509 : {
510 352720 : if (vr.varying_p () || vr.undefined_p ())
511 : break;
512 :
513 314924 : value_range res (op->type);
514 314924 : if (!op->val[0])
515 : {
516 103058 : value_range varying (op->type);
517 103058 : varying.set_varying (op->type);
518 103058 : range_op_handler handler (op->code);
519 103058 : if (!handler
520 103058 : || !res.supports_type_p (op->type)
521 206116 : || !handler.fold_range (res, op->type, vr, varying))
522 0 : res.set_varying (op->type);
523 103058 : }
524 211866 : else if (!op->val[1])
525 : {
526 211694 : value_range op0 (TREE_TYPE (op->val[0]));
527 211694 : range_op_handler handler (op->code);
528 :
529 211694 : ipa_get_range_from_ip_invariant (op0, op->val[0], node);
530 :
531 211694 : if (!handler
532 211694 : || !res.supports_type_p (op->type)
533 423388 : || !handler.fold_range (res, op->type,
534 211694 : op->index ? op0 : vr,
535 422800 : op->index ? vr : op0))
536 0 : res.set_varying (op->type);
537 211694 : }
538 : else
539 172 : res.set_varying (op->type);
540 314924 : vr = res;
541 314924 : }
542 1243192 : if (!vr.varying_p () && !vr.undefined_p ())
543 : {
544 1199923 : int_range<2> res;
545 1199923 : value_range val_vr (TREE_TYPE (c->val));
546 1199923 : range_op_handler handler (c->code);
547 :
548 1199923 : ipa_get_range_from_ip_invariant (val_vr, c->val, node);
549 :
550 1199923 : if (!handler
551 1199923 : || !val_vr.supports_type_p (TREE_TYPE (c->val))
552 2399846 : || !handler.fold_range (res, boolean_type_node, vr, val_vr))
553 0 : res.set_varying (boolean_type_node);
554 :
555 1199923 : if (res.zero_p ())
556 204101 : continue;
557 1199923 : }
558 : }
559 1906270 : }
560 :
561 14010373 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
562 14010373 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
563 : }
564 21755602 : *ret_clause = clause;
565 21755602 : if (ret_nonspec_clause)
566 18903831 : *ret_nonspec_clause = nonspec_clause;
567 21755602 : }
568 :
569 : /* Return true if VRP will be exectued on the function.
570 : We do not want to anticipate optimizations that will not happen.
571 :
572 : FIXME: This can be confused with -fdisable and debug counters and thus
573 : it should not be used for correctness (only to make heuristics work).
574 : This means that inliner should do its own optimizations of expressions
575 : that it predicts to be constant so wrong code can not be triggered by
576 : builtin_constant_p. */
577 :
578 : static bool
579 11911138 : vrp_will_run_p (struct cgraph_node *node)
580 : {
581 11911138 : return (opt_for_fn (node->decl, optimize)
582 11911138 : && !opt_for_fn (node->decl, optimize_debug)
583 23821552 : && opt_for_fn (node->decl, flag_tree_vrp));
584 : }
585 :
586 : /* Similarly about FRE. */
587 :
588 : static bool
589 14186665 : fre_will_run_p (struct cgraph_node *node)
590 : {
591 14186665 : return (opt_for_fn (node->decl, optimize)
592 14186665 : && !opt_for_fn (node->decl, optimize_debug)
593 28371736 : && opt_for_fn (node->decl, flag_tree_fre));
594 : }
595 :
596 : /* Work out what conditions might be true at invocation of E.
597 : Compute costs for inlined edge if INLINE_P is true.
598 :
599 : Return in CLAUSE_PTR the evaluated conditions and in NONSPEC_CLAUSE_PTR
600 : (if non-NULL) conditions evaluated for nonspecialized clone called
601 : in a given context.
602 :
603 : Vectors in AVALS will be populated with useful known information about
604 : argument values - information not known to have any uses will be omitted -
605 : except for m_known_contexts which will only be calculated if
606 : COMPUTE_CONTEXTS is true. */
607 :
608 : void
609 21458971 : evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p,
610 : clause_t *clause_ptr,
611 : clause_t *nonspec_clause_ptr,
612 : ipa_auto_call_arg_values *avals,
613 : bool compute_contexts)
614 : {
615 21458971 : struct cgraph_node *callee = e->callee->ultimate_alias_target ();
616 21458971 : class ipa_fn_summary *info = ipa_fn_summaries->get (callee);
617 21458971 : class ipa_edge_args *args;
618 21458971 : class ipa_call_summary *es = NULL;
619 :
620 21458971 : if (clause_ptr)
621 21458971 : *clause_ptr = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
622 :
623 21458971 : if (ipa_node_params_sum
624 9911191 : && !e->call_stmt_cannot_inline_p
625 9911191 : && (info->conds || compute_contexts)
626 31370162 : && (args = ipa_edge_args_sum->get (e)) != NULL)
627 : {
628 9856712 : struct cgraph_node *caller;
629 9856712 : class ipa_node_params *caller_parms_info, *callee_pi = NULL;
630 9856712 : int i, count = ipa_get_cs_argument_count (args);
631 9856712 : es = ipa_call_summaries->get (e);
632 :
633 9856712 : if (count)
634 : {
635 9161721 : if (e->caller->inlined_to)
636 : caller = e->caller->inlined_to;
637 : else
638 7372852 : caller = e->caller;
639 9161721 : caller_parms_info = ipa_node_params_sum->get (caller);
640 9161721 : callee_pi = ipa_node_params_sum->get (callee);
641 :
642 : /* Watch for thunks. */
643 9161721 : if (callee_pi)
644 : /* Watch for variadic functions. */
645 9161380 : count = MIN (count, ipa_get_param_count (callee_pi));
646 : }
647 :
648 9161380 : if (callee_pi)
649 30084290 : for (i = 0; i < count; i++)
650 : {
651 20922910 : struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i);
652 :
653 20922910 : if (ipa_is_param_used_by_indirect_call (callee_pi, i)
654 20922910 : || ipa_is_param_used_by_ipa_predicates (callee_pi, i))
655 : {
656 : /* Determine if we know constant value of the parameter. */
657 14186665 : tree type = ipa_get_type (callee_pi, i);
658 14186665 : tree cst = ipa_value_from_jfunc (caller_parms_info, jf, type);
659 :
660 11049453 : if (!cst && e->call_stmt
661 25185472 : && i < (int)gimple_call_num_args (e->call_stmt))
662 : {
663 10998807 : cst = gimple_call_arg (e->call_stmt, i);
664 10998807 : if (!is_gimple_min_invariant (cst))
665 : cst = NULL;
666 : }
667 3374361 : if (cst)
668 : {
669 3323715 : gcc_checking_assert (TREE_CODE (cst) != TREE_BINFO);
670 3323715 : if (!avals->m_known_vals.length ())
671 1806015 : avals->m_known_vals.safe_grow_cleared (count, true);
672 3323715 : avals->m_known_vals[i] = cst;
673 : }
674 10862950 : else if (inline_p && !es->param[i].change_prob)
675 : {
676 4119875 : if (!avals->m_known_vals.length ())
677 3416226 : avals->m_known_vals.safe_grow_cleared (count, true);
678 4119875 : avals->m_known_vals[i] = error_mark_node;
679 : }
680 :
681 : /* If we failed to get simple constant, try value range. */
682 3323715 : if ((!cst || TREE_CODE (cst) != INTEGER_CST)
683 11911138 : && vrp_will_run_p (caller)
684 25934703 : && ipa_is_param_used_by_ipa_predicates (callee_pi, i))
685 : {
686 11707665 : value_range vr (type);
687 :
688 11707665 : ipa_value_range_from_jfunc (vr, caller_parms_info, e, jf, type);
689 11707665 : if (!vr.undefined_p () && !vr.varying_p ())
690 : {
691 8352937 : if (!avals->m_known_value_ranges.length ())
692 : {
693 6018664 : avals->m_known_value_ranges.safe_grow_cleared (count,
694 : true);
695 19147302 : for (int i = 0; i < count; ++i)
696 13128638 : avals->m_known_value_ranges[i].set_range_class
697 13128638 : (void_type_node);
698 : }
699 8352937 : avals->m_known_value_ranges[i] = vr;
700 : }
701 11707665 : }
702 :
703 : /* Determine known aggregate values. */
704 14186665 : if (fre_will_run_p (caller))
705 14184667 : ipa_push_agg_values_from_jfunc (caller_parms_info,
706 : caller, &jf->agg, i,
707 : &avals->m_known_aggs);
708 : }
709 :
710 : /* For calls used in polymorphic calls we further determine
711 : polymorphic call context. */
712 20922910 : if (compute_contexts
713 20922910 : && ipa_is_param_used_by_polymorphic_call (callee_pi, i))
714 : {
715 163194 : ipa_polymorphic_call_context
716 163194 : ctx = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
717 326388 : if (!ctx.useless_p ())
718 : {
719 161246 : if (!avals->m_known_contexts.length ())
720 161109 : avals->m_known_contexts.safe_grow_cleared (count, true);
721 161246 : avals->m_known_contexts[i]
722 322492 : = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
723 : }
724 : }
725 : }
726 : else
727 695332 : gcc_assert (!count || callee->thunk);
728 : }
729 11602259 : else if (e->call_stmt && !e->call_stmt_cannot_inline_p && info->conds)
730 : {
731 9092985 : int i, count = (int)gimple_call_num_args (e->call_stmt);
732 :
733 25877680 : for (i = 0; i < count; i++)
734 : {
735 16784695 : tree cst = gimple_call_arg (e->call_stmt, i);
736 16784695 : if (!is_gimple_min_invariant (cst))
737 : cst = NULL;
738 5975722 : if (cst)
739 : {
740 5975722 : if (!avals->m_known_vals.length ())
741 4112612 : avals->m_known_vals.safe_grow_cleared (count, true);
742 5975722 : avals->m_known_vals[i] = cst;
743 : }
744 : }
745 : }
746 :
747 21458971 : evaluate_conditions_for_known_args (callee, inline_p, avals, clause_ptr,
748 : nonspec_clause_ptr, es);
749 21458971 : }
750 :
751 :
752 : /* Allocate the function summary. */
753 :
754 : static void
755 457941 : ipa_fn_summary_alloc (void)
756 : {
757 457941 : gcc_checking_assert (!ipa_fn_summaries);
758 457941 : ipa_size_summaries = new ipa_size_summary_t (symtab);
759 457941 : ipa_fn_summaries = ipa_fn_summary_t::create_ggc (symtab);
760 457941 : ipa_call_summaries = new ipa_call_summary_t (symtab);
761 457941 : }
762 :
763 26782345 : ipa_call_summary::~ipa_call_summary ()
764 : {
765 26782345 : if (predicate)
766 3213233 : edge_predicate_pool.remove (predicate);
767 :
768 26782345 : param.release ();
769 26782345 : }
770 :
771 9843736 : ipa_fn_summary::~ipa_fn_summary ()
772 : {
773 9843736 : unsigned len = vec_safe_length (loop_iterations);
774 9955297 : for (unsigned i = 0; i < len; i++)
775 111561 : edge_predicate_pool.remove ((*loop_iterations)[i].predicate);
776 9843736 : len = vec_safe_length (loop_strides);
777 9877275 : for (unsigned i = 0; i < len; i++)
778 33539 : edge_predicate_pool.remove ((*loop_strides)[i].predicate);
779 9843736 : vec_free (conds);
780 9843736 : call_size_time_table.release ();
781 9843736 : vec_free (loop_iterations);
782 9843736 : vec_free (loop_strides);
783 9843736 : builtin_constant_p_parms.release ();
784 9843736 : }
785 :
786 : void
787 7135062 : ipa_fn_summary_t::remove_callees (cgraph_node *node)
788 : {
789 7135062 : cgraph_edge *e;
790 29330877 : for (e = node->callees; e; e = e->next_callee)
791 22195815 : ipa_call_summaries->remove (e);
792 7611528 : for (e = node->indirect_calls; e; e = e->next_callee)
793 476466 : ipa_call_summaries->remove (e);
794 7135062 : }
795 :
796 : /* Duplicate predicates in loop hint vector, allocating memory for them and
797 : remove and deallocate any uninteresting (true or false) ones. Return the
798 : result. */
799 :
800 : static vec<ipa_freqcounting_predicate, va_gc> *
801 29460 : remap_freqcounting_preds_after_dup (vec<ipa_freqcounting_predicate, va_gc> *v,
802 : clause_t possible_truths)
803 : {
804 29460 : if (vec_safe_length (v) == 0)
805 : return NULL;
806 :
807 4699 : vec<ipa_freqcounting_predicate, va_gc> *res = v->copy ();
808 4699 : int len = res->length();
809 11001 : for (int i = len - 1; i >= 0; i--)
810 : {
811 6302 : ipa_predicate new_predicate
812 6302 : = (*res)[i].predicate->remap_after_duplication (possible_truths);
813 : /* We do not want to free previous predicate; it is used by node
814 : origin. */
815 6302 : (*res)[i].predicate = NULL;
816 6302 : set_hint_predicate (&(*res)[i].predicate, new_predicate);
817 :
818 6302 : if (!(*res)[i].predicate)
819 3015 : res->unordered_remove (i);
820 : }
821 :
822 : return res;
823 : }
824 :
825 :
826 : /* Hook that is called by cgraph.cc when a node is duplicated. */
827 : void
828 2621486 : ipa_fn_summary_t::duplicate (cgraph_node *src,
829 : cgraph_node *dst,
830 : ipa_fn_summary *src_info,
831 : ipa_fn_summary *info)
832 : {
833 2621486 : new (info) ipa_fn_summary (*src_info);
834 : /* TODO: as an optimization, we may avoid copying conditions
835 : that are known to be false or true. */
836 2621486 : info->conds = vec_safe_copy (info->conds);
837 :
838 2621486 : clone_info *cinfo = clone_info::get (dst);
839 : /* When there are any replacements in the function body, see if we can figure
840 : out that something was optimized out. */
841 2621486 : if (ipa_node_params_sum && cinfo && cinfo->tree_map)
842 : {
843 : /* Use SRC parm info since it may not be copied yet. */
844 14730 : ipa_node_params *parms_info = ipa_node_params_sum->get (src);
845 14730 : ipa_auto_call_arg_values avals;
846 14730 : int count = ipa_get_param_count (parms_info);
847 14730 : int i, j;
848 14730 : clause_t possible_truths;
849 14730 : ipa_predicate true_pred = true;
850 14730 : size_time_entry *e;
851 14730 : int optimized_out_size = 0;
852 14730 : bool inlined_to_p = false;
853 14730 : struct cgraph_edge *edge, *next;
854 :
855 14730 : info->size_time_table.release ();
856 14730 : avals.m_known_vals.safe_grow_cleared (count, true);
857 60068 : for (i = 0; i < count; i++)
858 : {
859 : struct ipa_replace_map *r;
860 :
861 149001 : for (j = 0; vec_safe_iterate (cinfo->tree_map, j, &r); j++)
862 : {
863 84011 : if (r->parm_num == i)
864 : {
865 25686 : avals.m_known_vals[i] = r->new_tree;
866 25686 : break;
867 : }
868 : }
869 : }
870 14730 : evaluate_conditions_for_known_args (dst, false,
871 : &avals,
872 : &possible_truths,
873 : /* We are going to specialize,
874 : so ignore nonspec truths. */
875 : NULL,
876 : NULL);
877 :
878 14730 : info->account_size_time (0, 0, true_pred, true_pred);
879 :
880 : /* Remap size_time vectors.
881 : Simplify the predicate by pruning out alternatives that are known
882 : to be false.
883 : TODO: as on optimization, we can also eliminate conditions known
884 : to be true. */
885 104894 : for (i = 0; src_info->size_time_table.iterate (i, &e); i++)
886 : {
887 90164 : ipa_predicate new_exec_pred;
888 90164 : ipa_predicate new_nonconst_pred;
889 90164 : new_exec_pred = e->exec_predicate.remap_after_duplication
890 90164 : (possible_truths);
891 90164 : new_nonconst_pred = e->nonconst_predicate.remap_after_duplication
892 90164 : (possible_truths);
893 90164 : if (new_exec_pred == false || new_nonconst_pred == false)
894 13021 : optimized_out_size += e->size;
895 : else
896 77143 : info->account_size_time (e->size, e->time, new_exec_pred,
897 : new_nonconst_pred);
898 : }
899 :
900 : /* Remap edge predicates with the same simplification as above.
901 : Also copy constantness arrays. */
902 190412 : for (edge = dst->callees; edge; edge = next)
903 : {
904 175682 : ipa_predicate new_predicate;
905 175682 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
906 175682 : next = edge->next_callee;
907 :
908 175682 : if (!edge->inline_failed)
909 0 : inlined_to_p = true;
910 175682 : if (!es->predicate)
911 160173 : continue;
912 15509 : new_predicate = es->predicate->remap_after_duplication
913 15509 : (possible_truths);
914 15509 : if (new_predicate == false && *es->predicate != false)
915 4496 : optimized_out_size += es->call_stmt_size * ipa_fn_summary::size_scale;
916 15509 : edge_set_predicate (edge, &new_predicate);
917 : }
918 :
919 : /* Remap indirect edge predicates with the same simplification as above.
920 : Also copy constantness arrays. */
921 15634 : for (edge = dst->indirect_calls; edge; edge = next)
922 : {
923 904 : ipa_predicate new_predicate;
924 904 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
925 904 : next = edge->next_callee;
926 :
927 904 : gcc_checking_assert (edge->inline_failed);
928 904 : if (!es->predicate)
929 799 : continue;
930 105 : new_predicate = es->predicate->remap_after_duplication
931 105 : (possible_truths);
932 105 : if (new_predicate == false && *es->predicate != false)
933 24 : optimized_out_size
934 24 : += es->call_stmt_size * ipa_fn_summary::size_scale;
935 105 : edge_set_predicate (edge, &new_predicate);
936 : }
937 14730 : info->loop_iterations
938 14730 : = remap_freqcounting_preds_after_dup (info->loop_iterations,
939 : possible_truths);
940 14730 : info->loop_strides
941 14730 : = remap_freqcounting_preds_after_dup (info->loop_strides,
942 : possible_truths);
943 14730 : if (info->builtin_constant_p_parms.length())
944 : {
945 22 : vec <int, va_heap, vl_ptr> parms = info->builtin_constant_p_parms;
946 22 : int ip;
947 22 : info->builtin_constant_p_parms = vNULL;
948 44 : for (i = 0; parms.iterate (i, &ip); i++)
949 22 : if (!avals.m_known_vals[ip])
950 4 : info->builtin_constant_p_parms.safe_push (ip);
951 : }
952 :
953 : /* If inliner or someone after inliner will ever start producing
954 : non-trivial clones, we will get trouble with lack of information
955 : about updating self sizes, because size vectors already contains
956 : sizes of the callees. */
957 14730 : gcc_assert (!inlined_to_p || !optimized_out_size);
958 14730 : }
959 : else
960 : {
961 2606756 : info->size_time_table = src_info->size_time_table.copy ();
962 2606756 : info->loop_iterations = vec_safe_copy (src_info->loop_iterations);
963 2606756 : info->loop_strides = vec_safe_copy (info->loop_strides);
964 :
965 2606756 : info->builtin_constant_p_parms
966 2606756 : = info->builtin_constant_p_parms.copy ();
967 :
968 2606756 : ipa_freqcounting_predicate *f;
969 2655012 : for (int i = 0; vec_safe_iterate (info->loop_iterations, i, &f); i++)
970 : {
971 48256 : ipa_predicate p = *f->predicate;
972 48256 : f->predicate = NULL;
973 48256 : set_hint_predicate (&f->predicate, p);
974 : }
975 2630725 : for (int i = 0; vec_safe_iterate (info->loop_strides, i, &f); i++)
976 : {
977 23969 : ipa_predicate p = *f->predicate;
978 23969 : f->predicate = NULL;
979 23969 : set_hint_predicate (&f->predicate, p);
980 : }
981 : }
982 2621486 : if (!dst->inlined_to)
983 135719 : ipa_update_overall_fn_summary (dst);
984 2621486 : }
985 :
986 :
987 : /* Hook that is called by cgraph.cc when a node is duplicated. */
988 :
989 : void
990 3757387 : ipa_call_summary_t::duplicate (struct cgraph_edge *src,
991 : struct cgraph_edge *dst,
992 : class ipa_call_summary *srcinfo,
993 : class ipa_call_summary *info)
994 : {
995 3757387 : new (info) ipa_call_summary (*srcinfo);
996 3757387 : info->predicate = NULL;
997 3757387 : edge_set_predicate (dst, srcinfo->predicate);
998 3757387 : info->param = srcinfo->param.copy ();
999 3757387 : if (!dst->indirect_unknown_callee && src->indirect_unknown_callee
1000 : /* Don't subtract the size when dealing with callback pairs, since the
1001 : edge has no real size. */
1002 21600 : && !src->has_callback && !dst->callback)
1003 : {
1004 21600 : info->call_stmt_size -= (eni_size_weights.indirect_call_cost
1005 21600 : - eni_size_weights.call_cost);
1006 21600 : info->call_stmt_time -= (eni_time_weights.indirect_call_cost
1007 21600 : - eni_time_weights.call_cost);
1008 : }
1009 3757387 : }
1010 :
1011 : /* Dump edge summaries associated to NODE and recursively to all clones.
1012 : Indent by INDENT. */
1013 :
1014 : static void
1015 1984 : dump_ipa_call_summary (FILE *f, int indent, struct cgraph_node *node,
1016 : class ipa_fn_summary *info)
1017 : {
1018 1984 : struct cgraph_edge *edge;
1019 9822 : for (edge = node->callees; edge; edge = edge->next_callee)
1020 : {
1021 7838 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
1022 7838 : struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
1023 7838 : int i;
1024 :
1025 15676 : fprintf (f,
1026 : "%*s%s %s\n%*s freq:%4.2f",
1027 : indent, "", callee->dump_name (),
1028 7838 : !edge->inline_failed
1029 7284 : ? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
1030 7838 : indent, "", edge->sreal_frequency ().to_double ());
1031 :
1032 7838 : if (cross_module_call_p (edge))
1033 4 : fprintf (f, " cross module");
1034 :
1035 7838 : if (es)
1036 7284 : fprintf (f, " loop depth:%2i size:%2i time: %2i",
1037 : es->loop_depth, es->call_stmt_size, es->call_stmt_time);
1038 :
1039 7838 : ipa_fn_summary *s = ipa_fn_summaries->get (callee);
1040 7838 : ipa_size_summary *ss = ipa_size_summaries->get (callee);
1041 7838 : if (s != NULL)
1042 623 : fprintf (f, " callee size:%2i stack:%2i",
1043 623 : (int) (ss->size / ipa_fn_summary::size_scale),
1044 623 : (int) s->estimated_stack_size);
1045 :
1046 7838 : if (es && es->predicate)
1047 : {
1048 4627 : fprintf (f, " predicate: ");
1049 4627 : es->predicate->dump (f, info->conds);
1050 : }
1051 : else
1052 3211 : fprintf (f, "\n");
1053 7838 : if (es && es->param.exists ())
1054 5390 : for (i = 0; i < (int) es->param.length (); i++)
1055 : {
1056 1590 : int prob = es->param[i].change_prob;
1057 :
1058 1590 : if (!prob)
1059 606 : fprintf (f, "%*s op%i is compile time invariant\n",
1060 : indent + 2, "", i);
1061 984 : else if (prob != REG_BR_PROB_BASE)
1062 35 : fprintf (f, "%*s op%i change %f%% of time\n", indent + 2, "", i,
1063 35 : prob * 100.0 / REG_BR_PROB_BASE);
1064 1590 : if (es->param[i].points_to_local_or_readonly_memory)
1065 406 : fprintf (f, "%*s op%i points to local or readonly memory\n",
1066 : indent + 2, "", i);
1067 1590 : if (es->param[i].points_to_possible_sra_candidate)
1068 225 : fprintf (f, "%*s op%i points to possible sra candidate\n",
1069 : indent + 2, "", i);
1070 : }
1071 7838 : if (!edge->inline_failed)
1072 : {
1073 554 : ipa_size_summary *ss = ipa_size_summaries->get (callee);
1074 554 : fprintf (f, "%*sStack frame offset %i, callee self size %i\n",
1075 : indent + 2, "",
1076 554 : (int) ipa_get_stack_frame_offset (callee),
1077 554 : (int) ss->estimated_self_stack_size);
1078 554 : dump_ipa_call_summary (f, indent + 2, callee, info);
1079 : }
1080 : }
1081 2282 : for (edge = node->indirect_calls; edge; edge = edge->next_callee)
1082 : {
1083 298 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
1084 298 : fprintf (f, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
1085 : " time: %2i",
1086 : indent, "",
1087 : es->loop_depth,
1088 298 : edge->sreal_frequency ().to_double (), es->call_stmt_size,
1089 : es->call_stmt_time);
1090 298 : if (es->predicate)
1091 : {
1092 6 : fprintf (f, "predicate: ");
1093 6 : es->predicate->dump (f, info->conds);
1094 : }
1095 : else
1096 292 : fprintf (f, "\n");
1097 : }
1098 1984 : }
1099 :
1100 :
1101 : void
1102 1633 : ipa_dump_fn_summary (FILE *f, struct cgraph_node *node)
1103 : {
1104 1633 : if (node->definition)
1105 : {
1106 1633 : class ipa_fn_summary *s = ipa_fn_summaries->get (node);
1107 1633 : class ipa_size_summary *ss = ipa_size_summaries->get (node);
1108 1633 : if (s != NULL)
1109 : {
1110 1430 : size_time_entry *e;
1111 1430 : int i;
1112 1430 : fprintf (f, "IPA function summary for %s", node->dump_name ());
1113 1430 : if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
1114 0 : fprintf (f, " always_inline");
1115 1430 : if (s->inlinable)
1116 1243 : fprintf (f, " inlinable");
1117 1430 : if (s->fp_expressions)
1118 45 : fprintf (f, " fp_expression");
1119 1430 : if (s->builtin_constant_p_parms.length ())
1120 : {
1121 2 : fprintf (f, " builtin_constant_p_parms");
1122 4 : for (unsigned int i = 0;
1123 4 : i < s->builtin_constant_p_parms.length (); i++)
1124 2 : fprintf (f, " %i", s->builtin_constant_p_parms[i]);
1125 : }
1126 1430 : fprintf (f, "\n global time: %f\n", s->time.to_double ());
1127 1430 : fprintf (f, " self size: %i\n", ss->self_size);
1128 1430 : fprintf (f, " global size: %i\n", ss->size);
1129 1430 : fprintf (f, " min size: %i\n", s->min_size);
1130 1430 : fprintf (f, " self stack: %i\n",
1131 1430 : (int) ss->estimated_self_stack_size);
1132 1430 : fprintf (f, " global stack: %i\n", (int) s->estimated_stack_size);
1133 1430 : if (s->growth)
1134 161 : fprintf (f, " estimated growth:%i\n", (int) s->growth);
1135 1430 : if (s->scc_no)
1136 5 : fprintf (f, " In SCC: %i\n", (int) s->scc_no);
1137 5972 : for (i = 0; s->size_time_table.iterate (i, &e); i++)
1138 : {
1139 4542 : fprintf (f, " size:%f, time:%f",
1140 4542 : (double) e->size / ipa_fn_summary::size_scale,
1141 : e->time.to_double ());
1142 4542 : if (e->exec_predicate != true)
1143 : {
1144 2655 : fprintf (f, ", executed if:");
1145 2655 : e->exec_predicate.dump (f, s->conds, 0);
1146 : }
1147 4542 : if (e->exec_predicate != e->nonconst_predicate)
1148 : {
1149 1187 : fprintf (f, ", nonconst if:");
1150 1187 : e->nonconst_predicate.dump (f, s->conds, 0);
1151 : }
1152 4542 : fprintf (f, "\n");
1153 : }
1154 : ipa_freqcounting_predicate *fcp;
1155 : bool first_fcp = true;
1156 1542 : for (int i = 0; vec_safe_iterate (s->loop_iterations, i, &fcp); i++)
1157 : {
1158 112 : if (first_fcp)
1159 : {
1160 79 : fprintf (f, " loop iterations:");
1161 79 : first_fcp = false;
1162 : }
1163 112 : fprintf (f, " %3.2f for ", fcp->freq.to_double ());
1164 112 : fcp->predicate->dump (f, s->conds);
1165 : }
1166 : first_fcp = true;
1167 1450 : for (int i = 0; vec_safe_iterate (s->loop_strides, i, &fcp); i++)
1168 : {
1169 20 : if (first_fcp)
1170 : {
1171 11 : fprintf (f, " loop strides:");
1172 11 : first_fcp = false;
1173 : }
1174 20 : fprintf (f, " %3.2f for :", fcp->freq.to_double ());
1175 20 : fcp->predicate->dump (f, s->conds);
1176 : }
1177 1430 : fprintf (f, " calls:\n");
1178 1430 : dump_ipa_call_summary (f, 4, node, s);
1179 1430 : fprintf (f, "\n");
1180 1430 : if (s->target_info)
1181 0 : fprintf (f, " target_info: %x\n", s->target_info);
1182 : }
1183 : else
1184 203 : fprintf (f, "IPA summary for %s is missing.\n", node->dump_name ());
1185 : }
1186 1633 : }
1187 :
1188 : DEBUG_FUNCTION void
1189 0 : ipa_debug_fn_summary (struct cgraph_node *node)
1190 : {
1191 0 : ipa_dump_fn_summary (stderr, node);
1192 0 : }
1193 :
1194 : void
1195 356 : ipa_dump_fn_summaries (FILE *f)
1196 : {
1197 356 : struct cgraph_node *node;
1198 :
1199 2296 : FOR_EACH_DEFINED_FUNCTION (node)
1200 1940 : if (!node->inlined_to)
1201 1386 : ipa_dump_fn_summary (f, node);
1202 356 : }
1203 :
1204 : /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1205 : boolean variable pointed to by DATA. */
1206 :
1207 : static bool
1208 778939 : mark_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED,
1209 : void *data)
1210 : {
1211 778939 : bool *b = (bool *) data;
1212 778939 : *b = true;
1213 778939 : return true;
1214 : }
1215 :
1216 : /* If OP refers to value of function parameter, return the corresponding
1217 : parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
1218 : PARM_DECL) will be stored to *SIZE_P in that case too. */
1219 :
1220 : static tree
1221 182656819 : unmodified_parm_1 (ipa_func_body_info *fbi, gimple *stmt, tree op,
1222 : poly_int64 *size_p)
1223 : {
1224 : /* SSA_NAME referring to parm default def? */
1225 182656819 : if (TREE_CODE (op) == SSA_NAME
1226 106831999 : && SSA_NAME_IS_DEFAULT_DEF (op)
1227 207737223 : && TREE_CODE (SSA_NAME_VAR (op)) == PARM_DECL)
1228 : {
1229 24881306 : if (size_p)
1230 0 : *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
1231 24881306 : return SSA_NAME_VAR (op);
1232 : }
1233 : /* Non-SSA parm reference? */
1234 157775513 : if (TREE_CODE (op) == PARM_DECL
1235 1438220 : && fbi->aa_walk_budget > 0)
1236 : {
1237 1435674 : bool modified = false;
1238 :
1239 1435674 : ao_ref refd;
1240 1435674 : ao_ref_init (&refd, op);
1241 2871348 : int walked = walk_aliased_vdefs (&refd, gimple_vuse (stmt),
1242 : mark_modified, &modified, NULL, NULL,
1243 : fbi->aa_walk_budget);
1244 1435674 : if (walked < 0)
1245 : {
1246 8 : fbi->aa_walk_budget = 0;
1247 926713 : return NULL_TREE;
1248 : }
1249 1435666 : fbi->aa_walk_budget -= walked;
1250 1435666 : if (!modified)
1251 : {
1252 926705 : if (size_p)
1253 0 : *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
1254 926705 : return op;
1255 : }
1256 : }
1257 : return NULL_TREE;
1258 : }
1259 :
1260 : /* If OP refers to value of function parameter, return the corresponding
1261 : parameter. Also traverse chains of SSA register assignments. If non-NULL,
1262 : the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
1263 : stored to *SIZE_P in that case too. */
1264 :
1265 : static tree
1266 119263040 : unmodified_parm (ipa_func_body_info *fbi, gimple *stmt, tree op,
1267 : poly_int64 *size_p)
1268 : {
1269 148484126 : tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
1270 148484126 : if (res)
1271 : return res;
1272 :
1273 123479197 : if (TREE_CODE (op) == SSA_NAME
1274 70104236 : && !SSA_NAME_IS_DEFAULT_DEF (op)
1275 193388898 : && gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1276 29221086 : return unmodified_parm (fbi, SSA_NAME_DEF_STMT (op),
1277 29221086 : gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op)),
1278 29221086 : size_p);
1279 : return NULL_TREE;
1280 : }
1281 :
1282 : /* If OP refers to a value of a function parameter or value loaded from an
1283 : aggregate passed to a parameter (either by value or reference), return TRUE
1284 : and store the number of the parameter to *INDEX_P, the access size into
1285 : *SIZE_P, and information whether and how it has been loaded from an
1286 : aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1287 : statement in which OP is used or loaded. */
1288 :
1289 : static bool
1290 32729835 : unmodified_parm_or_parm_agg_item (struct ipa_func_body_info *fbi,
1291 : gimple *stmt, tree op, int *index_p,
1292 : poly_int64 *size_p,
1293 : struct agg_position_info *aggpos)
1294 : {
1295 34172693 : tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
1296 :
1297 34172693 : gcc_checking_assert (aggpos);
1298 34172693 : if (res)
1299 : {
1300 803082 : *index_p = ipa_get_param_decl_index (fbi->info, res);
1301 803082 : if (*index_p < 0)
1302 : return false;
1303 802952 : aggpos->agg_contents = false;
1304 802952 : aggpos->by_ref = false;
1305 802952 : return true;
1306 : }
1307 :
1308 33369611 : if (TREE_CODE (op) == SSA_NAME)
1309 : {
1310 11846457 : if (SSA_NAME_IS_DEFAULT_DEF (op)
1311 11846457 : || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1312 : return false;
1313 4782884 : stmt = SSA_NAME_DEF_STMT (op);
1314 4782884 : op = gimple_assign_rhs1 (stmt);
1315 4782884 : if (!REFERENCE_CLASS_P (op))
1316 : return unmodified_parm_or_parm_agg_item (fbi, stmt, op, index_p, size_p,
1317 : aggpos);
1318 : }
1319 :
1320 24863180 : aggpos->agg_contents = true;
1321 24863180 : return ipa_load_from_parm_agg (fbi, fbi->info->descriptors,
1322 : stmt, op, index_p, &aggpos->offset,
1323 24863180 : size_p, &aggpos->by_ref);
1324 : }
1325 :
1326 : /* If stmt is simple load or store of value pointed to by a function parmaeter,
1327 : return its index. */
1328 :
1329 : static int
1330 110627062 : load_or_store_of_ptr_parameter (ipa_func_body_info *fbi, gimple *stmt)
1331 : {
1332 110627062 : if (!optimize)
1333 : return -1;
1334 158630531 : gassign *assign = dyn_cast <gassign *> (stmt);
1335 55140116 : if (!assign)
1336 : return -1;
1337 55140116 : tree param;
1338 55140116 : if (gimple_assign_load_p (stmt))
1339 20257247 : param = gimple_assign_rhs1 (stmt);
1340 34882869 : else if (gimple_store_p (stmt))
1341 18513108 : param = gimple_assign_lhs (stmt);
1342 : else
1343 : return -1;
1344 38770355 : tree base = get_base_address (param);
1345 38770355 : if (TREE_CODE (base) != MEM_REF
1346 13408792 : || TREE_CODE (TREE_OPERAND (base, 0)) != SSA_NAME
1347 52175043 : || !SSA_NAME_IS_DEFAULT_DEF (TREE_OPERAND (base, 0)))
1348 : return -1;
1349 7220926 : tree p = SSA_NAME_VAR (TREE_OPERAND (base, 0));
1350 7220926 : if (TREE_CODE (p) != PARM_DECL)
1351 : return -1;
1352 7136647 : return ipa_get_param_decl_index (fbi->info, p);
1353 : }
1354 :
1355 : /* See if statement might disappear after inlining.
1356 : 0 - means not eliminated
1357 : 1 - half of statements goes away
1358 : 2 - for sure it is eliminated.
1359 : We are not terribly sophisticated, basically looking for simple abstraction
1360 : penalty wrappers. */
1361 :
1362 : static int
1363 110627062 : eliminated_by_inlining_prob (ipa_func_body_info *fbi, gimple *stmt)
1364 : {
1365 110627062 : enum gimple_code code = gimple_code (stmt);
1366 110627062 : enum tree_code rhs_code;
1367 :
1368 110627062 : if (!optimize)
1369 : return 0;
1370 :
1371 92535668 : switch (code)
1372 : {
1373 : case GIMPLE_RETURN:
1374 : return 2;
1375 55140116 : case GIMPLE_ASSIGN:
1376 55140116 : if (gimple_num_ops (stmt) != 2)
1377 : return 0;
1378 :
1379 39572683 : rhs_code = gimple_assign_rhs_code (stmt);
1380 :
1381 : /* Casts of parameters, loads from parameters passed by reference
1382 : and stores to return value or parameters are often free after
1383 : inlining due to SRA and further combining.
1384 : Assume that half of statements goes away. */
1385 39572683 : if (CONVERT_EXPR_CODE_P (rhs_code)
1386 : || rhs_code == VIEW_CONVERT_EXPR
1387 : || rhs_code == ADDR_EXPR
1388 36795455 : || gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS)
1389 : {
1390 38770355 : tree rhs = gimple_assign_rhs1 (stmt);
1391 38770355 : tree lhs = gimple_assign_lhs (stmt);
1392 38770355 : tree inner_rhs = get_base_address (rhs);
1393 38770355 : tree inner_lhs = get_base_address (lhs);
1394 38770355 : bool rhs_free = false;
1395 38770355 : bool lhs_free = false;
1396 :
1397 38770355 : if (!inner_rhs)
1398 0 : inner_rhs = rhs;
1399 38770355 : if (!inner_lhs)
1400 0 : inner_lhs = lhs;
1401 :
1402 : /* Reads of parameter are expected to be free. */
1403 38770355 : if (unmodified_parm (fbi, stmt, inner_rhs, NULL))
1404 : rhs_free = true;
1405 : /* Match expressions of form &this->field. Those will most likely
1406 : combine with something upstream after inlining. */
1407 37512383 : else if (TREE_CODE (inner_rhs) == ADDR_EXPR)
1408 : {
1409 2478768 : tree op = get_base_address (TREE_OPERAND (inner_rhs, 0));
1410 2478768 : if (TREE_CODE (op) == PARM_DECL)
1411 : rhs_free = true;
1412 2440786 : else if (TREE_CODE (op) == MEM_REF
1413 2440786 : && unmodified_parm (fbi, stmt, TREE_OPERAND (op, 0),
1414 : NULL))
1415 : rhs_free = true;
1416 : }
1417 :
1418 : /* When parameter is not SSA register because its address is taken
1419 : and it is just copied into one, the statement will be completely
1420 : free after inlining (we will copy propagate backward). */
1421 1295954 : if (rhs_free && is_gimple_reg (lhs))
1422 : return 2;
1423 :
1424 : /* Reads of parameters passed by reference
1425 : expected to be free (i.e. optimized out after inlining). */
1426 38097246 : if (TREE_CODE (inner_rhs) == MEM_REF
1427 38097246 : && unmodified_parm (fbi, stmt, TREE_OPERAND (inner_rhs, 0), NULL))
1428 : rhs_free = true;
1429 :
1430 : /* Copying parameter passed by reference into gimple register is
1431 : probably also going to copy propagate, but we can't be quite
1432 : sure. */
1433 38097246 : if (rhs_free && is_gimple_reg (lhs))
1434 : lhs_free = true;
1435 :
1436 : /* Writes to parameters, parameters passed by value and return value
1437 : (either directly or passed via invisible reference) are free.
1438 :
1439 : TODO: We ought to handle testcase like
1440 : struct a {int a,b;};
1441 : struct a
1442 : returnstruct (void)
1443 : {
1444 : struct a a ={1,2};
1445 : return a;
1446 : }
1447 :
1448 : This translate into:
1449 :
1450 : returnstruct ()
1451 : {
1452 : int a$b;
1453 : int a$a;
1454 : struct a a;
1455 : struct a D.2739;
1456 :
1457 : <bb 2>:
1458 : D.2739.a = 1;
1459 : D.2739.b = 2;
1460 : return D.2739;
1461 :
1462 : }
1463 : For that we either need to copy ipa-split logic detecting writes
1464 : to return value. */
1465 38097246 : if (TREE_CODE (inner_lhs) == PARM_DECL
1466 38010096 : || TREE_CODE (inner_lhs) == RESULT_DECL
1467 75270355 : || (TREE_CODE (inner_lhs) == MEM_REF
1468 5012898 : && (unmodified_parm (fbi, stmt, TREE_OPERAND (inner_lhs, 0),
1469 : NULL)
1470 2586342 : || (TREE_CODE (TREE_OPERAND (inner_lhs, 0)) == SSA_NAME
1471 2585153 : && SSA_NAME_VAR (TREE_OPERAND (inner_lhs, 0))
1472 356005 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1473 : (inner_lhs,
1474 : 0))) == RESULT_DECL))))
1475 : lhs_free = true;
1476 34665531 : if (lhs_free
1477 38097246 : && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs)))
1478 : rhs_free = true;
1479 38097246 : if (lhs_free && rhs_free)
1480 : return 1;
1481 : }
1482 : return 0;
1483 : default:
1484 : return 0;
1485 : }
1486 : }
1487 :
1488 : /* Analyze EXPR if it represents a series of simple operations performed on
1489 : a function parameter and return true if so. FBI, STMT, EXPR, INDEX_P and
1490 : AGGPOS have the same meaning like in unmodified_parm_or_parm_agg_item.
1491 : Type of the parameter or load from an aggregate via the parameter is
1492 : stored in *TYPE_P. Operations on the parameter are recorded to
1493 : PARAM_OPS_P if it is not NULL. */
1494 :
1495 : static bool
1496 26961500 : decompose_param_expr (struct ipa_func_body_info *fbi,
1497 : gimple *stmt, tree expr,
1498 : int *index_p, tree *type_p,
1499 : struct agg_position_info *aggpos,
1500 : expr_eval_ops *param_ops_p = NULL)
1501 : {
1502 26961500 : int op_limit = opt_for_fn (fbi->node->decl, param_ipa_max_param_expr_ops);
1503 26961500 : int op_count = 0;
1504 :
1505 26961500 : if (param_ops_p)
1506 9219494 : *param_ops_p = NULL;
1507 :
1508 32729835 : while (true)
1509 : {
1510 32729835 : expr_eval_op eval_op;
1511 32729835 : unsigned rhs_count;
1512 32729835 : unsigned cst_count = 0;
1513 :
1514 32729835 : if (unmodified_parm_or_parm_agg_item (fbi, stmt, expr, index_p, NULL,
1515 : aggpos))
1516 : {
1517 4537148 : tree type = TREE_TYPE (expr);
1518 :
1519 4537148 : if (aggpos->agg_contents)
1520 : {
1521 : /* Stop if containing bit-field. */
1522 3734196 : if (TREE_CODE (expr) == BIT_FIELD_REF
1523 3734196 : || contains_bitfld_component_ref_p (expr))
1524 : break;
1525 : }
1526 :
1527 4514368 : *type_p = type;
1528 4514368 : return true;
1529 : }
1530 :
1531 28192687 : if (TREE_CODE (expr) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (expr))
1532 : break;
1533 10521435 : stmt = SSA_NAME_DEF_STMT (expr);
1534 :
1535 10521435 : if (gcall *call = dyn_cast <gcall *> (stmt))
1536 : {
1537 2480887 : int flags = gimple_call_return_flags (call);
1538 2480887 : if (!(flags & ERF_RETURNS_ARG))
1539 3073983 : goto fail;
1540 207898 : int arg = flags & ERF_RETURN_ARG_MASK;
1541 207898 : if (arg >= (int)gimple_call_num_args (call))
1542 0 : goto fail;
1543 207898 : expr = gimple_call_arg (stmt, arg);
1544 4006432 : continue;
1545 207898 : }
1546 :
1547 8040548 : if (!is_gimple_assign (stmt = SSA_NAME_DEF_STMT (expr)))
1548 : break;
1549 :
1550 6361506 : switch (gimple_assign_rhs_class (stmt))
1551 : {
1552 3798534 : case GIMPLE_SINGLE_RHS:
1553 3798534 : expr = gimple_assign_rhs1 (stmt);
1554 3798534 : continue;
1555 :
1556 : case GIMPLE_UNARY_RHS:
1557 : rhs_count = 1;
1558 : break;
1559 :
1560 1543786 : case GIMPLE_BINARY_RHS:
1561 1543786 : rhs_count = 2;
1562 1543786 : break;
1563 :
1564 642 : case GIMPLE_TERNARY_RHS:
1565 642 : rhs_count = 3;
1566 642 : break;
1567 :
1568 0 : default:
1569 0 : goto fail;
1570 : }
1571 :
1572 : /* Stop if expression is too complex. */
1573 2562972 : if (op_count++ == op_limit)
1574 : break;
1575 :
1576 2562897 : if (param_ops_p)
1577 : {
1578 2546245 : eval_op.code = gimple_assign_rhs_code (stmt);
1579 2546245 : eval_op.type = TREE_TYPE (gimple_assign_lhs (stmt));
1580 2546245 : eval_op.val[0] = NULL_TREE;
1581 2546245 : eval_op.val[1] = NULL_TREE;
1582 : }
1583 :
1584 : expr = NULL_TREE;
1585 5869242 : for (unsigned i = 0; i < rhs_count; i++)
1586 : {
1587 4107339 : tree op = gimple_op (stmt, i + 1);
1588 :
1589 4107339 : gcc_assert (op && !TYPE_P (op));
1590 4107339 : if (is_gimple_ip_invariant (op))
1591 : {
1592 745730 : if (++cst_count == rhs_count)
1593 1141 : goto fail;
1594 :
1595 744589 : eval_op.val[cst_count - 1] = op;
1596 : }
1597 3361609 : else if (!expr)
1598 : {
1599 : /* Found a non-constant operand, and record its index in rhs
1600 : operands. */
1601 2561756 : eval_op.index = i;
1602 2561756 : expr = op;
1603 : }
1604 : else
1605 : {
1606 : /* Found more than one non-constant operands. */
1607 799853 : goto fail;
1608 : }
1609 : }
1610 :
1611 1761903 : if (param_ops_p)
1612 1745696 : vec_safe_insert (*param_ops_p, 0, eval_op);
1613 : }
1614 :
1615 : /* Failed to decompose, free resource and return. */
1616 22447132 : fail:
1617 22447132 : if (param_ops_p)
1618 9041425 : vec_free (*param_ops_p);
1619 :
1620 : return false;
1621 : }
1622 :
1623 : /* Record to SUMMARY that PARM is used by builtin_constant_p. */
1624 :
1625 : static void
1626 10700 : add_builtin_constant_p_parm (class ipa_fn_summary *summary, int parm)
1627 : {
1628 10700 : int ip;
1629 :
1630 : /* Avoid duplicates. */
1631 10738 : for (unsigned int i = 0;
1632 10738 : summary->builtin_constant_p_parms.iterate (i, &ip); i++)
1633 5160 : if (ip == parm)
1634 10700 : return;
1635 5578 : summary->builtin_constant_p_parms.safe_push (parm);
1636 : }
1637 :
1638 : /* If BB ends by a conditional we can turn into predicates, attach corresponding
1639 : predicates to the CFG edges. */
1640 :
1641 : static void
1642 34781113 : set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
1643 : class ipa_fn_summary *summary,
1644 : class ipa_node_params *params_summary,
1645 : basic_block bb)
1646 : {
1647 34781113 : tree op, op2;
1648 34781113 : int index;
1649 34781113 : struct agg_position_info aggpos;
1650 34781113 : enum tree_code code, inverted_code;
1651 34781113 : edge e;
1652 34781113 : edge_iterator ei;
1653 34781113 : gimple *set_stmt;
1654 34781113 : tree param_type;
1655 34781113 : expr_eval_ops param_ops;
1656 :
1657 69562226 : gcond *last = safe_dyn_cast <gcond *> (*gsi_last_bb (bb));
1658 11548850 : if (!last)
1659 34769958 : return;
1660 11548850 : if (!is_gimple_ip_invariant (gimple_cond_rhs (last)))
1661 : return;
1662 9141215 : op = gimple_cond_lhs (last);
1663 :
1664 9141215 : if (decompose_param_expr (fbi, last, op, &index, ¶m_type, &aggpos,
1665 : ¶m_ops))
1666 : {
1667 1283970 : code = gimple_cond_code (last);
1668 1283970 : inverted_code = invert_tree_comparison (code, HONOR_NANS (op));
1669 :
1670 3851910 : FOR_EACH_EDGE (e, ei, bb->succs)
1671 : {
1672 1283970 : enum tree_code this_code = (e->flags & EDGE_TRUE_VALUE
1673 2567940 : ? code : inverted_code);
1674 : /* invert_tree_comparison will return ERROR_MARK on FP
1675 : comparisons that are not EQ/NE instead of returning proper
1676 : unordered one. Be sure it is not confused with NON_CONSTANT.
1677 :
1678 : And if the edge's target is the final block of diamond CFG graph
1679 : of this conditional statement, we do not need to compute
1680 : predicate for the edge because the final block's predicate must
1681 : be at least as that of the first block of the statement. */
1682 2567940 : if (this_code != ERROR_MARK
1683 2567940 : && !dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1684 : {
1685 2164247 : ipa_predicate p
1686 2164247 : = add_condition (summary, params_summary, index,
1687 : param_type, &aggpos,
1688 : this_code, gimple_cond_rhs (last), param_ops);
1689 2164247 : e->aux = edge_predicate_pool.allocate ();
1690 2164247 : *(ipa_predicate *) e->aux = p;
1691 : }
1692 : }
1693 1283970 : vec_free (param_ops);
1694 1283970 : return;
1695 : }
1696 :
1697 7857245 : if (TREE_CODE (op) != SSA_NAME)
1698 : return;
1699 : /* Special case
1700 : if (builtin_constant_p (op))
1701 : constant_code
1702 : else
1703 : nonconstant_code.
1704 : Here we can predicate nonconstant_code. We can't
1705 : really handle constant_code since we have no predicate
1706 : for this and also the constant code is not known to be
1707 : optimized away when inliner doesn't see operand is constant.
1708 : Other optimizers might think otherwise. */
1709 7856162 : if (gimple_cond_code (last) != NE_EXPR
1710 7856162 : || !integer_zerop (gimple_cond_rhs (last)))
1711 4451275 : return;
1712 3404887 : set_stmt = SSA_NAME_DEF_STMT (op);
1713 3404887 : if (!gimple_call_builtin_p (set_stmt, BUILT_IN_CONSTANT_P)
1714 3404887 : || gimple_call_num_args (set_stmt) != 1)
1715 : return;
1716 27664 : op2 = gimple_call_arg (set_stmt, 0);
1717 27664 : if (!decompose_param_expr (fbi, set_stmt, op2, &index, ¶m_type, &aggpos))
1718 : return;
1719 11155 : if (!aggpos.by_ref)
1720 10664 : add_builtin_constant_p_parm (summary, index);
1721 33465 : FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
1722 : {
1723 11155 : ipa_predicate p = add_condition (summary, params_summary, index,
1724 : param_type, &aggpos,
1725 : ipa_predicate::is_not_constant, NULL_TREE);
1726 11155 : e->aux = edge_predicate_pool.allocate ();
1727 11155 : *(ipa_predicate *) e->aux = p;
1728 : }
1729 : }
1730 :
1731 :
1732 : /* If BB ends by a switch we can turn into predicates, attach corresponding
1733 : predicates to the CFG edges. */
1734 :
1735 : static void
1736 34781113 : set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
1737 : class ipa_fn_summary *summary,
1738 : class ipa_node_params *params_summary,
1739 : basic_block bb)
1740 : {
1741 34781113 : tree op;
1742 34781113 : int index;
1743 34781113 : struct agg_position_info aggpos;
1744 34781113 : edge e;
1745 34781113 : edge_iterator ei;
1746 34781113 : size_t n;
1747 34781113 : size_t case_idx;
1748 34781113 : tree param_type;
1749 34781113 : expr_eval_ops param_ops;
1750 :
1751 69562226 : gswitch *last = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb));
1752 78279 : if (!last)
1753 34752113 : return;
1754 78279 : op = gimple_switch_index (last);
1755 78279 : if (!decompose_param_expr (fbi, last, op, &index, ¶m_type, &aggpos,
1756 : ¶m_ops))
1757 : return;
1758 :
1759 43971 : auto_vec<std::pair<tree, tree> > ranges;
1760 43971 : tree type = TREE_TYPE (op);
1761 43971 : int bound_limit = opt_for_fn (fbi->node->decl,
1762 : param_ipa_max_switch_predicate_bounds);
1763 43971 : int bound_count = 0;
1764 : // This can safely be an integer range, as switches can only hold
1765 : // integers.
1766 43971 : int_range<2> vr;
1767 :
1768 87942 : get_range_query (cfun)->range_of_expr (vr, op);
1769 43971 : if (vr.undefined_p ())
1770 0 : vr.set_varying (TREE_TYPE (op));
1771 43971 : tree vr_min, vr_max;
1772 : // TODO: This entire function could use a rewrite to use the irange
1773 : // API, instead of trying to recreate its intersection/union logic.
1774 : // Any use of get_legacy_range() is a serious code smell.
1775 43971 : value_range_kind vr_type = get_legacy_range (vr, vr_min, vr_max);
1776 43971 : wide_int vr_wmin = wi::to_wide (vr_min);
1777 43971 : wide_int vr_wmax = wi::to_wide (vr_max);
1778 :
1779 303816 : FOR_EACH_EDGE (e, ei, bb->succs)
1780 : {
1781 259845 : e->aux = edge_predicate_pool.allocate ();
1782 259845 : *(ipa_predicate *) e->aux = false;
1783 : }
1784 :
1785 43971 : e = gimple_switch_edge (cfun, last, 0);
1786 : /* Set BOUND_COUNT to maximum count to bypass computing predicate for
1787 : default case if its target basic block is in convergence point of all
1788 : switch cases, which can be determined by checking whether it
1789 : post-dominates the switch statement. */
1790 43971 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1791 12599 : bound_count = INT_MAX;
1792 :
1793 43971 : n = gimple_switch_num_labels (last);
1794 290286 : for (case_idx = 1; case_idx < n; ++case_idx)
1795 : {
1796 246315 : tree cl = gimple_switch_label (last, case_idx);
1797 246315 : tree min = CASE_LOW (cl);
1798 246315 : tree max = CASE_HIGH (cl);
1799 246315 : ipa_predicate p;
1800 :
1801 246315 : e = gimple_switch_edge (cfun, last, case_idx);
1802 :
1803 : /* The case value might not have same type as switch expression,
1804 : extend the value based on the expression type. */
1805 246315 : if (TREE_TYPE (min) != type)
1806 43891 : min = wide_int_to_tree (type, wi::to_wide (min));
1807 :
1808 246315 : if (!max)
1809 : max = min;
1810 11950 : else if (TREE_TYPE (max) != type)
1811 3297 : max = wide_int_to_tree (type, wi::to_wide (max));
1812 :
1813 : /* The case's target basic block is in convergence point of all switch
1814 : cases, its predicate should be at least as that of the switch
1815 : statement. */
1816 246315 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1817 4626 : p = true;
1818 241689 : else if (min == max)
1819 231275 : p = add_condition (summary, params_summary, index, param_type,
1820 : &aggpos, EQ_EXPR, min, param_ops);
1821 : else
1822 : {
1823 10414 : ipa_predicate p1, p2;
1824 10414 : p1 = add_condition (summary, params_summary, index, param_type,
1825 : &aggpos, GE_EXPR, min, param_ops);
1826 10414 : p2 = add_condition (summary, params_summary,index, param_type,
1827 : &aggpos, LE_EXPR, max, param_ops);
1828 10414 : p = p1 & p2;
1829 : }
1830 246315 : *(ipa_predicate *) e->aux
1831 246315 : = p.or_with (summary->conds, *(ipa_predicate *) e->aux);
1832 :
1833 : /* If there are too many disjoint case ranges, predicate for default
1834 : case might become too complicated. So add a limit here. */
1835 246315 : if (bound_count > bound_limit)
1836 96236 : continue;
1837 :
1838 150079 : bool new_range = true;
1839 :
1840 150079 : if (!ranges.is_empty ())
1841 : {
1842 118707 : wide_int curr_wmin = wi::to_wide (min);
1843 118707 : wide_int last_wmax = wi::to_wide (ranges.last ().second);
1844 :
1845 : /* Merge case ranges if they are continuous. */
1846 118707 : if (curr_wmin == last_wmax + 1)
1847 : new_range = false;
1848 40680 : else if (vr_type == VR_ANTI_RANGE)
1849 : {
1850 : /* If two disjoint case ranges can be connected by anti-range
1851 : of switch index, combine them to one range. */
1852 79 : if (wi::lt_p (vr_wmax, curr_wmin - 1, TYPE_SIGN (type)))
1853 : vr_type = VR_UNDEFINED;
1854 0 : else if (wi::le_p (vr_wmin, last_wmax + 1, TYPE_SIGN (type)))
1855 0 : new_range = false;
1856 : }
1857 118707 : }
1858 :
1859 : /* Create/extend a case range. And we count endpoints of range set,
1860 : this number nearly equals to number of conditions that we will create
1861 : for predicate of default case. */
1862 118707 : if (new_range)
1863 : {
1864 72052 : bound_count += (min == max) ? 1 : 2;
1865 72052 : ranges.safe_push (std::make_pair (min, max));
1866 : }
1867 : else
1868 : {
1869 78027 : bound_count += (ranges.last ().first == ranges.last ().second);
1870 78027 : ranges.last ().second = max;
1871 : }
1872 : }
1873 :
1874 43971 : e = gimple_switch_edge (cfun, last, 0);
1875 43971 : if (bound_count > bound_limit)
1876 : {
1877 14971 : *(ipa_predicate *) e->aux = true;
1878 14971 : vec_free (param_ops);
1879 14971 : return;
1880 : }
1881 :
1882 29000 : ipa_predicate p_seg = true;
1883 29000 : ipa_predicate p_all = false;
1884 :
1885 29000 : if (vr_type != VR_RANGE)
1886 : {
1887 28479 : vr_wmin = wi::to_wide (TYPE_MIN_VALUE (type));
1888 28479 : vr_wmax = wi::to_wide (TYPE_MAX_VALUE (type));
1889 : }
1890 :
1891 : /* Construct predicate to represent default range set that is negation of
1892 : all case ranges. Case range is classified as containing single/non-single
1893 : values. Suppose a piece of case ranges in the following.
1894 :
1895 : [D1...D2] [S1] ... [Sn] [D3...D4]
1896 :
1897 : To represent default case's range sets between two non-single value
1898 : case ranges (From D2 to D3), we construct predicate as:
1899 :
1900 : D2 < x < D3 && x != S1 && ... && x != Sn
1901 : */
1902 89294 : for (size_t i = 0; i < ranges.length (); i++)
1903 : {
1904 60389 : tree min = ranges[i].first;
1905 60389 : tree max = ranges[i].second;
1906 :
1907 60389 : if (min == max)
1908 74756 : p_seg &= add_condition (summary, params_summary, index,
1909 : param_type, &aggpos, NE_EXPR,
1910 37378 : min, param_ops);
1911 : else
1912 : {
1913 : /* Do not create sub-predicate for range that is beyond low bound
1914 : of switch index. */
1915 23011 : if (wi::lt_p (vr_wmin, wi::to_wide (min), TYPE_SIGN (type)))
1916 : {
1917 16073 : p_seg &= add_condition (summary, params_summary, index,
1918 : param_type, &aggpos,
1919 16073 : LT_EXPR, min, param_ops);
1920 16073 : p_all = p_all.or_with (summary->conds, p_seg);
1921 : }
1922 :
1923 : /* Do not create sub-predicate for range that is beyond up bound
1924 : of switch index. */
1925 23011 : if (wi::le_p (vr_wmax, wi::to_wide (max), TYPE_SIGN (type)))
1926 : {
1927 95 : p_seg = false;
1928 95 : break;
1929 : }
1930 :
1931 22916 : p_seg = add_condition (summary, params_summary, index,
1932 : param_type, &aggpos, GT_EXPR,
1933 : max, param_ops);
1934 : }
1935 : }
1936 :
1937 29000 : p_all = p_all.or_with (summary->conds, p_seg);
1938 29000 : *(ipa_predicate *) e->aux
1939 29000 : = p_all.or_with (summary->conds, *(ipa_predicate *) e->aux);
1940 :
1941 35400 : vec_free (param_ops);
1942 43971 : }
1943 :
1944 :
1945 : /* For each BB in NODE attach to its AUX pointer predicate under
1946 : which it is executable. */
1947 :
1948 : static void
1949 6241511 : compute_bb_predicates (struct ipa_func_body_info *fbi,
1950 : struct cgraph_node *node,
1951 : class ipa_fn_summary *summary,
1952 : class ipa_node_params *params_summary)
1953 : {
1954 6241511 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
1955 6241511 : bool done = false;
1956 6241511 : basic_block bb;
1957 :
1958 41022624 : FOR_EACH_BB_FN (bb, my_function)
1959 : {
1960 34781113 : set_cond_stmt_execution_predicate (fbi, summary, params_summary, bb);
1961 34781113 : set_switch_stmt_execution_predicate (fbi, summary, params_summary, bb);
1962 : }
1963 :
1964 : /* Entry block is always executable. */
1965 6241511 : ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
1966 6241511 : = edge_predicate_pool.allocate ();
1967 6241511 : *(ipa_predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux = true;
1968 :
1969 : /* A simple dataflow propagation of predicates forward in the CFG.
1970 : TODO: work in reverse postorder. */
1971 18843243 : while (!done)
1972 : {
1973 12601732 : done = true;
1974 87249835 : FOR_EACH_BB_FN (bb, my_function)
1975 : {
1976 74648103 : ipa_predicate p = false;
1977 74648103 : edge e;
1978 74648103 : edge_iterator ei;
1979 92444807 : FOR_EACH_EDGE (e, ei, bb->preds)
1980 : {
1981 79299759 : if (e->src->aux)
1982 : {
1983 77559688 : ipa_predicate this_bb_predicate
1984 : = *(ipa_predicate *) e->src->aux;
1985 77559688 : if (e->aux)
1986 5077019 : this_bb_predicate &= (*(ipa_predicate *) e->aux);
1987 77559688 : p = p.or_with (summary->conds, this_bb_predicate);
1988 77559688 : if (p == true)
1989 : break;
1990 : }
1991 : }
1992 74648103 : if (p != false)
1993 : {
1994 73512047 : basic_block pdom_bb;
1995 :
1996 73512047 : if (!bb->aux)
1997 : {
1998 27229537 : done = false;
1999 27229537 : bb->aux = edge_predicate_pool.allocate ();
2000 27229537 : *((ipa_predicate *) bb->aux) = p;
2001 : }
2002 46282510 : else if (p != *(ipa_predicate *) bb->aux)
2003 : {
2004 : /* This OR operation is needed to ensure monotonous data flow
2005 : in the case we hit the limit on number of clauses and the
2006 : and/or operations above give approximate answers. */
2007 126178 : p = p.or_with (summary->conds, *(ipa_predicate *)bb->aux);
2008 126178 : if (p != *(ipa_predicate *)bb->aux)
2009 : {
2010 103553 : done = false;
2011 103553 : *((ipa_predicate *)bb->aux) = p;
2012 : }
2013 : }
2014 :
2015 : /* For switch/if statement, we can OR-combine predicates of all
2016 : its cases/branches to get predicate for basic block in their
2017 : convergence point, but sometimes this will generate very
2018 : complicated predicate. Actually, we can get simplified
2019 : predicate in another way by using the fact that predicate
2020 : for a basic block must also hold true for its post dominators.
2021 : To be specific, basic block in convergence point of
2022 : conditional statement should include predicate of the
2023 : statement. */
2024 73512047 : pdom_bb = get_immediate_dominator (CDI_POST_DOMINATORS, bb);
2025 73512047 : if (pdom_bb == EXIT_BLOCK_PTR_FOR_FN (my_function) || !pdom_bb)
2026 : ;
2027 37899042 : else if (!pdom_bb->aux)
2028 : {
2029 7551562 : done = false;
2030 7551562 : pdom_bb->aux = edge_predicate_pool.allocate ();
2031 7551562 : *((ipa_predicate *)pdom_bb->aux) = p;
2032 : }
2033 30347480 : else if (p != *(ipa_predicate *)pdom_bb->aux)
2034 : {
2035 3305160 : p = p.or_with (summary->conds,
2036 : *(ipa_predicate *)pdom_bb->aux);
2037 3305160 : if (p != *(ipa_predicate *)pdom_bb->aux)
2038 : {
2039 140866 : done = false;
2040 140866 : *((ipa_predicate *)pdom_bb->aux) = p;
2041 : }
2042 : }
2043 : }
2044 : }
2045 : }
2046 6241511 : }
2047 :
2048 :
2049 : /* Return predicate specifying when the STMT might have result that is not
2050 : a compile time constant. */
2051 :
2052 : static ipa_predicate
2053 4236963 : will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
2054 : class ipa_fn_summary *summary,
2055 : class ipa_node_params *params_summary,
2056 : tree expr,
2057 : vec<ipa_predicate> nonconstant_names)
2058 : {
2059 4236963 : tree parm;
2060 4236963 : int index;
2061 :
2062 4496228 : while (UNARY_CLASS_P (expr))
2063 259265 : expr = TREE_OPERAND (expr, 0);
2064 :
2065 4236963 : parm = unmodified_parm (fbi, NULL, expr, NULL);
2066 4236963 : if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
2067 227286 : return add_condition (summary, params_summary, index, TREE_TYPE (parm), NULL,
2068 227286 : ipa_predicate::changed, NULL_TREE);
2069 4009677 : if (is_gimple_min_invariant (expr))
2070 62554 : return false;
2071 3947123 : if (TREE_CODE (expr) == SSA_NAME)
2072 3765612 : return nonconstant_names[SSA_NAME_VERSION (expr)];
2073 181511 : if (BINARY_CLASS_P (expr) || COMPARISON_CLASS_P (expr))
2074 : {
2075 180885 : ipa_predicate p1
2076 180885 : = will_be_nonconstant_expr_predicate (fbi, summary,
2077 : params_summary,
2078 180885 : TREE_OPERAND (expr, 0),
2079 : nonconstant_names);
2080 180885 : if (p1 == true)
2081 105420 : return p1;
2082 :
2083 75465 : ipa_predicate p2
2084 75465 : = will_be_nonconstant_expr_predicate (fbi, summary,
2085 : params_summary,
2086 75465 : TREE_OPERAND (expr, 1),
2087 : nonconstant_names);
2088 75465 : return p1.or_with (summary->conds, p2);
2089 : }
2090 626 : else if (TREE_CODE (expr) == COND_EXPR)
2091 : {
2092 184 : ipa_predicate p1
2093 184 : = will_be_nonconstant_expr_predicate (fbi, summary,
2094 : params_summary,
2095 184 : TREE_OPERAND (expr, 0),
2096 : nonconstant_names);
2097 184 : if (p1 == true)
2098 33 : return p1;
2099 :
2100 151 : ipa_predicate p2
2101 151 : = will_be_nonconstant_expr_predicate (fbi, summary,
2102 : params_summary,
2103 151 : TREE_OPERAND (expr, 1),
2104 : nonconstant_names);
2105 151 : if (p2 == true)
2106 151 : return p2;
2107 0 : p1 = p1.or_with (summary->conds, p2);
2108 0 : p2 = will_be_nonconstant_expr_predicate (fbi, summary,
2109 : params_summary,
2110 0 : TREE_OPERAND (expr, 2),
2111 : nonconstant_names);
2112 0 : return p2.or_with (summary->conds, p1);
2113 : }
2114 442 : else if (TREE_CODE (expr) == CALL_EXPR)
2115 442 : return true;
2116 : else
2117 : {
2118 0 : debug_tree (expr);
2119 0 : gcc_unreachable ();
2120 : }
2121 : }
2122 :
2123 :
2124 : /* Return predicate specifying when the STMT might have result that is not
2125 : a compile time constant. */
2126 :
2127 : static ipa_predicate
2128 112906028 : will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
2129 : class ipa_fn_summary *summary,
2130 : class ipa_node_params *params_summary,
2131 : gimple *stmt,
2132 : vec<ipa_predicate> nonconstant_names)
2133 : {
2134 112906028 : ipa_predicate p = true;
2135 112906028 : ssa_op_iter iter;
2136 112906028 : tree use;
2137 112906028 : tree param_type = NULL_TREE;
2138 112906028 : ipa_predicate op_non_const;
2139 112906028 : bool is_load;
2140 112906028 : int base_index;
2141 112906028 : struct agg_position_info aggpos;
2142 :
2143 : /* What statements might be optimized away
2144 : when their arguments are constant. */
2145 112906028 : if (gimple_code (stmt) != GIMPLE_ASSIGN
2146 38774005 : && gimple_code (stmt) != GIMPLE_COND
2147 27473917 : && gimple_code (stmt) != GIMPLE_SWITCH
2148 140301666 : && (gimple_code (stmt) != GIMPLE_CALL
2149 20562821 : || !(gimple_call_flags (stmt) & ECF_CONST)))
2150 25761687 : return p;
2151 :
2152 : /* Stores will stay anyway. */
2153 87144341 : if (gimple_store_p (stmt))
2154 25496487 : return p;
2155 :
2156 61647854 : is_load = gimple_assign_load_p (stmt);
2157 :
2158 : /* Loads can be optimized when the value is known. */
2159 61647854 : if (is_load)
2160 : {
2161 17714342 : tree op = gimple_assign_rhs1 (stmt);
2162 17714342 : if (!decompose_param_expr (fbi, stmt, op, &base_index, ¶m_type,
2163 : &aggpos))
2164 14539070 : return p;
2165 : }
2166 : else
2167 43933512 : base_index = -1;
2168 :
2169 : /* See if we understand all operands before we start
2170 : adding conditionals. */
2171 63663901 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2172 : {
2173 46785997 : tree parm = unmodified_parm (fbi, stmt, use, NULL);
2174 : /* For arguments we can build a condition. */
2175 46785997 : if (parm && ipa_get_param_decl_index (fbi->info, parm) >= 0)
2176 8298128 : continue;
2177 38487869 : if (TREE_CODE (use) != SSA_NAME)
2178 0 : return p;
2179 : /* If we know when operand is constant,
2180 : we still can say something useful. */
2181 38487869 : if (nonconstant_names[SSA_NAME_VERSION (use)] != true)
2182 8256989 : continue;
2183 30230880 : return p;
2184 : }
2185 :
2186 16877904 : if (is_load)
2187 3175191 : op_non_const =
2188 3175191 : add_condition (summary, params_summary,
2189 : base_index, param_type, &aggpos,
2190 : ipa_predicate::changed, NULL_TREE);
2191 : else
2192 13702713 : op_non_const = false;
2193 32408188 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2194 : {
2195 15530284 : tree parm = unmodified_parm (fbi, stmt, use, NULL);
2196 15530284 : int index;
2197 :
2198 15530284 : if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
2199 : {
2200 7809431 : if (index != base_index)
2201 5328944 : p = add_condition (summary, params_summary, index,
2202 5328944 : TREE_TYPE (parm), NULL,
2203 : ipa_predicate::changed, NULL_TREE);
2204 : else
2205 2480487 : continue;
2206 : }
2207 : else
2208 7720853 : p = nonconstant_names[SSA_NAME_VERSION (use)];
2209 13049797 : op_non_const = p.or_with (summary->conds, op_non_const);
2210 : }
2211 16877904 : if ((gimple_code (stmt) == GIMPLE_ASSIGN || gimple_code (stmt) == GIMPLE_CALL)
2212 14699858 : && gimple_op (stmt, 0)
2213 31294935 : && TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
2214 14417031 : nonconstant_names[SSA_NAME_VERSION (gimple_op (stmt, 0))]
2215 14417031 : = op_non_const;
2216 16877904 : return op_non_const;
2217 : }
2218 :
2219 : struct record_modified_bb_info
2220 : {
2221 : tree op;
2222 : bitmap bb_set;
2223 : gimple *stmt;
2224 : };
2225 :
2226 : /* Value is initialized in INIT_BB and used in USE_BB. We want to compute
2227 : probability how often it changes between USE_BB.
2228 : INIT_BB->count/USE_BB->count is an estimate, but if INIT_BB
2229 : is in different loop nest, we can do better.
2230 : This is all just estimate. In theory we look for minimal cut separating
2231 : INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
2232 : anyway. */
2233 :
2234 : static basic_block
2235 10311399 : get_minimal_bb (basic_block init_bb, basic_block use_bb)
2236 : {
2237 10311399 : class loop *l = find_common_loop (init_bb->loop_father, use_bb->loop_father);
2238 10311399 : if (l && l->header->count < init_bb->count)
2239 366397 : return l->header;
2240 : return init_bb;
2241 : }
2242 :
2243 : /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2244 : set except for info->stmt. */
2245 :
2246 : static bool
2247 4797080 : record_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef, void *data)
2248 : {
2249 4797080 : struct record_modified_bb_info *info =
2250 : (struct record_modified_bb_info *) data;
2251 4797080 : if (SSA_NAME_DEF_STMT (vdef) == info->stmt)
2252 : return false;
2253 4769152 : if (gimple_clobber_p (SSA_NAME_DEF_STMT (vdef)))
2254 : return false;
2255 4722915 : bitmap_set_bit (info->bb_set,
2256 4722915 : SSA_NAME_IS_DEFAULT_DEF (vdef)
2257 0 : ? ENTRY_BLOCK_PTR_FOR_FN (cfun)->index
2258 : : get_minimal_bb
2259 4722915 : (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2260 4722915 : gimple_bb (info->stmt))->index);
2261 4722915 : if (dump_file)
2262 : {
2263 0 : fprintf (dump_file, " Param ");
2264 0 : print_generic_expr (dump_file, info->op, TDF_SLIM);
2265 0 : fprintf (dump_file, " changed at bb %i, minimal: %i stmt: ",
2266 0 : gimple_bb (SSA_NAME_DEF_STMT (vdef))->index,
2267 : get_minimal_bb
2268 0 : (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2269 0 : gimple_bb (info->stmt))->index);
2270 0 : print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (vdef), 0);
2271 : }
2272 : return false;
2273 : }
2274 :
2275 : /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2276 : will change since last invocation of STMT.
2277 :
2278 : Value 0 is reserved for compile time invariants.
2279 : For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2280 : ought to be REG_BR_PROB_BASE / estimated_iters. */
2281 :
2282 : static int
2283 38040622 : param_change_prob (ipa_func_body_info *fbi, gimple *stmt, int i)
2284 : {
2285 38040622 : tree op = gimple_call_arg (stmt, i);
2286 38040622 : basic_block bb = gimple_bb (stmt);
2287 :
2288 38040622 : if (TREE_CODE (op) == WITH_SIZE_EXPR)
2289 299 : op = TREE_OPERAND (op, 0);
2290 :
2291 38040622 : tree base = get_base_address (op);
2292 :
2293 : /* Global invariants never change. */
2294 38040622 : if (is_gimple_min_invariant (base))
2295 : return 0;
2296 :
2297 : /* We would have to do non-trivial analysis to really work out what
2298 : is the probability of value to change (i.e. when init statement
2299 : is in a sibling loop of the call).
2300 :
2301 : We do an conservative estimate: when call is executed N times more often
2302 : than the statement defining value, we take the frequency 1/N. */
2303 19035620 : if (TREE_CODE (base) == SSA_NAME)
2304 : {
2305 16751123 : profile_count init_count;
2306 :
2307 25484865 : if (!bb->count.nonzero_p ())
2308 : return REG_BR_PROB_BASE;
2309 :
2310 8577023 : if (SSA_NAME_IS_DEFAULT_DEF (base))
2311 2988539 : init_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2312 : else
2313 5588484 : init_count = get_minimal_bb
2314 5588484 : (gimple_bb (SSA_NAME_DEF_STMT (base)),
2315 : gimple_bb (stmt))->count;
2316 :
2317 8577023 : if (init_count < bb->count)
2318 446755 : return MAX ((init_count.to_sreal_scale (bb->count)
2319 : * REG_BR_PROB_BASE).to_int (), 1);
2320 : return REG_BR_PROB_BASE;
2321 : }
2322 : else
2323 : {
2324 2284497 : ao_ref refd;
2325 2284497 : profile_count max = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2326 2284497 : struct record_modified_bb_info info;
2327 2284497 : tree init = ctor_for_folding (base);
2328 :
2329 2284497 : if (init != error_mark_node)
2330 : return 0;
2331 5071584 : if (!bb->count.nonzero_p () || fbi->aa_walk_budget == 0)
2332 : return REG_BR_PROB_BASE;
2333 1406214 : if (dump_file)
2334 : {
2335 0 : fprintf (dump_file, " Analyzing param change probability of ");
2336 0 : print_generic_expr (dump_file, op, TDF_SLIM);
2337 0 : fprintf (dump_file, "\n");
2338 : }
2339 1406214 : ao_ref_init (&refd, op);
2340 1406214 : info.op = op;
2341 1406214 : info.stmt = stmt;
2342 1406214 : info.bb_set = BITMAP_ALLOC (NULL);
2343 1406214 : int walked
2344 2812428 : = walk_aliased_vdefs (&refd, gimple_vuse (stmt), record_modified, &info,
2345 : NULL, NULL, fbi->aa_walk_budget);
2346 1406214 : if (walked > 0)
2347 1277993 : fbi->aa_walk_budget -= walked;
2348 1406214 : if (walked < 0 || bitmap_bit_p (info.bb_set, bb->index))
2349 : {
2350 866910 : if (walked < 0)
2351 206 : fbi->aa_walk_budget = 0;
2352 866910 : if (dump_file)
2353 : {
2354 0 : if (walked < 0)
2355 0 : fprintf (dump_file, " Ran out of AA walking budget.\n");
2356 : else
2357 0 : fprintf (dump_file, " Set in same BB as used.\n");
2358 : }
2359 866910 : BITMAP_FREE (info.bb_set);
2360 866910 : return REG_BR_PROB_BASE;
2361 : }
2362 :
2363 539304 : bitmap_iterator bi;
2364 539304 : unsigned index;
2365 : /* Lookup the most frequent update of the value and believe that
2366 : it dominates all the other; precise analysis here is difficult. */
2367 1520052 : EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi)
2368 980748 : max = profile_count::max_prefer_initialized
2369 980748 : (max, BASIC_BLOCK_FOR_FN (cfun, index)->count);
2370 539304 : if (dump_file)
2371 : {
2372 0 : fprintf (dump_file, " Set with count ");
2373 0 : max.dump (dump_file);
2374 0 : fprintf (dump_file, " and used with count ");
2375 0 : bb->count.dump (dump_file);
2376 0 : fprintf (dump_file, " freq %f\n",
2377 0 : max.to_sreal_scale (bb->count).to_double ());
2378 : }
2379 :
2380 539304 : BITMAP_FREE (info.bb_set);
2381 539304 : if (max < bb->count)
2382 90762 : return MAX ((max.to_sreal_scale (bb->count)
2383 : * REG_BR_PROB_BASE).to_int (), 1);
2384 : return REG_BR_PROB_BASE;
2385 : }
2386 : }
2387 :
2388 : /* Find whether a basic block BB is the final block of a (half) diamond CFG
2389 : sub-graph and if the predicate the condition depends on is known. If so,
2390 : return true and store the pointer the predicate in *P. */
2391 :
2392 : static bool
2393 6295313 : phi_result_unknown_predicate (ipa_func_body_info *fbi,
2394 : ipa_fn_summary *summary,
2395 : class ipa_node_params *params_summary,
2396 : basic_block bb,
2397 : ipa_predicate *p,
2398 : vec<ipa_predicate> nonconstant_names)
2399 : {
2400 6295313 : edge e;
2401 6295313 : edge_iterator ei;
2402 6295313 : basic_block first_bb = NULL;
2403 :
2404 6295313 : if (single_pred_p (bb))
2405 : {
2406 19727 : *p = false;
2407 19727 : return true;
2408 : }
2409 :
2410 14473450 : FOR_EACH_EDGE (e, ei, bb->preds)
2411 : {
2412 12350111 : if (single_succ_p (e->src))
2413 : {
2414 12959651 : if (!single_pred_p (e->src))
2415 : return false;
2416 7279027 : if (!first_bb)
2417 3186352 : first_bb = single_pred (e->src);
2418 4092675 : else if (single_pred (e->src) != first_bb)
2419 : return false;
2420 : }
2421 : else
2422 : {
2423 4087113 : if (!first_bb)
2424 : first_bb = e->src;
2425 1408758 : else if (e->src != first_bb)
2426 : return false;
2427 : }
2428 : }
2429 :
2430 2123339 : if (!first_bb)
2431 : return false;
2432 :
2433 4246678 : gcond *stmt = safe_dyn_cast <gcond *> (*gsi_last_bb (first_bb));
2434 2081244 : if (!stmt
2435 2081244 : || !is_gimple_ip_invariant (gimple_cond_rhs (stmt)))
2436 309381 : return false;
2437 :
2438 1813958 : *p = will_be_nonconstant_expr_predicate (fbi, summary, params_summary,
2439 : gimple_cond_lhs (stmt),
2440 : nonconstant_names);
2441 1813958 : if (*p == true)
2442 : return false;
2443 : else
2444 : return true;
2445 : }
2446 :
2447 : /* Given a PHI statement in a function described by inline properties SUMMARY
2448 : and *P being the predicate describing whether the selected PHI argument is
2449 : known, store a predicate for the result of the PHI statement into
2450 : NONCONSTANT_NAMES, if possible. */
2451 :
2452 : static void
2453 696489 : predicate_for_phi_result (class ipa_fn_summary *summary, gphi *phi,
2454 : ipa_predicate *p,
2455 : vec<ipa_predicate> nonconstant_names)
2456 : {
2457 696489 : unsigned i;
2458 :
2459 986472 : for (i = 0; i < gimple_phi_num_args (phi); i++)
2460 : {
2461 868568 : tree arg = gimple_phi_arg (phi, i)->def;
2462 868568 : if (!is_gimple_min_invariant (arg))
2463 : {
2464 750664 : gcc_assert (TREE_CODE (arg) == SSA_NAME);
2465 1501328 : *p = p->or_with (summary->conds,
2466 750664 : nonconstant_names[SSA_NAME_VERSION (arg)]);
2467 750664 : if (*p == true)
2468 : return;
2469 : }
2470 : }
2471 :
2472 117904 : if (dump_file && (dump_flags & TDF_DETAILS))
2473 : {
2474 3 : fprintf (dump_file, "\t\tphi predicate: ");
2475 3 : p->dump (dump_file, summary->conds);
2476 : }
2477 117904 : nonconstant_names[SSA_NAME_VERSION (gimple_phi_result (phi))] = *p;
2478 : }
2479 :
2480 : /* For a typical usage of __builtin_expect (a<b, 1), we
2481 : may introduce an extra relation stmt:
2482 : With the builtin, we have
2483 : t1 = a <= b;
2484 : t2 = (long int) t1;
2485 : t3 = __builtin_expect (t2, 1);
2486 : if (t3 != 0)
2487 : goto ...
2488 : Without the builtin, we have
2489 : if (a<=b)
2490 : goto...
2491 : This affects the size/time estimation and may have
2492 : an impact on the earlier inlining.
2493 : Here find this pattern and fix it up later. */
2494 :
2495 : static gimple *
2496 40681544 : find_foldable_builtin_expect (basic_block bb)
2497 : {
2498 40681544 : gimple_stmt_iterator bsi;
2499 :
2500 280188647 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
2501 : {
2502 198985931 : gimple *stmt = gsi_stmt (bsi);
2503 198985931 : if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)
2504 198775803 : || gimple_call_builtin_p (stmt, BUILT_IN_EXPECT_WITH_PROBABILITY)
2505 397761669 : || gimple_call_internal_p (stmt, IFN_BUILTIN_EXPECT))
2506 : {
2507 340199 : tree var = gimple_call_lhs (stmt);
2508 340199 : tree arg = gimple_call_arg (stmt, 0);
2509 340199 : use_operand_p use_p;
2510 340199 : gimple *use_stmt;
2511 340199 : bool match = false;
2512 340199 : bool done = false;
2513 :
2514 340199 : if (!var || !arg)
2515 3 : continue;
2516 340196 : gcc_assert (TREE_CODE (var) == SSA_NAME);
2517 :
2518 679098 : while (TREE_CODE (arg) == SSA_NAME)
2519 : {
2520 679098 : gimple *stmt_tmp = SSA_NAME_DEF_STMT (arg);
2521 679098 : if (!is_gimple_assign (stmt_tmp))
2522 : break;
2523 671946 : switch (gimple_assign_rhs_code (stmt_tmp))
2524 : {
2525 : case LT_EXPR:
2526 : case LE_EXPR:
2527 : case GT_EXPR:
2528 : case GE_EXPR:
2529 : case EQ_EXPR:
2530 : case NE_EXPR:
2531 : match = true;
2532 : done = true;
2533 : break;
2534 : CASE_CONVERT:
2535 : break;
2536 : default:
2537 : done = true;
2538 : break;
2539 : }
2540 338902 : if (done)
2541 : break;
2542 338902 : arg = gimple_assign_rhs1 (stmt_tmp);
2543 : }
2544 :
2545 304383 : if (match && single_imm_use (var, &use_p, &use_stmt)
2546 644158 : && gimple_code (use_stmt) == GIMPLE_COND)
2547 160372 : return use_stmt;
2548 : }
2549 : }
2550 : return NULL;
2551 : }
2552 :
2553 : /* Return true when the basic blocks contains only clobbers followed by RESX.
2554 : Such BBs are kept around to make removal of dead stores possible with
2555 : presence of EH and will be optimized out by optimize_clobbers later in the
2556 : game.
2557 :
2558 : NEED_EH is used to recurse in case the clobber has non-EH predecessors
2559 : that can be clobber only, too.. When it is false, the RESX is not necessary
2560 : on the end of basic block. */
2561 :
2562 : static bool
2563 41271290 : clobber_only_eh_bb_p (basic_block bb, bool need_eh = true)
2564 : {
2565 41271290 : gimple_stmt_iterator gsi = gsi_last_bb (bb);
2566 41271290 : edge_iterator ei;
2567 41271290 : edge e;
2568 :
2569 41271290 : if (need_eh)
2570 : {
2571 41203373 : if (gsi_end_p (gsi))
2572 : return false;
2573 40081200 : if (gimple_code (gsi_stmt (gsi)) != GIMPLE_RESX)
2574 : return false;
2575 1009677 : gsi_prev (&gsi);
2576 : }
2577 40749931 : else if (!single_succ_p (bb))
2578 : return false;
2579 :
2580 4483865 : for (; !gsi_end_p (gsi); gsi_prev (&gsi))
2581 : {
2582 2686570 : gimple *stmt = gsi_stmt (gsi);
2583 2686570 : if (is_gimple_debug (stmt))
2584 882503 : continue;
2585 1804067 : if (gimple_clobber_p (stmt))
2586 853626 : continue;
2587 950441 : if (gimple_code (stmt) == GIMPLE_LABEL)
2588 : break;
2589 : return false;
2590 : }
2591 :
2592 : /* See if all predecessors are either throws or clobber only BBs. */
2593 2969391 : FOR_EACH_EDGE (e, ei, bb->preds)
2594 2446102 : if (!(e->flags & EDGE_EH)
2595 2446102 : && !clobber_only_eh_bb_p (e->src, false))
2596 : return false;
2597 :
2598 : return true;
2599 : }
2600 :
2601 : /* Return true if STMT compute a floating point expression that may be affected
2602 : by -ffast-math and similar flags. */
2603 :
2604 : static bool
2605 93462329 : fp_expression_p (gimple *stmt)
2606 : {
2607 93462329 : ssa_op_iter i;
2608 93462329 : tree op;
2609 :
2610 209727409 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_DEF|SSA_OP_USE)
2611 116858854 : if (FLOAT_TYPE_P (TREE_TYPE (op)))
2612 : return true;
2613 : return false;
2614 : }
2615 :
2616 : /* Return true if T references memory location that is local
2617 : for the function (that means, dead after return) or read-only. */
2618 :
2619 : bool
2620 58101585 : refs_local_or_readonly_memory_p (tree t)
2621 : {
2622 : /* Non-escaping memory is fine. */
2623 58101585 : t = get_base_address (t);
2624 58101585 : if ((TREE_CODE (t) == MEM_REF
2625 58101585 : || TREE_CODE (t) == TARGET_MEM_REF))
2626 24524396 : return points_to_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
2627 :
2628 : /* Automatic variables are fine. */
2629 33577189 : if (DECL_P (t)
2630 33577189 : && auto_var_in_fn_p (t, current_function_decl))
2631 : return true;
2632 :
2633 : /* Read-only variables are fine. */
2634 11031970 : if (DECL_P (t) && TREE_READONLY (t))
2635 : return true;
2636 :
2637 : return false;
2638 : }
2639 :
2640 : /* Return true if T is a pointer pointing to memory location that is local
2641 : for the function (that means, dead after return) or read-only. */
2642 :
2643 : bool
2644 71008327 : points_to_local_or_readonly_memory_p (tree t)
2645 : {
2646 : /* See if memory location is clearly invalid. */
2647 71008327 : if (integer_zerop (t))
2648 2422171 : return flag_delete_null_pointer_checks;
2649 68586156 : if (TREE_CODE (t) == SSA_NAME)
2650 : {
2651 : /* For IPA passes we can consinder accesses to return slot local
2652 : even if it is not local in the sense that memory is dead by
2653 : the end of founction.
2654 : The outer function will see a store in the call assignment
2655 : and thus this will do right thing for all uses of this
2656 : function in the current IPA passes (modref, pure/const discovery
2657 : and inlining heuristics). */
2658 43372450 : if (DECL_RESULT (current_function_decl)
2659 43372450 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
2660 44394172 : && t == ssa_default_def (cfun, DECL_RESULT (current_function_decl)))
2661 : return true;
2662 43069075 : return !ptr_deref_may_alias_global_p (t, false);
2663 : }
2664 25213706 : if (TREE_CODE (t) == ADDR_EXPR
2665 25213706 : && (TREE_CODE (TREE_OPERAND (t, 0)) != TARGET_MEM_REF
2666 4690 : || TREE_CODE (TREE_OPERAND (TREE_OPERAND (t, 0), 0)) != INTEGER_CST))
2667 12988500 : return refs_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
2668 : return false;
2669 : }
2670 :
2671 : /* Return true if T is a pointer pointing to memory location that is possible
2672 : sra candidate if all functions it is passed to are inlined. */
2673 :
2674 : static bool
2675 38040622 : points_to_possible_sra_candidate_p (tree t)
2676 : {
2677 38040622 : if (TREE_CODE (t) != ADDR_EXPR)
2678 : return false;
2679 :
2680 11315560 : t = get_base_address (TREE_OPERAND (t, 0));
2681 :
2682 : /* Automatic variables are fine. */
2683 11315560 : if (DECL_P (t)
2684 11315560 : && auto_var_in_fn_p (t, current_function_decl))
2685 : return true;
2686 : return false;
2687 : }
2688 :
2689 : /* Return true if BB only calls builtin_unreachable.
2690 : We skip empty basic blocks, debug statements, clobbers and predicts.
2691 : CACHE is used to memoize already analyzed blocks. */
2692 :
2693 : static bool
2694 27074409 : builtin_unreachable_bb_p (basic_block bb, vec<unsigned char> &cache)
2695 : {
2696 27074409 : if (cache[bb->index])
2697 2550785 : return cache[bb->index] - 1;
2698 24523624 : gimple_stmt_iterator si;
2699 24523624 : auto_vec <basic_block, 4> visited_bbs;
2700 24523624 : bool ret = false;
2701 25317914 : while (true)
2702 : {
2703 25317914 : bool empty_bb = true;
2704 25317914 : visited_bbs.safe_push (bb);
2705 25317914 : cache[bb->index] = 3;
2706 25317914 : for (si = gsi_start_nondebug_bb (bb);
2707 26784139 : !gsi_end_p (si) && empty_bb;
2708 1466225 : gsi_next_nondebug (&si))
2709 : {
2710 25431916 : if (gimple_code (gsi_stmt (si)) != GIMPLE_PREDICT
2711 25065526 : && !gimple_clobber_p (gsi_stmt (si))
2712 49430903 : && !gimple_nop_p (gsi_stmt (si)))
2713 : {
2714 : empty_bb = false;
2715 : break;
2716 : }
2717 : }
2718 25317914 : if (!empty_bb)
2719 : break;
2720 : else
2721 1352223 : bb = single_succ_edge (bb)->dest;
2722 1352223 : if (cache[bb->index])
2723 : {
2724 557933 : ret = cache[bb->index] == 3 ? false : cache[bb->index] - 1;
2725 557933 : goto done;
2726 : }
2727 : }
2728 23965691 : if (gimple_call_builtin_p (gsi_stmt (si), BUILT_IN_UNREACHABLE)
2729 23965691 : || gimple_call_builtin_p (gsi_stmt (si), BUILT_IN_UNREACHABLE_TRAP))
2730 : ret = true;
2731 24523624 : done:
2732 98888786 : for (basic_block vbb:visited_bbs)
2733 25317914 : cache[vbb->index] = (unsigned char)ret + 1;
2734 24523624 : return ret;
2735 24523624 : }
2736 :
2737 : static bool
2738 13655343 : guards_builtin_unreachable (basic_block bb, vec<unsigned char> &cache)
2739 : {
2740 13655343 : edge_iterator ei;
2741 13655343 : edge e;
2742 40479192 : FOR_EACH_EDGE (e, ei, bb->succs)
2743 27074409 : if (builtin_unreachable_bb_p (e->dest, cache))
2744 : {
2745 250560 : if (dump_file && (dump_flags & TDF_DETAILS))
2746 1 : fprintf (dump_file,
2747 : "BB %i ends with conditional guarding __builtin_unreachable;"
2748 : " conditinal is unnecesary\n", bb->index);
2749 250560 : return true;
2750 : }
2751 : return false;
2752 : }
2753 :
2754 : #define STMT_NECESSARY GF_PLF_1
2755 :
2756 : /* If STMT is not already marked necessary, mark it, and add it to the
2757 : worklist if ADD_TO_WORKLIST is true. */
2758 :
2759 : static inline void
2760 170263296 : mark_stmt_necessary (gimple *stmt, auto_vec<gimple *> &worklist)
2761 : {
2762 170263296 : gcc_assert (stmt);
2763 :
2764 170263296 : if (gimple_plf (stmt, STMT_NECESSARY))
2765 : return;
2766 :
2767 141201795 : if (dump_file && (dump_flags & TDF_DETAILS))
2768 : {
2769 207 : fprintf (dump_file, "Marking useful stmt: ");
2770 207 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
2771 207 : fprintf (dump_file, "\n");
2772 : }
2773 :
2774 141201795 : gimple_set_plf (stmt, STMT_NECESSARY, true);
2775 141201795 : worklist.safe_push (stmt);
2776 : }
2777 :
2778 : /* Mark the statement defining operand OP as necessary. */
2779 :
2780 : static inline void
2781 119870514 : mark_operand_necessary (tree op, auto_vec<gimple *> &worklist)
2782 : {
2783 119870514 : gimple *stmt = SSA_NAME_DEF_STMT (op);
2784 119870514 : if (gimple_nop_p (stmt))
2785 : return;
2786 97362863 : mark_stmt_necessary (stmt, worklist);
2787 : }
2788 :
2789 : /* Mark all statements that will remain in the body after optimizing out
2790 : conditionals guarding __builtin_unreachable which we keep to preserve
2791 : value ranges. */
2792 :
2793 : static void
2794 7133774 : find_necessary_statements (struct cgraph_node *node)
2795 : {
2796 7133774 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
2797 7133774 : auto_vec<unsigned char, 10> cache;
2798 7133774 : basic_block bb;
2799 7133774 : auto_vec<gimple *> worklist;
2800 :
2801 7133774 : cache.safe_grow_cleared (last_basic_block_for_fn (cfun));
2802 : /* Mark all obviously necessary statements. */
2803 48337147 : FOR_EACH_BB_FN (bb, my_function)
2804 : {
2805 41203373 : for (gimple_stmt_iterator gsi = gsi_start_phis (bb);
2806 52686169 : !gsi_end_p (gsi); gsi_next (&gsi))
2807 11482796 : gimple_set_plf (gsi_stmt (gsi), STMT_NECESSARY, false);
2808 :
2809 230436876 : for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
2810 148030130 : gsi_next_nondebug (&bsi))
2811 : {
2812 148030130 : gimple *stmt = gsi_stmt (bsi);
2813 :
2814 148030130 : gimple_set_plf (stmt, STMT_NECESSARY, false);
2815 148030130 : if (gimple_has_side_effects (stmt)
2816 121332015 : || (is_ctrl_stmt (stmt)
2817 21789503 : && (gimple_code (stmt) != GIMPLE_COND
2818 13655343 : || !guards_builtin_unreachable (bb, cache)))
2819 99793072 : || gimple_store_p (stmt)
2820 223181517 : || gimple_code (stmt) == GIMPLE_ASM)
2821 72900433 : mark_stmt_necessary (stmt, worklist);
2822 : }
2823 : }
2824 148335569 : while (worklist.length () > 0)
2825 : {
2826 141201795 : gimple *stmt = worklist.pop ();
2827 :
2828 141201795 : if (dump_file && (dump_flags & TDF_DETAILS))
2829 : {
2830 207 : fprintf (dump_file, "processing: ");
2831 207 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
2832 207 : fprintf (dump_file, "\n");
2833 : }
2834 141201795 : if (gimple_code (stmt) == GIMPLE_PHI)
2835 18125367 : for (unsigned int k = 0; k < gimple_phi_num_args (stmt); k++)
2836 : {
2837 12827338 : tree arg = PHI_ARG_DEF (stmt, k);
2838 :
2839 12827338 : if (TREE_CODE (arg) == SSA_NAME)
2840 9848649 : mark_operand_necessary (arg, worklist);
2841 : }
2842 : else
2843 : {
2844 135903766 : ssa_op_iter iter;
2845 135903766 : tree use;
2846 :
2847 245925631 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2848 110021865 : mark_operand_necessary (use, worklist);
2849 : }
2850 : }
2851 7133774 : }
2852 :
2853 : /* Analyze function body for NODE.
2854 : EARLY indicates run from early optimization pipeline. */
2855 :
2856 : static void
2857 7133774 : analyze_function_body (struct cgraph_node *node, bool early)
2858 : {
2859 7133774 : sreal time = opt_for_fn (node->decl, param_uninlined_function_time);
2860 : /* Estimate static overhead for function prologue/epilogue and alignment. */
2861 7133774 : int size = opt_for_fn (node->decl, param_uninlined_function_insns);
2862 : /* Benefits are scaled by probability of elimination that is in range
2863 : <0,2>. */
2864 7133774 : basic_block bb;
2865 7133774 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
2866 7133774 : sreal freq;
2867 7133774 : class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
2868 7133774 : ipa_node_params *params_summary
2869 7133774 : = early ? NULL : ipa_node_params_sum->get (node);
2870 7133774 : ipa_predicate bb_predicate;
2871 7133774 : struct ipa_func_body_info fbi;
2872 7133774 : vec<ipa_predicate> nonconstant_names = vNULL;
2873 7133774 : int nblocks, n;
2874 7133774 : int *order;
2875 7133774 : gimple *fix_builtin_expect_stmt;
2876 :
2877 7133774 : gcc_assert (my_function && my_function->cfg);
2878 7133774 : gcc_assert (cfun == my_function);
2879 :
2880 7133774 : memset(&fbi, 0, sizeof(fbi));
2881 7133774 : vec_free (info->conds);
2882 7133774 : info->conds = NULL;
2883 7133774 : info->size_time_table.release ();
2884 7133774 : info->call_size_time_table.release ();
2885 :
2886 : /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2887 : so we can produce proper inline hints.
2888 :
2889 : When optimizing and analyzing for early inliner, initialize node params
2890 : so we can produce correct BB predicates. */
2891 :
2892 7133774 : if (opt_for_fn (node->decl, optimize))
2893 : {
2894 6241511 : calculate_dominance_info (CDI_DOMINATORS);
2895 6241511 : calculate_dominance_info (CDI_POST_DOMINATORS);
2896 6241511 : if (!early)
2897 1368866 : loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
2898 : else
2899 : {
2900 4872645 : ipa_check_create_node_params ();
2901 4872645 : ipa_initialize_node_params (node);
2902 : }
2903 :
2904 6241511 : if (ipa_node_params_sum)
2905 : {
2906 6241511 : fbi.node = node;
2907 6241511 : fbi.info = ipa_node_params_sum->get (node);
2908 6241511 : fbi.bb_infos = vNULL;
2909 6241511 : fbi.bb_infos.safe_grow_cleared (last_basic_block_for_fn (cfun), true);
2910 6241511 : fbi.param_count = count_formal_params (node->decl);
2911 6241511 : fbi.aa_walk_budget = opt_for_fn (node->decl, param_ipa_max_aa_steps);
2912 :
2913 6241511 : nonconstant_names.safe_grow_cleared
2914 6241511 : (SSANAMES (my_function)->length (), true);
2915 : }
2916 : }
2917 :
2918 7133774 : if (dump_file)
2919 247 : fprintf (dump_file, "\nAnalyzing function body size: %s\n",
2920 : node->dump_name ());
2921 :
2922 : /* When we run into maximal number of entries, we assign everything to the
2923 : constant truth case. Be sure to have it in list. */
2924 7133774 : bb_predicate = true;
2925 7133774 : info->account_size_time (0, 0, bb_predicate, bb_predicate);
2926 :
2927 7133774 : bb_predicate = ipa_predicate::not_inlined ();
2928 7133774 : info->account_size_time (opt_for_fn (node->decl,
2929 : param_uninlined_function_insns)
2930 : * ipa_fn_summary::size_scale,
2931 7133774 : opt_for_fn (node->decl,
2932 : param_uninlined_function_time),
2933 : bb_predicate,
2934 : bb_predicate);
2935 :
2936 : /* Only look for target information for inlinable functions. */
2937 7133774 : bool scan_for_target_info =
2938 7133774 : info->inlinable
2939 12803224 : && targetm.target_option.need_ipa_fn_target_info (node->decl,
2940 5669450 : info->target_info);
2941 :
2942 7133774 : if (fbi.info)
2943 6241511 : compute_bb_predicates (&fbi, node, info, params_summary);
2944 7133774 : find_necessary_statements (node);
2945 7133774 : const profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2946 7133774 : order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
2947 7133774 : nblocks = pre_and_rev_post_order_compute (NULL, order, false);
2948 48337147 : for (n = 0; n < nblocks; n++)
2949 : {
2950 41203373 : bb = BASIC_BLOCK_FOR_FN (cfun, order[n]);
2951 41203373 : freq = bb->count.to_sreal_scale (entry_count);
2952 41203373 : if (clobber_only_eh_bb_p (bb))
2953 : {
2954 521829 : if (dump_file && (dump_flags & TDF_DETAILS))
2955 0 : fprintf (dump_file, "\n Ignoring BB %i;"
2956 : " it will be optimized away by cleanup_clobbers\n",
2957 : bb->index);
2958 521829 : continue;
2959 : }
2960 :
2961 : /* TODO: Obviously predicates can be propagated down across CFG. */
2962 40681544 : if (fbi.info)
2963 : {
2964 34320028 : if (bb->aux)
2965 34320014 : bb_predicate = *(ipa_predicate *)bb->aux;
2966 : else
2967 14 : bb_predicate = false;
2968 : }
2969 : else
2970 6361516 : bb_predicate = true;
2971 :
2972 40681544 : if (dump_file && (dump_flags & TDF_DETAILS))
2973 : {
2974 67 : fprintf (dump_file, "\n BB %i predicate:", bb->index);
2975 67 : bb_predicate.dump (dump_file, info->conds);
2976 : }
2977 :
2978 40681544 : if (fbi.info && nonconstant_names.exists ())
2979 : {
2980 34320028 : ipa_predicate phi_predicate;
2981 34320028 : bool first_phi = true;
2982 :
2983 35016517 : for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
2984 696489 : gsi_next (&bsi))
2985 : {
2986 6377113 : if (first_phi
2987 6377113 : && !phi_result_unknown_predicate (&fbi, info,
2988 : params_summary,
2989 : bb,
2990 : &phi_predicate,
2991 : nonconstant_names))
2992 : break;
2993 696489 : first_phi = false;
2994 696489 : if (dump_file && (dump_flags & TDF_DETAILS))
2995 : {
2996 3 : fprintf (dump_file, " ");
2997 3 : print_gimple_stmt (dump_file, gsi_stmt (bsi), 0);
2998 : }
2999 696489 : predicate_for_phi_result (info, bsi.phi (), &phi_predicate,
3000 : nonconstant_names);
3001 : }
3002 : }
3003 :
3004 40681544 : fix_builtin_expect_stmt = find_foldable_builtin_expect (bb);
3005 :
3006 40681544 : for (gimple_stmt_iterator bsi = gsi_start_nondebug_bb (bb);
3007 180831122 : !gsi_end_p (bsi); gsi_next_nondebug (&bsi))
3008 : {
3009 140149578 : gimple *stmt = gsi_stmt (bsi);
3010 140149578 : if (!gimple_plf (stmt, STMT_NECESSARY))
3011 : {
3012 5485043 : if (dump_file && (dump_flags & TDF_DETAILS))
3013 : {
3014 19 : fprintf (dump_file, " skipping unnecesary stmt ");
3015 19 : print_gimple_stmt (dump_file, stmt, 0);
3016 : }
3017 : /* TODO: const calls used only to produce values for
3018 : builtion_unreachable guards should not be accounted. However
3019 : we still want to inline them and this does does not work well
3020 : with the cost model. For now account them as usual. */
3021 5485043 : if (!is_gimple_call (stmt)
3022 5485043 : || gimple_call_internal_p (stmt))
3023 5213741 : continue;
3024 : }
3025 134935837 : int this_size = estimate_num_insns (stmt, &eni_size_weights);
3026 134935837 : int this_time = estimate_num_insns (stmt, &eni_time_weights);
3027 134935837 : int prob;
3028 134935837 : ipa_predicate will_be_nonconstant;
3029 :
3030 : /* This relation stmt should be folded after we remove
3031 : __builtin_expect call. Adjust the cost here. */
3032 134935837 : if (stmt == fix_builtin_expect_stmt)
3033 : {
3034 159934 : this_size--;
3035 159934 : this_time--;
3036 : }
3037 :
3038 134935837 : if (dump_file && (dump_flags & TDF_DETAILS))
3039 : {
3040 201 : fprintf (dump_file, " ");
3041 201 : print_gimple_stmt (dump_file, stmt, 0);
3042 201 : fprintf (dump_file, "\t\tfreq:%3.2f size:%3i time:%3i\n",
3043 : freq.to_double (), this_size,
3044 : this_time);
3045 : }
3046 :
3047 134935837 : if (is_gimple_call (stmt)
3048 134935837 : && !gimple_call_internal_p (stmt))
3049 : {
3050 22655912 : struct cgraph_edge *edge = node->get_edge (stmt);
3051 22655912 : ipa_call_summary *es = ipa_call_summaries->get_create (edge);
3052 :
3053 : /* Special case: results of BUILT_IN_CONSTANT_P will be always
3054 : resolved as constant. We however don't want to optimize
3055 : out the cgraph edges. */
3056 22655912 : if (nonconstant_names.exists ()
3057 19792533 : && gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P)
3058 28317 : && gimple_call_lhs (stmt)
3059 22684229 : && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
3060 : {
3061 28317 : ipa_predicate false_p = false;
3062 28317 : nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
3063 28317 : = false_p;
3064 : }
3065 22655912 : if (ipa_node_params_sum)
3066 : {
3067 19805312 : int count = gimple_call_num_args (stmt);
3068 19805312 : int i;
3069 :
3070 19805312 : if (count)
3071 16914773 : es->param.safe_grow_cleared (count, true);
3072 57845934 : for (i = 0; i < count; i++)
3073 : {
3074 38040622 : int prob = param_change_prob (&fbi, stmt, i);
3075 38040622 : gcc_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
3076 38040622 : es->param[i].change_prob = prob;
3077 38040622 : es->param[i].points_to_local_or_readonly_memory
3078 38040622 : = points_to_local_or_readonly_memory_p
3079 38040622 : (gimple_call_arg (stmt, i));
3080 38040622 : es->param[i].points_to_possible_sra_candidate
3081 38040622 : = points_to_possible_sra_candidate_p
3082 38040622 : (gimple_call_arg (stmt, i));
3083 : }
3084 : }
3085 : /* We cannot setup VLA parameters during inlining. */
3086 66496009 : for (unsigned int i = 0; i < gimple_call_num_args (stmt); ++i)
3087 43840493 : if (TREE_CODE (gimple_call_arg (stmt, i)) == WITH_SIZE_EXPR)
3088 : {
3089 396 : edge->inline_failed = CIF_FUNCTION_NOT_INLINABLE;
3090 396 : break;
3091 : }
3092 22655912 : es->call_stmt_size = this_size;
3093 22655912 : es->call_stmt_time = this_time;
3094 22655912 : es->loop_depth = bb_loop_depth (bb);
3095 22655912 : edge_set_predicate (edge, &bb_predicate);
3096 22655912 : if (edge->speculative)
3097 : {
3098 0 : cgraph_edge *indirect
3099 0 : = edge->speculative_call_indirect_edge ();
3100 0 : ipa_call_summary *es2
3101 0 : = ipa_call_summaries->get_create (indirect);
3102 0 : ipa_call_summaries->duplicate (edge, indirect,
3103 : es, es2);
3104 :
3105 : /* Edge is the first direct call.
3106 : create and duplicate call summaries for multiple
3107 : speculative call targets. */
3108 0 : for (cgraph_edge *direct
3109 0 : = edge->next_speculative_call_target ();
3110 0 : direct;
3111 0 : direct = direct->next_speculative_call_target ())
3112 : {
3113 0 : ipa_call_summary *es3
3114 0 : = ipa_call_summaries->get_create (direct);
3115 0 : ipa_call_summaries->duplicate (edge, direct,
3116 : es, es3);
3117 : }
3118 : }
3119 :
3120 : /* If dealing with a carrying edge, copy its summary over to its
3121 : attached edges as well. */
3122 22655912 : if (edge->has_callback)
3123 : {
3124 15081 : cgraph_edge *cbe;
3125 30162 : for (cbe = edge->first_callback_edge (); cbe;
3126 15081 : cbe = cbe->next_callback_edge ())
3127 : {
3128 15081 : ipa_call_summary *es2 = ipa_call_summaries->get_create (cbe);
3129 15081 : ipa_call_summaries->duplicate (edge, cbe, es, es2);
3130 : /* Unlike speculative edges, callback edges have no real
3131 : size or time; the call doesn't exist. Reflect that in
3132 : their summaries. */
3133 15081 : es2->call_stmt_size = 0;
3134 15081 : es2->call_stmt_time = 0;
3135 : }
3136 : }
3137 : }
3138 :
3139 : /* TODO: When conditional jump or switch is known to be constant, but
3140 : we did not translate it into the predicates, we really can account
3141 : just maximum of the possible paths. */
3142 134935837 : if (fbi.info)
3143 112906028 : will_be_nonconstant
3144 112906028 : = will_be_nonconstant_predicate (&fbi, info, params_summary,
3145 : stmt, nonconstant_names);
3146 : else
3147 22029809 : will_be_nonconstant = true;
3148 134935837 : if (this_time || this_size)
3149 : {
3150 110627062 : sreal final_time = (sreal)this_time * freq;
3151 110627062 : prob = eliminated_by_inlining_prob (&fbi, stmt);
3152 110627062 : if (prob == 1 && dump_file && (dump_flags & TDF_DETAILS))
3153 11 : fprintf (dump_file,
3154 : "\t\t50%% will be eliminated by inlining\n");
3155 110627062 : if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
3156 18 : fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
3157 :
3158 110627062 : ipa_predicate p = bb_predicate & will_be_nonconstant;
3159 110627062 : int parm = load_or_store_of_ptr_parameter (&fbi, stmt);
3160 110627062 : ipa_predicate sra_predicate = true;
3161 110627062 : if (parm != -1)
3162 13959886 : sra_predicate &= add_condition (info, params_summary, parm,
3163 : ptr_type_node, NULL,
3164 6979943 : ipa_predicate::not_sra_candidate, NULL, 0);
3165 :
3166 : /* We can ignore statement when we proved it is never going
3167 : to happen, but we cannot do that for call statements
3168 : because edges are accounted specially. */
3169 :
3170 221254124 : if (*(is_gimple_call (stmt) ? &bb_predicate : &p) != false)
3171 : {
3172 109916047 : time += final_time;
3173 109916047 : size += this_size;
3174 : }
3175 :
3176 : /* We account everything but the calls. Calls have their own
3177 : size/time info attached to cgraph edges. This is necessary
3178 : in order to make the cost disappear after inlining. */
3179 110627062 : if (!is_gimple_call (stmt))
3180 : {
3181 88628564 : if (prob)
3182 : {
3183 14720320 : ipa_predicate ip
3184 14720320 : = bb_predicate & ipa_predicate::not_inlined () & sra_predicate;
3185 29440640 : info->account_size_time (this_size * prob,
3186 14720320 : (final_time * prob) / 2, ip,
3187 : p);
3188 : }
3189 14720320 : if (prob != 2)
3190 163634858 : info->account_size_time (this_size * (2 - prob),
3191 81817429 : (final_time * (2 - prob) / 2),
3192 163634858 : bb_predicate & sra_predicate,
3193 : p);
3194 : }
3195 :
3196 110627062 : if (!info->fp_expressions && fp_expression_p (stmt))
3197 : {
3198 593774 : info->fp_expressions = true;
3199 593774 : if (dump_file)
3200 9 : fprintf (dump_file, " fp_expression set\n");
3201 : }
3202 : }
3203 :
3204 : /* For target specific information, we want to scan all statements
3205 : rather than those statements with non-zero weights, to avoid
3206 : missing to scan something interesting for target information,
3207 : such as: internal function calls. */
3208 134935837 : if (scan_for_target_info)
3209 0 : scan_for_target_info =
3210 0 : targetm.target_option.update_ipa_fn_target_info
3211 0 : (info->target_info, stmt);
3212 :
3213 : /* Account cost of address calculations in the statements. */
3214 521690770 : for (unsigned int i = 0; i < gimple_num_ops (stmt); i++)
3215 : {
3216 386754933 : for (tree op = gimple_op (stmt, i);
3217 744603374 : op && handled_component_p (op);
3218 46647111 : op = TREE_OPERAND (op, 0))
3219 46647111 : if ((TREE_CODE (op) == ARRAY_REF
3220 46647111 : || TREE_CODE (op) == ARRAY_RANGE_REF)
3221 46647111 : && TREE_CODE (TREE_OPERAND (op, 1)) == SSA_NAME)
3222 : {
3223 2379614 : ipa_predicate p = bb_predicate;
3224 2379614 : if (fbi.info)
3225 1861156 : p = p & will_be_nonconstant_expr_predicate
3226 1861156 : (&fbi, info, params_summary,
3227 1861156 : TREE_OPERAND (op, 1),
3228 1861156 : nonconstant_names);
3229 2379614 : if (p != false)
3230 : {
3231 2373902 : time += freq;
3232 2373902 : size += 1;
3233 2373902 : if (dump_file)
3234 24 : fprintf (dump_file,
3235 : "\t\tAccounting address calculation.\n");
3236 2373902 : info->account_size_time (ipa_fn_summary::size_scale,
3237 : freq,
3238 : bb_predicate,
3239 : p);
3240 : }
3241 : }
3242 : }
3243 :
3244 : }
3245 : }
3246 7133774 : free (order);
3247 :
3248 7133774 : if (nonconstant_names.exists () && !early)
3249 : {
3250 1368866 : ipa_fn_summary *s = ipa_fn_summaries->get (node);
3251 1368866 : unsigned max_loop_predicates = opt_for_fn (node->decl,
3252 : param_ipa_max_loop_predicates);
3253 :
3254 1368866 : if (dump_file && (dump_flags & TDF_DETAILS))
3255 14 : flow_loops_dump (dump_file, NULL, 0);
3256 1368866 : scev_initialize ();
3257 4726608 : for (auto loop : loops_list (cfun, 0))
3258 : {
3259 620010 : ipa_predicate loop_iterations = true;
3260 620010 : sreal header_freq;
3261 620010 : edge ex;
3262 620010 : unsigned int j;
3263 620010 : class tree_niter_desc niter_desc;
3264 620010 : if (!loop->header->aux)
3265 0 : continue;
3266 :
3267 620010 : profile_count hdr_count = loop->header->count;
3268 620010 : sreal hdr_freq = hdr_count.to_sreal_scale (entry_count);
3269 :
3270 620010 : bb_predicate = *(ipa_predicate *)loop->header->aux;
3271 620010 : auto_vec<edge> exits = get_loop_exit_edges (loop);
3272 1619069 : FOR_EACH_VEC_ELT (exits, j, ex)
3273 999059 : if (number_of_iterations_exit (loop, ex, &niter_desc, false)
3274 999059 : && !is_gimple_min_invariant (niter_desc.niter))
3275 : {
3276 152805 : ipa_predicate will_be_nonconstant
3277 152805 : = will_be_nonconstant_expr_predicate (&fbi, info,
3278 : params_summary,
3279 : niter_desc.niter,
3280 : nonconstant_names);
3281 152805 : if (will_be_nonconstant != true)
3282 61369 : will_be_nonconstant = bb_predicate & will_be_nonconstant;
3283 152805 : if (will_be_nonconstant != true
3284 214174 : && will_be_nonconstant != false)
3285 60814 : loop_iterations &= will_be_nonconstant;
3286 : }
3287 620010 : add_freqcounting_predicate (&s->loop_iterations, loop_iterations,
3288 : hdr_freq, max_loop_predicates);
3289 620012 : }
3290 :
3291 : /* To avoid quadratic behavior we analyze stride predicates only
3292 : with respect to the containing loop. Thus we simply iterate
3293 : over all defs in the outermost loop body. */
3294 1368866 : for (class loop *loop = loops_for_fn (cfun)->tree_root->inner;
3295 1855539 : loop != NULL; loop = loop->next)
3296 : {
3297 486673 : ipa_predicate loop_stride = true;
3298 486673 : basic_block *body = get_loop_body (loop);
3299 486673 : profile_count hdr_count = loop->header->count;
3300 486673 : sreal hdr_freq = hdr_count.to_sreal_scale (entry_count);
3301 2869976 : for (unsigned i = 0; i < loop->num_nodes; i++)
3302 : {
3303 2383303 : gimple_stmt_iterator gsi;
3304 2383303 : if (!body[i]->aux)
3305 7 : continue;
3306 :
3307 2383296 : bb_predicate = *(ipa_predicate *)body[i]->aux;
3308 15703892 : for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
3309 10937300 : gsi_next (&gsi))
3310 : {
3311 10937300 : gimple *stmt = gsi_stmt (gsi);
3312 :
3313 10937300 : if (!is_gimple_assign (stmt))
3314 10784941 : continue;
3315 :
3316 5357485 : tree def = gimple_assign_lhs (stmt);
3317 5357485 : if (TREE_CODE (def) != SSA_NAME)
3318 1019880 : continue;
3319 :
3320 4337605 : affine_iv iv;
3321 8675210 : if (!simple_iv (loop_containing_stmt (stmt),
3322 : loop_containing_stmt (stmt),
3323 : def, &iv, true)
3324 4337605 : || is_gimple_min_invariant (iv.step))
3325 4185246 : continue;
3326 :
3327 152359 : ipa_predicate will_be_nonconstant
3328 152359 : = will_be_nonconstant_expr_predicate (&fbi, info,
3329 : params_summary,
3330 : iv.step,
3331 : nonconstant_names);
3332 152359 : if (will_be_nonconstant != true)
3333 53981 : will_be_nonconstant = bb_predicate & will_be_nonconstant;
3334 152359 : if (will_be_nonconstant != true
3335 206340 : && will_be_nonconstant != false)
3336 35103 : loop_stride = loop_stride & will_be_nonconstant;
3337 : }
3338 : }
3339 486673 : add_freqcounting_predicate (&s->loop_strides, loop_stride,
3340 : hdr_freq, max_loop_predicates);
3341 486673 : free (body);
3342 : }
3343 1368866 : scev_finalize ();
3344 : }
3345 62604695 : FOR_ALL_BB_FN (bb, my_function)
3346 : {
3347 55470921 : edge e;
3348 55470921 : edge_iterator ei;
3349 :
3350 55470921 : if (bb->aux)
3351 41022610 : edge_predicate_pool.remove ((ipa_predicate *)bb->aux);
3352 55470921 : bb->aux = NULL;
3353 118026434 : FOR_EACH_EDGE (e, ei, bb->succs)
3354 : {
3355 62555513 : if (e->aux)
3356 2435247 : edge_predicate_pool.remove ((ipa_predicate *)e->aux);
3357 62555513 : e->aux = NULL;
3358 : }
3359 : }
3360 7133774 : ipa_fn_summary *s = ipa_fn_summaries->get (node);
3361 7133774 : ipa_size_summary *ss = ipa_size_summaries->get (node);
3362 7133774 : s->time = time;
3363 7133774 : ss->self_size = size;
3364 7133774 : nonconstant_names.release ();
3365 7133774 : ipa_release_body_info (&fbi);
3366 7133774 : if (opt_for_fn (node->decl, optimize))
3367 : {
3368 6241511 : if (!early)
3369 1368866 : loop_optimizer_finalize ();
3370 4872645 : else if (!ipa_edge_args_sum)
3371 4872631 : ipa_free_all_node_params ();
3372 6241511 : free_dominance_info (CDI_DOMINATORS);
3373 6241511 : free_dominance_info (CDI_POST_DOMINATORS);
3374 : }
3375 7133774 : if (dump_file)
3376 : {
3377 247 : fprintf (dump_file, "\n");
3378 247 : ipa_dump_fn_summary (dump_file, node);
3379 : }
3380 7133774 : }
3381 :
3382 :
3383 : /* Compute function summary.
3384 : EARLY is true when we compute parameters during early opts. */
3385 :
3386 : void
3387 7135062 : compute_fn_summary (struct cgraph_node *node, bool early)
3388 : {
3389 7135062 : HOST_WIDE_INT self_stack_size;
3390 7135062 : struct cgraph_edge *e;
3391 :
3392 7135062 : gcc_assert (!node->inlined_to);
3393 :
3394 7135062 : if (!ipa_fn_summaries)
3395 213875 : ipa_fn_summary_alloc ();
3396 :
3397 : /* Create a new ipa_fn_summary. */
3398 7135062 : ((ipa_fn_summary_t *)ipa_fn_summaries)->remove_callees (node);
3399 7135062 : ipa_fn_summaries->remove (node);
3400 7135062 : class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
3401 7135062 : class ipa_size_summary *size_info = ipa_size_summaries->get_create (node);
3402 :
3403 : /* Estimate the stack size for the function if we're optimizing. */
3404 12484294 : self_stack_size = optimize && !node->thunk
3405 13376573 : ? estimated_stack_frame_size (node) : 0;
3406 7135062 : size_info->estimated_self_stack_size = self_stack_size;
3407 7135062 : info->estimated_stack_size = self_stack_size;
3408 :
3409 7135062 : if (node->thunk)
3410 : {
3411 1288 : ipa_call_summary *es = ipa_call_summaries->get_create (node->callees);
3412 1288 : ipa_predicate t = true;
3413 :
3414 1288 : node->can_change_signature = false;
3415 1288 : es->call_stmt_size = eni_size_weights.call_cost;
3416 1288 : es->call_stmt_time = eni_time_weights.call_cost;
3417 3864 : info->account_size_time (ipa_fn_summary::size_scale
3418 1288 : * opt_for_fn (node->decl,
3419 : param_uninlined_function_thunk_insns),
3420 1288 : opt_for_fn (node->decl,
3421 : param_uninlined_function_thunk_time), t, t);
3422 1288 : t = ipa_predicate::not_inlined ();
3423 1288 : info->account_size_time (2 * ipa_fn_summary::size_scale, 0, t, t);
3424 1288 : ipa_update_overall_fn_summary (node);
3425 1288 : size_info->self_size = size_info->size;
3426 1288 : if (stdarg_p (TREE_TYPE (node->decl)))
3427 : {
3428 9 : info->inlinable = false;
3429 9 : node->callees->inline_failed = CIF_VARIADIC_THUNK;
3430 : }
3431 : else
3432 1279 : info->inlinable = true;
3433 : }
3434 : else
3435 : {
3436 : /* Even is_gimple_min_invariant rely on current_function_decl. */
3437 7133774 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
3438 :
3439 : /* During IPA profile merging we may be called w/o virtual SSA form
3440 : built. */
3441 7133774 : update_ssa (TODO_update_ssa_only_virtuals);
3442 :
3443 : /* Can this function be inlined at all? */
3444 7133774 : if (!opt_for_fn (node->decl, optimize)
3445 8026037 : && !lookup_attribute ("always_inline",
3446 892263 : DECL_ATTRIBUTES (node->decl)))
3447 824675 : info->inlinable = false;
3448 : else
3449 6309099 : info->inlinable = tree_inlinable_function_p (node->decl);
3450 :
3451 7133774 : bool no_signature = false;
3452 :
3453 : /* Don't allow signature changes for functions which have
3454 : [[gnu::musttail]] or [[clang::musttail]] calls. Sometimes
3455 : (more often on targets which pass everything on the stack)
3456 : signature changes can result in tail calls being impossible
3457 : even when without the signature changes they would be ok.
3458 : See PR121023. */
3459 7133774 : if (cfun->has_musttail)
3460 : {
3461 1406 : if (dump_file)
3462 0 : fprintf (dump_file, "No signature change:"
3463 : " function has calls with musttail attribute.\n");
3464 : no_signature = true;
3465 : }
3466 :
3467 : /* Type attributes can use parameter indices to describe them.
3468 : Special case fn spec since we can safely preserve them in
3469 : modref summaries. */
3470 7133774 : for (tree list = TYPE_ATTRIBUTES (TREE_TYPE (node->decl));
3471 7524053 : list && !no_signature; list = TREE_CHAIN (list))
3472 390279 : if (!ipa_param_adjustments::type_attribute_allowed_p
3473 390279 : (get_attribute_name (list)))
3474 : {
3475 157489 : if (dump_file)
3476 : {
3477 0 : fprintf (dump_file, "No signature change:"
3478 : " function type has unhandled attribute %s.\n",
3479 0 : IDENTIFIER_POINTER (get_attribute_name (list)));
3480 : }
3481 : no_signature = true;
3482 : }
3483 7133774 : for (tree parm = DECL_ARGUMENTS (node->decl);
3484 21299983 : parm && !no_signature; parm = DECL_CHAIN (parm))
3485 14166209 : if (variably_modified_type_p (TREE_TYPE (parm), node->decl))
3486 : {
3487 15638 : if (dump_file)
3488 : {
3489 0 : fprintf (dump_file, "No signature change:"
3490 : " has parameter with variably modified type.\n");
3491 : }
3492 : no_signature = true;
3493 : }
3494 :
3495 : /* Likewise for #pragma omp declare simd functions or functions
3496 : with simd attribute. */
3497 7133774 : if (no_signature
3498 14093015 : || lookup_attribute ("omp declare simd",
3499 6959241 : DECL_ATTRIBUTES (node->decl)))
3500 176221 : node->can_change_signature = false;
3501 : else
3502 : {
3503 : /* Otherwise, inlinable functions always can change signature. */
3504 6957553 : if (info->inlinable)
3505 5590541 : node->can_change_signature = true;
3506 : else
3507 : {
3508 : /* Functions calling builtin_apply cannot change signature. */
3509 5364633 : for (e = node->callees; e; e = e->next_callee)
3510 : {
3511 4030149 : tree cdecl = e->callee->decl;
3512 4030149 : if (fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS,
3513 : BUILT_IN_VA_START))
3514 : break;
3515 : }
3516 1367012 : node->can_change_signature = !e;
3517 : }
3518 : }
3519 7133774 : analyze_function_body (node, early);
3520 7133774 : pop_cfun ();
3521 : }
3522 :
3523 : /* Inlining characteristics are maintained by the cgraph_mark_inline. */
3524 7135062 : size_info->size = size_info->self_size;
3525 7135062 : info->estimated_stack_size = size_info->estimated_self_stack_size;
3526 :
3527 : /* Code above should compute exactly the same result as
3528 : ipa_update_overall_fn_summary except for case when speculative
3529 : edges are present since these are accounted to size but not
3530 : self_size. Do not compare time since different order the roundoff
3531 : errors result in slight changes. */
3532 7135062 : ipa_update_overall_fn_summary (node);
3533 7135062 : if (flag_checking)
3534 : {
3535 7611446 : for (e = node->indirect_calls; e; e = e->next_callee)
3536 476466 : if (e->speculative)
3537 : break;
3538 7134980 : gcc_assert (e || size_info->size == size_info->self_size);
3539 : }
3540 7135062 : }
3541 :
3542 :
3543 : /* Compute parameters of functions used by inliner using
3544 : current_function_decl. */
3545 :
3546 : static unsigned int
3547 5706083 : compute_fn_summary_for_current (void)
3548 : {
3549 5706083 : compute_fn_summary (cgraph_node::get (current_function_decl), true);
3550 5706083 : return 0;
3551 : }
3552 :
3553 : /* Estimate benefit devirtualizing indirect edge IE and return true if it can
3554 : be devirtualized and inlined, provided m_known_vals, m_known_contexts and
3555 : m_known_aggs in AVALS. Return false straight away if AVALS is NULL. */
3556 :
3557 : static bool
3558 4740086 : estimate_edge_devirt_benefit (struct cgraph_edge *ie,
3559 : int *size, int *time,
3560 : ipa_call_arg_values *avals)
3561 : {
3562 4740086 : tree target;
3563 4740086 : struct cgraph_node *callee;
3564 4740086 : class ipa_fn_summary *isummary;
3565 4740086 : enum availability avail;
3566 4740086 : bool speculative;
3567 :
3568 4740086 : if (!avals
3569 5953526 : || (!avals->m_known_vals.length() && !avals->m_known_contexts.length ()))
3570 : return false;
3571 1409961 : if (!opt_for_fn (ie->caller->decl, flag_indirect_inlining))
3572 : return false;
3573 :
3574 1403594 : target = ipa_get_indirect_edge_target
3575 1403594 : (ie->callee ? ie->speculative_call_indirect_edge () : ie,
3576 : avals, &speculative);
3577 1403594 : if (!target || speculative)
3578 : return false;
3579 :
3580 : /* If this is speculative call, turn its cost into 0; we will account
3581 : the call when processing the indirect call. */
3582 217168 : if (ie->callee)
3583 : {
3584 4081 : gcc_checking_assert (ie->speculative && *size > 0);
3585 4081 : *size = 0;
3586 4081 : *time = 0;
3587 : }
3588 : else
3589 : {
3590 : /* Account for difference in cost between indirect and direct calls. */
3591 213087 : *size -= (eni_size_weights.indirect_call_cost - eni_size_weights.call_cost);
3592 213087 : *time -= (eni_time_weights.indirect_call_cost - eni_time_weights.call_cost);
3593 : }
3594 217168 : gcc_checking_assert (*time >= 0);
3595 217168 : gcc_checking_assert (*size >= 0);
3596 :
3597 217168 : callee = cgraph_node::get (target);
3598 217168 : if (!callee || !callee->definition)
3599 : return false;
3600 196559 : callee = callee->function_symbol (&avail);
3601 196559 : if (avail < AVAIL_AVAILABLE)
3602 : return false;
3603 196533 : isummary = ipa_fn_summaries->get (callee);
3604 196533 : if (isummary == NULL)
3605 : return false;
3606 :
3607 196521 : return isummary->inlinable;
3608 : }
3609 :
3610 : /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
3611 : handle edge E with probability PROB. Set HINTS accordingly if edge may be
3612 : devirtualized. AVALS, if non-NULL, describes the context of the call site
3613 : as far as values of parameters are concerened. */
3614 :
3615 : static inline void
3616 208403920 : estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *min_size,
3617 : sreal *time, ipa_call_arg_values *avals,
3618 : ipa_hints *hints)
3619 : {
3620 208403920 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3621 208403920 : int call_size = es->call_stmt_size;
3622 208403920 : int call_time = es->call_stmt_time;
3623 208403920 : int cur_size;
3624 :
3625 204225652 : if ((!e->callee || e->speculative)
3626 208965738 : && estimate_edge_devirt_benefit (e, &call_size, &call_time, avals))
3627 : {
3628 196179 : if (hints && e->maybe_hot_p ())
3629 186933 : *hints |= INLINE_HINT_indirect_call;
3630 : }
3631 208403920 : cur_size = call_size * ipa_fn_summary::size_scale;
3632 208403920 : *size += cur_size;
3633 208403920 : if (min_size)
3634 27734251 : *min_size += cur_size;
3635 208403920 : if (time)
3636 199073528 : *time += ((sreal)call_time) * e->sreal_frequency ();
3637 208403920 : }
3638 :
3639 :
3640 : /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3641 : calls in NODE. POSSIBLE_TRUTHS and AVALS describe the context of the call
3642 : site.
3643 :
3644 : Helper for estimate_calls_size_and_time which does the same but
3645 : (in most cases) faster. */
3646 :
3647 : static void
3648 73511790 : estimate_calls_size_and_time_1 (struct cgraph_node *node, int *size,
3649 : int *min_size, sreal *time,
3650 : ipa_hints *hints,
3651 : clause_t possible_truths,
3652 : ipa_call_arg_values *avals)
3653 : {
3654 73511790 : struct cgraph_edge *e;
3655 351609386 : for (e = node->callees; e; e = e->next_callee)
3656 : {
3657 278097596 : if (!e->inline_failed)
3658 : {
3659 53808483 : gcc_checking_assert (!ipa_call_summaries->get (e));
3660 53808483 : estimate_calls_size_and_time_1 (e->callee, size, min_size, time,
3661 : hints, possible_truths, avals);
3662 :
3663 53808483 : continue;
3664 : }
3665 224289113 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3666 :
3667 : /* Do not care about zero sized builtins. */
3668 224289113 : if (!es->call_stmt_size)
3669 : {
3670 28515658 : gcc_checking_assert (!es->call_stmt_time);
3671 28515658 : continue;
3672 : }
3673 195773455 : if (!es->predicate
3674 195773455 : || es->predicate->evaluate (possible_truths))
3675 : {
3676 : /* Predicates of calls shall not use NOT_CHANGED codes,
3677 : so we do not need to compute probabilities. */
3678 193285911 : estimate_edge_size_and_time (e, size,
3679 193285911 : es->predicate ? NULL : min_size,
3680 : time, avals, hints);
3681 : }
3682 : }
3683 77409600 : for (e = node->indirect_calls; e; e = e->next_callee)
3684 : {
3685 3897810 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3686 3897810 : if (!es->predicate
3687 3897810 : || es->predicate->evaluate (possible_truths))
3688 3868256 : estimate_edge_size_and_time (e, size,
3689 3868256 : es->predicate ? NULL : min_size,
3690 : time, avals, hints);
3691 : }
3692 73511790 : }
3693 :
3694 : /* Populate sum->call_size_time_table for edges from NODE. */
3695 :
3696 : static void
3697 3085758 : summarize_calls_size_and_time (struct cgraph_node *node,
3698 : ipa_fn_summary *sum)
3699 : {
3700 3085758 : struct cgraph_edge *e;
3701 14563666 : for (e = node->callees; e; e = e->next_callee)
3702 : {
3703 11477908 : if (!e->inline_failed)
3704 : {
3705 1382042 : gcc_checking_assert (!ipa_call_summaries->get (e));
3706 1382042 : summarize_calls_size_and_time (e->callee, sum);
3707 1382042 : continue;
3708 : }
3709 10095866 : int size = 0;
3710 10095866 : sreal time = 0;
3711 :
3712 10095866 : estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
3713 :
3714 10095866 : ipa_predicate pred = true;
3715 10095866 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3716 :
3717 10095866 : if (es->predicate)
3718 1810531 : pred = *es->predicate;
3719 10095866 : sum->account_size_time (size, time, pred, pred, true);
3720 : }
3721 3395770 : for (e = node->indirect_calls; e; e = e->next_callee)
3722 : {
3723 310012 : int size = 0;
3724 310012 : sreal time = 0;
3725 :
3726 310012 : estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
3727 310012 : ipa_predicate pred = true;
3728 310012 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3729 :
3730 310012 : if (es->predicate)
3731 78455 : pred = *es->predicate;
3732 310012 : sum->account_size_time (size, time, pred, pred, true);
3733 : }
3734 3085758 : }
3735 :
3736 : /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3737 : calls in NODE. POSSIBLE_TRUTHS and AVALS (the latter if non-NULL) describe
3738 : context of the call site. */
3739 :
3740 : static void
3741 19703328 : estimate_calls_size_and_time (struct cgraph_node *node, int *size,
3742 : int *min_size, sreal *time,
3743 : ipa_hints *hints,
3744 : clause_t possible_truths,
3745 : ipa_call_arg_values *avals)
3746 : {
3747 19703328 : class ipa_fn_summary *sum = ipa_fn_summaries->get (node);
3748 19703328 : bool use_table = true;
3749 :
3750 19703328 : gcc_assert (node->callees || node->indirect_calls);
3751 :
3752 : /* During early inlining we do not calculate info for very
3753 : large functions and thus there is no need for producing
3754 : summaries. */
3755 19703328 : if (!ipa_node_params_sum)
3756 : use_table = false;
3757 : /* Do not calculate summaries for simple wrappers; it is waste
3758 : of memory. */
3759 10751890 : else if (node->callees && !node->indirect_calls
3760 10016881 : && node->callees->inline_failed && !node->callees->next_callee)
3761 : use_table = false;
3762 : /* If there is an indirect edge that may be optimized, we need
3763 : to go the slow way. */
3764 8619403 : else if (avals
3765 8619403 : && (avals->m_known_vals.length ()
3766 3144542 : || avals->m_known_contexts.length ()
3767 3009206 : || avals->m_known_aggs.length ()))
3768 : {
3769 4113021 : ipa_node_params *params_summary = ipa_node_params_sum->get (node);
3770 4113021 : unsigned int nargs = params_summary
3771 4113021 : ? ipa_get_param_count (params_summary) : 0;
3772 :
3773 13816153 : for (unsigned int i = 0; i < nargs && use_table; i++)
3774 : {
3775 9703132 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3776 9703132 : && (avals->safe_sval_at (i)
3777 256749 : || (ipa_argagg_value_list (avals).value_for_index_p (i))))
3778 : use_table = false;
3779 9607737 : else if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3780 9607737 : && (avals->m_known_contexts.length () > i
3781 9856371 : && !avals->m_known_contexts[i].useless_p ()))
3782 : use_table = false;
3783 : }
3784 : }
3785 :
3786 : /* Fast path is via the call size time table. */
3787 4113021 : if (use_table)
3788 : {
3789 : /* Build summary if it is absent. */
3790 8370940 : if (!sum->call_size_time_table.length ())
3791 : {
3792 859841 : ipa_predicate true_pred = true;
3793 859841 : sum->account_size_time (0, 0, true_pred, true_pred, true);
3794 859841 : summarize_calls_size_and_time (node, sum);
3795 : }
3796 :
3797 8370940 : int old_size = *size;
3798 8370940 : sreal old_time = time ? *time : 0;
3799 :
3800 8370940 : if (min_size)
3801 8370940 : *min_size += sum->call_size_time_table[0].size;
3802 :
3803 : unsigned int i;
3804 : size_time_entry *e;
3805 :
3806 : /* Walk the table and account sizes and times. */
3807 22284690 : for (i = 0; sum->call_size_time_table.iterate (i, &e);
3808 : i++)
3809 13913750 : if (e->exec_predicate.evaluate (possible_truths))
3810 : {
3811 12853388 : *size += e->size;
3812 12853388 : if (time)
3813 10814278 : *time += e->time;
3814 : }
3815 :
3816 : /* Be careful and see if both methods agree. */
3817 21 : if ((flag_checking || dump_file)
3818 : /* Do not try to sanity check when we know we lost some
3819 : precision. */
3820 8370940 : && sum->call_size_time_table.length ()
3821 : < ipa_fn_summary::max_size_time_table_size)
3822 : {
3823 8370919 : estimate_calls_size_and_time_1 (node, &old_size, NULL, &old_time, NULL,
3824 : possible_truths, avals);
3825 8370919 : gcc_assert (*size == old_size);
3826 15396544 : if (time && (*time - old_time > 1 || *time - old_time < -1)
3827 8370948 : && dump_file)
3828 0 : fprintf (dump_file, "Time mismatch in call summary %f!=%f\n",
3829 : old_time.to_double (),
3830 : time->to_double ());
3831 : }
3832 : }
3833 : /* Slow path by walking all edges. */
3834 : else
3835 11332388 : estimate_calls_size_and_time_1 (node, size, min_size, time, hints,
3836 : possible_truths, avals);
3837 19703328 : }
3838 :
3839 : /* Main constructor for ipa call context. Memory allocation of ARG_VALUES
3840 : is owned by the caller. INLINE_PARAM_SUMMARY is also owned by the
3841 : caller. */
3842 :
3843 18903831 : ipa_call_context::ipa_call_context (cgraph_node *node, clause_t possible_truths,
3844 : clause_t nonspec_possible_truths,
3845 : vec<inline_param_summary>
3846 : inline_param_summary,
3847 18903831 : ipa_auto_call_arg_values *arg_values)
3848 18903831 : : m_node (node), m_possible_truths (possible_truths),
3849 18903831 : m_nonspec_possible_truths (nonspec_possible_truths),
3850 18903831 : m_inline_param_summary (inline_param_summary),
3851 18903831 : m_avals (arg_values)
3852 : {
3853 18903831 : }
3854 :
3855 : /* Set THIS to be a duplicate of CTX. Copy all relevant info. */
3856 :
3857 : void
3858 1866439 : ipa_cached_call_context::duplicate_from (const ipa_call_context &ctx)
3859 : {
3860 1866439 : m_node = ctx.m_node;
3861 1866439 : m_possible_truths = ctx.m_possible_truths;
3862 1866439 : m_nonspec_possible_truths = ctx.m_nonspec_possible_truths;
3863 1866439 : ipa_node_params *params_summary = ipa_node_params_sum->get (m_node);
3864 1866439 : unsigned int nargs = params_summary
3865 1866439 : ? ipa_get_param_count (params_summary) : 0;
3866 :
3867 1866439 : m_inline_param_summary = vNULL;
3868 : /* Copy the info only if there is at least one useful entry. */
3869 1866439 : if (ctx.m_inline_param_summary.exists ())
3870 : {
3871 1655361 : unsigned int n = MIN (ctx.m_inline_param_summary.length (), nargs);
3872 :
3873 3761571 : for (unsigned int i = 0; i < n; i++)
3874 3078106 : if (ipa_is_param_used_by_ipa_predicates (params_summary, i)
3875 4359518 : && !ctx.m_inline_param_summary[i].useless_p ())
3876 : {
3877 971896 : m_inline_param_summary
3878 971896 : = ctx.m_inline_param_summary.copy ();
3879 971896 : break;
3880 : }
3881 : }
3882 1866439 : m_avals.m_known_vals = vNULL;
3883 1866439 : if (ctx.m_avals.m_known_vals.exists ())
3884 : {
3885 1866439 : unsigned int n = MIN (ctx.m_avals.m_known_vals.length (), nargs);
3886 :
3887 4321928 : for (unsigned int i = 0; i < n; i++)
3888 2481887 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3889 2481887 : && ctx.m_avals.m_known_vals[i])
3890 : {
3891 26398 : m_avals.m_known_vals = ctx.m_avals.m_known_vals.copy ();
3892 26398 : break;
3893 : }
3894 : }
3895 :
3896 1866439 : m_avals.m_known_contexts = vNULL;
3897 1866439 : if (ctx.m_avals.m_known_contexts.exists ())
3898 : {
3899 1866439 : unsigned int n = MIN (ctx.m_avals.m_known_contexts.length (), nargs);
3900 :
3901 1868385 : for (unsigned int i = 0; i < n; i++)
3902 26501 : if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3903 26501 : && !ctx.m_avals.m_known_contexts[i].useless_p ())
3904 : {
3905 24555 : m_avals.m_known_contexts = ctx.m_avals.m_known_contexts.copy ();
3906 24555 : break;
3907 : }
3908 : }
3909 :
3910 1866439 : m_avals.m_known_aggs = vNULL;
3911 1866439 : if (ctx.m_avals.m_known_aggs.exists ())
3912 : {
3913 1866439 : const ipa_argagg_value_list avl (&ctx.m_avals);
3914 6169602 : for (unsigned int i = 0; i < nargs; i++)
3915 4306082 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3916 4306082 : && avl.value_for_index_p (i))
3917 : {
3918 2919 : m_avals.m_known_aggs = ctx.m_avals.m_known_aggs.copy ();
3919 2919 : break;
3920 : }
3921 : }
3922 :
3923 1866439 : m_avals.m_known_value_ranges = vNULL;
3924 1866439 : }
3925 :
3926 : /* Release memory used by known_vals/contexts/aggs vectors. and
3927 : inline_param_summary. */
3928 :
3929 : void
3930 3037570 : ipa_cached_call_context::release ()
3931 : {
3932 : /* See if context is initialized at first place. */
3933 3037570 : if (!m_node)
3934 : return;
3935 1866439 : m_avals.m_known_aggs.release ();
3936 1866439 : m_avals.m_known_vals.release ();
3937 1866439 : m_avals.m_known_contexts.release ();
3938 1866439 : m_inline_param_summary.release ();
3939 : }
3940 :
3941 : /* Return true if CTX describes the same call context as THIS. */
3942 :
3943 : bool
3944 6801986 : ipa_call_context::equal_to (const ipa_call_context &ctx)
3945 : {
3946 6801986 : if (m_node != ctx.m_node
3947 5630855 : || m_possible_truths != ctx.m_possible_truths
3948 5064821 : || m_nonspec_possible_truths != ctx.m_nonspec_possible_truths)
3949 : return false;
3950 :
3951 5064821 : ipa_node_params *params_summary = ipa_node_params_sum->get (m_node);
3952 5064821 : unsigned int nargs = params_summary
3953 5064821 : ? ipa_get_param_count (params_summary) : 0;
3954 :
3955 5064821 : if (m_inline_param_summary.exists () || ctx.m_inline_param_summary.exists ())
3956 : {
3957 15231779 : for (unsigned int i = 0; i < nargs; i++)
3958 : {
3959 10494629 : if (!ipa_is_param_used_by_ipa_predicates (params_summary, i))
3960 3337455 : continue;
3961 7157174 : if (i >= m_inline_param_summary.length ()
3962 5171884 : || m_inline_param_summary[i].useless_p ())
3963 : {
3964 2920505 : if (i < ctx.m_inline_param_summary.length ()
3965 2920505 : && !ctx.m_inline_param_summary[i].useless_p ())
3966 : return false;
3967 2875542 : continue;
3968 : }
3969 4236669 : if (i >= ctx.m_inline_param_summary.length ()
3970 4236666 : || ctx.m_inline_param_summary[i].useless_p ())
3971 : {
3972 44622 : if (i < m_inline_param_summary.length ()
3973 44622 : && !m_inline_param_summary[i].useless_p ())
3974 : return false;
3975 0 : continue;
3976 : }
3977 4192047 : if (!m_inline_param_summary[i].equal_to
3978 4192047 : (ctx.m_inline_param_summary[i]))
3979 : return false;
3980 : }
3981 : }
3982 4948848 : if (m_avals.m_known_vals.exists () || ctx.m_avals.m_known_vals.exists ())
3983 : {
3984 15241459 : for (unsigned int i = 0; i < nargs; i++)
3985 : {
3986 10303058 : if (!ipa_is_param_used_by_indirect_call (params_summary, i))
3987 10106334 : continue;
3988 196724 : if (i >= m_avals.m_known_vals.length () || !m_avals.m_known_vals[i])
3989 : {
3990 140621 : if (i < ctx.m_avals.m_known_vals.length ()
3991 140621 : && ctx.m_avals.m_known_vals[i])
3992 : return false;
3993 140540 : continue;
3994 : }
3995 56103 : if (i >= ctx.m_avals.m_known_vals.length ()
3996 56103 : || !ctx.m_avals.m_known_vals[i])
3997 : {
3998 : if (i < m_avals.m_known_vals.length () && m_avals.m_known_vals[i])
3999 : return false;
4000 : continue;
4001 : }
4002 56019 : if (m_avals.m_known_vals[i] != ctx.m_avals.m_known_vals[i])
4003 : return false;
4004 : }
4005 : }
4006 4938401 : if (m_avals.m_known_contexts.exists ()
4007 4938401 : || ctx.m_avals.m_known_contexts.exists ())
4008 : {
4009 15226553 : for (unsigned int i = 0; i < nargs; i++)
4010 : {
4011 10289534 : if (!ipa_is_param_used_by_polymorphic_call (params_summary, i))
4012 10164521 : continue;
4013 125013 : if (i >= m_avals.m_known_contexts.length ()
4014 123846 : || m_avals.m_known_contexts[i].useless_p ())
4015 : {
4016 1167 : if (i < ctx.m_avals.m_known_contexts.length ()
4017 1167 : && !ctx.m_avals.m_known_contexts[i].useless_p ())
4018 : return false;
4019 1158 : continue;
4020 : }
4021 123846 : if (i >= ctx.m_avals.m_known_contexts.length ()
4022 123846 : || ctx.m_avals.m_known_contexts[i].useless_p ())
4023 : {
4024 8 : if (i < m_avals.m_known_contexts.length ()
4025 1866447 : && !m_avals.m_known_contexts[i].useless_p ())
4026 : return false;
4027 0 : continue;
4028 : }
4029 123838 : if (!m_avals.m_known_contexts[i].equal_to
4030 123838 : (ctx.m_avals.m_known_contexts[i]))
4031 : return false;
4032 : }
4033 : }
4034 4937019 : if (m_avals.m_known_aggs.exists () || ctx.m_avals.m_known_aggs.exists ())
4035 : {
4036 : unsigned i = 0, j = 0;
4037 11395758 : while (i < m_avals.m_known_aggs.length ()
4038 5697176 : || j < ctx.m_avals.m_known_aggs.length ())
4039 : {
4040 761629 : if (i >= m_avals.m_known_aggs.length ())
4041 : {
4042 757137 : int idx2 = ctx.m_avals.m_known_aggs[j].index;
4043 757137 : if (ipa_is_param_used_by_indirect_call (params_summary, idx2))
4044 : return false;
4045 756537 : j++;
4046 756537 : continue;
4047 756537 : }
4048 4492 : if (j >= ctx.m_avals.m_known_aggs.length ())
4049 : {
4050 631 : int idx1 = m_avals.m_known_aggs[i].index;
4051 631 : if (ipa_is_param_used_by_indirect_call (params_summary, idx1))
4052 : return false;
4053 8 : i++;
4054 8 : continue;
4055 8 : }
4056 :
4057 3861 : int idx1 = m_avals.m_known_aggs[i].index;
4058 3861 : int idx2 = ctx.m_avals.m_known_aggs[j].index;
4059 3861 : if (idx1 < idx2)
4060 : {
4061 0 : if (ipa_is_param_used_by_indirect_call (params_summary, idx1))
4062 : return false;
4063 0 : i++;
4064 0 : continue;
4065 : }
4066 3861 : if (idx1 > idx2)
4067 : {
4068 0 : if (ipa_is_param_used_by_indirect_call (params_summary, idx2))
4069 : return false;
4070 0 : j++;
4071 0 : continue;
4072 : }
4073 3861 : if (!ipa_is_param_used_by_indirect_call (params_summary, idx1))
4074 : {
4075 476 : i++;
4076 476 : j++;
4077 476 : continue;
4078 : }
4079 :
4080 3385 : if ((m_avals.m_known_aggs[i].unit_offset
4081 3385 : != ctx.m_avals.m_known_aggs[j].unit_offset)
4082 3377 : || (m_avals.m_known_aggs[i].by_ref
4083 3377 : != ctx.m_avals.m_known_aggs[j].by_ref)
4084 6762 : || !operand_equal_p (m_avals.m_known_aggs[i].value,
4085 3377 : ctx.m_avals.m_known_aggs[j].value))
4086 249 : return false;
4087 3136 : i++;
4088 3136 : j++;
4089 : }
4090 : }
4091 : return true;
4092 : }
4093 :
4094 : /* Fill in the selected fields in ESTIMATES with value estimated for call in
4095 : this context. Always compute size and min_size. Only compute time and
4096 : nonspecialized_time if EST_TIMES is true. Only compute hints if EST_HINTS
4097 : is true. */
4098 :
4099 : void
4100 18876297 : ipa_call_context::estimate_size_and_time (ipa_call_estimates *estimates,
4101 : bool est_times, bool est_hints)
4102 : {
4103 18876297 : class ipa_fn_summary *info = ipa_fn_summaries->get (m_node);
4104 18876297 : size_time_entry *e;
4105 18876297 : int size = 0;
4106 18876297 : sreal time = 0;
4107 18876297 : int min_size = 0;
4108 18876297 : ipa_hints hints = 0;
4109 18876297 : sreal loops_with_known_iterations = 0;
4110 18876297 : sreal loops_with_known_strides = 0;
4111 18876297 : int i;
4112 :
4113 18876297 : if (dump_file && (dump_flags & TDF_DETAILS))
4114 : {
4115 1332 : bool found = false;
4116 1332 : fprintf (dump_file, " Estimating body: %s\n"
4117 : " Known to be false: ", m_node->dump_name ());
4118 :
4119 4287 : for (i = ipa_predicate::not_inlined_condition;
4120 8574 : i < (ipa_predicate::first_dynamic_condition
4121 7002 : + (int) vec_safe_length (info->conds)); i++)
4122 2955 : if (!(m_possible_truths & (1 << i)))
4123 : {
4124 1769 : if (found)
4125 504 : fprintf (dump_file, ", ");
4126 1769 : found = true;
4127 1769 : dump_condition (dump_file, info->conds, i);
4128 : }
4129 : }
4130 :
4131 18876297 : if (m_node->callees || m_node->indirect_calls)
4132 24717702 : estimate_calls_size_and_time (m_node, &size, &min_size,
4133 : est_times ? &time : NULL,
4134 : est_hints ? &hints : NULL, m_possible_truths,
4135 : &m_avals);
4136 :
4137 18876297 : sreal nonspecialized_time = time;
4138 :
4139 18876297 : min_size += info->size_time_table[0].size;
4140 129909740 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
4141 : {
4142 111033443 : bool exec = e->exec_predicate.evaluate (m_nonspec_possible_truths);
4143 :
4144 : /* Because predicates are conservative, it can happen that nonconst is 1
4145 : but exec is 0. */
4146 111033443 : if (exec)
4147 : {
4148 108600893 : bool nonconst = e->nonconst_predicate.evaluate (m_possible_truths);
4149 :
4150 108600893 : gcc_checking_assert (e->time >= 0);
4151 108600893 : gcc_checking_assert (time >= 0);
4152 :
4153 : /* We compute specialized size only because size of nonspecialized
4154 : copy is context independent.
4155 :
4156 : The difference between nonspecialized execution and specialized is
4157 : that nonspecialized is not going to have optimized out computations
4158 : known to be constant in a specialized setting. */
4159 108600893 : if (nonconst)
4160 54607513 : size += e->size;
4161 108600893 : if (!est_times)
4162 55379595 : continue;
4163 53221298 : nonspecialized_time += e->time;
4164 53221298 : if (!nonconst)
4165 : ;
4166 25556094 : else if (!m_inline_param_summary.exists ())
4167 : {
4168 2522953 : if (nonconst)
4169 2522953 : time += e->time;
4170 : }
4171 : else
4172 : {
4173 23033141 : int prob = e->nonconst_predicate.probability
4174 23033141 : (info->conds, m_possible_truths,
4175 : m_inline_param_summary);
4176 23033141 : gcc_checking_assert (prob >= 0);
4177 23033141 : gcc_checking_assert (prob <= REG_BR_PROB_BASE);
4178 23033141 : if (prob == REG_BR_PROB_BASE)
4179 19330768 : time += e->time;
4180 : else
4181 3702373 : time += e->time * prob / REG_BR_PROB_BASE;
4182 : }
4183 53221298 : gcc_checking_assert (time >= 0);
4184 : }
4185 : }
4186 18876297 : gcc_checking_assert (info->size_time_table[0].exec_predicate == true);
4187 18876297 : gcc_checking_assert (info->size_time_table[0].nonconst_predicate == true);
4188 18876297 : gcc_checking_assert (min_size >= 0);
4189 18876297 : gcc_checking_assert (size >= 0);
4190 18876297 : gcc_checking_assert (time >= 0);
4191 : /* nonspecialized_time should be always bigger than specialized time.
4192 : Roundoff issues however may get into the way. */
4193 18876297 : gcc_checking_assert ((nonspecialized_time - time * 99 / 100) >= -1);
4194 :
4195 : /* Roundoff issues may make specialized time bigger than nonspecialized
4196 : time. We do not really want that to happen because some heuristics
4197 : may get confused by seeing negative speedups. */
4198 18876297 : if (time > nonspecialized_time)
4199 0 : time = nonspecialized_time;
4200 :
4201 18876297 : if (est_hints)
4202 : {
4203 7060154 : if (info->scc_no)
4204 226831 : hints |= INLINE_HINT_in_scc;
4205 7060154 : if (DECL_DECLARED_INLINE_P (m_node->decl))
4206 4431585 : hints |= INLINE_HINT_declared_inline;
4207 7060154 : if (info->builtin_constant_p_parms.length ()
4208 8895 : && DECL_DECLARED_INLINE_P (m_node->decl))
4209 8837 : hints |= INLINE_HINT_builtin_constant_p;
4210 :
4211 : ipa_freqcounting_predicate *fcp;
4212 7609978 : for (i = 0; vec_safe_iterate (info->loop_iterations, i, &fcp); i++)
4213 549824 : if (!fcp->predicate->evaluate (m_possible_truths))
4214 : {
4215 369916 : hints |= INLINE_HINT_loop_iterations;
4216 369916 : loops_with_known_iterations += fcp->freq;
4217 : }
4218 7060154 : estimates->loops_with_known_iterations = loops_with_known_iterations;
4219 :
4220 7346106 : for (i = 0; vec_safe_iterate (info->loop_strides, i, &fcp); i++)
4221 285952 : if (!fcp->predicate->evaluate (m_possible_truths))
4222 : {
4223 261444 : hints |= INLINE_HINT_loop_stride;
4224 261444 : loops_with_known_strides += fcp->freq;
4225 : }
4226 7060154 : estimates->loops_with_known_strides = loops_with_known_strides;
4227 : }
4228 :
4229 18876297 : size = RDIV (size, ipa_fn_summary::size_scale);
4230 18876297 : min_size = RDIV (min_size, ipa_fn_summary::size_scale);
4231 :
4232 18876297 : if (dump_file && (dump_flags & TDF_DETAILS))
4233 : {
4234 1332 : fprintf (dump_file, "\n size:%i", (int) size);
4235 1332 : if (est_times)
4236 1090 : fprintf (dump_file, " time:%f nonspec time:%f",
4237 : time.to_double (), nonspecialized_time.to_double ());
4238 1332 : if (est_hints)
4239 1090 : fprintf (dump_file, " loops with known iterations:%f "
4240 : "known strides:%f", loops_with_known_iterations.to_double (),
4241 : loops_with_known_strides.to_double ());
4242 1332 : fprintf (dump_file, "\n");
4243 : }
4244 18876297 : if (est_times)
4245 : {
4246 7060154 : estimates->time = time;
4247 7060154 : estimates->nonspecialized_time = nonspecialized_time;
4248 : }
4249 18876297 : estimates->size = size;
4250 18876297 : estimates->min_size = min_size;
4251 18876297 : if (est_hints)
4252 7060154 : estimates->hints = hints;
4253 18876297 : return;
4254 : }
4255 :
4256 :
4257 : /* Estimate size and time needed to execute callee of EDGE assuming that
4258 : parameters known to be constant at caller of EDGE are propagated.
4259 : KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
4260 : and types for parameters. */
4261 :
4262 : void
4263 281901 : estimate_ipcp_clone_size_and_time (struct cgraph_node *node,
4264 : ipa_auto_call_arg_values *avals,
4265 : ipa_call_estimates *estimates)
4266 : {
4267 281901 : clause_t clause, nonspec_clause;
4268 :
4269 281901 : evaluate_conditions_for_known_args (node, false, avals, &clause,
4270 : &nonspec_clause, NULL);
4271 281901 : ipa_call_context ctx (node, clause, nonspec_clause, vNULL, avals);
4272 281901 : ctx.estimate_size_and_time (estimates);
4273 281901 : }
4274 :
4275 : /* Return stack frame offset where frame of NODE is supposed to start inside
4276 : of the function it is inlined to.
4277 : Return 0 for functions that are not inlined. */
4278 :
4279 : HOST_WIDE_INT
4280 5021703 : ipa_get_stack_frame_offset (struct cgraph_node *node)
4281 : {
4282 5021703 : HOST_WIDE_INT offset = 0;
4283 5021703 : if (!node->inlined_to)
4284 : return 0;
4285 3896768 : node = node->callers->caller;
4286 5055800 : while (true)
4287 : {
4288 5055800 : offset += ipa_size_summaries->get (node)->estimated_self_stack_size;
4289 4476284 : if (!node->inlined_to)
4290 : return offset;
4291 579516 : node = node->callers->caller;
4292 : }
4293 : }
4294 :
4295 :
4296 : /* Update summary information of inline clones after inlining.
4297 : Compute peak stack usage. */
4298 :
4299 : static void
4300 4574390 : inline_update_callee_summaries (struct cgraph_node *node, int depth)
4301 : {
4302 4574390 : struct cgraph_edge *e;
4303 :
4304 4574390 : ipa_propagate_frequency (node);
4305 8830776 : for (e = node->callees; e; e = e->next_callee)
4306 : {
4307 4256386 : if (!e->inline_failed)
4308 678176 : inline_update_callee_summaries (e->callee, depth);
4309 : else
4310 3578210 : ipa_call_summaries->get (e)->loop_depth += depth;
4311 : }
4312 4675114 : for (e = node->indirect_calls; e; e = e->next_callee)
4313 100724 : ipa_call_summaries->get (e)->loop_depth += depth;
4314 4574390 : }
4315 :
4316 : /* Update change_prob and points_to_local_or_readonly_memory of EDGE after
4317 : INLINED_EDGE has been inlined.
4318 :
4319 : When function A is inlined in B and A calls C with parameter that
4320 : changes with probability PROB1 and C is known to be passthrough
4321 : of argument if B that change with probability PROB2, the probability
4322 : of change is now PROB1*PROB2. */
4323 :
4324 : static void
4325 3679247 : remap_edge_params (struct cgraph_edge *inlined_edge,
4326 : struct cgraph_edge *edge)
4327 : {
4328 3679247 : if (ipa_node_params_sum)
4329 : {
4330 2647643 : int i;
4331 2647643 : ipa_edge_args *args = ipa_edge_args_sum->get (edge);
4332 2647643 : if (!args)
4333 : return;
4334 1485836 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
4335 1485836 : class ipa_call_summary *inlined_es
4336 1485836 : = ipa_call_summaries->get (inlined_edge);
4337 :
4338 1485836 : if (es->param.length () == 0)
4339 : return;
4340 :
4341 8648128 : for (i = 0; i < ipa_get_cs_argument_count (args); i++)
4342 : {
4343 2913435 : struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
4344 2913435 : if (jfunc->type == IPA_JF_PASS_THROUGH
4345 2132316 : || jfunc->type == IPA_JF_ANCESTOR)
4346 : {
4347 912983 : int id = jfunc->type == IPA_JF_PASS_THROUGH
4348 912983 : ? ipa_get_jf_pass_through_formal_id (jfunc)
4349 131864 : : ipa_get_jf_ancestor_formal_id (jfunc);
4350 1825874 : if (id < (int) inlined_es->param.length ())
4351 : {
4352 912883 : int prob1 = es->param[i].change_prob;
4353 912883 : int prob2 = inlined_es->param[id].change_prob;
4354 912883 : int prob = combine_probabilities (prob1, prob2);
4355 :
4356 912883 : if (prob1 && prob2 && !prob)
4357 912883 : prob = 1;
4358 :
4359 912883 : es->param[i].change_prob = prob;
4360 :
4361 1825766 : if (inlined_es
4362 912883 : ->param[id].points_to_local_or_readonly_memory)
4363 199338 : es->param[i].points_to_local_or_readonly_memory = true;
4364 1825766 : if (inlined_es
4365 912883 : ->param[id].points_to_possible_sra_candidate)
4366 162455 : es->param[i].points_to_possible_sra_candidate = true;
4367 : }
4368 912983 : if (!es->param[i].points_to_local_or_readonly_memory
4369 : && jfunc->type == IPA_JF_CONST
4370 : && points_to_local_or_readonly_memory_p
4371 : (ipa_get_jf_constant (jfunc)))
4372 : es->param[i].points_to_local_or_readonly_memory = true;
4373 : }
4374 : }
4375 : }
4376 : }
4377 :
4378 : /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
4379 :
4380 : Remap predicates of callees of NODE. Rest of arguments match
4381 : remap_predicate.
4382 :
4383 : Also update change probabilities. */
4384 :
4385 : static void
4386 4574390 : remap_edge_summaries (struct cgraph_edge *inlined_edge,
4387 : struct cgraph_node *node,
4388 : class ipa_fn_summary *info,
4389 : class ipa_node_params *params_summary,
4390 : class ipa_fn_summary *callee_info,
4391 : const vec<int> &operand_map,
4392 : const vec<HOST_WIDE_INT> &offset_map,
4393 : clause_t possible_truths,
4394 : ipa_predicate *toplev_predicate)
4395 : {
4396 4574390 : struct cgraph_edge *e, *next;
4397 8830351 : for (e = node->callees; e; e = next)
4398 : {
4399 4255961 : ipa_predicate p;
4400 4255961 : next = e->next_callee;
4401 :
4402 4255961 : if (e->inline_failed)
4403 : {
4404 3577785 : class ipa_call_summary *es = ipa_call_summaries->get (e);
4405 3577785 : remap_edge_params (inlined_edge, e);
4406 :
4407 3577785 : if (es->predicate)
4408 : {
4409 1236674 : p = es->predicate->remap_after_inlining
4410 1236674 : (info, params_summary,
4411 : callee_info, operand_map,
4412 : offset_map, possible_truths,
4413 : *toplev_predicate);
4414 1236674 : edge_set_predicate (e, &p);
4415 : }
4416 : else
4417 2341111 : edge_set_predicate (e, toplev_predicate);
4418 : }
4419 : else
4420 678176 : remap_edge_summaries (inlined_edge, e->callee, info,
4421 : params_summary, callee_info,
4422 : operand_map, offset_map, possible_truths,
4423 : toplev_predicate);
4424 : }
4425 4675852 : for (e = node->indirect_calls; e; e = next)
4426 : {
4427 101462 : class ipa_call_summary *es = ipa_call_summaries->get (e);
4428 101462 : ipa_predicate p;
4429 101462 : next = e->next_callee;
4430 :
4431 101462 : remap_edge_params (inlined_edge, e);
4432 101462 : if (es->predicate)
4433 : {
4434 25987 : p = es->predicate->remap_after_inlining
4435 25987 : (info, params_summary,
4436 : callee_info, operand_map, offset_map,
4437 : possible_truths, *toplev_predicate);
4438 25987 : edge_set_predicate (e, &p);
4439 : }
4440 : else
4441 75475 : edge_set_predicate (e, toplev_predicate);
4442 : }
4443 4574390 : }
4444 :
4445 : /* Run remap_after_inlining on each predicate in V. */
4446 :
4447 : static void
4448 7792428 : remap_freqcounting_predicate (class ipa_fn_summary *info,
4449 : class ipa_node_params *params_summary,
4450 : class ipa_fn_summary *callee_info,
4451 : vec<ipa_freqcounting_predicate, va_gc> *v,
4452 : const vec<int> &operand_map,
4453 : const vec<HOST_WIDE_INT> &offset_map,
4454 : clause_t possible_truths,
4455 : ipa_predicate *toplev_predicate)
4456 :
4457 : {
4458 7792428 : ipa_freqcounting_predicate *fcp;
4459 7819768 : for (int i = 0; vec_safe_iterate (v, i, &fcp); i++)
4460 : {
4461 27340 : ipa_predicate p
4462 27340 : = fcp->predicate->remap_after_inlining (info, params_summary,
4463 : callee_info, operand_map,
4464 : offset_map, possible_truths,
4465 : *toplev_predicate);
4466 38186 : if (p != false && p != true)
4467 2795 : *fcp->predicate &= p;
4468 : }
4469 7792428 : }
4470 :
4471 : /* We inlined EDGE. Update summary of the function we inlined into. */
4472 :
4473 : void
4474 3896214 : ipa_merge_fn_summary_after_inlining (struct cgraph_edge *edge)
4475 : {
4476 3896214 : ipa_fn_summary *callee_info = ipa_fn_summaries->get (edge->callee);
4477 3624805 : struct cgraph_node *to = (edge->caller->inlined_to
4478 3896214 : ? edge->caller->inlined_to : edge->caller);
4479 3896214 : class ipa_fn_summary *info = ipa_fn_summaries->get (to);
4480 3896214 : clause_t clause = 0; /* not_inline is known to be false. */
4481 3896214 : size_time_entry *e;
4482 3896214 : auto_vec<int, 8> operand_map;
4483 3896214 : auto_vec<HOST_WIDE_INT, 8> offset_map;
4484 3896214 : int i;
4485 3896214 : ipa_predicate toplev_predicate;
4486 3896214 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
4487 3896214 : ipa_node_params *params_summary = (ipa_node_params_sum
4488 3896214 : ? ipa_node_params_sum->get (to) : NULL);
4489 :
4490 3896214 : if (es->predicate)
4491 335221 : toplev_predicate = *es->predicate;
4492 : else
4493 3560993 : toplev_predicate = true;
4494 :
4495 3896214 : info->fp_expressions |= callee_info->fp_expressions;
4496 3896214 : info->target_info |= callee_info->target_info;
4497 :
4498 3896214 : if (callee_info->conds)
4499 : {
4500 2837041 : ipa_auto_call_arg_values avals;
4501 2837041 : evaluate_properties_for_edge (edge, true, &clause, NULL, &avals, false);
4502 2837041 : }
4503 3896214 : if (ipa_node_params_sum && callee_info->conds)
4504 : {
4505 784181 : ipa_edge_args *args = ipa_edge_args_sum->get (edge);
4506 784181 : int count = args ? ipa_get_cs_argument_count (args) : 0;
4507 784039 : int i;
4508 :
4509 784039 : if (count)
4510 : {
4511 784039 : operand_map.safe_grow_cleared (count, true);
4512 784039 : offset_map.safe_grow_cleared (count, true);
4513 : }
4514 2519886 : for (i = 0; i < count; i++)
4515 : {
4516 1735705 : struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
4517 1735705 : int map = -1;
4518 :
4519 : /* TODO: handle non-NOPs when merging. */
4520 1735705 : if (jfunc->type == IPA_JF_PASS_THROUGH)
4521 : {
4522 279921 : if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
4523 277061 : map = ipa_get_jf_pass_through_formal_id (jfunc);
4524 279921 : if (!ipa_get_jf_pass_through_agg_preserved (jfunc))
4525 187610 : offset_map[i] = -1;
4526 : }
4527 1455784 : else if (jfunc->type == IPA_JF_ANCESTOR)
4528 : {
4529 87064 : HOST_WIDE_INT offset = ipa_get_jf_ancestor_offset (jfunc);
4530 87064 : if (offset >= 0 && offset < INT_MAX)
4531 : {
4532 87064 : map = ipa_get_jf_ancestor_formal_id (jfunc);
4533 87064 : if (!ipa_get_jf_ancestor_agg_preserved (jfunc))
4534 53606 : offset = -1;
4535 87064 : offset_map[i] = offset;
4536 : }
4537 : }
4538 1735705 : operand_map[i] = map;
4539 3132023 : gcc_assert (map < ipa_get_param_count (params_summary));
4540 : }
4541 :
4542 : int ip;
4543 786762 : for (i = 0; callee_info->builtin_constant_p_parms.iterate (i, &ip); i++)
4544 2581 : if (ip < count && operand_map[ip] >= 0)
4545 36 : add_builtin_constant_p_parm (info, operand_map[ip]);
4546 : }
4547 3896214 : sreal freq = edge->sreal_frequency ();
4548 21534369 : for (i = 0; callee_info->size_time_table.iterate (i, &e); i++)
4549 : {
4550 17638155 : ipa_predicate p;
4551 17638155 : p = e->exec_predicate.remap_after_inlining
4552 17638155 : (info, params_summary,
4553 : callee_info, operand_map,
4554 : offset_map, clause,
4555 : toplev_predicate);
4556 17638155 : ipa_predicate nonconstp;
4557 17638155 : nonconstp = e->nonconst_predicate.remap_after_inlining
4558 17638155 : (info, params_summary,
4559 : callee_info, operand_map,
4560 : offset_map, clause,
4561 : toplev_predicate);
4562 26627371 : if (p != false && nonconstp != false)
4563 : {
4564 8679423 : sreal add_time = ((sreal)e->time * freq);
4565 8679423 : int prob = e->nonconst_predicate.probability (callee_info->conds,
4566 : clause, es->param);
4567 8679423 : if (prob != REG_BR_PROB_BASE)
4568 1070797 : add_time = add_time * prob / REG_BR_PROB_BASE;
4569 1070797 : if (prob != REG_BR_PROB_BASE
4570 1070797 : && dump_file && (dump_flags & TDF_DETAILS))
4571 : {
4572 13 : fprintf (dump_file, "\t\tScaling time by probability:%f\n",
4573 13 : (double) prob / REG_BR_PROB_BASE);
4574 : }
4575 8679423 : info->account_size_time (e->size, add_time, p, nonconstp);
4576 : }
4577 : }
4578 3896214 : remap_edge_summaries (edge, edge->callee, info, params_summary,
4579 : callee_info, operand_map,
4580 : offset_map, clause, &toplev_predicate);
4581 3896214 : remap_freqcounting_predicate (info, params_summary, callee_info,
4582 : info->loop_iterations, operand_map,
4583 : offset_map, clause, &toplev_predicate);
4584 3896214 : remap_freqcounting_predicate (info, params_summary, callee_info,
4585 : info->loop_strides, operand_map,
4586 : offset_map, clause, &toplev_predicate);
4587 :
4588 3896214 : HOST_WIDE_INT stack_frame_offset = ipa_get_stack_frame_offset (edge->callee);
4589 3896214 : HOST_WIDE_INT peak = stack_frame_offset + callee_info->estimated_stack_size;
4590 :
4591 3896214 : if (info->estimated_stack_size < peak)
4592 124686 : info->estimated_stack_size = peak;
4593 :
4594 3896214 : inline_update_callee_summaries (edge->callee, es->loop_depth);
4595 3896214 : if (info->call_size_time_table.length ())
4596 : {
4597 843875 : int edge_size = 0;
4598 843875 : sreal edge_time = 0;
4599 :
4600 843875 : estimate_edge_size_and_time (edge, &edge_size, NULL, &edge_time, NULL, 0);
4601 : /* Unaccount size and time of the optimized out call. */
4602 843875 : info->account_size_time (-edge_size, -edge_time,
4603 843875 : es->predicate ? *es->predicate : true,
4604 843875 : es->predicate ? *es->predicate : true,
4605 : true);
4606 : /* Account new calls. */
4607 843875 : summarize_calls_size_and_time (edge->callee, info);
4608 : }
4609 :
4610 : /* Free summaries that are not maintained for inline clones/edges. */
4611 3896214 : ipa_call_summaries->remove (edge);
4612 3896214 : ipa_fn_summaries->remove (edge->callee);
4613 3896214 : ipa_remove_from_growth_caches (edge);
4614 3896214 : }
4615 :
4616 : /* For performance reasons ipa_merge_fn_summary_after_inlining is not updating
4617 : overall size and time. Recompute it.
4618 : If RESET is true also recompute call_time_size_table. */
4619 :
4620 : void
4621 9317064 : ipa_update_overall_fn_summary (struct cgraph_node *node, bool reset)
4622 : {
4623 9317064 : class ipa_fn_summary *info = ipa_fn_summaries->get (node);
4624 9317064 : class ipa_size_summary *size_info = ipa_size_summaries->get (node);
4625 9317064 : size_time_entry *e;
4626 9317064 : int i;
4627 :
4628 9317064 : size_info->size = 0;
4629 9317064 : info->time = 0;
4630 47222915 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
4631 : {
4632 37905851 : size_info->size += e->size;
4633 37905851 : info->time += e->time;
4634 : }
4635 9317064 : info->min_size = info->size_time_table[0].size;
4636 9317064 : if (reset)
4637 8431438 : info->call_size_time_table.release ();
4638 9317064 : if (node->callees || node->indirect_calls)
4639 7089338 : estimate_calls_size_and_time (node, &size_info->size, &info->min_size,
4640 : &info->time, NULL,
4641 : ~(clause_t) (1 << ipa_predicate::false_condition),
4642 : NULL);
4643 9317064 : size_info->size = RDIV (size_info->size, ipa_fn_summary::size_scale);
4644 9317064 : info->min_size = RDIV (info->min_size, ipa_fn_summary::size_scale);
4645 9317064 : }
4646 :
4647 :
4648 : /* This function performs intraprocedural analysis in NODE that is required to
4649 : inline indirect calls. */
4650 :
4651 : static void
4652 1368862 : inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
4653 : {
4654 1368862 : ipa_analyze_node (node);
4655 1368862 : if (dump_file && (dump_flags & TDF_DETAILS))
4656 : {
4657 14 : ipa_print_node_params (dump_file, node);
4658 14 : ipa_print_node_jump_functions (dump_file, node);
4659 : }
4660 1368862 : }
4661 :
4662 :
4663 : /* Note function body size. */
4664 :
4665 : void
4666 1379552 : inline_analyze_function (struct cgraph_node *node)
4667 : {
4668 1379552 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
4669 :
4670 1379552 : if (dump_file)
4671 94 : fprintf (dump_file, "\nAnalyzing function: %s\n", node->dump_name ());
4672 1379552 : if (opt_for_fn (node->decl, optimize) && !node->thunk)
4673 1368862 : inline_indirect_intraprocedural_analysis (node);
4674 1379552 : compute_fn_summary (node, false);
4675 1379552 : if (!optimize)
4676 : {
4677 9418 : struct cgraph_edge *e;
4678 22138 : for (e = node->callees; e; e = e->next_callee)
4679 12720 : e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4680 9489 : for (e = node->indirect_calls; e; e = e->next_callee)
4681 71 : e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4682 : }
4683 :
4684 1379552 : pop_cfun ();
4685 1379552 : }
4686 :
4687 :
4688 : /* Called when new function is inserted to callgraph late. */
4689 :
4690 : void
4691 19215 : ipa_fn_summary_t::insert (struct cgraph_node *node, ipa_fn_summary *)
4692 : {
4693 19215 : inline_analyze_function (node);
4694 19215 : }
4695 :
4696 : /* Note function body size. */
4697 :
4698 : static void
4699 231782 : ipa_fn_summary_generate (void)
4700 : {
4701 231782 : struct cgraph_node *node;
4702 :
4703 2085564 : FOR_EACH_DEFINED_FUNCTION (node)
4704 1853782 : if (DECL_STRUCT_FUNCTION (node->decl))
4705 1840589 : node->versionable = tree_versionable_function_p (node->decl);
4706 :
4707 231782 : ipa_fn_summary_alloc ();
4708 :
4709 231782 : ipa_fn_summaries->enable_insertion_hook ();
4710 :
4711 231782 : ipa_register_cgraph_hooks ();
4712 :
4713 2085564 : FOR_EACH_DEFINED_FUNCTION (node)
4714 1853782 : if (!node->alias
4715 1853782 : && (flag_generate_lto || flag_generate_offload|| flag_wpa
4716 1663568 : || opt_for_fn (node->decl, optimize)))
4717 1342256 : inline_analyze_function (node);
4718 231782 : }
4719 :
4720 :
4721 : /* Write inline summary for edge E to OB. */
4722 :
4723 : static void
4724 347224 : read_ipa_call_summary (class lto_input_block *ib, struct cgraph_edge *e,
4725 : bool prevails)
4726 : {
4727 347224 : class ipa_call_summary *es = prevails
4728 347224 : ? ipa_call_summaries->get_create (e) : NULL;
4729 347224 : ipa_predicate p;
4730 347224 : int length, i;
4731 :
4732 347224 : int size = streamer_read_uhwi (ib);
4733 347224 : int time = streamer_read_uhwi (ib);
4734 347224 : int depth = streamer_read_uhwi (ib);
4735 :
4736 347224 : if (es)
4737 : {
4738 347175 : es->call_stmt_size = size;
4739 347175 : es->call_stmt_time = time;
4740 347175 : es->loop_depth = depth;
4741 : }
4742 :
4743 347224 : bitpack_d bp = streamer_read_bitpack (ib);
4744 347224 : if (es)
4745 347175 : es->is_return_callee_uncaptured = bp_unpack_value (&bp, 1);
4746 : else
4747 49 : bp_unpack_value (&bp, 1);
4748 :
4749 347224 : p.stream_in (ib);
4750 347224 : if (es)
4751 347175 : edge_set_predicate (e, &p);
4752 347224 : length = streamer_read_uhwi (ib);
4753 347224 : if (length && es
4754 347224 : && (e->possibly_call_in_translation_unit_p ()
4755 : /* Also stream in jump functions to builtins in hope that they
4756 : will get fnspecs. */
4757 118724 : || fndecl_built_in_p (e->callee->decl, BUILT_IN_NORMAL)))
4758 : {
4759 232318 : es->param.safe_grow_cleared (length, true);
4760 799636 : for (i = 0; i < length; i++)
4761 : {
4762 567318 : es->param[i].change_prob = streamer_read_uhwi (ib);
4763 567318 : bitpack_d bp = streamer_read_bitpack (ib);
4764 1701954 : es->param[i].points_to_local_or_readonly_memory
4765 567318 : = bp_unpack_value (&bp, 1);
4766 1701954 : es->param[i].points_to_possible_sra_candidate
4767 567318 : = bp_unpack_value (&bp, 1);
4768 : }
4769 : }
4770 : else
4771 : {
4772 137679 : for (i = 0; i < length; i++)
4773 : {
4774 22773 : streamer_read_uhwi (ib);
4775 22773 : streamer_read_uhwi (ib);
4776 : }
4777 : }
4778 347224 : }
4779 :
4780 :
4781 : /* Stream in inline summaries from the section. */
4782 :
4783 : static void
4784 13339 : inline_read_section (struct lto_file_decl_data *file_data, const char *data,
4785 : size_t len)
4786 : {
4787 13339 : const struct lto_function_header *header =
4788 : (const struct lto_function_header *) data;
4789 13339 : const int cfg_offset = sizeof (struct lto_function_header);
4790 13339 : const int main_offset = cfg_offset + header->cfg_size;
4791 13339 : const int string_offset = main_offset + header->main_size;
4792 13339 : class data_in *data_in;
4793 13339 : unsigned int i, count2, j;
4794 13339 : unsigned int f_count;
4795 :
4796 13339 : lto_input_block ib ((const char *) data + main_offset, header->main_size,
4797 13339 : file_data);
4798 :
4799 13339 : data_in =
4800 26678 : lto_data_in_create (file_data, (const char *) data + string_offset,
4801 13339 : header->string_size, vNULL);
4802 13339 : f_count = streamer_read_uhwi (&ib);
4803 99460 : for (i = 0; i < f_count; i++)
4804 : {
4805 86121 : unsigned int index;
4806 86121 : struct cgraph_node *node;
4807 86121 : class ipa_fn_summary *info;
4808 86121 : class ipa_node_params *params_summary;
4809 86121 : class ipa_size_summary *size_info;
4810 86121 : lto_symtab_encoder_t encoder;
4811 86121 : struct bitpack_d bp;
4812 86121 : struct cgraph_edge *e;
4813 86121 : ipa_predicate p;
4814 :
4815 86121 : index = streamer_read_uhwi (&ib);
4816 86121 : encoder = file_data->symtab_node_encoder;
4817 86121 : node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
4818 : index));
4819 86121 : info = node->prevailing_p () ? ipa_fn_summaries->get_create (node) : NULL;
4820 86121 : params_summary = node->prevailing_p ()
4821 86121 : ? ipa_node_params_sum->get (node) : NULL;
4822 86121 : size_info = node->prevailing_p ()
4823 86121 : ? ipa_size_summaries->get_create (node) : NULL;
4824 :
4825 86121 : int stack_size = streamer_read_uhwi (&ib);
4826 86121 : int size = streamer_read_uhwi (&ib);
4827 86121 : sreal time = sreal::stream_in (&ib);
4828 :
4829 86121 : if (info)
4830 : {
4831 86063 : info->estimated_stack_size
4832 86063 : = size_info->estimated_self_stack_size = stack_size;
4833 86063 : size_info->size = size_info->self_size = size;
4834 86063 : info->time = time;
4835 : }
4836 :
4837 86121 : bp = streamer_read_bitpack (&ib);
4838 86121 : if (info)
4839 : {
4840 86063 : info->inlinable = bp_unpack_value (&bp, 1);
4841 86063 : info->fp_expressions = bp_unpack_value (&bp, 1);
4842 86063 : if (!lto_stream_offload_p)
4843 86063 : info->target_info = streamer_read_uhwi (&ib);
4844 : }
4845 : else
4846 : {
4847 58 : bp_unpack_value (&bp, 1);
4848 58 : bp_unpack_value (&bp, 1);
4849 58 : if (!lto_stream_offload_p)
4850 58 : streamer_read_uhwi (&ib);
4851 : }
4852 :
4853 86121 : count2 = streamer_read_uhwi (&ib);
4854 86121 : gcc_assert (!info || !info->conds);
4855 86063 : if (info)
4856 86063 : vec_safe_reserve_exact (info->conds, count2);
4857 162986 : for (j = 0; j < count2; j++)
4858 : {
4859 76865 : struct condition c;
4860 76865 : unsigned int k, count3;
4861 76865 : c.operand_num = streamer_read_uhwi (&ib);
4862 76865 : c.code = (enum tree_code) streamer_read_uhwi (&ib);
4863 76865 : c.type = stream_read_tree (&ib, data_in);
4864 76865 : c.val = stream_read_tree (&ib, data_in);
4865 76865 : bp = streamer_read_bitpack (&ib);
4866 76865 : c.agg_contents = bp_unpack_value (&bp, 1);
4867 76865 : c.by_ref = bp_unpack_value (&bp, 1);
4868 76865 : if (c.agg_contents)
4869 12406 : c.offset = streamer_read_uhwi (&ib);
4870 76865 : count3 = streamer_read_uhwi (&ib);
4871 76865 : c.param_ops = NULL;
4872 76865 : if (info)
4873 76865 : vec_safe_reserve_exact (c.param_ops, count3);
4874 76865 : if (params_summary)
4875 76865 : ipa_set_param_used_by_ipa_predicates
4876 76865 : (params_summary, c.operand_num, true);
4877 80363 : for (k = 0; k < count3; k++)
4878 : {
4879 3498 : struct expr_eval_op op;
4880 3498 : enum gimple_rhs_class rhs_class;
4881 3498 : op.code = (enum tree_code) streamer_read_uhwi (&ib);
4882 3498 : op.type = stream_read_tree (&ib, data_in);
4883 3498 : switch (rhs_class = get_gimple_rhs_class (op.code))
4884 : {
4885 1401 : case GIMPLE_UNARY_RHS:
4886 1401 : op.index = 0;
4887 1401 : op.val[0] = NULL_TREE;
4888 1401 : op.val[1] = NULL_TREE;
4889 1401 : break;
4890 :
4891 2097 : case GIMPLE_BINARY_RHS:
4892 2097 : case GIMPLE_TERNARY_RHS:
4893 2097 : bp = streamer_read_bitpack (&ib);
4894 2097 : op.index = bp_unpack_value (&bp, 2);
4895 2097 : op.val[0] = stream_read_tree (&ib, data_in);
4896 2097 : if (rhs_class == GIMPLE_BINARY_RHS)
4897 2097 : op.val[1] = NULL_TREE;
4898 : else
4899 0 : op.val[1] = stream_read_tree (&ib, data_in);
4900 : break;
4901 :
4902 0 : default:
4903 0 : fatal_error (UNKNOWN_LOCATION,
4904 : "invalid fnsummary in LTO stream");
4905 : }
4906 3498 : if (info)
4907 3498 : c.param_ops->quick_push (op);
4908 : }
4909 76865 : if (info)
4910 76865 : info->conds->quick_push (c);
4911 : }
4912 86121 : count2 = streamer_read_uhwi (&ib);
4913 86121 : gcc_assert (!info || !info->size_time_table.length ());
4914 86121 : if (info && count2)
4915 86063 : info->size_time_table.reserve_exact (count2);
4916 327147 : for (j = 0; j < count2; j++)
4917 : {
4918 241026 : class size_time_entry e;
4919 :
4920 241026 : e.size = streamer_read_uhwi (&ib);
4921 241026 : e.time = sreal::stream_in (&ib);
4922 241026 : e.exec_predicate.stream_in (&ib);
4923 241026 : e.nonconst_predicate.stream_in (&ib);
4924 :
4925 241026 : if (info)
4926 240910 : info->size_time_table.quick_push (e);
4927 : }
4928 :
4929 86121 : count2 = streamer_read_uhwi (&ib);
4930 87470 : for (j = 0; j < count2; j++)
4931 : {
4932 1349 : p.stream_in (&ib);
4933 1349 : sreal fcp_freq = sreal::stream_in (&ib);
4934 1349 : if (info)
4935 : {
4936 1349 : ipa_freqcounting_predicate fcp;
4937 1349 : fcp.predicate = NULL;
4938 1349 : set_hint_predicate (&fcp.predicate, p);
4939 1349 : fcp.freq = fcp_freq;
4940 1349 : vec_safe_push (info->loop_iterations, fcp);
4941 : }
4942 : }
4943 86121 : count2 = streamer_read_uhwi (&ib);
4944 86408 : for (j = 0; j < count2; j++)
4945 : {
4946 287 : p.stream_in (&ib);
4947 287 : sreal fcp_freq = sreal::stream_in (&ib);
4948 287 : if (info)
4949 : {
4950 287 : ipa_freqcounting_predicate fcp;
4951 287 : fcp.predicate = NULL;
4952 287 : set_hint_predicate (&fcp.predicate, p);
4953 287 : fcp.freq = fcp_freq;
4954 287 : vec_safe_push (info->loop_strides, fcp);
4955 : }
4956 : }
4957 86121 : count2 = streamer_read_uhwi (&ib);
4958 86121 : if (info && count2)
4959 6 : info->builtin_constant_p_parms.reserve_exact (count2);
4960 86127 : for (j = 0; j < count2; j++)
4961 : {
4962 6 : int parm = streamer_read_uhwi (&ib);
4963 6 : if (info)
4964 6 : info->builtin_constant_p_parms.quick_push (parm);
4965 : }
4966 431865 : for (e = node->callees; e; e = e->next_callee)
4967 345744 : read_ipa_call_summary (&ib, e, info != NULL);
4968 87601 : for (e = node->indirect_calls; e; e = e->next_callee)
4969 1480 : read_ipa_call_summary (&ib, e, info != NULL);
4970 : }
4971 :
4972 13339 : lto_free_section_data (file_data, LTO_section_ipa_fn_summary, NULL, data,
4973 : len);
4974 13339 : lto_data_in_delete (data_in);
4975 13339 : }
4976 :
4977 :
4978 : /* Read inline summary. Jump functions are shared among ipa-cp
4979 : and inliner, so when ipa-cp is active, we don't need to write them
4980 : twice. */
4981 :
4982 : static void
4983 12284 : ipa_fn_summary_read (void)
4984 : {
4985 12284 : struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
4986 12284 : struct lto_file_decl_data *file_data;
4987 12284 : unsigned int j = 0;
4988 :
4989 12284 : ipa_prop_read_jump_functions ();
4990 12284 : ipa_fn_summary_alloc ();
4991 :
4992 37907 : while ((file_data = file_data_vec[j++]))
4993 : {
4994 13339 : size_t len;
4995 13339 : const char *data
4996 13339 : = lto_get_summary_section_data (file_data, LTO_section_ipa_fn_summary,
4997 : &len);
4998 13339 : if (data)
4999 13339 : inline_read_section (file_data, data, len);
5000 : else
5001 : /* Fatal error here. We do not want to support compiling ltrans units
5002 : with different version of compiler or different flags than the WPA
5003 : unit, so this should never happen. */
5004 0 : fatal_error (input_location,
5005 : "ipa inline summary is missing in input file");
5006 : }
5007 12284 : ipa_register_cgraph_hooks ();
5008 :
5009 12284 : gcc_assert (ipa_fn_summaries);
5010 12284 : ipa_fn_summaries->enable_insertion_hook ();
5011 12284 : }
5012 :
5013 :
5014 : /* Write inline summary for edge E to OB. */
5015 :
5016 : static void
5017 378658 : write_ipa_call_summary (struct output_block *ob, struct cgraph_edge *e)
5018 : {
5019 378658 : class ipa_call_summary *es = ipa_call_summaries->get (e);
5020 378658 : int i;
5021 :
5022 378658 : streamer_write_uhwi (ob, es->call_stmt_size);
5023 378658 : streamer_write_uhwi (ob, es->call_stmt_time);
5024 378658 : streamer_write_uhwi (ob, es->loop_depth);
5025 :
5026 378658 : bitpack_d bp = bitpack_create (ob->main_stream);
5027 378658 : bp_pack_value (&bp, es->is_return_callee_uncaptured, 1);
5028 378658 : streamer_write_bitpack (&bp);
5029 :
5030 378658 : if (es->predicate)
5031 10587 : es->predicate->stream_out (ob);
5032 : else
5033 368071 : streamer_write_uhwi (ob, 0);
5034 378658 : streamer_write_uhwi (ob, es->param.length ());
5035 2285896 : for (i = 0; i < (int) es->param.length (); i++)
5036 : {
5037 630393 : streamer_write_uhwi (ob, es->param[i].change_prob);
5038 630393 : bp = bitpack_create (ob->main_stream);
5039 630393 : bp_pack_value (&bp, es->param[i].points_to_local_or_readonly_memory, 1);
5040 630393 : bp_pack_value (&bp, es->param[i].points_to_possible_sra_candidate, 1);
5041 630393 : streamer_write_bitpack (&bp);
5042 : }
5043 378658 : }
5044 :
5045 :
5046 : /* Write inline summary for node in SET.
5047 : Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
5048 : active, we don't need to write them twice. */
5049 :
5050 : static void
5051 23187 : ipa_fn_summary_write (void)
5052 : {
5053 23187 : struct output_block *ob = create_output_block (LTO_section_ipa_fn_summary);
5054 23187 : lto_symtab_encoder_iterator lsei;
5055 23187 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
5056 23187 : unsigned int count = 0;
5057 :
5058 128147 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
5059 104960 : lsei_next_function_in_partition (&lsei))
5060 : {
5061 104960 : cgraph_node *cnode = lsei_cgraph_node (lsei);
5062 104960 : if (cnode->definition && !cnode->alias)
5063 102518 : count++;
5064 : }
5065 23187 : streamer_write_uhwi (ob, count);
5066 :
5067 128147 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
5068 104960 : lsei_next_function_in_partition (&lsei))
5069 : {
5070 104960 : cgraph_node *cnode = lsei_cgraph_node (lsei);
5071 104960 : if (cnode->definition && !cnode->alias)
5072 : {
5073 102518 : class ipa_fn_summary *info = ipa_fn_summaries->get (cnode);
5074 102518 : class ipa_size_summary *size_info = ipa_size_summaries->get (cnode);
5075 102518 : struct bitpack_d bp;
5076 102518 : struct cgraph_edge *edge;
5077 102518 : int i;
5078 102518 : size_time_entry *e;
5079 102518 : struct condition *c;
5080 :
5081 102518 : streamer_write_uhwi (ob, lto_symtab_encoder_encode (encoder, cnode));
5082 102518 : streamer_write_hwi (ob, size_info->estimated_self_stack_size);
5083 102518 : streamer_write_hwi (ob, size_info->self_size);
5084 102518 : info->time.stream_out (ob);
5085 102518 : bp = bitpack_create (ob->main_stream);
5086 102518 : bp_pack_value (&bp, info->inlinable, 1);
5087 102518 : bp_pack_value (&bp, info->fp_expressions, 1);
5088 102518 : streamer_write_bitpack (&bp);
5089 102518 : if (!lto_stream_offload_p)
5090 102518 : streamer_write_uhwi (ob, info->target_info);
5091 102518 : streamer_write_uhwi (ob, vec_safe_length (info->conds));
5092 195083 : for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
5093 : {
5094 92565 : int j;
5095 92565 : struct expr_eval_op *op;
5096 :
5097 92565 : streamer_write_uhwi (ob, c->operand_num);
5098 92565 : streamer_write_uhwi (ob, c->code);
5099 92565 : stream_write_tree (ob, c->type, true);
5100 92565 : stream_write_tree (ob, c->val, true);
5101 92565 : bp = bitpack_create (ob->main_stream);
5102 92565 : bp_pack_value (&bp, c->agg_contents, 1);
5103 92565 : bp_pack_value (&bp, c->by_ref, 1);
5104 92565 : streamer_write_bitpack (&bp);
5105 92565 : if (c->agg_contents)
5106 15577 : streamer_write_uhwi (ob, c->offset);
5107 92565 : streamer_write_uhwi (ob, vec_safe_length (c->param_ops));
5108 192121 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
5109 : {
5110 4091 : streamer_write_uhwi (ob, op->code);
5111 4091 : stream_write_tree (ob, op->type, true);
5112 4091 : if (op->val[0])
5113 : {
5114 2468 : bp = bitpack_create (ob->main_stream);
5115 2468 : bp_pack_value (&bp, op->index, 2);
5116 2468 : streamer_write_bitpack (&bp);
5117 2468 : stream_write_tree (ob, op->val[0], true);
5118 2468 : if (op->val[1])
5119 4 : stream_write_tree (ob, op->val[1], true);
5120 : }
5121 : }
5122 : }
5123 102518 : streamer_write_uhwi (ob, info->size_time_table.length ());
5124 495499 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
5125 : {
5126 290463 : streamer_write_uhwi (ob, e->size);
5127 290463 : e->time.stream_out (ob);
5128 290463 : e->exec_predicate.stream_out (ob);
5129 290463 : e->nonconst_predicate.stream_out (ob);
5130 : }
5131 102518 : ipa_freqcounting_predicate *fcp;
5132 102518 : streamer_write_uhwi (ob, vec_safe_length (info->loop_iterations));
5133 207013 : for (i = 0; vec_safe_iterate (info->loop_iterations, i, &fcp); i++)
5134 : {
5135 1977 : fcp->predicate->stream_out (ob);
5136 1977 : fcp->freq.stream_out (ob);
5137 : }
5138 102518 : streamer_write_uhwi (ob, vec_safe_length (info->loop_strides));
5139 205377 : for (i = 0; vec_safe_iterate (info->loop_strides, i, &fcp); i++)
5140 : {
5141 341 : fcp->predicate->stream_out (ob);
5142 341 : fcp->freq.stream_out (ob);
5143 : }
5144 102518 : streamer_write_uhwi (ob, info->builtin_constant_p_parms.length ());
5145 102518 : int ip;
5146 205050 : for (i = 0; info->builtin_constant_p_parms.iterate (i, &ip);
5147 : i++)
5148 14 : streamer_write_uhwi (ob, ip);
5149 478492 : for (edge = cnode->callees; edge; edge = edge->next_callee)
5150 375974 : write_ipa_call_summary (ob, edge);
5151 105202 : for (edge = cnode->indirect_calls; edge; edge = edge->next_callee)
5152 2684 : write_ipa_call_summary (ob, edge);
5153 : }
5154 : }
5155 23187 : streamer_write_char_stream (ob->main_stream, 0);
5156 23187 : produce_asm (ob);
5157 23187 : destroy_output_block (ob);
5158 :
5159 23187 : ipa_prop_write_jump_functions ();
5160 23187 : }
5161 :
5162 :
5163 : /* Release function summary. */
5164 :
5165 : void
5166 723573 : ipa_free_fn_summary (void)
5167 : {
5168 723573 : if (!ipa_call_summaries)
5169 : return;
5170 457936 : ggc_delete (ipa_fn_summaries);
5171 457936 : ipa_fn_summaries = NULL;
5172 457936 : delete ipa_call_summaries;
5173 457936 : ipa_call_summaries = NULL;
5174 457936 : edge_predicate_pool.release ();
5175 : /* During IPA this is one of largest datastructures to release. */
5176 457936 : if (flag_wpa)
5177 7860 : ggc_trim ();
5178 : }
5179 :
5180 : /* Release function summary. */
5181 :
5182 : void
5183 723573 : ipa_free_size_summary (void)
5184 : {
5185 723573 : if (!ipa_size_summaries)
5186 : return;
5187 457936 : delete ipa_size_summaries;
5188 457936 : ipa_size_summaries = NULL;
5189 : }
5190 :
5191 : namespace {
5192 :
5193 : const pass_data pass_data_local_fn_summary =
5194 : {
5195 : GIMPLE_PASS, /* type */
5196 : "local-fnsummary", /* name */
5197 : OPTGROUP_INLINE, /* optinfo_flags */
5198 : TV_INLINE_PARAMETERS, /* tv_id */
5199 : 0, /* properties_required */
5200 : 0, /* properties_provided */
5201 : 0, /* properties_destroyed */
5202 : 0, /* todo_flags_start */
5203 : 0, /* todo_flags_finish */
5204 : };
5205 :
5206 : class pass_local_fn_summary : public gimple_opt_pass
5207 : {
5208 : public:
5209 576094 : pass_local_fn_summary (gcc::context *ctxt)
5210 1152188 : : gimple_opt_pass (pass_data_local_fn_summary, ctxt)
5211 : {}
5212 :
5213 : /* opt_pass methods: */
5214 288047 : opt_pass * clone () final override
5215 : {
5216 288047 : return new pass_local_fn_summary (m_ctxt);
5217 : }
5218 5706083 : unsigned int execute (function *) final override
5219 : {
5220 5706083 : return compute_fn_summary_for_current ();
5221 : }
5222 :
5223 : }; // class pass_local_fn_summary
5224 :
5225 : } // anon namespace
5226 :
5227 : gimple_opt_pass *
5228 288047 : make_pass_local_fn_summary (gcc::context *ctxt)
5229 : {
5230 288047 : return new pass_local_fn_summary (ctxt);
5231 : }
5232 :
5233 :
5234 : /* Free inline summary. */
5235 :
5236 : namespace {
5237 :
5238 : const pass_data pass_data_ipa_free_fn_summary =
5239 : {
5240 : SIMPLE_IPA_PASS, /* type */
5241 : "free-fnsummary", /* name */
5242 : OPTGROUP_NONE, /* optinfo_flags */
5243 : TV_IPA_FREE_INLINE_SUMMARY, /* tv_id */
5244 : 0, /* properties_required */
5245 : 0, /* properties_provided */
5246 : 0, /* properties_destroyed */
5247 : 0, /* todo_flags_start */
5248 : 0, /* todo_flags_finish */
5249 : };
5250 :
5251 : class pass_ipa_free_fn_summary : public simple_ipa_opt_pass
5252 : {
5253 : public:
5254 576094 : pass_ipa_free_fn_summary (gcc::context *ctxt)
5255 576094 : : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary, ctxt),
5256 1152188 : small_p (false)
5257 : {}
5258 :
5259 : /* opt_pass methods: */
5260 288047 : opt_pass *clone () final override
5261 : {
5262 288047 : return new pass_ipa_free_fn_summary (m_ctxt);
5263 : }
5264 576094 : void set_pass_param (unsigned int n, bool param) final override
5265 : {
5266 576094 : gcc_assert (n == 0);
5267 576094 : small_p = param;
5268 576094 : }
5269 484500 : bool gate (function *) final override { return true; }
5270 463971 : unsigned int execute (function *) final override
5271 : {
5272 463971 : ipa_free_fn_summary ();
5273 : /* Free ipa-prop structures if they are no longer needed. */
5274 463971 : ipa_free_all_structures_after_iinln ();
5275 463971 : if (!flag_wpa)
5276 456111 : ipa_free_size_summary ();
5277 463971 : return 0;
5278 : }
5279 :
5280 : private:
5281 : bool small_p;
5282 : }; // class pass_ipa_free_fn_summary
5283 :
5284 : } // anon namespace
5285 :
5286 : simple_ipa_opt_pass *
5287 288047 : make_pass_ipa_free_fn_summary (gcc::context *ctxt)
5288 : {
5289 288047 : return new pass_ipa_free_fn_summary (ctxt);
5290 : }
5291 :
5292 : namespace {
5293 :
5294 : const pass_data pass_data_ipa_fn_summary =
5295 : {
5296 : IPA_PASS, /* type */
5297 : "fnsummary", /* name */
5298 : OPTGROUP_INLINE, /* optinfo_flags */
5299 : TV_IPA_FNSUMMARY, /* tv_id */
5300 : 0, /* properties_required */
5301 : 0, /* properties_provided */
5302 : 0, /* properties_destroyed */
5303 : 0, /* todo_flags_start */
5304 : ( TODO_dump_symtab ), /* todo_flags_finish */
5305 : };
5306 :
5307 : class pass_ipa_fn_summary : public ipa_opt_pass_d
5308 : {
5309 : public:
5310 288047 : pass_ipa_fn_summary (gcc::context *ctxt)
5311 : : ipa_opt_pass_d (pass_data_ipa_fn_summary, ctxt,
5312 : ipa_fn_summary_generate, /* generate_summary */
5313 : ipa_fn_summary_write, /* write_summary */
5314 : ipa_fn_summary_read, /* read_summary */
5315 : NULL, /* write_optimization_summary */
5316 : NULL, /* read_optimization_summary */
5317 : NULL, /* stmt_fixup */
5318 : 0, /* function_transform_todo_flags_start */
5319 : NULL, /* function_transform */
5320 288047 : NULL) /* variable_transform */
5321 288047 : {}
5322 :
5323 : /* opt_pass methods: */
5324 232040 : unsigned int execute (function *) final override { return 0; }
5325 :
5326 : }; // class pass_ipa_fn_summary
5327 :
5328 : } // anon namespace
5329 :
5330 : ipa_opt_pass_d *
5331 288047 : make_pass_ipa_fn_summary (gcc::context *ctxt)
5332 : {
5333 288047 : return new pass_ipa_fn_summary (ctxt);
5334 : }
5335 :
5336 : /* Reset all state within ipa-fnsummary.cc so that we can rerun the compiler
5337 : within the same process. For use by toplev::finalize. */
5338 :
5339 : void
5340 258746 : ipa_fnsummary_cc_finalize (void)
5341 : {
5342 258746 : ipa_free_fn_summary ();
5343 258746 : ipa_free_size_summary ();
5344 258746 : }
|
