Line data Source code
1 : /* Function summary pass.
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* Analysis of function bodies used by inter-procedural passes
22 :
23 : We estimate for each function
24 : - function body size and size after specializing into given context
25 : - average function execution time in a given context
26 : - function frame size
27 : For each call
28 : - call statement size, time and how often the parameters change
29 :
30 : ipa_fn_summary data structures store above information locally (i.e.
31 : parameters of the function itself) and globally (i.e. parameters of
32 : the function created by applying all the inline decisions already
33 : present in the callgraph).
34 :
35 : We provide access to the ipa_fn_summary data structure and
36 : basic logic updating the parameters when inlining is performed.
37 :
38 : The summaries are context sensitive. Context means
39 : 1) partial assignment of known constant values of operands
40 : 2) whether function is inlined into the call or not.
41 : It is easy to add more variants. To represent function size and time
42 : that depends on context (i.e. it is known to be optimized away when
43 : context is known either by inlining or from IP-CP and cloning),
44 : we use predicates.
45 :
46 : estimate_edge_size_and_time can be used to query
47 : function size/time in the given context. ipa_merge_fn_summary_after_inlining merges
48 : properties of caller and callee after inlining.
49 :
50 : Finally pass_inline_parameters is exported. This is used to drive
51 : computation of function parameters used by the early inliner. IPA
52 : inlined performs analysis via its analyze_function method. */
53 :
54 : #include "config.h"
55 : #define INCLUDE_VECTOR
56 : #include "system.h"
57 : #include "coretypes.h"
58 : #include "backend.h"
59 : #include "target.h"
60 : #include "tree.h"
61 : #include "gimple.h"
62 : #include "alloc-pool.h"
63 : #include "tree-pass.h"
64 : #include "ssa.h"
65 : #include "tree-streamer.h"
66 : #include "cgraph.h"
67 : #include "diagnostic.h"
68 : #include "fold-const.h"
69 : #include "print-tree.h"
70 : #include "tree-inline.h"
71 : #include "gimple-pretty-print.h"
72 : #include "cfganal.h"
73 : #include "gimple-iterator.h"
74 : #include "tree-cfg.h"
75 : #include "tree-ssa-loop-niter.h"
76 : #include "tree-ssa-loop.h"
77 : #include "symbol-summary.h"
78 : #include "sreal.h"
79 : #include "ipa-cp.h"
80 : #include "ipa-prop.h"
81 : #include "ipa-fnsummary.h"
82 : #include "cfgloop.h"
83 : #include "tree-scalar-evolution.h"
84 : #include "ipa-utils.h"
85 : #include "cfgexpand.h"
86 : #include "gimplify.h"
87 : #include "stringpool.h"
88 : #include "attribs.h"
89 : #include "tree-into-ssa.h"
90 : #include "symtab-clones.h"
91 : #include "gimple-range.h"
92 : #include "tree-dfa.h"
93 :
94 : /* Summaries. */
95 : fast_function_summary <ipa_fn_summary *, va_gc> *ipa_fn_summaries;
96 : fast_function_summary <ipa_size_summary *, va_heap> *ipa_size_summaries;
97 : fast_call_summary <ipa_call_summary *, va_heap> *ipa_call_summaries;
98 :
99 : /* Edge predicates goes here. */
100 : static object_allocator<ipa_predicate> edge_predicate_pool ("edge predicates");
101 :
102 :
103 : /* Dump IPA hints. */
104 : void
105 185 : ipa_dump_hints (FILE *f, ipa_hints hints)
106 : {
107 185 : if (!hints)
108 : return;
109 148 : fprintf (f, "IPA hints:");
110 148 : if (hints & INLINE_HINT_indirect_call)
111 : {
112 23 : hints &= ~INLINE_HINT_indirect_call;
113 23 : fprintf (f, " indirect_call");
114 : }
115 148 : if (hints & INLINE_HINT_loop_iterations)
116 : {
117 3 : hints &= ~INLINE_HINT_loop_iterations;
118 3 : fprintf (f, " loop_iterations");
119 : }
120 148 : if (hints & INLINE_HINT_loop_stride)
121 : {
122 3 : hints &= ~INLINE_HINT_loop_stride;
123 3 : fprintf (f, " loop_stride");
124 : }
125 148 : if (hints & INLINE_HINT_same_scc)
126 : {
127 4 : hints &= ~INLINE_HINT_same_scc;
128 4 : fprintf (f, " same_scc");
129 : }
130 148 : if (hints & INLINE_HINT_in_scc)
131 : {
132 11 : hints &= ~INLINE_HINT_in_scc;
133 11 : fprintf (f, " in_scc");
134 : }
135 148 : if (hints & INLINE_HINT_cross_module)
136 : {
137 2 : hints &= ~INLINE_HINT_cross_module;
138 2 : fprintf (f, " cross_module");
139 : }
140 148 : if (hints & INLINE_HINT_declared_inline)
141 : {
142 122 : hints &= ~INLINE_HINT_declared_inline;
143 122 : fprintf (f, " declared_inline");
144 : }
145 148 : if (hints & INLINE_HINT_known_hot)
146 : {
147 1 : hints &= ~INLINE_HINT_known_hot;
148 1 : fprintf (f, " known_hot");
149 : }
150 148 : if (hints & INLINE_HINT_builtin_constant_p)
151 : {
152 4 : hints &= ~INLINE_HINT_builtin_constant_p;
153 4 : fprintf (f, " builtin_constant_p");
154 : }
155 148 : gcc_assert (!hints);
156 : }
157 :
158 :
159 : /* Record SIZE and TIME to SUMMARY.
160 : The accounted code will be executed when EXEC_PRED is true.
161 : When NONCONST_PRED is false the code will evaluate to constant and
162 : will get optimized out in specialized clones of the function.
163 : If CALL is true account to call_size_time_table rather than
164 : size_time_table. */
165 :
166 : void
167 138789911 : ipa_fn_summary::account_size_time (int size, sreal time,
168 : const ipa_predicate &exec_pred,
169 : const ipa_predicate &nonconst_pred_in,
170 : bool call)
171 : {
172 138789911 : size_time_entry *e;
173 138789911 : bool found = false;
174 138789911 : int i;
175 138789911 : ipa_predicate nonconst_pred;
176 138789911 : vec<size_time_entry> *table = call ? &call_size_time_table : &size_time_table;
177 :
178 138789911 : if (exec_pred == false)
179 5053571 : return;
180 :
181 138495454 : nonconst_pred = nonconst_pred_in & exec_pred;
182 :
183 138495454 : if (nonconst_pred == false)
184 : return;
185 :
186 : /* We need to create initial empty unconditional clause, but otherwise
187 : we don't need to account empty times and sizes. */
188 162648813 : if (!size && time == 0 && table->length ())
189 4040815 : return;
190 :
191 : /* Only for calls we are unaccounting what we previously recorded. */
192 133736340 : gcc_checking_assert (time >= 0 || call);
193 :
194 485995163 : for (i = 0; table->iterate (i, &e); i++)
195 455660302 : if (e->exec_predicate == exec_pred
196 455660302 : && e->nonconst_predicate == nonconst_pred)
197 : {
198 : found = true;
199 : break;
200 : }
201 133736340 : if (i == max_size_time_table_size)
202 : {
203 4896 : i = 0;
204 4896 : found = true;
205 4896 : e = &(*table)[0];
206 4896 : if (dump_file && (dump_flags & TDF_DETAILS))
207 0 : fprintf (dump_file,
208 : "\t\tReached limit on number of entries, "
209 : "ignoring the predicate.");
210 : }
211 133740577 : if (dump_file && (dump_flags & TDF_DETAILS) && (time != 0 || size))
212 : {
213 3963 : fprintf (dump_file,
214 : "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
215 3730 : ((double) size) / ipa_fn_summary::size_scale,
216 : (time.to_double ()), found ? "" : "new ");
217 3730 : exec_pred.dump (dump_file, conds, 0);
218 3730 : if (exec_pred != nonconst_pred)
219 : {
220 84 : fprintf (dump_file, " nonconst:");
221 84 : nonconst_pred.dump (dump_file, conds);
222 : }
223 : else
224 3646 : fprintf (dump_file, "\n");
225 : }
226 133736340 : if (!found)
227 : {
228 30329965 : class size_time_entry new_entry;
229 30329965 : new_entry.size = size;
230 30329965 : new_entry.time = time;
231 30329965 : new_entry.exec_predicate = exec_pred;
232 30329965 : new_entry.nonconst_predicate = nonconst_pred;
233 30329965 : if (call)
234 1374024 : call_size_time_table.safe_push (new_entry);
235 : else
236 28955941 : size_time_table.safe_push (new_entry);
237 : }
238 : else
239 : {
240 103406375 : e->size += size;
241 103406375 : e->time += time;
242 : /* FIXME: PR bootstrap/92653 gcc_checking_assert (e->time >= -1); */
243 : /* Tolerate small roundoff issues. */
244 103406375 : if (e->time < 0)
245 219 : e->time = 0;
246 : }
247 : }
248 :
249 : /* We proved E to be unreachable, redirect it to __builtin_unreachable. */
250 :
251 : static struct cgraph_edge *
252 349998 : redirect_to_unreachable (struct cgraph_edge *e)
253 : {
254 349998 : struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
255 349998 : struct cgraph_node *target
256 349998 : = cgraph_node::get_create (builtin_decl_unreachable ());
257 :
258 349998 : gcc_checking_assert (lookup_attribute ("cold",
259 : DECL_ATTRIBUTES (target->decl)));
260 :
261 349998 : if (e->speculative)
262 311 : e = cgraph_edge::resolve_speculation (e, target->decl);
263 349687 : else if (!e->callee)
264 773 : e = cgraph_edge::make_direct (e, target);
265 : else
266 348914 : e->redirect_callee (target);
267 349998 : class ipa_call_summary *es = ipa_call_summaries->get (e);
268 349998 : e->inline_failed = CIF_UNREACHABLE;
269 349998 : e->count = profile_count::zero ();
270 349998 : es->call_stmt_size = 0;
271 349998 : es->call_stmt_time = 0;
272 349998 : if (callee)
273 0 : callee->remove_symbol_and_inline_clones ();
274 349998 : if (e->has_callback)
275 4 : for (cgraph_edge *cbe = e->first_callback_edge (); cbe;
276 1 : cbe = cbe->next_callback_edge ())
277 : /* If the carrying edge is unreachable, so are the callback calls. */
278 1 : redirect_to_unreachable (cbe);
279 349998 : return e;
280 : }
281 :
282 : /* Set predicate for edge E. */
283 :
284 : static void
285 32066193 : edge_set_predicate (struct cgraph_edge *e, ipa_predicate *predicate)
286 : {
287 : /* If the edge is determined to be never executed, redirect it
288 : to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
289 : be optimized out. */
290 29092643 : if (predicate && *predicate == false
291 : /* When handling speculative edges, we need to do the redirection
292 : just once. Do it always on the direct edge, so we do not
293 : attempt to resolve speculation while duplicating the edge. */
294 32416267 : && (!e->speculative || e->callee))
295 349997 : e = redirect_to_unreachable (e);
296 :
297 32066193 : class ipa_call_summary *es = ipa_call_summaries->get (e);
298 61158836 : if (predicate && *predicate != true)
299 : {
300 4758653 : if (!es->predicate)
301 4234400 : es->predicate = edge_predicate_pool.allocate ();
302 4758653 : *es->predicate = *predicate;
303 : }
304 : else
305 : {
306 27307540 : if (es->predicate)
307 866850 : edge_predicate_pool.remove (es->predicate);
308 27307540 : es->predicate = NULL;
309 : }
310 32066193 : }
311 :
312 : /* Set predicate for hint *P. */
313 :
314 : static void
315 152259 : set_hint_predicate (ipa_predicate **p, ipa_predicate new_predicate)
316 : {
317 152259 : if (new_predicate == false || new_predicate == true)
318 : {
319 2927 : if (*p)
320 0 : edge_predicate_pool.remove (*p);
321 2927 : *p = NULL;
322 : }
323 : else
324 : {
325 149332 : if (!*p)
326 149332 : *p = edge_predicate_pool.allocate ();
327 149332 : **p = new_predicate;
328 : }
329 152259 : }
330 :
331 : /* Find if NEW_PREDICATE is already in V and if so, increment its freq.
332 : Otherwise add a new item to the vector with this predicate and frerq equal
333 : to add_freq, unless the number of predicates would exceed MAX_NUM_PREDICATES
334 : in which case the function does nothing. */
335 :
336 : static void
337 1129057 : add_freqcounting_predicate (vec<ipa_freqcounting_predicate, va_gc> **v,
338 : const ipa_predicate &new_predicate, sreal add_freq,
339 : unsigned max_num_predicates)
340 : {
341 1129057 : if (new_predicate == false || new_predicate == true)
342 : return;
343 : ipa_freqcounting_predicate *f;
344 143760 : for (int i = 0; vec_safe_iterate (*v, i, &f); i++)
345 72424 : if (new_predicate == f->predicate)
346 : {
347 0 : f->freq += add_freq;
348 0 : return;
349 : }
350 97660 : if (vec_safe_length (*v) >= max_num_predicates)
351 : /* Too many different predicates to account for. */
352 : return;
353 :
354 71064 : ipa_freqcounting_predicate fcp;
355 71064 : fcp.predicate = NULL;
356 71064 : set_hint_predicate (&fcp.predicate, new_predicate);
357 71064 : fcp.freq = add_freq;
358 71064 : vec_safe_push (*v, fcp);
359 71064 : return;
360 : }
361 :
362 : /* Compute what conditions may or may not hold given information about
363 : parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
364 : while RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
365 : copy when called in a given context. It is a bitmask of conditions. Bit
366 : 0 means that condition is known to be false, while bit 1 means that condition
367 : may or may not be true. These differs - for example NOT_INLINED condition
368 : is always false in the second and also builtin_constant_p tests cannot use
369 : the fact that parameter is indeed a constant.
370 :
371 : When INLINE_P is true, assume that we are inlining. AVAL contains known
372 : information about argument values. The function does not modify its content
373 : and so AVALs could also be of type ipa_call_arg_values but so far all
374 : callers work with the auto version and so we avoid the conversion for
375 : convenience.
376 :
377 : ERROR_MARK value of an argument means compile time invariant. */
378 :
379 : static void
380 23037619 : evaluate_conditions_for_known_args (struct cgraph_node *node,
381 : bool inline_p,
382 : ipa_auto_call_arg_values *avals,
383 : clause_t *ret_clause,
384 : clause_t *ret_nonspec_clause,
385 : ipa_call_summary *es)
386 : {
387 23037619 : clause_t clause = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
388 23037619 : clause_t nonspec_clause = 1 << ipa_predicate::not_inlined_condition;
389 23037619 : class ipa_fn_summary *info = ipa_fn_summaries->get (node);
390 23037619 : int i;
391 23037619 : struct condition *c;
392 :
393 95447525 : for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
394 : {
395 72409906 : tree val = NULL;
396 72409906 : tree res;
397 72409906 : int j;
398 72409906 : struct expr_eval_op *op;
399 :
400 72409906 : if (c->code == ipa_predicate::not_sra_candidate)
401 : {
402 16895053 : if (!inline_p
403 16895053 : || !es
404 16757107 : || (int)es->param.length () <= c->operand_num
405 25273585 : || !es->param[c->operand_num].points_to_possible_sra_candidate)
406 12573335 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
407 16895053 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
408 72409906 : continue;
409 : }
410 :
411 55514853 : if (c->agg_contents)
412 : {
413 30853991 : if (c->code == ipa_predicate::changed
414 19746991 : && !c->by_ref
415 33707535 : && (avals->safe_sval_at(c->operand_num) == error_mark_node))
416 3545 : continue;
417 :
418 30846901 : if (tree sval = avals->safe_sval_at (c->operand_num))
419 14452239 : val = ipa_find_agg_cst_from_init (sval, c->offset, c->by_ref);
420 14452239 : if (!val)
421 : {
422 30831757 : ipa_argagg_value_list avs (avals);
423 30831757 : val = avs.get_value (c->operand_num, c->offset / BITS_PER_UNIT,
424 30831757 : c->by_ref);
425 : }
426 : }
427 : else
428 : {
429 24664407 : val = avals->safe_sval_at (c->operand_num);
430 24664407 : if (val && val == error_mark_node
431 2316410 : && c->code != ipa_predicate::changed)
432 : val = NULL_TREE;
433 : }
434 :
435 42151778 : if (!val
436 42013503 : && (c->code == ipa_predicate::changed
437 : || c->code == ipa_predicate::is_not_constant))
438 : {
439 26638618 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
440 26638618 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
441 26638618 : continue;
442 : }
443 28872690 : if (c->code == ipa_predicate::changed)
444 : {
445 7799558 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
446 7799558 : continue;
447 : }
448 :
449 21073132 : if (c->code == ipa_predicate::is_not_constant)
450 : {
451 8693 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
452 8693 : continue;
453 : }
454 :
455 21064439 : if (val && TYPE_SIZE (c->type) == TYPE_SIZE (TREE_TYPE (val)))
456 : {
457 5607720 : if (c->type != TREE_TYPE (val))
458 942851 : val = fold_unary (VIEW_CONVERT_EXPR, c->type, val);
459 6103544 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
460 : {
461 496289 : if (!val)
462 : break;
463 495824 : if (!op->val[0])
464 224718 : val = fold_unary (op->code, op->type, val);
465 271106 : else if (!op->val[1])
466 542212 : val = fold_binary (op->code, op->type,
467 : op->index ? op->val[0] : val,
468 : op->index ? val : op->val[0]);
469 0 : else if (op->index == 0)
470 0 : val = fold_ternary (op->code, op->type,
471 : val, op->val[0], op->val[1]);
472 0 : else if (op->index == 1)
473 0 : val = fold_ternary (op->code, op->type,
474 : op->val[0], val, op->val[1]);
475 0 : else if (op->index == 2)
476 0 : val = fold_ternary (op->code, op->type,
477 : op->val[0], op->val[1], val);
478 : else
479 : val = NULL_TREE;
480 : }
481 :
482 5607720 : res = val
483 5607720 : ? fold_binary_to_constant (c->code, boolean_type_node, val, c->val)
484 : : NULL;
485 :
486 5607235 : if (res && integer_zerop (res))
487 2846592 : continue;
488 2761128 : if (res && integer_onep (res))
489 : {
490 2738413 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
491 2738413 : nonspec_clause
492 2738413 : |= 1 << (i + ipa_predicate::first_dynamic_condition);
493 2738413 : continue;
494 : }
495 : }
496 15479434 : if (c->operand_num < (int) avals->m_known_value_ranges.length ()
497 9102729 : && !c->agg_contents
498 17670692 : && (!val || TREE_CODE (val) != INTEGER_CST))
499 : {
500 2191187 : value_range vr (avals->m_known_value_ranges[c->operand_num]);
501 2191187 : if (!vr.undefined_p ()
502 1538507 : && !vr.varying_p ()
503 3729694 : && (TYPE_SIZE (c->type) == TYPE_SIZE (vr.type ())))
504 : {
505 1537981 : if (!useless_type_conversion_p (c->type, vr.type ()))
506 508 : range_cast (vr, c->type);
507 :
508 1967583 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
509 : {
510 534167 : if (vr.varying_p () || vr.undefined_p ())
511 : break;
512 :
513 429602 : value_range res (op->type);
514 429602 : if (!op->val[0])
515 : {
516 154759 : value_range varying (op->type);
517 154759 : varying.set_varying (op->type);
518 154759 : range_op_handler handler (op->code);
519 154759 : if (!handler
520 154759 : || !res.supports_type_p (op->type)
521 309518 : || !handler.fold_range (res, op->type, vr, varying))
522 0 : res.set_varying (op->type);
523 154759 : }
524 274843 : else if (!op->val[1])
525 : {
526 274671 : value_range op0 (TREE_TYPE (op->val[0]));
527 274671 : range_op_handler handler (op->code);
528 :
529 274671 : ipa_get_range_from_ip_invariant (op0, op->val[0], node);
530 :
531 274671 : if (!handler
532 274671 : || !res.supports_type_p (op->type)
533 549342 : || !handler.fold_range (res, op->type,
534 274671 : op->index ? op0 : vr,
535 547662 : op->index ? vr : op0))
536 0 : res.set_varying (op->type);
537 274671 : }
538 : else
539 172 : res.set_varying (op->type);
540 429602 : vr = res;
541 429602 : }
542 1537981 : if (!vr.varying_p () && !vr.undefined_p ())
543 : {
544 1427675 : int_range<2> res;
545 1427675 : value_range val_vr (TREE_TYPE (c->val));
546 1427675 : range_op_handler handler (c->code);
547 :
548 1427675 : ipa_get_range_from_ip_invariant (val_vr, c->val, node);
549 :
550 1427675 : if (!handler
551 1427675 : || !val_vr.supports_type_p (TREE_TYPE (c->val))
552 2855350 : || !handler.fold_range (res, boolean_type_node, vr, val_vr))
553 0 : res.set_varying (boolean_type_node);
554 :
555 1427675 : if (res.zero_p ())
556 241927 : continue;
557 1427675 : }
558 : }
559 2191187 : }
560 :
561 15237507 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
562 15237507 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
563 : }
564 23037619 : *ret_clause = clause;
565 23037619 : if (ret_nonspec_clause)
566 19944344 : *ret_nonspec_clause = nonspec_clause;
567 23037619 : }
568 :
569 : /* Return true if VRP will be executed on the function.
570 : We do not want to anticipate optimizations that will not happen.
571 :
572 : FIXME: This can be confused with -fdisable and debug counters and thus
573 : it should not be used for correctness (only to make heuristics work).
574 : This means that inliner should do its own optimizations of expressions
575 : that it predicts to be constant so wrong code can not be triggered by
576 : builtin_constant_p. */
577 :
578 : static bool
579 12653856 : vrp_will_run_p (struct cgraph_node *node)
580 : {
581 12653856 : return (opt_for_fn (node->decl, optimize)
582 12653856 : && !opt_for_fn (node->decl, optimize_debug)
583 25306984 : && opt_for_fn (node->decl, flag_tree_vrp));
584 : }
585 :
586 : /* Similarly about FRE. */
587 :
588 : static bool
589 14920342 : fre_will_run_p (struct cgraph_node *node)
590 : {
591 14920342 : return (opt_for_fn (node->decl, optimize)
592 14920342 : && !opt_for_fn (node->decl, optimize_debug)
593 29839082 : && opt_for_fn (node->decl, flag_tree_fre));
594 : }
595 :
596 : /* Work out what conditions might be true at invocation of E.
597 : Compute costs for inlined edge if INLINE_P is true.
598 :
599 : Return in CLAUSE_PTR the evaluated conditions and in NONSPEC_CLAUSE_PTR
600 : (if non-NULL) conditions evaluated for nonspecialized clone called
601 : in a given context.
602 :
603 : Vectors in AVALS will be populated with useful known information about
604 : argument values - information not known to have any uses will be omitted -
605 : except for m_known_contexts which will only be calculated if
606 : COMPUTE_CONTEXTS is true. */
607 :
608 : void
609 22793639 : evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p,
610 : clause_t *clause_ptr,
611 : clause_t *nonspec_clause_ptr,
612 : ipa_auto_call_arg_values *avals,
613 : bool compute_contexts)
614 : {
615 22793639 : struct cgraph_node *callee = e->callee->ultimate_alias_target ();
616 22793639 : class ipa_fn_summary *info = ipa_fn_summaries->get (callee);
617 22793639 : class ipa_edge_args *args;
618 22793639 : class ipa_call_summary *es = NULL;
619 :
620 22793639 : if (clause_ptr)
621 22793639 : *clause_ptr = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
622 :
623 22793639 : if (ipa_node_params_sum
624 10387938 : && !e->call_stmt_cannot_inline_p
625 10387938 : && (info->conds || compute_contexts)
626 33181577 : && (args = ipa_edge_args_sum->get (e)) != NULL)
627 : {
628 10333403 : struct cgraph_node *caller;
629 10333403 : class ipa_node_params *caller_parms_info, *callee_pi = NULL;
630 10333403 : int i, count = ipa_get_cs_argument_count (args);
631 10333403 : es = ipa_call_summaries->get (e);
632 :
633 10333403 : if (count)
634 : {
635 9619938 : if (e->caller->inlined_to)
636 : caller = e->caller->inlined_to;
637 : else
638 7669341 : caller = e->caller;
639 9619938 : caller_parms_info = ipa_node_params_sum->get (caller);
640 9619938 : callee_pi = ipa_node_params_sum->get (callee);
641 :
642 : /* Watch for thunks. */
643 9619938 : if (callee_pi)
644 : /* Watch for variadic functions. */
645 9619564 : count = MIN (count, ipa_get_param_count (callee_pi));
646 : }
647 :
648 9619564 : if (callee_pi)
649 31468913 : for (i = 0; i < count; i++)
650 : {
651 21849349 : struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i);
652 :
653 21849349 : if (ipa_is_param_used_by_indirect_call (callee_pi, i)
654 21849349 : || ipa_is_param_used_by_ipa_predicates (callee_pi, i))
655 : {
656 : /* Determine if we know constant value of the parameter. */
657 14920342 : tree type = ipa_get_type (callee_pi, i);
658 14920342 : tree cst = ipa_value_from_jfunc (caller_parms_info, jf, type);
659 :
660 11955348 : if (!cst && e->call_stmt
661 26824418 : && i < (int)gimple_call_num_args (e->call_stmt))
662 : {
663 11904076 : cst = gimple_call_arg (e->call_stmt, i);
664 11904076 : if (!is_gimple_min_invariant (cst))
665 : cst = NULL;
666 : }
667 3214866 : if (cst)
668 : {
669 3163594 : gcc_checking_assert (TREE_CODE (cst) != TREE_BINFO);
670 3163594 : if (!avals->m_known_vals.length ())
671 1829422 : avals->m_known_vals.safe_grow_cleared (count, true);
672 3163594 : avals->m_known_vals[i] = cst;
673 : }
674 11756748 : else if (inline_p && !es->param[i].change_prob)
675 : {
676 4355329 : if (!avals->m_known_vals.length ())
677 3563967 : avals->m_known_vals.safe_grow_cleared (count, true);
678 4355329 : avals->m_known_vals[i] = error_mark_node;
679 : }
680 :
681 : /* If we failed to get simple constant, try value range. */
682 3163594 : if ((!cst || TREE_CODE (cst) != INTEGER_CST)
683 12653856 : && vrp_will_run_p (caller)
684 27409798 : && ipa_is_param_used_by_ipa_predicates (callee_pi, i))
685 : {
686 12448894 : value_range vr (type);
687 :
688 12448894 : ipa_value_range_from_jfunc (vr, caller_parms_info, e, jf, type);
689 12448894 : if (!vr.undefined_p () && !vr.varying_p ())
690 : {
691 8867286 : if (!avals->m_known_value_ranges.length ())
692 : {
693 6484418 : avals->m_known_value_ranges.safe_grow_cleared (count,
694 : true);
695 27005913 : for (int i = 0; i < count; ++i)
696 14037077 : avals->m_known_value_ranges[i].set_range_class
697 14037077 : (void_type_node);
698 : }
699 8867286 : avals->m_known_value_ranges[i] = vr;
700 : }
701 12448894 : }
702 :
703 : /* Determine known aggregate values. */
704 14920342 : if (fre_will_run_p (caller))
705 14918336 : ipa_push_agg_values_from_jfunc (caller_parms_info,
706 : caller, &jf->agg, i,
707 : &avals->m_known_aggs);
708 : }
709 :
710 : /* For calls used in polymorphic calls we further determine
711 : polymorphic call context. */
712 21849349 : if (compute_contexts
713 21849349 : && ipa_is_param_used_by_polymorphic_call (callee_pi, i))
714 : {
715 175364 : ipa_polymorphic_call_context
716 175364 : ctx = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
717 350728 : if (!ctx.useless_p ())
718 : {
719 173396 : if (!avals->m_known_contexts.length ())
720 173259 : avals->m_known_contexts.safe_grow_cleared (count, true);
721 173396 : avals->m_known_contexts[i]
722 346792 : = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
723 : }
724 : }
725 : }
726 : else
727 713839 : gcc_assert (!count || callee->thunk);
728 : }
729 12460236 : else if (e->call_stmt && !e->call_stmt_cannot_inline_p && info->conds)
730 : {
731 9799553 : int i, count = (int)gimple_call_num_args (e->call_stmt);
732 :
733 27857928 : for (i = 0; i < count; i++)
734 : {
735 18058375 : tree cst = gimple_call_arg (e->call_stmt, i);
736 18058375 : if (!is_gimple_min_invariant (cst))
737 : cst = NULL;
738 6421781 : if (cst)
739 : {
740 6421781 : if (!avals->m_known_vals.length ())
741 4456485 : avals->m_known_vals.safe_grow_cleared (count, true);
742 6421781 : avals->m_known_vals[i] = cst;
743 : }
744 : }
745 : }
746 :
747 22793639 : evaluate_conditions_for_known_args (callee, inline_p, avals, clause_ptr,
748 : nonspec_clause_ptr, es);
749 22793639 : }
750 :
751 :
752 : /* Allocate the function summary. */
753 :
754 : static void
755 468081 : ipa_fn_summary_alloc (void)
756 : {
757 468081 : gcc_checking_assert (!ipa_fn_summaries);
758 468081 : ipa_size_summaries = new ipa_size_summary_t (symtab);
759 468081 : ipa_fn_summaries = ipa_fn_summary_t::create_ggc (symtab);
760 468081 : ipa_call_summaries = new ipa_call_summary_t (symtab);
761 468081 : }
762 :
763 28053083 : ipa_call_summary::~ipa_call_summary ()
764 : {
765 28053083 : if (predicate)
766 3367550 : edge_predicate_pool.remove (predicate);
767 :
768 28053083 : param.release ();
769 28053083 : }
770 :
771 10406045 : ipa_fn_summary::~ipa_fn_summary ()
772 : {
773 10406045 : unsigned len = vec_safe_length (loop_iterations);
774 10519728 : for (unsigned i = 0; i < len; i++)
775 113683 : edge_predicate_pool.remove ((*loop_iterations)[i].predicate);
776 10406045 : len = vec_safe_length (loop_strides);
777 10441694 : for (unsigned i = 0; i < len; i++)
778 35649 : edge_predicate_pool.remove ((*loop_strides)[i].predicate);
779 10406045 : vec_free (conds);
780 10406045 : call_size_time_table.release ();
781 10406045 : vec_free (loop_iterations);
782 10406045 : vec_free (loop_strides);
783 10406045 : builtin_constant_p_parms.release ();
784 10406045 : }
785 :
786 : void
787 7487182 : ipa_fn_summary_t::remove_callees (cgraph_node *node)
788 : {
789 7487182 : cgraph_edge *e;
790 30533319 : for (e = node->callees; e; e = e->next_callee)
791 23046137 : ipa_call_summaries->remove (e);
792 7969547 : for (e = node->indirect_calls; e; e = e->next_callee)
793 482365 : ipa_call_summaries->remove (e);
794 7487182 : }
795 :
796 : /* Duplicate predicates in loop hint vector, allocating memory for them and
797 : remove and deallocate any uninteresting (true or false) ones. Return the
798 : result. */
799 :
800 : static vec<ipa_freqcounting_predicate, va_gc> *
801 28314 : remap_freqcounting_preds_after_dup (vec<ipa_freqcounting_predicate, va_gc> *v,
802 : clause_t possible_truths)
803 : {
804 28314 : if (vec_safe_length (v) == 0)
805 : return NULL;
806 :
807 4671 : vec<ipa_freqcounting_predicate, va_gc> *res = v->copy ();
808 4671 : int len = res->length();
809 10839 : for (int i = len - 1; i >= 0; i--)
810 : {
811 6168 : ipa_predicate new_predicate
812 6168 : = (*res)[i].predicate->remap_after_duplication (possible_truths);
813 : /* We do not want to free previous predicate; it is used by node
814 : origin. */
815 6168 : (*res)[i].predicate = NULL;
816 6168 : set_hint_predicate (&(*res)[i].predicate, new_predicate);
817 :
818 6168 : if (!(*res)[i].predicate)
819 2927 : res->unordered_remove (i);
820 : }
821 :
822 : return res;
823 : }
824 :
825 :
826 : /* Hook that is called by cgraph.cc when a node is duplicated. */
827 : void
828 2831149 : ipa_fn_summary_t::duplicate (cgraph_node *src,
829 : cgraph_node *dst,
830 : ipa_fn_summary *src_info,
831 : ipa_fn_summary *info)
832 : {
833 2831149 : new (info) ipa_fn_summary (*src_info);
834 : /* TODO: as an optimization, we may avoid copying conditions
835 : that are known to be false or true. */
836 2831149 : info->conds = vec_safe_copy (info->conds);
837 :
838 2831149 : clone_info *cinfo = clone_info::get (dst);
839 : /* When there are any replacements in the function body, see if we can figure
840 : out that something was optimized out. */
841 2831149 : if (ipa_node_params_sum && cinfo && cinfo->tree_map)
842 : {
843 : /* Use SRC parm info since it may not be copied yet. */
844 14157 : ipa_node_params *parms_info = ipa_node_params_sum->get (src);
845 14157 : ipa_auto_call_arg_values avals;
846 14157 : int count = ipa_get_param_count (parms_info);
847 14157 : int i, j;
848 14157 : clause_t possible_truths;
849 14157 : ipa_predicate true_pred = true;
850 14157 : size_time_entry *e;
851 14157 : int optimized_out_size = 0;
852 14157 : bool inlined_to_p = false;
853 14157 : struct cgraph_edge *edge, *next;
854 :
855 14157 : info->size_time_table.release ();
856 14157 : avals.m_known_vals.safe_grow_cleared (count, true);
857 71187 : for (i = 0; i < count; i++)
858 : {
859 : struct ipa_replace_map *r;
860 :
861 140172 : for (j = 0; vec_safe_iterate (cinfo->tree_map, j, &r); j++)
862 : {
863 78922 : if (r->parm_num == i)
864 : {
865 24496 : avals.m_known_vals[i] = r->new_tree;
866 24496 : break;
867 : }
868 : }
869 : }
870 14157 : evaluate_conditions_for_known_args (dst, false,
871 : &avals,
872 : &possible_truths,
873 : /* We are going to specialize,
874 : so ignore nonspec truths. */
875 : NULL,
876 : NULL);
877 :
878 14157 : info->account_size_time (0, 0, true_pred, true_pred);
879 :
880 : /* Remap size_time vectors.
881 : Simplify the predicate by pruning out alternatives that are known
882 : to be false.
883 : TODO: as on optimization, we can also eliminate conditions known
884 : to be true. */
885 98604 : for (i = 0; src_info->size_time_table.iterate (i, &e); i++)
886 : {
887 84447 : ipa_predicate new_exec_pred;
888 84447 : ipa_predicate new_nonconst_pred;
889 84447 : new_exec_pred = e->exec_predicate.remap_after_duplication
890 84447 : (possible_truths);
891 84447 : new_nonconst_pred = e->nonconst_predicate.remap_after_duplication
892 84447 : (possible_truths);
893 84447 : if (new_exec_pred == false || new_nonconst_pred == false)
894 11641 : optimized_out_size += e->size;
895 : else
896 72806 : info->account_size_time (e->size, e->time, new_exec_pred,
897 : new_nonconst_pred);
898 : }
899 :
900 : /* Remap edge predicates with the same simplification as above.
901 : Also copy constantness arrays. */
902 182936 : for (edge = dst->callees; edge; edge = next)
903 : {
904 168779 : ipa_predicate new_predicate;
905 168779 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
906 168779 : next = edge->next_callee;
907 :
908 168779 : if (!edge->inline_failed)
909 0 : inlined_to_p = true;
910 168779 : if (!es->predicate)
911 156858 : continue;
912 11921 : new_predicate = es->predicate->remap_after_duplication
913 11921 : (possible_truths);
914 11921 : if (new_predicate == false && *es->predicate != false)
915 3086 : optimized_out_size += es->call_stmt_size * ipa_fn_summary::size_scale;
916 11921 : edge_set_predicate (edge, &new_predicate);
917 : }
918 :
919 : /* Remap indirect edge predicates with the same simplification as above.
920 : Also copy constantness arrays. */
921 15053 : for (edge = dst->indirect_calls; edge; edge = next)
922 : {
923 896 : ipa_predicate new_predicate;
924 896 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
925 896 : next = edge->next_callee;
926 :
927 896 : gcc_checking_assert (edge->inline_failed);
928 896 : if (!es->predicate)
929 792 : continue;
930 104 : new_predicate = es->predicate->remap_after_duplication
931 104 : (possible_truths);
932 104 : if (new_predicate == false && *es->predicate != false)
933 21 : optimized_out_size
934 21 : += es->call_stmt_size * ipa_fn_summary::size_scale;
935 104 : edge_set_predicate (edge, &new_predicate);
936 : }
937 14157 : info->loop_iterations
938 14157 : = remap_freqcounting_preds_after_dup (info->loop_iterations,
939 : possible_truths);
940 14157 : info->loop_strides
941 14157 : = remap_freqcounting_preds_after_dup (info->loop_strides,
942 : possible_truths);
943 14157 : if (info->builtin_constant_p_parms.length())
944 : {
945 22 : vec <int, va_heap, vl_ptr> parms = info->builtin_constant_p_parms;
946 22 : int ip;
947 22 : info->builtin_constant_p_parms = vNULL;
948 44 : for (i = 0; parms.iterate (i, &ip); i++)
949 22 : if (!avals.m_known_vals[ip])
950 8 : info->builtin_constant_p_parms.safe_push (ip);
951 : }
952 :
953 : /* If inliner or someone after inliner will ever start producing
954 : non-trivial clones, we will get trouble with lack of information
955 : about updating self sizes, because size vectors already contains
956 : sizes of the callees. */
957 14157 : gcc_assert (!inlined_to_p || !optimized_out_size);
958 14157 : }
959 : else
960 : {
961 2816992 : info->size_time_table = src_info->size_time_table.copy ();
962 2816992 : info->loop_iterations = vec_safe_copy (src_info->loop_iterations);
963 2816992 : info->loop_strides = vec_safe_copy (info->loop_strides);
964 :
965 2816992 : info->builtin_constant_p_parms
966 2816992 : = info->builtin_constant_p_parms.copy ();
967 :
968 2816992 : ipa_freqcounting_predicate *f;
969 5683230 : for (int i = 0; vec_safe_iterate (info->loop_iterations, i, &f); i++)
970 : {
971 49246 : ipa_predicate p = *f->predicate;
972 49246 : f->predicate = NULL;
973 49246 : set_hint_predicate (&f->predicate, p);
974 : }
975 2841124 : for (int i = 0; vec_safe_iterate (info->loop_strides, i, &f); i++)
976 : {
977 24132 : ipa_predicate p = *f->predicate;
978 24132 : f->predicate = NULL;
979 24132 : set_hint_predicate (&f->predicate, p);
980 : }
981 : }
982 2831149 : if (!dst->inlined_to)
983 148049 : ipa_update_overall_fn_summary (dst);
984 2831149 : }
985 :
986 :
987 : /* Hook that is called by cgraph.cc when a node is duplicated. */
988 :
989 : void
990 4176428 : ipa_call_summary_t::duplicate (struct cgraph_edge *src,
991 : struct cgraph_edge *dst,
992 : class ipa_call_summary *srcinfo,
993 : class ipa_call_summary *info)
994 : {
995 4176428 : new (info) ipa_call_summary (*srcinfo);
996 4176428 : info->predicate = NULL;
997 4176428 : edge_set_predicate (dst, srcinfo->predicate);
998 4176428 : info->param = srcinfo->param.copy ();
999 4176428 : if (!dst->indirect_unknown_callee && src->indirect_unknown_callee
1000 : /* Don't subtract the size when dealing with callback pairs, since the
1001 : edge has no real size. */
1002 24087 : && !src->has_callback && !dst->callback)
1003 : {
1004 24087 : info->call_stmt_size -= (eni_size_weights.indirect_call_cost
1005 24087 : - eni_size_weights.call_cost);
1006 24087 : info->call_stmt_time -= (eni_time_weights.indirect_call_cost
1007 24087 : - eni_time_weights.call_cost);
1008 : }
1009 4176428 : }
1010 :
1011 : /* Dump edge summaries associated to NODE and recursively to all clones.
1012 : Indent by INDENT. */
1013 :
1014 : static void
1015 1969 : dump_ipa_call_summary (FILE *f, int indent, struct cgraph_node *node,
1016 : class ipa_fn_summary *info)
1017 : {
1018 1969 : struct cgraph_edge *edge;
1019 9787 : for (edge = node->callees; edge; edge = edge->next_callee)
1020 : {
1021 7818 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
1022 7818 : struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
1023 7818 : int i;
1024 :
1025 15636 : fprintf (f,
1026 : "%*s%s %s\n%*s freq:%4.2f",
1027 : indent, "", callee->dump_name (),
1028 7818 : !edge->inline_failed
1029 7265 : ? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
1030 7818 : indent, "", edge->sreal_frequency ().to_double ());
1031 :
1032 7818 : if (cross_module_call_p (edge))
1033 4 : fprintf (f, " cross module");
1034 :
1035 7818 : if (es)
1036 7265 : fprintf (f, " loop depth:%2i size:%2i time: %2i",
1037 : es->loop_depth, es->call_stmt_size, es->call_stmt_time);
1038 :
1039 7818 : ipa_fn_summary *s = ipa_fn_summaries->get (callee);
1040 7818 : ipa_size_summary *ss = ipa_size_summaries->get (callee);
1041 7818 : if (s != NULL)
1042 620 : fprintf (f, " callee size:%2i stack:%2i",
1043 620 : (int) (ss->size / ipa_fn_summary::size_scale),
1044 620 : (int) s->estimated_stack_size);
1045 :
1046 7818 : if (es && es->predicate)
1047 : {
1048 4627 : fprintf (f, " predicate: ");
1049 4627 : es->predicate->dump (f, info->conds);
1050 : }
1051 : else
1052 3191 : fprintf (f, "\n");
1053 7818 : if (es && es->param.exists ())
1054 5348 : for (i = 0; i < (int) es->param.length (); i++)
1055 : {
1056 1582 : int prob = es->param[i].change_prob;
1057 :
1058 1582 : if (!prob)
1059 620 : fprintf (f, "%*s op%i is compile time invariant\n",
1060 : indent + 2, "", i);
1061 962 : else if (prob != REG_BR_PROB_BASE)
1062 35 : fprintf (f, "%*s op%i change %f%% of time\n", indent + 2, "", i,
1063 35 : prob * 100.0 / REG_BR_PROB_BASE);
1064 1582 : if (es->param[i].points_to_local_or_readonly_memory)
1065 351 : fprintf (f, "%*s op%i points to local or readonly memory\n",
1066 : indent + 2, "", i);
1067 1582 : if (es->param[i].points_to_possible_sra_candidate)
1068 227 : fprintf (f, "%*s op%i points to possible sra candidate\n",
1069 : indent + 2, "", i);
1070 : }
1071 7818 : if (!edge->inline_failed)
1072 : {
1073 553 : ipa_size_summary *ss = ipa_size_summaries->get (callee);
1074 553 : fprintf (f, "%*sStack frame offset %i, callee self size %i\n",
1075 : indent + 2, "",
1076 553 : (int) ipa_get_stack_frame_offset (callee),
1077 553 : (int) ss->estimated_self_stack_size);
1078 553 : dump_ipa_call_summary (f, indent + 2, callee, info);
1079 : }
1080 : }
1081 2267 : for (edge = node->indirect_calls; edge; edge = edge->next_callee)
1082 : {
1083 298 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
1084 298 : fprintf (f, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
1085 : " time: %2i",
1086 : indent, "",
1087 : es->loop_depth,
1088 298 : edge->sreal_frequency ().to_double (), es->call_stmt_size,
1089 : es->call_stmt_time);
1090 298 : if (es->predicate)
1091 : {
1092 6 : fprintf (f, "predicate: ");
1093 6 : es->predicate->dump (f, info->conds);
1094 : }
1095 : else
1096 292 : fprintf (f, "\n");
1097 : }
1098 1969 : }
1099 :
1100 :
1101 : void
1102 1628 : ipa_dump_fn_summary (FILE *f, struct cgraph_node *node)
1103 : {
1104 1628 : if (node->definition)
1105 : {
1106 1628 : class ipa_fn_summary *s = ipa_fn_summaries->get (node);
1107 1628 : class ipa_size_summary *ss = ipa_size_summaries->get (node);
1108 1628 : if (s != NULL)
1109 : {
1110 1416 : size_time_entry *e;
1111 1416 : int i;
1112 1416 : fprintf (f, "IPA function summary for %s", node->dump_name ());
1113 1416 : if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
1114 0 : fprintf (f, " always_inline");
1115 1416 : if (s->inlinable)
1116 1235 : fprintf (f, " inlinable");
1117 1416 : if (s->fp_expressions)
1118 44 : fprintf (f, " fp_expression");
1119 1416 : if (s->builtin_constant_p_parms.length ())
1120 : {
1121 2 : fprintf (f, " builtin_constant_p_parms");
1122 4 : for (unsigned int i = 0;
1123 4 : i < s->builtin_constant_p_parms.length (); i++)
1124 2 : fprintf (f, " %i", s->builtin_constant_p_parms[i]);
1125 : }
1126 1416 : fprintf (f, "\n global time: %f\n", s->time.to_double ());
1127 1416 : fprintf (f, " self size: %i\n", ss->self_size);
1128 1416 : fprintf (f, " global size: %i\n", ss->size);
1129 1416 : fprintf (f, " min size: %i\n", s->min_size);
1130 1416 : fprintf (f, " self stack: %i\n",
1131 1416 : (int) ss->estimated_self_stack_size);
1132 1416 : fprintf (f, " global stack: %i\n", (int) s->estimated_stack_size);
1133 1416 : if (s->growth)
1134 168 : fprintf (f, " estimated growth:%i\n", (int) s->growth);
1135 1416 : if (s->scc_no)
1136 5 : fprintf (f, " In SCC: %i\n", (int) s->scc_no);
1137 5908 : for (i = 0; s->size_time_table.iterate (i, &e); i++)
1138 : {
1139 4492 : fprintf (f, " size:%f, time:%f",
1140 4492 : (double) e->size / ipa_fn_summary::size_scale,
1141 : e->time.to_double ());
1142 4492 : if (e->exec_predicate != true)
1143 : {
1144 2623 : fprintf (f, ", executed if:");
1145 2623 : e->exec_predicate.dump (f, s->conds, 0);
1146 : }
1147 4492 : if (e->exec_predicate != e->nonconst_predicate)
1148 : {
1149 1167 : fprintf (f, ", nonconst if:");
1150 1167 : e->nonconst_predicate.dump (f, s->conds, 0);
1151 : }
1152 4492 : fprintf (f, "\n");
1153 : }
1154 : ipa_freqcounting_predicate *fcp;
1155 : bool first_fcp = true;
1156 1522 : for (int i = 0; vec_safe_iterate (s->loop_iterations, i, &fcp); i++)
1157 : {
1158 106 : if (first_fcp)
1159 : {
1160 78 : fprintf (f, " loop iterations:");
1161 78 : first_fcp = false;
1162 : }
1163 106 : fprintf (f, " %3.2f for ", fcp->freq.to_double ());
1164 106 : fcp->predicate->dump (f, s->conds);
1165 : }
1166 : first_fcp = true;
1167 1434 : for (int i = 0; vec_safe_iterate (s->loop_strides, i, &fcp); i++)
1168 : {
1169 18 : if (first_fcp)
1170 : {
1171 10 : fprintf (f, " loop strides:");
1172 10 : first_fcp = false;
1173 : }
1174 18 : fprintf (f, " %3.2f for :", fcp->freq.to_double ());
1175 18 : fcp->predicate->dump (f, s->conds);
1176 : }
1177 1416 : fprintf (f, " calls:\n");
1178 1416 : dump_ipa_call_summary (f, 4, node, s);
1179 1416 : fprintf (f, "\n");
1180 1416 : if (s->target_info)
1181 0 : fprintf (f, " target_info: %x\n", s->target_info);
1182 : }
1183 : else
1184 212 : fprintf (f, "IPA summary for %s is missing.\n", node->dump_name ());
1185 : }
1186 1628 : }
1187 :
1188 : DEBUG_FUNCTION void
1189 0 : ipa_debug_fn_summary (struct cgraph_node *node)
1190 : {
1191 0 : ipa_dump_fn_summary (stderr, node);
1192 0 : }
1193 :
1194 : void
1195 356 : ipa_dump_fn_summaries (FILE *f)
1196 : {
1197 356 : struct cgraph_node *node;
1198 :
1199 2288 : FOR_EACH_DEFINED_FUNCTION (node)
1200 1932 : if (!node->inlined_to)
1201 1379 : ipa_dump_fn_summary (f, node);
1202 356 : }
1203 :
1204 : /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1205 : boolean variable pointed to by DATA. */
1206 :
1207 : static bool
1208 844780 : mark_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED,
1209 : void *data)
1210 : {
1211 844780 : bool *b = (bool *) data;
1212 844780 : *b = true;
1213 844780 : return true;
1214 : }
1215 :
1216 : /* If OP refers to value of function parameter, return the corresponding
1217 : parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
1218 : PARM_DECL) will be stored to *SIZE_P in that case too. */
1219 :
1220 : static tree
1221 189214795 : unmodified_parm_1 (ipa_func_body_info *fbi, gimple *stmt, tree op,
1222 : poly_int64 *size_p)
1223 : {
1224 : /* SSA_NAME referring to parm default def? */
1225 189214795 : if (TREE_CODE (op) == SSA_NAME
1226 110998469 : && SSA_NAME_IS_DEFAULT_DEF (op)
1227 215523435 : && TREE_CODE (SSA_NAME_VAR (op)) == PARM_DECL)
1228 : {
1229 26095311 : if (size_p)
1230 0 : *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
1231 26095311 : return SSA_NAME_VAR (op);
1232 : }
1233 : /* Non-SSA parm reference? */
1234 163119484 : if (TREE_CODE (op) == PARM_DECL
1235 1426897 : && fbi->aa_walk_budget > 0)
1236 : {
1237 1424351 : bool modified = false;
1238 :
1239 1424351 : ao_ref refd;
1240 1424351 : ao_ref_init (&refd, op);
1241 2848702 : int walked = walk_aliased_vdefs (&refd, gimple_vuse (stmt),
1242 : mark_modified, &modified, NULL, NULL,
1243 : fbi->aa_walk_budget);
1244 1424351 : if (walked < 0)
1245 : {
1246 8 : fbi->aa_walk_budget = 0;
1247 913174 : return NULL_TREE;
1248 : }
1249 1424343 : fbi->aa_walk_budget -= walked;
1250 1424343 : if (!modified)
1251 : {
1252 913166 : if (size_p)
1253 0 : *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
1254 : return op;
1255 : }
1256 : }
1257 : return NULL_TREE;
1258 : }
1259 :
1260 : /* If OP refers to value of function parameter, return the corresponding
1261 : parameter. Also traverse chains of SSA register assignments. If non-NULL,
1262 : the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
1263 : stored to *SIZE_P in that case too. */
1264 :
1265 : static tree
1266 123796031 : unmodified_parm (ipa_func_body_info *fbi, gimple *stmt, tree op,
1267 : poly_int64 *size_p)
1268 : {
1269 153935108 : tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
1270 153935108 : if (res)
1271 : return res;
1272 :
1273 127778174 : if (TREE_CODE (op) == SSA_NAME
1274 72631254 : && !SSA_NAME_IS_DEFAULT_DEF (op)
1275 200200826 : && gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1276 30139077 : return unmodified_parm (fbi, SSA_NAME_DEF_STMT (op),
1277 30139077 : gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op)),
1278 30139077 : size_p);
1279 : return NULL_TREE;
1280 : }
1281 :
1282 : /* If OP refers to a value of a function parameter or value loaded from an
1283 : aggregate passed to a parameter (either by value or reference), return TRUE
1284 : and store the number of the parameter to *INDEX_P, the access size into
1285 : *SIZE_P, and information whether and how it has been loaded from an
1286 : aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1287 : statement in which OP is used or loaded. */
1288 :
1289 : static bool
1290 33822834 : unmodified_parm_or_parm_agg_item (struct ipa_func_body_info *fbi,
1291 : gimple *stmt, tree op, int *index_p,
1292 : poly_int64 *size_p,
1293 : struct agg_position_info *aggpos)
1294 : {
1295 35279687 : tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
1296 :
1297 35279687 : gcc_checking_assert (aggpos);
1298 35279687 : if (res)
1299 : {
1300 851543 : *index_p = ipa_get_param_decl_index (fbi->info, res);
1301 851543 : if (*index_p < 0)
1302 : return false;
1303 851413 : aggpos->agg_contents = false;
1304 851413 : aggpos->by_ref = false;
1305 851413 : return true;
1306 : }
1307 :
1308 34428144 : if (TREE_CODE (op) == SSA_NAME)
1309 : {
1310 12271904 : if (SSA_NAME_IS_DEFAULT_DEF (op)
1311 12271904 : || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1312 : return false;
1313 4914560 : stmt = SSA_NAME_DEF_STMT (op);
1314 4914560 : op = gimple_assign_rhs1 (stmt);
1315 4914560 : if (!REFERENCE_CLASS_P (op))
1316 : return unmodified_parm_or_parm_agg_item (fbi, stmt, op, index_p, size_p,
1317 : aggpos);
1318 : }
1319 :
1320 25613947 : aggpos->agg_contents = true;
1321 25613947 : return ipa_load_from_parm_agg (fbi, fbi->info->descriptors,
1322 : stmt, op, index_p, &aggpos->offset,
1323 25613947 : size_p, &aggpos->by_ref);
1324 : }
1325 :
1326 : /* If stmt is simple load or store of value pointed to by a function parameter,
1327 : return its index. */
1328 :
1329 : static int
1330 113950412 : load_or_store_of_ptr_parameter (ipa_func_body_info *fbi, gimple *stmt)
1331 : {
1332 113950412 : if (!optimize)
1333 : return -1;
1334 95502859 : gassign *assign = dyn_cast <gassign *> (stmt);
1335 56774914 : if (!assign)
1336 : return -1;
1337 56774914 : tree param;
1338 56774914 : if (gimple_assign_load_p (stmt))
1339 20815000 : param = gimple_assign_rhs1 (stmt);
1340 35959914 : else if (gimple_store_p (stmt))
1341 19089240 : param = gimple_assign_lhs (stmt);
1342 : else
1343 : return -1;
1344 39904240 : tree base = get_base_address (param);
1345 39904240 : if (TREE_CODE (base) != MEM_REF
1346 13913147 : || TREE_CODE (TREE_OPERAND (base, 0)) != SSA_NAME
1347 53813265 : || !SSA_NAME_IS_DEFAULT_DEF (TREE_OPERAND (base, 0)))
1348 : return -1;
1349 7536395 : tree p = SSA_NAME_VAR (TREE_OPERAND (base, 0));
1350 7536395 : if (TREE_CODE (p) != PARM_DECL)
1351 : return -1;
1352 7444684 : return ipa_get_param_decl_index (fbi->info, p);
1353 : }
1354 :
1355 : /* See if statement might disappear after inlining.
1356 : 0 - means not eliminated
1357 : 1 - half of statements goes away
1358 : 2 - for sure it is eliminated.
1359 : We are not terribly sophisticated, basically looking for simple abstraction
1360 : penalty wrappers. */
1361 :
1362 : static int
1363 113950412 : eliminated_by_inlining_prob (ipa_func_body_info *fbi, gimple *stmt)
1364 : {
1365 113950412 : enum gimple_code code = gimple_code (stmt);
1366 113950412 : enum tree_code rhs_code;
1367 :
1368 113950412 : if (!optimize)
1369 : return 0;
1370 :
1371 95502859 : switch (code)
1372 : {
1373 : case GIMPLE_RETURN:
1374 : return 2;
1375 56774914 : case GIMPLE_ASSIGN:
1376 56774914 : if (gimple_num_ops (stmt) != 2)
1377 : return 0;
1378 :
1379 40726783 : rhs_code = gimple_assign_rhs_code (stmt);
1380 :
1381 : /* Casts of parameters, loads from parameters passed by reference
1382 : and stores to return value or parameters are often free after
1383 : inlining due to SRA and further combining.
1384 : Assume that half of statements goes away. */
1385 40726783 : if (CONVERT_EXPR_CODE_P (rhs_code)
1386 : || rhs_code == VIEW_CONVERT_EXPR
1387 : || rhs_code == ADDR_EXPR
1388 37858625 : || gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS)
1389 : {
1390 39904240 : tree rhs = gimple_assign_rhs1 (stmt);
1391 39904240 : tree lhs = gimple_assign_lhs (stmt);
1392 39904240 : tree inner_rhs = get_base_address (rhs);
1393 39904240 : tree inner_lhs = get_base_address (lhs);
1394 39904240 : bool rhs_free = false;
1395 39904240 : bool lhs_free = false;
1396 :
1397 39904240 : if (!inner_rhs)
1398 0 : inner_rhs = rhs;
1399 39904240 : if (!inner_lhs)
1400 0 : inner_lhs = lhs;
1401 :
1402 : /* Reads of parameter are expected to be free. */
1403 39904240 : if (unmodified_parm (fbi, stmt, inner_rhs, NULL))
1404 : rhs_free = true;
1405 : /* Match expressions of form &this->field. Those will most likely
1406 : combine with something upstream after inlining. */
1407 38585252 : else if (TREE_CODE (inner_rhs) == ADDR_EXPR)
1408 : {
1409 2579752 : tree op = get_base_address (TREE_OPERAND (inner_rhs, 0));
1410 2579752 : if (TREE_CODE (op) == PARM_DECL)
1411 : rhs_free = true;
1412 2535473 : else if (TREE_CODE (op) == MEM_REF
1413 2535473 : && unmodified_parm (fbi, stmt, TREE_OPERAND (op, 0),
1414 : NULL))
1415 : rhs_free = true;
1416 : }
1417 :
1418 : /* When parameter is not SSA register because its address is taken
1419 : and it is just copied into one, the statement will be completely
1420 : free after inlining (we will copy propagate backward). */
1421 1363267 : if (rhs_free && is_gimple_reg (lhs))
1422 : return 2;
1423 :
1424 : /* Reads of parameters passed by reference
1425 : expected to be free (i.e. optimized out after inlining). */
1426 39220004 : if (TREE_CODE (inner_rhs) == MEM_REF
1427 39220004 : && unmodified_parm (fbi, stmt, TREE_OPERAND (inner_rhs, 0), NULL))
1428 : rhs_free = true;
1429 :
1430 : /* Copying parameter passed by reference into gimple register is
1431 : probably also going to copy propagate, but we can't be quite
1432 : sure. */
1433 39220004 : if (rhs_free && is_gimple_reg (lhs))
1434 : lhs_free = true;
1435 :
1436 : /* Writes to parameters, parameters passed by value and return value
1437 : (either directly or passed via invisible reference) are free.
1438 :
1439 : TODO: We ought to handle testcase like
1440 : struct a {int a,b;};
1441 : struct a
1442 : returnstruct (void)
1443 : {
1444 : struct a a ={1,2};
1445 : return a;
1446 : }
1447 :
1448 : This translate into:
1449 :
1450 : returnstruct ()
1451 : {
1452 : int a$b;
1453 : int a$a;
1454 : struct a a;
1455 : struct a D.2739;
1456 :
1457 : <bb 2>:
1458 : D.2739.a = 1;
1459 : D.2739.b = 2;
1460 : return D.2739;
1461 :
1462 : }
1463 : For that we either need to copy ipa-split logic detecting writes
1464 : to return value. */
1465 39220004 : if (TREE_CODE (inner_lhs) == PARM_DECL
1466 39130224 : || TREE_CODE (inner_lhs) == RESULT_DECL
1467 77502446 : || (TREE_CODE (inner_lhs) == MEM_REF
1468 5203497 : && (unmodified_parm (fbi, stmt, TREE_OPERAND (inner_lhs, 0),
1469 : NULL)
1470 2677588 : || (TREE_CODE (TREE_OPERAND (inner_lhs, 0)) == SSA_NAME
1471 2676400 : && SSA_NAME_VAR (TREE_OPERAND (inner_lhs, 0))
1472 378400 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1473 : (inner_lhs,
1474 : 0))) == RESULT_DECL))))
1475 : lhs_free = true;
1476 35665021 : if (lhs_free
1477 39220004 : && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs)))
1478 : rhs_free = true;
1479 39220004 : if (lhs_free && rhs_free)
1480 8223095 : return 1;
1481 : }
1482 : return 0;
1483 : default:
1484 : return 0;
1485 : }
1486 : }
1487 :
1488 : /* Analyze EXPR if it represents a series of simple operations performed on
1489 : a function parameter and return true if so. FBI, STMT, EXPR, INDEX_P and
1490 : AGGPOS have the same meaning like in unmodified_parm_or_parm_agg_item.
1491 : Type of the parameter or load from an aggregate via the parameter is
1492 : stored in *TYPE_P. Operations on the parameter are recorded to
1493 : PARAM_OPS_P if it is not NULL. */
1494 :
1495 : static bool
1496 27785795 : decompose_param_expr (struct ipa_func_body_info *fbi,
1497 : gimple *stmt, tree expr,
1498 : int *index_p, tree *type_p,
1499 : struct agg_position_info *aggpos,
1500 : expr_eval_ops *param_ops_p = NULL)
1501 : {
1502 27785795 : int op_limit = opt_for_fn (fbi->node->decl, param_ipa_max_param_expr_ops);
1503 27785795 : int op_count = 0;
1504 :
1505 27785795 : if (param_ops_p)
1506 9509066 : *param_ops_p = NULL;
1507 :
1508 33822834 : while (true)
1509 : {
1510 33822834 : expr_eval_op eval_op;
1511 33822834 : unsigned rhs_count;
1512 33822834 : unsigned cst_count = 0;
1513 :
1514 33822834 : if (unmodified_parm_or_parm_agg_item (fbi, stmt, expr, index_p, NULL,
1515 : aggpos))
1516 : {
1517 4746451 : tree type = TREE_TYPE (expr);
1518 :
1519 4746451 : if (aggpos->agg_contents)
1520 : {
1521 : /* Stop if containing bit-field. */
1522 3895038 : if (TREE_CODE (expr) == BIT_FIELD_REF
1523 3895038 : || contains_bitfld_component_ref_p (expr))
1524 : break;
1525 : }
1526 :
1527 4719336 : *type_p = type;
1528 4719336 : return true;
1529 : }
1530 :
1531 29076383 : if (TREE_CODE (expr) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (expr))
1532 : break;
1533 10919309 : stmt = SSA_NAME_DEF_STMT (expr);
1534 :
1535 10919309 : if (gcall *call = dyn_cast <gcall *> (stmt))
1536 : {
1537 2585746 : int flags = gimple_call_return_flags (call);
1538 2585746 : if (!(flags & ERF_RETURNS_ARG))
1539 3169948 : goto fail;
1540 235107 : int arg = flags & ERF_RETURN_ARG_MASK;
1541 235107 : if (arg >= (int)gimple_call_num_args (call))
1542 0 : goto fail;
1543 235107 : expr = gimple_call_arg (stmt, arg);
1544 4145984 : continue;
1545 235107 : }
1546 :
1547 8333563 : if (!is_gimple_assign (stmt = SSA_NAME_DEF_STMT (expr)))
1548 : break;
1549 :
1550 6621327 : switch (gimple_assign_rhs_class (stmt))
1551 : {
1552 3910877 : case GIMPLE_SINGLE_RHS:
1553 3910877 : expr = gimple_assign_rhs1 (stmt);
1554 3910877 : continue;
1555 :
1556 : case GIMPLE_UNARY_RHS:
1557 : rhs_count = 1;
1558 : break;
1559 :
1560 1625312 : case GIMPLE_BINARY_RHS:
1561 1625312 : rhs_count = 2;
1562 1625312 : break;
1563 :
1564 696 : case GIMPLE_TERNARY_RHS:
1565 696 : rhs_count = 3;
1566 696 : break;
1567 :
1568 0 : default:
1569 0 : goto fail;
1570 : }
1571 :
1572 : /* Stop if expression is too complex. */
1573 2710450 : if (op_count++ == op_limit)
1574 : break;
1575 :
1576 2710364 : if (param_ops_p)
1577 : {
1578 2676079 : eval_op.code = gimple_assign_rhs_code (stmt);
1579 2676079 : eval_op.type = TREE_TYPE (gimple_assign_lhs (stmt));
1580 2676079 : eval_op.val[0] = NULL_TREE;
1581 2676079 : eval_op.val[1] = NULL_TREE;
1582 : }
1583 :
1584 2710364 : expr = NULL_TREE;
1585 6227434 : for (unsigned i = 0; i < rhs_count; i++)
1586 : {
1587 4336379 : tree op = gimple_op (stmt, i + 1);
1588 :
1589 4336379 : gcc_assert (op && !TYPE_P (op));
1590 4336379 : if (is_gimple_ip_invariant (op))
1591 : {
1592 809024 : if (++cst_count == rhs_count)
1593 1159 : goto fail;
1594 :
1595 807865 : eval_op.val[cst_count - 1] = op;
1596 : }
1597 3527355 : else if (!expr)
1598 : {
1599 : /* Found a non-constant operand, and record its index in rhs
1600 : operands. */
1601 2709205 : eval_op.index = i;
1602 2709205 : expr = op;
1603 : }
1604 : else
1605 : {
1606 : /* Found more than one non-constant operands. */
1607 818150 : goto fail;
1608 : }
1609 : }
1610 :
1611 1891055 : if (param_ops_p)
1612 1857386 : vec_safe_insert (*param_ops_p, 0, eval_op);
1613 : }
1614 :
1615 : /* Failed to decompose, free resource and return. */
1616 23066459 : fail:
1617 23066459 : if (param_ops_p)
1618 9284547 : vec_free (*param_ops_p);
1619 :
1620 : return false;
1621 : }
1622 :
1623 : /* Record to SUMMARY that PARM is used by builtin_constant_p. */
1624 :
1625 : static void
1626 11955 : add_builtin_constant_p_parm (class ipa_fn_summary *summary, int parm)
1627 : {
1628 11955 : int ip;
1629 :
1630 : /* Avoid duplicates. */
1631 12013 : for (unsigned int i = 0;
1632 12013 : summary->builtin_constant_p_parms.iterate (i, &ip); i++)
1633 5929 : if (ip == parm)
1634 11955 : return;
1635 6084 : summary->builtin_constant_p_parms.safe_push (parm);
1636 : }
1637 :
1638 : /* If BB ends by a conditional we can turn into predicates, attach corresponding
1639 : predicates to the CFG edges. */
1640 :
1641 : static void
1642 35991680 : set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
1643 : class ipa_fn_summary *summary,
1644 : class ipa_node_params *params_summary,
1645 : basic_block bb)
1646 : {
1647 35991680 : tree op, op2;
1648 35991680 : int index;
1649 35991680 : struct agg_position_info aggpos;
1650 35991680 : enum tree_code code, inverted_code;
1651 35991680 : edge e;
1652 35991680 : edge_iterator ei;
1653 35991680 : gimple *set_stmt;
1654 35991680 : tree param_type;
1655 35991680 : expr_eval_ops param_ops;
1656 :
1657 71983360 : gcond *last = safe_dyn_cast <gcond *> (*gsi_last_bb (bb));
1658 11879415 : if (!last)
1659 35978714 : return;
1660 11879415 : if (!is_gimple_ip_invariant (gimple_cond_rhs (last)))
1661 : return;
1662 9427219 : op = gimple_cond_lhs (last);
1663 :
1664 9427219 : if (decompose_param_expr (fbi, last, op, &index, ¶m_type, &aggpos,
1665 : ¶m_ops))
1666 : {
1667 1357518 : code = gimple_cond_code (last);
1668 1357518 : inverted_code = invert_tree_comparison (code, HONOR_NANS (op));
1669 :
1670 4072554 : FOR_EACH_EDGE (e, ei, bb->succs)
1671 : {
1672 1357518 : enum tree_code this_code = (e->flags & EDGE_TRUE_VALUE
1673 2715036 : ? code : inverted_code);
1674 : /* invert_tree_comparison will return ERROR_MARK on FP
1675 : comparisons that are not EQ/NE instead of returning proper
1676 : unordered one. Be sure it is not confused with NON_CONSTANT.
1677 :
1678 : And if the edge's target is the final block of diamond CFG graph
1679 : of this conditional statement, we do not need to compute
1680 : predicate for the edge because the final block's predicate must
1681 : be at least as that of the first block of the statement. */
1682 2715036 : if (this_code != ERROR_MARK
1683 2715036 : && !dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1684 : {
1685 2300695 : ipa_predicate p
1686 2300695 : = add_condition (summary, params_summary, index,
1687 : param_type, &aggpos,
1688 : this_code, gimple_cond_rhs (last), param_ops);
1689 2300695 : e->aux = edge_predicate_pool.allocate ();
1690 2300695 : *(ipa_predicate *) e->aux = p;
1691 : }
1692 : }
1693 36201907 : vec_free (param_ops);
1694 : return;
1695 : }
1696 :
1697 8069701 : if (TREE_CODE (op) != SSA_NAME)
1698 : return;
1699 : /* Special case
1700 : if (builtin_constant_p (op))
1701 : constant_code
1702 : else
1703 : nonconstant_code.
1704 : Here we can predicate nonconstant_code. We can't
1705 : really handle constant_code since we have no predicate
1706 : for this and also the constant code is not known to be
1707 : optimized away when inliner doesn't see operand is constant.
1708 : Other optimizers might think otherwise. */
1709 8068616 : if (gimple_cond_code (last) != NE_EXPR
1710 8068616 : || !integer_zerop (gimple_cond_rhs (last)))
1711 : return;
1712 3515942 : set_stmt = SSA_NAME_DEF_STMT (op);
1713 3515942 : if (!gimple_call_builtin_p (set_stmt, BUILT_IN_CONSTANT_P)
1714 3515942 : || gimple_call_num_args (set_stmt) != 1)
1715 : return;
1716 46342 : op2 = gimple_call_arg (set_stmt, 0);
1717 46342 : if (!decompose_param_expr (fbi, set_stmt, op2, &index, ¶m_type, &aggpos))
1718 : return;
1719 12966 : if (!aggpos.by_ref)
1720 11907 : add_builtin_constant_p_parm (summary, index);
1721 38898 : FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
1722 : {
1723 12966 : ipa_predicate p = add_condition (summary, params_summary, index,
1724 : param_type, &aggpos,
1725 : ipa_predicate::is_not_constant, NULL_TREE);
1726 12966 : e->aux = edge_predicate_pool.allocate ();
1727 12966 : *(ipa_predicate *) e->aux = p;
1728 : }
1729 : }
1730 :
1731 :
1732 : /* If BB ends by a switch we can turn into predicates, attach corresponding
1733 : predicates to the CFG edges. */
1734 :
1735 : static void
1736 35991680 : set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
1737 : class ipa_fn_summary *summary,
1738 : class ipa_node_params *params_summary,
1739 : basic_block bb)
1740 : {
1741 35991680 : tree op;
1742 35991680 : int index;
1743 35991680 : struct agg_position_info aggpos;
1744 35991680 : edge e;
1745 35991680 : edge_iterator ei;
1746 35991680 : size_t n;
1747 35991680 : size_t case_idx;
1748 35991680 : tree param_type;
1749 35991680 : expr_eval_ops param_ops;
1750 :
1751 71983360 : gswitch *last = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb));
1752 81847 : if (!last)
1753 35962157 : return;
1754 81847 : op = gimple_switch_index (last);
1755 81847 : if (!decompose_param_expr (fbi, last, op, &index, ¶m_type, &aggpos,
1756 : ¶m_ops))
1757 : return;
1758 :
1759 44635 : auto_vec<std::pair<tree, tree> > ranges;
1760 44635 : tree type = TREE_TYPE (op);
1761 44635 : int bound_limit = opt_for_fn (fbi->node->decl,
1762 : param_ipa_max_switch_predicate_bounds);
1763 44635 : int bound_count = 0;
1764 : // This can safely be an integer range, as switches can only hold
1765 : // integers.
1766 44635 : int_range<2> vr;
1767 :
1768 89270 : get_range_query (cfun)->range_of_expr (vr, op);
1769 44635 : if (vr.undefined_p ())
1770 0 : vr.set_varying (TREE_TYPE (op));
1771 44635 : tree vr_min, vr_max;
1772 : // TODO: This entire function could use a rewrite to use the irange
1773 : // API, instead of trying to recreate its intersection/union logic.
1774 : // Any use of get_legacy_range() is a serious code smell.
1775 44635 : value_range_kind vr_type = get_legacy_range (vr, vr_min, vr_max);
1776 44635 : wide_int vr_wmin = wi::to_wide (vr_min);
1777 44635 : wide_int vr_wmax = wi::to_wide (vr_max);
1778 :
1779 308831 : FOR_EACH_EDGE (e, ei, bb->succs)
1780 : {
1781 264196 : e->aux = edge_predicate_pool.allocate ();
1782 264196 : *(ipa_predicate *) e->aux = false;
1783 : }
1784 :
1785 44635 : e = gimple_switch_edge (cfun, last, 0);
1786 : /* Set BOUND_COUNT to maximum count to bypass computing predicate for
1787 : default case if its target basic block is in convergence point of all
1788 : switch cases, which can be determined by checking whether it
1789 : post-dominates the switch statement. */
1790 44635 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1791 12603 : bound_count = INT_MAX;
1792 :
1793 44635 : n = gimple_switch_num_labels (last);
1794 295271 : for (case_idx = 1; case_idx < n; ++case_idx)
1795 : {
1796 250636 : tree cl = gimple_switch_label (last, case_idx);
1797 250636 : tree min = CASE_LOW (cl);
1798 250636 : tree max = CASE_HIGH (cl);
1799 250636 : ipa_predicate p;
1800 :
1801 250636 : e = gimple_switch_edge (cfun, last, case_idx);
1802 :
1803 : /* The case value might not have same type as switch expression,
1804 : extend the value based on the expression type. */
1805 250636 : if (TREE_TYPE (min) != type)
1806 44908 : min = wide_int_to_tree (type, wi::to_wide (min));
1807 :
1808 250636 : if (!max)
1809 : max = min;
1810 12477 : else if (TREE_TYPE (max) != type)
1811 3427 : max = wide_int_to_tree (type, wi::to_wide (max));
1812 :
1813 : /* The case's target basic block is in convergence point of all switch
1814 : cases, its predicate should be at least as that of the switch
1815 : statement. */
1816 250636 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1817 5189 : p = true;
1818 245447 : else if (min == max)
1819 234886 : p = add_condition (summary, params_summary, index, param_type,
1820 : &aggpos, EQ_EXPR, min, param_ops);
1821 : else
1822 : {
1823 10561 : ipa_predicate p1, p2;
1824 10561 : p1 = add_condition (summary, params_summary, index, param_type,
1825 : &aggpos, GE_EXPR, min, param_ops);
1826 10561 : p2 = add_condition (summary, params_summary,index, param_type,
1827 : &aggpos, LE_EXPR, max, param_ops);
1828 10561 : p = p1 & p2;
1829 : }
1830 250636 : *(ipa_predicate *) e->aux
1831 250636 : = p.or_with (summary->conds, *(ipa_predicate *) e->aux);
1832 :
1833 : /* If there are too many disjoint case ranges, predicate for default
1834 : case might become too complicated. So add a limit here. */
1835 250636 : if (bound_count > bound_limit)
1836 97889 : continue;
1837 :
1838 152747 : bool new_range = true;
1839 :
1840 152747 : if (!ranges.is_empty ())
1841 : {
1842 120715 : wide_int curr_wmin = wi::to_wide (min);
1843 120715 : wide_int last_wmax = wi::to_wide (ranges.last ().second);
1844 :
1845 : /* Merge case ranges if they are continuous. */
1846 120715 : if (curr_wmin == last_wmax + 1)
1847 : new_range = false;
1848 41874 : else if (vr_type == VR_ANTI_RANGE)
1849 : {
1850 : /* If two disjoint case ranges can be connected by anti-range
1851 : of switch index, combine them to one range. */
1852 141 : if (wi::lt_p (vr_wmax, curr_wmin - 1, TYPE_SIGN (type)))
1853 : vr_type = VR_UNDEFINED;
1854 0 : else if (wi::le_p (vr_wmin, last_wmax + 1, TYPE_SIGN (type)))
1855 0 : new_range = false;
1856 : }
1857 120715 : }
1858 :
1859 : /* Create/extend a case range. And we count endpoints of range set,
1860 : this number nearly equals to number of conditions that we will create
1861 : for predicate of default case. */
1862 120715 : if (new_range)
1863 : {
1864 73906 : bound_count += (min == max) ? 1 : 2;
1865 73906 : ranges.safe_push (std::make_pair (min, max));
1866 : }
1867 : else
1868 : {
1869 78841 : bound_count += (ranges.last ().first == ranges.last ().second);
1870 78841 : ranges.last ().second = max;
1871 : }
1872 : }
1873 :
1874 44635 : e = gimple_switch_edge (cfun, last, 0);
1875 44635 : if (bound_count > bound_limit)
1876 : {
1877 15112 : *(ipa_predicate *) e->aux = true;
1878 15112 : vec_free (param_ops);
1879 15112 : return;
1880 : }
1881 :
1882 29523 : ipa_predicate p_seg = true;
1883 29523 : ipa_predicate p_all = false;
1884 :
1885 29523 : if (vr_type != VR_RANGE)
1886 : {
1887 28923 : vr_wmin = wi::to_wide (TYPE_MIN_VALUE (type));
1888 28923 : vr_wmax = wi::to_wide (TYPE_MAX_VALUE (type));
1889 : }
1890 :
1891 : /* Construct predicate to represent default range set that is negation of
1892 : all case ranges. Case range is classified as containing single/non-single
1893 : values. Suppose a piece of case ranges in the following.
1894 :
1895 : [D1...D2] [S1] ... [Sn] [D3...D4]
1896 :
1897 : To represent default case's range sets between two non-single value
1898 : case ranges (From D2 to D3), we construct predicate as:
1899 :
1900 : D2 < x < D3 && x != S1 && ... && x != Sn
1901 : */
1902 90996 : for (size_t i = 0; i < ranges.length (); i++)
1903 : {
1904 61568 : tree min = ranges[i].first;
1905 61568 : tree max = ranges[i].second;
1906 :
1907 61568 : if (min == max)
1908 73378 : p_seg &= add_condition (summary, params_summary, index,
1909 : param_type, &aggpos, NE_EXPR,
1910 36689 : min, param_ops);
1911 : else
1912 : {
1913 : /* Do not create sub-predicate for range that is beyond low bound
1914 : of switch index. */
1915 24879 : if (wi::lt_p (vr_wmin, wi::to_wide (min), TYPE_SIGN (type)))
1916 : {
1917 17797 : p_seg &= add_condition (summary, params_summary, index,
1918 : param_type, &aggpos,
1919 17797 : LT_EXPR, min, param_ops);
1920 17797 : p_all = p_all.or_with (summary->conds, p_seg);
1921 : }
1922 :
1923 : /* Do not create sub-predicate for range that is beyond up bound
1924 : of switch index. */
1925 24879 : if (wi::le_p (vr_wmax, wi::to_wide (max), TYPE_SIGN (type)))
1926 : {
1927 95 : p_seg = false;
1928 95 : break;
1929 : }
1930 :
1931 24784 : p_seg = add_condition (summary, params_summary, index,
1932 : param_type, &aggpos, GT_EXPR,
1933 : max, param_ops);
1934 : }
1935 : }
1936 :
1937 29523 : p_all = p_all.or_with (summary->conds, p_seg);
1938 29523 : *(ipa_predicate *) e->aux
1939 29523 : = p_all.or_with (summary->conds, *(ipa_predicate *) e->aux);
1940 :
1941 36100 : vec_free (param_ops);
1942 44635 : }
1943 :
1944 :
1945 : /* For each BB in NODE attach to its AUX pointer predicate under
1946 : which it is executable. */
1947 :
1948 : static void
1949 6552319 : compute_bb_predicates (struct ipa_func_body_info *fbi,
1950 : struct cgraph_node *node,
1951 : class ipa_fn_summary *summary,
1952 : class ipa_node_params *params_summary)
1953 : {
1954 6552319 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
1955 6552319 : bool done = false;
1956 6552319 : basic_block bb;
1957 :
1958 42543999 : FOR_EACH_BB_FN (bb, my_function)
1959 : {
1960 35991680 : set_cond_stmt_execution_predicate (fbi, summary, params_summary, bb);
1961 35991680 : set_switch_stmt_execution_predicate (fbi, summary, params_summary, bb);
1962 : }
1963 :
1964 : /* Entry block is always executable. */
1965 6552319 : ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
1966 6552319 : = edge_predicate_pool.allocate ();
1967 6552319 : *(ipa_predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux = true;
1968 :
1969 : /* A simple dataflow propagation of predicates forward in the CFG.
1970 : TODO: work in reverse postorder. */
1971 19776431 : while (!done)
1972 : {
1973 13224112 : done = true;
1974 90317563 : FOR_EACH_BB_FN (bb, my_function)
1975 : {
1976 77093451 : ipa_predicate p = false;
1977 77093451 : edge e;
1978 77093451 : edge_iterator ei;
1979 95324103 : FOR_EACH_EDGE (e, ei, bb->preds)
1980 : {
1981 81836477 : if (e->src->aux)
1982 : {
1983 80109973 : ipa_predicate this_bb_predicate
1984 : = *(ipa_predicate *) e->src->aux;
1985 80109973 : if (e->aux)
1986 5358292 : this_bb_predicate &= (*(ipa_predicate *) e->aux);
1987 80109973 : p = p.or_with (summary->conds, this_bb_predicate);
1988 80109973 : if (p == true)
1989 : break;
1990 : }
1991 : }
1992 77093451 : if (p != false)
1993 : {
1994 75964141 : basic_block pdom_bb;
1995 :
1996 75964141 : if (!bb->aux)
1997 : {
1998 28225835 : done = false;
1999 28225835 : bb->aux = edge_predicate_pool.allocate ();
2000 28225835 : *((ipa_predicate *) bb->aux) = p;
2001 : }
2002 47738306 : else if (p != *(ipa_predicate *) bb->aux)
2003 : {
2004 : /* This OR operation is needed to ensure monotonous data flow
2005 : in the case we hit the limit on number of clauses and the
2006 : and/or operations above give approximate answers. */
2007 129659 : p = p.or_with (summary->conds, *(ipa_predicate *)bb->aux);
2008 129659 : if (p != *(ipa_predicate *)bb->aux)
2009 : {
2010 105106 : done = false;
2011 105106 : *((ipa_predicate *)bb->aux) = p;
2012 : }
2013 : }
2014 :
2015 : /* For switch/if statement, we can OR-combine predicates of all
2016 : its cases/branches to get predicate for basic block in their
2017 : convergence point, but sometimes this will generate very
2018 : complicated predicate. Actually, we can get simplified
2019 : predicate in another way by using the fact that predicate
2020 : for a basic block must also hold true for its post dominators.
2021 : To be specific, basic block in convergence point of
2022 : conditional statement should include predicate of the
2023 : statement. */
2024 75964141 : pdom_bb = get_immediate_dominator (CDI_POST_DOMINATORS, bb);
2025 75964141 : if (pdom_bb == EXIT_BLOCK_PTR_FOR_FN (my_function) || !pdom_bb)
2026 : ;
2027 38915553 : else if (!pdom_bb->aux)
2028 : {
2029 7765831 : done = false;
2030 7765831 : pdom_bb->aux = edge_predicate_pool.allocate ();
2031 7765831 : *((ipa_predicate *)pdom_bb->aux) = p;
2032 : }
2033 31149722 : else if (p != *(ipa_predicate *)pdom_bb->aux)
2034 : {
2035 3404036 : p = p.or_with (summary->conds,
2036 : *(ipa_predicate *)pdom_bb->aux);
2037 3404036 : if (p != *(ipa_predicate *)pdom_bb->aux)
2038 : {
2039 140821 : done = false;
2040 140821 : *((ipa_predicate *)pdom_bb->aux) = p;
2041 : }
2042 : }
2043 : }
2044 : }
2045 : }
2046 6552319 : }
2047 :
2048 :
2049 : /* Return predicate specifying when the STMT might have result that is not
2050 : a compile time constant. */
2051 :
2052 : static ipa_predicate
2053 4319373 : will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
2054 : class ipa_fn_summary *summary,
2055 : class ipa_node_params *params_summary,
2056 : tree expr,
2057 : vec<ipa_predicate> nonconstant_names)
2058 : {
2059 4319373 : tree parm;
2060 4319373 : int index;
2061 :
2062 4563863 : while (UNARY_CLASS_P (expr))
2063 244490 : expr = TREE_OPERAND (expr, 0);
2064 :
2065 4319373 : parm = unmodified_parm (fbi, NULL, expr, NULL);
2066 4319373 : if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
2067 229216 : return add_condition (summary, params_summary, index, TREE_TYPE (parm), NULL,
2068 229216 : ipa_predicate::changed, NULL_TREE);
2069 4090157 : if (is_gimple_min_invariant (expr))
2070 62792 : return false;
2071 4027365 : if (TREE_CODE (expr) == SSA_NAME)
2072 3847317 : return nonconstant_names[SSA_NAME_VERSION (expr)];
2073 180048 : if (BINARY_CLASS_P (expr) || COMPARISON_CLASS_P (expr))
2074 : {
2075 179401 : ipa_predicate p1
2076 179401 : = will_be_nonconstant_expr_predicate (fbi, summary,
2077 : params_summary,
2078 179401 : TREE_OPERAND (expr, 0),
2079 : nonconstant_names);
2080 179401 : if (p1 == true)
2081 103367 : return p1;
2082 :
2083 76034 : ipa_predicate p2
2084 76034 : = will_be_nonconstant_expr_predicate (fbi, summary,
2085 : params_summary,
2086 76034 : TREE_OPERAND (expr, 1),
2087 : nonconstant_names);
2088 76034 : return p1.or_with (summary->conds, p2);
2089 : }
2090 647 : else if (TREE_CODE (expr) == COND_EXPR)
2091 : {
2092 184 : ipa_predicate p1
2093 184 : = will_be_nonconstant_expr_predicate (fbi, summary,
2094 : params_summary,
2095 184 : TREE_OPERAND (expr, 0),
2096 : nonconstant_names);
2097 184 : if (p1 == true)
2098 33 : return p1;
2099 :
2100 151 : ipa_predicate p2
2101 151 : = will_be_nonconstant_expr_predicate (fbi, summary,
2102 : params_summary,
2103 151 : TREE_OPERAND (expr, 1),
2104 : nonconstant_names);
2105 151 : if (p2 == true)
2106 151 : return p2;
2107 0 : p1 = p1.or_with (summary->conds, p2);
2108 0 : p2 = will_be_nonconstant_expr_predicate (fbi, summary,
2109 : params_summary,
2110 0 : TREE_OPERAND (expr, 2),
2111 : nonconstant_names);
2112 0 : return p2.or_with (summary->conds, p1);
2113 : }
2114 463 : else if (TREE_CODE (expr) == CALL_EXPR)
2115 463 : return true;
2116 : else
2117 : {
2118 0 : debug_tree (expr);
2119 0 : gcc_unreachable ();
2120 : }
2121 : }
2122 :
2123 :
2124 : /* Return predicate specifying when the STMT might have result that is not
2125 : a compile time constant. */
2126 :
2127 : static ipa_predicate
2128 116875971 : will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
2129 : class ipa_fn_summary *summary,
2130 : class ipa_node_params *params_summary,
2131 : gimple *stmt,
2132 : vec<ipa_predicate> nonconstant_names)
2133 : {
2134 116875971 : ipa_predicate p = true;
2135 116875971 : ssa_op_iter iter;
2136 116875971 : tree use;
2137 116875971 : tree param_type = NULL_TREE;
2138 116875971 : ipa_predicate op_non_const;
2139 116875971 : bool is_load;
2140 116875971 : int base_index;
2141 116875971 : struct agg_position_info aggpos;
2142 :
2143 : /* What statements might be optimized away
2144 : when their arguments are constant. */
2145 116875971 : if (gimple_code (stmt) != GIMPLE_ASSIGN
2146 : && gimple_code (stmt) != GIMPLE_COND
2147 : && gimple_code (stmt) != GIMPLE_SWITCH
2148 : && (gimple_code (stmt) != GIMPLE_CALL
2149 21379861 : || !(gimple_call_flags (stmt) & ECF_CONST)))
2150 26806246 : return p;
2151 :
2152 : /* Stores will stay anyway. */
2153 90069725 : if (gimple_store_p (stmt))
2154 26515746 : return p;
2155 :
2156 63553979 : is_load = gimple_assign_load_p (stmt);
2157 :
2158 : /* Loads can be optimized when the value is known. */
2159 63553979 : if (is_load)
2160 : {
2161 18230387 : tree op = gimple_assign_rhs1 (stmt);
2162 18230387 : if (!decompose_param_expr (fbi, stmt, op, &base_index, ¶m_type,
2163 : &aggpos))
2164 14926170 : return p;
2165 : }
2166 : else
2167 45323592 : base_index = -1;
2168 :
2169 : /* See if we understand all operands before we start
2170 : adding conditionals. */
2171 66318346 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2172 : {
2173 48471799 : tree parm = unmodified_parm (fbi, stmt, use, NULL);
2174 : /* For arguments we can build a condition. */
2175 48471799 : if (parm && ipa_get_param_decl_index (fbi->info, parm) >= 0)
2176 8689558 : continue;
2177 39782241 : if (TREE_CODE (use) != SSA_NAME)
2178 0 : return p;
2179 : /* If we know when operand is constant,
2180 : we still can say something useful. */
2181 39782241 : if (nonconstant_names[SSA_NAME_VERSION (use)] != true)
2182 9000979 : continue;
2183 30781262 : return p;
2184 : }
2185 :
2186 17846547 : if (is_load)
2187 3304136 : op_non_const =
2188 3304136 : add_condition (summary, params_summary,
2189 : base_index, param_type, &aggpos,
2190 : ipa_predicate::changed, NULL_TREE);
2191 : else
2192 14542411 : op_non_const = false;
2193 34492008 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2194 : {
2195 16645461 : tree parm = unmodified_parm (fbi, stmt, use, NULL);
2196 16645461 : int index;
2197 :
2198 16645461 : if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
2199 : {
2200 8190922 : if (index != base_index)
2201 5591424 : p = add_condition (summary, params_summary, index,
2202 5591424 : TREE_TYPE (parm), NULL,
2203 : ipa_predicate::changed, NULL_TREE);
2204 : else
2205 2599498 : continue;
2206 : }
2207 : else
2208 8454539 : p = nonconstant_names[SSA_NAME_VERSION (use)];
2209 14045963 : op_non_const = p.or_with (summary->conds, op_non_const);
2210 : }
2211 17846547 : if ((gimple_code (stmt) == GIMPLE_ASSIGN || gimple_code (stmt) == GIMPLE_CALL)
2212 15541908 : && gimple_op (stmt, 0)
2213 33083512 : && TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
2214 15236965 : nonconstant_names[SSA_NAME_VERSION (gimple_op (stmt, 0))]
2215 15236965 : = op_non_const;
2216 17846547 : return op_non_const;
2217 : }
2218 :
2219 : struct record_modified_bb_info
2220 : {
2221 : tree op;
2222 : bitmap bb_set;
2223 : gimple *stmt;
2224 : };
2225 :
2226 : /* Value is initialized in INIT_BB and used in USE_BB. We want to compute
2227 : probability how often it changes between USE_BB.
2228 : INIT_BB->count/USE_BB->count is an estimate, but if INIT_BB
2229 : is in different loop nest, we can do better.
2230 : This is all just estimate. In theory we look for minimal cut separating
2231 : INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
2232 : anyway. */
2233 :
2234 : static basic_block
2235 10687981 : get_minimal_bb (basic_block init_bb, basic_block use_bb)
2236 : {
2237 10687981 : class loop *l = find_common_loop (init_bb->loop_father, use_bb->loop_father);
2238 10687981 : if (l && l->header->count < init_bb->count)
2239 388783 : return l->header;
2240 : return init_bb;
2241 : }
2242 :
2243 : /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2244 : set except for info->stmt. */
2245 :
2246 : static bool
2247 4953537 : record_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef, void *data)
2248 : {
2249 4953537 : struct record_modified_bb_info *info =
2250 : (struct record_modified_bb_info *) data;
2251 4953537 : if (SSA_NAME_DEF_STMT (vdef) == info->stmt)
2252 : return false;
2253 4925350 : if (gimple_clobber_p (SSA_NAME_DEF_STMT (vdef)))
2254 : return false;
2255 4875831 : bitmap_set_bit (info->bb_set,
2256 4875831 : SSA_NAME_IS_DEFAULT_DEF (vdef)
2257 0 : ? ENTRY_BLOCK_PTR_FOR_FN (cfun)->index
2258 : : get_minimal_bb
2259 4875831 : (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2260 4875831 : gimple_bb (info->stmt))->index);
2261 4875831 : if (dump_file)
2262 : {
2263 0 : fprintf (dump_file, " Param ");
2264 0 : print_generic_expr (dump_file, info->op, TDF_SLIM);
2265 0 : fprintf (dump_file, " changed at bb %i, minimal: %i stmt: ",
2266 0 : gimple_bb (SSA_NAME_DEF_STMT (vdef))->index,
2267 : get_minimal_bb
2268 0 : (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2269 0 : gimple_bb (info->stmt))->index);
2270 0 : print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (vdef), 0);
2271 : }
2272 : return false;
2273 : }
2274 :
2275 : /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2276 : will change since last invocation of STMT.
2277 :
2278 : Value 0 is reserved for compile time invariants.
2279 : For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2280 : ought to be REG_BR_PROB_BASE / estimated_iters. */
2281 :
2282 : static int
2283 39594401 : param_change_prob (ipa_func_body_info *fbi, gimple *stmt, int i)
2284 : {
2285 39594401 : tree op = gimple_call_arg (stmt, i);
2286 39594401 : basic_block bb = gimple_bb (stmt);
2287 :
2288 39594401 : if (TREE_CODE (op) == WITH_SIZE_EXPR)
2289 299 : op = TREE_OPERAND (op, 0);
2290 :
2291 39594401 : tree base = get_base_address (op);
2292 :
2293 : /* Global invariants never change. */
2294 39594401 : if (is_gimple_min_invariant (base))
2295 : return 0;
2296 :
2297 : /* We would have to do non-trivial analysis to really work out what
2298 : is the probability of value to change (i.e. when init statement
2299 : is in a sibling loop of the call).
2300 :
2301 : We do an conservative estimate: when call is executed N times more often
2302 : than the statement defining value, we take the frequency 1/N. */
2303 19888084 : if (TREE_CODE (base) == SSA_NAME)
2304 : {
2305 17493035 : profile_count init_count;
2306 :
2307 26837144 : if (!bb->count.nonzero_p ())
2308 : return REG_BR_PROB_BASE;
2309 :
2310 8938202 : if (SSA_NAME_IS_DEFAULT_DEF (base))
2311 3126052 : init_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2312 : else
2313 5812150 : init_count = get_minimal_bb
2314 5812150 : (gimple_bb (SSA_NAME_DEF_STMT (base)),
2315 : gimple_bb (stmt))->count;
2316 :
2317 8938202 : if (init_count < bb->count)
2318 462345 : return MAX ((init_count.to_sreal_scale (bb->count)
2319 : * REG_BR_PROB_BASE).to_int (), 1);
2320 : return REG_BR_PROB_BASE;
2321 : }
2322 : else
2323 : {
2324 2395049 : ao_ref refd;
2325 2395049 : profile_count max = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2326 2395049 : struct record_modified_bb_info info;
2327 2395049 : tree init = ctor_for_folding (base);
2328 :
2329 2395049 : if (init != error_mark_node)
2330 : return 0;
2331 5353419 : if (!bb->count.nonzero_p () || fbi->aa_walk_budget == 0)
2332 : return REG_BR_PROB_BASE;
2333 1468058 : if (dump_file)
2334 : {
2335 0 : fprintf (dump_file, " Analyzing param change probability of ");
2336 0 : print_generic_expr (dump_file, op, TDF_SLIM);
2337 0 : fprintf (dump_file, "\n");
2338 : }
2339 1468058 : ao_ref_init (&refd, op);
2340 1468058 : info.op = op;
2341 1468058 : info.stmt = stmt;
2342 1468058 : info.bb_set = BITMAP_ALLOC (NULL);
2343 1468058 : int walked
2344 2936116 : = walk_aliased_vdefs (&refd, gimple_vuse (stmt), record_modified, &info,
2345 : NULL, NULL, fbi->aa_walk_budget);
2346 1468058 : if (walked > 0)
2347 1334347 : fbi->aa_walk_budget -= walked;
2348 1468058 : if (walked < 0 || bitmap_bit_p (info.bb_set, bb->index))
2349 : {
2350 893580 : if (walked < 0)
2351 206 : fbi->aa_walk_budget = 0;
2352 893580 : if (dump_file)
2353 : {
2354 0 : if (walked < 0)
2355 0 : fprintf (dump_file, " Ran out of AA walking budget.\n");
2356 : else
2357 0 : fprintf (dump_file, " Set in same BB as used.\n");
2358 : }
2359 893580 : BITMAP_FREE (info.bb_set);
2360 893580 : return REG_BR_PROB_BASE;
2361 : }
2362 :
2363 574478 : bitmap_iterator bi;
2364 574478 : unsigned index;
2365 : /* Lookup the most frequent update of the value and believe that
2366 : it dominates all the other; precise analysis here is difficult. */
2367 1614110 : EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi)
2368 1039632 : max = profile_count::max_prefer_initialized
2369 1039632 : (max, BASIC_BLOCK_FOR_FN (cfun, index)->count);
2370 574478 : if (dump_file)
2371 : {
2372 0 : fprintf (dump_file, " Set with count ");
2373 0 : max.dump (dump_file);
2374 0 : fprintf (dump_file, " and used with count ");
2375 0 : bb->count.dump (dump_file);
2376 0 : fprintf (dump_file, " freq %f\n",
2377 0 : max.to_sreal_scale (bb->count).to_double ());
2378 : }
2379 :
2380 574478 : BITMAP_FREE (info.bb_set);
2381 574478 : if (max < bb->count)
2382 95782 : return MAX ((max.to_sreal_scale (bb->count)
2383 : * REG_BR_PROB_BASE).to_int (), 1);
2384 : return REG_BR_PROB_BASE;
2385 : }
2386 : }
2387 :
2388 : /* Find whether a basic block BB is the final block of a (half) diamond CFG
2389 : sub-graph and if the predicate the condition depends on is known. If so,
2390 : return true and store the pointer the predicate in *P. */
2391 :
2392 : static bool
2393 6470083 : phi_result_unknown_predicate (ipa_func_body_info *fbi,
2394 : ipa_fn_summary *summary,
2395 : class ipa_node_params *params_summary,
2396 : basic_block bb,
2397 : ipa_predicate *p,
2398 : vec<ipa_predicate> nonconstant_names)
2399 : {
2400 6470083 : edge e;
2401 6470083 : edge_iterator ei;
2402 6470083 : basic_block first_bb = NULL;
2403 :
2404 6470083 : if (single_pred_p (bb))
2405 : {
2406 24210 : *p = false;
2407 24210 : return true;
2408 : }
2409 :
2410 14884217 : FOR_EACH_EDGE (e, ei, bb->preds)
2411 : {
2412 12704652 : if (single_succ_p (e->src))
2413 : {
2414 8491465 : if (!single_pred_p (e->src))
2415 : return false;
2416 7482259 : if (!first_bb)
2417 3259855 : first_bb = single_pred (e->src);
2418 4222404 : else if (single_pred (e->src) != first_bb)
2419 : return false;
2420 : }
2421 : else
2422 : {
2423 4213187 : if (!first_bb)
2424 : first_bb = e->src;
2425 1448735 : else if (e->src != first_bb)
2426 : return false;
2427 : }
2428 : }
2429 :
2430 2179565 : if (!first_bb)
2431 : return false;
2432 :
2433 4359130 : gcond *stmt = safe_dyn_cast <gcond *> (*gsi_last_bb (first_bb));
2434 2134771 : if (!stmt
2435 2134771 : || !is_gimple_ip_invariant (gimple_cond_rhs (stmt)))
2436 : return false;
2437 :
2438 1865884 : *p = will_be_nonconstant_expr_predicate (fbi, summary, params_summary,
2439 : gimple_cond_lhs (stmt),
2440 : nonconstant_names);
2441 1865884 : if (*p == true)
2442 : return false;
2443 : else
2444 616864 : return true;
2445 : }
2446 :
2447 : /* Given a PHI statement in a function described by inline properties SUMMARY
2448 : and *P being the predicate describing whether the selected PHI argument is
2449 : known, store a predicate for the result of the PHI statement into
2450 : NONCONSTANT_NAMES, if possible. */
2451 :
2452 : static void
2453 732210 : predicate_for_phi_result (class ipa_fn_summary *summary, gphi *phi,
2454 : ipa_predicate *p,
2455 : vec<ipa_predicate> nonconstant_names)
2456 : {
2457 732210 : unsigned i;
2458 :
2459 1022095 : for (i = 0; i < gimple_phi_num_args (phi); i++)
2460 : {
2461 902269 : tree arg = gimple_phi_arg (phi, i)->def;
2462 902269 : if (!is_gimple_min_invariant (arg))
2463 : {
2464 783655 : gcc_assert (TREE_CODE (arg) == SSA_NAME);
2465 1567310 : *p = p->or_with (summary->conds,
2466 783655 : nonconstant_names[SSA_NAME_VERSION (arg)]);
2467 783655 : if (*p == true)
2468 : return;
2469 : }
2470 : }
2471 :
2472 119826 : if (dump_file && (dump_flags & TDF_DETAILS))
2473 : {
2474 3 : fprintf (dump_file, "\t\tphi predicate: ");
2475 3 : p->dump (dump_file, summary->conds);
2476 : }
2477 119826 : nonconstant_names[SSA_NAME_VERSION (gimple_phi_result (phi))] = *p;
2478 : }
2479 :
2480 : /* For a typical usage of __builtin_expect (a<b, 1), we
2481 : may introduce an extra relation stmt:
2482 : With the builtin, we have
2483 : t1 = a <= b;
2484 : t2 = (long int) t1;
2485 : t3 = __builtin_expect (t2, 1);
2486 : if (t3 != 0)
2487 : goto ...
2488 : Without the builtin, we have
2489 : if (a<=b)
2490 : goto...
2491 : This affects the size/time estimation and may have
2492 : an impact on the earlier inlining.
2493 : Here find this pattern and fix it up later. */
2494 :
2495 : static gimple *
2496 42029392 : find_foldable_builtin_expect (basic_block bb)
2497 : {
2498 42029392 : gimple_stmt_iterator bsi;
2499 :
2500 294988496 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
2501 : {
2502 211117294 : gimple *stmt = gsi_stmt (bsi);
2503 211117294 : if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)
2504 210877635 : || gimple_call_builtin_p (stmt, BUILT_IN_EXPECT_WITH_PROBABILITY)
2505 421994852 : || gimple_call_internal_p (stmt, IFN_BUILTIN_EXPECT))
2506 : {
2507 372444 : tree var = gimple_call_lhs (stmt);
2508 372444 : tree arg = gimple_call_arg (stmt, 0);
2509 372444 : use_operand_p use_p;
2510 372444 : gimple *use_stmt;
2511 372444 : bool match = false;
2512 372444 : bool done = false;
2513 :
2514 372444 : if (!var || !arg)
2515 3 : continue;
2516 372441 : gcc_assert (TREE_CODE (var) == SSA_NAME);
2517 :
2518 743561 : while (TREE_CODE (arg) == SSA_NAME)
2519 : {
2520 743561 : gimple *stmt_tmp = SSA_NAME_DEF_STMT (arg);
2521 743561 : if (!is_gimple_assign (stmt_tmp))
2522 : break;
2523 736117 : switch (gimple_assign_rhs_code (stmt_tmp))
2524 : {
2525 : case LT_EXPR:
2526 : case LE_EXPR:
2527 : case GT_EXPR:
2528 : case GE_EXPR:
2529 : case EQ_EXPR:
2530 : case NE_EXPR:
2531 : match = true;
2532 : done = true;
2533 : break;
2534 : CASE_CONVERT:
2535 : break;
2536 : default:
2537 : done = true;
2538 : break;
2539 : }
2540 371120 : if (done)
2541 : break;
2542 371120 : arg = gimple_assign_rhs1 (stmt_tmp);
2543 : }
2544 :
2545 335095 : if (match && single_imm_use (var, &use_p, &use_stmt)
2546 707218 : && gimple_code (use_stmt) == GIMPLE_COND)
2547 187582 : return use_stmt;
2548 : }
2549 : }
2550 : return NULL;
2551 : }
2552 :
2553 : /* Return true when the basic blocks contains only clobbers followed by RESX.
2554 : Such BBs are kept around to make removal of dead stores possible with
2555 : presence of EH and will be optimized out by optimize_clobbers later in the
2556 : game.
2557 :
2558 : NEED_EH is used to recurse in case the clobber has non-EH predecessors
2559 : that can be clobber only, too.. When it is false, the RESX is not necessary
2560 : on the end of basic block. */
2561 :
2562 : static bool
2563 42681155 : clobber_only_eh_bb_p (basic_block bb, bool need_eh = true)
2564 : {
2565 42681155 : gimple_stmt_iterator gsi = gsi_last_bb (bb);
2566 42681155 : edge_iterator ei;
2567 42681155 : edge e;
2568 :
2569 42681155 : if (need_eh)
2570 : {
2571 42611156 : if (gsi_end_p (gsi))
2572 : return false;
2573 41464835 : if (gimple_code (gsi_stmt (gsi)) != GIMPLE_RESX)
2574 : return false;
2575 1161589 : gsi_prev (&gsi);
2576 : }
2577 69999 : else if (!single_succ_p (bb))
2578 : return false;
2579 :
2580 3065834 : for (; !gsi_end_p (gsi); gsi_prev (&gsi))
2581 : {
2582 2989496 : gimple *stmt = gsi_stmt (gsi);
2583 2989496 : if (is_gimple_debug (stmt))
2584 996456 : continue;
2585 1993040 : if (gimple_clobber_p (stmt))
2586 944849 : continue;
2587 1048191 : if (gimple_code (stmt) == GIMPLE_LABEL)
2588 : break;
2589 : return false;
2590 : }
2591 :
2592 : /* See if all predecessors are either throws or clobber only BBs. */
2593 3137313 : FOR_EACH_EDGE (e, ei, bb->preds)
2594 2554057 : if (!(e->flags & EDGE_EH)
2595 2554057 : && !clobber_only_eh_bb_p (e->src, false))
2596 : return false;
2597 :
2598 : return true;
2599 : }
2600 :
2601 : /* Return true if STMT compute a floating point expression that may be affected
2602 : by -ffast-math and similar flags. */
2603 :
2604 : static bool
2605 96291051 : fp_expression_p (gimple *stmt)
2606 : {
2607 96291051 : ssa_op_iter i;
2608 96291051 : tree op;
2609 :
2610 216152517 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_DEF|SSA_OP_USE)
2611 120479118 : if (FLOAT_TYPE_P (TREE_TYPE (op)))
2612 : return true;
2613 : return false;
2614 : }
2615 :
2616 : /* Return true if T references memory location that is local
2617 : for the function (that means, dead after return) or read-only. */
2618 :
2619 : bool
2620 60968439 : refs_local_or_readonly_memory_p (tree t)
2621 : {
2622 : /* Non-escaping memory is fine. */
2623 60968439 : t = get_base_address (t);
2624 60968439 : if ((TREE_CODE (t) == MEM_REF
2625 60968439 : || TREE_CODE (t) == TARGET_MEM_REF))
2626 25762811 : return points_to_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
2627 :
2628 : /* Automatic variables are fine. */
2629 35205628 : if (DECL_P (t)
2630 35205628 : && auto_var_in_fn_p (t, current_function_decl))
2631 : return true;
2632 :
2633 : /* Read-only variables are fine. */
2634 11342315 : if (DECL_P (t) && TREE_READONLY (t))
2635 1671664 : return true;
2636 :
2637 : return false;
2638 : }
2639 :
2640 : /* Return true if T is a pointer pointing to memory location that is local
2641 : for the function (that means, dead after return) or read-only. */
2642 :
2643 : bool
2644 74206575 : points_to_local_or_readonly_memory_p (tree t)
2645 : {
2646 : /* See if memory location is clearly invalid. */
2647 74206575 : if (integer_zerop (t))
2648 2480162 : return flag_delete_null_pointer_checks;
2649 71726413 : if (TREE_CODE (t) == SSA_NAME)
2650 : {
2651 : /* For IPA passes we can consinder accesses to return slot local
2652 : even if it is not local in the sense that memory is dead by
2653 : the end of founction.
2654 : The outer function will see a store in the call assignment
2655 : and thus this will do right thing for all uses of this
2656 : function in the current IPA passes (modref, pure/const discovery
2657 : and inlining heuristics). */
2658 45453909 : if (DECL_RESULT (current_function_decl)
2659 45453909 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
2660 46589237 : && t == ssa_default_def (cfun, DECL_RESULT (current_function_decl)))
2661 : return true;
2662 45117407 : return !ptr_deref_may_alias_global_p (t, false);
2663 : }
2664 26272504 : if (TREE_CODE (t) == ADDR_EXPR
2665 26272504 : && (TREE_CODE (TREE_OPERAND (t, 0)) != TARGET_MEM_REF
2666 4677 : || TREE_CODE (TREE_OPERAND (TREE_OPERAND (t, 0), 0)) != INTEGER_CST))
2667 13622181 : return refs_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
2668 : return false;
2669 : }
2670 :
2671 : /* Return true if T is a pointer pointing to memory location that is possible
2672 : sra candidate if all functions it is passed to are inlined. */
2673 :
2674 : static bool
2675 39594401 : points_to_possible_sra_candidate_p (tree t)
2676 : {
2677 39594401 : if (TREE_CODE (t) != ADDR_EXPR)
2678 : return false;
2679 :
2680 11829569 : t = get_base_address (TREE_OPERAND (t, 0));
2681 :
2682 : /* Automatic variables are fine. */
2683 11829569 : if (DECL_P (t)
2684 11829569 : && auto_var_in_fn_p (t, current_function_decl))
2685 : return true;
2686 : return false;
2687 : }
2688 :
2689 : /* Return true if BB only calls builtin_unreachable.
2690 : We skip empty basic blocks, debug statements, clobbers and predicts.
2691 : CACHE is used to memoize already analyzed blocks. */
2692 :
2693 : static bool
2694 27808243 : builtin_unreachable_bb_p (basic_block bb, vec<unsigned char> &cache)
2695 : {
2696 27808243 : if (cache[bb->index])
2697 2629997 : return cache[bb->index] - 1;
2698 25178246 : gimple_stmt_iterator si;
2699 25178246 : auto_vec <basic_block, 4> visited_bbs;
2700 25178246 : bool ret = false;
2701 25986928 : while (true)
2702 : {
2703 25986928 : bool empty_bb = true;
2704 25986928 : visited_bbs.safe_push (bb);
2705 25986928 : cache[bb->index] = 3;
2706 25986928 : for (si = gsi_start_nondebug_bb (bb);
2707 27503488 : !gsi_end_p (si) && empty_bb;
2708 1516560 : gsi_next_nondebug (&si))
2709 : {
2710 26128064 : if (gimple_code (gsi_stmt (si)) != GIMPLE_PREDICT
2711 25753053 : && !gimple_clobber_p (gsi_stmt (si))
2712 50772958 : && !gimple_nop_p (gsi_stmt (si)))
2713 : {
2714 : empty_bb = false;
2715 : break;
2716 : }
2717 : }
2718 25986928 : if (!empty_bb)
2719 : break;
2720 : else
2721 1375424 : bb = single_succ_edge (bb)->dest;
2722 1375424 : if (cache[bb->index])
2723 : {
2724 566742 : ret = cache[bb->index] == 3 ? false : cache[bb->index] - 1;
2725 566742 : goto done;
2726 : }
2727 : }
2728 24611504 : if (gimple_call_builtin_p (gsi_stmt (si), BUILT_IN_UNREACHABLE)
2729 24611504 : || gimple_call_builtin_p (gsi_stmt (si), BUILT_IN_UNREACHABLE_TRAP))
2730 : ret = true;
2731 25178246 : done:
2732 101521666 : for (basic_block vbb:visited_bbs)
2733 25986928 : cache[vbb->index] = (unsigned char)ret + 1;
2734 25178246 : return ret;
2735 25178246 : }
2736 :
2737 : static bool
2738 14033299 : guards_builtin_unreachable (basic_block bb, vec<unsigned char> &cache)
2739 : {
2740 14033299 : edge_iterator ei;
2741 14033299 : edge e;
2742 41568574 : FOR_EACH_EDGE (e, ei, bb->succs)
2743 27808243 : if (builtin_unreachable_bb_p (e->dest, cache))
2744 : {
2745 272968 : if (dump_file && (dump_flags & TDF_DETAILS))
2746 1 : fprintf (dump_file,
2747 : "BB %i ends with conditional guarding __builtin_unreachable;"
2748 : " conditinal is unnecessary\n", bb->index);
2749 : return true;
2750 : }
2751 : return false;
2752 : }
2753 :
2754 : #define STMT_NECESSARY GF_PLF_1
2755 :
2756 : /* If STMT is not already marked necessary, mark it, and add it to the
2757 : worklist if ADD_TO_WORKLIST is true. */
2758 :
2759 : static inline void
2760 175771303 : mark_stmt_necessary (gimple *stmt, auto_vec<gimple *> &worklist)
2761 : {
2762 175771303 : gcc_assert (stmt);
2763 :
2764 175771303 : if (gimple_plf (stmt, STMT_NECESSARY))
2765 : return;
2766 :
2767 145974510 : if (dump_file && (dump_flags & TDF_DETAILS))
2768 : {
2769 211 : fprintf (dump_file, "Marking useful stmt: ");
2770 211 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
2771 211 : fprintf (dump_file, "\n");
2772 : }
2773 :
2774 145974510 : gimple_set_plf (stmt, STMT_NECESSARY, true);
2775 145974510 : worklist.safe_push (stmt);
2776 : }
2777 :
2778 : /* Mark the statement defining operand OP as necessary. */
2779 :
2780 : static inline void
2781 123799528 : mark_operand_necessary (tree op, auto_vec<gimple *> &worklist)
2782 : {
2783 123799528 : gimple *stmt = SSA_NAME_DEF_STMT (op);
2784 123799528 : if (gimple_nop_p (stmt))
2785 : return;
2786 100194526 : mark_stmt_necessary (stmt, worklist);
2787 : }
2788 :
2789 : /* Mark all statements that will remain in the body after optimizing out
2790 : conditionals guarding __builtin_unreachable which we keep to preserve
2791 : value ranges. */
2792 :
2793 : static void
2794 7485874 : find_necessary_statements (struct cgraph_node *node)
2795 : {
2796 7485874 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
2797 7485874 : auto_vec<unsigned char, 10> cache;
2798 7485874 : basic_block bb;
2799 7485874 : auto_vec<gimple *> worklist;
2800 :
2801 7485874 : cache.safe_grow_cleared (last_basic_block_for_fn (cfun));
2802 : /* Mark all obviously necessary statements. */
2803 50097030 : FOR_EACH_BB_FN (bb, my_function)
2804 : {
2805 42611156 : for (gimple_stmt_iterator gsi = gsi_start_phis (bb);
2806 54404919 : !gsi_end_p (gsi); gsi_next (&gsi))
2807 11793763 : gimple_set_plf (gsi_stmt (gsi), STMT_NECESSARY, false);
2808 :
2809 238619323 : for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
2810 153397011 : gsi_next_nondebug (&bsi))
2811 : {
2812 153397011 : gimple *stmt = gsi_stmt (bsi);
2813 :
2814 153397011 : gimple_set_plf (stmt, STMT_NECESSARY, false);
2815 153397011 : if (gimple_has_side_effects (stmt)
2816 125583039 : || (is_ctrl_stmt (stmt)
2817 22618167 : && (gimple_code (stmt) != GIMPLE_COND
2818 14033299 : || !guards_builtin_unreachable (bb, cache)))
2819 103237840 : || gimple_store_p (stmt)
2820 231238920 : || gimple_code (stmt) == GIMPLE_ASM)
2821 75576777 : mark_stmt_necessary (stmt, worklist);
2822 : }
2823 : }
2824 153460384 : while (worklist.length () > 0)
2825 : {
2826 145974510 : gimple *stmt = worklist.pop ();
2827 :
2828 145974510 : if (dump_file && (dump_flags & TDF_DETAILS))
2829 : {
2830 211 : fprintf (dump_file, "processing: ");
2831 211 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
2832 211 : fprintf (dump_file, "\n");
2833 : }
2834 145974510 : if (gimple_code (stmt) == GIMPLE_PHI)
2835 18613454 : for (unsigned int k = 0; k < gimple_phi_num_args (stmt); k++)
2836 : {
2837 13180289 : tree arg = PHI_ARG_DEF (stmt, k);
2838 :
2839 13180289 : if (TREE_CODE (arg) == SSA_NAME)
2840 10113200 : mark_operand_necessary (arg, worklist);
2841 : }
2842 : else
2843 : {
2844 140541345 : ssa_op_iter iter;
2845 140541345 : tree use;
2846 :
2847 254227673 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2848 113686328 : mark_operand_necessary (use, worklist);
2849 : }
2850 : }
2851 7485874 : }
2852 :
2853 : /* Analyze function body for NODE.
2854 : EARLY indicates run from early optimization pipeline. */
2855 :
2856 : static void
2857 7485874 : analyze_function_body (struct cgraph_node *node, bool early)
2858 : {
2859 7485874 : sreal time = opt_for_fn (node->decl, param_uninlined_function_time);
2860 : /* Estimate static overhead for function prologue/epilogue and alignment. */
2861 7485874 : int size = opt_for_fn (node->decl, param_uninlined_function_insns);
2862 : /* Benefits are scaled by probability of elimination that is in range
2863 : <0,2>. */
2864 7485874 : basic_block bb;
2865 7485874 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
2866 7485874 : sreal freq;
2867 7485874 : class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
2868 7485874 : ipa_node_params *params_summary
2869 7485874 : = early ? NULL : ipa_node_params_sum->get (node);
2870 7485874 : ipa_predicate bb_predicate;
2871 7485874 : struct ipa_func_body_info fbi;
2872 7485874 : vec<ipa_predicate> nonconstant_names = vNULL;
2873 7485874 : int nblocks, n;
2874 7485874 : int *order;
2875 7485874 : gimple *fix_builtin_expect_stmt;
2876 :
2877 7485874 : gcc_assert (my_function && my_function->cfg);
2878 7485874 : gcc_assert (cfun == my_function);
2879 :
2880 7485874 : memset(&fbi, 0, sizeof(fbi));
2881 7485874 : vec_free (info->conds);
2882 7485874 : info->conds = NULL;
2883 7485874 : info->size_time_table.release ();
2884 7485874 : info->call_size_time_table.release ();
2885 :
2886 : /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2887 : so we can produce proper inline hints.
2888 :
2889 : When optimizing and analyzing for early inliner, initialize node params
2890 : so we can produce correct BB predicates. */
2891 :
2892 7485874 : if (opt_for_fn (node->decl, optimize))
2893 : {
2894 6552319 : calculate_dominance_info (CDI_DOMINATORS);
2895 6552319 : calculate_dominance_info (CDI_POST_DOMINATORS);
2896 6552319 : if (!early)
2897 1415850 : loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
2898 : else
2899 : {
2900 5136469 : ipa_check_create_node_params ();
2901 5136469 : ipa_initialize_node_params (node);
2902 : }
2903 :
2904 6552319 : if (ipa_node_params_sum)
2905 : {
2906 6552319 : fbi.node = node;
2907 6552319 : fbi.info = ipa_node_params_sum->get (node);
2908 6552319 : fbi.bb_infos = vNULL;
2909 6552319 : fbi.bb_infos.safe_grow_cleared (last_basic_block_for_fn (cfun), true);
2910 6552319 : fbi.param_count = count_formal_params (node->decl);
2911 6552319 : fbi.aa_walk_budget = opt_for_fn (node->decl, param_ipa_max_aa_steps);
2912 :
2913 6552319 : nonconstant_names.safe_grow_cleared
2914 6552319 : (SSANAMES (my_function)->length (), true);
2915 : }
2916 : }
2917 :
2918 7485874 : if (dump_file)
2919 249 : fprintf (dump_file, "\nAnalyzing function body size: %s\n",
2920 : node->dump_name ());
2921 :
2922 : /* When we run into maximal number of entries, we assign everything to the
2923 : constant truth case. Be sure to have it in list. */
2924 7485874 : bb_predicate = true;
2925 7485874 : info->account_size_time (0, 0, bb_predicate, bb_predicate);
2926 :
2927 7485874 : bb_predicate = ipa_predicate::not_inlined ();
2928 7485874 : info->account_size_time (opt_for_fn (node->decl,
2929 : param_uninlined_function_insns)
2930 : * ipa_fn_summary::size_scale,
2931 7485874 : opt_for_fn (node->decl,
2932 : param_uninlined_function_time),
2933 : bb_predicate,
2934 : bb_predicate);
2935 :
2936 : /* Only look for target information for inlinable functions. */
2937 7485874 : bool scan_for_target_info =
2938 7485874 : info->inlinable
2939 13449332 : && targetm.target_option.need_ipa_fn_target_info (node->decl,
2940 5963458 : info->target_info);
2941 :
2942 7485874 : if (fbi.info)
2943 6552319 : compute_bb_predicates (&fbi, node, info, params_summary);
2944 7485874 : find_necessary_statements (node);
2945 7485874 : const profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2946 7485874 : order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
2947 7485874 : nblocks = pre_and_rev_post_order_compute (NULL, order, false);
2948 57582904 : for (n = 0; n < nblocks; n++)
2949 : {
2950 42611156 : bb = BASIC_BLOCK_FOR_FN (cfun, order[n]);
2951 42611156 : freq = bb->count.to_sreal_scale (entry_count);
2952 42611156 : if (clobber_only_eh_bb_p (bb))
2953 : {
2954 581764 : if (dump_file && (dump_flags & TDF_DETAILS))
2955 0 : fprintf (dump_file, "\n Ignoring BB %i;"
2956 : " it will be optimized away by cleanup_clobbers\n",
2957 : bb->index);
2958 581764 : continue;
2959 : }
2960 :
2961 : /* TODO: Obviously predicates can be propagated down across CFG. */
2962 42029392 : if (fbi.info)
2963 : {
2964 35475605 : if (bb->aux)
2965 35475591 : bb_predicate = *(ipa_predicate *)bb->aux;
2966 : else
2967 14 : bb_predicate = false;
2968 : }
2969 : else
2970 6553787 : bb_predicate = true;
2971 :
2972 42029392 : if (dump_file && (dump_flags & TDF_DETAILS))
2973 : {
2974 69 : fprintf (dump_file, "\n BB %i predicate:", bb->index);
2975 69 : bb_predicate.dump (dump_file, info->conds);
2976 : }
2977 :
2978 42029392 : if (fbi.info && nonconstant_names.exists ())
2979 : {
2980 35475605 : ipa_predicate phi_predicate;
2981 35475605 : bool first_phi = true;
2982 :
2983 36207815 : for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
2984 732210 : gsi_next (&bsi))
2985 : {
2986 6561219 : if (first_phi
2987 6561219 : && !phi_result_unknown_predicate (&fbi, info,
2988 : params_summary,
2989 : bb,
2990 : &phi_predicate,
2991 : nonconstant_names))
2992 : break;
2993 732210 : first_phi = false;
2994 732210 : if (dump_file && (dump_flags & TDF_DETAILS))
2995 : {
2996 3 : fprintf (dump_file, " ");
2997 3 : print_gimple_stmt (dump_file, gsi_stmt (bsi), 0);
2998 : }
2999 732210 : predicate_for_phi_result (info, bsi.phi (), &phi_predicate,
3000 : nonconstant_names);
3001 : }
3002 : }
3003 :
3004 42029392 : fix_builtin_expect_stmt = find_foldable_builtin_expect (bb);
3005 :
3006 42029392 : for (gimple_stmt_iterator bsi = gsi_start_nondebug_bb (bb);
3007 186939847 : !gsi_end_p (bsi); gsi_next_nondebug (&bsi))
3008 : {
3009 144910455 : gimple *stmt = gsi_stmt (bsi);
3010 144910455 : if (!gimple_plf (stmt, STMT_NECESSARY))
3011 : {
3012 5742345 : if (dump_file && (dump_flags & TDF_DETAILS))
3013 : {
3014 19 : fprintf (dump_file, " skipping unnecessary stmt ");
3015 19 : print_gimple_stmt (dump_file, stmt, 0);
3016 : }
3017 : /* TODO: const calls used only to produce values for
3018 : builtion_unreachable guards should not be accounted. However
3019 : we still want to inline them and this does does not work well
3020 : with the cost model. For now account them as usual. */
3021 5742345 : if (!is_gimple_call (stmt)
3022 5742345 : || gimple_call_internal_p (stmt))
3023 5453370 : continue;
3024 : }
3025 139457085 : int this_size = estimate_num_insns (stmt, &eni_size_weights);
3026 139457085 : int this_time = estimate_num_insns (stmt, &eni_time_weights);
3027 139457085 : int prob;
3028 139457085 : ipa_predicate will_be_nonconstant;
3029 :
3030 : /* This relation stmt should be folded after we remove
3031 : __builtin_expect call. Adjust the cost here. */
3032 139457085 : if (stmt == fix_builtin_expect_stmt)
3033 : {
3034 187144 : this_size--;
3035 187144 : this_time--;
3036 : }
3037 :
3038 139457085 : if (dump_file && (dump_flags & TDF_DETAILS))
3039 : {
3040 205 : fprintf (dump_file, " ");
3041 205 : print_gimple_stmt (dump_file, stmt, 0);
3042 205 : fprintf (dump_file, "\t\tfreq:%3.2f size:%3i time:%3i\n",
3043 : freq.to_double (), this_size,
3044 : this_time);
3045 : }
3046 :
3047 139457085 : if (is_gimple_call (stmt)
3048 139457085 : && !gimple_call_internal_p (stmt))
3049 : {
3050 23512069 : struct cgraph_edge *edge = node->get_edge (stmt);
3051 23512069 : ipa_call_summary *es = ipa_call_summaries->get_create (edge);
3052 :
3053 : /* Special case: results of BUILT_IN_CONSTANT_P will be always
3054 : resolved as constant. We however don't want to optimize
3055 : out the cgraph edges. */
3056 23512069 : if (nonconstant_names.exists ()
3057 20559169 : && gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P)
3058 48190 : && gimple_call_lhs (stmt)
3059 23560259 : && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
3060 : {
3061 48190 : ipa_predicate false_p = false;
3062 48190 : nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
3063 48190 : = false_p;
3064 : }
3065 23512069 : if (ipa_node_params_sum)
3066 : {
3067 20571982 : int count = gimple_call_num_args (stmt);
3068 20571982 : int i;
3069 :
3070 20571982 : if (count)
3071 17684307 : es->param.safe_grow_cleared (count, true);
3072 60166383 : for (i = 0; i < count; i++)
3073 : {
3074 39594401 : int prob = param_change_prob (&fbi, stmt, i);
3075 39594401 : gcc_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
3076 39594401 : es->param[i].change_prob = prob;
3077 39594401 : es->param[i].points_to_local_or_readonly_memory
3078 39594401 : = points_to_local_or_readonly_memory_p
3079 39594401 : (gimple_call_arg (stmt, i));
3080 39594401 : es->param[i].points_to_possible_sra_candidate
3081 39594401 : = points_to_possible_sra_candidate_p
3082 39594401 : (gimple_call_arg (stmt, i));
3083 : }
3084 : }
3085 : /* We cannot setup VLA parameters during inlining. */
3086 69094460 : for (unsigned int i = 0; i < gimple_call_num_args (stmt); ++i)
3087 45582787 : if (TREE_CODE (gimple_call_arg (stmt, i)) == WITH_SIZE_EXPR)
3088 : {
3089 396 : edge->inline_failed = CIF_FUNCTION_NOT_INLINABLE;
3090 396 : break;
3091 : }
3092 23512069 : es->call_stmt_size = this_size;
3093 23512069 : es->call_stmt_time = this_time;
3094 23512069 : es->loop_depth = bb_loop_depth (bb);
3095 23512069 : edge_set_predicate (edge, &bb_predicate);
3096 23512069 : if (edge->speculative)
3097 : {
3098 0 : cgraph_edge *indirect
3099 0 : = edge->speculative_call_indirect_edge ();
3100 0 : ipa_call_summary *es2
3101 0 : = ipa_call_summaries->get_create (indirect);
3102 0 : ipa_call_summaries->duplicate (edge, indirect,
3103 : es, es2);
3104 :
3105 : /* Edge is the first direct call.
3106 : create and duplicate call summaries for multiple
3107 : speculative call targets. */
3108 0 : for (cgraph_edge *direct
3109 0 : = edge->next_speculative_call_target ();
3110 0 : direct;
3111 0 : direct = direct->next_speculative_call_target ())
3112 : {
3113 0 : ipa_call_summary *es3
3114 0 : = ipa_call_summaries->get_create (direct);
3115 0 : ipa_call_summaries->duplicate (edge, direct,
3116 : es, es3);
3117 : }
3118 : }
3119 :
3120 : /* If dealing with a carrying edge, copy its summary over to its
3121 : attached edges as well. */
3122 23512069 : if (edge->has_callback)
3123 : {
3124 15124 : cgraph_edge *cbe;
3125 30249 : for (cbe = edge->first_callback_edge (); cbe;
3126 15125 : cbe = cbe->next_callback_edge ())
3127 : {
3128 15125 : ipa_call_summary *es2 = ipa_call_summaries->get_create (cbe);
3129 15125 : ipa_call_summaries->duplicate (edge, cbe, es, es2);
3130 : /* Unlike speculative edges, callback edges have no real
3131 : size or time; the call doesn't exist. Reflect that in
3132 : their summaries. */
3133 15125 : es2->call_stmt_size = 0;
3134 15125 : es2->call_stmt_time = 0;
3135 : }
3136 : }
3137 : }
3138 :
3139 : /* TODO: When conditional jump or switch is known to be constant, but
3140 : we did not translate it into the predicates, we really can account
3141 : just maximum of the possible paths. */
3142 139457085 : if (fbi.info)
3143 116875971 : will_be_nonconstant
3144 116875971 : = will_be_nonconstant_predicate (&fbi, info, params_summary,
3145 : stmt, nonconstant_names);
3146 : else
3147 22581114 : will_be_nonconstant = true;
3148 139457085 : if (this_time || this_size)
3149 : {
3150 113950412 : sreal final_time = (sreal)this_time * freq;
3151 113950412 : prob = eliminated_by_inlining_prob (&fbi, stmt);
3152 113950412 : if (prob == 1 && dump_file && (dump_flags & TDF_DETAILS))
3153 11 : fprintf (dump_file,
3154 : "\t\t50%% will be eliminated by inlining\n");
3155 113950412 : if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
3156 20 : fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
3157 :
3158 113950412 : ipa_predicate p = bb_predicate & will_be_nonconstant;
3159 113950412 : int parm = load_or_store_of_ptr_parameter (&fbi, stmt);
3160 113950412 : ipa_predicate sra_predicate = true;
3161 113950412 : if (parm != -1)
3162 14572410 : sra_predicate &= add_condition (info, params_summary, parm,
3163 : ptr_type_node, NULL,
3164 7286205 : ipa_predicate::not_sra_candidate, NULL, 0);
3165 :
3166 : /* We can ignore statement when we proved it is never going
3167 : to happen, but we cannot do that for call statements
3168 : because edges are accounted specially. */
3169 :
3170 227900824 : if (*(is_gimple_call (stmt) ? &bb_predicate : &p) != false)
3171 : {
3172 113232113 : time += final_time;
3173 113232113 : size += this_size;
3174 : }
3175 :
3176 : /* We account everything but the calls. Calls have their own
3177 : size/time info attached to cgraph edges. This is necessary
3178 : in order to make the cost disappear after inlining. */
3179 113950412 : if (!is_gimple_call (stmt))
3180 : {
3181 91170618 : if (prob)
3182 : {
3183 15351583 : ipa_predicate ip
3184 15351583 : = bb_predicate & ipa_predicate::not_inlined () & sra_predicate;
3185 30703166 : info->account_size_time (this_size * prob,
3186 15351583 : (final_time * prob) / 2, ip,
3187 : p);
3188 : }
3189 15351583 : if (prob != 2)
3190 168084260 : info->account_size_time (this_size * (2 - prob),
3191 84042130 : (final_time * (2 - prob) / 2),
3192 168084260 : bb_predicate & sra_predicate,
3193 : p);
3194 : }
3195 :
3196 113950412 : if (!info->fp_expressions && fp_expression_p (stmt))
3197 : {
3198 617652 : info->fp_expressions = true;
3199 617652 : if (dump_file)
3200 9 : fprintf (dump_file, " fp_expression set\n");
3201 : }
3202 : }
3203 :
3204 : /* For target specific information, we want to scan all statements
3205 : rather than those statements with non-zero weights, to avoid
3206 : missing to scan something interesting for target information,
3207 : such as: internal function calls. */
3208 139457085 : if (scan_for_target_info)
3209 0 : scan_for_target_info =
3210 0 : targetm.target_option.update_ipa_fn_target_info
3211 0 : (info->target_info, stmt);
3212 :
3213 : /* Account cost of address calculations in the statements. */
3214 539328122 : for (unsigned int i = 0; i < gimple_num_ops (stmt); i++)
3215 : {
3216 399871037 : for (tree op = gimple_op (stmt, i);
3217 769764381 : op && handled_component_p (op);
3218 48293265 : op = TREE_OPERAND (op, 0))
3219 48293265 : if ((TREE_CODE (op) == ARRAY_REF
3220 48293265 : || TREE_CODE (op) == ARRAY_RANGE_REF)
3221 48293265 : && TREE_CODE (TREE_OPERAND (op, 1)) == SSA_NAME)
3222 : {
3223 2418761 : ipa_predicate p = bb_predicate;
3224 2418761 : if (fbi.info)
3225 1893261 : p = p & will_be_nonconstant_expr_predicate
3226 1893261 : (&fbi, info, params_summary,
3227 1893261 : TREE_OPERAND (op, 1),
3228 1893261 : nonconstant_names);
3229 2418761 : if (p != false)
3230 : {
3231 2413339 : time += freq;
3232 2413339 : size += 1;
3233 2413339 : if (dump_file)
3234 24 : fprintf (dump_file,
3235 : "\t\tAccounting address calculation.\n");
3236 2413339 : info->account_size_time (ipa_fn_summary::size_scale,
3237 : freq,
3238 : bb_predicate,
3239 : p);
3240 : }
3241 : }
3242 : }
3243 :
3244 : }
3245 : }
3246 7485874 : free (order);
3247 :
3248 7485874 : if (nonconstant_names.exists () && !early)
3249 : {
3250 1415850 : ipa_fn_summary *s = ipa_fn_summaries->get (node);
3251 1415850 : unsigned max_loop_predicates = opt_for_fn (node->decl,
3252 : param_ipa_max_loop_predicates);
3253 :
3254 1415850 : if (dump_file && (dump_flags & TDF_DETAILS))
3255 16 : flow_loops_dump (dump_file, NULL, 0);
3256 1415850 : scev_initialize ();
3257 4878271 : for (auto loop : loops_list (cfun, 0))
3258 : {
3259 630721 : ipa_predicate loop_iterations = true;
3260 630721 : sreal header_freq;
3261 630721 : edge ex;
3262 630721 : unsigned int j;
3263 630721 : class tree_niter_desc niter_desc;
3264 630721 : if (!loop->header->aux)
3265 0 : continue;
3266 :
3267 630721 : profile_count hdr_count = loop->header->count;
3268 630721 : sreal hdr_freq = hdr_count.to_sreal_scale (entry_count);
3269 :
3270 630721 : bb_predicate = *(ipa_predicate *)loop->header->aux;
3271 630721 : auto_vec<edge> exits = get_loop_exit_edges (loop);
3272 2281020 : FOR_EACH_VEC_ELT (exits, j, ex)
3273 1019578 : if (number_of_iterations_exit (loop, ex, &niter_desc, false)
3274 1019578 : && !is_gimple_min_invariant (niter_desc.niter))
3275 : {
3276 155273 : ipa_predicate will_be_nonconstant
3277 155273 : = will_be_nonconstant_expr_predicate (&fbi, info,
3278 : params_summary,
3279 : niter_desc.niter,
3280 : nonconstant_names);
3281 155273 : if (will_be_nonconstant != true)
3282 62476 : will_be_nonconstant = bb_predicate & will_be_nonconstant;
3283 155273 : if (will_be_nonconstant != true
3284 217749 : && will_be_nonconstant != false)
3285 61923 : loop_iterations &= will_be_nonconstant;
3286 : }
3287 630721 : add_freqcounting_predicate (&s->loop_iterations, loop_iterations,
3288 : hdr_freq, max_loop_predicates);
3289 630723 : }
3290 :
3291 : /* To avoid quadratic behavior we analyze stride predicates only
3292 : with respect to the containing loop. Thus we simply iterate
3293 : over all defs in the outermost loop body. */
3294 1415850 : for (class loop *loop = loops_for_fn (cfun)->tree_root->inner;
3295 1914186 : loop != NULL; loop = loop->next)
3296 : {
3297 498336 : ipa_predicate loop_stride = true;
3298 498336 : basic_block *body = get_loop_body (loop);
3299 498336 : profile_count hdr_count = loop->header->count;
3300 498336 : sreal hdr_freq = hdr_count.to_sreal_scale (entry_count);
3301 3433818 : for (unsigned i = 0; i < loop->num_nodes; i++)
3302 : {
3303 2437146 : gimple_stmt_iterator gsi;
3304 2437146 : if (!body[i]->aux)
3305 7 : continue;
3306 :
3307 2437139 : bb_predicate = *(ipa_predicate *)body[i]->aux;
3308 16460023 : for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
3309 11585745 : gsi_next (&gsi))
3310 : {
3311 11585745 : gimple *stmt = gsi_stmt (gsi);
3312 :
3313 11585745 : if (!is_gimple_assign (stmt))
3314 11436560 : continue;
3315 :
3316 5480018 : tree def = gimple_assign_lhs (stmt);
3317 5480018 : if (TREE_CODE (def) != SSA_NAME)
3318 1049848 : continue;
3319 :
3320 4430170 : affine_iv iv;
3321 8860340 : if (!simple_iv (loop_containing_stmt (stmt),
3322 : loop_containing_stmt (stmt),
3323 : def, &iv, true)
3324 4430170 : || is_gimple_min_invariant (iv.step))
3325 4280985 : continue;
3326 :
3327 149185 : ipa_predicate will_be_nonconstant
3328 149185 : = will_be_nonconstant_expr_predicate (&fbi, info,
3329 : params_summary,
3330 : iv.step,
3331 : nonconstant_names);
3332 149185 : if (will_be_nonconstant != true)
3333 54491 : will_be_nonconstant = bb_predicate & will_be_nonconstant;
3334 149185 : if (will_be_nonconstant != true
3335 203676 : && will_be_nonconstant != false)
3336 44896 : loop_stride = loop_stride & will_be_nonconstant;
3337 : }
3338 : }
3339 498336 : add_freqcounting_predicate (&s->loop_strides, loop_stride,
3340 : hdr_freq, max_loop_predicates);
3341 498336 : free (body);
3342 : }
3343 1415850 : scev_finalize ();
3344 : }
3345 65068778 : FOR_ALL_BB_FN (bb, my_function)
3346 : {
3347 57582904 : edge e;
3348 57582904 : edge_iterator ei;
3349 :
3350 57582904 : if (bb->aux)
3351 42543985 : edge_predicate_pool.remove ((ipa_predicate *)bb->aux);
3352 57582904 : bb->aux = NULL;
3353 122293823 : FOR_EACH_EDGE (e, ei, bb->succs)
3354 : {
3355 64710919 : if (e->aux)
3356 2577857 : edge_predicate_pool.remove ((ipa_predicate *)e->aux);
3357 64710919 : e->aux = NULL;
3358 : }
3359 : }
3360 7485874 : ipa_fn_summary *s = ipa_fn_summaries->get (node);
3361 7485874 : ipa_size_summary *ss = ipa_size_summaries->get (node);
3362 7485874 : s->time = time;
3363 7485874 : ss->self_size = size;
3364 7485874 : nonconstant_names.release ();
3365 7485874 : ipa_release_body_info (&fbi);
3366 7485874 : if (opt_for_fn (node->decl, optimize))
3367 : {
3368 6552319 : if (!early)
3369 1415850 : loop_optimizer_finalize ();
3370 5136469 : else if (!ipa_edge_args_sum)
3371 5136455 : ipa_free_all_node_params ();
3372 6552319 : free_dominance_info (CDI_DOMINATORS);
3373 6552319 : free_dominance_info (CDI_POST_DOMINATORS);
3374 : }
3375 7485874 : if (dump_file)
3376 : {
3377 249 : fprintf (dump_file, "\n");
3378 249 : ipa_dump_fn_summary (dump_file, node);
3379 : }
3380 7485874 : }
3381 :
3382 :
3383 : /* Compute function summary.
3384 : EARLY is true when we compute parameters during early opts. */
3385 :
3386 : void
3387 7487182 : compute_fn_summary (struct cgraph_node *node, bool early)
3388 : {
3389 7487182 : HOST_WIDE_INT self_stack_size;
3390 7487182 : struct cgraph_edge *e;
3391 :
3392 7487182 : gcc_assert (!node->inlined_to);
3393 :
3394 7487182 : if (!ipa_fn_summaries)
3395 218788 : ipa_fn_summary_alloc ();
3396 :
3397 : /* Create a new ipa_fn_summary. */
3398 7487182 : ((ipa_fn_summary_t *)ipa_fn_summaries)->remove_callees (node);
3399 7487182 : ipa_fn_summaries->remove (node);
3400 7487182 : class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
3401 7487182 : class ipa_size_summary *size_info = ipa_size_summaries->get_create (node);
3402 :
3403 : /* Estimate the stack size for the function if we're optimizing. */
3404 13105930 : self_stack_size = optimize && !node->thunk
3405 14039501 : ? estimated_stack_frame_size (node) : 0;
3406 7487182 : size_info->estimated_self_stack_size = self_stack_size;
3407 7487182 : info->estimated_stack_size = self_stack_size;
3408 :
3409 7487182 : if (node->thunk)
3410 : {
3411 1308 : ipa_call_summary *es = ipa_call_summaries->get_create (node->callees);
3412 1308 : ipa_predicate t = true;
3413 :
3414 1308 : node->can_change_signature = false;
3415 1308 : es->call_stmt_size = eni_size_weights.call_cost;
3416 1308 : es->call_stmt_time = eni_time_weights.call_cost;
3417 3924 : info->account_size_time (ipa_fn_summary::size_scale
3418 1308 : * opt_for_fn (node->decl,
3419 : param_uninlined_function_thunk_insns),
3420 1308 : opt_for_fn (node->decl,
3421 : param_uninlined_function_thunk_time), t, t);
3422 1308 : t = ipa_predicate::not_inlined ();
3423 1308 : info->account_size_time (2 * ipa_fn_summary::size_scale, 0, t, t);
3424 1308 : ipa_update_overall_fn_summary (node);
3425 1308 : size_info->self_size = size_info->size;
3426 1308 : if (stdarg_p (TREE_TYPE (node->decl)))
3427 : {
3428 9 : info->inlinable = false;
3429 9 : node->callees->inline_failed = CIF_VARIADIC_THUNK;
3430 : }
3431 : else
3432 1299 : info->inlinable = true;
3433 : }
3434 : else
3435 : {
3436 : /* Even is_gimple_min_invariant rely on current_function_decl. */
3437 7485874 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
3438 :
3439 : /* During IPA profile merging we may be called w/o virtual SSA form
3440 : built. */
3441 7485874 : update_ssa (TODO_update_ssa_only_virtuals);
3442 :
3443 : /* Can this function be inlined at all? */
3444 7485874 : if (!opt_for_fn (node->decl, optimize)
3445 8419429 : && !lookup_attribute ("always_inline",
3446 933555 : DECL_ATTRIBUTES (node->decl)))
3447 862997 : info->inlinable = false;
3448 : else
3449 6622877 : info->inlinable = tree_inlinable_function_p (node->decl);
3450 :
3451 7485874 : bool no_signature = false;
3452 :
3453 : /* Don't allow signature changes for functions which have
3454 : [[gnu::musttail]] or [[clang::musttail]] calls. Sometimes
3455 : (more often on targets which pass everything on the stack)
3456 : signature changes can result in tail calls being impossible
3457 : even when without the signature changes they would be ok.
3458 : See PR121023. */
3459 7485874 : if (cfun->has_musttail)
3460 : {
3461 1441 : if (dump_file)
3462 0 : fprintf (dump_file, "No signature change:"
3463 : " function has calls with musttail attribute.\n");
3464 : no_signature = true;
3465 : }
3466 :
3467 : /* Type attributes can use parameter indices to describe them.
3468 : Special case fn spec since we can safely preserve them in
3469 : modref summaries. */
3470 7485874 : for (tree list = TYPE_ATTRIBUTES (TREE_TYPE (node->decl));
3471 7884298 : list && !no_signature; list = TREE_CHAIN (list))
3472 398424 : if (!ipa_param_adjustments::type_attribute_allowed_p
3473 398424 : (get_attribute_name (list)))
3474 : {
3475 159119 : if (dump_file)
3476 : {
3477 0 : fprintf (dump_file, "No signature change:"
3478 : " function type has unhandled attribute %s.\n",
3479 0 : IDENTIFIER_POINTER (get_attribute_name (list)));
3480 : }
3481 : no_signature = true;
3482 : }
3483 7485874 : for (tree parm = DECL_ARGUMENTS (node->decl);
3484 22327789 : parm && !no_signature; parm = DECL_CHAIN (parm))
3485 14841915 : if (variably_modified_type_p (TREE_TYPE (parm), node->decl))
3486 : {
3487 15786 : if (dump_file)
3488 : {
3489 0 : fprintf (dump_file, "No signature change:"
3490 : " has parameter with variably modified type.\n");
3491 : }
3492 : no_signature = true;
3493 : }
3494 :
3495 : /* Likewise for #pragma omp declare simd functions or functions
3496 : with simd attribute. */
3497 7485874 : if (no_signature
3498 14795402 : || lookup_attribute ("omp declare simd",
3499 7309528 : DECL_ATTRIBUTES (node->decl)))
3500 178037 : node->can_change_signature = false;
3501 : else
3502 : {
3503 : /* Otherwise, inlinable functions always can change signature. */
3504 7307837 : if (info->inlinable)
3505 5883724 : node->can_change_signature = true;
3506 : else
3507 : {
3508 : /* Functions calling builtin_apply cannot change signature. */
3509 5494381 : for (e = node->callees; e; e = e->next_callee)
3510 : {
3511 4103062 : tree cdecl = e->callee->decl;
3512 4103062 : if (fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS,
3513 : BUILT_IN_VA_START))
3514 : break;
3515 : }
3516 1424113 : node->can_change_signature = !e;
3517 : }
3518 : }
3519 7485874 : analyze_function_body (node, early);
3520 7485874 : pop_cfun ();
3521 : }
3522 :
3523 : /* Inlining characteristics are maintained by the cgraph_mark_inline. */
3524 7487182 : size_info->size = size_info->self_size;
3525 7487182 : info->estimated_stack_size = size_info->estimated_self_stack_size;
3526 :
3527 : /* Code above should compute exactly the same result as
3528 : ipa_update_overall_fn_summary except for case when speculative
3529 : edges are present since these are accounted to size but not
3530 : self_size. Do not compare time since different order the roundoff
3531 : errors result in slight changes. */
3532 7487182 : ipa_update_overall_fn_summary (node);
3533 7487182 : if (flag_checking)
3534 : {
3535 7969465 : for (e = node->indirect_calls; e; e = e->next_callee)
3536 482365 : if (e->speculative)
3537 : break;
3538 7487100 : gcc_assert (e || size_info->size == size_info->self_size);
3539 : }
3540 7487182 : }
3541 :
3542 :
3543 : /* Compute parameters of functions used by inliner using
3544 : current_function_decl. */
3545 :
3546 : static unsigned int
3547 6007418 : compute_fn_summary_for_current (void)
3548 : {
3549 6007418 : compute_fn_summary (cgraph_node::get (current_function_decl), true);
3550 6007418 : return 0;
3551 : }
3552 :
3553 : /* Estimate benefit devirtualizing indirect edge IE and return true if it can
3554 : be devirtualized and inlined, provided m_known_vals, m_known_contexts and
3555 : m_known_aggs in AVALS. Return false straight away if AVALS is NULL. */
3556 :
3557 : static bool
3558 4902127 : estimate_edge_devirt_benefit (struct cgraph_edge *ie,
3559 : int *size, int *time,
3560 : ipa_call_arg_values *avals)
3561 : {
3562 4902127 : tree target;
3563 4902127 : struct cgraph_node *callee;
3564 4902127 : class ipa_fn_summary *isummary;
3565 4902127 : enum availability avail;
3566 4902127 : bool speculative;
3567 :
3568 4902127 : if (!avals
3569 4902127 : || (!avals->m_known_vals.length() && !avals->m_known_contexts.length ()))
3570 : return false;
3571 1476318 : if (!opt_for_fn (ie->caller->decl, flag_indirect_inlining))
3572 : return false;
3573 :
3574 1469855 : target = ipa_get_indirect_edge_target
3575 1469855 : (ie->callee ? ie->speculative_call_indirect_edge () : ie,
3576 : avals, &speculative);
3577 1469855 : if (!target || speculative)
3578 : return false;
3579 :
3580 : /* If this is speculative call, turn its cost into 0; we will account
3581 : the call when processing the indirect call. */
3582 217901 : if (ie->callee)
3583 : {
3584 4402 : gcc_checking_assert (ie->speculative && *size > 0);
3585 4402 : *size = 0;
3586 4402 : *time = 0;
3587 : }
3588 : else
3589 : {
3590 : /* Account for difference in cost between indirect and direct calls. */
3591 213499 : *size -= (eni_size_weights.indirect_call_cost - eni_size_weights.call_cost);
3592 213499 : *time -= (eni_time_weights.indirect_call_cost - eni_time_weights.call_cost);
3593 : }
3594 217901 : gcc_checking_assert (*time >= 0);
3595 217901 : gcc_checking_assert (*size >= 0);
3596 :
3597 217901 : callee = cgraph_node::get (target);
3598 217901 : if (!callee || !callee->definition)
3599 : return false;
3600 197266 : callee = callee->function_symbol (&avail);
3601 197266 : if (avail < AVAIL_AVAILABLE)
3602 : return false;
3603 197240 : isummary = ipa_fn_summaries->get (callee);
3604 197240 : if (isummary == NULL)
3605 : return false;
3606 :
3607 197226 : return isummary->inlinable;
3608 : }
3609 :
3610 : /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
3611 : handle edge E with probability PROB. Set HINTS accordingly if edge may be
3612 : devirtualized. AVALS, if non-NULL, describes the context of the call site
3613 : as far as values of parameters are concerned. */
3614 :
3615 : static inline void
3616 220088265 : estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *min_size,
3617 : sreal *time, ipa_call_arg_values *avals,
3618 : ipa_hints *hints)
3619 : {
3620 220088265 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3621 220088265 : int call_size = es->call_stmt_size;
3622 220088265 : int call_time = es->call_stmt_time;
3623 220088265 : int cur_size;
3624 :
3625 215802097 : if ((!e->callee || e->speculative)
3626 220704224 : && estimate_edge_devirt_benefit (e, &call_size, &call_time, avals))
3627 : {
3628 196980 : if (hints && e->maybe_hot_p ())
3629 187838 : *hints |= INLINE_HINT_indirect_call;
3630 : }
3631 220088265 : cur_size = call_size * ipa_fn_summary::size_scale;
3632 220088265 : *size += cur_size;
3633 220088265 : if (min_size)
3634 29194774 : *min_size += cur_size;
3635 220088265 : if (time)
3636 210122182 : *time += ((sreal)call_time) * e->sreal_frequency ();
3637 220088265 : }
3638 :
3639 :
3640 : /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3641 : calls in NODE. POSSIBLE_TRUTHS and AVALS describe the context of the call
3642 : site.
3643 :
3644 : Helper for estimate_calls_size_and_time which does the same but
3645 : (in most cases) faster. */
3646 :
3647 : static void
3648 80000357 : estimate_calls_size_and_time_1 (struct cgraph_node *node, int *size,
3649 : int *min_size, sreal *time,
3650 : ipa_hints *hints,
3651 : clause_t possible_truths,
3652 : ipa_call_arg_values *avals)
3653 : {
3654 80000357 : struct cgraph_edge *e;
3655 379669047 : for (e = node->callees; e; e = e->next_callee)
3656 : {
3657 299668690 : if (!e->inline_failed)
3658 : {
3659 59110750 : gcc_checking_assert (!ipa_call_summaries->get (e));
3660 59110750 : estimate_calls_size_and_time_1 (e->callee, size, min_size, time,
3661 : hints, possible_truths, avals);
3662 :
3663 59110750 : continue;
3664 : }
3665 240557940 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3666 :
3667 : /* Do not care about zero sized builtins. */
3668 240557940 : if (!es->call_stmt_size)
3669 : {
3670 33596539 : gcc_checking_assert (!es->call_stmt_time);
3671 33596539 : continue;
3672 : }
3673 206961401 : if (!es->predicate
3674 206961401 : || es->predicate->evaluate (possible_truths))
3675 : {
3676 : /* Predicates of calls shall not use NOT_CHANGED codes,
3677 : so we do not need to compute probabilities. */
3678 204494961 : estimate_edge_size_and_time (e, size,
3679 204494961 : es->predicate ? NULL : min_size,
3680 : time, avals, hints);
3681 : }
3682 : }
3683 84001608 : for (e = node->indirect_calls; e; e = e->next_callee)
3684 : {
3685 4001251 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3686 4001251 : if (!es->predicate
3687 4001251 : || es->predicate->evaluate (possible_truths))
3688 3969124 : estimate_edge_size_and_time (e, size,
3689 3969124 : es->predicate ? NULL : min_size,
3690 : time, avals, hints);
3691 : }
3692 80000357 : }
3693 :
3694 : /* Populate sum->call_size_time_table for edges from NODE. */
3695 :
3696 : static void
3697 3328047 : summarize_calls_size_and_time (struct cgraph_node *node,
3698 : ipa_fn_summary *sum)
3699 : {
3700 3328047 : struct cgraph_edge *e;
3701 15254387 : for (e = node->callees; e; e = e->next_callee)
3702 : {
3703 11926340 : if (!e->inline_failed)
3704 : {
3705 1526134 : gcc_checking_assert (!ipa_call_summaries->get (e));
3706 1526134 : summarize_calls_size_and_time (e->callee, sum);
3707 1526134 : continue;
3708 : }
3709 10400206 : int size = 0;
3710 10400206 : sreal time = 0;
3711 :
3712 10400206 : estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
3713 :
3714 10400206 : ipa_predicate pred = true;
3715 10400206 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3716 :
3717 10400206 : if (es->predicate)
3718 1907187 : pred = *es->predicate;
3719 10400206 : sum->account_size_time (size, time, pred, pred, true);
3720 : }
3721 3645091 : for (e = node->indirect_calls; e; e = e->next_callee)
3722 : {
3723 317044 : int size = 0;
3724 317044 : sreal time = 0;
3725 :
3726 317044 : estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
3727 317044 : ipa_predicate pred = true;
3728 317044 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3729 :
3730 317044 : if (es->predicate)
3731 81003 : pred = *es->predicate;
3732 317044 : sum->account_size_time (size, time, pred, pred, true);
3733 : }
3734 3328047 : }
3735 :
3736 : /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3737 : calls in NODE. POSSIBLE_TRUTHS and AVALS (the latter if non-NULL) describe
3738 : context of the call site. */
3739 :
3740 : static void
3741 20889628 : estimate_calls_size_and_time (struct cgraph_node *node, int *size,
3742 : int *min_size, sreal *time,
3743 : ipa_hints *hints,
3744 : clause_t possible_truths,
3745 : ipa_call_arg_values *avals)
3746 : {
3747 20889628 : class ipa_fn_summary *sum = ipa_fn_summaries->get (node);
3748 20889628 : bool use_table = true;
3749 :
3750 20889628 : gcc_assert (node->callees || node->indirect_calls);
3751 :
3752 : /* During early inlining we do not calculate info for very
3753 : large functions and thus there is no need for producing
3754 : summaries. */
3755 20889628 : if (!ipa_node_params_sum)
3756 : use_table = false;
3757 : /* Do not calculate summaries for simple wrappers; it is waste
3758 : of memory. */
3759 11323239 : else if (node->callees && !node->indirect_calls
3760 10563845 : && node->callees->inline_failed && !node->callees->next_callee)
3761 : use_table = false;
3762 : /* If there is an indirect edge that may be optimized, we need
3763 : to go the slow way. */
3764 9100232 : else if (avals
3765 9100232 : && (avals->m_known_vals.length ()
3766 3397337 : || avals->m_known_contexts.length ()
3767 3251324 : || avals->m_known_aggs.length ()))
3768 : {
3769 4301163 : ipa_node_params *params_summary = ipa_node_params_sum->get (node);
3770 4301163 : unsigned int nargs = params_summary
3771 4301163 : ? ipa_get_param_count (params_summary) : 0;
3772 :
3773 14281025 : for (unsigned int i = 0; i < nargs && use_table; i++)
3774 : {
3775 9979862 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3776 9979862 : && (avals->safe_sval_at (i)
3777 269659 : || (ipa_argagg_value_list (avals).value_for_index_p (i))))
3778 : use_table = false;
3779 9883478 : else if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3780 9883478 : && (avals->m_known_contexts.length () > i
3781 10121632 : && !avals->m_known_contexts[i].useless_p ()))
3782 : use_table = false;
3783 : }
3784 : }
3785 :
3786 : /* Fast path is via the call size time table. */
3787 4301163 : if (use_table)
3788 : {
3789 : /* Build summary if it is absent. */
3790 8838956 : if (!sum->call_size_time_table.length ())
3791 : {
3792 894983 : ipa_predicate true_pred = true;
3793 894983 : sum->account_size_time (0, 0, true_pred, true_pred, true);
3794 894983 : summarize_calls_size_and_time (node, sum);
3795 : }
3796 :
3797 8838956 : int old_size = *size;
3798 8838956 : sreal old_time = time ? *time : 0;
3799 :
3800 8838956 : if (min_size)
3801 8838956 : *min_size += sum->call_size_time_table[0].size;
3802 :
3803 8838956 : unsigned int i;
3804 8838956 : size_time_entry *e;
3805 :
3806 : /* Walk the table and account sizes and times. */
3807 23293427 : for (i = 0; sum->call_size_time_table.iterate (i, &e);
3808 : i++)
3809 14454471 : if (e->exec_predicate.evaluate (possible_truths))
3810 : {
3811 13531565 : *size += e->size;
3812 13531565 : if (time)
3813 11363104 : *time += e->time;
3814 : }
3815 :
3816 : /* Be careful and see if both methods agree. */
3817 21 : if ((flag_checking || dump_file)
3818 : /* Do not try to sanity check when we know we lost some
3819 : precision. */
3820 8838956 : && sum->call_size_time_table.length ()
3821 : < ipa_fn_summary::max_size_time_table_size)
3822 : {
3823 8838935 : estimate_calls_size_and_time_1 (node, &old_size, NULL, &old_time, NULL,
3824 : possible_truths, avals);
3825 8838935 : gcc_assert (*size == old_size);
3826 16248141 : if (time && (*time - old_time > 1 || *time - old_time < -1)
3827 8838963 : && dump_file)
3828 0 : fprintf (dump_file, "Time mismatch in call summary %f!=%f\n",
3829 : old_time.to_double (),
3830 : time->to_double ());
3831 : }
3832 : }
3833 : /* Slow path by walking all edges. */
3834 : else
3835 12050672 : estimate_calls_size_and_time_1 (node, size, min_size, time, hints,
3836 : possible_truths, avals);
3837 20889628 : }
3838 :
3839 : /* Main constructor for ipa call context. Memory allocation of ARG_VALUES
3840 : is owned by the caller. INLINE_PARAM_SUMMARY is also owned by the
3841 : caller. */
3842 :
3843 19944344 : ipa_call_context::ipa_call_context (cgraph_node *node, clause_t possible_truths,
3844 : clause_t nonspec_possible_truths,
3845 : vec<inline_param_summary>
3846 : inline_param_summary,
3847 : ipa_auto_call_arg_values *arg_values)
3848 19944344 : : m_node (node), m_possible_truths (possible_truths),
3849 19944344 : m_nonspec_possible_truths (nonspec_possible_truths),
3850 19944344 : m_inline_param_summary (inline_param_summary),
3851 19944344 : m_avals (arg_values)
3852 : {
3853 19944344 : }
3854 :
3855 : /* Set THIS to be a duplicate of CTX. Copy all relevant info. */
3856 :
3857 : void
3858 2008617 : ipa_cached_call_context::duplicate_from (const ipa_call_context &ctx)
3859 : {
3860 2008617 : m_node = ctx.m_node;
3861 2008617 : m_possible_truths = ctx.m_possible_truths;
3862 2008617 : m_nonspec_possible_truths = ctx.m_nonspec_possible_truths;
3863 2008617 : ipa_node_params *params_summary = ipa_node_params_sum->get (m_node);
3864 2008617 : unsigned int nargs = params_summary
3865 2008617 : ? ipa_get_param_count (params_summary) : 0;
3866 :
3867 2008617 : m_inline_param_summary = vNULL;
3868 : /* Copy the info only if there is at least one useful entry. */
3869 2008617 : if (ctx.m_inline_param_summary.exists ())
3870 : {
3871 1789288 : unsigned int n = MIN (ctx.m_inline_param_summary.length (), nargs);
3872 :
3873 4094512 : for (unsigned int i = 0; i < n; i++)
3874 3345026 : if (ipa_is_param_used_by_ipa_predicates (params_summary, i)
3875 4772685 : && !ctx.m_inline_param_summary[i].useless_p ())
3876 : {
3877 1039802 : m_inline_param_summary
3878 1039802 : = ctx.m_inline_param_summary.copy ();
3879 1039802 : break;
3880 : }
3881 : }
3882 2008617 : m_avals.m_known_vals = vNULL;
3883 2008617 : if (ctx.m_avals.m_known_vals.exists ())
3884 : {
3885 2008617 : unsigned int n = MIN (ctx.m_avals.m_known_vals.length (), nargs);
3886 :
3887 4666586 : for (unsigned int i = 0; i < n; i++)
3888 2684512 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3889 2684512 : && ctx.m_avals.m_known_vals[i])
3890 : {
3891 26543 : m_avals.m_known_vals = ctx.m_avals.m_known_vals.copy ();
3892 26543 : break;
3893 : }
3894 : }
3895 :
3896 2008617 : m_avals.m_known_contexts = vNULL;
3897 2008617 : if (ctx.m_avals.m_known_contexts.exists ())
3898 : {
3899 2008617 : unsigned int n = MIN (ctx.m_avals.m_known_contexts.length (), nargs);
3900 :
3901 2010634 : for (unsigned int i = 0; i < n; i++)
3902 26780 : if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3903 26780 : && !ctx.m_avals.m_known_contexts[i].useless_p ())
3904 : {
3905 24763 : m_avals.m_known_contexts = ctx.m_avals.m_known_contexts.copy ();
3906 24763 : break;
3907 : }
3908 : }
3909 :
3910 2008617 : m_avals.m_known_aggs = vNULL;
3911 2008617 : if (ctx.m_avals.m_known_aggs.exists ())
3912 : {
3913 2008617 : const ipa_argagg_value_list avl (&ctx.m_avals);
3914 6672848 : for (unsigned int i = 0; i < nargs; i++)
3915 4667155 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3916 4667155 : && avl.value_for_index_p (i))
3917 : {
3918 2924 : m_avals.m_known_aggs = ctx.m_avals.m_known_aggs.copy ();
3919 2924 : break;
3920 : }
3921 : }
3922 :
3923 2008617 : m_avals.m_known_value_ranges = vNULL;
3924 2008617 : }
3925 :
3926 : /* Release memory used by known_vals/contexts/aggs vectors. and
3927 : inline_param_summary. */
3928 :
3929 : void
3930 3269456 : ipa_cached_call_context::release ()
3931 : {
3932 : /* See if context is initialized at first place. */
3933 3269456 : if (!m_node)
3934 : return;
3935 2008617 : m_avals.m_known_aggs.release ();
3936 2008617 : m_avals.m_known_vals.release ();
3937 2008617 : m_avals.m_known_contexts.release ();
3938 2008617 : m_inline_param_summary.release ();
3939 : }
3940 :
3941 : /* Return true if CTX describes the same call context as THIS. */
3942 :
3943 : bool
3944 7101682 : ipa_call_context::equal_to (const ipa_call_context &ctx)
3945 : {
3946 7101682 : if (m_node != ctx.m_node
3947 5840843 : || m_possible_truths != ctx.m_possible_truths
3948 5224324 : || m_nonspec_possible_truths != ctx.m_nonspec_possible_truths)
3949 : return false;
3950 :
3951 5224324 : ipa_node_params *params_summary = ipa_node_params_sum->get (m_node);
3952 5224324 : unsigned int nargs = params_summary
3953 5224324 : ? ipa_get_param_count (params_summary) : 0;
3954 :
3955 5224324 : if (m_inline_param_summary.exists () || ctx.m_inline_param_summary.exists ())
3956 : {
3957 15559820 : for (unsigned int i = 0; i < nargs; i++)
3958 : {
3959 10671676 : if (!ipa_is_param_used_by_ipa_predicates (params_summary, i))
3960 3347933 : continue;
3961 7323743 : if (i >= m_inline_param_summary.length ()
3962 5098029 : || m_inline_param_summary[i].useless_p ())
3963 : {
3964 3230145 : if (i < ctx.m_inline_param_summary.length ()
3965 3230145 : && !ctx.m_inline_param_summary[i].useless_p ())
3966 : return false;
3967 3184559 : continue;
3968 : }
3969 4093598 : if (i >= ctx.m_inline_param_summary.length ()
3970 4093595 : || ctx.m_inline_param_summary[i].useless_p ())
3971 : {
3972 45031 : if (i < m_inline_param_summary.length ()
3973 45031 : && !m_inline_param_summary[i].useless_p ())
3974 : return false;
3975 0 : continue;
3976 : }
3977 4048567 : if (!m_inline_param_summary[i].equal_to
3978 4048567 : (ctx.m_inline_param_summary[i]))
3979 : return false;
3980 : }
3981 : }
3982 5106354 : if (m_avals.m_known_vals.exists () || ctx.m_avals.m_known_vals.exists ())
3983 : {
3984 15583057 : for (unsigned int i = 0; i < nargs; i++)
3985 : {
3986 10487174 : if (!ipa_is_param_used_by_indirect_call (params_summary, i))
3987 10278002 : continue;
3988 209172 : if (i >= m_avals.m_known_vals.length () || !m_avals.m_known_vals[i])
3989 : {
3990 151945 : if (i < ctx.m_avals.m_known_vals.length ()
3991 151945 : && ctx.m_avals.m_known_vals[i])
3992 : return false;
3993 151856 : continue;
3994 : }
3995 57227 : if (i >= ctx.m_avals.m_known_vals.length ()
3996 57227 : || !ctx.m_avals.m_known_vals[i])
3997 : {
3998 : if (i < m_avals.m_known_vals.length () && m_avals.m_known_vals[i])
3999 : return false;
4000 : continue;
4001 : }
4002 57137 : if (m_avals.m_known_vals[i] != ctx.m_avals.m_known_vals[i])
4003 : return false;
4004 : }
4005 : }
4006 5095883 : if (m_avals.m_known_contexts.exists ()
4007 5095883 : || ctx.m_avals.m_known_contexts.exists ())
4008 : {
4009 15568062 : for (unsigned int i = 0; i < nargs; i++)
4010 : {
4011 10473592 : if (!ipa_is_param_used_by_polymorphic_call (params_summary, i))
4012 10337672 : continue;
4013 135920 : if (i >= m_avals.m_known_contexts.length ()
4014 134758 : || m_avals.m_known_contexts[i].useless_p ())
4015 : {
4016 1162 : if (i < ctx.m_avals.m_known_contexts.length ()
4017 1162 : && !ctx.m_avals.m_known_contexts[i].useless_p ())
4018 : return false;
4019 1150 : continue;
4020 : }
4021 134758 : if (i >= ctx.m_avals.m_known_contexts.length ()
4022 134758 : || ctx.m_avals.m_known_contexts[i].useless_p ())
4023 : {
4024 8 : if (i < m_avals.m_known_contexts.length ()
4025 8 : && !m_avals.m_known_contexts[i].useless_p ())
4026 : return false;
4027 0 : continue;
4028 : }
4029 134750 : if (!m_avals.m_known_contexts[i].equal_to
4030 134750 : (ctx.m_avals.m_known_contexts[i]))
4031 : return false;
4032 : }
4033 : }
4034 5094470 : if (m_avals.m_known_aggs.exists () || ctx.m_avals.m_known_aggs.exists ())
4035 : {
4036 : unsigned i = 0, j = 0;
4037 12073136 : while (i < m_avals.m_known_aggs.length ()
4038 6035809 : || j < ctx.m_avals.m_known_aggs.length ())
4039 : {
4040 942744 : if (i >= m_avals.m_known_aggs.length ())
4041 : {
4042 937846 : int idx2 = ctx.m_avals.m_known_aggs[j].index;
4043 937846 : if (ipa_is_param_used_by_indirect_call (params_summary, idx2))
4044 : return false;
4045 937292 : j++;
4046 937292 : continue;
4047 937292 : }
4048 4898 : if (j >= ctx.m_avals.m_known_aggs.length ())
4049 : {
4050 600 : int idx1 = m_avals.m_known_aggs[i].index;
4051 600 : if (ipa_is_param_used_by_indirect_call (params_summary, idx1))
4052 : return false;
4053 8 : i++;
4054 8 : continue;
4055 8 : }
4056 :
4057 4298 : int idx1 = m_avals.m_known_aggs[i].index;
4058 4298 : int idx2 = ctx.m_avals.m_known_aggs[j].index;
4059 4298 : if (idx1 < idx2)
4060 : {
4061 0 : if (ipa_is_param_used_by_indirect_call (params_summary, idx1))
4062 : return false;
4063 0 : i++;
4064 0 : continue;
4065 : }
4066 4298 : if (idx1 > idx2)
4067 : {
4068 0 : if (ipa_is_param_used_by_indirect_call (params_summary, idx2))
4069 : return false;
4070 0 : j++;
4071 0 : continue;
4072 : }
4073 4298 : if (!ipa_is_param_used_by_indirect_call (params_summary, idx1))
4074 : {
4075 482 : i++;
4076 482 : j++;
4077 482 : continue;
4078 : }
4079 :
4080 3816 : if ((m_avals.m_known_aggs[i].unit_offset
4081 3816 : != ctx.m_avals.m_known_aggs[j].unit_offset)
4082 3808 : || (m_avals.m_known_aggs[i].by_ref
4083 3808 : != ctx.m_avals.m_known_aggs[j].by_ref)
4084 7624 : || !operand_equal_p (m_avals.m_known_aggs[i].value,
4085 3808 : ctx.m_avals.m_known_aggs[j].value))
4086 : return false;
4087 3557 : i++;
4088 3557 : j++;
4089 : }
4090 : }
4091 : return true;
4092 : }
4093 :
4094 : /* Fill in the selected fields in ESTIMATES with value estimated for call in
4095 : this context. Always compute size and min_size. Only compute time and
4096 : nonspecialized_time if EST_TIMES is true. Only compute hints if EST_HINTS
4097 : is true. */
4098 :
4099 : void
4100 19912788 : ipa_call_context::estimate_size_and_time (ipa_call_estimates *estimates,
4101 : bool est_times, bool est_hints)
4102 : {
4103 19912788 : class ipa_fn_summary *info = ipa_fn_summaries->get (m_node);
4104 19912788 : size_time_entry *e;
4105 19912788 : int size = 0;
4106 19912788 : sreal time = 0;
4107 19912788 : int min_size = 0;
4108 19912788 : ipa_hints hints = 0;
4109 19912788 : sreal loops_with_known_iterations = 0;
4110 19912788 : sreal loops_with_known_strides = 0;
4111 19912788 : int i;
4112 :
4113 19912788 : if (dump_file && (dump_flags & TDF_DETAILS))
4114 : {
4115 1317 : bool found = false;
4116 1317 : fprintf (dump_file, " Estimating body: %s\n"
4117 : " Known to be false: ", m_node->dump_name ());
4118 :
4119 4267 : for (i = ipa_predicate::not_inlined_condition;
4120 8534 : i < (ipa_predicate::first_dynamic_condition
4121 6984 : + (int) vec_safe_length (info->conds)); i++)
4122 2950 : if (!(m_possible_truths & (1 << i)))
4123 : {
4124 1783 : if (found)
4125 519 : fprintf (dump_file, ", ");
4126 1783 : found = true;
4127 1783 : dump_condition (dump_file, info->conds, i);
4128 : }
4129 : }
4130 :
4131 19912788 : if (m_node->callees || m_node->indirect_calls)
4132 26393465 : estimate_calls_size_and_time (m_node, &size, &min_size,
4133 : est_times ? &time : NULL,
4134 : est_hints ? &hints : NULL, m_possible_truths,
4135 : &m_avals);
4136 :
4137 19912788 : sreal nonspecialized_time = time;
4138 :
4139 19912788 : min_size += info->size_time_table[0].size;
4140 136785424 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
4141 : {
4142 116872636 : bool exec = e->exec_predicate.evaluate (m_nonspec_possible_truths);
4143 :
4144 : /* Because predicates are conservative, it can happen that nonconst is 1
4145 : but exec is 0. */
4146 116872636 : if (exec)
4147 : {
4148 114381421 : bool nonconst = e->nonconst_predicate.evaluate (m_possible_truths);
4149 :
4150 114381421 : gcc_checking_assert (e->time >= 0);
4151 114381421 : gcc_checking_assert (time >= 0);
4152 :
4153 : /* We compute specialized size only because size of nonspecialized
4154 : copy is context independent.
4155 :
4156 : The difference between nonspecialized execution and specialized is
4157 : that nonspecialized is not going to have optimized out computations
4158 : known to be constant in a specialized setting. */
4159 114381421 : if (nonconst)
4160 57566388 : size += e->size;
4161 114381421 : if (!est_times)
4162 59095524 : continue;
4163 55285897 : nonspecialized_time += e->time;
4164 55285897 : if (!nonconst)
4165 : ;
4166 26599161 : else if (!m_inline_param_summary.exists ())
4167 : {
4168 2414756 : if (nonconst)
4169 2414756 : time += e->time;
4170 : }
4171 : else
4172 : {
4173 24184405 : int prob = e->nonconst_predicate.probability
4174 24184405 : (info->conds, m_possible_truths,
4175 : m_inline_param_summary);
4176 24184405 : gcc_checking_assert (prob >= 0);
4177 24184405 : gcc_checking_assert (prob <= REG_BR_PROB_BASE);
4178 24184405 : if (prob == REG_BR_PROB_BASE)
4179 19980729 : time += e->time;
4180 : else
4181 4203676 : time += e->time * prob / REG_BR_PROB_BASE;
4182 : }
4183 55285897 : gcc_checking_assert (time >= 0);
4184 : }
4185 : }
4186 19912788 : gcc_checking_assert (info->size_time_table[0].exec_predicate == true);
4187 19912788 : gcc_checking_assert (info->size_time_table[0].nonconst_predicate == true);
4188 19912788 : gcc_checking_assert (min_size >= 0);
4189 19912788 : gcc_checking_assert (size >= 0);
4190 19912788 : gcc_checking_assert (time >= 0);
4191 : /* nonspecialized_time should be always bigger than specialized time.
4192 : Roundoff issues however may get into the way. */
4193 19912788 : gcc_checking_assert ((nonspecialized_time - time * 99 / 100) >= -1);
4194 :
4195 : /* Roundoff issues may make specialized time bigger than nonspecialized
4196 : time. We do not really want that to happen because some heuristics
4197 : may get confused by seeing negative speedups. */
4198 19912788 : if (time > nonspecialized_time)
4199 0 : time = nonspecialized_time;
4200 :
4201 19912788 : if (est_hints)
4202 : {
4203 7303750 : if (info->scc_no)
4204 233772 : hints |= INLINE_HINT_in_scc;
4205 7303750 : if (DECL_DECLARED_INLINE_P (m_node->decl))
4206 4661195 : hints |= INLINE_HINT_declared_inline;
4207 7303750 : if (info->builtin_constant_p_parms.length ()
4208 9529 : && DECL_DECLARED_INLINE_P (m_node->decl))
4209 9471 : hints |= INLINE_HINT_builtin_constant_p;
4210 :
4211 7303750 : ipa_freqcounting_predicate *fcp;
4212 7855522 : for (i = 0; vec_safe_iterate (info->loop_iterations, i, &fcp); i++)
4213 551772 : if (!fcp->predicate->evaluate (m_possible_truths))
4214 : {
4215 370881 : hints |= INLINE_HINT_loop_iterations;
4216 370881 : loops_with_known_iterations += fcp->freq;
4217 : }
4218 7303750 : estimates->loops_with_known_iterations = loops_with_known_iterations;
4219 :
4220 7595836 : for (i = 0; vec_safe_iterate (info->loop_strides, i, &fcp); i++)
4221 292086 : if (!fcp->predicate->evaluate (m_possible_truths))
4222 : {
4223 262389 : hints |= INLINE_HINT_loop_stride;
4224 262389 : loops_with_known_strides += fcp->freq;
4225 : }
4226 7303750 : estimates->loops_with_known_strides = loops_with_known_strides;
4227 : }
4228 :
4229 19912788 : size = RDIV (size, ipa_fn_summary::size_scale);
4230 19912788 : min_size = RDIV (min_size, ipa_fn_summary::size_scale);
4231 :
4232 19912788 : if (dump_file && (dump_flags & TDF_DETAILS))
4233 : {
4234 1317 : fprintf (dump_file, "\n size:%i", (int) size);
4235 1317 : if (est_times)
4236 1080 : fprintf (dump_file, " time:%f nonspec time:%f",
4237 : time.to_double (), nonspecialized_time.to_double ());
4238 1317 : if (est_hints)
4239 1080 : fprintf (dump_file, " loops with known iterations:%f "
4240 : "known strides:%f", loops_with_known_iterations.to_double (),
4241 : loops_with_known_strides.to_double ());
4242 1317 : fprintf (dump_file, "\n");
4243 : }
4244 19912788 : if (est_times)
4245 : {
4246 7303750 : estimates->time = time;
4247 7303750 : estimates->nonspecialized_time = nonspecialized_time;
4248 : }
4249 19912788 : estimates->size = size;
4250 19912788 : estimates->min_size = min_size;
4251 19912788 : if (est_hints)
4252 7303750 : estimates->hints = hints;
4253 19912788 : return;
4254 : }
4255 :
4256 :
4257 : /* Estimate size and time needed to execute callee of EDGE assuming that
4258 : parameters known to be constant at caller of EDGE are propagated.
4259 : KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
4260 : and types for parameters. */
4261 :
4262 : void
4263 229823 : estimate_ipcp_clone_size_and_time (struct cgraph_node *node,
4264 : ipa_auto_call_arg_values *avals,
4265 : ipa_call_estimates *estimates)
4266 : {
4267 229823 : clause_t clause, nonspec_clause;
4268 :
4269 229823 : evaluate_conditions_for_known_args (node, false, avals, &clause,
4270 : &nonspec_clause, NULL);
4271 229823 : ipa_call_context ctx (node, clause, nonspec_clause, vNULL, avals);
4272 229823 : ctx.estimate_size_and_time (estimates);
4273 229823 : }
4274 :
4275 : /* Return stack frame offset where frame of NODE is supposed to start inside
4276 : of the function it is inlined to.
4277 : Return 0 for functions that are not inlined. */
4278 :
4279 : HOST_WIDE_INT
4280 5271281 : ipa_get_stack_frame_offset (struct cgraph_node *node)
4281 : {
4282 5271281 : HOST_WIDE_INT offset = 0;
4283 5271281 : if (!node->inlined_to)
4284 : return 0;
4285 4200405 : node = node->callers->caller;
4286 5442461 : while (true)
4287 : {
4288 4821433 : offset += ipa_size_summaries->get (node)->estimated_self_stack_size;
4289 4821433 : if (!node->inlined_to)
4290 : return offset;
4291 621028 : node = node->callers->caller;
4292 : }
4293 : }
4294 :
4295 :
4296 : /* Update summary information of inline clones after inlining.
4297 : Compute peak stack usage. */
4298 :
4299 : static void
4300 4958925 : inline_update_callee_summaries (struct cgraph_node *node, int depth)
4301 : {
4302 4958925 : struct cgraph_edge *e;
4303 :
4304 4958925 : ipa_propagate_frequency (node);
4305 9629490 : for (e = node->callees; e; e = e->next_callee)
4306 : {
4307 4670565 : if (!e->inline_failed)
4308 759073 : inline_update_callee_summaries (e->callee, depth);
4309 : else
4310 3911492 : ipa_call_summaries->get (e)->loop_depth += depth;
4311 : }
4312 5065091 : for (e = node->indirect_calls; e; e = e->next_callee)
4313 106166 : ipa_call_summaries->get (e)->loop_depth += depth;
4314 4958925 : }
4315 :
4316 : /* Update change_prob and points_to_local_or_readonly_memory of EDGE after
4317 : INLINED_EDGE has been inlined.
4318 :
4319 : When function A is inlined in B and A calls C with parameter that
4320 : changes with probability PROB1 and C is known to be passthrough
4321 : of argument if B that change with probability PROB2, the probability
4322 : of change is now PROB1*PROB2. */
4323 :
4324 : static void
4325 4017968 : remap_edge_params (struct cgraph_edge *inlined_edge,
4326 : struct cgraph_edge *edge)
4327 : {
4328 4017968 : if (ipa_node_params_sum)
4329 : {
4330 2832421 : int i;
4331 2832421 : ipa_edge_args *args = ipa_edge_args_sum->get (edge);
4332 2832421 : if (!args)
4333 : return;
4334 1609300 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
4335 1609300 : class ipa_call_summary *inlined_es
4336 1609300 : = ipa_call_summaries->get (inlined_edge);
4337 :
4338 1609300 : if (es->param.length () == 0)
4339 : return;
4340 :
4341 9381806 : for (i = 0; i < ipa_get_cs_argument_count (args); i++)
4342 : {
4343 3161601 : struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
4344 3161601 : if (jfunc->type == IPA_JF_PASS_THROUGH
4345 2321297 : || jfunc->type == IPA_JF_ANCESTOR)
4346 : {
4347 985839 : int id = jfunc->type == IPA_JF_PASS_THROUGH
4348 985839 : ? ipa_get_jf_pass_through_formal_id (jfunc)
4349 145535 : : ipa_get_jf_ancestor_formal_id (jfunc);
4350 1971595 : if (id < (int) inlined_es->param.length ())
4351 : {
4352 985748 : int prob1 = es->param[i].change_prob;
4353 985748 : int prob2 = inlined_es->param[id].change_prob;
4354 985748 : int prob = combine_probabilities (prob1, prob2);
4355 :
4356 985748 : if (prob1 && prob2 && !prob)
4357 985748 : prob = 1;
4358 :
4359 985748 : es->param[i].change_prob = prob;
4360 :
4361 1971496 : if (inlined_es
4362 985748 : ->param[id].points_to_local_or_readonly_memory)
4363 215290 : es->param[i].points_to_local_or_readonly_memory = true;
4364 1971496 : if (inlined_es
4365 985748 : ->param[id].points_to_possible_sra_candidate)
4366 178735 : es->param[i].points_to_possible_sra_candidate = true;
4367 : }
4368 985839 : if (!es->param[i].points_to_local_or_readonly_memory
4369 : && jfunc->type == IPA_JF_CONST
4370 : && points_to_local_or_readonly_memory_p
4371 : (ipa_get_jf_constant (jfunc)))
4372 : es->param[i].points_to_local_or_readonly_memory = true;
4373 : }
4374 : }
4375 : }
4376 : }
4377 :
4378 : /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
4379 :
4380 : Remap predicates of callees of NODE. Rest of arguments match
4381 : remap_predicate.
4382 :
4383 : Also update change probabilities. */
4384 :
4385 : static void
4386 4958925 : remap_edge_summaries (struct cgraph_edge *inlined_edge,
4387 : struct cgraph_node *node,
4388 : class ipa_fn_summary *info,
4389 : class ipa_node_params *params_summary,
4390 : class ipa_fn_summary *callee_info,
4391 : const vec<int> &operand_map,
4392 : const vec<HOST_WIDE_INT> &offset_map,
4393 : clause_t possible_truths,
4394 : ipa_predicate *toplev_predicate)
4395 : {
4396 4958925 : struct cgraph_edge *e, *next;
4397 9629049 : for (e = node->callees; e; e = next)
4398 : {
4399 4670124 : ipa_predicate p;
4400 4670124 : next = e->next_callee;
4401 :
4402 4670124 : if (e->inline_failed)
4403 : {
4404 3911051 : class ipa_call_summary *es = ipa_call_summaries->get (e);
4405 3911051 : remap_edge_params (inlined_edge, e);
4406 :
4407 3911051 : if (es->predicate)
4408 : {
4409 1351801 : p = es->predicate->remap_after_inlining
4410 1351801 : (info, params_summary,
4411 : callee_info, operand_map,
4412 : offset_map, possible_truths,
4413 : *toplev_predicate);
4414 1351801 : edge_set_predicate (e, &p);
4415 : }
4416 : else
4417 2559250 : edge_set_predicate (e, toplev_predicate);
4418 : }
4419 : else
4420 759073 : remap_edge_summaries (inlined_edge, e->callee, info,
4421 : params_summary, callee_info,
4422 : operand_map, offset_map, possible_truths,
4423 : toplev_predicate);
4424 : }
4425 5065842 : for (e = node->indirect_calls; e; e = next)
4426 : {
4427 106917 : class ipa_call_summary *es = ipa_call_summaries->get (e);
4428 106917 : ipa_predicate p;
4429 106917 : next = e->next_callee;
4430 :
4431 106917 : remap_edge_params (inlined_edge, e);
4432 106917 : if (es->predicate)
4433 : {
4434 27277 : p = es->predicate->remap_after_inlining
4435 27277 : (info, params_summary,
4436 : callee_info, operand_map, offset_map,
4437 : possible_truths, *toplev_predicate);
4438 27277 : edge_set_predicate (e, &p);
4439 : }
4440 : else
4441 79640 : edge_set_predicate (e, toplev_predicate);
4442 : }
4443 4958925 : }
4444 :
4445 : /* Run remap_after_inlining on each predicate in V. */
4446 :
4447 : static void
4448 8399704 : remap_freqcounting_predicate (class ipa_fn_summary *info,
4449 : class ipa_node_params *params_summary,
4450 : class ipa_fn_summary *callee_info,
4451 : vec<ipa_freqcounting_predicate, va_gc> *v,
4452 : const vec<int> &operand_map,
4453 : const vec<HOST_WIDE_INT> &offset_map,
4454 : clause_t possible_truths,
4455 : ipa_predicate *toplev_predicate)
4456 :
4457 : {
4458 8399704 : ipa_freqcounting_predicate *fcp;
4459 8427025 : for (int i = 0; vec_safe_iterate (v, i, &fcp); i++)
4460 : {
4461 27321 : ipa_predicate p
4462 27321 : = fcp->predicate->remap_after_inlining (info, params_summary,
4463 : callee_info, operand_map,
4464 : offset_map, possible_truths,
4465 : *toplev_predicate);
4466 37801 : if (p != false && p != true)
4467 2616 : *fcp->predicate &= p;
4468 : }
4469 8399704 : }
4470 :
4471 : /* We inlined EDGE. Update summary of the function we inlined into. */
4472 :
4473 : void
4474 4199852 : ipa_merge_fn_summary_after_inlining (struct cgraph_edge *edge)
4475 : {
4476 4199852 : ipa_fn_summary *callee_info = ipa_fn_summaries->get (edge->callee);
4477 3904882 : struct cgraph_node *to = (edge->caller->inlined_to
4478 4199852 : ? edge->caller->inlined_to : edge->caller);
4479 4199852 : class ipa_fn_summary *info = ipa_fn_summaries->get (to);
4480 4199852 : clause_t clause = 0; /* not_inline is known to be false. */
4481 4199852 : size_time_entry *e;
4482 4199852 : auto_vec<int, 8> operand_map;
4483 4199852 : auto_vec<HOST_WIDE_INT, 8> offset_map;
4484 4199852 : int i;
4485 4199852 : ipa_predicate toplev_predicate;
4486 4199852 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
4487 4199852 : ipa_node_params *params_summary = (ipa_node_params_sum
4488 4199852 : ? ipa_node_params_sum->get (to) : NULL);
4489 :
4490 4199852 : if (es->predicate)
4491 358517 : toplev_predicate = *es->predicate;
4492 : else
4493 3841335 : toplev_predicate = true;
4494 :
4495 4199852 : info->fp_expressions |= callee_info->fp_expressions;
4496 4199852 : info->target_info |= callee_info->target_info;
4497 :
4498 4199852 : if (callee_info->conds)
4499 : {
4500 3079118 : ipa_auto_call_arg_values avals;
4501 3079118 : evaluate_properties_for_edge (edge, true, &clause, NULL, &avals, false);
4502 3079118 : }
4503 4199852 : if (ipa_node_params_sum && callee_info->conds)
4504 : {
4505 845687 : ipa_edge_args *args = ipa_edge_args_sum->get (edge);
4506 845687 : int count = args ? ipa_get_cs_argument_count (args) : 0;
4507 845548 : int i;
4508 :
4509 845548 : if (count)
4510 : {
4511 845548 : operand_map.safe_grow_cleared (count, true);
4512 845548 : offset_map.safe_grow_cleared (count, true);
4513 : }
4514 2708325 : for (i = 0; i < count; i++)
4515 : {
4516 1862638 : struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
4517 1862638 : int map = -1;
4518 :
4519 : /* TODO: handle non-NOPs when merging. */
4520 1862638 : if (jfunc->type == IPA_JF_PASS_THROUGH)
4521 : {
4522 300231 : if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
4523 296667 : map = ipa_get_jf_pass_through_formal_id (jfunc);
4524 300231 : if (!ipa_get_jf_pass_through_agg_preserved (jfunc))
4525 201850 : offset_map[i] = -1;
4526 : }
4527 1562407 : else if (jfunc->type == IPA_JF_ANCESTOR)
4528 : {
4529 95503 : HOST_WIDE_INT offset = ipa_get_jf_ancestor_offset (jfunc);
4530 95503 : if (offset >= 0 && offset < INT_MAX)
4531 : {
4532 95503 : map = ipa_get_jf_ancestor_formal_id (jfunc);
4533 95503 : if (!ipa_get_jf_ancestor_agg_preserved (jfunc))
4534 56415 : offset = -1;
4535 95503 : offset_map[i] = offset;
4536 : }
4537 : }
4538 1862638 : operand_map[i] = map;
4539 3374299 : gcc_assert (map < ipa_get_param_count (params_summary));
4540 : }
4541 :
4542 : int ip;
4543 848544 : for (i = 0; callee_info->builtin_constant_p_parms.iterate (i, &ip); i++)
4544 2857 : if (ip < count && operand_map[ip] >= 0)
4545 48 : add_builtin_constant_p_parm (info, operand_map[ip]);
4546 : }
4547 4199852 : sreal freq = edge->sreal_frequency ();
4548 27398536 : for (i = 0; callee_info->size_time_table.iterate (i, &e); i++)
4549 : {
4550 18998832 : ipa_predicate p;
4551 18998832 : p = e->exec_predicate.remap_after_inlining
4552 18998832 : (info, params_summary,
4553 : callee_info, operand_map,
4554 : offset_map, clause,
4555 : toplev_predicate);
4556 18998832 : ipa_predicate nonconstp;
4557 18998832 : nonconstp = e->nonconst_predicate.remap_after_inlining
4558 18998832 : (info, params_summary,
4559 : callee_info, operand_map,
4560 : offset_map, clause,
4561 : toplev_predicate);
4562 28735996 : if (p != false && nonconstp != false)
4563 : {
4564 9402369 : sreal add_time = ((sreal)e->time * freq);
4565 9402369 : int prob = e->nonconst_predicate.probability (callee_info->conds,
4566 : clause, es->param);
4567 9402369 : if (prob != REG_BR_PROB_BASE)
4568 1139825 : add_time = add_time * prob / REG_BR_PROB_BASE;
4569 1139825 : if (prob != REG_BR_PROB_BASE
4570 1139825 : && dump_file && (dump_flags & TDF_DETAILS))
4571 : {
4572 3 : fprintf (dump_file, "\t\tScaling time by probability:%f\n",
4573 3 : (double) prob / REG_BR_PROB_BASE);
4574 : }
4575 9402369 : info->account_size_time (e->size, add_time, p, nonconstp);
4576 : }
4577 : }
4578 4199852 : remap_edge_summaries (edge, edge->callee, info, params_summary,
4579 : callee_info, operand_map,
4580 : offset_map, clause, &toplev_predicate);
4581 4199852 : remap_freqcounting_predicate (info, params_summary, callee_info,
4582 : info->loop_iterations, operand_map,
4583 : offset_map, clause, &toplev_predicate);
4584 4199852 : remap_freqcounting_predicate (info, params_summary, callee_info,
4585 : info->loop_strides, operand_map,
4586 : offset_map, clause, &toplev_predicate);
4587 :
4588 4199852 : HOST_WIDE_INT stack_frame_offset = ipa_get_stack_frame_offset (edge->callee);
4589 4199852 : HOST_WIDE_INT peak = stack_frame_offset + callee_info->estimated_stack_size;
4590 :
4591 4199852 : if (info->estimated_stack_size < peak)
4592 123961 : info->estimated_stack_size = peak;
4593 :
4594 4199852 : inline_update_callee_summaries (edge->callee, es->loop_depth);
4595 4199852 : if (info->call_size_time_table.length ())
4596 : {
4597 906930 : int edge_size = 0;
4598 906930 : sreal edge_time = 0;
4599 :
4600 906930 : estimate_edge_size_and_time (edge, &edge_size, NULL, &edge_time, NULL, 0);
4601 : /* Unaccount size and time of the optimized out call. */
4602 906930 : info->account_size_time (-edge_size, -edge_time,
4603 906930 : es->predicate ? *es->predicate : true,
4604 906930 : es->predicate ? *es->predicate : true,
4605 : true);
4606 : /* Account new calls. */
4607 906930 : summarize_calls_size_and_time (edge->callee, info);
4608 : }
4609 :
4610 : /* Free summaries that are not maintained for inline clones/edges. */
4611 4199852 : ipa_call_summaries->remove (edge);
4612 4199852 : ipa_fn_summaries->remove (edge->callee);
4613 4199852 : ipa_remove_from_growth_caches (edge);
4614 4199852 : }
4615 :
4616 : /* For performance reasons ipa_merge_fn_summary_after_inlining is not updating
4617 : overall size and time. Recompute it.
4618 : If RESET is true also recompute call_time_size_table. */
4619 :
4620 : void
4621 9833720 : ipa_update_overall_fn_summary (struct cgraph_node *node, bool reset)
4622 : {
4623 9833720 : class ipa_fn_summary *info = ipa_fn_summaries->get (node);
4624 9833720 : class ipa_size_summary *size_info = ipa_size_summaries->get (node);
4625 9833720 : size_time_entry *e;
4626 9833720 : int i;
4627 :
4628 9833720 : size_info->size = 0;
4629 9833720 : info->time = 0;
4630 49861428 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
4631 : {
4632 40027708 : size_info->size += e->size;
4633 40027708 : info->time += e->time;
4634 : }
4635 9833720 : info->min_size = info->size_time_table[0].size;
4636 9833720 : if (reset)
4637 8883238 : info->call_size_time_table.release ();
4638 9833720 : if (node->callees || node->indirect_calls)
4639 7489435 : estimate_calls_size_and_time (node, &size_info->size, &info->min_size,
4640 : &info->time, NULL,
4641 : ~(clause_t) (1 << ipa_predicate::false_condition),
4642 : NULL);
4643 9833720 : size_info->size = RDIV (size_info->size, ipa_fn_summary::size_scale);
4644 9833720 : info->min_size = RDIV (info->min_size, ipa_fn_summary::size_scale);
4645 9833720 : }
4646 :
4647 :
4648 : /* This function performs intraprocedural analysis in NODE that is required to
4649 : inline indirect calls. */
4650 :
4651 : static void
4652 1415847 : inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
4653 : {
4654 1415847 : ipa_analyze_node (node);
4655 1415847 : if (dump_file && (dump_flags & TDF_DETAILS))
4656 : {
4657 16 : ipa_print_node_params (dump_file, node);
4658 16 : ipa_print_node_jump_functions (dump_file, node);
4659 : }
4660 1415847 : }
4661 :
4662 :
4663 : /* Note function body size. */
4664 :
4665 : void
4666 1426505 : inline_analyze_function (struct cgraph_node *node)
4667 : {
4668 1426505 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
4669 :
4670 1426505 : if (dump_file)
4671 96 : fprintf (dump_file, "\nAnalyzing function: %s\n", node->dump_name ());
4672 1426505 : if (opt_for_fn (node->decl, optimize) && !node->thunk)
4673 1415847 : inline_indirect_intraprocedural_analysis (node);
4674 1426505 : compute_fn_summary (node, false);
4675 1426505 : if (!optimize)
4676 : {
4677 9366 : struct cgraph_edge *e;
4678 22117 : for (e = node->callees; e; e = e->next_callee)
4679 12751 : e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4680 9440 : for (e = node->indirect_calls; e; e = e->next_callee)
4681 74 : e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4682 : }
4683 :
4684 1426505 : pop_cfun ();
4685 1426505 : }
4686 :
4687 :
4688 : /* Called when new function is inserted to callgraph late. */
4689 :
4690 : void
4691 19605 : ipa_fn_summary_t::insert (struct cgraph_node *node, ipa_fn_summary *)
4692 : {
4693 19605 : inline_analyze_function (node);
4694 19605 : }
4695 :
4696 : /* Note function body size. */
4697 :
4698 : static void
4699 236913 : ipa_fn_summary_generate (void)
4700 : {
4701 236913 : struct cgraph_node *node;
4702 :
4703 2161454 : FOR_EACH_DEFINED_FUNCTION (node)
4704 1924541 : if (DECL_STRUCT_FUNCTION (node->decl))
4705 1910890 : node->versionable = tree_versionable_function_p (node->decl);
4706 :
4707 236913 : ipa_fn_summary_alloc ();
4708 :
4709 236913 : ipa_fn_summaries->enable_insertion_hook ();
4710 :
4711 236913 : ipa_register_cgraph_hooks ();
4712 :
4713 2161454 : FOR_EACH_DEFINED_FUNCTION (node)
4714 1924541 : if (!node->alias
4715 1924541 : && (flag_generate_lto || flag_generate_offload|| flag_wpa
4716 1729145 : || opt_for_fn (node->decl, optimize)))
4717 1388447 : inline_analyze_function (node);
4718 236913 : }
4719 :
4720 :
4721 : /* Write inline summary for edge E to OB. */
4722 :
4723 : static void
4724 347752 : read_ipa_call_summary (class lto_input_block *ib, struct cgraph_edge *e,
4725 : bool prevails)
4726 : {
4727 347752 : class ipa_call_summary *es = prevails
4728 347752 : ? ipa_call_summaries->get_create (e) : NULL;
4729 347752 : ipa_predicate p;
4730 347752 : int length, i;
4731 :
4732 347752 : int size = streamer_read_uhwi (ib);
4733 347752 : int time = streamer_read_uhwi (ib);
4734 347752 : int depth = streamer_read_uhwi (ib);
4735 :
4736 347752 : if (es)
4737 : {
4738 347703 : es->call_stmt_size = size;
4739 347703 : es->call_stmt_time = time;
4740 347703 : es->loop_depth = depth;
4741 : }
4742 :
4743 347752 : bitpack_d bp = streamer_read_bitpack (ib);
4744 347752 : if (es)
4745 347703 : es->is_return_callee_uncaptured = bp_unpack_value (&bp, 1);
4746 : else
4747 49 : bp_unpack_value (&bp, 1);
4748 :
4749 347752 : p.stream_in (ib);
4750 347752 : if (es)
4751 347703 : edge_set_predicate (e, &p);
4752 347752 : length = streamer_read_uhwi (ib);
4753 347752 : if (length && es
4754 347752 : && (e->possibly_call_in_translation_unit_p ()
4755 : /* Also stream in jump functions to builtins in hope that they
4756 : will get fnspecs. */
4757 118930 : || fndecl_built_in_p (e->callee->decl, BUILT_IN_NORMAL)))
4758 : {
4759 232702 : es->param.safe_grow_cleared (length, true);
4760 1033320 : for (i = 0; i < length; i++)
4761 : {
4762 567916 : es->param[i].change_prob = streamer_read_uhwi (ib);
4763 567916 : bitpack_d bp = streamer_read_bitpack (ib);
4764 1703748 : es->param[i].points_to_local_or_readonly_memory
4765 567916 : = bp_unpack_value (&bp, 1);
4766 1703748 : es->param[i].points_to_possible_sra_candidate
4767 567916 : = bp_unpack_value (&bp, 1);
4768 : }
4769 : }
4770 : else
4771 : {
4772 137940 : for (i = 0; i < length; i++)
4773 : {
4774 22890 : streamer_read_uhwi (ib);
4775 22890 : streamer_read_uhwi (ib);
4776 : }
4777 : }
4778 347752 : }
4779 :
4780 :
4781 : /* Stream in inline summaries from the section. */
4782 :
4783 : static void
4784 13466 : inline_read_section (struct lto_file_decl_data *file_data, const char *data,
4785 : size_t len)
4786 : {
4787 13466 : const struct lto_function_header *header =
4788 : (const struct lto_function_header *) data;
4789 13466 : const int cfg_offset = sizeof (struct lto_function_header);
4790 13466 : const int main_offset = cfg_offset + header->cfg_size;
4791 13466 : const int string_offset = main_offset + header->main_size;
4792 13466 : class data_in *data_in;
4793 13466 : unsigned int i, count2, j;
4794 13466 : unsigned int f_count;
4795 :
4796 13466 : lto_input_block ib ((const char *) data + main_offset, header->main_size,
4797 13466 : file_data);
4798 :
4799 13466 : data_in =
4800 26932 : lto_data_in_create (file_data, (const char *) data + string_offset,
4801 13466 : header->string_size, vNULL);
4802 13466 : f_count = streamer_read_uhwi (&ib);
4803 100095 : for (i = 0; i < f_count; i++)
4804 : {
4805 86629 : unsigned int index;
4806 86629 : struct cgraph_node *node;
4807 86629 : class ipa_fn_summary *info;
4808 86629 : class ipa_node_params *params_summary;
4809 86629 : class ipa_size_summary *size_info;
4810 86629 : lto_symtab_encoder_t encoder;
4811 86629 : struct bitpack_d bp;
4812 86629 : struct cgraph_edge *e;
4813 86629 : ipa_predicate p;
4814 :
4815 86629 : index = streamer_read_uhwi (&ib);
4816 86629 : encoder = file_data->symtab_node_encoder;
4817 86629 : node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
4818 : index));
4819 86629 : info = node->prevailing_p () ? ipa_fn_summaries->get_create (node) : NULL;
4820 86629 : params_summary = node->prevailing_p ()
4821 86629 : ? ipa_node_params_sum->get (node) : NULL;
4822 86629 : size_info = node->prevailing_p ()
4823 86629 : ? ipa_size_summaries->get_create (node) : NULL;
4824 :
4825 86629 : int stack_size = streamer_read_uhwi (&ib);
4826 86629 : int size = streamer_read_uhwi (&ib);
4827 86629 : sreal time = sreal::stream_in (&ib);
4828 :
4829 86629 : if (info)
4830 : {
4831 86571 : info->estimated_stack_size
4832 86571 : = size_info->estimated_self_stack_size = stack_size;
4833 86571 : size_info->size = size_info->self_size = size;
4834 86571 : info->time = time;
4835 : }
4836 :
4837 86629 : bp = streamer_read_bitpack (&ib);
4838 86629 : if (info)
4839 : {
4840 86571 : info->inlinable = bp_unpack_value (&bp, 1);
4841 86571 : info->fp_expressions = bp_unpack_value (&bp, 1);
4842 86571 : if (!lto_stream_offload_p)
4843 86571 : info->target_info = streamer_read_uhwi (&ib);
4844 : }
4845 : else
4846 : {
4847 58 : bp_unpack_value (&bp, 1);
4848 58 : bp_unpack_value (&bp, 1);
4849 58 : if (!lto_stream_offload_p)
4850 58 : streamer_read_uhwi (&ib);
4851 : }
4852 :
4853 86629 : count2 = streamer_read_uhwi (&ib);
4854 86629 : gcc_assert (!info || !info->conds);
4855 86571 : if (info)
4856 86571 : vec_safe_reserve_exact (info->conds, count2);
4857 164001 : for (j = 0; j < count2; j++)
4858 : {
4859 77372 : struct condition c;
4860 77372 : unsigned int k, count3;
4861 77372 : c.operand_num = streamer_read_uhwi (&ib);
4862 77372 : c.code = (enum tree_code) streamer_read_uhwi (&ib);
4863 77372 : c.type = stream_read_tree (&ib, data_in);
4864 77372 : c.val = stream_read_tree (&ib, data_in);
4865 77372 : bp = streamer_read_bitpack (&ib);
4866 77372 : c.agg_contents = bp_unpack_value (&bp, 1);
4867 77372 : c.by_ref = bp_unpack_value (&bp, 1);
4868 77372 : if (c.agg_contents)
4869 12426 : c.offset = streamer_read_uhwi (&ib);
4870 77372 : count3 = streamer_read_uhwi (&ib);
4871 77372 : c.param_ops = NULL;
4872 77372 : if (info)
4873 77372 : vec_safe_reserve_exact (c.param_ops, count3);
4874 77372 : if (params_summary)
4875 77372 : ipa_set_param_used_by_ipa_predicates
4876 77372 : (params_summary, c.operand_num, true);
4877 81031 : for (k = 0; k < count3; k++)
4878 : {
4879 3659 : struct expr_eval_op op;
4880 3659 : enum gimple_rhs_class rhs_class;
4881 3659 : op.code = (enum tree_code) streamer_read_uhwi (&ib);
4882 3659 : op.type = stream_read_tree (&ib, data_in);
4883 3659 : switch (rhs_class = get_gimple_rhs_class (op.code))
4884 : {
4885 1534 : case GIMPLE_UNARY_RHS:
4886 1534 : op.index = 0;
4887 1534 : op.val[0] = NULL_TREE;
4888 1534 : op.val[1] = NULL_TREE;
4889 1534 : break;
4890 :
4891 2125 : case GIMPLE_BINARY_RHS:
4892 2125 : case GIMPLE_TERNARY_RHS:
4893 2125 : bp = streamer_read_bitpack (&ib);
4894 2125 : op.index = bp_unpack_value (&bp, 2);
4895 2125 : op.val[0] = stream_read_tree (&ib, data_in);
4896 2125 : if (rhs_class == GIMPLE_BINARY_RHS)
4897 2125 : op.val[1] = NULL_TREE;
4898 : else
4899 0 : op.val[1] = stream_read_tree (&ib, data_in);
4900 : break;
4901 :
4902 0 : default:
4903 0 : fatal_error (UNKNOWN_LOCATION,
4904 : "invalid fnsummary in LTO stream");
4905 : }
4906 3659 : if (info)
4907 3659 : c.param_ops->quick_push (op);
4908 : }
4909 77372 : if (info)
4910 77372 : info->conds->quick_push (c);
4911 : }
4912 86629 : count2 = streamer_read_uhwi (&ib);
4913 86629 : gcc_assert (!info || !info->size_time_table.length ());
4914 86629 : if (info && count2)
4915 86571 : info->size_time_table.reserve_exact (count2);
4916 328970 : for (j = 0; j < count2; j++)
4917 : {
4918 242341 : class size_time_entry e;
4919 :
4920 242341 : e.size = streamer_read_uhwi (&ib);
4921 242341 : e.time = sreal::stream_in (&ib);
4922 242341 : e.exec_predicate.stream_in (&ib);
4923 242341 : e.nonconst_predicate.stream_in (&ib);
4924 :
4925 242341 : if (info)
4926 242225 : info->size_time_table.quick_push (e);
4927 : }
4928 :
4929 86629 : count2 = streamer_read_uhwi (&ib);
4930 87989 : for (j = 0; j < count2; j++)
4931 : {
4932 1360 : p.stream_in (&ib);
4933 1360 : sreal fcp_freq = sreal::stream_in (&ib);
4934 1360 : if (info)
4935 : {
4936 1360 : ipa_freqcounting_predicate fcp;
4937 1360 : fcp.predicate = NULL;
4938 1360 : set_hint_predicate (&fcp.predicate, p);
4939 1360 : fcp.freq = fcp_freq;
4940 1360 : vec_safe_push (info->loop_iterations, fcp);
4941 : }
4942 : }
4943 86629 : count2 = streamer_read_uhwi (&ib);
4944 86918 : for (j = 0; j < count2; j++)
4945 : {
4946 289 : p.stream_in (&ib);
4947 289 : sreal fcp_freq = sreal::stream_in (&ib);
4948 289 : if (info)
4949 : {
4950 289 : ipa_freqcounting_predicate fcp;
4951 289 : fcp.predicate = NULL;
4952 289 : set_hint_predicate (&fcp.predicate, p);
4953 289 : fcp.freq = fcp_freq;
4954 289 : vec_safe_push (info->loop_strides, fcp);
4955 : }
4956 : }
4957 86629 : count2 = streamer_read_uhwi (&ib);
4958 86629 : if (info && count2)
4959 6 : info->builtin_constant_p_parms.reserve_exact (count2);
4960 86635 : for (j = 0; j < count2; j++)
4961 : {
4962 6 : int parm = streamer_read_uhwi (&ib);
4963 6 : if (info)
4964 6 : info->builtin_constant_p_parms.quick_push (parm);
4965 : }
4966 432888 : for (e = node->callees; e; e = e->next_callee)
4967 346259 : read_ipa_call_summary (&ib, e, info != NULL);
4968 88122 : for (e = node->indirect_calls; e; e = e->next_callee)
4969 1493 : read_ipa_call_summary (&ib, e, info != NULL);
4970 : }
4971 :
4972 13466 : lto_free_section_data (file_data, LTO_section_ipa_fn_summary, NULL, data,
4973 : len);
4974 13466 : lto_data_in_delete (data_in);
4975 13466 : }
4976 :
4977 :
4978 : /* Read inline summary. Jump functions are shared among ipa-cp
4979 : and inliner, so when ipa-cp is active, we don't need to write them
4980 : twice. */
4981 :
4982 : static void
4983 12380 : ipa_fn_summary_read (void)
4984 : {
4985 12380 : struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
4986 12380 : struct lto_file_decl_data *file_data;
4987 12380 : unsigned int j = 0;
4988 :
4989 12380 : ipa_prop_read_jump_functions ();
4990 12380 : ipa_fn_summary_alloc ();
4991 :
4992 38226 : while ((file_data = file_data_vec[j++]))
4993 : {
4994 13466 : size_t len;
4995 13466 : const char *data
4996 13466 : = lto_get_summary_section_data (file_data, LTO_section_ipa_fn_summary,
4997 : &len);
4998 13466 : if (data)
4999 13466 : inline_read_section (file_data, data, len);
5000 : else
5001 : /* Fatal error here. We do not want to support compiling ltrans units
5002 : with different version of compiler or different flags than the WPA
5003 : unit, so this should never happen. */
5004 0 : fatal_error (input_location,
5005 : "ipa inline summary is missing in input file");
5006 : }
5007 12380 : ipa_register_cgraph_hooks ();
5008 :
5009 12380 : gcc_assert (ipa_fn_summaries);
5010 12380 : ipa_fn_summaries->enable_insertion_hook ();
5011 12380 : }
5012 :
5013 :
5014 : /* Write inline summary for edge E to OB. */
5015 :
5016 : static void
5017 379437 : write_ipa_call_summary (struct output_block *ob, struct cgraph_edge *e)
5018 : {
5019 379437 : class ipa_call_summary *es = ipa_call_summaries->get (e);
5020 379437 : int i;
5021 :
5022 379437 : streamer_write_uhwi (ob, es->call_stmt_size);
5023 379437 : streamer_write_uhwi (ob, es->call_stmt_time);
5024 379437 : streamer_write_uhwi (ob, es->loop_depth);
5025 :
5026 379437 : bitpack_d bp = bitpack_create (ob->main_stream);
5027 379437 : bp_pack_value (&bp, es->is_return_callee_uncaptured, 1);
5028 379437 : streamer_write_bitpack (&bp);
5029 :
5030 379437 : if (es->predicate)
5031 10633 : es->predicate->stream_out (ob);
5032 : else
5033 368804 : streamer_write_uhwi (ob, 0);
5034 379437 : streamer_write_uhwi (ob, es->param.length ());
5035 2290297 : for (i = 0; i < (int) es->param.length (); i++)
5036 : {
5037 631485 : streamer_write_uhwi (ob, es->param[i].change_prob);
5038 631485 : bp = bitpack_create (ob->main_stream);
5039 631485 : bp_pack_value (&bp, es->param[i].points_to_local_or_readonly_memory, 1);
5040 631485 : bp_pack_value (&bp, es->param[i].points_to_possible_sra_candidate, 1);
5041 631485 : streamer_write_bitpack (&bp);
5042 : }
5043 379437 : }
5044 :
5045 :
5046 : /* Write inline summary for node in SET.
5047 : Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
5048 : active, we don't need to write them twice. */
5049 :
5050 : static void
5051 23420 : ipa_fn_summary_write (void)
5052 : {
5053 23420 : struct output_block *ob = create_output_block (LTO_section_ipa_fn_summary);
5054 23420 : lto_symtab_encoder_iterator lsei;
5055 23420 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
5056 23420 : unsigned int count = 0;
5057 :
5058 129081 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
5059 105661 : lsei_next_function_in_partition (&lsei))
5060 : {
5061 105661 : cgraph_node *cnode = lsei_cgraph_node (lsei);
5062 105661 : if (cnode->definition && !cnode->alias)
5063 103213 : count++;
5064 : }
5065 23420 : streamer_write_uhwi (ob, count);
5066 :
5067 129081 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
5068 105661 : lsei_next_function_in_partition (&lsei))
5069 : {
5070 105661 : cgraph_node *cnode = lsei_cgraph_node (lsei);
5071 105661 : if (cnode->definition && !cnode->alias)
5072 : {
5073 103213 : class ipa_fn_summary *info = ipa_fn_summaries->get (cnode);
5074 103213 : class ipa_size_summary *size_info = ipa_size_summaries->get (cnode);
5075 103213 : struct bitpack_d bp;
5076 103213 : struct cgraph_edge *edge;
5077 103213 : int i;
5078 103213 : size_time_entry *e;
5079 103213 : struct condition *c;
5080 :
5081 103213 : streamer_write_uhwi (ob, lto_symtab_encoder_encode (encoder, cnode));
5082 103213 : streamer_write_hwi (ob, size_info->estimated_self_stack_size);
5083 103213 : streamer_write_hwi (ob, size_info->self_size);
5084 103213 : info->time.stream_out (ob);
5085 103213 : bp = bitpack_create (ob->main_stream);
5086 103213 : bp_pack_value (&bp, info->inlinable, 1);
5087 103213 : bp_pack_value (&bp, info->fp_expressions, 1);
5088 103213 : streamer_write_bitpack (&bp);
5089 103213 : if (!lto_stream_offload_p)
5090 103213 : streamer_write_uhwi (ob, info->target_info);
5091 103213 : streamer_write_uhwi (ob, vec_safe_length (info->conds));
5092 299809 : for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
5093 : {
5094 93383 : int j;
5095 93383 : struct expr_eval_op *op;
5096 :
5097 93383 : streamer_write_uhwi (ob, c->operand_num);
5098 93383 : streamer_write_uhwi (ob, c->code);
5099 93383 : stream_write_tree (ob, c->type, true);
5100 93383 : stream_write_tree (ob, c->val, true);
5101 93383 : bp = bitpack_create (ob->main_stream);
5102 93383 : bp_pack_value (&bp, c->agg_contents, 1);
5103 93383 : bp_pack_value (&bp, c->by_ref, 1);
5104 93383 : streamer_write_bitpack (&bp);
5105 93383 : if (c->agg_contents)
5106 15680 : streamer_write_uhwi (ob, c->offset);
5107 93383 : streamer_write_uhwi (ob, vec_safe_length (c->param_ops));
5108 194092 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
5109 : {
5110 4329 : streamer_write_uhwi (ob, op->code);
5111 4329 : stream_write_tree (ob, op->type, true);
5112 4329 : if (op->val[0])
5113 : {
5114 2521 : bp = bitpack_create (ob->main_stream);
5115 2521 : bp_pack_value (&bp, op->index, 2);
5116 2521 : streamer_write_bitpack (&bp);
5117 2521 : stream_write_tree (ob, op->val[0], true);
5118 2521 : if (op->val[1])
5119 4 : stream_write_tree (ob, op->val[1], true);
5120 : }
5121 : }
5122 : }
5123 103213 : streamer_write_uhwi (ob, info->size_time_table.length ());
5124 498882 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
5125 : {
5126 292456 : streamer_write_uhwi (ob, e->size);
5127 292456 : e->time.stream_out (ob);
5128 292456 : e->exec_predicate.stream_out (ob);
5129 292456 : e->nonconst_predicate.stream_out (ob);
5130 : }
5131 103213 : ipa_freqcounting_predicate *fcp;
5132 103213 : streamer_write_uhwi (ob, vec_safe_length (info->loop_iterations));
5133 208432 : for (i = 0; vec_safe_iterate (info->loop_iterations, i, &fcp); i++)
5134 : {
5135 2006 : fcp->predicate->stream_out (ob);
5136 2006 : fcp->freq.stream_out (ob);
5137 : }
5138 103213 : streamer_write_uhwi (ob, vec_safe_length (info->loop_strides));
5139 206771 : for (i = 0; vec_safe_iterate (info->loop_strides, i, &fcp); i++)
5140 : {
5141 345 : fcp->predicate->stream_out (ob);
5142 345 : fcp->freq.stream_out (ob);
5143 : }
5144 103213 : streamer_write_uhwi (ob, info->builtin_constant_p_parms.length ());
5145 103213 : int ip;
5146 206440 : for (i = 0; info->builtin_constant_p_parms.iterate (i, &ip);
5147 : i++)
5148 14 : streamer_write_uhwi (ob, ip);
5149 479949 : for (edge = cnode->callees; edge; edge = edge->next_callee)
5150 376736 : write_ipa_call_summary (ob, edge);
5151 105914 : for (edge = cnode->indirect_calls; edge; edge = edge->next_callee)
5152 2701 : write_ipa_call_summary (ob, edge);
5153 : }
5154 : }
5155 23420 : streamer_write_char_stream (ob->main_stream, 0);
5156 23420 : produce_asm (ob);
5157 23420 : destroy_output_block (ob);
5158 :
5159 23420 : ipa_prop_write_jump_functions ();
5160 23420 : }
5161 :
5162 :
5163 : /* Release function summary. */
5164 :
5165 : void
5166 739481 : ipa_free_fn_summary (void)
5167 : {
5168 739481 : if (!ipa_call_summaries)
5169 : return;
5170 468076 : ggc_delete (ipa_fn_summaries);
5171 468076 : ipa_fn_summaries = NULL;
5172 468076 : delete ipa_call_summaries;
5173 468076 : ipa_call_summaries = NULL;
5174 468076 : edge_predicate_pool.release ();
5175 : /* During IPA this is one of largest datastructures to release. */
5176 468076 : if (flag_wpa)
5177 7889 : ggc_trim ();
5178 : }
5179 :
5180 : /* Release function summary. */
5181 :
5182 : void
5183 739481 : ipa_free_size_summary (void)
5184 : {
5185 739481 : if (!ipa_size_summaries)
5186 : return;
5187 468076 : delete ipa_size_summaries;
5188 468076 : ipa_size_summaries = NULL;
5189 : }
5190 :
5191 : namespace {
5192 :
5193 : const pass_data pass_data_local_fn_summary =
5194 : {
5195 : GIMPLE_PASS, /* type */
5196 : "local-fnsummary", /* name */
5197 : OPTGROUP_INLINE, /* optinfo_flags */
5198 : TV_INLINE_PARAMETERS, /* tv_id */
5199 : 0, /* properties_required */
5200 : 0, /* properties_provided */
5201 : 0, /* properties_destroyed */
5202 : 0, /* todo_flags_start */
5203 : 0, /* todo_flags_finish */
5204 : };
5205 :
5206 : class pass_local_fn_summary : public gimple_opt_pass
5207 : {
5208 : public:
5209 588392 : pass_local_fn_summary (gcc::context *ctxt)
5210 1176784 : : gimple_opt_pass (pass_data_local_fn_summary, ctxt)
5211 : {}
5212 :
5213 : /* opt_pass methods: */
5214 294196 : opt_pass * clone () final override
5215 : {
5216 294196 : return new pass_local_fn_summary (m_ctxt);
5217 : }
5218 6007418 : unsigned int execute (function *) final override
5219 : {
5220 6007418 : return compute_fn_summary_for_current ();
5221 : }
5222 :
5223 : }; // class pass_local_fn_summary
5224 :
5225 : } // anon namespace
5226 :
5227 : gimple_opt_pass *
5228 294196 : make_pass_local_fn_summary (gcc::context *ctxt)
5229 : {
5230 294196 : return new pass_local_fn_summary (ctxt);
5231 : }
5232 :
5233 :
5234 : /* Free inline summary. */
5235 :
5236 : namespace {
5237 :
5238 : const pass_data pass_data_ipa_free_fn_summary =
5239 : {
5240 : SIMPLE_IPA_PASS, /* type */
5241 : "free-fnsummary", /* name */
5242 : OPTGROUP_NONE, /* optinfo_flags */
5243 : TV_IPA_FREE_INLINE_SUMMARY, /* tv_id */
5244 : 0, /* properties_required */
5245 : 0, /* properties_provided */
5246 : 0, /* properties_destroyed */
5247 : 0, /* todo_flags_start */
5248 : 0, /* todo_flags_finish */
5249 : };
5250 :
5251 : class pass_ipa_free_fn_summary : public simple_ipa_opt_pass
5252 : {
5253 : public:
5254 588392 : pass_ipa_free_fn_summary (gcc::context *ctxt)
5255 : : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary, ctxt),
5256 1176784 : small_p (false)
5257 : {}
5258 :
5259 : /* opt_pass methods: */
5260 294196 : opt_pass *clone () final override
5261 : {
5262 294196 : return new pass_ipa_free_fn_summary (m_ctxt);
5263 : }
5264 588392 : void set_pass_param (unsigned int n, bool param) final override
5265 : {
5266 588392 : gcc_assert (n == 0);
5267 588392 : small_p = param;
5268 588392 : }
5269 494930 : bool gate (function *) final override { return true; }
5270 474266 : unsigned int execute (function *) final override
5271 : {
5272 474266 : ipa_free_fn_summary ();
5273 : /* Free ipa-prop structures if they are no longer needed. */
5274 474266 : ipa_free_all_structures_after_iinln ();
5275 474266 : if (!flag_wpa)
5276 466377 : ipa_free_size_summary ();
5277 474266 : return 0;
5278 : }
5279 :
5280 : private:
5281 : bool small_p;
5282 : }; // class pass_ipa_free_fn_summary
5283 :
5284 : } // anon namespace
5285 :
5286 : simple_ipa_opt_pass *
5287 294196 : make_pass_ipa_free_fn_summary (gcc::context *ctxt)
5288 : {
5289 294196 : return new pass_ipa_free_fn_summary (ctxt);
5290 : }
5291 :
5292 : namespace {
5293 :
5294 : const pass_data pass_data_ipa_fn_summary =
5295 : {
5296 : IPA_PASS, /* type */
5297 : "fnsummary", /* name */
5298 : OPTGROUP_INLINE, /* optinfo_flags */
5299 : TV_IPA_FNSUMMARY, /* tv_id */
5300 : 0, /* properties_required */
5301 : 0, /* properties_provided */
5302 : 0, /* properties_destroyed */
5303 : 0, /* todo_flags_start */
5304 : ( TODO_dump_symtab ), /* todo_flags_finish */
5305 : };
5306 :
5307 : class pass_ipa_fn_summary : public ipa_opt_pass_d
5308 : {
5309 : public:
5310 294196 : pass_ipa_fn_summary (gcc::context *ctxt)
5311 : : ipa_opt_pass_d (pass_data_ipa_fn_summary, ctxt,
5312 : ipa_fn_summary_generate, /* generate_summary */
5313 : ipa_fn_summary_write, /* write_summary */
5314 : ipa_fn_summary_read, /* read_summary */
5315 : NULL, /* write_optimization_summary */
5316 : NULL, /* read_optimization_summary */
5317 : NULL, /* stmt_fixup */
5318 : 0, /* function_transform_todo_flags_start */
5319 : NULL, /* function_transform */
5320 294196 : NULL) /* variable_transform */
5321 294196 : {}
5322 :
5323 : /* opt_pass methods: */
5324 237204 : unsigned int execute (function *) final override { return 0; }
5325 :
5326 : }; // class pass_ipa_fn_summary
5327 :
5328 : } // anon namespace
5329 :
5330 : ipa_opt_pass_d *
5331 294196 : make_pass_ipa_fn_summary (gcc::context *ctxt)
5332 : {
5333 294196 : return new pass_ipa_fn_summary (ctxt);
5334 : }
5335 :
5336 : /* Reset all state within ipa-fnsummary.cc so that we can rerun the compiler
5337 : within the same process. For use by toplev::finalize. */
5338 :
5339 : void
5340 264319 : ipa_fnsummary_cc_finalize (void)
5341 : {
5342 264319 : ipa_free_fn_summary ();
5343 264319 : ipa_free_size_summary ();
5344 264319 : }
|
