Line data Source code
1 : /* Function summary pass.
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* Analysis of function bodies used by inter-procedural passes
22 :
23 : We estimate for each function
24 : - function body size and size after specializing into given context
25 : - average function execution time in a given context
26 : - function frame size
27 : For each call
28 : - call statement size, time and how often the parameters change
29 :
30 : ipa_fn_summary data structures store above information locally (i.e.
31 : parameters of the function itself) and globally (i.e. parameters of
32 : the function created by applying all the inline decisions already
33 : present in the callgraph).
34 :
35 : We provide access to the ipa_fn_summary data structure and
36 : basic logic updating the parameters when inlining is performed.
37 :
38 : The summaries are context sensitive. Context means
39 : 1) partial assignment of known constant values of operands
40 : 2) whether function is inlined into the call or not.
41 : It is easy to add more variants. To represent function size and time
42 : that depends on context (i.e. it is known to be optimized away when
43 : context is known either by inlining or from IP-CP and cloning),
44 : we use predicates.
45 :
46 : estimate_edge_size_and_time can be used to query
47 : function size/time in the given context. ipa_merge_fn_summary_after_inlining merges
48 : properties of caller and callee after inlining.
49 :
50 : Finally pass_inline_parameters is exported. This is used to drive
51 : computation of function parameters used by the early inliner. IPA
52 : inlined performs analysis via its analyze_function method. */
53 :
54 : #include "config.h"
55 : #define INCLUDE_VECTOR
56 : #include "system.h"
57 : #include "coretypes.h"
58 : #include "backend.h"
59 : #include "target.h"
60 : #include "tree.h"
61 : #include "gimple.h"
62 : #include "alloc-pool.h"
63 : #include "tree-pass.h"
64 : #include "ssa.h"
65 : #include "tree-streamer.h"
66 : #include "cgraph.h"
67 : #include "diagnostic.h"
68 : #include "fold-const.h"
69 : #include "print-tree.h"
70 : #include "tree-inline.h"
71 : #include "gimple-pretty-print.h"
72 : #include "cfganal.h"
73 : #include "gimple-iterator.h"
74 : #include "tree-cfg.h"
75 : #include "tree-ssa-loop-niter.h"
76 : #include "tree-ssa-loop.h"
77 : #include "symbol-summary.h"
78 : #include "sreal.h"
79 : #include "ipa-cp.h"
80 : #include "ipa-prop.h"
81 : #include "ipa-fnsummary.h"
82 : #include "cfgloop.h"
83 : #include "tree-scalar-evolution.h"
84 : #include "ipa-utils.h"
85 : #include "cfgexpand.h"
86 : #include "gimplify.h"
87 : #include "stringpool.h"
88 : #include "attribs.h"
89 : #include "tree-into-ssa.h"
90 : #include "symtab-clones.h"
91 : #include "gimple-range.h"
92 : #include "tree-dfa.h"
93 : #include "attr-callback.h"
94 : #include "callback-info.h"
95 :
96 : /* Summaries. */
97 : fast_function_summary <ipa_fn_summary *, va_gc> *ipa_fn_summaries;
98 : fast_function_summary <ipa_size_summary *, va_heap> *ipa_size_summaries;
99 : fast_call_summary <ipa_call_summary *, va_heap> *ipa_call_summaries;
100 :
101 : /* Edge predicates goes here. */
102 : static object_allocator<ipa_predicate> edge_predicate_pool ("edge predicates");
103 :
104 :
105 : /* Dump IPA hints. */
106 : void
107 185 : ipa_dump_hints (FILE *f, ipa_hints hints)
108 : {
109 185 : if (!hints)
110 : return;
111 148 : fprintf (f, "IPA hints:");
112 148 : if (hints & INLINE_HINT_indirect_call)
113 : {
114 23 : hints &= ~INLINE_HINT_indirect_call;
115 23 : fprintf (f, " indirect_call");
116 : }
117 148 : if (hints & INLINE_HINT_loop_iterations)
118 : {
119 3 : hints &= ~INLINE_HINT_loop_iterations;
120 3 : fprintf (f, " loop_iterations");
121 : }
122 148 : if (hints & INLINE_HINT_loop_stride)
123 : {
124 3 : hints &= ~INLINE_HINT_loop_stride;
125 3 : fprintf (f, " loop_stride");
126 : }
127 148 : if (hints & INLINE_HINT_same_scc)
128 : {
129 4 : hints &= ~INLINE_HINT_same_scc;
130 4 : fprintf (f, " same_scc");
131 : }
132 148 : if (hints & INLINE_HINT_in_scc)
133 : {
134 11 : hints &= ~INLINE_HINT_in_scc;
135 11 : fprintf (f, " in_scc");
136 : }
137 148 : if (hints & INLINE_HINT_cross_module)
138 : {
139 2 : hints &= ~INLINE_HINT_cross_module;
140 2 : fprintf (f, " cross_module");
141 : }
142 148 : if (hints & INLINE_HINT_declared_inline)
143 : {
144 122 : hints &= ~INLINE_HINT_declared_inline;
145 122 : fprintf (f, " declared_inline");
146 : }
147 148 : if (hints & INLINE_HINT_known_hot)
148 : {
149 1 : hints &= ~INLINE_HINT_known_hot;
150 1 : fprintf (f, " known_hot");
151 : }
152 148 : if (hints & INLINE_HINT_builtin_constant_p)
153 : {
154 4 : hints &= ~INLINE_HINT_builtin_constant_p;
155 4 : fprintf (f, " builtin_constant_p");
156 : }
157 148 : gcc_assert (!hints);
158 : }
159 :
160 :
161 : /* Record SIZE and TIME to SUMMARY.
162 : The accounted code will be executed when EXEC_PRED is true.
163 : When NONCONST_PRED is false the code will evaluate to constant and
164 : will get optimized out in specialized clones of the function.
165 : If CALL is true account to call_size_time_table rather than
166 : size_time_table. */
167 :
168 : void
169 139417274 : ipa_fn_summary::account_size_time (int size, sreal time,
170 : const ipa_predicate &exec_pred,
171 : const ipa_predicate &nonconst_pred_in,
172 : bool call)
173 : {
174 139417274 : size_time_entry *e;
175 139417274 : bool found = false;
176 139417274 : int i;
177 139417274 : ipa_predicate nonconst_pred;
178 139417274 : vec<size_time_entry> *table = call ? &call_size_time_table : &size_time_table;
179 :
180 139417274 : if (exec_pred == false)
181 5032789 : return;
182 :
183 139126637 : nonconst_pred = nonconst_pred_in & exec_pred;
184 :
185 139126637 : if (nonconst_pred == false)
186 : return;
187 :
188 : /* We need to create initial empty unconditional clause, but otherwise
189 : we don't need to account empty times and sizes. */
190 163023156 : if (!size && time == 0 && table->length ())
191 3974553 : return;
192 :
193 : /* Only for calls we are unaccounting what we previously recorded. */
194 134384485 : gcc_checking_assert (time >= 0 || call);
195 :
196 486892267 : for (i = 0; table->iterate (i, &e); i++)
197 456684811 : if (e->exec_predicate == exec_pred
198 456684811 : && e->nonconst_predicate == nonconst_pred)
199 : {
200 : found = true;
201 : break;
202 : }
203 134384485 : if (i == max_size_time_table_size)
204 : {
205 4896 : i = 0;
206 4896 : found = true;
207 4896 : e = &(*table)[0];
208 4896 : if (dump_file && (dump_flags & TDF_DETAILS))
209 0 : fprintf (dump_file,
210 : "\t\tReached limit on number of entries, "
211 : "ignoring the predicate.");
212 : }
213 134388846 : if (dump_file && (dump_flags & TDF_DETAILS) && (time != 0 || size))
214 : {
215 4037 : fprintf (dump_file,
216 : "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
217 3777 : ((double) size) / ipa_fn_summary::size_scale,
218 : (time.to_double ()), found ? "" : "new ");
219 3777 : exec_pred.dump (dump_file, conds, 0);
220 3777 : if (exec_pred != nonconst_pred)
221 : {
222 96 : fprintf (dump_file, " nonconst:");
223 96 : nonconst_pred.dump (dump_file, conds);
224 : }
225 : else
226 3681 : fprintf (dump_file, "\n");
227 : }
228 134384485 : if (!found)
229 : {
230 30202560 : class size_time_entry new_entry;
231 30202560 : new_entry.size = size;
232 30202560 : new_entry.time = time;
233 30202560 : new_entry.exec_predicate = exec_pred;
234 30202560 : new_entry.nonconst_predicate = nonconst_pred;
235 30202560 : if (call)
236 1364926 : call_size_time_table.safe_push (new_entry);
237 : else
238 28837634 : size_time_table.safe_push (new_entry);
239 : }
240 : else
241 : {
242 104181925 : e->size += size;
243 104181925 : e->time += time;
244 : /* FIXME: PR bootstrap/92653 gcc_checking_assert (e->time >= -1); */
245 : /* Tolerate small roundoff issues. */
246 104181925 : if (e->time < 0)
247 231 : e->time = 0;
248 : }
249 : }
250 :
251 : /* We proved E to be unreachable, redirect it to __builtin_unreachable. */
252 :
253 : static struct cgraph_edge *
254 344800 : redirect_to_unreachable (struct cgraph_edge *e)
255 : {
256 344800 : struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
257 344800 : struct cgraph_node *target
258 344800 : = cgraph_node::get_create (builtin_decl_unreachable ());
259 :
260 344800 : gcc_checking_assert (lookup_attribute ("cold",
261 : DECL_ATTRIBUTES (target->decl)));
262 :
263 344800 : if (e->speculative)
264 311 : e = cgraph_edge::resolve_speculation (e, target->decl);
265 344489 : else if (!e->callee)
266 789 : e = cgraph_edge::make_direct (e, target);
267 : else
268 343700 : e->redirect_callee (target);
269 344800 : class ipa_call_summary *es = ipa_call_summaries->get (e);
270 344800 : e->inline_failed = CIF_UNREACHABLE;
271 344800 : e->count = profile_count::zero ();
272 344800 : es->call_stmt_size = 0;
273 344800 : es->call_stmt_time = 0;
274 344800 : if (callee)
275 0 : callee->remove_symbol_and_inline_clones ();
276 344800 : if (e->has_callback)
277 6 : for (cgraph_edge *cbe = e->first_callback_edge (); cbe;
278 2 : cbe = cbe->next_callback_edge ())
279 : /* If the carrying edge is unreachable, so are the callback calls. */
280 2 : redirect_to_unreachable (cbe);
281 344800 : return e;
282 : }
283 :
284 : /* Set predicate for edge E. */
285 :
286 : static void
287 32127114 : edge_set_predicate (struct cgraph_edge *e, ipa_predicate *predicate)
288 : {
289 : /* If the edge is determined to be never executed, redirect it
290 : to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
291 : be optimized out. */
292 29120555 : if (predicate && *predicate == false
293 : /* When handling speculative edges, we need to do the redirection
294 : just once. Do it always on the direct edge, so we do not
295 : attempt to resolve speculation while duplicating the edge. */
296 32471989 : && (!e->speculative || e->callee))
297 344798 : e = redirect_to_unreachable (e);
298 :
299 32127114 : class ipa_call_summary *es = ipa_call_summaries->get (e);
300 61247669 : if (predicate && *predicate != true)
301 : {
302 4677916 : if (!es->predicate)
303 4167570 : es->predicate = edge_predicate_pool.allocate ();
304 4677916 : *es->predicate = *predicate;
305 : }
306 : else
307 : {
308 27449198 : if (es->predicate)
309 828903 : edge_predicate_pool.remove (es->predicate);
310 27449198 : es->predicate = NULL;
311 : }
312 32127114 : }
313 :
314 : /* Set predicate for hint *P. */
315 :
316 : static void
317 155428 : set_hint_predicate (ipa_predicate **p, ipa_predicate new_predicate)
318 : {
319 155428 : if (new_predicate == false || new_predicate == true)
320 : {
321 2986 : if (*p)
322 0 : edge_predicate_pool.remove (*p);
323 2986 : *p = NULL;
324 : }
325 : else
326 : {
327 152442 : if (!*p)
328 152442 : *p = edge_predicate_pool.allocate ();
329 152442 : **p = new_predicate;
330 : }
331 155428 : }
332 :
333 : /* Find if NEW_PREDICATE is already in V and if so, increment its freq.
334 : Otherwise add a new item to the vector with this predicate and frerq equal
335 : to add_freq, unless the number of predicates would exceed MAX_NUM_PREDICATES
336 : in which case the function does nothing. */
337 :
338 : static void
339 1145232 : add_freqcounting_predicate (vec<ipa_freqcounting_predicate, va_gc> **v,
340 : const ipa_predicate &new_predicate, sreal add_freq,
341 : unsigned max_num_predicates)
342 : {
343 1145232 : if (new_predicate == false || new_predicate == true)
344 : return;
345 : ipa_freqcounting_predicate *f;
346 145384 : for (int i = 0; vec_safe_iterate (*v, i, &f); i++)
347 72577 : if (new_predicate == f->predicate)
348 : {
349 0 : f->freq += add_freq;
350 0 : return;
351 : }
352 99396 : if (vec_safe_length (*v) >= max_num_predicates)
353 : /* Too many different predicates to account for. */
354 : return;
355 :
356 72535 : ipa_freqcounting_predicate fcp;
357 72535 : fcp.predicate = NULL;
358 72535 : set_hint_predicate (&fcp.predicate, new_predicate);
359 72535 : fcp.freq = add_freq;
360 72535 : vec_safe_push (*v, fcp);
361 72535 : return;
362 : }
363 :
364 : /* Compute what conditions may or may not hold given information about
365 : parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
366 : while RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
367 : copy when called in a given context. It is a bitmask of conditions. Bit
368 : 0 means that condition is known to be false, while bit 1 means that condition
369 : may or may not be true. These differs - for example NOT_INLINED condition
370 : is always false in the second and also builtin_constant_p tests cannot use
371 : the fact that parameter is indeed a constant.
372 :
373 : When INLINE_P is true, assume that we are inlining. AVAL contains known
374 : information about argument values. The function does not modify its content
375 : and so AVALs could also be of type ipa_call_arg_values but so far all
376 : callers work with the auto version and so we avoid the conversion for
377 : convenience.
378 :
379 : ERROR_MARK value of an argument means compile time invariant. */
380 :
381 : static void
382 23001217 : evaluate_conditions_for_known_args (struct cgraph_node *node,
383 : bool inline_p,
384 : ipa_auto_call_arg_values *avals,
385 : clause_t *ret_clause,
386 : clause_t *ret_nonspec_clause,
387 : ipa_call_summary *es)
388 : {
389 23001217 : clause_t clause = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
390 23001217 : clause_t nonspec_clause = 1 << ipa_predicate::not_inlined_condition;
391 23001217 : class ipa_fn_summary *info = ipa_fn_summaries->get (node);
392 23001217 : int i;
393 23001217 : struct condition *c;
394 :
395 94622881 : for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
396 : {
397 71621664 : tree val = NULL;
398 71621664 : tree res;
399 71621664 : int j;
400 71621664 : struct expr_eval_op *op;
401 :
402 71621664 : if (c->code == ipa_predicate::not_sra_candidate)
403 : {
404 16768842 : if (!inline_p
405 16768842 : || !es
406 16446671 : || (int)es->param.length () <= c->operand_num
407 24992150 : || !es->param[c->operand_num].points_to_possible_sra_candidate)
408 12552918 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
409 16768842 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
410 71621664 : continue;
411 : }
412 :
413 54852822 : if (c->agg_contents)
414 : {
415 30259725 : if (c->code == ipa_predicate::changed
416 19453027 : && !c->by_ref
417 33147260 : && (avals->safe_sval_at(c->operand_num) == error_mark_node))
418 3462 : continue;
419 :
420 30252801 : if (tree sval = avals->safe_sval_at (c->operand_num))
421 13982161 : val = ipa_find_agg_cst_from_init (sval, c->offset, c->by_ref);
422 : /* ipa_argagg_value_list is indexed by byte offsets, so a condition
423 : which does not start at a byte boundary (a bit-field) cannot be
424 : looked up in it; the containing byte would be reinterpreted as
425 : the whole field below. */
426 30252801 : if (!val && (c->offset % BITS_PER_UNIT) == 0)
427 : {
428 30215402 : ipa_argagg_value_list avs (avals);
429 30215402 : val = avs.get_value (c->operand_num, c->offset / BITS_PER_UNIT,
430 30215402 : c->by_ref);
431 : }
432 : }
433 : else
434 : {
435 24596559 : val = avals->safe_sval_at (c->operand_num);
436 24596559 : if (val && val == error_mark_node
437 2192626 : && c->code != ipa_predicate::changed)
438 : val = NULL_TREE;
439 : }
440 :
441 41507525 : if (!val
442 41426822 : && (c->code == ipa_predicate::changed
443 : || c->code == ipa_predicate::is_not_constant))
444 : {
445 26294782 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
446 26294782 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
447 26294782 : continue;
448 : }
449 28554578 : if (c->code == ipa_predicate::changed)
450 : {
451 7675844 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
452 7675844 : continue;
453 : }
454 :
455 20878734 : if (c->code == ipa_predicate::is_not_constant)
456 : {
457 8779 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
458 8779 : continue;
459 : }
460 :
461 20869955 : if (val
462 20869955 : && (c->type == TREE_TYPE (val)
463 1074614 : || (TYPE_SIZE (c->type) == TYPE_SIZE (TREE_TYPE (val))
464 : /* Avoid precision mismatch like with bit-fields where the
465 : VIEW_CONVERT_EXPR does not truncate excess bits
466 : appropriately. */
467 1011996 : && ((!INTEGRAL_TYPE_P (c->type)
468 240648 : || type_has_mode_precision_p (c->type))
469 1010273 : && (!INTEGRAL_TYPE_P (TREE_TYPE (val))
470 239105 : || type_has_mode_precision_p (TREE_TYPE (val)))))))
471 : {
472 5673560 : if (c->type != TREE_TYPE (val))
473 1010259 : val = fold_unary (VIEW_CONVERT_EXPR, c->type, val);
474 6197521 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
475 : {
476 524416 : if (!val)
477 : break;
478 523961 : if (!op->val[0])
479 226150 : val = fold_unary (op->code, op->type, val);
480 297811 : else if (!op->val[1])
481 595622 : val = fold_binary (op->code, op->type,
482 : op->index ? op->val[0] : val,
483 : op->index ? val : op->val[0]);
484 0 : else if (op->index == 0)
485 0 : val = fold_ternary (op->code, op->type,
486 : val, op->val[0], op->val[1]);
487 0 : else if (op->index == 1)
488 0 : val = fold_ternary (op->code, op->type,
489 : op->val[0], val, op->val[1]);
490 0 : else if (op->index == 2)
491 0 : val = fold_ternary (op->code, op->type,
492 : op->val[0], op->val[1], val);
493 : else
494 : val = NULL_TREE;
495 : }
496 :
497 5673560 : res = val
498 5673560 : ? fold_binary_to_constant (c->code, boolean_type_node, val, c->val)
499 : : NULL;
500 :
501 5673085 : if (res && integer_zerop (res))
502 2882058 : continue;
503 2791502 : if (res && integer_onep (res))
504 : {
505 2768840 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
506 2768840 : nonspec_clause
507 2768840 : |= 1 << (i + ipa_predicate::first_dynamic_condition);
508 2768840 : continue;
509 : }
510 : }
511 15219057 : if (c->operand_num < (int) avals->m_known_value_ranges.length ()
512 8825161 : && !c->agg_contents
513 17426315 : && (!val || TREE_CODE (val) != INTEGER_CST))
514 : {
515 2206523 : value_range vr (avals->m_known_value_ranges[c->operand_num]);
516 2206523 : if (!vr.undefined_p ()
517 1550254 : && !vr.varying_p ()
518 3756777 : && (TYPE_SIZE (c->type) == TYPE_SIZE (vr.type ())))
519 : {
520 1549717 : if (!useless_type_conversion_p (c->type, vr.type ()))
521 536 : range_cast (vr, c->type);
522 :
523 1974235 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
524 : {
525 526581 : if (vr.varying_p () || vr.undefined_p ())
526 : break;
527 :
528 424518 : value_range res (op->type);
529 424518 : if (!op->val[0])
530 : {
531 150506 : value_range varying (op->type);
532 150506 : varying.set_varying (op->type);
533 150506 : range_op_handler handler (op->code);
534 150506 : if (!handler
535 150506 : || !res.supports_type_p (op->type)
536 301012 : || !handler.fold_range (res, op->type, vr, varying))
537 0 : res.set_varying (op->type);
538 150506 : }
539 274012 : else if (!op->val[1])
540 : {
541 273840 : value_range op0 (TREE_TYPE (op->val[0]));
542 273840 : range_op_handler handler (op->code);
543 :
544 273840 : ipa_get_range_from_ip_invariant (op0, op->val[0], node);
545 :
546 273840 : if (!handler
547 273840 : || !res.supports_type_p (op->type)
548 547680 : || !handler.fold_range (res, op->type,
549 273840 : op->index ? op0 : vr,
550 545912 : op->index ? vr : op0))
551 0 : res.set_varying (op->type);
552 273840 : }
553 : else
554 172 : res.set_varying (op->type);
555 424518 : vr = res;
556 424518 : }
557 1549717 : if (!vr.varying_p () && !vr.undefined_p ())
558 : {
559 1438479 : int_range<2> res;
560 1438479 : value_range val_vr (TREE_TYPE (c->val));
561 1438479 : range_op_handler handler (c->code);
562 :
563 1438479 : ipa_get_range_from_ip_invariant (val_vr, c->val, node);
564 :
565 1438479 : if (!handler
566 1438479 : || !val_vr.supports_type_p (TREE_TYPE (c->val))
567 2876958 : || !handler.fold_range (res, boolean_type_node, vr, val_vr))
568 0 : res.set_varying (boolean_type_node);
569 :
570 1438479 : if (res.zero_p ())
571 243469 : continue;
572 1438479 : }
573 : }
574 2206523 : }
575 :
576 14975588 : clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
577 14975588 : nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
578 : }
579 23001217 : *ret_clause = clause;
580 23001217 : if (ret_nonspec_clause)
581 19920174 : *ret_nonspec_clause = nonspec_clause;
582 23001217 : }
583 :
584 : /* Return true if VRP will be executed on the function.
585 : We do not want to anticipate optimizations that will not happen.
586 :
587 : FIXME: This can be confused with -fdisable and debug counters and thus
588 : it should not be used for correctness (only to make heuristics work).
589 : This means that inliner should do its own optimizations of expressions
590 : that it predicts to be constant so wrong code can not be triggered by
591 : builtin_constant_p. */
592 :
593 : static bool
594 12520813 : vrp_will_run_p (struct cgraph_node *node)
595 : {
596 12520813 : return (opt_for_fn (node->decl, optimize)
597 12520813 : && !opt_for_fn (node->decl, optimize_debug)
598 25040858 : && opt_for_fn (node->decl, flag_tree_vrp));
599 : }
600 :
601 : /* Similarly about FRE. */
602 :
603 : static bool
604 14746263 : fre_will_run_p (struct cgraph_node *node)
605 : {
606 14746263 : return (opt_for_fn (node->decl, optimize)
607 14746263 : && !opt_for_fn (node->decl, optimize_debug)
608 29490878 : && opt_for_fn (node->decl, flag_tree_fre));
609 : }
610 :
611 : /* Work out what conditions might be true at invocation of E.
612 : Compute costs for inlined edge if INLINE_P is true.
613 :
614 : Return in CLAUSE_PTR the evaluated conditions and in NONSPEC_CLAUSE_PTR
615 : (if non-NULL) conditions evaluated for nonspecialized clone called
616 : in a given context.
617 :
618 : Vectors in AVALS will be populated with useful known information about
619 : argument values - information not known to have any uses will be omitted -
620 : except for m_known_contexts which will only be calculated if
621 : COMPUTE_CONTEXTS is true. */
622 :
623 : void
624 22751247 : evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p,
625 : clause_t *clause_ptr,
626 : clause_t *nonspec_clause_ptr,
627 : ipa_auto_call_arg_values *avals,
628 : bool compute_contexts)
629 : {
630 22751247 : struct cgraph_node *callee = e->callee->ultimate_alias_target ();
631 22751247 : class ipa_fn_summary *info = ipa_fn_summaries->get (callee);
632 22751247 : class ipa_edge_args *args;
633 22751247 : class ipa_call_summary *es = NULL;
634 :
635 22751247 : if (clause_ptr)
636 22751247 : *clause_ptr = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
637 :
638 22751247 : if (ipa_node_params_sum
639 10294637 : && !e->call_stmt_cannot_inline_p
640 10294637 : && (info->conds || compute_contexts)
641 33045884 : && (args = ipa_edge_args_sum->get (e)) != NULL)
642 : {
643 10239368 : struct cgraph_node *caller;
644 10239368 : class ipa_node_params *caller_parms_info, *callee_pi = NULL;
645 10239368 : int i, count = ipa_get_cs_argument_count (args);
646 10239368 : es = ipa_call_summaries->get (e);
647 :
648 10239368 : if (count)
649 : {
650 9501470 : if (e->caller->inlined_to)
651 : caller = e->caller->inlined_to;
652 : else
653 7557704 : caller = e->caller;
654 9501470 : caller_parms_info = ipa_node_params_sum->get (caller);
655 9501470 : callee_pi = ipa_node_params_sum->get (callee);
656 :
657 : /* Watch for thunks. */
658 9501470 : if (callee_pi)
659 : /* Watch for variadic functions. */
660 9501096 : count = MIN (count, ipa_get_param_count (callee_pi));
661 : }
662 :
663 9501096 : if (callee_pi)
664 31328562 : for (i = 0; i < count; i++)
665 : {
666 21827466 : struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i);
667 :
668 21827466 : if (ipa_is_param_used_by_indirect_call (callee_pi, i)
669 21827466 : || ipa_is_param_used_by_ipa_predicates (callee_pi, i))
670 : {
671 : /* Determine if we know constant value of the parameter. */
672 14746263 : tree type = ipa_get_type (callee_pi, i);
673 14746263 : tree cst = ipa_value_from_jfunc (caller_parms_info, jf, type);
674 :
675 11833776 : if (!cst && e->call_stmt
676 26526317 : && i < (int)gimple_call_num_args (e->call_stmt))
677 : {
678 11780054 : cst = gimple_call_arg (e->call_stmt, i);
679 11780054 : if (!is_gimple_min_invariant (cst))
680 : cst = NULL;
681 : }
682 3160836 : if (cst)
683 : {
684 3107114 : gcc_checking_assert (TREE_CODE (cst) != TREE_BINFO);
685 3107114 : if (!avals->m_known_vals.length ())
686 1787425 : avals->m_known_vals.safe_grow_cleared (count, true);
687 3107114 : avals->m_known_vals[i] = cst;
688 : }
689 11639149 : else if (inline_p && !es->param[i].change_prob)
690 : {
691 4251013 : if (!avals->m_known_vals.length ())
692 3449580 : avals->m_known_vals.safe_grow_cleared (count, true);
693 4251013 : avals->m_known_vals[i] = error_mark_node;
694 : }
695 :
696 : /* If we failed to get simple constant, try value range. */
697 3107114 : if ((!cst || TREE_CODE (cst) != INTEGER_CST)
698 12520813 : && vrp_will_run_p (caller)
699 27099154 : && ipa_is_param_used_by_ipa_predicates (callee_pi, i))
700 : {
701 12311984 : value_range vr (type);
702 :
703 12311984 : ipa_value_range_from_jfunc (vr, caller_parms_info, e, jf, type);
704 12311984 : if (!vr.undefined_p () && !vr.varying_p ())
705 : {
706 8746581 : if (!avals->m_known_value_ranges.length ())
707 : {
708 6357043 : avals->m_known_value_ranges.safe_grow_cleared (count,
709 : true);
710 26671039 : for (int i = 0; i < count; ++i)
711 13956953 : avals->m_known_value_ranges[i].set_range_class
712 13956953 : (void_type_node);
713 : }
714 8746581 : avals->m_known_value_ranges[i] = vr;
715 : }
716 12311984 : }
717 :
718 : /* Determine known aggregate values. */
719 14746263 : if (fre_will_run_p (caller))
720 14744211 : ipa_push_agg_values_from_jfunc (caller_parms_info,
721 : caller, &jf->agg, i,
722 : &avals->m_known_aggs);
723 : }
724 :
725 : /* For calls used in polymorphic calls we further determine
726 : polymorphic call context. */
727 21827466 : if (compute_contexts
728 21827466 : && ipa_is_param_used_by_polymorphic_call (callee_pi, i))
729 : {
730 185376 : ipa_polymorphic_call_context
731 185376 : ctx = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
732 370752 : if (!ctx.useless_p ())
733 : {
734 183410 : if (!avals->m_known_contexts.length ())
735 183236 : avals->m_known_contexts.safe_grow_cleared (count, true);
736 183410 : avals->m_known_contexts[i]
737 366820 : = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
738 : }
739 : }
740 : }
741 : else
742 738272 : gcc_assert (!count || callee->thunk);
743 : }
744 12511879 : else if (e->call_stmt && !e->call_stmt_cannot_inline_p && info->conds)
745 : {
746 9840205 : int i, count = (int)gimple_call_num_args (e->call_stmt);
747 :
748 27997048 : for (i = 0; i < count; i++)
749 : {
750 18156843 : tree cst = gimple_call_arg (e->call_stmt, i);
751 18156843 : if (!is_gimple_min_invariant (cst))
752 : cst = NULL;
753 6509928 : if (cst)
754 : {
755 6509928 : if (!avals->m_known_vals.length ())
756 4514956 : avals->m_known_vals.safe_grow_cleared (count, true);
757 6509928 : avals->m_known_vals[i] = cst;
758 : }
759 : }
760 : }
761 :
762 22751247 : evaluate_conditions_for_known_args (callee, inline_p, avals, clause_ptr,
763 : nonspec_clause_ptr, es);
764 22751247 : }
765 :
766 :
767 : /* Allocate the function summary. */
768 :
769 : static void
770 472283 : ipa_fn_summary_alloc (void)
771 : {
772 472283 : gcc_checking_assert (!ipa_fn_summaries);
773 472283 : ipa_size_summaries = new ipa_size_summary_t (symtab);
774 472283 : ipa_fn_summaries = ipa_fn_summary_t::create_ggc (symtab);
775 472283 : ipa_call_summaries = new ipa_call_summary_t (symtab);
776 472283 : }
777 :
778 28143343 : ipa_call_summary::~ipa_call_summary ()
779 : {
780 28143343 : if (predicate)
781 3338667 : edge_predicate_pool.remove (predicate);
782 :
783 28143343 : param.release ();
784 28143343 : }
785 :
786 10369843 : ipa_fn_summary::~ipa_fn_summary ()
787 : {
788 10369843 : unsigned len = vec_safe_length (loop_iterations);
789 10486068 : for (unsigned i = 0; i < len; i++)
790 116225 : edge_predicate_pool.remove ((*loop_iterations)[i].predicate);
791 10369843 : len = vec_safe_length (loop_strides);
792 10406060 : for (unsigned i = 0; i < len; i++)
793 36217 : edge_predicate_pool.remove ((*loop_strides)[i].predicate);
794 10369843 : vec_free (conds);
795 10369843 : call_size_time_table.release ();
796 10369843 : vec_free (loop_iterations);
797 10369843 : vec_free (loop_strides);
798 10369843 : builtin_constant_p_parms.release ();
799 10369843 : }
800 :
801 : void
802 7452066 : ipa_fn_summary_t::remove_callees (cgraph_node *node)
803 : {
804 7452066 : cgraph_edge *e;
805 30589425 : for (e = node->callees; e; e = e->next_callee)
806 23137359 : ipa_call_summaries->remove (e);
807 7938393 : for (e = node->indirect_calls; e; e = e->next_callee)
808 486327 : ipa_call_summaries->remove (e);
809 7452066 : }
810 :
811 : /* Duplicate predicates in loop hint vector, allocating memory for them and
812 : remove and deallocate any uninteresting (true or false) ones. Return the
813 : result. */
814 :
815 : static vec<ipa_freqcounting_predicate, va_gc> *
816 29076 : remap_freqcounting_preds_after_dup (vec<ipa_freqcounting_predicate, va_gc> *v,
817 : clause_t possible_truths)
818 : {
819 29076 : if (vec_safe_length (v) == 0)
820 : return NULL;
821 :
822 4771 : vec<ipa_freqcounting_predicate, va_gc> *res = v->copy ();
823 4771 : int len = res->length();
824 11095 : for (int i = len - 1; i >= 0; i--)
825 : {
826 6324 : ipa_predicate new_predicate
827 6324 : = (*res)[i].predicate->remap_after_duplication (possible_truths);
828 : /* We do not want to free previous predicate; it is used by node
829 : origin. */
830 6324 : (*res)[i].predicate = NULL;
831 6324 : set_hint_predicate (&(*res)[i].predicate, new_predicate);
832 :
833 6324 : if (!(*res)[i].predicate)
834 2986 : res->unordered_remove (i);
835 : }
836 :
837 : return res;
838 : }
839 :
840 :
841 : /* Hook that is called by cgraph.cc when a node is duplicated. */
842 : void
843 2828353 : ipa_fn_summary_t::duplicate (cgraph_node *src,
844 : cgraph_node *dst,
845 : ipa_fn_summary *src_info,
846 : ipa_fn_summary *info)
847 : {
848 2828353 : new (info) ipa_fn_summary (*src_info);
849 : /* TODO: as an optimization, we may avoid copying conditions
850 : that are known to be false or true. */
851 2828353 : info->conds = vec_safe_copy (info->conds);
852 :
853 2828353 : clone_info *cinfo = clone_info::get (dst);
854 : /* When there are any replacements in the function body, see if we can figure
855 : out that something was optimized out. */
856 2828353 : if (ipa_node_params_sum && cinfo && cinfo->tree_map)
857 : {
858 : /* Use SRC parm info since it may not be copied yet. */
859 14538 : ipa_node_params *parms_info = ipa_node_params_sum->get (src);
860 14538 : ipa_auto_call_arg_values avals;
861 14538 : int count = ipa_get_param_count (parms_info);
862 14538 : int i, j;
863 14538 : clause_t possible_truths;
864 14538 : ipa_predicate true_pred = true;
865 14538 : size_time_entry *e;
866 14538 : int optimized_out_size = 0;
867 14538 : bool inlined_to_p = false;
868 14538 : struct cgraph_edge *edge, *next;
869 :
870 14538 : info->size_time_table.release ();
871 14538 : avals.m_known_vals.safe_grow_cleared (count, true);
872 72865 : for (i = 0; i < count; i++)
873 : {
874 : struct ipa_replace_map *r;
875 :
876 142706 : for (j = 0; vec_safe_iterate (cinfo->tree_map, j, &r); j++)
877 : {
878 80100 : if (r->parm_num == i)
879 : {
880 24972 : avals.m_known_vals[i] = r->new_tree;
881 24972 : break;
882 : }
883 : }
884 : }
885 14538 : evaluate_conditions_for_known_args (dst, false,
886 : &avals,
887 : &possible_truths,
888 : /* We are going to specialize,
889 : so ignore nonspec truths. */
890 : NULL,
891 : NULL);
892 :
893 14538 : info->account_size_time (0, 0, true_pred, true_pred);
894 :
895 : /* Remap size_time vectors.
896 : Simplify the predicate by pruning out alternatives that are known
897 : to be false.
898 : TODO: as on optimization, we can also eliminate conditions known
899 : to be true. */
900 100913 : for (i = 0; src_info->size_time_table.iterate (i, &e); i++)
901 : {
902 86375 : ipa_predicate new_exec_pred;
903 86375 : ipa_predicate new_nonconst_pred;
904 86375 : new_exec_pred = e->exec_predicate.remap_after_duplication
905 86375 : (possible_truths);
906 86375 : new_nonconst_pred = e->nonconst_predicate.remap_after_duplication
907 86375 : (possible_truths);
908 86375 : if (new_exec_pred == false || new_nonconst_pred == false)
909 11913 : optimized_out_size += e->size;
910 : else
911 74462 : info->account_size_time (e->size, e->time, new_exec_pred,
912 : new_nonconst_pred);
913 : }
914 :
915 : /* Remap edge predicates with the same simplification as above.
916 : Also copy constantness arrays. */
917 184580 : for (edge = dst->callees; edge; edge = next)
918 : {
919 170042 : ipa_predicate new_predicate;
920 170042 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
921 170042 : next = edge->next_callee;
922 :
923 170042 : if (!edge->inline_failed)
924 0 : inlined_to_p = true;
925 170042 : if (!es->predicate)
926 157551 : continue;
927 12491 : new_predicate = es->predicate->remap_after_duplication
928 12491 : (possible_truths);
929 12491 : if (new_predicate == false && *es->predicate != false)
930 3247 : optimized_out_size += es->call_stmt_size * ipa_fn_summary::size_scale;
931 12491 : edge_set_predicate (edge, &new_predicate);
932 : }
933 :
934 : /* Remap indirect edge predicates with the same simplification as above.
935 : Also copy constantness arrays. */
936 15457 : for (edge = dst->indirect_calls; edge; edge = next)
937 : {
938 919 : ipa_predicate new_predicate;
939 919 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
940 919 : next = edge->next_callee;
941 :
942 919 : gcc_checking_assert (edge->inline_failed);
943 919 : if (!es->predicate)
944 815 : continue;
945 104 : new_predicate = es->predicate->remap_after_duplication
946 104 : (possible_truths);
947 104 : if (new_predicate == false && *es->predicate != false)
948 21 : optimized_out_size
949 21 : += es->call_stmt_size * ipa_fn_summary::size_scale;
950 104 : edge_set_predicate (edge, &new_predicate);
951 : }
952 14538 : info->loop_iterations
953 14538 : = remap_freqcounting_preds_after_dup (info->loop_iterations,
954 : possible_truths);
955 14538 : info->loop_strides
956 14538 : = remap_freqcounting_preds_after_dup (info->loop_strides,
957 : possible_truths);
958 14538 : if (info->builtin_constant_p_parms.length())
959 : {
960 23 : vec <int, va_heap, vl_ptr> parms = info->builtin_constant_p_parms;
961 23 : int ip;
962 23 : info->builtin_constant_p_parms = vNULL;
963 46 : for (i = 0; parms.iterate (i, &ip); i++)
964 23 : if (!avals.m_known_vals[ip])
965 8 : info->builtin_constant_p_parms.safe_push (ip);
966 : }
967 :
968 : /* If inliner or someone after inliner will ever start producing
969 : non-trivial clones, we will get trouble with lack of information
970 : about updating self sizes, because size vectors already contains
971 : sizes of the callees. */
972 14538 : gcc_assert (!inlined_to_p || !optimized_out_size);
973 14538 : }
974 : else
975 : {
976 2813815 : info->size_time_table = src_info->size_time_table.copy ();
977 2813815 : info->loop_iterations = vec_safe_copy (src_info->loop_iterations);
978 2813815 : info->loop_strides = vec_safe_copy (info->loop_strides);
979 :
980 2813815 : info->builtin_constant_p_parms
981 2813815 : = info->builtin_constant_p_parms.copy ();
982 :
983 2813815 : ipa_freqcounting_predicate *f;
984 5677915 : for (int i = 0; vec_safe_iterate (info->loop_iterations, i, &f); i++)
985 : {
986 50285 : ipa_predicate p = *f->predicate;
987 50285 : f->predicate = NULL;
988 50285 : set_hint_predicate (&f->predicate, p);
989 : }
990 2838288 : for (int i = 0; vec_safe_iterate (info->loop_strides, i, &f); i++)
991 : {
992 24473 : ipa_predicate p = *f->predicate;
993 24473 : f->predicate = NULL;
994 24473 : set_hint_predicate (&f->predicate, p);
995 : }
996 : }
997 2828353 : if (!dst->inlined_to)
998 145390 : ipa_update_overall_fn_summary (dst);
999 2828353 : }
1000 :
1001 :
1002 : /* Hook that is called by cgraph.cc when a node is duplicated. */
1003 :
1004 : void
1005 4168582 : ipa_call_summary_t::duplicate (struct cgraph_edge *src,
1006 : struct cgraph_edge *dst,
1007 : class ipa_call_summary *srcinfo,
1008 : class ipa_call_summary *info)
1009 : {
1010 4168582 : new (info) ipa_call_summary (*srcinfo);
1011 4168582 : info->predicate = NULL;
1012 4168582 : edge_set_predicate (dst, srcinfo->predicate);
1013 4168582 : info->param = srcinfo->param.copy ();
1014 4168582 : if (!dst->indirect_unknown_callee && src->indirect_unknown_callee
1015 : /* Don't subtract the size when dealing with callback pairs, since the
1016 : edge has no real size. */
1017 36734 : && !src->has_callback && !dst->callback)
1018 : {
1019 36734 : info->call_stmt_size -= (eni_size_weights.indirect_call_cost
1020 36734 : - eni_size_weights.call_cost);
1021 36734 : info->call_stmt_time -= (eni_time_weights.indirect_call_cost
1022 36734 : - eni_time_weights.call_cost);
1023 : }
1024 4168582 : }
1025 :
1026 : /* Dump edge summaries associated to NODE and recursively to all clones.
1027 : Indent by INDENT. */
1028 :
1029 : static void
1030 1969 : dump_ipa_call_summary (FILE *f, int indent, struct cgraph_node *node,
1031 : class ipa_fn_summary *info)
1032 : {
1033 1969 : struct cgraph_edge *edge;
1034 9787 : for (edge = node->callees; edge; edge = edge->next_callee)
1035 : {
1036 7818 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
1037 7818 : struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
1038 7818 : int i;
1039 :
1040 15636 : fprintf (f,
1041 : "%*s%s %s\n%*s freq:%4.2f",
1042 : indent, "", callee->dump_name (),
1043 7818 : !edge->inline_failed
1044 7265 : ? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
1045 7818 : indent, "", edge->sreal_frequency ().to_double ());
1046 :
1047 7818 : if (cross_module_call_p (edge))
1048 4 : fprintf (f, " cross module");
1049 :
1050 7818 : if (es)
1051 7265 : fprintf (f, " loop depth:%2i size:%2i time: %2i",
1052 : es->loop_depth, es->call_stmt_size, es->call_stmt_time);
1053 :
1054 7818 : ipa_fn_summary *s = ipa_fn_summaries->get (callee);
1055 7818 : ipa_size_summary *ss = ipa_size_summaries->get (callee);
1056 7818 : if (s != NULL)
1057 620 : fprintf (f, " callee size:%2i stack:%2i",
1058 620 : (int) (ss->size / ipa_fn_summary::size_scale),
1059 620 : (int) s->estimated_stack_size);
1060 :
1061 7818 : if (es && es->predicate)
1062 : {
1063 4627 : fprintf (f, " predicate: ");
1064 4627 : es->predicate->dump (f, info->conds);
1065 : }
1066 : else
1067 3191 : fprintf (f, "\n");
1068 7818 : if (es && es->param.exists ())
1069 5348 : for (i = 0; i < (int) es->param.length (); i++)
1070 : {
1071 1582 : int prob = es->param[i].change_prob;
1072 :
1073 1582 : if (!prob)
1074 620 : fprintf (f, "%*s op%i is compile time invariant\n",
1075 : indent + 2, "", i);
1076 962 : else if (prob != REG_BR_PROB_BASE)
1077 35 : fprintf (f, "%*s op%i change %f%% of time\n", indent + 2, "", i,
1078 35 : prob * 100.0 / REG_BR_PROB_BASE);
1079 1582 : if (es->param[i].points_to_local_or_readonly_memory)
1080 351 : fprintf (f, "%*s op%i points to local or readonly memory\n",
1081 : indent + 2, "", i);
1082 1582 : if (es->param[i].points_to_possible_sra_candidate)
1083 227 : fprintf (f, "%*s op%i points to possible sra candidate\n",
1084 : indent + 2, "", i);
1085 : }
1086 7818 : if (!edge->inline_failed)
1087 : {
1088 553 : ipa_size_summary *ss = ipa_size_summaries->get (callee);
1089 553 : fprintf (f, "%*sStack frame offset %i, callee self size %i\n",
1090 : indent + 2, "",
1091 553 : (int) ipa_get_stack_frame_offset (callee),
1092 553 : (int) ss->estimated_self_stack_size);
1093 553 : dump_ipa_call_summary (f, indent + 2, callee, info);
1094 : }
1095 : }
1096 2267 : for (edge = node->indirect_calls; edge; edge = edge->next_callee)
1097 : {
1098 298 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
1099 298 : fprintf (f, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
1100 : " time: %2i",
1101 : indent, "",
1102 : es->loop_depth,
1103 298 : edge->sreal_frequency ().to_double (), es->call_stmt_size,
1104 : es->call_stmt_time);
1105 298 : if (es->predicate)
1106 : {
1107 6 : fprintf (f, "predicate: ");
1108 6 : es->predicate->dump (f, info->conds);
1109 : }
1110 : else
1111 292 : fprintf (f, "\n");
1112 : }
1113 1969 : }
1114 :
1115 :
1116 : void
1117 1628 : ipa_dump_fn_summary (FILE *f, struct cgraph_node *node)
1118 : {
1119 1628 : if (node->definition)
1120 : {
1121 1628 : class ipa_fn_summary *s = ipa_fn_summaries->get (node);
1122 1628 : class ipa_size_summary *ss = ipa_size_summaries->get (node);
1123 1628 : if (s != NULL)
1124 : {
1125 1416 : size_time_entry *e;
1126 1416 : int i;
1127 1416 : fprintf (f, "IPA function summary for %s", node->dump_name ());
1128 1416 : if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
1129 0 : fprintf (f, " always_inline");
1130 1416 : if (s->inlinable)
1131 1235 : fprintf (f, " inlinable");
1132 1416 : if (s->fp_expressions)
1133 44 : fprintf (f, " fp_expression");
1134 1416 : if (s->builtin_constant_p_parms.length ())
1135 : {
1136 2 : fprintf (f, " builtin_constant_p_parms");
1137 4 : for (unsigned int i = 0;
1138 4 : i < s->builtin_constant_p_parms.length (); i++)
1139 2 : fprintf (f, " %i", s->builtin_constant_p_parms[i]);
1140 : }
1141 1416 : fprintf (f, "\n global time: %f\n", s->time.to_double ());
1142 1416 : fprintf (f, " self size: %i\n", ss->self_size);
1143 1416 : fprintf (f, " global size: %i\n", ss->size);
1144 1416 : fprintf (f, " min size: %i\n", s->min_size);
1145 1416 : fprintf (f, " self stack: %i\n",
1146 1416 : (int) ss->estimated_self_stack_size);
1147 1416 : fprintf (f, " global stack: %i\n", (int) s->estimated_stack_size);
1148 1416 : if (s->growth)
1149 168 : fprintf (f, " estimated growth:%i\n", (int) s->growth);
1150 1416 : if (s->scc_no)
1151 5 : fprintf (f, " In SCC: %i\n", (int) s->scc_no);
1152 5908 : for (i = 0; s->size_time_table.iterate (i, &e); i++)
1153 : {
1154 4492 : fprintf (f, " size:%f, time:%f",
1155 4492 : (double) e->size / ipa_fn_summary::size_scale,
1156 : e->time.to_double ());
1157 4492 : if (e->exec_predicate != true)
1158 : {
1159 2623 : fprintf (f, ", executed if:");
1160 2623 : e->exec_predicate.dump (f, s->conds, 0);
1161 : }
1162 4492 : if (e->exec_predicate != e->nonconst_predicate)
1163 : {
1164 1167 : fprintf (f, ", nonconst if:");
1165 1167 : e->nonconst_predicate.dump (f, s->conds, 0);
1166 : }
1167 4492 : fprintf (f, "\n");
1168 : }
1169 : ipa_freqcounting_predicate *fcp;
1170 : bool first_fcp = true;
1171 1522 : for (int i = 0; vec_safe_iterate (s->loop_iterations, i, &fcp); i++)
1172 : {
1173 106 : if (first_fcp)
1174 : {
1175 78 : fprintf (f, " loop iterations:");
1176 78 : first_fcp = false;
1177 : }
1178 106 : fprintf (f, " %3.2f for ", fcp->freq.to_double ());
1179 106 : fcp->predicate->dump (f, s->conds);
1180 : }
1181 : first_fcp = true;
1182 1434 : for (int i = 0; vec_safe_iterate (s->loop_strides, i, &fcp); i++)
1183 : {
1184 18 : if (first_fcp)
1185 : {
1186 10 : fprintf (f, " loop strides:");
1187 10 : first_fcp = false;
1188 : }
1189 18 : fprintf (f, " %3.2f for :", fcp->freq.to_double ());
1190 18 : fcp->predicate->dump (f, s->conds);
1191 : }
1192 1416 : fprintf (f, " calls:\n");
1193 1416 : dump_ipa_call_summary (f, 4, node, s);
1194 1416 : fprintf (f, "\n");
1195 1416 : if (s->target_info)
1196 0 : fprintf (f, " target_info: %x\n", s->target_info);
1197 : }
1198 : else
1199 212 : fprintf (f, "IPA summary for %s is missing.\n", node->dump_name ());
1200 : }
1201 1628 : }
1202 :
1203 : DEBUG_FUNCTION void
1204 0 : ipa_debug_fn_summary (struct cgraph_node *node)
1205 : {
1206 0 : ipa_dump_fn_summary (stderr, node);
1207 0 : }
1208 :
1209 : void
1210 356 : ipa_dump_fn_summaries (FILE *f)
1211 : {
1212 356 : struct cgraph_node *node;
1213 :
1214 2288 : FOR_EACH_DEFINED_FUNCTION (node)
1215 1932 : if (!node->inlined_to)
1216 1379 : ipa_dump_fn_summary (f, node);
1217 356 : }
1218 :
1219 : /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1220 : boolean variable pointed to by DATA. */
1221 :
1222 : static bool
1223 848988 : mark_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED,
1224 : void *data)
1225 : {
1226 848988 : bool *b = (bool *) data;
1227 848988 : *b = true;
1228 848988 : return true;
1229 : }
1230 :
1231 : /* If OP refers to value of function parameter, return the corresponding
1232 : parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
1233 : PARM_DECL) will be stored to *SIZE_P in that case too. */
1234 :
1235 : static tree
1236 190535823 : unmodified_parm_1 (ipa_func_body_info *fbi, gimple *stmt, tree op,
1237 : poly_int64 *size_p)
1238 : {
1239 : /* SSA_NAME referring to parm default def? */
1240 190535823 : if (TREE_CODE (op) == SSA_NAME
1241 111640166 : && SSA_NAME_IS_DEFAULT_DEF (op)
1242 216791108 : && TREE_CODE (SSA_NAME_VAR (op)) == PARM_DECL)
1243 : {
1244 26044255 : if (size_p)
1245 0 : *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
1246 26044255 : return SSA_NAME_VAR (op);
1247 : }
1248 : /* Non-SSA parm reference? */
1249 164491568 : if (TREE_CODE (op) == PARM_DECL
1250 1431397 : && fbi->aa_walk_budget > 0)
1251 : {
1252 1428851 : bool modified = false;
1253 :
1254 1428851 : ao_ref refd;
1255 1428851 : ao_ref_init (&refd, op);
1256 2857702 : int walked = walk_aliased_vdefs (&refd, gimple_vuse (stmt),
1257 : mark_modified, &modified, NULL, NULL,
1258 : fbi->aa_walk_budget);
1259 1428851 : if (walked < 0)
1260 : {
1261 8 : fbi->aa_walk_budget = 0;
1262 916996 : return NULL_TREE;
1263 : }
1264 1428843 : fbi->aa_walk_budget -= walked;
1265 1428843 : if (!modified)
1266 : {
1267 916988 : if (size_p)
1268 0 : *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
1269 : return op;
1270 : }
1271 : }
1272 : return NULL_TREE;
1273 : }
1274 :
1275 : /* If OP refers to value of function parameter, return the corresponding
1276 : parameter. Also traverse chains of SSA register assignments. If non-NULL,
1277 : the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
1278 : stored to *SIZE_P in that case too. */
1279 :
1280 : static tree
1281 124640169 : unmodified_parm (ipa_func_body_info *fbi, gimple *stmt, tree op,
1282 : poly_int64 *size_p)
1283 : {
1284 154907438 : tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
1285 154907438 : if (res)
1286 : return res;
1287 :
1288 128795591 : if (TREE_CODE (op) == SSA_NAME
1289 73175822 : && !SSA_NAME_IS_DEFAULT_DEF (op)
1290 201765144 : && gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1291 30267269 : return unmodified_parm (fbi, SSA_NAME_DEF_STMT (op),
1292 30267269 : gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op)),
1293 30267269 : size_p);
1294 : return NULL_TREE;
1295 : }
1296 :
1297 : /* If OP refers to a value of a function parameter or value loaded from an
1298 : aggregate passed to a parameter (either by value or reference), return TRUE
1299 : and store the number of the parameter to *INDEX_P, the access size into
1300 : *SIZE_P, and information whether and how it has been loaded from an
1301 : aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1302 : statement in which OP is used or loaded. */
1303 :
1304 : static bool
1305 34162100 : unmodified_parm_or_parm_agg_item (struct ipa_func_body_info *fbi,
1306 : gimple *stmt, tree op, int *index_p,
1307 : poly_int64 *size_p,
1308 : struct agg_position_info *aggpos)
1309 : {
1310 35628385 : tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
1311 :
1312 35628385 : gcc_checking_assert (aggpos);
1313 35628385 : if (res)
1314 : {
1315 849396 : *index_p = ipa_get_param_decl_index (fbi->info, res);
1316 849396 : if (*index_p < 0)
1317 : return false;
1318 849266 : aggpos->agg_contents = false;
1319 849266 : aggpos->by_ref = false;
1320 849266 : return true;
1321 : }
1322 :
1323 34778989 : if (TREE_CODE (op) == SSA_NAME)
1324 : {
1325 12420089 : if (SSA_NAME_IS_DEFAULT_DEF (op)
1326 12420089 : || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1327 : return false;
1328 4966412 : stmt = SSA_NAME_DEF_STMT (op);
1329 4966412 : op = gimple_assign_rhs1 (stmt);
1330 4966412 : if (!REFERENCE_CLASS_P (op))
1331 : return unmodified_parm_or_parm_agg_item (fbi, stmt, op, index_p, size_p,
1332 : aggpos);
1333 : }
1334 :
1335 25859027 : aggpos->agg_contents = true;
1336 25859027 : return ipa_load_from_parm_agg (fbi, fbi->info->descriptors,
1337 : stmt, op, index_p, &aggpos->offset,
1338 25859027 : size_p, &aggpos->by_ref);
1339 : }
1340 :
1341 : /* If stmt is simple load or store of value pointed to by a function parameter,
1342 : return its index. */
1343 :
1344 : static int
1345 114735321 : load_or_store_of_ptr_parameter (ipa_func_body_info *fbi, gimple *stmt)
1346 : {
1347 114735321 : if (!optimize)
1348 : return -1;
1349 96226725 : gassign *assign = dyn_cast <gassign *> (stmt);
1350 57315590 : if (!assign)
1351 : return -1;
1352 57315590 : tree param;
1353 57315590 : if (gimple_assign_load_p (stmt))
1354 20960047 : param = gimple_assign_rhs1 (stmt);
1355 36355543 : else if (gimple_store_p (stmt))
1356 19338585 : param = gimple_assign_lhs (stmt);
1357 : else
1358 : return -1;
1359 40298632 : tree base = get_base_address (param);
1360 40298632 : if (TREE_CODE (base) != MEM_REF
1361 13984879 : || TREE_CODE (TREE_OPERAND (base, 0)) != SSA_NAME
1362 54279381 : || !SSA_NAME_IS_DEFAULT_DEF (TREE_OPERAND (base, 0)))
1363 : return -1;
1364 7545765 : tree p = SSA_NAME_VAR (TREE_OPERAND (base, 0));
1365 7545765 : if (TREE_CODE (p) != PARM_DECL)
1366 : return -1;
1367 7457750 : return ipa_get_param_decl_index (fbi->info, p);
1368 : }
1369 :
1370 : /* See if statement might disappear after inlining.
1371 : 0 - means not eliminated
1372 : 1 - half of statements goes away
1373 : 2 - for sure it is eliminated.
1374 : We are not terribly sophisticated, basically looking for simple abstraction
1375 : penalty wrappers. */
1376 :
1377 : static int
1378 114735321 : eliminated_by_inlining_prob (ipa_func_body_info *fbi, gimple *stmt)
1379 : {
1380 114735321 : enum gimple_code code = gimple_code (stmt);
1381 114735321 : enum tree_code rhs_code;
1382 :
1383 114735321 : if (!optimize)
1384 : return 0;
1385 :
1386 96226725 : switch (code)
1387 : {
1388 : case GIMPLE_RETURN:
1389 : return 2;
1390 57315590 : case GIMPLE_ASSIGN:
1391 57315590 : if (gimple_num_ops (stmt) != 2)
1392 : return 0;
1393 :
1394 41135677 : rhs_code = gimple_assign_rhs_code (stmt);
1395 :
1396 : /* Casts of parameters, loads from parameters passed by reference
1397 : and stores to return value or parameters are often free after
1398 : inlining due to SRA and further combining.
1399 : Assume that half of statements goes away. */
1400 41135677 : if (CONVERT_EXPR_CODE_P (rhs_code)
1401 : || rhs_code == VIEW_CONVERT_EXPR
1402 : || rhs_code == ADDR_EXPR
1403 38258075 : || gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS)
1404 : {
1405 40298632 : tree rhs = gimple_assign_rhs1 (stmt);
1406 40298632 : tree lhs = gimple_assign_lhs (stmt);
1407 40298632 : tree inner_rhs = get_base_address (rhs);
1408 40298632 : tree inner_lhs = get_base_address (lhs);
1409 40298632 : bool rhs_free = false;
1410 40298632 : bool lhs_free = false;
1411 :
1412 40298632 : if (!inner_rhs)
1413 0 : inner_rhs = rhs;
1414 40298632 : if (!inner_lhs)
1415 0 : inner_lhs = lhs;
1416 :
1417 : /* Reads of parameter are expected to be free. */
1418 40298632 : if (unmodified_parm (fbi, stmt, inner_rhs, NULL))
1419 : rhs_free = true;
1420 : /* Match expressions of form &this->field. Those will most likely
1421 : combine with something upstream after inlining. */
1422 38982221 : else if (TREE_CODE (inner_rhs) == ADDR_EXPR)
1423 : {
1424 2589068 : tree op = get_base_address (TREE_OPERAND (inner_rhs, 0));
1425 2589068 : if (TREE_CODE (op) == PARM_DECL)
1426 : rhs_free = true;
1427 2544958 : else if (TREE_CODE (op) == MEM_REF
1428 2544958 : && unmodified_parm (fbi, stmt, TREE_OPERAND (op, 0),
1429 : NULL))
1430 : rhs_free = true;
1431 : }
1432 :
1433 : /* When parameter is not SSA register because its address is taken
1434 : and it is just copied into one, the statement will be completely
1435 : free after inlining (we will copy propagate backward). */
1436 1360521 : if (rhs_free && is_gimple_reg (lhs))
1437 : return 2;
1438 :
1439 : /* Reads of parameters passed by reference
1440 : expected to be free (i.e. optimized out after inlining). */
1441 39610657 : if (TREE_CODE (inner_rhs) == MEM_REF
1442 39610657 : && unmodified_parm (fbi, stmt, TREE_OPERAND (inner_rhs, 0), NULL))
1443 : rhs_free = true;
1444 :
1445 : /* Copying parameter passed by reference into gimple register is
1446 : probably also going to copy propagate, but we can't be quite
1447 : sure. */
1448 39610657 : if (rhs_free && is_gimple_reg (lhs))
1449 : lhs_free = true;
1450 :
1451 : /* Writes to parameters, parameters passed by value and return value
1452 : (either directly or passed via invisible reference) are free.
1453 :
1454 : TODO: We ought to handle testcase like
1455 : struct a {int a,b;};
1456 : struct a
1457 : returnstruct (void)
1458 : {
1459 : struct a a ={1,2};
1460 : return a;
1461 : }
1462 :
1463 : This translate into:
1464 :
1465 : returnstruct ()
1466 : {
1467 : int a$b;
1468 : int a$a;
1469 : struct a a;
1470 : struct a D.2739;
1471 :
1472 : <bb 2>:
1473 : D.2739.a = 1;
1474 : D.2739.b = 2;
1475 : return D.2739;
1476 :
1477 : }
1478 : For that we either need to copy ipa-split logic detecting writes
1479 : to return value. */
1480 39610657 : if (TREE_CODE (inner_lhs) == PARM_DECL
1481 39520998 : || TREE_CODE (inner_lhs) == RESULT_DECL
1482 78283105 : || (TREE_CODE (inner_lhs) == MEM_REF
1483 5209625 : && (unmodified_parm (fbi, stmt, TREE_OPERAND (inner_lhs, 0),
1484 : NULL)
1485 2692445 : || (TREE_CODE (TREE_OPERAND (inner_lhs, 0)) == SSA_NAME
1486 2691247 : && SSA_NAME_VAR (TREE_OPERAND (inner_lhs, 0))
1487 378745 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1488 : (inner_lhs,
1489 : 0))) == RESULT_DECL))))
1490 : lhs_free = true;
1491 36067362 : if (lhs_free
1492 39610657 : && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs)))
1493 : rhs_free = true;
1494 39610657 : if (lhs_free && rhs_free)
1495 8229825 : return 1;
1496 : }
1497 : return 0;
1498 : default:
1499 : return 0;
1500 : }
1501 : }
1502 :
1503 : /* Analyze EXPR if it represents a series of simple operations performed on
1504 : a function parameter and return true if so. FBI, STMT, EXPR, INDEX_P and
1505 : AGGPOS have the same meaning like in unmodified_parm_or_parm_agg_item.
1506 : Type of the parameter or load from an aggregate via the parameter is
1507 : stored in *TYPE_P. Operations on the parameter are recorded to
1508 : PARAM_OPS_P if it is not NULL. */
1509 :
1510 : static bool
1511 28022724 : decompose_param_expr (struct ipa_func_body_info *fbi,
1512 : gimple *stmt, tree expr,
1513 : int *index_p, tree *type_p,
1514 : struct agg_position_info *aggpos,
1515 : expr_eval_ops *param_ops_p = NULL)
1516 : {
1517 28022724 : int op_limit = opt_for_fn (fbi->node->decl, param_ipa_max_param_expr_ops);
1518 28022724 : int op_count = 0;
1519 :
1520 28022724 : if (param_ops_p)
1521 9607774 : *param_ops_p = NULL;
1522 :
1523 34162100 : while (true)
1524 : {
1525 34162100 : expr_eval_op eval_op;
1526 34162100 : unsigned rhs_count;
1527 34162100 : unsigned cst_count = 0;
1528 :
1529 34162100 : if (unmodified_parm_or_parm_agg_item (fbi, stmt, expr, index_p, NULL,
1530 : aggpos))
1531 : {
1532 4736180 : tree type = TREE_TYPE (expr);
1533 :
1534 4736180 : if (aggpos->agg_contents)
1535 : {
1536 : /* Stop if containing bit-field. */
1537 3886914 : if (TREE_CODE (expr) == BIT_FIELD_REF
1538 3886914 : || contains_bitfld_component_ref_p (expr))
1539 : break;
1540 : }
1541 :
1542 4708687 : *type_p = type;
1543 4708687 : return true;
1544 : }
1545 :
1546 29425920 : if (TREE_CODE (expr) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (expr))
1547 : break;
1548 11070801 : stmt = SSA_NAME_DEF_STMT (expr);
1549 :
1550 11070801 : if (gcall *call = dyn_cast <gcall *> (stmt))
1551 : {
1552 2598465 : int flags = gimple_call_return_flags (call);
1553 2598465 : if (!(flags & ERF_RETURNS_ARG))
1554 3192517 : goto fail;
1555 231555 : int arg = flags & ERF_RETURN_ARG_MASK;
1556 231555 : if (arg >= (int)gimple_call_num_args (call))
1557 0 : goto fail;
1558 231555 : expr = gimple_call_arg (stmt, arg);
1559 4197963 : continue;
1560 231555 : }
1561 :
1562 8472336 : if (!is_gimple_assign (stmt = SSA_NAME_DEF_STMT (expr)))
1563 : break;
1564 :
1565 6733516 : switch (gimple_assign_rhs_class (stmt))
1566 : {
1567 3966408 : case GIMPLE_SINGLE_RHS:
1568 3966408 : expr = gimple_assign_rhs1 (stmt);
1569 3966408 : continue;
1570 :
1571 : case GIMPLE_UNARY_RHS:
1572 : rhs_count = 1;
1573 : break;
1574 :
1575 1679378 : case GIMPLE_BINARY_RHS:
1576 1679378 : rhs_count = 2;
1577 1679378 : break;
1578 :
1579 696 : case GIMPLE_TERNARY_RHS:
1580 696 : rhs_count = 3;
1581 696 : break;
1582 :
1583 0 : default:
1584 0 : goto fail;
1585 : }
1586 :
1587 : /* Stop if expression is too complex. */
1588 2767108 : if (op_count++ == op_limit)
1589 : break;
1590 :
1591 2767020 : if (param_ops_p)
1592 : {
1593 2732723 : eval_op.code = gimple_assign_rhs_code (stmt);
1594 2732723 : eval_op.type = TREE_TYPE (gimple_assign_lhs (stmt));
1595 2732723 : eval_op.val[0] = NULL_TREE;
1596 2732723 : eval_op.val[1] = NULL_TREE;
1597 : }
1598 :
1599 2767020 : expr = NULL_TREE;
1600 6388512 : for (unsigned i = 0; i < rhs_count; i++)
1601 : {
1602 4447099 : tree op = gimple_op (stmt, i + 1);
1603 :
1604 4447099 : gcc_assert (op && !TYPE_P (op));
1605 4447099 : if (is_gimple_ip_invariant (op))
1606 : {
1607 857214 : if (++cst_count == rhs_count)
1608 1371 : goto fail;
1609 :
1610 855843 : eval_op.val[cst_count - 1] = op;
1611 : }
1612 3589885 : else if (!expr)
1613 : {
1614 : /* Found a non-constant operand, and record its index in rhs
1615 : operands. */
1616 2765649 : eval_op.index = i;
1617 2765649 : expr = op;
1618 : }
1619 : else
1620 : {
1621 : /* Found more than one non-constant operands. */
1622 824236 : goto fail;
1623 : }
1624 : }
1625 :
1626 1941413 : if (param_ops_p)
1627 1907736 : vec_safe_insert (*param_ops_p, 0, eval_op);
1628 : }
1629 :
1630 : /* Failed to decompose, free resource and return. */
1631 23314037 : fail:
1632 23314037 : if (param_ops_p)
1633 9433954 : vec_free (*param_ops_p);
1634 :
1635 : return false;
1636 : }
1637 :
1638 : /* Record to SUMMARY that PARM is used by builtin_constant_p. */
1639 :
1640 : static void
1641 12085 : add_builtin_constant_p_parm (class ipa_fn_summary *summary, int parm)
1642 : {
1643 12085 : int ip;
1644 :
1645 : /* Avoid duplicates. */
1646 12143 : for (unsigned int i = 0;
1647 12143 : summary->builtin_constant_p_parms.iterate (i, &ip); i++)
1648 5987 : if (ip == parm)
1649 12085 : return;
1650 6156 : summary->builtin_constant_p_parms.safe_push (parm);
1651 : }
1652 :
1653 : /* If BB ends by a conditional we can turn into predicates, attach corresponding
1654 : predicates to the CFG edges. */
1655 :
1656 : static void
1657 36155811 : set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
1658 : class ipa_fn_summary *summary,
1659 : class ipa_node_params *params_summary,
1660 : basic_block bb)
1661 : {
1662 36155811 : tree op, op2;
1663 36155811 : int index;
1664 36155811 : struct agg_position_info aggpos;
1665 36155811 : enum tree_code code, inverted_code;
1666 36155811 : edge e;
1667 36155811 : edge_iterator ei;
1668 36155811 : gimple *set_stmt;
1669 36155811 : tree param_type;
1670 36155811 : expr_eval_ops param_ops;
1671 :
1672 72311622 : gcond *last = safe_dyn_cast <gcond *> (*gsi_last_bb (bb));
1673 11986371 : if (!last)
1674 36142729 : return;
1675 11986371 : if (!is_gimple_ip_invariant (gimple_cond_rhs (last)))
1676 : return;
1677 9524196 : op = gimple_cond_lhs (last);
1678 :
1679 9524196 : if (decompose_param_expr (fbi, last, op, &index, ¶m_type, &aggpos,
1680 : ¶m_ops))
1681 : {
1682 1351686 : code = gimple_cond_code (last);
1683 1351686 : inverted_code = invert_tree_comparison (code, HONOR_NANS (op));
1684 :
1685 4055058 : FOR_EACH_EDGE (e, ei, bb->succs)
1686 : {
1687 1351686 : enum tree_code this_code = (e->flags & EDGE_TRUE_VALUE
1688 2703372 : ? code : inverted_code);
1689 : /* invert_tree_comparison will return ERROR_MARK on FP
1690 : comparisons that are not EQ/NE instead of returning proper
1691 : unordered one. Be sure it is not confused with NON_CONSTANT.
1692 :
1693 : And if the edge's target is the final block of diamond CFG graph
1694 : of this conditional statement, we do not need to compute
1695 : predicate for the edge because the final block's predicate must
1696 : be at least as that of the first block of the statement. */
1697 2703372 : if (this_code != ERROR_MARK
1698 2703372 : && !dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1699 : {
1700 2300602 : ipa_predicate p
1701 2300602 : = add_condition (summary, params_summary, index,
1702 : param_type, &aggpos,
1703 : this_code, gimple_cond_rhs (last), param_ops);
1704 2300602 : e->aux = edge_predicate_pool.allocate ();
1705 2300602 : *(ipa_predicate *) e->aux = p;
1706 : }
1707 : }
1708 36371085 : vec_free (param_ops);
1709 : return;
1710 : }
1711 :
1712 8172510 : if (TREE_CODE (op) != SSA_NAME)
1713 : return;
1714 : /* Special case
1715 : if (builtin_constant_p (op))
1716 : constant_code
1717 : else
1718 : nonconstant_code.
1719 : Here we can predicate nonconstant_code. We can't
1720 : really handle constant_code since we have no predicate
1721 : for this and also the constant code is not known to be
1722 : optimized away when inliner doesn't see operand is constant.
1723 : Other optimizers might think otherwise. */
1724 8171425 : if (gimple_cond_code (last) != NE_EXPR
1725 8171425 : || !integer_zerop (gimple_cond_rhs (last)))
1726 : return;
1727 3600173 : set_stmt = SSA_NAME_DEF_STMT (op);
1728 3600173 : if (!gimple_call_builtin_p (set_stmt, BUILT_IN_CONSTANT_P)
1729 3600173 : || gimple_call_num_args (set_stmt) != 1)
1730 : return;
1731 46462 : op2 = gimple_call_arg (set_stmt, 0);
1732 46462 : if (!decompose_param_expr (fbi, set_stmt, op2, &index, ¶m_type, &aggpos))
1733 : return;
1734 13082 : if (!aggpos.by_ref)
1735 12023 : add_builtin_constant_p_parm (summary, index);
1736 39246 : FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
1737 : {
1738 13082 : ipa_predicate p = add_condition (summary, params_summary, index,
1739 : param_type, &aggpos,
1740 : ipa_predicate::is_not_constant, NULL_TREE);
1741 13082 : e->aux = edge_predicate_pool.allocate ();
1742 13082 : *(ipa_predicate *) e->aux = p;
1743 : }
1744 : }
1745 :
1746 :
1747 : /* If BB ends by a switch we can turn into predicates, attach corresponding
1748 : predicates to the CFG edges. */
1749 :
1750 : static void
1751 36155811 : set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
1752 : class ipa_fn_summary *summary,
1753 : class ipa_node_params *params_summary,
1754 : basic_block bb)
1755 : {
1756 36155811 : tree op;
1757 36155811 : int index;
1758 36155811 : struct agg_position_info aggpos;
1759 36155811 : edge e;
1760 36155811 : edge_iterator ei;
1761 36155811 : size_t n;
1762 36155811 : size_t case_idx;
1763 36155811 : tree param_type;
1764 36155811 : expr_eval_ops param_ops;
1765 :
1766 72311622 : gswitch *last = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb));
1767 83578 : if (!last)
1768 36126071 : return;
1769 83578 : op = gimple_switch_index (last);
1770 83578 : if (!decompose_param_expr (fbi, last, op, &index, ¶m_type, &aggpos,
1771 : ¶m_ops))
1772 : return;
1773 :
1774 44966 : auto_vec<std::pair<tree, tree> > ranges;
1775 44966 : tree type = TREE_TYPE (op);
1776 44966 : int bound_limit = opt_for_fn (fbi->node->decl,
1777 : param_ipa_max_switch_predicate_bounds);
1778 44966 : int bound_count = 0;
1779 : // This can safely be an integer range, as switches can only hold
1780 : // integers.
1781 44966 : int_range<2> vr;
1782 :
1783 89932 : get_range_query (cfun)->range_of_expr (vr, op);
1784 44966 : if (vr.undefined_p ())
1785 0 : vr.set_varying (TREE_TYPE (op));
1786 44966 : tree vr_min, vr_max;
1787 : // TODO: This entire function could use a rewrite to use the irange
1788 : // API, instead of trying to recreate its intersection/union logic.
1789 : // Any use of get_legacy_range() is a serious code smell.
1790 44966 : value_range_kind vr_type = get_legacy_range (vr, vr_min, vr_max);
1791 44966 : wide_int vr_wmin = wi::to_wide (vr_min);
1792 44966 : wide_int vr_wmax = wi::to_wide (vr_max);
1793 :
1794 311620 : FOR_EACH_EDGE (e, ei, bb->succs)
1795 : {
1796 266654 : e->aux = edge_predicate_pool.allocate ();
1797 266654 : *(ipa_predicate *) e->aux = false;
1798 : }
1799 :
1800 44966 : e = gimple_switch_edge (cfun, last, 0);
1801 : /* Set BOUND_COUNT to maximum count to bypass computing predicate for
1802 : default case if its target basic block is in convergence point of all
1803 : switch cases, which can be determined by checking whether it
1804 : post-dominates the switch statement. */
1805 44966 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1806 12658 : bound_count = INT_MAX;
1807 :
1808 44966 : n = gimple_switch_num_labels (last);
1809 297923 : for (case_idx = 1; case_idx < n; ++case_idx)
1810 : {
1811 252957 : tree cl = gimple_switch_label (last, case_idx);
1812 252957 : tree min = CASE_LOW (cl);
1813 252957 : tree max = CASE_HIGH (cl);
1814 252957 : ipa_predicate p;
1815 :
1816 252957 : e = gimple_switch_edge (cfun, last, case_idx);
1817 :
1818 : /* The case value might not have same type as switch expression,
1819 : extend the value based on the expression type. */
1820 252957 : if (TREE_TYPE (min) != type)
1821 45173 : min = wide_int_to_tree (type, wi::to_widest (min));
1822 :
1823 252957 : if (!max)
1824 : max = min;
1825 12691 : else if (TREE_TYPE (max) != type)
1826 3522 : max = wide_int_to_tree (type, wi::to_widest (max));
1827 :
1828 : /* The case's target basic block is in convergence point of all switch
1829 : cases, its predicate should be at least as that of the switch
1830 : statement. */
1831 252957 : if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1832 5229 : p = true;
1833 247728 : else if (min == max)
1834 236977 : p = add_condition (summary, params_summary, index, param_type,
1835 : &aggpos, EQ_EXPR, min, param_ops);
1836 : else
1837 : {
1838 10751 : ipa_predicate p1, p2;
1839 10751 : p1 = add_condition (summary, params_summary, index, param_type,
1840 : &aggpos, GE_EXPR, min, param_ops);
1841 10751 : p2 = add_condition (summary, params_summary,index, param_type,
1842 : &aggpos, LE_EXPR, max, param_ops);
1843 10751 : p = p1 & p2;
1844 : }
1845 252957 : *(ipa_predicate *) e->aux
1846 252957 : = p.or_with (summary->conds, *(ipa_predicate *) e->aux);
1847 :
1848 : /* If there are too many disjoint case ranges, predicate for default
1849 : case might become too complicated. So add a limit here. */
1850 252957 : if (bound_count > bound_limit)
1851 98136 : continue;
1852 :
1853 154821 : bool new_range = true;
1854 :
1855 154821 : if (!ranges.is_empty ())
1856 : {
1857 122513 : wide_int curr_wmin = wi::to_wide (min);
1858 122513 : wide_int last_wmax = wi::to_wide (ranges.last ().second);
1859 :
1860 : /* Merge case ranges if they are continuous. */
1861 122513 : if (curr_wmin == last_wmax + 1)
1862 : new_range = false;
1863 42314 : else if (vr_type == VR_ANTI_RANGE)
1864 : {
1865 : /* If two disjoint case ranges can be connected by anti-range
1866 : of switch index, combine them to one range. */
1867 141 : if (wi::lt_p (vr_wmax, curr_wmin - 1, TYPE_SIGN (type)))
1868 : vr_type = VR_UNDEFINED;
1869 0 : else if (wi::le_p (vr_wmin, last_wmax + 1, TYPE_SIGN (type)))
1870 0 : new_range = false;
1871 : }
1872 122513 : }
1873 :
1874 : /* Create/extend a case range. And we count endpoints of range set,
1875 : this number nearly equals to number of conditions that we will create
1876 : for predicate of default case. */
1877 122513 : if (new_range)
1878 : {
1879 74622 : bound_count += (min == max) ? 1 : 2;
1880 74622 : ranges.safe_push (std::make_pair (min, max));
1881 : }
1882 : else
1883 : {
1884 80199 : bound_count += (ranges.last ().first == ranges.last ().second);
1885 80199 : ranges.last ().second = max;
1886 : }
1887 : }
1888 :
1889 44966 : e = gimple_switch_edge (cfun, last, 0);
1890 44966 : if (bound_count > bound_limit)
1891 : {
1892 15226 : *(ipa_predicate *) e->aux = true;
1893 15226 : vec_free (param_ops);
1894 15226 : return;
1895 : }
1896 :
1897 29740 : ipa_predicate p_seg = true;
1898 29740 : ipa_predicate p_all = false;
1899 :
1900 29740 : if (vr_type != VR_RANGE)
1901 : {
1902 29140 : vr_wmin = wi::to_wide (TYPE_MIN_VALUE (type));
1903 29140 : vr_wmax = wi::to_wide (TYPE_MAX_VALUE (type));
1904 : }
1905 :
1906 : /* Construct predicate to represent default range set that is negation of
1907 : all case ranges. Case range is classified as containing single/non-single
1908 : values. Suppose a piece of case ranges in the following.
1909 :
1910 : [D1...D2] [S1] ... [Sn] [D3...D4]
1911 :
1912 : To represent default case's range sets between two non-single value
1913 : case ranges (From D2 to D3), we construct predicate as:
1914 :
1915 : D2 < x < D3 && x != S1 && ... && x != Sn
1916 : */
1917 91716 : for (size_t i = 0; i < ranges.length (); i++)
1918 : {
1919 62073 : tree min = ranges[i].first;
1920 62073 : tree max = ranges[i].second;
1921 :
1922 62073 : if (min == max)
1923 73902 : p_seg &= add_condition (summary, params_summary, index,
1924 : param_type, &aggpos, NE_EXPR,
1925 36951 : min, param_ops);
1926 : else
1927 : {
1928 : /* Do not create sub-predicate for range that is beyond low bound
1929 : of switch index. */
1930 25122 : if (wi::lt_p (vr_wmin, wi::to_wide (min), TYPE_SIGN (type)))
1931 : {
1932 18012 : p_seg &= add_condition (summary, params_summary, index,
1933 : param_type, &aggpos,
1934 18012 : LT_EXPR, min, param_ops);
1935 18012 : p_all = p_all.or_with (summary->conds, p_seg);
1936 : }
1937 :
1938 : /* Do not create sub-predicate for range that is beyond up bound
1939 : of switch index. */
1940 25122 : if (wi::le_p (vr_wmax, wi::to_wide (max), TYPE_SIGN (type)))
1941 : {
1942 97 : p_seg = false;
1943 97 : break;
1944 : }
1945 :
1946 25025 : p_seg = add_condition (summary, params_summary, index,
1947 : param_type, &aggpos, GT_EXPR,
1948 : max, param_ops);
1949 : }
1950 : }
1951 :
1952 29740 : p_all = p_all.or_with (summary->conds, p_seg);
1953 29740 : *(ipa_predicate *) e->aux
1954 29740 : = p_all.or_with (summary->conds, *(ipa_predicate *) e->aux);
1955 :
1956 36333 : vec_free (param_ops);
1957 44966 : }
1958 :
1959 :
1960 : /* For each BB in NODE attach to its AUX pointer predicate under
1961 : which it is executable. */
1962 :
1963 : static void
1964 6517766 : compute_bb_predicates (struct ipa_func_body_info *fbi,
1965 : struct cgraph_node *node,
1966 : class ipa_fn_summary *summary,
1967 : class ipa_node_params *params_summary)
1968 : {
1969 6517766 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
1970 6517766 : bool done = false;
1971 6517766 : basic_block bb;
1972 :
1973 42673577 : FOR_EACH_BB_FN (bb, my_function)
1974 : {
1975 36155811 : set_cond_stmt_execution_predicate (fbi, summary, params_summary, bb);
1976 36155811 : set_switch_stmt_execution_predicate (fbi, summary, params_summary, bb);
1977 : }
1978 :
1979 : /* Entry block is always executable. */
1980 6517766 : ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
1981 6517766 : = edge_predicate_pool.allocate ();
1982 6517766 : *(ipa_predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux = true;
1983 :
1984 : /* A simple dataflow propagation of predicates forward in the CFG.
1985 : TODO: work in reverse postorder. */
1986 19678448 : while (!done)
1987 : {
1988 13160682 : done = true;
1989 90850511 : FOR_EACH_BB_FN (bb, my_function)
1990 : {
1991 77689829 : ipa_predicate p = false;
1992 77689829 : edge e;
1993 77689829 : edge_iterator ei;
1994 96214953 : FOR_EACH_EDGE (e, ei, bb->preds)
1995 : {
1996 82570356 : if (e->src->aux)
1997 : {
1998 80663836 : ipa_predicate this_bb_predicate
1999 : = *(ipa_predicate *) e->src->aux;
2000 80663836 : if (e->aux)
2001 5367678 : this_bb_predicate &= (*(ipa_predicate *) e->aux);
2002 80663836 : p = p.or_with (summary->conds, this_bb_predicate);
2003 80663836 : if (p == true)
2004 : break;
2005 : }
2006 : }
2007 77689829 : if (p != false)
2008 : {
2009 76423338 : basic_block pdom_bb;
2010 :
2011 76423338 : if (!bb->aux)
2012 : {
2013 28399642 : done = false;
2014 28399642 : bb->aux = edge_predicate_pool.allocate ();
2015 28399642 : *((ipa_predicate *) bb->aux) = p;
2016 : }
2017 48023696 : else if (p != *(ipa_predicate *) bb->aux)
2018 : {
2019 : /* This OR operation is needed to ensure monotonous data flow
2020 : in the case we hit the limit on number of clauses and the
2021 : and/or operations above give approximate answers. */
2022 128262 : p = p.or_with (summary->conds, *(ipa_predicate *)bb->aux);
2023 128262 : if (p != *(ipa_predicate *)bb->aux)
2024 : {
2025 103477 : done = false;
2026 103477 : *((ipa_predicate *)bb->aux) = p;
2027 : }
2028 : }
2029 :
2030 : /* For switch/if statement, we can OR-combine predicates of all
2031 : its cases/branches to get predicate for basic block in their
2032 : convergence point, but sometimes this will generate very
2033 : complicated predicate. Actually, we can get simplified
2034 : predicate in another way by using the fact that predicate
2035 : for a basic block must also hold true for its post dominators.
2036 : To be specific, basic block in convergence point of
2037 : conditional statement should include predicate of the
2038 : statement. */
2039 76423338 : pdom_bb = get_immediate_dominator (CDI_POST_DOMINATORS, bb);
2040 76423338 : if (pdom_bb == EXIT_BLOCK_PTR_FOR_FN (my_function) || !pdom_bb)
2041 : ;
2042 39199082 : else if (!pdom_bb->aux)
2043 : {
2044 7756153 : done = false;
2045 7756153 : pdom_bb->aux = edge_predicate_pool.allocate ();
2046 7756153 : *((ipa_predicate *)pdom_bb->aux) = p;
2047 : }
2048 31442929 : else if (p != *(ipa_predicate *)pdom_bb->aux)
2049 : {
2050 3436261 : p = p.or_with (summary->conds,
2051 : *(ipa_predicate *)pdom_bb->aux);
2052 3436261 : if (p != *(ipa_predicate *)pdom_bb->aux)
2053 : {
2054 140980 : done = false;
2055 140980 : *((ipa_predicate *)pdom_bb->aux) = p;
2056 : }
2057 : }
2058 : }
2059 : }
2060 : }
2061 6517766 : }
2062 :
2063 :
2064 : /* Return predicate specifying when the STMT might have result that is not
2065 : a compile time constant. */
2066 :
2067 : static ipa_predicate
2068 4302669 : will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
2069 : class ipa_fn_summary *summary,
2070 : class ipa_node_params *params_summary,
2071 : tree expr,
2072 : vec<ipa_predicate> nonconstant_names)
2073 : {
2074 4302669 : tree parm;
2075 4302669 : int index;
2076 :
2077 4551444 : while (UNARY_CLASS_P (expr))
2078 248775 : expr = TREE_OPERAND (expr, 0);
2079 :
2080 4302669 : parm = unmodified_parm (fbi, NULL, expr, NULL);
2081 4302669 : if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
2082 222390 : return add_condition (summary, params_summary, index, TREE_TYPE (parm), NULL,
2083 222390 : ipa_predicate::changed, NULL_TREE);
2084 4080279 : if (is_gimple_min_invariant (expr))
2085 64369 : return false;
2086 4015910 : if (TREE_CODE (expr) == SSA_NAME)
2087 3832269 : return nonconstant_names[SSA_NAME_VERSION (expr)];
2088 183641 : if (BINARY_CLASS_P (expr) || COMPARISON_CLASS_P (expr))
2089 : {
2090 183039 : ipa_predicate p1
2091 183039 : = will_be_nonconstant_expr_predicate (fbi, summary,
2092 : params_summary,
2093 183039 : TREE_OPERAND (expr, 0),
2094 : nonconstant_names);
2095 183039 : if (p1 == true)
2096 105073 : return p1;
2097 :
2098 77966 : ipa_predicate p2
2099 77966 : = will_be_nonconstant_expr_predicate (fbi, summary,
2100 : params_summary,
2101 77966 : TREE_OPERAND (expr, 1),
2102 : nonconstant_names);
2103 77966 : return p1.or_with (summary->conds, p2);
2104 : }
2105 602 : else if (TREE_CODE (expr) == COND_EXPR)
2106 : {
2107 89 : ipa_predicate p1
2108 89 : = will_be_nonconstant_expr_predicate (fbi, summary,
2109 : params_summary,
2110 89 : TREE_OPERAND (expr, 0),
2111 : nonconstant_names);
2112 89 : if (p1 == true)
2113 27 : return p1;
2114 :
2115 62 : ipa_predicate p2
2116 62 : = will_be_nonconstant_expr_predicate (fbi, summary,
2117 : params_summary,
2118 62 : TREE_OPERAND (expr, 1),
2119 : nonconstant_names);
2120 62 : if (p2 == true)
2121 62 : return p2;
2122 0 : p1 = p1.or_with (summary->conds, p2);
2123 0 : p2 = will_be_nonconstant_expr_predicate (fbi, summary,
2124 : params_summary,
2125 0 : TREE_OPERAND (expr, 2),
2126 : nonconstant_names);
2127 0 : return p2.or_with (summary->conds, p1);
2128 : }
2129 513 : else if (TREE_CODE (expr) == CALL_EXPR)
2130 513 : return true;
2131 : else
2132 : {
2133 0 : debug_tree (expr);
2134 0 : gcc_unreachable ();
2135 : }
2136 : }
2137 :
2138 :
2139 : /* Return predicate specifying when the STMT might have result that is not
2140 : a compile time constant. */
2141 :
2142 : static ipa_predicate
2143 117712834 : will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
2144 : class ipa_fn_summary *summary,
2145 : class ipa_node_params *params_summary,
2146 : gimple *stmt,
2147 : vec<ipa_predicate> nonconstant_names)
2148 : {
2149 117712834 : ipa_predicate p = true;
2150 117712834 : ssa_op_iter iter;
2151 117712834 : tree use;
2152 117712834 : tree param_type = NULL_TREE;
2153 117712834 : ipa_predicate op_non_const;
2154 117712834 : bool is_load;
2155 117712834 : int base_index;
2156 117712834 : struct agg_position_info aggpos;
2157 :
2158 : /* What statements might be optimized away
2159 : when their arguments are constant. */
2160 117712834 : if (gimple_code (stmt) != GIMPLE_ASSIGN
2161 : && gimple_code (stmt) != GIMPLE_COND
2162 : && gimple_code (stmt) != GIMPLE_SWITCH
2163 : && (gimple_code (stmt) != GIMPLE_CALL
2164 21478764 : || !(gimple_call_flags (stmt) & ECF_CONST)))
2165 26872855 : return p;
2166 :
2167 : /* Stores will stay anyway. */
2168 90839979 : if (gimple_store_p (stmt))
2169 26790466 : return p;
2170 :
2171 64049513 : is_load = gimple_assign_load_p (stmt);
2172 :
2173 : /* Loads can be optimized when the value is known. */
2174 64049513 : if (is_load)
2175 : {
2176 18368488 : tree op = gimple_assign_rhs1 (stmt);
2177 18368488 : if (!decompose_param_expr (fbi, stmt, op, &base_index, ¶m_type,
2178 : &aggpos))
2179 15069535 : return p;
2180 : }
2181 : else
2182 45681025 : base_index = -1;
2183 :
2184 : /* See if we understand all operands before we start
2185 : adding conditionals. */
2186 66726533 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2187 : {
2188 48811163 : tree parm = unmodified_parm (fbi, stmt, use, NULL);
2189 : /* For arguments we can build a condition. */
2190 48811163 : if (parm && ipa_get_param_decl_index (fbi->info, parm) >= 0)
2191 8664044 : continue;
2192 40147119 : if (TREE_CODE (use) != SSA_NAME)
2193 0 : return p;
2194 : /* If we know when operand is constant,
2195 : we still can say something useful. */
2196 40147119 : if (nonconstant_names[SSA_NAME_VERSION (use)] != true)
2197 9082511 : continue;
2198 31064608 : return p;
2199 : }
2200 :
2201 17915370 : if (is_load)
2202 3298872 : op_non_const =
2203 3298872 : add_condition (summary, params_summary,
2204 : base_index, param_type, &aggpos,
2205 : ipa_predicate::changed, NULL_TREE);
2206 : else
2207 14616498 : op_non_const = false;
2208 34613195 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2209 : {
2210 16697825 : tree parm = unmodified_parm (fbi, stmt, use, NULL);
2211 16697825 : int index;
2212 :
2213 16697825 : if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
2214 : {
2215 8165933 : if (index != base_index)
2216 5575651 : p = add_condition (summary, params_summary, index,
2217 5575651 : TREE_TYPE (parm), NULL,
2218 : ipa_predicate::changed, NULL_TREE);
2219 : else
2220 2590282 : continue;
2221 : }
2222 : else
2223 8531892 : p = nonconstant_names[SSA_NAME_VERSION (use)];
2224 14107543 : op_non_const = p.or_with (summary->conds, op_non_const);
2225 : }
2226 17915370 : if ((gimple_code (stmt) == GIMPLE_ASSIGN || gimple_code (stmt) == GIMPLE_CALL)
2227 15571717 : && gimple_op (stmt, 0)
2228 33190345 : && TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
2229 15274975 : nonconstant_names[SSA_NAME_VERSION (gimple_op (stmt, 0))]
2230 15274975 : = op_non_const;
2231 17915370 : return op_non_const;
2232 : }
2233 :
2234 : struct record_modified_bb_info
2235 : {
2236 : tree op;
2237 : bitmap bb_set;
2238 : gimple *stmt;
2239 : };
2240 :
2241 : /* Value is initialized in INIT_BB and used in USE_BB. We want to compute
2242 : probability how often it changes between USE_BB.
2243 : INIT_BB->count/USE_BB->count is an estimate, but if INIT_BB
2244 : is in different loop nest, we can do better.
2245 : This is all just estimate. In theory we look for minimal cut separating
2246 : INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
2247 : anyway. */
2248 :
2249 : static basic_block
2250 10705206 : get_minimal_bb (basic_block init_bb, basic_block use_bb)
2251 : {
2252 10705206 : class loop *l = find_common_loop (init_bb->loop_father, use_bb->loop_father);
2253 10705206 : if (l && l->header->count < init_bb->count)
2254 386292 : return l->header;
2255 : return init_bb;
2256 : }
2257 :
2258 : /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2259 : set except for info->stmt. */
2260 :
2261 : static bool
2262 4954181 : record_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef, void *data)
2263 : {
2264 4954181 : struct record_modified_bb_info *info =
2265 : (struct record_modified_bb_info *) data;
2266 4954181 : if (SSA_NAME_DEF_STMT (vdef) == info->stmt)
2267 : return false;
2268 4926074 : if (gimple_clobber_p (SSA_NAME_DEF_STMT (vdef)))
2269 : return false;
2270 4876922 : bitmap_set_bit (info->bb_set,
2271 4876922 : SSA_NAME_IS_DEFAULT_DEF (vdef)
2272 0 : ? ENTRY_BLOCK_PTR_FOR_FN (cfun)->index
2273 : : get_minimal_bb
2274 4876922 : (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2275 4876922 : gimple_bb (info->stmt))->index);
2276 4876922 : if (dump_file)
2277 : {
2278 0 : fprintf (dump_file, " Param ");
2279 0 : print_generic_expr (dump_file, info->op, TDF_SLIM);
2280 0 : fprintf (dump_file, " changed at bb %i, minimal: %i stmt: ",
2281 0 : gimple_bb (SSA_NAME_DEF_STMT (vdef))->index,
2282 : get_minimal_bb
2283 0 : (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2284 0 : gimple_bb (info->stmt))->index);
2285 0 : print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (vdef), 0);
2286 : }
2287 : return false;
2288 : }
2289 :
2290 : /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2291 : will change since last invocation of STMT.
2292 :
2293 : Value 0 is reserved for compile time invariants.
2294 : For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2295 : ought to be REG_BR_PROB_BASE / estimated_iters. */
2296 :
2297 : static int
2298 39808244 : param_change_prob (ipa_func_body_info *fbi, gimple *stmt, int i)
2299 : {
2300 39808244 : tree op = gimple_call_arg (stmt, i);
2301 39808244 : basic_block bb = gimple_bb (stmt);
2302 :
2303 39808244 : if (TREE_CODE (op) == WITH_SIZE_EXPR)
2304 299 : op = TREE_OPERAND (op, 0);
2305 :
2306 39808244 : tree base = get_base_address (op);
2307 :
2308 : /* Global invariants never change. */
2309 39808244 : if (is_gimple_min_invariant (base))
2310 : return 0;
2311 :
2312 : /* We would have to do non-trivial analysis to really work out what
2313 : is the probability of value to change (i.e. when init statement
2314 : is in a sibling loop of the call).
2315 :
2316 : We do an conservative estimate: when call is executed N times more often
2317 : than the statement defining value, we take the frequency 1/N. */
2318 19924240 : if (TREE_CODE (base) == SSA_NAME)
2319 : {
2320 17511040 : profile_count init_count;
2321 :
2322 26870259 : if (!bb->count.nonzero_p ())
2323 : return REG_BR_PROB_BASE;
2324 :
2325 8932553 : if (SSA_NAME_IS_DEFAULT_DEF (base))
2326 3104269 : init_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2327 : else
2328 5828284 : init_count = get_minimal_bb
2329 5828284 : (gimple_bb (SSA_NAME_DEF_STMT (base)),
2330 : gimple_bb (stmt))->count;
2331 :
2332 8932553 : if (init_count < bb->count)
2333 466787 : return MAX ((init_count.to_sreal_scale (bb->count)
2334 : * REG_BR_PROB_BASE).to_int (), 1);
2335 : return REG_BR_PROB_BASE;
2336 : }
2337 : else
2338 : {
2339 2413200 : ao_ref refd;
2340 2413200 : profile_count max = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2341 2413200 : struct record_modified_bb_info info;
2342 2413200 : tree init = ctor_for_folding (base);
2343 :
2344 2413200 : if (init != error_mark_node)
2345 : return 0;
2346 5386314 : if (!bb->count.nonzero_p () || fbi->aa_walk_budget == 0)
2347 : return REG_BR_PROB_BASE;
2348 1474298 : if (dump_file)
2349 : {
2350 0 : fprintf (dump_file, " Analyzing param change probability of ");
2351 0 : print_generic_expr (dump_file, op, TDF_SLIM);
2352 0 : fprintf (dump_file, "\n");
2353 : }
2354 1474298 : ao_ref_init (&refd, op);
2355 1474298 : info.op = op;
2356 1474298 : info.stmt = stmt;
2357 1474298 : info.bb_set = BITMAP_ALLOC (NULL);
2358 1474298 : int walked
2359 2948596 : = walk_aliased_vdefs (&refd, gimple_vuse (stmt), record_modified, &info,
2360 : NULL, NULL, fbi->aa_walk_budget);
2361 1474298 : if (walked > 0)
2362 1337686 : fbi->aa_walk_budget -= walked;
2363 1474298 : if (walked < 0 || bitmap_bit_p (info.bb_set, bb->index))
2364 : {
2365 894484 : if (walked < 0)
2366 212 : fbi->aa_walk_budget = 0;
2367 894484 : if (dump_file)
2368 : {
2369 0 : if (walked < 0)
2370 0 : fprintf (dump_file, " Ran out of AA walking budget.\n");
2371 : else
2372 0 : fprintf (dump_file, " Set in same BB as used.\n");
2373 : }
2374 894484 : BITMAP_FREE (info.bb_set);
2375 894484 : return REG_BR_PROB_BASE;
2376 : }
2377 :
2378 579814 : bitmap_iterator bi;
2379 579814 : unsigned index;
2380 : /* Lookup the most frequent update of the value and believe that
2381 : it dominates all the other; precise analysis here is difficult. */
2382 1620360 : EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi)
2383 1040546 : max = profile_count::max_prefer_initialized
2384 1040546 : (max, BASIC_BLOCK_FOR_FN (cfun, index)->count);
2385 579814 : if (dump_file)
2386 : {
2387 0 : fprintf (dump_file, " Set with count ");
2388 0 : max.dump (dump_file);
2389 0 : fprintf (dump_file, " and used with count ");
2390 0 : bb->count.dump (dump_file);
2391 0 : fprintf (dump_file, " freq %f\n",
2392 0 : max.to_sreal_scale (bb->count).to_double ());
2393 : }
2394 :
2395 579814 : BITMAP_FREE (info.bb_set);
2396 579814 : if (max < bb->count)
2397 97070 : return MAX ((max.to_sreal_scale (bb->count)
2398 : * REG_BR_PROB_BASE).to_int (), 1);
2399 : return REG_BR_PROB_BASE;
2400 : }
2401 : }
2402 :
2403 : /* Find whether a basic block BB is the final block of a (half) diamond CFG
2404 : sub-graph and if the predicate the condition depends on is known. If so,
2405 : return true and store the pointer the predicate in *P. */
2406 :
2407 : static bool
2408 6516640 : phi_result_unknown_predicate (ipa_func_body_info *fbi,
2409 : ipa_fn_summary *summary,
2410 : class ipa_node_params *params_summary,
2411 : basic_block bb,
2412 : ipa_predicate *p,
2413 : vec<ipa_predicate> nonconstant_names)
2414 : {
2415 6516640 : edge e;
2416 6516640 : edge_iterator ei;
2417 6516640 : basic_block first_bb = NULL;
2418 :
2419 6516640 : if (single_pred_p (bb))
2420 : {
2421 82650 : *p = false;
2422 82650 : return true;
2423 : }
2424 :
2425 14791857 : FOR_EACH_EDGE (e, ei, bb->preds)
2426 : {
2427 12670138 : if (single_succ_p (e->src))
2428 : {
2429 8486617 : if (!single_pred_p (e->src))
2430 : return false;
2431 7453240 : if (!first_bb)
2432 3278918 : first_bb = single_pred (e->src);
2433 4174322 : else if (single_pred (e->src) != first_bb)
2434 : return false;
2435 : }
2436 : else
2437 : {
2438 4183521 : if (!first_bb)
2439 : first_bb = e->src;
2440 1464771 : else if (e->src != first_bb)
2441 : return false;
2442 : }
2443 : }
2444 :
2445 2121719 : if (!first_bb)
2446 : return false;
2447 :
2448 4243438 : gcond *stmt = safe_dyn_cast <gcond *> (*gsi_last_bb (first_bb));
2449 2076085 : if (!stmt
2450 2076085 : || !is_gimple_ip_invariant (gimple_cond_rhs (stmt)))
2451 : return false;
2452 :
2453 1819311 : *p = will_be_nonconstant_expr_predicate (fbi, summary, params_summary,
2454 : gimple_cond_lhs (stmt),
2455 : nonconstant_names);
2456 1819311 : if (*p == true)
2457 : return false;
2458 : else
2459 601470 : return true;
2460 : }
2461 :
2462 : /* Given a PHI statement in a function described by inline properties SUMMARY
2463 : and *P being the predicate describing whether the selected PHI argument is
2464 : known, store a predicate for the result of the PHI statement into
2465 : NONCONSTANT_NAMES, if possible. */
2466 :
2467 : static void
2468 780422 : predicate_for_phi_result (class ipa_fn_summary *summary, gphi *phi,
2469 : ipa_predicate *p,
2470 : vec<ipa_predicate> nonconstant_names)
2471 : {
2472 780422 : unsigned i;
2473 :
2474 1102837 : for (i = 0; i < gimple_phi_num_args (phi); i++)
2475 : {
2476 942619 : tree arg = gimple_phi_arg (phi, i)->def;
2477 942619 : if (!is_gimple_min_invariant (arg))
2478 : {
2479 790419 : gcc_assert (TREE_CODE (arg) == SSA_NAME);
2480 1580838 : *p = p->or_with (summary->conds,
2481 790419 : nonconstant_names[SSA_NAME_VERSION (arg)]);
2482 790419 : if (*p == true)
2483 : return;
2484 : }
2485 : }
2486 :
2487 160218 : if (dump_file && (dump_flags & TDF_DETAILS))
2488 : {
2489 3 : fprintf (dump_file, "\t\tphi predicate: ");
2490 3 : p->dump (dump_file, summary->conds);
2491 : }
2492 160218 : nonconstant_names[SSA_NAME_VERSION (gimple_phi_result (phi))] = *p;
2493 : }
2494 :
2495 : /* For a typical usage of __builtin_expect (a<b, 1), we
2496 : may introduce an extra relation stmt:
2497 : With the builtin, we have
2498 : t1 = a <= b;
2499 : t2 = (long int) t1;
2500 : t3 = __builtin_expect (t2, 1);
2501 : if (t3 != 0)
2502 : goto ...
2503 : Without the builtin, we have
2504 : if (a<=b)
2505 : goto...
2506 : This affects the size/time estimation and may have
2507 : an impact on the earlier inlining.
2508 : Here find this pattern and fix it up later. */
2509 :
2510 : static gimple *
2511 42202673 : find_foldable_builtin_expect (basic_block bb)
2512 : {
2513 42202673 : gimple_stmt_iterator bsi;
2514 :
2515 296391931 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
2516 : {
2517 212171316 : gimple *stmt = gsi_stmt (bsi);
2518 212171316 : if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)
2519 211935181 : || gimple_call_builtin_p (stmt, BUILT_IN_EXPECT_WITH_PROBABILITY)
2520 424106420 : || gimple_call_internal_p (stmt, IFN_BUILTIN_EXPECT))
2521 : {
2522 372052 : tree var = gimple_call_lhs (stmt);
2523 372052 : tree arg = gimple_call_arg (stmt, 0);
2524 372052 : use_operand_p use_p;
2525 372052 : gimple *use_stmt;
2526 372052 : bool match = false;
2527 372052 : bool done = false;
2528 :
2529 372052 : if (!var || !arg)
2530 3 : continue;
2531 372049 : gcc_assert (TREE_CODE (var) == SSA_NAME);
2532 :
2533 742715 : while (TREE_CODE (arg) == SSA_NAME)
2534 : {
2535 742715 : gimple *stmt_tmp = SSA_NAME_DEF_STMT (arg);
2536 742715 : if (!is_gimple_assign (stmt_tmp))
2537 : break;
2538 735364 : switch (gimple_assign_rhs_code (stmt_tmp))
2539 : {
2540 : case LT_EXPR:
2541 : case LE_EXPR:
2542 : case GT_EXPR:
2543 : case GE_EXPR:
2544 : case EQ_EXPR:
2545 : case NE_EXPR:
2546 : match = true;
2547 : done = true;
2548 : break;
2549 : CASE_CONVERT:
2550 : break;
2551 : default:
2552 : done = true;
2553 : break;
2554 : }
2555 370666 : if (done)
2556 : break;
2557 370666 : arg = gimple_assign_rhs1 (stmt_tmp);
2558 : }
2559 :
2560 332083 : if (match && single_imm_use (var, &use_p, &use_stmt)
2561 703814 : && gimple_code (use_stmt) == GIMPLE_COND)
2562 184731 : return use_stmt;
2563 : }
2564 : }
2565 : return NULL;
2566 : }
2567 :
2568 : /* Return true when the basic blocks contains only clobbers followed by RESX.
2569 : Such BBs are kept around to make removal of dead stores possible with
2570 : presence of EH and will be optimized out by optimize_clobbers later in the
2571 : game.
2572 :
2573 : NEED_EH is used to recurse in case the clobber has non-EH predecessors
2574 : that can be clobber only, too.. When it is false, the RESX is not necessary
2575 : on the end of basic block. */
2576 :
2577 : static bool
2578 42856097 : clobber_only_eh_bb_p (basic_block bb, bool need_eh = true)
2579 : {
2580 42856097 : gimple_stmt_iterator gsi = gsi_last_bb (bb);
2581 42856097 : edge_iterator ei;
2582 42856097 : edge e;
2583 :
2584 42856097 : if (need_eh)
2585 : {
2586 42781790 : if (gsi_end_p (gsi))
2587 : return false;
2588 41626932 : if (gimple_code (gsi_stmt (gsi)) != GIMPLE_RESX)
2589 : return false;
2590 1167325 : gsi_prev (&gsi);
2591 : }
2592 74307 : else if (!single_succ_p (bb))
2593 : return false;
2594 :
2595 3081412 : for (; !gsi_end_p (gsi); gsi_prev (&gsi))
2596 : {
2597 3000622 : gimple *stmt = gsi_stmt (gsi);
2598 3000622 : if (is_gimple_debug (stmt))
2599 1014289 : continue;
2600 1986333 : if (gimple_clobber_p (stmt))
2601 940830 : continue;
2602 1045503 : if (gimple_code (stmt) == GIMPLE_LABEL)
2603 : break;
2604 : return false;
2605 : }
2606 :
2607 : /* See if all predecessors are either throws or clobber only BBs. */
2608 3142110 : FOR_EACH_EDGE (e, ei, bb->preds)
2609 2561491 : if (!(e->flags & EDGE_EH)
2610 2561491 : && !clobber_only_eh_bb_p (e->src, false))
2611 : return false;
2612 :
2613 : return true;
2614 : }
2615 :
2616 : /* Return true if STMT compute a floating point expression that may be affected
2617 : by -ffast-math and similar flags. */
2618 :
2619 : static bool
2620 96869056 : fp_expression_p (gimple *stmt)
2621 : {
2622 96869056 : ssa_op_iter i;
2623 96869056 : tree op;
2624 :
2625 217271714 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_DEF|SSA_OP_USE)
2626 121026959 : if (FLOAT_TYPE_P (TREE_TYPE (op)))
2627 : return true;
2628 : return false;
2629 : }
2630 :
2631 : /* Return true if T references memory location that is local
2632 : for the function (that means, dead after return) or read-only. */
2633 :
2634 : bool
2635 61262979 : refs_local_or_readonly_memory_p (tree t)
2636 : {
2637 : /* Non-escaping memory is fine. */
2638 61262979 : t = get_base_address (t);
2639 61262979 : if ((TREE_CODE (t) == MEM_REF
2640 61262979 : || TREE_CODE (t) == TARGET_MEM_REF))
2641 25870491 : return points_to_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
2642 :
2643 : /* Automatic variables are fine. */
2644 35392488 : if (DECL_P (t)
2645 35392488 : && auto_var_in_fn_p (t, current_function_decl))
2646 : return true;
2647 :
2648 : /* Read-only variables are fine. */
2649 11484497 : if (DECL_P (t) && TREE_READONLY (t))
2650 1684997 : return true;
2651 :
2652 : return false;
2653 : }
2654 :
2655 : /* Return true if T is a pointer pointing to memory location that is local
2656 : for the function (that means, dead after return) or read-only. */
2657 :
2658 : bool
2659 74515856 : points_to_local_or_readonly_memory_p (tree t)
2660 : {
2661 : /* See if memory location is clearly invalid. */
2662 74515856 : if (integer_zerop (t))
2663 2490287 : return flag_delete_null_pointer_checks;
2664 72025569 : if (TREE_CODE (t) == SSA_NAME)
2665 : {
2666 : /* For IPA passes we can consinder accesses to return slot local
2667 : even if it is not local in the sense that memory is dead by
2668 : the end of founction.
2669 : The outer function will see a store in the call assignment
2670 : and thus this will do right thing for all uses of this
2671 : function in the current IPA passes (modref, pure/const discovery
2672 : and inlining heuristics). */
2673 45571767 : if (DECL_RESULT (current_function_decl)
2674 45571767 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
2675 46693071 : && t == ssa_default_def (cfun, DECL_RESULT (current_function_decl)))
2676 : return true;
2677 45246374 : return !ptr_deref_may_alias_global_p (t, false);
2678 : }
2679 26453802 : if (TREE_CODE (t) == ADDR_EXPR
2680 26453802 : && (TREE_CODE (TREE_OPERAND (t, 0)) != TARGET_MEM_REF
2681 4447 : || TREE_CODE (TREE_OPERAND (TREE_OPERAND (t, 0), 0)) != INTEGER_CST))
2682 13704139 : return refs_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
2683 : return false;
2684 : }
2685 :
2686 : /* Return true if T is a pointer pointing to memory location that is possible
2687 : sra candidate if all functions it is passed to are inlined. */
2688 :
2689 : static bool
2690 39808244 : points_to_possible_sra_candidate_p (tree t)
2691 : {
2692 39808244 : if (TREE_CODE (t) != ADDR_EXPR)
2693 : return false;
2694 :
2695 11940558 : t = get_base_address (TREE_OPERAND (t, 0));
2696 :
2697 : /* Automatic variables are fine. */
2698 11940558 : if (DECL_P (t)
2699 11940558 : && auto_var_in_fn_p (t, current_function_decl))
2700 : return true;
2701 : return false;
2702 : }
2703 :
2704 : /* Return true if BB only calls builtin_unreachable.
2705 : We skip empty basic blocks, debug statements, clobbers and predicts.
2706 : CACHE is used to memoize already analyzed blocks. */
2707 :
2708 : static bool
2709 28032769 : builtin_unreachable_bb_p (basic_block bb, vec<unsigned char> &cache)
2710 : {
2711 28032769 : if (cache[bb->index])
2712 2770482 : return cache[bb->index] - 1;
2713 25262287 : gimple_stmt_iterator si;
2714 25262287 : auto_vec <basic_block, 4> visited_bbs;
2715 25262287 : bool ret = false;
2716 26083621 : while (true)
2717 : {
2718 26083621 : bool empty_bb = true;
2719 26083621 : visited_bbs.safe_push (bb);
2720 26083621 : cache[bb->index] = 3;
2721 26083621 : for (si = gsi_start_nondebug_bb (bb);
2722 27603224 : !gsi_end_p (si) && empty_bb;
2723 1519603 : gsi_next_nondebug (&si))
2724 : {
2725 26221378 : if (gimple_code (gsi_stmt (si)) != GIMPLE_PREDICT
2726 25852564 : && !gimple_clobber_p (gsi_stmt (si))
2727 50956660 : && !gimple_nop_p (gsi_stmt (si)))
2728 : {
2729 : empty_bb = false;
2730 : break;
2731 : }
2732 : }
2733 26083621 : if (!empty_bb)
2734 : break;
2735 : else
2736 1381846 : bb = single_succ_edge (bb)->dest;
2737 1381846 : if (cache[bb->index])
2738 : {
2739 560512 : ret = cache[bb->index] == 3 ? false : cache[bb->index] - 1;
2740 560512 : goto done;
2741 : }
2742 : }
2743 24701775 : if (gimple_call_builtin_p (gsi_stmt (si), BUILT_IN_UNREACHABLE)
2744 24701775 : || gimple_call_builtin_p (gsi_stmt (si), BUILT_IN_UNREACHABLE_TRAP))
2745 : ret = true;
2746 25262287 : done:
2747 101870482 : for (basic_block vbb:visited_bbs)
2748 26083621 : cache[vbb->index] = (unsigned char)ret + 1;
2749 25262287 : return ret;
2750 25262287 : }
2751 :
2752 : static bool
2753 14141245 : guards_builtin_unreachable (basic_block bb, vec<unsigned char> &cache)
2754 : {
2755 14141245 : edge_iterator ei;
2756 14141245 : edge e;
2757 41909929 : FOR_EACH_EDGE (e, ei, bb->succs)
2758 28032769 : if (builtin_unreachable_bb_p (e->dest, cache))
2759 : {
2760 264085 : if (dump_file && (dump_flags & TDF_DETAILS))
2761 1 : fprintf (dump_file,
2762 : "BB %i ends with conditional guarding __builtin_unreachable;"
2763 : " conditinal is unnecessary\n", bb->index);
2764 : return true;
2765 : }
2766 : return false;
2767 : }
2768 :
2769 : #define STMT_NECESSARY GF_PLF_1
2770 :
2771 : /* If STMT is not already marked necessary, mark it, and add it to the
2772 : worklist if ADD_TO_WORKLIST is true. */
2773 :
2774 : static inline void
2775 176976858 : mark_stmt_necessary (gimple *stmt, auto_vec<gimple *> &worklist)
2776 : {
2777 176976858 : gcc_assert (stmt);
2778 :
2779 176976858 : if (gimple_plf (stmt, STMT_NECESSARY))
2780 : return;
2781 :
2782 146926089 : if (dump_file && (dump_flags & TDF_DETAILS))
2783 : {
2784 211 : fprintf (dump_file, "Marking useful stmt: ");
2785 211 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
2786 211 : fprintf (dump_file, "\n");
2787 : }
2788 :
2789 146926089 : gimple_set_plf (stmt, STMT_NECESSARY, true);
2790 146926089 : worklist.safe_push (stmt);
2791 : }
2792 :
2793 : /* Mark the statement defining operand OP as necessary. */
2794 :
2795 : static inline void
2796 124422530 : mark_operand_necessary (tree op, auto_vec<gimple *> &worklist)
2797 : {
2798 124422530 : gimple *stmt = SSA_NAME_DEF_STMT (op);
2799 124422530 : if (gimple_nop_p (stmt))
2800 : return;
2801 100878314 : mark_stmt_necessary (stmt, worklist);
2802 : }
2803 :
2804 : /* Mark all statements that will remain in the body after optimizing out
2805 : conditionals guarding __builtin_unreachable which we keep to preserve
2806 : value ranges. */
2807 :
2808 : static void
2809 7450714 : find_necessary_statements (struct cgraph_node *node)
2810 : {
2811 7450714 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
2812 7450714 : auto_vec<unsigned char, 10> cache;
2813 7450714 : basic_block bb;
2814 7450714 : auto_vec<gimple *> worklist;
2815 :
2816 7450714 : cache.safe_grow_cleared (last_basic_block_for_fn (cfun));
2817 : /* Mark all obviously necessary statements. */
2818 50232504 : FOR_EACH_BB_FN (bb, my_function)
2819 : {
2820 42781790 : for (gimple_stmt_iterator gsi = gsi_start_phis (bb);
2821 54675539 : !gsi_end_p (gsi); gsi_next (&gsi))
2822 11893749 : gimple_set_plf (gsi_stmt (gsi), STMT_NECESSARY, false);
2823 :
2824 239829341 : for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
2825 154265761 : gsi_next_nondebug (&bsi))
2826 : {
2827 154265761 : gimple *stmt = gsi_stmt (bsi);
2828 :
2829 154265761 : gimple_set_plf (stmt, STMT_NECESSARY, false);
2830 154265761 : if (gimple_has_side_effects (stmt)
2831 126142079 : || (is_ctrl_stmt (stmt)
2832 22693948 : && (gimple_code (stmt) != GIMPLE_COND
2833 14141245 : || !guards_builtin_unreachable (bb, cache)))
2834 103712216 : || gimple_store_p (stmt)
2835 232454679 : || gimple_code (stmt) == GIMPLE_ASM)
2836 76098544 : mark_stmt_necessary (stmt, worklist);
2837 : }
2838 : }
2839 154376803 : while (worklist.length () > 0)
2840 : {
2841 146926089 : gimple *stmt = worklist.pop ();
2842 :
2843 146926089 : if (dump_file && (dump_flags & TDF_DETAILS))
2844 : {
2845 211 : fprintf (dump_file, "processing: ");
2846 211 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
2847 211 : fprintf (dump_file, "\n");
2848 : }
2849 146926089 : if (gimple_code (stmt) == GIMPLE_PHI)
2850 18818596 : for (unsigned int k = 0; k < gimple_phi_num_args (stmt); k++)
2851 : {
2852 13317578 : tree arg = PHI_ARG_DEF (stmt, k);
2853 :
2854 13317578 : if (TREE_CODE (arg) == SSA_NAME)
2855 10197430 : mark_operand_necessary (arg, worklist);
2856 : }
2857 : else
2858 : {
2859 141425071 : ssa_op_iter iter;
2860 141425071 : tree use;
2861 :
2862 255650171 : FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2863 114225100 : mark_operand_necessary (use, worklist);
2864 : }
2865 : }
2866 7450714 : }
2867 :
2868 : /* Analyze function body for NODE.
2869 : EARLY indicates run from early optimization pipeline. */
2870 :
2871 : static void
2872 7450714 : analyze_function_body (struct cgraph_node *node, bool early)
2873 : {
2874 7450714 : sreal time = opt_for_fn (node->decl, param_uninlined_function_time);
2875 : /* Estimate static overhead for function prologue/epilogue and alignment. */
2876 7450714 : int size = opt_for_fn (node->decl, param_uninlined_function_insns);
2877 : /* Benefits are scaled by probability of elimination that is in range
2878 : <0,2>. */
2879 7450714 : basic_block bb;
2880 7450714 : struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
2881 7450714 : sreal freq;
2882 7450714 : class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
2883 7450714 : ipa_node_params *params_summary
2884 7450714 : = early ? NULL : ipa_node_params_sum->get (node);
2885 7450714 : ipa_predicate bb_predicate;
2886 7450714 : struct ipa_func_body_info fbi;
2887 7450714 : vec<ipa_predicate> nonconstant_names = vNULL;
2888 7450714 : int nblocks, n;
2889 7450714 : int *order;
2890 7450714 : gimple *fix_builtin_expect_stmt;
2891 :
2892 7450714 : gcc_assert (my_function && my_function->cfg);
2893 7450714 : gcc_assert (cfun == my_function);
2894 :
2895 7450714 : memset(&fbi, 0, sizeof(fbi));
2896 7450714 : vec_free (info->conds);
2897 7450714 : info->conds = NULL;
2898 7450714 : info->size_time_table.release ();
2899 7450714 : info->call_size_time_table.release ();
2900 :
2901 : /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2902 : so we can produce proper inline hints.
2903 :
2904 : When optimizing and analyzing for early inliner, initialize node params
2905 : so we can produce correct BB predicates. */
2906 :
2907 7450714 : if (opt_for_fn (node->decl, optimize))
2908 : {
2909 6517766 : calculate_dominance_info (CDI_DOMINATORS);
2910 6517766 : calculate_dominance_info (CDI_POST_DOMINATORS);
2911 6517766 : if (!early)
2912 1415887 : loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
2913 : else
2914 : {
2915 5101879 : ipa_check_create_node_params ();
2916 5101879 : ipa_initialize_node_params (node);
2917 : }
2918 :
2919 6517766 : if (ipa_node_params_sum)
2920 : {
2921 6517766 : fbi.node = node;
2922 6517766 : fbi.info = ipa_node_params_sum->get (node);
2923 6517766 : fbi.bb_infos = vNULL;
2924 6517766 : fbi.bb_infos.safe_grow_cleared (last_basic_block_for_fn (cfun), true);
2925 6517766 : fbi.param_count = count_formal_params (node->decl);
2926 6517766 : fbi.aa_walk_budget = opt_for_fn (node->decl, param_ipa_max_aa_steps);
2927 :
2928 6517766 : nonconstant_names.safe_grow_cleared
2929 6517766 : (SSANAMES (my_function)->length (), true);
2930 : }
2931 : }
2932 :
2933 7450714 : if (dump_file)
2934 249 : fprintf (dump_file, "\nAnalyzing function body size: %s\n",
2935 : node->dump_name ());
2936 :
2937 : /* When we run into maximal number of entries, we assign everything to the
2938 : constant truth case. Be sure to have it in list. */
2939 7450714 : bb_predicate = true;
2940 7450714 : info->account_size_time (0, 0, bb_predicate, bb_predicate);
2941 :
2942 7450714 : bb_predicate = ipa_predicate::not_inlined ();
2943 7450714 : info->account_size_time (opt_for_fn (node->decl,
2944 : param_uninlined_function_insns)
2945 : * ipa_fn_summary::size_scale,
2946 7450714 : opt_for_fn (node->decl,
2947 : param_uninlined_function_time),
2948 : bb_predicate,
2949 : bb_predicate);
2950 :
2951 : /* Only look for target information for inlinable functions. */
2952 7450714 : bool scan_for_target_info =
2953 7450714 : info->inlinable
2954 13399567 : && targetm.target_option.need_ipa_fn_target_info (node->decl,
2955 5948853 : info->target_info);
2956 :
2957 7450714 : if (fbi.info)
2958 6517766 : compute_bb_predicates (&fbi, node, info, params_summary);
2959 7450714 : find_necessary_statements (node);
2960 7450714 : const profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2961 7450714 : order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
2962 7450714 : nblocks = pre_and_rev_post_order_compute (NULL, order, false);
2963 57683218 : for (n = 0; n < nblocks; n++)
2964 : {
2965 42781790 : bb = BASIC_BLOCK_FOR_FN (cfun, order[n]);
2966 42781790 : freq = bb->count.to_sreal_scale (entry_count);
2967 42781790 : if (clobber_only_eh_bb_p (bb))
2968 : {
2969 579117 : if (dump_file && (dump_flags & TDF_DETAILS))
2970 0 : fprintf (dump_file, "\n Ignoring BB %i;"
2971 : " it will be optimized away by cleanup_clobbers\n",
2972 : bb->index);
2973 579117 : continue;
2974 : }
2975 :
2976 : /* TODO: Obviously predicates can be propagated down across CFG. */
2977 42202673 : if (fbi.info)
2978 : {
2979 35642723 : if (bb->aux)
2980 35642707 : bb_predicate = *(ipa_predicate *)bb->aux;
2981 : else
2982 16 : bb_predicate = false;
2983 : }
2984 : else
2985 6559950 : bb_predicate = true;
2986 :
2987 42202673 : if (dump_file && (dump_flags & TDF_DETAILS))
2988 : {
2989 69 : fprintf (dump_file, "\n BB %i predicate:", bb->index);
2990 69 : bb_predicate.dump (dump_file, info->conds);
2991 : }
2992 :
2993 42202673 : if (fbi.info && nonconstant_names.exists ())
2994 : {
2995 35642723 : ipa_predicate phi_predicate;
2996 35642723 : bool first_phi = true;
2997 :
2998 36423145 : for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
2999 780422 : gsi_next (&bsi))
3000 : {
3001 6612942 : if (first_phi
3002 6612942 : && !phi_result_unknown_predicate (&fbi, info,
3003 : params_summary,
3004 : bb,
3005 : &phi_predicate,
3006 : nonconstant_names))
3007 : break;
3008 780422 : first_phi = false;
3009 780422 : if (dump_file && (dump_flags & TDF_DETAILS))
3010 : {
3011 3 : fprintf (dump_file, " ");
3012 3 : print_gimple_stmt (dump_file, gsi_stmt (bsi), 0);
3013 : }
3014 780422 : predicate_for_phi_result (info, bsi.phi (), &phi_predicate,
3015 : nonconstant_names);
3016 : }
3017 : }
3018 :
3019 42202673 : fix_builtin_expect_stmt = find_foldable_builtin_expect (bb);
3020 :
3021 42202673 : for (gimple_stmt_iterator bsi = gsi_start_nondebug_bb (bb);
3022 188032986 : !gsi_end_p (bsi); gsi_next_nondebug (&bsi))
3023 : {
3024 145830313 : gimple *stmt = gsi_stmt (bsi);
3025 145830313 : if (!gimple_plf (stmt, STMT_NECESSARY))
3026 : {
3027 5772472 : if (dump_file && (dump_flags & TDF_DETAILS))
3028 : {
3029 19 : fprintf (dump_file, " skipping unnecessary stmt ");
3030 19 : print_gimple_stmt (dump_file, stmt, 0);
3031 : }
3032 : /* TODO: const calls used only to produce values for
3033 : builtion_unreachable guards should not be accounted. However
3034 : we still want to inline them and this does does not work well
3035 : with the cost model. For now account them as usual. */
3036 5772472 : if (!is_gimple_call (stmt)
3037 5772472 : || gimple_call_internal_p (stmt))
3038 5492092 : continue;
3039 : }
3040 140338221 : int this_size = estimate_num_insns (stmt, &eni_size_weights);
3041 140338221 : int this_time = estimate_num_insns (stmt, &eni_time_weights);
3042 140338221 : int prob;
3043 140338221 : ipa_predicate will_be_nonconstant;
3044 :
3045 : /* This relation stmt should be folded after we remove
3046 : __builtin_expect call. Adjust the cost here. */
3047 140338221 : if (stmt == fix_builtin_expect_stmt)
3048 : {
3049 184293 : this_size--;
3050 184293 : this_time--;
3051 : }
3052 :
3053 140338221 : if (dump_file && (dump_flags & TDF_DETAILS))
3054 : {
3055 205 : fprintf (dump_file, " ");
3056 205 : print_gimple_stmt (dump_file, stmt, 0);
3057 205 : fprintf (dump_file, "\t\tfreq:%3.2f size:%3i time:%3i\n",
3058 : freq.to_double (), this_size,
3059 : this_time);
3060 : }
3061 :
3062 140338221 : struct cgraph_edge *edge;
3063 140338221 : if (is_gimple_call (stmt)
3064 24756881 : && (!gimple_call_internal_p (stmt)
3065 1149925 : || gimple_call_internal_fn (stmt) == IFN_ASSUME)
3066 163945693 : && (edge = node->get_edge (stmt)))
3067 : {
3068 23607131 : ipa_call_summary *es = ipa_call_summaries->get_create (edge);
3069 :
3070 : /* Special case: results of BUILT_IN_CONSTANT_P will be always
3071 : resolved as constant. We however don't want to optimize
3072 : out the cgraph edges. */
3073 23607131 : if (nonconstant_names.exists ()
3074 20650988 : && gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P)
3075 48326 : && gimple_call_lhs (stmt)
3076 23655457 : && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
3077 : {
3078 48326 : ipa_predicate false_p = false;
3079 48326 : nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
3080 48326 : = false_p;
3081 : }
3082 23607131 : if (ipa_node_params_sum)
3083 : {
3084 20663821 : int count = gimple_call_num_args (stmt);
3085 20663821 : int i;
3086 :
3087 20663821 : if (count)
3088 17757936 : es->param.safe_grow_cleared (count, true);
3089 60472065 : for (i = 0; i < count; i++)
3090 : {
3091 39808244 : int prob = param_change_prob (&fbi, stmt, i);
3092 39808244 : gcc_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
3093 39808244 : es->param[i].change_prob = prob;
3094 39808244 : es->param[i].points_to_local_or_readonly_memory
3095 39808244 : = points_to_local_or_readonly_memory_p
3096 39808244 : (gimple_call_arg (stmt, i));
3097 39808244 : es->param[i].points_to_possible_sra_candidate
3098 39808244 : = points_to_possible_sra_candidate_p
3099 39808244 : (gimple_call_arg (stmt, i));
3100 : }
3101 : }
3102 : /* We cannot setup VLA parameters during inlining. */
3103 69395857 : for (unsigned int i = 0; i < gimple_call_num_args (stmt); ++i)
3104 45789122 : if (TREE_CODE (gimple_call_arg (stmt, i)) == WITH_SIZE_EXPR)
3105 : {
3106 396 : edge->inline_failed = CIF_FUNCTION_NOT_INLINABLE;
3107 396 : break;
3108 : }
3109 23607131 : es->call_stmt_size = this_size;
3110 23607131 : es->call_stmt_time = this_time;
3111 23607131 : es->loop_depth = bb_loop_depth (bb);
3112 23607131 : edge_set_predicate (edge, &bb_predicate);
3113 23607131 : if (edge->speculative)
3114 : {
3115 0 : cgraph_edge *indirect
3116 0 : = edge->speculative_call_indirect_edge ();
3117 0 : ipa_call_summary *es2
3118 0 : = ipa_call_summaries->get_create (indirect);
3119 0 : ipa_call_summaries->duplicate (edge, indirect,
3120 : es, es2);
3121 :
3122 : /* Edge is the first direct call.
3123 : create and duplicate call summaries for multiple
3124 : speculative call targets. */
3125 0 : for (cgraph_edge *direct
3126 0 : = edge->next_speculative_call_target ();
3127 0 : direct;
3128 0 : direct = direct->next_speculative_call_target ())
3129 : {
3130 0 : ipa_call_summary *es3
3131 0 : = ipa_call_summaries->get_create (direct);
3132 0 : ipa_call_summaries->duplicate (edge, direct,
3133 : es, es3);
3134 : }
3135 : }
3136 :
3137 : /* If dealing with a carrying edge, copy its summary over to its
3138 : attached edges as well. */
3139 23607131 : if (edge->has_callback)
3140 : {
3141 15202 : cgraph_edge *cbe;
3142 30405 : for (cbe = edge->first_callback_edge (); cbe;
3143 15203 : cbe = cbe->next_callback_edge ())
3144 : {
3145 15203 : ipa_call_summary *es2 = ipa_call_summaries->get_create (cbe);
3146 15203 : ipa_call_summaries->duplicate (edge, cbe, es, es2);
3147 : /* Unlike speculative edges, callback edges have no real
3148 : size or time; the call doesn't exist. Reflect that in
3149 : their summaries. */
3150 15203 : es2->call_stmt_size = 0;
3151 15203 : es2->call_stmt_time = 0;
3152 : }
3153 : }
3154 : /* Internal calls are not in the callgraph, so edges pointing to
3155 : .ASSUME bodies are not covered by the above branch, as the
3156 : dispatching edge doesn't exist. */
3157 23591929 : else if (edge->callback)
3158 : {
3159 175 : gcc_checking_assert (edge->callee->assume_function);
3160 175 : es->call_stmt_size = 0;
3161 175 : es->call_stmt_time = 0;
3162 : }
3163 : }
3164 :
3165 : /* TODO: When conditional jump or switch is known to be constant, but
3166 : we did not translate it into the predicates, we really can account
3167 : just maximum of the possible paths. */
3168 140338221 : if (fbi.info)
3169 117712834 : will_be_nonconstant
3170 117712834 : = will_be_nonconstant_predicate (&fbi, info, params_summary,
3171 : stmt, nonconstant_names);
3172 : else
3173 22625387 : will_be_nonconstant = true;
3174 140338221 : if (this_time || this_size)
3175 : {
3176 114735321 : sreal final_time = (sreal)this_time * freq;
3177 114735321 : prob = eliminated_by_inlining_prob (&fbi, stmt);
3178 114735321 : if (prob == 1 && dump_file && (dump_flags & TDF_DETAILS))
3179 11 : fprintf (dump_file,
3180 : "\t\t50%% will be eliminated by inlining\n");
3181 114735321 : if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
3182 20 : fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
3183 :
3184 114735321 : ipa_predicate p = bb_predicate & will_be_nonconstant;
3185 114735321 : int parm = load_or_store_of_ptr_parameter (&fbi, stmt);
3186 114735321 : ipa_predicate sra_predicate = true;
3187 114735321 : if (parm != -1)
3188 14598576 : sra_predicate &= add_condition (info, params_summary, parm,
3189 : ptr_type_node, NULL,
3190 7299288 : ipa_predicate::not_sra_candidate, NULL, 0);
3191 :
3192 : /* We can ignore statement when we proved it is never going
3193 : to happen, but we cannot do that for call statements
3194 : because edges are accounted specially. */
3195 :
3196 229470642 : if (*(is_gimple_call (stmt) ? &bb_predicate : &p) != false)
3197 : {
3198 113967722 : time += final_time;
3199 113967722 : size += this_size;
3200 : }
3201 :
3202 : /* We account everything but the calls. Calls have their own
3203 : size/time info attached to cgraph edges. This is necessary
3204 : in order to make the cost disappear after inlining. */
3205 114735321 : if (!is_gimple_call (stmt))
3206 : {
3207 91854922 : if (prob)
3208 : {
3209 15327071 : ipa_predicate ip
3210 15327071 : = bb_predicate & ipa_predicate::not_inlined () & sra_predicate;
3211 30654142 : info->account_size_time (this_size * prob,
3212 15327071 : (final_time * prob) / 2, ip,
3213 : p);
3214 : }
3215 15327071 : if (prob != 2)
3216 169515352 : info->account_size_time (this_size * (2 - prob),
3217 84757676 : (final_time * (2 - prob) / 2),
3218 169515352 : bb_predicate & sra_predicate,
3219 : p);
3220 : }
3221 :
3222 114735321 : if (!info->fp_expressions && fp_expression_p (stmt))
3223 : {
3224 624301 : info->fp_expressions = true;
3225 624301 : if (dump_file)
3226 9 : fprintf (dump_file, " fp_expression set\n");
3227 : }
3228 : }
3229 :
3230 : /* For target specific information, we want to scan all statements
3231 : rather than those statements with non-zero weights, to avoid
3232 : missing to scan something interesting for target information,
3233 : such as: internal function calls. */
3234 140338221 : if (scan_for_target_info)
3235 0 : scan_for_target_info =
3236 0 : targetm.target_option.update_ipa_fn_target_info
3237 0 : (info->target_info, stmt);
3238 :
3239 : /* Account cost of address calculations in the statements. */
3240 542692720 : for (unsigned int i = 0; i < gimple_num_ops (stmt); i++)
3241 : {
3242 402354499 : for (tree op = gimple_op (stmt, i);
3243 774927880 : op && handled_component_p (op);
3244 48818043 : op = TREE_OPERAND (op, 0))
3245 48818043 : if ((TREE_CODE (op) == ARRAY_REF
3246 48818043 : || TREE_CODE (op) == ARRAY_RANGE_REF)
3247 48818043 : && TREE_CODE (TREE_OPERAND (op, 1)) == SSA_NAME)
3248 : {
3249 2440173 : ipa_predicate p = bb_predicate;
3250 2440173 : if (fbi.info)
3251 1911815 : p = p & will_be_nonconstant_expr_predicate
3252 1911815 : (&fbi, info, params_summary,
3253 1911815 : TREE_OPERAND (op, 1),
3254 1911815 : nonconstant_names);
3255 2440173 : if (p != false)
3256 : {
3257 2434748 : time += freq;
3258 2434748 : size += 1;
3259 2434748 : if (dump_file)
3260 24 : fprintf (dump_file,
3261 : "\t\tAccounting address calculation.\n");
3262 2434748 : info->account_size_time (ipa_fn_summary::size_scale,
3263 : freq,
3264 : bb_predicate,
3265 : p);
3266 : }
3267 : }
3268 : }
3269 :
3270 : }
3271 : }
3272 7450714 : free (order);
3273 :
3274 7450714 : if (nonconstant_names.exists () && !early)
3275 : {
3276 1415887 : ipa_fn_summary *s = ipa_fn_summaries->get (node);
3277 1415887 : unsigned max_loop_predicates = opt_for_fn (node->decl,
3278 : param_ipa_max_loop_predicates);
3279 :
3280 1415887 : if (dump_file && (dump_flags & TDF_DETAILS))
3281 16 : flow_loops_dump (dump_file, NULL, 0);
3282 1415887 : scev_initialize ();
3283 4888219 : for (auto loop : loops_list (cfun, 0))
3284 : {
3285 640558 : ipa_predicate loop_iterations = true;
3286 640558 : sreal header_freq;
3287 640558 : edge ex;
3288 640558 : unsigned int j;
3289 640558 : class tree_niter_desc niter_desc;
3290 640558 : if (!loop->header->aux)
3291 0 : continue;
3292 :
3293 640558 : profile_count hdr_count = loop->header->count;
3294 640558 : sreal hdr_freq = hdr_count.to_sreal_scale (entry_count);
3295 :
3296 640558 : bb_predicate = *(ipa_predicate *)loop->header->aux;
3297 640558 : auto_vec<edge> exits = get_loop_exit_edges (loop);
3298 2320536 : FOR_EACH_VEC_ELT (exits, j, ex)
3299 1039420 : if (number_of_iterations_exit (loop, ex, &niter_desc, false)
3300 1039420 : && !is_gimple_min_invariant (niter_desc.niter))
3301 : {
3302 157712 : ipa_predicate will_be_nonconstant
3303 157712 : = will_be_nonconstant_expr_predicate (&fbi, info,
3304 : params_summary,
3305 : niter_desc.niter,
3306 : nonconstant_names);
3307 157712 : if (will_be_nonconstant != true)
3308 63739 : will_be_nonconstant = bb_predicate & will_be_nonconstant;
3309 157712 : if (will_be_nonconstant != true
3310 221451 : && will_be_nonconstant != false)
3311 63185 : loop_iterations &= will_be_nonconstant;
3312 : }
3313 640558 : add_freqcounting_predicate (&s->loop_iterations, loop_iterations,
3314 : hdr_freq, max_loop_predicates);
3315 640560 : }
3316 :
3317 : /* To avoid quadratic behavior we analyze stride predicates only
3318 : with respect to the containing loop. Thus we simply iterate
3319 : over all defs in the outermost loop body. */
3320 1415887 : for (class loop *loop = loops_for_fn (cfun)->tree_root->inner;
3321 1920561 : loop != NULL; loop = loop->next)
3322 : {
3323 504674 : ipa_predicate loop_stride = true;
3324 504674 : basic_block *body = get_loop_body (loop);
3325 504674 : profile_count hdr_count = loop->header->count;
3326 504674 : sreal hdr_freq = hdr_count.to_sreal_scale (entry_count);
3327 3472618 : for (unsigned i = 0; i < loop->num_nodes; i++)
3328 : {
3329 2463270 : gimple_stmt_iterator gsi;
3330 2463270 : if (!body[i]->aux)
3331 8 : continue;
3332 :
3333 2463262 : bb_predicate = *(ipa_predicate *)body[i]->aux;
3334 16710533 : for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
3335 11784009 : gsi_next (&gsi))
3336 : {
3337 11784009 : gimple *stmt = gsi_stmt (gsi);
3338 :
3339 11784009 : if (!is_gimple_assign (stmt))
3340 11631334 : continue;
3341 :
3342 5609597 : tree def = gimple_assign_lhs (stmt);
3343 5609597 : if (TREE_CODE (def) != SSA_NAME)
3344 1109027 : continue;
3345 :
3346 4500570 : affine_iv iv;
3347 9001140 : if (!simple_iv (loop_containing_stmt (stmt),
3348 : loop_containing_stmt (stmt),
3349 : def, &iv, true)
3350 4500570 : || is_gimple_min_invariant (iv.step))
3351 4347895 : continue;
3352 :
3353 152675 : ipa_predicate will_be_nonconstant
3354 152675 : = will_be_nonconstant_expr_predicate (&fbi, info,
3355 : params_summary,
3356 : iv.step,
3357 : nonconstant_names);
3358 152675 : if (will_be_nonconstant != true)
3359 56512 : will_be_nonconstant = bb_predicate & will_be_nonconstant;
3360 152675 : if (will_be_nonconstant != true
3361 209187 : && will_be_nonconstant != false)
3362 46914 : loop_stride = loop_stride & will_be_nonconstant;
3363 : }
3364 : }
3365 504674 : add_freqcounting_predicate (&s->loop_strides, loop_stride,
3366 : hdr_freq, max_loop_predicates);
3367 504674 : free (body);
3368 : }
3369 1415887 : scev_finalize ();
3370 : }
3371 65133932 : FOR_ALL_BB_FN (bb, my_function)
3372 : {
3373 57683218 : edge e;
3374 57683218 : edge_iterator ei;
3375 :
3376 57683218 : if (bb->aux)
3377 42673561 : edge_predicate_pool.remove ((ipa_predicate *)bb->aux);
3378 57683218 : bb->aux = NULL;
3379 122704966 : FOR_EACH_EDGE (e, ei, bb->succs)
3380 : {
3381 65021748 : if (e->aux)
3382 2580338 : edge_predicate_pool.remove ((ipa_predicate *)e->aux);
3383 65021748 : e->aux = NULL;
3384 : }
3385 : }
3386 7450714 : ipa_fn_summary *s = ipa_fn_summaries->get (node);
3387 7450714 : ipa_size_summary *ss = ipa_size_summaries->get (node);
3388 7450714 : s->time = time;
3389 7450714 : ss->self_size = size;
3390 7450714 : nonconstant_names.release ();
3391 7450714 : ipa_release_body_info (&fbi);
3392 7450714 : if (opt_for_fn (node->decl, optimize))
3393 : {
3394 6517766 : if (!early)
3395 1415887 : loop_optimizer_finalize ();
3396 5101879 : else if (!ipa_edge_args_sum)
3397 5101865 : ipa_free_all_node_params ();
3398 6517766 : free_dominance_info (CDI_DOMINATORS);
3399 6517766 : free_dominance_info (CDI_POST_DOMINATORS);
3400 : }
3401 7450714 : if (dump_file)
3402 : {
3403 249 : fprintf (dump_file, "\n");
3404 249 : ipa_dump_fn_summary (dump_file, node);
3405 : }
3406 7450714 : }
3407 :
3408 :
3409 : /* Compute function summary.
3410 : EARLY is true when we compute parameters during early opts. */
3411 :
3412 : void
3413 7452066 : compute_fn_summary (struct cgraph_node *node, bool early)
3414 : {
3415 7452066 : HOST_WIDE_INT self_stack_size;
3416 7452066 : struct cgraph_edge *e;
3417 :
3418 7452066 : gcc_assert (!node->inlined_to);
3419 :
3420 7452066 : if (!ipa_fn_summaries)
3421 220583 : ipa_fn_summary_alloc ();
3422 :
3423 : /* Create a new ipa_fn_summary. */
3424 7452066 : ((ipa_fn_summary_t *)ipa_fn_summaries)->remove_callees (node);
3425 7452066 : ipa_fn_summaries->remove (node);
3426 7452066 : class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
3427 7452066 : class ipa_size_summary *size_info = ipa_size_summaries->get_create (node);
3428 :
3429 : /* Estimate the stack size for the function if we're optimizing. */
3430 13036868 : self_stack_size = optimize && !node->thunk
3431 13969832 : ? estimated_stack_frame_size (node) : 0;
3432 7452066 : size_info->estimated_self_stack_size = self_stack_size;
3433 7452066 : info->estimated_stack_size = self_stack_size;
3434 :
3435 7452066 : if (node->thunk)
3436 : {
3437 1352 : ipa_call_summary *es = ipa_call_summaries->get_create (node->callees);
3438 1352 : ipa_predicate t = true;
3439 :
3440 1352 : node->can_change_signature = false;
3441 1352 : es->call_stmt_size = eni_size_weights.call_cost;
3442 1352 : es->call_stmt_time = eni_time_weights.call_cost;
3443 4056 : info->account_size_time (ipa_fn_summary::size_scale
3444 1352 : * opt_for_fn (node->decl,
3445 : param_uninlined_function_thunk_insns),
3446 1352 : opt_for_fn (node->decl,
3447 : param_uninlined_function_thunk_time), t, t);
3448 1352 : t = ipa_predicate::not_inlined ();
3449 1352 : info->account_size_time (2 * ipa_fn_summary::size_scale, 0, t, t);
3450 1352 : ipa_update_overall_fn_summary (node);
3451 1352 : size_info->self_size = size_info->size;
3452 1352 : if (stdarg_p (TREE_TYPE (node->decl)))
3453 : {
3454 9 : info->inlinable = false;
3455 9 : node->callees->inline_failed = CIF_VARIADIC_THUNK;
3456 : }
3457 : else
3458 1343 : info->inlinable = true;
3459 : }
3460 : else
3461 : {
3462 : /* Even is_gimple_min_invariant rely on current_function_decl. */
3463 7450714 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
3464 :
3465 : /* During IPA profile merging we may be called w/o virtual SSA form
3466 : built. */
3467 7450714 : update_ssa (TODO_update_ssa_only_virtuals);
3468 :
3469 : /* Can this function be inlined at all? */
3470 7450714 : if (!opt_for_fn (node->decl, optimize)
3471 8383662 : && !lookup_attribute ("always_inline",
3472 932948 : DECL_ATTRIBUTES (node->decl)))
3473 861830 : info->inlinable = false;
3474 : else
3475 6588884 : info->inlinable = tree_inlinable_function_p (node->decl);
3476 :
3477 7450714 : bool no_signature = false;
3478 :
3479 : /* Don't allow signature changes for functions which have
3480 : [[gnu::musttail]] or [[clang::musttail]] calls. Sometimes
3481 : (more often on targets which pass everything on the stack)
3482 : signature changes can result in tail calls being impossible
3483 : even when without the signature changes they would be ok.
3484 : See PR121023. */
3485 7450714 : if (cfun->has_musttail)
3486 : {
3487 1441 : if (dump_file)
3488 0 : fprintf (dump_file, "No signature change:"
3489 : " function has calls with musttail attribute.\n");
3490 : no_signature = true;
3491 : }
3492 :
3493 : /* Type attributes can use parameter indices to describe them.
3494 : Special case fn spec since we can safely preserve them in
3495 : modref summaries. */
3496 7450714 : for (tree list = TYPE_ATTRIBUTES (TREE_TYPE (node->decl));
3497 7855136 : list && !no_signature; list = TREE_CHAIN (list))
3498 404422 : if (!ipa_param_adjustments::type_attribute_allowed_p
3499 404422 : (get_attribute_name (list)))
3500 : {
3501 159430 : if (dump_file)
3502 : {
3503 0 : fprintf (dump_file, "No signature change:"
3504 : " function type has unhandled attribute %s.\n",
3505 0 : IDENTIFIER_POINTER (get_attribute_name (list)));
3506 : }
3507 : no_signature = true;
3508 : }
3509 7450714 : for (tree parm = DECL_ARGUMENTS (node->decl);
3510 22216768 : parm && !no_signature; parm = DECL_CHAIN (parm))
3511 14766054 : if (variably_modified_type_p (TREE_TYPE (parm), node->decl))
3512 : {
3513 16036 : if (dump_file)
3514 : {
3515 0 : fprintf (dump_file, "No signature change:"
3516 : " has parameter with variably modified type.\n");
3517 : }
3518 : no_signature = true;
3519 : }
3520 :
3521 : /* Likewise for #pragma omp declare simd functions or functions
3522 : with simd attribute. */
3523 7450714 : if (no_signature
3524 14724521 : || lookup_attribute ("omp declare simd",
3525 7273807 : DECL_ATTRIBUTES (node->decl)))
3526 178622 : node->can_change_signature = false;
3527 : else
3528 : {
3529 : /* Otherwise, inlinable functions always can change signature. */
3530 7272092 : if (info->inlinable)
3531 5868974 : node->can_change_signature = true;
3532 : else
3533 : {
3534 : /* Functions calling builtin_apply cannot change signature. */
3535 5428658 : for (e = node->callees; e; e = e->next_callee)
3536 : {
3537 4058325 : tree cdecl = e->callee->decl;
3538 4058325 : if (fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS,
3539 : BUILT_IN_VA_START))
3540 : break;
3541 : }
3542 1403118 : node->can_change_signature = !e;
3543 : }
3544 : }
3545 7450714 : analyze_function_body (node, early);
3546 7450714 : pop_cfun ();
3547 : }
3548 :
3549 : /* Inlining characteristics are maintained by the cgraph_mark_inline. */
3550 7452066 : size_info->size = size_info->self_size;
3551 7452066 : info->estimated_stack_size = size_info->estimated_self_stack_size;
3552 :
3553 : /* Code above should compute exactly the same result as
3554 : ipa_update_overall_fn_summary except for case when speculative
3555 : edges are present since these are accounted to size but not
3556 : self_size. Do not compare time since different order the roundoff
3557 : errors result in slight changes. */
3558 7452066 : ipa_update_overall_fn_summary (node);
3559 7452066 : if (flag_checking)
3560 : {
3561 7938311 : for (e = node->indirect_calls; e; e = e->next_callee)
3562 486327 : if (e->speculative)
3563 : break;
3564 7451984 : gcc_assert (e || size_info->size == size_info->self_size);
3565 : }
3566 7452066 : }
3567 :
3568 :
3569 : /* Compute parameters of functions used by inliner using
3570 : current_function_decl. */
3571 :
3572 : static unsigned int
3573 5972550 : compute_fn_summary_for_current (void)
3574 : {
3575 5972550 : compute_fn_summary (cgraph_node::get (current_function_decl), true);
3576 5972550 : return 0;
3577 : }
3578 :
3579 : /* Estimate benefit devirtualizing indirect edge IE and return true if it can
3580 : be devirtualized and inlined, provided m_known_vals, m_known_contexts and
3581 : m_known_aggs in AVALS. Return false straight away if AVALS is NULL. */
3582 :
3583 : static bool
3584 4937894 : estimate_edge_devirt_benefit (struct cgraph_edge *ie,
3585 : int *size, int *time,
3586 : ipa_call_arg_values *avals)
3587 : {
3588 4937894 : tree target;
3589 4937894 : struct cgraph_node *callee;
3590 4937894 : class ipa_fn_summary *isummary;
3591 4937894 : enum availability avail;
3592 4937894 : bool speculative;
3593 :
3594 4937894 : if (!avals
3595 4937894 : || (!avals->m_known_vals.length() && !avals->m_known_contexts.length ()))
3596 : return false;
3597 1506826 : if (!opt_for_fn (ie->caller->decl, flag_indirect_inlining))
3598 : return false;
3599 :
3600 1500279 : target = ipa_get_indirect_edge_target
3601 1500279 : (ie->callee ? ie->speculative_call_indirect_edge () : ie,
3602 : avals, &speculative);
3603 1500279 : if (!target || speculative)
3604 : return false;
3605 :
3606 : /* If this is speculative call, turn its cost into 0; we will account
3607 : the call when processing the indirect call. */
3608 219653 : if (ie->callee)
3609 : {
3610 4549 : gcc_checking_assert (ie->speculative && *size > 0);
3611 4549 : *size = 0;
3612 4549 : *time = 0;
3613 : }
3614 : else
3615 : {
3616 : /* Account for difference in cost between indirect and direct calls. */
3617 215104 : *size -= (eni_size_weights.indirect_call_cost - eni_size_weights.call_cost);
3618 215104 : *time -= (eni_time_weights.indirect_call_cost - eni_time_weights.call_cost);
3619 : }
3620 219653 : gcc_checking_assert (*time >= 0);
3621 219653 : gcc_checking_assert (*size >= 0);
3622 :
3623 219653 : callee = cgraph_node::get (target);
3624 219653 : if (!callee || !callee->definition)
3625 : return false;
3626 198961 : callee = callee->function_symbol (&avail);
3627 198961 : if (avail < AVAIL_AVAILABLE)
3628 : return false;
3629 198911 : isummary = ipa_fn_summaries->get (callee);
3630 198911 : if (isummary == NULL)
3631 : return false;
3632 :
3633 198897 : return isummary->inlinable;
3634 : }
3635 :
3636 : /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
3637 : handle edge E with probability PROB. Set HINTS accordingly if edge may be
3638 : devirtualized. AVALS, if non-NULL, describes the context of the call site
3639 : as far as values of parameters are concerned. */
3640 :
3641 : static inline void
3642 208139933 : estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *min_size,
3643 : sreal *time, ipa_call_arg_values *avals,
3644 : ipa_hints *hints)
3645 : {
3646 208139933 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3647 208139933 : int call_size = es->call_stmt_size;
3648 208139933 : int call_time = es->call_stmt_time;
3649 208139933 : int cur_size;
3650 :
3651 203815627 : if ((!e->callee || e->speculative)
3652 208753521 : && estimate_edge_devirt_benefit (e, &call_size, &call_time, avals))
3653 : {
3654 198651 : if (hints && e->maybe_hot_p ())
3655 189281 : *hints |= INLINE_HINT_indirect_call;
3656 : }
3657 208139933 : cur_size = call_size * ipa_fn_summary::size_scale;
3658 208139933 : *size += cur_size;
3659 208139933 : if (min_size)
3660 29628319 : *min_size += cur_size;
3661 208139933 : if (time)
3662 197992578 : *time += ((sreal)call_time) * e->sreal_frequency ();
3663 208139933 : }
3664 :
3665 :
3666 : /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3667 : calls in NODE. POSSIBLE_TRUTHS and AVALS describe the context of the call
3668 : site.
3669 :
3670 : Helper for estimate_calls_size_and_time which does the same but
3671 : (in most cases) faster. */
3672 :
3673 : static void
3674 73737375 : estimate_calls_size_and_time_1 (struct cgraph_node *node, int *size,
3675 : int *min_size, sreal *time,
3676 : ipa_hints *hints,
3677 : clause_t possible_truths,
3678 : ipa_call_arg_values *avals)
3679 : {
3680 73737375 : struct cgraph_edge *e;
3681 351585049 : for (e = node->callees; e; e = e->next_callee)
3682 : {
3683 277847674 : if (!e->inline_failed)
3684 : {
3685 52948228 : gcc_checking_assert (!ipa_call_summaries->get (e));
3686 52948228 : estimate_calls_size_and_time_1 (e->callee, size, min_size, time,
3687 : hints, possible_truths, avals);
3688 :
3689 52948228 : continue;
3690 : }
3691 224899446 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3692 :
3693 : /* Do not care about zero sized builtins. */
3694 224899446 : if (!es->call_stmt_size)
3695 : {
3696 30178771 : gcc_checking_assert (!es->call_stmt_time);
3697 30178771 : continue;
3698 : }
3699 194720675 : if (!es->predicate
3700 194720675 : || es->predicate->evaluate (possible_truths))
3701 : {
3702 : /* Predicates of calls shall not use NOT_CHANGED codes,
3703 : so we do not need to compute probabilities. */
3704 192452679 : estimate_edge_size_and_time (e, size,
3705 192452679 : es->predicate ? NULL : min_size,
3706 : time, avals, hints);
3707 : }
3708 : }
3709 77771228 : for (e = node->indirect_calls; e; e = e->next_callee)
3710 : {
3711 4033853 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3712 4033853 : if (!es->predicate
3713 4033853 : || es->predicate->evaluate (possible_truths))
3714 4006339 : estimate_edge_size_and_time (e, size,
3715 4006339 : es->predicate ? NULL : min_size,
3716 : time, avals, hints);
3717 : }
3718 73737375 : }
3719 :
3720 : /* Populate sum->call_size_time_table for edges from NODE. */
3721 :
3722 : static void
3723 3247786 : summarize_calls_size_and_time (struct cgraph_node *node,
3724 : ipa_fn_summary *sum)
3725 : {
3726 3247786 : struct cgraph_edge *e;
3727 15180740 : for (e = node->callees; e; e = e->next_callee)
3728 : {
3729 11932954 : if (!e->inline_failed)
3730 : {
3731 1462769 : gcc_checking_assert (!ipa_call_summaries->get (e));
3732 1462769 : summarize_calls_size_and_time (e->callee, sum);
3733 1462769 : continue;
3734 : }
3735 10470185 : int size = 0;
3736 10470185 : sreal time = 0;
3737 :
3738 10470185 : estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
3739 :
3740 10470185 : ipa_predicate pred = true;
3741 10470185 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3742 :
3743 10470185 : if (es->predicate)
3744 1887532 : pred = *es->predicate;
3745 10470185 : sum->account_size_time (size, time, pred, pred, true);
3746 : }
3747 3565753 : for (e = node->indirect_calls; e; e = e->next_callee)
3748 : {
3749 317967 : int size = 0;
3750 317967 : sreal time = 0;
3751 :
3752 317967 : estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
3753 317967 : ipa_predicate pred = true;
3754 317967 : class ipa_call_summary *es = ipa_call_summaries->get (e);
3755 :
3756 317967 : if (es->predicate)
3757 81270 : pred = *es->predicate;
3758 317967 : sum->account_size_time (size, time, pred, pred, true);
3759 : }
3760 3247786 : }
3761 :
3762 : /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3763 : calls in NODE. POSSIBLE_TRUTHS and AVALS (the latter if non-NULL) describe
3764 : context of the call site. */
3765 :
3766 : static void
3767 20789168 : estimate_calls_size_and_time (struct cgraph_node *node, int *size,
3768 : int *min_size, sreal *time,
3769 : ipa_hints *hints,
3770 : clause_t possible_truths,
3771 : ipa_call_arg_values *avals)
3772 : {
3773 20789168 : class ipa_fn_summary *sum = ipa_fn_summaries->get (node);
3774 20789168 : bool use_table = true;
3775 :
3776 20789168 : gcc_assert (node->callees || node->indirect_calls);
3777 :
3778 : /* During early inlining we do not calculate info for very
3779 : large functions and thus there is no need for producing
3780 : summaries. */
3781 20789168 : if (!ipa_node_params_sum)
3782 : use_table = false;
3783 : /* Do not calculate summaries for simple wrappers; it is waste
3784 : of memory. */
3785 11218326 : else if (node->callees && !node->indirect_calls
3786 10442302 : && node->callees->inline_failed && !node->callees->next_callee)
3787 : use_table = false;
3788 : /* If there is an indirect edge that may be optimized, we need
3789 : to go the slow way. */
3790 8995535 : else if (avals
3791 8995535 : && (avals->m_known_vals.length ()
3792 3456440 : || avals->m_known_contexts.length ()
3793 3302093 : || avals->m_known_aggs.length ()))
3794 : {
3795 4163717 : ipa_node_params *params_summary = ipa_node_params_sum->get (node);
3796 4163717 : unsigned int nargs = params_summary
3797 4163717 : ? ipa_get_param_count (params_summary) : 0;
3798 :
3799 14004378 : for (unsigned int i = 0; i < nargs && use_table; i++)
3800 : {
3801 9840661 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3802 9840661 : && (avals->safe_sval_at (i)
3803 280202 : || (ipa_argagg_value_list (avals).value_for_index_p (i))))
3804 : use_table = false;
3805 9743674 : else if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3806 9743674 : && (avals->m_known_contexts.length () > i
3807 9992304 : && !avals->m_known_contexts[i].useless_p ()))
3808 : use_table = false;
3809 : }
3810 : }
3811 :
3812 : /* Fast path is via the call size time table. */
3813 4163717 : if (use_table)
3814 : {
3815 : /* Build summary if it is absent. */
3816 8723868 : if (!sum->call_size_time_table.length ())
3817 : {
3818 892254 : ipa_predicate true_pred = true;
3819 892254 : sum->account_size_time (0, 0, true_pred, true_pred, true);
3820 892254 : summarize_calls_size_and_time (node, sum);
3821 : }
3822 :
3823 8723868 : int old_size = *size;
3824 8723868 : sreal old_time = time ? *time : 0;
3825 :
3826 8723868 : if (min_size)
3827 8723868 : *min_size += sum->call_size_time_table[0].size;
3828 :
3829 8723868 : unsigned int i;
3830 8723868 : size_time_entry *e;
3831 :
3832 : /* Walk the table and account sizes and times. */
3833 23007212 : for (i = 0; sum->call_size_time_table.iterate (i, &e);
3834 : i++)
3835 14283344 : if (e->exec_predicate.evaluate (possible_truths))
3836 : {
3837 13366556 : *size += e->size;
3838 13366556 : if (time)
3839 11180275 : *time += e->time;
3840 : }
3841 :
3842 : /* Be careful and see if both methods agree. */
3843 21 : if ((flag_checking || dump_file)
3844 : /* Do not try to sanity check when we know we lost some
3845 : precision. */
3846 8723868 : && sum->call_size_time_table.length ()
3847 : < ipa_fn_summary::max_size_time_table_size)
3848 : {
3849 8723847 : estimate_calls_size_and_time_1 (node, &old_size, NULL, &old_time, NULL,
3850 : possible_truths, avals);
3851 8723847 : gcc_assert (*size == old_size);
3852 16003122 : if (time && (*time - old_time > 1 || *time - old_time < -1)
3853 8723880 : && dump_file)
3854 0 : fprintf (dump_file, "Time mismatch in call summary %f!=%f\n",
3855 : old_time.to_double (),
3856 : time->to_double ());
3857 : }
3858 : }
3859 : /* Slow path by walking all edges. */
3860 : else
3861 12065300 : estimate_calls_size_and_time_1 (node, size, min_size, time, hints,
3862 : possible_truths, avals);
3863 20789168 : }
3864 :
3865 : /* Main constructor for ipa call context. Memory allocation of ARG_VALUES
3866 : is owned by the caller. INLINE_PARAM_SUMMARY is also owned by the
3867 : caller. */
3868 :
3869 19920174 : ipa_call_context::ipa_call_context (cgraph_node *node, clause_t possible_truths,
3870 : clause_t nonspec_possible_truths,
3871 : vec<inline_param_summary>
3872 : inline_param_summary,
3873 : ipa_auto_call_arg_values *arg_values)
3874 19920174 : : m_node (node), m_possible_truths (possible_truths),
3875 19920174 : m_nonspec_possible_truths (nonspec_possible_truths),
3876 19920174 : m_inline_param_summary (inline_param_summary),
3877 19920174 : m_avals (arg_values)
3878 : {
3879 19920174 : }
3880 :
3881 : /* Set THIS to be a duplicate of CTX. Copy all relevant info. */
3882 :
3883 : void
3884 1995701 : ipa_cached_call_context::duplicate_from (const ipa_call_context &ctx)
3885 : {
3886 1995701 : m_node = ctx.m_node;
3887 1995701 : m_possible_truths = ctx.m_possible_truths;
3888 1995701 : m_nonspec_possible_truths = ctx.m_nonspec_possible_truths;
3889 1995701 : ipa_node_params *params_summary = ipa_node_params_sum->get (m_node);
3890 1995701 : unsigned int nargs = params_summary
3891 1995701 : ? ipa_get_param_count (params_summary) : 0;
3892 :
3893 1995701 : m_inline_param_summary = vNULL;
3894 : /* Copy the info only if there is at least one useful entry. */
3895 1995701 : if (ctx.m_inline_param_summary.exists ())
3896 : {
3897 1772624 : unsigned int n = MIN (ctx.m_inline_param_summary.length (), nargs);
3898 :
3899 4058482 : for (unsigned int i = 0; i < n; i++)
3900 3311171 : if (ipa_is_param_used_by_ipa_predicates (params_summary, i)
3901 4712938 : && !ctx.m_inline_param_summary[i].useless_p ())
3902 : {
3903 1025313 : m_inline_param_summary
3904 1025313 : = ctx.m_inline_param_summary.copy ();
3905 1025313 : break;
3906 : }
3907 : }
3908 1995701 : m_avals.m_known_vals = vNULL;
3909 1995701 : if (ctx.m_avals.m_known_vals.exists ())
3910 : {
3911 1995701 : unsigned int n = MIN (ctx.m_avals.m_known_vals.length (), nargs);
3912 :
3913 4615157 : for (unsigned int i = 0; i < n; i++)
3914 2646421 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3915 2646421 : && ctx.m_avals.m_known_vals[i])
3916 : {
3917 26965 : m_avals.m_known_vals = ctx.m_avals.m_known_vals.copy ();
3918 26965 : break;
3919 : }
3920 : }
3921 :
3922 1995701 : m_avals.m_known_contexts = vNULL;
3923 1995701 : if (ctx.m_avals.m_known_contexts.exists ())
3924 : {
3925 1995701 : unsigned int n = MIN (ctx.m_avals.m_known_contexts.length (), nargs);
3926 :
3927 1998569 : for (unsigned int i = 0; i < n; i++)
3928 28497 : if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3929 28497 : && !ctx.m_avals.m_known_contexts[i].useless_p ())
3930 : {
3931 25629 : m_avals.m_known_contexts = ctx.m_avals.m_known_contexts.copy ();
3932 25629 : break;
3933 : }
3934 : }
3935 :
3936 1995701 : m_avals.m_known_aggs = vNULL;
3937 1995701 : if (ctx.m_avals.m_known_aggs.exists ())
3938 : {
3939 1995701 : const ipa_argagg_value_list avl (&ctx.m_avals);
3940 6643054 : for (unsigned int i = 0; i < nargs; i++)
3941 4650304 : if (ipa_is_param_used_by_indirect_call (params_summary, i)
3942 4650304 : && avl.value_for_index_p (i))
3943 : {
3944 2951 : m_avals.m_known_aggs = ctx.m_avals.m_known_aggs.copy ();
3945 2951 : break;
3946 : }
3947 : }
3948 :
3949 1995701 : m_avals.m_known_value_ranges = vNULL;
3950 1995701 : }
3951 :
3952 : /* Release memory used by known_vals/contexts/aggs vectors. and
3953 : inline_param_summary. */
3954 :
3955 : void
3956 3248610 : ipa_cached_call_context::release ()
3957 : {
3958 : /* See if context is initialized at first place. */
3959 3248610 : if (!m_node)
3960 : return;
3961 1995701 : m_avals.m_known_aggs.release ();
3962 1995701 : m_avals.m_known_vals.release ();
3963 1995701 : m_avals.m_known_contexts.release ();
3964 1995701 : m_inline_param_summary.release ();
3965 : }
3966 :
3967 : /* Return true if CTX describes the same call context as THIS. */
3968 :
3969 : bool
3970 7004800 : ipa_call_context::equal_to (const ipa_call_context &ctx)
3971 : {
3972 7004800 : if (m_node != ctx.m_node
3973 5751891 : || m_possible_truths != ctx.m_possible_truths
3974 5150317 : || m_nonspec_possible_truths != ctx.m_nonspec_possible_truths)
3975 : return false;
3976 :
3977 5150317 : ipa_node_params *params_summary = ipa_node_params_sum->get (m_node);
3978 5150317 : unsigned int nargs = params_summary
3979 5150317 : ? ipa_get_param_count (params_summary) : 0;
3980 :
3981 5150317 : if (m_inline_param_summary.exists () || ctx.m_inline_param_summary.exists ())
3982 : {
3983 15456113 : for (unsigned int i = 0; i < nargs; i++)
3984 : {
3985 10668448 : if (!ipa_is_param_used_by_ipa_predicates (params_summary, i))
3986 3469426 : continue;
3987 7199022 : if (i >= m_inline_param_summary.length ()
3988 4947324 : || m_inline_param_summary[i].useless_p ())
3989 : {
3990 3240434 : if (i < ctx.m_inline_param_summary.length ()
3991 3240434 : && !ctx.m_inline_param_summary[i].useless_p ())
3992 : return false;
3993 3190896 : continue;
3994 : }
3995 3958588 : if (i >= ctx.m_inline_param_summary.length ()
3996 3958585 : || ctx.m_inline_param_summary[i].useless_p ())
3997 : {
3998 47589 : if (i < m_inline_param_summary.length ()
3999 47589 : && !m_inline_param_summary[i].useless_p ())
4000 : return false;
4001 0 : continue;
4002 : }
4003 3910999 : if (!m_inline_param_summary[i].equal_to
4004 3910999 : (ctx.m_inline_param_summary[i]))
4005 : return false;
4006 : }
4007 : }
4008 5022718 : if (m_avals.m_known_vals.exists () || ctx.m_avals.m_known_vals.exists ())
4009 : {
4010 15484459 : for (unsigned int i = 0; i < nargs; i++)
4011 : {
4012 10472468 : if (!ipa_is_param_used_by_indirect_call (params_summary, i))
4013 10254262 : continue;
4014 218206 : if (i >= m_avals.m_known_vals.length () || !m_avals.m_known_vals[i])
4015 : {
4016 160662 : if (i < ctx.m_avals.m_known_vals.length ()
4017 160662 : && ctx.m_avals.m_known_vals[i])
4018 : return false;
4019 160573 : continue;
4020 : }
4021 57544 : if (i >= ctx.m_avals.m_known_vals.length ()
4022 57544 : || !ctx.m_avals.m_known_vals[i])
4023 : {
4024 : if (i < m_avals.m_known_vals.length () && m_avals.m_known_vals[i])
4025 : return false;
4026 : continue;
4027 : }
4028 57454 : if (m_avals.m_known_vals[i] != ctx.m_avals.m_known_vals[i])
4029 : return false;
4030 : }
4031 : }
4032 5011991 : if (m_avals.m_known_contexts.exists ()
4033 5011991 : || ctx.m_avals.m_known_contexts.exists ())
4034 : {
4035 15469078 : for (unsigned int i = 0; i < nargs; i++)
4036 : {
4037 10458565 : if (!ipa_is_param_used_by_polymorphic_call (params_summary, i))
4038 10314385 : continue;
4039 144180 : if (i >= m_avals.m_known_contexts.length ()
4040 143019 : || m_avals.m_known_contexts[i].useless_p ())
4041 : {
4042 1161 : if (i < ctx.m_avals.m_known_contexts.length ()
4043 1161 : && !ctx.m_avals.m_known_contexts[i].useless_p ())
4044 : return false;
4045 1149 : continue;
4046 : }
4047 143019 : if (i >= ctx.m_avals.m_known_contexts.length ()
4048 143019 : || ctx.m_avals.m_known_contexts[i].useless_p ())
4049 : {
4050 8 : if (i < m_avals.m_known_contexts.length ()
4051 8 : && !m_avals.m_known_contexts[i].useless_p ())
4052 : return false;
4053 0 : continue;
4054 : }
4055 143011 : if (!m_avals.m_known_contexts[i].equal_to
4056 143011 : (ctx.m_avals.m_known_contexts[i]))
4057 : return false;
4058 : }
4059 : }
4060 5010513 : if (m_avals.m_known_aggs.exists () || ctx.m_avals.m_known_aggs.exists ())
4061 : {
4062 : unsigned i = 0, j = 0;
4063 11880060 : while (i < m_avals.m_known_aggs.length ()
4064 5939276 : || j < ctx.m_avals.m_known_aggs.length ())
4065 : {
4066 930177 : if (i >= m_avals.m_known_aggs.length ())
4067 : {
4068 925276 : int idx2 = ctx.m_avals.m_known_aggs[j].index;
4069 925276 : if (ipa_is_param_used_by_indirect_call (params_summary, idx2))
4070 : return false;
4071 924716 : j++;
4072 924716 : continue;
4073 924716 : }
4074 4901 : if (j >= ctx.m_avals.m_known_aggs.length ())
4075 : {
4076 603 : int idx1 = m_avals.m_known_aggs[i].index;
4077 603 : if (ipa_is_param_used_by_indirect_call (params_summary, idx1))
4078 : return false;
4079 8 : i++;
4080 8 : continue;
4081 8 : }
4082 :
4083 4298 : int idx1 = m_avals.m_known_aggs[i].index;
4084 4298 : int idx2 = ctx.m_avals.m_known_aggs[j].index;
4085 4298 : if (idx1 < idx2)
4086 : {
4087 0 : if (ipa_is_param_used_by_indirect_call (params_summary, idx1))
4088 : return false;
4089 0 : i++;
4090 0 : continue;
4091 : }
4092 4298 : if (idx1 > idx2)
4093 : {
4094 0 : if (ipa_is_param_used_by_indirect_call (params_summary, idx2))
4095 : return false;
4096 0 : j++;
4097 0 : continue;
4098 : }
4099 4298 : if (!ipa_is_param_used_by_indirect_call (params_summary, idx1))
4100 : {
4101 482 : i++;
4102 482 : j++;
4103 482 : continue;
4104 : }
4105 :
4106 3816 : if ((m_avals.m_known_aggs[i].unit_offset
4107 3816 : != ctx.m_avals.m_known_aggs[j].unit_offset)
4108 3808 : || (m_avals.m_known_aggs[i].by_ref
4109 3808 : != ctx.m_avals.m_known_aggs[j].by_ref)
4110 7624 : || !operand_equal_p (m_avals.m_known_aggs[i].value,
4111 3808 : ctx.m_avals.m_known_aggs[j].value))
4112 : return false;
4113 3557 : i++;
4114 3557 : j++;
4115 : }
4116 : }
4117 : return true;
4118 : }
4119 :
4120 : /* Fill in the selected fields in ESTIMATES with value estimated for call in
4121 : this context. Always compute size and min_size. Only compute time and
4122 : nonspecialized_time if EST_TIMES is true. Only compute hints if EST_HINTS
4123 : is true. */
4124 :
4125 : void
4126 19887656 : ipa_call_context::estimate_size_and_time (ipa_call_estimates *estimates,
4127 : bool est_times, bool est_hints)
4128 : {
4129 19887656 : class ipa_fn_summary *info = ipa_fn_summaries->get (m_node);
4130 19887656 : size_time_entry *e;
4131 19887656 : int size = 0;
4132 19887656 : sreal time = 0;
4133 19887656 : int min_size = 0;
4134 19887656 : ipa_hints hints = 0;
4135 19887656 : sreal loops_with_known_iterations = 0;
4136 19887656 : sreal loops_with_known_strides = 0;
4137 19887656 : int i;
4138 :
4139 19887656 : if (dump_file && (dump_flags & TDF_DETAILS))
4140 : {
4141 1333 : bool found = false;
4142 1333 : fprintf (dump_file, " Estimating body: %s\n"
4143 : " Known to be false: ", m_node->dump_name ());
4144 :
4145 4347 : for (i = ipa_predicate::not_inlined_condition;
4146 8694 : i < (ipa_predicate::first_dynamic_condition
4147 7144 : + (int) vec_safe_length (info->conds)); i++)
4148 3014 : if (!(m_possible_truths & (1 << i)))
4149 : {
4150 1795 : if (found)
4151 519 : fprintf (dump_file, ", ");
4152 1795 : found = true;
4153 1795 : dump_condition (dump_file, info->conds, i);
4154 : }
4155 : }
4156 :
4157 19887656 : if (m_node->callees || m_node->indirect_calls)
4158 26432821 : estimate_calls_size_and_time (m_node, &size, &min_size,
4159 : est_times ? &time : NULL,
4160 : est_hints ? &hints : NULL, m_possible_truths,
4161 : &m_avals);
4162 :
4163 19887656 : sreal nonspecialized_time = time;
4164 :
4165 19887656 : min_size += info->size_time_table[0].size;
4166 135594765 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
4167 : {
4168 115707109 : bool exec = e->exec_predicate.evaluate (m_nonspec_possible_truths);
4169 :
4170 : /* Because predicates are conservative, it can happen that nonconst is 1
4171 : but exec is 0. */
4172 115707109 : if (exec)
4173 : {
4174 113269235 : bool nonconst = e->nonconst_predicate.evaluate (m_possible_truths);
4175 :
4176 113269235 : gcc_checking_assert (e->time >= 0);
4177 113269235 : gcc_checking_assert (time >= 0);
4178 :
4179 : /* We compute specialized size only because size of nonspecialized
4180 : copy is context independent.
4181 :
4182 : The difference between nonspecialized execution and specialized is
4183 : that nonspecialized is not going to have optimized out computations
4184 : known to be constant in a specialized setting. */
4185 113269235 : if (nonconst)
4186 56992416 : size += e->size;
4187 113269235 : if (!est_times)
4188 59317853 : continue;
4189 53951382 : nonspecialized_time += e->time;
4190 53951382 : if (!nonconst)
4191 : ;
4192 25946873 : else if (!m_inline_param_summary.exists ())
4193 : {
4194 2464399 : if (nonconst)
4195 2464399 : time += e->time;
4196 : }
4197 : else
4198 : {
4199 23482474 : int prob = e->nonconst_predicate.probability
4200 23482474 : (info->conds, m_possible_truths,
4201 : m_inline_param_summary);
4202 23482474 : gcc_checking_assert (prob >= 0);
4203 23482474 : gcc_checking_assert (prob <= REG_BR_PROB_BASE);
4204 23482474 : if (prob == REG_BR_PROB_BASE)
4205 19623522 : time += e->time;
4206 : else
4207 3858952 : time += e->time * prob / REG_BR_PROB_BASE;
4208 : }
4209 53951382 : gcc_checking_assert (time >= 0);
4210 : }
4211 : }
4212 19887656 : gcc_checking_assert (info->size_time_table[0].exec_predicate == true);
4213 19887656 : gcc_checking_assert (info->size_time_table[0].nonconst_predicate == true);
4214 19887656 : gcc_checking_assert (min_size >= 0);
4215 19887656 : gcc_checking_assert (size >= 0);
4216 19887656 : gcc_checking_assert (time >= 0);
4217 : /* nonspecialized_time should be always bigger than specialized time.
4218 : Roundoff issues however may get into the way. */
4219 19887656 : gcc_checking_assert ((nonspecialized_time - time * 99 / 100) >= -1);
4220 :
4221 : /* Roundoff issues may make specialized time bigger than nonspecialized
4222 : time. We do not really want that to happen because some heuristics
4223 : may get confused by seeing negative speedups. */
4224 19887656 : if (time > nonspecialized_time)
4225 0 : time = nonspecialized_time;
4226 :
4227 19887656 : if (est_hints)
4228 : {
4229 7211515 : if (info->scc_no)
4230 234170 : hints |= INLINE_HINT_in_scc;
4231 7211515 : if (DECL_DECLARED_INLINE_P (m_node->decl))
4232 4540092 : hints |= INLINE_HINT_declared_inline;
4233 7211515 : if (info->builtin_constant_p_parms.length ()
4234 9826 : && DECL_DECLARED_INLINE_P (m_node->decl))
4235 9637 : hints |= INLINE_HINT_builtin_constant_p;
4236 :
4237 7211515 : ipa_freqcounting_predicate *fcp;
4238 7771705 : for (i = 0; vec_safe_iterate (info->loop_iterations, i, &fcp); i++)
4239 560190 : if (!fcp->predicate->evaluate (m_possible_truths))
4240 : {
4241 372728 : hints |= INLINE_HINT_loop_iterations;
4242 372728 : loops_with_known_iterations += fcp->freq;
4243 : }
4244 7211515 : estimates->loops_with_known_iterations = loops_with_known_iterations;
4245 :
4246 7506112 : for (i = 0; vec_safe_iterate (info->loop_strides, i, &fcp); i++)
4247 294597 : if (!fcp->predicate->evaluate (m_possible_truths))
4248 : {
4249 263751 : hints |= INLINE_HINT_loop_stride;
4250 263751 : loops_with_known_strides += fcp->freq;
4251 : }
4252 7211515 : estimates->loops_with_known_strides = loops_with_known_strides;
4253 : }
4254 :
4255 19887656 : size = RDIV (size, ipa_fn_summary::size_scale);
4256 19887656 : min_size = RDIV (min_size, ipa_fn_summary::size_scale);
4257 :
4258 19887656 : if (dump_file && (dump_flags & TDF_DETAILS))
4259 : {
4260 1333 : fprintf (dump_file, "\n size:%i", (int) size);
4261 1333 : if (est_times)
4262 1096 : fprintf (dump_file, " time:%f nonspec time:%f",
4263 : time.to_double (), nonspecialized_time.to_double ());
4264 1333 : if (est_hints)
4265 1096 : fprintf (dump_file, " loops with known iterations:%f "
4266 : "known strides:%f", loops_with_known_iterations.to_double (),
4267 : loops_with_known_strides.to_double ());
4268 1333 : fprintf (dump_file, "\n");
4269 : }
4270 19887656 : if (est_times)
4271 : {
4272 7211515 : estimates->time = time;
4273 7211515 : estimates->nonspecialized_time = nonspecialized_time;
4274 : }
4275 19887656 : estimates->size = size;
4276 19887656 : estimates->min_size = min_size;
4277 19887656 : if (est_hints)
4278 7211515 : estimates->hints = hints;
4279 19887656 : return;
4280 : }
4281 :
4282 :
4283 : /* Estimate size and time needed to execute callee of EDGE assuming that
4284 : parameters known to be constant at caller of EDGE are propagated.
4285 : KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
4286 : and types for parameters. */
4287 :
4288 : void
4289 235432 : estimate_ipcp_clone_size_and_time (struct cgraph_node *node,
4290 : ipa_auto_call_arg_values *avals,
4291 : ipa_call_estimates *estimates)
4292 : {
4293 235432 : clause_t clause, nonspec_clause;
4294 :
4295 235432 : evaluate_conditions_for_known_args (node, false, avals, &clause,
4296 : &nonspec_clause, NULL);
4297 235432 : ipa_call_context ctx (node, clause, nonspec_clause, vNULL, avals);
4298 235432 : ctx.estimate_size_and_time (estimates);
4299 235432 : }
4300 :
4301 : /* Return stack frame offset where frame of NODE is supposed to start inside
4302 : of the function it is inlined to.
4303 : Return 0 for functions that are not inlined. */
4304 :
4305 : HOST_WIDE_INT
4306 5265656 : ipa_get_stack_frame_offset (struct cgraph_node *node)
4307 : {
4308 5265656 : HOST_WIDE_INT offset = 0;
4309 5265656 : if (!node->inlined_to)
4310 : return 0;
4311 4185744 : node = node->callers->caller;
4312 5420028 : while (true)
4313 : {
4314 4802886 : offset += ipa_size_summaries->get (node)->estimated_self_stack_size;
4315 4802886 : if (!node->inlined_to)
4316 : return offset;
4317 617142 : node = node->callers->caller;
4318 : }
4319 : }
4320 :
4321 :
4322 : /* Update summary information of inline clones after inlining.
4323 : Compute peak stack usage. */
4324 :
4325 : static void
4326 4913614 : inline_update_callee_summaries (struct cgraph_node *node, int depth)
4327 : {
4328 4913614 : struct cgraph_edge *e;
4329 :
4330 4913614 : ipa_propagate_frequency (node);
4331 9513012 : for (e = node->callees; e; e = e->next_callee)
4332 : {
4333 4599398 : if (!e->inline_failed)
4334 728423 : inline_update_callee_summaries (e->callee, depth);
4335 : else
4336 3870975 : ipa_call_summaries->get (e)->loop_depth += depth;
4337 : }
4338 5030603 : for (e = node->indirect_calls; e; e = e->next_callee)
4339 116989 : ipa_call_summaries->get (e)->loop_depth += depth;
4340 4913614 : }
4341 :
4342 : /* Update change_prob and points_to_local_or_readonly_memory of EDGE after
4343 : INLINED_EDGE has been inlined.
4344 :
4345 : When function A is inlined in B and A calls C with parameter that
4346 : changes with probability PROB1 and C is known to be passthrough
4347 : of argument if B that change with probability PROB2, the probability
4348 : of change is now PROB1*PROB2. */
4349 :
4350 : static void
4351 3988274 : remap_edge_params (struct cgraph_edge *inlined_edge,
4352 : struct cgraph_edge *edge)
4353 : {
4354 3988274 : if (ipa_node_params_sum)
4355 : {
4356 2797138 : int i;
4357 2797138 : ipa_edge_args *args = ipa_edge_args_sum->get (edge);
4358 2797138 : if (!args)
4359 : return;
4360 1598264 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
4361 1598264 : class ipa_call_summary *inlined_es
4362 1598264 : = ipa_call_summaries->get (inlined_edge);
4363 :
4364 1598264 : if (es->param.length () == 0)
4365 : return;
4366 :
4367 9286697 : for (i = 0; i < ipa_get_cs_argument_count (args); i++)
4368 : {
4369 3129944 : struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
4370 3129944 : if (jfunc->type == IPA_JF_PASS_THROUGH
4371 2302034 : || jfunc->type == IPA_JF_ANCESTOR)
4372 : {
4373 970972 : int id = jfunc->type == IPA_JF_PASS_THROUGH
4374 970972 : ? ipa_get_jf_pass_through_formal_id (jfunc)
4375 143062 : : ipa_get_jf_ancestor_formal_id (jfunc);
4376 1941861 : if (id < (int) inlined_es->param.length ())
4377 : {
4378 970881 : int prob1 = es->param[i].change_prob;
4379 970881 : int prob2 = inlined_es->param[id].change_prob;
4380 970881 : int prob = combine_probabilities (prob1, prob2);
4381 :
4382 970881 : if (prob1 && prob2 && !prob)
4383 970881 : prob = 1;
4384 :
4385 970881 : es->param[i].change_prob = prob;
4386 :
4387 1941762 : if (inlined_es
4388 970881 : ->param[id].points_to_local_or_readonly_memory)
4389 215325 : es->param[i].points_to_local_or_readonly_memory = true;
4390 1941762 : if (inlined_es
4391 970881 : ->param[id].points_to_possible_sra_candidate)
4392 179288 : es->param[i].points_to_possible_sra_candidate = true;
4393 : }
4394 970972 : if (!es->param[i].points_to_local_or_readonly_memory
4395 : && jfunc->type == IPA_JF_CONST
4396 : && points_to_local_or_readonly_memory_p
4397 : (ipa_get_jf_constant (jfunc)))
4398 : es->param[i].points_to_local_or_readonly_memory = true;
4399 : }
4400 : }
4401 : }
4402 : }
4403 :
4404 : /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
4405 :
4406 : Remap predicates of callees of NODE. Rest of arguments match
4407 : remap_predicate.
4408 :
4409 : Also update change probabilities. */
4410 :
4411 : static void
4412 4913614 : remap_edge_summaries (struct cgraph_edge *inlined_edge,
4413 : struct cgraph_node *node,
4414 : class ipa_fn_summary *info,
4415 : class ipa_node_params *params_summary,
4416 : class ipa_fn_summary *callee_info,
4417 : const vec<int> &operand_map,
4418 : const vec<HOST_WIDE_INT> &offset_map,
4419 : clause_t possible_truths,
4420 : ipa_predicate *toplev_predicate)
4421 : {
4422 4913614 : struct cgraph_edge *e, *next;
4423 9512555 : for (e = node->callees; e; e = next)
4424 : {
4425 4598941 : ipa_predicate p;
4426 4598941 : next = e->next_callee;
4427 :
4428 4598941 : if (e->inline_failed)
4429 : {
4430 3870518 : class ipa_call_summary *es = ipa_call_summaries->get (e);
4431 3870518 : remap_edge_params (inlined_edge, e);
4432 :
4433 3870518 : if (es->predicate)
4434 : {
4435 1301434 : p = es->predicate->remap_after_inlining
4436 1301434 : (info, params_summary,
4437 : callee_info, operand_map,
4438 : offset_map, possible_truths,
4439 : *toplev_predicate);
4440 1301434 : edge_set_predicate (e, &p);
4441 : }
4442 : else
4443 2569084 : edge_set_predicate (e, toplev_predicate);
4444 : }
4445 : else
4446 728423 : remap_edge_summaries (inlined_edge, e->callee, info,
4447 : params_summary, callee_info,
4448 : operand_map, offset_map, possible_truths,
4449 : toplev_predicate);
4450 : }
4451 5031370 : for (e = node->indirect_calls; e; e = next)
4452 : {
4453 117756 : class ipa_call_summary *es = ipa_call_summaries->get (e);
4454 117756 : ipa_predicate p;
4455 117756 : next = e->next_callee;
4456 :
4457 117756 : remap_edge_params (inlined_edge, e);
4458 117756 : if (es->predicate)
4459 : {
4460 25220 : p = es->predicate->remap_after_inlining
4461 25220 : (info, params_summary,
4462 : callee_info, operand_map, offset_map,
4463 : possible_truths, *toplev_predicate);
4464 25220 : edge_set_predicate (e, &p);
4465 : }
4466 : else
4467 92536 : edge_set_predicate (e, toplev_predicate);
4468 : }
4469 4913614 : }
4470 :
4471 : /* Run remap_after_inlining on each predicate in V. */
4472 :
4473 : static void
4474 8370382 : remap_freqcounting_predicate (class ipa_fn_summary *info,
4475 : class ipa_node_params *params_summary,
4476 : class ipa_fn_summary *callee_info,
4477 : vec<ipa_freqcounting_predicate, va_gc> *v,
4478 : const vec<int> &operand_map,
4479 : const vec<HOST_WIDE_INT> &offset_map,
4480 : clause_t possible_truths,
4481 : ipa_predicate *toplev_predicate)
4482 :
4483 : {
4484 8370382 : ipa_freqcounting_predicate *fcp;
4485 8398134 : for (int i = 0; vec_safe_iterate (v, i, &fcp); i++)
4486 : {
4487 27752 : ipa_predicate p
4488 27752 : = fcp->predicate->remap_after_inlining (info, params_summary,
4489 : callee_info, operand_map,
4490 : offset_map, possible_truths,
4491 : *toplev_predicate);
4492 39015 : if (p != false && p != true)
4493 2679 : *fcp->predicate &= p;
4494 : }
4495 8370382 : }
4496 :
4497 : /* We inlined EDGE. Update summary of the function we inlined into. */
4498 :
4499 : void
4500 4185191 : ipa_merge_fn_summary_after_inlining (struct cgraph_edge *edge)
4501 : {
4502 4185191 : ipa_fn_summary *callee_info = ipa_fn_summaries->get (edge->callee);
4503 3894428 : struct cgraph_node *to = (edge->caller->inlined_to
4504 4185191 : ? edge->caller->inlined_to : edge->caller);
4505 4185191 : class ipa_fn_summary *info = ipa_fn_summaries->get (to);
4506 4185191 : clause_t clause = 0; /* not_inline is known to be false. */
4507 4185191 : size_time_entry *e;
4508 4185191 : auto_vec<int, 8> operand_map;
4509 4185191 : auto_vec<HOST_WIDE_INT, 8> offset_map;
4510 4185191 : int i;
4511 4185191 : ipa_predicate toplev_predicate;
4512 4185191 : class ipa_call_summary *es = ipa_call_summaries->get (edge);
4513 4185191 : ipa_node_params *params_summary = (ipa_node_params_sum
4514 4185191 : ? ipa_node_params_sum->get (to) : NULL);
4515 :
4516 4185191 : if (es->predicate)
4517 352046 : toplev_predicate = *es->predicate;
4518 : else
4519 3833145 : toplev_predicate = true;
4520 :
4521 4185191 : info->fp_expressions |= callee_info->fp_expressions;
4522 4185191 : info->target_info |= callee_info->target_info;
4523 :
4524 4185191 : if (callee_info->conds)
4525 : {
4526 3066505 : ipa_auto_call_arg_values avals;
4527 3066505 : evaluate_properties_for_edge (edge, true, &clause, NULL, &avals, false);
4528 3066505 : }
4529 4185191 : if (ipa_node_params_sum && callee_info->conds)
4530 : {
4531 829494 : ipa_edge_args *args = ipa_edge_args_sum->get (edge);
4532 829494 : int count = args ? ipa_get_cs_argument_count (args) : 0;
4533 829355 : int i;
4534 :
4535 829355 : if (count)
4536 : {
4537 829355 : operand_map.safe_grow_cleared (count, true);
4538 829355 : offset_map.safe_grow_cleared (count, true);
4539 : }
4540 2664641 : for (i = 0; i < count; i++)
4541 : {
4542 1835147 : struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
4543 1835147 : int map = -1;
4544 :
4545 : /* TODO: handle non-NOPs when merging. */
4546 1835147 : if (jfunc->type == IPA_JF_PASS_THROUGH)
4547 : {
4548 298640 : if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
4549 295125 : map = ipa_get_jf_pass_through_formal_id (jfunc);
4550 298640 : if (!ipa_get_jf_pass_through_agg_preserved (jfunc))
4551 200918 : offset_map[i] = -1;
4552 : }
4553 1536507 : else if (jfunc->type == IPA_JF_ANCESTOR)
4554 : {
4555 95954 : HOST_WIDE_INT offset = ipa_get_jf_ancestor_offset (jfunc);
4556 95954 : if (offset >= 0 && offset < INT_MAX)
4557 : {
4558 95954 : map = ipa_get_jf_ancestor_formal_id (jfunc);
4559 95954 : if (!ipa_get_jf_ancestor_agg_preserved (jfunc))
4560 57702 : offset = -1;
4561 95954 : offset_map[i] = offset;
4562 : }
4563 : }
4564 1835147 : operand_map[i] = map;
4565 3312684 : gcc_assert (map < ipa_get_param_count (params_summary));
4566 : }
4567 :
4568 : int ip;
4569 832416 : for (i = 0; callee_info->builtin_constant_p_parms.iterate (i, &ip); i++)
4570 2922 : if (ip < count && operand_map[ip] >= 0)
4571 62 : add_builtin_constant_p_parm (info, operand_map[ip]);
4572 : }
4573 4185191 : sreal freq = edge->sreal_frequency ();
4574 27250755 : for (i = 0; callee_info->size_time_table.iterate (i, &e); i++)
4575 : {
4576 18880373 : ipa_predicate p;
4577 18880373 : p = e->exec_predicate.remap_after_inlining
4578 18880373 : (info, params_summary,
4579 : callee_info, operand_map,
4580 : offset_map, clause,
4581 : toplev_predicate);
4582 18880373 : ipa_predicate nonconstp;
4583 18880373 : nonconstp = e->nonconst_predicate.remap_after_inlining
4584 18880373 : (info, params_summary,
4585 : callee_info, operand_map,
4586 : offset_map, clause,
4587 : toplev_predicate);
4588 28543717 : if (p != false && nonconstp != false)
4589 : {
4590 9331478 : sreal add_time = ((sreal)e->time * freq);
4591 9331478 : int prob = e->nonconst_predicate.probability (callee_info->conds,
4592 : clause, es->param);
4593 9331478 : if (prob != REG_BR_PROB_BASE)
4594 1137559 : add_time = add_time * prob / REG_BR_PROB_BASE;
4595 1137559 : if (prob != REG_BR_PROB_BASE
4596 1137559 : && dump_file && (dump_flags & TDF_DETAILS))
4597 : {
4598 3 : fprintf (dump_file, "\t\tScaling time by probability:%f\n",
4599 3 : (double) prob / REG_BR_PROB_BASE);
4600 : }
4601 9331478 : info->account_size_time (e->size, add_time, p, nonconstp);
4602 : }
4603 : }
4604 4185191 : remap_edge_summaries (edge, edge->callee, info, params_summary,
4605 : callee_info, operand_map,
4606 : offset_map, clause, &toplev_predicate);
4607 4185191 : remap_freqcounting_predicate (info, params_summary, callee_info,
4608 : info->loop_iterations, operand_map,
4609 : offset_map, clause, &toplev_predicate);
4610 4185191 : remap_freqcounting_predicate (info, params_summary, callee_info,
4611 : info->loop_strides, operand_map,
4612 : offset_map, clause, &toplev_predicate);
4613 :
4614 4185191 : HOST_WIDE_INT stack_frame_offset = ipa_get_stack_frame_offset (edge->callee);
4615 4185191 : HOST_WIDE_INT peak = stack_frame_offset + callee_info->estimated_stack_size;
4616 :
4617 4185191 : if (info->estimated_stack_size < peak)
4618 126222 : info->estimated_stack_size = peak;
4619 :
4620 4185191 : inline_update_callee_summaries (edge->callee, es->loop_depth);
4621 4185191 : if (info->call_size_time_table.length ())
4622 : {
4623 892763 : int edge_size = 0;
4624 892763 : sreal edge_time = 0;
4625 :
4626 892763 : estimate_edge_size_and_time (edge, &edge_size, NULL, &edge_time, NULL, 0);
4627 : /* Unaccount size and time of the optimized out call. */
4628 892763 : info->account_size_time (-edge_size, -edge_time,
4629 892763 : es->predicate ? *es->predicate : true,
4630 892763 : es->predicate ? *es->predicate : true,
4631 : true);
4632 : /* Account new calls. */
4633 892763 : summarize_calls_size_and_time (edge->callee, info);
4634 : }
4635 :
4636 : /* Free summaries that are not maintained for inline clones/edges. */
4637 4185191 : ipa_call_summaries->remove (edge);
4638 4185191 : ipa_fn_summaries->remove (edge->callee);
4639 4185191 : ipa_remove_from_growth_caches (edge);
4640 4185191 : }
4641 :
4642 : /* For performance reasons ipa_merge_fn_summary_after_inlining is not updating
4643 : overall size and time. Recompute it.
4644 : If RESET is true also recompute call_time_size_table. */
4645 :
4646 : void
4647 9776806 : ipa_update_overall_fn_summary (struct cgraph_node *node, bool reset)
4648 : {
4649 9776806 : class ipa_fn_summary *info = ipa_fn_summaries->get (node);
4650 9776806 : class ipa_size_summary *size_info = ipa_size_summaries->get (node);
4651 9776806 : size_time_entry *e;
4652 9776806 : int i;
4653 :
4654 9776806 : size_info->size = 0;
4655 9776806 : info->time = 0;
4656 49505031 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
4657 : {
4658 39728225 : size_info->size += e->size;
4659 39728225 : info->time += e->time;
4660 : }
4661 9776806 : info->min_size = info->size_time_table[0].size;
4662 9776806 : if (reset)
4663 8840453 : info->call_size_time_table.release ();
4664 9776806 : if (node->callees || node->indirect_calls)
4665 7443231 : estimate_calls_size_and_time (node, &size_info->size, &info->min_size,
4666 : &info->time, NULL,
4667 : ~(clause_t) (1 << ipa_predicate::false_condition),
4668 : NULL);
4669 9776806 : size_info->size = RDIV (size_info->size, ipa_fn_summary::size_scale);
4670 9776806 : info->min_size = RDIV (info->min_size, ipa_fn_summary::size_scale);
4671 9776806 : }
4672 :
4673 :
4674 : /* This function performs intraprocedural analysis in NODE that is required to
4675 : inline indirect calls. */
4676 :
4677 : static void
4678 1415883 : inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
4679 : {
4680 1415883 : ipa_analyze_node (node);
4681 1415883 : if (dump_file && (dump_flags & TDF_DETAILS))
4682 : {
4683 16 : ipa_print_node_params (dump_file, node);
4684 16 : ipa_print_node_jump_functions (dump_file, node);
4685 : }
4686 1415883 : }
4687 :
4688 :
4689 : /* Note function body size. */
4690 :
4691 : void
4692 1426686 : inline_analyze_function (struct cgraph_node *node)
4693 : {
4694 1426686 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
4695 :
4696 1426686 : if (dump_file)
4697 96 : fprintf (dump_file, "\nAnalyzing function: %s\n", node->dump_name ());
4698 1426686 : if (opt_for_fn (node->decl, optimize) && !node->thunk)
4699 1415883 : inline_indirect_intraprocedural_analysis (node);
4700 1426686 : compute_fn_summary (node, false);
4701 1426686 : if (!optimize)
4702 : {
4703 9467 : struct cgraph_edge *e;
4704 22238 : for (e = node->callees; e; e = e->next_callee)
4705 12771 : e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4706 9541 : for (e = node->indirect_calls; e; e = e->next_callee)
4707 74 : e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4708 : }
4709 :
4710 1426686 : pop_cfun ();
4711 1426686 : }
4712 :
4713 :
4714 : /* Called when new function is inserted to callgraph late. */
4715 :
4716 : void
4717 20092 : ipa_fn_summary_t::insert (struct cgraph_node *node, ipa_fn_summary *)
4718 : {
4719 20092 : inline_analyze_function (node);
4720 20092 : }
4721 :
4722 : /* Note function body size. */
4723 :
4724 : static void
4725 238918 : ipa_fn_summary_generate (void)
4726 : {
4727 238918 : struct cgraph_node *node;
4728 :
4729 2160704 : FOR_EACH_DEFINED_FUNCTION (node)
4730 1921786 : if (DECL_STRUCT_FUNCTION (node->decl))
4731 1907972 : node->versionable = tree_versionable_function_p (node->decl);
4732 :
4733 238918 : ipa_fn_summary_alloc ();
4734 :
4735 238918 : ipa_fn_summaries->enable_insertion_hook ();
4736 :
4737 238918 : ipa_register_cgraph_hooks ();
4738 :
4739 2160704 : FOR_EACH_DEFINED_FUNCTION (node)
4740 1921786 : if (!node->alias
4741 1921786 : && (flag_generate_lto || flag_generate_offload|| flag_wpa
4742 1725870 : || opt_for_fn (node->decl, optimize)))
4743 1387698 : inline_analyze_function (node);
4744 238918 : }
4745 :
4746 :
4747 : /* Write inline summary for edge E to OB. */
4748 :
4749 : static void
4750 350581 : read_ipa_call_summary (class lto_input_block *ib, struct cgraph_edge *e,
4751 : bool prevails)
4752 : {
4753 350581 : class ipa_call_summary *es = prevails
4754 350581 : ? ipa_call_summaries->get_create (e) : NULL;
4755 350581 : ipa_predicate p;
4756 350581 : int length, i;
4757 :
4758 350581 : int size = streamer_read_uhwi (ib);
4759 350581 : int time = streamer_read_uhwi (ib);
4760 350581 : int depth = streamer_read_uhwi (ib);
4761 :
4762 350581 : if (es)
4763 : {
4764 350532 : es->call_stmt_size = size;
4765 350532 : es->call_stmt_time = time;
4766 350532 : es->loop_depth = depth;
4767 : }
4768 :
4769 350581 : bitpack_d bp = streamer_read_bitpack (ib);
4770 350581 : if (es)
4771 350532 : es->is_return_callee_uncaptured = bp_unpack_value (&bp, 1);
4772 : else
4773 49 : bp_unpack_value (&bp, 1);
4774 :
4775 350581 : p.stream_in (ib);
4776 350581 : if (es)
4777 350532 : edge_set_predicate (e, &p);
4778 350581 : length = streamer_read_uhwi (ib);
4779 350581 : if (length && es
4780 350581 : && (e->possibly_call_in_translation_unit_p ()
4781 : /* Also stream in jump functions to builtins in hope that they
4782 : will get fnspecs. */
4783 119553 : || fndecl_built_in_p (e->callee->decl, BUILT_IN_NORMAL)))
4784 : {
4785 234428 : es->param.safe_grow_cleared (length, true);
4786 1040909 : for (i = 0; i < length; i++)
4787 : {
4788 572053 : es->param[i].change_prob = streamer_read_uhwi (ib);
4789 572053 : bitpack_d bp = streamer_read_bitpack (ib);
4790 1716159 : es->param[i].points_to_local_or_readonly_memory
4791 572053 : = bp_unpack_value (&bp, 1);
4792 1716159 : es->param[i].points_to_possible_sra_candidate
4793 572053 : = bp_unpack_value (&bp, 1);
4794 : }
4795 : }
4796 : else
4797 : {
4798 139316 : for (i = 0; i < length; i++)
4799 : {
4800 23163 : streamer_read_uhwi (ib);
4801 23163 : streamer_read_uhwi (ib);
4802 : }
4803 : }
4804 350581 : }
4805 :
4806 :
4807 : /* Stream in inline summaries from the section. */
4808 :
4809 : static void
4810 13896 : inline_read_section (struct lto_file_decl_data *file_data, const char *data,
4811 : size_t len)
4812 : {
4813 13896 : const struct lto_function_header *header =
4814 : (const struct lto_function_header *) data;
4815 13896 : const int cfg_offset = sizeof (struct lto_function_header);
4816 13896 : const int main_offset = cfg_offset + header->cfg_size;
4817 13896 : const int string_offset = main_offset + header->main_size;
4818 13896 : class data_in *data_in;
4819 13896 : unsigned int i, count2, j;
4820 13896 : unsigned int f_count;
4821 :
4822 13896 : lto_input_block ib ((const char *) data + main_offset, header->main_size,
4823 13896 : file_data);
4824 :
4825 13896 : data_in =
4826 27792 : lto_data_in_create (file_data, (const char *) data + string_offset,
4827 13896 : header->string_size, vNULL);
4828 13896 : f_count = streamer_read_uhwi (&ib);
4829 102191 : for (i = 0; i < f_count; i++)
4830 : {
4831 88295 : unsigned int index;
4832 88295 : struct cgraph_node *node;
4833 88295 : class ipa_fn_summary *info;
4834 88295 : class ipa_node_params *params_summary;
4835 88295 : class ipa_size_summary *size_info;
4836 88295 : lto_symtab_encoder_t encoder;
4837 88295 : struct bitpack_d bp;
4838 88295 : struct cgraph_edge *e;
4839 88295 : ipa_predicate p;
4840 :
4841 88295 : index = streamer_read_uhwi (&ib);
4842 88295 : encoder = file_data->symtab_node_encoder;
4843 88295 : node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
4844 : index));
4845 88295 : info = node->prevailing_p () ? ipa_fn_summaries->get_create (node) : NULL;
4846 88295 : params_summary = node->prevailing_p ()
4847 88295 : ? ipa_node_params_sum->get (node) : NULL;
4848 88295 : size_info = node->prevailing_p ()
4849 88295 : ? ipa_size_summaries->get_create (node) : NULL;
4850 :
4851 88295 : int stack_size = streamer_read_uhwi (&ib);
4852 88295 : int size = streamer_read_uhwi (&ib);
4853 88295 : sreal time = sreal::stream_in (&ib);
4854 :
4855 88295 : if (info)
4856 : {
4857 88237 : info->estimated_stack_size
4858 88237 : = size_info->estimated_self_stack_size = stack_size;
4859 88237 : size_info->size = size_info->self_size = size;
4860 88237 : info->time = time;
4861 : }
4862 :
4863 88295 : bp = streamer_read_bitpack (&ib);
4864 88295 : if (info)
4865 : {
4866 88237 : info->inlinable = bp_unpack_value (&bp, 1);
4867 88237 : info->fp_expressions = bp_unpack_value (&bp, 1);
4868 88237 : if (!lto_stream_offload_p)
4869 88237 : info->target_info = streamer_read_uhwi (&ib);
4870 : }
4871 : else
4872 : {
4873 58 : bp_unpack_value (&bp, 1);
4874 58 : bp_unpack_value (&bp, 1);
4875 58 : if (!lto_stream_offload_p)
4876 58 : streamer_read_uhwi (&ib);
4877 : }
4878 :
4879 88295 : count2 = streamer_read_uhwi (&ib);
4880 88295 : gcc_assert (!info || !info->conds);
4881 88237 : if (info)
4882 88237 : vec_safe_reserve_exact (info->conds, count2);
4883 167677 : for (j = 0; j < count2; j++)
4884 : {
4885 79382 : struct condition c;
4886 79382 : unsigned int k, count3;
4887 79382 : c.operand_num = streamer_read_uhwi (&ib);
4888 79382 : c.code = (enum tree_code) streamer_read_uhwi (&ib);
4889 79382 : c.type = stream_read_tree (&ib, data_in);
4890 79382 : c.val = stream_read_tree (&ib, data_in);
4891 79382 : bp = streamer_read_bitpack (&ib);
4892 79382 : c.agg_contents = bp_unpack_value (&bp, 1);
4893 79382 : c.by_ref = bp_unpack_value (&bp, 1);
4894 79382 : if (c.agg_contents)
4895 12683 : c.offset = streamer_read_uhwi (&ib);
4896 79382 : count3 = streamer_read_uhwi (&ib);
4897 79382 : c.param_ops = NULL;
4898 79382 : if (info)
4899 79382 : vec_safe_reserve_exact (c.param_ops, count3);
4900 79382 : if (params_summary)
4901 79382 : ipa_set_param_used_by_ipa_predicates
4902 79382 : (params_summary, c.operand_num, true);
4903 83187 : for (k = 0; k < count3; k++)
4904 : {
4905 3805 : struct expr_eval_op op;
4906 3805 : enum gimple_rhs_class rhs_class;
4907 3805 : op.code = (enum tree_code) streamer_read_uhwi (&ib);
4908 3805 : op.type = stream_read_tree (&ib, data_in);
4909 3805 : switch (rhs_class = get_gimple_rhs_class (op.code))
4910 : {
4911 1577 : case GIMPLE_UNARY_RHS:
4912 1577 : op.index = 0;
4913 1577 : op.val[0] = NULL_TREE;
4914 1577 : op.val[1] = NULL_TREE;
4915 1577 : break;
4916 :
4917 2228 : case GIMPLE_BINARY_RHS:
4918 2228 : case GIMPLE_TERNARY_RHS:
4919 2228 : bp = streamer_read_bitpack (&ib);
4920 2228 : op.index = bp_unpack_value (&bp, 2);
4921 2228 : op.val[0] = stream_read_tree (&ib, data_in);
4922 2228 : if (rhs_class == GIMPLE_BINARY_RHS)
4923 2228 : op.val[1] = NULL_TREE;
4924 : else
4925 0 : op.val[1] = stream_read_tree (&ib, data_in);
4926 : break;
4927 :
4928 0 : default:
4929 0 : fatal_error (UNKNOWN_LOCATION,
4930 : "invalid fnsummary in LTO stream");
4931 : }
4932 3805 : if (info)
4933 3805 : c.param_ops->quick_push (op);
4934 : }
4935 79382 : if (info)
4936 79382 : info->conds->quick_push (c);
4937 : }
4938 88295 : count2 = streamer_read_uhwi (&ib);
4939 88295 : gcc_assert (!info || !info->size_time_table.length ());
4940 88295 : if (info && count2)
4941 88237 : info->size_time_table.reserve_exact (count2);
4942 336110 : for (j = 0; j < count2; j++)
4943 : {
4944 247815 : class size_time_entry e;
4945 :
4946 247815 : e.size = streamer_read_uhwi (&ib);
4947 247815 : e.time = sreal::stream_in (&ib);
4948 247815 : e.exec_predicate.stream_in (&ib);
4949 247815 : e.nonconst_predicate.stream_in (&ib);
4950 :
4951 247815 : if (info)
4952 247699 : info->size_time_table.quick_push (e);
4953 : }
4954 :
4955 88295 : count2 = streamer_read_uhwi (&ib);
4956 89816 : for (j = 0; j < count2; j++)
4957 : {
4958 1521 : p.stream_in (&ib);
4959 1521 : sreal fcp_freq = sreal::stream_in (&ib);
4960 1521 : if (info)
4961 : {
4962 1521 : ipa_freqcounting_predicate fcp;
4963 1521 : fcp.predicate = NULL;
4964 1521 : set_hint_predicate (&fcp.predicate, p);
4965 1521 : fcp.freq = fcp_freq;
4966 1521 : vec_safe_push (info->loop_iterations, fcp);
4967 : }
4968 : }
4969 88295 : count2 = streamer_read_uhwi (&ib);
4970 88585 : for (j = 0; j < count2; j++)
4971 : {
4972 290 : p.stream_in (&ib);
4973 290 : sreal fcp_freq = sreal::stream_in (&ib);
4974 290 : if (info)
4975 : {
4976 290 : ipa_freqcounting_predicate fcp;
4977 290 : fcp.predicate = NULL;
4978 290 : set_hint_predicate (&fcp.predicate, p);
4979 290 : fcp.freq = fcp_freq;
4980 290 : vec_safe_push (info->loop_strides, fcp);
4981 : }
4982 : }
4983 88295 : count2 = streamer_read_uhwi (&ib);
4984 88295 : if (info && count2)
4985 8 : info->builtin_constant_p_parms.reserve_exact (count2);
4986 88303 : for (j = 0; j < count2; j++)
4987 : {
4988 8 : int parm = streamer_read_uhwi (&ib);
4989 8 : if (info)
4990 8 : info->builtin_constant_p_parms.quick_push (parm);
4991 : }
4992 437373 : for (e = node->callees; e; e = e->next_callee)
4993 349078 : read_ipa_call_summary (&ib, e, info != NULL);
4994 89798 : for (e = node->indirect_calls; e; e = e->next_callee)
4995 1503 : read_ipa_call_summary (&ib, e, info != NULL);
4996 : }
4997 :
4998 13896 : lto_free_section_data (file_data, LTO_section_ipa_fn_summary, NULL, data,
4999 : len);
5000 13896 : lto_data_in_delete (data_in);
5001 13896 : }
5002 :
5003 :
5004 : /* Read inline summary. Jump functions are shared among ipa-cp
5005 : and inliner, so when ipa-cp is active, we don't need to write them
5006 : twice. */
5007 :
5008 : static void
5009 12782 : ipa_fn_summary_read (void)
5010 : {
5011 12782 : struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
5012 12782 : struct lto_file_decl_data *file_data;
5013 12782 : unsigned int j = 0;
5014 :
5015 12782 : ipa_prop_read_jump_functions ();
5016 12782 : ipa_fn_summary_alloc ();
5017 :
5018 39460 : while ((file_data = file_data_vec[j++]))
5019 : {
5020 13896 : size_t len;
5021 13896 : const char *data
5022 13896 : = lto_get_summary_section_data (file_data, LTO_section_ipa_fn_summary,
5023 : &len);
5024 13896 : if (data)
5025 13896 : inline_read_section (file_data, data, len);
5026 : else
5027 : /* Fatal error here. We do not want to support compiling ltrans units
5028 : with different version of compiler or different flags than the WPA
5029 : unit, so this should never happen. */
5030 0 : fatal_error (input_location,
5031 : "ipa inline summary is missing in input file");
5032 : }
5033 12782 : ipa_register_cgraph_hooks ();
5034 :
5035 12782 : gcc_assert (ipa_fn_summaries);
5036 12782 : ipa_fn_summaries->enable_insertion_hook ();
5037 12782 : }
5038 :
5039 :
5040 : /* Write inline summary for edge E to OB. */
5041 :
5042 : static void
5043 382410 : write_ipa_call_summary (struct output_block *ob, struct cgraph_edge *e)
5044 : {
5045 382410 : class ipa_call_summary *es = ipa_call_summaries->get (e);
5046 382410 : int i;
5047 :
5048 382410 : streamer_write_uhwi (ob, es->call_stmt_size);
5049 382410 : streamer_write_uhwi (ob, es->call_stmt_time);
5050 382410 : streamer_write_uhwi (ob, es->loop_depth);
5051 :
5052 382410 : bitpack_d bp = bitpack_create (ob->main_stream);
5053 382410 : bp_pack_value (&bp, es->is_return_callee_uncaptured, 1);
5054 382410 : streamer_write_bitpack (&bp);
5055 :
5056 382410 : if (es->predicate)
5057 10821 : es->predicate->stream_out (ob);
5058 : else
5059 371589 : streamer_write_uhwi (ob, 0);
5060 382410 : streamer_write_uhwi (ob, es->param.length ());
5061 2307327 : for (i = 0; i < (int) es->param.length (); i++)
5062 : {
5063 636043 : streamer_write_uhwi (ob, es->param[i].change_prob);
5064 636043 : bp = bitpack_create (ob->main_stream);
5065 636043 : bp_pack_value (&bp, es->param[i].points_to_local_or_readonly_memory, 1);
5066 636043 : bp_pack_value (&bp, es->param[i].points_to_possible_sra_candidate, 1);
5067 636043 : streamer_write_bitpack (&bp);
5068 : }
5069 382410 : }
5070 :
5071 :
5072 : /* Write inline summary for node in SET.
5073 : Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
5074 : active, we don't need to write them twice. */
5075 :
5076 : static void
5077 23943 : ipa_fn_summary_write (void)
5078 : {
5079 23943 : struct output_block *ob = create_output_block (LTO_section_ipa_fn_summary);
5080 23943 : lto_symtab_encoder_iterator lsei;
5081 23943 : lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
5082 23943 : unsigned int count = 0;
5083 :
5084 131446 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
5085 107503 : lsei_next_function_in_partition (&lsei))
5086 : {
5087 107503 : cgraph_node *cnode = lsei_cgraph_node (lsei);
5088 107503 : if (cnode->definition && !cnode->alias)
5089 105023 : count++;
5090 : }
5091 23943 : streamer_write_uhwi (ob, count);
5092 :
5093 131446 : for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
5094 107503 : lsei_next_function_in_partition (&lsei))
5095 : {
5096 107503 : cgraph_node *cnode = lsei_cgraph_node (lsei);
5097 107503 : if (cnode->definition && !cnode->alias)
5098 : {
5099 105023 : class ipa_fn_summary *info = ipa_fn_summaries->get (cnode);
5100 105023 : class ipa_size_summary *size_info = ipa_size_summaries->get (cnode);
5101 105023 : struct bitpack_d bp;
5102 105023 : struct cgraph_edge *edge;
5103 105023 : int i;
5104 105023 : size_time_entry *e;
5105 105023 : struct condition *c;
5106 :
5107 105023 : streamer_write_uhwi (ob, lto_symtab_encoder_encode (encoder, cnode));
5108 105023 : streamer_write_hwi (ob, size_info->estimated_self_stack_size);
5109 105023 : streamer_write_hwi (ob, size_info->self_size);
5110 105023 : info->time.stream_out (ob);
5111 105023 : bp = bitpack_create (ob->main_stream);
5112 105023 : bp_pack_value (&bp, info->inlinable, 1);
5113 105023 : bp_pack_value (&bp, info->fp_expressions, 1);
5114 105023 : streamer_write_bitpack (&bp);
5115 105023 : if (!lto_stream_offload_p)
5116 105023 : streamer_write_uhwi (ob, info->target_info);
5117 105023 : streamer_write_uhwi (ob, vec_safe_length (info->conds));
5118 305675 : for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
5119 : {
5120 95629 : int j;
5121 95629 : struct expr_eval_op *op;
5122 :
5123 95629 : streamer_write_uhwi (ob, c->operand_num);
5124 95629 : streamer_write_uhwi (ob, c->code);
5125 95629 : stream_write_tree (ob, c->type, true);
5126 95629 : stream_write_tree (ob, c->val, true);
5127 95629 : bp = bitpack_create (ob->main_stream);
5128 95629 : bp_pack_value (&bp, c->agg_contents, 1);
5129 95629 : bp_pack_value (&bp, c->by_ref, 1);
5130 95629 : streamer_write_bitpack (&bp);
5131 95629 : if (c->agg_contents)
5132 15956 : streamer_write_uhwi (ob, c->offset);
5133 95629 : streamer_write_uhwi (ob, vec_safe_length (c->param_ops));
5134 198809 : for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
5135 : {
5136 4467 : streamer_write_uhwi (ob, op->code);
5137 4467 : stream_write_tree (ob, op->type, true);
5138 4467 : if (op->val[0])
5139 : {
5140 2616 : bp = bitpack_create (ob->main_stream);
5141 2616 : bp_pack_value (&bp, op->index, 2);
5142 2616 : streamer_write_bitpack (&bp);
5143 2616 : stream_write_tree (ob, op->val[0], true);
5144 2616 : if (op->val[1])
5145 4 : stream_write_tree (ob, op->val[1], true);
5146 : }
5147 : }
5148 : }
5149 105023 : streamer_write_uhwi (ob, info->size_time_table.length ());
5150 508387 : for (i = 0; info->size_time_table.iterate (i, &e); i++)
5151 : {
5152 298341 : streamer_write_uhwi (ob, e->size);
5153 298341 : e->time.stream_out (ob);
5154 298341 : e->exec_predicate.stream_out (ob);
5155 298341 : e->nonconst_predicate.stream_out (ob);
5156 : }
5157 105023 : ipa_freqcounting_predicate *fcp;
5158 105023 : streamer_write_uhwi (ob, vec_safe_length (info->loop_iterations));
5159 212257 : for (i = 0; vec_safe_iterate (info->loop_iterations, i, &fcp); i++)
5160 : {
5161 2211 : fcp->predicate->stream_out (ob);
5162 2211 : fcp->freq.stream_out (ob);
5163 : }
5164 105023 : streamer_write_uhwi (ob, vec_safe_length (info->loop_strides));
5165 210392 : for (i = 0; vec_safe_iterate (info->loop_strides, i, &fcp); i++)
5166 : {
5167 346 : fcp->predicate->stream_out (ob);
5168 346 : fcp->freq.stream_out (ob);
5169 : }
5170 105023 : streamer_write_uhwi (ob, info->builtin_constant_p_parms.length ());
5171 105023 : int ip;
5172 210062 : for (i = 0; info->builtin_constant_p_parms.iterate (i, &ip);
5173 : i++)
5174 16 : streamer_write_uhwi (ob, ip);
5175 484722 : for (edge = cnode->callees; edge; edge = edge->next_callee)
5176 379699 : write_ipa_call_summary (ob, edge);
5177 107734 : for (edge = cnode->indirect_calls; edge; edge = edge->next_callee)
5178 2711 : write_ipa_call_summary (ob, edge);
5179 : }
5180 : }
5181 23943 : streamer_write_char_stream (ob->main_stream, 0);
5182 23943 : produce_asm (ob);
5183 23943 : destroy_output_block (ob);
5184 :
5185 23943 : ipa_prop_write_jump_functions ();
5186 23943 : }
5187 :
5188 :
5189 : /* Release function summary. */
5190 :
5191 : void
5192 746266 : ipa_free_fn_summary (void)
5193 : {
5194 746266 : if (!ipa_call_summaries)
5195 : return;
5196 472278 : ggc_delete (ipa_fn_summaries);
5197 472278 : ipa_fn_summaries = NULL;
5198 472278 : delete ipa_call_summaries;
5199 472278 : ipa_call_summaries = NULL;
5200 472278 : edge_predicate_pool.release ();
5201 : /* During IPA this is one of largest datastructures to release. */
5202 472278 : if (flag_wpa)
5203 8147 : ggc_trim ();
5204 : }
5205 :
5206 : /* Release function summary. */
5207 :
5208 : void
5209 746266 : ipa_free_size_summary (void)
5210 : {
5211 746266 : if (!ipa_size_summaries)
5212 : return;
5213 472278 : delete ipa_size_summaries;
5214 472278 : ipa_size_summaries = NULL;
5215 : }
5216 :
5217 : namespace {
5218 :
5219 : const pass_data pass_data_local_fn_summary =
5220 : {
5221 : GIMPLE_PASS, /* type */
5222 : "local-fnsummary", /* name */
5223 : OPTGROUP_INLINE, /* optinfo_flags */
5224 : TV_INLINE_PARAMETERS, /* tv_id */
5225 : 0, /* properties_required */
5226 : 0, /* properties_provided */
5227 : 0, /* properties_destroyed */
5228 : 0, /* todo_flags_start */
5229 : 0, /* todo_flags_finish */
5230 : };
5231 :
5232 : class pass_local_fn_summary : public gimple_opt_pass
5233 : {
5234 : public:
5235 594360 : pass_local_fn_summary (gcc::context *ctxt)
5236 1188720 : : gimple_opt_pass (pass_data_local_fn_summary, ctxt)
5237 : {}
5238 :
5239 : /* opt_pass methods: */
5240 297180 : opt_pass * clone () final override
5241 : {
5242 297180 : return new pass_local_fn_summary (m_ctxt);
5243 : }
5244 5972550 : unsigned int execute (function *) final override
5245 : {
5246 5972550 : return compute_fn_summary_for_current ();
5247 : }
5248 :
5249 : }; // class pass_local_fn_summary
5250 :
5251 : } // anon namespace
5252 :
5253 : gimple_opt_pass *
5254 297180 : make_pass_local_fn_summary (gcc::context *ctxt)
5255 : {
5256 297180 : return new pass_local_fn_summary (ctxt);
5257 : }
5258 :
5259 :
5260 : /* Free inline summary. */
5261 :
5262 : namespace {
5263 :
5264 : const pass_data pass_data_ipa_free_fn_summary =
5265 : {
5266 : SIMPLE_IPA_PASS, /* type */
5267 : "free-fnsummary", /* name */
5268 : OPTGROUP_NONE, /* optinfo_flags */
5269 : TV_IPA_FREE_INLINE_SUMMARY, /* tv_id */
5270 : 0, /* properties_required */
5271 : 0, /* properties_provided */
5272 : 0, /* properties_destroyed */
5273 : 0, /* todo_flags_start */
5274 : 0, /* todo_flags_finish */
5275 : };
5276 :
5277 : class pass_ipa_free_fn_summary : public simple_ipa_opt_pass
5278 : {
5279 : public:
5280 594360 : pass_ipa_free_fn_summary (gcc::context *ctxt)
5281 : : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary, ctxt),
5282 1188720 : small_p (false)
5283 : {}
5284 :
5285 : /* opt_pass methods: */
5286 297180 : opt_pass *clone () final override
5287 : {
5288 297180 : return new pass_ipa_free_fn_summary (m_ctxt);
5289 : }
5290 594360 : void set_pass_param (unsigned int n, bool param) final override
5291 : {
5292 594360 : gcc_assert (n == 0);
5293 594360 : small_p = param;
5294 594360 : }
5295 499756 : bool gate (function *) final override { return true; }
5296 478426 : unsigned int execute (function *) final override
5297 : {
5298 478426 : ipa_free_fn_summary ();
5299 : /* Free ipa-prop structures if they are no longer needed. */
5300 478426 : ipa_free_all_structures_after_iinln ();
5301 478426 : if (!flag_wpa)
5302 470279 : ipa_free_size_summary ();
5303 478426 : return 0;
5304 : }
5305 :
5306 : private:
5307 : bool small_p;
5308 : }; // class pass_ipa_free_fn_summary
5309 :
5310 : } // anon namespace
5311 :
5312 : simple_ipa_opt_pass *
5313 297180 : make_pass_ipa_free_fn_summary (gcc::context *ctxt)
5314 : {
5315 297180 : return new pass_ipa_free_fn_summary (ctxt);
5316 : }
5317 :
5318 : namespace {
5319 :
5320 : const pass_data pass_data_ipa_fn_summary =
5321 : {
5322 : IPA_PASS, /* type */
5323 : "fnsummary", /* name */
5324 : OPTGROUP_INLINE, /* optinfo_flags */
5325 : TV_IPA_FNSUMMARY, /* tv_id */
5326 : 0, /* properties_required */
5327 : 0, /* properties_provided */
5328 : 0, /* properties_destroyed */
5329 : 0, /* todo_flags_start */
5330 : ( TODO_dump_symtab ), /* todo_flags_finish */
5331 : };
5332 :
5333 : class pass_ipa_fn_summary : public ipa_opt_pass_d
5334 : {
5335 : public:
5336 297180 : pass_ipa_fn_summary (gcc::context *ctxt)
5337 : : ipa_opt_pass_d (pass_data_ipa_fn_summary, ctxt,
5338 : ipa_fn_summary_generate, /* generate_summary */
5339 : ipa_fn_summary_write, /* write_summary */
5340 : ipa_fn_summary_read, /* read_summary */
5341 : NULL, /* write_optimization_summary */
5342 : NULL, /* read_optimization_summary */
5343 : NULL, /* stmt_fixup */
5344 : 0, /* function_transform_todo_flags_start */
5345 : NULL, /* function_transform */
5346 297180 : NULL) /* variable_transform */
5347 297180 : {}
5348 :
5349 : /* opt_pass methods: */
5350 239358 : unsigned int execute (function *) final override { return 0; }
5351 :
5352 : }; // class pass_ipa_fn_summary
5353 :
5354 : } // anon namespace
5355 :
5356 : ipa_opt_pass_d *
5357 297180 : make_pass_ipa_fn_summary (gcc::context *ctxt)
5358 : {
5359 297180 : return new pass_ipa_fn_summary (ctxt);
5360 : }
5361 :
5362 : /* Reset all state within ipa-fnsummary.cc so that we can rerun the compiler
5363 : within the same process. For use by toplev::finalize. */
5364 :
5365 : void
5366 266934 : ipa_fnsummary_cc_finalize (void)
5367 : {
5368 266934 : ipa_free_fn_summary ();
5369 266934 : ipa_free_size_summary ();
5370 266934 : }
|