Line data Source code
1 : /* Tree inlining.
2 : Copyright (C) 2001-2026 Free Software Foundation, Inc.
3 : Contributed by Alexandre Oliva <aoliva@redhat.com>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify
8 : it under the terms of the GNU General Public License as published by
9 : the Free Software Foundation; either version 3, or (at your option)
10 : any later version.
11 :
12 : GCC is distributed in the hope that it will be useful,
13 : but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 : GNU General Public License for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "target.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "gimple.h"
29 : #include "cfghooks.h"
30 : #include "tree-pass.h"
31 : #include "ssa.h"
32 : #include "cgraph.h"
33 : #include "tree-pretty-print.h"
34 : #include "diagnostic-core.h"
35 : #include "gimple-predict.h"
36 : #include "fold-const.h"
37 : #include "stor-layout.h"
38 : #include "calls.h"
39 : #include "tree-inline.h"
40 : #include "langhooks.h"
41 : #include "cfganal.h"
42 : #include "tree-iterator.h"
43 : #include "intl.h"
44 : #include "gimple-iterator.h"
45 : #include "gimple-fold.h"
46 : #include "tree-eh.h"
47 : #include "gimplify.h"
48 : #include "gimplify-me.h"
49 : #include "gimple-walk.h"
50 : #include "tree-cfg.h"
51 : #include "tree-into-ssa.h"
52 : #include "tree-dfa.h"
53 : #include "tree-ssa.h"
54 : #include "except.h"
55 : #include "debug.h"
56 : #include "value-prof.h"
57 : #include "cfgloop.h"
58 : #include "builtins.h"
59 : #include "stringpool.h"
60 : #include "attribs.h"
61 : #include "sreal.h"
62 : #include "tree-cfgcleanup.h"
63 : #include "tree-ssa-live.h"
64 : #include "alloc-pool.h"
65 : #include "symbol-summary.h"
66 : #include "symtab-thunks.h"
67 : #include "symtab-clones.h"
68 : #include "asan.h"
69 :
70 : /* I'm not real happy about this, but we need to handle gimple and
71 : non-gimple trees. */
72 :
73 : /* Inlining, Cloning, Versioning, Parallelization
74 :
75 : Inlining: a function body is duplicated, but the PARM_DECLs are
76 : remapped into VAR_DECLs, and non-void RETURN_EXPRs become
77 : MODIFY_EXPRs that store to a dedicated returned-value variable.
78 : The duplicated eh_region info of the copy will later be appended
79 : to the info for the caller; the eh_region info in copied throwing
80 : statements and RESX statements are adjusted accordingly.
81 :
82 : Cloning: (only in C++) We have one body for a con/de/structor, and
83 : multiple function decls, each with a unique parameter list.
84 : Duplicate the body, using the given splay tree; some parameters
85 : will become constants (like 0 or 1).
86 :
87 : Versioning: a function body is duplicated and the result is a new
88 : function rather than into blocks of an existing function as with
89 : inlining. Some parameters will become constants.
90 :
91 : Parallelization: a region of a function is duplicated resulting in
92 : a new function. Variables may be replaced with complex expressions
93 : to enable shared variable semantics.
94 :
95 : All of these will simultaneously lookup any callgraph edges. If
96 : we're going to inline the duplicated function body, and the given
97 : function has some cloned callgraph nodes (one for each place this
98 : function will be inlined) those callgraph edges will be duplicated.
99 : If we're cloning the body, those callgraph edges will be
100 : updated to point into the new body. (Note that the original
101 : callgraph node and edge list will not be altered.)
102 :
103 : See the CALL_EXPR handling case in copy_tree_body_r (). */
104 :
105 : /* To Do:
106 :
107 : o In order to make inlining-on-trees work, we pessimized
108 : function-local static constants. In particular, they are now
109 : always output, even when not addressed. Fix this by treating
110 : function-local static constants just like global static
111 : constants; the back-end already knows not to output them if they
112 : are not needed.
113 :
114 : o Provide heuristics to clamp inlining of recursive template
115 : calls? */
116 :
117 :
118 : /* Weights that estimate_num_insns uses to estimate the size of the
119 : produced code. */
120 :
121 : eni_weights eni_size_weights;
122 :
123 : /* Weights that estimate_num_insns uses to estimate the time necessary
124 : to execute the produced code. */
125 :
126 : eni_weights eni_time_weights;
127 :
128 : /* Prototypes. */
129 :
130 : static tree declare_return_variable (copy_body_data *, tree, tree,
131 : basic_block);
132 : static void remap_block (tree *, copy_body_data *);
133 : static void copy_bind_expr (tree *, int *, copy_body_data *);
134 : static void declare_inline_vars (tree, tree);
135 : static void remap_save_expr (tree *, hash_map<tree, tree> *, int *);
136 : static void prepend_lexical_block (tree current_block, tree new_block);
137 : static tree copy_result_decl_to_var (tree, copy_body_data *);
138 : static tree copy_decl_maybe_to_var (tree, copy_body_data *);
139 : static gimple_seq remap_gimple_stmt (gimple *, copy_body_data *);
140 : static void insert_init_stmt (copy_body_data *, basic_block, gimple *);
141 :
142 : /* Insert a tree->tree mapping for ID. Despite the name suggests
143 : that the trees should be variables, it is used for more than that. */
144 :
145 : void
146 697818841 : insert_decl_map (copy_body_data *id, tree key, tree value)
147 : {
148 697818841 : id->decl_map->put (key, value);
149 :
150 : /* Always insert an identity map as well. If we see this same new
151 : node again, we won't want to duplicate it a second time. */
152 697818841 : if (key != value && value)
153 213167617 : id->decl_map->put (value, value);
154 697818841 : }
155 :
156 : /* If nonzero, we're remapping the contents of inlined debug
157 : statements. If negative, an error has occurred, such as a
158 : reference to a variable that isn't available in the inlined
159 : context. */
160 : static int processing_debug_stmt = 0;
161 :
162 : /* Construct new SSA name for old NAME. ID is the inline context. */
163 :
164 : static tree
165 71736790 : remap_ssa_name (tree name, copy_body_data *id)
166 : {
167 71736790 : tree new_tree, var;
168 71736790 : tree *n;
169 :
170 71736790 : gcc_assert (TREE_CODE (name) == SSA_NAME);
171 :
172 71736790 : n = id->decl_map->get (name);
173 71736790 : if (n)
174 : {
175 : /* When we perform edge redirection as part of CFG copy, IPA-SRA can
176 : remove an unused LHS from a call statement. Such LHS can however
177 : still appear in debug statements, but their value is lost in this
178 : function and we do not want to map them. */
179 52691656 : if (id->killed_new_ssa_names
180 52691656 : && id->killed_new_ssa_names->contains (*n))
181 : {
182 386 : gcc_assert (processing_debug_stmt);
183 386 : processing_debug_stmt = -1;
184 386 : return name;
185 : }
186 :
187 52691270 : return unshare_expr (*n);
188 : }
189 :
190 19045134 : if (processing_debug_stmt)
191 : {
192 99365 : if (SSA_NAME_IS_DEFAULT_DEF (name)
193 99322 : && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL
194 99275 : && id->entry_bb == NULL
195 198639 : && single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
196 : {
197 99274 : gimple *def_temp;
198 99274 : gimple_stmt_iterator gsi;
199 99274 : tree val = SSA_NAME_VAR (name);
200 :
201 99274 : n = id->decl_map->get (val);
202 99274 : if (n != NULL)
203 99274 : val = *n;
204 99274 : if (TREE_CODE (val) != PARM_DECL
205 99274 : && !(VAR_P (val) && DECL_ABSTRACT_ORIGIN (val)))
206 : {
207 0 : processing_debug_stmt = -1;
208 0 : return name;
209 : }
210 99274 : n = id->decl_map->get (val);
211 99274 : if (n && TREE_CODE (*n) == DEBUG_EXPR_DECL)
212 : return *n;
213 34699 : tree vexpr = build_debug_expr_decl (TREE_TYPE (name));
214 : /* FIXME: Is setting the mode really necessary? */
215 34699 : SET_DECL_MODE (vexpr, DECL_MODE (SSA_NAME_VAR (name)));
216 34699 : def_temp = gimple_build_debug_source_bind (vexpr, val, NULL);
217 34699 : gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
218 34699 : gsi_insert_before (&gsi, def_temp, GSI_SAME_STMT);
219 34699 : insert_decl_map (id, val, vexpr);
220 34699 : return vexpr;
221 : }
222 :
223 91 : processing_debug_stmt = -1;
224 91 : return name;
225 : }
226 :
227 : /* Remap anonymous SSA names or SSA names of anonymous decls. */
228 18945769 : var = SSA_NAME_VAR (name);
229 3764591 : if (!var
230 3764591 : || (!SSA_NAME_IS_DEFAULT_DEF (name)
231 3359417 : && VAR_P (var)
232 3149444 : && !VAR_DECL_IS_VIRTUAL_OPERAND (var)
233 3149444 : && DECL_ARTIFICIAL (var)
234 579133 : && DECL_IGNORED_P (var)
235 161474 : && !DECL_NAME (var)))
236 : {
237 15186622 : const pt_solution *pt;
238 15186622 : new_tree = make_ssa_name (remap_type (TREE_TYPE (name), id));
239 15186622 : if (!var && SSA_NAME_IDENTIFIER (name))
240 1969612 : SET_SSA_NAME_VAR_OR_IDENTIFIER (new_tree, SSA_NAME_IDENTIFIER (name));
241 15186622 : insert_decl_map (id, name, new_tree);
242 30373244 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
243 15186622 : = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
244 : /* Range info can be transferred, if local addresses are removed. */
245 15186622 : if (SSA_NAME_RANGE_INFO (name))
246 8741841 : duplicate_ssa_name_range_info (new_tree, name, true);
247 : /* At least IPA points-to info can be directly transferred. */
248 15186622 : if (id->src_cfun->gimple_df
249 15186622 : && id->src_cfun->gimple_df->ipa_pta
250 15022 : && POINTER_TYPE_P (TREE_TYPE (name))
251 5191 : && (pt = ssa_name_pt_solution (name))
252 15189215 : && !pt->anything)
253 2572 : set_ssa_name_pt_solution (new_tree, *pt);
254 : return new_tree;
255 : }
256 :
257 : /* Do not set DEF_STMT yet as statement is not copied yet. We do that
258 : in copy_bb. */
259 3759147 : new_tree = remap_decl (var, id);
260 :
261 : /* We might've substituted constant or another SSA_NAME for
262 : the variable.
263 :
264 : Replace the SSA name representing RESULT_DECL by variable during
265 : inlining: this saves us from need to introduce PHI node in a case
266 : return value is just partly initialized. */
267 388806 : if ((VAR_P (new_tree) || TREE_CODE (new_tree) == PARM_DECL)
268 4147953 : && (!SSA_NAME_VAR (name)
269 3759147 : || TREE_CODE (SSA_NAME_VAR (name)) != RESULT_DECL
270 0 : || !id->transform_return_to_modify))
271 : {
272 3759147 : const pt_solution *pt;
273 3759147 : new_tree = make_ssa_name (new_tree);
274 3759147 : insert_decl_map (id, name, new_tree);
275 7518294 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
276 3759147 : = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
277 :
278 3759147 : tree zero = NULL_TREE;
279 3759147 : if (SSA_NAME_IS_DEFAULT_DEF (name))
280 : {
281 : /* By inlining function having uninitialized variable, we might
282 : extend the lifetime (variable might get reused). This cause
283 : ICE in the case we end up extending lifetime of SSA name across
284 : abnormal edge, but also increase register pressure.
285 :
286 : We simply initialize all uninitialized vars by 0 except
287 : for case we are inlining to very first BB. We can avoid
288 : this for all BBs that are not inside strongly connected
289 : regions of the CFG, but this is expensive to test. */
290 405174 : if (id->entry_bb
291 45077 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name)
292 0 : && (!SSA_NAME_VAR (name)
293 0 : || TREE_CODE (SSA_NAME_VAR (name)) != PARM_DECL)
294 405174 : && (id->entry_bb != EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun),
295 0 : 0)->dest
296 0 : || EDGE_COUNT (id->entry_bb->preds) != 1))
297 : {
298 0 : gimple_stmt_iterator gsi = gsi_last_bb (id->entry_bb);
299 0 : gimple *init_stmt;
300 0 : zero = build_zero_cst (TREE_TYPE (new_tree));
301 :
302 0 : init_stmt = gimple_build_assign (new_tree, zero);
303 0 : gsi_insert_after (&gsi, init_stmt, GSI_NEW_STMT);
304 0 : SSA_NAME_IS_DEFAULT_DEF (new_tree) = 0;
305 : }
306 : else
307 : {
308 405174 : SSA_NAME_DEF_STMT (new_tree) = gimple_build_nop ();
309 405174 : set_ssa_default_def (cfun, SSA_NAME_VAR (new_tree), new_tree);
310 : }
311 : }
312 : /* Range info can be transferred if local addresses are removed
313 : and no uninitialized name was processed and set to zero. */
314 3759147 : if (!zero && SSA_NAME_RANGE_INFO (name))
315 1928452 : duplicate_ssa_name_range_info (new_tree, name, true);
316 : /* At least IPA points-to info can be directly transferred. */
317 3759147 : if (id->src_cfun->gimple_df
318 3759147 : && id->src_cfun->gimple_df->ipa_pta
319 2518 : && POINTER_TYPE_P (TREE_TYPE (name))
320 560 : && (pt = ssa_name_pt_solution (name))
321 3759424 : && !pt->anything)
322 277 : set_ssa_name_pt_solution (new_tree, *pt);
323 : }
324 : else
325 0 : insert_decl_map (id, name, new_tree);
326 : return new_tree;
327 : }
328 :
329 : /* Remap DECL during the copying of the BLOCK tree for the function. */
330 :
331 : tree
332 355218806 : remap_decl (tree decl, copy_body_data *id)
333 : {
334 355218806 : tree *n;
335 :
336 : /* We only remap local variables in the current function. */
337 :
338 : /* See if we have remapped this declaration. */
339 :
340 355218806 : n = id->decl_map->get (decl);
341 :
342 355218806 : if (!n && processing_debug_stmt)
343 : {
344 677738 : processing_debug_stmt = -1;
345 677738 : return decl;
346 : }
347 :
348 : /* When remapping a type within copy_gimple_seq_and_replace_locals, all
349 : necessary DECLs have already been remapped and we do not want to duplicate
350 : a decl coming from outside of the sequence we are copying. */
351 130929579 : if (!n
352 130929579 : && id->prevent_decl_creation_for_types
353 0 : && id->remapping_type_depth > 0
354 0 : && (VAR_P (decl) || TREE_CODE (decl) == PARM_DECL))
355 : return decl;
356 :
357 : /* If we didn't already have an equivalent for this declaration, create one
358 : now. */
359 354541068 : if (!n)
360 : {
361 : /* Make a copy of the variable or label. */
362 130929579 : tree t = id->copy_decl (decl, id);
363 :
364 : /* Remember it, so that if we encounter this local entity again
365 : we can reuse this copy. Do this early because remap_type may
366 : need this decl for TYPE_STUB_DECL. */
367 130929579 : insert_decl_map (id, decl, t);
368 :
369 130929579 : if (!DECL_P (t) || t == decl)
370 : return t;
371 :
372 : /* Remap types, if necessary. */
373 128569485 : TREE_TYPE (t) = remap_type (TREE_TYPE (t), id);
374 128569485 : if (TREE_CODE (t) == TYPE_DECL)
375 : {
376 1607754 : DECL_ORIGINAL_TYPE (t) = remap_type (DECL_ORIGINAL_TYPE (t), id);
377 :
378 : /* Preserve the invariant that DECL_ORIGINAL_TYPE != TREE_TYPE,
379 : which is enforced in gen_typedef_die when DECL_ABSTRACT_ORIGIN
380 : is not set on the TYPE_DECL, for example in LTO mode. */
381 1607754 : if (DECL_ORIGINAL_TYPE (t) == TREE_TYPE (t))
382 : {
383 10 : tree x = build_variant_type_copy (TREE_TYPE (t));
384 10 : TYPE_STUB_DECL (x) = TYPE_STUB_DECL (TREE_TYPE (t));
385 10 : TYPE_NAME (x) = TYPE_NAME (TREE_TYPE (t));
386 10 : DECL_ORIGINAL_TYPE (t) = x;
387 : }
388 : }
389 :
390 : /* Remap sizes as necessary. */
391 128569485 : walk_tree (&DECL_SIZE (t), copy_tree_body_r, id, NULL);
392 128569485 : walk_tree (&DECL_SIZE_UNIT (t), copy_tree_body_r, id, NULL);
393 :
394 : /* If fields, do likewise for offset and qualifier. */
395 128569485 : if (TREE_CODE (t) == FIELD_DECL)
396 : {
397 773 : walk_tree (&DECL_FIELD_OFFSET (t), copy_tree_body_r, id, NULL);
398 773 : if (TREE_CODE (DECL_CONTEXT (t)) == QUAL_UNION_TYPE)
399 0 : walk_tree (&DECL_QUALIFIER (t), copy_tree_body_r, id, NULL);
400 : }
401 :
402 : return t;
403 : }
404 :
405 223611489 : if (id->do_not_unshare)
406 141511691 : return *n;
407 : else
408 82099798 : return unshare_expr (*n);
409 : }
410 :
411 : static tree
412 104561 : remap_type_1 (tree type, copy_body_data *id)
413 : {
414 104561 : tree new_tree, t;
415 :
416 : /* We do need a copy. build and register it now. If this is a pointer or
417 : reference type, remap the designated type and make a new pointer or
418 : reference type. */
419 104561 : if (TREE_CODE (type) == POINTER_TYPE)
420 : {
421 28450 : new_tree = build_pointer_type_for_mode (remap_type (TREE_TYPE (type), id),
422 28450 : TYPE_MODE (type),
423 28450 : TYPE_REF_CAN_ALIAS_ALL (type));
424 28450 : if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type))
425 15521 : new_tree = build_type_attribute_qual_variant (new_tree,
426 15521 : TYPE_ATTRIBUTES (type),
427 15521 : TYPE_QUALS (type));
428 28450 : insert_decl_map (id, type, new_tree);
429 28450 : return new_tree;
430 : }
431 76111 : else if (TREE_CODE (type) == REFERENCE_TYPE)
432 : {
433 8061 : new_tree = build_reference_type_for_mode (remap_type (TREE_TYPE (type), id),
434 8061 : TYPE_MODE (type),
435 8061 : TYPE_REF_CAN_ALIAS_ALL (type));
436 8061 : if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type))
437 5481 : new_tree = build_type_attribute_qual_variant (new_tree,
438 5481 : TYPE_ATTRIBUTES (type),
439 5481 : TYPE_QUALS (type));
440 8061 : insert_decl_map (id, type, new_tree);
441 8061 : return new_tree;
442 : }
443 : else
444 68050 : new_tree = copy_node (type);
445 :
446 68050 : insert_decl_map (id, type, new_tree);
447 :
448 : /* This is a new type, not a copy of an old type. Need to reassociate
449 : variants. We can handle everything except the main variant lazily. */
450 68050 : t = TYPE_MAIN_VARIANT (type);
451 68050 : if (type != t)
452 : {
453 79 : t = remap_type (t, id);
454 79 : TYPE_MAIN_VARIANT (new_tree) = t;
455 79 : TYPE_NEXT_VARIANT (new_tree) = TYPE_NEXT_VARIANT (t);
456 79 : TYPE_NEXT_VARIANT (t) = new_tree;
457 : }
458 : else
459 : {
460 67971 : TYPE_MAIN_VARIANT (new_tree) = new_tree;
461 67971 : TYPE_NEXT_VARIANT (new_tree) = NULL;
462 : }
463 :
464 68050 : if (TYPE_STUB_DECL (type))
465 182 : TYPE_STUB_DECL (new_tree) = remap_decl (TYPE_STUB_DECL (type), id);
466 :
467 : /* Lazily create pointer and reference types. */
468 68050 : TYPE_POINTER_TO (new_tree) = NULL;
469 68050 : TYPE_REFERENCE_TO (new_tree) = NULL;
470 :
471 : /* Copy all types that may contain references to local variables; be sure to
472 : preserve sharing in between type and its main variant when possible. */
473 68050 : switch (TREE_CODE (new_tree))
474 : {
475 32689 : case INTEGER_TYPE:
476 32689 : case REAL_TYPE:
477 32689 : case FIXED_POINT_TYPE:
478 32689 : case ENUMERAL_TYPE:
479 32689 : case BOOLEAN_TYPE:
480 32689 : if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
481 : {
482 0 : gcc_checking_assert (TYPE_MIN_VALUE (type) == TYPE_MIN_VALUE (TYPE_MAIN_VARIANT (type)));
483 0 : gcc_checking_assert (TYPE_MAX_VALUE (type) == TYPE_MAX_VALUE (TYPE_MAIN_VARIANT (type)));
484 :
485 0 : TYPE_MIN_VALUE (new_tree) = TYPE_MIN_VALUE (TYPE_MAIN_VARIANT (new_tree));
486 0 : TYPE_MAX_VALUE (new_tree) = TYPE_MAX_VALUE (TYPE_MAIN_VARIANT (new_tree));
487 : }
488 : else
489 : {
490 32689 : t = TYPE_MIN_VALUE (new_tree);
491 32689 : if (t && TREE_CODE (t) != INTEGER_CST)
492 0 : walk_tree (&TYPE_MIN_VALUE (new_tree), copy_tree_body_r, id, NULL);
493 :
494 32689 : t = TYPE_MAX_VALUE (new_tree);
495 32689 : if (t && TREE_CODE (t) != INTEGER_CST)
496 32689 : walk_tree (&TYPE_MAX_VALUE (new_tree), copy_tree_body_r, id, NULL);
497 : }
498 : return new_tree;
499 :
500 16 : case FUNCTION_TYPE:
501 16 : if (TYPE_MAIN_VARIANT (new_tree) != new_tree
502 16 : && TREE_TYPE (type) == TREE_TYPE (TYPE_MAIN_VARIANT (type)))
503 0 : TREE_TYPE (new_tree) = TREE_TYPE (TYPE_MAIN_VARIANT (new_tree));
504 : else
505 16 : TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
506 16 : if (TYPE_MAIN_VARIANT (new_tree) != new_tree
507 16 : && TYPE_ARG_TYPES (type) == TYPE_ARG_TYPES (TYPE_MAIN_VARIANT (type)))
508 0 : TYPE_ARG_TYPES (new_tree) = TYPE_ARG_TYPES (TYPE_MAIN_VARIANT (new_tree));
509 : else
510 16 : walk_tree (&TYPE_ARG_TYPES (new_tree), copy_tree_body_r, id, NULL);
511 : return new_tree;
512 :
513 35114 : case ARRAY_TYPE:
514 35114 : if (TYPE_MAIN_VARIANT (new_tree) != new_tree
515 35114 : && TREE_TYPE (type) == TREE_TYPE (TYPE_MAIN_VARIANT (type)))
516 29 : TREE_TYPE (new_tree) = TREE_TYPE (TYPE_MAIN_VARIANT (new_tree));
517 : else
518 35085 : TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
519 :
520 35114 : if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
521 : {
522 62 : gcc_checking_assert (TYPE_DOMAIN (type)
523 : == TYPE_DOMAIN (TYPE_MAIN_VARIANT (type)));
524 62 : TYPE_DOMAIN (new_tree) = TYPE_DOMAIN (TYPE_MAIN_VARIANT (new_tree));
525 : }
526 : else
527 : {
528 35052 : TYPE_DOMAIN (new_tree) = remap_type (TYPE_DOMAIN (new_tree), id);
529 : /* For array bounds where we have decided not to copy over the bounds
530 : variable which isn't used in OpenMP/OpenACC region, change them to
531 : an uninitialized VAR_DECL temporary. */
532 35052 : if (id->adjust_array_error_bounds
533 3751 : && TYPE_DOMAIN (new_tree)
534 3751 : && TYPE_MAX_VALUE (TYPE_DOMAIN (new_tree)) == error_mark_node
535 37951 : && TYPE_MAX_VALUE (TYPE_DOMAIN (type)) != error_mark_node)
536 : {
537 2899 : tree v = create_tmp_var (TREE_TYPE (TYPE_DOMAIN (new_tree)));
538 2899 : DECL_ATTRIBUTES (v)
539 2899 : = tree_cons (get_identifier ("omp dummy var"), NULL_TREE,
540 2899 : DECL_ATTRIBUTES (v));
541 2899 : TYPE_MAX_VALUE (TYPE_DOMAIN (new_tree)) = v;
542 : }
543 : }
544 : break;
545 :
546 231 : case RECORD_TYPE:
547 231 : case UNION_TYPE:
548 231 : case QUAL_UNION_TYPE:
549 231 : if (TYPE_MAIN_VARIANT (type) != type
550 231 : && TYPE_FIELDS (type) == TYPE_FIELDS (TYPE_MAIN_VARIANT (type)))
551 17 : TYPE_FIELDS (new_tree) = TYPE_FIELDS (TYPE_MAIN_VARIANT (new_tree));
552 : else
553 : {
554 214 : tree f, nf = NULL;
555 :
556 987 : for (f = TYPE_FIELDS (new_tree); f ; f = DECL_CHAIN (f))
557 : {
558 773 : t = remap_decl (f, id);
559 773 : DECL_CONTEXT (t) = new_tree;
560 773 : DECL_CHAIN (t) = nf;
561 773 : nf = t;
562 : }
563 214 : TYPE_FIELDS (new_tree) = nreverse (nf);
564 : }
565 : break;
566 :
567 0 : case OFFSET_TYPE:
568 0 : default:
569 : /* Shouldn't have been thought variable sized. */
570 0 : gcc_unreachable ();
571 : }
572 :
573 : /* All variants of type share the same size, so use the already remapped data. */
574 35345 : if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
575 : {
576 79 : tree s = TYPE_SIZE (type);
577 79 : tree mvs = TYPE_SIZE (TYPE_MAIN_VARIANT (type));
578 79 : tree su = TYPE_SIZE_UNIT (type);
579 79 : tree mvsu = TYPE_SIZE_UNIT (TYPE_MAIN_VARIANT (type));
580 79 : gcc_checking_assert ((TREE_CODE (s) == PLACEHOLDER_EXPR
581 : && (TREE_CODE (mvs) == PLACEHOLDER_EXPR))
582 : || s == mvs);
583 79 : gcc_checking_assert ((TREE_CODE (su) == PLACEHOLDER_EXPR
584 : && (TREE_CODE (mvsu) == PLACEHOLDER_EXPR))
585 : || su == mvsu);
586 79 : TYPE_SIZE (new_tree) = TYPE_SIZE (TYPE_MAIN_VARIANT (new_tree));
587 79 : TYPE_SIZE_UNIT (new_tree) = TYPE_SIZE_UNIT (TYPE_MAIN_VARIANT (new_tree));
588 : }
589 : else
590 : {
591 35266 : walk_tree (&TYPE_SIZE (new_tree), copy_tree_body_r, id, NULL);
592 35266 : walk_tree (&TYPE_SIZE_UNIT (new_tree), copy_tree_body_r, id, NULL);
593 : }
594 :
595 : return new_tree;
596 : }
597 :
598 : /* Helper function for remap_type_2, called through walk_tree. */
599 :
600 : static tree
601 31050 : remap_type_3 (tree *tp, int *walk_subtrees, void *data)
602 : {
603 31050 : copy_body_data *id = (copy_body_data *) data;
604 :
605 31050 : if (TYPE_P (*tp))
606 0 : *walk_subtrees = 0;
607 :
608 31050 : else if (DECL_P (*tp) && remap_decl (*tp, id) != *tp)
609 9 : return *tp;
610 :
611 : return NULL_TREE;
612 : }
613 :
614 : /* Return true if TYPE needs to be remapped because remap_decl on any
615 : needed embedded decl returns something other than that decl. */
616 :
617 : static bool
618 51969 : remap_type_2 (tree type, copy_body_data *id)
619 : {
620 62694 : tree t;
621 :
622 : #define RETURN_TRUE_IF_VAR(T) \
623 : do \
624 : { \
625 : tree _t = (T); \
626 : if (_t) \
627 : { \
628 : if (DECL_P (_t) && remap_decl (_t, id) != _t) \
629 : return true; \
630 : if (!TYPE_SIZES_GIMPLIFIED (type) \
631 : && walk_tree (&_t, remap_type_3, id, NULL)) \
632 : return true; \
633 : } \
634 : } \
635 : while (0)
636 :
637 62694 : switch (TREE_CODE (type))
638 : {
639 10725 : case POINTER_TYPE:
640 10725 : case REFERENCE_TYPE:
641 10725 : case FUNCTION_TYPE:
642 10725 : case METHOD_TYPE:
643 10725 : return remap_type_2 (TREE_TYPE (type), id);
644 :
645 34935 : case INTEGER_TYPE:
646 34935 : case REAL_TYPE:
647 34935 : case FIXED_POINT_TYPE:
648 34935 : case ENUMERAL_TYPE:
649 34935 : case BOOLEAN_TYPE:
650 34935 : RETURN_TRUE_IF_VAR (TYPE_MIN_VALUE (type));
651 34935 : RETURN_TRUE_IF_VAR (TYPE_MAX_VALUE (type));
652 23038 : return false;
653 :
654 16559 : case ARRAY_TYPE:
655 16559 : if (remap_type_2 (TREE_TYPE (type), id)
656 32464 : || (TYPE_DOMAIN (type) && remap_type_2 (TYPE_DOMAIN (type), id)))
657 : return true;
658 : break;
659 :
660 357 : case RECORD_TYPE:
661 357 : case UNION_TYPE:
662 357 : case QUAL_UNION_TYPE:
663 3561 : for (t = TYPE_FIELDS (type); t; t = DECL_CHAIN (t))
664 3204 : if (TREE_CODE (t) == FIELD_DECL)
665 : {
666 1242 : RETURN_TRUE_IF_VAR (DECL_FIELD_OFFSET (t));
667 1242 : RETURN_TRUE_IF_VAR (DECL_SIZE (t));
668 1242 : RETURN_TRUE_IF_VAR (DECL_SIZE_UNIT (t));
669 1242 : if (TREE_CODE (type) == QUAL_UNION_TYPE)
670 0 : RETURN_TRUE_IF_VAR (DECL_QUALIFIER (t));
671 : }
672 : break;
673 :
674 : default:
675 : return false;
676 : }
677 :
678 7802 : RETURN_TRUE_IF_VAR (TYPE_SIZE (type));
679 7793 : RETURN_TRUE_IF_VAR (TYPE_SIZE_UNIT (type));
680 7057 : return false;
681 : #undef RETURN_TRUE_IF_VAR
682 : }
683 :
684 : tree
685 1414057772 : remap_type (tree type, copy_body_data *id)
686 : {
687 1414057772 : tree *node;
688 1414057772 : tree tmp;
689 :
690 1414057772 : if (type == NULL)
691 : return type;
692 :
693 : /* See if we have remapped this type. */
694 1413703382 : node = id->decl_map->get (type);
695 1413703382 : if (node)
696 931840138 : return *node;
697 :
698 : /* The type only needs remapping if it's variably modified. */
699 481863244 : if (! variably_modified_type_p (type, id->src_fn)
700 : /* Don't remap if copy_decl method doesn't always return a new
701 : decl and for all embedded decls returns the passed in decl. */
702 481863244 : || (id->dont_remap_vla_if_no_change && !remap_type_2 (type, id)))
703 : {
704 481758746 : insert_decl_map (id, type, type);
705 481758746 : return type;
706 : }
707 :
708 104498 : id->remapping_type_depth++;
709 104498 : tmp = remap_type_1 (type, id);
710 104498 : id->remapping_type_depth--;
711 :
712 104498 : return tmp;
713 : }
714 :
715 : /* Decide if DECL can be put into BLOCK_NONLOCAL_VARs. */
716 :
717 : static bool
718 50877091 : can_be_nonlocal (tree decl, copy_body_data *id)
719 : {
720 : /* We cannot duplicate function decls. */
721 50877091 : if (TREE_CODE (decl) == FUNCTION_DECL)
722 : return true;
723 :
724 : /* Local static vars must be non-local or we get multiple declaration
725 : problems. */
726 50858228 : if (VAR_P (decl) && !auto_var_in_fn_p (decl, id->src_fn))
727 231675 : return true;
728 :
729 : return false;
730 : }
731 :
732 : static tree
733 52590864 : remap_decls (tree decls, vec<tree, va_gc> **nonlocalized_list,
734 : copy_body_data *id)
735 : {
736 52590864 : tree old_var;
737 52590864 : tree new_decls = NULL_TREE;
738 :
739 : /* Remap its variables. */
740 98993705 : for (old_var = decls; old_var; old_var = DECL_CHAIN (old_var))
741 : {
742 46402841 : tree new_var;
743 :
744 46402841 : if (can_be_nonlocal (old_var, id))
745 : {
746 : /* We need to add this variable to the local decls as otherwise
747 : nothing else will do so. */
748 163244 : if (VAR_P (old_var) && ! DECL_EXTERNAL (old_var) && cfun)
749 144125 : add_local_decl (cfun, old_var);
750 100403 : if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
751 135457 : && !DECL_IGNORED_P (old_var)
752 298458 : && nonlocalized_list)
753 84401 : vec_safe_push (*nonlocalized_list, old_var);
754 163244 : continue;
755 : }
756 :
757 : /* Remap the variable. */
758 46239597 : new_var = remap_decl (old_var, id);
759 :
760 : /* If we didn't remap this variable, we can't mess with its
761 : TREE_CHAIN. If we remapped this variable to the return slot, it's
762 : already declared somewhere else, so don't declare it here. */
763 :
764 46239597 : if (new_var == old_var || new_var == id->retvar)
765 : ;
766 41959415 : else if (!new_var)
767 : {
768 0 : if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
769 0 : && !DECL_IGNORED_P (old_var)
770 0 : && nonlocalized_list)
771 0 : vec_safe_push (*nonlocalized_list, old_var);
772 : }
773 : else
774 : {
775 41959415 : gcc_assert (DECL_P (new_var));
776 41959415 : DECL_CHAIN (new_var) = new_decls;
777 41959415 : new_decls = new_var;
778 :
779 : /* Also copy value-expressions. */
780 41959415 : if (VAR_P (new_var) && DECL_HAS_VALUE_EXPR_P (new_var))
781 : {
782 1621134 : tree tem = DECL_VALUE_EXPR (new_var);
783 1621134 : bool old_regimplify = id->regimplify;
784 1621134 : id->remapping_type_depth++;
785 1621134 : walk_tree (&tem, copy_tree_body_r, id, NULL);
786 1621134 : id->remapping_type_depth--;
787 1621134 : id->regimplify = old_regimplify;
788 1621134 : SET_DECL_VALUE_EXPR (new_var, tem);
789 : }
790 : }
791 : }
792 :
793 52590864 : return nreverse (new_decls);
794 : }
795 :
796 : /* Copy the BLOCK to contain remapped versions of the variables
797 : therein. And hook the new block into the block-tree. */
798 :
799 : static void
800 47254865 : remap_block (tree *block, copy_body_data *id)
801 : {
802 47254865 : tree old_block;
803 47254865 : tree new_block;
804 :
805 : /* Make the new block. */
806 47254865 : old_block = *block;
807 47254865 : new_block = make_node (BLOCK);
808 47254865 : TREE_USED (new_block) = TREE_USED (old_block);
809 47254865 : BLOCK_ABSTRACT_ORIGIN (new_block) = BLOCK_ORIGIN (old_block);
810 47254865 : BLOCK_SOURCE_LOCATION (new_block) = BLOCK_SOURCE_LOCATION (old_block);
811 47254865 : BLOCK_NONLOCALIZED_VARS (new_block)
812 47336977 : = vec_safe_copy (BLOCK_NONLOCALIZED_VARS (old_block));
813 47254865 : *block = new_block;
814 :
815 : /* Remap its variables. */
816 94509730 : BLOCK_VARS (new_block) = remap_decls (BLOCK_VARS (old_block),
817 47254865 : &BLOCK_NONLOCALIZED_VARS (new_block),
818 : id);
819 :
820 : /* Remember the remapped block. */
821 47254865 : insert_decl_map (id, old_block, new_block);
822 47254865 : }
823 :
824 : /* Copy the whole block tree and root it in id->block. */
825 :
826 : static tree
827 26684537 : remap_blocks (tree block, copy_body_data *id)
828 : {
829 26684537 : tree t;
830 26684537 : tree new_tree = block;
831 :
832 26684537 : if (!block)
833 : return NULL;
834 :
835 26684537 : remap_block (&new_tree, id);
836 26684537 : gcc_assert (new_tree != block);
837 48496342 : for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
838 21811805 : prepend_lexical_block (new_tree, remap_blocks (t, id));
839 : /* Blocks are in arbitrary order, but make things slightly prettier and do
840 : not swap order when producing a copy. */
841 26684537 : BLOCK_SUBBLOCKS (new_tree) = blocks_nreverse (BLOCK_SUBBLOCKS (new_tree));
842 26684537 : return new_tree;
843 : }
844 :
845 : /* Remap the block tree rooted at BLOCK to nothing. */
846 :
847 : static void
848 75063 : remap_blocks_to_null (tree block, copy_body_data *id)
849 : {
850 75063 : tree t;
851 75063 : insert_decl_map (id, block, NULL_TREE);
852 123218 : for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
853 48155 : remap_blocks_to_null (t, id);
854 75063 : }
855 :
856 : /* Remap the location info pointed to by LOCUS. */
857 :
858 : static location_t
859 26666311 : remap_location (location_t locus, copy_body_data *id)
860 : {
861 26666311 : if (LOCATION_BLOCK (locus))
862 : {
863 10780342 : tree *n = id->decl_map->get (LOCATION_BLOCK (locus));
864 10780342 : gcc_assert (n);
865 10780342 : if (*n)
866 10747302 : return set_block (locus, *n);
867 : }
868 :
869 15919009 : locus = LOCATION_LOCUS (locus);
870 :
871 15919009 : if (locus != UNKNOWN_LOCATION && id->block)
872 15 : return set_block (locus, id->block);
873 :
874 : return locus;
875 : }
876 :
877 : static void
878 37767602 : copy_statement_list (tree *tp)
879 : {
880 37767602 : tree_stmt_iterator oi, ni;
881 37767602 : tree new_tree;
882 :
883 37767602 : new_tree = alloc_stmt_list ();
884 37767602 : ni = tsi_start (new_tree);
885 37767602 : oi = tsi_start (*tp);
886 37767602 : TREE_TYPE (new_tree) = TREE_TYPE (*tp);
887 37767602 : *tp = new_tree;
888 :
889 124057058 : for (; !tsi_end_p (oi); tsi_next (&oi))
890 : {
891 86289456 : tree stmt = tsi_stmt (oi);
892 86289456 : if (TREE_CODE (stmt) == STATEMENT_LIST)
893 : /* This copy is not redundant; tsi_link_after will smash this
894 : STATEMENT_LIST into the end of the one we're building, and we
895 : don't want to do that with the original. */
896 77695 : copy_statement_list (&stmt);
897 86289456 : tsi_link_after (&ni, stmt, TSI_CONTINUE_LINKING);
898 : }
899 37767602 : }
900 :
901 : static void
902 20570611 : copy_bind_expr (tree *tp, int *walk_subtrees, copy_body_data *id)
903 : {
904 20570611 : tree block = BIND_EXPR_BLOCK (*tp);
905 : /* Copy (and replace) the statement. */
906 20570611 : copy_tree_r (tp, walk_subtrees, NULL);
907 20570611 : if (block)
908 : {
909 20570033 : remap_block (&block, id);
910 20570033 : BIND_EXPR_BLOCK (*tp) = block;
911 : }
912 :
913 20570611 : if (BIND_EXPR_VARS (*tp))
914 : /* This will remap a lot of the same decls again, but this should be
915 : harmless. */
916 5335831 : BIND_EXPR_VARS (*tp) = remap_decls (BIND_EXPR_VARS (*tp), NULL, id);
917 20570611 : }
918 :
919 :
920 : /* Create a new gimple_seq by remapping all the statements in BODY
921 : using the inlining information in ID. */
922 :
923 : static gimple_seq
924 67 : remap_gimple_seq (gimple_seq body, copy_body_data *id)
925 : {
926 67 : gimple_stmt_iterator si;
927 67 : gimple_seq new_body = NULL;
928 :
929 67 : for (si = gsi_start (body); !gsi_end_p (si); gsi_next (&si))
930 : {
931 0 : gimple_seq new_stmts = remap_gimple_stmt (gsi_stmt (si), id);
932 0 : gimple_seq_add_seq (&new_body, new_stmts);
933 : }
934 :
935 67 : return new_body;
936 : }
937 :
938 :
939 : /* Copy a GIMPLE_BIND statement STMT, remapping all the symbols in its
940 : block using the mapping information in ID. */
941 :
942 : static gimple *
943 0 : copy_gimple_bind (gbind *stmt, copy_body_data *id)
944 : {
945 0 : gimple *new_bind;
946 0 : tree new_block, new_vars;
947 0 : gimple_seq body, new_body;
948 :
949 : /* Copy the statement. Note that we purposely don't use copy_stmt
950 : here because we need to remap statements as we copy. */
951 0 : body = gimple_bind_body (stmt);
952 0 : new_body = remap_gimple_seq (body, id);
953 :
954 0 : new_block = gimple_bind_block (stmt);
955 0 : if (new_block)
956 0 : remap_block (&new_block, id);
957 :
958 : /* This will remap a lot of the same decls again, but this should be
959 : harmless. */
960 0 : new_vars = gimple_bind_vars (stmt);
961 0 : if (new_vars)
962 0 : new_vars = remap_decls (new_vars, NULL, id);
963 :
964 0 : new_bind = gimple_build_bind (new_vars, new_body, new_block);
965 :
966 0 : return new_bind;
967 : }
968 :
969 : /* Return true if DECL is a parameter or a SSA_NAME for a parameter. */
970 :
971 : static bool
972 31946 : is_parm (tree decl)
973 : {
974 31946 : if (TREE_CODE (decl) == SSA_NAME)
975 : {
976 30064 : decl = SSA_NAME_VAR (decl);
977 : if (!decl)
978 : return false;
979 : }
980 :
981 19385 : return (TREE_CODE (decl) == PARM_DECL);
982 : }
983 :
984 : /* Copy the TREE_THIS_NOTRAP flag from OLD to T if it is appropriate to do so.
985 : T and OLD must be both either INDIRECT_REF or MEM_REF. */
986 :
987 : static void
988 21685080 : maybe_copy_this_notrap (copy_body_data *id, tree t, tree old)
989 : {
990 21685080 : gcc_assert (TREE_CODE (t) == TREE_CODE (old));
991 :
992 : /* We cannot blindly propagate the TREE_THIS_NOTRAP flag if we have remapped
993 : a parameter as the property might be valid only for the parameter itself,
994 : typically when it is passed by reference. But we propagate the flag when
995 : this is the dereference of an entire object done in a type that has self-
996 : referential size, to avoid the static size check in tree_could_trap_p. */
997 21685080 : if (TREE_THIS_NOTRAP (old)
998 21685080 : && (!is_parm (TREE_OPERAND (old, 0))
999 13614 : || (!id->transform_parameter && is_parm (TREE_OPERAND (t, 0)))
1000 9994 : || ((TREE_CODE (t) == INDIRECT_REF
1001 9994 : || integer_zerop (TREE_OPERAND (t, 1)))
1002 9994 : && TREE_CODE (TREE_OPERAND (t, 0)) == ADDR_EXPR
1003 9329 : && DECL_P (TREE_OPERAND (TREE_OPERAND (t, 0), 0))
1004 9329 : && type_contains_placeholder_p (TREE_TYPE (t)))))
1005 18332 : TREE_THIS_NOTRAP (t) = 1;
1006 21685080 : }
1007 :
1008 : /* Remap the dependence CLIQUE from the source to the destination function
1009 : as specified in ID. */
1010 :
1011 : static unsigned short
1012 2838204 : remap_dependence_clique (copy_body_data *id, unsigned short clique)
1013 : {
1014 2838204 : if (clique == 0 || processing_debug_stmt)
1015 : return 0;
1016 2803881 : if (!id->dependence_map)
1017 733457 : id->dependence_map = new hash_map<dependence_hash, unsigned short>;
1018 2803881 : bool existed;
1019 2803881 : unsigned short &newc = id->dependence_map->get_or_insert (clique, &existed);
1020 2803881 : if (!existed)
1021 : {
1022 : /* Clique 1 is reserved for local ones set by PTA. */
1023 1100819 : if (cfun->last_clique == 0)
1024 340306 : cfun->last_clique = 1;
1025 2201638 : newc = get_new_clique (cfun);
1026 : }
1027 2803881 : return newc;
1028 : }
1029 :
1030 : /* Remap the GIMPLE operand pointed to by *TP. DATA is really a
1031 : 'struct walk_stmt_info *'. DATA->INFO is a 'copy_body_data *'.
1032 : WALK_SUBTREES is used to indicate walk_gimple_op whether to keep
1033 : recursing into the children nodes of *TP. */
1034 :
1035 : static tree
1036 212509986 : remap_gimple_op_r (tree *tp, int *walk_subtrees, void *data)
1037 : {
1038 212509986 : struct walk_stmt_info *wi_p = (struct walk_stmt_info *) data;
1039 212509986 : copy_body_data *id = (copy_body_data *) wi_p->info;
1040 212509986 : tree fn = id->src_fn;
1041 :
1042 : /* For recursive invocations this is no longer the LHS itself. */
1043 212509986 : bool is_lhs = wi_p->is_lhs;
1044 212509986 : wi_p->is_lhs = false;
1045 :
1046 212509986 : if (TREE_CODE (*tp) == SSA_NAME)
1047 : {
1048 67631228 : *tp = remap_ssa_name (*tp, id);
1049 67631228 : *walk_subtrees = 0;
1050 67631228 : if (is_lhs)
1051 16821869 : SSA_NAME_DEF_STMT (*tp) = wi_p->stmt;
1052 : return NULL;
1053 : }
1054 144878758 : else if (auto_var_in_fn_p (*tp, fn))
1055 : {
1056 : /* Local variables and labels need to be replaced by equivalent
1057 : variables. We don't want to copy static variables; there's
1058 : only one of those, no matter how many times we inline the
1059 : containing function. Similarly for globals from an outer
1060 : function. */
1061 47331685 : tree new_decl;
1062 :
1063 : /* Remap the declaration. */
1064 47331685 : new_decl = remap_decl (*tp, id);
1065 47331685 : gcc_assert (new_decl);
1066 : /* Replace this variable with the copy. */
1067 47331685 : STRIP_TYPE_NOPS (new_decl);
1068 : /* ??? The C++ frontend uses void * pointer zero to initialize
1069 : any other type. This confuses the middle-end type verification.
1070 : As cloned bodies do not go through gimplification again the fixup
1071 : there doesn't trigger. */
1072 47331685 : if (TREE_CODE (new_decl) == INTEGER_CST
1073 47331685 : && !useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (new_decl)))
1074 0 : new_decl = fold_convert (TREE_TYPE (*tp), new_decl);
1075 47331685 : *tp = new_decl;
1076 47331685 : *walk_subtrees = 0;
1077 : }
1078 97547073 : else if (TREE_CODE (*tp) == STATEMENT_LIST)
1079 0 : gcc_unreachable ();
1080 97547073 : else if (TREE_CODE (*tp) == SAVE_EXPR)
1081 0 : gcc_unreachable ();
1082 97547073 : else if (TREE_CODE (*tp) == LABEL_DECL
1083 97547073 : && (!DECL_CONTEXT (*tp)
1084 461 : || decl_function_context (*tp) == id->src_fn))
1085 : /* These may need to be remapped for EH handling. */
1086 0 : *tp = remap_decl (*tp, id);
1087 97547073 : else if (TREE_CODE (*tp) == FIELD_DECL)
1088 : {
1089 : /* If the enclosing record type is variably_modified_type_p, the field
1090 : has already been remapped. Otherwise, it need not be. */
1091 16924391 : tree *n = id->decl_map->get (*tp);
1092 16924391 : if (n)
1093 83 : *tp = *n;
1094 16924391 : *walk_subtrees = 0;
1095 : }
1096 80622682 : else if (TYPE_P (*tp))
1097 : /* Types may need remapping as well. */
1098 0 : *tp = remap_type (*tp, id);
1099 80622682 : else if (CONSTANT_CLASS_P (*tp))
1100 : {
1101 : /* If this is a constant, we have to copy the node iff the type
1102 : will be remapped. copy_tree_r will not copy a constant. */
1103 13968856 : tree new_type = remap_type (TREE_TYPE (*tp), id);
1104 :
1105 13968856 : if (new_type == TREE_TYPE (*tp))
1106 13965667 : *walk_subtrees = 0;
1107 :
1108 3189 : else if (TREE_CODE (*tp) == INTEGER_CST)
1109 3189 : *tp = wide_int_to_tree (new_type, wi::to_wide (*tp));
1110 : else
1111 : {
1112 0 : *tp = copy_node (*tp);
1113 0 : TREE_TYPE (*tp) = new_type;
1114 : }
1115 : }
1116 : else
1117 : {
1118 : /* Otherwise, just copy the node. Note that copy_tree_r already
1119 : knows not to copy VAR_DECLs, etc., so this is safe. */
1120 :
1121 66653826 : if (TREE_CODE (*tp) == MEM_REF && !id->do_not_fold)
1122 : {
1123 : /* We need to re-canonicalize MEM_REFs from inline substitutions
1124 : that can happen when a pointer argument is an ADDR_EXPR.
1125 : Recurse here manually to allow that. */
1126 14302520 : tree ptr = TREE_OPERAND (*tp, 0);
1127 14302520 : tree type = remap_type (TREE_TYPE (*tp), id);
1128 14302520 : tree old = *tp;
1129 14302520 : walk_tree (&ptr, remap_gimple_op_r, data, NULL);
1130 14302520 : *tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
1131 14302520 : TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1132 14302520 : TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1133 14302520 : copy_warning (*tp, old);
1134 14302520 : if (MR_DEPENDENCE_CLIQUE (old) != 0)
1135 : {
1136 2751903 : MR_DEPENDENCE_CLIQUE (*tp)
1137 2751903 : = remap_dependence_clique (id, MR_DEPENDENCE_CLIQUE (old));
1138 2751903 : MR_DEPENDENCE_BASE (*tp) = MR_DEPENDENCE_BASE (old);
1139 : }
1140 14302520 : maybe_copy_this_notrap (id, *tp, old);
1141 14302520 : REF_REVERSE_STORAGE_ORDER (*tp) = REF_REVERSE_STORAGE_ORDER (old);
1142 14302520 : *walk_subtrees = 0;
1143 14302520 : return NULL;
1144 : }
1145 :
1146 : /* Here is the "usual case". Copy this tree node, and then
1147 : tweak some special cases. */
1148 52351306 : copy_tree_r (tp, walk_subtrees, NULL);
1149 :
1150 52351306 : if (TREE_CODE (*tp) != OMP_CLAUSE)
1151 52351306 : TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
1152 :
1153 52351306 : if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
1154 : {
1155 : /* The copied TARGET_EXPR has never been expanded, even if the
1156 : original node was expanded already. */
1157 0 : TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
1158 0 : TREE_OPERAND (*tp, 3) = NULL_TREE;
1159 : }
1160 52351306 : else if (TREE_CODE (*tp) == ADDR_EXPR)
1161 : {
1162 : /* Variable substitution need not be simple. In particular,
1163 : the MEM_REF substitution above. Make sure that
1164 : TREE_CONSTANT and friends are up-to-date. */
1165 16108920 : int invariant = is_gimple_min_invariant (*tp);
1166 16108920 : walk_tree (&TREE_OPERAND (*tp, 0), remap_gimple_op_r, data, NULL);
1167 16108920 : recompute_tree_invariant_for_addr_expr (*tp);
1168 :
1169 : /* If this used to be invariant, but is not any longer,
1170 : then regimplification is probably needed. */
1171 16108920 : if (invariant && !is_gimple_min_invariant (*tp))
1172 4628 : id->regimplify = true;
1173 :
1174 16108920 : *walk_subtrees = 0;
1175 : }
1176 36242386 : else if (TREE_CODE (*tp) == OMP_NEXT_VARIANT)
1177 : {
1178 : /* Neither operand is interesting, and walking the selector
1179 : causes problems because it's not an expression. */
1180 288 : gcc_assert (TREE_CODE (TREE_OPERAND (*tp, 0)) == INTEGER_CST);
1181 288 : *walk_subtrees = 0;
1182 : }
1183 : }
1184 :
1185 : /* Update the TREE_BLOCK for the cloned expr. */
1186 130576238 : if (EXPR_P (*tp))
1187 : {
1188 34546460 : tree new_block = id->remapping_type_depth == 0 ? id->block : NULL;
1189 34546460 : tree old_block = TREE_BLOCK (*tp);
1190 34546460 : if (old_block)
1191 : {
1192 19937182 : tree *n;
1193 19937182 : n = id->decl_map->get (TREE_BLOCK (*tp));
1194 19937182 : if (n)
1195 19932161 : new_block = *n;
1196 : }
1197 34546460 : TREE_SET_BLOCK (*tp, new_block);
1198 : }
1199 :
1200 : /* Keep iterating. */
1201 : return NULL_TREE;
1202 : }
1203 :
1204 :
1205 : /* Called from copy_body_id via walk_tree. DATA is really a
1206 : `copy_body_data *'. */
1207 :
1208 : tree
1209 1432405920 : copy_tree_body_r (tree *tp, int *walk_subtrees, void *data)
1210 : {
1211 1432405920 : copy_body_data *id = (copy_body_data *) data;
1212 1432405920 : tree fn = id->src_fn;
1213 1432405920 : tree new_block;
1214 :
1215 : /* Begin by recognizing trees that we'll completely rewrite for the
1216 : inlining context. Our output for these trees is completely
1217 : different from out input (e.g. RETURN_EXPR is deleted, and morphs
1218 : into an edge). Further down, we'll handle trees that get
1219 : duplicated and/or tweaked. */
1220 :
1221 : /* When requested, RETURN_EXPRs should be transformed to just the
1222 : contained MODIFY_EXPR. The branch semantics of the return will
1223 : be handled elsewhere by manipulating the CFG rather than a statement. */
1224 1432405920 : if (TREE_CODE (*tp) == RETURN_EXPR && id->transform_return_to_modify)
1225 : {
1226 0 : tree assignment = TREE_OPERAND (*tp, 0);
1227 :
1228 : /* If we're returning something, just turn that into an
1229 : assignment into the equivalent of the original RESULT_DECL.
1230 : If the "assignment" is just the result decl, the result
1231 : decl has already been set (e.g. a recent "foo (&result_decl,
1232 : ...)"); just toss the entire RETURN_EXPR. */
1233 0 : if (assignment && TREE_CODE (assignment) == MODIFY_EXPR)
1234 : {
1235 : /* Replace the RETURN_EXPR with (a copy of) the
1236 : MODIFY_EXPR hanging underneath. */
1237 0 : *tp = copy_node (assignment);
1238 : }
1239 : else /* Else the RETURN_EXPR returns no value. */
1240 : {
1241 0 : *tp = NULL;
1242 0 : return (tree) (void *)1;
1243 : }
1244 0 : }
1245 1432405920 : else if (TREE_CODE (*tp) == SSA_NAME)
1246 : {
1247 4058027 : *tp = remap_ssa_name (*tp, id);
1248 4058027 : *walk_subtrees = 0;
1249 4058027 : return NULL;
1250 : }
1251 :
1252 : /* Local variables and labels need to be replaced by equivalent
1253 : variables. We don't want to copy static variables; there's only
1254 : one of those, no matter how many times we inline the containing
1255 : function. Similarly for globals from an outer function. */
1256 1428347893 : else if (auto_var_in_fn_p (*tp, fn))
1257 : {
1258 175848316 : tree new_decl;
1259 :
1260 : /* Remap the declaration. */
1261 175848316 : new_decl = remap_decl (*tp, id);
1262 175848316 : gcc_assert (new_decl);
1263 : /* Replace this variable with the copy. */
1264 175848316 : STRIP_TYPE_NOPS (new_decl);
1265 175848316 : *tp = new_decl;
1266 175848316 : *walk_subtrees = 0;
1267 : }
1268 1252499577 : else if (TREE_CODE (*tp) == STATEMENT_LIST)
1269 37689709 : copy_statement_list (tp);
1270 1214809868 : else if (TREE_CODE (*tp) == SAVE_EXPR
1271 1214119491 : || TREE_CODE (*tp) == TARGET_EXPR)
1272 15698316 : remap_save_expr (tp, id->decl_map, walk_subtrees);
1273 1199111552 : else if (TREE_CODE (*tp) == LABEL_DECL
1274 1199111552 : && (! DECL_CONTEXT (*tp)
1275 7 : || decl_function_context (*tp) == id->src_fn))
1276 : /* These may need to be remapped for EH handling. */
1277 0 : *tp = remap_decl (*tp, id);
1278 1199111552 : else if (TREE_CODE (*tp) == BIND_EXPR)
1279 20570611 : copy_bind_expr (tp, walk_subtrees, id);
1280 : /* Types may need remapping as well. */
1281 1178540941 : else if (TYPE_P (*tp))
1282 223369 : *tp = remap_type (*tp, id);
1283 :
1284 : /* If this is a constant, we have to copy the node iff the type will be
1285 : remapped. copy_tree_r will not copy a constant. */
1286 1178317572 : else if (CONSTANT_CLASS_P (*tp))
1287 : {
1288 361673153 : tree new_type = remap_type (TREE_TYPE (*tp), id);
1289 :
1290 361673153 : if (new_type == TREE_TYPE (*tp))
1291 361671998 : *walk_subtrees = 0;
1292 :
1293 1155 : else if (TREE_CODE (*tp) == INTEGER_CST)
1294 1155 : *tp = wide_int_to_tree (new_type, wi::to_wide (*tp));
1295 : else
1296 : {
1297 0 : *tp = copy_node (*tp);
1298 0 : TREE_TYPE (*tp) = new_type;
1299 : }
1300 : }
1301 :
1302 : /* Otherwise, just copy the node. Note that copy_tree_r already
1303 : knows not to copy VAR_DECLs, etc., so this is safe. */
1304 : else
1305 : {
1306 : /* Here we handle trees that are not completely rewritten.
1307 : First we detect some inlining-induced bogosities for
1308 : discarding. */
1309 816644419 : if (TREE_CODE (*tp) == MODIFY_EXPR
1310 10960876 : && TREE_OPERAND (*tp, 0) == TREE_OPERAND (*tp, 1)
1311 816644440 : && (auto_var_in_fn_p (TREE_OPERAND (*tp, 0), fn)))
1312 : {
1313 : /* Some assignments VAR = VAR; don't generate any rtl code
1314 : and thus don't count as variable modification. Avoid
1315 : keeping bogosities like 0 = 0. */
1316 0 : tree decl = TREE_OPERAND (*tp, 0), value;
1317 0 : tree *n;
1318 :
1319 0 : n = id->decl_map->get (decl);
1320 0 : if (n)
1321 : {
1322 0 : value = *n;
1323 0 : STRIP_TYPE_NOPS (value);
1324 0 : if (TREE_CONSTANT (value) || TREE_READONLY (value))
1325 : {
1326 0 : *tp = build_empty_stmt (EXPR_LOCATION (*tp));
1327 0 : return copy_tree_body_r (tp, walk_subtrees, data);
1328 : }
1329 : }
1330 : }
1331 816644419 : else if (INDIRECT_REF_P (*tp))
1332 : {
1333 : /* Get rid of *& from inline substitutions that can happen when a
1334 : pointer argument is an ADDR_EXPR. */
1335 44361250 : tree decl = TREE_OPERAND (*tp, 0);
1336 44361250 : tree *n = id->decl_map->get (decl);
1337 44361250 : if (n)
1338 : {
1339 : /* If we happen to get an ADDR_EXPR in n->value, strip
1340 : it manually here as we'll eventually get ADDR_EXPRs
1341 : which lie about their types pointed to. In this case
1342 : build_fold_indirect_ref wouldn't strip the INDIRECT_REF,
1343 : but we absolutely rely on that. As fold_indirect_ref
1344 : does other useful transformations, try that first, though. */
1345 7310361 : tree type = TREE_TYPE (*tp);
1346 7310361 : tree ptr = id->do_not_unshare ? *n : unshare_expr (*n);
1347 7310361 : tree old = *tp;
1348 7310361 : *tp = id->do_not_fold ? NULL : gimple_fold_indirect_ref (ptr);
1349 7310361 : if (! *tp)
1350 : {
1351 7310226 : type = remap_type (type, id);
1352 7310226 : if (TREE_CODE (ptr) == ADDR_EXPR && !id->do_not_fold)
1353 : {
1354 0 : *tp
1355 0 : = fold_indirect_ref_1 (EXPR_LOCATION (ptr), type, ptr);
1356 : /* ??? We should either assert here or build
1357 : a VIEW_CONVERT_EXPR instead of blindly leaking
1358 : incompatible types to our IL. */
1359 0 : if (! *tp)
1360 0 : *tp = TREE_OPERAND (ptr, 0);
1361 : }
1362 : else
1363 : {
1364 7310226 : *tp = build1 (INDIRECT_REF, type, ptr);
1365 7310226 : TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1366 7310226 : TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1367 7310226 : TREE_READONLY (*tp) = TREE_READONLY (old);
1368 7310226 : maybe_copy_this_notrap (id, *tp, old);
1369 : }
1370 : }
1371 7310361 : *walk_subtrees = 0;
1372 7310361 : return NULL;
1373 : }
1374 : }
1375 772283169 : else if (TREE_CODE (*tp) == MEM_REF && !id->do_not_fold)
1376 : {
1377 : /* We need to re-canonicalize MEM_REFs from inline substitutions
1378 : that can happen when a pointer argument is an ADDR_EXPR.
1379 : Recurse here manually to allow that. */
1380 72334 : tree ptr = TREE_OPERAND (*tp, 0);
1381 72334 : tree type = remap_type (TREE_TYPE (*tp), id);
1382 72334 : tree old = *tp;
1383 72334 : walk_tree (&ptr, copy_tree_body_r, data, NULL);
1384 72334 : *tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
1385 72334 : TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1386 72334 : TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1387 72334 : copy_warning (*tp, old);
1388 72334 : if (MR_DEPENDENCE_CLIQUE (old) != 0)
1389 : {
1390 5878 : MR_DEPENDENCE_CLIQUE (*tp)
1391 5878 : = remap_dependence_clique (id, MR_DEPENDENCE_CLIQUE (old));
1392 5878 : MR_DEPENDENCE_BASE (*tp) = MR_DEPENDENCE_BASE (old);
1393 : }
1394 72334 : maybe_copy_this_notrap (id, *tp, old);
1395 72334 : REF_REVERSE_STORAGE_ORDER (*tp) = REF_REVERSE_STORAGE_ORDER (old);
1396 72334 : *walk_subtrees = 0;
1397 72334 : return NULL;
1398 : }
1399 :
1400 : /* Here is the "usual case". Copy this tree node, and then
1401 : tweak some special cases. */
1402 809261724 : copy_tree_r (tp, walk_subtrees, NULL);
1403 :
1404 : /* If EXPR has block defined, map it to newly constructed block.
1405 : When inlining we want EXPRs without block appear in the block
1406 : of function call if we are not remapping a type. */
1407 809261724 : if (EXPR_P (*tp))
1408 : {
1409 706022525 : new_block = id->remapping_type_depth == 0 ? id->block : NULL;
1410 706022525 : if (TREE_BLOCK (*tp))
1411 : {
1412 14023 : tree *n;
1413 14023 : n = id->decl_map->get (TREE_BLOCK (*tp));
1414 14023 : if (n)
1415 14023 : new_block = *n;
1416 : }
1417 706022525 : TREE_SET_BLOCK (*tp, new_block);
1418 : }
1419 :
1420 809261724 : if (TREE_CODE (*tp) != OMP_CLAUSE)
1421 809261633 : TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
1422 :
1423 : /* The copied TARGET_EXPR has never been expanded, even if the
1424 : original node was expanded already. */
1425 809261724 : if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
1426 : {
1427 0 : TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
1428 0 : TREE_OPERAND (*tp, 3) = NULL_TREE;
1429 : }
1430 :
1431 : /* Variable substitution need not be simple. In particular, the
1432 : INDIRECT_REF substitution above. Make sure that TREE_CONSTANT
1433 : and friends are up-to-date. */
1434 809261724 : else if (TREE_CODE (*tp) == ADDR_EXPR)
1435 : {
1436 80072409 : int invariant = is_gimple_min_invariant (*tp);
1437 80072409 : walk_tree (&TREE_OPERAND (*tp, 0), copy_tree_body_r, id, NULL);
1438 :
1439 : /* Handle the case where we substituted an INDIRECT_REF
1440 : into the operand of the ADDR_EXPR. */
1441 80072409 : if (INDIRECT_REF_P (TREE_OPERAND (*tp, 0))
1442 80072409 : && !id->do_not_fold)
1443 : {
1444 195 : tree t = TREE_OPERAND (TREE_OPERAND (*tp, 0), 0);
1445 195 : if (TREE_TYPE (t) != TREE_TYPE (*tp))
1446 195 : t = fold_convert (remap_type (TREE_TYPE (*tp), id), t);
1447 195 : *tp = t;
1448 : }
1449 : else
1450 80072214 : recompute_tree_invariant_for_addr_expr (*tp);
1451 :
1452 : /* If this used to be invariant, but is not any longer,
1453 : then regimplification is probably needed. */
1454 80072409 : if (invariant && !is_gimple_min_invariant (*tp))
1455 19 : id->regimplify = true;
1456 :
1457 80072409 : *walk_subtrees = 0;
1458 : }
1459 729189315 : else if (TREE_CODE (*tp) == OMP_CLAUSE
1460 729189315 : && (OMP_CLAUSE_CODE (*tp) == OMP_CLAUSE_AFFINITY
1461 76 : || OMP_CLAUSE_CODE (*tp) == OMP_CLAUSE_DEPEND))
1462 : {
1463 30 : tree t = OMP_CLAUSE_DECL (*tp);
1464 30 : if (t && OMP_ITERATOR_DECL_P (t))
1465 : {
1466 18 : *walk_subtrees = 0;
1467 18 : OMP_CLAUSE_DECL (*tp) = copy_node (t);
1468 18 : t = OMP_CLAUSE_DECL (*tp);
1469 18 : TREE_PURPOSE (t) = copy_node (TREE_PURPOSE (t));
1470 108 : for (int i = 0; i <= 4; i++)
1471 90 : walk_tree (&TREE_VEC_ELT (TREE_PURPOSE (t), i),
1472 : copy_tree_body_r, id, NULL);
1473 18 : if (OMP_ITERATOR_BLOCK (TREE_PURPOSE (t)))
1474 18 : remap_block (&OMP_ITERATOR_BLOCK (TREE_PURPOSE (t)), id);
1475 18 : walk_tree (&TREE_VALUE (t), copy_tree_body_r, id, NULL);
1476 : }
1477 : }
1478 : }
1479 :
1480 : /* Keep iterating. */
1481 : return NULL_TREE;
1482 : }
1483 :
1484 : /* Helper for remap_gimple_stmt. Given an EH region number for the
1485 : source function, map that to the duplicate EH region number in
1486 : the destination function. */
1487 :
1488 : static int
1489 102195 : remap_eh_region_nr (int old_nr, copy_body_data *id)
1490 : {
1491 102195 : eh_region old_r, new_r;
1492 :
1493 102195 : old_r = get_eh_region_from_number_fn (id->src_cfun, old_nr);
1494 102195 : new_r = static_cast<eh_region> (*id->eh_map->get (old_r));
1495 :
1496 102195 : return new_r->index;
1497 : }
1498 :
1499 : /* Similar, but operate on INTEGER_CSTs. */
1500 :
1501 : static tree
1502 9735 : remap_eh_region_tree_nr (tree old_t_nr, copy_body_data *id)
1503 : {
1504 9735 : int old_nr, new_nr;
1505 :
1506 9735 : old_nr = tree_to_shwi (old_t_nr);
1507 9735 : new_nr = remap_eh_region_nr (old_nr, id);
1508 :
1509 9735 : return build_int_cst (integer_type_node, new_nr);
1510 : }
1511 :
1512 : /* Helper for copy_bb. Remap statement STMT using the inlining
1513 : information in ID. Return the new statement copy. */
1514 :
1515 : static gimple_seq
1516 98598216 : remap_gimple_stmt (gimple *stmt, copy_body_data *id)
1517 : {
1518 98598216 : gimple *copy = NULL;
1519 98598216 : struct walk_stmt_info wi;
1520 98598216 : bool skip_first = false;
1521 98598216 : gimple_seq stmts = NULL;
1522 :
1523 98598216 : if (is_gimple_debug (stmt)
1524 98598216 : && (gimple_debug_nonbind_marker_p (stmt)
1525 14038228 : ? !DECL_STRUCT_FUNCTION (id->dst_fn)->debug_nonbind_markers
1526 47993531 : : !opt_for_fn (id->dst_fn, flag_var_tracking_assignments)))
1527 : return NULL;
1528 :
1529 98566342 : if (!is_gimple_debug (stmt)
1530 36566457 : && id->param_body_adjs
1531 102285089 : && id->param_body_adjs->m_dead_stmts.contains (stmt))
1532 : {
1533 6832 : tree *dval = id->param_body_adjs->m_dead_stmt_debug_equiv.get (stmt);
1534 6832 : if (!dval)
1535 : return NULL;
1536 :
1537 3340 : gcc_assert (is_gimple_assign (stmt));
1538 3340 : tree lhs = gimple_assign_lhs (stmt);
1539 3340 : tree *dvar = id->param_body_adjs->m_dead_ssa_debug_equiv.get (lhs);
1540 3340 : gdebug *bind = gimple_build_debug_bind (*dvar, *dval, stmt);
1541 3340 : if (id->reset_location)
1542 0 : gimple_set_location (bind, input_location);
1543 3340 : id->debug_stmts.safe_push (bind);
1544 3340 : gimple_seq_add_stmt_without_update (&stmts, bind);
1545 3340 : return stmts;
1546 : }
1547 :
1548 : /* Begin by recognizing trees that we'll completely rewrite for the
1549 : inlining context. Our output for these trees is completely
1550 : different from our input (e.g. RETURN_EXPR is deleted and morphs
1551 : into an edge). Further down, we'll handle trees that get
1552 : duplicated and/or tweaked. */
1553 :
1554 : /* When requested, GIMPLE_RETURN should be transformed to just the
1555 : contained GIMPLE_ASSIGN. The branch semantics of the return will
1556 : be handled elsewhere by manipulating the CFG rather than the
1557 : statement. */
1558 98559510 : if (gimple_code (stmt) == GIMPLE_RETURN && id->transform_return_to_modify)
1559 : {
1560 4607572 : tree retval = gimple_return_retval (as_a <greturn *> (stmt));
1561 :
1562 : /* If we're returning something, just turn that into an
1563 : assignment to the equivalent of the original RESULT_DECL.
1564 : If RETVAL is just the result decl, the result decl has
1565 : already been set (e.g. a recent "foo (&result_decl, ...)");
1566 : just toss the entire GIMPLE_RETURN. Likewise for when the
1567 : call doesn't want the return value. */
1568 4607572 : if (retval
1569 4607572 : && (TREE_CODE (retval) != RESULT_DECL
1570 2411398 : && (!id->call_stmt
1571 2411398 : || gimple_call_lhs (id->call_stmt) != NULL_TREE)
1572 2276425 : && (TREE_CODE (retval) != SSA_NAME
1573 1793125 : || ! SSA_NAME_VAR (retval)
1574 425314 : || TREE_CODE (SSA_NAME_VAR (retval)) != RESULT_DECL)))
1575 : {
1576 4386682 : copy = gimple_build_assign (id->do_not_unshare
1577 2193341 : ? id->retvar : unshare_expr (id->retvar),
1578 : retval);
1579 : /* id->retvar is already substituted. Skip it on later remapping. */
1580 2193341 : skip_first = true;
1581 : }
1582 : else
1583 : return NULL;
1584 : }
1585 93951938 : else if (gimple_has_substatements (stmt))
1586 : {
1587 67 : gimple_seq s1, s2;
1588 :
1589 : /* When cloning bodies from the C++ front end, we will be handed bodies
1590 : in High GIMPLE form. Handle here all the High GIMPLE statements that
1591 : have embedded statements. */
1592 67 : switch (gimple_code (stmt))
1593 : {
1594 0 : case GIMPLE_BIND:
1595 0 : copy = copy_gimple_bind (as_a <gbind *> (stmt), id);
1596 0 : break;
1597 :
1598 0 : case GIMPLE_CATCH:
1599 0 : {
1600 0 : gcatch *catch_stmt = as_a <gcatch *> (stmt);
1601 0 : s1 = remap_gimple_seq (gimple_catch_handler (catch_stmt), id);
1602 0 : copy = gimple_build_catch (gimple_catch_types (catch_stmt), s1);
1603 : }
1604 0 : break;
1605 :
1606 0 : case GIMPLE_EH_FILTER:
1607 0 : s1 = remap_gimple_seq (gimple_eh_filter_failure (stmt), id);
1608 0 : copy = gimple_build_eh_filter (gimple_eh_filter_types (stmt), s1);
1609 0 : break;
1610 :
1611 0 : case GIMPLE_TRY:
1612 0 : s1 = remap_gimple_seq (gimple_try_eval (stmt), id);
1613 0 : s2 = remap_gimple_seq (gimple_try_cleanup (stmt), id);
1614 0 : copy = gimple_build_try (s1, s2, gimple_try_kind (stmt));
1615 0 : break;
1616 :
1617 0 : case GIMPLE_WITH_CLEANUP_EXPR:
1618 0 : s1 = remap_gimple_seq (gimple_wce_cleanup (stmt), id);
1619 0 : copy = gimple_build_wce (s1);
1620 0 : break;
1621 :
1622 0 : case GIMPLE_OMP_PARALLEL:
1623 0 : {
1624 0 : gomp_parallel *omp_par_stmt = as_a <gomp_parallel *> (stmt);
1625 0 : s1 = remap_gimple_seq (gimple_omp_body (omp_par_stmt), id);
1626 0 : copy = gimple_build_omp_parallel
1627 0 : (s1,
1628 : gimple_omp_parallel_clauses (omp_par_stmt),
1629 : gimple_omp_parallel_child_fn (omp_par_stmt),
1630 : gimple_omp_parallel_data_arg (omp_par_stmt));
1631 : }
1632 0 : break;
1633 :
1634 0 : case GIMPLE_OMP_TASK:
1635 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1636 0 : copy = gimple_build_omp_task
1637 0 : (s1,
1638 : gimple_omp_task_clauses (stmt),
1639 : gimple_omp_task_child_fn (stmt),
1640 : gimple_omp_task_data_arg (stmt),
1641 : gimple_omp_task_copy_fn (stmt),
1642 : gimple_omp_task_arg_size (stmt),
1643 : gimple_omp_task_arg_align (stmt));
1644 0 : break;
1645 :
1646 0 : case GIMPLE_OMP_FOR:
1647 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1648 0 : s2 = remap_gimple_seq (gimple_omp_for_pre_body (stmt), id);
1649 0 : copy = gimple_build_omp_for (s1, gimple_omp_for_kind (stmt),
1650 : gimple_omp_for_clauses (stmt),
1651 : gimple_omp_for_collapse (stmt), s2);
1652 0 : {
1653 0 : size_t i;
1654 0 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
1655 : {
1656 0 : gimple_omp_for_set_index (copy, i,
1657 : gimple_omp_for_index (stmt, i));
1658 0 : gimple_omp_for_set_initial (copy, i,
1659 : gimple_omp_for_initial (stmt, i));
1660 0 : gimple_omp_for_set_final (copy, i,
1661 : gimple_omp_for_final (stmt, i));
1662 0 : gimple_omp_for_set_incr (copy, i,
1663 : gimple_omp_for_incr (stmt, i));
1664 0 : gimple_omp_for_set_cond (copy, i,
1665 : gimple_omp_for_cond (stmt, i));
1666 : }
1667 : }
1668 : break;
1669 :
1670 0 : case GIMPLE_OMP_MASTER:
1671 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1672 0 : copy = gimple_build_omp_master (s1);
1673 0 : break;
1674 :
1675 0 : case GIMPLE_OMP_MASKED:
1676 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1677 0 : copy = gimple_build_omp_masked
1678 0 : (s1, gimple_omp_masked_clauses (stmt));
1679 0 : break;
1680 :
1681 0 : case GIMPLE_OMP_SCOPE:
1682 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1683 0 : copy = gimple_build_omp_scope
1684 0 : (s1, gimple_omp_scope_clauses (stmt));
1685 0 : break;
1686 :
1687 0 : case GIMPLE_OMP_DISPATCH:
1688 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1689 0 : copy = gimple_build_omp_dispatch (s1,
1690 : gimple_omp_dispatch_clauses (stmt));
1691 0 : break;
1692 :
1693 0 : case GIMPLE_OMP_TASKGROUP:
1694 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1695 0 : copy = gimple_build_omp_taskgroup
1696 0 : (s1, gimple_omp_taskgroup_clauses (stmt));
1697 0 : break;
1698 :
1699 0 : case GIMPLE_OMP_ORDERED:
1700 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1701 0 : copy = gimple_build_omp_ordered
1702 0 : (s1,
1703 0 : gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt)));
1704 0 : break;
1705 :
1706 0 : case GIMPLE_OMP_SCAN:
1707 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1708 0 : copy = gimple_build_omp_scan
1709 0 : (s1, gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)));
1710 0 : break;
1711 :
1712 0 : case GIMPLE_OMP_SECTION:
1713 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1714 0 : copy = gimple_build_omp_section (s1);
1715 0 : break;
1716 :
1717 0 : case GIMPLE_OMP_SECTIONS:
1718 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1719 0 : copy = gimple_build_omp_sections
1720 0 : (s1, gimple_omp_sections_clauses (stmt));
1721 0 : break;
1722 :
1723 0 : case GIMPLE_OMP_STRUCTURED_BLOCK:
1724 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1725 0 : copy = gimple_build_omp_structured_block (s1);
1726 0 : break;
1727 :
1728 0 : case GIMPLE_OMP_SINGLE:
1729 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1730 0 : copy = gimple_build_omp_single
1731 0 : (s1, gimple_omp_single_clauses (stmt));
1732 0 : break;
1733 :
1734 0 : case GIMPLE_OMP_TARGET:
1735 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1736 0 : copy = gimple_build_omp_target
1737 0 : (s1, gimple_omp_target_kind (stmt),
1738 : gimple_omp_target_clauses (stmt));
1739 0 : break;
1740 :
1741 0 : case GIMPLE_OMP_TEAMS:
1742 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1743 0 : copy = gimple_build_omp_teams
1744 0 : (s1, gimple_omp_teams_clauses (stmt));
1745 0 : break;
1746 :
1747 0 : case GIMPLE_OMP_CRITICAL:
1748 0 : s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1749 0 : copy = gimple_build_omp_critical (s1,
1750 : gimple_omp_critical_name
1751 0 : (as_a <gomp_critical *> (stmt)),
1752 : gimple_omp_critical_clauses
1753 0 : (as_a <gomp_critical *> (stmt)));
1754 0 : break;
1755 :
1756 0 : case GIMPLE_ASSUME:
1757 0 : s1 = remap_gimple_seq (gimple_assume_body (stmt), id);
1758 0 : copy = gimple_build_assume (gimple_assume_guard (stmt), s1);
1759 0 : break;
1760 :
1761 67 : case GIMPLE_TRANSACTION:
1762 67 : {
1763 67 : gtransaction *old_trans_stmt = as_a <gtransaction *> (stmt);
1764 67 : gtransaction *new_trans_stmt;
1765 67 : s1 = remap_gimple_seq (gimple_transaction_body (old_trans_stmt),
1766 : id);
1767 67 : copy = new_trans_stmt = gimple_build_transaction (s1);
1768 67 : gimple_transaction_set_subcode (new_trans_stmt,
1769 : gimple_transaction_subcode (old_trans_stmt));
1770 67 : gimple_transaction_set_label_norm (new_trans_stmt,
1771 : gimple_transaction_label_norm (old_trans_stmt));
1772 67 : gimple_transaction_set_label_uninst (new_trans_stmt,
1773 : gimple_transaction_label_uninst (old_trans_stmt));
1774 67 : gimple_transaction_set_label_over (new_trans_stmt,
1775 : gimple_transaction_label_over (old_trans_stmt));
1776 : }
1777 67 : break;
1778 :
1779 0 : default:
1780 0 : gcc_unreachable ();
1781 : }
1782 : }
1783 : else
1784 : {
1785 93951871 : if (gimple_assign_single_p (stmt)
1786 14002086 : && gimple_assign_lhs (stmt) == gimple_assign_rhs1 (stmt)
1787 93951871 : && auto_var_in_fn_p (gimple_assign_lhs (stmt), id->src_fn))
1788 : {
1789 : /* Here we handle statements that are not completely rewritten.
1790 : First we detect some inlining-induced bogosities for
1791 : discarding. */
1792 :
1793 : /* Some assignments VAR = VAR; don't generate any rtl code
1794 : and thus don't count as variable modification. Avoid
1795 : keeping bogosities like 0 = 0. */
1796 0 : tree decl = gimple_assign_lhs (stmt), value;
1797 0 : tree *n;
1798 :
1799 0 : n = id->decl_map->get (decl);
1800 0 : if (n)
1801 : {
1802 0 : value = *n;
1803 0 : STRIP_TYPE_NOPS (value);
1804 0 : if (TREE_CONSTANT (value) || TREE_READONLY (value))
1805 0 : return NULL;
1806 : }
1807 : }
1808 :
1809 : /* For *ptr_N ={v} {CLOBBER}, if ptr_N is SSA_NAME defined
1810 : in a block that we aren't copying during tree_function_versioning,
1811 : just drop the clobber stmt. */
1812 93951871 : if (id->blocks_to_copy && gimple_clobber_p (stmt))
1813 : {
1814 17701 : tree lhs = gimple_assign_lhs (stmt);
1815 17701 : if (TREE_CODE (lhs) == MEM_REF
1816 17701 : && TREE_CODE (TREE_OPERAND (lhs, 0)) == SSA_NAME)
1817 : {
1818 1136 : gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (lhs, 0));
1819 1136 : if (gimple_bb (def_stmt)
1820 1476 : && !bitmap_bit_p (id->blocks_to_copy,
1821 340 : gimple_bb (def_stmt)->index))
1822 : return NULL;
1823 : }
1824 : }
1825 :
1826 : /* We do not allow CLOBBERs of handled components. In case
1827 : returned value is stored via such handled component, remove
1828 : the clobber so stmt verifier is happy. */
1829 93951868 : if (gimple_clobber_p (stmt)
1830 93951868 : && TREE_CODE (gimple_assign_lhs (stmt)) == RESULT_DECL)
1831 : {
1832 0 : tree remapped = remap_decl (gimple_assign_lhs (stmt), id);
1833 0 : if (!DECL_P (remapped)
1834 0 : && TREE_CODE (remapped) != MEM_REF)
1835 : return NULL;
1836 : }
1837 :
1838 93951868 : if (gimple_debug_bind_p (stmt))
1839 : {
1840 47429563 : tree var = gimple_debug_bind_get_var (stmt);
1841 47429563 : tree value = gimple_debug_bind_get_value (stmt);
1842 47429563 : if (id->param_body_adjs
1843 47429563 : && id->param_body_adjs->m_dead_stmts.contains (stmt))
1844 : {
1845 16554 : value = unshare_expr_without_location (value);
1846 16554 : id->param_body_adjs->remap_with_debug_expressions (&value);
1847 : }
1848 :
1849 47429563 : gdebug *copy = gimple_build_debug_bind (var, value, stmt);
1850 47429563 : if (id->reset_location)
1851 4 : gimple_set_location (copy, input_location);
1852 47429563 : id->debug_stmts.safe_push (copy);
1853 47429563 : gimple_seq_add_stmt_without_update (&stmts, copy);
1854 47429563 : return stmts;
1855 : }
1856 46522305 : if (gimple_debug_source_bind_p (stmt))
1857 : {
1858 563968 : gdebug *copy = gimple_build_debug_source_bind
1859 563968 : (gimple_debug_source_bind_get_var (stmt),
1860 : gimple_debug_source_bind_get_value (stmt),
1861 563968 : stmt);
1862 563968 : if (id->reset_location)
1863 0 : gimple_set_location (copy, input_location);
1864 563968 : id->debug_stmts.safe_push (copy);
1865 563968 : gimple_seq_add_stmt_without_update (&stmts, copy);
1866 563968 : return stmts;
1867 : }
1868 45958337 : if (gimple_debug_nonbind_marker_p (stmt))
1869 : {
1870 : /* If the inlined function has too many debug markers,
1871 : don't copy them. */
1872 14006354 : if (id->src_cfun->debug_marker_count
1873 14006354 : > param_max_debug_marker_count
1874 14006354 : || id->reset_location)
1875 0 : return stmts;
1876 :
1877 14006354 : gdebug *copy = as_a <gdebug *> (gimple_copy (stmt));
1878 14006354 : id->debug_stmts.safe_push (copy);
1879 14006354 : gimple_seq_add_stmt_without_update (&stmts, copy);
1880 14006354 : return stmts;
1881 : }
1882 :
1883 : /* Create a new deep copy of the statement. */
1884 31951983 : copy = gimple_copy (stmt);
1885 :
1886 : /* Clear flags that need revisiting. */
1887 31951983 : if (gcall *call_stmt = dyn_cast <gcall *> (copy))
1888 : {
1889 5203294 : if (gimple_call_tail_p (call_stmt))
1890 148 : gimple_call_set_tail (call_stmt, false);
1891 5203294 : if (gimple_call_from_thunk_p (call_stmt))
1892 187 : gimple_call_set_from_thunk (call_stmt, false);
1893 : /* Silently clear musttail flag when inlining a function
1894 : with must tail call from a non-musttail call. The inlining
1895 : removes one frame so acts like musttail's intent, and we
1896 : can be inlining a function with musttail calls in the middle
1897 : of caller where musttail will always error. */
1898 5203294 : if (gimple_call_must_tail_p (call_stmt)
1899 49 : && id->call_stmt
1900 5203335 : && !gimple_call_must_tail_p (id->call_stmt))
1901 14 : gimple_call_set_must_tail (call_stmt, false);
1902 5203294 : if (gimple_call_internal_p (call_stmt))
1903 60076 : switch (gimple_call_internal_fn (call_stmt))
1904 : {
1905 166 : case IFN_GOMP_SIMD_LANE:
1906 166 : case IFN_GOMP_SIMD_VF:
1907 166 : case IFN_GOMP_SIMD_LAST_LANE:
1908 166 : case IFN_GOMP_SIMD_ORDERED_START:
1909 166 : case IFN_GOMP_SIMD_ORDERED_END:
1910 166 : DECL_STRUCT_FUNCTION (id->dst_fn)->has_simduid_loops = true;
1911 166 : break;
1912 : default:
1913 : break;
1914 : }
1915 : }
1916 :
1917 : /* Remap the region numbers for __builtin_eh_{pointer,filter},
1918 : RESX and EH_DISPATCH. */
1919 31951983 : if (id->eh_map)
1920 31951983 : switch (gimple_code (copy))
1921 : {
1922 5203294 : case GIMPLE_CALL:
1923 5203294 : {
1924 5203294 : tree r, fndecl = gimple_call_fndecl (copy);
1925 5203294 : if (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
1926 1437402 : switch (DECL_FUNCTION_CODE (fndecl))
1927 : {
1928 0 : case BUILT_IN_EH_COPY_VALUES:
1929 0 : r = gimple_call_arg (copy, 1);
1930 0 : r = remap_eh_region_tree_nr (r, id);
1931 0 : gimple_call_set_arg (copy, 1, r);
1932 : /* FALLTHRU */
1933 :
1934 9735 : case BUILT_IN_EH_POINTER:
1935 9735 : case BUILT_IN_EH_FILTER:
1936 9735 : r = gimple_call_arg (copy, 0);
1937 9735 : r = remap_eh_region_tree_nr (r, id);
1938 9735 : gimple_call_set_arg (copy, 0, r);
1939 9735 : break;
1940 :
1941 : default:
1942 : break;
1943 : }
1944 :
1945 : /* Reset alias info if we didn't apply measures to
1946 : keep it valid over inlining by setting DECL_PT_UID. */
1947 5203294 : if (!id->src_cfun->gimple_df
1948 5203294 : || !id->src_cfun->gimple_df->ipa_pta)
1949 5195941 : gimple_call_reset_alias_info (as_a <gcall *> (copy));
1950 : }
1951 : break;
1952 :
1953 81634 : case GIMPLE_RESX:
1954 81634 : {
1955 81634 : gresx *resx_stmt = as_a <gresx *> (copy);
1956 81634 : int r = gimple_resx_region (resx_stmt);
1957 81634 : r = remap_eh_region_nr (r, id);
1958 81634 : gimple_resx_set_region (resx_stmt, r);
1959 : }
1960 81634 : break;
1961 :
1962 10826 : case GIMPLE_EH_DISPATCH:
1963 10826 : {
1964 10826 : geh_dispatch *eh_dispatch = as_a <geh_dispatch *> (copy);
1965 10826 : int r = gimple_eh_dispatch_region (eh_dispatch);
1966 10826 : r = remap_eh_region_nr (r, id);
1967 10826 : gimple_eh_dispatch_set_region (eh_dispatch, r);
1968 : }
1969 10826 : break;
1970 :
1971 : default:
1972 : break;
1973 : }
1974 : }
1975 :
1976 : /* If STMT has a block defined, map it to the newly constructed block. */
1977 34145391 : if (tree block = gimple_block (copy))
1978 : {
1979 30852165 : tree *n;
1980 30852165 : n = id->decl_map->get (block);
1981 30852165 : gcc_assert (n);
1982 30852165 : gimple_set_block (copy, *n);
1983 : }
1984 34145391 : if (id->param_body_adjs)
1985 : {
1986 3711915 : gimple_seq extra_stmts = NULL;
1987 3711915 : id->param_body_adjs->modify_gimple_stmt (©, &extra_stmts, stmt);
1988 3711915 : if (!gimple_seq_empty_p (extra_stmts))
1989 : {
1990 58 : memset (&wi, 0, sizeof (wi));
1991 58 : wi.info = id;
1992 58 : for (gimple_stmt_iterator egsi = gsi_start (extra_stmts);
1993 172 : !gsi_end_p (egsi);
1994 114 : gsi_next (&egsi))
1995 114 : walk_gimple_op (gsi_stmt (egsi), remap_gimple_op_r, &wi);
1996 58 : gimple_seq_add_seq_without_update (&stmts, extra_stmts);
1997 : }
1998 : }
1999 :
2000 34145391 : if (id->reset_location)
2001 696 : gimple_set_location (copy, input_location);
2002 :
2003 : /* Debug statements ought to be rebuilt and not copied. */
2004 34145391 : gcc_checking_assert (!is_gimple_debug (copy));
2005 :
2006 : /* Remap all the operands in COPY. */
2007 34145391 : memset (&wi, 0, sizeof (wi));
2008 34145391 : wi.info = id;
2009 34145391 : if (skip_first)
2010 2193341 : walk_tree (gimple_op_ptr (copy, 1), remap_gimple_op_r, &wi, NULL);
2011 : else
2012 31952050 : walk_gimple_op (copy, remap_gimple_op_r, &wi);
2013 :
2014 : /* Clear the copied virtual operands. We are not remapping them here
2015 : but are going to recreate them from scratch. */
2016 34145391 : if (gimple_has_mem_ops (copy))
2017 : {
2018 29296872 : gimple_set_vdef (copy, NULL_TREE);
2019 29296872 : gimple_set_vuse (copy, NULL_TREE);
2020 : }
2021 :
2022 34145391 : if (cfun->can_throw_non_call_exceptions)
2023 : {
2024 : /* When inlining a function which does not have non-call exceptions
2025 : enabled into a function that has (which only happens with
2026 : always-inline) we have to fixup stmts that cannot throw. */
2027 1712947 : if (gcond *cond = dyn_cast <gcond *> (copy))
2028 207927 : if (gimple_could_trap_p (cond))
2029 : {
2030 1 : gassign *cmp
2031 1 : = gimple_build_assign (make_ssa_name (boolean_type_node),
2032 : gimple_cond_code (cond),
2033 : gimple_cond_lhs (cond),
2034 : gimple_cond_rhs (cond));
2035 1 : gimple_seq_add_stmt_without_update (&stmts, cmp);
2036 1 : gimple_cond_set_code (cond, NE_EXPR);
2037 1 : gimple_cond_set_lhs (cond, gimple_assign_lhs (cmp));
2038 1 : gimple_cond_set_rhs (cond, boolean_false_node);
2039 : }
2040 : }
2041 :
2042 34145391 : gimple_seq_add_stmt_without_update (&stmts, copy);
2043 34145391 : return stmts;
2044 : }
2045 :
2046 :
2047 : /* Copy basic block, scale profile accordingly. Edges will be taken care of
2048 : later */
2049 :
2050 : static basic_block
2051 14815247 : copy_bb (copy_body_data *id, basic_block bb,
2052 : profile_count num, profile_count den)
2053 : {
2054 14815247 : gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
2055 14815247 : basic_block copy_basic_block;
2056 14815247 : tree decl;
2057 14815247 : basic_block prev;
2058 :
2059 14815247 : profile_count::adjust_for_ipa_scaling (&num, &den);
2060 :
2061 : /* Search for previous copied basic block. */
2062 14815247 : prev = bb->prev_bb;
2063 14977665 : while (!prev->aux)
2064 162418 : prev = prev->prev_bb;
2065 :
2066 : /* create_basic_block() will append every new block to
2067 : basic_block_info automatically. */
2068 14815247 : copy_basic_block = create_basic_block (NULL, (basic_block) prev->aux);
2069 14815247 : copy_basic_block->count = bb->count.apply_scale (num, den);
2070 :
2071 14815247 : copy_gsi = gsi_start_bb (copy_basic_block);
2072 :
2073 14815247 : unsigned min_cond_uid = 0;
2074 14815247 : if (id->src_cfun->cond_uids)
2075 : {
2076 63 : if (!cfun->cond_uids)
2077 7 : cfun->cond_uids = new hash_map <gcond*, unsigned> ();
2078 :
2079 193 : for (auto itr : *id->src_cfun->cond_uids)
2080 67 : if (itr.second >= min_cond_uid)
2081 67 : min_cond_uid = itr.second + 1;
2082 : }
2083 :
2084 128228710 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2085 : {
2086 98598216 : gimple_seq stmts;
2087 98598216 : gimple *stmt = gsi_stmt (gsi);
2088 98598216 : gimple *orig_stmt = stmt;
2089 98598216 : gimple_stmt_iterator stmts_gsi;
2090 98598216 : bool stmt_added = false;
2091 :
2092 98598216 : id->regimplify = false;
2093 98598216 : stmts = remap_gimple_stmt (stmt, id);
2094 :
2095 98598216 : if (gimple_seq_empty_p (stmts))
2096 2449783 : continue;
2097 :
2098 96148616 : seq_gsi = copy_gsi;
2099 :
2100 96148616 : for (stmts_gsi = gsi_start (stmts);
2101 192297347 : !gsi_end_p (stmts_gsi); )
2102 : {
2103 96148731 : stmt = gsi_stmt (stmts_gsi);
2104 :
2105 : /* Advance iterator now before stmt is moved to seq_gsi. */
2106 96148731 : gsi_next (&stmts_gsi);
2107 :
2108 96148731 : if (gimple_nop_p (stmt))
2109 183 : continue;
2110 :
2111 : /* If -fcondition-coverage is used, register the inlined conditions
2112 : in the cond->expression mapping of the caller. The expression tag
2113 : is shifted conditions from the two bodies are not mixed. */
2114 96148548 : if (id->src_cfun->cond_uids && is_a <gcond*> (stmt))
2115 : {
2116 19 : gcond *orig_cond = as_a <gcond*> (orig_stmt);
2117 19 : gcond *cond = as_a <gcond*> (stmt);
2118 19 : unsigned *v = id->src_cfun->cond_uids->get (orig_cond);
2119 19 : if (v)
2120 19 : cfun->cond_uids->put (cond, *v + min_cond_uid);
2121 : }
2122 :
2123 96148548 : gimple_duplicate_stmt_histograms (cfun, stmt, id->src_cfun,
2124 : orig_stmt);
2125 :
2126 96148548 : gsi_insert_after (&seq_gsi, stmt, GSI_NEW_STMT);
2127 :
2128 96148548 : if (id->regimplify)
2129 4554 : gimple_regimplify_operands (stmt, &seq_gsi);
2130 :
2131 96148731 : stmt_added = true;
2132 : }
2133 :
2134 96148616 : if (!stmt_added)
2135 183 : continue;
2136 :
2137 : /* If copy_basic_block has been empty at the start of this iteration,
2138 : call gsi_start_bb again to get at the newly added statements. */
2139 96148433 : if (gsi_end_p (copy_gsi))
2140 26200230 : copy_gsi = gsi_start_bb (copy_basic_block);
2141 : else
2142 83048318 : gsi_next (©_gsi);
2143 :
2144 : /* Process the new statement. The call to gimple_regimplify_operands
2145 : possibly turned the statement into multiple statements, we
2146 : need to process all of them. */
2147 96148552 : do
2148 : {
2149 96148552 : tree fn;
2150 96148552 : gcall *call_stmt;
2151 :
2152 96148552 : stmt = gsi_stmt (copy_gsi);
2153 96148552 : call_stmt = dyn_cast <gcall *> (stmt);
2154 5203294 : if (call_stmt
2155 5203294 : && gimple_call_va_arg_pack_p (call_stmt)
2156 324 : && id->call_stmt
2157 322 : && ! gimple_call_va_arg_pack_p (id->call_stmt))
2158 : {
2159 : /* __builtin_va_arg_pack () should be replaced by
2160 : all arguments corresponding to ... in the caller. */
2161 308 : tree p;
2162 308 : gcall *new_call;
2163 308 : vec<tree> argarray;
2164 308 : size_t nargs_caller = gimple_call_num_args (id->call_stmt);
2165 308 : size_t nargs = nargs_caller;
2166 :
2167 792 : for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
2168 : {
2169 : /* Avoid crashing on invalid IL that doesn't have a
2170 : varargs function or that passes not enough arguments. */
2171 500 : if (nargs == 0)
2172 : break;
2173 484 : nargs--;
2174 : }
2175 :
2176 : /* Create the new array of arguments. */
2177 308 : size_t nargs_callee = gimple_call_num_args (call_stmt);
2178 308 : size_t n = nargs + nargs_callee;
2179 308 : argarray.create (n);
2180 308 : argarray.safe_grow_cleared (n, true);
2181 :
2182 : /* Copy all the arguments before '...' */
2183 308 : if (nargs_callee)
2184 612 : memcpy (argarray.address (),
2185 306 : gimple_call_arg_ptr (call_stmt, 0),
2186 : nargs_callee * sizeof (tree));
2187 :
2188 : /* Append the arguments passed in '...' */
2189 308 : if (nargs)
2190 174 : memcpy (argarray.address () + nargs_callee,
2191 174 : gimple_call_arg_ptr (id->call_stmt, 0)
2192 174 : + (nargs_caller - nargs), nargs * sizeof (tree));
2193 :
2194 308 : new_call = gimple_build_call_vec (gimple_call_fn (call_stmt),
2195 : argarray);
2196 :
2197 308 : argarray.release ();
2198 :
2199 : /* Copy all GIMPLE_CALL flags, location and block, except
2200 : GF_CALL_VA_ARG_PACK. */
2201 308 : gimple_call_copy_flags (new_call, call_stmt);
2202 308 : gimple_call_set_va_arg_pack (new_call, false);
2203 616 : gimple_call_set_fntype (new_call, gimple_call_fntype (call_stmt));
2204 : /* location includes block. */
2205 308 : gimple_set_location (new_call, gimple_location (stmt));
2206 308 : gimple_call_set_lhs (new_call, gimple_call_lhs (call_stmt));
2207 :
2208 308 : gsi_replace (©_gsi, new_call, false);
2209 308 : stmt = new_call;
2210 : }
2211 96148244 : else if (call_stmt
2212 5202986 : && id->call_stmt
2213 4232495 : && (decl = gimple_call_fndecl (stmt))
2214 100246978 : && fndecl_built_in_p (decl, BUILT_IN_VA_ARG_PACK_LEN))
2215 : {
2216 : /* __builtin_va_arg_pack_len () should be replaced by
2217 : the number of anonymous arguments. */
2218 147 : size_t nargs = gimple_call_num_args (id->call_stmt);
2219 147 : tree count, p;
2220 147 : gimple *new_stmt;
2221 :
2222 423 : for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
2223 276 : nargs--;
2224 :
2225 147 : if (!gimple_call_lhs (stmt))
2226 : {
2227 : /* Drop unused calls. */
2228 1 : gsi_remove (©_gsi, false);
2229 1 : continue;
2230 : }
2231 146 : else if (!gimple_call_va_arg_pack_p (id->call_stmt))
2232 : {
2233 124 : count = build_int_cst (integer_type_node, nargs);
2234 124 : new_stmt = gimple_build_assign (gimple_call_lhs (stmt), count);
2235 124 : gsi_replace (©_gsi, new_stmt, false);
2236 124 : stmt = new_stmt;
2237 : }
2238 22 : else if (nargs != 0)
2239 : {
2240 7 : tree newlhs = make_ssa_name (integer_type_node);
2241 7 : count = build_int_cst (integer_type_node, nargs);
2242 7 : new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
2243 : PLUS_EXPR, newlhs, count);
2244 7 : gimple_call_set_lhs (stmt, newlhs);
2245 7 : gsi_insert_after (©_gsi, new_stmt, GSI_NEW_STMT);
2246 : }
2247 : }
2248 96148097 : else if (call_stmt
2249 5202839 : && id->call_stmt
2250 100380445 : && gimple_call_internal_p (stmt))
2251 45970 : switch (gimple_call_internal_fn (stmt))
2252 : {
2253 154 : case IFN_TSAN_FUNC_EXIT:
2254 : /* Drop .TSAN_FUNC_EXIT () internal calls during inlining. */
2255 154 : gsi_remove (©_gsi, false);
2256 154 : continue;
2257 1396 : case IFN_ASAN_MARK:
2258 : /* Drop .ASAN_MARK internal calls during inlining into
2259 : no_sanitize functions. */
2260 1396 : if (!sanitize_flags_p (SANITIZE_ADDRESS, id->dst_fn)
2261 1396 : && !sanitize_flags_p (SANITIZE_HWADDRESS, id->dst_fn))
2262 : {
2263 14 : gsi_remove (©_gsi, false);
2264 14 : continue;
2265 : }
2266 : break;
2267 : default:
2268 : break;
2269 : }
2270 :
2271 : /* Statements produced by inlining can be unfolded, especially
2272 : when we constant propagated some operands. We can't fold
2273 : them right now for two reasons:
2274 : 1) folding require SSA_NAME_DEF_STMTs to be correct
2275 : 2) we can't change function calls to builtins.
2276 : So we just mark statement for later folding. We mark
2277 : all new statements, instead just statements that has changed
2278 : by some nontrivial substitution so even statements made
2279 : foldable indirectly are updated. If this turns out to be
2280 : expensive, copy_body can be told to watch for nontrivial
2281 : changes. */
2282 96148383 : if (id->statements_to_fold)
2283 96148383 : id->statements_to_fold->add (stmt);
2284 :
2285 : /* We're duplicating a CALL_EXPR. Find any corresponding
2286 : callgraph edges and update or duplicate them. */
2287 96148383 : if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
2288 : {
2289 5203001 : struct cgraph_edge *edge;
2290 :
2291 5203001 : switch (id->transform_call_graph_edges)
2292 : {
2293 4232510 : case CB_CGE_DUPLICATE:
2294 4232510 : edge = id->src_node->get_edge (orig_stmt);
2295 4232510 : if (edge)
2296 : {
2297 4186711 : struct cgraph_edge *old_edge = edge;
2298 :
2299 : /* A speculative call is consist of multiple
2300 : edges - indirect edge and one or more direct edges
2301 : Duplicate the whole thing and distribute frequencies
2302 : accordingly. */
2303 4186711 : if (edge->speculative)
2304 : {
2305 20423 : int n = 0;
2306 20423 : profile_count direct_cnt
2307 20423 : = profile_count::zero ();
2308 :
2309 : /* First figure out the distribution of counts
2310 : so we can re-scale BB profile accordingly. */
2311 47735 : for (cgraph_edge *e = old_edge; e;
2312 27312 : e = e->next_speculative_call_target ())
2313 27312 : direct_cnt = direct_cnt + e->count;
2314 :
2315 20423 : cgraph_edge *indirect
2316 20423 : = old_edge->speculative_call_indirect_edge ();
2317 20423 : profile_count indir_cnt = indirect->count;
2318 :
2319 : /* Next iterate all direct edges, clone it and its
2320 : corresponding reference and update profile. */
2321 20423 : for (cgraph_edge *e = old_edge;
2322 47735 : e;
2323 27312 : e = e->next_speculative_call_target ())
2324 : {
2325 27312 : profile_count cnt = e->count;
2326 :
2327 27312 : id->dst_node->clone_reference
2328 27312 : (e->speculative_call_target_ref (), stmt);
2329 27312 : edge = e->clone (id->dst_node, call_stmt,
2330 : gimple_uid (stmt), num, den,
2331 : true);
2332 27312 : profile_probability prob
2333 27312 : = cnt.probability_in (direct_cnt
2334 : + indir_cnt);
2335 27312 : edge->count
2336 : = copy_basic_block->count.apply_probability
2337 27312 : (prob);
2338 27312 : n++;
2339 : }
2340 20423 : gcc_checking_assert
2341 : (indirect->num_speculative_call_targets_p ()
2342 : == n);
2343 :
2344 : /* Duplicate the indirect edge after all direct edges
2345 : cloned. */
2346 20423 : indirect = indirect->clone (id->dst_node, call_stmt,
2347 : gimple_uid (stmt),
2348 : num, den,
2349 : true);
2350 :
2351 20423 : profile_probability prob
2352 20423 : = indir_cnt.probability_in (direct_cnt
2353 : + indir_cnt);
2354 20423 : indirect->count
2355 20423 : = copy_basic_block->count.apply_probability (prob);
2356 : }
2357 : /* If edge is a callback-carrying edge, copy all its
2358 : attached edges as well. */
2359 4166288 : else if (edge->has_callback)
2360 : {
2361 2394 : edge
2362 2394 : = edge->clone (id->dst_node, call_stmt,
2363 : gimple_uid (stmt), num, den, true);
2364 2394 : cgraph_edge *e;
2365 2846 : for (e = old_edge->first_callback_edge (); e;
2366 452 : e = e->next_callback_edge ())
2367 452 : edge = e->clone (id->dst_node, call_stmt,
2368 : gimple_uid (stmt), num, den, true);
2369 : }
2370 : else
2371 : {
2372 4163894 : edge = edge->clone (id->dst_node, call_stmt,
2373 : gimple_uid (stmt),
2374 : num, den,
2375 : true);
2376 4163894 : edge->count = copy_basic_block->count;
2377 : }
2378 : }
2379 : break;
2380 :
2381 773661 : case CB_CGE_MOVE_CLONES:
2382 773661 : id->dst_node->set_call_stmt_including_clones (orig_stmt,
2383 : call_stmt);
2384 773661 : edge = id->dst_node->get_edge (stmt);
2385 773661 : break;
2386 :
2387 196830 : case CB_CGE_MOVE:
2388 196830 : edge = id->dst_node->get_edge (orig_stmt);
2389 196830 : if (edge)
2390 196464 : edge = cgraph_edge::set_call_stmt (edge, call_stmt);
2391 : break;
2392 :
2393 0 : default:
2394 0 : gcc_unreachable ();
2395 : }
2396 :
2397 : /* Constant propagation on argument done during inlining
2398 : may create new direct call. Produce an edge for it. */
2399 4163894 : if ((!edge
2400 5044777 : || (edge->indirect_inlining_edge
2401 4048 : && id->transform_call_graph_edges == CB_CGE_MOVE_CLONES))
2402 158720 : && id->dst_node->definition
2403 1052282 : && (fn = gimple_call_fndecl (stmt)) != NULL)
2404 : {
2405 0 : struct cgraph_node *dest = cgraph_node::get_create (fn);
2406 :
2407 : /* We have missing edge in the callgraph. This can happen
2408 : when previous inlining turned an indirect call into a
2409 : direct call by constant propagating arguments or we are
2410 : producing dead clone (for further cloning). In all
2411 : other cases we hit a bug (incorrect node sharing is the
2412 : most common reason for missing edges). */
2413 0 : gcc_assert (!dest->definition
2414 : || dest->address_taken
2415 : || !id->src_node->definition
2416 : || !id->dst_node->definition);
2417 0 : if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
2418 0 : id->dst_node->create_edge_including_clones
2419 0 : (dest, orig_stmt, call_stmt, bb->count,
2420 : CIF_ORIGINALLY_INDIRECT_CALL);
2421 : else
2422 0 : id->dst_node->create_edge (dest, call_stmt,
2423 : bb->count)->inline_failed
2424 0 : = CIF_ORIGINALLY_INDIRECT_CALL;
2425 0 : if (dump_file)
2426 : {
2427 0 : fprintf (dump_file, "Created new direct edge to %s\n",
2428 : dest->dump_name ());
2429 : }
2430 : }
2431 :
2432 5203001 : notice_special_calls (as_a <gcall *> (stmt));
2433 : }
2434 :
2435 96148383 : maybe_duplicate_eh_stmt_fn (cfun, stmt, id->src_cfun, orig_stmt,
2436 : id->eh_map, id->eh_lp_nr);
2437 :
2438 96148383 : gsi_next (©_gsi);
2439 : }
2440 96148552 : while (!gsi_end_p (copy_gsi));
2441 :
2442 192296866 : copy_gsi = gsi_last_bb (copy_basic_block);
2443 : }
2444 :
2445 14815247 : return copy_basic_block;
2446 : }
2447 :
2448 : /* Inserting Single Entry Multiple Exit region in SSA form into code in SSA
2449 : form is quite easy, since dominator relationship for old basic blocks does
2450 : not change.
2451 :
2452 : There is however exception where inlining might change dominator relation
2453 : across EH edges from basic block within inlined functions destinating
2454 : to landing pads in function we inline into.
2455 :
2456 : The function fills in PHI_RESULTs of such PHI nodes if they refer
2457 : to gimple regs. Otherwise, the function mark PHI_RESULT of such
2458 : PHI nodes for renaming. For non-gimple regs, renaming is safe: the
2459 : EH edges are abnormal and SSA_NAME_OCCURS_IN_ABNORMAL_PHI must be
2460 : set, and this means that there will be no overlapping live ranges
2461 : for the underlying symbol.
2462 :
2463 : This might change in future if we allow redirecting of EH edges and
2464 : we might want to change way build CFG pre-inlining to include
2465 : all the possible edges then. */
2466 : static void
2467 785592 : update_ssa_across_abnormal_edges (basic_block bb, basic_block ret_bb,
2468 : bool can_throw, bool nonlocal_goto)
2469 : {
2470 785592 : edge e;
2471 785592 : edge_iterator ei;
2472 :
2473 2166739 : FOR_EACH_EDGE (e, ei, bb->succs)
2474 1381147 : if (!e->dest->aux
2475 755196 : || ((basic_block)e->dest->aux)->index == ENTRY_BLOCK)
2476 : {
2477 625951 : gphi *phi;
2478 625951 : gphi_iterator si;
2479 :
2480 625951 : if (!nonlocal_goto)
2481 625513 : gcc_assert (e->flags & EDGE_EH);
2482 :
2483 625951 : if (!can_throw)
2484 229 : gcc_assert (!(e->flags & EDGE_EH));
2485 :
2486 1041053 : for (si = gsi_start_phis (e->dest); !gsi_end_p (si); gsi_next (&si))
2487 : {
2488 415102 : edge re;
2489 :
2490 415102 : phi = si.phi ();
2491 :
2492 : /* For abnormal goto/call edges the receiver can be the
2493 : ENTRY_BLOCK. Do not assert this cannot happen. */
2494 :
2495 415102 : gcc_assert ((e->flags & EDGE_EH)
2496 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)));
2497 :
2498 415102 : re = find_edge (ret_bb, e->dest);
2499 415102 : gcc_checking_assert (re);
2500 415102 : gcc_assert ((re->flags & (EDGE_EH | EDGE_ABNORMAL))
2501 : == (e->flags & (EDGE_EH | EDGE_ABNORMAL)));
2502 :
2503 415102 : SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e),
2504 : USE_FROM_PTR (PHI_ARG_DEF_PTR_FROM_EDGE (phi, re)));
2505 : }
2506 : }
2507 785592 : }
2508 :
2509 : /* Insert clobbers for automatic variables of inlined ID->src_fn
2510 : function at the start of basic block ID->eh_landing_pad_dest. */
2511 :
2512 : static void
2513 453637 : add_clobbers_to_eh_landing_pad (copy_body_data *id)
2514 : {
2515 453637 : tree var;
2516 453637 : basic_block bb = id->eh_landing_pad_dest;
2517 453637 : live_vars_map *vars = NULL;
2518 453637 : unsigned int cnt = 0;
2519 453637 : unsigned int i;
2520 1563814 : FOR_EACH_VEC_SAFE_ELT (id->src_cfun->local_decls, i, var)
2521 700337 : if (VAR_P (var)
2522 700337 : && !DECL_HARD_REGISTER (var)
2523 700337 : && !TREE_THIS_VOLATILE (var)
2524 700317 : && !DECL_HAS_VALUE_EXPR_P (var)
2525 691348 : && !is_gimple_reg (var)
2526 297527 : && auto_var_in_fn_p (var, id->src_fn)
2527 995526 : && !lookup_attribute ("omp simd array", DECL_ATTRIBUTES (var)))
2528 : {
2529 295189 : tree *t = id->decl_map->get (var);
2530 295189 : if (!t)
2531 0 : continue;
2532 295189 : tree new_var = *t;
2533 295189 : if (VAR_P (new_var)
2534 295189 : && !DECL_HARD_REGISTER (new_var)
2535 295189 : && !TREE_THIS_VOLATILE (new_var)
2536 295189 : && !DECL_HAS_VALUE_EXPR_P (new_var)
2537 295189 : && !is_gimple_reg (new_var)
2538 590378 : && auto_var_in_fn_p (new_var, id->dst_fn))
2539 : {
2540 295189 : if (vars == NULL)
2541 184243 : vars = new live_vars_map;
2542 295189 : vars->put (DECL_UID (var), cnt++);
2543 : }
2544 : }
2545 453637 : if (vars == NULL)
2546 269394 : return;
2547 :
2548 184243 : vec<bitmap_head> live = compute_live_vars (id->src_cfun, vars);
2549 1069544 : FOR_EACH_VEC_SAFE_ELT (id->src_cfun->local_decls, i, var)
2550 516815 : if (VAR_P (var))
2551 : {
2552 516815 : edge e;
2553 516815 : edge_iterator ei;
2554 516815 : bool needed = false;
2555 516815 : unsigned int *v = vars->get (DECL_UID (var));
2556 516815 : if (v == NULL)
2557 221626 : continue;
2558 5191331 : FOR_EACH_EDGE (e, ei, bb->preds)
2559 5058838 : if ((e->flags & EDGE_EH) != 0
2560 5058798 : && e->src->index >= id->add_clobbers_to_eh_landing_pads)
2561 : {
2562 485630 : basic_block src_bb = (basic_block) e->src->aux;
2563 :
2564 485630 : if (bitmap_bit_p (&live[src_bb->index], *v))
2565 : {
2566 : needed = true;
2567 : break;
2568 : }
2569 : }
2570 295189 : if (needed)
2571 : {
2572 162696 : tree new_var = *id->decl_map->get (var);
2573 162696 : gimple_stmt_iterator gsi = gsi_after_labels (bb);
2574 162696 : tree clobber = build_clobber (TREE_TYPE (new_var));
2575 162696 : gimple *clobber_stmt = gimple_build_assign (new_var, clobber);
2576 162696 : gsi_insert_before (&gsi, clobber_stmt, GSI_NEW_STMT);
2577 : }
2578 : }
2579 184243 : destroy_live_vars (live);
2580 184243 : delete vars;
2581 : }
2582 :
2583 : /* Copy edges from BB into its copy constructed earlier, scale profile
2584 : accordingly. Edges will be taken care of later. Assume aux
2585 : pointers to point to the copies of each BB. Return true if any
2586 : debug stmts are left after a statement that must end the basic block. */
2587 :
2588 : static bool
2589 24509317 : copy_edges_for_bb (basic_block bb, profile_count num, profile_count den,
2590 : basic_block ret_bb, basic_block abnormal_goto_dest,
2591 : copy_body_data *id)
2592 : {
2593 24509317 : basic_block new_bb = (basic_block) bb->aux;
2594 24509317 : edge_iterator ei;
2595 24509317 : edge old_edge;
2596 24509317 : gimple_stmt_iterator si;
2597 24509317 : bool need_debug_cleanup = false;
2598 :
2599 : /* Use the indices from the original blocks to create edges for the
2600 : new ones. */
2601 47443122 : FOR_EACH_EDGE (old_edge, ei, bb->succs)
2602 22933805 : if (!(old_edge->flags & EDGE_EH))
2603 : {
2604 22773854 : edge new_edge;
2605 22773854 : int flags = old_edge->flags;
2606 22773854 : location_t locus = old_edge->goto_locus;
2607 :
2608 : /* Return edges do get a FALLTHRU flag when they get inlined. */
2609 22773854 : if (old_edge->dest->index == EXIT_BLOCK
2610 4845071 : && !(flags & (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE|EDGE_FAKE))
2611 4845071 : && old_edge->dest->aux != EXIT_BLOCK_PTR_FOR_FN (cfun))
2612 4607572 : flags |= EDGE_FALLTHRU;
2613 :
2614 22773854 : new_edge
2615 22773854 : = make_edge (new_bb, (basic_block) old_edge->dest->aux, flags);
2616 22773854 : new_edge->probability = old_edge->probability;
2617 22773854 : if (!id->reset_location)
2618 22773466 : new_edge->goto_locus = remap_location (locus, id);
2619 : }
2620 :
2621 24509317 : if (bb->index == ENTRY_BLOCK || bb->index == EXIT_BLOCK)
2622 : return false;
2623 :
2624 : /* When doing function splitting, we must decrease count of the return block
2625 : which was previously reachable by block we did not copy. */
2626 14815247 : if (single_succ_p (bb) && single_succ_edge (bb)->dest->index == EXIT_BLOCK)
2627 11520273 : FOR_EACH_EDGE (old_edge, ei, bb->preds)
2628 6675202 : if (old_edge->src->index != ENTRY_BLOCK
2629 3829449 : && !old_edge->src->aux)
2630 47093 : new_bb->count -= old_edge->count ().apply_scale (num, den);
2631 :
2632 : /* Walk stmts from end to start so that splitting will adjust the BB
2633 : pointer for each stmt at most once, even when we split the block
2634 : multiple times. */
2635 14815247 : bool seen_nondebug = false;
2636 125778884 : for (si = gsi_last_bb (new_bb); !gsi_end_p (si);)
2637 : {
2638 96148390 : bool can_throw, nonlocal_goto;
2639 96148390 : gimple *copy_stmt = gsi_stmt (si);
2640 :
2641 : /* Do this before the possible split_block. */
2642 96148390 : gsi_prev (&si);
2643 :
2644 : /* If this tree could throw an exception, there are two
2645 : cases where we need to add abnormal edge(s): the
2646 : tree wasn't in a region and there is a "current
2647 : region" in the caller; or the original tree had
2648 : EH edges. In both cases split the block after the tree,
2649 : and add abnormal edge(s) as needed; we need both
2650 : those from the callee and the caller.
2651 : We check whether the copy can throw, because the const
2652 : propagation can change an INDIRECT_REF which throws
2653 : into a COMPONENT_REF which doesn't. If the copy
2654 : can throw, the original could also throw. */
2655 96148390 : can_throw = stmt_can_throw_internal (cfun, copy_stmt);
2656 96148390 : nonlocal_goto
2657 96148390 : = (stmt_can_make_abnormal_goto (copy_stmt)
2658 96148390 : && !computed_goto_p (copy_stmt));
2659 :
2660 96147894 : if (can_throw || nonlocal_goto)
2661 : {
2662 : /* If there's only debug insns after copy_stmt don't split
2663 : the block but instead mark the block for cleanup. */
2664 785730 : if (!seen_nondebug)
2665 : need_debug_cleanup = true;
2666 : else
2667 : {
2668 : /* Note that bb's predecessor edges aren't necessarily
2669 : right at this point; split_block doesn't care. */
2670 289559 : edge e = split_block (new_bb, copy_stmt);
2671 289559 : e->dest->aux = new_bb->aux;
2672 289559 : seen_nondebug = false;
2673 : }
2674 : }
2675 :
2676 96148390 : if (!is_gimple_debug (copy_stmt))
2677 34145165 : seen_nondebug = true;
2678 :
2679 96148390 : bool update_probs = false;
2680 :
2681 96148390 : if (gimple_code (copy_stmt) == GIMPLE_EH_DISPATCH)
2682 : {
2683 10826 : make_eh_dispatch_edges (as_a <geh_dispatch *> (copy_stmt));
2684 10826 : update_probs = true;
2685 : }
2686 96137564 : else if (can_throw)
2687 : {
2688 785363 : make_eh_edge (copy_stmt);
2689 785363 : update_probs = true;
2690 : }
2691 :
2692 : /* EH edges may not match old edges. Copy as much as possible. */
2693 796189 : if (update_probs)
2694 : {
2695 796189 : edge e;
2696 796189 : edge_iterator ei;
2697 796189 : basic_block copy_stmt_bb = gimple_bb (copy_stmt);
2698 :
2699 1602956 : FOR_EACH_EDGE (old_edge, ei, bb->succs)
2700 806767 : if ((old_edge->flags & EDGE_EH)
2701 169017 : && (e = find_edge (copy_stmt_bb,
2702 169017 : (basic_block) old_edge->dest->aux))
2703 966517 : && (e->flags & EDGE_EH))
2704 159750 : e->probability = old_edge->probability;
2705 :
2706 2189293 : FOR_EACH_EDGE (e, ei, copy_stmt_bb->succs)
2707 1393104 : if (e->flags & EDGE_EH)
2708 : {
2709 785363 : if (!e->probability.initialized_p ())
2710 625999 : e->probability = profile_probability::never ();
2711 785363 : if (e->dest->index < id->add_clobbers_to_eh_landing_pads)
2712 : {
2713 624578 : if (id->eh_landing_pad_dest == NULL)
2714 453637 : id->eh_landing_pad_dest = e->dest;
2715 : else
2716 170941 : gcc_assert (id->eh_landing_pad_dest == e->dest);
2717 : }
2718 : }
2719 : }
2720 :
2721 :
2722 : /* If the call we inline cannot make abnormal goto do not add
2723 : additional abnormal edges but only retain those already present
2724 : in the original function body. */
2725 96148390 : if (abnormal_goto_dest == NULL)
2726 : nonlocal_goto = false;
2727 1101 : if (nonlocal_goto)
2728 : {
2729 338 : basic_block copy_stmt_bb = gimple_bb (copy_stmt);
2730 :
2731 338 : if (get_abnormal_succ_dispatcher (copy_stmt_bb))
2732 : nonlocal_goto = false;
2733 : /* ABNORMAL_DISPATCHER (1) is for longjmp/setjmp or nonlocal gotos
2734 : in OpenMP regions which aren't allowed to be left abnormally.
2735 : So, no need to add abnormal edge in that case. */
2736 338 : else if (is_gimple_call (copy_stmt)
2737 338 : && gimple_call_internal_p (copy_stmt)
2738 0 : && (gimple_call_internal_fn (copy_stmt)
2739 : == IFN_ABNORMAL_DISPATCHER)
2740 338 : && gimple_call_arg (copy_stmt, 0) == boolean_true_node)
2741 : nonlocal_goto = false;
2742 : else
2743 : {
2744 338 : make_single_succ_edge (copy_stmt_bb, abnormal_goto_dest,
2745 : EDGE_ABNORMAL);
2746 338 : gimple_call_set_ctrl_altering (copy_stmt, true);
2747 338 : if (is_a <gcall *> (copy_stmt)
2748 338 : && (gimple_call_flags (copy_stmt) & ECF_NORETURN))
2749 78 : fixup_noreturn_call (copy_stmt);
2750 : }
2751 : }
2752 :
2753 96148390 : if ((can_throw || nonlocal_goto)
2754 96933982 : && gimple_in_ssa_p (cfun))
2755 785592 : update_ssa_across_abnormal_edges (gimple_bb (copy_stmt), ret_bb,
2756 : can_throw, nonlocal_goto);
2757 : }
2758 : return need_debug_cleanup;
2759 : }
2760 :
2761 : /* Copy the PHIs. All blocks and edges are copied, some blocks
2762 : was possibly split and new outgoing EH edges inserted.
2763 : BB points to the block of original function and AUX pointers links
2764 : the original and newly copied blocks. */
2765 :
2766 : static void
2767 24509317 : copy_phis_for_bb (basic_block bb, copy_body_data *id)
2768 : {
2769 24509317 : basic_block const new_bb = (basic_block) bb->aux;
2770 24509317 : edge_iterator ei;
2771 24509317 : gphi *phi;
2772 24509317 : gphi_iterator si;
2773 24509317 : edge new_edge;
2774 24509317 : bool inserted = false;
2775 :
2776 27578151 : for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
2777 : {
2778 3068834 : tree res, new_res;
2779 3068834 : gphi *new_phi;
2780 :
2781 3068834 : phi = si.phi ();
2782 3068834 : res = PHI_RESULT (phi);
2783 3068834 : new_res = res;
2784 3068834 : if (!virtual_operand_p (res)
2785 3068834 : && (!id->param_body_adjs
2786 3068834 : || !id->param_body_adjs->m_dead_stmts.contains (phi)))
2787 : {
2788 1706419 : walk_tree (&new_res, copy_tree_body_r, id, NULL);
2789 1706419 : if (EDGE_COUNT (new_bb->preds) == 0)
2790 : {
2791 : /* Technically we'd want a SSA_DEFAULT_DEF here... */
2792 0 : SSA_NAME_DEF_STMT (new_res) = gimple_build_nop ();
2793 : }
2794 : else
2795 : {
2796 1706419 : new_phi = create_phi_node (new_res, new_bb);
2797 5599264 : FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
2798 : {
2799 3892845 : edge old_edge = find_edge ((basic_block) new_edge->src->aux,
2800 3892845 : bb);
2801 3892845 : tree arg;
2802 3892845 : tree new_arg;
2803 3892845 : edge_iterator ei2;
2804 3892845 : location_t locus;
2805 :
2806 : /* When doing partial cloning, we allow PHIs on the entry
2807 : block as long as all the arguments are the same.
2808 : Find any input edge to see argument to copy. */
2809 3892845 : if (!old_edge)
2810 2057 : FOR_EACH_EDGE (old_edge, ei2, bb->preds)
2811 2057 : if (!old_edge->src->aux)
2812 : break;
2813 :
2814 3892845 : arg = PHI_ARG_DEF_FROM_EDGE (phi, old_edge);
2815 3892845 : new_arg = arg;
2816 3892845 : walk_tree (&new_arg, copy_tree_body_r, id, NULL);
2817 3892845 : gcc_assert (new_arg);
2818 : /* With return slot optimization we can end up with
2819 : non-gimple (foo *)&this->m, fix that here. */
2820 3892845 : if (TREE_CODE (new_arg) != SSA_NAME
2821 1582803 : && TREE_CODE (new_arg) != FUNCTION_DECL
2822 5475648 : && !is_gimple_val (new_arg))
2823 : {
2824 12 : gimple_seq stmts = NULL;
2825 12 : new_arg = force_gimple_operand (new_arg, &stmts, true,
2826 : NULL);
2827 12 : gsi_insert_seq_on_edge (new_edge, stmts);
2828 12 : inserted = true;
2829 : }
2830 3892845 : locus = gimple_phi_arg_location_from_edge (phi, old_edge);
2831 3892845 : if (id->reset_location)
2832 0 : locus = input_location;
2833 : else
2834 3892845 : locus = remap_location (locus, id);
2835 3892845 : add_phi_arg (new_phi, new_arg, new_edge, locus);
2836 : }
2837 : }
2838 : }
2839 : }
2840 :
2841 : /* Commit the delayed edge insertions. */
2842 24509317 : if (inserted)
2843 36 : FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
2844 24 : gsi_commit_one_edge_insert (new_edge, NULL);
2845 24509317 : }
2846 :
2847 :
2848 : /* Wrapper for remap_decl so it can be used as a callback. */
2849 :
2850 : static tree
2851 104529 : remap_decl_1 (tree decl, void *data)
2852 : {
2853 104529 : return remap_decl (decl, (copy_body_data *) data);
2854 : }
2855 :
2856 : /* Build struct function and associated datastructures for the new clone
2857 : NEW_FNDECL to be build. CALLEE_FNDECL is the original. Function changes
2858 : the cfun to the function of new_fndecl (and current_function_decl too). */
2859 :
2860 : static void
2861 253174 : initialize_cfun (tree new_fndecl, tree callee_fndecl, profile_count count)
2862 : {
2863 253174 : struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
2864 :
2865 : /* Register specific tree functions. */
2866 253174 : gimple_register_cfg_hooks ();
2867 :
2868 : /* Get clean struct function. */
2869 253174 : push_struct_function (new_fndecl, true);
2870 253174 : targetm.target_option.relayout_function (new_fndecl);
2871 :
2872 : /* We will rebuild these, so just sanity check that they are empty. */
2873 253174 : gcc_assert (VALUE_HISTOGRAMS (cfun) == NULL);
2874 253174 : gcc_assert (cfun->local_decls == NULL);
2875 253174 : gcc_assert (cfun->cfg == NULL);
2876 253174 : gcc_assert (cfun->decl == new_fndecl);
2877 :
2878 : /* Copy items we preserve during cloning. */
2879 253174 : cfun->static_chain_decl = src_cfun->static_chain_decl;
2880 253174 : cfun->nonlocal_goto_save_area = src_cfun->nonlocal_goto_save_area;
2881 253174 : cfun->function_end_locus = src_cfun->function_end_locus;
2882 253174 : cfun->curr_properties = src_cfun->curr_properties;
2883 253174 : cfun->va_list_gpr_size = src_cfun->va_list_gpr_size;
2884 253174 : cfun->va_list_fpr_size = src_cfun->va_list_fpr_size;
2885 253174 : cfun->has_nonlocal_label = src_cfun->has_nonlocal_label;
2886 253174 : cfun->calls_eh_return = src_cfun->calls_eh_return;
2887 253174 : cfun->stdarg = src_cfun->stdarg;
2888 253174 : cfun->after_inlining = src_cfun->after_inlining;
2889 253174 : cfun->can_throw_non_call_exceptions
2890 253174 : = src_cfun->can_throw_non_call_exceptions;
2891 253174 : cfun->can_delete_dead_exceptions = src_cfun->can_delete_dead_exceptions;
2892 253174 : cfun->returns_struct = src_cfun->returns_struct;
2893 253174 : cfun->returns_pcc_struct = src_cfun->returns_pcc_struct;
2894 253174 : cfun->assume_function = src_cfun->assume_function;
2895 :
2896 253174 : init_empty_tree_cfg ();
2897 :
2898 253174 : profile_status_for_fn (cfun) = profile_status_for_fn (src_cfun);
2899 253174 : cfun->cfg->full_profile = src_cfun->cfg->full_profile;
2900 :
2901 253174 : profile_count num = count;
2902 253174 : profile_count den = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count;
2903 253174 : profile_count::adjust_for_ipa_scaling (&num, &den);
2904 :
2905 253174 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->count =
2906 253174 : ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num, den);
2907 253174 : EXIT_BLOCK_PTR_FOR_FN (cfun)->count =
2908 253174 : EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num, den);
2909 253174 : if (src_cfun->eh)
2910 253174 : init_eh_for_function ();
2911 :
2912 253174 : if (src_cfun->gimple_df)
2913 : {
2914 253174 : init_tree_ssa (cfun);
2915 253174 : cfun->gimple_df->in_ssa_p = src_cfun->gimple_df->in_ssa_p;
2916 253174 : if (cfun->gimple_df->in_ssa_p)
2917 253174 : init_ssa_operands (cfun);
2918 : }
2919 253174 : }
2920 :
2921 : /* Helper function for copy_cfg_body. Move debug stmts from the end
2922 : of NEW_BB to the beginning of successor basic blocks when needed. If the
2923 : successor has multiple predecessors, reset them, otherwise keep
2924 : their value. */
2925 :
2926 : static void
2927 1649537 : maybe_move_debug_stmts_to_successors (copy_body_data *id, basic_block new_bb)
2928 : {
2929 1649537 : edge e;
2930 1649537 : edge_iterator ei;
2931 1649537 : gimple_stmt_iterator si = gsi_last_nondebug_bb (new_bb);
2932 :
2933 1649537 : if (gsi_end_p (si)
2934 1720417 : || gsi_one_before_end_p (si)
2935 1819392 : || !(stmt_can_throw_internal (cfun, gsi_stmt (si))
2936 70881 : || stmt_can_make_abnormal_goto (gsi_stmt (si))))
2937 1621443 : return;
2938 :
2939 84284 : FOR_EACH_EDGE (e, ei, new_bb->succs)
2940 : {
2941 56190 : gimple_stmt_iterator ssi = gsi_last_bb (new_bb);
2942 56190 : gimple_stmt_iterator dsi = gsi_after_labels (e->dest);
2943 177061 : while (is_gimple_debug (gsi_stmt (ssi)))
2944 : {
2945 120871 : gimple *stmt = gsi_stmt (ssi);
2946 120871 : gdebug *new_stmt;
2947 120871 : tree var;
2948 120871 : tree value;
2949 :
2950 : /* For the last edge move the debug stmts instead of copying
2951 : them. */
2952 120871 : if (ei_one_before_end_p (ei))
2953 : {
2954 60434 : si = ssi;
2955 60434 : gsi_prev (&ssi);
2956 60434 : if (!single_pred_p (e->dest) && gimple_debug_bind_p (stmt))
2957 : {
2958 58912 : gimple_debug_bind_reset_value (stmt);
2959 117185 : gimple_set_location (stmt, UNKNOWN_LOCATION);
2960 : }
2961 60434 : gsi_remove (&si, false);
2962 60434 : gsi_insert_before (&dsi, stmt, GSI_NEW_STMT);
2963 60434 : continue;
2964 : }
2965 :
2966 60437 : if (gimple_debug_bind_p (stmt))
2967 : {
2968 58915 : var = gimple_debug_bind_get_var (stmt);
2969 58915 : if (single_pred_p (e->dest))
2970 : {
2971 49230 : value = gimple_debug_bind_get_value (stmt);
2972 49230 : value = unshare_expr (value);
2973 49230 : new_stmt = gimple_build_debug_bind (var, value, stmt);
2974 : }
2975 : else
2976 9685 : new_stmt = gimple_build_debug_bind (var, NULL_TREE, NULL);
2977 : }
2978 1522 : else if (gimple_debug_source_bind_p (stmt))
2979 : {
2980 0 : var = gimple_debug_source_bind_get_var (stmt);
2981 0 : value = gimple_debug_source_bind_get_value (stmt);
2982 0 : new_stmt = gimple_build_debug_source_bind (var, value, stmt);
2983 : }
2984 1522 : else if (gimple_debug_nonbind_marker_p (stmt))
2985 1522 : new_stmt = as_a <gdebug *> (gimple_copy (stmt));
2986 : else
2987 0 : gcc_unreachable ();
2988 60437 : gsi_insert_before (&dsi, new_stmt, GSI_NEW_STMT);
2989 60437 : id->debug_stmts.safe_push (new_stmt);
2990 60437 : gsi_prev (&ssi);
2991 : }
2992 : }
2993 : }
2994 :
2995 : /* Make a copy of the sub-loops of SRC_PARENT and place them
2996 : as siblings of DEST_PARENT. */
2997 :
2998 : static void
2999 5519654 : copy_loops (copy_body_data *id,
3000 : class loop *dest_parent, class loop *src_parent)
3001 : {
3002 5519654 : class loop *src_loop = src_parent->inner;
3003 6143841 : while (src_loop)
3004 : {
3005 624187 : if (!id->blocks_to_copy
3006 624187 : || bitmap_bit_p (id->blocks_to_copy, src_loop->header->index))
3007 : {
3008 620014 : class loop *dest_loop = alloc_loop ();
3009 :
3010 : /* Assign the new loop its header and latch and associate
3011 : those with the new loop. */
3012 620014 : dest_loop->header = (basic_block)src_loop->header->aux;
3013 620014 : dest_loop->header->loop_father = dest_loop;
3014 620014 : if (src_loop->latch != NULL)
3015 : {
3016 619998 : dest_loop->latch = (basic_block)src_loop->latch->aux;
3017 619998 : dest_loop->latch->loop_father = dest_loop;
3018 : }
3019 :
3020 : /* Copy loop meta-data. */
3021 620014 : copy_loop_info (src_loop, dest_loop);
3022 620014 : if (dest_loop->unroll)
3023 2192 : cfun->has_unroll = true;
3024 620014 : if (dest_loop->force_vectorize)
3025 154 : cfun->has_force_vectorize_loops = true;
3026 620014 : if (id->src_cfun->last_clique != 0)
3027 80423 : dest_loop->owned_clique
3028 80423 : = remap_dependence_clique (id,
3029 80423 : src_loop->owned_clique
3030 : ? src_loop->owned_clique : 1);
3031 :
3032 : /* Finally place it into the loop array and the loop tree. */
3033 620014 : place_new_loop (cfun, dest_loop);
3034 620014 : flow_loop_tree_node_add (dest_parent, dest_loop);
3035 :
3036 620014 : if (src_loop->simduid)
3037 : {
3038 68 : dest_loop->simduid = remap_decl (src_loop->simduid, id);
3039 68 : cfun->has_simduid_loops = true;
3040 : }
3041 :
3042 : /* Recurse. */
3043 620014 : copy_loops (id, dest_loop, src_loop);
3044 : }
3045 624187 : src_loop = src_loop->next;
3046 : }
3047 5519654 : }
3048 :
3049 : /* Call redirect_call_stmt_to_callee on all calls in BB. */
3050 :
3051 : void
3052 13249894 : redirect_all_calls (copy_body_data * id, basic_block bb)
3053 : {
3054 13249894 : gimple_stmt_iterator si;
3055 13249894 : gimple *last = last_nondebug_stmt (bb);
3056 109176106 : for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
3057 : {
3058 82676318 : gimple *stmt = gsi_stmt (si);
3059 82676318 : if (is_gimple_call (stmt))
3060 : {
3061 4287134 : struct cgraph_edge *edge = id->dst_node->get_edge (stmt);
3062 4287134 : if (edge)
3063 : {
3064 4241335 : if (!id->killed_new_ssa_names)
3065 2260969 : id->killed_new_ssa_names = new hash_set<tree> (16);
3066 4241335 : cgraph_edge::redirect_call_stmt_to_callee (
3067 : edge, id->killed_new_ssa_names);
3068 4241335 : if (edge->has_callback)
3069 : {
3070 : /* When redirecting a carrying edge, we need to redirect its
3071 : attached edges as well. */
3072 2394 : cgraph_edge *cbe;
3073 2846 : for (cbe = edge->first_callback_edge (); cbe;
3074 452 : cbe = cbe->next_callback_edge ())
3075 452 : cgraph_edge::redirect_call_stmt_to_callee (
3076 : cbe, id->killed_new_ssa_names);
3077 : }
3078 :
3079 4241335 : if (stmt == last && id->call_stmt && maybe_clean_eh_stmt (stmt))
3080 59996 : gimple_purge_dead_eh_edges (bb);
3081 : }
3082 : }
3083 : }
3084 13249894 : }
3085 :
3086 : /* Make a copy of the body of FN so that it can be inserted inline in
3087 : another function. Walks FN via CFG, returns new fndecl. */
3088 :
3089 : static tree
3090 4899640 : copy_cfg_body (copy_body_data * id,
3091 : basic_block entry_block_map, basic_block exit_block_map,
3092 : basic_block new_entry)
3093 : {
3094 4899640 : tree callee_fndecl = id->src_fn;
3095 : /* Original cfun for the callee, doesn't change. */
3096 4899640 : struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
3097 4899640 : struct function *cfun_to_copy;
3098 4899640 : basic_block bb;
3099 4899640 : tree new_fndecl = NULL;
3100 4899640 : bool need_debug_cleanup = false;
3101 4899640 : int last;
3102 4899640 : profile_count den = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count;
3103 4899640 : profile_count num = entry_block_map->count;
3104 :
3105 4899640 : cfun_to_copy = id->src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
3106 :
3107 : /* Register specific tree functions. */
3108 4899640 : gimple_register_cfg_hooks ();
3109 :
3110 : /* If we are offlining region of the function, make sure to connect
3111 : new entry to ENTRY_BLOCK_PTR_FOR_FN (cfun). Since new entry can be
3112 : part of loop, we must compute frequency and probability of
3113 : ENTRY_BLOCK_PTR_FOR_FN (cfun) based on the frequencies and
3114 : probabilities of edges incoming from nonduplicated region. */
3115 4899640 : if (new_entry)
3116 : {
3117 52605 : edge e;
3118 52605 : edge_iterator ei;
3119 52605 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = profile_count::zero ();
3120 :
3121 113283 : FOR_EACH_EDGE (e, ei, new_entry->preds)
3122 60678 : if (!e->src->aux)
3123 60678 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->count += e->count ();
3124 : /* Do not scale - the profile of offlined region should
3125 : remain unchanged. */
3126 52605 : num = den = profile_count::one ();
3127 : }
3128 :
3129 4899640 : profile_count::adjust_for_ipa_scaling (&num, &den);
3130 :
3131 : /* Must have a CFG here at this point. */
3132 4899640 : gcc_assert (ENTRY_BLOCK_PTR_FOR_FN
3133 : (DECL_STRUCT_FUNCTION (callee_fndecl)));
3134 :
3135 :
3136 4899640 : ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = entry_block_map;
3137 4899640 : EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = exit_block_map;
3138 4899640 : entry_block_map->aux = ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy);
3139 4899640 : exit_block_map->aux = EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy);
3140 :
3141 : /* Duplicate any exception-handling regions. */
3142 4899640 : if (cfun->eh)
3143 4899640 : id->eh_map = duplicate_eh_regions (cfun_to_copy, NULL, id->eh_lp_nr,
3144 : remap_decl_1, id);
3145 :
3146 : /* Use aux pointers to map the original blocks to copy. */
3147 19898457 : FOR_EACH_BB_FN (bb, cfun_to_copy)
3148 14998817 : if (!id->blocks_to_copy || bitmap_bit_p (id->blocks_to_copy, bb->index))
3149 : {
3150 14815247 : basic_block new_bb = copy_bb (id, bb, num, den);
3151 14815247 : bb->aux = new_bb;
3152 14815247 : new_bb->aux = bb;
3153 14815247 : new_bb->loop_father = entry_block_map->loop_father;
3154 : }
3155 :
3156 4899640 : last = last_basic_block_for_fn (cfun);
3157 :
3158 : /* Now that we've duplicated the blocks, duplicate their edges. */
3159 4899640 : basic_block abnormal_goto_dest = NULL;
3160 4899640 : if (id->call_stmt
3161 4899640 : && stmt_can_make_abnormal_goto (id->call_stmt))
3162 : {
3163 327 : gimple_stmt_iterator gsi = gsi_for_stmt (id->call_stmt);
3164 :
3165 327 : bb = gimple_bb (id->call_stmt);
3166 327 : gsi_next (&gsi);
3167 327 : if (gsi_end_p (gsi))
3168 327 : abnormal_goto_dest = get_abnormal_succ_dispatcher (bb);
3169 : }
3170 29697737 : FOR_ALL_BB_FN (bb, cfun_to_copy)
3171 24798097 : if (!id->blocks_to_copy
3172 24798097 : || (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
3173 24509317 : need_debug_cleanup |= copy_edges_for_bb (bb, num, den, exit_block_map,
3174 : abnormal_goto_dest, id);
3175 :
3176 4899640 : if (id->eh_landing_pad_dest)
3177 : {
3178 453637 : add_clobbers_to_eh_landing_pad (id);
3179 453637 : id->eh_landing_pad_dest = NULL;
3180 : }
3181 :
3182 4899640 : if (new_entry)
3183 : {
3184 52605 : edge e = make_edge (entry_block_map, (basic_block)new_entry->aux,
3185 : EDGE_FALLTHRU);
3186 52605 : e->probability = profile_probability::always ();
3187 : }
3188 :
3189 : /* Duplicate the loop tree, if available and wanted. */
3190 4899640 : if (loops_for_fn (src_cfun) != NULL
3191 4899640 : && current_loops != NULL)
3192 : {
3193 4899640 : copy_loops (id, entry_block_map->loop_father,
3194 : get_loop (src_cfun, 0));
3195 : /* Defer to cfgcleanup to update loop-father fields of basic-blocks. */
3196 4899640 : loops_state_set (LOOPS_NEED_FIXUP);
3197 : }
3198 :
3199 : /* If the loop tree in the source function needed fixup, mark the
3200 : destination loop tree for fixup, too. */
3201 4899640 : if (loops_for_fn (src_cfun)->state & LOOPS_NEED_FIXUP)
3202 0 : loops_state_set (LOOPS_NEED_FIXUP);
3203 :
3204 4899640 : if (gimple_in_ssa_p (cfun))
3205 29697737 : FOR_ALL_BB_FN (bb, cfun_to_copy)
3206 24798097 : if (!id->blocks_to_copy
3207 24798097 : || (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
3208 24509317 : copy_phis_for_bb (bb, id);
3209 :
3210 29697737 : FOR_ALL_BB_FN (bb, cfun_to_copy)
3211 24798097 : if (bb->aux)
3212 : {
3213 24614527 : if (need_debug_cleanup
3214 2169495 : && bb->index != ENTRY_BLOCK
3215 1848474 : && bb->index != EXIT_BLOCK)
3216 1527453 : maybe_move_debug_stmts_to_successors (id, (basic_block) bb->aux);
3217 : /* Update call edge destinations. This cannot be done before loop
3218 : info is updated, because we may split basic blocks. */
3219 24614527 : if (id->transform_call_graph_edges == CB_CGE_DUPLICATE
3220 22171331 : && bb->index != ENTRY_BLOCK
3221 17524865 : && bb->index != EXIT_BLOCK)
3222 12878399 : redirect_all_calls (id, (basic_block)bb->aux);
3223 24614527 : ((basic_block)bb->aux)->aux = NULL;
3224 24614527 : bb->aux = NULL;
3225 : }
3226 :
3227 : /* Zero out AUX fields of newly created block during EH edge
3228 : insertion. */
3229 5271135 : for (; last < last_basic_block_for_fn (cfun); last++)
3230 : {
3231 371495 : if (need_debug_cleanup)
3232 122084 : maybe_move_debug_stmts_to_successors (id,
3233 122084 : BASIC_BLOCK_FOR_FN (cfun, last));
3234 371495 : BASIC_BLOCK_FOR_FN (cfun, last)->aux = NULL;
3235 : /* Update call edge destinations. This cannot be done before loop
3236 : info is updated, because we may split basic blocks. */
3237 371495 : if (id->transform_call_graph_edges == CB_CGE_DUPLICATE)
3238 371495 : redirect_all_calls (id, BASIC_BLOCK_FOR_FN (cfun, last));
3239 : }
3240 4899640 : entry_block_map->aux = NULL;
3241 4899640 : exit_block_map->aux = NULL;
3242 :
3243 4899640 : if (id->eh_map)
3244 : {
3245 4899640 : delete id->eh_map;
3246 4899640 : id->eh_map = NULL;
3247 : }
3248 4899640 : if (id->dependence_map)
3249 : {
3250 733457 : delete id->dependence_map;
3251 733457 : id->dependence_map = NULL;
3252 : }
3253 :
3254 4899640 : return new_fndecl;
3255 : }
3256 :
3257 : /* Copy the debug STMT using ID. We deal with these statements in a
3258 : special way: if any variable in their VALUE expression wasn't
3259 : remapped yet, we won't remap it, because that would get decl uids
3260 : out of sync, causing codegen differences between -g and -g0. If
3261 : this arises, we drop the VALUE expression altogether. */
3262 :
3263 : static void
3264 62063662 : copy_debug_stmt (gdebug *stmt, copy_body_data *id)
3265 : {
3266 62063662 : tree t, *n;
3267 62063662 : struct walk_stmt_info wi;
3268 :
3269 62063662 : if (tree block = gimple_block (stmt))
3270 : {
3271 40601259 : n = id->decl_map->get (block);
3272 40601259 : gimple_set_block (stmt, n ? *n : id->block);
3273 : }
3274 :
3275 62063662 : if (gimple_debug_nonbind_marker_p (stmt))
3276 : {
3277 14007876 : if (id->call_stmt && !gimple_block (stmt))
3278 : {
3279 37451 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3280 37451 : gsi_remove (&gsi, true);
3281 : }
3282 14007876 : return;
3283 : }
3284 :
3285 : /* Remap all the operands in COPY. */
3286 48055786 : memset (&wi, 0, sizeof (wi));
3287 48055786 : wi.info = id;
3288 :
3289 48055786 : processing_debug_stmt = 1;
3290 :
3291 48055786 : if (gimple_debug_source_bind_p (stmt))
3292 563968 : t = gimple_debug_source_bind_get_var (stmt);
3293 47491818 : else if (gimple_debug_bind_p (stmt))
3294 47491818 : t = gimple_debug_bind_get_var (stmt);
3295 : else
3296 0 : gcc_unreachable ();
3297 :
3298 48055786 : if (TREE_CODE (t) == PARM_DECL
3299 266888 : && id->debug_map
3300 48057622 : && (n = id->debug_map->get (t)))
3301 : {
3302 0 : gcc_assert (VAR_P (*n));
3303 0 : t = *n;
3304 : }
3305 48055786 : else if (VAR_P (t) && !is_global_var (t) && !id->decl_map->get (t))
3306 : /* T is a non-localized variable. */;
3307 : else
3308 41531059 : walk_tree (&t, remap_gimple_op_r, &wi, NULL);
3309 :
3310 48055786 : if (gimple_debug_bind_p (stmt))
3311 : {
3312 47491818 : gimple_debug_bind_set_var (stmt, t);
3313 :
3314 47491818 : if (gimple_debug_bind_has_value_p (stmt))
3315 27033318 : walk_tree (gimple_debug_bind_get_value_ptr (stmt),
3316 : remap_gimple_op_r, &wi, NULL);
3317 :
3318 : /* Punt if any decl couldn't be remapped. */
3319 47491818 : if (processing_debug_stmt < 0)
3320 678185 : gimple_debug_bind_reset_value (stmt);
3321 : }
3322 563968 : else if (gimple_debug_source_bind_p (stmt))
3323 : {
3324 563968 : gimple_debug_source_bind_set_var (stmt, t);
3325 : /* When inlining and source bind refers to one of the optimized
3326 : away parameters, change the source bind into normal debug bind
3327 : referring to the corresponding DEBUG_EXPR_DECL that should have
3328 : been bound before the call stmt. */
3329 563968 : t = gimple_debug_source_bind_get_value (stmt);
3330 563968 : if (t != NULL_TREE
3331 563968 : && TREE_CODE (t) == PARM_DECL
3332 429389 : && id->call_stmt)
3333 : {
3334 424322 : vec<tree, va_gc> **debug_args = decl_debug_args_lookup (id->src_fn);
3335 424322 : unsigned int i;
3336 424322 : if (debug_args != NULL)
3337 : {
3338 535901 : for (i = 0; i < vec_safe_length (*debug_args); i += 2)
3339 535900 : if ((**debug_args)[i] == DECL_ORIGIN (t)
3340 535900 : && TREE_CODE ((**debug_args)[i + 1]) == DEBUG_EXPR_DECL)
3341 : {
3342 418474 : t = (**debug_args)[i + 1];
3343 418474 : stmt->subcode = GIMPLE_DEBUG_BIND;
3344 418474 : gimple_debug_bind_set_value (stmt, t);
3345 418474 : break;
3346 : }
3347 : }
3348 : }
3349 563968 : if (gimple_debug_source_bind_p (stmt))
3350 145494 : walk_tree (gimple_debug_source_bind_get_value_ptr (stmt),
3351 : remap_gimple_op_r, &wi, NULL);
3352 : }
3353 :
3354 48055786 : processing_debug_stmt = 0;
3355 :
3356 48055786 : update_stmt (stmt);
3357 : }
3358 :
3359 : /* Process deferred debug stmts. In order to give values better odds
3360 : of being successfully remapped, we delay the processing of debug
3361 : stmts until all other stmts that might require remapping are
3362 : processed. */
3363 :
3364 : static void
3365 4899640 : copy_debug_stmts (copy_body_data *id)
3366 : {
3367 4899640 : if (!id->debug_stmts.exists ())
3368 : return;
3369 :
3370 65163648 : for (gdebug *stmt : id->debug_stmts)
3371 : /* But avoid re-processing debug stmts that have been elided. */
3372 62063662 : if (gimple_bb (stmt))
3373 62063662 : copy_debug_stmt (stmt, id);
3374 :
3375 3099986 : id->debug_stmts.release ();
3376 : }
3377 :
3378 : /* Make a copy of the body of SRC_FN so that it can be inserted inline in
3379 : another function. */
3380 :
3381 : static tree
3382 31020400 : copy_tree_body (copy_body_data *id)
3383 : {
3384 31020400 : tree fndecl = id->src_fn;
3385 31020400 : tree body = DECL_SAVED_TREE (fndecl);
3386 :
3387 31020400 : walk_tree (&body, copy_tree_body_r, id, NULL);
3388 :
3389 31020400 : return body;
3390 : }
3391 :
3392 : /* Make a copy of the body of FN so that it can be inserted inline in
3393 : another function. */
3394 :
3395 : static tree
3396 4899640 : copy_body (copy_body_data *id,
3397 : basic_block entry_block_map, basic_block exit_block_map,
3398 : basic_block new_entry)
3399 : {
3400 4899640 : tree fndecl = id->src_fn;
3401 4899640 : tree body;
3402 :
3403 : /* If this body has a CFG, walk CFG and copy. */
3404 4899640 : gcc_assert (ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (fndecl)));
3405 4899640 : body = copy_cfg_body (id, entry_block_map, exit_block_map,
3406 : new_entry);
3407 4899640 : copy_debug_stmts (id);
3408 4899640 : if (id->killed_new_ssa_names)
3409 : {
3410 2260969 : ipa_release_ssas_in_hash (id->killed_new_ssa_names);
3411 4521938 : delete id->killed_new_ssa_names;
3412 2260969 : id->killed_new_ssa_names = NULL;
3413 : }
3414 :
3415 4899640 : return body;
3416 : }
3417 :
3418 : /* Return true if VALUE is an ADDR_EXPR of an automatic variable
3419 : defined in function FN, or of a data member thereof. */
3420 :
3421 : static bool
3422 168330 : self_inlining_addr_expr (tree value, tree fn)
3423 : {
3424 168330 : tree var;
3425 :
3426 168330 : if (TREE_CODE (value) != ADDR_EXPR)
3427 : return false;
3428 :
3429 161254 : var = get_base_address (TREE_OPERAND (value, 0));
3430 :
3431 161254 : return var && auto_var_in_fn_p (var, fn);
3432 : }
3433 :
3434 : /* Append to BB a debug annotation that binds VAR to VALUE, inheriting
3435 : lexical block and line number information from base_stmt, if given,
3436 : or from the last stmt of the block otherwise. */
3437 :
3438 : static gimple *
3439 7970348 : insert_init_debug_bind (copy_body_data *id,
3440 : basic_block bb, tree var, tree value,
3441 : gimple *base_stmt)
3442 : {
3443 7970348 : gimple *note;
3444 7970348 : gimple_stmt_iterator gsi;
3445 7970348 : tree tracked_var;
3446 :
3447 7970348 : if (!gimple_in_ssa_p (id->src_cfun))
3448 : return NULL;
3449 :
3450 7970348 : if (!opt_for_fn (id->dst_fn, flag_var_tracking_assignments))
3451 : return NULL;
3452 :
3453 7051751 : tracked_var = target_for_debug_bind (var);
3454 7051751 : if (!tracked_var)
3455 : return NULL;
3456 :
3457 6382752 : if (bb)
3458 : {
3459 6375741 : gsi = gsi_last_bb (bb);
3460 6375741 : if (!base_stmt && !gsi_end_p (gsi))
3461 : base_stmt = gsi_stmt (gsi);
3462 : }
3463 :
3464 6382752 : note = gimple_build_debug_bind (tracked_var,
3465 6382752 : value == error_mark_node
3466 6382752 : ? NULL_TREE : unshare_expr (value),
3467 : base_stmt);
3468 :
3469 6382752 : if (bb)
3470 : {
3471 6375741 : if (!gsi_end_p (gsi))
3472 5533234 : gsi_insert_after (&gsi, note, GSI_SAME_STMT);
3473 : else
3474 842507 : gsi_insert_before (&gsi, note, GSI_SAME_STMT);
3475 : }
3476 :
3477 : return note;
3478 : }
3479 :
3480 : static void
3481 483876 : insert_init_stmt (copy_body_data *id, basic_block bb, gimple *init_stmt)
3482 : {
3483 : /* If VAR represents a zero-sized variable, it's possible that the
3484 : assignment statement may result in no gimple statements. */
3485 483876 : if (init_stmt)
3486 : {
3487 483876 : gimple_stmt_iterator si = gsi_last_bb (bb);
3488 :
3489 : /* We can end up with init statements that store to a non-register
3490 : from a rhs with a conversion. Handle that here by forcing the
3491 : rhs into a temporary. gimple_regimplify_operands is not
3492 : prepared to do this for us. */
3493 483876 : if (!is_gimple_debug (init_stmt)
3494 476865 : && !is_gimple_reg (gimple_assign_lhs (init_stmt))
3495 368178 : && is_gimple_reg_type (TREE_TYPE (gimple_assign_lhs (init_stmt)))
3496 510089 : && gimple_assign_rhs_class (init_stmt) == GIMPLE_UNARY_RHS)
3497 : {
3498 2 : tree rhs = build1 (gimple_assign_rhs_code (init_stmt),
3499 1 : TREE_TYPE (gimple_assign_lhs (init_stmt)),
3500 : gimple_assign_rhs1 (init_stmt));
3501 1 : rhs = force_gimple_operand_gsi (&si, rhs, true, NULL_TREE, false,
3502 : GSI_NEW_STMT);
3503 1 : gimple_assign_set_rhs_code (init_stmt, TREE_CODE (rhs));
3504 1 : gimple_assign_set_rhs1 (init_stmt, rhs);
3505 : }
3506 483876 : gsi_insert_after (&si, init_stmt, GSI_NEW_STMT);
3507 483876 : if (!is_gimple_debug (init_stmt))
3508 : {
3509 476865 : gimple_regimplify_operands (init_stmt, &si);
3510 :
3511 476865 : tree def = gimple_assign_lhs (init_stmt);
3512 476865 : insert_init_debug_bind (id, bb, def, def, init_stmt);
3513 : }
3514 : }
3515 483876 : }
3516 :
3517 : /* Deal with mismatched formal/actual parameters, in a rather brute-force way
3518 : if need be (which should only be necessary for invalid programs). Attempt
3519 : to convert VAL to TYPE and return the result if it is possible, just return
3520 : a zero constant of the given type if it fails. */
3521 :
3522 : tree
3523 973 : force_value_to_type (tree type, tree value)
3524 : {
3525 : /* If we can match up types by promotion/demotion do so. */
3526 973 : if (fold_convertible_p (type, value))
3527 549 : return fold_convert (type, value);
3528 :
3529 : /* ??? For valid programs we should not end up here.
3530 : Still if we end up with truly mismatched types here, fall back
3531 : to using a VIEW_CONVERT_EXPR or a literal zero to not leak invalid
3532 : GIMPLE to the following passes. */
3533 424 : if (TREE_CODE (value) == WITH_SIZE_EXPR)
3534 0 : return error_mark_node;
3535 424 : else if (!is_gimple_reg_type (TREE_TYPE (value))
3536 424 : || TYPE_SIZE (type) == TYPE_SIZE (TREE_TYPE (value)))
3537 133 : return fold_build1 (VIEW_CONVERT_EXPR, type, value);
3538 : else
3539 291 : return build_zero_cst (type);
3540 : }
3541 :
3542 : /* Initialize parameter P with VALUE. If needed, produce init statement
3543 : at the end of BB. When BB is NULL, we return init statement to be
3544 : output later. */
3545 : static gimple *
3546 7960136 : setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn,
3547 : basic_block bb, tree *vars)
3548 : {
3549 7960136 : gimple *init_stmt = NULL;
3550 7960136 : tree var;
3551 7960136 : tree def = (gimple_in_ssa_p (cfun)
3552 7960136 : ? ssa_default_def (id->src_cfun, p) : NULL);
3553 :
3554 : /* Make an equivalent VAR_DECL. Note that we must NOT remap the type
3555 : here since the type of this decl must be visible to the calling
3556 : function. */
3557 7960136 : var = copy_decl_to_var (p, id);
3558 :
3559 : /* Declare this new variable. */
3560 7960136 : DECL_CHAIN (var) = *vars;
3561 7960136 : *vars = var;
3562 :
3563 : /* Make gimplifier happy about this variable. */
3564 7960136 : DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
3565 :
3566 : /* If the parameter is never assigned to, has no SSA_NAMEs created,
3567 : we would not need to create a new variable here at all, if it
3568 : weren't for debug info. Still, we can just use the argument
3569 : value. */
3570 7960136 : if (TREE_READONLY (p)
3571 4212762 : && !TREE_ADDRESSABLE (p)
3572 4211764 : && value
3573 4211750 : && !TREE_SIDE_EFFECTS (value)
3574 12171880 : && !def)
3575 : {
3576 : /* We may produce non-gimple trees by adding NOPs or introduce invalid
3577 : sharing when the value is not constant or DECL. And we need to make
3578 : sure that it cannot be modified from another path in the callee. */
3579 381590 : if (((is_gimple_min_invariant (value)
3580 : /* When the parameter is used in a context that forces it to
3581 : not be a GIMPLE register avoid substituting something that
3582 : is not a decl there. */
3583 167880 : && ! DECL_NOT_GIMPLE_REG_P (p))
3584 213721 : || (DECL_P (value) && TREE_READONLY (value))
3585 213637 : || (auto_var_in_fn_p (value, id->dst_fn)
3586 385 : && !TREE_ADDRESSABLE (value)))
3587 168335 : && useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value))
3588 : /* We have to be very careful about ADDR_EXPR. Make sure
3589 : the base variable isn't a local variable of the inlined
3590 : function, e.g., when doing recursive inlining, direct or
3591 : mutually-recursive or whatever, which is why we don't
3592 : just test whether fn == current_function_decl. */
3593 549920 : && ! self_inlining_addr_expr (value, fn))
3594 : {
3595 168330 : insert_decl_map (id, p, value);
3596 168330 : if (!id->debug_map)
3597 163107 : id->debug_map = new hash_map<tree, tree>;
3598 168330 : id->debug_map->put (p, var);
3599 168330 : return insert_init_debug_bind (id, bb, var, value, NULL);
3600 : }
3601 : }
3602 :
3603 : /* Register the VAR_DECL as the equivalent for the PARM_DECL;
3604 : that way, when the PARM_DECL is encountered, it will be
3605 : automatically replaced by the VAR_DECL. */
3606 7791806 : insert_decl_map (id, p, var);
3607 :
3608 : /* Even if P was TREE_READONLY, the new VAR should not be. In the original
3609 : code, we would have constructed a temporary, and then the function body
3610 : would have never changed the value of P. However, now, we will be
3611 : constructing VAR directly. Therefore, it must not be TREE_READONLY. */
3612 7791806 : TREE_READONLY (var) = 0;
3613 :
3614 7791806 : tree rhs = value;
3615 7791806 : if (value
3616 7790865 : && value != error_mark_node
3617 15582671 : && !useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value)))
3618 962 : rhs = force_value_to_type (TREE_TYPE (p), value);
3619 :
3620 : /* If there is no setup required and we are in SSA, take the easy route
3621 : replacing all SSA names representing the function parameter by the
3622 : SSA name passed to function.
3623 :
3624 : We need to construct map for the variable anyway as it might be used
3625 : in different SSA names when parameter is set in function.
3626 :
3627 : Do replacement at -O0 for const arguments replaced by constant.
3628 : This is important for builtin_constant_p and other construct requiring
3629 : constant argument to be visible in inlined function body. */
3630 15583612 : if (gimple_in_ssa_p (cfun) && rhs && def && is_gimple_reg (p)
3631 6992316 : && (optimize
3632 26275 : || (TREE_READONLY (p)
3633 12584 : && is_gimple_min_invariant (rhs)))
3634 6969677 : && (TREE_CODE (rhs) == SSA_NAME
3635 2610558 : || is_gimple_min_invariant (rhs))
3636 6969395 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def))
3637 : {
3638 6969395 : insert_decl_map (id, def, rhs);
3639 6969395 : return insert_init_debug_bind (id, bb, var, rhs, NULL);
3640 : }
3641 :
3642 : /* If the value of argument is never used, don't care about initializing
3643 : it. */
3644 1618702 : if (optimize && gimple_in_ssa_p (cfun) && !def && is_gimple_reg (p))
3645 : {
3646 : /* When there's a gross type mismatch between the passed value
3647 : and the declared argument type drop it on the floor and do
3648 : not bother to insert a debug bind. */
3649 355772 : if (value && !is_gimple_reg_type (TREE_TYPE (value)))
3650 : return NULL;
3651 355758 : return insert_init_debug_bind (id, bb, var, rhs, NULL);
3652 : }
3653 :
3654 : /* Initialize this VAR_DECL from the equivalent argument. Convert
3655 : the argument to the proper type in case it was promoted. */
3656 466639 : if (value)
3657 : {
3658 466544 : if (rhs == error_mark_node)
3659 : {
3660 0 : insert_decl_map (id, p, var);
3661 0 : return insert_init_debug_bind (id, bb, var, rhs, NULL);
3662 : }
3663 :
3664 466544 : STRIP_USELESS_TYPE_CONVERSION (rhs);
3665 :
3666 : /* If we are in SSA form properly remap the default definition. */
3667 933088 : if (gimple_in_ssa_p (cfun) && is_gimple_reg (p))
3668 : {
3669 25547 : if (def)
3670 : {
3671 22921 : def = remap_ssa_name (def, id);
3672 22921 : init_stmt = gimple_build_assign (def, rhs);
3673 22921 : SSA_NAME_IS_DEFAULT_DEF (def) = 0;
3674 22921 : set_ssa_default_def (cfun, var, NULL);
3675 : }
3676 : }
3677 440997 : else if (!is_empty_type (TREE_TYPE (var)))
3678 368178 : init_stmt = gimple_build_assign (var, rhs);
3679 :
3680 466544 : if (bb && init_stmt)
3681 390800 : insert_init_stmt (id, bb, init_stmt);
3682 : }
3683 : return init_stmt;
3684 : }
3685 :
3686 : /* Generate code to initialize the parameters of the function at the
3687 : top of the stack in ID from the GIMPLE_CALL STMT. */
3688 :
3689 : static void
3690 4646466 : initialize_inlined_parameters (copy_body_data *id, gimple *stmt,
3691 : tree fn, basic_block bb)
3692 : {
3693 4646466 : tree parms;
3694 4646466 : size_t i;
3695 4646466 : tree p;
3696 4646466 : tree vars = NULL_TREE;
3697 4646466 : tree static_chain = gimple_call_chain (stmt);
3698 :
3699 : /* Figure out what the parameters are. */
3700 4646466 : parms = DECL_ARGUMENTS (fn);
3701 :
3702 : /* Loop through the parameter declarations, replacing each with an
3703 : equivalent VAR_DECL, appropriately initialized. */
3704 12579349 : for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++)
3705 : {
3706 7932883 : tree val;
3707 7932883 : val = i < gimple_call_num_args (stmt) ? gimple_call_arg (stmt, i) : NULL;
3708 7932883 : setup_one_parameter (id, p, val, fn, bb, &vars);
3709 : }
3710 : /* After remapping parameters remap their types. This has to be done
3711 : in a second loop over all parameters to appropriately remap
3712 : variable sized arrays when the size is specified in a
3713 : parameter following the array. */
3714 12579349 : for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++)
3715 : {
3716 7932883 : tree *varp = id->decl_map->get (p);
3717 7932883 : if (varp && VAR_P (*varp))
3718 : {
3719 15532998 : tree def = (gimple_in_ssa_p (cfun) && is_gimple_reg (p)
3720 7325385 : ? ssa_default_def (id->src_cfun, p) : NULL);
3721 7766499 : tree var = *varp;
3722 7766499 : TREE_TYPE (var) = remap_type (TREE_TYPE (var), id);
3723 : /* Also remap the default definition if it was remapped
3724 : to the default definition of the parameter replacement
3725 : by the parameter setup. */
3726 7766499 : if (def)
3727 : {
3728 6968643 : tree *defp = id->decl_map->get (def);
3729 6968643 : if (defp
3730 6968549 : && TREE_CODE (*defp) == SSA_NAME
3731 13677590 : && SSA_NAME_VAR (*defp) == var)
3732 22921 : TREE_TYPE (*defp) = TREE_TYPE (var);
3733 : }
3734 : /* When not optimizing and the parameter is unused, assign to
3735 : a dummy SSA name. Do this after remapping the type above. */
3736 797856 : else if (!optimize
3737 3481 : && is_gimple_reg (p)
3738 800482 : && i < gimple_call_num_args (stmt))
3739 : {
3740 2626 : tree val = gimple_call_arg (stmt, i);
3741 2626 : if (val != error_mark_node)
3742 : {
3743 2626 : if (!useless_type_conversion_p (TREE_TYPE (p),
3744 2626 : TREE_TYPE (val)))
3745 0 : val = force_value_to_type (TREE_TYPE (p), val);
3746 2626 : def = make_ssa_name (var);
3747 2626 : gimple *init_stmt = gimple_build_assign (def, val);
3748 2626 : insert_init_stmt (id, bb, init_stmt);
3749 : }
3750 : }
3751 : }
3752 : }
3753 :
3754 : /* Initialize the static chain. */
3755 4646466 : p = DECL_STRUCT_FUNCTION (fn)->static_chain_decl;
3756 4646466 : gcc_assert (fn != current_function_decl);
3757 4646466 : if (p)
3758 : {
3759 : /* Calling a function that requires a static chain directly
3760 : without setting up the static chain has undefined behavior
3761 : at run-time. */
3762 2340 : if (!static_chain)
3763 0 : static_chain = null_pointer_node;
3764 2340 : setup_one_parameter (id, p, static_chain, fn, bb, &vars);
3765 : }
3766 :
3767 : /* Reverse so the variables appear in the correct order in DWARF
3768 : debug info. */
3769 4646466 : vars = nreverse (vars);
3770 :
3771 4646466 : declare_inline_vars (id->block, vars);
3772 4646466 : }
3773 :
3774 :
3775 : /* Declare a return variable to replace the RESULT_DECL for the
3776 : function we are calling. An appropriate DECL_STMT is returned.
3777 : The USE_STMT is filled to contain a use of the declaration to
3778 : indicate the return value of the function.
3779 :
3780 : RETURN_SLOT, if non-null is place where to store the result. It
3781 : is set only for CALL_EXPR_RETURN_SLOT_OPT. MODIFY_DEST, if non-null,
3782 : was the LHS of the MODIFY_EXPR to which this call is the RHS.
3783 :
3784 : The return value is a (possibly null) value that holds the result
3785 : as seen by the caller. */
3786 :
3787 : static tree
3788 4646466 : declare_return_variable (copy_body_data *id, tree return_slot, tree modify_dest,
3789 : basic_block entry_bb)
3790 : {
3791 4646466 : tree callee = id->src_fn;
3792 4646466 : tree result = DECL_RESULT (callee);
3793 4646466 : tree callee_type = TREE_TYPE (result);
3794 4646466 : tree caller_type;
3795 4646466 : tree var, use;
3796 :
3797 : /* Handle type-mismatches in the function declaration return type
3798 : vs. the call expression. */
3799 4646466 : if (modify_dest)
3800 2176285 : caller_type = TREE_TYPE (modify_dest);
3801 2470181 : else if (return_slot)
3802 143027 : caller_type = TREE_TYPE (return_slot);
3803 : else /* No LHS on the call. */
3804 2327154 : caller_type = TREE_TYPE (TREE_TYPE (callee));
3805 :
3806 : /* We don't need to do anything for functions that don't return anything. */
3807 4646466 : if (VOID_TYPE_P (callee_type))
3808 : return NULL_TREE;
3809 :
3810 : /* If there was a return slot, then the return value is the
3811 : dereferenced address of that object. */
3812 2489501 : if (return_slot)
3813 : {
3814 : /* The front end shouldn't have used both return_slot and
3815 : a modify expression. */
3816 143027 : gcc_assert (!modify_dest);
3817 143027 : if (DECL_BY_REFERENCE (result))
3818 : {
3819 83140 : tree return_slot_addr = build_fold_addr_expr (return_slot);
3820 83140 : STRIP_USELESS_TYPE_CONVERSION (return_slot_addr);
3821 :
3822 : /* We are going to construct *&return_slot and we can't do that
3823 : for variables believed to be not addressable.
3824 :
3825 : FIXME: This check possibly can match, because values returned
3826 : via return slot optimization are not believed to have address
3827 : taken by alias analysis. */
3828 83140 : gcc_assert (TREE_CODE (return_slot) != SSA_NAME);
3829 83140 : var = return_slot_addr;
3830 83140 : mark_addressable (return_slot);
3831 : }
3832 : else
3833 : {
3834 59887 : var = return_slot;
3835 59887 : gcc_assert (TREE_CODE (var) != SSA_NAME);
3836 59887 : if (TREE_ADDRESSABLE (result))
3837 23338 : mark_addressable (var);
3838 : }
3839 143027 : if (DECL_NOT_GIMPLE_REG_P (result)
3840 143027 : && DECL_P (var))
3841 0 : DECL_NOT_GIMPLE_REG_P (var) = 1;
3842 :
3843 143027 : if (!useless_type_conversion_p (callee_type, caller_type))
3844 83141 : var = build1 (VIEW_CONVERT_EXPR, callee_type, var);
3845 :
3846 143027 : use = NULL;
3847 143027 : goto done;
3848 : }
3849 :
3850 : /* All types requiring non-trivial constructors should have been handled. */
3851 2346474 : gcc_assert (!TREE_ADDRESSABLE (callee_type));
3852 :
3853 : /* Attempt to avoid creating a new temporary variable. */
3854 2346474 : if (modify_dest
3855 2176264 : && TREE_CODE (modify_dest) != SSA_NAME)
3856 : {
3857 387060 : bool use_it = false;
3858 :
3859 : /* We can't use MODIFY_DEST if there's type promotion involved. */
3860 387060 : if (!useless_type_conversion_p (callee_type, caller_type))
3861 : use_it = false;
3862 :
3863 : /* ??? If we're assigning to a variable sized type, then we must
3864 : reuse the destination variable, because we've no good way to
3865 : create variable sized temporaries at this point. */
3866 387058 : else if (!poly_int_tree_p (TYPE_SIZE_UNIT (caller_type)))
3867 : use_it = true;
3868 :
3869 : /* If the callee cannot possibly modify MODIFY_DEST, then we can
3870 : reuse it as the result of the call directly. Don't do this if
3871 : it would promote MODIFY_DEST to addressable. */
3872 387058 : else if (TREE_ADDRESSABLE (result))
3873 : use_it = false;
3874 : else
3875 : {
3876 385732 : tree base_m = get_base_address (modify_dest);
3877 :
3878 : /* If the base isn't a decl, then it's a pointer, and we don't
3879 : know where that's going to go. */
3880 385732 : if (!DECL_P (base_m))
3881 : use_it = false;
3882 381710 : else if (is_global_var (base_m))
3883 : use_it = false;
3884 381255 : else if (DECL_NOT_GIMPLE_REG_P (result)
3885 381255 : && !DECL_NOT_GIMPLE_REG_P (base_m))
3886 : use_it = false;
3887 381255 : else if (!TREE_ADDRESSABLE (base_m))
3888 : use_it = true;
3889 : }
3890 :
3891 : if (use_it)
3892 : {
3893 279730 : var = modify_dest;
3894 279730 : use = NULL;
3895 279730 : goto done;
3896 : }
3897 : }
3898 :
3899 2066744 : gcc_assert (poly_int_tree_p (TYPE_SIZE_UNIT (callee_type)));
3900 :
3901 2066744 : var = copy_result_decl_to_var (result, id);
3902 2066744 : DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
3903 :
3904 : /* Do not have the rest of GCC warn about this variable as it should
3905 : not be visible to the user. */
3906 2066744 : suppress_warning (var /* OPT_Wuninitialized? */);
3907 :
3908 2066744 : declare_inline_vars (id->block, var);
3909 :
3910 : /* Build the use expr. If the return type of the function was
3911 : promoted, convert it back to the expected type. */
3912 2066744 : use = var;
3913 2066744 : if (!useless_type_conversion_p (caller_type, TREE_TYPE (var)))
3914 : {
3915 : /* If we can match up types by promotion/demotion do so. */
3916 7 : if (fold_convertible_p (caller_type, var))
3917 0 : use = fold_convert (caller_type, var);
3918 : else
3919 : {
3920 : /* ??? For valid programs we should not end up here.
3921 : Still if we end up with truly mismatched types here, fall back
3922 : to using a MEM_REF to not leak invalid GIMPLE to the following
3923 : passes. */
3924 : /* Prevent var from being written into SSA form. */
3925 7 : if (is_gimple_reg_type (TREE_TYPE (var)))
3926 7 : DECL_NOT_GIMPLE_REG_P (var) = true;
3927 7 : use = fold_build2 (MEM_REF, caller_type,
3928 : build_fold_addr_expr (var),
3929 : build_int_cst (ptr_type_node, 0));
3930 : }
3931 : }
3932 :
3933 2066744 : STRIP_USELESS_TYPE_CONVERSION (use);
3934 :
3935 2066744 : if (DECL_BY_REFERENCE (result))
3936 : {
3937 0 : TREE_ADDRESSABLE (var) = 1;
3938 0 : var = build_fold_addr_expr (var);
3939 : }
3940 :
3941 2066744 : done:
3942 : /* Register the VAR_DECL as the equivalent for the RESULT_DECL; that
3943 : way, when the RESULT_DECL is encountered, it will be
3944 : automatically replaced by the VAR_DECL.
3945 :
3946 : When returning by reference, ensure that RESULT_DECL remaps to
3947 : gimple_val. */
3948 2489501 : if (DECL_BY_REFERENCE (result)
3949 2489501 : && !is_gimple_val (var))
3950 : {
3951 83140 : tree temp = create_tmp_var (TREE_TYPE (result), "retvalptr");
3952 83140 : insert_decl_map (id, result, temp);
3953 : /* When RESULT_DECL is in SSA form, we need to remap and initialize
3954 : it's default_def SSA_NAME. */
3955 83140 : if (gimple_in_ssa_p (id->src_cfun)
3956 83140 : && is_gimple_reg (result))
3957 83140 : if (tree default_def = ssa_default_def (id->src_cfun, result))
3958 : {
3959 83103 : temp = make_ssa_name (temp);
3960 83103 : insert_decl_map (id, default_def, temp);
3961 : }
3962 83140 : insert_init_stmt (id, entry_bb, gimple_build_assign (temp, var));
3963 : }
3964 : else
3965 2406361 : insert_decl_map (id, result, var);
3966 :
3967 : /* Remember this so we can ignore it in remap_decls. */
3968 2489501 : id->retvar = var;
3969 2489501 : return use;
3970 : }
3971 :
3972 : /* Determine if the function can be copied. If so return NULL. If
3973 : not return a string describing the reason for failure. */
3974 :
3975 : const char *
3976 18567568 : copy_forbidden (struct function *fun)
3977 : {
3978 18567568 : const char *reason = fun->cannot_be_copied_reason;
3979 :
3980 : /* Only examine the function once. */
3981 18567568 : if (fun->cannot_be_copied_set)
3982 : return reason;
3983 :
3984 : /* We cannot copy a function that receives a non-local goto
3985 : because we cannot remap the destination label used in the
3986 : function that is performing the non-local goto. */
3987 : /* ??? Actually, this should be possible, if we work at it.
3988 : No doubt there's just a handful of places that simply
3989 : assume it doesn't happen and don't substitute properly. */
3990 11079116 : if (fun->has_nonlocal_label)
3991 : {
3992 782 : reason = G_("function %q+F can never be copied "
3993 : "because it receives a non-local goto");
3994 782 : goto fail;
3995 : }
3996 :
3997 11078334 : if (fun->has_forced_label_in_static)
3998 : {
3999 230 : reason = G_("function %q+F can never be copied because it saves "
4000 : "address of local label in a static variable");
4001 230 : goto fail;
4002 : }
4003 :
4004 11078104 : fail:
4005 11079116 : fun->cannot_be_copied_reason = reason;
4006 11079116 : fun->cannot_be_copied_set = true;
4007 11079116 : return reason;
4008 : }
4009 :
4010 :
4011 : static const char *inline_forbidden_reason;
4012 :
4013 : /* A callback for walk_gimple_seq to handle statements. Returns non-null
4014 : iff a function cannot be inlined. Also sets the reason why. */
4015 :
4016 : static tree
4017 177993119 : inline_forbidden_p_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
4018 : struct walk_stmt_info *wip)
4019 : {
4020 177993119 : tree fn = (tree) wip->info;
4021 177993119 : tree t;
4022 177993119 : gimple *stmt = gsi_stmt (*gsi);
4023 :
4024 177993119 : switch (gimple_code (stmt))
4025 : {
4026 19446322 : case GIMPLE_CALL:
4027 : /* Refuse to inline alloca call unless user explicitly forced so as
4028 : this may change program's memory overhead drastically when the
4029 : function using alloca is called in loop. In GCC present in
4030 : SPEC2000 inlining into schedule_block cause it to require 2GB of
4031 : RAM instead of 256MB. Don't do so for alloca calls emitted for
4032 : VLA objects as those can't cause unbounded growth (they're always
4033 : wrapped inside stack_save/stack_restore regions. */
4034 19446322 : if (gimple_maybe_alloca_call_p (stmt)
4035 9558 : && !gimple_call_alloca_for_var_p (as_a <gcall *> (stmt))
4036 19451095 : && !lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn)))
4037 : {
4038 4645 : inline_forbidden_reason
4039 4645 : = G_("function %q+F can never be inlined because it uses "
4040 : "alloca (override using the always_inline attribute)");
4041 4645 : *handled_ops_p = true;
4042 4645 : return fn;
4043 : }
4044 :
4045 19441677 : t = gimple_call_fndecl (stmt);
4046 19441677 : if (t == NULL_TREE)
4047 : break;
4048 :
4049 : /* We cannot inline functions that call setjmp. */
4050 18713495 : if (setjmp_call_p (t))
4051 : {
4052 1045 : inline_forbidden_reason
4053 1045 : = G_("function %q+F can never be inlined because it uses setjmp");
4054 1045 : *handled_ops_p = true;
4055 1045 : return t;
4056 : }
4057 :
4058 18712450 : if (DECL_BUILT_IN_CLASS (t) == BUILT_IN_NORMAL)
4059 3845638 : switch (DECL_FUNCTION_CODE (t))
4060 : {
4061 : /* We cannot inline functions that take a variable number of
4062 : arguments. */
4063 2814 : case BUILT_IN_VA_START:
4064 2814 : case BUILT_IN_NEXT_ARG:
4065 2814 : case BUILT_IN_VA_END:
4066 2814 : inline_forbidden_reason
4067 2814 : = G_("function %q+F can never be inlined because it "
4068 : "uses variable argument lists");
4069 2814 : *handled_ops_p = true;
4070 2814 : return t;
4071 :
4072 226 : case BUILT_IN_LONGJMP:
4073 : /* We can't inline functions that call __builtin_longjmp at
4074 : all. The non-local goto machinery really requires the
4075 : destination be in a different function. If we allow the
4076 : function calling __builtin_longjmp to be inlined into the
4077 : function calling __builtin_setjmp, Things will Go Awry. */
4078 226 : inline_forbidden_reason
4079 226 : = G_("function %q+F can never be inlined because "
4080 : "it uses setjmp-longjmp exception handling");
4081 226 : *handled_ops_p = true;
4082 226 : return t;
4083 :
4084 276 : case BUILT_IN_NONLOCAL_GOTO:
4085 : /* Similarly. */
4086 276 : inline_forbidden_reason
4087 276 : = G_("function %q+F can never be inlined because "
4088 : "it uses non-local goto");
4089 276 : *handled_ops_p = true;
4090 276 : return t;
4091 :
4092 384 : case BUILT_IN_RETURN:
4093 384 : case BUILT_IN_APPLY_ARGS:
4094 : /* If a __builtin_apply_args caller would be inlined,
4095 : it would be saving arguments of the function it has
4096 : been inlined into. Similarly __builtin_return would
4097 : return from the function the inline has been inlined into. */
4098 384 : inline_forbidden_reason
4099 384 : = G_("function %q+F can never be inlined because "
4100 : "it uses %<__builtin_return%> or %<__builtin_apply_args%>");
4101 384 : *handled_ops_p = true;
4102 384 : return t;
4103 :
4104 : default:
4105 : break;
4106 : }
4107 : break;
4108 :
4109 313 : case GIMPLE_GOTO:
4110 313 : t = gimple_goto_dest (stmt);
4111 :
4112 : /* We will not inline a function which uses computed goto. The
4113 : addresses of its local labels, which may be tucked into
4114 : global storage, are of course not constant across
4115 : instantiations, which causes unexpected behavior. */
4116 313 : if (TREE_CODE (t) != LABEL_DECL)
4117 : {
4118 313 : inline_forbidden_reason
4119 313 : = G_("function %q+F can never be inlined "
4120 : "because it contains a computed goto");
4121 313 : *handled_ops_p = true;
4122 313 : return t;
4123 : }
4124 : break;
4125 :
4126 : default:
4127 : break;
4128 : }
4129 :
4130 177983416 : *handled_ops_p = false;
4131 177983416 : return NULL_TREE;
4132 : }
4133 :
4134 : /* Return true if FNDECL is a function that cannot be inlined into
4135 : another one. */
4136 :
4137 : static bool
4138 5959263 : inline_forbidden_p (tree fndecl)
4139 : {
4140 5959263 : struct function *fun = DECL_STRUCT_FUNCTION (fndecl);
4141 5959263 : struct walk_stmt_info wi;
4142 5959263 : basic_block bb;
4143 5959263 : bool forbidden_p = false;
4144 :
4145 : /* First check for shared reasons not to copy the code. */
4146 5959263 : inline_forbidden_reason = copy_forbidden (fun);
4147 5959263 : if (inline_forbidden_reason != NULL)
4148 : return true;
4149 :
4150 : /* Next, walk the statements of the function looking for
4151 : constructs we can't handle, or are non-optimal for inlining. */
4152 5958558 : hash_set<tree> visited_nodes;
4153 5958558 : memset (&wi, 0, sizeof (wi));
4154 5958558 : wi.info = (void *) fndecl;
4155 5958558 : wi.pset = &visited_nodes;
4156 :
4157 : /* We cannot inline a function with a variable-sized parameter because we
4158 : cannot materialize a temporary of such a type in the caller if need be.
4159 : Note that the return case is not symmetrical because we can guarantee
4160 : that a temporary is not needed by means of CALL_EXPR_RETURN_SLOT_OPT. */
4161 18926757 : for (tree parm = DECL_ARGUMENTS (fndecl); parm; parm = DECL_CHAIN (parm))
4162 12968201 : if (!poly_int_tree_p (DECL_SIZE (parm)))
4163 : {
4164 2 : inline_forbidden_reason
4165 2 : = G_("function %q+F can never be inlined because "
4166 : "it has a VLA argument");
4167 2 : return true;
4168 : }
4169 :
4170 38668944 : FOR_EACH_BB_FN (bb, fun)
4171 : {
4172 32720091 : gimple *ret;
4173 32720091 : gimple_seq seq = bb_seq (bb);
4174 32720091 : ret = walk_gimple_seq (seq, inline_forbidden_p_stmt, NULL, &wi);
4175 32720091 : forbidden_p = (ret != NULL);
4176 32720091 : if (forbidden_p)
4177 : break;
4178 : }
4179 :
4180 : return forbidden_p;
4181 5958558 : }
4182 :
4183 : /* Return false if the function FNDECL cannot be inlined on account of its
4184 : attributes, true otherwise. */
4185 : static bool
4186 5959263 : function_attribute_inlinable_p (const_tree fndecl)
4187 : {
4188 11893499 : for (auto scoped_attributes : targetm.attribute_table)
4189 : {
4190 5959263 : const_tree a;
4191 :
4192 7851390 : for (a = DECL_ATTRIBUTES (fndecl); a; a = TREE_CHAIN (a))
4193 : {
4194 1917154 : const_tree name = get_attribute_name (a);
4195 :
4196 54973484 : for (const attribute_spec &attribute : scoped_attributes->attributes)
4197 53081357 : if (is_attribute_p (attribute.name, name))
4198 25027 : return targetm.function_attribute_inlinable_p (fndecl);
4199 : }
4200 : }
4201 :
4202 : return true;
4203 : }
4204 :
4205 : /* Returns nonzero if FN is a function that does not have any
4206 : fundamental inline blocking properties. */
4207 :
4208 : bool
4209 6588884 : tree_inlinable_function_p (tree fn)
4210 : {
4211 6588884 : bool inlinable = true;
4212 6588884 : bool do_warning;
4213 6588884 : tree always_inline;
4214 :
4215 : /* If we've already decided this function shouldn't be inlined,
4216 : there's no need to check again. */
4217 6588884 : if (DECL_UNINLINABLE (fn))
4218 : return false;
4219 :
4220 : /* We only warn for functions declared `inline' by the user. */
4221 5979781 : do_warning = (opt_for_fn (fn, warn_inline)
4222 637 : && DECL_DECLARED_INLINE_P (fn)
4223 508 : && !DECL_NO_INLINE_WARNING_P (fn)
4224 5980159 : && !DECL_IN_SYSTEM_HEADER (fn));
4225 :
4226 5979781 : always_inline = lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn));
4227 :
4228 5979781 : if (flag_no_inline
4229 97128 : && always_inline == NULL)
4230 : {
4231 20518 : if (do_warning)
4232 0 : warning (OPT_Winline, "function %q+F can never be inlined because it "
4233 : "is suppressed using %<-fno-inline%>", fn);
4234 : inlinable = false;
4235 : }
4236 :
4237 5959263 : else if (!function_attribute_inlinable_p (fn))
4238 : {
4239 0 : if (do_warning)
4240 0 : warning (OPT_Winline, "function %q+F can never be inlined because it "
4241 : "uses attributes conflicting with inlining", fn);
4242 : inlinable = false;
4243 : }
4244 :
4245 5959263 : else if (inline_forbidden_p (fn))
4246 : {
4247 : /* See if we should warn about uninlinable functions. Previously,
4248 : some of these warnings would be issued while trying to expand
4249 : the function inline, but that would cause multiple warnings
4250 : about functions that would for example call alloca. But since
4251 : this a property of the function, just one warning is enough.
4252 : As a bonus we can now give more details about the reason why a
4253 : function is not inlinable. */
4254 10410 : if (always_inline)
4255 2 : error (inline_forbidden_reason, fn);
4256 10408 : else if (do_warning)
4257 2 : warning (OPT_Winline, inline_forbidden_reason, fn);
4258 :
4259 : inlinable = false;
4260 : }
4261 :
4262 : /* Squirrel away the result so that we don't have to check again. */
4263 5979781 : DECL_UNINLINABLE (fn) = !inlinable;
4264 :
4265 5979781 : return inlinable;
4266 : }
4267 :
4268 : /* Estimate the cost of a memory move of type TYPE. Use machine dependent
4269 : word size and take possible memcpy call into account and return
4270 : cost based on whether optimizing for size or speed according to SPEED_P. */
4271 :
4272 : int
4273 337293683 : estimate_move_cost (tree type, bool ARG_UNUSED (speed_p))
4274 : {
4275 337293683 : HOST_WIDE_INT size;
4276 :
4277 337293683 : gcc_assert (!VOID_TYPE_P (type));
4278 :
4279 337293683 : if (VECTOR_TYPE_P (type))
4280 : {
4281 4343862 : scalar_mode inner = SCALAR_TYPE_MODE (TREE_TYPE (type));
4282 4343862 : machine_mode simd = targetm.vectorize.preferred_simd_mode (inner);
4283 4343862 : int orig_mode_size
4284 8687724 : = estimated_poly_value (GET_MODE_SIZE (TYPE_MODE (type)));
4285 8687724 : int simd_mode_size = estimated_poly_value (GET_MODE_SIZE (simd));
4286 4343862 : return ((orig_mode_size + simd_mode_size - 1)
4287 4343862 : / simd_mode_size);
4288 : }
4289 :
4290 332949821 : size = int_size_in_bytes (type);
4291 :
4292 332952286 : if (size < 0 || size > MOVE_MAX_PIECES * MOVE_RATIO (speed_p))
4293 : /* Cost of a memcpy call, 3 arguments and the call. */
4294 : return 4;
4295 : else
4296 331996988 : return ((size + MOVE_MAX_PIECES - 1) / MOVE_MAX_PIECES);
4297 : }
4298 :
4299 : /* Returns cost of operation CODE, according to WEIGHTS */
4300 :
4301 : static int
4302 405112190 : estimate_operator_cost (enum tree_code code, eni_weights *weights,
4303 : tree op1 ATTRIBUTE_UNUSED, tree op2)
4304 : {
4305 405112190 : switch (code)
4306 : {
4307 : /* These are "free" conversions, or their presumed cost
4308 : is folded into other operations. */
4309 : case RANGE_EXPR:
4310 : CASE_CONVERT:
4311 : case COMPLEX_EXPR:
4312 : case PAREN_EXPR:
4313 : case VIEW_CONVERT_EXPR:
4314 : return 0;
4315 :
4316 : /* Assign cost of 1 to usual operations.
4317 : ??? We may consider mapping RTL costs to this. */
4318 : case COND_EXPR:
4319 : case VEC_COND_EXPR:
4320 : case VEC_PERM_EXPR:
4321 :
4322 : case PLUS_EXPR:
4323 : case POINTER_PLUS_EXPR:
4324 : case POINTER_DIFF_EXPR:
4325 : case MINUS_EXPR:
4326 : case MULT_EXPR:
4327 : case MULT_HIGHPART_EXPR:
4328 :
4329 : case ADDR_SPACE_CONVERT_EXPR:
4330 : case FIXED_CONVERT_EXPR:
4331 : case FIX_TRUNC_EXPR:
4332 :
4333 : case NEGATE_EXPR:
4334 : case FLOAT_EXPR:
4335 : case MIN_EXPR:
4336 : case MAX_EXPR:
4337 : case ABS_EXPR:
4338 : case ABSU_EXPR:
4339 :
4340 : case LSHIFT_EXPR:
4341 : case RSHIFT_EXPR:
4342 : case LROTATE_EXPR:
4343 : case RROTATE_EXPR:
4344 :
4345 : case BIT_IOR_EXPR:
4346 : case BIT_XOR_EXPR:
4347 : case BIT_AND_EXPR:
4348 : case BIT_NOT_EXPR:
4349 :
4350 : case TRUTH_ANDIF_EXPR:
4351 : case TRUTH_ORIF_EXPR:
4352 : case TRUTH_AND_EXPR:
4353 : case TRUTH_OR_EXPR:
4354 : case TRUTH_XOR_EXPR:
4355 : case TRUTH_NOT_EXPR:
4356 :
4357 : case LT_EXPR:
4358 : case LE_EXPR:
4359 : case GT_EXPR:
4360 : case GE_EXPR:
4361 : case EQ_EXPR:
4362 : case NE_EXPR:
4363 : case ORDERED_EXPR:
4364 : case UNORDERED_EXPR:
4365 :
4366 : case UNLT_EXPR:
4367 : case UNLE_EXPR:
4368 : case UNGT_EXPR:
4369 : case UNGE_EXPR:
4370 : case UNEQ_EXPR:
4371 : case LTGT_EXPR:
4372 :
4373 : case CONJ_EXPR:
4374 :
4375 : case PREDECREMENT_EXPR:
4376 : case PREINCREMENT_EXPR:
4377 : case POSTDECREMENT_EXPR:
4378 : case POSTINCREMENT_EXPR:
4379 :
4380 : case REALIGN_LOAD_EXPR:
4381 :
4382 : case WIDEN_SUM_EXPR:
4383 : case WIDEN_MULT_EXPR:
4384 : case DOT_PROD_EXPR:
4385 : case SAD_EXPR:
4386 : case WIDEN_MULT_PLUS_EXPR:
4387 : case WIDEN_MULT_MINUS_EXPR:
4388 : case WIDEN_LSHIFT_EXPR:
4389 :
4390 : case VEC_WIDEN_MULT_HI_EXPR:
4391 : case VEC_WIDEN_MULT_LO_EXPR:
4392 : case VEC_WIDEN_MULT_EVEN_EXPR:
4393 : case VEC_WIDEN_MULT_ODD_EXPR:
4394 : case VEC_UNPACK_HI_EXPR:
4395 : case VEC_UNPACK_LO_EXPR:
4396 : case VEC_UNPACK_FLOAT_HI_EXPR:
4397 : case VEC_UNPACK_FLOAT_LO_EXPR:
4398 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
4399 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
4400 : case VEC_PACK_TRUNC_EXPR:
4401 : case VEC_PACK_SAT_EXPR:
4402 : case VEC_PACK_FIX_TRUNC_EXPR:
4403 : case VEC_PACK_FLOAT_EXPR:
4404 : case VEC_WIDEN_LSHIFT_HI_EXPR:
4405 : case VEC_WIDEN_LSHIFT_LO_EXPR:
4406 : case VEC_DUPLICATE_EXPR:
4407 : case VEC_SERIES_EXPR:
4408 :
4409 : return 1;
4410 :
4411 : /* Few special cases of expensive operations. This is useful
4412 : to avoid inlining on functions having too many of these. */
4413 2687246 : case TRUNC_DIV_EXPR:
4414 2687246 : case CEIL_DIV_EXPR:
4415 2687246 : case FLOOR_DIV_EXPR:
4416 2687246 : case ROUND_DIV_EXPR:
4417 2687246 : case EXACT_DIV_EXPR:
4418 2687246 : case TRUNC_MOD_EXPR:
4419 2687246 : case CEIL_MOD_EXPR:
4420 2687246 : case FLOOR_MOD_EXPR:
4421 2687246 : case ROUND_MOD_EXPR:
4422 2687246 : case RDIV_EXPR:
4423 2687246 : if (TREE_CODE (op2) != INTEGER_CST)
4424 1344213 : return weights->div_mod_cost;
4425 : return 1;
4426 :
4427 : /* Bit-field insertion needs several shift and mask operations. */
4428 : case BIT_INSERT_EXPR:
4429 : return 3;
4430 :
4431 182827348 : default:
4432 : /* We expect a copy assignment with no operator. */
4433 182827348 : gcc_assert (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS);
4434 : return 0;
4435 : }
4436 : }
4437 :
4438 :
4439 : /* Estimate number of instructions that will be created by expanding
4440 : the statements in the statement sequence STMTS.
4441 : WEIGHTS contains weights attributed to various constructs. */
4442 :
4443 : int
4444 232102 : estimate_num_insns_seq (gimple_seq stmts, eni_weights *weights)
4445 : {
4446 232102 : int cost;
4447 232102 : gimple_stmt_iterator gsi;
4448 :
4449 232102 : cost = 0;
4450 616425 : for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
4451 384323 : cost += estimate_num_insns (gsi_stmt (gsi), weights);
4452 :
4453 232102 : return cost;
4454 : }
4455 :
4456 :
4457 : /* Estimate number of instructions that will be created by expanding STMT.
4458 : WEIGHTS contains weights attributed to various constructs. */
4459 :
4460 : int
4461 576322197 : estimate_num_insns (gimple *stmt, eni_weights *weights)
4462 : {
4463 576322197 : unsigned cost, i;
4464 576322197 : enum gimple_code code = gimple_code (stmt);
4465 576322197 : tree lhs;
4466 576322197 : tree rhs;
4467 :
4468 576322197 : switch (code)
4469 : {
4470 328838360 : case GIMPLE_ASSIGN:
4471 : /* Try to estimate the cost of assignments. We have three cases to
4472 : deal with:
4473 : 1) Simple assignments to registers;
4474 : 2) Stores to things that must live in memory. This includes
4475 : "normal" stores to scalars, but also assignments of large
4476 : structures, or constructors of big arrays;
4477 :
4478 : Let us look at the first two cases, assuming we have "a = b + C":
4479 : <GIMPLE_ASSIGN <var_decl "a">
4480 : <plus_expr <var_decl "b"> <constant C>>
4481 : If "a" is a GIMPLE register, the assignment to it is free on almost
4482 : any target, because "a" usually ends up in a real register. Hence
4483 : the only cost of this expression comes from the PLUS_EXPR, and we
4484 : can ignore the GIMPLE_ASSIGN.
4485 : If "a" is not a GIMPLE register, the assignment to "a" will most
4486 : likely be a real store, so the cost of the GIMPLE_ASSIGN is the cost
4487 : of moving something into "a", which we compute using the function
4488 : estimate_move_cost. */
4489 328838360 : if (gimple_clobber_p (stmt))
4490 : return 0; /* ={v} {CLOBBER} stmt expands to nothing. */
4491 :
4492 312128463 : lhs = gimple_assign_lhs (stmt);
4493 312128463 : rhs = gimple_assign_rhs1 (stmt);
4494 :
4495 312128463 : cost = 0;
4496 :
4497 : /* Account for the cost of moving to / from memory. */
4498 312128463 : if (gimple_store_p (stmt))
4499 76864340 : cost += estimate_move_cost (TREE_TYPE (lhs), weights->time_based);
4500 312128463 : if (gimple_assign_load_p (stmt))
4501 85610398 : cost += estimate_move_cost (TREE_TYPE (rhs), weights->time_based);
4502 :
4503 404601232 : cost += estimate_operator_cost (gimple_assign_rhs_code (stmt), weights,
4504 : gimple_assign_rhs1 (stmt),
4505 312128463 : get_gimple_rhs_class (gimple_assign_rhs_code (stmt))
4506 : == GIMPLE_BINARY_RHS
4507 92472769 : ? gimple_assign_rhs2 (stmt) : NULL);
4508 312128463 : break;
4509 :
4510 92983727 : case GIMPLE_COND:
4511 92983727 : cost = 1 + estimate_operator_cost (gimple_cond_code (stmt), weights,
4512 : gimple_op (stmt, 0),
4513 : gimple_op (stmt, 1));
4514 92983727 : break;
4515 :
4516 695540 : case GIMPLE_SWITCH:
4517 695540 : {
4518 695540 : gswitch *switch_stmt = as_a <gswitch *> (stmt);
4519 : /* Take into account cost of the switch + guess 2 conditional jumps for
4520 : each case label.
4521 :
4522 : TODO: once the switch expansion logic is sufficiently separated, we can
4523 : do better job on estimating cost of the switch. */
4524 695540 : if (weights->time_based)
4525 127672 : cost = floor_log2 (gimple_switch_num_labels (switch_stmt)) * 2;
4526 : else
4527 567868 : cost = gimple_switch_num_labels (switch_stmt) * 2;
4528 : }
4529 : break;
4530 :
4531 73906421 : case GIMPLE_CALL:
4532 73906421 : {
4533 73906421 : tree decl;
4534 :
4535 73906421 : if (gimple_call_internal_p (stmt))
4536 : return 0;
4537 71028648 : else if ((decl = gimple_call_fndecl (stmt))
4538 71028648 : && fndecl_built_in_p (decl))
4539 : {
4540 : /* Do not special case builtins where we see the body.
4541 : This just confuse inliner. */
4542 18129407 : struct cgraph_node *node;
4543 18129407 : if ((node = cgraph_node::get (decl))
4544 18129407 : && node->definition)
4545 : ;
4546 : /* For builtins that are likely expanded to nothing or
4547 : inlined do not account operand costs. */
4548 18099359 : else if (is_simple_builtin (decl))
4549 : return 0;
4550 15215993 : else if (is_inexpensive_builtin (decl))
4551 1937593 : return weights->target_builtin_call_cost;
4552 13278400 : else if (gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
4553 : {
4554 : /* We canonicalize x * x to pow (x, 2.0) with -ffast-math, so
4555 : specialize the cheap expansion we do here.
4556 : ??? This asks for a more general solution. */
4557 13033408 : switch (DECL_FUNCTION_CODE (decl))
4558 : {
4559 9729 : case BUILT_IN_POW:
4560 9729 : case BUILT_IN_POWF:
4561 9729 : case BUILT_IN_POWL:
4562 9729 : if (TREE_CODE (gimple_call_arg (stmt, 1)) == REAL_CST
4563 13683 : && (real_equal
4564 3954 : (&TREE_REAL_CST (gimple_call_arg (stmt, 1)),
4565 : &dconst2)))
4566 541 : return estimate_operator_cost
4567 541 : (MULT_EXPR, weights, gimple_call_arg (stmt, 0),
4568 541 : gimple_call_arg (stmt, 0));
4569 : break;
4570 :
4571 : default:
4572 : break;
4573 : }
4574 : }
4575 : }
4576 :
4577 66207148 : cost = decl ? weights->call_cost : weights->indirect_call_cost;
4578 66207148 : if (gimple_call_lhs (stmt))
4579 25883411 : cost += estimate_move_cost (TREE_TYPE (gimple_call_lhs (stmt)),
4580 : weights->time_based);
4581 200449995 : for (i = 0; i < gimple_call_num_args (stmt); i++)
4582 : {
4583 134242847 : tree arg = gimple_call_arg (stmt, i);
4584 134242847 : cost += estimate_move_cost (TREE_TYPE (arg),
4585 : weights->time_based);
4586 : }
4587 : break;
4588 : }
4589 :
4590 18341305 : case GIMPLE_RETURN:
4591 18341305 : return weights->return_cost;
4592 :
4593 : case GIMPLE_GOTO:
4594 : case GIMPLE_LABEL:
4595 : case GIMPLE_NOP:
4596 : case GIMPLE_PHI:
4597 : case GIMPLE_PREDICT:
4598 : case GIMPLE_DEBUG:
4599 : return 0;
4600 :
4601 664928 : case GIMPLE_ASM:
4602 664928 : {
4603 664928 : int count = asm_str_count (gimple_asm_string (as_a <gasm *> (stmt)));
4604 : /* 1000 means infinity. This avoids overflows later
4605 : with very long asm statements. */
4606 664928 : if (count > 1000)
4607 : count = 1000;
4608 : /* If this asm is asm inline, count anything as minimum size. */
4609 664928 : if (gimple_asm_inline_p (as_a <gasm *> (stmt)))
4610 709 : count = MIN (1, count);
4611 664928 : return MAX (1, count);
4612 : }
4613 :
4614 : case GIMPLE_RESX:
4615 : /* This is either going to be an external function call with one
4616 : argument, or two register copy statements plus a goto. */
4617 : return 2;
4618 :
4619 18758 : case GIMPLE_EH_DISPATCH:
4620 : /* ??? This is going to turn into a switch statement. Ideally
4621 : we'd have a look at the eh region and estimate the number of
4622 : edges involved. */
4623 18758 : return 10;
4624 :
4625 0 : case GIMPLE_BIND:
4626 0 : return estimate_num_insns_seq (
4627 0 : gimple_bind_body (as_a <gbind *> (stmt)),
4628 0 : weights);
4629 :
4630 0 : case GIMPLE_EH_FILTER:
4631 0 : return estimate_num_insns_seq (gimple_eh_filter_failure (stmt), weights);
4632 :
4633 8292 : case GIMPLE_CATCH:
4634 8292 : return estimate_num_insns_seq (gimple_catch_handler (
4635 8292 : as_a <gcatch *> (stmt)),
4636 8292 : weights);
4637 :
4638 8297 : case GIMPLE_TRY:
4639 8297 : return (estimate_num_insns_seq (gimple_try_eval (stmt), weights)
4640 8297 : + estimate_num_insns_seq (gimple_try_cleanup (stmt), weights));
4641 :
4642 : /* OMP directives are generally very expensive. */
4643 :
4644 : case GIMPLE_OMP_RETURN:
4645 : case GIMPLE_OMP_SECTIONS_SWITCH:
4646 : case GIMPLE_OMP_ATOMIC_STORE:
4647 : case GIMPLE_OMP_CONTINUE:
4648 : /* ...except these, which are cheap. */
4649 : return 0;
4650 :
4651 0 : case GIMPLE_OMP_ATOMIC_LOAD:
4652 0 : return weights->omp_cost;
4653 :
4654 0 : case GIMPLE_OMP_FOR:
4655 0 : return (weights->omp_cost
4656 0 : + estimate_num_insns_seq (gimple_omp_body (stmt), weights)
4657 0 : + estimate_num_insns_seq (gimple_omp_for_pre_body (stmt), weights));
4658 :
4659 0 : case GIMPLE_OMP_PARALLEL:
4660 0 : case GIMPLE_OMP_TASK:
4661 0 : case GIMPLE_OMP_CRITICAL:
4662 0 : case GIMPLE_OMP_MASTER:
4663 0 : case GIMPLE_OMP_MASKED:
4664 0 : case GIMPLE_OMP_SCOPE:
4665 0 : case GIMPLE_OMP_DISPATCH:
4666 0 : case GIMPLE_OMP_TASKGROUP:
4667 0 : case GIMPLE_OMP_ORDERED:
4668 0 : case GIMPLE_OMP_SCAN:
4669 0 : case GIMPLE_OMP_SECTION:
4670 0 : case GIMPLE_OMP_SECTIONS:
4671 0 : case GIMPLE_OMP_STRUCTURED_BLOCK:
4672 0 : case GIMPLE_OMP_SINGLE:
4673 0 : case GIMPLE_OMP_TARGET:
4674 0 : case GIMPLE_OMP_TEAMS:
4675 0 : return (weights->omp_cost
4676 0 : + estimate_num_insns_seq (gimple_omp_body (stmt), weights));
4677 :
4678 75 : case GIMPLE_TRANSACTION:
4679 75 : return (weights->tm_cost
4680 75 : + estimate_num_insns_seq (gimple_transaction_body (
4681 75 : as_a <gtransaction *> (stmt)),
4682 75 : weights));
4683 :
4684 0 : default:
4685 0 : gcc_unreachable ();
4686 : }
4687 :
4688 472014878 : return cost;
4689 : }
4690 :
4691 : /* Estimate number of instructions that will be created by expanding
4692 : function FNDECL. WEIGHTS contains weights attributed to various
4693 : constructs. */
4694 :
4695 : int
4696 0 : estimate_num_insns_fn (tree fndecl, eni_weights *weights)
4697 : {
4698 0 : struct function *my_function = DECL_STRUCT_FUNCTION (fndecl);
4699 0 : gimple_stmt_iterator bsi;
4700 0 : basic_block bb;
4701 0 : int n = 0;
4702 :
4703 0 : gcc_assert (my_function && my_function->cfg);
4704 0 : FOR_EACH_BB_FN (bb, my_function)
4705 : {
4706 0 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
4707 0 : n += estimate_num_insns (gsi_stmt (bsi), weights);
4708 : }
4709 :
4710 0 : return n;
4711 : }
4712 :
4713 :
4714 : /* Initializes weights used by estimate_num_insns. */
4715 :
4716 : void
4717 289641 : init_inline_once (void)
4718 : {
4719 289641 : eni_size_weights.call_cost = 1;
4720 289641 : eni_size_weights.indirect_call_cost = 3;
4721 289641 : eni_size_weights.target_builtin_call_cost = 1;
4722 289641 : eni_size_weights.div_mod_cost = 1;
4723 289641 : eni_size_weights.omp_cost = 40;
4724 289641 : eni_size_weights.tm_cost = 10;
4725 289641 : eni_size_weights.time_based = false;
4726 289641 : eni_size_weights.return_cost = 1;
4727 :
4728 : /* Estimating time for call is difficult, since we have no idea what the
4729 : called function does. In the current uses of eni_time_weights,
4730 : underestimating the cost does less harm than overestimating it, so
4731 : we choose a rather small value here. */
4732 289641 : eni_time_weights.call_cost = 10;
4733 289641 : eni_time_weights.indirect_call_cost = 15;
4734 289641 : eni_time_weights.target_builtin_call_cost = 1;
4735 289641 : eni_time_weights.div_mod_cost = 10;
4736 289641 : eni_time_weights.omp_cost = 40;
4737 289641 : eni_time_weights.tm_cost = 40;
4738 289641 : eni_time_weights.time_based = true;
4739 289641 : eni_time_weights.return_cost = 2;
4740 289641 : }
4741 :
4742 :
4743 : /* Install new lexical TREE_BLOCK underneath 'current_block'. */
4744 :
4745 : static void
4746 31050921 : prepend_lexical_block (tree current_block, tree new_block)
4747 : {
4748 31050921 : BLOCK_CHAIN (new_block) = BLOCK_SUBBLOCKS (current_block);
4749 31050921 : BLOCK_SUBBLOCKS (current_block) = new_block;
4750 31050921 : BLOCK_SUPERCONTEXT (new_block) = current_block;
4751 31050921 : }
4752 :
4753 : /* Add local variables from CALLEE to CALLER. */
4754 :
4755 : static inline void
4756 4798334 : add_local_variables (struct function *callee, struct function *caller,
4757 : copy_body_data *id)
4758 : {
4759 4798334 : tree var;
4760 4798334 : unsigned ix;
4761 :
4762 13299696 : FOR_EACH_LOCAL_DECL (callee, ix, var)
4763 4473890 : if (!can_be_nonlocal (var, id))
4764 : {
4765 4386596 : tree new_var = remap_decl (var, id);
4766 :
4767 : /* Remap debug-expressions. */
4768 4386596 : if (VAR_P (new_var)
4769 4386596 : && DECL_HAS_DEBUG_EXPR_P (var)
4770 4706088 : && new_var != var)
4771 : {
4772 319492 : tree tem = DECL_DEBUG_EXPR (var);
4773 319492 : bool old_regimplify = id->regimplify;
4774 319492 : id->remapping_type_depth++;
4775 319492 : walk_tree (&tem, copy_tree_body_r, id, NULL);
4776 319492 : id->remapping_type_depth--;
4777 319492 : id->regimplify = old_regimplify;
4778 319492 : SET_DECL_DEBUG_EXPR (new_var, tem);
4779 319492 : DECL_HAS_DEBUG_EXPR_P (new_var) = 1;
4780 : }
4781 4386596 : add_local_decl (caller, new_var);
4782 : }
4783 4798334 : }
4784 :
4785 : /* Add to BINDINGS a debug stmt resetting SRCVAR if inlining might
4786 : have brought in or introduced any debug stmts for SRCVAR. */
4787 :
4788 : static inline void
4789 9952718 : reset_debug_binding (copy_body_data *id, tree srcvar, gimple_seq *bindings)
4790 : {
4791 9952718 : tree *remappedvarp = id->decl_map->get (srcvar);
4792 :
4793 9952718 : if (!remappedvarp)
4794 : return;
4795 :
4796 9896856 : if (!VAR_P (*remappedvarp))
4797 : return;
4798 :
4799 9745789 : if (*remappedvarp == id->retvar)
4800 : return;
4801 :
4802 9745789 : tree tvar = target_for_debug_bind (*remappedvarp);
4803 9745789 : if (!tvar)
4804 : return;
4805 :
4806 15668200 : gdebug *stmt = gimple_build_debug_bind (tvar, NULL_TREE,
4807 7834100 : id->call_stmt);
4808 7834100 : gimple_seq_add_stmt (bindings, stmt);
4809 : }
4810 :
4811 : /* For each inlined variable for which we may have debug bind stmts,
4812 : add before GSI a final debug stmt resetting it, marking the end of
4813 : its life, so that var-tracking knows it doesn't have to compute
4814 : further locations for it. */
4815 :
4816 : static inline void
4817 4646466 : reset_debug_bindings (copy_body_data *id, gimple_stmt_iterator gsi)
4818 : {
4819 4646466 : tree var;
4820 4646466 : unsigned ix;
4821 4646466 : gimple_seq bindings = NULL;
4822 :
4823 4646466 : if (!gimple_in_ssa_p (id->src_cfun))
4824 556339 : return;
4825 :
4826 4646466 : if (!opt_for_fn (id->dst_fn, flag_var_tracking_assignments))
4827 : return;
4828 :
4829 4090127 : for (var = DECL_ARGUMENTS (id->src_fn);
4830 11124914 : var; var = DECL_CHAIN (var))
4831 7034787 : reset_debug_binding (id, var, &bindings);
4832 :
4833 10440983 : FOR_EACH_LOCAL_DECL (id->src_cfun, ix, var)
4834 2917931 : reset_debug_binding (id, var, &bindings);
4835 :
4836 4090127 : gsi_insert_seq_before_without_update (&gsi, bindings, GSI_SAME_STMT);
4837 : }
4838 :
4839 : /* If STMT is a GIMPLE_CALL, replace it with its inline expansion. */
4840 :
4841 : static bool
4842 15948101 : expand_call_inline (basic_block bb, gimple *stmt, copy_body_data *id,
4843 : bitmap to_purge)
4844 : {
4845 15948101 : tree use_retvar;
4846 15948101 : tree fn;
4847 15948101 : hash_map<tree, tree> *dst;
4848 15948101 : hash_map<tree, tree> *st = NULL;
4849 15948101 : tree return_slot;
4850 15948101 : tree modify_dest;
4851 15948101 : struct cgraph_edge *cg_edge;
4852 15948101 : cgraph_inline_failed_t reason;
4853 15948101 : basic_block return_block;
4854 15948101 : edge e;
4855 15948101 : gimple_stmt_iterator gsi, stmt_gsi;
4856 15948101 : bool successfully_inlined = false;
4857 15948101 : bool purge_dead_abnormal_edges;
4858 15948101 : gcall *call_stmt;
4859 15948101 : unsigned int prop_mask, src_properties;
4860 15948101 : struct function *dst_cfun;
4861 15948101 : tree simduid;
4862 15948101 : use_operand_p use;
4863 15948101 : gimple *simtenter_stmt = NULL;
4864 15948101 : vec<tree> *simtvars_save;
4865 15948101 : tree save_stack = NULL_TREE;
4866 :
4867 : /* The gimplifier uses input_location in too many places, such as
4868 : internal_get_tmp_var (). */
4869 15948101 : location_t saved_location = input_location;
4870 15948101 : input_location = gimple_location (stmt);
4871 :
4872 : /* From here on, we're only interested in CALL_EXPRs. */
4873 15948101 : call_stmt = dyn_cast <gcall *> (stmt);
4874 15948101 : if (!call_stmt)
4875 0 : goto egress;
4876 :
4877 15948101 : cg_edge = id->dst_node->get_edge (stmt);
4878 : /* Edge should exist and speculations should be resolved at this
4879 : stage. */
4880 15948101 : gcc_checking_assert (cg_edge && !cg_edge->speculative);
4881 : /* First, see if we can figure out what function is being called.
4882 : If we cannot, then there is no hope of inlining the function. */
4883 15948101 : if (cg_edge->indirect_unknown_callee)
4884 240570 : goto egress;
4885 15707531 : fn = cg_edge->callee->decl;
4886 15707531 : gcc_checking_assert (fn);
4887 :
4888 : /* If FN is a declaration of a function in a nested scope that was
4889 : globally declared inline, we don't set its DECL_INITIAL.
4890 : However, we can't blindly follow DECL_ABSTRACT_ORIGIN because the
4891 : C++ front-end uses it for cdtors to refer to their internal
4892 : declarations, that are not real functions. Fortunately those
4893 : don't have trees to be saved, so we can tell by checking their
4894 : gimple_body. */
4895 15707531 : if (!DECL_INITIAL (fn)
4896 6945034 : && DECL_ABSTRACT_ORIGIN (fn)
4897 15843064 : && gimple_has_body_p (DECL_ABSTRACT_ORIGIN (fn)))
4898 288 : fn = DECL_ABSTRACT_ORIGIN (fn);
4899 :
4900 : /* Don't try to inline functions that are not well-suited to inlining. */
4901 15707531 : if (cg_edge->inline_failed)
4902 : {
4903 11060867 : reason = cg_edge->inline_failed;
4904 : /* If this call was originally indirect, we do not want to emit any
4905 : inlining related warnings or sorry messages because there are no
4906 : guarantees regarding those. */
4907 11060867 : if (cg_edge->indirect_inlining_edge)
4908 1965 : goto egress;
4909 :
4910 11058902 : if (lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn))
4911 : /* For extern inline functions that get redefined we always
4912 : silently ignored always_inline flag. Better behavior would
4913 : be to be able to keep both bodies and use extern inline body
4914 : for inlining, but we can't do that because frontends overwrite
4915 : the body. */
4916 63 : && !cg_edge->callee->redefined_extern_inline
4917 : /* During early inline pass, report only when optimization is
4918 : not turned on. */
4919 63 : && (symtab->global_info_ready
4920 61 : || !optimize
4921 51 : || cgraph_inline_failed_type (reason) == CIF_FINAL_ERROR)
4922 : /* PR 20090218-1_0.c. Body can be provided by another module. */
4923 11058936 : && (reason != CIF_BODY_NOT_AVAILABLE || !flag_generate_lto))
4924 : {
4925 27 : error ("inlining failed in call to %<always_inline%> %q+F: %s", fn,
4926 : cgraph_inline_failed_string (reason));
4927 27 : if (gimple_location (stmt) != UNKNOWN_LOCATION)
4928 27 : inform (gimple_location (stmt), "called from here");
4929 0 : else if (DECL_SOURCE_LOCATION (cfun->decl) != UNKNOWN_LOCATION)
4930 0 : inform (DECL_SOURCE_LOCATION (cfun->decl),
4931 : "called from this function");
4932 : }
4933 11058875 : else if (opt_for_fn (fn, warn_inline)
4934 339 : && DECL_DECLARED_INLINE_P (fn)
4935 110 : && !DECL_NO_INLINE_WARNING_P (fn)
4936 77 : && !DECL_IN_SYSTEM_HEADER (fn)
4937 60 : && reason != CIF_UNSPECIFIED
4938 60 : && !lookup_attribute ("noinline", DECL_ATTRIBUTES (fn))
4939 : /* Do not warn about not inlined recursive calls. */
4940 60 : && !cg_edge->recursive_p ()
4941 : /* Avoid warnings during early inline pass. */
4942 11058935 : && symtab->global_info_ready)
4943 : {
4944 8 : auto_diagnostic_group d;
4945 8 : if (warning (OPT_Winline, "inlining failed in call to %q+F: %s",
4946 : fn, _(cgraph_inline_failed_string (reason))))
4947 : {
4948 8 : if (gimple_location (stmt) != UNKNOWN_LOCATION)
4949 8 : inform (gimple_location (stmt), "called from here");
4950 0 : else if (DECL_SOURCE_LOCATION (cfun->decl) != UNKNOWN_LOCATION)
4951 0 : inform (DECL_SOURCE_LOCATION (cfun->decl),
4952 : "called from this function");
4953 : }
4954 8 : }
4955 11058902 : goto egress;
4956 : }
4957 4646664 : id->src_node = cg_edge->callee;
4958 :
4959 : /* If callee is thunk, all we need is to adjust the THIS pointer
4960 : and redirect to function being thunked. */
4961 4646664 : if (id->src_node->thunk)
4962 : {
4963 198 : cgraph_edge *edge;
4964 198 : tree virtual_offset = NULL;
4965 198 : profile_count count = cg_edge->count;
4966 198 : tree op;
4967 198 : gimple_stmt_iterator iter = gsi_for_stmt (stmt);
4968 198 : thunk_info *info = thunk_info::get (id->src_node);
4969 :
4970 198 : cgraph_edge::remove (cg_edge);
4971 396 : edge = id->src_node->callees->clone (id->dst_node, call_stmt,
4972 : gimple_uid (stmt),
4973 : profile_count::one (),
4974 : profile_count::one (),
4975 : true);
4976 198 : edge->count = count;
4977 198 : if (info->virtual_offset_p)
4978 7 : virtual_offset = size_int (info->virtual_value);
4979 198 : op = create_tmp_reg_fn (cfun, TREE_TYPE (gimple_call_arg (stmt, 0)),
4980 : NULL);
4981 198 : gsi_insert_before (&iter, gimple_build_assign (op,
4982 : gimple_call_arg (stmt, 0)),
4983 : GSI_NEW_STMT);
4984 198 : gcc_assert (info->this_adjusting);
4985 198 : op = thunk_adjust (&iter, op, 1, info->fixed_offset,
4986 : virtual_offset, info->indirect_offset);
4987 :
4988 198 : gimple_call_set_arg (stmt, 0, op);
4989 198 : gimple_call_set_fndecl (stmt, edge->callee->decl);
4990 198 : update_stmt (stmt);
4991 198 : id->src_node->remove ();
4992 198 : successfully_inlined = expand_call_inline (bb, stmt, id, to_purge);
4993 198 : maybe_remove_unused_call_args (cfun, stmt);
4994 : /* This used to return true even though we do fail to inline in
4995 : some cases. See PR98525. */
4996 198 : goto egress;
4997 : }
4998 4646466 : fn = cg_edge->callee->decl;
4999 4646466 : cg_edge->callee->get_untransformed_body ();
5000 :
5001 4646466 : if (flag_checking && cg_edge->callee->decl != id->dst_node->decl)
5002 4646457 : cg_edge->callee->verify ();
5003 :
5004 : /* We will be inlining this callee. */
5005 4646466 : id->eh_lp_nr = lookup_stmt_eh_lp (stmt);
5006 :
5007 : /* Update the callers EH personality. */
5008 4646466 : if (DECL_FUNCTION_PERSONALITY (fn))
5009 301900 : DECL_FUNCTION_PERSONALITY (cg_edge->caller->decl)
5010 150950 : = DECL_FUNCTION_PERSONALITY (fn);
5011 :
5012 : /* Split the block before the GIMPLE_CALL. */
5013 4646466 : stmt_gsi = gsi_for_stmt (stmt);
5014 4646466 : gsi_prev (&stmt_gsi);
5015 4646466 : e = split_block (bb, gsi_end_p (stmt_gsi) ? NULL : gsi_stmt (stmt_gsi));
5016 4646466 : bb = e->src;
5017 4646466 : return_block = e->dest;
5018 4646466 : remove_edge (e);
5019 :
5020 : /* If the GIMPLE_CALL was in the last statement of BB, it may have
5021 : been the source of abnormal edges. In this case, schedule
5022 : the removal of dead abnormal edges. */
5023 4646466 : gsi = gsi_start_bb (return_block);
5024 4646466 : gsi_next (&gsi);
5025 4646466 : purge_dead_abnormal_edges = gsi_end_p (gsi);
5026 :
5027 4646466 : stmt_gsi = gsi_start_bb (return_block);
5028 :
5029 : /* Build a block containing code to initialize the arguments, the
5030 : actual inline expansion of the body, and a label for the return
5031 : statements within the function to jump to. The type of the
5032 : statement expression is the return type of the function call.
5033 : ??? If the call does not have an associated block then we will
5034 : remap all callee blocks to NULL, effectively dropping most of
5035 : its debug information. This should only happen for calls to
5036 : artificial decls inserted by the compiler itself. We need to
5037 : either link the inlined blocks into the caller block tree or
5038 : not refer to them in any way to not break GC for locations. */
5039 4646466 : if (tree block = gimple_block (stmt))
5040 : {
5041 : /* We do want to assign a not UNKNOWN_LOCATION BLOCK_SOURCE_LOCATION
5042 : to make inlined_function_outer_scope_p return true on this BLOCK. */
5043 4619558 : location_t loc = LOCATION_LOCUS (gimple_location (stmt));
5044 4619558 : if (loc == UNKNOWN_LOCATION)
5045 262152 : loc = LOCATION_LOCUS (DECL_SOURCE_LOCATION (fn));
5046 262152 : if (loc == UNKNOWN_LOCATION)
5047 : loc = BUILTINS_LOCATION;
5048 4619558 : if (has_discriminator (gimple_location (stmt)))
5049 1615829 : loc = location_with_discriminator
5050 1615829 : (loc, get_discriminator_from_loc (gimple_location (stmt)));
5051 4619558 : id->block = make_node (BLOCK);
5052 7954062 : BLOCK_ABSTRACT_ORIGIN (id->block) = DECL_ORIGIN (fn);
5053 4619558 : BLOCK_SOURCE_LOCATION (id->block) = loc;
5054 4619558 : prepend_lexical_block (block, id->block);
5055 : }
5056 :
5057 : /* Local declarations will be replaced by their equivalents in this map. */
5058 4646466 : st = id->decl_map;
5059 4646466 : id->decl_map = new hash_map<tree, tree>;
5060 4646466 : dst = id->debug_map;
5061 4646466 : id->debug_map = NULL;
5062 4646466 : if (flag_stack_reuse != SR_NONE)
5063 4639528 : id->add_clobbers_to_eh_landing_pads = last_basic_block_for_fn (cfun);
5064 :
5065 : /* Record the function we are about to inline. */
5066 4646466 : id->src_fn = fn;
5067 4646466 : id->src_cfun = DECL_STRUCT_FUNCTION (fn);
5068 4646466 : id->reset_location = DECL_IGNORED_P (fn);
5069 4646466 : id->call_stmt = call_stmt;
5070 4646466 : cfun->cfg->full_profile &= id->src_cfun->cfg->full_profile;
5071 :
5072 : /* When inlining into an OpenMP SIMD-on-SIMT loop, arrange for new automatic
5073 : variables to be added to IFN_GOMP_SIMT_ENTER argument list. */
5074 4646466 : dst_cfun = DECL_STRUCT_FUNCTION (id->dst_fn);
5075 4646466 : simtvars_save = id->dst_simt_vars;
5076 4646466 : if (!(dst_cfun->curr_properties & PROP_gimple_lomp_dev)
5077 21469 : && (simduid = bb->loop_father->simduid) != NULL_TREE
5078 0 : && (simduid = ssa_default_def (dst_cfun, simduid)) != NULL_TREE
5079 0 : && single_imm_use (simduid, &use, &simtenter_stmt)
5080 0 : && is_gimple_call (simtenter_stmt)
5081 4646466 : && gimple_call_internal_p (simtenter_stmt, IFN_GOMP_SIMT_ENTER))
5082 0 : vec_alloc (id->dst_simt_vars, 0);
5083 : else
5084 4646466 : id->dst_simt_vars = NULL;
5085 :
5086 4646466 : if (profile_status_for_fn (id->src_cfun) == PROFILE_ABSENT)
5087 31291 : profile_status_for_fn (dst_cfun) = PROFILE_ABSENT;
5088 :
5089 : /* If the src function contains an IFN_VA_ARG, then so will the dst
5090 : function after inlining. Likewise for IFN_GOMP_USE_SIMT. */
5091 4646466 : prop_mask = PROP_gimple_lva | PROP_gimple_lomp_dev;
5092 4646466 : src_properties = id->src_cfun->curr_properties & prop_mask;
5093 4646466 : if (src_properties != prop_mask)
5094 1063 : dst_cfun->curr_properties &= src_properties | ~prop_mask;
5095 4646466 : dst_cfun->calls_eh_return |= id->src_cfun->calls_eh_return;
5096 4646466 : id->dst_node->has_omp_variant_constructs
5097 4646466 : |= id->src_node->has_omp_variant_constructs;
5098 :
5099 4646466 : gcc_assert (!id->src_cfun->after_inlining);
5100 :
5101 4646466 : id->entry_bb = bb;
5102 4646466 : if (lookup_attribute ("cold", DECL_ATTRIBUTES (fn)))
5103 : {
5104 1462 : gimple_stmt_iterator si = gsi_last_bb (bb);
5105 1462 : gsi_insert_after (&si, gimple_build_predict (PRED_COLD_FUNCTION,
5106 : NOT_TAKEN),
5107 : GSI_NEW_STMT);
5108 : }
5109 4646466 : initialize_inlined_parameters (id, stmt, fn, bb);
5110 4090145 : if (debug_nonbind_markers_p && debug_inline_points && id->block
5111 8724530 : && inlined_function_outer_scope_p (id->block))
5112 : {
5113 4078064 : gimple_stmt_iterator si = gsi_last_bb (bb);
5114 4078064 : gsi_insert_after (&si, gimple_build_debug_inline_entry
5115 4078064 : (id->block, DECL_SOURCE_LOCATION (id->src_fn)),
5116 : GSI_NEW_STMT);
5117 : }
5118 :
5119 : /* If function to be inlined calls alloca, wrap the inlined function
5120 : in between save_stack = __builtin_stack_save (); and
5121 : __builtin_stack_restore (save_stack); calls. */
5122 4646466 : if (id->src_cfun->calls_alloca && !gimple_call_noreturn_p (stmt))
5123 : /* Don't do this for VLA allocations though, just for user alloca
5124 : calls. */
5125 4703 : for (struct cgraph_edge *e = id->src_node->callees; e; e = e->next_callee)
5126 4293 : if (gimple_maybe_alloca_call_p (e->call_stmt)
5127 4293 : && !gimple_call_alloca_for_var_p (e->call_stmt))
5128 : {
5129 88 : tree fn = builtin_decl_implicit (BUILT_IN_STACK_SAVE);
5130 88 : gcall *call = gimple_build_call (fn, 0);
5131 88 : save_stack = make_ssa_name (ptr_type_node);
5132 88 : gimple_call_set_lhs (call, save_stack);
5133 88 : gimple_stmt_iterator si = gsi_last_bb (bb);
5134 88 : gsi_insert_after (&si, call, GSI_NEW_STMT);
5135 88 : struct cgraph_node *dest = cgraph_node::get_create (fn);
5136 88 : id->dst_node->create_edge (dest, call, bb->count)->inline_failed
5137 88 : = CIF_BODY_NOT_AVAILABLE;
5138 88 : break;
5139 : }
5140 :
5141 4646466 : if (DECL_INITIAL (fn))
5142 : {
5143 4646466 : if (gimple_block (stmt))
5144 : {
5145 4619558 : tree *var;
5146 :
5147 4619558 : prepend_lexical_block (id->block,
5148 4619558 : remap_blocks (DECL_INITIAL (fn), id));
5149 4619558 : gcc_checking_assert (BLOCK_SUBBLOCKS (id->block)
5150 : && (BLOCK_CHAIN (BLOCK_SUBBLOCKS (id->block))
5151 : == NULL_TREE));
5152 : /* Move vars for PARM_DECLs from DECL_INITIAL block to id->block,
5153 : otherwise for DWARF DW_TAG_formal_parameter will not be children of
5154 : DW_TAG_inlined_subroutine, but of a DW_TAG_lexical_block
5155 : under it. The parameters can be then evaluated in the debugger,
5156 : but don't show in backtraces. */
5157 6792310 : for (var = &BLOCK_VARS (BLOCK_SUBBLOCKS (id->block)); *var; )
5158 2172752 : if (TREE_CODE (DECL_ORIGIN (*var)) == PARM_DECL)
5159 : {
5160 587210 : tree v = *var;
5161 587210 : *var = TREE_CHAIN (v);
5162 587210 : TREE_CHAIN (v) = BLOCK_VARS (id->block);
5163 587210 : BLOCK_VARS (id->block) = v;
5164 : }
5165 : else
5166 1585542 : var = &TREE_CHAIN (*var);
5167 : }
5168 : else
5169 26908 : remap_blocks_to_null (DECL_INITIAL (fn), id);
5170 : }
5171 :
5172 : /* Return statements in the function body will be replaced by jumps
5173 : to the RET_LABEL. */
5174 4646466 : gcc_assert (DECL_INITIAL (fn));
5175 4646466 : gcc_assert (TREE_CODE (DECL_INITIAL (fn)) == BLOCK);
5176 :
5177 : /* Find the LHS to which the result of this call is assigned. */
5178 4646466 : return_slot = NULL;
5179 4646466 : if (gimple_call_lhs (stmt))
5180 : {
5181 2339587 : modify_dest = gimple_call_lhs (stmt);
5182 :
5183 : /* The function which we are inlining might not return a value,
5184 : in which case we should issue a warning that the function
5185 : does not return a value. In that case the optimizers will
5186 : see that the variable to which the value is assigned was not
5187 : initialized. We do not want to issue a warning about that
5188 : uninitialized variable. */
5189 2339587 : if (DECL_P (modify_dest))
5190 503192 : suppress_warning (modify_dest, OPT_Wuninitialized);
5191 :
5192 : /* If we have a return slot, we can assign it the result directly,
5193 : except in the case where it is a global variable that is only
5194 : written to because, the callee being permitted to read or take
5195 : the address of its DECL_RESULT, this could invalidate the flag
5196 : on the global variable; instead we preventively remove the store,
5197 : which would have happened later if the call was not inlined. */
5198 2339587 : if (gimple_call_return_slot_opt_p (call_stmt))
5199 : {
5200 143027 : tree base = get_base_address (modify_dest);
5201 :
5202 143027 : if (VAR_P (base)
5203 118360 : && (TREE_STATIC (base) || DECL_EXTERNAL (base))
5204 143099 : && varpool_node::get (base)->writeonly)
5205 : return_slot = NULL;
5206 : else
5207 : return_slot = modify_dest;
5208 :
5209 : modify_dest = NULL;
5210 : }
5211 : }
5212 : else
5213 : modify_dest = NULL;
5214 :
5215 : /* If we are inlining a call to the C++ operator new, we don't want
5216 : to use type based alias analysis on the return value. Otherwise
5217 : we may get confused if the compiler sees that the inlined new
5218 : function returns a pointer which was just deleted. See bug
5219 : 33407. */
5220 4646466 : if (DECL_IS_OPERATOR_NEW_P (fn))
5221 : {
5222 20284 : return_slot = NULL;
5223 20284 : modify_dest = NULL;
5224 : }
5225 :
5226 : /* Declare the return variable for the function. */
5227 4646466 : use_retvar = declare_return_variable (id, return_slot, modify_dest, bb);
5228 :
5229 : /* Add local vars in this inlined callee to caller. */
5230 4646466 : add_local_variables (id->src_cfun, cfun, id);
5231 :
5232 4646466 : if (dump_enabled_p ())
5233 : {
5234 680 : char buf[128];
5235 680 : snprintf (buf, sizeof(buf), "%4.2f",
5236 680 : cg_edge->sreal_frequency ().to_double ());
5237 680 : dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
5238 680 : call_stmt,
5239 : "Inlining %C to %C with frequency %s\n",
5240 : id->src_node, id->dst_node, buf);
5241 680 : if (dump_file && (dump_flags & TDF_DETAILS))
5242 : {
5243 198 : id->src_node->dump (dump_file);
5244 198 : id->dst_node->dump (dump_file);
5245 : }
5246 : }
5247 :
5248 : /* This is it. Duplicate the callee body. Assume callee is
5249 : pre-gimplified. Note that we must not alter the caller
5250 : function in any way before this point, as this CALL_EXPR may be
5251 : a self-referential call; if we're calling ourselves, we need to
5252 : duplicate our body before altering anything. */
5253 4646466 : copy_body (id, bb, return_block, NULL);
5254 :
5255 4646466 : reset_debug_bindings (id, stmt_gsi);
5256 :
5257 4646466 : if (flag_stack_reuse != SR_NONE)
5258 12562143 : for (tree p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
5259 7922615 : if (!TREE_THIS_VOLATILE (p))
5260 : {
5261 : /* The value associated with P is a local temporary only if
5262 : there is no value associated with P in the debug map. */
5263 7922342 : tree *varp = id->decl_map->get (p);
5264 7922342 : if (varp
5265 7922342 : && VAR_P (*varp)
5266 7756106 : && !is_gimple_reg (*varp)
5267 8362962 : && !(id->debug_map && id->debug_map->get (p)))
5268 : {
5269 440205 : tree clobber = build_clobber (TREE_TYPE (*varp),
5270 : CLOBBER_STORAGE_END);
5271 440205 : gimple *clobber_stmt;
5272 440205 : clobber_stmt = gimple_build_assign (*varp, clobber);
5273 440205 : gimple_set_location (clobber_stmt, gimple_location (stmt));
5274 440205 : gsi_insert_before (&stmt_gsi, clobber_stmt, GSI_SAME_STMT);
5275 : }
5276 : }
5277 :
5278 4646466 : if (save_stack)
5279 : {
5280 88 : tree fn = builtin_decl_implicit (BUILT_IN_STACK_RESTORE);
5281 88 : gcall *call = gimple_build_call (fn, 1, save_stack);
5282 88 : gsi_insert_before (&stmt_gsi, call, GSI_SAME_STMT);
5283 88 : struct cgraph_node *dest = cgraph_node::get_create (fn);
5284 88 : id->dst_node->create_edge (dest, call,
5285 : return_block->count)->inline_failed
5286 88 : = CIF_BODY_NOT_AVAILABLE;
5287 : }
5288 :
5289 : /* Reset the escaped solution. */
5290 4646466 : if (cfun->gimple_df)
5291 : {
5292 4646466 : pt_solution_reset (&cfun->gimple_df->escaped);
5293 4646466 : pt_solution_reset (&cfun->gimple_df->escaped_return);
5294 : }
5295 :
5296 : /* Add new automatic variables to IFN_GOMP_SIMT_ENTER arguments. */
5297 4646466 : if (id->dst_simt_vars && id->dst_simt_vars->length () > 0)
5298 : {
5299 0 : size_t nargs = gimple_call_num_args (simtenter_stmt);
5300 0 : vec<tree> *vars = id->dst_simt_vars;
5301 0 : auto_vec<tree> newargs (nargs + vars->length ());
5302 0 : for (size_t i = 0; i < nargs; i++)
5303 0 : newargs.quick_push (gimple_call_arg (simtenter_stmt, i));
5304 0 : for (tree *pvar = vars->begin (); pvar != vars->end (); pvar++)
5305 : {
5306 0 : tree ptrtype = build_pointer_type (TREE_TYPE (*pvar));
5307 0 : newargs.quick_push (build1 (ADDR_EXPR, ptrtype, *pvar));
5308 : }
5309 0 : gcall *g = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, newargs);
5310 0 : gimple_call_set_lhs (g, gimple_call_lhs (simtenter_stmt));
5311 0 : gimple_stmt_iterator gsi = gsi_for_stmt (simtenter_stmt);
5312 0 : gsi_replace (&gsi, g, false);
5313 0 : }
5314 4646466 : vec_free (id->dst_simt_vars);
5315 4646466 : id->dst_simt_vars = simtvars_save;
5316 :
5317 : /* Clean up. */
5318 4646466 : if (id->debug_map)
5319 : {
5320 161859 : delete id->debug_map;
5321 161859 : id->debug_map = dst;
5322 : }
5323 9292932 : delete id->decl_map;
5324 4646466 : id->decl_map = st;
5325 :
5326 : /* Unlink the calls virtual operands before replacing it. */
5327 4646466 : unlink_stmt_vdef (stmt);
5328 4646466 : if (gimple_vdef (stmt)
5329 4646466 : && TREE_CODE (gimple_vdef (stmt)) == SSA_NAME)
5330 2162548 : release_ssa_name (gimple_vdef (stmt));
5331 :
5332 : /* If the inlined function returns a result that we care about,
5333 : substitute the GIMPLE_CALL with an assignment of the return
5334 : variable to the LHS of the call. That is, if STMT was
5335 : 'a = foo (...)', substitute the call with 'a = USE_RETVAR'. */
5336 4646466 : if (use_retvar && gimple_call_lhs (stmt))
5337 : {
5338 1916809 : gimple *old_stmt = stmt;
5339 1916809 : tree lhs = gimple_call_lhs (stmt);
5340 1916809 : if (!is_gimple_reg (lhs)
5341 107330 : && !is_gimple_reg (use_retvar)
5342 2024097 : && is_gimple_reg_type (TREE_TYPE (lhs)))
5343 : {
5344 : /* If both lhs and use_retvar aren't gimple regs, yet have
5345 : gimple reg type, copy through a temporary SSA_NAME. */
5346 0 : gimple *g = gimple_build_assign (make_ssa_name (TREE_TYPE (lhs)),
5347 : use_retvar);
5348 0 : gimple_set_location (g, gimple_location (old_stmt));
5349 0 : gsi_insert_before (&stmt_gsi, g, GSI_SAME_STMT);
5350 0 : use_retvar = gimple_assign_lhs (g);
5351 : }
5352 1916809 : stmt = gimple_build_assign (lhs, use_retvar);
5353 1916809 : gimple_set_location (stmt, gimple_location (old_stmt));
5354 1916809 : gsi_replace (&stmt_gsi, stmt, false);
5355 1916809 : maybe_clean_or_replace_eh_stmt (old_stmt, stmt);
5356 : /* Append a clobber for id->retvar if easily possible. */
5357 1916809 : if (flag_stack_reuse != SR_NONE
5358 1914661 : && id->retvar
5359 1914661 : && VAR_P (id->retvar)
5360 1914661 : && id->retvar != return_slot
5361 1914661 : && id->retvar != modify_dest
5362 1914661 : && !TREE_THIS_VOLATILE (id->retvar)
5363 1914631 : && !is_gimple_reg (id->retvar)
5364 2025155 : && !stmt_ends_bb_p (stmt))
5365 : {
5366 108343 : tree clobber = build_clobber (TREE_TYPE (id->retvar),
5367 : CLOBBER_STORAGE_END);
5368 108343 : gimple *clobber_stmt;
5369 108343 : clobber_stmt = gimple_build_assign (id->retvar, clobber);
5370 108343 : gimple_set_location (clobber_stmt, gimple_location (old_stmt));
5371 108343 : gsi_insert_after (&stmt_gsi, clobber_stmt, GSI_SAME_STMT);
5372 : }
5373 : }
5374 : else
5375 : {
5376 : /* Handle the case of inlining a function with no return
5377 : statement, which causes the return value to become undefined. */
5378 2729657 : if (gimple_call_lhs (stmt)
5379 2729657 : && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
5380 : {
5381 21 : tree name = gimple_call_lhs (stmt);
5382 21 : tree var = SSA_NAME_VAR (name);
5383 6 : tree def = var ? ssa_default_def (cfun, var) : NULL;
5384 :
5385 6 : if (def)
5386 : {
5387 : /* If the variable is used undefined, make this name
5388 : undefined via a move. */
5389 0 : stmt = gimple_build_assign (gimple_call_lhs (stmt), def);
5390 0 : gsi_replace (&stmt_gsi, stmt, true);
5391 : }
5392 : else
5393 : {
5394 21 : if (!var)
5395 : {
5396 15 : var = create_tmp_reg_fn (cfun, TREE_TYPE (name), NULL);
5397 30 : SET_SSA_NAME_VAR_OR_IDENTIFIER (name, var);
5398 : }
5399 : /* Otherwise make this variable undefined. */
5400 21 : gsi_remove (&stmt_gsi, true);
5401 21 : set_ssa_default_def (cfun, var, name);
5402 21 : SSA_NAME_DEF_STMT (name) = gimple_build_nop ();
5403 : }
5404 : }
5405 : /* Replace with a clobber for id->retvar. */
5406 2729636 : else if (flag_stack_reuse != SR_NONE
5407 2724846 : && id->retvar
5408 572002 : && VAR_P (id->retvar)
5409 480418 : && id->retvar != return_slot
5410 428087 : && id->retvar != modify_dest
5411 149716 : && !TREE_THIS_VOLATILE (id->retvar)
5412 2879352 : && !is_gimple_reg (id->retvar))
5413 : {
5414 15172 : tree clobber = build_clobber (TREE_TYPE (id->retvar));
5415 15172 : gimple *clobber_stmt;
5416 15172 : clobber_stmt = gimple_build_assign (id->retvar, clobber);
5417 15172 : gimple_set_location (clobber_stmt, gimple_location (stmt));
5418 15172 : gsi_replace (&stmt_gsi, clobber_stmt, false);
5419 15172 : maybe_clean_or_replace_eh_stmt (stmt, clobber_stmt);
5420 : }
5421 : else
5422 2714464 : gsi_remove (&stmt_gsi, true);
5423 : }
5424 :
5425 4646466 : if (purge_dead_abnormal_edges)
5426 1132568 : bitmap_set_bit (to_purge, return_block->index);
5427 :
5428 : /* If the value of the new expression is ignored, that's OK. We
5429 : don't warn about this for CALL_EXPRs, so we shouldn't warn about
5430 : the equivalent inlined version either. */
5431 4646466 : if (is_gimple_assign (stmt))
5432 : {
5433 1916809 : gcc_assert (gimple_assign_single_p (stmt)
5434 : || CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt)));
5435 1916809 : TREE_USED (gimple_assign_rhs1 (stmt)) = 1;
5436 : }
5437 :
5438 4646466 : id->add_clobbers_to_eh_landing_pads = 0;
5439 :
5440 : /* Output the inlining info for this abstract function, since it has been
5441 : inlined. If we don't do this now, we can lose the information about the
5442 : variables in the function when the blocks get blown away as soon as we
5443 : remove the cgraph node. */
5444 4646466 : if (gimple_block (stmt))
5445 4619558 : (*debug_hooks->outlining_inline_function) (fn);
5446 :
5447 : /* Update callgraph if needed. */
5448 4646466 : cg_edge->callee->remove ();
5449 :
5450 4646466 : id->block = NULL_TREE;
5451 4646466 : id->retvar = NULL_TREE;
5452 4646466 : successfully_inlined = true;
5453 :
5454 15948101 : egress:
5455 15948101 : input_location = saved_location;
5456 15948101 : return successfully_inlined;
5457 : }
5458 :
5459 : /* Expand call statements reachable from STMT_P.
5460 : We can only have CALL_EXPRs as the "toplevel" tree code or nested
5461 : in a MODIFY_EXPR. */
5462 :
5463 : static bool
5464 32849170 : gimple_expand_calls_inline (basic_block bb, copy_body_data *id,
5465 : bitmap to_purge)
5466 : {
5467 32849170 : gimple_stmt_iterator gsi;
5468 32849170 : bool inlined = false;
5469 :
5470 244240220 : for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi);)
5471 : {
5472 178541880 : gimple *stmt = gsi_stmt (gsi);
5473 178541880 : gsi_prev (&gsi);
5474 :
5475 178541880 : if (is_gimple_call (stmt)
5476 178541880 : && !gimple_call_internal_p (stmt))
5477 15947903 : inlined |= expand_call_inline (bb, stmt, id, to_purge);
5478 : }
5479 :
5480 32849170 : return inlined;
5481 : }
5482 :
5483 :
5484 : /* Walk all basic blocks created after FIRST and try to fold every statement
5485 : in the STATEMENTS pointer set. */
5486 :
5487 : static void
5488 1669662 : fold_marked_statements (int first, hash_set<gimple *> *statements)
5489 : {
5490 1669662 : auto_bitmap to_purge;
5491 1669662 : auto_bitmap to_purge_abnormal;
5492 :
5493 1669662 : auto_vec<edge, 20> stack (n_basic_blocks_for_fn (cfun) + 2);
5494 1669662 : auto_sbitmap visited (last_basic_block_for_fn (cfun));
5495 1669662 : bitmap_clear (visited);
5496 :
5497 1669662 : stack.quick_push (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
5498 40604498 : while (!stack.is_empty ())
5499 : {
5500 : /* Look at the edge on the top of the stack. */
5501 38934836 : edge e = stack.pop ();
5502 38934836 : basic_block dest = e->dest;
5503 :
5504 47933798 : if (dest == EXIT_BLOCK_PTR_FOR_FN (cfun)
5505 37290789 : || bitmap_bit_p (visited, dest->index))
5506 8998962 : continue;
5507 :
5508 29935874 : bitmap_set_bit (visited, dest->index);
5509 :
5510 29935874 : if (dest->index >= first)
5511 38931840 : for (gimple_stmt_iterator gsi = gsi_start_bb (dest);
5512 136901469 : !gsi_end_p (gsi); gsi_next (&gsi))
5513 : {
5514 117435549 : if (!statements->contains (gsi_stmt (gsi)))
5515 24044849 : continue;
5516 :
5517 93390700 : gimple *old_stmt = gsi_stmt (gsi);
5518 93390700 : bool can_make_abnormal_goto = false;
5519 93390700 : tree old_decl = NULL_TREE;
5520 :
5521 93390700 : if (is_gimple_call (old_stmt))
5522 : {
5523 4153602 : old_decl = gimple_call_fndecl (old_stmt);
5524 4153602 : if (stmt_can_make_abnormal_goto (old_stmt))
5525 : can_make_abnormal_goto = true;
5526 : }
5527 :
5528 4153602 : if (old_decl && fndecl_built_in_p (old_decl))
5529 : {
5530 : /* Folding builtins can create multiple instructions,
5531 : we need to look at all of them. */
5532 1568715 : gimple_stmt_iterator i2 = gsi;
5533 1568715 : gsi_prev (&i2);
5534 1568715 : if (fold_stmt (&gsi))
5535 : {
5536 93544 : gimple *new_stmt;
5537 : /* If a builtin at the end of a bb folded into nothing,
5538 : the following loop won't work. */
5539 93544 : if (gsi_end_p (gsi))
5540 : {
5541 0 : cgraph_update_edges_for_call_stmt (old_stmt,
5542 : old_decl, NULL);
5543 0 : if (can_make_abnormal_goto)
5544 0 : bitmap_set_bit (to_purge_abnormal, dest->index);
5545 19465920 : break;
5546 : }
5547 93544 : if (gsi_end_p (i2))
5548 125456 : i2 = gsi_start_bb (dest);
5549 : else
5550 30816 : gsi_next (&i2);
5551 94472 : while (1)
5552 : {
5553 464 : new_stmt = gsi_stmt (i2);
5554 94008 : update_stmt (new_stmt);
5555 94008 : cgraph_update_edges_for_call_stmt (old_stmt, old_decl,
5556 : new_stmt);
5557 :
5558 94008 : if (new_stmt == gsi_stmt (gsi))
5559 : {
5560 : /* It is okay to check only for the very last
5561 : of these statements. If it is a throwing
5562 : statement nothing will change. If it isn't
5563 : this can remove EH edges. If that weren't
5564 : correct then because some intermediate stmts
5565 : throw, but not the last one. That would mean
5566 : we'd have to split the block, which we can't
5567 : here and we'd loose anyway. And as builtins
5568 : probably never throw, this all
5569 : is mood anyway. */
5570 93544 : if (maybe_clean_or_replace_eh_stmt (old_stmt,
5571 : new_stmt))
5572 7 : bitmap_set_bit (to_purge, dest->index);
5573 93544 : if (can_make_abnormal_goto
5574 93544 : && !stmt_can_make_abnormal_goto (new_stmt))
5575 0 : bitmap_set_bit (to_purge_abnormal, dest->index);
5576 : break;
5577 : }
5578 464 : gsi_next (&i2);
5579 : }
5580 : }
5581 : }
5582 91821985 : else if (fold_stmt (&gsi))
5583 : {
5584 : /* Re-read the statement from GSI as fold_stmt() may
5585 : have changed it. */
5586 3010004 : gimple *new_stmt = gsi_stmt (gsi);
5587 3010004 : update_stmt (new_stmt);
5588 :
5589 3010004 : if (is_gimple_call (old_stmt)
5590 3010004 : || is_gimple_call (new_stmt))
5591 4867 : cgraph_update_edges_for_call_stmt (old_stmt, old_decl,
5592 : new_stmt);
5593 :
5594 3010004 : if (maybe_clean_or_replace_eh_stmt (old_stmt, new_stmt))
5595 154 : bitmap_set_bit (to_purge, dest->index);
5596 3010004 : if (can_make_abnormal_goto
5597 3010004 : && !stmt_can_make_abnormal_goto (new_stmt))
5598 7 : bitmap_set_bit (to_purge_abnormal, dest->index);
5599 : }
5600 : }
5601 :
5602 58205861 : if (EDGE_COUNT (dest->succs) > 0)
5603 : {
5604 : /* Avoid warnings emitted from folding statements that
5605 : became unreachable because of inlined function parameter
5606 : propagation. */
5607 28269987 : e = find_taken_edge (dest, NULL_TREE);
5608 28269987 : if (e)
5609 19467391 : stack.quick_push (e);
5610 : else
5611 : {
5612 8802596 : edge_iterator ei;
5613 26600379 : FOR_EACH_EDGE (e, ei, dest->succs)
5614 17797783 : stack.safe_push (e);
5615 : }
5616 : }
5617 : }
5618 :
5619 1669662 : gimple_purge_all_dead_eh_edges (to_purge);
5620 1669662 : gimple_purge_all_dead_abnormal_call_edges (to_purge_abnormal);
5621 1669662 : }
5622 :
5623 : /* Expand calls to inline functions in the body of FN. */
5624 :
5625 : unsigned int
5626 2041742 : optimize_inline_calls (tree fn)
5627 : {
5628 2041742 : copy_body_data id;
5629 2041742 : basic_block bb;
5630 2041742 : int last = n_basic_blocks_for_fn (cfun);
5631 2041742 : bool inlined_p = false;
5632 :
5633 : /* Clear out ID. */
5634 2041742 : memset (&id, 0, sizeof (id));
5635 :
5636 2041742 : id.src_node = id.dst_node = cgraph_node::get (fn);
5637 2041742 : gcc_assert (id.dst_node->definition);
5638 2041742 : id.dst_fn = fn;
5639 : /* Or any functions that aren't finished yet. */
5640 2041742 : if (current_function_decl)
5641 2041742 : id.dst_fn = current_function_decl;
5642 :
5643 2041742 : id.copy_decl = copy_decl_maybe_to_var;
5644 2041742 : id.transform_call_graph_edges = CB_CGE_DUPLICATE;
5645 2041742 : id.transform_new_cfg = false;
5646 2041742 : id.transform_return_to_modify = true;
5647 2041742 : id.transform_parameter = true;
5648 2041742 : id.statements_to_fold = new hash_set<gimple *>;
5649 :
5650 2041742 : push_gimplify_context ();
5651 :
5652 : /* We make no attempts to keep dominance info up-to-date. */
5653 2041742 : free_dominance_info (CDI_DOMINATORS);
5654 2041742 : free_dominance_info (CDI_POST_DOMINATORS);
5655 :
5656 : /* Register specific gimple functions. */
5657 2041742 : gimple_register_cfg_hooks ();
5658 :
5659 : /* Reach the trees by walking over the CFG, and note the
5660 : enclosing basic-blocks in the call edges. */
5661 : /* We walk the blocks going forward, because inlined function bodies
5662 : will split id->current_basic_block, and the new blocks will
5663 : follow it; we'll trudge through them, processing their CALL_EXPRs
5664 : along the way. */
5665 2041742 : auto_bitmap to_purge;
5666 34890912 : FOR_EACH_BB_FN (bb, cfun)
5667 32849170 : inlined_p |= gimple_expand_calls_inline (bb, &id, to_purge);
5668 :
5669 2041742 : pop_gimplify_context (NULL);
5670 :
5671 2041742 : if (flag_checking)
5672 : {
5673 2041724 : struct cgraph_edge *e;
5674 :
5675 2041724 : id.dst_node->verify ();
5676 :
5677 : /* Double check that we inlined everything we are supposed to inline. */
5678 12389986 : for (e = id.dst_node->callees; e; e = e->next_callee)
5679 10348262 : gcc_assert (e->inline_failed);
5680 : }
5681 :
5682 : /* If we didn't inline into the function there is nothing to do. */
5683 2041742 : if (!inlined_p)
5684 : {
5685 1250508 : delete id.statements_to_fold;
5686 : return 0;
5687 : }
5688 :
5689 : /* Fold queued statements. */
5690 1416488 : update_max_bb_count ();
5691 1416488 : fold_marked_statements (last, id.statements_to_fold);
5692 2832976 : delete id.statements_to_fold;
5693 :
5694 : /* Finally purge EH and abnormal edges from the call stmts we inlined.
5695 : We need to do this after fold_marked_statements since that may walk
5696 : the SSA use-def chain. */
5697 1416488 : unsigned i;
5698 1416488 : bitmap_iterator bi;
5699 2549056 : EXECUTE_IF_SET_IN_BITMAP (to_purge, 0, i, bi)
5700 : {
5701 1132568 : basic_block bb = BASIC_BLOCK_FOR_FN (cfun, i);
5702 1132568 : if (bb)
5703 : {
5704 1132568 : gimple_purge_dead_eh_edges (bb);
5705 1132568 : gimple_purge_dead_abnormal_call_edges (bb);
5706 : }
5707 : }
5708 :
5709 1416488 : gcc_assert (!id.debug_stmts.exists ());
5710 :
5711 : /* Renumber the lexical scoping (non-code) blocks consecutively. */
5712 1416488 : number_blocks (fn);
5713 :
5714 1416488 : delete_unreachable_blocks_update_callgraph (id.dst_node, false);
5715 1416488 : id.dst_node->calls_comdat_local = id.dst_node->check_calls_comdat_local_p ();
5716 :
5717 1416488 : if (flag_checking)
5718 1416479 : id.dst_node->verify ();
5719 :
5720 : /* It would be nice to check SSA/CFG/statement consistency here, but it is
5721 : not possible yet - the IPA passes might make various functions to not
5722 : throw and they don't care to proactively update local EH info. This is
5723 : done later in fixup_cfg pass that also execute the verification. */
5724 1416488 : return (TODO_update_ssa
5725 : | TODO_cleanup_cfg
5726 1416488 : | (gimple_in_ssa_p (cfun) ? TODO_remove_unused_locals : 0)
5727 2832976 : | (gimple_in_ssa_p (cfun) ? TODO_update_address_taken : 0));
5728 2041742 : }
5729 :
5730 : /* Passed to walk_tree. Copies the node pointed to, if appropriate. */
5731 :
5732 : tree
5733 1987613217 : copy_tree_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED)
5734 : {
5735 1987613217 : enum tree_code code = TREE_CODE (*tp);
5736 1987613217 : enum tree_code_class cl = TREE_CODE_CLASS (code);
5737 :
5738 : /* We make copies of most nodes. */
5739 1987613217 : if (IS_EXPR_CODE_CLASS (cl)
5740 : || code == TREE_LIST
5741 304128766 : || code == TREE_VEC
5742 299143816 : || code == TYPE_DECL
5743 299143816 : || code == OMP_CLAUSE)
5744 : {
5745 : /* Because the chain gets clobbered when we make a copy, we save it
5746 : here. */
5747 1688493864 : tree chain = NULL_TREE, new_tree;
5748 :
5749 1688493864 : if (CODE_CONTAINS_STRUCT (code, TS_COMMON))
5750 5009413 : chain = TREE_CHAIN (*tp);
5751 :
5752 : /* Copy the node. */
5753 1688493864 : new_tree = copy_node (*tp);
5754 :
5755 1688493864 : *tp = new_tree;
5756 :
5757 : /* Now, restore the chain, if appropriate. That will cause
5758 : walk_tree to walk into the chain as well. */
5759 1688493864 : if (code == PARM_DECL
5760 1688493864 : || code == TREE_LIST
5761 1686266751 : || code == OMP_CLAUSE)
5762 2251548 : TREE_CHAIN (*tp) = chain;
5763 :
5764 : /* For now, we don't update BLOCKs when we make copies. So, we
5765 : have to nullify all BIND_EXPRs. */
5766 1688493864 : if (TREE_CODE (*tp) == BIND_EXPR)
5767 20570612 : BIND_EXPR_BLOCK (*tp) = NULL_TREE;
5768 : }
5769 299119353 : else if (code == CONSTRUCTOR)
5770 : {
5771 : /* CONSTRUCTOR nodes need special handling because
5772 : we need to duplicate the vector of elements. */
5773 18300018 : tree new_tree;
5774 :
5775 18300018 : new_tree = copy_node (*tp);
5776 26667790 : CONSTRUCTOR_ELTS (new_tree) = vec_safe_copy (CONSTRUCTOR_ELTS (*tp));
5777 18300018 : *tp = new_tree;
5778 : }
5779 280819335 : else if (code == STATEMENT_LIST)
5780 : /* We used to just abort on STATEMENT_LIST, but we can run into them
5781 : with statement-expressions (c++/40975). */
5782 198 : copy_statement_list (tp);
5783 280819137 : else if (TREE_CODE_CLASS (code) == tcc_type)
5784 278 : *walk_subtrees = 0;
5785 280818859 : else if (TREE_CODE_CLASS (code) == tcc_declaration)
5786 121368302 : *walk_subtrees = 0;
5787 159450557 : else if (TREE_CODE_CLASS (code) == tcc_constant)
5788 0 : *walk_subtrees = 0;
5789 1987613217 : return NULL_TREE;
5790 : }
5791 :
5792 : /* The SAVE_EXPR pointed to by TP is being copied. If ST contains
5793 : information indicating to what new SAVE_EXPR this one should be mapped,
5794 : use that one. Otherwise, create a new node and enter it in ST. FN is
5795 : the function into which the copy will be placed. */
5796 :
5797 : static void
5798 15698316 : remap_save_expr (tree *tp, hash_map<tree, tree> *st, int *walk_subtrees)
5799 : {
5800 15698316 : tree *n;
5801 15698316 : tree t;
5802 :
5803 : /* See if we already encountered this SAVE_EXPR. */
5804 15698316 : n = st->get (*tp);
5805 :
5806 : /* If we didn't already remap this SAVE_EXPR, do so now. */
5807 15698316 : if (!n)
5808 : {
5809 15240381 : t = copy_node (*tp);
5810 :
5811 : /* Remember this SAVE_EXPR. */
5812 15240381 : st->put (*tp, t);
5813 : /* Make sure we don't remap an already-remapped SAVE_EXPR. */
5814 15240381 : st->put (t, t);
5815 : }
5816 : else
5817 : {
5818 : /* We've already walked into this SAVE_EXPR; don't do it again. */
5819 457935 : *walk_subtrees = 0;
5820 457935 : t = *n;
5821 : }
5822 :
5823 : /* Replace this SAVE_EXPR with the copy. */
5824 15698316 : *tp = t;
5825 15698316 : }
5826 :
5827 : /* Called via walk_gimple_seq. If *GSIP points to a GIMPLE_LABEL for a local
5828 : label, copies the declaration and enters it in the splay_tree in DATA (which
5829 : is really a 'copy_body_data *'. */
5830 :
5831 : static tree
5832 1635702 : mark_local_labels_stmt (gimple_stmt_iterator *gsip,
5833 : bool *handled_ops_p ATTRIBUTE_UNUSED,
5834 : struct walk_stmt_info *wi)
5835 : {
5836 1635702 : copy_body_data *id = (copy_body_data *) wi->info;
5837 1635702 : glabel *stmt = dyn_cast <glabel *> (gsi_stmt (*gsip));
5838 :
5839 53944 : if (stmt)
5840 : {
5841 53944 : tree decl = gimple_label_label (stmt);
5842 :
5843 : /* Copy the decl and remember the copy. */
5844 53944 : insert_decl_map (id, decl, id->copy_decl (decl, id));
5845 : }
5846 :
5847 1635702 : return NULL_TREE;
5848 : }
5849 :
5850 : static gimple_seq duplicate_remap_omp_clause_seq (gimple_seq seq,
5851 : struct walk_stmt_info *wi);
5852 :
5853 : /* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
5854 : Using the splay_tree pointed to by ST (which is really a `splay_tree'),
5855 : remaps all local declarations to appropriate replacements in gimple
5856 : operands. */
5857 :
5858 : static tree
5859 3576578 : replace_locals_op (tree *tp, int *walk_subtrees, void *data)
5860 : {
5861 3576578 : struct walk_stmt_info *wi = (struct walk_stmt_info*) data;
5862 3576578 : copy_body_data *id = (copy_body_data *) wi->info;
5863 3576578 : hash_map<tree, tree> *st = id->decl_map;
5864 3576578 : tree *n;
5865 3576578 : tree expr = *tp;
5866 :
5867 : /* For recursive invocations this is no longer the LHS itself. */
5868 3576578 : bool is_lhs = wi->is_lhs;
5869 3576578 : wi->is_lhs = false;
5870 :
5871 3576578 : if (TREE_CODE (expr) == SSA_NAME)
5872 : {
5873 24614 : *tp = remap_ssa_name (*tp, id);
5874 24614 : *walk_subtrees = 0;
5875 24614 : if (is_lhs)
5876 12307 : SSA_NAME_DEF_STMT (*tp) = gsi_stmt (wi->gsi);
5877 : }
5878 : /* Only a local declaration (variable or label). */
5879 3551964 : else if ((VAR_P (expr) && !TREE_STATIC (expr))
5880 2071405 : || TREE_CODE (expr) == LABEL_DECL)
5881 : {
5882 : /* Lookup the declaration. */
5883 1590503 : n = st->get (expr);
5884 :
5885 : /* If it's there, remap it. */
5886 1590503 : if (n)
5887 108602 : *tp = *n;
5888 1590503 : *walk_subtrees = 0;
5889 : }
5890 1961461 : else if (TREE_CODE (expr) == STATEMENT_LIST
5891 1961461 : || TREE_CODE (expr) == BIND_EXPR
5892 1961461 : || TREE_CODE (expr) == SAVE_EXPR)
5893 0 : gcc_unreachable ();
5894 1961461 : else if (TREE_CODE (expr) == TARGET_EXPR)
5895 : {
5896 : /* Don't mess with a TARGET_EXPR that hasn't been expanded.
5897 : It's OK for this to happen if it was part of a subtree that
5898 : isn't immediately expanded, such as operand 2 of another
5899 : TARGET_EXPR. */
5900 0 : if (!TREE_OPERAND (expr, 1))
5901 : {
5902 0 : TREE_OPERAND (expr, 1) = TREE_OPERAND (expr, 3);
5903 0 : TREE_OPERAND (expr, 3) = NULL_TREE;
5904 : }
5905 : }
5906 1961461 : else if (TREE_CODE (expr) == OMP_CLAUSE)
5907 : {
5908 : /* Before the omplower pass completes, some OMP clauses can contain
5909 : sequences that are neither copied by gimple_seq_copy nor walked by
5910 : walk_gimple_seq. To make copy_gimple_seq_and_replace_locals work even
5911 : in those situations, we have to copy and process them explicitly. */
5912 :
5913 588 : if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_LASTPRIVATE)
5914 : {
5915 14 : gimple_seq seq = OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (expr);
5916 14 : seq = duplicate_remap_omp_clause_seq (seq, wi);
5917 14 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (expr) = seq;
5918 : }
5919 574 : else if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_LINEAR)
5920 : {
5921 85 : gimple_seq seq = OMP_CLAUSE_LINEAR_GIMPLE_SEQ (expr);
5922 85 : seq = duplicate_remap_omp_clause_seq (seq, wi);
5923 85 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (expr) = seq;
5924 : }
5925 489 : else if (OMP_CLAUSE_CODE (expr) == OMP_CLAUSE_REDUCTION)
5926 : {
5927 107 : gimple_seq seq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (expr);
5928 107 : seq = duplicate_remap_omp_clause_seq (seq, wi);
5929 107 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (expr) = seq;
5930 107 : seq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (expr);
5931 107 : seq = duplicate_remap_omp_clause_seq (seq, wi);
5932 107 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (expr) = seq;
5933 : }
5934 : }
5935 :
5936 : /* Keep iterating. */
5937 3576578 : return NULL_TREE;
5938 : }
5939 :
5940 :
5941 : /* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
5942 : Using the splay_tree pointed to by ST (which is really a `splay_tree'),
5943 : remaps all local declarations to appropriate replacements in gimple
5944 : statements. */
5945 :
5946 : static tree
5947 1635702 : replace_locals_stmt (gimple_stmt_iterator *gsip,
5948 : bool *handled_ops_p ATTRIBUTE_UNUSED,
5949 : struct walk_stmt_info *wi)
5950 : {
5951 1635702 : copy_body_data *id = (copy_body_data *) wi->info;
5952 1635702 : gimple *gs = gsi_stmt (*gsip);
5953 :
5954 1635702 : if (gbind *stmt = dyn_cast <gbind *> (gs))
5955 : {
5956 357 : tree block = gimple_bind_block (stmt);
5957 :
5958 357 : if (block)
5959 : {
5960 277 : remap_block (&block, id);
5961 277 : gimple_bind_set_block (stmt, block);
5962 : }
5963 :
5964 : /* This will remap a lot of the same decls again, but this should be
5965 : harmless. */
5966 357 : if (gimple_bind_vars (stmt))
5967 : {
5968 : tree old_var, decls = gimple_bind_vars (stmt);
5969 :
5970 528 : for (old_var = decls; old_var; old_var = DECL_CHAIN (old_var))
5971 360 : if (!can_be_nonlocal (old_var, id)
5972 360 : && ! variably_modified_type_p (TREE_TYPE (old_var), id->src_fn))
5973 360 : remap_decl (old_var, id);
5974 :
5975 168 : gcc_checking_assert (!id->prevent_decl_creation_for_types);
5976 168 : id->prevent_decl_creation_for_types = true;
5977 168 : gimple_bind_set_vars (stmt, remap_decls (decls, NULL, id));
5978 168 : id->prevent_decl_creation_for_types = false;
5979 : }
5980 : }
5981 :
5982 : /* Keep iterating. */
5983 1635702 : return NULL_TREE;
5984 : }
5985 :
5986 : /* Create a copy of SEQ and remap all decls in it. */
5987 :
5988 : static gimple_seq
5989 313 : duplicate_remap_omp_clause_seq (gimple_seq seq, struct walk_stmt_info *wi)
5990 : {
5991 313 : if (!seq)
5992 : return NULL;
5993 :
5994 : /* If there are any labels in OMP sequences, they can be only referred to in
5995 : the sequence itself and therefore we can do both here. */
5996 60 : walk_gimple_seq (seq, mark_local_labels_stmt, NULL, wi);
5997 60 : gimple_seq copy = gimple_seq_copy (seq);
5998 60 : walk_gimple_seq (copy, replace_locals_stmt, replace_locals_op, wi);
5999 60 : return copy;
6000 : }
6001 :
6002 : /* Copies everything in SEQ and replaces variables and labels local to
6003 : current_function_decl. */
6004 :
6005 : gimple_seq
6006 1105484 : copy_gimple_seq_and_replace_locals (gimple_seq seq)
6007 : {
6008 1105484 : copy_body_data id;
6009 1105484 : struct walk_stmt_info wi;
6010 1105484 : gimple_seq copy;
6011 :
6012 : /* There's nothing to do for NULL_TREE. */
6013 1105484 : if (seq == NULL)
6014 : return seq;
6015 :
6016 : /* Set up ID. */
6017 1105468 : memset (&id, 0, sizeof (id));
6018 1105468 : id.src_fn = current_function_decl;
6019 1105468 : id.dst_fn = current_function_decl;
6020 1105468 : id.src_cfun = cfun;
6021 1105468 : id.decl_map = new hash_map<tree, tree>;
6022 1105468 : id.debug_map = NULL;
6023 :
6024 1105468 : id.copy_decl = copy_decl_no_change;
6025 1105468 : id.transform_call_graph_edges = CB_CGE_DUPLICATE;
6026 1105468 : id.transform_new_cfg = false;
6027 1105468 : id.transform_return_to_modify = false;
6028 1105468 : id.transform_parameter = false;
6029 :
6030 : /* Walk the tree once to find local labels. */
6031 1105468 : memset (&wi, 0, sizeof (wi));
6032 1105468 : hash_set<tree> visited;
6033 1105468 : wi.info = &id;
6034 1105468 : wi.pset = &visited;
6035 1105468 : walk_gimple_seq (seq, mark_local_labels_stmt, NULL, &wi);
6036 :
6037 1105468 : copy = gimple_seq_copy (seq);
6038 :
6039 : /* Walk the copy, remapping decls. */
6040 1105468 : memset (&wi, 0, sizeof (wi));
6041 1105468 : wi.info = &id;
6042 1105468 : walk_gimple_seq (copy, replace_locals_stmt, replace_locals_op, &wi);
6043 :
6044 : /* Clean up. */
6045 2210936 : delete id.decl_map;
6046 1105468 : if (id.debug_map)
6047 0 : delete id.debug_map;
6048 1105468 : if (id.dependence_map)
6049 : {
6050 0 : delete id.dependence_map;
6051 0 : id.dependence_map = NULL;
6052 : }
6053 :
6054 1105468 : return copy;
6055 1105468 : }
6056 :
6057 :
6058 : /* Declare the variables created by the inliner. Add all the variables in
6059 : VARS to BIND_EXPR. */
6060 :
6061 : static void
6062 7011885 : declare_inline_vars (tree block, tree vars)
6063 : {
6064 7011885 : tree t;
6065 17250569 : for (t = vars; t; t = DECL_CHAIN (t))
6066 : {
6067 10238684 : DECL_SEEN_IN_BIND_EXPR_P (t) = 1;
6068 10238684 : gcc_assert (!TREE_STATIC (t) && !TREE_ASM_WRITTEN (t));
6069 10238684 : add_local_decl (cfun, t);
6070 : }
6071 :
6072 7011885 : if (block)
6073 6928263 : BLOCK_VARS (block) = chainon (BLOCK_VARS (block), vars);
6074 7011885 : }
6075 :
6076 : /* Copy NODE (which must be a DECL). The DECL originally was in the FROM_FN,
6077 : but now it will be in the TO_FN. PARM_TO_VAR means enable PARM_DECL to
6078 : VAR_DECL translation. */
6079 :
6080 : tree
6081 138618439 : copy_decl_for_dup_finish (copy_body_data *id, tree decl, tree copy)
6082 : {
6083 : /* Don't generate debug information for the copy if we wouldn't have
6084 : generated it for the copy either. */
6085 138618439 : DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (decl);
6086 138618439 : DECL_IGNORED_P (copy) = DECL_IGNORED_P (decl);
6087 :
6088 : /* Set the DECL_ABSTRACT_ORIGIN so the debugging routines know what
6089 : declaration inspired this copy. */
6090 171900405 : DECL_ABSTRACT_ORIGIN (copy) = DECL_ORIGIN (decl);
6091 :
6092 : /* The new variable/label has no RTL, yet. */
6093 138618439 : if (HAS_RTL_P (copy)
6094 138618439 : && !TREE_STATIC (copy) && !DECL_EXTERNAL (copy))
6095 138612903 : SET_DECL_RTL (copy, 0);
6096 : /* For vector typed decls make sure to update DECL_MODE according
6097 : to the new function context. */
6098 138618439 : if (VECTOR_TYPE_P (TREE_TYPE (copy)))
6099 336492 : SET_DECL_MODE (copy, TYPE_MODE (TREE_TYPE (copy)));
6100 :
6101 : /* These args would always appear unused, if not for this. */
6102 138618439 : TREE_USED (copy) = 1;
6103 :
6104 : /* Set the context for the new declaration. */
6105 138618439 : if (!DECL_CONTEXT (decl))
6106 : /* Globals stay global. */
6107 : ;
6108 138618288 : else if (DECL_CONTEXT (decl) != id->src_fn)
6109 : /* Things that weren't in the scope of the function we're inlining
6110 : from aren't in the scope we're inlining to, either. */
6111 : ;
6112 138615556 : else if (TREE_STATIC (decl))
6113 : /* Function-scoped static variables should stay in the original
6114 : function. */
6115 : ;
6116 : else
6117 : {
6118 : /* Ordinary automatic local variables are now in the scope of the
6119 : new function. */
6120 138612022 : DECL_CONTEXT (copy) = id->dst_fn;
6121 138612022 : if (VAR_P (copy) && id->dst_simt_vars && !is_gimple_reg (copy))
6122 : {
6123 0 : if (!lookup_attribute ("omp simt private", DECL_ATTRIBUTES (copy)))
6124 0 : DECL_ATTRIBUTES (copy)
6125 0 : = tree_cons (get_identifier ("omp simt private"), NULL,
6126 0 : DECL_ATTRIBUTES (copy));
6127 0 : id->dst_simt_vars->safe_push (copy);
6128 : }
6129 : }
6130 :
6131 138618439 : return copy;
6132 : }
6133 :
6134 : /* Create a new VAR_DECL that is identical in all respect to DECL except that
6135 : DECL can be either a VAR_DECL, a PARM_DECL or RESULT_DECL. The original
6136 : DECL must come from ID->src_fn and the copy will be part of ID->dst_fn. */
6137 :
6138 : tree
6139 8140715 : copy_decl_to_var (tree decl, copy_body_data *id)
6140 : {
6141 8140715 : tree copy, type;
6142 :
6143 8140715 : gcc_assert (TREE_CODE (decl) == PARM_DECL
6144 : || TREE_CODE (decl) == RESULT_DECL);
6145 :
6146 8140715 : type = TREE_TYPE (decl);
6147 :
6148 8140715 : copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
6149 8140715 : VAR_DECL, DECL_NAME (decl), type);
6150 8140715 : if (DECL_PT_UID_SET_P (decl))
6151 289 : SET_DECL_PT_UID (copy, DECL_PT_UID (decl));
6152 8140715 : TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
6153 8140715 : TREE_READONLY (copy) = TREE_READONLY (decl);
6154 8140715 : TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
6155 8140715 : DECL_NOT_GIMPLE_REG_P (copy) = DECL_NOT_GIMPLE_REG_P (decl);
6156 8140715 : DECL_BY_REFERENCE (copy) = DECL_BY_REFERENCE (decl);
6157 :
6158 8140715 : return copy_decl_for_dup_finish (id, decl, copy);
6159 : }
6160 :
6161 : /* Like copy_decl_to_var, but create a return slot object instead of a
6162 : pointer variable for return by invisible reference. */
6163 :
6164 : static tree
6165 2112245 : copy_result_decl_to_var (tree decl, copy_body_data *id)
6166 : {
6167 2112245 : tree copy, type;
6168 :
6169 2112245 : gcc_assert (TREE_CODE (decl) == PARM_DECL
6170 : || TREE_CODE (decl) == RESULT_DECL);
6171 :
6172 2112245 : type = TREE_TYPE (decl);
6173 2112245 : if (DECL_BY_REFERENCE (decl))
6174 141 : type = TREE_TYPE (type);
6175 :
6176 2112245 : copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
6177 2112245 : VAR_DECL, DECL_NAME (decl), type);
6178 2112245 : if (DECL_PT_UID_SET_P (decl))
6179 0 : SET_DECL_PT_UID (copy, DECL_PT_UID (decl));
6180 2112245 : TREE_READONLY (copy) = TREE_READONLY (decl);
6181 2112245 : TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
6182 2112245 : if (!DECL_BY_REFERENCE (decl))
6183 : {
6184 2112104 : TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
6185 2112104 : DECL_NOT_GIMPLE_REG_P (copy)
6186 4224208 : = (DECL_NOT_GIMPLE_REG_P (decl)
6187 : /* RESULT_DECLs are treated special by needs_to_live_in_memory,
6188 : mirror that to the created VAR_DECL. */
6189 2112104 : || (TREE_CODE (decl) == RESULT_DECL
6190 2111937 : && aggregate_value_p (decl, id->src_fn)));
6191 : }
6192 :
6193 2112245 : return copy_decl_for_dup_finish (id, decl, copy);
6194 : }
6195 :
6196 : tree
6197 128365236 : copy_decl_no_change (tree decl, copy_body_data *id)
6198 : {
6199 128365236 : tree copy;
6200 :
6201 128365236 : copy = copy_node (decl);
6202 :
6203 : /* The COPY is not abstract; it will be generated in DST_FN. */
6204 128365236 : DECL_ABSTRACT_P (copy) = false;
6205 128365236 : lang_hooks.dup_lang_specific_decl (copy);
6206 :
6207 : /* TREE_ADDRESSABLE isn't used to indicate that a label's address has
6208 : been taken; it's for internal bookkeeping in expand_goto_internal. */
6209 128365236 : if (TREE_CODE (copy) == LABEL_DECL)
6210 : {
6211 1695906 : TREE_ADDRESSABLE (copy) = 0;
6212 1695906 : LABEL_DECL_UID (copy) = -1;
6213 : }
6214 :
6215 128365236 : return copy_decl_for_dup_finish (id, decl, copy);
6216 : }
6217 :
6218 : static tree
6219 24577016 : copy_decl_maybe_to_var (tree decl, copy_body_data *id)
6220 : {
6221 24577016 : if (TREE_CODE (decl) == PARM_DECL || TREE_CODE (decl) == RESULT_DECL)
6222 14276 : return copy_decl_to_var (decl, id);
6223 : else
6224 24562740 : return copy_decl_no_change (decl, id);
6225 : }
6226 :
6227 : /* Return a copy of the function's argument tree without any modifications. */
6228 :
6229 : static tree
6230 80424 : copy_arguments_nochange (tree orig_parm, copy_body_data * id)
6231 : {
6232 80424 : tree arg, *parg;
6233 80424 : tree new_parm = NULL;
6234 :
6235 80424 : parg = &new_parm;
6236 242887 : for (arg = orig_parm; arg; arg = DECL_CHAIN (arg))
6237 : {
6238 162463 : tree new_tree = remap_decl (arg, id);
6239 162463 : if (TREE_CODE (new_tree) != PARM_DECL)
6240 3880 : new_tree = id->copy_decl (arg, id);
6241 162463 : lang_hooks.dup_lang_specific_decl (new_tree);
6242 162463 : *parg = new_tree;
6243 162463 : parg = &DECL_CHAIN (new_tree);
6244 : }
6245 80424 : return new_parm;
6246 : }
6247 :
6248 : /* Return a copy of the function's static chain. */
6249 : static tree
6250 984 : copy_static_chain (tree static_chain, copy_body_data * id)
6251 : {
6252 984 : tree *chain_copy, *pvar;
6253 :
6254 984 : chain_copy = &static_chain;
6255 1968 : for (pvar = chain_copy; *pvar; pvar = &DECL_CHAIN (*pvar))
6256 : {
6257 984 : tree new_tree = remap_decl (*pvar, id);
6258 984 : lang_hooks.dup_lang_specific_decl (new_tree);
6259 984 : DECL_CHAIN (new_tree) = DECL_CHAIN (*pvar);
6260 984 : *pvar = new_tree;
6261 : }
6262 984 : return static_chain;
6263 : }
6264 :
6265 : /* Return true if the function is allowed to be versioned.
6266 : This is a guard for the versioning functionality. */
6267 :
6268 : bool
6269 12887955 : tree_versionable_function_p (tree fndecl)
6270 : {
6271 12887955 : return (!lookup_attribute ("noclone", DECL_ATTRIBUTES (fndecl))
6272 25496258 : && copy_forbidden (DECL_STRUCT_FUNCTION (fndecl)) == NULL);
6273 : }
6274 :
6275 : /* Update clone info after duplication. */
6276 :
6277 : static void
6278 253174 : update_clone_info (copy_body_data * id)
6279 : {
6280 253174 : struct cgraph_node *this_node = id->dst_node;
6281 253174 : if (!this_node->clones)
6282 : return;
6283 644312 : for (cgraph_node *node = this_node->clones; node != this_node;)
6284 : {
6285 : /* First update replace maps to match the new body. */
6286 559304 : clone_info *info = clone_info::get (node);
6287 559304 : if (info && info->tree_map)
6288 : {
6289 : unsigned int i;
6290 0 : for (i = 0; i < vec_safe_length (info->tree_map); i++)
6291 : {
6292 0 : struct ipa_replace_map *replace_info;
6293 0 : replace_info = (*info->tree_map)[i];
6294 0 : walk_tree (&replace_info->new_tree, copy_tree_body_r, id, NULL);
6295 : }
6296 : }
6297 :
6298 559304 : if (node->clones)
6299 : node = node->clones;
6300 534311 : else if (node->next_sibling_clone)
6301 : node = node->next_sibling_clone;
6302 : else
6303 : {
6304 207550 : while (node != id->dst_node && !node->next_sibling_clone)
6305 110001 : node = node->clone_of;
6306 97549 : if (node != id->dst_node)
6307 12541 : node = node->next_sibling_clone;
6308 : }
6309 : }
6310 : }
6311 :
6312 : /* Create a copy of a function's tree.
6313 : OLD_DECL and NEW_DECL are FUNCTION_DECL tree nodes
6314 : of the original function and the new copied function
6315 : respectively. In case we want to replace a DECL
6316 : tree with another tree while duplicating the function's
6317 : body, TREE_MAP represents the mapping between these
6318 : trees. If UPDATE_CLONES is set, the call_stmt fields
6319 : of edges of clones of the function will be updated.
6320 :
6321 : If non-NULL PARAM_ADJUSTMENTS determines how function prototype (i.e. the
6322 : function parameters and return value) should be modified).
6323 : If non-NULL BLOCKS_TO_COPY determine what basic blocks to copy.
6324 : If non_NULL NEW_ENTRY determine new entry BB of the clone.
6325 : */
6326 : void
6327 253174 : tree_function_versioning (tree old_decl, tree new_decl,
6328 : vec<ipa_replace_map *, va_gc> *tree_map,
6329 : ipa_param_adjustments *param_adjustments,
6330 : bool update_clones, bitmap blocks_to_copy,
6331 : basic_block new_entry)
6332 : {
6333 253174 : struct cgraph_node *old_version_node;
6334 253174 : struct cgraph_node *new_version_node;
6335 253174 : copy_body_data id;
6336 253174 : tree p;
6337 253174 : unsigned i;
6338 253174 : struct ipa_replace_map *replace_info;
6339 253174 : basic_block old_entry_block, bb;
6340 253174 : auto_vec<gimple *, 10> init_stmts;
6341 253174 : tree vars = NULL_TREE;
6342 :
6343 : /* We can get called recursively from expand_call_inline via clone
6344 : materialization. While expand_call_inline maintains input_location
6345 : we cannot tolerate it to leak into the materialized clone. */
6346 253174 : location_t saved_location = input_location;
6347 253174 : input_location = UNKNOWN_LOCATION;
6348 :
6349 253174 : gcc_assert (TREE_CODE (old_decl) == FUNCTION_DECL
6350 : && TREE_CODE (new_decl) == FUNCTION_DECL);
6351 253174 : DECL_POSSIBLY_INLINED (old_decl) = 1;
6352 :
6353 253174 : old_version_node = cgraph_node::get (old_decl);
6354 253174 : gcc_checking_assert (old_version_node);
6355 253174 : new_version_node = cgraph_node::get (new_decl);
6356 253174 : gcc_checking_assert (new_version_node);
6357 :
6358 : /* Copy over debug args. */
6359 253174 : if (DECL_HAS_DEBUG_ARGS_P (old_decl))
6360 : {
6361 3961 : vec<tree, va_gc> **new_debug_args, **old_debug_args;
6362 3961 : gcc_checking_assert (decl_debug_args_lookup (new_decl) == NULL);
6363 3961 : DECL_HAS_DEBUG_ARGS_P (new_decl) = 0;
6364 3961 : old_debug_args = decl_debug_args_lookup (old_decl);
6365 3961 : if (old_debug_args)
6366 : {
6367 3961 : new_debug_args = decl_debug_args_insert (new_decl);
6368 7922 : *new_debug_args = vec_safe_copy (*old_debug_args);
6369 : }
6370 : }
6371 :
6372 : /* Output the inlining info for this abstract function, since it has been
6373 : inlined. If we don't do this now, we can lose the information about the
6374 : variables in the function when the blocks get blown away as soon as we
6375 : remove the cgraph node. */
6376 253174 : (*debug_hooks->outlining_inline_function) (old_decl);
6377 :
6378 253174 : DECL_ARTIFICIAL (new_decl) = 1;
6379 458109 : DECL_ABSTRACT_ORIGIN (new_decl) = DECL_ORIGIN (old_decl);
6380 458109 : if (DECL_ORIGIN (old_decl) == old_decl)
6381 224547 : old_version_node->used_as_abstract_origin = true;
6382 253174 : DECL_FUNCTION_PERSONALITY (new_decl) = DECL_FUNCTION_PERSONALITY (old_decl);
6383 :
6384 : /* Prepare the data structures for the tree copy. */
6385 253174 : memset (&id, 0, sizeof (id));
6386 :
6387 : /* Generate a new name for the new version. */
6388 253174 : id.statements_to_fold = new hash_set<gimple *>;
6389 :
6390 253174 : id.decl_map = new hash_map<tree, tree>;
6391 253174 : id.debug_map = NULL;
6392 253174 : id.src_fn = old_decl;
6393 253174 : id.dst_fn = new_decl;
6394 253174 : id.src_node = old_version_node;
6395 253174 : id.dst_node = new_version_node;
6396 253174 : id.src_cfun = DECL_STRUCT_FUNCTION (old_decl);
6397 253174 : id.blocks_to_copy = blocks_to_copy;
6398 :
6399 253174 : id.copy_decl = copy_decl_no_change;
6400 253174 : id.transform_call_graph_edges
6401 253174 : = update_clones ? CB_CGE_MOVE_CLONES : CB_CGE_MOVE;
6402 253174 : id.transform_new_cfg = true;
6403 253174 : id.transform_return_to_modify = false;
6404 253174 : id.transform_parameter = false;
6405 :
6406 253174 : old_entry_block = ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (old_decl));
6407 253174 : DECL_RESULT (new_decl) = DECL_RESULT (old_decl);
6408 253174 : DECL_ARGUMENTS (new_decl) = DECL_ARGUMENTS (old_decl);
6409 253174 : initialize_cfun (new_decl, old_decl,
6410 253174 : new_entry ? new_entry->count : old_entry_block->count);
6411 253174 : new_version_node->has_omp_variant_constructs
6412 253174 : = old_version_node->has_omp_variant_constructs;
6413 253174 : if (DECL_STRUCT_FUNCTION (new_decl)->gimple_df)
6414 253174 : DECL_STRUCT_FUNCTION (new_decl)->gimple_df->ipa_pta
6415 253174 : = id.src_cfun->gimple_df->ipa_pta;
6416 :
6417 : /* Copy the function's static chain. */
6418 253174 : p = DECL_STRUCT_FUNCTION (old_decl)->static_chain_decl;
6419 253174 : if (p)
6420 1968 : DECL_STRUCT_FUNCTION (new_decl)->static_chain_decl
6421 984 : = copy_static_chain (p, &id);
6422 :
6423 253174 : auto_vec<int, 16> new_param_indices;
6424 253174 : clone_info *info = clone_info::get (old_version_node);
6425 260075 : ipa_param_adjustments *old_param_adjustments
6426 253174 : = info ? info->param_adjustments : NULL;
6427 6901 : if (old_param_adjustments)
6428 6812 : old_param_adjustments->get_updated_indices (&new_param_indices);
6429 :
6430 : /* If there's a tree_map, prepare for substitution. */
6431 253174 : if (tree_map)
6432 39394 : for (i = 0; i < tree_map->length (); i++)
6433 : {
6434 24913 : gimple *init;
6435 24913 : replace_info = (*tree_map)[i];
6436 :
6437 24913 : int p = replace_info->parm_num;
6438 24913 : if (old_param_adjustments)
6439 0 : p = new_param_indices[p];
6440 :
6441 24913 : tree parm;
6442 71801 : for (parm = DECL_ARGUMENTS (old_decl); p;
6443 46888 : parm = DECL_CHAIN (parm))
6444 46888 : p--;
6445 24913 : gcc_assert (parm);
6446 24913 : init = setup_one_parameter (&id, parm, replace_info->new_tree,
6447 : id.src_fn, NULL, &vars);
6448 24913 : if (init)
6449 7310 : init_stmts.safe_push (init);
6450 : }
6451 :
6452 253174 : ipa_param_body_adjustments *param_body_adjs = NULL;
6453 253174 : if (param_adjustments)
6454 : {
6455 333312 : param_body_adjs = new ipa_param_body_adjustments (param_adjustments,
6456 : new_decl, old_decl,
6457 166656 : &id, &vars, tree_map);
6458 166656 : id.param_body_adjs = param_body_adjs;
6459 166656 : DECL_ARGUMENTS (new_decl) = param_body_adjs->get_new_param_chain ();
6460 : }
6461 86518 : else if (DECL_ARGUMENTS (old_decl) != NULL_TREE)
6462 160848 : DECL_ARGUMENTS (new_decl)
6463 80424 : = copy_arguments_nochange (DECL_ARGUMENTS (old_decl), &id);
6464 :
6465 253174 : DECL_INITIAL (new_decl) = remap_blocks (DECL_INITIAL (id.src_fn), &id);
6466 253174 : BLOCK_SUPERCONTEXT (DECL_INITIAL (new_decl)) = new_decl;
6467 :
6468 253174 : declare_inline_vars (DECL_INITIAL (new_decl), vars);
6469 :
6470 253174 : if (!vec_safe_is_empty (DECL_STRUCT_FUNCTION (old_decl)->local_decls))
6471 : /* Add local vars. */
6472 151868 : add_local_variables (DECL_STRUCT_FUNCTION (old_decl), cfun, &id);
6473 :
6474 253174 : if (DECL_RESULT (old_decl) == NULL_TREE)
6475 : ;
6476 166656 : else if (param_adjustments && param_adjustments->m_skip_return
6477 306099 : && !VOID_TYPE_P (TREE_TYPE (DECL_RESULT (old_decl))))
6478 : {
6479 45501 : tree resdecl_repl = copy_result_decl_to_var (DECL_RESULT (old_decl),
6480 : &id);
6481 45501 : declare_inline_vars (NULL, resdecl_repl);
6482 45501 : if (DECL_BY_REFERENCE (DECL_RESULT (old_decl)))
6483 141 : resdecl_repl = build_fold_addr_expr (resdecl_repl);
6484 45501 : insert_decl_map (&id, DECL_RESULT (old_decl), resdecl_repl);
6485 :
6486 91002 : DECL_RESULT (new_decl)
6487 45501 : = build_decl (DECL_SOURCE_LOCATION (DECL_RESULT (old_decl)),
6488 : RESULT_DECL, NULL_TREE, void_type_node);
6489 45501 : DECL_CONTEXT (DECL_RESULT (new_decl)) = new_decl;
6490 45501 : DECL_IS_MALLOC (new_decl) = false;
6491 45501 : cfun->returns_struct = 0;
6492 45501 : cfun->returns_pcc_struct = 0;
6493 : }
6494 : else
6495 : {
6496 207673 : tree old_name;
6497 207673 : DECL_RESULT (new_decl) = remap_decl (DECL_RESULT (old_decl), &id);
6498 207673 : lang_hooks.dup_lang_specific_decl (DECL_RESULT (new_decl));
6499 415346 : if (gimple_in_ssa_p (id.src_cfun)
6500 207673 : && DECL_BY_REFERENCE (DECL_RESULT (old_decl))
6501 4884 : && (old_name = ssa_default_def (id.src_cfun, DECL_RESULT (old_decl))))
6502 : {
6503 4865 : tree new_name = make_ssa_name (DECL_RESULT (new_decl));
6504 4865 : insert_decl_map (&id, old_name, new_name);
6505 4865 : SSA_NAME_DEF_STMT (new_name) = gimple_build_nop ();
6506 4865 : set_ssa_default_def (cfun, DECL_RESULT (new_decl), new_name);
6507 : }
6508 : }
6509 :
6510 : /* Set up the destination functions loop tree. */
6511 253174 : if (loops_for_fn (DECL_STRUCT_FUNCTION (old_decl)) != NULL)
6512 : {
6513 253174 : cfun->curr_properties &= ~PROP_loops;
6514 253174 : loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
6515 253174 : cfun->curr_properties |= PROP_loops;
6516 : }
6517 :
6518 : /* Copy the Function's body. */
6519 253174 : copy_body (&id, ENTRY_BLOCK_PTR_FOR_FN (cfun), EXIT_BLOCK_PTR_FOR_FN (cfun),
6520 : new_entry);
6521 :
6522 : /* Renumber the lexical scoping (non-code) blocks consecutively. */
6523 253174 : number_blocks (new_decl);
6524 :
6525 : /* We want to create the BB unconditionally, so that the addition of
6526 : debug stmts doesn't affect BB count, which may in the end cause
6527 : codegen differences. */
6528 253174 : bb = split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
6529 513658 : while (init_stmts.length ())
6530 7310 : insert_init_stmt (&id, bb, init_stmts.pop ());
6531 253174 : if (param_body_adjs)
6532 166656 : param_body_adjs->append_init_stmts (bb);
6533 253174 : update_clone_info (&id);
6534 :
6535 : /* Remap the nonlocal_goto_save_area, if any. */
6536 253174 : if (cfun->nonlocal_goto_save_area)
6537 : {
6538 0 : struct walk_stmt_info wi;
6539 :
6540 0 : memset (&wi, 0, sizeof (wi));
6541 0 : wi.info = &id;
6542 0 : walk_tree (&cfun->nonlocal_goto_save_area, remap_gimple_op_r, &wi, NULL);
6543 : }
6544 :
6545 : /* Clean up. */
6546 506348 : delete id.decl_map;
6547 253174 : if (id.debug_map)
6548 1248 : delete id.debug_map;
6549 253174 : free_dominance_info (CDI_DOMINATORS);
6550 253174 : free_dominance_info (CDI_POST_DOMINATORS);
6551 :
6552 253174 : update_max_bb_count ();
6553 253174 : fold_marked_statements (0, id.statements_to_fold);
6554 506348 : delete id.statements_to_fold;
6555 253174 : delete_unreachable_blocks_update_callgraph (id.dst_node, update_clones);
6556 253174 : if (id.dst_node->definition)
6557 248667 : cgraph_edge::rebuild_references ();
6558 253174 : if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
6559 : {
6560 253174 : calculate_dominance_info (CDI_DOMINATORS);
6561 253174 : fix_loop_structure (NULL);
6562 : }
6563 253174 : update_ssa (TODO_update_ssa);
6564 :
6565 : /* After partial cloning we need to rescale frequencies, so they are
6566 : within proper range in the cloned function. */
6567 253174 : if (new_entry)
6568 : {
6569 52605 : struct cgraph_edge *e;
6570 52605 : rebuild_frequencies ();
6571 :
6572 52605 : new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
6573 236967 : for (e = new_version_node->callees; e; e = e->next_callee)
6574 : {
6575 184362 : basic_block bb = gimple_bb (e->call_stmt);
6576 184362 : e->count = bb->count;
6577 : }
6578 57615 : for (e = new_version_node->indirect_calls; e; e = e->next_callee)
6579 : {
6580 5010 : basic_block bb = gimple_bb (e->call_stmt);
6581 5010 : e->count = bb->count;
6582 : }
6583 : }
6584 :
6585 253174 : if (param_body_adjs && MAY_HAVE_DEBUG_BIND_STMTS)
6586 : {
6587 142690 : vec<tree, va_gc> **debug_args = NULL;
6588 142690 : unsigned int len = 0;
6589 142690 : unsigned reset_len = param_body_adjs->m_reset_debug_decls.length ();
6590 :
6591 254869 : for (i = 0; i < reset_len; i++)
6592 : {
6593 112179 : tree parm = param_body_adjs->m_reset_debug_decls[i];
6594 112179 : gcc_assert (is_gimple_reg (parm));
6595 112179 : tree ddecl;
6596 :
6597 112179 : if (debug_args == NULL)
6598 : {
6599 85236 : debug_args = decl_debug_args_insert (new_decl);
6600 85236 : len = vec_safe_length (*debug_args);
6601 : }
6602 112179 : ddecl = build_debug_expr_decl (TREE_TYPE (parm));
6603 : /* FIXME: Is setting the mode really necessary? */
6604 112179 : SET_DECL_MODE (ddecl, DECL_MODE (parm));
6605 112179 : vec_safe_push (*debug_args, DECL_ORIGIN (parm));
6606 112179 : vec_safe_push (*debug_args, ddecl);
6607 : }
6608 142690 : if (debug_args != NULL)
6609 : {
6610 : /* On the callee side, add
6611 : DEBUG D#Y s=> parm
6612 : DEBUG var => D#Y
6613 : stmts to the first bb where var is a VAR_DECL created for the
6614 : optimized away parameter in DECL_INITIAL block. This hints
6615 : in the debug info that var (whole DECL_ORIGIN is the parm
6616 : PARM_DECL) is optimized away, but could be looked up at the
6617 : call site as value of D#X there. */
6618 85236 : gimple_stmt_iterator cgsi
6619 85236 : = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
6620 85236 : gimple *def_temp;
6621 85236 : tree var = vars;
6622 170472 : i = vec_safe_length (*debug_args);
6623 112179 : do
6624 : {
6625 112179 : tree vexpr = NULL_TREE;
6626 112179 : i -= 2;
6627 112179 : while (var != NULL_TREE
6628 148442 : && DECL_ABSTRACT_ORIGIN (var) != (**debug_args)[i])
6629 36263 : var = TREE_CHAIN (var);
6630 112179 : if (var == NULL_TREE)
6631 : break;
6632 112179 : tree parm = (**debug_args)[i];
6633 112179 : if (tree parm_ddef = ssa_default_def (id.src_cfun, parm))
6634 149910 : if (tree *d
6635 74955 : = param_body_adjs->m_dead_ssa_debug_equiv.get (parm_ddef))
6636 30908 : vexpr = *d;
6637 112179 : if (!vexpr)
6638 : {
6639 81271 : vexpr = build_debug_expr_decl (TREE_TYPE (parm));
6640 : /* FIXME: Is setting the mode really necessary? */
6641 81271 : SET_DECL_MODE (vexpr, DECL_MODE (parm));
6642 : }
6643 112179 : def_temp = gimple_build_debug_bind (var, vexpr, NULL);
6644 112179 : gsi_insert_before (&cgsi, def_temp, GSI_NEW_STMT);
6645 112179 : def_temp = gimple_build_debug_source_bind (vexpr, parm, NULL);
6646 112179 : gsi_insert_before (&cgsi, def_temp, GSI_NEW_STMT);
6647 : }
6648 112179 : while (i > len);
6649 : }
6650 : }
6651 166656 : delete param_body_adjs;
6652 253174 : free_dominance_info (CDI_DOMINATORS);
6653 253174 : free_dominance_info (CDI_POST_DOMINATORS);
6654 :
6655 253174 : gcc_assert (!id.debug_stmts.exists ());
6656 253174 : pop_cfun ();
6657 253174 : input_location = saved_location;
6658 253174 : return;
6659 253174 : }
6660 :
6661 : /* EXP is CALL_EXPR present in a GENERIC expression tree. Try to integrate
6662 : the callee and return the inlined body on success. */
6663 :
6664 : tree
6665 0 : maybe_inline_call_in_expr (tree exp)
6666 : {
6667 0 : tree fn = get_callee_fndecl (exp);
6668 :
6669 : /* We can only try to inline "const" functions. */
6670 0 : if (fn && TREE_READONLY (fn) && DECL_SAVED_TREE (fn))
6671 : {
6672 0 : call_expr_arg_iterator iter;
6673 0 : copy_body_data id;
6674 0 : tree param, arg, t;
6675 0 : hash_map<tree, tree> decl_map;
6676 :
6677 : /* Remap the parameters. */
6678 0 : for (param = DECL_ARGUMENTS (fn), arg = first_call_expr_arg (exp, &iter);
6679 0 : param;
6680 0 : param = DECL_CHAIN (param), arg = next_call_expr_arg (&iter))
6681 0 : decl_map.put (param, arg);
6682 :
6683 0 : memset (&id, 0, sizeof (id));
6684 0 : id.src_fn = fn;
6685 0 : id.dst_fn = current_function_decl;
6686 0 : id.src_cfun = DECL_STRUCT_FUNCTION (fn);
6687 0 : id.decl_map = &decl_map;
6688 :
6689 0 : id.copy_decl = copy_decl_no_change;
6690 0 : id.transform_call_graph_edges = CB_CGE_DUPLICATE;
6691 0 : id.transform_new_cfg = false;
6692 0 : id.transform_return_to_modify = true;
6693 0 : id.transform_parameter = true;
6694 :
6695 : /* Make sure not to unshare trees behind the front-end's back
6696 : since front-end specific mechanisms may rely on sharing. */
6697 0 : id.regimplify = false;
6698 0 : id.do_not_unshare = true;
6699 :
6700 : /* We're not inside any EH region. */
6701 0 : id.eh_lp_nr = 0;
6702 :
6703 0 : t = copy_tree_body (&id);
6704 :
6705 : /* We can only return something suitable for use in a GENERIC
6706 : expression tree. */
6707 0 : if (TREE_CODE (t) == MODIFY_EXPR)
6708 0 : return TREE_OPERAND (t, 1);
6709 0 : }
6710 :
6711 : return NULL_TREE;
6712 : }
6713 :
6714 : /* Duplicate a type, fields and all. */
6715 :
6716 : tree
6717 63 : build_duplicate_type (tree type)
6718 : {
6719 63 : struct copy_body_data id;
6720 :
6721 63 : memset (&id, 0, sizeof (id));
6722 63 : id.src_fn = current_function_decl;
6723 63 : id.dst_fn = current_function_decl;
6724 63 : id.src_cfun = cfun;
6725 63 : id.decl_map = new hash_map<tree, tree>;
6726 63 : id.debug_map = NULL;
6727 63 : id.copy_decl = copy_decl_no_change;
6728 :
6729 63 : type = remap_type_1 (type, &id);
6730 :
6731 126 : delete id.decl_map;
6732 63 : if (id.debug_map)
6733 0 : delete id.debug_map;
6734 :
6735 63 : TYPE_CANONICAL (type) = type;
6736 :
6737 63 : return type;
6738 : }
6739 :
6740 : /* Unshare the entire DECL_SAVED_TREE of FN and return the remapped
6741 : parameters and RESULT_DECL in PARMS and RESULT. Used by C++ constexpr
6742 : evaluation. */
6743 :
6744 : tree
6745 31020400 : copy_fn (tree fn, tree& parms, tree& result)
6746 : {
6747 31020400 : copy_body_data id;
6748 31020400 : tree param;
6749 31020400 : hash_map<tree, tree> decl_map;
6750 :
6751 31020400 : tree *p = &parms;
6752 31020400 : *p = NULL_TREE;
6753 :
6754 31020400 : memset (&id, 0, sizeof (id));
6755 31020400 : id.src_fn = fn;
6756 31020400 : id.dst_fn = current_function_decl;
6757 31020400 : id.src_cfun = DECL_STRUCT_FUNCTION (fn);
6758 31020400 : id.decl_map = &decl_map;
6759 :
6760 130407729 : id.copy_decl = [] (tree decl, copy_body_data *id)
6761 : {
6762 99387329 : if (TREE_CODE (decl) == TYPE_DECL || TREE_CODE (decl) == CONST_DECL)
6763 : /* Don't make copies of local types or injected enumerators,
6764 : the C++ constexpr evaluator doesn't need them and they
6765 : confuse modules streaming. */
6766 : return decl;
6767 97247238 : return copy_decl_no_change (decl, id);
6768 : };
6769 31020400 : id.transform_call_graph_edges = CB_CGE_DUPLICATE;
6770 31020400 : id.transform_new_cfg = false;
6771 31020400 : id.transform_return_to_modify = false;
6772 31020400 : id.transform_parameter = true;
6773 :
6774 : /* Make sure not to unshare trees behind the front-end's back
6775 : since front-end specific mechanisms may rely on sharing. */
6776 31020400 : id.regimplify = false;
6777 31020400 : id.do_not_unshare = true;
6778 31020400 : id.do_not_fold = true;
6779 :
6780 : /* We're not inside any EH region. */
6781 31020400 : id.eh_lp_nr = 0;
6782 :
6783 : /* Remap the parameters and result and return them to the caller. */
6784 31020400 : for (param = DECL_ARGUMENTS (fn);
6785 74725434 : param;
6786 43705034 : param = DECL_CHAIN (param))
6787 : {
6788 43705034 : *p = remap_decl (param, &id);
6789 43705034 : p = &DECL_CHAIN (*p);
6790 : }
6791 :
6792 31020400 : if (DECL_RESULT (fn))
6793 31020400 : result = remap_decl (DECL_RESULT (fn), &id);
6794 : else
6795 : result = NULL_TREE;
6796 :
6797 31020400 : return copy_tree_body (&id);
6798 31020400 : }
|