Line data Source code
1 : /* Function splitting pass
2 : Copyright (C) 2010-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka <jh@suse.cz>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* The purpose of this pass is to split function bodies to improve
22 : inlining. I.e. for function of the form:
23 :
24 : func (...)
25 : {
26 : if (cheap_test)
27 : something_small
28 : else
29 : something_big
30 : }
31 :
32 : Produce:
33 :
34 : func.part (...)
35 : {
36 : something_big
37 : }
38 :
39 : func (...)
40 : {
41 : if (cheap_test)
42 : something_small
43 : else
44 : func.part (...);
45 : }
46 :
47 : When func becomes inlinable and when cheap_test is often true, inlining func,
48 : but not fund.part leads to performance improvement similar as inlining
49 : original func while the code size growth is smaller.
50 :
51 : The pass is organized in three stages:
52 : 1) Collect local info about basic block into BB_INFO structure and
53 : compute function body estimated size and time.
54 : 2) Via DFS walk find all possible basic blocks where we can split
55 : and chose best one.
56 : 3) If split point is found, split at the specified BB by creating a clone
57 : and updating function to call it.
58 :
59 : The decisions what functions to split are in execute_split_functions
60 : and consider_split.
61 :
62 : There are several possible future improvements for this pass including:
63 :
64 : 1) Splitting to break up large functions
65 : 2) Splitting to reduce stack frame usage
66 : 3) Allow split part of function to use values computed in the header part.
67 : The values needs to be passed to split function, perhaps via same
68 : interface as for nested functions or as argument.
69 : 4) Support for simple rematerialization. I.e. when split part use
70 : value computed in header from function parameter in very cheap way, we
71 : can just recompute it.
72 : 5) Support splitting of nested functions.
73 : 6) Support non-SSA arguments.
74 : 7) There is nothing preventing us from producing multiple parts of single function
75 : when needed or splitting also the parts. */
76 :
77 : #include "config.h"
78 : #include "system.h"
79 : #include "coretypes.h"
80 : #include "backend.h"
81 : #include "rtl.h"
82 : #include "tree.h"
83 : #include "gimple.h"
84 : #include "cfghooks.h"
85 : #include "alloc-pool.h"
86 : #include "tree-pass.h"
87 : #include "ssa.h"
88 : #include "cgraph.h"
89 : #include "diagnostic.h"
90 : #include "fold-const.h"
91 : #include "cfganal.h"
92 : #include "calls.h"
93 : #include "gimplify.h"
94 : #include "gimple-iterator.h"
95 : #include "gimplify-me.h"
96 : #include "gimple-walk.h"
97 : #include "symbol-summary.h"
98 : #include "sreal.h"
99 : #include "ipa-cp.h"
100 : #include "ipa-prop.h"
101 : #include "tree-cfg.h"
102 : #include "tree-into-ssa.h"
103 : #include "tree-dfa.h"
104 : #include "tree-inline.h"
105 : #include "gimple-pretty-print.h"
106 : #include "ipa-fnsummary.h"
107 : #include "cfgloop.h"
108 : #include "attribs.h"
109 : #include "ipa-strub.h"
110 :
111 : /* Per basic block info. */
112 :
113 8272270 : class split_bb_info
114 : {
115 : public:
116 : unsigned int size;
117 : sreal time;
118 : };
119 :
120 : static vec<split_bb_info> bb_info_vec;
121 :
122 : /* Description of split point. */
123 :
124 1235789 : class split_point
125 : {
126 : public:
127 : /* Size of the partitions. */
128 : sreal header_time, split_time;
129 : unsigned int header_size, split_size;
130 :
131 : /* SSA names that need to be passed into spit function. */
132 : bitmap ssa_names_to_pass;
133 :
134 : /* Basic block where we split (that will become entry point of new function. */
135 : basic_block entry_bb;
136 :
137 : /* Count for entering the split part.
138 : This is not count of the entry_bb because it may be in loop. */
139 : profile_count count;
140 :
141 : /* Basic blocks we are splitting away. */
142 : bitmap split_bbs;
143 :
144 : /* True when return value is computed on split part and thus it needs
145 : to be returned. */
146 : bool split_part_set_retval;
147 : };
148 :
149 : /* Best split point found. */
150 :
151 : class split_point best_split_point;
152 :
153 : /* Set of basic blocks that are not allowed to dominate a split point. */
154 :
155 : static bitmap forbidden_dominators;
156 :
157 : static tree find_retval (basic_block return_bb);
158 :
159 : /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
160 : variable, check it if it is present in bitmap passed via DATA. */
161 :
162 : static bool
163 28563 : test_nonssa_use (gimple *, tree t, tree, void *data)
164 : {
165 28563 : t = get_base_address (t);
166 :
167 28563 : if (!t || is_gimple_reg (t))
168 0 : return false;
169 :
170 28563 : if (TREE_CODE (t) == PARM_DECL
171 28497 : || (VAR_P (t)
172 5397 : && auto_var_in_fn_p (t, current_function_decl))
173 23947 : || TREE_CODE (t) == RESULT_DECL
174 : /* Normal labels are part of CFG and will be handled gratefully.
175 : Forced labels however can be used directly by statements and
176 : need to stay in one partition along with their uses. */
177 51952 : || (TREE_CODE (t) == LABEL_DECL
178 6005 : && FORCED_LABEL (t)))
179 5177 : return bitmap_bit_p ((bitmap)data, DECL_UID (t));
180 :
181 : /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
182 : to pretend that the value pointed to is actual result decl. */
183 8341 : if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t))
184 15045 : && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME
185 15041 : && SSA_NAME_VAR (TREE_OPERAND (t, 0))
186 11610 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL
187 23386 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
188 0 : return
189 0 : bitmap_bit_p ((bitmap)data,
190 0 : DECL_UID (DECL_RESULT (current_function_decl)));
191 :
192 : return false;
193 : }
194 :
195 : /* Dump split point CURRENT. */
196 :
197 : static void
198 13 : dump_split_point (FILE * file, class split_point *current)
199 : {
200 13 : fprintf (file,
201 : "Split point at BB %i\n"
202 : " header time: %f header size: %i\n"
203 : " split time: %f split size: %i\n bbs: ",
204 13 : current->entry_bb->index, current->header_time.to_double (),
205 : current->header_size, current->split_time.to_double (),
206 : current->split_size);
207 13 : dump_bitmap (file, current->split_bbs);
208 13 : fprintf (file, " SSA names to pass: ");
209 13 : dump_bitmap (file, current->ssa_names_to_pass);
210 13 : }
211 :
212 : /* Look for all BBs in header that might lead to the split part and verify
213 : that they are not defining any non-SSA var used by the split part.
214 : Parameters are the same as for consider_split. */
215 :
216 : static bool
217 12564 : verify_non_ssa_vars (class split_point *current, bitmap non_ssa_vars,
218 : basic_block return_bb)
219 : {
220 12564 : bitmap seen = BITMAP_ALLOC (NULL);
221 12564 : vec<basic_block> worklist = vNULL;
222 12564 : edge e;
223 12564 : edge_iterator ei;
224 12564 : bool ok = true;
225 12564 : basic_block bb;
226 :
227 26886 : FOR_EACH_EDGE (e, ei, current->entry_bb->preds)
228 14322 : if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)
229 14322 : && !bitmap_bit_p (current->split_bbs, e->src->index))
230 : {
231 13807 : worklist.safe_push (e->src);
232 13807 : bitmap_set_bit (seen, e->src->index);
233 : }
234 :
235 29988 : while (!worklist.is_empty ())
236 : {
237 20219 : bb = worklist.pop ();
238 42222 : FOR_EACH_EDGE (e, ei, bb->preds)
239 22003 : if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)
240 22003 : && bitmap_set_bit (seen, e->src->index))
241 : {
242 8410 : gcc_checking_assert (!bitmap_bit_p (current->split_bbs,
243 : e->src->index));
244 8410 : worklist.safe_push (e->src);
245 : }
246 337307 : for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
247 296869 : gsi_next (&bsi))
248 : {
249 299664 : gimple *stmt = gsi_stmt (bsi);
250 299664 : if (is_gimple_debug (stmt))
251 251402 : continue;
252 48262 : if (walk_stmt_load_store_addr_ops
253 48262 : (stmt, non_ssa_vars, test_nonssa_use, test_nonssa_use,
254 : test_nonssa_use))
255 : {
256 2795 : ok = false;
257 2795 : goto done;
258 : }
259 296955 : if (glabel *label_stmt = dyn_cast <glabel *> (stmt))
260 86 : if (test_nonssa_use (stmt, gimple_label_label (label_stmt),
261 : NULL_TREE, non_ssa_vars))
262 : {
263 0 : ok = false;
264 0 : goto done;
265 : }
266 : }
267 18388 : for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
268 964 : gsi_next (&bsi))
269 : {
270 964 : if (walk_stmt_load_store_addr_ops
271 964 : (gsi_stmt (bsi), non_ssa_vars, test_nonssa_use, test_nonssa_use,
272 : test_nonssa_use))
273 : {
274 0 : ok = false;
275 0 : goto done;
276 : }
277 : }
278 53016 : FOR_EACH_EDGE (e, ei, bb->succs)
279 : {
280 35592 : if (e->dest != return_bb)
281 33099 : continue;
282 2493 : for (gphi_iterator bsi = gsi_start_phis (return_bb);
283 5693 : !gsi_end_p (bsi);
284 3200 : gsi_next (&bsi))
285 : {
286 3200 : gphi *stmt = bsi.phi ();
287 3200 : tree op = gimple_phi_arg_def (stmt, e->dest_idx);
288 :
289 6400 : if (virtual_operand_p (gimple_phi_result (stmt)))
290 2379 : continue;
291 821 : if (TREE_CODE (op) != SSA_NAME
292 821 : && test_nonssa_use (stmt, op, op, non_ssa_vars))
293 : {
294 0 : ok = false;
295 0 : goto done;
296 : }
297 : }
298 : }
299 : }
300 :
301 : /* Verify that the rest of function does not define any label
302 : used by the split part. */
303 104552 : FOR_EACH_BB_FN (bb, cfun)
304 94786 : if (!bitmap_bit_p (current->split_bbs, bb->index)
305 94786 : && !bitmap_bit_p (seen, bb->index))
306 : {
307 33282 : gimple_stmt_iterator bsi;
308 72480 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
309 100702 : if (glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (bsi)))
310 : {
311 5919 : if (test_nonssa_use (label_stmt,
312 : gimple_label_label (label_stmt),
313 : NULL_TREE, non_ssa_vars))
314 : {
315 3 : ok = false;
316 3 : goto done;
317 : }
318 : }
319 : else
320 : break;
321 : }
322 :
323 9766 : done:
324 12564 : BITMAP_FREE (seen);
325 12564 : worklist.release ();
326 12564 : return ok;
327 : }
328 :
329 : /* If STMT is a call, check the callee against a list of forbidden
330 : predicate functions. If a match is found, look for uses of the
331 : call result in condition statements that compare against zero.
332 : For each such use, find the block targeted by the condition
333 : statement for the nonzero result, and set the bit for this block
334 : in the forbidden dominators bitmap. The purpose of this is to avoid
335 : selecting a split point where we are likely to lose the chance
336 : to optimize away an unused function call. */
337 :
338 : static void
339 37885451 : check_forbidden_calls (gimple *stmt)
340 : {
341 37885451 : imm_use_iterator use_iter;
342 37885451 : use_operand_p use_p;
343 37885451 : tree lhs;
344 :
345 : /* At the moment, __builtin_constant_p is the only forbidden
346 : predicate function call (see PR49642). */
347 37885451 : if (!gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P))
348 37875361 : return;
349 :
350 10090 : lhs = gimple_call_lhs (stmt);
351 :
352 10090 : if (!lhs || TREE_CODE (lhs) != SSA_NAME)
353 : return;
354 :
355 32521 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, lhs)
356 : {
357 12341 : tree op1;
358 12341 : basic_block use_bb, forbidden_bb;
359 12341 : enum tree_code code;
360 12341 : edge true_edge, false_edge;
361 12341 : gcond *use_stmt;
362 :
363 12341 : use_stmt = dyn_cast <gcond *> (USE_STMT (use_p));
364 12341 : if (!use_stmt)
365 29 : continue;
366 :
367 : /* Assuming canonical form for GIMPLE_COND here, with constant
368 : in second position. */
369 12312 : op1 = gimple_cond_rhs (use_stmt);
370 12312 : code = gimple_cond_code (use_stmt);
371 12312 : use_bb = gimple_bb (use_stmt);
372 :
373 12312 : extract_true_false_edges_from_block (use_bb, &true_edge, &false_edge);
374 :
375 : /* We're only interested in comparisons that distinguish
376 : unambiguously from zero. */
377 12312 : if (!integer_zerop (op1) || code == LE_EXPR || code == GE_EXPR)
378 0 : continue;
379 :
380 12312 : if (code == EQ_EXPR)
381 1935 : forbidden_bb = false_edge->dest;
382 : else
383 10377 : forbidden_bb = true_edge->dest;
384 :
385 12312 : bitmap_set_bit (forbidden_dominators, forbidden_bb->index);
386 10090 : }
387 : }
388 :
389 : /* If BB is dominated by any block in the forbidden dominators set,
390 : return TRUE; else FALSE. */
391 :
392 : static bool
393 1294 : dominated_by_forbidden (basic_block bb)
394 : {
395 1294 : unsigned dom_bb;
396 1294 : bitmap_iterator bi;
397 :
398 3896 : EXECUTE_IF_SET_IN_BITMAP (forbidden_dominators, 1, dom_bb, bi)
399 : {
400 2617 : if (dominated_by_p (CDI_DOMINATORS, bb,
401 2617 : BASIC_BLOCK_FOR_FN (cfun, dom_bb)))
402 : return true;
403 : }
404 :
405 : return false;
406 : }
407 :
408 : /* For give split point CURRENT and return block RETURN_BB return 1
409 : if ssa name VAL is set by split part and 0 otherwise. */
410 : static bool
411 19377 : split_part_set_ssa_name_p (tree val, class split_point *current,
412 : basic_block return_bb)
413 : {
414 19377 : if (TREE_CODE (val) != SSA_NAME)
415 : return false;
416 :
417 19377 : return (!SSA_NAME_IS_DEFAULT_DEF (val)
418 19377 : && (bitmap_bit_p (current->split_bbs,
419 16039 : gimple_bb (SSA_NAME_DEF_STMT (val))->index)
420 16037 : || gimple_bb (SSA_NAME_DEF_STMT (val)) == return_bb));
421 : }
422 :
423 : /* We found an split_point CURRENT. NON_SSA_VARS is bitmap of all non ssa
424 : variables used and RETURN_BB is return basic block.
425 : See if we can split function here. */
426 :
427 : static void
428 2519223 : consider_split (class split_point *current, bitmap non_ssa_vars,
429 : basic_block return_bb)
430 : {
431 2519223 : tree parm;
432 2519223 : unsigned int num_args = 0;
433 2519223 : unsigned int call_overhead;
434 2519223 : edge e;
435 2519223 : edge_iterator ei;
436 2519223 : gphi_iterator bsi;
437 2519223 : unsigned int i;
438 2519223 : tree retval;
439 2519223 : bool back_edge = false;
440 :
441 2519223 : if (dump_file && (dump_flags & TDF_DETAILS))
442 6 : dump_split_point (dump_file, current);
443 :
444 2519223 : current->count = profile_count::zero ();
445 5481802 : FOR_EACH_EDGE (e, ei, current->entry_bb->preds)
446 : {
447 2962579 : if (e->flags & EDGE_DFS_BACK)
448 143410 : back_edge = true;
449 2962579 : if (!bitmap_bit_p (current->split_bbs, e->src->index))
450 2819168 : current->count += e->count ();
451 : }
452 :
453 : /* Do not split when we would end up calling function anyway.
454 : Compares are three state, use !(...<...) to also give up when outcome
455 : is unknown. */
456 5038446 : if (!(current->count
457 2519223 : < (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.apply_scale
458 5038446 : (param_partial_inlining_entry_probability, 100))))
459 : {
460 : /* When profile is guessed, we cannot expect it to give us
461 : realistic estimate on likeliness of function taking the
462 : complex path. As a special case, when tail of the function is
463 : a loop, enable splitting since inlining code skipping the loop
464 : is likely noticeable win. */
465 1405742 : if (back_edge
466 83175 : && profile_status_for_fn (cfun) != PROFILE_READ
467 1488880 : && current->count
468 83138 : < ENTRY_BLOCK_PTR_FOR_FN (cfun)->count)
469 : {
470 14357 : if (dump_file && (dump_flags & TDF_DETAILS))
471 : {
472 0 : fprintf (dump_file,
473 : " Split before loop, accepting despite low counts");
474 0 : current->count.dump (dump_file);
475 0 : fprintf (dump_file, " ");
476 0 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.dump (dump_file);
477 : }
478 : }
479 : else
480 : {
481 1391385 : if (dump_file && (dump_flags & TDF_DETAILS))
482 3 : fprintf (dump_file,
483 : " Refused: incoming frequency is too large.\n");
484 2456715 : return;
485 : }
486 : }
487 :
488 1127838 : if (!current->header_size)
489 : {
490 0 : if (dump_file && (dump_flags & TDF_DETAILS))
491 0 : fprintf (dump_file, " Refused: header empty\n");
492 0 : return;
493 : }
494 :
495 : /* Verify that PHI args on entry are either virtual or all their operands
496 : incoming from header are the same. */
497 1304527 : for (bsi = gsi_start_phis (current->entry_bb); !gsi_end_p (bsi); gsi_next (&bsi))
498 : {
499 192293 : gphi *stmt = bsi.phi ();
500 192293 : tree val = NULL;
501 :
502 384586 : if (virtual_operand_p (gimple_phi_result (stmt)))
503 48312 : continue;
504 416420 : for (i = 0; i < gimple_phi_num_args (stmt); i++)
505 : {
506 288043 : edge e = gimple_phi_arg_edge (stmt, i);
507 288043 : if (!bitmap_bit_p (current->split_bbs, e->src->index))
508 : {
509 159808 : tree edge_val = gimple_phi_arg_def (stmt, i);
510 159808 : if (val && edge_val != val)
511 : {
512 15604 : if (dump_file && (dump_flags & TDF_DETAILS))
513 0 : fprintf (dump_file,
514 : " Refused: entry BB has PHI with multiple variants\n");
515 15604 : return;
516 : }
517 : val = edge_val;
518 : }
519 : }
520 : }
521 :
522 :
523 : /* See what argument we will pass to the split function and compute
524 : call overhead. */
525 1112234 : call_overhead = eni_size_weights.call_cost;
526 3253321 : for (parm = DECL_ARGUMENTS (current_function_decl); parm;
527 2141087 : parm = DECL_CHAIN (parm))
528 : {
529 2141087 : if (!is_gimple_reg (parm))
530 : {
531 148635 : if (bitmap_bit_p (non_ssa_vars, DECL_UID (parm)))
532 : {
533 0 : if (dump_file && (dump_flags & TDF_DETAILS))
534 0 : fprintf (dump_file,
535 : " Refused: need to pass non-ssa param values\n");
536 0 : return;
537 : }
538 : }
539 : else
540 : {
541 1992452 : tree ddef = ssa_default_def (cfun, parm);
542 1992452 : if (ddef
543 3931160 : && bitmap_bit_p (current->ssa_names_to_pass,
544 1938708 : SSA_NAME_VERSION (ddef)))
545 : {
546 659730 : if (!VOID_TYPE_P (TREE_TYPE (parm)))
547 659730 : call_overhead += estimate_move_cost (TREE_TYPE (parm), false);
548 659730 : num_args++;
549 : }
550 : }
551 : }
552 1112234 : if (!VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
553 739587 : call_overhead += estimate_move_cost (TREE_TYPE (TREE_TYPE
554 : (current_function_decl)),
555 : false);
556 :
557 1112234 : if (current->split_size <= call_overhead)
558 : {
559 538841 : if (dump_file && (dump_flags & TDF_DETAILS))
560 0 : fprintf (dump_file,
561 : " Refused: split size is smaller than call overhead\n");
562 538841 : return;
563 : }
564 : /* FIXME: The logic here is not very precise, because inliner does use
565 : inline predicates to reduce function body size. We add 10 to anticipate
566 : that. Next stage1 we should try to be more meaningful here. */
567 573393 : if (current->header_size + call_overhead
568 573393 : >= (unsigned int)(DECL_DECLARED_INLINE_P (current_function_decl)
569 143422 : ? param_max_inline_insns_single
570 573393 : : param_max_inline_insns_auto) + 10)
571 : {
572 406640 : if (dump_file && (dump_flags & TDF_DETAILS))
573 0 : fprintf (dump_file,
574 : " Refused: header size is too large for inline candidate\n");
575 406640 : return;
576 : }
577 :
578 : /* Splitting functions brings the target out of comdat group; this will
579 : lead to code duplication if the function is reused by other unit.
580 : Limit this duplication. This is consistent with limit in tree-sra.cc
581 : FIXME: with LTO we ought to be able to do better! */
582 301408 : if (DECL_ONE_ONLY (current_function_decl)
583 301402 : && current->split_size >= (unsigned int) param_max_inline_insns_auto + 10)
584 : {
585 13814 : if (dump_file && (dump_flags & TDF_DETAILS))
586 0 : fprintf (dump_file,
587 : " Refused: function is COMDAT and tail is too large\n");
588 13814 : return;
589 : }
590 : /* For comdat functions also reject very small tails; those will likely get
591 : inlined back and we do not want to risk the duplication overhead.
592 : FIXME: with LTO we ought to be able to do better! */
593 273780 : if (DECL_ONE_ONLY (current_function_decl)
594 152939 : && current->split_size
595 120835 : <= (unsigned int) param_early_inlining_insns / 2)
596 : {
597 30631 : if (dump_file && (dump_flags & TDF_DETAILS))
598 0 : fprintf (dump_file,
599 : " Refused: function is COMDAT and tail is too small\n");
600 30631 : return;
601 : }
602 :
603 : /* FIXME: we currently can pass only SSA function parameters to the split
604 : arguments. Once parm_adjustment infrastructure is supported by cloning,
605 : we can pass more than that. */
606 122308 : if (num_args != bitmap_count_bits (current->ssa_names_to_pass))
607 : {
608 :
609 56987 : if (dump_file && (dump_flags & TDF_DETAILS))
610 0 : fprintf (dump_file,
611 : " Refused: need to pass non-param values\n");
612 56987 : return;
613 : }
614 :
615 : /* When there are non-ssa vars used in the split region, see if they
616 : are used in the header region. If so, reject the split.
617 : FIXME: we can use nested function support to access both. */
618 65321 : if (!bitmap_empty_p (non_ssa_vars)
619 65321 : && !verify_non_ssa_vars (current, non_ssa_vars, return_bb))
620 : {
621 2798 : if (dump_file && (dump_flags & TDF_DETAILS))
622 0 : fprintf (dump_file,
623 : " Refused: split part has non-ssa uses\n");
624 2798 : return;
625 : }
626 :
627 : /* If the split point is dominated by a forbidden block, reject
628 : the split. */
629 62523 : if (!bitmap_empty_p (forbidden_dominators)
630 62523 : && dominated_by_forbidden (current->entry_bb))
631 : {
632 15 : if (dump_file && (dump_flags & TDF_DETAILS))
633 0 : fprintf (dump_file,
634 : " Refused: split point dominated by forbidden block\n");
635 15 : return;
636 : }
637 :
638 : /* See if retval used by return bb is computed by header or split part.
639 : When it is computed by split part, we need to produce return statement
640 : in the split part and add code to header to pass it around.
641 :
642 : This is bit tricky to test:
643 : 1) When there is no return_bb or no return value, we always pass
644 : value around.
645 : 2) Invariants are always computed by caller.
646 : 3) For SSA we need to look if defining statement is in header or split part
647 : 4) For non-SSA we need to look where the var is computed. */
648 62508 : retval = find_retval (return_bb);
649 62508 : if (!retval)
650 : {
651 : /* If there is a return_bb with no return value in function returning
652 : value by reference, also make the split part return void, otherwise
653 : we expansion would try to create a non-POD temporary, which is
654 : invalid. */
655 37292 : if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)
656 28950 : && DECL_RESULT (current_function_decl)
657 66242 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
658 0 : current->split_part_set_retval = false;
659 : else
660 37292 : current->split_part_set_retval = true;
661 : }
662 25216 : else if (is_gimple_min_invariant (retval))
663 2109 : current->split_part_set_retval = false;
664 : /* Special case is value returned by reference we record as if it was non-ssa
665 : set to result_decl. */
666 23107 : else if (TREE_CODE (retval) == SSA_NAME
667 19378 : && SSA_NAME_VAR (retval)
668 5436 : && TREE_CODE (SSA_NAME_VAR (retval)) == RESULT_DECL
669 23108 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
670 1 : current->split_part_set_retval
671 1 : = bitmap_bit_p (non_ssa_vars, DECL_UID (SSA_NAME_VAR (retval)));
672 23106 : else if (TREE_CODE (retval) == SSA_NAME)
673 19377 : current->split_part_set_retval
674 19377 : = split_part_set_ssa_name_p (retval, current, return_bb);
675 3729 : else if (TREE_CODE (retval) == PARM_DECL)
676 0 : current->split_part_set_retval = false;
677 3729 : else if (VAR_P (retval)
678 434 : || TREE_CODE (retval) == RESULT_DECL)
679 3729 : current->split_part_set_retval
680 3729 : = bitmap_bit_p (non_ssa_vars, DECL_UID (retval));
681 : else
682 0 : current->split_part_set_retval = true;
683 :
684 : /* split_function fixes up at most one PHI non-virtual PHI node in return_bb,
685 : for the return value. If there are other PHIs, give up. */
686 62508 : if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
687 : {
688 54166 : gphi_iterator psi;
689 :
690 111617 : for (psi = gsi_start_phis (return_bb); !gsi_end_p (psi); gsi_next (&psi))
691 57451 : if (!virtual_operand_p (gimple_phi_result (psi.phi ()))
692 57451 : && !(retval
693 14020 : && current->split_part_set_retval
694 14020 : && TREE_CODE (retval) == SSA_NAME
695 14020 : && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
696 14020 : && SSA_NAME_DEF_STMT (retval) == psi.phi ()))
697 : {
698 0 : if (dump_file && (dump_flags & TDF_DETAILS))
699 0 : fprintf (dump_file,
700 : " Refused: return bb has extra PHIs\n");
701 0 : return;
702 : }
703 : }
704 :
705 62508 : if (dump_file && (dump_flags & TDF_DETAILS))
706 3 : fprintf (dump_file, " Accepted!\n");
707 :
708 : /* At the moment chose split point with lowest count and that leaves
709 : out smallest size of header.
710 : In future we might re-consider this heuristics. */
711 62508 : if (!best_split_point.split_bbs
712 15743 : || best_split_point.count
713 15743 : > current->count
714 129553 : || (best_split_point.count == current->count
715 4537 : && best_split_point.split_size < current->split_size))
716 :
717 : {
718 50147 : if (dump_file && (dump_flags & TDF_DETAILS))
719 2 : fprintf (dump_file, " New best split point!\n");
720 50147 : if (best_split_point.ssa_names_to_pass)
721 : {
722 3382 : BITMAP_FREE (best_split_point.ssa_names_to_pass);
723 3382 : BITMAP_FREE (best_split_point.split_bbs);
724 : }
725 50147 : best_split_point = *current;
726 50147 : best_split_point.ssa_names_to_pass = BITMAP_ALLOC (NULL);
727 50147 : bitmap_copy (best_split_point.ssa_names_to_pass,
728 50147 : current->ssa_names_to_pass);
729 50147 : best_split_point.split_bbs = BITMAP_ALLOC (NULL);
730 50147 : bitmap_copy (best_split_point.split_bbs, current->split_bbs);
731 : }
732 : }
733 :
734 : /* Return basic block containing RETURN statement. We allow basic blocks
735 : of the form:
736 : <retval> = tmp_var;
737 : return <retval>
738 : but return_bb cannot be more complex than this (except for
739 : -fsanitize=thread we allow TSAN_FUNC_EXIT () internal call in there).
740 : If nothing is found, return the exit block.
741 :
742 : When there are multiple RETURN statement, chose one with return value,
743 : since that one is more likely shared by multiple code paths.
744 :
745 : Return BB is special, because for function splitting it is the only
746 : basic block that is duplicated in between header and split part of the
747 : function.
748 :
749 : TODO: We might support multiple return blocks. */
750 :
751 : static basic_block
752 1235805 : find_return_bb (void)
753 : {
754 1235805 : edge e;
755 1235805 : basic_block return_bb = EXIT_BLOCK_PTR_FOR_FN (cfun);
756 1235805 : gimple_stmt_iterator bsi;
757 1235805 : bool found_return = false;
758 1235805 : tree retval = NULL_TREE;
759 :
760 2075061 : if (!single_pred_p (EXIT_BLOCK_PTR_FOR_FN (cfun)))
761 : return return_bb;
762 :
763 1235584 : e = single_pred_edge (EXIT_BLOCK_PTR_FOR_FN (cfun));
764 9361200 : for (bsi = gsi_last_bb (e->src); !gsi_end_p (bsi); gsi_prev (&bsi))
765 : {
766 4284272 : gimple *stmt = gsi_stmt (bsi);
767 4284272 : if (gimple_code (stmt) == GIMPLE_LABEL
768 4273363 : || is_gimple_debug (stmt)
769 6636700 : || gimple_clobber_p (stmt))
770 : ;
771 2085768 : else if (gimple_code (stmt) == GIMPLE_ASSIGN
772 586525 : && found_return
773 586525 : && gimple_assign_single_p (stmt)
774 491721 : && (auto_var_in_fn_p (gimple_assign_rhs1 (stmt),
775 : current_function_decl)
776 473513 : || is_gimple_min_invariant (gimple_assign_rhs1 (stmt)))
777 2190869 : && retval == gimple_assign_lhs (stmt))
778 : ;
779 2074968 : else if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
780 : {
781 1235584 : found_return = true;
782 1235584 : retval = gimple_return_retval (return_stmt);
783 : }
784 : /* For -fsanitize=thread, allow also TSAN_FUNC_EXIT () in the return
785 : bb. */
786 839384 : else if ((flag_sanitize & SANITIZE_THREAD)
787 839384 : && gimple_call_internal_p (stmt, IFN_TSAN_FUNC_EXIT))
788 : ;
789 : else
790 : break;
791 : }
792 1235584 : if (gsi_end_p (bsi) && found_return)
793 396328 : return_bb = e->src;
794 :
795 : return return_bb;
796 : }
797 :
798 : /* Given return basic block RETURN_BB, see where return value is really
799 : stored. */
800 : static tree
801 96853 : find_retval (basic_block return_bb)
802 : {
803 96853 : gimple_stmt_iterator bsi;
804 278272 : for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi); gsi_next (&bsi))
805 173077 : if (greturn *return_stmt = dyn_cast <greturn *> (gsi_stmt (bsi)))
806 88427 : return gimple_return_retval (return_stmt);
807 84650 : else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN
808 84650 : && !gimple_clobber_p (gsi_stmt (bsi)))
809 84 : return gimple_assign_rhs1 (gsi_stmt (bsi));
810 : return NULL;
811 : }
812 :
813 : /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
814 : variable, mark it as used in bitmap passed via DATA.
815 : Return true when access to T prevents splitting the function. */
816 :
817 : static bool
818 7551865 : mark_nonssa_use (gimple *, tree t, tree, void *data)
819 : {
820 7551865 : t = get_base_address (t);
821 :
822 7551865 : if (!t || is_gimple_reg (t))
823 0 : return false;
824 :
825 : /* At present we can't pass non-SSA arguments to split function.
826 : FIXME: this can be relaxed by passing references to arguments. */
827 7551865 : if (TREE_CODE (t) == PARM_DECL)
828 : {
829 235385 : if (dump_file && (dump_flags & TDF_DETAILS))
830 0 : fprintf (dump_file,
831 : "Cannot split: use of non-ssa function parameter.\n");
832 235385 : return true;
833 : }
834 :
835 3046192 : if ((VAR_P (t) && auto_var_in_fn_p (t, current_function_decl))
836 4942796 : || TREE_CODE (t) == RESULT_DECL
837 12214053 : || (TREE_CODE (t) == LABEL_DECL && FORCED_LABEL (t)))
838 2418943 : bitmap_set_bit ((bitmap)data, DECL_UID (t));
839 :
840 : /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
841 : to pretend that the value pointed to is actual result decl. */
842 3986767 : if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t))
843 3329713 : && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME
844 3329134 : && SSA_NAME_VAR (TREE_OPERAND (t, 0))
845 2679229 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL
846 7383722 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
847 67242 : return
848 67242 : bitmap_bit_p ((bitmap)data,
849 67242 : DECL_UID (DECL_RESULT (current_function_decl)));
850 :
851 : return false;
852 : }
853 :
854 : /* Compute local properties of basic block BB we collect when looking for
855 : split points. We look for ssa defs and store them in SET_SSA_NAMES,
856 : for ssa uses and store them in USED_SSA_NAMES and for any non-SSA automatic
857 : vars stored in NON_SSA_VARS.
858 :
859 : When BB has edge to RETURN_BB, collect uses in RETURN_BB too.
860 :
861 : Return false when BB contains something that prevents it from being put into
862 : split function. */
863 :
864 : static bool
865 4168461 : visit_bb (basic_block bb, basic_block return_bb,
866 : bitmap set_ssa_names, bitmap used_ssa_names,
867 : bitmap non_ssa_vars)
868 : {
869 4168461 : edge e;
870 4168461 : edge_iterator ei;
871 4168461 : bool can_split = true;
872 :
873 45022021 : for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
874 36685099 : gsi_next (&bsi))
875 : {
876 36685099 : gimple *stmt = gsi_stmt (bsi);
877 36685099 : tree op;
878 36685099 : ssa_op_iter iter;
879 :
880 36685099 : if (is_gimple_debug (stmt))
881 24041552 : continue;
882 :
883 13435304 : if (gimple_clobber_p (stmt))
884 791757 : continue;
885 :
886 : /* FIXME: We can split regions containing EH. We cannot however
887 : split RESX, EH_DISPATCH and EH_POINTER referring to same region
888 : into different partitions. This would require tracking of
889 : EH regions and checking in consider_split_point if they
890 : are not used elsewhere. */
891 12643547 : if (gimple_code (stmt) == GIMPLE_RESX)
892 : {
893 99969 : if (dump_file && (dump_flags & TDF_DETAILS))
894 0 : fprintf (dump_file, "Cannot split: resx.\n");
895 : can_split = false;
896 : }
897 12643547 : if (gimple_code (stmt) == GIMPLE_EH_DISPATCH)
898 : {
899 6687 : if (dump_file && (dump_flags & TDF_DETAILS))
900 0 : fprintf (dump_file, "Cannot split: eh dispatch.\n");
901 : can_split = false;
902 : }
903 :
904 : /* Check calls that would prevent splitting. */
905 12643547 : if (gimple_code (stmt) == GIMPLE_CALL)
906 : {
907 2237552 : if (tree decl = gimple_call_fndecl (stmt))
908 : {
909 : /* Check builtins that would prevent splitting. */
910 2157934 : if (fndecl_built_in_p (decl, BUILT_IN_NORMAL))
911 659243 : switch (DECL_FUNCTION_CODE (decl))
912 : {
913 : /* FIXME: once we will allow passing non-parm values to
914 : split part, we need to be sure to handle correct
915 : builtin_stack_save and builtin_stack_restore. At the
916 : moment we are safe; there is no way to store
917 : builtin_stack_save result in non-SSA variable since all
918 : calls to those are compiler generated. */
919 10 : case BUILT_IN_APPLY:
920 10 : case BUILT_IN_APPLY_ARGS:
921 10 : case BUILT_IN_VA_START:
922 10 : if (dump_file && (dump_flags & TDF_DETAILS))
923 0 : fprintf (dump_file,
924 : "Cannot split: builtin_apply and va_start.\n");
925 : can_split = false;
926 : break;
927 7611 : case BUILT_IN_EH_POINTER:
928 7611 : if (dump_file && (dump_flags & TDF_DETAILS))
929 0 : fprintf (dump_file,
930 : "Cannot split: builtin_eh_pointer.\n");
931 : can_split = false;
932 : break;
933 : default:
934 : break;
935 : }
936 :
937 : /* Calls to functions (which have the warning or error
938 : attribute on them) should not be split off into another
939 : function. */
940 2157934 : if (lookup_attribute ("warning", DECL_ATTRIBUTES (decl))
941 2157934 : || lookup_attribute ("error", DECL_ATTRIBUTES (decl)))
942 : {
943 20 : if (dump_file && (dump_flags & TDF_DETAILS))
944 0 : fprintf (dump_file,
945 : "Cannot split: warning or error attribute.\n");
946 : can_split = false;
947 : }
948 : }
949 : }
950 :
951 18657504 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_DEF)
952 6013957 : bitmap_set_bit (set_ssa_names, SSA_NAME_VERSION (op));
953 24900814 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
954 12257267 : bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op));
955 12643547 : can_split &= !walk_stmt_load_store_addr_ops (stmt, non_ssa_vars,
956 : mark_nonssa_use,
957 : mark_nonssa_use,
958 : mark_nonssa_use);
959 : }
960 5041284 : for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
961 872823 : gsi_next (&bsi))
962 : {
963 872823 : gphi *stmt = bsi.phi ();
964 872823 : unsigned int i;
965 :
966 1745646 : if (virtual_operand_p (gimple_phi_result (stmt)))
967 358318 : continue;
968 1029010 : bitmap_set_bit (set_ssa_names,
969 514505 : SSA_NAME_VERSION (gimple_phi_result (stmt)));
970 1689003 : for (i = 0; i < gimple_phi_num_args (stmt); i++)
971 : {
972 1174498 : tree op = gimple_phi_arg_def (stmt, i);
973 1174498 : if (TREE_CODE (op) == SSA_NAME)
974 797599 : bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op));
975 : }
976 514505 : can_split &= !walk_stmt_load_store_addr_ops (stmt, non_ssa_vars,
977 : mark_nonssa_use,
978 : mark_nonssa_use,
979 : mark_nonssa_use);
980 : }
981 : /* Record also uses coming from PHI operand in return BB. */
982 9701182 : FOR_EACH_EDGE (e, ei, bb->succs)
983 5532721 : if (e->dest == return_bb)
984 : {
985 1563354 : for (gphi_iterator bsi = gsi_start_phis (return_bb);
986 2334423 : !gsi_end_p (bsi);
987 771069 : gsi_next (&bsi))
988 : {
989 771069 : gphi *stmt = bsi.phi ();
990 771069 : tree op = gimple_phi_arg_def (stmt, e->dest_idx);
991 :
992 1542138 : if (virtual_operand_p (gimple_phi_result (stmt)))
993 454839 : continue;
994 316230 : if (TREE_CODE (op) == SSA_NAME)
995 146521 : bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op));
996 : else
997 169709 : can_split &= !mark_nonssa_use (stmt, op, op, non_ssa_vars);
998 : }
999 : }
1000 4168461 : return can_split;
1001 : }
1002 :
1003 : /* Stack entry for recursive DFS walk in find_split_point. */
1004 :
1005 5404250 : class stack_entry
1006 : {
1007 : public:
1008 : /* Basic block we are examining. */
1009 : basic_block bb;
1010 :
1011 : /* SSA names set and used by the BB and all BBs reachable
1012 : from it via DFS walk. */
1013 : bitmap set_ssa_names, used_ssa_names;
1014 : bitmap non_ssa_vars;
1015 :
1016 : /* All BBS visited from this BB via DFS walk. */
1017 : bitmap bbs_visited;
1018 :
1019 : /* Last examined edge in DFS walk. Since we walk unoriented graph,
1020 : the value is up to sum of incoming and outgoing edges of BB. */
1021 : unsigned int edge_num;
1022 :
1023 : /* Stack entry index of earliest BB reachable from current BB
1024 : or any BB visited later in DFS walk. */
1025 : int earliest;
1026 :
1027 : /* Overall time and size of all BBs reached from this BB in DFS walk. */
1028 : sreal overall_time;
1029 : int overall_size;
1030 :
1031 : /* When false we cannot split on this BB. */
1032 : bool can_split;
1033 : };
1034 :
1035 :
1036 : /* Find all articulations and call consider_split on them.
1037 : OVERALL_TIME and OVERALL_SIZE is time and size of the function.
1038 :
1039 : We perform basic algorithm for finding an articulation in a graph
1040 : created from CFG by considering it to be an unoriented graph.
1041 :
1042 : The articulation is discovered via DFS walk. We collect earliest
1043 : basic block on stack that is reachable via backward edge. Articulation
1044 : is any basic block such that there is no backward edge bypassing it.
1045 : To reduce stack usage we maintain heap allocated stack in STACK vector.
1046 : AUX pointer of BB is set to index it appears in the stack or -1 once
1047 : it is visited and popped off the stack.
1048 :
1049 : The algorithm finds articulation after visiting the whole component
1050 : reachable by it. This makes it convenient to collect information about
1051 : the component used by consider_split. */
1052 :
1053 : static void
1054 1235789 : find_split_points (basic_block return_bb, sreal overall_time, int overall_size)
1055 : {
1056 1235789 : stack_entry first;
1057 1235789 : vec<stack_entry> stack = vNULL;
1058 1235789 : basic_block bb;
1059 1235789 : class split_point current;
1060 :
1061 1235789 : current.header_time = overall_time;
1062 1235789 : current.header_size = overall_size;
1063 1235789 : current.split_time = 0;
1064 1235789 : current.split_size = 0;
1065 1235789 : current.ssa_names_to_pass = BITMAP_ALLOC (NULL);
1066 :
1067 1235789 : first.bb = ENTRY_BLOCK_PTR_FOR_FN (cfun);
1068 1235789 : first.edge_num = 0;
1069 1235789 : first.overall_time = 0;
1070 1235789 : first.overall_size = 0;
1071 1235789 : first.earliest = INT_MAX;
1072 1235789 : first.set_ssa_names = 0;
1073 1235789 : first.used_ssa_names = 0;
1074 1235789 : first.non_ssa_vars = 0;
1075 1235789 : first.bbs_visited = 0;
1076 1235789 : first.can_split = false;
1077 1235789 : stack.safe_push (first);
1078 1235789 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->aux = (void *)(intptr_t)-1;
1079 :
1080 18483879 : while (!stack.is_empty ())
1081 : {
1082 17248090 : stack_entry *entry = &stack.last ();
1083 :
1084 : /* We are walking an acyclic graph, so edge_num counts
1085 : succ and pred edges together. However when considering
1086 : articulation, we want to have processed everything reachable
1087 : from articulation but nothing that reaches into it. */
1088 17248090 : if (entry->edge_num == EDGE_COUNT (entry->bb->succs)
1089 17248090 : && entry->bb != ENTRY_BLOCK_PTR_FOR_FN (cfun))
1090 : {
1091 4168461 : int pos = stack.length ();
1092 4168461 : entry->can_split &= visit_bb (entry->bb, return_bb,
1093 : entry->set_ssa_names,
1094 : entry->used_ssa_names,
1095 : entry->non_ssa_vars);
1096 2803807 : if (pos <= entry->earliest && !entry->can_split
1097 4453045 : && dump_file && (dump_flags & TDF_DETAILS))
1098 0 : fprintf (dump_file,
1099 : "found articulation at bb %i but cannot split\n",
1100 0 : entry->bb->index);
1101 4168461 : if (pos <= entry->earliest && entry->can_split)
1102 : {
1103 2519223 : if (dump_file && (dump_flags & TDF_DETAILS))
1104 6 : fprintf (dump_file, "found articulation at bb %i\n",
1105 6 : entry->bb->index);
1106 2519223 : current.entry_bb = entry->bb;
1107 2519223 : current.ssa_names_to_pass = BITMAP_ALLOC (NULL);
1108 2519223 : bitmap_and_compl (current.ssa_names_to_pass,
1109 2519223 : entry->used_ssa_names, entry->set_ssa_names);
1110 2519223 : current.header_time = overall_time - entry->overall_time;
1111 2519223 : current.header_size = overall_size - entry->overall_size;
1112 2519223 : current.split_time = entry->overall_time;
1113 2519223 : current.split_size = entry->overall_size;
1114 2519223 : current.split_bbs = entry->bbs_visited;
1115 2519223 : consider_split (¤t, entry->non_ssa_vars, return_bb);
1116 2519223 : BITMAP_FREE (current.ssa_names_to_pass);
1117 : }
1118 : }
1119 : /* Do actual DFS walk. */
1120 34496180 : if (entry->edge_num
1121 17248090 : < (EDGE_COUNT (entry->bb->succs)
1122 32024602 : + EDGE_COUNT (entry->bb->preds)))
1123 : {
1124 11843840 : edge e;
1125 11843840 : basic_block dest;
1126 11843840 : if (entry->edge_num < EDGE_COUNT (entry->bb->succs))
1127 : {
1128 6768510 : e = EDGE_SUCC (entry->bb, entry->edge_num);
1129 6768510 : dest = e->dest;
1130 : }
1131 : else
1132 : {
1133 5075330 : e = EDGE_PRED (entry->bb, entry->edge_num
1134 : - EDGE_COUNT (entry->bb->succs));
1135 5075330 : dest = e->src;
1136 : }
1137 :
1138 11843840 : entry->edge_num++;
1139 :
1140 : /* New BB to visit, push it to the stack. */
1141 11843840 : if (dest != return_bb && dest != EXIT_BLOCK_PTR_FOR_FN (cfun)
1142 10150660 : && !dest->aux)
1143 : {
1144 4168461 : stack_entry new_entry;
1145 :
1146 4168461 : new_entry.bb = dest;
1147 4168461 : new_entry.edge_num = 0;
1148 4168461 : new_entry.overall_time
1149 4168461 : = bb_info_vec[dest->index].time;
1150 4168461 : new_entry.overall_size
1151 4168461 : = bb_info_vec[dest->index].size;
1152 4168461 : new_entry.earliest = INT_MAX;
1153 4168461 : new_entry.set_ssa_names = BITMAP_ALLOC (NULL);
1154 4168461 : new_entry.used_ssa_names = BITMAP_ALLOC (NULL);
1155 4168461 : new_entry.bbs_visited = BITMAP_ALLOC (NULL);
1156 4168461 : new_entry.non_ssa_vars = BITMAP_ALLOC (NULL);
1157 4168461 : new_entry.can_split = true;
1158 4168461 : bitmap_set_bit (new_entry.bbs_visited, dest->index);
1159 4168461 : stack.safe_push (new_entry);
1160 4168461 : dest->aux = (void *)(intptr_t)stack.length ();
1161 4168461 : }
1162 : /* Back edge found, record the earliest point. */
1163 7675379 : else if ((intptr_t)dest->aux > 0
1164 3969395 : && (intptr_t)dest->aux < entry->earliest)
1165 2614920 : entry->earliest = (intptr_t)dest->aux;
1166 : }
1167 : /* We are done with examining the edges. Pop off the value from stack
1168 : and merge stuff we accumulate during the walk. */
1169 5404250 : else if (entry->bb != ENTRY_BLOCK_PTR_FOR_FN (cfun))
1170 : {
1171 4168461 : stack_entry *prev = &stack[stack.length () - 2];
1172 :
1173 4168461 : entry->bb->aux = (void *)(intptr_t)-1;
1174 4168461 : prev->can_split &= entry->can_split;
1175 4168461 : if (prev->set_ssa_names)
1176 : {
1177 3062498 : bitmap_ior_into (prev->set_ssa_names, entry->set_ssa_names);
1178 3062498 : bitmap_ior_into (prev->used_ssa_names, entry->used_ssa_names);
1179 3062498 : bitmap_ior_into (prev->bbs_visited, entry->bbs_visited);
1180 3062498 : bitmap_ior_into (prev->non_ssa_vars, entry->non_ssa_vars);
1181 : }
1182 4168461 : if (prev->earliest > entry->earliest)
1183 2586127 : prev->earliest = entry->earliest;
1184 4168461 : prev->overall_time += entry->overall_time;
1185 4168461 : prev->overall_size += entry->overall_size;
1186 4168461 : BITMAP_FREE (entry->set_ssa_names);
1187 4168461 : BITMAP_FREE (entry->used_ssa_names);
1188 4168461 : BITMAP_FREE (entry->bbs_visited);
1189 4168461 : BITMAP_FREE (entry->non_ssa_vars);
1190 4168461 : stack.pop ();
1191 : }
1192 : else
1193 1235789 : stack.pop ();
1194 : }
1195 1235789 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->aux = NULL;
1196 5800567 : FOR_EACH_BB_FN (bb, cfun)
1197 4564778 : bb->aux = NULL;
1198 1235789 : stack.release ();
1199 1235789 : BITMAP_FREE (current.ssa_names_to_pass);
1200 1235789 : }
1201 :
1202 : /* Split function at SPLIT_POINT. */
1203 :
1204 : static void
1205 46765 : split_function (basic_block return_bb, class split_point *split_point,
1206 : bool add_tsan_func_exit)
1207 : {
1208 46765 : vec<tree> args_to_pass = vNULL;
1209 46765 : bitmap args_to_skip;
1210 46765 : tree parm;
1211 46765 : int num = 0;
1212 46765 : cgraph_node *node, *cur_node = cgraph_node::get (current_function_decl);
1213 46765 : basic_block call_bb;
1214 46765 : gcall *call, *tsan_func_exit_call = NULL;
1215 46765 : edge e;
1216 46765 : edge_iterator ei;
1217 46765 : tree retval = NULL, real_retval = NULL;
1218 46765 : gimple *last_stmt = NULL;
1219 46765 : unsigned int i;
1220 46765 : tree arg, ddef;
1221 :
1222 46765 : if (dump_file)
1223 : {
1224 7 : fprintf (dump_file, "\n\nSplitting function at:\n");
1225 7 : dump_split_point (dump_file, split_point);
1226 : }
1227 :
1228 46765 : if (cur_node->can_change_signature)
1229 46033 : args_to_skip = BITMAP_ALLOC (NULL);
1230 : else
1231 : args_to_skip = NULL;
1232 :
1233 : /* Collect the parameters of new function and args_to_skip bitmap. */
1234 46765 : for (parm = DECL_ARGUMENTS (current_function_decl);
1235 150638 : parm; parm = DECL_CHAIN (parm), num++)
1236 103873 : if (args_to_skip
1237 103873 : && (!is_gimple_reg (parm)
1238 98671 : || (ddef = ssa_default_def (cfun, parm)) == NULL_TREE
1239 96423 : || !bitmap_bit_p (split_point->ssa_names_to_pass,
1240 96423 : SSA_NAME_VERSION (ddef))))
1241 35861 : bitmap_set_bit (args_to_skip, num);
1242 : else
1243 : {
1244 : /* This parm might not have been used up to now, but is going to be
1245 : used, hence register it. */
1246 68012 : if (is_gimple_reg (parm))
1247 67997 : arg = get_or_create_ssa_default_def (cfun, parm);
1248 : else
1249 15 : arg = parm;
1250 :
1251 68012 : if (!useless_type_conversion_p (DECL_ARG_TYPE (parm), TREE_TYPE (arg)))
1252 0 : arg = fold_convert (DECL_ARG_TYPE (parm), arg);
1253 68012 : args_to_pass.safe_push (arg);
1254 : }
1255 :
1256 : /* See if the split function will return. */
1257 46765 : bool split_part_return_p = false;
1258 246270 : FOR_EACH_EDGE (e, ei, return_bb->preds)
1259 : {
1260 199505 : if (bitmap_bit_p (split_point->split_bbs, e->src->index))
1261 134852 : split_part_return_p = true;
1262 : }
1263 :
1264 : /* Add return block to what will become the split function.
1265 : We do not return; no return block is needed. */
1266 46765 : if (!split_part_return_p)
1267 : ;
1268 : /* We have no return block, so nothing is needed. */
1269 34474 : else if (return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
1270 : ;
1271 : /* When we do not want to return value, we need to construct
1272 : new return block with empty return statement.
1273 : FIXME: Once we are able to change return type, we should change function
1274 : to return void instead of just outputting function with undefined return
1275 : value. For structures this affects quality of codegen. */
1276 34345 : else if ((retval = find_retval (return_bb))
1277 34345 : && !split_point->split_part_set_retval)
1278 : {
1279 3788 : bool redirected = true;
1280 3788 : basic_block new_return_bb = create_basic_block (NULL, 0, return_bb);
1281 3788 : gimple_stmt_iterator gsi = gsi_start_bb (new_return_bb);
1282 3788 : gsi_insert_after (&gsi, gimple_build_return (NULL), GSI_NEW_STMT);
1283 3788 : new_return_bb->count = profile_count::zero ();
1284 16026 : while (redirected)
1285 : {
1286 8450 : redirected = false;
1287 21560 : FOR_EACH_EDGE (e, ei, return_bb->preds)
1288 17772 : if (bitmap_bit_p (split_point->split_bbs, e->src->index))
1289 : {
1290 4662 : new_return_bb->count += e->count ();
1291 4662 : redirect_edge_and_branch (e, new_return_bb);
1292 4662 : redirected = true;
1293 4662 : break;
1294 : }
1295 : }
1296 3788 : e = make_single_succ_edge (new_return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1297 3788 : add_bb_to_loop (new_return_bb, current_loops->tree_root);
1298 3788 : bitmap_set_bit (split_point->split_bbs, new_return_bb->index);
1299 : }
1300 : /* When we pass around the value, use existing return block. */
1301 : else
1302 30557 : bitmap_set_bit (split_point->split_bbs, return_bb->index);
1303 :
1304 : /* If RETURN_BB has virtual operand PHIs, they must be removed and the
1305 : virtual operand marked for renaming as we change the CFG in a way that
1306 : tree-inline is not able to compensate for.
1307 :
1308 : Note this can happen whether or not we have a return value. If we have
1309 : a return value, then RETURN_BB may have PHIs for real operands too. */
1310 46765 : if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
1311 : {
1312 40861 : bool phi_p = false;
1313 40861 : for (gphi_iterator gsi = gsi_start_phis (return_bb);
1314 84142 : !gsi_end_p (gsi);)
1315 : {
1316 43281 : gphi *stmt = gsi.phi ();
1317 86562 : if (!virtual_operand_p (gimple_phi_result (stmt)))
1318 : {
1319 10731 : gsi_next (&gsi);
1320 10731 : continue;
1321 : }
1322 32550 : mark_virtual_phi_result_for_renaming (stmt);
1323 32550 : remove_phi_node (&gsi, true);
1324 32550 : phi_p = true;
1325 : }
1326 : /* In reality we have to rename the reaching definition of the
1327 : virtual operand at return_bb as we will eventually release it
1328 : when we remove the code region we outlined.
1329 : So we have to rename all immediate virtual uses of that region
1330 : if we didn't see a PHI definition yet. */
1331 : /* ??? In real reality we want to set the reaching vdef of the
1332 : entry of the SESE region as the vuse of the call and the reaching
1333 : vdef of the exit of the SESE region as the vdef of the call. */
1334 40861 : if (!phi_p)
1335 16622 : for (gimple_stmt_iterator gsi = gsi_start_bb (return_bb);
1336 29338 : !gsi_end_p (gsi);
1337 21027 : gsi_next (&gsi))
1338 : {
1339 21798 : gimple *stmt = gsi_stmt (gsi);
1340 30109 : if (gimple_vuse (stmt))
1341 : {
1342 8311 : gimple_set_vuse (stmt, NULL_TREE);
1343 8311 : update_stmt (stmt);
1344 : }
1345 30109 : if (gimple_vdef (stmt))
1346 : break;
1347 : }
1348 : }
1349 :
1350 46765 : ipa_param_adjustments *adjustments;
1351 93530 : bool skip_return = (!split_part_return_p
1352 46765 : || !split_point->split_part_set_retval);
1353 : /* TODO: Perhaps get rid of args_to_skip entirely, after we make sure the
1354 : debug info generation and discrepancy avoiding works well too. */
1355 46033 : if ((args_to_skip && !bitmap_empty_p (args_to_skip))
1356 70187 : || skip_return)
1357 : {
1358 26516 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
1359 26516 : unsigned j;
1360 26516 : for (parm = DECL_ARGUMENTS (current_function_decl), j = 0;
1361 88110 : parm; parm = DECL_CHAIN (parm), j++)
1362 61594 : if (!args_to_skip || !bitmap_bit_p (args_to_skip, j))
1363 : {
1364 25733 : ipa_adjusted_param adj;
1365 25733 : memset (&adj, 0, sizeof (adj));
1366 25733 : adj.op = IPA_PARAM_OP_COPY;
1367 25733 : adj.base_index = j;
1368 25733 : adj.prev_clone_index = j;
1369 25733 : vec_safe_push (new_params, adj);
1370 : }
1371 26516 : adjustments = new ipa_param_adjustments (new_params, j, skip_return);
1372 : }
1373 : else
1374 : adjustments = NULL;
1375 :
1376 : /* Now create the actual clone. */
1377 46765 : cgraph_edge::rebuild_edges ();
1378 46765 : node = cur_node->create_version_clone_with_body
1379 46765 : (vNULL, NULL, adjustments,
1380 : split_point->split_bbs, split_point->entry_bb, "part");
1381 46765 : delete adjustments;
1382 46765 : node->split_part = true;
1383 :
1384 46765 : if (cur_node->same_comdat_group)
1385 : {
1386 : /* TODO: call is versionable if we make sure that all
1387 : callers are inside of a comdat group. */
1388 2059 : cur_node->calls_comdat_local = true;
1389 2059 : node->add_to_same_comdat_group (cur_node);
1390 : }
1391 :
1392 :
1393 : /* Let's take a time profile for splitted function. */
1394 46765 : if (cur_node->tp_first_run)
1395 2 : node->tp_first_run = cur_node->tp_first_run + 1;
1396 :
1397 : /* For usual cloning it is enough to clear builtin only when signature
1398 : changes. For partial inlining we however cannot expect the part
1399 : of builtin implementation to have same semantic as the whole. */
1400 46765 : if (fndecl_built_in_p (node->decl))
1401 32 : set_decl_built_in_function (node->decl, NOT_BUILT_IN, 0);
1402 :
1403 : /* Drop "clobber *this" attribute from first argument of the split
1404 : function if any. Code before that might be initializing the
1405 : members. */
1406 46765 : if (tree arg = DECL_ARGUMENTS (node->decl))
1407 34071 : if (lookup_attribute ("clobber *this", DECL_ATTRIBUTES (arg)))
1408 893 : DECL_ATTRIBUTES (arg)
1409 1786 : = remove_attribute ("clobber *this",
1410 893 : copy_list (DECL_ATTRIBUTES (arg)));
1411 :
1412 : /* If return_bb contains any clobbers that refer to SSA_NAMEs
1413 : set in the split part, remove them. Also reset debug stmts that
1414 : refer to SSA_NAMEs set in the split part. */
1415 46765 : if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
1416 : {
1417 40861 : gimple_stmt_iterator gsi = gsi_start_bb (return_bb);
1418 126420 : while (!gsi_end_p (gsi))
1419 : {
1420 85559 : tree op;
1421 85559 : ssa_op_iter iter;
1422 85559 : gimple *stmt = gsi_stmt (gsi);
1423 85559 : bool remove = false;
1424 85559 : if (gimple_clobber_p (stmt) || is_gimple_debug (stmt))
1425 44541 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
1426 : {
1427 2454 : basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (op));
1428 2454 : if (op != retval
1429 2454 : && bb
1430 202 : && bb != return_bb
1431 2647 : && bitmap_bit_p (split_point->split_bbs, bb->index))
1432 : {
1433 0 : if (is_gimple_debug (stmt))
1434 : {
1435 0 : gimple_debug_bind_reset_value (stmt);
1436 0 : update_stmt (stmt);
1437 : }
1438 : else
1439 : remove = true;
1440 : break;
1441 : }
1442 : }
1443 0 : if (remove)
1444 0 : gsi_remove (&gsi, true);
1445 : else
1446 85559 : gsi_next (&gsi);
1447 : }
1448 : }
1449 :
1450 : /* If the original function is declared inline, there is no point in issuing
1451 : a warning for the non-inlinable part. */
1452 46765 : DECL_NO_INLINE_WARNING_P (node->decl) = 1;
1453 46765 : cur_node->remove_callees ();
1454 46765 : cur_node->remove_all_references ();
1455 46765 : if (!split_part_return_p)
1456 12291 : TREE_THIS_VOLATILE (node->decl) = 1;
1457 46765 : if (dump_file)
1458 7 : dump_function_to_file (node->decl, dump_file, dump_flags);
1459 :
1460 : /* Create the basic block we place call into. It is the entry basic block
1461 : split after last label and after the last eos clobber and debug stmt. */
1462 46765 : call_bb = split_point->entry_bb;
1463 193670 : for (gimple_stmt_iterator gsi = gsi_start_bb (call_bb); !gsi_end_p (gsi);)
1464 146876 : if (gimple_code (gsi_stmt (gsi)) == GIMPLE_LABEL
1465 145370 : || gimple_clobber_p (gsi_stmt (gsi), CLOBBER_STORAGE_END)
1466 292096 : || is_gimple_debug (gsi_stmt (gsi)))
1467 : {
1468 100140 : last_stmt = gsi_stmt (gsi);
1469 100140 : gsi_next (&gsi);
1470 : }
1471 : else
1472 : break;
1473 :
1474 : /* Find the old return bb, it might contain some clobbers
1475 : which we want to copy back after the call. */
1476 46765 : basic_block old_return = nullptr;
1477 46765 : if (split_part_return_p)
1478 : {
1479 34474 : bool one_return_bb = true;
1480 113591 : FOR_EACH_EDGE (e, ei, return_bb->preds)
1481 89980 : if (bitmap_bit_p (split_point->split_bbs, e->src->index))
1482 : {
1483 41549 : if (old_return != nullptr)
1484 : {
1485 : one_return_bb = false;
1486 : break;
1487 : }
1488 30686 : old_return = e->src;
1489 : }
1490 34474 : if (!one_return_bb)
1491 23154 : old_return = nullptr;
1492 : }
1493 :
1494 46765 : call_bb->count = split_point->count;
1495 46765 : e = split_block (split_point->entry_bb, last_stmt);
1496 46765 : if (old_return == e->src)
1497 17504 : old_return = e->dest;
1498 46765 : remove_edge (e);
1499 :
1500 : /* Produce the call statement. */
1501 46765 : gimple_stmt_iterator gsi = gsi_last_bb (call_bb);
1502 114777 : FOR_EACH_VEC_ELT (args_to_pass, i, arg)
1503 68012 : if (!is_gimple_val (arg))
1504 : {
1505 7 : arg = force_gimple_operand_gsi (&gsi, arg, true, NULL_TREE,
1506 : false, GSI_CONTINUE_LINKING);
1507 7 : args_to_pass[i] = arg;
1508 : }
1509 46765 : call = gimple_build_call_vec (node->decl, args_to_pass);
1510 46765 : if (cur_node->get_fun ()->has_musttail && !add_tsan_func_exit)
1511 4 : gimple_call_set_must_tail (call, true);
1512 46765 : gimple_set_block (call, DECL_INITIAL (current_function_decl));
1513 46765 : args_to_pass.release ();
1514 :
1515 : /* For optimized away parameters, add on the caller side
1516 : before the call
1517 : DEBUG D#X => parm_Y(D)
1518 : stmts and associate D#X with parm in decl_debug_args_lookup
1519 : vector to say for debug info that if parameter parm had been passed,
1520 : it would have value parm_Y(D). */
1521 46765 : if (args_to_skip)
1522 : {
1523 46033 : vec<tree, va_gc> **debug_args = NULL;
1524 46033 : unsigned i = 0, len = 0;
1525 46033 : if (MAY_HAVE_DEBUG_BIND_STMTS)
1526 : {
1527 39816 : debug_args = decl_debug_args_lookup (node->decl);
1528 39816 : if (debug_args)
1529 17652 : len = vec_safe_length (*debug_args);
1530 : }
1531 46033 : for (parm = DECL_ARGUMENTS (current_function_decl), num = 0;
1532 147509 : parm; parm = DECL_CHAIN (parm), num++)
1533 101476 : if (bitmap_bit_p (args_to_skip, num) && is_gimple_reg (parm))
1534 : {
1535 33056 : tree ddecl;
1536 33056 : gimple *def_temp;
1537 :
1538 : /* This needs to be done even without
1539 : MAY_HAVE_DEBUG_BIND_STMTS, otherwise if it didn't exist
1540 : before, we'd end up with different SSA_NAME_VERSIONs
1541 : between -g and -g0. */
1542 33056 : arg = get_or_create_ssa_default_def (cfun, parm);
1543 33056 : if (!MAY_HAVE_DEBUG_BIND_STMTS || debug_args == NULL)
1544 6236 : continue;
1545 :
1546 71976 : while (i < len && (**debug_args)[i] != DECL_ORIGIN (parm))
1547 9168 : i += 2;
1548 26820 : if (i >= len)
1549 0 : continue;
1550 26820 : ddecl = (**debug_args)[i + 1];
1551 26820 : def_temp
1552 26820 : = gimple_build_debug_bind (ddecl, unshare_expr (arg), call);
1553 26820 : gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
1554 : }
1555 46033 : BITMAP_FREE (args_to_skip);
1556 : }
1557 :
1558 : /* We avoid address being taken on any variable used by split part,
1559 : so return slot optimization is always possible. Moreover this is
1560 : required to make DECL_BY_REFERENCE work. */
1561 46765 : if (aggregate_value_p (DECL_RESULT (current_function_decl),
1562 46765 : TREE_TYPE (current_function_decl))
1563 46765 : && (!is_gimple_reg_type (TREE_TYPE (DECL_RESULT (current_function_decl)))
1564 125 : || DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))))
1565 390 : gimple_call_set_return_slot_opt (call, true);
1566 :
1567 46765 : if (add_tsan_func_exit)
1568 2 : tsan_func_exit_call = gimple_build_call_internal (IFN_TSAN_FUNC_EXIT, 0);
1569 :
1570 : /* Update return value. This is bit tricky. When we do not return,
1571 : do nothing. When we return we might need to update return_bb
1572 : or produce a new return statement. */
1573 46765 : if (!split_part_return_p)
1574 : {
1575 12291 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1576 12291 : if (tsan_func_exit_call)
1577 0 : gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
1578 : }
1579 : else
1580 : {
1581 68948 : e = make_single_succ_edge (call_bb, return_bb,
1582 34474 : return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
1583 34474 : ? 0 : EDGE_FALLTHRU);
1584 :
1585 : /* If there is return basic block, see what value we need to store
1586 : return value into and put call just before it. */
1587 34474 : if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
1588 : {
1589 34345 : real_retval = retval;
1590 34345 : if (real_retval && split_point->split_part_set_retval)
1591 : {
1592 12651 : gphi_iterator psi;
1593 :
1594 : /* See if we need new SSA_NAME for the result.
1595 : When DECL_BY_REFERENCE is true, retval is actually pointer to
1596 : return value and it is constant in whole function. */
1597 12651 : if (TREE_CODE (retval) == SSA_NAME
1598 12651 : && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
1599 : {
1600 10444 : retval = copy_ssa_name (retval, call);
1601 :
1602 : /* See if there is PHI defining return value. */
1603 10444 : for (psi = gsi_start_phis (return_bb);
1604 10444 : !gsi_end_p (psi); gsi_next (&psi))
1605 20888 : if (!virtual_operand_p (gimple_phi_result (psi.phi ())))
1606 : break;
1607 :
1608 : /* When there is PHI, just update its value. */
1609 10444 : if (TREE_CODE (retval) == SSA_NAME
1610 10444 : && !gsi_end_p (psi))
1611 10444 : add_phi_arg (psi.phi (), retval, e, UNKNOWN_LOCATION);
1612 : /* Otherwise update the return BB itself.
1613 : find_return_bb allows at most one assignment to return value,
1614 : so update first statement. */
1615 : else
1616 : {
1617 0 : gimple_stmt_iterator bsi;
1618 0 : for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi);
1619 0 : gsi_next (&bsi))
1620 0 : if (greturn *return_stmt
1621 0 : = dyn_cast <greturn *> (gsi_stmt (bsi)))
1622 : {
1623 0 : gimple_return_set_retval (return_stmt, retval);
1624 0 : break;
1625 : }
1626 0 : else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN
1627 0 : && !gimple_clobber_p (gsi_stmt (bsi)))
1628 : {
1629 0 : gimple_assign_set_rhs1 (gsi_stmt (bsi), retval);
1630 0 : break;
1631 : }
1632 0 : update_stmt (gsi_stmt (bsi));
1633 : /* Also adjust clobbers and debug stmts in return_bb. */
1634 0 : for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi);
1635 0 : gsi_next (&bsi))
1636 : {
1637 0 : gimple *stmt = gsi_stmt (bsi);
1638 0 : if (gimple_clobber_p (stmt)
1639 0 : || is_gimple_debug (stmt))
1640 : {
1641 0 : ssa_op_iter iter;
1642 0 : use_operand_p use_p;
1643 0 : bool update = false;
1644 0 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter,
1645 : SSA_OP_USE)
1646 0 : if (USE_FROM_PTR (use_p) == real_retval)
1647 : {
1648 0 : SET_USE (use_p, retval);
1649 0 : update = true;
1650 : }
1651 0 : if (update)
1652 0 : update_stmt (stmt);
1653 : }
1654 : }
1655 : }
1656 : }
1657 12651 : if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
1658 : {
1659 0 : gimple_call_set_lhs (call, build_simple_mem_ref (retval));
1660 0 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1661 : }
1662 : else
1663 : {
1664 12651 : tree restype;
1665 12651 : restype = TREE_TYPE (DECL_RESULT (current_function_decl));
1666 12651 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1667 12651 : if (!useless_type_conversion_p (TREE_TYPE (retval), restype))
1668 : {
1669 0 : gimple *cpy;
1670 0 : tree tem = create_tmp_reg (restype);
1671 0 : tem = make_ssa_name (tem, call);
1672 0 : cpy = gimple_build_assign (retval, NOP_EXPR, tem);
1673 0 : gsi_insert_after (&gsi, cpy, GSI_NEW_STMT);
1674 0 : retval = tem;
1675 : }
1676 12651 : gimple_call_set_lhs (call, retval);
1677 12651 : update_stmt (call);
1678 : }
1679 12651 : }
1680 : else
1681 21694 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1682 34345 : if (tsan_func_exit_call)
1683 2 : gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
1684 : }
1685 : /* We don't use return block (there is either no return in function or
1686 : multiple of them). So create new basic block with return statement.
1687 : */
1688 : else
1689 : {
1690 129 : greturn *ret;
1691 129 : if (split_point->split_part_set_retval
1692 129 : && !VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
1693 : {
1694 83 : retval = DECL_RESULT (current_function_decl);
1695 :
1696 : /* We use temporary register to hold value when aggregate_value_p
1697 : is false. Similarly for DECL_BY_REFERENCE we must avoid extra
1698 : copy. */
1699 83 : if (!aggregate_value_p (retval, TREE_TYPE (current_function_decl))
1700 83 : && !DECL_BY_REFERENCE (retval))
1701 83 : retval = create_tmp_reg (TREE_TYPE (retval));
1702 83 : if (is_gimple_reg (retval))
1703 : {
1704 : /* When returning by reference, there is only one SSA name
1705 : assigned to RESULT_DECL (that is pointer to return value).
1706 : Look it up or create new one if it is missing. */
1707 65 : if (DECL_BY_REFERENCE (retval))
1708 0 : retval = get_or_create_ssa_default_def (cfun, retval);
1709 : /* Otherwise produce new SSA name for return value. */
1710 : else
1711 65 : retval = make_ssa_name (retval, call);
1712 : }
1713 83 : if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
1714 0 : gimple_call_set_lhs (call, build_simple_mem_ref (retval));
1715 : else
1716 83 : gimple_call_set_lhs (call, retval);
1717 83 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1718 : }
1719 : else
1720 : {
1721 46 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1722 46 : if (retval
1723 0 : && is_gimple_reg_type (TREE_TYPE (retval))
1724 46 : && !is_gimple_val (retval))
1725 : {
1726 0 : gassign *g
1727 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (retval)),
1728 : retval);
1729 0 : retval = gimple_assign_lhs (g);
1730 0 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
1731 : }
1732 : }
1733 129 : if (tsan_func_exit_call)
1734 0 : gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
1735 129 : ret = gimple_build_return (retval);
1736 129 : gsi_insert_after (&gsi, ret, GSI_NEW_STMT);
1737 : }
1738 : }
1739 :
1740 : /* Move the clobbers from the old return bb to after the call. */
1741 46765 : if (old_return)
1742 : {
1743 19823 : gimple_stmt_iterator ngsi = gsi_last_bb (call_bb);
1744 19823 : gsi_next (&ngsi);
1745 19823 : for (gimple_stmt_iterator ogsi = gsi_last_bb (old_return);
1746 53362 : !gsi_end_p (ogsi); )
1747 : {
1748 53298 : gimple *stmt = *ogsi;
1749 53298 : if (is_gimple_debug (stmt))
1750 : {
1751 31116 : gsi_prev (&ogsi);
1752 31116 : continue;
1753 : }
1754 22182 : if (!gimple_clobber_p (stmt, CLOBBER_STORAGE_END))
1755 : break;
1756 : /* Change the vdef/vuse of the clobber to be renamed. */
1757 2423 : unlink_stmt_vdef (stmt);
1758 4846 : release_ssa_name (gimple_vdef (stmt));
1759 2423 : gimple_set_vuse (stmt, gimple_vop (cfun));
1760 2423 : gimple_set_vdef (stmt, gimple_vop (cfun));
1761 2423 : gimple_stmt_iterator nogsi = ogsi;
1762 :
1763 : /* Move to the previous stmt before the move happens. */
1764 2423 : gsi_prev (&ogsi);
1765 2423 : gsi_move_before (&nogsi, &ngsi, GSI_NEW_STMT);
1766 2423 : update_stmt (stmt);
1767 : }
1768 : }
1769 46765 : free_dominance_info (CDI_DOMINATORS);
1770 46765 : free_dominance_info (CDI_POST_DOMINATORS);
1771 46765 : compute_fn_summary (node, true);
1772 46765 : }
1773 :
1774 : /* Execute function splitting pass. */
1775 :
1776 : static unsigned int
1777 2036062 : execute_split_functions (void)
1778 : {
1779 2036062 : gimple_stmt_iterator bsi;
1780 2036062 : basic_block bb;
1781 2036062 : sreal overall_time = 0;
1782 2036062 : int overall_size = 0;
1783 2036062 : int todo = 0;
1784 2036062 : struct cgraph_node *node = cgraph_node::get (current_function_decl);
1785 :
1786 2036062 : if (flags_from_decl_or_type (current_function_decl)
1787 2036062 : & (ECF_NORETURN|ECF_MALLOC|ECF_RETURNS_TWICE))
1788 : {
1789 61607 : if (dump_file)
1790 0 : fprintf (dump_file, "Not splitting: noreturn/malloc/returns_twice "
1791 : "function.\n");
1792 61607 : return 0;
1793 : }
1794 1974455 : if (MAIN_NAME_P (DECL_NAME (current_function_decl)))
1795 : {
1796 58045 : if (dump_file)
1797 3 : fprintf (dump_file, "Not splitting: main function.\n");
1798 58045 : return 0;
1799 : }
1800 1916410 : if (node->frequency == NODE_FREQUENCY_UNLIKELY_EXECUTED)
1801 : {
1802 678 : if (dump_file)
1803 0 : fprintf (dump_file, "Not splitting: function is unlikely executed.\n");
1804 678 : return 0;
1805 : }
1806 : /* This can be relaxed; function might become inlinable after splitting
1807 : away the uninlinable part. */
1808 1915732 : if (ipa_fn_summaries
1809 1915697 : && ipa_fn_summaries->get (node)
1810 1915694 : && !ipa_fn_summaries->get (node)->inlinable)
1811 : {
1812 186771 : if (dump_file)
1813 0 : fprintf (dump_file, "Not splitting: not inlinable.\n");
1814 186771 : return 0;
1815 : }
1816 1728961 : if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
1817 : {
1818 272517 : if (dump_file)
1819 0 : fprintf (dump_file, "Not splitting: disregarding inline limits.\n");
1820 272517 : return 0;
1821 : }
1822 : /* This can be relaxed; most of versioning tests actually prevents
1823 : a duplication. */
1824 1456444 : if (!tree_versionable_function_p (current_function_decl))
1825 : {
1826 33 : if (dump_file)
1827 0 : fprintf (dump_file, "Not splitting: not versionable.\n");
1828 33 : return 0;
1829 : }
1830 : /* FIXME: we could support this. */
1831 1456411 : if (DECL_STRUCT_FUNCTION (current_function_decl)->static_chain_decl)
1832 : {
1833 1774 : if (dump_file)
1834 0 : fprintf (dump_file, "Not splitting: nested function.\n");
1835 1774 : return 0;
1836 : }
1837 :
1838 : /* See if it makes sense to try to split.
1839 : It makes sense to split if we inline, that is if we have direct calls to
1840 : handle or direct calls are possibly going to appear as result of indirect
1841 : inlining or LTO. Also handle -fprofile-generate as LTO to allow non-LTO
1842 : training for LTO -fprofile-use build.
1843 :
1844 : Note that we are not completely conservative about disqualifying functions
1845 : called once. It is possible that the caller is called more then once and
1846 : then inlining would still benefit. */
1847 1454637 : if ((!node->callers
1848 : /* Local functions called once will be completely inlined most of time. */
1849 1017766 : || (!node->callers->next_caller && node->local))
1850 508249 : && !node->address_taken
1851 418895 : && !node->has_aliases_p ()
1852 1685633 : && (!flag_lto || !node->externally_visible))
1853 : {
1854 218661 : if (dump_file)
1855 21 : fprintf (dump_file, "Not splitting: not called directly "
1856 : "or called once.\n");
1857 218661 : return 0;
1858 : }
1859 :
1860 : /* FIXME: We can actually split if splitting reduces call overhead. */
1861 1235976 : if (!flag_inline_small_functions
1862 1235976 : && !DECL_DECLARED_INLINE_P (current_function_decl))
1863 : {
1864 22 : if (dump_file)
1865 0 : fprintf (dump_file, "Not splitting: not autoinlining and function"
1866 : " is not inline.\n");
1867 22 : return 0;
1868 : }
1869 :
1870 1235954 : if (lookup_attribute ("noinline", DECL_ATTRIBUTES (current_function_decl)))
1871 : {
1872 0 : if (dump_file)
1873 0 : fprintf (dump_file, "Not splitting: function is noinline.\n");
1874 0 : return 0;
1875 : }
1876 1235954 : if (lookup_attribute ("section", DECL_ATTRIBUTES (current_function_decl)))
1877 : {
1878 15 : if (dump_file)
1879 0 : fprintf (dump_file, "Not splitting: function is in user defined "
1880 : "section.\n");
1881 15 : return 0;
1882 : }
1883 1235939 : if (!strub_splittable_p (node))
1884 : {
1885 134 : if (dump_file)
1886 0 : fprintf (dump_file, "Not splitting: function is a strub context.\n");
1887 134 : return 0;
1888 : }
1889 :
1890 : /* We enforce splitting after loop headers when profile info is not
1891 : available. */
1892 1235805 : if (profile_status_for_fn (cfun) != PROFILE_READ)
1893 1235733 : mark_dfs_back_edges ();
1894 :
1895 : /* Initialize bitmap to track forbidden calls. */
1896 1235805 : forbidden_dominators = BITMAP_ALLOC (NULL);
1897 1235805 : calculate_dominance_info (CDI_DOMINATORS);
1898 :
1899 : /* Compute local info about basic blocks and determine function size/time. */
1900 1235805 : bb_info_vec.safe_grow_cleared (last_basic_block_for_fn (cfun) + 1, true);
1901 1235805 : best_split_point.split_bbs = NULL;
1902 1235805 : basic_block return_bb = find_return_bb ();
1903 1235805 : int tsan_exit_found = -1;
1904 5800642 : FOR_EACH_BB_FN (bb, cfun)
1905 : {
1906 4564853 : sreal time = 0;
1907 4564853 : int size = 0;
1908 4564853 : sreal freq = bb->count.to_sreal_scale
1909 4564853 : (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
1910 :
1911 4564853 : if (dump_file && (dump_flags & TDF_DETAILS))
1912 9 : fprintf (dump_file, "Basic block %i\n", bb->index);
1913 :
1914 47015141 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
1915 : {
1916 37885451 : sreal this_time;
1917 37885451 : int this_size;
1918 37885451 : gimple *stmt = gsi_stmt (bsi);
1919 :
1920 37885451 : this_size = estimate_num_insns (stmt, &eni_size_weights);
1921 37885451 : this_time = (sreal)estimate_num_insns (stmt, &eni_time_weights)
1922 37885451 : * freq;
1923 37885451 : size += this_size;
1924 37885451 : time += this_time;
1925 37885451 : check_forbidden_calls (stmt);
1926 :
1927 37885451 : if (dump_file && (dump_flags & TDF_DETAILS))
1928 : {
1929 17 : fprintf (dump_file, " freq:%4.2f size:%3i time:%4.2f ",
1930 : freq.to_double (), this_size, this_time.to_double ());
1931 17 : print_gimple_stmt (dump_file, stmt, 0);
1932 : }
1933 :
1934 37885451 : if ((flag_sanitize & SANITIZE_THREAD)
1935 37885451 : && gimple_call_internal_p (stmt, IFN_TSAN_FUNC_EXIT))
1936 : {
1937 : /* We handle TSAN_FUNC_EXIT for splitting either in the
1938 : return_bb, or in its immediate predecessors. */
1939 121 : if ((bb != return_bb && !find_edge (bb, return_bb))
1940 255 : || (tsan_exit_found != -1
1941 2 : && tsan_exit_found != (bb != return_bb)))
1942 : {
1943 16 : if (dump_file)
1944 0 : fprintf (dump_file, "Not splitting: TSAN_FUNC_EXIT"
1945 : " in unexpected basic block.\n");
1946 16 : BITMAP_FREE (forbidden_dominators);
1947 16 : bb_info_vec.release ();
1948 16 : return 0;
1949 : }
1950 134 : tsan_exit_found = bb != return_bb;
1951 : }
1952 : }
1953 4564837 : overall_time += time;
1954 4564837 : overall_size += size;
1955 4564837 : bb_info_vec[bb->index].time = time;
1956 4564837 : bb_info_vec[bb->index].size = size;
1957 : }
1958 1235789 : find_split_points (return_bb, overall_time, overall_size);
1959 1235789 : if (best_split_point.split_bbs)
1960 : {
1961 46765 : split_function (return_bb, &best_split_point, tsan_exit_found == 1);
1962 46765 : BITMAP_FREE (best_split_point.ssa_names_to_pass);
1963 46765 : BITMAP_FREE (best_split_point.split_bbs);
1964 46765 : todo = TODO_update_ssa | TODO_cleanup_cfg;
1965 : }
1966 1235789 : BITMAP_FREE (forbidden_dominators);
1967 1235789 : bb_info_vec.release ();
1968 1235789 : return todo;
1969 : }
1970 :
1971 : namespace {
1972 :
1973 : const pass_data pass_data_split_functions =
1974 : {
1975 : GIMPLE_PASS, /* type */
1976 : "fnsplit", /* name */
1977 : OPTGROUP_NONE, /* optinfo_flags */
1978 : TV_IPA_FNSPLIT, /* tv_id */
1979 : PROP_cfg, /* properties_required */
1980 : 0, /* properties_provided */
1981 : 0, /* properties_destroyed */
1982 : 0, /* todo_flags_start */
1983 : 0, /* todo_flags_finish */
1984 : };
1985 :
1986 : class pass_split_functions : public gimple_opt_pass
1987 : {
1988 : public:
1989 287872 : pass_split_functions (gcc::context *ctxt)
1990 575744 : : gimple_opt_pass (pass_data_split_functions, ctxt)
1991 : {}
1992 :
1993 : /* opt_pass methods: */
1994 : bool gate (function *) final override;
1995 2035624 : unsigned int execute (function *) final override
1996 : {
1997 2035624 : return execute_split_functions ();
1998 : }
1999 :
2000 : }; // class pass_split_functions
2001 :
2002 : bool
2003 2403979 : pass_split_functions::gate (function *)
2004 : {
2005 : /* When doing profile feedback, we want to execute the pass after profiling
2006 : is read. So disable one in early optimization. */
2007 2403979 : return (flag_partial_inlining
2008 2037877 : && !profile_arc_flag
2009 2036169 : && !flag_branch_probabilities
2010 4439664 : && !flag_auto_profile);
2011 : }
2012 :
2013 : } // anon namespace
2014 :
2015 : gimple_opt_pass *
2016 287872 : make_pass_split_functions (gcc::context *ctxt)
2017 : {
2018 287872 : return new pass_split_functions (ctxt);
2019 : }
2020 :
2021 : /* Execute function splitting pass. */
2022 :
2023 : static unsigned int
2024 438 : execute_feedback_split_functions (void)
2025 : {
2026 0 : unsigned int retval = execute_split_functions ();
2027 438 : if (retval)
2028 2 : retval |= TODO_rebuild_cgraph_edges;
2029 438 : return retval;
2030 : }
2031 :
2032 : namespace {
2033 :
2034 : const pass_data pass_data_feedback_split_functions =
2035 : {
2036 : GIMPLE_PASS, /* type */
2037 : "feedback_fnsplit", /* name */
2038 : OPTGROUP_NONE, /* optinfo_flags */
2039 : TV_IPA_FNSPLIT, /* tv_id */
2040 : PROP_cfg, /* properties_required */
2041 : 0, /* properties_provided */
2042 : 0, /* properties_destroyed */
2043 : 0, /* todo_flags_start */
2044 : 0, /* todo_flags_finish */
2045 : };
2046 :
2047 : class pass_feedback_split_functions : public gimple_opt_pass
2048 : {
2049 : public:
2050 575744 : pass_feedback_split_functions (gcc::context *ctxt)
2051 1151488 : : gimple_opt_pass (pass_data_feedback_split_functions, ctxt)
2052 : {}
2053 :
2054 : /* opt_pass methods: */
2055 : bool gate (function *) final override;
2056 438 : unsigned int execute (function *) final override
2057 : {
2058 438 : return execute_feedback_split_functions ();
2059 : }
2060 :
2061 287872 : opt_pass * clone () final override
2062 : {
2063 287872 : return new pass_feedback_split_functions (m_ctxt);
2064 : }
2065 : }; // class pass_feedback_split_functions
2066 :
2067 : bool
2068 2572 : pass_feedback_split_functions::gate (function *)
2069 : {
2070 : /* We don't need to split when profiling at all, we are producing
2071 : lousy code anyway. */
2072 2572 : return (flag_partial_inlining
2073 2572 : && (flag_branch_probabilities || flag_auto_profile));
2074 : }
2075 :
2076 : } // anon namespace
2077 :
2078 : gimple_opt_pass *
2079 287872 : make_pass_feedback_split_functions (gcc::context *ctxt)
2080 : {
2081 287872 : return new pass_feedback_split_functions (ctxt);
2082 : }
|
