Line data Source code
1 : /* Function splitting pass
2 : Copyright (C) 2010-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka <jh@suse.cz>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* The purpose of this pass is to split function bodies to improve
22 : inlining. I.e. for function of the form:
23 :
24 : func (...)
25 : {
26 : if (cheap_test)
27 : something_small
28 : else
29 : something_big
30 : }
31 :
32 : Produce:
33 :
34 : func.part (...)
35 : {
36 : something_big
37 : }
38 :
39 : func (...)
40 : {
41 : if (cheap_test)
42 : something_small
43 : else
44 : func.part (...);
45 : }
46 :
47 : When func becomes inlinable and when cheap_test is often true, inlining func,
48 : but not fund.part leads to performance improvement similar as inlining
49 : original func while the code size growth is smaller.
50 :
51 : The pass is organized in three stages:
52 : 1) Collect local info about basic block into BB_INFO structure and
53 : compute function body estimated size and time.
54 : 2) Via DFS walk find all possible basic blocks where we can split
55 : and chose best one.
56 : 3) If split point is found, split at the specified BB by creating a clone
57 : and updating function to call it.
58 :
59 : The decisions what functions to split are in execute_split_functions
60 : and consider_split.
61 :
62 : There are several possible future improvements for this pass including:
63 :
64 : 1) Splitting to break up large functions
65 : 2) Splitting to reduce stack frame usage
66 : 3) Allow split part of function to use values computed in the header part.
67 : The values needs to be passed to split function, perhaps via same
68 : interface as for nested functions or as argument.
69 : 4) Support for simple rematerialization. I.e. when split part use
70 : value computed in header from function parameter in very cheap way, we
71 : can just recompute it.
72 : 5) Support splitting of nested functions.
73 : 6) Support non-SSA arguments.
74 : 7) There is nothing preventing us from producing multiple parts of single function
75 : when needed or splitting also the parts. */
76 :
77 : #include "config.h"
78 : #include "system.h"
79 : #include "coretypes.h"
80 : #include "backend.h"
81 : #include "rtl.h"
82 : #include "tree.h"
83 : #include "gimple.h"
84 : #include "cfghooks.h"
85 : #include "alloc-pool.h"
86 : #include "tree-pass.h"
87 : #include "ssa.h"
88 : #include "cgraph.h"
89 : #include "diagnostic.h"
90 : #include "fold-const.h"
91 : #include "cfganal.h"
92 : #include "calls.h"
93 : #include "gimplify.h"
94 : #include "gimple-iterator.h"
95 : #include "gimplify-me.h"
96 : #include "gimple-walk.h"
97 : #include "symbol-summary.h"
98 : #include "sreal.h"
99 : #include "ipa-cp.h"
100 : #include "ipa-prop.h"
101 : #include "tree-cfg.h"
102 : #include "tree-into-ssa.h"
103 : #include "tree-dfa.h"
104 : #include "tree-inline.h"
105 : #include "gimple-pretty-print.h"
106 : #include "ipa-fnsummary.h"
107 : #include "cfgloop.h"
108 : #include "attribs.h"
109 : #include "ipa-strub.h"
110 :
111 : /* Per basic block info. */
112 :
113 8267160 : class split_bb_info
114 : {
115 : public:
116 : unsigned int size;
117 : sreal time;
118 : };
119 :
120 : static vec<split_bb_info> bb_info_vec;
121 :
122 : /* Description of split point. */
123 :
124 1240821 : class split_point
125 : {
126 : public:
127 : /* Size of the partitions. */
128 : sreal header_time, split_time;
129 : unsigned int header_size, split_size;
130 :
131 : /* SSA names that need to be passed into spit function. */
132 : bitmap ssa_names_to_pass;
133 :
134 : /* Basic block where we split (that will become entry point of new function. */
135 : basic_block entry_bb;
136 :
137 : /* Count for entering the split part.
138 : This is not count of the entry_bb because it may be in loop. */
139 : profile_count count;
140 :
141 : /* Basic blocks we are splitting away. */
142 : bitmap split_bbs;
143 :
144 : /* True when return value is computed on split part and thus it needs
145 : to be returned. */
146 : bool split_part_set_retval;
147 : };
148 :
149 : /* Best split point found. */
150 :
151 : class split_point best_split_point;
152 :
153 : /* Set of basic blocks that are not allowed to dominate a split point. */
154 :
155 : static bitmap forbidden_dominators;
156 :
157 : static tree find_retval (basic_block return_bb);
158 :
159 : /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
160 : variable, check it if it is present in bitmap passed via DATA. */
161 :
162 : static bool
163 28442 : test_nonssa_use (gimple *, tree t, tree, void *data)
164 : {
165 28442 : t = get_base_address (t);
166 :
167 28442 : if (!t || is_gimple_reg (t))
168 0 : return false;
169 :
170 28442 : if (TREE_CODE (t) == PARM_DECL
171 28376 : || (VAR_P (t)
172 5052 : && auto_var_in_fn_p (t, current_function_decl))
173 24171 : || TREE_CODE (t) == RESULT_DECL
174 : /* Normal labels are part of CFG and will be handled gratefully.
175 : Forced labels however can be used directly by statements and
176 : need to stay in one partition along with their uses. */
177 52055 : || (TREE_CODE (t) == LABEL_DECL
178 6005 : && FORCED_LABEL (t)))
179 4832 : return bitmap_bit_p ((bitmap)data, DECL_UID (t));
180 :
181 : /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
182 : to pretend that the value pointed to is actual result decl. */
183 8331 : if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t))
184 15279 : && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME
185 15275 : && SSA_NAME_VAR (TREE_OPERAND (t, 0))
186 11845 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL
187 23610 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
188 0 : return
189 0 : bitmap_bit_p ((bitmap)data,
190 0 : DECL_UID (DECL_RESULT (current_function_decl)));
191 :
192 : return false;
193 : }
194 :
195 : /* Dump split point CURRENT. */
196 :
197 : static void
198 13 : dump_split_point (FILE * file, class split_point *current)
199 : {
200 13 : fprintf (file,
201 : "Split point at BB %i\n"
202 : " header time: %f header size: %i\n"
203 : " split time: %f split size: %i\n bbs: ",
204 13 : current->entry_bb->index, current->header_time.to_double (),
205 : current->header_size, current->split_time.to_double (),
206 : current->split_size);
207 13 : dump_bitmap (file, current->split_bbs);
208 13 : fprintf (file, " SSA names to pass: ");
209 13 : dump_bitmap (file, current->ssa_names_to_pass);
210 13 : }
211 :
212 : /* Look for all BBs in header that might lead to the split part and verify
213 : that they are not defining any non-SSA var used by the split part.
214 : Parameters are the same as for consider_split. */
215 :
216 : static bool
217 12569 : verify_non_ssa_vars (class split_point *current, bitmap non_ssa_vars,
218 : basic_block return_bb)
219 : {
220 12569 : bitmap seen = BITMAP_ALLOC (NULL);
221 12569 : vec<basic_block> worklist = vNULL;
222 12569 : edge e;
223 12569 : edge_iterator ei;
224 12569 : bool ok = true;
225 12569 : basic_block bb;
226 :
227 26885 : FOR_EACH_EDGE (e, ei, current->entry_bb->preds)
228 14316 : if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)
229 14316 : && !bitmap_bit_p (current->split_bbs, e->src->index))
230 : {
231 13801 : worklist.safe_push (e->src);
232 13801 : bitmap_set_bit (seen, e->src->index);
233 : }
234 :
235 30013 : while (!worklist.is_empty ())
236 : {
237 20218 : bb = worklist.pop ();
238 42164 : FOR_EACH_EDGE (e, ei, bb->preds)
239 21946 : if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)
240 21946 : && bitmap_set_bit (seen, e->src->index))
241 : {
242 8333 : gcc_checking_assert (!bitmap_bit_p (current->split_bbs,
243 : e->src->index));
244 8333 : worklist.safe_push (e->src);
245 : }
246 336218 : for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
247 295782 : gsi_next (&bsi))
248 : {
249 298556 : gimple *stmt = gsi_stmt (bsi);
250 298556 : if (is_gimple_debug (stmt))
251 250520 : continue;
252 48036 : if (walk_stmt_load_store_addr_ops
253 48036 : (stmt, non_ssa_vars, test_nonssa_use, test_nonssa_use,
254 : test_nonssa_use))
255 : {
256 2774 : ok = false;
257 2774 : goto done;
258 : }
259 295868 : if (glabel *label_stmt = dyn_cast <glabel *> (stmt))
260 86 : if (test_nonssa_use (stmt, gimple_label_label (label_stmt),
261 : NULL_TREE, non_ssa_vars))
262 : {
263 0 : ok = false;
264 0 : goto done;
265 : }
266 : }
267 18408 : for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
268 964 : gsi_next (&bsi))
269 : {
270 964 : if (walk_stmt_load_store_addr_ops
271 964 : (gsi_stmt (bsi), non_ssa_vars, test_nonssa_use, test_nonssa_use,
272 : test_nonssa_use))
273 : {
274 0 : ok = false;
275 0 : goto done;
276 : }
277 : }
278 53077 : FOR_EACH_EDGE (e, ei, bb->succs)
279 : {
280 35633 : if (e->dest != return_bb)
281 33152 : continue;
282 2481 : for (gphi_iterator bsi = gsi_start_phis (return_bb);
283 5669 : !gsi_end_p (bsi);
284 3188 : gsi_next (&bsi))
285 : {
286 3188 : gphi *stmt = bsi.phi ();
287 3188 : tree op = gimple_phi_arg_def (stmt, e->dest_idx);
288 :
289 6376 : if (virtual_operand_p (gimple_phi_result (stmt)))
290 2367 : continue;
291 821 : if (TREE_CODE (op) != SSA_NAME
292 821 : && test_nonssa_use (stmt, op, op, non_ssa_vars))
293 : {
294 0 : ok = false;
295 0 : goto done;
296 : }
297 : }
298 : }
299 : }
300 :
301 : /* Verify that the rest of function does not define any label
302 : used by the split part. */
303 104725 : FOR_EACH_BB_FN (bb, cfun)
304 94933 : if (!bitmap_bit_p (current->split_bbs, bb->index)
305 94933 : && !bitmap_bit_p (seen, bb->index))
306 : {
307 33264 : gimple_stmt_iterator bsi;
308 72444 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
309 100849 : if (glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (bsi)))
310 : {
311 5919 : if (test_nonssa_use (label_stmt,
312 : gimple_label_label (label_stmt),
313 : NULL_TREE, non_ssa_vars))
314 : {
315 3 : ok = false;
316 3 : goto done;
317 : }
318 : }
319 : else
320 : break;
321 : }
322 :
323 9792 : done:
324 12569 : BITMAP_FREE (seen);
325 12569 : worklist.release ();
326 12569 : return ok;
327 : }
328 :
329 : /* If STMT is a call, check the callee against a list of forbidden
330 : predicate functions. If a match is found, look for uses of the
331 : call result in condition statements that compare against zero.
332 : For each such use, find the block targeted by the condition
333 : statement for the nonzero result, and set the bit for this block
334 : in the forbidden dominators bitmap. The purpose of this is to avoid
335 : selecting a split point where we are likely to lose the chance
336 : to optimize away an unused function call. */
337 :
338 : static void
339 37519892 : check_forbidden_calls (gimple *stmt)
340 : {
341 37519892 : imm_use_iterator use_iter;
342 37519892 : use_operand_p use_p;
343 37519892 : tree lhs;
344 :
345 : /* At the moment, __builtin_constant_p is the only forbidden
346 : predicate function call (see PR49642). */
347 37519892 : if (!gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P))
348 37518010 : return;
349 :
350 1882 : lhs = gimple_call_lhs (stmt);
351 :
352 1882 : if (!lhs || TREE_CODE (lhs) != SSA_NAME)
353 : return;
354 :
355 7340 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, lhs)
356 : {
357 3576 : tree op1;
358 3576 : basic_block use_bb, forbidden_bb;
359 3576 : enum tree_code code;
360 3576 : edge true_edge, false_edge;
361 3576 : gcond *use_stmt;
362 :
363 3576 : use_stmt = dyn_cast <gcond *> (USE_STMT (use_p));
364 3576 : if (!use_stmt)
365 29 : continue;
366 :
367 : /* Assuming canonical form for GIMPLE_COND here, with constant
368 : in second position. */
369 3547 : op1 = gimple_cond_rhs (use_stmt);
370 3547 : code = gimple_cond_code (use_stmt);
371 3547 : use_bb = gimple_bb (use_stmt);
372 :
373 3547 : extract_true_false_edges_from_block (use_bb, &true_edge, &false_edge);
374 :
375 : /* We're only interested in comparisons that distinguish
376 : unambiguously from zero. */
377 3547 : if (!integer_zerop (op1) || code == LE_EXPR || code == GE_EXPR)
378 0 : continue;
379 :
380 3547 : if (code == EQ_EXPR)
381 5 : forbidden_bb = false_edge->dest;
382 : else
383 3542 : forbidden_bb = true_edge->dest;
384 :
385 3547 : bitmap_set_bit (forbidden_dominators, forbidden_bb->index);
386 1882 : }
387 : }
388 :
389 : /* If BB is dominated by any block in the forbidden dominators set,
390 : return TRUE; else FALSE. */
391 :
392 : static bool
393 1230 : dominated_by_forbidden (basic_block bb)
394 : {
395 1230 : unsigned dom_bb;
396 1230 : bitmap_iterator bi;
397 :
398 3666 : EXECUTE_IF_SET_IN_BITMAP (forbidden_dominators, 1, dom_bb, bi)
399 : {
400 2448 : if (dominated_by_p (CDI_DOMINATORS, bb,
401 2448 : BASIC_BLOCK_FOR_FN (cfun, dom_bb)))
402 : return true;
403 : }
404 :
405 : return false;
406 : }
407 :
408 : /* For give split point CURRENT and return block RETURN_BB return 1
409 : if ssa name VAL is set by split part and 0 otherwise. */
410 : static bool
411 19540 : split_part_set_ssa_name_p (tree val, class split_point *current,
412 : basic_block return_bb)
413 : {
414 19540 : if (TREE_CODE (val) != SSA_NAME)
415 : return false;
416 :
417 19540 : return (!SSA_NAME_IS_DEFAULT_DEF (val)
418 19540 : && (bitmap_bit_p (current->split_bbs,
419 16092 : gimple_bb (SSA_NAME_DEF_STMT (val))->index)
420 16090 : || gimple_bb (SSA_NAME_DEF_STMT (val)) == return_bb));
421 : }
422 :
423 : /* We found an split_point CURRENT. NON_SSA_VARS is bitmap of all non ssa
424 : variables used and RETURN_BB is return basic block.
425 : See if we can split function here. */
426 :
427 : static void
428 2525815 : consider_split (class split_point *current, bitmap non_ssa_vars,
429 : basic_block return_bb)
430 : {
431 2525815 : tree parm;
432 2525815 : unsigned int num_args = 0;
433 2525815 : unsigned int call_overhead;
434 2525815 : edge e;
435 2525815 : edge_iterator ei;
436 2525815 : gphi_iterator bsi;
437 2525815 : unsigned int i;
438 2525815 : tree retval;
439 2525815 : bool back_edge = false;
440 :
441 2525815 : if (dump_file && (dump_flags & TDF_DETAILS))
442 6 : dump_split_point (dump_file, current);
443 :
444 2525815 : current->count = profile_count::zero ();
445 5488998 : FOR_EACH_EDGE (e, ei, current->entry_bb->preds)
446 : {
447 2963183 : if (e->flags & EDGE_DFS_BACK)
448 143301 : back_edge = true;
449 2963183 : if (!bitmap_bit_p (current->split_bbs, e->src->index))
450 2819881 : current->count += e->count ();
451 : }
452 :
453 : /* Do not split when we would end up calling function anyway.
454 : Compares are three state, use !(...<...) to also give up when outcome
455 : is unknown. */
456 5051630 : if (!(current->count
457 2525815 : < (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.apply_scale
458 5051630 : (param_partial_inlining_entry_probability, 100))))
459 : {
460 : /* When profile is guessed, we cannot expect it to give us
461 : realistic estimate on likeliness of function taking the
462 : complex path. As a special case, when tail of the function is
463 : a loop, enable splitting since inlining code skipping the loop
464 : is likely noticeable win. */
465 1411839 : if (back_edge
466 83261 : && profile_status_for_fn (cfun) != PROFILE_READ
467 1495063 : && current->count
468 83224 : < ENTRY_BLOCK_PTR_FOR_FN (cfun)->count)
469 : {
470 14298 : if (dump_file && (dump_flags & TDF_DETAILS))
471 : {
472 0 : fprintf (dump_file,
473 : " Split before loop, accepting despite low counts");
474 0 : current->count.dump (dump_file);
475 0 : fprintf (dump_file, " ");
476 0 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.dump (dump_file);
477 : }
478 : }
479 : else
480 : {
481 1397541 : if (dump_file && (dump_flags & TDF_DETAILS))
482 3 : fprintf (dump_file,
483 : " Refused: incoming frequency is too large.\n");
484 2463168 : return;
485 : }
486 : }
487 :
488 1128274 : if (!current->header_size)
489 : {
490 0 : if (dump_file && (dump_flags & TDF_DETAILS))
491 0 : fprintf (dump_file, " Refused: header empty\n");
492 0 : return;
493 : }
494 :
495 : /* Verify that PHI args on entry are either virtual or all their operands
496 : incoming from header are the same. */
497 1304475 : for (bsi = gsi_start_phis (current->entry_bb); !gsi_end_p (bsi); gsi_next (&bsi))
498 : {
499 191427 : gphi *stmt = bsi.phi ();
500 191427 : tree val = NULL;
501 :
502 382854 : if (virtual_operand_p (gimple_phi_result (stmt)))
503 48028 : continue;
504 415052 : for (i = 0; i < gimple_phi_num_args (stmt); i++)
505 : {
506 286879 : edge e = gimple_phi_arg_edge (stmt, i);
507 286879 : if (!bitmap_bit_p (current->split_bbs, e->src->index))
508 : {
509 158848 : tree edge_val = gimple_phi_arg_def (stmt, i);
510 158848 : if (val && edge_val != val)
511 : {
512 15226 : if (dump_file && (dump_flags & TDF_DETAILS))
513 0 : fprintf (dump_file,
514 : " Refused: entry BB has PHI with multiple variants\n");
515 15226 : return;
516 : }
517 : val = edge_val;
518 : }
519 : }
520 : }
521 :
522 :
523 : /* See what argument we will pass to the split function and compute
524 : call overhead. */
525 1113048 : call_overhead = eni_size_weights.call_cost;
526 3257065 : for (parm = DECL_ARGUMENTS (current_function_decl); parm;
527 2144017 : parm = DECL_CHAIN (parm))
528 : {
529 2144017 : if (!is_gimple_reg (parm))
530 : {
531 146617 : if (bitmap_bit_p (non_ssa_vars, DECL_UID (parm)))
532 : {
533 0 : if (dump_file && (dump_flags & TDF_DETAILS))
534 0 : fprintf (dump_file,
535 : " Refused: need to pass non-ssa param values\n");
536 0 : return;
537 : }
538 : }
539 : else
540 : {
541 1997400 : tree ddef = ssa_default_def (cfun, parm);
542 1997400 : if (ddef
543 3943046 : && bitmap_bit_p (current->ssa_names_to_pass,
544 1945646 : SSA_NAME_VERSION (ddef)))
545 : {
546 662242 : if (!VOID_TYPE_P (TREE_TYPE (parm)))
547 662242 : call_overhead += estimate_move_cost (TREE_TYPE (parm), false);
548 662242 : num_args++;
549 : }
550 : }
551 : }
552 1113048 : if (!VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
553 740875 : call_overhead += estimate_move_cost (TREE_TYPE (TREE_TYPE
554 : (current_function_decl)),
555 : false);
556 :
557 1113048 : if (current->split_size <= call_overhead)
558 : {
559 540762 : if (dump_file && (dump_flags & TDF_DETAILS))
560 0 : fprintf (dump_file,
561 : " Refused: split size is smaller than call overhead\n");
562 540762 : return;
563 : }
564 : /* FIXME: The logic here is not very precise, because inliner does use
565 : inline predicates to reduce function body size. We add 10 to anticipate
566 : that. Next stage1 we should try to be more meaningful here. */
567 572286 : if (current->header_size + call_overhead
568 572286 : >= (unsigned int)(DECL_DECLARED_INLINE_P (current_function_decl)
569 142241 : ? param_max_inline_insns_single
570 572286 : : param_max_inline_insns_auto) + 10)
571 : {
572 403899 : if (dump_file && (dump_flags & TDF_DETAILS))
573 0 : fprintf (dump_file,
574 : " Refused: header size is too large for inline candidate\n");
575 403899 : return;
576 : }
577 :
578 : /* Splitting functions brings the target out of comdat group; this will
579 : lead to code duplication if the function is reused by other unit.
580 : Limit this duplication. This is consistent with limit in tree-sra.cc
581 : FIXME: with LTO we ought to be able to do better! */
582 304129 : if (DECL_ONE_ONLY (current_function_decl)
583 304123 : && current->split_size >= (unsigned int) param_max_inline_insns_auto + 10)
584 : {
585 13787 : if (dump_file && (dump_flags & TDF_DETAILS))
586 0 : fprintf (dump_file,
587 : " Refused: function is COMDAT and tail is too large\n");
588 13787 : return;
589 : }
590 : /* For comdat functions also reject very small tails; those will likely get
591 : inlined back and we do not want to risk the duplication overhead.
592 : FIXME: with LTO we ought to be able to do better! */
593 276555 : if (DECL_ONE_ONLY (current_function_decl)
594 154600 : && current->split_size
595 121949 : <= (unsigned int) param_early_inlining_insns / 2)
596 : {
597 30963 : if (dump_file && (dump_flags & TDF_DETAILS))
598 0 : fprintf (dump_file,
599 : " Refused: function is COMDAT and tail is too small\n");
600 30963 : return;
601 : }
602 :
603 : /* FIXME: we currently can pass only SSA function parameters to the split
604 : arguments. Once parm_adjustment infrastructure is supported by cloning,
605 : we can pass more than that. */
606 123637 : if (num_args != bitmap_count_bits (current->ssa_names_to_pass))
607 : {
608 :
609 58201 : if (dump_file && (dump_flags & TDF_DETAILS))
610 0 : fprintf (dump_file,
611 : " Refused: need to pass non-param values\n");
612 58201 : return;
613 : }
614 :
615 : /* When there are non-ssa vars used in the split region, see if they
616 : are used in the header region. If so, reject the split.
617 : FIXME: we can use nested function support to access both. */
618 65436 : if (!bitmap_empty_p (non_ssa_vars)
619 65436 : && !verify_non_ssa_vars (current, non_ssa_vars, return_bb))
620 : {
621 2777 : if (dump_file && (dump_flags & TDF_DETAILS))
622 0 : fprintf (dump_file,
623 : " Refused: split part has non-ssa uses\n");
624 2777 : return;
625 : }
626 :
627 : /* If the split point is dominated by a forbidden block, reject
628 : the split. */
629 62659 : if (!bitmap_empty_p (forbidden_dominators)
630 62659 : && dominated_by_forbidden (current->entry_bb))
631 : {
632 12 : if (dump_file && (dump_flags & TDF_DETAILS))
633 0 : fprintf (dump_file,
634 : " Refused: split point dominated by forbidden block\n");
635 12 : return;
636 : }
637 :
638 : /* See if retval used by return bb is computed by header or split part.
639 : When it is computed by split part, we need to produce return statement
640 : in the split part and add code to header to pass it around.
641 :
642 : This is bit tricky to test:
643 : 1) When there is no return_bb or no return value, we always pass
644 : value around.
645 : 2) Invariants are always computed by caller.
646 : 3) For SSA we need to look if defining statement is in header or split part
647 : 4) For non-SSA we need to look where the var is computed. */
648 62647 : retval = find_retval (return_bb);
649 62647 : if (!retval)
650 : {
651 : /* If there is a return_bb with no return value in function returning
652 : value by reference, also make the split part return void, otherwise
653 : we expansion would try to create a non-POD temporary, which is
654 : invalid. */
655 37243 : if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)
656 29185 : && DECL_RESULT (current_function_decl)
657 66428 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
658 0 : current->split_part_set_retval = false;
659 : else
660 37243 : current->split_part_set_retval = true;
661 : }
662 25404 : else if (is_gimple_min_invariant (retval))
663 2102 : current->split_part_set_retval = false;
664 : /* Special case is value returned by reference we record as if it was non-ssa
665 : set to result_decl. */
666 23302 : else if (TREE_CODE (retval) == SSA_NAME
667 19541 : && SSA_NAME_VAR (retval)
668 5546 : && TREE_CODE (SSA_NAME_VAR (retval)) == RESULT_DECL
669 23303 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
670 1 : current->split_part_set_retval
671 1 : = bitmap_bit_p (non_ssa_vars, DECL_UID (SSA_NAME_VAR (retval)));
672 23301 : else if (TREE_CODE (retval) == SSA_NAME)
673 19540 : current->split_part_set_retval
674 19540 : = split_part_set_ssa_name_p (retval, current, return_bb);
675 3761 : else if (TREE_CODE (retval) == PARM_DECL)
676 0 : current->split_part_set_retval = false;
677 3761 : else if (VAR_P (retval)
678 428 : || TREE_CODE (retval) == RESULT_DECL)
679 3761 : current->split_part_set_retval
680 3761 : = bitmap_bit_p (non_ssa_vars, DECL_UID (retval));
681 : else
682 0 : current->split_part_set_retval = true;
683 :
684 : /* split_function fixes up at most one PHI non-virtual PHI node in return_bb,
685 : for the return value. If there are other PHIs, give up. */
686 62647 : if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
687 : {
688 54589 : gphi_iterator psi;
689 :
690 112523 : for (psi = gsi_start_phis (return_bb); !gsi_end_p (psi); gsi_next (&psi))
691 57934 : if (!virtual_operand_p (gimple_phi_result (psi.phi ()))
692 57934 : && !(retval
693 14073 : && current->split_part_set_retval
694 14073 : && TREE_CODE (retval) == SSA_NAME
695 14073 : && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
696 14073 : && SSA_NAME_DEF_STMT (retval) == psi.phi ()))
697 : {
698 0 : if (dump_file && (dump_flags & TDF_DETAILS))
699 0 : fprintf (dump_file,
700 : " Refused: return bb has extra PHIs\n");
701 0 : return;
702 : }
703 : }
704 :
705 62647 : if (dump_file && (dump_flags & TDF_DETAILS))
706 3 : fprintf (dump_file, " Accepted!\n");
707 :
708 : /* At the moment chose split point with lowest count and that leaves
709 : out smallest size of header.
710 : In future we might re-consider this heuristics. */
711 62647 : if (!best_split_point.split_bbs
712 15801 : || best_split_point.count
713 15801 : > current->count
714 129904 : || (best_split_point.count == current->count
715 4610 : && best_split_point.split_size < current->split_size))
716 :
717 : {
718 50249 : if (dump_file && (dump_flags & TDF_DETAILS))
719 2 : fprintf (dump_file, " New best split point!\n");
720 50249 : if (best_split_point.ssa_names_to_pass)
721 : {
722 3403 : BITMAP_FREE (best_split_point.ssa_names_to_pass);
723 3403 : BITMAP_FREE (best_split_point.split_bbs);
724 : }
725 50249 : best_split_point = *current;
726 50249 : best_split_point.ssa_names_to_pass = BITMAP_ALLOC (NULL);
727 50249 : bitmap_copy (best_split_point.ssa_names_to_pass,
728 50249 : current->ssa_names_to_pass);
729 50249 : best_split_point.split_bbs = BITMAP_ALLOC (NULL);
730 50249 : bitmap_copy (best_split_point.split_bbs, current->split_bbs);
731 : }
732 : }
733 :
734 : /* Return basic block containing RETURN statement. We allow basic blocks
735 : of the form:
736 : <retval> = tmp_var;
737 : return <retval>
738 : but return_bb cannot be more complex than this (except for
739 : -fsanitize=thread we allow TSAN_FUNC_EXIT () internal call in there).
740 : If nothing is found, return the exit block.
741 :
742 : When there are multiple RETURN statement, chose one with return value,
743 : since that one is more likely shared by multiple code paths.
744 :
745 : Return BB is special, because for function splitting it is the only
746 : basic block that is duplicated in between header and split part of the
747 : function.
748 :
749 : TODO: We might support multiple return blocks. */
750 :
751 : static basic_block
752 1240837 : find_return_bb (void)
753 : {
754 1240837 : edge e;
755 1240837 : basic_block return_bb = EXIT_BLOCK_PTR_FOR_FN (cfun);
756 1240837 : gimple_stmt_iterator bsi;
757 1240837 : bool found_return = false;
758 1240837 : tree retval = NULL_TREE;
759 :
760 2082320 : if (!single_pred_p (EXIT_BLOCK_PTR_FOR_FN (cfun)))
761 : return return_bb;
762 :
763 1240616 : e = single_pred_edge (EXIT_BLOCK_PTR_FOR_FN (cfun));
764 9393408 : for (bsi = gsi_last_bb (e->src); !gsi_end_p (bsi); gsi_prev (&bsi))
765 : {
766 4297571 : gimple *stmt = gsi_stmt (bsi);
767 4297571 : if (gimple_code (stmt) == GIMPLE_LABEL
768 4286640 : || is_gimple_debug (stmt)
769 6656062 : || gimple_clobber_p (stmt))
770 : ;
771 2093187 : else if (gimple_code (stmt) == GIMPLE_ASSIGN
772 587654 : && found_return
773 587654 : && gimple_assign_single_p (stmt)
774 492570 : && (auto_var_in_fn_p (gimple_assign_rhs1 (stmt),
775 : current_function_decl)
776 474075 : || is_gimple_min_invariant (gimple_assign_rhs1 (stmt)))
777 2198236 : && retval == gimple_assign_lhs (stmt))
778 : ;
779 2082217 : else if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
780 : {
781 1240616 : found_return = true;
782 1240616 : retval = gimple_return_retval (return_stmt);
783 : }
784 : /* For -fsanitize=thread, allow also TSAN_FUNC_EXIT () in the return
785 : bb. */
786 841601 : else if ((flag_sanitize & SANITIZE_THREAD)
787 841601 : && gimple_call_internal_p (stmt, IFN_TSAN_FUNC_EXIT))
788 : ;
789 : else
790 : break;
791 : }
792 1240616 : if (gsi_end_p (bsi) && found_return)
793 399133 : return_bb = e->src;
794 :
795 : return return_bb;
796 : }
797 :
798 : /* Given return basic block RETURN_BB, see where return value is really
799 : stored. */
800 : static tree
801 97377 : find_retval (basic_block return_bb)
802 : {
803 97377 : gimple_stmt_iterator bsi;
804 279903 : for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi); gsi_next (&bsi))
805 174468 : if (greturn *return_stmt = dyn_cast <greturn *> (gsi_stmt (bsi)))
806 89255 : return gimple_return_retval (return_stmt);
807 85213 : else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN
808 85213 : && !gimple_clobber_p (gsi_stmt (bsi)))
809 64 : return gimple_assign_rhs1 (gsi_stmt (bsi));
810 : return NULL;
811 : }
812 :
813 : /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
814 : variable, mark it as used in bitmap passed via DATA.
815 : Return true when access to T prevents splitting the function. */
816 :
817 : static bool
818 7490254 : mark_nonssa_use (gimple *, tree t, tree, void *data)
819 : {
820 7490254 : t = get_base_address (t);
821 :
822 7490254 : if (!t || is_gimple_reg (t))
823 0 : return false;
824 :
825 : /* At present we can't pass non-SSA arguments to split function.
826 : FIXME: this can be relaxed by passing references to arguments. */
827 7490254 : if (TREE_CODE (t) == PARM_DECL)
828 : {
829 231722 : if (dump_file && (dump_flags & TDF_DETAILS))
830 0 : fprintf (dump_file,
831 : "Cannot split: use of non-ssa function parameter.\n");
832 231722 : return true;
833 : }
834 :
835 3005600 : if ((VAR_P (t) && auto_var_in_fn_p (t, current_function_decl))
836 4915040 : || TREE_CODE (t) == RESULT_DECL
837 12128322 : || (TREE_CODE (t) == LABEL_DECL && FORCED_LABEL (t)))
838 2388778 : bitmap_set_bit ((bitmap)data, DECL_UID (t));
839 :
840 : /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
841 : to pretend that the value pointed to is actual result decl. */
842 3932458 : if ((TREE_CODE (t) == MEM_REF || INDIRECT_REF_P (t))
843 3326074 : && TREE_CODE (TREE_OPERAND (t, 0)) == SSA_NAME
844 3325495 : && SSA_NAME_VAR (TREE_OPERAND (t, 0))
845 2684900 : && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t, 0))) == RESULT_DECL
846 7326657 : && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
847 68125 : return
848 68125 : bitmap_bit_p ((bitmap)data,
849 68125 : DECL_UID (DECL_RESULT (current_function_decl)));
850 :
851 : return false;
852 : }
853 :
854 : /* Compute local properties of basic block BB we collect when looking for
855 : split points. We look for ssa defs and store them in SET_SSA_NAMES,
856 : for ssa uses and store them in USED_SSA_NAMES and for any non-SSA automatic
857 : vars stored in NON_SSA_VARS.
858 :
859 : When BB has edge to RETURN_BB, collect uses in RETURN_BB too.
860 :
861 : Return false when BB contains something that prevents it from being put into
862 : split function. */
863 :
864 : static bool
865 4145450 : visit_bb (basic_block bb, basic_block return_bb,
866 : bitmap set_ssa_names, bitmap used_ssa_names,
867 : bitmap non_ssa_vars)
868 : {
869 4145450 : edge e;
870 4145450 : edge_iterator ei;
871 4145450 : bool can_split = true;
872 :
873 44605057 : for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
874 36314157 : gsi_next (&bsi))
875 : {
876 36314157 : gimple *stmt = gsi_stmt (bsi);
877 36314157 : tree op;
878 36314157 : ssa_op_iter iter;
879 :
880 36314157 : if (is_gimple_debug (stmt))
881 23769817 : continue;
882 :
883 13330170 : if (gimple_clobber_p (stmt))
884 785830 : continue;
885 :
886 : /* FIXME: We can split regions containing EH. We cannot however
887 : split RESX, EH_DISPATCH and EH_POINTER referring to same region
888 : into different partitions. This would require tracking of
889 : EH regions and checking in consider_split_point if they
890 : are not used elsewhere. */
891 12544340 : if (gimple_code (stmt) == GIMPLE_RESX)
892 : {
893 101422 : if (dump_file && (dump_flags & TDF_DETAILS))
894 0 : fprintf (dump_file, "Cannot split: resx.\n");
895 : can_split = false;
896 : }
897 12544340 : if (gimple_code (stmt) == GIMPLE_EH_DISPATCH)
898 : {
899 6685 : if (dump_file && (dump_flags & TDF_DETAILS))
900 0 : fprintf (dump_file, "Cannot split: eh dispatch.\n");
901 : can_split = false;
902 : }
903 :
904 : /* Check calls that would prevent splitting. */
905 12544340 : if (gimple_code (stmt) == GIMPLE_CALL)
906 : {
907 2224414 : if (tree decl = gimple_call_fndecl (stmt))
908 : {
909 : /* Check builtins that would prevent splitting. */
910 2146401 : if (fndecl_built_in_p (decl, BUILT_IN_NORMAL))
911 649967 : switch (DECL_FUNCTION_CODE (decl))
912 : {
913 : /* FIXME: once we will allow passing non-parm values to
914 : split part, we need to be sure to handle correct
915 : builtin_stack_save and builtin_stack_restore. At the
916 : moment we are safe; there is no way to store
917 : builtin_stack_save result in non-SSA variable since all
918 : calls to those are compiler generated. */
919 10 : case BUILT_IN_APPLY:
920 10 : case BUILT_IN_APPLY_ARGS:
921 10 : case BUILT_IN_VA_START:
922 10 : if (dump_file && (dump_flags & TDF_DETAILS))
923 0 : fprintf (dump_file,
924 : "Cannot split: builtin_apply and va_start.\n");
925 : can_split = false;
926 : break;
927 7591 : case BUILT_IN_EH_POINTER:
928 7591 : if (dump_file && (dump_flags & TDF_DETAILS))
929 0 : fprintf (dump_file,
930 : "Cannot split: builtin_eh_pointer.\n");
931 : can_split = false;
932 : break;
933 : default:
934 : break;
935 : }
936 :
937 : /* Calls to functions (which have the warning or error
938 : attribute on them) should not be split off into another
939 : function. */
940 2146401 : if (lookup_attribute ("warning", DECL_ATTRIBUTES (decl))
941 2146401 : || lookup_attribute ("error", DECL_ATTRIBUTES (decl)))
942 : {
943 20 : if (dump_file && (dump_flags & TDF_DETAILS))
944 0 : fprintf (dump_file,
945 : "Cannot split: warning or error attribute.\n");
946 : can_split = false;
947 : }
948 : }
949 : }
950 :
951 18499542 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_DEF)
952 5955202 : bitmap_set_bit (set_ssa_names, SSA_NAME_VERSION (op));
953 24723945 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
954 12179605 : bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op));
955 12544340 : can_split &= !walk_stmt_load_store_addr_ops (stmt, non_ssa_vars,
956 : mark_nonssa_use,
957 : mark_nonssa_use,
958 : mark_nonssa_use);
959 : }
960 5012800 : for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
961 867350 : gsi_next (&bsi))
962 : {
963 867350 : gphi *stmt = bsi.phi ();
964 867350 : unsigned int i;
965 :
966 1734700 : if (virtual_operand_p (gimple_phi_result (stmt)))
967 356685 : continue;
968 1021330 : bitmap_set_bit (set_ssa_names,
969 510665 : SSA_NAME_VERSION (gimple_phi_result (stmt)));
970 1674740 : for (i = 0; i < gimple_phi_num_args (stmt); i++)
971 : {
972 1164075 : tree op = gimple_phi_arg_def (stmt, i);
973 1164075 : if (TREE_CODE (op) == SSA_NAME)
974 791999 : bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op));
975 : }
976 510665 : can_split &= !walk_stmt_load_store_addr_ops (stmt, non_ssa_vars,
977 : mark_nonssa_use,
978 : mark_nonssa_use,
979 : mark_nonssa_use);
980 : }
981 : /* Record also uses coming from PHI operand in return BB. */
982 9634050 : FOR_EACH_EDGE (e, ei, bb->succs)
983 5488600 : if (e->dest == return_bb)
984 : {
985 1570075 : for (gphi_iterator bsi = gsi_start_phis (return_bb);
986 2345782 : !gsi_end_p (bsi);
987 775707 : gsi_next (&bsi))
988 : {
989 775707 : gphi *stmt = bsi.phi ();
990 775707 : tree op = gimple_phi_arg_def (stmt, e->dest_idx);
991 :
992 1551414 : if (virtual_operand_p (gimple_phi_result (stmt)))
993 458053 : continue;
994 317654 : if (TREE_CODE (op) == SSA_NAME)
995 147170 : bitmap_set_bit (used_ssa_names, SSA_NAME_VERSION (op));
996 : else
997 170484 : can_split &= !mark_nonssa_use (stmt, op, op, non_ssa_vars);
998 : }
999 : }
1000 4145450 : return can_split;
1001 : }
1002 :
1003 : /* Stack entry for recursive DFS walk in find_split_point. */
1004 :
1005 5386271 : class stack_entry
1006 : {
1007 : public:
1008 : /* Basic block we are examining. */
1009 : basic_block bb;
1010 :
1011 : /* SSA names set and used by the BB and all BBs reachable
1012 : from it via DFS walk. */
1013 : bitmap set_ssa_names, used_ssa_names;
1014 : bitmap non_ssa_vars;
1015 :
1016 : /* All BBS visited from this BB via DFS walk. */
1017 : bitmap bbs_visited;
1018 :
1019 : /* Last examined edge in DFS walk. Since we walk unoriented graph,
1020 : the value is up to sum of incoming and outgoing edges of BB. */
1021 : unsigned int edge_num;
1022 :
1023 : /* Stack entry index of earliest BB reachable from current BB
1024 : or any BB visited later in DFS walk. */
1025 : int earliest;
1026 :
1027 : /* Overall time and size of all BBs reached from this BB in DFS walk. */
1028 : sreal overall_time;
1029 : int overall_size;
1030 :
1031 : /* When false we cannot split on this BB. */
1032 : bool can_split;
1033 : };
1034 :
1035 :
1036 : /* Find all articulations and call consider_split on them.
1037 : OVERALL_TIME and OVERALL_SIZE is time and size of the function.
1038 :
1039 : We perform basic algorithm for finding an articulation in a graph
1040 : created from CFG by considering it to be an unoriented graph.
1041 :
1042 : The articulation is discovered via DFS walk. We collect earliest
1043 : basic block on stack that is reachable via backward edge. Articulation
1044 : is any basic block such that there is no backward edge bypassing it.
1045 : To reduce stack usage we maintain heap allocated stack in STACK vector.
1046 : AUX pointer of BB is set to index it appears in the stack or -1 once
1047 : it is visited and popped off the stack.
1048 :
1049 : The algorithm finds articulation after visiting the whole component
1050 : reachable by it. This makes it convenient to collect information about
1051 : the component used by consider_split. */
1052 :
1053 : static void
1054 1240821 : find_split_points (basic_block return_bb, sreal overall_time, int overall_size)
1055 : {
1056 1240821 : stack_entry first;
1057 1240821 : vec<stack_entry> stack = vNULL;
1058 1240821 : basic_block bb;
1059 1240821 : class split_point current;
1060 :
1061 1240821 : current.header_time = overall_time;
1062 1240821 : current.header_size = overall_size;
1063 1240821 : current.split_time = 0;
1064 1240821 : current.split_size = 0;
1065 1240821 : current.ssa_names_to_pass = BITMAP_ALLOC (NULL);
1066 :
1067 1240821 : first.bb = ENTRY_BLOCK_PTR_FOR_FN (cfun);
1068 1240821 : first.edge_num = 0;
1069 1240821 : first.overall_time = 0;
1070 1240821 : first.overall_size = 0;
1071 1240821 : first.earliest = INT_MAX;
1072 1240821 : first.set_ssa_names = 0;
1073 1240821 : first.used_ssa_names = 0;
1074 1240821 : first.non_ssa_vars = 0;
1075 1240821 : first.bbs_visited = 0;
1076 1240821 : first.can_split = false;
1077 1240821 : stack.safe_push (first);
1078 1240821 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->aux = (void *)(intptr_t)-1;
1079 :
1080 18384902 : while (!stack.is_empty ())
1081 : {
1082 17144081 : stack_entry *entry = &stack.last ();
1083 :
1084 : /* We are walking an acyclic graph, so edge_num counts
1085 : succ and pred edges together. However when considering
1086 : articulation, we want to have processed everything reachable
1087 : from articulation but nothing that reaches into it. */
1088 17144081 : if (entry->edge_num == EDGE_COUNT (entry->bb->succs)
1089 17144081 : && entry->bb != ENTRY_BLOCK_PTR_FOR_FN (cfun))
1090 : {
1091 4145450 : int pos = stack.length ();
1092 4145450 : entry->can_split &= visit_bb (entry->bb, return_bb,
1093 : entry->set_ssa_names,
1094 : entry->used_ssa_names,
1095 : entry->non_ssa_vars);
1096 2809769 : if (pos <= entry->earliest && !entry->can_split
1097 4429404 : && dump_file && (dump_flags & TDF_DETAILS))
1098 0 : fprintf (dump_file,
1099 : "found articulation at bb %i but cannot split\n",
1100 0 : entry->bb->index);
1101 4145450 : if (pos <= entry->earliest && entry->can_split)
1102 : {
1103 2525815 : if (dump_file && (dump_flags & TDF_DETAILS))
1104 6 : fprintf (dump_file, "found articulation at bb %i\n",
1105 6 : entry->bb->index);
1106 2525815 : current.entry_bb = entry->bb;
1107 2525815 : current.ssa_names_to_pass = BITMAP_ALLOC (NULL);
1108 2525815 : bitmap_and_compl (current.ssa_names_to_pass,
1109 2525815 : entry->used_ssa_names, entry->set_ssa_names);
1110 2525815 : current.header_time = overall_time - entry->overall_time;
1111 2525815 : current.header_size = overall_size - entry->overall_size;
1112 2525815 : current.split_time = entry->overall_time;
1113 2525815 : current.split_size = entry->overall_size;
1114 2525815 : current.split_bbs = entry->bbs_visited;
1115 2525815 : consider_split (¤t, entry->non_ssa_vars, return_bb);
1116 2525815 : BITMAP_FREE (current.ssa_names_to_pass);
1117 : }
1118 : }
1119 : /* Do actual DFS walk. */
1120 34288162 : if (entry->edge_num
1121 17144081 : < (EDGE_COUNT (entry->bb->succs)
1122 31806520 : + EDGE_COUNT (entry->bb->preds)))
1123 : {
1124 11757810 : edge e;
1125 11757810 : basic_block dest;
1126 11757810 : if (entry->edge_num < EDGE_COUNT (entry->bb->succs))
1127 : {
1128 6729421 : e = EDGE_SUCC (entry->bb, entry->edge_num);
1129 6729421 : dest = e->dest;
1130 : }
1131 : else
1132 : {
1133 5028389 : e = EDGE_PRED (entry->bb, entry->edge_num
1134 : - EDGE_COUNT (entry->bb->succs));
1135 5028389 : dest = e->src;
1136 : }
1137 :
1138 11757810 : entry->edge_num++;
1139 :
1140 : /* New BB to visit, push it to the stack. */
1141 11757810 : if (dest != return_bb && dest != EXIT_BLOCK_PTR_FOR_FN (cfun)
1142 10056778 : && !dest->aux)
1143 : {
1144 4145450 : stack_entry new_entry;
1145 :
1146 4145450 : new_entry.bb = dest;
1147 4145450 : new_entry.edge_num = 0;
1148 4145450 : new_entry.overall_time
1149 4145450 : = bb_info_vec[dest->index].time;
1150 4145450 : new_entry.overall_size
1151 4145450 : = bb_info_vec[dest->index].size;
1152 4145450 : new_entry.earliest = INT_MAX;
1153 4145450 : new_entry.set_ssa_names = BITMAP_ALLOC (NULL);
1154 4145450 : new_entry.used_ssa_names = BITMAP_ALLOC (NULL);
1155 4145450 : new_entry.bbs_visited = BITMAP_ALLOC (NULL);
1156 4145450 : new_entry.non_ssa_vars = BITMAP_ALLOC (NULL);
1157 4145450 : new_entry.can_split = true;
1158 4145450 : bitmap_set_bit (new_entry.bbs_visited, dest->index);
1159 4145450 : stack.safe_push (new_entry);
1160 4145450 : dest->aux = (void *)(intptr_t)stack.length ();
1161 4145450 : }
1162 : /* Back edge found, record the earliest point. */
1163 7612360 : else if ((intptr_t)dest->aux > 0
1164 3918553 : && (intptr_t)dest->aux < entry->earliest)
1165 2606118 : entry->earliest = (intptr_t)dest->aux;
1166 : }
1167 : /* We are done with examining the edges. Pop off the value from stack
1168 : and merge stuff we accumulate during the walk. */
1169 5386271 : else if (entry->bb != ENTRY_BLOCK_PTR_FOR_FN (cfun))
1170 : {
1171 4145450 : stack_entry *prev = &stack[stack.length () - 2];
1172 :
1173 4145450 : entry->bb->aux = (void *)(intptr_t)-1;
1174 4145450 : prev->can_split &= entry->can_split;
1175 4145450 : if (prev->set_ssa_names)
1176 : {
1177 3035586 : bitmap_ior_into (prev->set_ssa_names, entry->set_ssa_names);
1178 3035586 : bitmap_ior_into (prev->used_ssa_names, entry->used_ssa_names);
1179 3035586 : bitmap_ior_into (prev->bbs_visited, entry->bbs_visited);
1180 3035586 : bitmap_ior_into (prev->non_ssa_vars, entry->non_ssa_vars);
1181 : }
1182 4145450 : if (prev->earliest > entry->earliest)
1183 2561328 : prev->earliest = entry->earliest;
1184 4145450 : prev->overall_time += entry->overall_time;
1185 4145450 : prev->overall_size += entry->overall_size;
1186 4145450 : BITMAP_FREE (entry->set_ssa_names);
1187 4145450 : BITMAP_FREE (entry->used_ssa_names);
1188 4145450 : BITMAP_FREE (entry->bbs_visited);
1189 4145450 : BITMAP_FREE (entry->non_ssa_vars);
1190 4145450 : stack.pop ();
1191 : }
1192 : else
1193 1240821 : stack.pop ();
1194 : }
1195 1240821 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->aux = NULL;
1196 5785393 : FOR_EACH_BB_FN (bb, cfun)
1197 4544572 : bb->aux = NULL;
1198 1240821 : stack.release ();
1199 1240821 : BITMAP_FREE (current.ssa_names_to_pass);
1200 1240821 : }
1201 :
1202 : /* Split function at SPLIT_POINT. */
1203 :
1204 : static void
1205 46846 : split_function (basic_block return_bb, class split_point *split_point,
1206 : bool add_tsan_func_exit)
1207 : {
1208 46846 : vec<tree> args_to_pass = vNULL;
1209 46846 : bitmap args_to_skip;
1210 46846 : tree parm;
1211 46846 : int num = 0;
1212 46846 : cgraph_node *node, *cur_node = cgraph_node::get (current_function_decl);
1213 46846 : basic_block call_bb;
1214 46846 : gcall *call, *tsan_func_exit_call = NULL;
1215 46846 : edge e;
1216 46846 : edge_iterator ei;
1217 46846 : tree retval = NULL, real_retval = NULL;
1218 46846 : gimple *last_stmt = NULL;
1219 46846 : unsigned int i;
1220 46846 : tree arg, ddef;
1221 :
1222 46846 : if (dump_file)
1223 : {
1224 7 : fprintf (dump_file, "\n\nSplitting function at:\n");
1225 7 : dump_split_point (dump_file, split_point);
1226 : }
1227 :
1228 46846 : if (cur_node->can_change_signature)
1229 46114 : args_to_skip = BITMAP_ALLOC (NULL);
1230 : else
1231 : args_to_skip = NULL;
1232 :
1233 : /* Collect the parameters of new function and args_to_skip bitmap. */
1234 46846 : for (parm = DECL_ARGUMENTS (current_function_decl);
1235 151158 : parm; parm = DECL_CHAIN (parm), num++)
1236 104312 : if (args_to_skip
1237 104312 : && (!is_gimple_reg (parm)
1238 99091 : || (ddef = ssa_default_def (cfun, parm)) == NULL_TREE
1239 96847 : || !bitmap_bit_p (split_point->ssa_names_to_pass,
1240 96847 : SSA_NAME_VERSION (ddef))))
1241 35536 : bitmap_set_bit (args_to_skip, num);
1242 : else
1243 : {
1244 : /* This parm might not have been used up to now, but is going to be
1245 : used, hence register it. */
1246 68776 : if (is_gimple_reg (parm))
1247 68761 : arg = get_or_create_ssa_default_def (cfun, parm);
1248 : else
1249 15 : arg = parm;
1250 :
1251 68776 : if (!useless_type_conversion_p (DECL_ARG_TYPE (parm), TREE_TYPE (arg)))
1252 0 : arg = fold_convert (DECL_ARG_TYPE (parm), arg);
1253 68776 : args_to_pass.safe_push (arg);
1254 : }
1255 :
1256 : /* See if the split function will return. */
1257 46846 : bool split_part_return_p = false;
1258 246895 : FOR_EACH_EDGE (e, ei, return_bb->preds)
1259 : {
1260 200049 : if (bitmap_bit_p (split_point->split_bbs, e->src->index))
1261 135343 : split_part_return_p = true;
1262 : }
1263 :
1264 : /* Add return block to what will become the split function.
1265 : We do not return; no return block is needed. */
1266 46846 : if (!split_part_return_p)
1267 : ;
1268 : /* We have no return block, so nothing is needed. */
1269 34855 : else if (return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
1270 : ;
1271 : /* When we do not want to return value, we need to construct
1272 : new return block with empty return statement.
1273 : FIXME: Once we are able to change return type, we should change function
1274 : to return void instead of just outputting function with undefined return
1275 : value. For structures this affects quality of codegen. */
1276 34730 : else if ((retval = find_retval (return_bb))
1277 34730 : && !split_point->split_part_set_retval)
1278 : {
1279 3905 : bool redirected = true;
1280 3905 : basic_block new_return_bb = create_basic_block (NULL, 0, return_bb);
1281 3905 : gimple_stmt_iterator gsi = gsi_start_bb (new_return_bb);
1282 3905 : gsi_insert_after (&gsi, gimple_build_return (NULL), GSI_NEW_STMT);
1283 3905 : new_return_bb->count = profile_count::zero ();
1284 16494 : while (redirected)
1285 : {
1286 8684 : redirected = false;
1287 21956 : FOR_EACH_EDGE (e, ei, return_bb->preds)
1288 18051 : if (bitmap_bit_p (split_point->split_bbs, e->src->index))
1289 : {
1290 4779 : new_return_bb->count += e->count ();
1291 4779 : redirect_edge_and_branch (e, new_return_bb);
1292 4779 : redirected = true;
1293 4779 : break;
1294 : }
1295 : }
1296 3905 : e = make_single_succ_edge (new_return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1297 3905 : add_bb_to_loop (new_return_bb, current_loops->tree_root);
1298 3905 : bitmap_set_bit (split_point->split_bbs, new_return_bb->index);
1299 : }
1300 : /* When we pass around the value, use existing return block. */
1301 : else
1302 30825 : bitmap_set_bit (split_point->split_bbs, return_bb->index);
1303 :
1304 : /* If RETURN_BB has virtual operand PHIs, they must be removed and the
1305 : virtual operand marked for renaming as we change the CFG in a way that
1306 : tree-inline is not able to compensate for.
1307 :
1308 : Note this can happen whether or not we have a return value. If we have
1309 : a return value, then RETURN_BB may have PHIs for real operands too. */
1310 46846 : if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
1311 : {
1312 41226 : bool phi_p = false;
1313 41226 : for (gphi_iterator gsi = gsi_start_phis (return_bb);
1314 84949 : !gsi_end_p (gsi);)
1315 : {
1316 43723 : gphi *stmt = gsi.phi ();
1317 87446 : if (!virtual_operand_p (gimple_phi_result (stmt)))
1318 : {
1319 10787 : gsi_next (&gsi);
1320 10787 : continue;
1321 : }
1322 32936 : mark_virtual_phi_result_for_renaming (stmt);
1323 32936 : remove_phi_node (&gsi, true);
1324 32936 : phi_p = true;
1325 : }
1326 : /* In reality we have to rename the reaching definition of the
1327 : virtual operand at return_bb as we will eventually release it
1328 : when we remove the code region we outlined.
1329 : So we have to rename all immediate virtual uses of that region
1330 : if we didn't see a PHI definition yet. */
1331 : /* ??? In real reality we want to set the reaching vdef of the
1332 : entry of the SESE region as the vuse of the call and the reaching
1333 : vdef of the exit of the SESE region as the vdef of the call. */
1334 41226 : if (!phi_p)
1335 16580 : for (gimple_stmt_iterator gsi = gsi_start_bb (return_bb);
1336 29101 : !gsi_end_p (gsi);
1337 20811 : gsi_next (&gsi))
1338 : {
1339 21601 : gimple *stmt = gsi_stmt (gsi);
1340 29891 : if (gimple_vuse (stmt))
1341 : {
1342 8290 : gimple_set_vuse (stmt, NULL_TREE);
1343 8290 : update_stmt (stmt);
1344 : }
1345 29891 : if (gimple_vdef (stmt))
1346 : break;
1347 : }
1348 : }
1349 :
1350 46846 : ipa_param_adjustments *adjustments;
1351 93692 : bool skip_return = (!split_part_return_p
1352 46846 : || !split_point->split_part_set_retval);
1353 : /* TODO: Perhaps get rid of args_to_skip entirely, after we make sure the
1354 : debug info generation and discrepancy avoiding works well too. */
1355 46114 : if ((args_to_skip && !bitmap_empty_p (args_to_skip))
1356 70476 : || skip_return)
1357 : {
1358 26433 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
1359 26433 : unsigned j;
1360 26433 : for (parm = DECL_ARGUMENTS (current_function_decl), j = 0;
1361 88005 : parm; parm = DECL_CHAIN (parm), j++)
1362 61572 : if (!args_to_skip || !bitmap_bit_p (args_to_skip, j))
1363 : {
1364 26036 : ipa_adjusted_param adj;
1365 26036 : memset (&adj, 0, sizeof (adj));
1366 26036 : adj.op = IPA_PARAM_OP_COPY;
1367 26036 : adj.base_index = j;
1368 26036 : adj.prev_clone_index = j;
1369 26036 : vec_safe_push (new_params, adj);
1370 : }
1371 26433 : adjustments = new ipa_param_adjustments (new_params, j, skip_return);
1372 : }
1373 : else
1374 : adjustments = NULL;
1375 :
1376 : /* Now create the actual clone. */
1377 46846 : cgraph_edge::rebuild_edges ();
1378 46846 : node = cur_node->create_version_clone_with_body
1379 46846 : (vNULL, NULL, adjustments,
1380 : split_point->split_bbs, split_point->entry_bb, "part");
1381 46846 : delete adjustments;
1382 46846 : node->split_part = true;
1383 :
1384 46846 : if (cur_node->same_comdat_group)
1385 : {
1386 : /* TODO: call is versionable if we make sure that all
1387 : callers are inside of a comdat group. */
1388 2033 : cur_node->calls_comdat_local = true;
1389 2033 : node->add_to_same_comdat_group (cur_node);
1390 : }
1391 :
1392 :
1393 : /* Let's take a time profile for splitted function. */
1394 46846 : if (cur_node->tp_first_run)
1395 2 : node->tp_first_run = cur_node->tp_first_run + 1;
1396 :
1397 : /* For usual cloning it is enough to clear builtin only when signature
1398 : changes. For partial inlining we however cannot expect the part
1399 : of builtin implementation to have same semantic as the whole. */
1400 46846 : if (fndecl_built_in_p (node->decl))
1401 32 : set_decl_built_in_function (node->decl, NOT_BUILT_IN, 0);
1402 :
1403 : /* Drop "clobber *this" attribute from first argument of the split
1404 : function if any. Code before that might be initializing the
1405 : members. */
1406 46846 : if (tree arg = DECL_ARGUMENTS (node->decl))
1407 34462 : if (lookup_attribute ("clobber *this", DECL_ATTRIBUTES (arg)))
1408 893 : DECL_ATTRIBUTES (arg)
1409 1786 : = remove_attribute ("clobber *this",
1410 893 : copy_list (DECL_ATTRIBUTES (arg)));
1411 :
1412 : /* If return_bb contains any clobbers that refer to SSA_NAMEs
1413 : set in the split part, remove them. Also reset debug stmts that
1414 : refer to SSA_NAMEs set in the split part. */
1415 46846 : if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
1416 : {
1417 41226 : gimple_stmt_iterator gsi = gsi_start_bb (return_bb);
1418 127330 : while (!gsi_end_p (gsi))
1419 : {
1420 86104 : tree op;
1421 86104 : ssa_op_iter iter;
1422 86104 : gimple *stmt = gsi_stmt (gsi);
1423 86104 : bool remove = false;
1424 86104 : if (gimple_clobber_p (stmt) || is_gimple_debug (stmt))
1425 44710 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
1426 : {
1427 2433 : basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (op));
1428 2433 : if (op != retval
1429 2433 : && bb
1430 181 : && bb != return_bb
1431 2605 : && bitmap_bit_p (split_point->split_bbs, bb->index))
1432 : {
1433 0 : if (is_gimple_debug (stmt))
1434 : {
1435 0 : gimple_debug_bind_reset_value (stmt);
1436 0 : update_stmt (stmt);
1437 : }
1438 : else
1439 : remove = true;
1440 : break;
1441 : }
1442 : }
1443 0 : if (remove)
1444 0 : gsi_remove (&gsi, true);
1445 : else
1446 86104 : gsi_next (&gsi);
1447 : }
1448 : }
1449 :
1450 : /* If the original function is declared inline, there is no point in issuing
1451 : a warning for the non-inlinable part. */
1452 46846 : DECL_NO_INLINE_WARNING_P (node->decl) = 1;
1453 46846 : cur_node->remove_callees ();
1454 46846 : cur_node->remove_all_references ();
1455 46846 : if (!split_part_return_p)
1456 11991 : TREE_THIS_VOLATILE (node->decl) = 1;
1457 46846 : if (dump_file)
1458 7 : dump_function_to_file (node->decl, dump_file, dump_flags);
1459 :
1460 : /* Create the basic block we place call into. It is the entry basic block
1461 : split after last label and after the last eos clobber and debug stmt. */
1462 46846 : call_bb = split_point->entry_bb;
1463 195162 : for (gimple_stmt_iterator gsi = gsi_start_bb (call_bb); !gsi_end_p (gsi);)
1464 148287 : if (gimple_code (gsi_stmt (gsi)) == GIMPLE_LABEL
1465 146782 : || gimple_clobber_p (gsi_stmt (gsi), CLOBBER_STORAGE_END)
1466 294919 : || is_gimple_debug (gsi_stmt (gsi)))
1467 : {
1468 101470 : last_stmt = gsi_stmt (gsi);
1469 101470 : gsi_next (&gsi);
1470 : }
1471 : else
1472 : break;
1473 :
1474 : /* Find the old return bb, it might contain some clobbers
1475 : which we want to copy back after the call. */
1476 46846 : basic_block old_return = nullptr;
1477 46846 : if (split_part_return_p)
1478 : {
1479 34855 : bool one_return_bb = true;
1480 114597 : FOR_EACH_EDGE (e, ei, return_bb->preds)
1481 90715 : if (bitmap_bit_p (split_point->split_bbs, e->src->index))
1482 : {
1483 41923 : if (old_return != nullptr)
1484 : {
1485 : one_return_bb = false;
1486 : break;
1487 : }
1488 30950 : old_return = e->src;
1489 : }
1490 34855 : if (!one_return_bb)
1491 22964 : old_return = nullptr;
1492 : }
1493 :
1494 46846 : call_bb->count = split_point->count;
1495 46846 : e = split_block (split_point->entry_bb, last_stmt);
1496 46846 : if (old_return == e->src)
1497 17644 : old_return = e->dest;
1498 46846 : remove_edge (e);
1499 :
1500 : /* Produce the call statement. */
1501 46846 : gimple_stmt_iterator gsi = gsi_last_bb (call_bb);
1502 115622 : FOR_EACH_VEC_ELT (args_to_pass, i, arg)
1503 68776 : if (!is_gimple_val (arg))
1504 : {
1505 7 : arg = force_gimple_operand_gsi (&gsi, arg, true, NULL_TREE,
1506 : false, GSI_CONTINUE_LINKING);
1507 7 : args_to_pass[i] = arg;
1508 : }
1509 46846 : call = gimple_build_call_vec (node->decl, args_to_pass);
1510 46846 : if (cur_node->get_fun ()->has_musttail && !add_tsan_func_exit)
1511 4 : gimple_call_set_must_tail (call, true);
1512 46846 : gimple_set_block (call, DECL_INITIAL (current_function_decl));
1513 46846 : args_to_pass.release ();
1514 :
1515 : /* For optimized away parameters, add on the caller side
1516 : before the call
1517 : DEBUG D#X => parm_Y(D)
1518 : stmts and associate D#X with parm in decl_debug_args_lookup
1519 : vector to say for debug info that if parameter parm had been passed,
1520 : it would have value parm_Y(D). */
1521 46846 : if (args_to_skip)
1522 : {
1523 46114 : vec<tree, va_gc> **debug_args = NULL;
1524 46114 : unsigned i = 0, len = 0;
1525 46114 : if (MAY_HAVE_DEBUG_BIND_STMTS)
1526 : {
1527 39903 : debug_args = decl_debug_args_lookup (node->decl);
1528 39903 : if (debug_args)
1529 17533 : len = vec_safe_length (*debug_args);
1530 : }
1531 46114 : for (parm = DECL_ARGUMENTS (current_function_decl), num = 0;
1532 148029 : parm; parm = DECL_CHAIN (parm), num++)
1533 101915 : if (bitmap_bit_p (args_to_skip, num) && is_gimple_reg (parm))
1534 : {
1535 32712 : tree ddecl;
1536 32712 : gimple *def_temp;
1537 :
1538 : /* This needs to be done even without
1539 : MAY_HAVE_DEBUG_BIND_STMTS, otherwise if it didn't exist
1540 : before, we'd end up with different SSA_NAME_VERSIONs
1541 : between -g and -g0. */
1542 32712 : arg = get_or_create_ssa_default_def (cfun, parm);
1543 32712 : if (!MAY_HAVE_DEBUG_BIND_STMTS || debug_args == NULL)
1544 6228 : continue;
1545 :
1546 70870 : while (i < len && (**debug_args)[i] != DECL_ORIGIN (parm))
1547 8951 : i += 2;
1548 26484 : if (i >= len)
1549 0 : continue;
1550 26484 : ddecl = (**debug_args)[i + 1];
1551 26484 : def_temp
1552 26484 : = gimple_build_debug_bind (ddecl, unshare_expr (arg), call);
1553 26484 : gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
1554 : }
1555 46114 : BITMAP_FREE (args_to_skip);
1556 : }
1557 :
1558 : /* We avoid address being taken on any variable used by split part,
1559 : so return slot optimization is always possible. Moreover this is
1560 : required to make DECL_BY_REFERENCE work. */
1561 46846 : if (aggregate_value_p (DECL_RESULT (current_function_decl),
1562 46846 : TREE_TYPE (current_function_decl))
1563 46846 : && (!is_gimple_reg_type (TREE_TYPE (DECL_RESULT (current_function_decl)))
1564 123 : || DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))))
1565 384 : gimple_call_set_return_slot_opt (call, true);
1566 :
1567 46846 : if (add_tsan_func_exit)
1568 2 : tsan_func_exit_call = gimple_build_call_internal (IFN_TSAN_FUNC_EXIT, 0);
1569 :
1570 : /* Update return value. This is bit tricky. When we do not return,
1571 : do nothing. When we return we might need to update return_bb
1572 : or produce a new return statement. */
1573 46846 : if (!split_part_return_p)
1574 : {
1575 11991 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1576 11991 : if (tsan_func_exit_call)
1577 0 : gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
1578 : }
1579 : else
1580 : {
1581 69710 : e = make_single_succ_edge (call_bb, return_bb,
1582 34855 : return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
1583 34855 : ? 0 : EDGE_FALLTHRU);
1584 :
1585 : /* If there is return basic block, see what value we need to store
1586 : return value into and put call just before it. */
1587 34855 : if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
1588 : {
1589 34730 : real_retval = retval;
1590 34730 : if (real_retval && split_point->split_part_set_retval)
1591 : {
1592 12709 : gphi_iterator psi;
1593 :
1594 : /* See if we need new SSA_NAME for the result.
1595 : When DECL_BY_REFERENCE is true, retval is actually pointer to
1596 : return value and it is constant in whole function. */
1597 12709 : if (TREE_CODE (retval) == SSA_NAME
1598 12709 : && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
1599 : {
1600 10508 : retval = copy_ssa_name (retval, call);
1601 :
1602 : /* See if there is PHI defining return value. */
1603 10508 : for (psi = gsi_start_phis (return_bb);
1604 10508 : !gsi_end_p (psi); gsi_next (&psi))
1605 21016 : if (!virtual_operand_p (gimple_phi_result (psi.phi ())))
1606 : break;
1607 :
1608 : /* When there is PHI, just update its value. */
1609 10508 : if (TREE_CODE (retval) == SSA_NAME
1610 10508 : && !gsi_end_p (psi))
1611 10508 : add_phi_arg (psi.phi (), retval, e, UNKNOWN_LOCATION);
1612 : /* Otherwise update the return BB itself.
1613 : find_return_bb allows at most one assignment to return value,
1614 : so update first statement. */
1615 : else
1616 : {
1617 0 : gimple_stmt_iterator bsi;
1618 0 : for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi);
1619 0 : gsi_next (&bsi))
1620 0 : if (greturn *return_stmt
1621 0 : = dyn_cast <greturn *> (gsi_stmt (bsi)))
1622 : {
1623 0 : gimple_return_set_retval (return_stmt, retval);
1624 0 : break;
1625 : }
1626 0 : else if (gimple_code (gsi_stmt (bsi)) == GIMPLE_ASSIGN
1627 0 : && !gimple_clobber_p (gsi_stmt (bsi)))
1628 : {
1629 0 : gimple_assign_set_rhs1 (gsi_stmt (bsi), retval);
1630 0 : break;
1631 : }
1632 0 : update_stmt (gsi_stmt (bsi));
1633 : /* Also adjust clobbers and debug stmts in return_bb. */
1634 0 : for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi);
1635 0 : gsi_next (&bsi))
1636 : {
1637 0 : gimple *stmt = gsi_stmt (bsi);
1638 0 : if (gimple_clobber_p (stmt)
1639 0 : || is_gimple_debug (stmt))
1640 : {
1641 0 : ssa_op_iter iter;
1642 0 : use_operand_p use_p;
1643 0 : bool update = false;
1644 0 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter,
1645 : SSA_OP_USE)
1646 0 : if (USE_FROM_PTR (use_p) == real_retval)
1647 : {
1648 0 : SET_USE (use_p, retval);
1649 0 : update = true;
1650 : }
1651 0 : if (update)
1652 0 : update_stmt (stmt);
1653 : }
1654 : }
1655 : }
1656 : }
1657 12709 : if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
1658 : {
1659 0 : gimple_call_set_lhs (call, build_simple_mem_ref (retval));
1660 0 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1661 : }
1662 : else
1663 : {
1664 12709 : tree restype;
1665 12709 : restype = TREE_TYPE (DECL_RESULT (current_function_decl));
1666 12709 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1667 12709 : if (!useless_type_conversion_p (TREE_TYPE (retval), restype))
1668 : {
1669 0 : gimple *cpy;
1670 0 : tree tem = create_tmp_reg (restype);
1671 0 : tem = make_ssa_name (tem, call);
1672 0 : cpy = gimple_build_assign (retval, NOP_EXPR, tem);
1673 0 : gsi_insert_after (&gsi, cpy, GSI_NEW_STMT);
1674 0 : retval = tem;
1675 : }
1676 12709 : gimple_call_set_lhs (call, retval);
1677 12709 : update_stmt (call);
1678 : }
1679 12709 : }
1680 : else
1681 22021 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1682 34730 : if (tsan_func_exit_call)
1683 2 : gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
1684 : }
1685 : /* We don't use return block (there is either no return in function or
1686 : multiple of them). So create new basic block with return statement.
1687 : */
1688 : else
1689 : {
1690 125 : greturn *ret;
1691 125 : if (split_point->split_part_set_retval
1692 125 : && !VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
1693 : {
1694 83 : retval = DECL_RESULT (current_function_decl);
1695 :
1696 : /* We use temporary register to hold value when aggregate_value_p
1697 : is false. Similarly for DECL_BY_REFERENCE we must avoid extra
1698 : copy. */
1699 83 : if (!aggregate_value_p (retval, TREE_TYPE (current_function_decl))
1700 83 : && !DECL_BY_REFERENCE (retval))
1701 83 : retval = create_tmp_reg (TREE_TYPE (retval));
1702 83 : if (is_gimple_reg (retval))
1703 : {
1704 : /* When returning by reference, there is only one SSA name
1705 : assigned to RESULT_DECL (that is pointer to return value).
1706 : Look it up or create new one if it is missing. */
1707 65 : if (DECL_BY_REFERENCE (retval))
1708 0 : retval = get_or_create_ssa_default_def (cfun, retval);
1709 : /* Otherwise produce new SSA name for return value. */
1710 : else
1711 65 : retval = make_ssa_name (retval, call);
1712 : }
1713 83 : if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
1714 0 : gimple_call_set_lhs (call, build_simple_mem_ref (retval));
1715 : else
1716 83 : gimple_call_set_lhs (call, retval);
1717 83 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1718 : }
1719 : else
1720 : {
1721 42 : gsi_insert_after (&gsi, call, GSI_NEW_STMT);
1722 42 : if (retval
1723 0 : && is_gimple_reg_type (TREE_TYPE (retval))
1724 42 : && !is_gimple_val (retval))
1725 : {
1726 0 : gassign *g
1727 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (retval)),
1728 : retval);
1729 0 : retval = gimple_assign_lhs (g);
1730 0 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
1731 : }
1732 : }
1733 125 : if (tsan_func_exit_call)
1734 0 : gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
1735 125 : ret = gimple_build_return (retval);
1736 125 : gsi_insert_after (&gsi, ret, GSI_NEW_STMT);
1737 : }
1738 : }
1739 :
1740 : /* Move the clobbers from the old return bb to after the call. */
1741 46846 : if (old_return)
1742 : {
1743 19977 : gimple_stmt_iterator ngsi = gsi_last_bb (call_bb);
1744 19977 : gsi_next (&ngsi);
1745 19977 : for (gimple_stmt_iterator ogsi = gsi_last_bb (old_return);
1746 54070 : !gsi_end_p (ogsi); )
1747 : {
1748 54006 : gimple *stmt = *ogsi;
1749 54006 : if (is_gimple_debug (stmt))
1750 : {
1751 31671 : gsi_prev (&ogsi);
1752 31671 : continue;
1753 : }
1754 22335 : if (!gimple_clobber_p (stmt, CLOBBER_STORAGE_END))
1755 : break;
1756 : /* Change the vdef/vuse of the clobber to be renamed. */
1757 2422 : unlink_stmt_vdef (stmt);
1758 4844 : release_ssa_name (gimple_vdef (stmt));
1759 2422 : gimple_set_vuse (stmt, gimple_vop (cfun));
1760 2422 : gimple_set_vdef (stmt, gimple_vop (cfun));
1761 2422 : gimple_stmt_iterator nogsi = ogsi;
1762 :
1763 : /* Move to the previous stmt before the move happens. */
1764 2422 : gsi_prev (&ogsi);
1765 2422 : gsi_move_before (&nogsi, &ngsi, GSI_NEW_STMT);
1766 2422 : update_stmt (stmt);
1767 : }
1768 : }
1769 46846 : free_dominance_info (CDI_DOMINATORS);
1770 46846 : free_dominance_info (CDI_POST_DOMINATORS);
1771 46846 : compute_fn_summary (node, true);
1772 46846 : }
1773 :
1774 : /* Execute function splitting pass. */
1775 :
1776 : static unsigned int
1777 2044962 : execute_split_functions (void)
1778 : {
1779 2044962 : gimple_stmt_iterator bsi;
1780 2044962 : basic_block bb;
1781 2044962 : sreal overall_time = 0;
1782 2044962 : int overall_size = 0;
1783 2044962 : int todo = 0;
1784 2044962 : struct cgraph_node *node = cgraph_node::get (current_function_decl);
1785 :
1786 2044962 : if (flags_from_decl_or_type (current_function_decl)
1787 2044962 : & (ECF_NORETURN|ECF_MALLOC|ECF_RETURNS_TWICE))
1788 : {
1789 62465 : if (dump_file)
1790 0 : fprintf (dump_file, "Not splitting: noreturn/malloc/returns_twice "
1791 : "function.\n");
1792 62465 : return 0;
1793 : }
1794 1982497 : if (MAIN_NAME_P (DECL_NAME (current_function_decl)))
1795 : {
1796 57998 : if (dump_file)
1797 3 : fprintf (dump_file, "Not splitting: main function.\n");
1798 57998 : return 0;
1799 : }
1800 1924499 : if (node->frequency == NODE_FREQUENCY_UNLIKELY_EXECUTED)
1801 : {
1802 673 : if (dump_file)
1803 0 : fprintf (dump_file, "Not splitting: function is unlikely executed.\n");
1804 673 : return 0;
1805 : }
1806 : /* This can be relaxed; function might become inlinable after splitting
1807 : away the uninlinable part. */
1808 1923826 : if (ipa_fn_summaries
1809 1923791 : && ipa_fn_summaries->get (node)
1810 1923788 : && !ipa_fn_summaries->get (node)->inlinable)
1811 : {
1812 191467 : if (dump_file)
1813 0 : fprintf (dump_file, "Not splitting: not inlinable.\n");
1814 191467 : return 0;
1815 : }
1816 1732359 : if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
1817 : {
1818 270202 : if (dump_file)
1819 0 : fprintf (dump_file, "Not splitting: disregarding inline limits.\n");
1820 270202 : return 0;
1821 : }
1822 : /* This can be relaxed; most of versioning tests actually prevents
1823 : a duplication. */
1824 1462157 : if (!tree_versionable_function_p (current_function_decl))
1825 : {
1826 33 : if (dump_file)
1827 0 : fprintf (dump_file, "Not splitting: not versionable.\n");
1828 33 : return 0;
1829 : }
1830 : /* FIXME: we could support this. */
1831 1462124 : if (DECL_STRUCT_FUNCTION (current_function_decl)->static_chain_decl)
1832 : {
1833 1774 : if (dump_file)
1834 0 : fprintf (dump_file, "Not splitting: nested function.\n");
1835 1774 : return 0;
1836 : }
1837 :
1838 : /* See if it makes sense to try to split.
1839 : It makes sense to split if we inline, that is if we have direct calls to
1840 : handle or direct calls are possibly going to appear as result of indirect
1841 : inlining or LTO. Also handle -fprofile-generate as LTO to allow non-LTO
1842 : training for LTO -fprofile-use build.
1843 :
1844 : Note that we are not completely conservative about disqualifying functions
1845 : called once. It is possible that the caller is called more then once and
1846 : then inlining would still benefit. */
1847 1460350 : if ((!node->callers
1848 : /* Local functions called once will be completely inlined most of time. */
1849 1021265 : || (!node->callers->next_caller && node->local))
1850 510446 : && !node->address_taken
1851 420550 : && !node->has_aliases_p ()
1852 1692024 : && (!flag_lto || !node->externally_visible))
1853 : {
1854 219342 : if (dump_file)
1855 21 : fprintf (dump_file, "Not splitting: not called directly "
1856 : "or called once.\n");
1857 219342 : return 0;
1858 : }
1859 :
1860 : /* FIXME: We can actually split if splitting reduces call overhead. */
1861 1241008 : if (!flag_inline_small_functions
1862 1241008 : && !DECL_DECLARED_INLINE_P (current_function_decl))
1863 : {
1864 22 : if (dump_file)
1865 0 : fprintf (dump_file, "Not splitting: not autoinlining and function"
1866 : " is not inline.\n");
1867 22 : return 0;
1868 : }
1869 :
1870 1240986 : if (lookup_attribute ("noinline", DECL_ATTRIBUTES (current_function_decl)))
1871 : {
1872 0 : if (dump_file)
1873 0 : fprintf (dump_file, "Not splitting: function is noinline.\n");
1874 0 : return 0;
1875 : }
1876 1240986 : if (lookup_attribute ("section", DECL_ATTRIBUTES (current_function_decl)))
1877 : {
1878 15 : if (dump_file)
1879 0 : fprintf (dump_file, "Not splitting: function is in user defined "
1880 : "section.\n");
1881 15 : return 0;
1882 : }
1883 1240971 : if (!strub_splittable_p (node))
1884 : {
1885 134 : if (dump_file)
1886 0 : fprintf (dump_file, "Not splitting: function is a strub context.\n");
1887 134 : return 0;
1888 : }
1889 :
1890 : /* We enforce splitting after loop headers when profile info is not
1891 : available. */
1892 1240837 : if (profile_status_for_fn (cfun) != PROFILE_READ)
1893 1240765 : mark_dfs_back_edges ();
1894 :
1895 : /* Initialize bitmap to track forbidden calls. */
1896 1240837 : forbidden_dominators = BITMAP_ALLOC (NULL);
1897 1240837 : calculate_dominance_info (CDI_DOMINATORS);
1898 :
1899 : /* Compute local info about basic blocks and determine function size/time. */
1900 1240837 : bb_info_vec.safe_grow_cleared (last_basic_block_for_fn (cfun) + 1, true);
1901 1240837 : best_split_point.split_bbs = NULL;
1902 1240837 : basic_block return_bb = find_return_bb ();
1903 1240837 : int tsan_exit_found = -1;
1904 5785468 : FOR_EACH_BB_FN (bb, cfun)
1905 : {
1906 4544647 : sreal time = 0;
1907 4544647 : int size = 0;
1908 4544647 : sreal freq = bb->count.to_sreal_scale
1909 4544647 : (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
1910 :
1911 4544647 : if (dump_file && (dump_flags & TDF_DETAILS))
1912 9 : fprintf (dump_file, "Basic block %i\n", bb->index);
1913 :
1914 46609170 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
1915 : {
1916 37519892 : sreal this_time;
1917 37519892 : int this_size;
1918 37519892 : gimple *stmt = gsi_stmt (bsi);
1919 :
1920 37519892 : this_size = estimate_num_insns (stmt, &eni_size_weights);
1921 37519892 : this_time = (sreal)estimate_num_insns (stmt, &eni_time_weights)
1922 37519892 : * freq;
1923 37519892 : size += this_size;
1924 37519892 : time += this_time;
1925 37519892 : check_forbidden_calls (stmt);
1926 :
1927 37519892 : if (dump_file && (dump_flags & TDF_DETAILS))
1928 : {
1929 17 : fprintf (dump_file, " freq:%4.2f size:%3i time:%4.2f ",
1930 : freq.to_double (), this_size, this_time.to_double ());
1931 17 : print_gimple_stmt (dump_file, stmt, 0);
1932 : }
1933 :
1934 37519892 : if ((flag_sanitize & SANITIZE_THREAD)
1935 37519892 : && gimple_call_internal_p (stmt, IFN_TSAN_FUNC_EXIT))
1936 : {
1937 : /* We handle TSAN_FUNC_EXIT for splitting either in the
1938 : return_bb, or in its immediate predecessors. */
1939 111 : if ((bb != return_bb && !find_edge (bb, return_bb))
1940 235 : || (tsan_exit_found != -1
1941 2 : && tsan_exit_found != (bb != return_bb)))
1942 : {
1943 16 : if (dump_file)
1944 0 : fprintf (dump_file, "Not splitting: TSAN_FUNC_EXIT"
1945 : " in unexpected basic block.\n");
1946 16 : BITMAP_FREE (forbidden_dominators);
1947 16 : bb_info_vec.release ();
1948 16 : return 0;
1949 : }
1950 124 : tsan_exit_found = bb != return_bb;
1951 : }
1952 : }
1953 4544631 : overall_time += time;
1954 4544631 : overall_size += size;
1955 4544631 : bb_info_vec[bb->index].time = time;
1956 4544631 : bb_info_vec[bb->index].size = size;
1957 : }
1958 1240821 : find_split_points (return_bb, overall_time, overall_size);
1959 1240821 : if (best_split_point.split_bbs)
1960 : {
1961 46846 : split_function (return_bb, &best_split_point, tsan_exit_found == 1);
1962 46846 : BITMAP_FREE (best_split_point.ssa_names_to_pass);
1963 46846 : BITMAP_FREE (best_split_point.split_bbs);
1964 46846 : todo = TODO_update_ssa | TODO_cleanup_cfg;
1965 : }
1966 1240821 : BITMAP_FREE (forbidden_dominators);
1967 1240821 : bb_info_vec.release ();
1968 1240821 : return todo;
1969 : }
1970 :
1971 : namespace {
1972 :
1973 : const pass_data pass_data_split_functions =
1974 : {
1975 : GIMPLE_PASS, /* type */
1976 : "fnsplit", /* name */
1977 : OPTGROUP_NONE, /* optinfo_flags */
1978 : TV_IPA_FNSPLIT, /* tv_id */
1979 : PROP_cfg, /* properties_required */
1980 : 0, /* properties_provided */
1981 : 0, /* properties_destroyed */
1982 : 0, /* todo_flags_start */
1983 : 0, /* todo_flags_finish */
1984 : };
1985 :
1986 : class pass_split_functions : public gimple_opt_pass
1987 : {
1988 : public:
1989 285722 : pass_split_functions (gcc::context *ctxt)
1990 571444 : : gimple_opt_pass (pass_data_split_functions, ctxt)
1991 : {}
1992 :
1993 : /* opt_pass methods: */
1994 : bool gate (function *) final override;
1995 2044524 : unsigned int execute (function *) final override
1996 : {
1997 2044524 : return execute_split_functions ();
1998 : }
1999 :
2000 : }; // class pass_split_functions
2001 :
2002 : bool
2003 2412428 : pass_split_functions::gate (function *)
2004 : {
2005 : /* When doing profile feedback, we want to execute the pass after profiling
2006 : is read. So disable one in early optimization. */
2007 2412428 : return (flag_partial_inlining
2008 2046774 : && !profile_arc_flag
2009 2045069 : && !flag_branch_probabilities
2010 4457013 : && !flag_auto_profile);
2011 : }
2012 :
2013 : } // anon namespace
2014 :
2015 : gimple_opt_pass *
2016 285722 : make_pass_split_functions (gcc::context *ctxt)
2017 : {
2018 285722 : return new pass_split_functions (ctxt);
2019 : }
2020 :
2021 : /* Execute function splitting pass. */
2022 :
2023 : static unsigned int
2024 438 : execute_feedback_split_functions (void)
2025 : {
2026 0 : unsigned int retval = execute_split_functions ();
2027 438 : if (retval)
2028 2 : retval |= TODO_rebuild_cgraph_edges;
2029 438 : return retval;
2030 : }
2031 :
2032 : namespace {
2033 :
2034 : const pass_data pass_data_feedback_split_functions =
2035 : {
2036 : GIMPLE_PASS, /* type */
2037 : "feedback_fnsplit", /* name */
2038 : OPTGROUP_NONE, /* optinfo_flags */
2039 : TV_IPA_FNSPLIT, /* tv_id */
2040 : PROP_cfg, /* properties_required */
2041 : 0, /* properties_provided */
2042 : 0, /* properties_destroyed */
2043 : 0, /* todo_flags_start */
2044 : 0, /* todo_flags_finish */
2045 : };
2046 :
2047 : class pass_feedback_split_functions : public gimple_opt_pass
2048 : {
2049 : public:
2050 571444 : pass_feedback_split_functions (gcc::context *ctxt)
2051 1142888 : : gimple_opt_pass (pass_data_feedback_split_functions, ctxt)
2052 : {}
2053 :
2054 : /* opt_pass methods: */
2055 : bool gate (function *) final override;
2056 438 : unsigned int execute (function *) final override
2057 : {
2058 438 : return execute_feedback_split_functions ();
2059 : }
2060 :
2061 285722 : opt_pass * clone () final override
2062 : {
2063 285722 : return new pass_feedback_split_functions (m_ctxt);
2064 : }
2065 : }; // class pass_feedback_split_functions
2066 :
2067 : bool
2068 2568 : pass_feedback_split_functions::gate (function *)
2069 : {
2070 : /* We don't need to split when profiling at all, we are producing
2071 : lousy code anyway. */
2072 2568 : return (flag_partial_inlining
2073 2568 : && (flag_branch_probabilities || flag_auto_profile));
2074 : }
2075 :
2076 : } // anon namespace
2077 :
2078 : gimple_opt_pass *
2079 285722 : make_pass_feedback_split_functions (gcc::context *ctxt)
2080 : {
2081 285722 : return new pass_feedback_split_functions (ctxt);
2082 : }
|
