Line data Source code
1 : /* coroutine expansion and optimisation passes.
2 :
3 : Copyright (C) 2018-2026 Free Software Foundation, Inc.
4 :
5 : Contributed by Iain Sandoe <iain@sandoe.co.uk> under contract to Facebook.
6 :
7 : This file is part of GCC.
8 :
9 : GCC is free software; you can redistribute it and/or modify it under
10 : the terms of the GNU General Public License as published by the Free
11 : Software Foundation; either version 3, or (at your option) any later
12 : version.
13 :
14 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 : for more details.
18 :
19 : You should have received a copy of the GNU General Public License
20 : along with GCC; see the file COPYING3. If not see
21 : <http://www.gnu.org/licenses/>. */
22 :
23 : #include "config.h"
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "target.h"
28 : #include "tree.h"
29 : #include "gimple.h"
30 : #include "tree-pass.h"
31 : #include "ssa.h"
32 : #include "calls.h"
33 : #include "cgraph.h"
34 : #include "pretty-print.h"
35 : #include "diagnostic-core.h"
36 : #include "fold-const.h"
37 : #include "internal-fn.h"
38 : #include "langhooks.h"
39 : #include "gimplify.h"
40 : #include "gimple-iterator.h"
41 : #include "gimplify-me.h"
42 : #include "gimple-walk.h"
43 : #include "gimple-fold.h"
44 : #include "tree-cfg.h"
45 : #include "tree-into-ssa.h"
46 : #include "tree-ssa-propagate.h"
47 : #include "gimple-pretty-print.h"
48 : #include "cfghooks.h"
49 :
50 : /* Here we:
51 : * lower the internal function that implements an exit from scope.
52 : * expand the builtins that are used to implement the library
53 : interfaces to the coroutine frame. */
54 :
55 : static tree
56 26226555 : lower_coro_builtin (gimple_stmt_iterator *gsi, bool *handled_ops_p,
57 : struct walk_stmt_info *wi ATTRIBUTE_UNUSED)
58 : {
59 26226555 : gimple *stmt = gsi_stmt (*gsi);
60 26226555 : *handled_ops_p = !gimple_has_substatements (stmt);
61 :
62 26226555 : if (gimple_code (stmt) != GIMPLE_CALL)
63 : return NULL_TREE;
64 :
65 : /* This internal function implements an exit from scope without
66 : performing any cleanups; it jumps directly to the label provided. */
67 4894985 : if (gimple_call_internal_p (stmt)
68 4894985 : && gimple_call_internal_fn (stmt) == IFN_CO_SUSPN)
69 : {
70 4102 : tree dest = TREE_OPERAND (gimple_call_arg (stmt, 0), 0);
71 4102 : ggoto *g = gimple_build_goto (dest);
72 4102 : gsi_replace (gsi, g, /* do EH */ false);
73 4102 : *handled_ops_p = true;
74 4102 : return NULL_TREE;
75 : }
76 :
77 4890883 : tree decl = gimple_call_fndecl (stmt);
78 4890883 : if (!decl || !fndecl_built_in_p (decl, BUILT_IN_NORMAL))
79 : return NULL_TREE;
80 :
81 : /* The remaining builtins implement the library interfaces to the coro
82 : frame. */
83 791283 : unsigned call_idx = 0;
84 :
85 791283 : switch (DECL_FUNCTION_CODE (decl))
86 : {
87 : default:
88 : break;
89 1789 : case BUILT_IN_CORO_PROMISE:
90 1789 : {
91 : /* If we are discarding this, then skip it; the function has no
92 : side-effects. */
93 1789 : tree lhs = gimple_call_lhs (stmt);
94 1789 : if (!lhs)
95 : {
96 0 : gsi_remove (gsi, true);
97 0 : *handled_ops_p = true;
98 0 : return NULL_TREE;
99 : }
100 : /* The coro frame starts with two pointers (to the resume and
101 : destroy() functions). These are followed by the promise which
102 : is aligned as per type [or user attribute].
103 : The input pointer is the first argument.
104 : The promise alignment is the second and the third is a bool
105 : that is true when we are converting from a promise ptr to a
106 : frame pointer, and false for the inverse. */
107 1789 : tree ptr = gimple_call_arg (stmt, 0);
108 1789 : tree align_t = gimple_call_arg (stmt, 1);
109 1789 : tree from = gimple_call_arg (stmt, 2);
110 1789 : gcc_checking_assert (TREE_CODE (align_t) == INTEGER_CST);
111 1789 : gcc_checking_assert (TREE_CODE (from) == INTEGER_CST);
112 1789 : bool dir = wi::to_wide (from) != 0;
113 1789 : HOST_WIDE_INT promise_align = TREE_INT_CST_LOW (align_t);
114 1789 : HOST_WIDE_INT psize =
115 1789 : TREE_INT_CST_LOW (TYPE_SIZE_UNIT (ptr_type_node));
116 1789 : HOST_WIDE_INT align = TYPE_ALIGN_UNIT (ptr_type_node);
117 1789 : align = MAX (align, promise_align);
118 1789 : psize *= 2; /* Start with two pointers. */
119 1789 : psize = ROUND_UP (psize, align);
120 1789 : HOST_WIDE_INT offs = dir ? -psize : psize;
121 1789 : tree repl = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
122 1789 : size_int (offs));
123 1789 : gassign *grpl = gimple_build_assign (lhs, repl);
124 1789 : gsi_replace (gsi, grpl, true);
125 1789 : *handled_ops_p = true;
126 : }
127 1789 : break;
128 896 : case BUILT_IN_CORO_DESTROY:
129 896 : call_idx = 1;
130 : /* FALLTHROUGH */
131 3202 : case BUILT_IN_CORO_RESUME:
132 3202 : {
133 3202 : tree ptr = gimple_call_arg (stmt, 0); /* frame ptr. */
134 3202 : HOST_WIDE_INT psize =
135 3202 : TREE_INT_CST_LOW (TYPE_SIZE_UNIT (ptr_type_node));
136 3202 : HOST_WIDE_INT offset = call_idx * psize;
137 3202 : tree fntype = TREE_TYPE (decl);
138 3202 : tree fntype_ptr = build_pointer_type (fntype);
139 3202 : tree fntype_ppp = build_pointer_type (fntype_ptr);
140 3202 : tree indirect = fold_build2 (MEM_REF, fntype_ptr, ptr,
141 : build_int_cst (fntype_ppp, offset));
142 3202 : tree f_ptr_tmp = make_ssa_name (TYPE_MAIN_VARIANT (fntype_ptr));
143 3202 : gassign *get_fptr = gimple_build_assign (f_ptr_tmp, indirect);
144 3202 : gsi_insert_before (gsi, get_fptr, GSI_SAME_STMT);
145 3202 : gimple_call_set_fn (static_cast<gcall *> (stmt), f_ptr_tmp);
146 3202 : *handled_ops_p = true;
147 : }
148 3202 : break;
149 810 : case BUILT_IN_CORO_DONE:
150 810 : {
151 : /* If we are discarding this, then skip it; the function has no
152 : side-effects. */
153 810 : tree lhs = gimple_call_lhs (stmt);
154 810 : if (!lhs)
155 : {
156 0 : gsi_remove (gsi, true);
157 0 : *handled_ops_p = true;
158 0 : return NULL_TREE;
159 : }
160 : /* When we're done, the resume fn is set to NULL. */
161 810 : tree ptr = gimple_call_arg (stmt, 0); /* frame ptr. */
162 810 : tree vpp = build_pointer_type (ptr_type_node);
163 810 : tree indirect
164 810 : = fold_build2 (MEM_REF, vpp, ptr, build_int_cst (vpp, 0));
165 810 : tree d_ptr_tmp = make_ssa_name (ptr_type_node);
166 810 : gassign *get_dptr = gimple_build_assign (d_ptr_tmp, indirect);
167 810 : gsi_insert_before (gsi, get_dptr, GSI_SAME_STMT);
168 810 : tree done = fold_build2 (EQ_EXPR, boolean_type_node, d_ptr_tmp,
169 : null_pointer_node);
170 810 : gassign *get_res = gimple_build_assign (lhs, done);
171 810 : gsi_replace (gsi, get_res, true);
172 810 : *handled_ops_p = true;
173 : }
174 810 : break;
175 : }
176 : return NULL_TREE;
177 : }
178 :
179 : /* Main entry point for lowering coroutine FE builtins. */
180 :
181 : static unsigned int
182 1580669 : execute_lower_coro_builtins (void)
183 : {
184 1580669 : struct walk_stmt_info wi;
185 1580669 : gimple_seq body;
186 :
187 1580669 : body = gimple_body (current_function_decl);
188 1580669 : memset (&wi, 0, sizeof (wi));
189 1580669 : walk_gimple_seq_mod (&body, lower_coro_builtin, NULL, &wi);
190 1580669 : gimple_set_body (current_function_decl, body);
191 :
192 1580669 : return 0;
193 : }
194 :
195 : namespace {
196 :
197 : const pass_data pass_data_coroutine_lower_builtins = {
198 : GIMPLE_PASS, /* type */
199 : "coro-lower-builtins", /* name */
200 : OPTGROUP_NONE, /* optinfo_flags */
201 : TV_NONE, /* tv_id */
202 : 0, /* properties_required */
203 : 0, /* properties_provided */
204 : 0, /* properties_destroyed */
205 : 0, /* todo_flags_start */
206 : 0 /* todo_flags_finish */
207 : };
208 :
209 : class pass_coroutine_lower_builtins : public gimple_opt_pass
210 : {
211 : public:
212 285722 : pass_coroutine_lower_builtins (gcc::context *ctxt)
213 571444 : : gimple_opt_pass (pass_data_coroutine_lower_builtins, ctxt)
214 : {}
215 :
216 : /* opt_pass methods: */
217 2869205 : bool gate (function *) final override { return flag_coroutines; };
218 :
219 1580669 : unsigned int execute (function *f ATTRIBUTE_UNUSED) final override
220 : {
221 1580669 : return execute_lower_coro_builtins ();
222 : }
223 :
224 : }; // class pass_coroutine_lower_builtins
225 :
226 : } // namespace
227 :
228 : gimple_opt_pass *
229 285722 : make_pass_coroutine_lower_builtins (gcc::context *ctxt)
230 : {
231 285722 : return new pass_coroutine_lower_builtins (ctxt);
232 : }
233 :
234 : /* Expand the remaining coroutine IFNs.
235 :
236 : In the front end we construct a single actor function that contains
237 : the coroutine state machine.
238 :
239 : The actor function has three entry conditions:
240 : 1. from the ramp, resume point 0 - to initial-suspend.
241 : 2. when resume () is executed (resume point N).
242 : 3. from the destroy () shim when that is executed.
243 :
244 : The actor function begins with two dispatchers; one for resume and
245 : one for destroy (where the initial entry from the ramp is a special-
246 : case of resume point 0).
247 :
248 : Each suspend point and each dispatch entry is marked with an IFN such
249 : that we can connect the relevant dispatchers to their target labels.
250 :
251 : So, if we have:
252 :
253 : CO_YIELD (NUM, FINAL, RES_LAB, DEST_LAB, FRAME_PTR)
254 :
255 : This is await point NUM, and is the final await if FINAL is non-zero.
256 : The resume point is RES_LAB, and the destroy point is DEST_LAB.
257 :
258 : We expect to find a CO_ACTOR (NUM) in the resume dispatcher and a
259 : CO_ACTOR (NUM+1) in the destroy dispatcher.
260 :
261 : Initially, the intent of keeping the resume and destroy paths together
262 : is that the conditionals controlling them are identical, and thus there
263 : would be duplication of any optimisation of those paths if the split
264 : were earlier.
265 :
266 : Subsequent inlining of the actor (and DCE) is then able to extract the
267 : resume and destroy paths as separate functions if that is found
268 : profitable by the optimisers.
269 :
270 : Once we have remade the connections to their correct postions, we elide
271 : the labels that the front end inserted. */
272 :
273 : static void
274 8134 : move_edge_and_update (edge e, basic_block old_bb, basic_block new_bb)
275 : {
276 8134 : if (dump_file)
277 0 : fprintf (dump_file, "redirecting edge from bb %u to bb %u\n", old_bb->index,
278 : new_bb->index);
279 :
280 8134 : e = redirect_edge_and_branch (e, new_bb);
281 8134 : if (!e && dump_file)
282 0 : fprintf (dump_file, "failed to redirect edge .. \n");
283 :
284 : /* Die if we failed. */
285 8134 : gcc_checking_assert (e);
286 8134 : }
287 :
288 : static unsigned int
289 4496 : execute_early_expand_coro_ifns (void)
290 : {
291 : /* Don't rebuild stuff unless we have to. */
292 4496 : unsigned int todoflags = 0;
293 4496 : bool changed = false;
294 : /* Some of the possible YIELD points will hopefully have been removed by
295 : earlier optimisations; record the ones that are still present. */
296 4496 : hash_map<int_hash<HOST_WIDE_INT, -1, -2>, tree> destinations;
297 : /* List of dispatch points to update. */
298 4496 : auto_vec<gimple_stmt_iterator, 16> actor_worklist;
299 4496 : basic_block bb;
300 4496 : gimple_stmt_iterator gsi;
301 :
302 69485 : FOR_EACH_BB_FN (bb, cfun)
303 360427 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
304 : {
305 230449 : gimple *stmt = gsi_stmt (gsi);
306 :
307 : /* Tell the user about 'alloca', we don't support it yet. */
308 230449 : if (gimple_alloca_call_p (stmt))
309 : {
310 6 : sorry_at (gimple_location (stmt),
311 : "%<alloca%> is not yet supported in coroutines");
312 6 : gsi_next (&gsi);
313 6 : continue;
314 : }
315 :
316 230443 : if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
317 : {
318 216374 : gsi_next (&gsi);
319 216374 : continue;
320 : }
321 14069 : switch (gimple_call_internal_fn (stmt))
322 : {
323 1470 : case IFN_CO_FRAME:
324 1470 : {
325 : /* This internal function is a placeholder for the frame
326 : size. In principle, we might lower it later (after some
327 : optimisation had reduced the frame size). At present,
328 : without any such optimisation, we just set it here. */
329 1470 : tree lhs = gimple_call_lhs (stmt);
330 1470 : tree size = gimple_call_arg (stmt, 0);
331 : /* Right now, this is a trivial operation - copy through
332 : the size computed during initial layout. */
333 1470 : gassign *grpl = gimple_build_assign (lhs, size);
334 1470 : gsi_replace (&gsi, grpl, true);
335 1470 : gsi_next (&gsi);
336 : }
337 1470 : break;
338 8140 : case IFN_CO_ACTOR:
339 8140 : changed = true;
340 8140 : actor_worklist.safe_push (gsi); /* Save for later. */
341 8140 : gsi_next (&gsi);
342 8140 : break;
343 4067 : case IFN_CO_YIELD:
344 4067 : {
345 4067 : changed = true;
346 : /* .CO_YIELD (NUM, FINAL, RES_LAB, DEST_LAB, FRAME_PTR);
347 : NUM = await number.
348 : FINAL = 1 if this is the final_suspend() await.
349 : RES_LAB = resume point label.
350 : DEST_LAB = destroy point label.
351 : FRAME_PTR = is a null pointer with the type of the coro
352 : frame, so that we can resize, if needed. */
353 4067 : if (dump_file)
354 0 : fprintf (dump_file, "saw CO_YIELD in BB %u\n", bb->index);
355 4067 : tree num = gimple_call_arg (stmt, 0); /* yield point. */
356 4067 : HOST_WIDE_INT idx = TREE_INT_CST_LOW (num);
357 4067 : bool existed;
358 4067 : tree res_tgt = TREE_OPERAND (gimple_call_arg (stmt, 2), 0);
359 4067 : tree &res_dest = destinations.get_or_insert (idx, &existed);
360 4067 : if (existed && dump_file)
361 : {
362 0 : fprintf (
363 : dump_file,
364 : "duplicate YIELD RESUME point (" HOST_WIDE_INT_PRINT_DEC
365 : ") ?\n",
366 : idx);
367 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_VOPS|TDF_MEMSYMS);
368 : }
369 : else
370 4067 : res_dest = res_tgt;
371 4067 : tree dst_tgt = TREE_OPERAND (gimple_call_arg (stmt, 3), 0);
372 4067 : tree &dst_dest = destinations.get_or_insert (idx + 1, &existed);
373 4067 : if (existed && dump_file)
374 : {
375 0 : fprintf (
376 : dump_file,
377 : "duplicate YIELD DESTROY point (" HOST_WIDE_INT_PRINT_DEC
378 : ") ?\n",
379 : idx + 1);
380 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_VOPS|TDF_MEMSYMS);
381 : }
382 : else
383 4067 : dst_dest = dst_tgt;
384 : /* lose the co_yield. */
385 4067 : gsi_remove (&gsi, true);
386 4067 : stmt = gsi_stmt (gsi); /* next. */
387 : /* lose the copy present at O0. */
388 4067 : if (is_gimple_assign (stmt))
389 : {
390 0 : gsi_remove (&gsi, true);
391 0 : stmt = gsi_stmt (gsi);
392 : }
393 : /* Simplify the switch or if following. */
394 4067 : if (gswitch *gsw = dyn_cast<gswitch *> (stmt))
395 : {
396 4067 : gimple_switch_set_index (gsw, integer_zero_node);
397 4067 : fold_stmt (&gsi);
398 : }
399 0 : else if (gcond *gif = dyn_cast<gcond *> (stmt))
400 : {
401 0 : if (gimple_cond_code (gif) == EQ_EXPR)
402 0 : gimple_cond_make_true (gif);
403 : else
404 0 : gimple_cond_make_false (gif);
405 0 : fold_stmt (&gsi);
406 : }
407 0 : else if (dump_file)
408 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_VOPS|TDF_MEMSYMS);
409 4067 : if (gsi_end_p (gsi))
410 : break;
411 4067 : continue;
412 4067 : }
413 392 : default:
414 392 : gsi_next (&gsi);
415 392 : break;
416 : }
417 : }
418 :
419 4496 : if (!changed)
420 : {
421 3015 : if (dump_file)
422 0 : fprintf (dump_file, "coro: nothing to do\n");
423 3015 : return todoflags;
424 : }
425 :
426 9621 : while (!actor_worklist.is_empty ())
427 : {
428 8140 : gsi = actor_worklist.pop ();
429 8140 : gimple *stmt = gsi_stmt (gsi);
430 8140 : gcc_checking_assert (is_gimple_call (stmt)
431 : && gimple_call_internal_p (stmt)
432 : && gimple_call_internal_fn (stmt) == IFN_CO_ACTOR);
433 8140 : bb = gsi_bb (gsi);
434 8140 : HOST_WIDE_INT idx = TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
435 8140 : tree *seen = destinations.get (idx);
436 8140 : changed = true;
437 :
438 8140 : if (dump_file)
439 0 : fprintf (dump_file, "saw CO_ACTOR in BB %u\n", bb->index);
440 :
441 8140 : if (!seen)
442 : {
443 : /* If we never saw this index, it means that the CO_YIELD
444 : associated was elided during earlier optimisations, so we
445 : don't need to fix up the switch targets. */
446 6 : if (dump_file)
447 0 : fprintf (dump_file, "yield point " HOST_WIDE_INT_PRINT_DEC
448 : " not used, removing it .. \n", idx);
449 6 : gsi_remove (&gsi, true);
450 6 : release_defs (stmt);
451 : }
452 : else
453 : {
454 : /* So we need to switch the target of this switch case to the
455 : relevant BB. */
456 8134 : basic_block new_bb = label_to_block (cfun, *seen);
457 : /* We expect the block we're modifying to contain a single
458 : CO_ACTOR() followed by a goto <switch default bb>. */
459 8134 : gcc_checking_assert (EDGE_COUNT (bb->succs) == 1);
460 8134 : edge e;
461 8134 : edge_iterator ei;
462 16268 : FOR_EACH_EDGE (e, ei, bb->succs)
463 : {
464 8134 : basic_block old_bb = e->dest;
465 8134 : move_edge_and_update (e, old_bb, new_bb);
466 : }
467 8134 : gsi_remove (&gsi, true);
468 : }
469 : }
470 :
471 : /* Changed the CFG. */
472 4496 : todoflags |= TODO_cleanup_cfg;
473 : return todoflags;
474 4496 : }
475 :
476 : namespace {
477 :
478 : const pass_data pass_data_coroutine_early_expand_ifns = {
479 : GIMPLE_PASS, /* type */
480 : "coro-early-expand-ifns", /* name */
481 : OPTGROUP_NONE, /* optinfo_flags */
482 : TV_NONE, /* tv_id */
483 : (PROP_cfg), /* properties_required */
484 : 0, /* properties_provided */
485 : 0, /* properties_destroyed */
486 : 0, /* todo_flags_start */
487 : 0 /* todo_flags_finish, set this in the fn. */
488 : };
489 :
490 : class pass_coroutine_early_expand_ifns : public gimple_opt_pass
491 : {
492 : public:
493 285722 : pass_coroutine_early_expand_ifns (gcc::context *ctxt)
494 571444 : : gimple_opt_pass (pass_data_coroutine_early_expand_ifns, ctxt)
495 : {}
496 :
497 : /* opt_pass methods: */
498 2869205 : bool gate (function *f) final override
499 : {
500 2869205 : return flag_coroutines && f->coroutine_component;
501 : }
502 :
503 4496 : unsigned int execute (function *f ATTRIBUTE_UNUSED) final override
504 : {
505 4496 : return execute_early_expand_coro_ifns ();
506 : }
507 :
508 : }; // class pass_coroutine_expand_ifns
509 :
510 : } // namespace
511 :
512 : gimple_opt_pass *
513 285722 : make_pass_coroutine_early_expand_ifns (gcc::context *ctxt)
514 : {
515 285722 : return new pass_coroutine_early_expand_ifns (ctxt);
516 : }
|
