Branch data Line data Source code
1 : : /* coroutine expansion and optimisation passes.
2 : :
3 : : Copyright (C) 2018-2024 Free Software Foundation, Inc.
4 : :
5 : : Contributed by Iain Sandoe <iain@sandoe.co.uk> under contract to Facebook.
6 : :
7 : : This file is part of GCC.
8 : :
9 : : GCC is free software; you can redistribute it and/or modify it under
10 : : the terms of the GNU General Public License as published by the Free
11 : : Software Foundation; either version 3, or (at your option) any later
12 : : version.
13 : :
14 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 : : for more details.
18 : :
19 : : You should have received a copy of the GNU General Public License
20 : : along with GCC; see the file COPYING3. If not see
21 : : <http://www.gnu.org/licenses/>. */
22 : :
23 : : #include "config.h"
24 : : #include "system.h"
25 : : #include "coretypes.h"
26 : : #include "backend.h"
27 : : #include "target.h"
28 : : #include "tree.h"
29 : : #include "gimple.h"
30 : : #include "tree-pass.h"
31 : : #include "ssa.h"
32 : : #include "calls.h"
33 : : #include "cgraph.h"
34 : : #include "pretty-print.h"
35 : : #include "diagnostic-core.h"
36 : : #include "fold-const.h"
37 : : #include "internal-fn.h"
38 : : #include "langhooks.h"
39 : : #include "gimplify.h"
40 : : #include "gimple-iterator.h"
41 : : #include "gimplify-me.h"
42 : : #include "gimple-walk.h"
43 : : #include "gimple-fold.h"
44 : : #include "tree-cfg.h"
45 : : #include "tree-into-ssa.h"
46 : : #include "tree-ssa-propagate.h"
47 : : #include "gimple-pretty-print.h"
48 : : #include "cfghooks.h"
49 : :
50 : : /* Here we:
51 : : * lower the internal function that implements an exit from scope.
52 : : * expand the builtins that are used to implement the library
53 : : interfaces to the coroutine frame. */
54 : :
55 : : static tree
56 : 7223795 : lower_coro_builtin (gimple_stmt_iterator *gsi, bool *handled_ops_p,
57 : : struct walk_stmt_info *wi ATTRIBUTE_UNUSED)
58 : : {
59 : 7223795 : gimple *stmt = gsi_stmt (*gsi);
60 : 7223795 : *handled_ops_p = !gimple_has_substatements (stmt);
61 : :
62 : 7223795 : if (gimple_code (stmt) != GIMPLE_CALL)
63 : : return NULL_TREE;
64 : :
65 : : /* This internal function implements an exit from scope without
66 : : performing any cleanups; it jumps directly to the label provided. */
67 : 1341873 : if (gimple_call_internal_p (stmt)
68 : 1341873 : && gimple_call_internal_fn (stmt) == IFN_CO_SUSPN)
69 : : {
70 : 3661 : tree dest = TREE_OPERAND (gimple_call_arg (stmt, 0), 0);
71 : 3661 : ggoto *g = gimple_build_goto (dest);
72 : 3661 : gsi_replace (gsi, g, /* do EH */ false);
73 : 3661 : *handled_ops_p = true;
74 : 3661 : return NULL_TREE;
75 : : }
76 : :
77 : 1338212 : tree decl = gimple_call_fndecl (stmt);
78 : 1338212 : if (!decl || !fndecl_built_in_p (decl, BUILT_IN_NORMAL))
79 : : return NULL_TREE;
80 : :
81 : : /* The remaining builtins implement the library interfaces to the coro
82 : : frame. */
83 : 213707 : unsigned call_idx = 0;
84 : :
85 : 213707 : switch (DECL_FUNCTION_CODE (decl))
86 : : {
87 : : default:
88 : : break;
89 : 1878 : case BUILT_IN_CORO_PROMISE:
90 : 1878 : {
91 : : /* If we are discarding this, then skip it; the function has no
92 : : side-effects. */
93 : 1878 : tree lhs = gimple_call_lhs (stmt);
94 : 1878 : if (!lhs)
95 : : {
96 : 0 : gsi_remove (gsi, true);
97 : 0 : *handled_ops_p = true;
98 : 0 : return NULL_TREE;
99 : : }
100 : : /* The coro frame starts with two pointers (to the resume and
101 : : destroy() functions). These are followed by the promise which
102 : : is aligned as per type [or user attribute].
103 : : The input pointer is the first argument.
104 : : The promise alignment is the second and the third is a bool
105 : : that is true when we are converting from a promise ptr to a
106 : : frame pointer, and false for the inverse. */
107 : 1878 : tree ptr = gimple_call_arg (stmt, 0);
108 : 1878 : tree align_t = gimple_call_arg (stmt, 1);
109 : 1878 : tree from = gimple_call_arg (stmt, 2);
110 : 1878 : gcc_checking_assert (TREE_CODE (align_t) == INTEGER_CST);
111 : 1878 : gcc_checking_assert (TREE_CODE (from) == INTEGER_CST);
112 : 1878 : bool dir = wi::to_wide (from) != 0;
113 : 1878 : HOST_WIDE_INT promise_align = TREE_INT_CST_LOW (align_t);
114 : 1878 : HOST_WIDE_INT psize =
115 : 1878 : TREE_INT_CST_LOW (TYPE_SIZE_UNIT (ptr_type_node));
116 : 1878 : HOST_WIDE_INT align = TYPE_ALIGN_UNIT (ptr_type_node);
117 : 1878 : align = MAX (align, promise_align);
118 : 1878 : psize *= 2; /* Start with two pointers. */
119 : 1878 : psize = ROUND_UP (psize, align);
120 : 1878 : HOST_WIDE_INT offs = dir ? -psize : psize;
121 : 1878 : tree repl = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
122 : 1878 : size_int (offs));
123 : 1878 : gassign *grpl = gimple_build_assign (lhs, repl);
124 : 1878 : gsi_replace (gsi, grpl, true);
125 : 1878 : *handled_ops_p = true;
126 : : }
127 : 1878 : break;
128 : 899 : case BUILT_IN_CORO_DESTROY:
129 : 899 : call_idx = 1;
130 : : /* FALLTHROUGH */
131 : 3084 : case BUILT_IN_CORO_RESUME:
132 : 3084 : {
133 : 3084 : tree ptr = gimple_call_arg (stmt, 0); /* frame ptr. */
134 : 3084 : HOST_WIDE_INT psize =
135 : 3084 : TREE_INT_CST_LOW (TYPE_SIZE_UNIT (ptr_type_node));
136 : 3084 : HOST_WIDE_INT offset = call_idx * psize;
137 : 3084 : tree fntype = TREE_TYPE (decl);
138 : 3084 : tree fntype_ptr = build_pointer_type (fntype);
139 : 3084 : tree fntype_ppp = build_pointer_type (fntype_ptr);
140 : 3084 : tree indirect = fold_build2 (MEM_REF, fntype_ptr, ptr,
141 : : build_int_cst (fntype_ppp, offset));
142 : 3084 : tree f_ptr_tmp = make_ssa_name (TYPE_MAIN_VARIANT (fntype_ptr));
143 : 3084 : gassign *get_fptr = gimple_build_assign (f_ptr_tmp, indirect);
144 : 3084 : gsi_insert_before (gsi, get_fptr, GSI_SAME_STMT);
145 : 3084 : gimple_call_set_fn (static_cast<gcall *> (stmt), f_ptr_tmp);
146 : 3084 : *handled_ops_p = true;
147 : : }
148 : 3084 : break;
149 : 812 : case BUILT_IN_CORO_DONE:
150 : 812 : {
151 : : /* If we are discarding this, then skip it; the function has no
152 : : side-effects. */
153 : 812 : tree lhs = gimple_call_lhs (stmt);
154 : 812 : if (!lhs)
155 : : {
156 : 0 : gsi_remove (gsi, true);
157 : 0 : *handled_ops_p = true;
158 : 0 : return NULL_TREE;
159 : : }
160 : : /* When we're done, the resume fn is set to NULL. */
161 : 812 : tree ptr = gimple_call_arg (stmt, 0); /* frame ptr. */
162 : 812 : tree vpp = build_pointer_type (ptr_type_node);
163 : 812 : tree indirect
164 : 812 : = fold_build2 (MEM_REF, vpp, ptr, build_int_cst (vpp, 0));
165 : 812 : tree d_ptr_tmp = make_ssa_name (ptr_type_node);
166 : 812 : gassign *get_dptr = gimple_build_assign (d_ptr_tmp, indirect);
167 : 812 : gsi_insert_before (gsi, get_dptr, GSI_SAME_STMT);
168 : 812 : tree done = fold_build2 (EQ_EXPR, boolean_type_node, d_ptr_tmp,
169 : : null_pointer_node);
170 : 812 : gassign *get_res = gimple_build_assign (lhs, done);
171 : 812 : gsi_replace (gsi, get_res, true);
172 : 812 : *handled_ops_p = true;
173 : : }
174 : 812 : break;
175 : : }
176 : : return NULL_TREE;
177 : : }
178 : :
179 : : /* Main entry point for lowering coroutine FE builtins. */
180 : :
181 : : static unsigned int
182 : 419585 : execute_lower_coro_builtins (void)
183 : : {
184 : 419585 : struct walk_stmt_info wi;
185 : 419585 : gimple_seq body;
186 : :
187 : 419585 : body = gimple_body (current_function_decl);
188 : 419585 : memset (&wi, 0, sizeof (wi));
189 : 419585 : walk_gimple_seq_mod (&body, lower_coro_builtin, NULL, &wi);
190 : 419585 : gimple_set_body (current_function_decl, body);
191 : :
192 : 419585 : return 0;
193 : : }
194 : :
195 : : namespace {
196 : :
197 : : const pass_data pass_data_coroutine_lower_builtins = {
198 : : GIMPLE_PASS, /* type */
199 : : "coro-lower-builtins", /* name */
200 : : OPTGROUP_NONE, /* optinfo_flags */
201 : : TV_NONE, /* tv_id */
202 : : 0, /* properties_required */
203 : : 0, /* properties_provided */
204 : : 0, /* properties_destroyed */
205 : : 0, /* todo_flags_start */
206 : : 0 /* todo_flags_finish */
207 : : };
208 : :
209 : : class pass_coroutine_lower_builtins : public gimple_opt_pass
210 : : {
211 : : public:
212 : 279404 : pass_coroutine_lower_builtins (gcc::context *ctxt)
213 : 558808 : : gimple_opt_pass (pass_data_coroutine_lower_builtins, ctxt)
214 : : {}
215 : :
216 : : /* opt_pass methods: */
217 : 2695795 : bool gate (function *) final override { return flag_coroutines; };
218 : :
219 : 419585 : unsigned int execute (function *f ATTRIBUTE_UNUSED) final override
220 : : {
221 : 419585 : return execute_lower_coro_builtins ();
222 : : }
223 : :
224 : : }; // class pass_coroutine_lower_builtins
225 : :
226 : : } // namespace
227 : :
228 : : gimple_opt_pass *
229 : 279404 : make_pass_coroutine_lower_builtins (gcc::context *ctxt)
230 : : {
231 : 279404 : return new pass_coroutine_lower_builtins (ctxt);
232 : : }
233 : :
234 : : /* Expand the remaining coroutine IFNs.
235 : :
236 : : In the front end we construct a single actor function that contains
237 : : the coroutine state machine.
238 : :
239 : : The actor function has three entry conditions:
240 : : 1. from the ramp, resume point 0 - to initial-suspend.
241 : : 2. when resume () is executed (resume point N).
242 : : 3. from the destroy () shim when that is executed.
243 : :
244 : : The actor function begins with two dispatchers; one for resume and
245 : : one for destroy (where the initial entry from the ramp is a special-
246 : : case of resume point 0).
247 : :
248 : : Each suspend point and each dispatch entry is marked with an IFN such
249 : : that we can connect the relevant dispatchers to their target labels.
250 : :
251 : : So, if we have:
252 : :
253 : : CO_YIELD (NUM, FINAL, RES_LAB, DEST_LAB, FRAME_PTR)
254 : :
255 : : This is await point NUM, and is the final await if FINAL is non-zero.
256 : : The resume point is RES_LAB, and the destroy point is DEST_LAB.
257 : :
258 : : We expect to find a CO_ACTOR (NUM) in the resume dispatcher and a
259 : : CO_ACTOR (NUM+1) in the destroy dispatcher.
260 : :
261 : : Initially, the intent of keeping the resume and destroy paths together
262 : : is that the conditionals controlling them are identical, and thus there
263 : : would be duplication of any optimisation of those paths if the split
264 : : were earlier.
265 : :
266 : : Subsequent inlining of the actor (and DCE) is then able to extract the
267 : : resume and destroy paths as separate functions if that is found
268 : : profitable by the optimisers.
269 : :
270 : : Once we have remade the connections to their correct postions, we elide
271 : : the labels that the front end inserted. */
272 : :
273 : : static void
274 : 7262 : move_edge_and_update (edge e, basic_block old_bb, basic_block new_bb)
275 : : {
276 : 7262 : if (dump_file)
277 : 0 : fprintf (dump_file, "redirecting edge from bb %u to bb %u\n", old_bb->index,
278 : : new_bb->index);
279 : :
280 : 7262 : e = redirect_edge_and_branch (e, new_bb);
281 : 7262 : if (!e && dump_file)
282 : 0 : fprintf (dump_file, "failed to redirect edge .. \n");
283 : :
284 : : /* Die if we failed. */
285 : 7262 : gcc_checking_assert (e);
286 : 7262 : }
287 : :
288 : : static unsigned int
289 : 3955 : execute_early_expand_coro_ifns (void)
290 : : {
291 : : /* Don't rebuild stuff unless we have to. */
292 : 3955 : unsigned int todoflags = 0;
293 : 3955 : bool changed = false;
294 : : /* Some of the possible YIELD points will hopefully have been removed by
295 : : earlier optimisations; record the ones that are still present. */
296 : 3955 : hash_map<int_hash<HOST_WIDE_INT, -1, -2>, tree> destinations;
297 : : /* List of dispatch points to update. */
298 : 3955 : auto_vec<gimple_stmt_iterator, 16> actor_worklist;
299 : 3955 : basic_block bb;
300 : 3955 : gimple_stmt_iterator gsi;
301 : :
302 : 55994 : FOR_EACH_BB_FN (bb, cfun)
303 : 293224 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
304 : : {
305 : 189146 : gimple *stmt = gsi_stmt (gsi);
306 : :
307 : : /* Tell the user about 'alloca', we don't support it yet. */
308 : 189146 : if (gimple_alloca_call_p (stmt))
309 : : {
310 : 2 : sorry_at (gimple_location (stmt),
311 : : "%<alloca%> is not yet supported in coroutines");
312 : 2 : gsi_next (&gsi);
313 : 2 : continue;
314 : : }
315 : :
316 : 189144 : if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
317 : : {
318 : 176856 : gsi_next (&gsi);
319 : 176856 : continue;
320 : : }
321 : 12288 : switch (gimple_call_internal_fn (stmt))
322 : : {
323 : 1308 : case IFN_CO_FRAME:
324 : 1308 : {
325 : : /* This internal function is a placeholder for the frame
326 : : size. In principle, we might lower it later (after some
327 : : optimisation had reduced the frame size). At present,
328 : : without any such optimisation, we just set it here. */
329 : 1308 : tree lhs = gimple_call_lhs (stmt);
330 : 1308 : tree size = gimple_call_arg (stmt, 0);
331 : : /* Right now, this is a trivial operation - copy through
332 : : the size computed during initial layout. */
333 : 1308 : gassign *grpl = gimple_build_assign (lhs, size);
334 : 1308 : gsi_replace (&gsi, grpl, true);
335 : 1308 : gsi_next (&gsi);
336 : : }
337 : 1308 : break;
338 : 7264 : case IFN_CO_ACTOR:
339 : 7264 : changed = true;
340 : 7264 : actor_worklist.safe_push (gsi); /* Save for later. */
341 : 7264 : gsi_next (&gsi);
342 : 7264 : break;
343 : 3631 : case IFN_CO_YIELD:
344 : 3631 : {
345 : 3631 : changed = true;
346 : : /* .CO_YIELD (NUM, FINAL, RES_LAB, DEST_LAB, FRAME_PTR);
347 : : NUM = await number.
348 : : FINAL = 1 if this is the final_suspend() await.
349 : : RES_LAB = resume point label.
350 : : DEST_LAB = destroy point label.
351 : : FRAME_PTR = is a null pointer with the type of the coro
352 : : frame, so that we can resize, if needed. */
353 : 3631 : if (dump_file)
354 : 0 : fprintf (dump_file, "saw CO_YIELD in BB %u\n", bb->index);
355 : 3631 : tree num = gimple_call_arg (stmt, 0); /* yield point. */
356 : 3631 : HOST_WIDE_INT idx = TREE_INT_CST_LOW (num);
357 : 3631 : bool existed;
358 : 3631 : tree res_tgt = TREE_OPERAND (gimple_call_arg (stmt, 2), 0);
359 : 3631 : tree &res_dest = destinations.get_or_insert (idx, &existed);
360 : 3631 : if (existed && dump_file)
361 : : {
362 : 0 : fprintf (
363 : : dump_file,
364 : : "duplicate YIELD RESUME point (" HOST_WIDE_INT_PRINT_DEC
365 : : ") ?\n",
366 : : idx);
367 : 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_VOPS|TDF_MEMSYMS);
368 : : }
369 : : else
370 : 3631 : res_dest = res_tgt;
371 : 3631 : tree dst_tgt = TREE_OPERAND (gimple_call_arg (stmt, 3), 0);
372 : 3631 : tree &dst_dest = destinations.get_or_insert (idx + 1, &existed);
373 : 3631 : if (existed && dump_file)
374 : : {
375 : 0 : fprintf (
376 : : dump_file,
377 : : "duplicate YIELD DESTROY point (" HOST_WIDE_INT_PRINT_DEC
378 : : ") ?\n",
379 : : idx + 1);
380 : 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_VOPS|TDF_MEMSYMS);
381 : : }
382 : : else
383 : 3631 : dst_dest = dst_tgt;
384 : : /* lose the co_yield. */
385 : 3631 : gsi_remove (&gsi, true);
386 : 3631 : stmt = gsi_stmt (gsi); /* next. */
387 : : /* lose the copy present at O0. */
388 : 3631 : if (is_gimple_assign (stmt))
389 : : {
390 : 0 : gsi_remove (&gsi, true);
391 : 0 : stmt = gsi_stmt (gsi);
392 : : }
393 : : /* Simplify the switch or if following. */
394 : 3631 : if (gswitch *gsw = dyn_cast<gswitch *> (stmt))
395 : : {
396 : 3631 : gimple_switch_set_index (gsw, integer_zero_node);
397 : 3631 : fold_stmt (&gsi);
398 : : }
399 : 0 : else if (gcond *gif = dyn_cast<gcond *> (stmt))
400 : : {
401 : 0 : if (gimple_cond_code (gif) == EQ_EXPR)
402 : 0 : gimple_cond_make_true (gif);
403 : : else
404 : 0 : gimple_cond_make_false (gif);
405 : 0 : fold_stmt (&gsi);
406 : : }
407 : 0 : else if (dump_file)
408 : 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_VOPS|TDF_MEMSYMS);
409 : 3631 : if (gsi_end_p (gsi))
410 : : break;
411 : 3631 : continue;
412 : 3631 : }
413 : 85 : default:
414 : 85 : gsi_next (&gsi);
415 : 85 : break;
416 : : }
417 : : }
418 : :
419 : 3955 : if (!changed)
420 : : {
421 : 2641 : if (dump_file)
422 : 0 : fprintf (dump_file, "coro: nothing to do\n");
423 : 2641 : return todoflags;
424 : : }
425 : :
426 : 8578 : while (!actor_worklist.is_empty ())
427 : : {
428 : 7264 : gsi = actor_worklist.pop ();
429 : 7264 : gimple *stmt = gsi_stmt (gsi);
430 : 7264 : gcc_checking_assert (is_gimple_call (stmt)
431 : : && gimple_call_internal_p (stmt)
432 : : && gimple_call_internal_fn (stmt) == IFN_CO_ACTOR);
433 : 7264 : bb = gsi_bb (gsi);
434 : 7264 : HOST_WIDE_INT idx = TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
435 : 7264 : tree *seen = destinations.get (idx);
436 : 7264 : changed = true;
437 : :
438 : 7264 : if (dump_file)
439 : 0 : fprintf (dump_file, "saw CO_ACTOR in BB %u\n", bb->index);
440 : :
441 : 7264 : if (!seen)
442 : : {
443 : : /* If we never saw this index, it means that the CO_YIELD
444 : : associated was elided during earlier optimisations, so we
445 : : don't need to fix up the switch targets. */
446 : 2 : if (dump_file)
447 : 0 : fprintf (dump_file, "yield point " HOST_WIDE_INT_PRINT_DEC
448 : : " not used, removing it .. \n", idx);
449 : 2 : gsi_remove (&gsi, true);
450 : 2 : release_defs (stmt);
451 : : }
452 : : else
453 : : {
454 : : /* So we need to switch the target of this switch case to the
455 : : relevant BB. */
456 : 7262 : basic_block new_bb = label_to_block (cfun, *seen);
457 : : /* We expect the block we're modifying to contain a single
458 : : CO_ACTOR() followed by a goto <switch default bb>. */
459 : 7262 : gcc_checking_assert (EDGE_COUNT (bb->succs) == 1);
460 : 7262 : edge e;
461 : 7262 : edge_iterator ei;
462 : 14524 : FOR_EACH_EDGE (e, ei, bb->succs)
463 : : {
464 : 7262 : basic_block old_bb = e->dest;
465 : 7262 : move_edge_and_update (e, old_bb, new_bb);
466 : : }
467 : 7262 : gsi_remove (&gsi, true);
468 : : }
469 : : }
470 : :
471 : : /* Changed the CFG. */
472 : 3955 : todoflags |= TODO_cleanup_cfg;
473 : : return todoflags;
474 : 3955 : }
475 : :
476 : : namespace {
477 : :
478 : : const pass_data pass_data_coroutine_early_expand_ifns = {
479 : : GIMPLE_PASS, /* type */
480 : : "coro-early-expand-ifns", /* name */
481 : : OPTGROUP_NONE, /* optinfo_flags */
482 : : TV_NONE, /* tv_id */
483 : : (PROP_cfg), /* properties_required */
484 : : 0, /* properties_provided */
485 : : 0, /* properties_destroyed */
486 : : 0, /* todo_flags_start */
487 : : 0 /* todo_flags_finish, set this in the fn. */
488 : : };
489 : :
490 : : class pass_coroutine_early_expand_ifns : public gimple_opt_pass
491 : : {
492 : : public:
493 : 279404 : pass_coroutine_early_expand_ifns (gcc::context *ctxt)
494 : 558808 : : gimple_opt_pass (pass_data_coroutine_early_expand_ifns, ctxt)
495 : : {}
496 : :
497 : : /* opt_pass methods: */
498 : 2695795 : bool gate (function *f) final override
499 : : {
500 : 2695795 : return flag_coroutines && f->coroutine_component;
501 : : }
502 : :
503 : 3955 : unsigned int execute (function *f ATTRIBUTE_UNUSED) final override
504 : : {
505 : 3955 : return execute_early_expand_coro_ifns ();
506 : : }
507 : :
508 : : }; // class pass_coroutine_expand_ifns
509 : :
510 : : } // namespace
511 : :
512 : : gimple_opt_pass *
513 : 279404 : make_pass_coroutine_early_expand_ifns (gcc::context *ctxt)
514 : : {
515 : 279404 : return new pass_coroutine_early_expand_ifns (ctxt);
516 : : }
|
