Line data Source code
1 : /* OpenACC worker partitioning via middle end neutering/broadcasting scheme
2 :
3 : Copyright (C) 2015-2026 Free Software Foundation, Inc.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it
8 : under the terms of the GNU General Public License as published
9 : by the Free Software Foundation; either version 3, or (at your
10 : option) any later version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT
13 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 : or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
15 : License for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "tree-pass.h"
29 : #include "ssa.h"
30 : #include "cgraph.h"
31 : #include "pretty-print.h"
32 : #include "fold-const.h"
33 : #include "gimplify.h"
34 : #include "gimple-iterator.h"
35 : #include "gimple-walk.h"
36 : #include "tree-inline.h"
37 : #include "langhooks.h"
38 : #include "omp-general.h"
39 : #include "omp-low.h"
40 : #include "gimple-pretty-print.h"
41 : #include "cfghooks.h"
42 : #include "insn-config.h"
43 : #include "recog.h"
44 : #include "internal-fn.h"
45 : #include "bitmap.h"
46 : #include "tree-nested.h"
47 : #include "stor-layout.h"
48 : #include "tree-ssa-threadupdate.h"
49 : #include "tree-into-ssa.h"
50 : #include "splay-tree.h"
51 : #include "target.h"
52 : #include "cfgloop.h"
53 : #include "tree-cfg.h"
54 : #include "omp-offload.h"
55 : #include "attribs.h"
56 : #include "targhooks.h"
57 : #include "diagnostic-core.h"
58 :
59 : /* Loop structure of the function. The entire function is described as
60 : a NULL loop. */
61 : /* Adapted from 'gcc/config/nvptx/nvptx.cc:struct parallel'. */
62 :
63 : struct parallel_g
64 : {
65 : /* Parent parallel. */
66 : parallel_g *parent;
67 :
68 : /* Next sibling parallel. */
69 : parallel_g *next;
70 :
71 : /* First child parallel. */
72 : parallel_g *inner;
73 :
74 : /* Partitioning mask of the parallel. */
75 : unsigned mask;
76 :
77 : /* Partitioning used within inner parallels. */
78 : unsigned inner_mask;
79 :
80 : /* Location of parallel forked and join. The forked is the first
81 : block in the parallel and the join is the first block after of
82 : the partition. */
83 : basic_block forked_block;
84 : basic_block join_block;
85 :
86 : gimple *forked_stmt;
87 : gimple *join_stmt;
88 :
89 : gimple *fork_stmt;
90 : gimple *joining_stmt;
91 :
92 : /* Basic blocks in this parallel, but not in child parallels. The
93 : FORKED and JOINING blocks are in the partition. The FORK and JOIN
94 : blocks are not. */
95 : auto_vec<basic_block> blocks;
96 :
97 : tree record_type;
98 : tree sender_decl;
99 : tree receiver_decl;
100 :
101 : public:
102 : parallel_g (parallel_g *parent, unsigned mode);
103 : ~parallel_g ();
104 : };
105 :
106 : /* Constructor links the new parallel into it's parent's chain of
107 : children. */
108 :
109 0 : parallel_g::parallel_g (parallel_g *parent_, unsigned mask_)
110 0 : :parent (parent_), next (0), inner (0), mask (mask_), inner_mask (0)
111 : {
112 0 : forked_block = join_block = 0;
113 0 : forked_stmt = join_stmt = NULL;
114 0 : fork_stmt = joining_stmt = NULL;
115 :
116 0 : record_type = NULL_TREE;
117 0 : sender_decl = NULL_TREE;
118 0 : receiver_decl = NULL_TREE;
119 :
120 0 : if (parent)
121 : {
122 0 : next = parent->inner;
123 0 : parent->inner = this;
124 : }
125 0 : }
126 :
127 0 : parallel_g::~parallel_g ()
128 : {
129 0 : delete inner;
130 0 : delete next;
131 0 : }
132 :
133 : static bool
134 0 : local_var_based_p (tree decl)
135 : {
136 0 : switch (TREE_CODE (decl))
137 : {
138 0 : case VAR_DECL:
139 0 : return !is_global_var (decl);
140 :
141 0 : case COMPONENT_REF:
142 0 : case BIT_FIELD_REF:
143 0 : case ARRAY_REF:
144 0 : return local_var_based_p (TREE_OPERAND (decl, 0));
145 :
146 : default:
147 : return false;
148 : }
149 : }
150 :
151 : /* Map of basic blocks to gimple stmts. */
152 : typedef hash_map<basic_block, gimple *> bb_stmt_map_t;
153 :
154 : /* Calls to OpenACC routines are made by all workers/wavefronts/warps, since
155 : the routine likely contains partitioned loops (else will do its own
156 : neutering and variable propagation). Return TRUE if a function call CALL
157 : should be made in (worker) single mode instead, rather than redundant
158 : mode. */
159 :
160 : static bool
161 0 : omp_sese_active_worker_call (gcall *call)
162 : {
163 : #define GOMP_DIM_SEQ GOMP_DIM_MAX
164 0 : tree fndecl = gimple_call_fndecl (call);
165 :
166 0 : if (!fndecl)
167 : return true;
168 :
169 0 : tree attrs = oacc_get_fn_attrib (fndecl);
170 :
171 0 : if (!attrs)
172 : return true;
173 :
174 0 : int level = oacc_fn_attrib_level (attrs);
175 :
176 : /* Neither regular functions nor "seq" routines should be run by all threads
177 : in worker-single mode. */
178 0 : return level == -1 || level == GOMP_DIM_SEQ;
179 : #undef GOMP_DIM_SEQ
180 : }
181 :
182 : /* Split basic blocks such that each forked and join unspecs are at
183 : the start of their basic blocks. Thus afterwards each block will
184 : have a single partitioning mode. We also do the same for return
185 : insns, as they are executed by every thread. Return the
186 : partitioning mode of the function as a whole. Populate MAP with
187 : head and tail blocks. We also clear the BB visited flag, which is
188 : used when finding partitions. */
189 : /* Adapted from 'gcc/config/nvptx/nvptx.cc:nvptx_split_blocks'. */
190 :
191 : static void
192 0 : omp_sese_split_blocks (bb_stmt_map_t *map)
193 : {
194 0 : auto_vec<gimple *> worklist;
195 0 : basic_block block;
196 :
197 : /* Locate all the reorg instructions of interest. */
198 0 : FOR_ALL_BB_FN (block, cfun)
199 : {
200 : /* Clear visited flag, for use by parallel locator */
201 0 : block->flags &= ~BB_VISITED;
202 :
203 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (block);
204 0 : !gsi_end_p (gsi);
205 0 : gsi_next (&gsi))
206 : {
207 0 : gimple *stmt = gsi_stmt (gsi);
208 :
209 0 : if (gimple_call_internal_p (stmt, IFN_UNIQUE))
210 : {
211 0 : enum ifn_unique_kind k = ((enum ifn_unique_kind)
212 0 : TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
213 :
214 0 : if (k == IFN_UNIQUE_OACC_JOIN)
215 0 : worklist.safe_push (stmt);
216 0 : else if (k == IFN_UNIQUE_OACC_FORK)
217 : {
218 0 : gcc_assert (gsi_one_before_end_p (gsi));
219 0 : basic_block forked_block = single_succ (block);
220 0 : gimple_stmt_iterator gsi2 = gsi_start_bb (forked_block);
221 :
222 : /* We push a NOP as a placeholder for the "forked" stmt.
223 : This is then recognized in omp_sese_find_par. */
224 0 : gimple *nop = gimple_build_nop ();
225 0 : gsi_insert_before (&gsi2, nop, GSI_SAME_STMT);
226 :
227 0 : worklist.safe_push (nop);
228 : }
229 : }
230 0 : else if (gimple_code (stmt) == GIMPLE_RETURN
231 0 : || gimple_code (stmt) == GIMPLE_COND
232 0 : || gimple_code (stmt) == GIMPLE_SWITCH
233 0 : || (gimple_code (stmt) == GIMPLE_CALL
234 0 : && !gimple_call_internal_p (stmt)
235 0 : && !omp_sese_active_worker_call (as_a <gcall *> (stmt))))
236 0 : worklist.safe_push (stmt);
237 0 : else if (is_gimple_assign (stmt))
238 : {
239 0 : tree lhs = gimple_assign_lhs (stmt);
240 :
241 : /* Force assignments to components/fields/elements of local
242 : aggregates into fully-partitioned (redundant) mode. This
243 : avoids having to broadcast the whole aggregate. The RHS of
244 : the assignment will be propagated using the normal
245 : mechanism. */
246 :
247 0 : switch (TREE_CODE (lhs))
248 : {
249 0 : case COMPONENT_REF:
250 0 : case BIT_FIELD_REF:
251 0 : case ARRAY_REF:
252 0 : {
253 0 : tree aggr = TREE_OPERAND (lhs, 0);
254 :
255 0 : if (local_var_based_p (aggr))
256 0 : worklist.safe_push (stmt);
257 : }
258 : break;
259 :
260 : default:
261 : ;
262 : }
263 : }
264 : }
265 : }
266 :
267 : /* Split blocks on the worklist. */
268 : unsigned ix;
269 : gimple *stmt;
270 :
271 0 : for (ix = 0; worklist.iterate (ix, &stmt); ix++)
272 : {
273 0 : basic_block block = gimple_bb (stmt);
274 :
275 0 : if (gimple_code (stmt) == GIMPLE_COND)
276 : {
277 0 : gcond *orig_cond = as_a <gcond *> (stmt);
278 0 : tree_code code = gimple_expr_code (orig_cond);
279 0 : tree pred = make_ssa_name (boolean_type_node);
280 0 : gimple *asgn = gimple_build_assign (pred, code,
281 : gimple_cond_lhs (orig_cond),
282 : gimple_cond_rhs (orig_cond));
283 0 : gcond *new_cond
284 0 : = gimple_build_cond (NE_EXPR, pred, boolean_false_node,
285 : gimple_cond_true_label (orig_cond),
286 : gimple_cond_false_label (orig_cond));
287 :
288 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
289 0 : gsi_insert_before (&gsi, asgn, GSI_SAME_STMT);
290 0 : gsi_replace (&gsi, new_cond, true);
291 :
292 0 : edge e = split_block (block, asgn);
293 0 : block = e->dest;
294 0 : map->get_or_insert (block) = new_cond;
295 : }
296 0 : else if ((gimple_code (stmt) == GIMPLE_CALL
297 0 : && !gimple_call_internal_p (stmt))
298 0 : || is_gimple_assign (stmt))
299 : {
300 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
301 0 : gsi_prev (&gsi);
302 :
303 0 : edge call = split_block (block, gsi_stmt (gsi));
304 :
305 0 : gimple *call_stmt = gsi_stmt (gsi_start_bb (call->dest));
306 :
307 0 : edge call_to_ret = split_block (call->dest, call_stmt);
308 :
309 0 : map->get_or_insert (call_to_ret->src) = call_stmt;
310 : }
311 : else
312 : {
313 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
314 0 : gsi_prev (&gsi);
315 :
316 0 : if (gsi_end_p (gsi))
317 0 : map->get_or_insert (block) = stmt;
318 : else
319 : {
320 : /* Split block before insn. The insn is in the new block. */
321 0 : edge e = split_block (block, gsi_stmt (gsi));
322 :
323 0 : block = e->dest;
324 0 : map->get_or_insert (block) = stmt;
325 : }
326 : }
327 : }
328 0 : }
329 :
330 : static const char *
331 0 : mask_name (unsigned mask)
332 : {
333 0 : switch (mask)
334 : {
335 : case 0: return "gang redundant";
336 0 : case 1: return "gang partitioned";
337 0 : case 2: return "worker partitioned";
338 0 : case 3: return "gang+worker partitioned";
339 0 : case 4: return "vector partitioned";
340 0 : case 5: return "gang+vector partitioned";
341 0 : case 6: return "worker+vector partitioned";
342 0 : case 7: return "fully partitioned";
343 0 : default: return "<illegal>";
344 : }
345 : }
346 :
347 : /* Dump this parallel and all its inner parallels. */
348 : /* Adapted from 'gcc/config/nvptx/nvptx.cc:nvptx_dump_pars'. */
349 :
350 : static void
351 0 : omp_sese_dump_pars (parallel_g *par, unsigned depth)
352 : {
353 0 : fprintf (dump_file, "%u: mask %d (%s) head=%d, tail=%d\n",
354 : depth, par->mask, mask_name (par->mask),
355 0 : par->forked_block ? par->forked_block->index : -1,
356 0 : par->join_block ? par->join_block->index : -1);
357 :
358 0 : fprintf (dump_file, " blocks:");
359 :
360 0 : basic_block block;
361 0 : for (unsigned ix = 0; par->blocks.iterate (ix, &block); ix++)
362 0 : fprintf (dump_file, " %d", block->index);
363 0 : fprintf (dump_file, "\n");
364 0 : if (par->inner)
365 0 : omp_sese_dump_pars (par->inner, depth + 1);
366 :
367 0 : if (par->next)
368 : omp_sese_dump_pars (par->next, depth);
369 0 : }
370 :
371 : /* If BLOCK contains a fork/join marker, process it to create or
372 : terminate a loop structure. Add this block to the current loop,
373 : and then walk successor blocks. */
374 : /* Adapted from 'gcc/config/nvptx/nvptx.cc:nvptx_find_par'. */
375 :
376 : static parallel_g *
377 0 : omp_sese_find_par (bb_stmt_map_t *map, parallel_g *par, basic_block block)
378 : {
379 0 : if (block->flags & BB_VISITED)
380 : return par;
381 0 : block->flags |= BB_VISITED;
382 :
383 0 : if (gimple **stmtp = map->get (block))
384 : {
385 0 : gimple *stmt = *stmtp;
386 :
387 0 : if (gimple_code (stmt) == GIMPLE_COND
388 0 : || gimple_code (stmt) == GIMPLE_SWITCH
389 0 : || gimple_code (stmt) == GIMPLE_RETURN
390 0 : || (gimple_code (stmt) == GIMPLE_CALL
391 0 : && !gimple_call_internal_p (stmt))
392 0 : || is_gimple_assign (stmt))
393 : {
394 : /* A single block that is forced to be at the maximum partition
395 : level. Make a singleton par for it. */
396 0 : par = new parallel_g (par, GOMP_DIM_MASK (GOMP_DIM_GANG)
397 : | GOMP_DIM_MASK (GOMP_DIM_WORKER)
398 0 : | GOMP_DIM_MASK (GOMP_DIM_VECTOR));
399 0 : par->forked_block = block;
400 0 : par->forked_stmt = stmt;
401 0 : par->blocks.safe_push (block);
402 0 : par = par->parent;
403 0 : goto walk_successors;
404 : }
405 0 : else if (gimple_nop_p (stmt))
406 : {
407 0 : basic_block pred = single_pred (block);
408 0 : gcc_assert (pred);
409 0 : gimple_stmt_iterator gsi = gsi_last_bb (pred);
410 0 : gimple *final_stmt = gsi_stmt (gsi);
411 :
412 0 : if (gimple_call_internal_p (final_stmt, IFN_UNIQUE))
413 : {
414 0 : gcall *call = as_a <gcall *> (final_stmt);
415 0 : enum ifn_unique_kind k = ((enum ifn_unique_kind)
416 0 : TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
417 :
418 0 : if (k == IFN_UNIQUE_OACC_FORK)
419 : {
420 0 : HOST_WIDE_INT dim
421 0 : = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
422 0 : unsigned mask = (dim >= 0) ? GOMP_DIM_MASK (dim) : 0;
423 :
424 0 : par = new parallel_g (par, mask);
425 0 : par->forked_block = block;
426 0 : par->forked_stmt = final_stmt;
427 0 : par->fork_stmt = stmt;
428 : }
429 : else
430 0 : gcc_unreachable ();
431 : }
432 : else
433 0 : gcc_unreachable ();
434 : }
435 0 : else if (gimple_call_internal_p (stmt, IFN_UNIQUE))
436 : {
437 0 : gcall *call = as_a <gcall *> (stmt);
438 0 : enum ifn_unique_kind k = ((enum ifn_unique_kind)
439 0 : TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
440 0 : if (k == IFN_UNIQUE_OACC_JOIN)
441 : {
442 0 : HOST_WIDE_INT dim = TREE_INT_CST_LOW (gimple_call_arg (stmt, 2));
443 0 : unsigned mask = (dim >= 0) ? GOMP_DIM_MASK (dim) : 0;
444 :
445 0 : gcc_assert (par->mask == mask);
446 0 : par->join_block = block;
447 0 : par->join_stmt = stmt;
448 0 : par = par->parent;
449 : }
450 : else
451 0 : gcc_unreachable ();
452 : }
453 : else
454 0 : gcc_unreachable ();
455 : }
456 :
457 0 : if (par)
458 : /* Add this block onto the current loop's list of blocks. */
459 0 : par->blocks.safe_push (block);
460 : else
461 : /* This must be the entry block. Create a NULL parallel. */
462 0 : par = new parallel_g (0, 0);
463 :
464 0 : walk_successors:
465 : /* Walk successor blocks. */
466 0 : edge e;
467 0 : edge_iterator ei;
468 :
469 0 : FOR_EACH_EDGE (e, ei, block->succs)
470 0 : omp_sese_find_par (map, par, e->dest);
471 :
472 : return par;
473 : }
474 :
475 : /* DFS walk the CFG looking for fork & join markers. Construct
476 : loop structures as we go. MAP is a mapping of basic blocks
477 : to head & tail markers, discovered when splitting blocks. This
478 : speeds up the discovery. We rely on the BB visited flag having
479 : been cleared when splitting blocks. */
480 : /* Adapted from 'gcc/config/nvptx/nvptx.cc:nvptx_discover_pars'. */
481 :
482 : static parallel_g *
483 0 : omp_sese_discover_pars (bb_stmt_map_t *map)
484 : {
485 0 : basic_block block;
486 :
487 : /* Mark exit blocks as visited. */
488 0 : block = EXIT_BLOCK_PTR_FOR_FN (cfun);
489 0 : block->flags |= BB_VISITED;
490 :
491 : /* And entry block as not. */
492 0 : block = ENTRY_BLOCK_PTR_FOR_FN (cfun);
493 0 : block->flags &= ~BB_VISITED;
494 :
495 0 : parallel_g *par = omp_sese_find_par (map, 0, block);
496 :
497 0 : if (dump_file)
498 : {
499 0 : fprintf (dump_file, "\nLoops\n");
500 0 : omp_sese_dump_pars (par, 0);
501 0 : fprintf (dump_file, "\n");
502 : }
503 :
504 0 : return par;
505 : }
506 :
507 : static void
508 0 : populate_single_mode_bitmaps (parallel_g *par, bitmap worker_single,
509 : bitmap vector_single, unsigned outer_mask,
510 : int depth)
511 : {
512 0 : unsigned mask = outer_mask | par->mask;
513 :
514 0 : basic_block block;
515 :
516 0 : for (unsigned i = 0; par->blocks.iterate (i, &block); i++)
517 : {
518 0 : if ((mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0)
519 0 : bitmap_set_bit (worker_single, block->index);
520 :
521 0 : if ((mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR)) == 0)
522 0 : bitmap_set_bit (vector_single, block->index);
523 : }
524 :
525 0 : if (par->inner)
526 0 : populate_single_mode_bitmaps (par->inner, worker_single, vector_single,
527 : mask, depth + 1);
528 0 : if (par->next)
529 : populate_single_mode_bitmaps (par->next, worker_single, vector_single,
530 : outer_mask, depth);
531 0 : }
532 :
533 : /* A map from SSA names or var decls to record fields. */
534 :
535 : typedef hash_map<tree, tree> field_map_t;
536 :
537 : /* For each propagation record type, this is a map from SSA names or var decls
538 : to propagate, to the field in the record type that should be used for
539 : transmission and reception. */
540 :
541 : typedef hash_map<tree, field_map_t> record_field_map_t;
542 :
543 : static void
544 0 : install_var_field (tree var, tree record_type, field_map_t *fields)
545 : {
546 0 : tree name;
547 0 : char tmp[20];
548 :
549 0 : if (TREE_CODE (var) == SSA_NAME)
550 : {
551 0 : name = SSA_NAME_IDENTIFIER (var);
552 0 : if (!name)
553 : {
554 0 : sprintf (tmp, "_%u", (unsigned) SSA_NAME_VERSION (var));
555 0 : name = get_identifier (tmp);
556 : }
557 : }
558 0 : else if (VAR_P (var))
559 : {
560 0 : name = DECL_NAME (var);
561 0 : if (!name)
562 : {
563 0 : sprintf (tmp, "D_%u", (unsigned) DECL_UID (var));
564 0 : name = get_identifier (tmp);
565 : }
566 : }
567 : else
568 0 : gcc_unreachable ();
569 :
570 0 : gcc_assert (!fields->get (var));
571 :
572 0 : tree type = TREE_TYPE (var);
573 :
574 0 : if (POINTER_TYPE_P (type)
575 0 : && TYPE_RESTRICT (type))
576 0 : type = build_qualified_type (type,
577 0 : TYPE_QUALS (type)
578 : .without (TYPE_QUAL_RESTRICT));
579 :
580 0 : tree field = build_decl (BUILTINS_LOCATION, FIELD_DECL, name, type);
581 :
582 0 : if (VAR_P (var) && type == TREE_TYPE (var))
583 : {
584 0 : SET_DECL_ALIGN (field, DECL_ALIGN (var));
585 0 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
586 0 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
587 : }
588 : else
589 0 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
590 :
591 0 : fields->put (var, field);
592 :
593 0 : insert_field_into_struct (record_type, field);
594 0 : }
595 :
596 : /* Sets of SSA_NAMES or VAR_DECLs to propagate. */
597 : typedef hash_set<tree> propagation_set;
598 :
599 : static void
600 0 : find_ssa_names_to_propagate (parallel_g *par, unsigned outer_mask,
601 : bitmap worker_single, bitmap vector_single,
602 : vec<propagation_set *> *prop_set)
603 : {
604 0 : unsigned mask = outer_mask | par->mask;
605 :
606 0 : if (par->inner)
607 0 : find_ssa_names_to_propagate (par->inner, mask, worker_single,
608 : vector_single, prop_set);
609 0 : if (par->next)
610 0 : find_ssa_names_to_propagate (par->next, outer_mask, worker_single,
611 : vector_single, prop_set);
612 :
613 0 : if (mask & GOMP_DIM_MASK (GOMP_DIM_WORKER))
614 : {
615 : basic_block block;
616 : int ix;
617 :
618 0 : for (ix = 0; par->blocks.iterate (ix, &block); ix++)
619 : {
620 0 : for (gphi_iterator psi = gsi_start_phis (block);
621 0 : !gsi_end_p (psi); gsi_next (&psi))
622 : {
623 0 : gphi *phi = psi.phi ();
624 0 : use_operand_p use;
625 0 : ssa_op_iter iter;
626 :
627 0 : FOR_EACH_PHI_ARG (use, phi, iter, SSA_OP_USE)
628 : {
629 0 : tree var = USE_FROM_PTR (use);
630 :
631 0 : if (TREE_CODE (var) != SSA_NAME)
632 0 : continue;
633 :
634 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (var);
635 :
636 0 : if (gimple_nop_p (def_stmt))
637 0 : continue;
638 :
639 0 : basic_block def_bb = gimple_bb (def_stmt);
640 :
641 0 : if (bitmap_bit_p (worker_single, def_bb->index))
642 : {
643 0 : if (!(*prop_set)[def_bb->index])
644 0 : (*prop_set)[def_bb->index] = new propagation_set;
645 :
646 0 : propagation_set *ws_prop = (*prop_set)[def_bb->index];
647 :
648 0 : ws_prop->add (var);
649 : }
650 : }
651 : }
652 :
653 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (block);
654 0 : !gsi_end_p (gsi); gsi_next (&gsi))
655 : {
656 0 : use_operand_p use;
657 0 : ssa_op_iter iter;
658 0 : gimple *stmt = gsi_stmt (gsi);
659 :
660 0 : FOR_EACH_SSA_USE_OPERAND (use, stmt, iter, SSA_OP_USE)
661 : {
662 0 : tree var = USE_FROM_PTR (use);
663 :
664 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (var);
665 :
666 0 : if (gimple_nop_p (def_stmt))
667 0 : continue;
668 :
669 0 : basic_block def_bb = gimple_bb (def_stmt);
670 :
671 0 : if (bitmap_bit_p (worker_single, def_bb->index))
672 : {
673 0 : if (!(*prop_set)[def_bb->index])
674 0 : (*prop_set)[def_bb->index] = new propagation_set;
675 :
676 0 : propagation_set *ws_prop = (*prop_set)[def_bb->index];
677 :
678 0 : ws_prop->add (var);
679 : }
680 : }
681 : }
682 : }
683 : }
684 0 : }
685 :
686 : /* Callback for walk_gimple_stmt to find RHS VAR_DECLs (uses) in a
687 : statement. */
688 :
689 : static tree
690 0 : find_partitioned_var_uses_1 (tree *node, int *, void *data)
691 : {
692 0 : walk_stmt_info *wi = (walk_stmt_info *) data;
693 0 : hash_set<tree> *partitioned_var_uses = (hash_set<tree> *) wi->info;
694 :
695 0 : if (!wi->is_lhs && VAR_P (*node))
696 0 : partitioned_var_uses->add (*node);
697 :
698 0 : return NULL_TREE;
699 : }
700 :
701 : static void
702 0 : find_partitioned_var_uses (parallel_g *par, unsigned outer_mask,
703 : hash_set<tree> *partitioned_var_uses)
704 : {
705 0 : unsigned mask = outer_mask | par->mask;
706 :
707 0 : if (par->inner)
708 0 : find_partitioned_var_uses (par->inner, mask, partitioned_var_uses);
709 0 : if (par->next)
710 0 : find_partitioned_var_uses (par->next, outer_mask, partitioned_var_uses);
711 :
712 0 : if (mask & GOMP_DIM_MASK (GOMP_DIM_WORKER))
713 : {
714 : basic_block block;
715 : int ix;
716 :
717 0 : for (ix = 0; par->blocks.iterate (ix, &block); ix++)
718 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (block);
719 0 : !gsi_end_p (gsi); gsi_next (&gsi))
720 : {
721 0 : walk_stmt_info wi;
722 0 : memset (&wi, 0, sizeof (wi));
723 0 : wi.info = (void *) partitioned_var_uses;
724 0 : walk_gimple_stmt (&gsi, NULL, find_partitioned_var_uses_1, &wi);
725 : }
726 : }
727 0 : }
728 :
729 : /* Gang-private variables (typically placed in a GPU's shared memory) do not
730 : need to be processed by the worker-propagation mechanism. Populate the
731 : GANG_PRIVATE_VARS set with any such variables found in the current
732 : function. */
733 :
734 : static void
735 0 : find_gang_private_vars (hash_set<tree> *gang_private_vars)
736 : {
737 0 : basic_block block;
738 :
739 0 : FOR_EACH_BB_FN (block, cfun)
740 : {
741 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (block);
742 0 : !gsi_end_p (gsi);
743 0 : gsi_next (&gsi))
744 : {
745 0 : gimple *stmt = gsi_stmt (gsi);
746 :
747 0 : if (gimple_call_internal_p (stmt, IFN_UNIQUE))
748 : {
749 0 : enum ifn_unique_kind k = ((enum ifn_unique_kind)
750 0 : TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
751 0 : if (k == IFN_UNIQUE_OACC_PRIVATE)
752 : {
753 0 : HOST_WIDE_INT level
754 0 : = TREE_INT_CST_LOW (gimple_call_arg (stmt, 2));
755 0 : if (level != GOMP_DIM_GANG)
756 0 : continue;
757 0 : for (unsigned i = 3; i < gimple_call_num_args (stmt); i++)
758 : {
759 0 : tree arg = gimple_call_arg (stmt, i);
760 0 : gcc_assert (TREE_CODE (arg) == ADDR_EXPR);
761 0 : tree decl = TREE_OPERAND (arg, 0);
762 0 : gang_private_vars->add (decl);
763 : }
764 : }
765 : }
766 : }
767 : }
768 0 : }
769 :
770 : static void
771 0 : find_local_vars_to_propagate (parallel_g *par, unsigned outer_mask,
772 : hash_set<tree> *partitioned_var_uses,
773 : hash_set<tree> *gang_private_vars,
774 : bitmap writes_gang_private,
775 : vec<propagation_set *> *prop_set)
776 : {
777 0 : unsigned mask = outer_mask | par->mask;
778 :
779 0 : if (par->inner)
780 0 : find_local_vars_to_propagate (par->inner, mask, partitioned_var_uses,
781 : gang_private_vars, writes_gang_private,
782 : prop_set);
783 0 : if (par->next)
784 0 : find_local_vars_to_propagate (par->next, outer_mask, partitioned_var_uses,
785 : gang_private_vars, writes_gang_private,
786 : prop_set);
787 :
788 0 : if (!(mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)))
789 : {
790 : basic_block block;
791 : int ix;
792 :
793 0 : for (ix = 0; par->blocks.iterate (ix, &block); ix++)
794 : {
795 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (block);
796 0 : !gsi_end_p (gsi); gsi_next (&gsi))
797 : {
798 0 : gimple *stmt = gsi_stmt (gsi);
799 0 : tree var;
800 0 : unsigned i;
801 :
802 0 : FOR_EACH_LOCAL_DECL (cfun, i, var)
803 : {
804 0 : if (!VAR_P (var)
805 0 : || is_global_var (var)
806 0 : || AGGREGATE_TYPE_P (TREE_TYPE (var))
807 0 : || !partitioned_var_uses->contains (var))
808 0 : continue;
809 :
810 0 : if (stmt_may_clobber_ref_p (stmt, var))
811 : {
812 0 : if (dump_file)
813 : {
814 0 : fprintf (dump_file, "bb %u: local variable may be "
815 : "clobbered in %s mode: ", block->index,
816 : mask_name (mask));
817 0 : print_generic_expr (dump_file, var, TDF_SLIM);
818 0 : fprintf (dump_file, "\n");
819 : }
820 :
821 0 : if (gang_private_vars->contains (var))
822 : {
823 : /* If we write a gang-private variable, we want a
824 : barrier at the end of the block. */
825 0 : bitmap_set_bit (writes_gang_private, block->index);
826 0 : continue;
827 : }
828 :
829 0 : if (!(*prop_set)[block->index])
830 0 : (*prop_set)[block->index] = new propagation_set;
831 :
832 0 : propagation_set *ws_prop
833 0 : = (*prop_set)[block->index];
834 :
835 0 : ws_prop->add (var);
836 : }
837 : }
838 : }
839 : }
840 : }
841 0 : }
842 :
843 : /* Transform basic blocks FROM, TO (which may be the same block) into:
844 : if (GOACC_single_start ())
845 : BLOCK;
846 : GOACC_barrier ();
847 : \ | /
848 : +----+
849 : | | (new) predicate block
850 : +----+--
851 : \ | / \ | / |t \
852 : +----+ +----+ +----+ |
853 : | | | | ===> | | | f (old) from block
854 : +----+ +----+ +----+ |
855 : | t/ \f | /
856 : +----+/
857 : (split (split before | | skip block
858 : at end) condition) +----+
859 : t/ \f
860 : */
861 :
862 : static void
863 0 : worker_single_simple (basic_block from, basic_block to,
864 : hash_set<tree> *def_escapes_block)
865 : {
866 0 : gimple *call, *cond;
867 0 : tree lhs, decl;
868 0 : basic_block skip_block;
869 :
870 0 : gimple_stmt_iterator gsi = gsi_last_bb (to);
871 0 : if (EDGE_COUNT (to->succs) > 1)
872 : {
873 0 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_COND);
874 0 : gsi_prev (&gsi);
875 : }
876 0 : edge e = split_block (to, gsi_stmt (gsi));
877 0 : skip_block = e->dest;
878 :
879 0 : gimple_stmt_iterator start = gsi_after_labels (from);
880 :
881 0 : decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_START);
882 0 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
883 0 : call = gimple_build_call (decl, 0);
884 0 : gimple_call_set_lhs (call, lhs);
885 0 : gsi_insert_before (&start, call, GSI_NEW_STMT);
886 0 : update_stmt (call);
887 :
888 0 : cond = gimple_build_cond (EQ_EXPR, lhs,
889 : fold_convert_loc (UNKNOWN_LOCATION,
890 0 : TREE_TYPE (lhs),
891 : boolean_true_node),
892 : NULL_TREE, NULL_TREE);
893 0 : gsi_insert_after (&start, cond, GSI_NEW_STMT);
894 0 : update_stmt (cond);
895 :
896 0 : edge et = split_block (from, cond);
897 0 : et->flags &= ~EDGE_FALLTHRU;
898 0 : et->flags |= EDGE_TRUE_VALUE;
899 : /* Make the active worker the more probable path so we prefer fallthrough
900 : (letting the idle workers jump around more). */
901 0 : et->probability = profile_probability::likely ();
902 :
903 0 : edge ef = make_edge (from, skip_block, EDGE_FALSE_VALUE);
904 0 : ef->probability = et->probability.invert ();
905 :
906 0 : basic_block neutered = split_edge (ef);
907 0 : gimple_stmt_iterator neut_gsi = gsi_last_bb (neutered);
908 :
909 0 : for (gsi = gsi_start_bb (et->dest); !gsi_end_p (gsi); gsi_next (&gsi))
910 : {
911 0 : gimple *stmt = gsi_stmt (gsi);
912 0 : ssa_op_iter iter;
913 0 : tree var;
914 :
915 0 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
916 : {
917 0 : if (def_escapes_block->contains (var))
918 : {
919 0 : gphi *join_phi = create_phi_node (NULL_TREE, skip_block);
920 0 : create_new_def_for (var, join_phi,
921 : gimple_phi_result_ptr (join_phi));
922 0 : add_phi_arg (join_phi, var, e, UNKNOWN_LOCATION);
923 :
924 0 : tree neutered_def = copy_ssa_name (var, NULL);
925 : /* We really want "don't care" or some value representing
926 : undefined here, but optimizers will probably get rid of the
927 : zero-assignments anyway. */
928 0 : gassign *zero = gimple_build_assign (neutered_def,
929 0 : build_zero_cst (TREE_TYPE (neutered_def)));
930 :
931 0 : gsi_insert_after (&neut_gsi, zero, GSI_CONTINUE_LINKING);
932 0 : update_stmt (zero);
933 :
934 0 : add_phi_arg (join_phi, neutered_def, single_succ_edge (neutered),
935 : UNKNOWN_LOCATION);
936 0 : update_stmt (join_phi);
937 : }
938 : }
939 : }
940 0 : }
941 :
942 : static tree
943 0 : build_receiver_ref (tree var, tree receiver_decl, field_map_t *fields)
944 : {
945 0 : tree x = build_simple_mem_ref (receiver_decl);
946 0 : tree field = *fields->get (var);
947 0 : TREE_THIS_NOTRAP (x) = 1;
948 0 : x = omp_build_component_ref (x, field);
949 0 : return x;
950 : }
951 :
952 : static tree
953 0 : build_sender_ref (tree var, tree sender_decl, field_map_t *fields)
954 : {
955 0 : if (POINTER_TYPE_P (TREE_TYPE (sender_decl)))
956 0 : sender_decl = build_simple_mem_ref (sender_decl);
957 0 : tree field = *fields->get (var);
958 0 : return omp_build_component_ref (sender_decl, field);
959 : }
960 :
961 : static int
962 0 : sort_by_ssa_version_or_uid (const void *p1, const void *p2)
963 : {
964 0 : const tree t1 = *(const tree *)p1;
965 0 : const tree t2 = *(const tree *)p2;
966 :
967 0 : if (TREE_CODE (t1) == SSA_NAME && TREE_CODE (t2) == SSA_NAME)
968 0 : return SSA_NAME_VERSION (t1) - SSA_NAME_VERSION (t2);
969 0 : else if (TREE_CODE (t1) == SSA_NAME && TREE_CODE (t2) != SSA_NAME)
970 : return -1;
971 0 : else if (TREE_CODE (t1) != SSA_NAME && TREE_CODE (t2) == SSA_NAME)
972 : return 1;
973 : else
974 0 : return DECL_UID (t1) - DECL_UID (t2);
975 : }
976 :
977 : static int
978 0 : sort_by_size_then_ssa_version_or_uid (const void *p1, const void *p2)
979 : {
980 0 : const tree t1 = *(const tree *)p1;
981 0 : const tree t2 = *(const tree *)p2;
982 0 : unsigned HOST_WIDE_INT s1 = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (t1)));
983 0 : unsigned HOST_WIDE_INT s2 = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (t2)));
984 0 : if (s1 != s2)
985 0 : return s2 - s1;
986 : else
987 0 : return sort_by_ssa_version_or_uid (p1, p2);
988 : }
989 :
990 : static void
991 0 : worker_single_copy (basic_block from, basic_block to,
992 : hash_set<tree> *def_escapes_block,
993 : hash_set<tree> *worker_partitioned_uses,
994 : tree record_type, record_field_map_t *record_field_map,
995 : unsigned HOST_WIDE_INT placement,
996 : bool isolate_broadcasts, bool has_gang_private_write)
997 : {
998 : /* If we only have virtual defs, we'll have no record type, but we still want
999 : to emit single_copy_start and (particularly) single_copy_end to act as
1000 : a vdef source on the neutered edge representing memory writes on the
1001 : non-neutered edge. */
1002 0 : if (!record_type)
1003 0 : record_type = char_type_node;
1004 :
1005 0 : tree sender_decl
1006 0 : = targetm.goacc.create_worker_broadcast_record (record_type, true,
1007 : ".oacc_worker_o",
1008 : placement);
1009 0 : tree receiver_decl
1010 0 : = targetm.goacc.create_worker_broadcast_record (record_type, false,
1011 : ".oacc_worker_i",
1012 : placement);
1013 :
1014 0 : gimple_stmt_iterator gsi = gsi_last_bb (to);
1015 0 : if (EDGE_COUNT (to->succs) > 1)
1016 0 : gsi_prev (&gsi);
1017 0 : edge e = split_block (to, gsi_stmt (gsi));
1018 0 : basic_block barrier_block = e->dest;
1019 :
1020 0 : gimple_stmt_iterator start = gsi_after_labels (from);
1021 :
1022 0 : tree decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_COPY_START);
1023 :
1024 0 : tree lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
1025 :
1026 0 : gimple *call
1027 0 : = gimple_build_call (decl, 1,
1028 0 : POINTER_TYPE_P (TREE_TYPE (sender_decl))
1029 0 : ? sender_decl : build_fold_addr_expr (sender_decl));
1030 0 : gimple_call_set_lhs (call, lhs);
1031 0 : gsi_insert_before (&start, call, GSI_NEW_STMT);
1032 0 : update_stmt (call);
1033 :
1034 : /* The shared-memory range for this block overflowed. Add a barrier before
1035 : the GOACC_single_copy_start call. */
1036 0 : if (isolate_broadcasts)
1037 : {
1038 0 : decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
1039 0 : gimple *acc_bar = gimple_build_call (decl, 0);
1040 0 : gsi_insert_before (&start, acc_bar, GSI_SAME_STMT);
1041 : }
1042 :
1043 0 : tree conv_tmp = make_ssa_name (TREE_TYPE (receiver_decl));
1044 :
1045 0 : gimple *conv = gimple_build_assign (conv_tmp,
1046 0 : fold_convert (TREE_TYPE (receiver_decl),
1047 : lhs));
1048 0 : update_stmt (conv);
1049 0 : gsi_insert_after (&start, conv, GSI_NEW_STMT);
1050 0 : gimple *asgn = gimple_build_assign (receiver_decl, conv_tmp);
1051 0 : gsi_insert_after (&start, asgn, GSI_NEW_STMT);
1052 0 : update_stmt (asgn);
1053 :
1054 0 : tree zero_ptr = build_int_cst (TREE_TYPE (receiver_decl), 0);
1055 :
1056 0 : tree recv_tmp = make_ssa_name (TREE_TYPE (receiver_decl));
1057 0 : asgn = gimple_build_assign (recv_tmp, receiver_decl);
1058 0 : gsi_insert_after (&start, asgn, GSI_NEW_STMT);
1059 0 : update_stmt (asgn);
1060 :
1061 0 : gimple *cond = gimple_build_cond (EQ_EXPR, recv_tmp, zero_ptr, NULL_TREE,
1062 : NULL_TREE);
1063 0 : update_stmt (cond);
1064 :
1065 0 : gsi_insert_after (&start, cond, GSI_NEW_STMT);
1066 :
1067 0 : edge et = split_block (from, cond);
1068 0 : et->flags &= ~EDGE_FALLTHRU;
1069 0 : et->flags |= EDGE_TRUE_VALUE;
1070 : /* Make the active worker the more probable path so we prefer fallthrough
1071 : (letting the idle workers jump around more). */
1072 0 : et->probability = profile_probability::likely ();
1073 :
1074 0 : basic_block body = et->dest;
1075 :
1076 0 : edge ef = make_edge (from, barrier_block, EDGE_FALSE_VALUE);
1077 0 : ef->probability = et->probability.invert ();
1078 :
1079 0 : gimple_stmt_iterator bar_gsi = gsi_start_bb (barrier_block);
1080 0 : cond = gimple_build_cond (NE_EXPR, recv_tmp, zero_ptr, NULL_TREE, NULL_TREE);
1081 :
1082 0 : if (record_type != char_type_node || has_gang_private_write)
1083 : {
1084 0 : decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
1085 0 : gimple *acc_bar = gimple_build_call (decl, 0);
1086 :
1087 0 : gsi_insert_before (&bar_gsi, acc_bar, GSI_NEW_STMT);
1088 0 : gsi_insert_after (&bar_gsi, cond, GSI_NEW_STMT);
1089 0 : }
1090 : else
1091 0 : gsi_insert_before (&bar_gsi, cond, GSI_NEW_STMT);
1092 :
1093 0 : edge et2 = split_block (barrier_block, cond);
1094 0 : et2->flags &= ~EDGE_FALLTHRU;
1095 0 : et2->flags |= EDGE_TRUE_VALUE;
1096 0 : et2->probability = profile_probability::unlikely ();
1097 :
1098 0 : basic_block exit_block = et2->dest;
1099 :
1100 0 : basic_block copyout_block = split_edge (et2);
1101 0 : edge ef2 = make_edge (barrier_block, exit_block, EDGE_FALSE_VALUE);
1102 0 : ef2->probability = et2->probability.invert ();
1103 :
1104 0 : gimple_stmt_iterator copyout_gsi = gsi_start_bb (copyout_block);
1105 :
1106 0 : edge copyout_to_exit = single_succ_edge (copyout_block);
1107 :
1108 0 : gimple_seq sender_seq = NULL;
1109 :
1110 : /* Make sure we iterate over definitions in a stable order. */
1111 0 : auto_vec<tree> escape_vec (def_escapes_block->elements ());
1112 0 : for (hash_set<tree>::iterator it = def_escapes_block->begin ();
1113 0 : it != def_escapes_block->end (); ++it)
1114 0 : escape_vec.quick_push (*it);
1115 0 : escape_vec.qsort (sort_by_ssa_version_or_uid);
1116 :
1117 0 : for (unsigned i = 0; i < escape_vec.length (); i++)
1118 : {
1119 0 : tree var = escape_vec[i];
1120 :
1121 0 : if (TREE_CODE (var) == SSA_NAME && SSA_NAME_IS_VIRTUAL_OPERAND (var))
1122 0 : continue;
1123 :
1124 0 : tree barrier_def = 0;
1125 :
1126 0 : if (TREE_CODE (var) == SSA_NAME)
1127 : {
1128 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (var);
1129 :
1130 0 : if (gimple_nop_p (def_stmt))
1131 0 : continue;
1132 :
1133 : /* The barrier phi takes one result from the actual work of the
1134 : block we're neutering, and the other result is constant zero of
1135 : the same type. */
1136 :
1137 0 : gphi *barrier_phi = create_phi_node (NULL_TREE, barrier_block);
1138 0 : barrier_def = create_new_def_for (var, barrier_phi,
1139 : gimple_phi_result_ptr (barrier_phi));
1140 :
1141 0 : add_phi_arg (barrier_phi, var, e, UNKNOWN_LOCATION);
1142 0 : add_phi_arg (barrier_phi, build_zero_cst (TREE_TYPE (var)), ef,
1143 : UNKNOWN_LOCATION);
1144 :
1145 0 : update_stmt (barrier_phi);
1146 : }
1147 : else
1148 0 : gcc_assert (VAR_P (var));
1149 :
1150 : /* If we had no record type, we will have no fields map. */
1151 0 : field_map_t *fields = record_field_map->get (record_type);
1152 :
1153 0 : if (worker_partitioned_uses->contains (var)
1154 0 : && fields
1155 0 : && fields->get (var))
1156 : {
1157 0 : tree neutered_def = make_ssa_name (TREE_TYPE (var));
1158 :
1159 : /* Receive definition from shared memory block. */
1160 :
1161 0 : tree receiver_ref = build_receiver_ref (var, receiver_decl, fields);
1162 0 : gassign *recv = gimple_build_assign (neutered_def,
1163 : receiver_ref);
1164 0 : gsi_insert_after (©out_gsi, recv, GSI_CONTINUE_LINKING);
1165 0 : update_stmt (recv);
1166 :
1167 0 : if (VAR_P (var))
1168 : {
1169 : /* If it's a VAR_DECL, we only copied to an SSA temporary. Copy
1170 : to the final location now. */
1171 0 : gassign *asgn = gimple_build_assign (var, neutered_def);
1172 0 : gsi_insert_after (©out_gsi, asgn, GSI_CONTINUE_LINKING);
1173 0 : update_stmt (asgn);
1174 : }
1175 : else
1176 : {
1177 : /* If it's an SSA name, create a new phi at the join node to
1178 : represent either the output from the active worker (the
1179 : barrier) or the inactive workers (the copyout block). */
1180 0 : gphi *join_phi = create_phi_node (NULL_TREE, exit_block);
1181 0 : create_new_def_for (barrier_def, join_phi,
1182 : gimple_phi_result_ptr (join_phi));
1183 0 : add_phi_arg (join_phi, barrier_def, ef2, UNKNOWN_LOCATION);
1184 0 : add_phi_arg (join_phi, neutered_def, copyout_to_exit,
1185 : UNKNOWN_LOCATION);
1186 0 : update_stmt (join_phi);
1187 : }
1188 :
1189 : /* Send definition to shared memory block. */
1190 :
1191 0 : tree sender_ref = build_sender_ref (var, sender_decl, fields);
1192 :
1193 0 : if (TREE_CODE (var) == SSA_NAME)
1194 : {
1195 0 : gassign *send = gimple_build_assign (sender_ref, var);
1196 0 : gimple_seq_add_stmt (&sender_seq, send);
1197 0 : update_stmt (send);
1198 : }
1199 0 : else if (VAR_P (var))
1200 : {
1201 0 : tree tmp = make_ssa_name (TREE_TYPE (var));
1202 0 : gassign *send = gimple_build_assign (tmp, var);
1203 0 : gimple_seq_add_stmt (&sender_seq, send);
1204 0 : update_stmt (send);
1205 0 : send = gimple_build_assign (sender_ref, tmp);
1206 0 : gimple_seq_add_stmt (&sender_seq, send);
1207 0 : update_stmt (send);
1208 : }
1209 : else
1210 0 : gcc_unreachable ();
1211 : }
1212 : }
1213 :
1214 : /* The shared-memory range for this block overflowed. Add a barrier at the
1215 : end. */
1216 0 : if (isolate_broadcasts)
1217 : {
1218 0 : gsi = gsi_start_bb (exit_block);
1219 0 : decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
1220 0 : gimple *acc_bar = gimple_build_call (decl, 0);
1221 0 : gsi_insert_before (&gsi, acc_bar, GSI_SAME_STMT);
1222 : }
1223 :
1224 : /* It's possible for the ET->DEST block (the work done by the active thread)
1225 : to finish with a control-flow insn, e.g. a UNIQUE function call. Split
1226 : the block and add SENDER_SEQ in the latter part to avoid having control
1227 : flow in the middle of a BB. */
1228 :
1229 0 : decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_COPY_END);
1230 0 : call = gimple_build_call (decl, 1,
1231 0 : POINTER_TYPE_P (TREE_TYPE (sender_decl))
1232 : ? sender_decl
1233 0 : : build_fold_addr_expr (sender_decl));
1234 0 : gimple_seq_add_stmt (&sender_seq, call);
1235 :
1236 0 : gsi = gsi_last_bb (body);
1237 0 : gimple *last = gsi_stmt (gsi);
1238 0 : basic_block sender_block = split_block (body, last)->dest;
1239 0 : gsi = gsi_last_bb (sender_block);
1240 0 : gsi_insert_seq_after (&gsi, sender_seq, GSI_CONTINUE_LINKING);
1241 0 : }
1242 :
1243 : typedef hash_map<basic_block, std::pair<unsigned HOST_WIDE_INT, bool> >
1244 : blk_offset_map_t;
1245 :
1246 : static void
1247 0 : neuter_worker_single (parallel_g *par, unsigned outer_mask,
1248 : bitmap worker_single, bitmap vector_single,
1249 : vec<propagation_set *> *prop_set,
1250 : hash_set<tree> *partitioned_var_uses,
1251 : record_field_map_t *record_field_map,
1252 : blk_offset_map_t *blk_offset_map,
1253 : bitmap writes_gang_private)
1254 : {
1255 0 : unsigned mask = outer_mask | par->mask;
1256 :
1257 0 : if ((mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0)
1258 : {
1259 : basic_block block;
1260 :
1261 0 : for (unsigned i = 0; par->blocks.iterate (i, &block); i++)
1262 : {
1263 0 : bool has_defs = false;
1264 0 : hash_set<tree> def_escapes_block;
1265 0 : hash_set<tree> worker_partitioned_uses;
1266 0 : unsigned j;
1267 0 : tree var;
1268 :
1269 0 : FOR_EACH_SSA_NAME (j, var, cfun)
1270 : {
1271 0 : if (SSA_NAME_IS_VIRTUAL_OPERAND (var))
1272 : {
1273 0 : has_defs = true;
1274 0 : continue;
1275 : }
1276 :
1277 0 : gimple *def_stmt = SSA_NAME_DEF_STMT (var);
1278 :
1279 0 : if (gimple_nop_p (def_stmt))
1280 0 : continue;
1281 :
1282 0 : if (gimple_bb (def_stmt)->index != block->index)
1283 0 : continue;
1284 :
1285 0 : gimple *use_stmt;
1286 0 : imm_use_iterator use_iter;
1287 0 : bool uses_outside_block = false;
1288 0 : bool worker_partitioned_use = false;
1289 :
1290 0 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, var)
1291 : {
1292 0 : int blocknum = gimple_bb (use_stmt)->index;
1293 :
1294 : /* Don't propagate SSA names that are only used in the
1295 : current block, unless the usage is in a phi node: that
1296 : means the name left the block, then came back in at the
1297 : top. */
1298 0 : if (blocknum != block->index
1299 0 : || gimple_code (use_stmt) == GIMPLE_PHI)
1300 : uses_outside_block = true;
1301 0 : if (!bitmap_bit_p (worker_single, blocknum))
1302 0 : worker_partitioned_use = true;
1303 0 : }
1304 :
1305 0 : if (uses_outside_block)
1306 0 : def_escapes_block.add (var);
1307 :
1308 0 : if (worker_partitioned_use)
1309 : {
1310 0 : worker_partitioned_uses.add (var);
1311 0 : has_defs = true;
1312 : }
1313 : }
1314 :
1315 0 : propagation_set *ws_prop = (*prop_set)[block->index];
1316 :
1317 0 : if (ws_prop)
1318 : {
1319 0 : for (propagation_set::iterator it = ws_prop->begin ();
1320 0 : it != ws_prop->end ();
1321 0 : ++it)
1322 : {
1323 0 : tree var = *it;
1324 0 : if (TREE_CODE (var) == VAR_DECL)
1325 : {
1326 0 : def_escapes_block.add (var);
1327 0 : if (partitioned_var_uses->contains (var))
1328 : {
1329 0 : worker_partitioned_uses.add (var);
1330 0 : has_defs = true;
1331 : }
1332 : }
1333 : }
1334 :
1335 0 : delete ws_prop;
1336 0 : (*prop_set)[block->index] = 0;
1337 : }
1338 :
1339 0 : bool only_marker_fns = true;
1340 0 : bool join_block = false;
1341 :
1342 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (block);
1343 0 : !gsi_end_p (gsi);
1344 0 : gsi_next (&gsi))
1345 : {
1346 0 : gimple *stmt = gsi_stmt (gsi);
1347 0 : if (gimple_code (stmt) == GIMPLE_CALL
1348 0 : && gimple_call_internal_p (stmt, IFN_UNIQUE))
1349 : {
1350 0 : enum ifn_unique_kind k = ((enum ifn_unique_kind)
1351 0 : TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
1352 0 : if (k != IFN_UNIQUE_OACC_PRIVATE
1353 0 : && k != IFN_UNIQUE_OACC_JOIN
1354 0 : && k != IFN_UNIQUE_OACC_FORK
1355 0 : && k != IFN_UNIQUE_OACC_HEAD_MARK
1356 0 : && k != IFN_UNIQUE_OACC_TAIL_MARK)
1357 : only_marker_fns = false;
1358 0 : else if (k == IFN_UNIQUE_OACC_JOIN)
1359 : /* The JOIN marker is special in that it *cannot* be
1360 : predicated for worker zero, because it may be lowered
1361 : to a barrier instruction and all workers must typically
1362 : execute that barrier. We shouldn't be doing any
1363 : broadcasts from the join block anyway. */
1364 0 : join_block = true;
1365 : }
1366 0 : else if (gimple_code (stmt) == GIMPLE_CALL
1367 0 : && gimple_call_internal_p (stmt, IFN_GOACC_LOOP))
1368 : /* Empty. */;
1369 0 : else if (gimple_nop_p (stmt))
1370 : /* Empty. */;
1371 : else
1372 0 : only_marker_fns = false;
1373 : }
1374 :
1375 : /* We can skip predicating this block for worker zero if the only
1376 : thing it contains is marker functions that will be removed in the
1377 : oaccdevlow pass anyway.
1378 : Don't do this if the block has (any) phi nodes, because those
1379 : might define SSA names that need broadcasting.
1380 : TODO: We might be able to skip transforming blocks that only
1381 : contain some other trivial statements too. */
1382 0 : if (only_marker_fns && !phi_nodes (block))
1383 0 : continue;
1384 :
1385 0 : gcc_assert (!join_block);
1386 :
1387 0 : if (has_defs)
1388 : {
1389 0 : tree record_type = (tree) block->aux;
1390 0 : std::pair<unsigned HOST_WIDE_INT, bool> *off_rngalloc
1391 0 : = blk_offset_map->get (block);
1392 0 : gcc_assert (!record_type || off_rngalloc);
1393 0 : unsigned HOST_WIDE_INT offset
1394 0 : = off_rngalloc ? off_rngalloc->first : 0;
1395 0 : bool range_allocated
1396 0 : = off_rngalloc ? off_rngalloc->second : true;
1397 0 : bool has_gang_private_write
1398 0 : = bitmap_bit_p (writes_gang_private, block->index);
1399 0 : worker_single_copy (block, block, &def_escapes_block,
1400 : &worker_partitioned_uses, record_type,
1401 : record_field_map,
1402 : offset, !range_allocated,
1403 : has_gang_private_write);
1404 : }
1405 : else
1406 0 : worker_single_simple (block, block, &def_escapes_block);
1407 0 : }
1408 : }
1409 :
1410 0 : if ((outer_mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0)
1411 : {
1412 : basic_block block;
1413 :
1414 0 : for (unsigned i = 0; par->blocks.iterate (i, &block); i++)
1415 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (block);
1416 0 : !gsi_end_p (gsi);
1417 0 : gsi_next (&gsi))
1418 : {
1419 0 : gimple *stmt = gsi_stmt (gsi);
1420 :
1421 0 : if (gimple_code (stmt) == GIMPLE_CALL
1422 0 : && !gimple_call_internal_p (stmt)
1423 0 : && !omp_sese_active_worker_call (as_a <gcall *> (stmt)))
1424 : {
1425 : /* If we have an OpenACC routine call in worker-single mode,
1426 : place barriers before and afterwards to prevent
1427 : clobbering re-used shared memory regions (as are used
1428 : for AMDGCN at present, for example). */
1429 0 : tree decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
1430 0 : gsi_insert_before (&gsi, gimple_build_call (decl, 0),
1431 : GSI_SAME_STMT);
1432 0 : gsi_insert_after (&gsi, gimple_build_call (decl, 0),
1433 : GSI_NEW_STMT);
1434 : }
1435 : }
1436 : }
1437 :
1438 0 : if (par->inner)
1439 0 : neuter_worker_single (par->inner, mask, worker_single, vector_single,
1440 : prop_set, partitioned_var_uses, record_field_map,
1441 : blk_offset_map, writes_gang_private);
1442 0 : if (par->next)
1443 : neuter_worker_single (par->next, outer_mask, worker_single, vector_single,
1444 : prop_set, partitioned_var_uses, record_field_map,
1445 : blk_offset_map, writes_gang_private);
1446 0 : }
1447 :
1448 : static void
1449 0 : dfs_broadcast_reachable_1 (basic_block bb, sbitmap reachable)
1450 : {
1451 0 : if (bb->flags & BB_VISITED)
1452 : return;
1453 :
1454 0 : bb->flags |= BB_VISITED;
1455 :
1456 0 : if (bb->succs)
1457 : {
1458 0 : edge e;
1459 0 : edge_iterator ei;
1460 0 : FOR_EACH_EDGE (e, ei, bb->succs)
1461 : {
1462 0 : basic_block dest = e->dest;
1463 0 : if (dest->aux)
1464 0 : bitmap_set_bit (reachable, dest->index);
1465 : else
1466 0 : dfs_broadcast_reachable_1 (dest, reachable);
1467 : }
1468 : }
1469 : }
1470 :
1471 : typedef std::pair<int, tree> idx_decl_pair_t;
1472 :
1473 : typedef auto_vec<splay_tree> used_range_vec_t;
1474 :
1475 : static int
1476 0 : sort_size_descending (const void *a, const void *b)
1477 : {
1478 0 : const idx_decl_pair_t *pa = (const idx_decl_pair_t *) a;
1479 0 : const idx_decl_pair_t *pb = (const idx_decl_pair_t *) b;
1480 0 : unsigned HOST_WIDE_INT asize = tree_to_uhwi (TYPE_SIZE_UNIT (pa->second));
1481 0 : unsigned HOST_WIDE_INT bsize = tree_to_uhwi (TYPE_SIZE_UNIT (pb->second));
1482 0 : return bsize - asize;
1483 : }
1484 :
1485 : class addr_range
1486 : {
1487 : public:
1488 0 : addr_range (unsigned HOST_WIDE_INT addr_lo, unsigned HOST_WIDE_INT addr_hi)
1489 0 : : lo (addr_lo), hi (addr_hi)
1490 : { }
1491 0 : addr_range (const addr_range &ar) : lo (ar.lo), hi (ar.hi)
1492 : { }
1493 0 : addr_range () : lo (0), hi (0)
1494 : { }
1495 :
1496 0 : bool invalid () { return lo == 0 && hi == 0; }
1497 :
1498 : unsigned HOST_WIDE_INT lo;
1499 : unsigned HOST_WIDE_INT hi;
1500 : };
1501 :
1502 : static int
1503 0 : splay_tree_compare_addr_range (splay_tree_key a, splay_tree_key b)
1504 : {
1505 0 : addr_range *ar = (addr_range *) a;
1506 0 : addr_range *br = (addr_range *) b;
1507 0 : if (ar->lo == br->lo && ar->hi == br->hi)
1508 : return 0;
1509 0 : if (ar->hi <= br->lo)
1510 : return -1;
1511 0 : else if (ar->lo >= br->hi)
1512 0 : return 1;
1513 : return 0;
1514 : }
1515 :
1516 : static void
1517 0 : splay_tree_free_key (splay_tree_key k)
1518 : {
1519 0 : addr_range *ar = (addr_range *) k;
1520 0 : delete ar;
1521 0 : }
1522 :
1523 : static addr_range
1524 0 : first_fit_range (splay_tree s, unsigned HOST_WIDE_INT size,
1525 : unsigned HOST_WIDE_INT align, addr_range *bounds)
1526 : {
1527 0 : splay_tree_node min = splay_tree_min (s);
1528 0 : if (min)
1529 : {
1530 : splay_tree_node next;
1531 0 : while ((next = splay_tree_successor (s, min->key)))
1532 : {
1533 0 : unsigned HOST_WIDE_INT lo = ((addr_range *) min->key)->hi;
1534 0 : unsigned HOST_WIDE_INT hi = ((addr_range *) next->key)->lo;
1535 0 : unsigned HOST_WIDE_INT base = (lo + align - 1) & ~(align - 1);
1536 0 : if (base + size <= hi)
1537 0 : return addr_range (base, base + size);
1538 : min = next;
1539 : }
1540 :
1541 0 : unsigned HOST_WIDE_INT base = ((addr_range *)min->key)->hi;
1542 0 : base = (base + align - 1) & ~(align - 1);
1543 0 : if (base + size <= bounds->hi)
1544 0 : return addr_range (base, base + size);
1545 : else
1546 0 : return addr_range ();
1547 : }
1548 : else
1549 : {
1550 0 : unsigned HOST_WIDE_INT lo = bounds->lo;
1551 0 : lo = (lo + align - 1) & ~(align - 1);
1552 0 : if (lo + size <= bounds->hi)
1553 0 : return addr_range (lo, lo + size);
1554 : else
1555 0 : return addr_range ();
1556 : }
1557 : }
1558 :
1559 : static int
1560 0 : merge_ranges_1 (splay_tree_node n, void *ptr)
1561 : {
1562 0 : splay_tree accum = (splay_tree) ptr;
1563 0 : addr_range ar = *(addr_range *) n->key;
1564 :
1565 0 : splay_tree_node old = splay_tree_lookup (accum, n->key);
1566 :
1567 : /* We might have an overlap. Create a new range covering the
1568 : overlapping parts. */
1569 0 : if (old)
1570 : {
1571 0 : addr_range *old_ar = (addr_range *) old->key;
1572 0 : ar.lo = MIN (old_ar->lo, ar.lo);
1573 0 : ar.hi = MAX (old_ar->hi, ar.hi);
1574 0 : splay_tree_remove (accum, old->key);
1575 : }
1576 :
1577 0 : addr_range *new_ar = new addr_range (ar);
1578 :
1579 0 : splay_tree_insert (accum, (splay_tree_key) new_ar, n->value);
1580 :
1581 0 : return 0;
1582 : }
1583 :
1584 : static void
1585 0 : merge_ranges (splay_tree accum, splay_tree sp)
1586 : {
1587 0 : splay_tree_foreach (sp, merge_ranges_1, (void *) accum);
1588 0 : }
1589 :
1590 : static void
1591 0 : oacc_do_neutering (unsigned HOST_WIDE_INT bounds_lo,
1592 : unsigned HOST_WIDE_INT bounds_hi)
1593 : {
1594 0 : bb_stmt_map_t bb_stmt_map;
1595 0 : auto_bitmap worker_single, vector_single;
1596 :
1597 0 : omp_sese_split_blocks (&bb_stmt_map);
1598 :
1599 0 : if (dump_file)
1600 : {
1601 0 : fprintf (dump_file, "\n\nAfter splitting:\n\n");
1602 0 : dump_function_to_file (current_function_decl, dump_file, dump_flags);
1603 : }
1604 :
1605 0 : unsigned mask = 0;
1606 :
1607 : /* If this is a routine, calculate MASK as if the outer levels are already
1608 : partitioned. */
1609 0 : {
1610 0 : tree attr = oacc_get_fn_attrib (current_function_decl);
1611 0 : tree dims = TREE_VALUE (attr);
1612 0 : unsigned ix;
1613 0 : for (ix = 0; ix != GOMP_DIM_MAX; ix++, dims = TREE_CHAIN (dims))
1614 : {
1615 0 : tree allowed = TREE_PURPOSE (dims);
1616 0 : if (allowed && integer_zerop (allowed))
1617 0 : mask |= GOMP_DIM_MASK (ix);
1618 : }
1619 : }
1620 :
1621 0 : parallel_g *par = omp_sese_discover_pars (&bb_stmt_map);
1622 0 : populate_single_mode_bitmaps (par, worker_single, vector_single, mask, 0);
1623 :
1624 0 : basic_block bb;
1625 0 : FOR_ALL_BB_FN (bb, cfun)
1626 0 : bb->aux = NULL;
1627 :
1628 0 : vec<propagation_set *> prop_set (vNULL);
1629 0 : prop_set.safe_grow_cleared (last_basic_block_for_fn (cfun), true);
1630 :
1631 0 : find_ssa_names_to_propagate (par, mask, worker_single, vector_single,
1632 : &prop_set);
1633 :
1634 0 : hash_set<tree> partitioned_var_uses;
1635 0 : hash_set<tree> gang_private_vars;
1636 0 : auto_bitmap writes_gang_private;
1637 :
1638 0 : find_gang_private_vars (&gang_private_vars);
1639 0 : find_partitioned_var_uses (par, mask, &partitioned_var_uses);
1640 0 : find_local_vars_to_propagate (par, mask, &partitioned_var_uses,
1641 : &gang_private_vars, writes_gang_private,
1642 : &prop_set);
1643 :
1644 0 : record_field_map_t record_field_map;
1645 :
1646 0 : FOR_ALL_BB_FN (bb, cfun)
1647 : {
1648 0 : propagation_set *ws_prop = prop_set[bb->index];
1649 0 : if (ws_prop)
1650 : {
1651 0 : tree record_type = lang_hooks.types.make_type (RECORD_TYPE);
1652 0 : tree name = create_tmp_var_name (".oacc_ws_data_s");
1653 0 : name = build_decl (UNKNOWN_LOCATION, TYPE_DECL, name, record_type);
1654 0 : DECL_ARTIFICIAL (name) = 1;
1655 0 : DECL_NAMELESS (name) = 1;
1656 0 : TYPE_NAME (record_type) = name;
1657 0 : TYPE_ARTIFICIAL (record_type) = 1;
1658 :
1659 0 : auto_vec<tree> field_vec (ws_prop->elements ());
1660 0 : for (hash_set<tree>::iterator it = ws_prop->begin ();
1661 0 : it != ws_prop->end (); ++it)
1662 0 : field_vec.quick_push (*it);
1663 :
1664 0 : field_vec.qsort (sort_by_size_then_ssa_version_or_uid);
1665 :
1666 0 : bool existed;
1667 0 : field_map_t *fields
1668 0 : = &record_field_map.get_or_insert (record_type, &existed);
1669 0 : gcc_checking_assert (!existed);
1670 :
1671 : /* Insert var fields in reverse order, so the last inserted element
1672 : is the first in the structure. */
1673 0 : for (int i = field_vec.length () - 1; i >= 0; i--)
1674 0 : install_var_field (field_vec[i], record_type, fields);
1675 :
1676 0 : layout_type (record_type);
1677 :
1678 0 : bb->aux = (tree) record_type;
1679 0 : }
1680 : }
1681 :
1682 0 : sbitmap *reachable
1683 0 : = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
1684 0 : last_basic_block_for_fn (cfun));
1685 :
1686 0 : bitmap_vector_clear (reachable, last_basic_block_for_fn (cfun));
1687 :
1688 0 : auto_vec<std::pair<int, tree> > priority;
1689 :
1690 0 : FOR_ALL_BB_FN (bb, cfun)
1691 : {
1692 0 : if (bb->aux)
1693 : {
1694 0 : tree record_type = (tree) bb->aux;
1695 :
1696 0 : basic_block bb2;
1697 0 : FOR_ALL_BB_FN (bb2, cfun)
1698 0 : bb2->flags &= ~BB_VISITED;
1699 :
1700 0 : priority.safe_push (std::make_pair (bb->index, record_type));
1701 0 : dfs_broadcast_reachable_1 (bb, reachable[bb->index]);
1702 : }
1703 : }
1704 :
1705 0 : sbitmap *inverted
1706 0 : = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
1707 0 : last_basic_block_for_fn (cfun));
1708 :
1709 0 : bitmap_vector_clear (inverted, last_basic_block_for_fn (cfun));
1710 :
1711 0 : for (int i = 0; i < last_basic_block_for_fn (cfun); i++)
1712 : {
1713 0 : sbitmap_iterator bi;
1714 0 : unsigned int j;
1715 0 : EXECUTE_IF_SET_IN_BITMAP (reachable[i], 0, j, bi)
1716 0 : bitmap_set_bit (inverted[j], i);
1717 : }
1718 :
1719 0 : for (int i = 0; i < last_basic_block_for_fn (cfun); i++)
1720 0 : bitmap_ior (reachable[i], reachable[i], inverted[i]);
1721 :
1722 0 : sbitmap_vector_free (inverted);
1723 :
1724 0 : used_range_vec_t used_ranges;
1725 :
1726 0 : used_ranges.safe_grow_cleared (last_basic_block_for_fn (cfun));
1727 :
1728 0 : blk_offset_map_t blk_offset_map;
1729 :
1730 0 : addr_range worker_shm_bounds (bounds_lo, bounds_hi);
1731 :
1732 0 : priority.qsort (sort_size_descending);
1733 0 : for (unsigned int i = 0; i < priority.length (); i++)
1734 : {
1735 0 : idx_decl_pair_t p = priority[i];
1736 0 : int blkno = p.first;
1737 0 : tree record_type = p.second;
1738 0 : HOST_WIDE_INT size = tree_to_uhwi (TYPE_SIZE_UNIT (record_type));
1739 0 : HOST_WIDE_INT align = TYPE_ALIGN_UNIT (record_type);
1740 :
1741 0 : splay_tree conflicts = splay_tree_new (splay_tree_compare_addr_range,
1742 : splay_tree_free_key, NULL);
1743 :
1744 0 : if (!used_ranges[blkno])
1745 0 : used_ranges[blkno] = splay_tree_new (splay_tree_compare_addr_range,
1746 : splay_tree_free_key, NULL);
1747 : else
1748 0 : merge_ranges (conflicts, used_ranges[blkno]);
1749 :
1750 0 : sbitmap_iterator bi;
1751 0 : unsigned int j;
1752 0 : EXECUTE_IF_SET_IN_BITMAP (reachable[blkno], 0, j, bi)
1753 0 : if (used_ranges[j])
1754 0 : merge_ranges (conflicts, used_ranges[j]);
1755 :
1756 0 : addr_range ar
1757 0 : = first_fit_range (conflicts, size, align, &worker_shm_bounds);
1758 :
1759 0 : splay_tree_delete (conflicts);
1760 :
1761 0 : if (ar.invalid ())
1762 : {
1763 0 : unsigned HOST_WIDE_INT base
1764 0 : = (bounds_lo + align - 1) & ~(align - 1);
1765 0 : if (base + size > bounds_hi)
1766 0 : error_at (UNKNOWN_LOCATION, "shared-memory region overflow");
1767 0 : std::pair<unsigned HOST_WIDE_INT, bool> base_inrng
1768 0 : = std::make_pair (base, false);
1769 0 : blk_offset_map.put (BASIC_BLOCK_FOR_FN (cfun, blkno), base_inrng);
1770 : }
1771 : else
1772 : {
1773 0 : splay_tree_node old = splay_tree_lookup (used_ranges[blkno],
1774 : (splay_tree_key) &ar);
1775 0 : if (old)
1776 : {
1777 0 : fprintf (stderr, "trying to map [%d..%d] but [%d..%d] is "
1778 0 : "already mapped in block %d\n", (int) ar.lo,
1779 0 : (int) ar.hi, (int) ((addr_range *) old->key)->lo,
1780 0 : (int) ((addr_range *) old->key)->hi, blkno);
1781 0 : abort ();
1782 : }
1783 :
1784 0 : addr_range *arp = new addr_range (ar);
1785 0 : splay_tree_insert (used_ranges[blkno], (splay_tree_key) arp,
1786 : (splay_tree_value) blkno);
1787 0 : std::pair<unsigned HOST_WIDE_INT, bool> base_inrng
1788 0 : = std::make_pair (ar.lo, true);
1789 0 : blk_offset_map.put (BASIC_BLOCK_FOR_FN (cfun, blkno), base_inrng);
1790 : }
1791 : }
1792 :
1793 0 : sbitmap_vector_free (reachable);
1794 :
1795 0 : neuter_worker_single (par, mask, worker_single, vector_single, &prop_set,
1796 : &partitioned_var_uses, &record_field_map,
1797 : &blk_offset_map, writes_gang_private);
1798 :
1799 0 : record_field_map.empty ();
1800 :
1801 : /* These are supposed to have been 'delete'd by 'neuter_worker_single'. */
1802 0 : for (auto it : prop_set)
1803 0 : gcc_checking_assert (!it);
1804 0 : prop_set.release ();
1805 :
1806 0 : delete par;
1807 :
1808 : /* This doesn't seem to make a difference. */
1809 0 : loops_state_clear (LOOP_CLOSED_SSA);
1810 :
1811 : /* Neutering worker-single neutered blocks will invalidate dominance info.
1812 : It may be possible to incrementally update just the affected blocks, but
1813 : obliterate everything for now. */
1814 0 : free_dominance_info (CDI_DOMINATORS);
1815 0 : free_dominance_info (CDI_POST_DOMINATORS);
1816 :
1817 0 : if (dump_file)
1818 : {
1819 0 : fprintf (dump_file, "\n\nAfter neutering:\n\n");
1820 0 : dump_function_to_file (current_function_decl, dump_file, dump_flags);
1821 : }
1822 0 : }
1823 :
1824 : static int
1825 0 : execute_omp_oacc_neuter_broadcast ()
1826 : {
1827 0 : unsigned HOST_WIDE_INT reduction_size[GOMP_DIM_MAX];
1828 0 : unsigned HOST_WIDE_INT private_size[GOMP_DIM_MAX];
1829 :
1830 0 : for (unsigned i = 0; i < GOMP_DIM_MAX; i++)
1831 : {
1832 0 : reduction_size[i] = 0;
1833 0 : private_size[i] = 0;
1834 : }
1835 :
1836 : /* Calculate shared memory size required for reduction variables and
1837 : gang-private memory for this offloaded function. */
1838 0 : basic_block bb;
1839 0 : FOR_ALL_BB_FN (bb, cfun)
1840 : {
1841 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
1842 0 : !gsi_end_p (gsi);
1843 0 : gsi_next (&gsi))
1844 : {
1845 0 : gimple *stmt = gsi_stmt (gsi);
1846 0 : if (!is_gimple_call (stmt))
1847 0 : continue;
1848 0 : gcall *call = as_a <gcall *> (stmt);
1849 0 : if (!gimple_call_internal_p (call))
1850 0 : continue;
1851 0 : enum internal_fn ifn_code = gimple_call_internal_fn (call);
1852 0 : switch (ifn_code)
1853 : {
1854 : default: break;
1855 0 : case IFN_GOACC_REDUCTION:
1856 0 : if (integer_minus_onep (gimple_call_arg (call, 3)))
1857 0 : continue;
1858 : else
1859 : {
1860 0 : unsigned code = TREE_INT_CST_LOW (gimple_call_arg (call, 0));
1861 : /* Only count reduction variables once: the choice to pick
1862 : the setup call is fairly arbitrary. */
1863 0 : if (code == IFN_GOACC_REDUCTION_SETUP)
1864 : {
1865 0 : int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3));
1866 0 : tree var = gimple_call_arg (call, 2);
1867 0 : tree offset = gimple_call_arg (call, 5);
1868 0 : tree var_type = TREE_TYPE (var);
1869 0 : unsigned HOST_WIDE_INT limit
1870 0 : = (tree_to_uhwi (offset)
1871 0 : + tree_to_uhwi (TYPE_SIZE_UNIT (var_type)));
1872 0 : reduction_size[level]
1873 0 : = MAX (reduction_size[level], limit);
1874 : }
1875 : }
1876 : break;
1877 0 : case IFN_UNIQUE:
1878 0 : {
1879 0 : enum ifn_unique_kind kind
1880 : = ((enum ifn_unique_kind)
1881 0 : TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
1882 :
1883 0 : if (kind == IFN_UNIQUE_OACC_PRIVATE)
1884 : {
1885 0 : HOST_WIDE_INT level
1886 0 : = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
1887 0 : if (level == -1)
1888 : break;
1889 0 : for (unsigned i = 3;
1890 0 : i < gimple_call_num_args (call);
1891 : i++)
1892 : {
1893 0 : tree arg = gimple_call_arg (call, i);
1894 0 : gcc_assert (TREE_CODE (arg) == ADDR_EXPR);
1895 0 : tree decl = TREE_OPERAND (arg, 0);
1896 0 : unsigned HOST_WIDE_INT align = DECL_ALIGN_UNIT (decl);
1897 0 : private_size[level] = ((private_size[level] + align - 1)
1898 0 : & ~(align - 1));
1899 0 : unsigned HOST_WIDE_INT decl_size
1900 0 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (decl)));
1901 0 : private_size[level] += decl_size;
1902 : }
1903 : }
1904 : }
1905 : break;
1906 : }
1907 : }
1908 : }
1909 :
1910 : int dims[GOMP_DIM_MAX];
1911 0 : for (unsigned i = 0; i < GOMP_DIM_MAX; i++)
1912 0 : dims[i] = oacc_get_fn_dim_size (current_function_decl, i);
1913 :
1914 : /* Find bounds of shared-memory buffer space we can use. */
1915 0 : unsigned HOST_WIDE_INT bounds_lo = 0, bounds_hi = 0;
1916 0 : if (targetm.goacc.shared_mem_layout)
1917 0 : targetm.goacc.shared_mem_layout (&bounds_lo, &bounds_hi, dims,
1918 : private_size, reduction_size);
1919 :
1920 : /* Perform worker partitioning unless we know 'num_workers(1)'. */
1921 0 : if (dims[GOMP_DIM_WORKER] != 1)
1922 0 : oacc_do_neutering (bounds_lo, bounds_hi);
1923 :
1924 0 : return 0;
1925 : }
1926 :
1927 : namespace {
1928 :
1929 : const pass_data pass_data_omp_oacc_neuter_broadcast =
1930 : {
1931 : GIMPLE_PASS, /* type */
1932 : "omp_oacc_neuter_broadcast", /* name */
1933 : OPTGROUP_OMP, /* optinfo_flags */
1934 : TV_NONE, /* tv_id */
1935 : PROP_cfg, /* properties_required */
1936 : 0, /* properties_provided */
1937 : 0, /* properties_destroyed */
1938 : 0, /* todo_flags_start */
1939 : TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */
1940 : };
1941 :
1942 : class pass_omp_oacc_neuter_broadcast : public gimple_opt_pass
1943 : {
1944 : public:
1945 297180 : pass_omp_oacc_neuter_broadcast (gcc::context *ctxt)
1946 594360 : : gimple_opt_pass (pass_data_omp_oacc_neuter_broadcast, ctxt)
1947 : {}
1948 :
1949 : /* opt_pass methods: */
1950 1518061 : bool gate (function *fun) final override
1951 : {
1952 1518061 : if (!flag_openacc)
1953 : return false;
1954 :
1955 15275 : if (!targetm.goacc.create_worker_broadcast_record)
1956 : return false;
1957 :
1958 : /* Only relevant for OpenACC offloaded functions. */
1959 0 : tree attr = oacc_get_fn_attrib (fun->decl);
1960 0 : if (!attr)
1961 0 : return false;
1962 :
1963 : return true;
1964 : }
1965 :
1966 0 : unsigned int execute (function *) final override
1967 : {
1968 0 : return execute_omp_oacc_neuter_broadcast ();
1969 : }
1970 :
1971 : }; // class pass_omp_oacc_neuter_broadcast
1972 :
1973 : } // anon namespace
1974 :
1975 : gimple_opt_pass *
1976 297180 : make_pass_omp_oacc_neuter_broadcast (gcc::context *ctxt)
1977 : {
1978 297180 : return new pass_omp_oacc_neuter_broadcast (ctxt);
1979 : }
|