Line data Source code
1 : /* Bits of OpenMP and OpenACC handling that is specific to device offloading
2 : and a lowering pass for OpenACC device directives.
3 :
4 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "target.h"
27 : #include "tree.h"
28 : #include "gimple.h"
29 : #include "tree-pass.h"
30 : #include "ssa.h"
31 : #include "cgraph.h"
32 : #include "pretty-print.h"
33 : #include "diagnostic-core.h"
34 : #include "fold-const.h"
35 : #include "internal-fn.h"
36 : #include "langhooks.h"
37 : #include "gimplify.h"
38 : #include "gimple-iterator.h"
39 : #include "gimplify-me.h"
40 : #include "gimple-walk.h"
41 : #include "tree-cfg.h"
42 : #include "tree-into-ssa.h"
43 : #include "tree-nested.h"
44 : #include "stor-layout.h"
45 : #include "common/common-target.h"
46 : #include "omp-general.h"
47 : #include "omp-offload.h"
48 : #include "lto-section-names.h"
49 : #include "gomp-constants.h"
50 : #include "gimple-pretty-print.h"
51 : #include "intl.h"
52 : #include "stringpool.h"
53 : #include "attribs.h"
54 : #include "cfgloop.h"
55 : #include "context.h"
56 : #include "convert.h"
57 : #include "opts.h"
58 :
59 : /* Describe the OpenACC looping structure of a function. The entire
60 : function is held in a 'NULL' loop. */
61 :
62 : struct oacc_loop
63 : {
64 : oacc_loop *parent; /* Containing loop. */
65 :
66 : oacc_loop *child; /* First inner loop. */
67 :
68 : oacc_loop *sibling; /* Next loop within same parent. */
69 :
70 : location_t loc; /* Location of the loop start. */
71 :
72 : gcall *marker; /* Initial head marker. */
73 :
74 : gcall *heads[GOMP_DIM_MAX]; /* Head marker functions. */
75 : gcall *tails[GOMP_DIM_MAX]; /* Tail marker functions. */
76 :
77 : tree routine; /* Pseudo-loop enclosing a routine. */
78 :
79 : unsigned mask; /* Partitioning mask. */
80 : unsigned e_mask; /* Partitioning of element loops (when tiling). */
81 : unsigned inner; /* Partitioning of inner loops. */
82 : unsigned flags; /* Partitioning flags. */
83 : vec<gcall *> ifns; /* Contained loop abstraction functions. */
84 : tree chunk_size; /* Chunk size. */
85 : gcall *head_end; /* Final marker of head sequence. */
86 : };
87 :
88 : /* Holds offload tables with decls. */
89 : vec<tree, va_gc> *offload_funcs, *offload_vars, *offload_ind_funcs;
90 :
91 : /* Return level at which oacc routine may spawn a partitioned loop, or
92 : -1 if it is not a routine (i.e. is an offload fn). */
93 :
94 : int
95 11142 : oacc_fn_attrib_level (tree attr)
96 : {
97 11142 : tree pos = TREE_VALUE (attr);
98 :
99 11142 : if (!TREE_PURPOSE (pos))
100 : return -1;
101 :
102 : int ix = 0;
103 5366 : for (ix = 0; ix != GOMP_DIM_MAX;
104 3612 : ix++, pos = TREE_CHAIN (pos))
105 4398 : if (!integer_zerop (TREE_PURPOSE (pos)))
106 : break;
107 :
108 : return ix;
109 : }
110 :
111 : /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
112 : adds their addresses and sizes to constructor-vector V_CTOR. */
113 :
114 : static void
115 87 : add_decls_addresses_to_decl_constructor (vec<tree, va_gc> *v_decls,
116 : vec<constructor_elt, va_gc> *v_ctor)
117 : {
118 87 : unsigned len = vec_safe_length (v_decls);
119 162 : for (unsigned i = 0; i < len; i++)
120 : {
121 75 : tree it = (*v_decls)[i];
122 75 : bool is_var = VAR_P (it);
123 75 : bool is_link_var
124 : = is_var
125 : #ifdef ACCEL_COMPILER
126 : && DECL_HAS_VALUE_EXPR_P (it)
127 : #endif
128 75 : && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it));
129 :
130 : /* See also omp_finish_file and output_offload_tables in lto-cgraph.cc. */
131 75 : if (!in_lto_p && !symtab_node::get (it))
132 0 : continue;
133 :
134 75 : tree size = NULL_TREE;
135 75 : if (is_var)
136 0 : size = fold_convert (const_ptr_type_node, DECL_SIZE_UNIT (it));
137 :
138 75 : tree addr;
139 75 : if (!is_link_var)
140 75 : addr = build_fold_addr_expr (it);
141 : else
142 : {
143 : #ifdef ACCEL_COMPILER
144 : /* For "omp declare target link" vars add address of the pointer to
145 : the target table, instead of address of the var. */
146 : tree value_expr = DECL_VALUE_EXPR (it);
147 : tree link_ptr_decl = TREE_OPERAND (value_expr, 0);
148 : varpool_node::finalize_decl (link_ptr_decl);
149 : addr = build_fold_addr_expr (link_ptr_decl);
150 : #else
151 0 : addr = build_fold_addr_expr (it);
152 : #endif
153 :
154 : /* Most significant bit of the size marks "omp declare target link"
155 : vars in host and target tables. */
156 0 : unsigned HOST_WIDE_INT isize = tree_to_uhwi (size);
157 0 : isize |= 1ULL << (int_size_in_bytes (const_ptr_type_node)
158 0 : * BITS_PER_UNIT - 1);
159 0 : size = wide_int_to_tree (const_ptr_type_node, isize);
160 : }
161 :
162 75 : CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, addr);
163 75 : if (is_var)
164 0 : CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, size);
165 : }
166 87 : }
167 :
168 : /* Return true if DECL is a function for which its references should be
169 : analyzed. */
170 :
171 : static bool
172 200499 : omp_declare_target_fn_p (tree decl)
173 : {
174 200499 : return (TREE_CODE (decl) == FUNCTION_DECL
175 200499 : && lookup_attribute ("omp declare target", DECL_ATTRIBUTES (decl))
176 24610 : && !lookup_attribute ("omp declare target host",
177 24610 : DECL_ATTRIBUTES (decl))
178 225065 : && (!flag_openacc
179 45 : || oacc_get_fn_attrib (decl) == NULL_TREE));
180 : }
181 :
182 : /* Return true if DECL Is a variable for which its initializer references
183 : should be analyzed. */
184 :
185 : static bool
186 110556 : omp_declare_target_var_p (tree decl)
187 : {
188 110556 : return (VAR_P (decl)
189 110556 : && lookup_attribute ("omp declare target", DECL_ATTRIBUTES (decl))
190 111103 : && !lookup_attribute ("omp declare target link",
191 547 : DECL_ATTRIBUTES (decl)));
192 : }
193 :
194 : /* Helper function for omp_discover_implicit_declare_target, called through
195 : walk_tree. Mark referenced FUNCTION_DECLs implicitly as
196 : declare target to. */
197 :
198 : static tree
199 996629 : omp_discover_declare_target_tgt_fn_r (tree *tp, int *walk_subtrees, void *data)
200 : {
201 996629 : if (TREE_CODE (*tp) == CALL_EXPR
202 26064 : && CALL_EXPR_FN (*tp)
203 26018 : && TREE_CODE (CALL_EXPR_FN (*tp)) == ADDR_EXPR
204 25969 : && TREE_CODE (TREE_OPERAND (CALL_EXPR_FN (*tp), 0)) == FUNCTION_DECL
205 1022598 : && lookup_attribute ("omp declare variant base",
206 25969 : DECL_ATTRIBUTES (TREE_OPERAND (CALL_EXPR_FN (*tp),
207 : 0))))
208 : {
209 83 : tree fn = TREE_OPERAND (CALL_EXPR_FN (*tp), 0);
210 196 : for (tree attr = DECL_ATTRIBUTES (fn); attr; attr = TREE_CHAIN (attr))
211 : {
212 114 : attr = lookup_attribute ("omp declare variant base", attr);
213 114 : if (attr == NULL_TREE)
214 : break;
215 113 : tree purpose = TREE_PURPOSE (TREE_VALUE (attr));
216 113 : if (TREE_CODE (purpose) == FUNCTION_DECL)
217 113 : omp_discover_declare_target_tgt_fn_r (&purpose, walk_subtrees, data);
218 : }
219 : }
220 996546 : else if (TREE_CODE (*tp) == FUNCTION_DECL)
221 : {
222 20745 : tree decl = *tp;
223 20745 : tree id = get_identifier ("omp declare target");
224 20745 : symtab_node *node = symtab_node::get (*tp);
225 20745 : if (node != NULL)
226 : {
227 13874 : while (node->alias_target
228 13874 : && TREE_CODE (node->alias_target) == FUNCTION_DECL)
229 : {
230 4 : if (!omp_declare_target_fn_p (node->decl)
231 8 : && !lookup_attribute ("omp declare target host",
232 4 : DECL_ATTRIBUTES (node->decl)))
233 : {
234 4 : node->offloadable = 1;
235 4 : DECL_ATTRIBUTES (node->decl)
236 8 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (node->decl));
237 : }
238 4 : node = symtab_node::get (node->alias_target);
239 : }
240 13870 : symtab_node *new_node = node->ultimate_alias_target ();
241 13870 : decl = new_node->decl;
242 15605 : while (node != new_node)
243 : {
244 1735 : if (!omp_declare_target_fn_p (node->decl)
245 2620 : && !lookup_attribute ("omp declare target host",
246 885 : DECL_ATTRIBUTES (node->decl)))
247 : {
248 885 : node->offloadable = 1;
249 885 : DECL_ATTRIBUTES (node->decl)
250 1770 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (node->decl));
251 : }
252 1735 : gcc_assert (node->alias && node->analyzed);
253 1735 : node = node->get_alias_target ();
254 : }
255 13870 : node->offloadable = 1;
256 13870 : if (ENABLE_OFFLOADING)
257 : g->have_offload = true;
258 : }
259 20745 : if (omp_declare_target_fn_p (decl)
260 29548 : || lookup_attribute ("omp declare target host",
261 8803 : DECL_ATTRIBUTES (decl)))
262 11942 : return NULL_TREE;
263 :
264 8803 : if (DECL_SAVED_TREE (decl)
265 8803 : && (!DECL_EXTERNAL (decl) || DECL_DECLARED_INLINE_P (decl)))
266 6675 : ((vec<tree> *) data)->safe_push (decl);
267 8803 : DECL_ATTRIBUTES (decl) = tree_cons (id, NULL_TREE,
268 8803 : DECL_ATTRIBUTES (decl));
269 : }
270 975801 : else if (TYPE_P (*tp))
271 39 : *walk_subtrees = 0;
272 975762 : else if (TREE_CODE (*tp) == OMP_TARGET)
273 : {
274 1672 : tree c = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE);
275 1672 : tree c2 = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE_TYPE);
276 43 : if ((c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
277 1672 : || (c2 && (OMP_CLAUSE_DEVICE_TYPE_KIND (c2)
278 : == OMP_CLAUSE_DEVICE_TYPE_HOST)))
279 43 : *walk_subtrees = 0;
280 : }
281 : return NULL_TREE;
282 : }
283 :
284 : /* Similarly, but ignore references outside of OMP_TARGET regions. */
285 :
286 : static tree
287 723280 : omp_discover_declare_target_fn_r (tree *tp, int *walk_subtrees, void *data)
288 : {
289 723280 : if (TREE_CODE (*tp) == OMP_TARGET)
290 : {
291 12047 : tree c = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE);
292 12047 : tree c2 = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE_TYPE);
293 865 : if ((!c || !OMP_CLAUSE_DEVICE_ANCESTOR (c))
294 12854 : && (!c2 || (OMP_CLAUSE_DEVICE_TYPE_KIND (c2)
295 : != OMP_CLAUSE_DEVICE_TYPE_HOST)))
296 11983 : walk_tree_without_duplicates (&OMP_TARGET_BODY (*tp),
297 : omp_discover_declare_target_tgt_fn_r,
298 : data);
299 12047 : *walk_subtrees = 0;
300 : }
301 711233 : else if (TYPE_P (*tp))
302 371 : *walk_subtrees = 0;
303 723280 : return NULL_TREE;
304 : }
305 :
306 : /* Helper function for omp_discover_implicit_declare_target, called through
307 : walk_tree. Mark referenced FUNCTION_DECLs implicitly as
308 : declare target to. */
309 :
310 : static tree
311 564 : omp_discover_declare_target_var_r (tree *tp, int *walk_subtrees, void *data)
312 : {
313 564 : if (TREE_CODE (*tp) == FUNCTION_DECL)
314 24 : return omp_discover_declare_target_tgt_fn_r (tp, walk_subtrees, data);
315 540 : else if (VAR_P (*tp)
316 54 : && is_global_var (*tp)
317 583 : && !omp_declare_target_var_p (*tp))
318 : {
319 15 : tree id = get_identifier ("omp declare target");
320 15 : if (lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (*tp)))
321 : {
322 0 : error_at (DECL_SOURCE_LOCATION (*tp),
323 : "%qD specified both in declare target %<link%> and "
324 : "implicitly in %<to%> clauses", *tp);
325 0 : DECL_ATTRIBUTES (*tp)
326 0 : = remove_attribute ("omp declare target link", DECL_ATTRIBUTES (*tp));
327 : }
328 15 : if (TREE_STATIC (*tp) && lang_hooks.decls.omp_get_decl_init (*tp))
329 15 : ((vec<tree> *) data)->safe_push (*tp);
330 15 : DECL_ATTRIBUTES (*tp) = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (*tp));
331 15 : symtab_node *node = symtab_node::get (*tp);
332 15 : if (node != NULL && !node->offloadable)
333 : {
334 15 : node->offloadable = 1;
335 15 : if (ENABLE_OFFLOADING)
336 : {
337 : g->have_offload = true;
338 : if (is_a <varpool_node *> (node))
339 : vec_safe_push (offload_vars, node->decl);
340 : }
341 : }
342 : }
343 525 : else if (TYPE_P (*tp))
344 0 : *walk_subtrees = 0;
345 : return NULL_TREE;
346 : }
347 :
348 : /* Perform the OpenMP implicit declare target to discovery. */
349 :
350 : void
351 9737 : omp_discover_implicit_declare_target (void)
352 : {
353 9737 : cgraph_node *node;
354 9737 : varpool_node *vnode;
355 9737 : auto_vec<tree> worklist;
356 :
357 169658 : FOR_EACH_DEFINED_FUNCTION (node)
358 159921 : if (DECL_SAVED_TREE (node->decl))
359 : {
360 159461 : struct cgraph_node *cgn;
361 159461 : if (lookup_attribute ("omp declare target indirect",
362 159461 : DECL_ATTRIBUTES (node->decl)))
363 123 : vec_safe_push (offload_ind_funcs, node->decl);
364 159461 : if (omp_declare_target_fn_p (node->decl))
365 2507 : worklist.safe_push (node->decl);
366 156954 : else if (DECL_STRUCT_FUNCTION (node->decl)
367 156954 : && DECL_STRUCT_FUNCTION (node->decl)->has_omp_target)
368 6792 : worklist.safe_push (node->decl);
369 321035 : for (cgn = first_nested_function (node);
370 161574 : cgn; cgn = next_nested_function (cgn))
371 2113 : if (omp_declare_target_fn_p (cgn->decl))
372 33 : worklist.safe_push (cgn->decl);
373 2080 : else if (DECL_STRUCT_FUNCTION (cgn->decl)
374 2080 : && DECL_STRUCT_FUNCTION (cgn->decl)->has_omp_target)
375 434 : worklist.safe_push (cgn->decl);
376 : }
377 128496 : FOR_EACH_VARIABLE (vnode)
378 118759 : if (lang_hooks.decls.omp_get_decl_init (vnode->decl)
379 118759 : && omp_declare_target_var_p (vnode->decl))
380 519 : worklist.safe_push (vnode->decl);
381 26712 : while (!worklist.is_empty ())
382 : {
383 16975 : tree decl = worklist.pop ();
384 16975 : if (VAR_P (decl))
385 534 : walk_tree_without_duplicates (lang_hooks.decls.omp_get_decl_init (decl),
386 : omp_discover_declare_target_var_r,
387 : &worklist);
388 16441 : else if (omp_declare_target_fn_p (decl))
389 9215 : walk_tree_without_duplicates (&DECL_SAVED_TREE (decl),
390 : omp_discover_declare_target_tgt_fn_r,
391 : &worklist);
392 : else
393 7226 : walk_tree_without_duplicates (&DECL_SAVED_TREE (decl),
394 : omp_discover_declare_target_fn_r,
395 : &worklist);
396 : }
397 :
398 9737 : if (omp_requires_mask
399 : & (OMP_REQUIRES_SELF_MAPS | OMP_REQUIRES_UNIFIED_SHARED_MEMORY))
400 77 : FOR_EACH_VARIABLE (vnode)
401 : {
402 29 : tree attr;
403 : /* If 'self_maps' or 'unified_shared_memory' is enabled,
404 : remove 'enter/to' and add 'link'.
405 : Note that "declare target local" is preserved and not converted. */
406 29 : if ((attr = lookup_attribute ("omp declare target",
407 29 : DECL_ATTRIBUTES (vnode->decl)))
408 45 : && !value_member (get_identifier ("local"), TREE_VALUE (attr)))
409 : {
410 16 : DECL_ATTRIBUTES (vnode->decl)
411 16 : = remove_attribute ("omp declare target",
412 16 : DECL_ATTRIBUTES (vnode->decl));
413 16 : if (!lookup_attribute ("omp declare target link",
414 16 : DECL_ATTRIBUTES (vnode->decl)))
415 16 : DECL_ATTRIBUTES (vnode->decl)
416 32 : = tree_cons (get_identifier ("omp declare target link"),
417 16 : NULL_TREE, DECL_ATTRIBUTES (vnode->decl));
418 : }
419 : }
420 :
421 9737 : lang_hooks.decls.omp_finish_decl_inits ();
422 9737 : }
423 :
424 :
425 : /* Create new symbols containing (address, size) pairs for global variables,
426 : marked with "omp declare target" attribute, as well as addresses for the
427 : functions, which are outlined offloading regions. */
428 : void
429 237544 : omp_finish_file (void)
430 : {
431 237544 : unsigned num_funcs = vec_safe_length (offload_funcs);
432 237544 : unsigned num_vars = vec_safe_length (offload_vars);
433 237544 : unsigned num_ind_funcs = vec_safe_length (offload_ind_funcs);
434 :
435 237544 : if (num_funcs == 0 && num_vars == 0 && num_ind_funcs == 0)
436 237544 : return;
437 :
438 29 : if (targetm_common.have_named_sections)
439 : {
440 29 : vec<constructor_elt, va_gc> *v_f, *v_v, *v_if;
441 29 : vec_alloc (v_f, num_funcs);
442 29 : vec_alloc (v_v, num_vars * 2);
443 29 : vec_alloc (v_if, num_ind_funcs);
444 :
445 29 : add_decls_addresses_to_decl_constructor (offload_funcs, v_f);
446 29 : add_decls_addresses_to_decl_constructor (offload_vars, v_v);
447 29 : add_decls_addresses_to_decl_constructor (offload_ind_funcs, v_if);
448 :
449 29 : tree vars_decl_type = build_array_type_nelts (pointer_sized_int_node,
450 29 : vec_safe_length (v_v));
451 29 : tree funcs_decl_type = build_array_type_nelts (pointer_sized_int_node,
452 29 : num_funcs);
453 29 : tree ind_funcs_decl_type = build_array_type_nelts (pointer_sized_int_node,
454 29 : num_ind_funcs);
455 :
456 29 : SET_TYPE_ALIGN (vars_decl_type, TYPE_ALIGN (pointer_sized_int_node));
457 29 : SET_TYPE_ALIGN (funcs_decl_type, TYPE_ALIGN (pointer_sized_int_node));
458 29 : SET_TYPE_ALIGN (ind_funcs_decl_type, TYPE_ALIGN (pointer_sized_int_node));
459 29 : tree ctor_v = build_constructor (vars_decl_type, v_v);
460 29 : tree ctor_f = build_constructor (funcs_decl_type, v_f);
461 29 : tree ctor_if = build_constructor (ind_funcs_decl_type, v_if);
462 29 : TREE_CONSTANT (ctor_v) = TREE_CONSTANT (ctor_f) = TREE_CONSTANT (ctor_if) = 1;
463 29 : TREE_STATIC (ctor_v) = TREE_STATIC (ctor_f) = TREE_STATIC (ctor_if) = 1;
464 29 : tree funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
465 : get_identifier (".offload_func_table"),
466 : funcs_decl_type);
467 29 : tree vars_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
468 : get_identifier (".offload_var_table"),
469 : vars_decl_type);
470 29 : tree ind_funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
471 : get_identifier (".offload_ind_func_table"),
472 : ind_funcs_decl_type);
473 29 : TREE_STATIC (funcs_decl) = TREE_STATIC (ind_funcs_decl) = 1;
474 29 : TREE_STATIC (vars_decl) = 1;
475 : /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
476 : otherwise a joint table in a binary will contain padding between
477 : tables from multiple object files. */
478 29 : DECL_USER_ALIGN (funcs_decl) = DECL_USER_ALIGN (ind_funcs_decl) = 1;
479 29 : DECL_USER_ALIGN (vars_decl) = 1;
480 29 : SET_DECL_ALIGN (funcs_decl, TYPE_ALIGN (funcs_decl_type));
481 29 : SET_DECL_ALIGN (vars_decl, TYPE_ALIGN (vars_decl_type));
482 29 : SET_DECL_ALIGN (ind_funcs_decl, TYPE_ALIGN (ind_funcs_decl_type));
483 29 : DECL_INITIAL (funcs_decl) = ctor_f;
484 29 : DECL_INITIAL (vars_decl) = ctor_v;
485 29 : DECL_INITIAL (ind_funcs_decl) = ctor_if;
486 29 : set_decl_section_name (funcs_decl, OFFLOAD_FUNC_TABLE_SECTION_NAME);
487 29 : set_decl_section_name (vars_decl, OFFLOAD_VAR_TABLE_SECTION_NAME);
488 29 : set_decl_section_name (ind_funcs_decl,
489 : OFFLOAD_IND_FUNC_TABLE_SECTION_NAME);
490 29 : varpool_node::finalize_decl (vars_decl);
491 29 : varpool_node::finalize_decl (funcs_decl);
492 29 : varpool_node::finalize_decl (ind_funcs_decl);
493 : }
494 : else
495 : {
496 0 : for (unsigned i = 0; i < num_funcs; i++)
497 : {
498 0 : tree it = (*offload_funcs)[i];
499 : /* See also add_decls_addresses_to_decl_constructor
500 : and output_offload_tables in lto-cgraph.cc. */
501 0 : if (!in_lto_p && !symtab_node::get (it))
502 0 : continue;
503 0 : targetm.record_offload_symbol (it);
504 : }
505 0 : for (unsigned i = 0; i < num_vars; i++)
506 : {
507 0 : tree it = (*offload_vars)[i];
508 0 : if (!in_lto_p && !symtab_node::get (it))
509 0 : continue;
510 : #ifdef ACCEL_COMPILER
511 : if (DECL_HAS_VALUE_EXPR_P (it)
512 : && lookup_attribute ("omp declare target link",
513 : DECL_ATTRIBUTES (it)))
514 : {
515 : tree value_expr = DECL_VALUE_EXPR (it);
516 : tree link_ptr_decl = TREE_OPERAND (value_expr, 0);
517 : targetm.record_offload_symbol (link_ptr_decl);
518 : varpool_node::finalize_decl (link_ptr_decl);
519 : }
520 : else
521 : #endif
522 0 : targetm.record_offload_symbol (it);
523 : }
524 0 : for (unsigned i = 0; i < num_ind_funcs; i++)
525 : {
526 0 : tree it = (*offload_ind_funcs)[i];
527 : /* See also add_decls_addresses_to_decl_constructor
528 : and output_offload_tables in lto-cgraph.cc. */
529 0 : if (!in_lto_p && !symtab_node::get (it))
530 0 : continue;
531 0 : targetm.record_offload_symbol (it);
532 : }
533 : }
534 : }
535 :
536 : /* Call dim_pos (POS == true) or dim_size (POS == false) builtins for
537 : axis DIM. Return a tmp var holding the result. */
538 :
539 : static tree
540 30709 : oacc_dim_call (bool pos, int dim, gimple_seq *seq)
541 : {
542 30709 : tree arg = build_int_cst (unsigned_type_node, dim);
543 30709 : tree size = create_tmp_var (integer_type_node);
544 30709 : enum internal_fn fn = pos ? IFN_GOACC_DIM_POS : IFN_GOACC_DIM_SIZE;
545 30709 : gimple *call = gimple_build_call_internal (fn, 1, arg);
546 :
547 30709 : gimple_call_set_lhs (call, size);
548 30709 : gimple_seq_add_stmt (seq, call);
549 :
550 30709 : return size;
551 : }
552 :
553 : /* Find the number of threads (POS = false), or thread number (POS =
554 : true) for an OpenACC region partitioned as MASK. Setup code
555 : required for the calculation is added to SEQ. */
556 :
557 : static tree
558 23610 : oacc_thread_numbers (bool pos, int mask, gimple_seq *seq)
559 : {
560 23610 : tree res = pos ? NULL_TREE : build_int_cst (unsigned_type_node, 1);
561 23610 : unsigned ix;
562 :
563 : /* Start at gang level, and examine relevant dimension indices. */
564 94440 : for (ix = GOMP_DIM_GANG; ix != GOMP_DIM_MAX; ix++)
565 70830 : if (GOMP_DIM_MASK (ix) & mask)
566 : {
567 26569 : if (res)
568 : {
569 : /* We had an outer index, so scale that by the size of
570 : this dimension. */
571 17369 : tree n = oacc_dim_call (false, ix, seq);
572 17369 : res = fold_build2 (MULT_EXPR, integer_type_node, res, n);
573 : }
574 26569 : if (pos)
575 : {
576 : /* Determine index in this dimension. */
577 13340 : tree id = oacc_dim_call (true, ix, seq);
578 13340 : if (res)
579 4140 : res = fold_build2 (PLUS_EXPR, integer_type_node, res, id);
580 : else
581 : res = id;
582 : }
583 : }
584 :
585 23610 : if (res == NULL_TREE)
586 2657 : res = integer_zero_node;
587 :
588 23610 : return res;
589 : }
590 :
591 : /* Transform IFN_GOACC_LOOP calls to actual code. See
592 : expand_oacc_for for where these are generated. At the vector
593 : level, we stride loops, such that each member of a warp will
594 : operate on adjacent iterations. At the worker and gang level,
595 : each gang/warp executes a set of contiguous iterations. Chunking
596 : can override this such that each iteration engine executes a
597 : contiguous chunk, and then moves on to stride to the next chunk. */
598 :
599 : static void
600 46702 : oacc_xform_loop (gcall *call)
601 : {
602 46702 : gimple_stmt_iterator gsi = gsi_for_stmt (call);
603 46702 : enum ifn_goacc_loop_kind code
604 46702 : = (enum ifn_goacc_loop_kind) TREE_INT_CST_LOW (gimple_call_arg (call, 0));
605 46702 : tree dir = gimple_call_arg (call, 1);
606 46702 : tree range = gimple_call_arg (call, 2);
607 46702 : tree step = gimple_call_arg (call, 3);
608 46702 : tree chunk_size = NULL_TREE;
609 46702 : unsigned mask = (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call, 5));
610 46702 : tree lhs = gimple_call_lhs (call);
611 46702 : tree type = NULL_TREE;
612 46702 : tree diff_type = TREE_TYPE (range);
613 46702 : tree r = NULL_TREE;
614 46702 : gimple_seq seq = NULL;
615 46702 : bool chunking = false, striding = true;
616 46702 : unsigned outer_mask = mask & (~mask + 1); // Outermost partitioning
617 46702 : unsigned inner_mask = mask & ~outer_mask; // Inner partitioning (if any)
618 :
619 : /* Skip lowering if return value of IFN_GOACC_LOOP call is not used. */
620 46702 : if (!lhs)
621 : {
622 8 : gsi_replace_with_seq (&gsi, seq, true);
623 8 : return;
624 : }
625 :
626 46694 : type = TREE_TYPE (lhs);
627 :
628 : #ifdef ACCEL_COMPILER
629 : chunk_size = gimple_call_arg (call, 4);
630 : if (integer_minus_onep (chunk_size) /* Force static allocation. */
631 : || integer_zerop (chunk_size)) /* Default (also static). */
632 : {
633 : /* If we're at the gang level, we want each to execute a
634 : contiguous run of iterations. Otherwise we want each element
635 : to stride. */
636 : striding = !(outer_mask & GOMP_DIM_MASK (GOMP_DIM_GANG));
637 : chunking = false;
638 : }
639 : else
640 : {
641 : /* Chunk of size 1 is striding. */
642 : striding = integer_onep (chunk_size);
643 : chunking = !striding;
644 : }
645 : #endif
646 :
647 : /* striding=true, chunking=true
648 : -> invalid.
649 : striding=true, chunking=false
650 : -> chunks=1
651 : striding=false,chunking=true
652 : -> chunks=ceil (range/(chunksize*threads*step))
653 : striding=false,chunking=false
654 : -> chunk_size=ceil(range/(threads*step)),chunks=1 */
655 46694 : push_gimplify_context (true);
656 :
657 46694 : switch (code)
658 : {
659 0 : default: gcc_unreachable ();
660 :
661 11231 : case IFN_GOACC_LOOP_CHUNKS:
662 11231 : if (!chunking)
663 11231 : r = build_int_cst (type, 1);
664 : else
665 : {
666 : /* chunk_max
667 : = (range - dir) / (chunks * step * num_threads) + dir */
668 : tree per = oacc_thread_numbers (false, mask, &seq);
669 : per = fold_convert (type, per);
670 : chunk_size = fold_convert (type, chunk_size);
671 : per = fold_build2 (MULT_EXPR, type, per, chunk_size);
672 : per = fold_build2 (MULT_EXPR, type, per, step);
673 : r = build2 (MINUS_EXPR, type, range, dir);
674 : r = build2 (PLUS_EXPR, type, r, per);
675 : r = build2 (TRUNC_DIV_EXPR, type, r, per);
676 : }
677 : break;
678 :
679 11753 : case IFN_GOACC_LOOP_STEP:
680 11753 : {
681 : /* If striding, step by the entire compute volume, otherwise
682 : step by the inner volume. */
683 11753 : unsigned volume = striding ? mask : inner_mask;
684 :
685 11753 : r = oacc_thread_numbers (false, volume, &seq);
686 11753 : r = build2 (MULT_EXPR, type, fold_convert (type, r), step);
687 : }
688 11753 : break;
689 :
690 11857 : case IFN_GOACC_LOOP_OFFSET:
691 : /* Enable vectorization on non-SIMT targets. */
692 11857 : if (!targetm.simt.vf
693 11857 : && outer_mask == GOMP_DIM_MASK (GOMP_DIM_VECTOR)
694 : /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
695 : the loop. */
696 1951 : && (flag_tree_loop_vectorize
697 1463 : || !OPTION_SET_P (flag_tree_loop_vectorize)))
698 : {
699 1951 : basic_block bb = gsi_bb (gsi);
700 1951 : class loop *parent = bb->loop_father;
701 1951 : class loop *body = parent->inner;
702 :
703 1951 : parent->force_vectorize = true;
704 1951 : parent->safelen = INT_MAX;
705 :
706 : /* "Chunking loops" may have inner loops. */
707 1951 : if (parent->inner)
708 : {
709 1939 : body->force_vectorize = true;
710 1939 : body->safelen = INT_MAX;
711 : }
712 :
713 1951 : cfun->has_force_vectorize_loops = true;
714 : }
715 11857 : if (striding)
716 : {
717 11857 : r = oacc_thread_numbers (true, mask, &seq);
718 11857 : r = fold_convert (diff_type, r);
719 : }
720 : else
721 : {
722 : tree inner_size = oacc_thread_numbers (false, inner_mask, &seq);
723 : tree outer_size = oacc_thread_numbers (false, outer_mask, &seq);
724 : tree volume = fold_build2 (MULT_EXPR, TREE_TYPE (inner_size),
725 : inner_size, outer_size);
726 :
727 : volume = fold_convert (diff_type, volume);
728 : if (chunking)
729 : chunk_size = fold_convert (diff_type, chunk_size);
730 : else
731 : {
732 : tree per = fold_build2 (MULT_EXPR, diff_type, volume, step);
733 :
734 : chunk_size = build2 (MINUS_EXPR, diff_type, range, dir);
735 : chunk_size = build2 (PLUS_EXPR, diff_type, chunk_size, per);
736 : chunk_size = build2 (TRUNC_DIV_EXPR, diff_type, chunk_size, per);
737 : }
738 :
739 : tree span = build2 (MULT_EXPR, diff_type, chunk_size,
740 : fold_convert (diff_type, inner_size));
741 : r = oacc_thread_numbers (true, outer_mask, &seq);
742 : r = fold_convert (diff_type, r);
743 : r = build2 (MULT_EXPR, diff_type, r, span);
744 :
745 : tree inner = oacc_thread_numbers (true, inner_mask, &seq);
746 : inner = fold_convert (diff_type, inner);
747 : r = fold_build2 (PLUS_EXPR, diff_type, r, inner);
748 :
749 : if (chunking)
750 : {
751 : tree chunk = fold_convert (diff_type, gimple_call_arg (call, 6));
752 : tree per
753 : = fold_build2 (MULT_EXPR, diff_type, volume, chunk_size);
754 : per = build2 (MULT_EXPR, diff_type, per, chunk);
755 :
756 : r = build2 (PLUS_EXPR, diff_type, r, per);
757 : }
758 : }
759 11857 : r = fold_build2 (MULT_EXPR, diff_type, r, step);
760 11857 : if (type != diff_type)
761 178 : r = fold_convert (type, r);
762 : break;
763 :
764 11853 : case IFN_GOACC_LOOP_BOUND:
765 11853 : if (striding)
766 11853 : r = range;
767 : else
768 : {
769 : tree inner_size = oacc_thread_numbers (false, inner_mask, &seq);
770 : tree outer_size = oacc_thread_numbers (false, outer_mask, &seq);
771 : tree volume = fold_build2 (MULT_EXPR, TREE_TYPE (inner_size),
772 : inner_size, outer_size);
773 :
774 : volume = fold_convert (diff_type, volume);
775 : if (chunking)
776 : chunk_size = fold_convert (diff_type, chunk_size);
777 : else
778 : {
779 : tree per = fold_build2 (MULT_EXPR, diff_type, volume, step);
780 :
781 : chunk_size = build2 (MINUS_EXPR, diff_type, range, dir);
782 : chunk_size = build2 (PLUS_EXPR, diff_type, chunk_size, per);
783 : chunk_size = build2 (TRUNC_DIV_EXPR, diff_type, chunk_size, per);
784 : }
785 :
786 : tree span = build2 (MULT_EXPR, diff_type, chunk_size,
787 : fold_convert (diff_type, inner_size));
788 :
789 : r = fold_build2 (MULT_EXPR, diff_type, span, step);
790 :
791 : tree offset = gimple_call_arg (call, 6);
792 : r = build2 (PLUS_EXPR, diff_type, r,
793 : fold_convert (diff_type, offset));
794 : r = build2 (integer_onep (dir) ? MIN_EXPR : MAX_EXPR,
795 : diff_type, r, range);
796 : }
797 11853 : if (diff_type != type)
798 178 : r = fold_convert (type, r);
799 : break;
800 : }
801 :
802 46694 : gimplify_assign (lhs, r, &seq);
803 :
804 46694 : pop_gimplify_context (NULL);
805 :
806 46694 : gsi_replace_with_seq (&gsi, seq, true);
807 : }
808 :
809 : /* Transform a GOACC_TILE call. Determines the element loop span for
810 : the specified loop of the nest. This is 1 if we're not tiling.
811 :
812 : GOACC_TILE (collapse_count, loop_no, tile_arg, gwv_tile, gwv_element); */
813 :
814 : static void
815 284 : oacc_xform_tile (gcall *call)
816 : {
817 284 : gimple_stmt_iterator gsi = gsi_for_stmt (call);
818 284 : unsigned collapse = tree_to_uhwi (gimple_call_arg (call, 0));
819 : /* Inner loops have higher loop_nos. */
820 284 : unsigned loop_no = tree_to_uhwi (gimple_call_arg (call, 1));
821 284 : tree tile_size = gimple_call_arg (call, 2);
822 284 : unsigned e_mask = tree_to_uhwi (gimple_call_arg (call, 4));
823 284 : tree lhs = gimple_call_lhs (call);
824 284 : tree type = TREE_TYPE (lhs);
825 284 : gimple_seq seq = NULL;
826 284 : tree span = build_int_cst (type, 1);
827 :
828 284 : gcc_assert (!(e_mask
829 : & ~(GOMP_DIM_MASK (GOMP_DIM_VECTOR)
830 : | GOMP_DIM_MASK (GOMP_DIM_WORKER))));
831 284 : push_gimplify_context (!seen_error ());
832 :
833 : #ifndef ACCEL_COMPILER
834 : /* Partitioning disabled on host compilers. */
835 284 : e_mask = 0;
836 : #endif
837 284 : if (!e_mask)
838 : /* Not partitioning. */
839 284 : span = integer_one_node;
840 : else if (!integer_zerop (tile_size))
841 : /* User explicitly specified size. */
842 : span = tile_size;
843 : else
844 : {
845 : /* Pick a size based on the partitioning of the element loop and
846 : the number of loop nests. */
847 : tree first_size = NULL_TREE;
848 : tree second_size = NULL_TREE;
849 :
850 : if (e_mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR))
851 : first_size = oacc_dim_call (false, GOMP_DIM_VECTOR, &seq);
852 : if (e_mask & GOMP_DIM_MASK (GOMP_DIM_WORKER))
853 : second_size = oacc_dim_call (false, GOMP_DIM_WORKER, &seq);
854 :
855 : if (!first_size)
856 : {
857 : first_size = second_size;
858 : second_size = NULL_TREE;
859 : }
860 :
861 : if (loop_no + 1 == collapse)
862 : {
863 : span = first_size;
864 : if (!loop_no && second_size)
865 : span = fold_build2 (MULT_EXPR, TREE_TYPE (span),
866 : span, second_size);
867 : }
868 : else if (loop_no + 2 == collapse)
869 : span = second_size;
870 : else
871 : span = NULL_TREE;
872 :
873 : if (!span)
874 : /* There's no obvious element size for this loop. Options
875 : are 1, first_size or some non-unity constant (32 is my
876 : favourite). We should gather some statistics. */
877 : span = first_size;
878 : }
879 :
880 284 : span = fold_convert (type, span);
881 284 : gimplify_assign (lhs, span, &seq);
882 :
883 284 : pop_gimplify_context (NULL);
884 :
885 284 : gsi_replace_with_seq (&gsi, seq, true);
886 284 : }
887 :
888 : /* Default partitioned and minimum partitioned dimensions. */
889 :
890 : static int oacc_default_dims[GOMP_DIM_MAX];
891 : static int oacc_min_dims[GOMP_DIM_MAX];
892 :
893 : int
894 0 : oacc_get_default_dim (int dim)
895 : {
896 0 : gcc_assert (0 <= dim && dim < GOMP_DIM_MAX);
897 0 : return oacc_default_dims[dim];
898 : }
899 :
900 : int
901 0 : oacc_get_min_dim (int dim)
902 : {
903 0 : gcc_assert (0 <= dim && dim < GOMP_DIM_MAX);
904 0 : return oacc_min_dims[dim];
905 : }
906 :
907 : /* Parse the default dimension parameter. This is a set of
908 : :-separated optional compute dimensions. Each specified dimension
909 : is a positive integer. When device type support is added, it is
910 : planned to be a comma separated list of such compute dimensions,
911 : with all but the first prefixed by the colon-terminated device
912 : type. */
913 :
914 : static void
915 2281 : oacc_parse_default_dims (const char *dims)
916 : {
917 2281 : int ix;
918 :
919 9124 : for (ix = GOMP_DIM_MAX; ix--;)
920 : {
921 6843 : oacc_default_dims[ix] = -1;
922 6843 : oacc_min_dims[ix] = 1;
923 : }
924 :
925 : #ifndef ACCEL_COMPILER
926 : /* Cannot be overridden on the host. */
927 2281 : dims = NULL;
928 : #endif
929 2281 : if (dims)
930 : {
931 : const char *pos = dims;
932 :
933 : for (ix = 0; *pos && ix != GOMP_DIM_MAX; ix++)
934 : {
935 : if (ix)
936 : {
937 : if (*pos != ':')
938 : goto malformed;
939 : pos++;
940 : }
941 :
942 : if (*pos != ':')
943 : {
944 : long val;
945 : const char *eptr;
946 :
947 : errno = 0;
948 : val = strtol (pos, const_cast<char **> (&eptr), 10);
949 : if (errno || val <= 0 || (int) val != val)
950 : goto malformed;
951 : pos = eptr;
952 : oacc_default_dims[ix] = (int) val;
953 : }
954 : }
955 : if (*pos)
956 : {
957 : malformed:
958 : error_at (UNKNOWN_LOCATION,
959 : "%<-fopenacc-dim%> operand is malformed at %qs", pos);
960 : }
961 : }
962 :
963 : /* Allow the backend to validate the dimensions. */
964 2281 : targetm.goacc.validate_dims (NULL_TREE, oacc_default_dims, -1, 0);
965 2281 : targetm.goacc.validate_dims (NULL_TREE, oacc_min_dims, -2, 0);
966 2281 : }
967 :
968 : /* Validate and update the dimensions for offloaded FN. ATTRS is the
969 : raw attribute. DIMS is an array of dimensions, which is filled in.
970 : LEVEL is the partitioning level of a routine, or -1 for an offload
971 : region itself. USED is the mask of partitioned execution in the
972 : function. */
973 :
974 : static void
975 9879 : oacc_validate_dims (tree fn, tree attrs, int *dims, int level, unsigned used)
976 : {
977 9879 : tree purpose[GOMP_DIM_MAX];
978 9879 : unsigned ix;
979 9879 : tree pos = TREE_VALUE (attrs);
980 :
981 : /* Make sure the attribute creator attached the dimension
982 : information. */
983 9879 : gcc_assert (pos);
984 :
985 39516 : for (ix = 0; ix != GOMP_DIM_MAX; ix++)
986 : {
987 29637 : purpose[ix] = TREE_PURPOSE (pos);
988 29637 : tree val = TREE_VALUE (pos);
989 29637 : dims[ix] = val ? TREE_INT_CST_LOW (val) : -1;
990 29637 : pos = TREE_CHAIN (pos);
991 : }
992 :
993 9879 : bool check = true;
994 : #ifdef ACCEL_COMPILER
995 : check = false;
996 : #endif
997 9879 : if (check
998 9879 : && warn_openacc_parallelism
999 1371 : && !lookup_attribute ("oacc kernels", DECL_ATTRIBUTES (fn)))
1000 : {
1001 1268 : static char const *const axes[] =
1002 : /* Must be kept in sync with GOMP_DIM enumeration. */
1003 : { "gang", "worker", "vector" };
1004 4775 : for (ix = level >= 0 ? level : 0; ix != GOMP_DIM_MAX; ix++)
1005 3507 : if (dims[ix] < 0)
1006 : ; /* Defaulting axis. */
1007 1970 : else if ((used & GOMP_DIM_MASK (ix)) && dims[ix] == 1)
1008 : /* There is partitioned execution, but the user requested a
1009 : dimension size of 1. They're probably confused. */
1010 94 : warning_at (DECL_SOURCE_LOCATION (fn), OPT_Wopenacc_parallelism,
1011 : "region contains %s partitioned code but"
1012 94 : " is not %s partitioned", axes[ix], axes[ix]);
1013 1876 : else if (!(used & GOMP_DIM_MASK (ix)) && dims[ix] != 1)
1014 : /* The dimension is explicitly partitioned to non-unity, but
1015 : no use is made within the region. */
1016 500 : warning_at (DECL_SOURCE_LOCATION (fn), OPT_Wopenacc_parallelism,
1017 : "region is %s partitioned but"
1018 : " does not contain %s partitioned code",
1019 500 : axes[ix], axes[ix]);
1020 : }
1021 :
1022 9879 : bool changed = targetm.goacc.validate_dims (fn, dims, level, used);
1023 :
1024 : /* Default anything left to 1 or a partitioned default. */
1025 49395 : for (ix = 0; ix != GOMP_DIM_MAX; ix++)
1026 29637 : if (dims[ix] < 0)
1027 : {
1028 : /* The OpenACC spec says 'If the [num_gangs] clause is not
1029 : specified, an implementation-defined default will be used;
1030 : the default may depend on the code within the construct.'
1031 : (2.5.6). Thus an implementation is free to choose
1032 : non-unity default for a parallel region that doesn't have
1033 : any gang-partitioned loops. However, it appears that there
1034 : is a sufficient body of user code that expects non-gang
1035 : partitioned regions to not execute in gang-redundant mode.
1036 : So we (a) don't warn about the non-portability and (b) pick
1037 : the minimum permissible dimension size when there is no
1038 : partitioned execution. Otherwise we pick the global
1039 : default for the dimension, which the user can control. The
1040 : same wording and logic applies to num_workers and
1041 : vector_length, however the worker- or vector- single
1042 : execution doesn't have the same impact as gang-redundant
1043 : execution. (If the minimum gang-level partitioning is not 1,
1044 : the target is probably too confusing.) */
1045 0 : dims[ix] = (used & GOMP_DIM_MASK (ix)
1046 0 : ? oacc_default_dims[ix] : oacc_min_dims[ix]);
1047 0 : changed = true;
1048 : }
1049 :
1050 9879 : if (changed)
1051 : {
1052 : /* Replace the attribute with new values. */
1053 : pos = NULL_TREE;
1054 35428 : for (ix = GOMP_DIM_MAX; ix--;)
1055 26571 : pos = tree_cons (purpose[ix],
1056 26571 : build_int_cst (integer_type_node, dims[ix]), pos);
1057 8857 : oacc_replace_fn_attrib (fn, pos);
1058 : }
1059 9879 : }
1060 :
1061 : /* Create an empty OpenACC loop structure at LOC. */
1062 :
1063 : static oacc_loop *
1064 21347 : new_oacc_loop_raw (oacc_loop *parent, location_t loc)
1065 : {
1066 10835 : oacc_loop *loop = XCNEW (oacc_loop);
1067 :
1068 21347 : loop->parent = parent;
1069 :
1070 10835 : if (parent)
1071 : {
1072 10835 : loop->sibling = parent->child;
1073 10835 : parent->child = loop;
1074 : }
1075 :
1076 21347 : loop->loc = loc;
1077 21347 : return loop;
1078 : }
1079 :
1080 : /* Create an outermost, dummy OpenACC loop for offloaded function
1081 : DECL. */
1082 :
1083 : static oacc_loop *
1084 9879 : new_oacc_loop_outer (tree decl)
1085 : {
1086 9879 : return new_oacc_loop_raw (NULL, DECL_SOURCE_LOCATION (decl));
1087 : }
1088 :
1089 : /* Start a new OpenACC loop structure beginning at head marker HEAD.
1090 : Link into PARENT loop. Return the new loop. */
1091 :
1092 : static oacc_loop *
1093 9634 : new_oacc_loop (oacc_loop *parent, gcall *marker)
1094 : {
1095 9634 : oacc_loop *loop = new_oacc_loop_raw (parent, gimple_location (marker));
1096 :
1097 9634 : loop->marker = marker;
1098 :
1099 : /* TODO: This is where device_type flattening would occur for the loop
1100 : flags. */
1101 :
1102 9634 : loop->flags = TREE_INT_CST_LOW (gimple_call_arg (marker, 3));
1103 :
1104 9634 : tree chunk_size = integer_zero_node;
1105 9634 : if (loop->flags & OLF_GANG_STATIC)
1106 146 : chunk_size = gimple_call_arg (marker, 4);
1107 9634 : loop->chunk_size = chunk_size;
1108 :
1109 9634 : return loop;
1110 : }
1111 :
1112 : /* Create a dummy loop encompassing a call to a openACC routine.
1113 : Extract the routine's partitioning requirements. */
1114 :
1115 : static void
1116 1201 : new_oacc_loop_routine (oacc_loop *parent, gcall *call, tree decl, tree attrs)
1117 : {
1118 1201 : oacc_loop *loop = new_oacc_loop_raw (parent, gimple_location (call));
1119 1201 : int level = oacc_fn_attrib_level (attrs);
1120 :
1121 1201 : gcc_assert (level >= 0);
1122 :
1123 1201 : loop->marker = call;
1124 1201 : loop->routine = decl;
1125 1201 : loop->mask = ((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1)
1126 1201 : ^ (GOMP_DIM_MASK (level) - 1));
1127 1201 : }
1128 :
1129 : /* Finish off the current OpenACC loop ending at tail marker TAIL.
1130 : Return the parent loop. */
1131 :
1132 : static oacc_loop *
1133 9634 : finish_oacc_loop (oacc_loop *loop)
1134 : {
1135 : /* If the loop has been collapsed, don't partition it. */
1136 0 : if (loop->ifns.is_empty ())
1137 0 : loop->mask = loop->flags = 0;
1138 9634 : return loop->parent;
1139 : }
1140 :
1141 : /* Free all OpenACC loop structures within LOOP (inclusive). */
1142 :
1143 : static void
1144 21347 : free_oacc_loop (oacc_loop *loop)
1145 : {
1146 21347 : if (loop->sibling)
1147 2194 : free_oacc_loop (loop->sibling);
1148 21347 : if (loop->child)
1149 8641 : free_oacc_loop (loop->child);
1150 :
1151 21347 : loop->ifns.release ();
1152 21347 : free (loop);
1153 21347 : }
1154 :
1155 : /* Dump out the OpenACC loop head or tail beginning at FROM. */
1156 :
1157 : static void
1158 238 : dump_oacc_loop_part (FILE *file, gcall *from, int depth,
1159 : const char *title, int level)
1160 : {
1161 238 : enum ifn_unique_kind kind
1162 238 : = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (from, 0));
1163 :
1164 238 : fprintf (file, "%*s%s-%d:\n", depth * 2, "", title, level);
1165 238 : for (gimple_stmt_iterator gsi = gsi_for_stmt (from);;)
1166 : {
1167 719 : gimple *stmt = gsi_stmt (gsi);
1168 :
1169 719 : if (gimple_call_internal_p (stmt, IFN_UNIQUE))
1170 : {
1171 719 : enum ifn_unique_kind k
1172 719 : = ((enum ifn_unique_kind) TREE_INT_CST_LOW
1173 719 : (gimple_call_arg (stmt, 0)));
1174 :
1175 719 : if (k == kind && stmt != from)
1176 : break;
1177 : }
1178 481 : print_gimple_stmt (file, stmt, depth * 2 + 2);
1179 :
1180 481 : gsi_next (&gsi);
1181 962 : while (gsi_end_p (gsi))
1182 962 : gsi = gsi_start_bb (single_succ (gsi_bb (gsi)));
1183 : }
1184 238 : }
1185 :
1186 : /* Dump OpenACC loop LOOP, its children, and its siblings. */
1187 :
1188 : static void
1189 183 : dump_oacc_loop (FILE *file, oacc_loop *loop, int depth)
1190 : {
1191 222 : int ix;
1192 :
1193 222 : fprintf (file, "%*sLoop %x(%x) %s:%u\n", depth * 2, "",
1194 : loop->flags, loop->mask,
1195 222 : LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc));
1196 :
1197 222 : if (loop->marker)
1198 108 : print_gimple_stmt (file, loop->marker, depth * 2);
1199 :
1200 222 : if (loop->routine)
1201 48 : fprintf (file, "%*sRoutine %s:%u:%s\n",
1202 48 : depth * 2, "", DECL_SOURCE_FILE (loop->routine),
1203 96 : DECL_SOURCE_LINE (loop->routine),
1204 48 : IDENTIFIER_POINTER (DECL_NAME (loop->routine)));
1205 :
1206 888 : for (ix = GOMP_DIM_GANG; ix != GOMP_DIM_MAX; ix++)
1207 666 : if (loop->heads[ix])
1208 119 : dump_oacc_loop_part (file, loop->heads[ix], depth, "Head", ix);
1209 888 : for (ix = GOMP_DIM_MAX; ix--;)
1210 666 : if (loop->tails[ix])
1211 119 : dump_oacc_loop_part (file, loop->tails[ix], depth, "Tail", ix);
1212 :
1213 222 : if (loop->child)
1214 69 : dump_oacc_loop (file, loop->child, depth + 1);
1215 222 : if (loop->sibling)
1216 : dump_oacc_loop (file, loop->sibling, depth);
1217 183 : }
1218 :
1219 : void debug_oacc_loop (oacc_loop *);
1220 :
1221 : /* Dump loops to stderr. */
1222 :
1223 : DEBUG_FUNCTION void
1224 0 : debug_oacc_loop (oacc_loop *loop)
1225 : {
1226 0 : dump_oacc_loop (stderr, loop, 0);
1227 0 : }
1228 :
1229 : /* Provide diagnostics on OpenACC loop LOOP, its children, and its
1230 : siblings. */
1231 :
1232 : static void
1233 2741 : inform_oacc_loop (const oacc_loop *loop)
1234 : {
1235 1796 : const char *gang
1236 2741 : = loop->mask & GOMP_DIM_MASK (GOMP_DIM_GANG) ? " gang" : "";
1237 2226 : const char *worker
1238 2741 : = loop->mask & GOMP_DIM_MASK (GOMP_DIM_WORKER) ? " worker" : "";
1239 1747 : const char *vector
1240 2741 : = loop->mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR) ? " vector" : "";
1241 2741 : const char *seq = loop->mask == 0 ? " seq" : "";
1242 2741 : const dump_user_location_t loc
1243 2741 : = dump_user_location_t::from_location_t (loop->loc);
1244 2741 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, loc,
1245 : "assigned OpenACC%s%s%s%s loop parallelism\n", gang, worker,
1246 : vector, seq);
1247 :
1248 2741 : if (loop->child)
1249 698 : inform_oacc_loop (loop->child);
1250 2741 : if (loop->sibling)
1251 280 : inform_oacc_loop (loop->sibling);
1252 2741 : }
1253 :
1254 : /* DFS walk of basic blocks BB onwards, creating OpenACC loop
1255 : structures as we go. By construction these loops are properly
1256 : nested. */
1257 :
1258 : static void
1259 176002 : oacc_loop_discover_walk (oacc_loop *loop, basic_block bb)
1260 : {
1261 176002 : int marker = 0;
1262 176002 : int remaining = 0;
1263 :
1264 176002 : if (bb->flags & BB_VISITED)
1265 40061 : return;
1266 :
1267 135941 : follow:
1268 202126 : bb->flags |= BB_VISITED;
1269 :
1270 : /* Scan for loop markers. */
1271 788600 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
1272 384348 : gsi_next (&gsi))
1273 : {
1274 384348 : gimple *stmt = gsi_stmt (gsi);
1275 :
1276 384348 : if (!is_gimple_call (stmt))
1277 220013 : continue;
1278 :
1279 169098 : gcall *call = as_a <gcall *> (stmt);
1280 :
1281 : /* If this is a routine, make a dummy loop for it. */
1282 169098 : if (tree decl = gimple_call_fndecl (call))
1283 4761 : if (tree attrs = oacc_get_fn_attrib (decl))
1284 : {
1285 1201 : gcc_assert (!marker);
1286 1201 : new_oacc_loop_routine (loop, call, decl, attrs);
1287 : }
1288 :
1289 169098 : if (!gimple_call_internal_p (call))
1290 4763 : continue;
1291 :
1292 164335 : switch (gimple_call_internal_fn (call))
1293 : {
1294 : default:
1295 : break;
1296 :
1297 46986 : case IFN_GOACC_LOOP:
1298 46986 : case IFN_GOACC_TILE:
1299 : /* Record the abstraction function, so we can manipulate it
1300 : later. */
1301 46986 : loop->ifns.safe_push (call);
1302 46986 : break;
1303 :
1304 85483 : case IFN_UNIQUE:
1305 85483 : enum ifn_unique_kind kind
1306 85483 : = (enum ifn_unique_kind) (TREE_INT_CST_LOW
1307 85483 : (gimple_call_arg (call, 0)));
1308 85483 : if (kind == IFN_UNIQUE_OACC_HEAD_MARK
1309 85483 : || kind == IFN_UNIQUE_OACC_TAIL_MARK)
1310 : {
1311 52246 : if (gimple_call_num_args (call) == 2)
1312 : {
1313 19268 : gcc_assert (marker && !remaining);
1314 19268 : marker = 0;
1315 19268 : if (kind == IFN_UNIQUE_OACC_TAIL_MARK)
1316 19268 : loop = finish_oacc_loop (loop);
1317 : else
1318 9634 : loop->head_end = call;
1319 : }
1320 : else
1321 : {
1322 32978 : int count = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
1323 :
1324 32978 : if (!marker)
1325 : {
1326 19268 : if (kind == IFN_UNIQUE_OACC_HEAD_MARK)
1327 9634 : loop = new_oacc_loop (loop, call);
1328 : remaining = count;
1329 : }
1330 32978 : gcc_assert (count == remaining);
1331 32978 : if (remaining)
1332 : {
1333 32978 : remaining--;
1334 32978 : if (kind == IFN_UNIQUE_OACC_HEAD_MARK)
1335 16489 : loop->heads[marker] = call;
1336 : else
1337 16489 : loop->tails[remaining] = call;
1338 : }
1339 32978 : marker++;
1340 : }
1341 : }
1342 : }
1343 : }
1344 202126 : if (remaining || marker)
1345 : {
1346 66185 : bb = single_succ (bb);
1347 66185 : gcc_assert (single_pred_p (bb) && !(bb->flags & BB_VISITED));
1348 66185 : goto follow;
1349 : }
1350 :
1351 : /* Walk successor blocks. */
1352 135941 : edge e;
1353 135941 : edge_iterator ei;
1354 :
1355 302064 : FOR_EACH_EDGE (e, ei, bb->succs)
1356 166123 : oacc_loop_discover_walk (loop, e->dest);
1357 : }
1358 :
1359 : /* LOOP is the first sibling. Reverse the order in place and return
1360 : the new first sibling. Recurse to child loops. */
1361 :
1362 : static oacc_loop *
1363 18520 : oacc_loop_sibling_nreverse (oacc_loop *loop)
1364 : {
1365 18520 : oacc_loop *last = NULL;
1366 20714 : do
1367 : {
1368 20714 : if (loop->child)
1369 8641 : loop->child = oacc_loop_sibling_nreverse (loop->child);
1370 :
1371 20714 : oacc_loop *next = loop->sibling;
1372 20714 : loop->sibling = last;
1373 20714 : last = loop;
1374 20714 : loop = next;
1375 : }
1376 20714 : while (loop);
1377 :
1378 18520 : return last;
1379 : }
1380 :
1381 : /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
1382 : the current function. */
1383 :
1384 : static oacc_loop *
1385 9879 : oacc_loop_discovery ()
1386 : {
1387 : /* Clear basic block flags, in particular BB_VISITED which we're going to use
1388 : in the following. */
1389 9879 : clear_bb_flags ();
1390 :
1391 9879 : oacc_loop *top = new_oacc_loop_outer (current_function_decl);
1392 9879 : oacc_loop_discover_walk (top, ENTRY_BLOCK_PTR_FOR_FN (cfun));
1393 :
1394 : /* The siblings were constructed in reverse order, reverse them so
1395 : that diagnostics come out in an unsurprising order. */
1396 9879 : top = oacc_loop_sibling_nreverse (top);
1397 :
1398 9879 : return top;
1399 : }
1400 :
1401 : /* Transform the abstract internal function markers starting at FROM
1402 : to be for partitioning level LEVEL. Stop when we meet another HEAD
1403 : or TAIL marker. */
1404 :
1405 : static void
1406 25614 : oacc_loop_xform_head_tail (gcall *from, int level)
1407 : {
1408 25614 : enum ifn_unique_kind kind
1409 25614 : = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (from, 0));
1410 25614 : tree replacement = build_int_cst (unsigned_type_node, level);
1411 :
1412 25614 : for (gimple_stmt_iterator gsi = gsi_for_stmt (from);;)
1413 : {
1414 107435 : gimple *stmt = gsi_stmt (gsi);
1415 :
1416 107435 : if (gimple_call_internal_p (stmt, IFN_UNIQUE))
1417 : {
1418 77068 : enum ifn_unique_kind k
1419 : = ((enum ifn_unique_kind)
1420 77068 : TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
1421 :
1422 77068 : if (k == IFN_UNIQUE_OACC_FORK
1423 77068 : || k == IFN_UNIQUE_OACC_JOIN
1424 77068 : || k == IFN_UNIQUE_OACC_PRIVATE)
1425 25840 : *gimple_call_arg_ptr (stmt, 2) = replacement;
1426 51228 : else if (k == kind && stmt != from)
1427 : break;
1428 : }
1429 30367 : else if (gimple_call_internal_p (stmt, IFN_GOACC_REDUCTION))
1430 19228 : *gimple_call_arg_ptr (stmt, 3) = replacement;
1431 81821 : update_stmt (stmt);
1432 :
1433 81821 : gsi_next (&gsi);
1434 133275 : while (gsi_end_p (gsi))
1435 102908 : gsi = gsi_start_bb (single_succ (gsi_bb (gsi)));
1436 : }
1437 25614 : }
1438 :
1439 : /* Process the discovered OpenACC loops, setting the correct
1440 : partitioning level etc. */
1441 :
1442 : static void
1443 20714 : oacc_loop_process (oacc_loop *loop, int fn_level)
1444 : {
1445 20714 : if (loop->child)
1446 8641 : oacc_loop_process (loop->child, fn_level);
1447 :
1448 20714 : if (loop->mask && !loop->routine)
1449 : {
1450 8645 : int ix;
1451 8645 : tree mask_arg = build_int_cst (unsigned_type_node, loop->mask);
1452 8645 : tree e_mask_arg = build_int_cst (unsigned_type_node, loop->e_mask);
1453 8645 : tree chunk_arg = loop->chunk_size;
1454 8645 : gcall *call;
1455 :
1456 43708 : for (ix = 0; loop->ifns.iterate (ix, &call); ix++)
1457 : {
1458 35063 : switch (gimple_call_internal_fn (call))
1459 : {
1460 34857 : case IFN_GOACC_LOOP:
1461 34857 : {
1462 34857 : bool is_e = gimple_call_arg (call, 5) == integer_minus_one_node;
1463 69335 : gimple_call_set_arg (call, 5, is_e ? e_mask_arg : mask_arg);
1464 34857 : if (!is_e)
1465 34478 : gimple_call_set_arg (call, 4, chunk_arg);
1466 : }
1467 : break;
1468 :
1469 206 : case IFN_GOACC_TILE:
1470 206 : gimple_call_set_arg (call, 3, mask_arg);
1471 206 : gimple_call_set_arg (call, 4, e_mask_arg);
1472 206 : break;
1473 :
1474 0 : default:
1475 0 : gcc_unreachable ();
1476 : }
1477 35063 : update_stmt (call);
1478 : }
1479 :
1480 8645 : unsigned dim = GOMP_DIM_GANG;
1481 8645 : unsigned mask = loop->mask | loop->e_mask;
1482 21452 : for (ix = 0; ix != GOMP_DIM_MAX && mask; ix++)
1483 : {
1484 25113 : while (!(GOMP_DIM_MASK (dim) & mask))
1485 12306 : dim++;
1486 :
1487 12807 : oacc_loop_xform_head_tail (loop->heads[ix], dim);
1488 12807 : oacc_loop_xform_head_tail (loop->tails[ix], dim);
1489 :
1490 12807 : mask ^= GOMP_DIM_MASK (dim);
1491 : }
1492 : }
1493 :
1494 20714 : if (loop->sibling)
1495 2194 : oacc_loop_process (loop->sibling, fn_level);
1496 :
1497 :
1498 : /* OpenACC 2.6, 2.9.11. "reduction clause" places a restriction such that
1499 : "The 'reduction' clause may not be specified on an orphaned 'loop'
1500 : construct with the 'gang' clause, or on an orphaned 'loop' construct that
1501 : will generate gang parallelism in a procedure that is compiled with the
1502 : 'routine gang' clause." */
1503 20714 : if (fn_level == GOMP_DIM_GANG
1504 624 : && (loop->mask & GOMP_DIM_MASK (GOMP_DIM_GANG))
1505 209 : && (loop->flags & OLF_REDUCTION))
1506 106 : error_at (loop->loc,
1507 : "gang reduction on an orphan loop");
1508 20714 : }
1509 :
1510 : /* Walk the OpenACC loop hierarchy checking and assigning the
1511 : programmer-specified partitionings. OUTER_MASK is the partitioning
1512 : this loop is contained within. Return mask of partitioning
1513 : encountered. If any auto loops are discovered, set GOMP_DIM_MAX
1514 : bit. */
1515 :
1516 : static unsigned
1517 20714 : oacc_loop_fixed_partitions (oacc_loop *loop, unsigned outer_mask)
1518 : {
1519 20714 : unsigned this_mask = loop->mask;
1520 20714 : unsigned mask_all = 0;
1521 20714 : bool noisy = true;
1522 :
1523 : #ifdef ACCEL_COMPILER
1524 : /* When device_type is supported, we want the device compiler to be
1525 : noisy, if the loop parameters are device_type-specific. */
1526 : noisy = false;
1527 : #endif
1528 :
1529 20714 : if (!loop->routine)
1530 : {
1531 19513 : bool auto_par = (loop->flags & OLF_AUTO) != 0;
1532 19513 : bool seq_par = (loop->flags & OLF_SEQ) != 0;
1533 19513 : bool tiling = (loop->flags & OLF_TILE) != 0;
1534 :
1535 19513 : this_mask = ((loop->flags >> OLF_DIM_BASE)
1536 : & (GOMP_DIM_MASK (GOMP_DIM_MAX) - 1));
1537 :
1538 : /* Apply auto partitioning if this is a non-partitioned regular
1539 : loop, or (no more than) single axis tiled loop. */
1540 39026 : bool maybe_auto
1541 19513 : = !seq_par && this_mask == (tiling ? this_mask & -this_mask : 0);
1542 :
1543 19513 : if ((this_mask != 0) + auto_par + seq_par > 1)
1544 : {
1545 170 : if (noisy)
1546 250 : error_at (loop->loc,
1547 : seq_par
1548 : ? G_("%<seq%> overrides other OpenACC loop specifiers")
1549 : : G_("%<auto%> conflicts with other OpenACC loop "
1550 : "specifiers"));
1551 170 : maybe_auto = false;
1552 170 : loop->flags &= ~OLF_AUTO;
1553 170 : if (seq_par)
1554 : {
1555 90 : loop->flags
1556 90 : &= ~((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1) << OLF_DIM_BASE);
1557 90 : this_mask = 0;
1558 : }
1559 : }
1560 :
1561 19433 : if (maybe_auto && (loop->flags & OLF_INDEPENDENT))
1562 : {
1563 5699 : loop->flags |= OLF_AUTO;
1564 5699 : mask_all |= GOMP_DIM_MASK (GOMP_DIM_MAX);
1565 : }
1566 : }
1567 :
1568 20714 : if (this_mask & outer_mask)
1569 : {
1570 248 : const oacc_loop *outer;
1571 350 : for (outer = loop->parent; outer; outer = outer->parent)
1572 248 : if ((outer->mask | outer->e_mask) & this_mask)
1573 : break;
1574 :
1575 248 : if (noisy)
1576 : {
1577 248 : if (outer)
1578 : {
1579 146 : error_at (loop->loc,
1580 146 : loop->routine
1581 : ? G_("routine call uses same OpenACC parallelism"
1582 : " as containing loop")
1583 : : G_("inner loop uses same OpenACC parallelism"
1584 : " as containing loop"));
1585 146 : inform (outer->loc, "containing loop here");
1586 : }
1587 : else
1588 102 : error_at (loop->loc,
1589 102 : loop->routine
1590 : ? G_("routine call uses OpenACC parallelism disallowed"
1591 : " by containing routine")
1592 : : G_("loop uses OpenACC parallelism disallowed"
1593 : " by containing routine"));
1594 :
1595 248 : if (loop->routine)
1596 154 : inform (DECL_SOURCE_LOCATION (loop->routine),
1597 : "routine %qD declared here", loop->routine);
1598 : }
1599 248 : this_mask &= ~outer_mask;
1600 : }
1601 : else
1602 : {
1603 20466 : unsigned outermost = least_bit_hwi (this_mask);
1604 :
1605 20466 : if (outermost && outermost <= outer_mask)
1606 : {
1607 40 : if (noisy)
1608 : {
1609 40 : error_at (loop->loc,
1610 : "incorrectly nested OpenACC loop parallelism");
1611 :
1612 40 : const oacc_loop *outer;
1613 40 : for (outer = loop->parent;
1614 40 : outer->flags && outer->flags < outermost;
1615 0 : outer = outer->parent)
1616 0 : continue;
1617 40 : inform (outer->loc, "containing loop here");
1618 0 : }
1619 :
1620 40 : this_mask &= ~outermost;
1621 : }
1622 : }
1623 :
1624 20714 : mask_all |= this_mask;
1625 :
1626 20714 : if (loop->flags & OLF_TILE)
1627 : {
1628 : /* When tiling, vector goes to the element loop, and failing
1629 : that we put worker there. The std doesn't contemplate
1630 : specifying all three. We choose to put worker and vector on
1631 : the element loops in that case. */
1632 136 : unsigned this_e_mask = this_mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR);
1633 136 : if (!this_e_mask || this_mask & GOMP_DIM_MASK (GOMP_DIM_GANG))
1634 120 : this_e_mask |= this_mask & GOMP_DIM_MASK (GOMP_DIM_WORKER);
1635 :
1636 136 : loop->e_mask = this_e_mask;
1637 136 : this_mask ^= this_e_mask;
1638 : }
1639 :
1640 20714 : loop->mask = this_mask;
1641 :
1642 20714 : if (dump_file)
1643 222 : fprintf (dump_file, "Loop %s:%d user specified %d & %d\n",
1644 444 : LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc),
1645 : loop->mask, loop->e_mask);
1646 :
1647 20714 : if (loop->child)
1648 : {
1649 8641 : unsigned tmp_mask = outer_mask | this_mask | loop->e_mask;
1650 8641 : loop->inner = oacc_loop_fixed_partitions (loop->child, tmp_mask);
1651 8641 : mask_all |= loop->inner;
1652 : }
1653 :
1654 20714 : if (loop->sibling)
1655 2194 : mask_all |= oacc_loop_fixed_partitions (loop->sibling, outer_mask);
1656 :
1657 20714 : return mask_all;
1658 : }
1659 :
1660 : /* Walk the OpenACC loop hierarchy to assign auto-partitioned loops.
1661 : OUTER_MASK is the partitioning this loop is contained within.
1662 : OUTER_ASSIGN is true if an outer loop is being auto-partitioned.
1663 : Return the cumulative partitioning used by this loop, siblings and
1664 : children. */
1665 :
1666 : static unsigned
1667 10192 : oacc_loop_auto_partitions (oacc_loop *loop, unsigned outer_mask,
1668 : bool outer_assign)
1669 : {
1670 10192 : bool assign = (loop->flags & OLF_AUTO) && (loop->flags & OLF_INDEPENDENT);
1671 10192 : bool noisy = true;
1672 10192 : bool tiling = loop->flags & OLF_TILE;
1673 :
1674 : #ifdef ACCEL_COMPILER
1675 : /* When device_type is supported, we want the device compiler to be
1676 : noisy, if the loop parameters are device_type-specific. */
1677 : noisy = false;
1678 : #endif
1679 :
1680 10192 : if (assign && (!outer_assign || loop->inner))
1681 : {
1682 : /* Allocate outermost and non-innermost loops at the outermost
1683 : non-innermost available level. */
1684 : unsigned this_mask = GOMP_DIM_MASK (GOMP_DIM_GANG);
1685 :
1686 : /* Find the first outermost available partition. */
1687 6659 : while (this_mask <= outer_mask)
1688 1923 : this_mask <<= 1;
1689 :
1690 : /* Grab two axes if tiling, and we've not assigned anything */
1691 4736 : if (tiling && !(loop->mask | loop->e_mask))
1692 94 : this_mask |= this_mask << 1;
1693 :
1694 : /* Prohibit the innermost partitioning at the moment. */
1695 4736 : this_mask &= GOMP_DIM_MASK (GOMP_DIM_MAX - 1) - 1;
1696 :
1697 : /* Don't use any dimension explicitly claimed by an inner loop. */
1698 4736 : this_mask &= ~loop->inner;
1699 :
1700 4736 : if (tiling && !loop->e_mask)
1701 : {
1702 : /* If we got two axes, allocate the inner one to the element
1703 : loop. */
1704 98 : loop->e_mask = this_mask & (this_mask << 1);
1705 98 : this_mask ^= loop->e_mask;
1706 : }
1707 :
1708 4736 : loop->mask |= this_mask;
1709 : }
1710 :
1711 10192 : if (loop->child)
1712 : {
1713 5360 : unsigned tmp_mask = outer_mask | loop->mask | loop->e_mask;
1714 5360 : loop->inner = oacc_loop_auto_partitions (loop->child, tmp_mask,
1715 5360 : outer_assign | assign);
1716 : }
1717 :
1718 10192 : if (assign && (!loop->mask || (tiling && !loop->e_mask) || !outer_assign))
1719 : {
1720 : /* Allocate the loop at the innermost available level. Note
1721 : that we do this even if we already assigned this loop the
1722 : outermost available level above. That way we'll partition
1723 : this along 2 axes, if they are available. */
1724 5044 : unsigned this_mask = 0;
1725 :
1726 : /* Determine the outermost partitioning used within this loop. */
1727 5044 : this_mask = loop->inner | GOMP_DIM_MASK (GOMP_DIM_MAX);
1728 5044 : this_mask = least_bit_hwi (this_mask);
1729 :
1730 : /* Pick the partitioning just inside that one. */
1731 5044 : this_mask >>= 1;
1732 :
1733 : /* And avoid picking one use by an outer loop. */
1734 5044 : this_mask &= ~outer_mask;
1735 :
1736 : /* If tiling and we failed completely above, grab the next one
1737 : too. Making sure it doesn't hit an outer loop. */
1738 5044 : if (tiling)
1739 : {
1740 110 : this_mask &= ~(loop->e_mask | loop->mask);
1741 110 : unsigned tile_mask = ((this_mask >> 1)
1742 110 : & ~(outer_mask | loop->e_mask | loop->mask));
1743 :
1744 110 : if (tile_mask || loop->mask)
1745 : {
1746 102 : loop->e_mask |= this_mask;
1747 102 : this_mask = tile_mask;
1748 : }
1749 110 : if (!loop->e_mask && noisy)
1750 8 : warning_at (loop->loc, 0,
1751 : "insufficient partitioning available"
1752 : " to parallelize element loop");
1753 : }
1754 :
1755 5044 : loop->mask |= this_mask;
1756 5044 : if (!loop->mask && noisy)
1757 1078 : warning_at (loop->loc, 0,
1758 : tiling
1759 : ? G_("insufficient partitioning available"
1760 : " to parallelize tile loop")
1761 : : G_("insufficient partitioning available"
1762 : " to parallelize loop"));
1763 : }
1764 :
1765 5699 : if (assign && dump_file)
1766 41 : fprintf (dump_file, "Auto loop %s:%d assigned %d & %d\n",
1767 82 : LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc),
1768 : loop->mask, loop->e_mask);
1769 :
1770 10192 : unsigned inner_mask = 0;
1771 :
1772 10192 : if (loop->sibling)
1773 1714 : inner_mask |= oacc_loop_auto_partitions (loop->sibling,
1774 : outer_mask, outer_assign);
1775 :
1776 10192 : inner_mask |= loop->inner | loop->mask | loop->e_mask;
1777 :
1778 10192 : return inner_mask;
1779 : }
1780 :
1781 : /* Walk the OpenACC loop hierarchy to check and assign partitioning
1782 : axes. Return mask of partitioning. */
1783 :
1784 : static unsigned
1785 9879 : oacc_loop_partition (oacc_loop *loop, unsigned outer_mask)
1786 : {
1787 9879 : unsigned mask_all = oacc_loop_fixed_partitions (loop, outer_mask);
1788 :
1789 9879 : if (mask_all & GOMP_DIM_MASK (GOMP_DIM_MAX))
1790 : {
1791 3118 : mask_all ^= GOMP_DIM_MASK (GOMP_DIM_MAX);
1792 3118 : mask_all |= oacc_loop_auto_partitions (loop, outer_mask, false);
1793 : }
1794 9879 : return mask_all;
1795 : }
1796 :
1797 : /* Default fork/join early expander. Delete the function calls if
1798 : there is no RTL expander. */
1799 :
1800 : bool
1801 25614 : default_goacc_fork_join (gcall *ARG_UNUSED (call),
1802 : const int *ARG_UNUSED (dims), bool is_fork)
1803 : {
1804 25614 : if (is_fork)
1805 12807 : return targetm.have_oacc_fork ();
1806 : else
1807 12807 : return targetm.have_oacc_join ();
1808 : }
1809 :
1810 : /* Default goacc.reduction early expander.
1811 :
1812 : LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
1813 : If RES_PTR is not integer-zerop:
1814 : SETUP - emit 'LHS = *RES_PTR', LHS = NULL
1815 : TEARDOWN - emit '*RES_PTR = VAR'
1816 : If LHS is not NULL
1817 : emit 'LHS = VAR' */
1818 :
1819 : void
1820 30884 : default_goacc_reduction (gcall *call)
1821 : {
1822 30884 : unsigned code = (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call, 0));
1823 30884 : gimple_stmt_iterator gsi = gsi_for_stmt (call);
1824 30884 : tree lhs = gimple_call_lhs (call);
1825 30884 : tree var = gimple_call_arg (call, 2);
1826 30884 : gimple_seq seq = NULL;
1827 :
1828 30884 : if (code == IFN_GOACC_REDUCTION_SETUP
1829 30884 : || code == IFN_GOACC_REDUCTION_TEARDOWN)
1830 : {
1831 : /* Setup and Teardown need to copy from/to the receiver object,
1832 : if there is one. */
1833 15442 : tree ref_to_res = gimple_call_arg (call, 1);
1834 :
1835 15442 : if (!integer_zerop (ref_to_res))
1836 : {
1837 5086 : tree dst = build_simple_mem_ref (ref_to_res);
1838 5086 : tree src = var;
1839 :
1840 5086 : if (code == IFN_GOACC_REDUCTION_SETUP)
1841 : {
1842 2543 : src = dst;
1843 2543 : dst = lhs;
1844 2543 : lhs = NULL;
1845 : }
1846 5086 : gimple_seq_add_stmt (&seq, gimple_build_assign (dst, src));
1847 : }
1848 : }
1849 :
1850 : /* Copy VAR to LHS, if there is an LHS. */
1851 30884 : if (lhs)
1852 26734 : gimple_seq_add_stmt (&seq, gimple_build_assign (lhs, var));
1853 :
1854 30884 : gsi_replace_with_seq (&gsi, seq, true);
1855 30884 : }
1856 :
1857 : struct var_decl_rewrite_info
1858 : {
1859 : gimple *stmt;
1860 : hash_map<tree, tree> *adjusted_vars;
1861 : bool avoid_pointer_conversion;
1862 : bool modified;
1863 : };
1864 :
1865 : /* Helper function for execute_oacc_device_lower. Rewrite VAR_DECLs (by
1866 : themselves or wrapped in various other nodes) according to ADJUSTED_VARS in
1867 : the var_decl_rewrite_info pointed to via DATA. Used as part of coercing
1868 : gang-private variables in OpenACC offload regions to reside in GPU shared
1869 : memory. */
1870 :
1871 : static tree
1872 0 : oacc_rewrite_var_decl (tree *tp, int *walk_subtrees, void *data)
1873 : {
1874 0 : walk_stmt_info *wi = (walk_stmt_info *) data;
1875 0 : var_decl_rewrite_info *info = (var_decl_rewrite_info *) wi->info;
1876 :
1877 0 : if (TREE_CODE (*tp) == ADDR_EXPR)
1878 : {
1879 0 : tree arg = TREE_OPERAND (*tp, 0);
1880 0 : tree *new_arg = info->adjusted_vars->get (arg);
1881 :
1882 0 : if (new_arg)
1883 : {
1884 0 : if (info->avoid_pointer_conversion)
1885 : {
1886 0 : *tp = build_fold_addr_expr (*new_arg);
1887 0 : info->modified = true;
1888 0 : *walk_subtrees = 0;
1889 : }
1890 : else
1891 : {
1892 0 : gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
1893 0 : tree repl = build_fold_addr_expr (*new_arg);
1894 0 : gimple *stmt1
1895 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
1896 0 : tree conv = convert_to_pointer (TREE_TYPE (*tp),
1897 : gimple_assign_lhs (stmt1));
1898 0 : gimple *stmt2
1899 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (*tp)), conv);
1900 0 : gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
1901 0 : gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
1902 0 : *tp = gimple_assign_lhs (stmt2);
1903 0 : info->modified = true;
1904 0 : *walk_subtrees = 0;
1905 : }
1906 : }
1907 : }
1908 0 : else if (TREE_CODE (*tp) == COMPONENT_REF || TREE_CODE (*tp) == ARRAY_REF)
1909 : {
1910 0 : tree *base = &TREE_OPERAND (*tp, 0);
1911 :
1912 0 : while (TREE_CODE (*base) == COMPONENT_REF
1913 0 : || TREE_CODE (*base) == ARRAY_REF)
1914 0 : base = &TREE_OPERAND (*base, 0);
1915 :
1916 0 : if (TREE_CODE (*base) != VAR_DECL)
1917 : return NULL;
1918 :
1919 0 : tree *new_decl = info->adjusted_vars->get (*base);
1920 0 : if (!new_decl)
1921 : return NULL;
1922 :
1923 0 : int base_quals = TYPE_QUALS (TREE_TYPE (*new_decl));
1924 0 : tree field = TREE_OPERAND (*tp, 1);
1925 :
1926 : /* Adjust the type of the field. */
1927 0 : int field_quals = TYPE_QUALS (TREE_TYPE (field));
1928 0 : if (TREE_CODE (field) == FIELD_DECL && field_quals != base_quals)
1929 : {
1930 0 : tree *field_type = &TREE_TYPE (field);
1931 0 : while (TREE_CODE (*field_type) == ARRAY_TYPE)
1932 0 : field_type = &TREE_TYPE (*field_type);
1933 0 : field_quals |= base_quals;
1934 0 : *field_type = build_qualified_type (*field_type, field_quals);
1935 : }
1936 :
1937 : /* Adjust the type of the component ref itself. */
1938 0 : tree comp_type = TREE_TYPE (*tp);
1939 0 : int comp_quals = TYPE_QUALS (comp_type);
1940 0 : if (TREE_CODE (*tp) == COMPONENT_REF && comp_quals != base_quals)
1941 : {
1942 0 : comp_quals |= base_quals;
1943 0 : TREE_TYPE (*tp)
1944 0 : = build_qualified_type (comp_type, comp_quals);
1945 : }
1946 :
1947 0 : *base = *new_decl;
1948 0 : info->modified = true;
1949 0 : }
1950 0 : else if (VAR_P (*tp))
1951 : {
1952 0 : tree *new_decl = info->adjusted_vars->get (*tp);
1953 0 : if (new_decl)
1954 : {
1955 0 : *tp = *new_decl;
1956 0 : info->modified = true;
1957 : }
1958 : }
1959 :
1960 : return NULL_TREE;
1961 : }
1962 :
1963 : /* Return TRUE if CALL is a call to a builtin atomic/sync operation. */
1964 :
1965 : static bool
1966 0 : is_sync_builtin_call (gcall *call)
1967 : {
1968 0 : tree callee = gimple_call_fndecl (call);
1969 :
1970 0 : if (callee != NULL_TREE
1971 0 : && gimple_call_builtin_p (call, BUILT_IN_NORMAL))
1972 0 : switch (DECL_FUNCTION_CODE (callee))
1973 : {
1974 : #undef DEF_SYNC_BUILTIN
1975 : #define DEF_SYNC_BUILTIN(ENUM, NAME, TYPE, ATTRS) case ENUM:
1976 : #include "sync-builtins.def"
1977 : #undef DEF_SYNC_BUILTIN
1978 0 : return true;
1979 :
1980 : default:
1981 : ;
1982 : }
1983 :
1984 : return false;
1985 : }
1986 :
1987 : /* Main entry point for oacc transformations which run on the device
1988 : compiler after LTO, so we know what the target device is at this
1989 : point (including the host fallback). */
1990 :
1991 : static unsigned int
1992 15341 : execute_oacc_loop_designation ()
1993 : {
1994 15341 : tree attrs = oacc_get_fn_attrib (current_function_decl);
1995 :
1996 15341 : if (!attrs)
1997 : /* Not an offloaded function. */
1998 : return 0;
1999 :
2000 : /* Parse the default dim argument exactly once. */
2001 9941 : if ((const void *)flag_openacc_dims != &flag_openacc_dims)
2002 : {
2003 2281 : oacc_parse_default_dims (flag_openacc_dims);
2004 2281 : flag_openacc_dims = (char *)&flag_openacc_dims;
2005 : }
2006 :
2007 9941 : bool is_oacc_parallel
2008 9941 : = (lookup_attribute ("oacc parallel",
2009 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2010 9941 : bool is_oacc_kernels
2011 9941 : = (lookup_attribute ("oacc kernels",
2012 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2013 9941 : bool is_oacc_serial
2014 9941 : = (lookup_attribute ("oacc serial",
2015 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2016 9941 : bool is_oacc_parallel_kernels_parallelized
2017 9941 : = (lookup_attribute ("oacc parallel_kernels_parallelized",
2018 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2019 9941 : bool is_oacc_parallel_kernels_gang_single
2020 9941 : = (lookup_attribute ("oacc parallel_kernels_gang_single",
2021 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2022 9941 : int fn_level = oacc_fn_attrib_level (attrs);
2023 9941 : bool is_oacc_routine = (fn_level >= 0);
2024 9941 : gcc_checking_assert (is_oacc_parallel
2025 : + is_oacc_kernels
2026 : + is_oacc_serial
2027 : + is_oacc_parallel_kernels_parallelized
2028 : + is_oacc_parallel_kernels_gang_single
2029 : + is_oacc_routine
2030 : == 1);
2031 :
2032 9941 : bool is_oacc_kernels_parallelized
2033 9941 : = (lookup_attribute ("oacc kernels parallelized",
2034 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2035 9941 : if (is_oacc_kernels_parallelized)
2036 386 : gcc_checking_assert (is_oacc_kernels);
2037 :
2038 9941 : if (dump_file)
2039 : {
2040 154 : if (is_oacc_parallel)
2041 34 : fprintf (dump_file, "Function is OpenACC parallel offload\n");
2042 120 : else if (is_oacc_kernels)
2043 66 : fprintf (dump_file, "Function is %s OpenACC kernels offload\n",
2044 : (is_oacc_kernels_parallelized
2045 : ? "parallelized" : "unparallelized"));
2046 82 : else if (is_oacc_serial)
2047 26 : fprintf (dump_file, "Function is OpenACC serial offload\n");
2048 56 : else if (is_oacc_parallel_kernels_parallelized)
2049 0 : fprintf (dump_file, "Function is %s OpenACC kernels offload\n",
2050 : "parallel_kernels_parallelized");
2051 56 : else if (is_oacc_parallel_kernels_gang_single)
2052 0 : fprintf (dump_file, "Function is %s OpenACC kernels offload\n",
2053 : "parallel_kernels_gang_single");
2054 56 : else if (is_oacc_routine)
2055 56 : fprintf (dump_file, "Function is OpenACC routine level %d\n",
2056 : fn_level);
2057 : else
2058 0 : gcc_unreachable ();
2059 : }
2060 :
2061 : /* This doesn't belong into 'pass_oacc_loop_designation' conceptually, but
2062 : it's a convenient place, so... */
2063 9941 : if (is_oacc_routine)
2064 : {
2065 553 : tree attr = lookup_attribute ("omp declare target",
2066 553 : DECL_ATTRIBUTES (current_function_decl));
2067 553 : gcc_checking_assert (attr);
2068 553 : tree clauses = TREE_VALUE (attr);
2069 553 : gcc_checking_assert (clauses);
2070 :
2071 : /* Should this OpenACC routine be discarded? */
2072 553 : bool discard = false;
2073 :
2074 553 : tree clause_nohost = omp_find_clause (clauses, OMP_CLAUSE_NOHOST);
2075 553 : if (dump_file)
2076 56 : fprintf (dump_file,
2077 : "OpenACC routine '%s' %s '%s' clause.\n",
2078 56 : lang_hooks.decl_printable_name (current_function_decl, 2),
2079 : clause_nohost ? "has" : "doesn't have",
2080 56 : omp_clause_code_name[OMP_CLAUSE_NOHOST]);
2081 : /* Host compiler, 'nohost' clause? */
2082 : #ifndef ACCEL_COMPILER
2083 553 : if (clause_nohost)
2084 62 : discard = true;
2085 : #endif
2086 :
2087 553 : if (dump_file)
2088 112 : fprintf (dump_file,
2089 : "OpenACC routine '%s' %sdiscarded.\n",
2090 56 : lang_hooks.decl_printable_name (current_function_decl, 2),
2091 : discard ? "" : "not ");
2092 553 : if (discard)
2093 : {
2094 62 : TREE_ASM_WRITTEN (current_function_decl) = 1;
2095 62 : return TODO_discard_function;
2096 : }
2097 : }
2098 :
2099 : /* Unparallelized OpenACC kernels constructs must get launched as 1 x 1 x 1
2100 : kernels, so remove the parallelism dimensions function attributes
2101 : potentially set earlier on. */
2102 9879 : if (is_oacc_kernels && !is_oacc_kernels_parallelized)
2103 : {
2104 1265 : oacc_set_fn_attrib (current_function_decl, NULL, NULL);
2105 1265 : attrs = oacc_get_fn_attrib (current_function_decl);
2106 : }
2107 :
2108 : /* Discover, partition and process the loops. */
2109 9879 : oacc_loop *loops = oacc_loop_discovery ();
2110 :
2111 9879 : unsigned outer_mask = 0;
2112 9879 : if (is_oacc_routine)
2113 491 : outer_mask = GOMP_DIM_MASK (fn_level) - 1;
2114 9879 : unsigned used_mask = oacc_loop_partition (loops, outer_mask);
2115 : /* OpenACC kernels constructs are special: they currently don't use the
2116 : generic oacc_loop infrastructure and attribute/dimension processing. */
2117 9879 : if (is_oacc_kernels && is_oacc_kernels_parallelized)
2118 : {
2119 : /* Parallelized OpenACC kernels constructs use gang parallelism. See
2120 : also tree-parloops.cc:create_parallel_loop. */
2121 386 : used_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
2122 : }
2123 :
2124 9879 : int dims[GOMP_DIM_MAX];
2125 9879 : oacc_validate_dims (current_function_decl, attrs, dims, fn_level, used_mask);
2126 :
2127 9879 : if (dump_file)
2128 : {
2129 : const char *comma = "Compute dimensions [";
2130 456 : for (int ix = 0; ix != GOMP_DIM_MAX; ix++, comma = ", ")
2131 342 : fprintf (dump_file, "%s%d", comma, dims[ix]);
2132 114 : fprintf (dump_file, "]\n");
2133 : }
2134 :
2135 : /* Verify that for OpenACC 'kernels' decomposed "gang-single" parts we launch
2136 : a single gang only. */
2137 9879 : if (is_oacc_parallel_kernels_gang_single)
2138 109 : gcc_checking_assert (dims[GOMP_DIM_GANG] == 1);
2139 :
2140 9879 : oacc_loop_process (loops, fn_level);
2141 9879 : if (dump_file)
2142 : {
2143 114 : fprintf (dump_file, "OpenACC loops\n");
2144 114 : dump_oacc_loop (dump_file, loops, 0);
2145 114 : fprintf (dump_file, "\n");
2146 : }
2147 9879 : if (dump_enabled_p ())
2148 : {
2149 2280 : oacc_loop *l = loops;
2150 : /* OpenACC kernels constructs are special: they currently don't use the
2151 : generic oacc_loop infrastructure. */
2152 2280 : if (is_oacc_kernels)
2153 : {
2154 : /* Create a fake oacc_loop for diagnostic purposes. */
2155 633 : l = new_oacc_loop_raw (NULL,
2156 633 : DECL_SOURCE_LOCATION (current_function_decl));
2157 633 : l->mask = used_mask;
2158 : }
2159 : else
2160 : {
2161 : /* Skip the outermost, dummy OpenACC loop */
2162 1647 : l = l->child;
2163 : }
2164 2280 : if (l)
2165 1763 : inform_oacc_loop (l);
2166 2280 : if (is_oacc_kernels)
2167 633 : free_oacc_loop (l);
2168 : }
2169 :
2170 9879 : free_oacc_loop (loops);
2171 :
2172 9879 : return 0;
2173 : }
2174 :
2175 : static unsigned int
2176 15279 : execute_oacc_device_lower ()
2177 : {
2178 15279 : tree attrs = oacc_get_fn_attrib (current_function_decl);
2179 :
2180 15279 : if (!attrs)
2181 : /* Not an offloaded function. */
2182 : return 0;
2183 :
2184 : int dims[GOMP_DIM_MAX];
2185 39516 : for (unsigned i = 0; i < GOMP_DIM_MAX; i++)
2186 29637 : dims[i] = oacc_get_fn_dim_size (current_function_decl, i);
2187 :
2188 9879 : hash_map<tree, tree> adjusted_vars;
2189 :
2190 : /* Now lower internal loop functions to target-specific code
2191 : sequences. */
2192 9879 : basic_block bb;
2193 185094 : FOR_ALL_BB_FN (bb, cfun)
2194 933264 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
2195 : {
2196 582834 : gimple *stmt = gsi_stmt (gsi);
2197 582834 : if (!is_gimple_call (stmt))
2198 : {
2199 383027 : gsi_next (&gsi);
2200 383027 : continue;
2201 : }
2202 :
2203 199807 : gcall *call = as_a <gcall *> (stmt);
2204 199807 : if (!gimple_call_internal_p (call))
2205 : {
2206 4763 : gsi_next (&gsi);
2207 4763 : continue;
2208 : }
2209 :
2210 : /* Rewind to allow rescan. */
2211 195044 : gsi_prev (&gsi);
2212 195044 : bool rescan = false, remove = false;
2213 195044 : enum internal_fn ifn_code = gimple_call_internal_fn (call);
2214 :
2215 195044 : switch (ifn_code)
2216 : {
2217 : default: break;
2218 :
2219 284 : case IFN_GOACC_TILE:
2220 284 : oacc_xform_tile (call);
2221 284 : rescan = true;
2222 284 : break;
2223 :
2224 46702 : case IFN_GOACC_LOOP:
2225 46702 : oacc_xform_loop (call);
2226 46702 : rescan = true;
2227 46702 : break;
2228 :
2229 30884 : case IFN_GOACC_REDUCTION:
2230 : /* Mark the function for SSA renaming. */
2231 30884 : mark_virtual_operands_for_renaming (cfun);
2232 :
2233 : /* If the level is -1, this ended up being an unused
2234 : axis. Handle as a default. */
2235 30884 : if (integer_minus_onep (gimple_call_arg (call, 3)))
2236 8528 : default_goacc_reduction (call);
2237 : else
2238 22356 : targetm.goacc.reduction (call);
2239 : rescan = true;
2240 : break;
2241 :
2242 85483 : case IFN_UNIQUE:
2243 85483 : {
2244 85483 : enum ifn_unique_kind kind
2245 : = ((enum ifn_unique_kind)
2246 85483 : TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
2247 :
2248 85483 : switch (kind)
2249 : {
2250 : default:
2251 : break;
2252 :
2253 32978 : case IFN_UNIQUE_OACC_FORK:
2254 32978 : case IFN_UNIQUE_OACC_JOIN:
2255 32978 : if (integer_minus_onep (gimple_call_arg (call, 2)))
2256 : remove = true;
2257 25614 : else if (!targetm.goacc.fork_join
2258 25614 : (call, dims, kind == IFN_UNIQUE_OACC_FORK))
2259 85483 : remove = true;
2260 : break;
2261 :
2262 : case IFN_UNIQUE_OACC_HEAD_MARK:
2263 : case IFN_UNIQUE_OACC_TAIL_MARK:
2264 85483 : remove = true;
2265 : break;
2266 :
2267 259 : case IFN_UNIQUE_OACC_PRIVATE:
2268 259 : {
2269 259 : dump_flags_t l_dump_flags
2270 259 : = get_openacc_privatization_dump_flags ();
2271 :
2272 259 : location_t loc = gimple_location (stmt);
2273 259 : if (LOCATION_LOCUS (loc) == UNKNOWN_LOCATION)
2274 30 : loc = DECL_SOURCE_LOCATION (current_function_decl);
2275 259 : const dump_user_location_t d_u_loc
2276 259 : = dump_user_location_t::from_location_t (loc);
2277 :
2278 259 : HOST_WIDE_INT level
2279 259 : = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
2280 259 : gcc_checking_assert (level == -1
2281 : || (level >= 0
2282 : && level < GOMP_DIM_MAX));
2283 339 : for (unsigned i = 3;
2284 598 : i < gimple_call_num_args (call);
2285 : i++)
2286 : {
2287 339 : static char const *const axes[] =
2288 : /* Must be kept in sync with GOMP_DIM enumeration. */
2289 : { "gang", "worker", "vector" };
2290 :
2291 339 : tree arg = gimple_call_arg (call, i);
2292 339 : gcc_checking_assert (TREE_CODE (arg) == ADDR_EXPR);
2293 339 : tree decl = TREE_OPERAND (arg, 0);
2294 339 : if (dump_enabled_p ())
2295 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
2296 : #if __GNUC__ >= 10
2297 318 : # pragma GCC diagnostic push
2298 318 : # pragma GCC diagnostic ignored "-Wformat"
2299 : #endif
2300 318 : dump_printf_loc (l_dump_flags, d_u_loc,
2301 : "variable %<%T%> ought to be"
2302 : " adjusted for OpenACC"
2303 : " privatization level: %qs\n",
2304 : decl,
2305 : (level == -1
2306 : ? "UNKNOWN" : axes[level]));
2307 : #if __GNUC__ >= 10
2308 339 : # pragma GCC diagnostic pop
2309 : #endif
2310 339 : bool adjusted;
2311 339 : if (level == -1)
2312 : adjusted = false;
2313 336 : else if (!targetm.goacc.adjust_private_decl)
2314 : adjusted = false;
2315 0 : else if (level == GOMP_DIM_VECTOR)
2316 : {
2317 : /* That's the default behavior. */
2318 : adjusted = true;
2319 : }
2320 : else
2321 : {
2322 0 : tree oldtype = TREE_TYPE (decl);
2323 0 : tree newdecl
2324 0 : = targetm.goacc.adjust_private_decl (loc, decl,
2325 0 : level);
2326 0 : adjusted = (TREE_TYPE (newdecl) != oldtype
2327 0 : || newdecl != decl);
2328 0 : if (adjusted)
2329 0 : adjusted_vars.put (decl, newdecl);
2330 : }
2331 0 : if (adjusted
2332 0 : && dump_enabled_p ())
2333 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
2334 : #if __GNUC__ >= 10
2335 0 : # pragma GCC diagnostic push
2336 0 : # pragma GCC diagnostic ignored "-Wformat"
2337 : #endif
2338 0 : dump_printf_loc (l_dump_flags, d_u_loc,
2339 : "variable %<%T%> adjusted for"
2340 : " OpenACC privatization level:"
2341 : " %qs\n",
2342 0 : decl, axes[level]);
2343 : #if __GNUC__ >= 10
2344 339 : # pragma GCC diagnostic pop
2345 : #endif
2346 : }
2347 259 : remove = true;
2348 : }
2349 259 : break;
2350 : }
2351 : break;
2352 : }
2353 : }
2354 :
2355 195044 : if (gsi_end_p (gsi))
2356 : /* We rewound past the beginning of the BB. */
2357 188830 : gsi = gsi_start_bb (bb);
2358 : else
2359 : /* Undo the rewind. */
2360 100629 : gsi_next (&gsi);
2361 :
2362 195044 : if (remove)
2363 : {
2364 170966 : if (gimple_vdef (call))
2365 85483 : replace_uses_by (gimple_vdef (call), gimple_vuse (call));
2366 85483 : if (gimple_call_lhs (call))
2367 : {
2368 : /* Propagate the data dependency var. */
2369 79856 : gimple *ass = gimple_build_assign (gimple_call_lhs (call),
2370 : gimple_call_arg (call, 1));
2371 79856 : gsi_replace (&gsi, ass, false);
2372 : }
2373 : else
2374 5627 : gsi_remove (&gsi, true);
2375 : }
2376 109561 : else if (!rescan)
2377 : /* If not rescanning, advance over the call. */
2378 31691 : gsi_next (&gsi);
2379 : }
2380 :
2381 : /* Regarding the OpenACC privatization level, we're currently only looking at
2382 : making the gang-private level work. Regarding that, we have the following
2383 : configurations:
2384 :
2385 : - GCN offloading: 'targetm.goacc.adjust_private_decl' does the work (in
2386 : particular, change 'TREE_TYPE', etc.) and there is no
2387 : 'targetm.goacc.expand_var_decl'.
2388 :
2389 : - nvptx offloading: 'targetm.goacc.adjust_private_decl' only sets a
2390 : marker and then 'targetm.goacc.expand_var_decl' does the work.
2391 :
2392 : Eventually (in particular, for worker-private level?), both
2393 : 'targetm.goacc.adjust_private_decl' and 'targetm.goacc.expand_var_decl'
2394 : may need to do things, but that's currently not meant to be addressed, and
2395 : thus not fully worked out and implemented, and thus untested. Hence,
2396 : 'assert' what currently is implemented/tested, only. */
2397 :
2398 9879 : if (targetm.goacc.expand_var_decl)
2399 0 : gcc_assert (adjusted_vars.is_empty ());
2400 :
2401 : /* Make adjustments to gang-private local variables if required by the
2402 : target, e.g. forcing them into a particular address space. Afterwards,
2403 : ADDR_EXPR nodes which have adjusted variables as their argument need to
2404 : be modified in one of two ways:
2405 :
2406 : 1. They can be recreated, making a pointer to the variable in the new
2407 : address space, or
2408 :
2409 : 2. The address of the variable in the new address space can be taken,
2410 : converted to the default (original) address space, and the result of
2411 : that conversion substituted in place of the original ADDR_EXPR node.
2412 :
2413 : Which of these is done depends on the gimple statement being processed.
2414 : At present atomic operations and inline asms use (1), and everything else
2415 : uses (2). At least on AMD GCN, there are atomic operations that work
2416 : directly in the LDS address space.
2417 :
2418 : COMPONENT_REFS, ARRAY_REFS and plain VAR_DECLs are also rewritten to use
2419 : the new decl, adjusting types of appropriate tree nodes as necessary. */
2420 :
2421 9879 : if (targetm.goacc.adjust_private_decl
2422 9879 : && !adjusted_vars.is_empty ())
2423 : {
2424 0 : FOR_ALL_BB_FN (bb, cfun)
2425 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2426 0 : !gsi_end_p (gsi);
2427 0 : gsi_next (&gsi))
2428 : {
2429 0 : gimple *stmt = gsi_stmt (gsi);
2430 0 : walk_stmt_info wi;
2431 0 : var_decl_rewrite_info info;
2432 :
2433 0 : info.avoid_pointer_conversion
2434 0 : = (is_gimple_call (stmt)
2435 0 : && is_sync_builtin_call (as_a <gcall *> (stmt)))
2436 0 : || gimple_code (stmt) == GIMPLE_ASM;
2437 0 : info.stmt = stmt;
2438 0 : info.modified = false;
2439 0 : info.adjusted_vars = &adjusted_vars;
2440 :
2441 0 : memset (&wi, 0, sizeof (wi));
2442 0 : wi.info = &info;
2443 :
2444 0 : walk_gimple_op (stmt, oacc_rewrite_var_decl, &wi);
2445 :
2446 0 : if (info.modified)
2447 0 : update_stmt (stmt);
2448 : }
2449 : }
2450 :
2451 9879 : return 0;
2452 9879 : }
2453 :
2454 : /* Default launch dimension validator. Force everything to 1. A
2455 : backend that wants to provide larger dimensions must override this
2456 : hook. */
2457 :
2458 : bool
2459 14441 : default_goacc_validate_dims (tree ARG_UNUSED (decl), int *dims,
2460 : int ARG_UNUSED (fn_level),
2461 : unsigned ARG_UNUSED (used))
2462 : {
2463 14441 : bool changed = false;
2464 :
2465 57764 : for (unsigned ix = 0; ix != GOMP_DIM_MAX; ix++)
2466 : {
2467 43323 : if (dims[ix] != 1)
2468 : {
2469 33054 : dims[ix] = 1;
2470 33054 : changed = true;
2471 : }
2472 : }
2473 :
2474 14441 : return changed;
2475 : }
2476 :
2477 : /* Default dimension bound is unknown on accelerator and 1 on host. */
2478 :
2479 : int
2480 0 : default_goacc_dim_limit (int ARG_UNUSED (axis))
2481 : {
2482 : #ifdef ACCEL_COMPILER
2483 : return 0;
2484 : #else
2485 0 : return 1;
2486 : #endif
2487 : }
2488 :
2489 : namespace {
2490 :
2491 : const pass_data pass_data_oacc_loop_designation =
2492 : {
2493 : GIMPLE_PASS, /* type */
2494 : "oaccloops", /* name */
2495 : OPTGROUP_OMP, /* optinfo_flags */
2496 : TV_NONE, /* tv_id */
2497 : PROP_cfg, /* properties_required */
2498 : 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
2499 : 0, /* properties_destroyed */
2500 : 0, /* todo_flags_start */
2501 : TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */
2502 : };
2503 :
2504 : class pass_oacc_loop_designation : public gimple_opt_pass
2505 : {
2506 : public:
2507 294536 : pass_oacc_loop_designation (gcc::context *ctxt)
2508 589072 : : gimple_opt_pass (pass_data_oacc_loop_designation, ctxt)
2509 : {}
2510 :
2511 : /* opt_pass methods: */
2512 1516885 : bool gate (function *) final override { return flag_openacc; };
2513 :
2514 15341 : unsigned int execute (function *) final override
2515 : {
2516 15341 : return execute_oacc_loop_designation ();
2517 : }
2518 :
2519 : }; // class pass_oacc_loop_designation
2520 :
2521 : const pass_data pass_data_oacc_device_lower =
2522 : {
2523 : GIMPLE_PASS, /* type */
2524 : "oaccdevlow", /* name */
2525 : OPTGROUP_OMP, /* optinfo_flags */
2526 : TV_NONE, /* tv_id */
2527 : PROP_cfg, /* properties_required */
2528 : 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
2529 : 0, /* properties_destroyed */
2530 : 0, /* todo_flags_start */
2531 : TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */
2532 : };
2533 :
2534 : class pass_oacc_device_lower : public gimple_opt_pass
2535 : {
2536 : public:
2537 294536 : pass_oacc_device_lower (gcc::context *ctxt)
2538 589072 : : gimple_opt_pass (pass_data_oacc_device_lower, ctxt)
2539 : {}
2540 :
2541 : /* opt_pass methods: */
2542 1516823 : bool gate (function *) final override { return flag_openacc; };
2543 :
2544 15279 : unsigned int execute (function *) final override
2545 : {
2546 15279 : return execute_oacc_device_lower ();
2547 : }
2548 :
2549 : }; // class pass_oacc_device_lower
2550 :
2551 : } // anon namespace
2552 :
2553 : gimple_opt_pass *
2554 294536 : make_pass_oacc_loop_designation (gcc::context *ctxt)
2555 : {
2556 294536 : return new pass_oacc_loop_designation (ctxt);
2557 : }
2558 :
2559 : gimple_opt_pass *
2560 294536 : make_pass_oacc_device_lower (gcc::context *ctxt)
2561 : {
2562 294536 : return new pass_oacc_device_lower (ctxt);
2563 : }
2564 :
2565 :
2566 : /* Rewrite GOMP_SIMT_ENTER_ALLOC call given by GSI and remove the preceding
2567 : GOMP_SIMT_ENTER call identifying the privatized variables, which are
2568 : turned to structure fields and receive a DECL_VALUE_EXPR accordingly.
2569 : Set *REGIMPLIFY to true, except if no privatized variables were seen. */
2570 :
2571 : static void
2572 0 : ompdevlow_adjust_simt_enter (gimple_stmt_iterator *gsi, bool *regimplify)
2573 : {
2574 0 : gimple *alloc_stmt = gsi_stmt (*gsi);
2575 0 : tree simtrec = gimple_call_lhs (alloc_stmt);
2576 0 : tree simduid = gimple_call_arg (alloc_stmt, 0);
2577 0 : gimple *enter_stmt = SSA_NAME_DEF_STMT (simduid);
2578 0 : gcc_assert (gimple_call_internal_p (enter_stmt, IFN_GOMP_SIMT_ENTER));
2579 0 : tree rectype = lang_hooks.types.make_type (RECORD_TYPE);
2580 0 : TYPE_ARTIFICIAL (rectype) = TYPE_NAMELESS (rectype) = 1;
2581 0 : TREE_ADDRESSABLE (rectype) = 1;
2582 0 : TREE_TYPE (simtrec) = build_pointer_type (rectype);
2583 0 : for (unsigned i = 1; i < gimple_call_num_args (enter_stmt); i++)
2584 : {
2585 0 : tree *argp = gimple_call_arg_ptr (enter_stmt, i);
2586 0 : if (*argp == null_pointer_node)
2587 0 : continue;
2588 0 : gcc_assert (TREE_CODE (*argp) == ADDR_EXPR
2589 : && VAR_P (TREE_OPERAND (*argp, 0)));
2590 0 : tree var = TREE_OPERAND (*argp, 0);
2591 :
2592 0 : tree field = build_decl (DECL_SOURCE_LOCATION (var), FIELD_DECL,
2593 0 : DECL_NAME (var), TREE_TYPE (var));
2594 0 : SET_DECL_ALIGN (field, DECL_ALIGN (var));
2595 0 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
2596 0 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
2597 :
2598 0 : insert_field_into_struct (rectype, field);
2599 :
2600 0 : tree t = build_simple_mem_ref (simtrec);
2601 0 : t = build3 (COMPONENT_REF, TREE_TYPE (var), t, field, NULL);
2602 0 : TREE_THIS_VOLATILE (t) = TREE_THIS_VOLATILE (var);
2603 0 : SET_DECL_VALUE_EXPR (var, t);
2604 0 : DECL_HAS_VALUE_EXPR_P (var) = 1;
2605 0 : *regimplify = true;
2606 : }
2607 0 : layout_type (rectype);
2608 0 : tree size = TYPE_SIZE_UNIT (rectype);
2609 0 : tree align = build_int_cst (TREE_TYPE (size), TYPE_ALIGN_UNIT (rectype));
2610 :
2611 0 : alloc_stmt
2612 0 : = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC, 2, size, align);
2613 0 : gimple_call_set_lhs (alloc_stmt, simtrec);
2614 0 : gsi_replace (gsi, alloc_stmt, false);
2615 0 : gimple_stmt_iterator enter_gsi = gsi_for_stmt (enter_stmt);
2616 0 : enter_stmt = gimple_build_assign (simduid, gimple_call_arg (enter_stmt, 0));
2617 0 : gsi_replace (&enter_gsi, enter_stmt, false);
2618 :
2619 0 : use_operand_p use;
2620 0 : gimple *exit_stmt;
2621 0 : if (single_imm_use (simtrec, &use, &exit_stmt))
2622 : {
2623 0 : gcc_assert (gimple_call_internal_p (exit_stmt, IFN_GOMP_SIMT_EXIT));
2624 0 : gimple_stmt_iterator exit_gsi = gsi_for_stmt (exit_stmt);
2625 0 : tree clobber = build_clobber (rectype);
2626 0 : exit_stmt = gimple_build_assign (build_simple_mem_ref (simtrec), clobber);
2627 0 : gsi_insert_before (&exit_gsi, exit_stmt, GSI_SAME_STMT);
2628 : }
2629 : else
2630 0 : gcc_checking_assert (has_zero_uses (simtrec));
2631 0 : }
2632 :
2633 : /* Callback for walk_gimple_stmt used to scan for SIMT-privatized variables. */
2634 :
2635 : static tree
2636 0 : find_simtpriv_var_op (tree *tp, int *walk_subtrees, void *)
2637 : {
2638 0 : tree t = *tp;
2639 :
2640 0 : if (VAR_P (t)
2641 0 : && DECL_HAS_VALUE_EXPR_P (t)
2642 0 : && lookup_attribute ("omp simt private", DECL_ATTRIBUTES (t)))
2643 : {
2644 0 : *walk_subtrees = 0;
2645 0 : return t;
2646 : }
2647 : return NULL_TREE;
2648 : }
2649 :
2650 : /* Helper function for execute_omp_device_lower, invoked via walk_gimple_op.
2651 : Resolve any OMP_TARGET_DEVICE_MATCHES and OMP_NEXT_VARIANT exprs to
2652 : constants. */
2653 : static tree
2654 13877 : resolve_omp_variant_cookies (tree *tp, int *walk_subtrees,
2655 : void *data ATTRIBUTE_UNUSED)
2656 : {
2657 13877 : if (TREE_CODE (*tp) == OMP_TARGET_DEVICE_MATCHES)
2658 : {
2659 0 : *tp = resolve_omp_target_device_matches (*tp);
2660 0 : *walk_subtrees = 0;
2661 0 : return NULL_TREE;
2662 : }
2663 :
2664 13877 : if (TREE_CODE (*tp) != OMP_NEXT_VARIANT)
2665 : return NULL_TREE;
2666 324 : tree index = OMP_NEXT_VARIANT_INDEX (*tp);
2667 324 : tree state = OMP_NEXT_VARIANT_STATE (*tp);
2668 :
2669 : /* State is a triplet of (result-vector, construct_context, selector_vec).
2670 : If result-vector has already been computed, just use it. Otherwise we
2671 : must resolve the variant and fill in that part of the state object.
2672 : All OMP_NEXT_VARIANT exprs for the same variant construct are supposed
2673 : to share the same state object, but if something bad happens and we end
2674 : up with copies, that is OK, it will just cause the result-vector to be
2675 : computed multiple times. */
2676 324 : tree result_vector = TREE_PURPOSE (state);
2677 324 : if (!result_vector)
2678 : {
2679 304 : tree construct_context = TREE_VALUE (state);
2680 304 : tree selectors = TREE_CHAIN (state);
2681 :
2682 304 : vec<struct omp_variant> candidates
2683 304 : = omp_resolve_variant_construct (construct_context, selectors);
2684 304 : int n = TREE_VEC_LENGTH (selectors);
2685 304 : TREE_PURPOSE (state) = result_vector = make_tree_vec (n + 1);
2686 : /* The result vector maps the index of each element of the original
2687 : selectors vector onto the index of the next element of the filtered/
2688 : sorted candidates vector. Since some of the original variants may
2689 : have been discarded as non-matching in candidates, initialize the
2690 : whole array to zero so that we have a placeholder "next" value for
2691 : those elements. Hopefully dead code elimination will take care of
2692 : subsequently discarding the unreachable cases in the already-generated
2693 : switch statement. */
2694 2108 : for (int i = 1; i <= n; i++)
2695 1804 : TREE_VEC_ELT (result_vector, i) = integer_zero_node;
2696 : /* Element 0 is the case label of the first variant in the sorted
2697 : list. */
2698 304 : if (dump_file)
2699 0 : fprintf (dump_file, "Computing case map for variant directive\n");
2700 304 : int j = 0;
2701 1588 : for (unsigned int i = 0; i < candidates.length(); i++)
2702 : {
2703 1284 : if (dump_file)
2704 0 : fprintf (dump_file, " %d -> case %d\n",
2705 0 : j, (int) tree_to_shwi (candidates[i].alternative));
2706 1284 : TREE_VEC_ELT (result_vector, j) = candidates[i].alternative;
2707 1284 : j = (int) tree_to_shwi (candidates[i].alternative);
2708 : }
2709 : }
2710 :
2711 : /* Now just grab the value out of the precomputed array. */
2712 324 : gcc_assert (TREE_CODE (index) == INTEGER_CST);
2713 324 : int indexval = (int) tree_to_shwi (index);
2714 324 : *tp = TREE_VEC_ELT (result_vector, indexval);
2715 324 : *walk_subtrees = 0;
2716 324 : return NULL_TREE;
2717 : }
2718 :
2719 :
2720 : /* Cleanup uses of SIMT placeholder internal functions: on non-SIMT targets,
2721 : VF is 1 and LANE is 0; on SIMT targets, VF is folded to a constant, and
2722 : LANE is kept to be expanded to RTL later on. Also cleanup all other SIMT
2723 : internal functions on non-SIMT targets, and likewise some SIMD internal
2724 : functions on SIMT targets. */
2725 :
2726 : static unsigned int
2727 21312 : execute_omp_device_lower ()
2728 : {
2729 21312 : int vf = targetm.simt.vf ? targetm.simt.vf () : 1;
2730 21312 : bool regimplify = false;
2731 21312 : basic_block bb;
2732 21312 : gimple_stmt_iterator gsi;
2733 : #ifdef ACCEL_COMPILER
2734 : bool omp_redirect_indirect_calls = vec_safe_length (offload_ind_funcs) > 0;
2735 : tree map_ptr_fn
2736 : = builtin_decl_explicit (BUILT_IN_GOMP_TARGET_MAP_INDIRECT_PTR);
2737 : #endif
2738 :
2739 : /* Handle expansion of magic cookies for variant constructs first. */
2740 21312 : if (cgraph_node::get (cfun->decl)->has_omp_variant_constructs)
2741 1822 : FOR_EACH_BB_FN (bb, cfun)
2742 : {
2743 6873 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2744 3409 : walk_gimple_op (gsi_stmt (gsi), resolve_omp_variant_cookies, NULL);
2745 2510 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2746 778 : walk_gimple_op (gsi_stmt (gsi), resolve_omp_variant_cookies, NULL);
2747 : }
2748 :
2749 57524 : FOR_EACH_BB_FN (bb, cfun)
2750 248473 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2751 : {
2752 176049 : gimple *stmt = gsi_stmt (gsi);
2753 176049 : if (!is_gimple_call (stmt))
2754 159088 : continue;
2755 16961 : if (!gimple_call_internal_p (stmt))
2756 : {
2757 : #ifdef ACCEL_COMPILER
2758 : if (omp_redirect_indirect_calls
2759 : && gimple_call_fndecl (stmt) == NULL_TREE)
2760 : {
2761 : gcall *orig_call = dyn_cast <gcall *> (stmt);
2762 : tree call_fn = gimple_call_fn (stmt);
2763 : tree fn_ty = TREE_TYPE (call_fn);
2764 :
2765 : if (TREE_CODE (call_fn) == OBJ_TYPE_REF)
2766 : {
2767 : tree obj_ref = create_tmp_reg (TREE_TYPE (call_fn),
2768 : ".ind_fn_objref");
2769 : gimple *gassign = gimple_build_assign (obj_ref, call_fn);
2770 : gsi_insert_before (&gsi, gassign, GSI_SAME_STMT);
2771 : call_fn = obj_ref;
2772 : }
2773 : tree mapped_fn = create_tmp_reg (fn_ty, ".ind_fn");
2774 : gimple *gcall =
2775 : gimple_build_call (map_ptr_fn, 1, call_fn);
2776 : gimple_set_location (gcall, gimple_location (stmt));
2777 : gimple_call_set_lhs (gcall, mapped_fn);
2778 : gsi_insert_before (&gsi, gcall, GSI_SAME_STMT);
2779 :
2780 : gimple_call_set_fn (orig_call, mapped_fn);
2781 : update_stmt (orig_call);
2782 : }
2783 : #endif
2784 16687 : continue;
2785 : }
2786 274 : tree lhs = gimple_call_lhs (stmt), rhs = NULL_TREE;
2787 274 : tree type = lhs ? TREE_TYPE (lhs) : integer_type_node;
2788 274 : switch (gimple_call_internal_fn (stmt))
2789 : {
2790 0 : case IFN_GOMP_TARGET_REV:
2791 0 : {
2792 : #ifndef ACCEL_COMPILER
2793 0 : gimple_stmt_iterator gsi2 = gsi;
2794 0 : gsi_next (&gsi2);
2795 0 : gcc_assert (!gsi_end_p (gsi2));
2796 0 : gcc_assert (gimple_call_builtin_p (gsi_stmt (gsi2),
2797 : BUILT_IN_GOMP_TARGET));
2798 0 : tree old_decl
2799 0 : = TREE_OPERAND (gimple_call_arg (gsi_stmt (gsi2), 1), 0);
2800 0 : tree new_decl = gimple_call_arg (gsi_stmt (gsi), 0);
2801 0 : gimple_call_set_arg (gsi_stmt (gsi2), 1, new_decl);
2802 0 : update_stmt (gsi_stmt (gsi2));
2803 0 : new_decl = TREE_OPERAND (new_decl, 0);
2804 0 : unsigned i;
2805 0 : unsigned num_funcs = vec_safe_length (offload_funcs);
2806 0 : for (i = 0; i < num_funcs; i++)
2807 : {
2808 0 : if ((*offload_funcs)[i] == old_decl)
2809 : {
2810 0 : (*offload_funcs)[i] = new_decl;
2811 0 : break;
2812 : }
2813 0 : else if ((*offload_funcs)[i] == new_decl)
2814 : break; /* This can happen due to inlining. */
2815 : }
2816 0 : gcc_assert (i < num_funcs);
2817 : #else
2818 : tree old_decl = TREE_OPERAND (gimple_call_arg (gsi_stmt (gsi), 0),
2819 : 0);
2820 : #endif
2821 : /* FIXME: Find a way to actually prevent outputting the empty-body
2822 : old_decl as debug symbol + function in the assembly file. */
2823 0 : cgraph_node *node = cgraph_node::get (old_decl);
2824 0 : node->address_taken = false;
2825 0 : node->need_lto_streaming = false;
2826 0 : node->offloadable = false;
2827 :
2828 0 : unlink_stmt_vdef (stmt);
2829 : }
2830 0 : break;
2831 0 : case IFN_GOMP_USE_SIMT:
2832 0 : rhs = vf == 1 ? integer_zero_node : integer_one_node;
2833 : break;
2834 0 : case IFN_GOMP_SIMT_ENTER:
2835 0 : rhs = vf == 1 ? gimple_call_arg (stmt, 0) : NULL_TREE;
2836 0 : goto simtreg_enter_exit;
2837 0 : case IFN_GOMP_SIMT_ENTER_ALLOC:
2838 0 : if (vf != 1)
2839 0 : ompdevlow_adjust_simt_enter (&gsi, ®implify);
2840 0 : rhs = vf == 1 ? null_pointer_node : NULL_TREE;
2841 0 : goto simtreg_enter_exit;
2842 0 : case IFN_GOMP_SIMT_EXIT:
2843 0 : simtreg_enter_exit:
2844 0 : if (vf != 1)
2845 0 : continue;
2846 0 : unlink_stmt_vdef (stmt);
2847 0 : break;
2848 0 : case IFN_GOMP_SIMT_LANE:
2849 0 : case IFN_GOMP_SIMT_LAST_LANE:
2850 0 : rhs = vf == 1 ? build_zero_cst (type) : NULL_TREE;
2851 : break;
2852 0 : case IFN_GOMP_SIMT_VF:
2853 0 : rhs = build_int_cst (type, vf);
2854 0 : break;
2855 2 : case IFN_GOMP_MAX_VF:
2856 2 : rhs = build_int_cst (type, omp_max_vf (false));
2857 2 : break;
2858 0 : case IFN_GOMP_SIMT_ORDERED_PRED:
2859 0 : rhs = vf == 1 ? integer_zero_node : NULL_TREE;
2860 0 : if (rhs || !lhs)
2861 0 : unlink_stmt_vdef (stmt);
2862 : break;
2863 0 : case IFN_GOMP_SIMT_VOTE_ANY:
2864 0 : case IFN_GOMP_SIMT_XCHG_BFLY:
2865 0 : case IFN_GOMP_SIMT_XCHG_IDX:
2866 0 : rhs = vf == 1 ? gimple_call_arg (stmt, 0) : NULL_TREE;
2867 : break;
2868 0 : case IFN_GOMP_SIMD_LANE:
2869 0 : case IFN_GOMP_SIMD_LAST_LANE:
2870 0 : rhs = vf != 1 ? build_zero_cst (type) : NULL_TREE;
2871 : break;
2872 0 : case IFN_GOMP_SIMD_VF:
2873 0 : rhs = vf != 1 ? build_one_cst (type) : NULL_TREE;
2874 : break;
2875 272 : default:
2876 272 : continue;
2877 272 : }
2878 2 : if (lhs && !rhs)
2879 0 : continue;
2880 2 : stmt = lhs ? gimple_build_assign (lhs, rhs) : gimple_build_nop ();
2881 2 : gsi_replace (&gsi, stmt, false);
2882 : }
2883 21312 : if (regimplify)
2884 0 : FOR_EACH_BB_REVERSE_FN (bb, cfun)
2885 0 : for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
2886 0 : if (walk_gimple_stmt (&gsi, NULL, find_simtpriv_var_op, NULL))
2887 : {
2888 0 : if (gimple_clobber_p (gsi_stmt (gsi)))
2889 0 : gsi_remove (&gsi, true);
2890 : else
2891 0 : gimple_regimplify_operands (gsi_stmt (gsi), &gsi);
2892 : }
2893 21312 : if (vf != 1)
2894 0 : cfun->has_force_vectorize_loops = false;
2895 21312 : return 0;
2896 : }
2897 :
2898 : namespace {
2899 :
2900 : const pass_data pass_data_omp_device_lower =
2901 : {
2902 : GIMPLE_PASS, /* type */
2903 : "ompdevlow", /* name */
2904 : OPTGROUP_OMP, /* optinfo_flags */
2905 : TV_NONE, /* tv_id */
2906 : PROP_cfg, /* properties_required */
2907 : PROP_gimple_lomp_dev, /* properties_provided */
2908 : 0, /* properties_destroyed */
2909 : 0, /* todo_flags_start */
2910 : TODO_update_ssa, /* todo_flags_finish */
2911 : };
2912 :
2913 : class pass_omp_device_lower : public gimple_opt_pass
2914 : {
2915 : public:
2916 294536 : pass_omp_device_lower (gcc::context *ctxt)
2917 589072 : : gimple_opt_pass (pass_data_omp_device_lower, ctxt)
2918 : {}
2919 :
2920 : /* opt_pass methods: */
2921 1516823 : bool gate (function *fun) final override
2922 : {
2923 1516823 : cgraph_node *node = cgraph_node::get (fun->decl);
2924 : #ifdef ACCEL_COMPILER
2925 : bool offload_ind_funcs_p = vec_safe_length (offload_ind_funcs) > 0;
2926 : #else
2927 1516823 : bool offload_ind_funcs_p = false;
2928 : #endif
2929 1516823 : return (!(fun->curr_properties & PROP_gimple_lomp_dev)
2930 1516823 : || (flag_openmp
2931 66254 : && (node->has_omp_variant_constructs || offload_ind_funcs_p)));
2932 : }
2933 21312 : unsigned int execute (function *) final override
2934 : {
2935 21312 : return execute_omp_device_lower ();
2936 : }
2937 :
2938 : }; // class pass_expand_omp_ssa
2939 :
2940 : } // anon namespace
2941 :
2942 : gimple_opt_pass *
2943 294536 : make_pass_omp_device_lower (gcc::context *ctxt)
2944 : {
2945 294536 : return new pass_omp_device_lower (ctxt);
2946 : }
2947 :
2948 : /* "omp declare target link" handling pass. */
2949 :
2950 : namespace {
2951 :
2952 : const pass_data pass_data_omp_target_link =
2953 : {
2954 : GIMPLE_PASS, /* type */
2955 : "omptargetlink", /* name */
2956 : OPTGROUP_OMP, /* optinfo_flags */
2957 : TV_NONE, /* tv_id */
2958 : PROP_ssa, /* properties_required */
2959 : 0, /* properties_provided */
2960 : 0, /* properties_destroyed */
2961 : 0, /* todo_flags_start */
2962 : TODO_update_ssa, /* todo_flags_finish */
2963 : };
2964 :
2965 : class pass_omp_target_link : public gimple_opt_pass
2966 : {
2967 : public:
2968 294536 : pass_omp_target_link (gcc::context *ctxt)
2969 589072 : : gimple_opt_pass (pass_data_omp_target_link, ctxt)
2970 : {}
2971 :
2972 : /* opt_pass methods: */
2973 1516823 : bool gate (function *fun) final override
2974 : {
2975 : #ifdef ACCEL_COMPILER
2976 : return offloading_function_p (fun->decl);
2977 : #else
2978 1516823 : (void) fun;
2979 1516823 : return false;
2980 : #endif
2981 : }
2982 :
2983 : unsigned execute (function *) final override;
2984 : };
2985 :
2986 : /* Callback for walk_gimple_stmt used to scan for link var operands. */
2987 :
2988 : static tree
2989 0 : process_link_var_op (tree *tp, int *walk_subtrees, void *data)
2990 : {
2991 0 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
2992 0 : tree t = *tp;
2993 :
2994 0 : if (VAR_P (t)
2995 0 : && DECL_HAS_VALUE_EXPR_P (t)
2996 0 : && is_global_var (t)
2997 0 : && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t)))
2998 : {
2999 0 : wi->info = *tp = unshare_expr (DECL_VALUE_EXPR (t));
3000 0 : *walk_subtrees = 0;
3001 0 : return NULL_TREE;
3002 : }
3003 :
3004 : return NULL_TREE;
3005 : }
3006 :
3007 : unsigned
3008 0 : pass_omp_target_link::execute (function *fun)
3009 : {
3010 0 : basic_block bb;
3011 0 : FOR_EACH_BB_FN (bb, fun)
3012 : {
3013 0 : gimple_stmt_iterator gsi;
3014 0 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
3015 : {
3016 0 : if (gimple_call_builtin_p (gsi_stmt (gsi), BUILT_IN_GOMP_TARGET))
3017 : {
3018 0 : tree dev = gimple_call_arg (gsi_stmt (gsi), 0);
3019 0 : tree fn = gimple_call_arg (gsi_stmt (gsi), 1);
3020 0 : if (POINTER_TYPE_P (TREE_TYPE (fn)))
3021 0 : fn = TREE_OPERAND (fn, 0);
3022 0 : if (TREE_CODE (dev) == INTEGER_CST
3023 0 : && wi::to_wide (dev) == GOMP_DEVICE_HOST_FALLBACK
3024 0 : && lookup_attribute ("omp target device_ancestor_nohost",
3025 0 : DECL_ATTRIBUTES (fn)) != NULL_TREE)
3026 0 : continue; /* ancestor:1 */
3027 : /* Nullify the second argument of __builtin_GOMP_target_ext. */
3028 0 : gimple_call_set_arg (gsi_stmt (gsi), 1, null_pointer_node);
3029 0 : update_stmt (gsi_stmt (gsi));
3030 : }
3031 0 : struct walk_stmt_info wi;
3032 0 : memset (&wi, 0, sizeof (wi));
3033 0 : walk_gimple_stmt (&gsi, NULL, process_link_var_op, &wi);
3034 0 : if (wi.info)
3035 0 : gimple_regimplify_operands (gsi_stmt (gsi), &gsi);
3036 : }
3037 : }
3038 :
3039 0 : return 0;
3040 : }
3041 :
3042 : } // anon namespace
3043 :
3044 : gimple_opt_pass *
3045 294536 : make_pass_omp_target_link (gcc::context *ctxt)
3046 : {
3047 294536 : return new pass_omp_target_link (ctxt);
3048 : }
|
