Line data Source code
1 : /* Bits of OpenMP and OpenACC handling that is specific to device offloading
2 : and a lowering pass for OpenACC device directives.
3 :
4 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "target.h"
27 : #include "tree.h"
28 : #include "gimple.h"
29 : #include "tree-pass.h"
30 : #include "ssa.h"
31 : #include "cgraph.h"
32 : #include "pretty-print.h"
33 : #include "diagnostic-core.h"
34 : #include "fold-const.h"
35 : #include "internal-fn.h"
36 : #include "langhooks.h"
37 : #include "gimplify.h"
38 : #include "gimple-iterator.h"
39 : #include "gimplify-me.h"
40 : #include "gimple-walk.h"
41 : #include "tree-cfg.h"
42 : #include "tree-into-ssa.h"
43 : #include "tree-nested.h"
44 : #include "stor-layout.h"
45 : #include "common/common-target.h"
46 : #include "omp-general.h"
47 : #include "omp-offload.h"
48 : #include "lto-section-names.h"
49 : #include "gomp-constants.h"
50 : #include "gimple-pretty-print.h"
51 : #include "intl.h"
52 : #include "stringpool.h"
53 : #include "attribs.h"
54 : #include "cfgloop.h"
55 : #include "context.h"
56 : #include "convert.h"
57 : #include "opts.h"
58 :
59 : /* Describe the OpenACC looping structure of a function. The entire
60 : function is held in a 'NULL' loop. */
61 :
62 : struct oacc_loop
63 : {
64 : oacc_loop *parent; /* Containing loop. */
65 :
66 : oacc_loop *child; /* First inner loop. */
67 :
68 : oacc_loop *sibling; /* Next loop within same parent. */
69 :
70 : location_t loc; /* Location of the loop start. */
71 :
72 : gcall *marker; /* Initial head marker. */
73 :
74 : gcall *heads[GOMP_DIM_MAX]; /* Head marker functions. */
75 : gcall *tails[GOMP_DIM_MAX]; /* Tail marker functions. */
76 :
77 : tree routine; /* Pseudo-loop enclosing a routine. */
78 :
79 : unsigned mask; /* Partitioning mask. */
80 : unsigned e_mask; /* Partitioning of element loops (when tiling). */
81 : unsigned inner; /* Partitioning of inner loops. */
82 : unsigned flags; /* Partitioning flags. */
83 : vec<gcall *> ifns; /* Contained loop abstraction functions. */
84 : tree chunk_size; /* Chunk size. */
85 : gcall *head_end; /* Final marker of head sequence. */
86 : };
87 :
88 : /* Holds offload tables with decls. */
89 : vec<tree, va_gc> *offload_funcs, *offload_vars, *offload_ind_funcs;
90 :
91 : /* Return level at which oacc routine may spawn a partitioned loop, or
92 : -1 if it is not a routine (i.e. is an offload fn). */
93 :
94 : int
95 11142 : oacc_fn_attrib_level (tree attr)
96 : {
97 11142 : tree pos = TREE_VALUE (attr);
98 :
99 11142 : if (!TREE_PURPOSE (pos))
100 : return -1;
101 :
102 : int ix = 0;
103 5366 : for (ix = 0; ix != GOMP_DIM_MAX;
104 3612 : ix++, pos = TREE_CHAIN (pos))
105 4398 : if (!integer_zerop (TREE_PURPOSE (pos)))
106 : break;
107 :
108 : return ix;
109 : }
110 :
111 : /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
112 : adds their addresses and sizes to constructor-vector V_CTOR. */
113 :
114 : static void
115 87 : add_decls_addresses_to_decl_constructor (vec<tree, va_gc> *v_decls,
116 : vec<constructor_elt, va_gc> *v_ctor)
117 : {
118 87 : unsigned len = vec_safe_length (v_decls);
119 162 : for (unsigned i = 0; i < len; i++)
120 : {
121 75 : tree it = (*v_decls)[i];
122 75 : bool is_var = VAR_P (it);
123 75 : bool is_link_var
124 : = is_var
125 : #ifdef ACCEL_COMPILER
126 : && DECL_HAS_VALUE_EXPR_P (it)
127 : #endif
128 75 : && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it));
129 :
130 : /* See also omp_finish_file and output_offload_tables in lto-cgraph.cc. */
131 75 : if (!in_lto_p && !symtab_node::get (it))
132 0 : continue;
133 :
134 75 : tree size = NULL_TREE;
135 75 : if (is_var)
136 0 : size = fold_convert (const_ptr_type_node, DECL_SIZE_UNIT (it));
137 :
138 75 : tree addr;
139 75 : if (!is_link_var)
140 75 : addr = build_fold_addr_expr (it);
141 : else
142 : {
143 : #ifdef ACCEL_COMPILER
144 : /* For "omp declare target link" vars add address of the pointer to
145 : the target table, instead of address of the var. */
146 : tree value_expr = DECL_VALUE_EXPR (it);
147 : tree link_ptr_decl = TREE_OPERAND (value_expr, 0);
148 : varpool_node::finalize_decl (link_ptr_decl);
149 : addr = build_fold_addr_expr (link_ptr_decl);
150 : #else
151 0 : addr = build_fold_addr_expr (it);
152 : #endif
153 :
154 : /* Most significant bit of the size marks "omp declare target link"
155 : vars in host and target tables. */
156 0 : unsigned HOST_WIDE_INT isize = tree_to_uhwi (size);
157 0 : isize |= 1ULL << (int_size_in_bytes (const_ptr_type_node)
158 0 : * BITS_PER_UNIT - 1);
159 0 : size = wide_int_to_tree (const_ptr_type_node, isize);
160 : }
161 :
162 75 : CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, addr);
163 75 : if (is_var)
164 0 : CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, size);
165 : }
166 87 : }
167 :
168 : /* Return true if DECL is a function for which its references should be
169 : analyzed. */
170 :
171 : static bool
172 200281 : omp_declare_target_fn_p (tree decl)
173 : {
174 200281 : return (TREE_CODE (decl) == FUNCTION_DECL
175 200281 : && lookup_attribute ("omp declare target", DECL_ATTRIBUTES (decl))
176 24590 : && !lookup_attribute ("omp declare target host",
177 24590 : DECL_ATTRIBUTES (decl))
178 224827 : && (!flag_openacc
179 45 : || oacc_get_fn_attrib (decl) == NULL_TREE));
180 : }
181 :
182 : /* Return true if DECL Is a variable for which its initializer references
183 : should be analyzed. */
184 :
185 : static bool
186 110533 : omp_declare_target_var_p (tree decl)
187 : {
188 110533 : return (VAR_P (decl)
189 110533 : && lookup_attribute ("omp declare target", DECL_ATTRIBUTES (decl))
190 111051 : && !lookup_attribute ("omp declare target link",
191 518 : DECL_ATTRIBUTES (decl)));
192 : }
193 :
194 : /* Helper function for omp_discover_implicit_declare_target, called through
195 : walk_tree. Mark referenced FUNCTION_DECLs implicitly as
196 : declare target to. */
197 :
198 : static tree
199 996077 : omp_discover_declare_target_tgt_fn_r (tree *tp, int *walk_subtrees, void *data)
200 : {
201 996077 : if (TREE_CODE (*tp) == CALL_EXPR
202 26027 : && CALL_EXPR_FN (*tp)
203 25981 : && TREE_CODE (CALL_EXPR_FN (*tp)) == ADDR_EXPR
204 25932 : && TREE_CODE (TREE_OPERAND (CALL_EXPR_FN (*tp), 0)) == FUNCTION_DECL
205 1022009 : && lookup_attribute ("omp declare variant base",
206 25932 : DECL_ATTRIBUTES (TREE_OPERAND (CALL_EXPR_FN (*tp),
207 : 0))))
208 : {
209 83 : tree fn = TREE_OPERAND (CALL_EXPR_FN (*tp), 0);
210 196 : for (tree attr = DECL_ATTRIBUTES (fn); attr; attr = TREE_CHAIN (attr))
211 : {
212 114 : attr = lookup_attribute ("omp declare variant base", attr);
213 114 : if (attr == NULL_TREE)
214 : break;
215 113 : tree purpose = TREE_PURPOSE (TREE_VALUE (attr));
216 113 : if (TREE_CODE (purpose) == FUNCTION_DECL)
217 113 : omp_discover_declare_target_tgt_fn_r (&purpose, walk_subtrees, data);
218 : }
219 : }
220 995994 : else if (TREE_CODE (*tp) == FUNCTION_DECL)
221 : {
222 20709 : tree decl = *tp;
223 20709 : tree id = get_identifier ("omp declare target");
224 20709 : symtab_node *node = symtab_node::get (*tp);
225 20709 : if (node != NULL)
226 : {
227 13856 : while (node->alias_target
228 13856 : && TREE_CODE (node->alias_target) == FUNCTION_DECL)
229 : {
230 4 : if (!omp_declare_target_fn_p (node->decl)
231 8 : && !lookup_attribute ("omp declare target host",
232 4 : DECL_ATTRIBUTES (node->decl)))
233 : {
234 4 : node->offloadable = 1;
235 4 : DECL_ATTRIBUTES (node->decl)
236 8 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (node->decl));
237 : }
238 4 : node = symtab_node::get (node->alias_target);
239 : }
240 13852 : symtab_node *new_node = node->ultimate_alias_target ();
241 13852 : decl = new_node->decl;
242 15587 : while (node != new_node)
243 : {
244 1735 : if (!omp_declare_target_fn_p (node->decl)
245 2620 : && !lookup_attribute ("omp declare target host",
246 885 : DECL_ATTRIBUTES (node->decl)))
247 : {
248 885 : node->offloadable = 1;
249 885 : DECL_ATTRIBUTES (node->decl)
250 1770 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (node->decl));
251 : }
252 1735 : gcc_assert (node->alias && node->analyzed);
253 1735 : node = node->get_alias_target ();
254 : }
255 13852 : node->offloadable = 1;
256 13852 : if (ENABLE_OFFLOADING)
257 : g->have_offload = true;
258 : }
259 20709 : if (omp_declare_target_fn_p (decl)
260 29478 : || lookup_attribute ("omp declare target host",
261 8769 : DECL_ATTRIBUTES (decl)))
262 11940 : return NULL_TREE;
263 :
264 8769 : if (DECL_SAVED_TREE (decl)
265 8769 : && (!DECL_EXTERNAL (decl) || DECL_DECLARED_INLINE_P (decl)))
266 6657 : ((vec<tree> *) data)->safe_push (decl);
267 8769 : DECL_ATTRIBUTES (decl) = tree_cons (id, NULL_TREE,
268 8769 : DECL_ATTRIBUTES (decl));
269 : }
270 975285 : else if (TYPE_P (*tp))
271 39 : *walk_subtrees = 0;
272 975246 : else if (TREE_CODE (*tp) == OMP_TARGET)
273 : {
274 1672 : tree c = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE);
275 1672 : tree c2 = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE_TYPE);
276 43 : if ((c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
277 1672 : || (c2 && (OMP_CLAUSE_DEVICE_TYPE_KIND (c2)
278 : == OMP_CLAUSE_DEVICE_TYPE_HOST)))
279 43 : *walk_subtrees = 0;
280 : }
281 : return NULL_TREE;
282 : }
283 :
284 : /* Similarly, but ignore references outside of OMP_TARGET regions. */
285 :
286 : static tree
287 722253 : omp_discover_declare_target_fn_r (tree *tp, int *walk_subtrees, void *data)
288 : {
289 722253 : if (TREE_CODE (*tp) == OMP_TARGET)
290 : {
291 12018 : tree c = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE);
292 12018 : tree c2 = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE_TYPE);
293 863 : if ((!c || !OMP_CLAUSE_DEVICE_ANCESTOR (c))
294 12823 : && (!c2 || (OMP_CLAUSE_DEVICE_TYPE_KIND (c2)
295 : != OMP_CLAUSE_DEVICE_TYPE_HOST)))
296 11954 : walk_tree_without_duplicates (&OMP_TARGET_BODY (*tp),
297 : omp_discover_declare_target_tgt_fn_r,
298 : data);
299 12018 : *walk_subtrees = 0;
300 : }
301 710235 : else if (TYPE_P (*tp))
302 371 : *walk_subtrees = 0;
303 722253 : return NULL_TREE;
304 : }
305 :
306 : /* Helper function for omp_discover_implicit_declare_target, called through
307 : walk_tree. Mark referenced FUNCTION_DECLs implicitly as
308 : declare target to. */
309 :
310 : static tree
311 550 : omp_discover_declare_target_var_r (tree *tp, int *walk_subtrees, void *data)
312 : {
313 550 : if (TREE_CODE (*tp) == FUNCTION_DECL)
314 24 : return omp_discover_declare_target_tgt_fn_r (tp, walk_subtrees, data);
315 526 : else if (VAR_P (*tp)
316 54 : && is_global_var (*tp)
317 569 : && !omp_declare_target_var_p (*tp))
318 : {
319 15 : tree id = get_identifier ("omp declare target");
320 15 : if (lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (*tp)))
321 : {
322 0 : error_at (DECL_SOURCE_LOCATION (*tp),
323 : "%qD specified both in declare target %<link%> and "
324 : "implicitly in %<to%> clauses", *tp);
325 0 : DECL_ATTRIBUTES (*tp)
326 0 : = remove_attribute ("omp declare target link", DECL_ATTRIBUTES (*tp));
327 : }
328 15 : if (TREE_STATIC (*tp) && lang_hooks.decls.omp_get_decl_init (*tp))
329 15 : ((vec<tree> *) data)->safe_push (*tp);
330 15 : DECL_ATTRIBUTES (*tp) = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (*tp));
331 15 : symtab_node *node = symtab_node::get (*tp);
332 15 : if (node != NULL && !node->offloadable)
333 : {
334 15 : node->offloadable = 1;
335 15 : if (ENABLE_OFFLOADING)
336 : {
337 : g->have_offload = true;
338 : if (is_a <varpool_node *> (node))
339 : vec_safe_push (offload_vars, node->decl);
340 : }
341 : }
342 : }
343 511 : else if (TYPE_P (*tp))
344 0 : *walk_subtrees = 0;
345 : return NULL_TREE;
346 : }
347 :
348 : /* Perform the OpenMP implicit declare target to discovery. */
349 :
350 : void
351 9678 : omp_discover_implicit_declare_target (void)
352 : {
353 9678 : cgraph_node *node;
354 9678 : varpool_node *vnode;
355 9678 : auto_vec<tree> worklist;
356 :
357 169459 : FOR_EACH_DEFINED_FUNCTION (node)
358 159781 : if (DECL_SAVED_TREE (node->decl))
359 : {
360 159321 : struct cgraph_node *cgn;
361 159321 : if (lookup_attribute ("omp declare target indirect",
362 159321 : DECL_ATTRIBUTES (node->decl)))
363 123 : vec_safe_push (offload_ind_funcs, node->decl);
364 159321 : if (omp_declare_target_fn_p (node->decl))
365 2507 : worklist.safe_push (node->decl);
366 156814 : else if (DECL_STRUCT_FUNCTION (node->decl)
367 156814 : && DECL_STRUCT_FUNCTION (node->decl)->has_omp_target)
368 6768 : worklist.safe_push (node->decl);
369 320755 : for (cgn = first_nested_function (node);
370 161434 : cgn; cgn = next_nested_function (cgn))
371 2113 : if (omp_declare_target_fn_p (cgn->decl))
372 33 : worklist.safe_push (cgn->decl);
373 2080 : else if (DECL_STRUCT_FUNCTION (cgn->decl)
374 2080 : && DECL_STRUCT_FUNCTION (cgn->decl)->has_omp_target)
375 434 : worklist.safe_push (cgn->decl);
376 : }
377 128390 : FOR_EACH_VARIABLE (vnode)
378 118712 : if (lang_hooks.decls.omp_get_decl_init (vnode->decl)
379 118712 : && omp_declare_target_var_p (vnode->decl))
380 490 : worklist.safe_push (vnode->decl);
381 26582 : while (!worklist.is_empty ())
382 : {
383 16904 : tree decl = worklist.pop ();
384 16904 : if (VAR_P (decl))
385 505 : walk_tree_without_duplicates (lang_hooks.decls.omp_get_decl_init (decl),
386 : omp_discover_declare_target_var_r,
387 : &worklist);
388 16399 : else if (omp_declare_target_fn_p (decl))
389 9197 : walk_tree_without_duplicates (&DECL_SAVED_TREE (decl),
390 : omp_discover_declare_target_tgt_fn_r,
391 : &worklist);
392 : else
393 7202 : walk_tree_without_duplicates (&DECL_SAVED_TREE (decl),
394 : omp_discover_declare_target_fn_r,
395 : &worklist);
396 : }
397 :
398 9678 : if (omp_requires_mask
399 : & (OMP_REQUIRES_SELF_MAPS | OMP_REQUIRES_UNIFIED_SHARED_MEMORY))
400 74 : FOR_EACH_VARIABLE (vnode)
401 : {
402 : /* If 'self_maps' or 'unified_shared_memory' is enabled,
403 : remove 'enter/to' and add 'link'. */
404 28 : if (lookup_attribute ("omp declare target",
405 28 : DECL_ATTRIBUTES (vnode->decl)))
406 : {
407 16 : DECL_ATTRIBUTES (vnode->decl)
408 16 : = remove_attribute ("omp declare target",
409 16 : DECL_ATTRIBUTES (vnode->decl));
410 16 : if (!lookup_attribute ("omp declare target link",
411 16 : DECL_ATTRIBUTES (vnode->decl)))
412 16 : DECL_ATTRIBUTES (vnode->decl)
413 32 : = tree_cons (get_identifier ("omp declare target link"),
414 16 : NULL_TREE, DECL_ATTRIBUTES (vnode->decl));
415 : }
416 : }
417 :
418 9678 : lang_hooks.decls.omp_finish_decl_inits ();
419 9678 : }
420 :
421 :
422 : /* Create new symbols containing (address, size) pairs for global variables,
423 : marked with "omp declare target" attribute, as well as addresses for the
424 : functions, which are outlined offloading regions. */
425 : void
426 237364 : omp_finish_file (void)
427 : {
428 237364 : unsigned num_funcs = vec_safe_length (offload_funcs);
429 237364 : unsigned num_vars = vec_safe_length (offload_vars);
430 237364 : unsigned num_ind_funcs = vec_safe_length (offload_ind_funcs);
431 :
432 237364 : if (num_funcs == 0 && num_vars == 0 && num_ind_funcs == 0)
433 237364 : return;
434 :
435 29 : if (targetm_common.have_named_sections)
436 : {
437 29 : vec<constructor_elt, va_gc> *v_f, *v_v, *v_if;
438 29 : vec_alloc (v_f, num_funcs);
439 29 : vec_alloc (v_v, num_vars * 2);
440 29 : vec_alloc (v_if, num_ind_funcs);
441 :
442 29 : add_decls_addresses_to_decl_constructor (offload_funcs, v_f);
443 29 : add_decls_addresses_to_decl_constructor (offload_vars, v_v);
444 29 : add_decls_addresses_to_decl_constructor (offload_ind_funcs, v_if);
445 :
446 29 : tree vars_decl_type = build_array_type_nelts (pointer_sized_int_node,
447 29 : vec_safe_length (v_v));
448 29 : tree funcs_decl_type = build_array_type_nelts (pointer_sized_int_node,
449 29 : num_funcs);
450 29 : tree ind_funcs_decl_type = build_array_type_nelts (pointer_sized_int_node,
451 29 : num_ind_funcs);
452 :
453 29 : SET_TYPE_ALIGN (vars_decl_type, TYPE_ALIGN (pointer_sized_int_node));
454 29 : SET_TYPE_ALIGN (funcs_decl_type, TYPE_ALIGN (pointer_sized_int_node));
455 29 : SET_TYPE_ALIGN (ind_funcs_decl_type, TYPE_ALIGN (pointer_sized_int_node));
456 29 : tree ctor_v = build_constructor (vars_decl_type, v_v);
457 29 : tree ctor_f = build_constructor (funcs_decl_type, v_f);
458 29 : tree ctor_if = build_constructor (ind_funcs_decl_type, v_if);
459 29 : TREE_CONSTANT (ctor_v) = TREE_CONSTANT (ctor_f) = TREE_CONSTANT (ctor_if) = 1;
460 29 : TREE_STATIC (ctor_v) = TREE_STATIC (ctor_f) = TREE_STATIC (ctor_if) = 1;
461 29 : tree funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
462 : get_identifier (".offload_func_table"),
463 : funcs_decl_type);
464 29 : tree vars_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
465 : get_identifier (".offload_var_table"),
466 : vars_decl_type);
467 29 : tree ind_funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
468 : get_identifier (".offload_ind_func_table"),
469 : ind_funcs_decl_type);
470 29 : TREE_STATIC (funcs_decl) = TREE_STATIC (ind_funcs_decl) = 1;
471 29 : TREE_STATIC (vars_decl) = 1;
472 : /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
473 : otherwise a joint table in a binary will contain padding between
474 : tables from multiple object files. */
475 29 : DECL_USER_ALIGN (funcs_decl) = DECL_USER_ALIGN (ind_funcs_decl) = 1;
476 29 : DECL_USER_ALIGN (vars_decl) = 1;
477 29 : SET_DECL_ALIGN (funcs_decl, TYPE_ALIGN (funcs_decl_type));
478 29 : SET_DECL_ALIGN (vars_decl, TYPE_ALIGN (vars_decl_type));
479 29 : SET_DECL_ALIGN (ind_funcs_decl, TYPE_ALIGN (ind_funcs_decl_type));
480 29 : DECL_INITIAL (funcs_decl) = ctor_f;
481 29 : DECL_INITIAL (vars_decl) = ctor_v;
482 29 : DECL_INITIAL (ind_funcs_decl) = ctor_if;
483 29 : set_decl_section_name (funcs_decl, OFFLOAD_FUNC_TABLE_SECTION_NAME);
484 29 : set_decl_section_name (vars_decl, OFFLOAD_VAR_TABLE_SECTION_NAME);
485 29 : set_decl_section_name (ind_funcs_decl,
486 : OFFLOAD_IND_FUNC_TABLE_SECTION_NAME);
487 29 : varpool_node::finalize_decl (vars_decl);
488 29 : varpool_node::finalize_decl (funcs_decl);
489 29 : varpool_node::finalize_decl (ind_funcs_decl);
490 : }
491 : else
492 : {
493 0 : for (unsigned i = 0; i < num_funcs; i++)
494 : {
495 0 : tree it = (*offload_funcs)[i];
496 : /* See also add_decls_addresses_to_decl_constructor
497 : and output_offload_tables in lto-cgraph.cc. */
498 0 : if (!in_lto_p && !symtab_node::get (it))
499 0 : continue;
500 0 : targetm.record_offload_symbol (it);
501 : }
502 0 : for (unsigned i = 0; i < num_vars; i++)
503 : {
504 0 : tree it = (*offload_vars)[i];
505 0 : if (!in_lto_p && !symtab_node::get (it))
506 0 : continue;
507 : #ifdef ACCEL_COMPILER
508 : if (DECL_HAS_VALUE_EXPR_P (it)
509 : && lookup_attribute ("omp declare target link",
510 : DECL_ATTRIBUTES (it)))
511 : {
512 : tree value_expr = DECL_VALUE_EXPR (it);
513 : tree link_ptr_decl = TREE_OPERAND (value_expr, 0);
514 : targetm.record_offload_symbol (link_ptr_decl);
515 : varpool_node::finalize_decl (link_ptr_decl);
516 : }
517 : else
518 : #endif
519 0 : targetm.record_offload_symbol (it);
520 : }
521 0 : for (unsigned i = 0; i < num_ind_funcs; i++)
522 : {
523 0 : tree it = (*offload_ind_funcs)[i];
524 : /* See also add_decls_addresses_to_decl_constructor
525 : and output_offload_tables in lto-cgraph.cc. */
526 0 : if (!in_lto_p && !symtab_node::get (it))
527 0 : continue;
528 0 : targetm.record_offload_symbol (it);
529 : }
530 : }
531 : }
532 :
533 : /* Call dim_pos (POS == true) or dim_size (POS == false) builtins for
534 : axis DIM. Return a tmp var holding the result. */
535 :
536 : static tree
537 30709 : oacc_dim_call (bool pos, int dim, gimple_seq *seq)
538 : {
539 30709 : tree arg = build_int_cst (unsigned_type_node, dim);
540 30709 : tree size = create_tmp_var (integer_type_node);
541 30709 : enum internal_fn fn = pos ? IFN_GOACC_DIM_POS : IFN_GOACC_DIM_SIZE;
542 30709 : gimple *call = gimple_build_call_internal (fn, 1, arg);
543 :
544 30709 : gimple_call_set_lhs (call, size);
545 30709 : gimple_seq_add_stmt (seq, call);
546 :
547 30709 : return size;
548 : }
549 :
550 : /* Find the number of threads (POS = false), or thread number (POS =
551 : true) for an OpenACC region partitioned as MASK. Setup code
552 : required for the calculation is added to SEQ. */
553 :
554 : static tree
555 23610 : oacc_thread_numbers (bool pos, int mask, gimple_seq *seq)
556 : {
557 23610 : tree res = pos ? NULL_TREE : build_int_cst (unsigned_type_node, 1);
558 23610 : unsigned ix;
559 :
560 : /* Start at gang level, and examine relevant dimension indices. */
561 94440 : for (ix = GOMP_DIM_GANG; ix != GOMP_DIM_MAX; ix++)
562 70830 : if (GOMP_DIM_MASK (ix) & mask)
563 : {
564 26569 : if (res)
565 : {
566 : /* We had an outer index, so scale that by the size of
567 : this dimension. */
568 17369 : tree n = oacc_dim_call (false, ix, seq);
569 17369 : res = fold_build2 (MULT_EXPR, integer_type_node, res, n);
570 : }
571 26569 : if (pos)
572 : {
573 : /* Determine index in this dimension. */
574 13340 : tree id = oacc_dim_call (true, ix, seq);
575 13340 : if (res)
576 4140 : res = fold_build2 (PLUS_EXPR, integer_type_node, res, id);
577 : else
578 : res = id;
579 : }
580 : }
581 :
582 23610 : if (res == NULL_TREE)
583 2657 : res = integer_zero_node;
584 :
585 23610 : return res;
586 : }
587 :
588 : /* Transform IFN_GOACC_LOOP calls to actual code. See
589 : expand_oacc_for for where these are generated. At the vector
590 : level, we stride loops, such that each member of a warp will
591 : operate on adjacent iterations. At the worker and gang level,
592 : each gang/warp executes a set of contiguous iterations. Chunking
593 : can override this such that each iteration engine executes a
594 : contiguous chunk, and then moves on to stride to the next chunk. */
595 :
596 : static void
597 46702 : oacc_xform_loop (gcall *call)
598 : {
599 46702 : gimple_stmt_iterator gsi = gsi_for_stmt (call);
600 46702 : enum ifn_goacc_loop_kind code
601 46702 : = (enum ifn_goacc_loop_kind) TREE_INT_CST_LOW (gimple_call_arg (call, 0));
602 46702 : tree dir = gimple_call_arg (call, 1);
603 46702 : tree range = gimple_call_arg (call, 2);
604 46702 : tree step = gimple_call_arg (call, 3);
605 46702 : tree chunk_size = NULL_TREE;
606 46702 : unsigned mask = (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call, 5));
607 46702 : tree lhs = gimple_call_lhs (call);
608 46702 : tree type = NULL_TREE;
609 46702 : tree diff_type = TREE_TYPE (range);
610 46702 : tree r = NULL_TREE;
611 46702 : gimple_seq seq = NULL;
612 46702 : bool chunking = false, striding = true;
613 46702 : unsigned outer_mask = mask & (~mask + 1); // Outermost partitioning
614 46702 : unsigned inner_mask = mask & ~outer_mask; // Inner partitioning (if any)
615 :
616 : /* Skip lowering if return value of IFN_GOACC_LOOP call is not used. */
617 46702 : if (!lhs)
618 : {
619 8 : gsi_replace_with_seq (&gsi, seq, true);
620 8 : return;
621 : }
622 :
623 46694 : type = TREE_TYPE (lhs);
624 :
625 : #ifdef ACCEL_COMPILER
626 : chunk_size = gimple_call_arg (call, 4);
627 : if (integer_minus_onep (chunk_size) /* Force static allocation. */
628 : || integer_zerop (chunk_size)) /* Default (also static). */
629 : {
630 : /* If we're at the gang level, we want each to execute a
631 : contiguous run of iterations. Otherwise we want each element
632 : to stride. */
633 : striding = !(outer_mask & GOMP_DIM_MASK (GOMP_DIM_GANG));
634 : chunking = false;
635 : }
636 : else
637 : {
638 : /* Chunk of size 1 is striding. */
639 : striding = integer_onep (chunk_size);
640 : chunking = !striding;
641 : }
642 : #endif
643 :
644 : /* striding=true, chunking=true
645 : -> invalid.
646 : striding=true, chunking=false
647 : -> chunks=1
648 : striding=false,chunking=true
649 : -> chunks=ceil (range/(chunksize*threads*step))
650 : striding=false,chunking=false
651 : -> chunk_size=ceil(range/(threads*step)),chunks=1 */
652 46694 : push_gimplify_context (true);
653 :
654 46694 : switch (code)
655 : {
656 0 : default: gcc_unreachable ();
657 :
658 11231 : case IFN_GOACC_LOOP_CHUNKS:
659 11231 : if (!chunking)
660 11231 : r = build_int_cst (type, 1);
661 : else
662 : {
663 : /* chunk_max
664 : = (range - dir) / (chunks * step * num_threads) + dir */
665 : tree per = oacc_thread_numbers (false, mask, &seq);
666 : per = fold_convert (type, per);
667 : chunk_size = fold_convert (type, chunk_size);
668 : per = fold_build2 (MULT_EXPR, type, per, chunk_size);
669 : per = fold_build2 (MULT_EXPR, type, per, step);
670 : r = build2 (MINUS_EXPR, type, range, dir);
671 : r = build2 (PLUS_EXPR, type, r, per);
672 : r = build2 (TRUNC_DIV_EXPR, type, r, per);
673 : }
674 : break;
675 :
676 11753 : case IFN_GOACC_LOOP_STEP:
677 11753 : {
678 : /* If striding, step by the entire compute volume, otherwise
679 : step by the inner volume. */
680 11753 : unsigned volume = striding ? mask : inner_mask;
681 :
682 11753 : r = oacc_thread_numbers (false, volume, &seq);
683 11753 : r = build2 (MULT_EXPR, type, fold_convert (type, r), step);
684 : }
685 11753 : break;
686 :
687 11857 : case IFN_GOACC_LOOP_OFFSET:
688 : /* Enable vectorization on non-SIMT targets. */
689 11857 : if (!targetm.simt.vf
690 11857 : && outer_mask == GOMP_DIM_MASK (GOMP_DIM_VECTOR)
691 : /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
692 : the loop. */
693 1951 : && (flag_tree_loop_vectorize
694 1463 : || !OPTION_SET_P (flag_tree_loop_vectorize)))
695 : {
696 1951 : basic_block bb = gsi_bb (gsi);
697 1951 : class loop *parent = bb->loop_father;
698 1951 : class loop *body = parent->inner;
699 :
700 1951 : parent->force_vectorize = true;
701 1951 : parent->safelen = INT_MAX;
702 :
703 : /* "Chunking loops" may have inner loops. */
704 1951 : if (parent->inner)
705 : {
706 1939 : body->force_vectorize = true;
707 1939 : body->safelen = INT_MAX;
708 : }
709 :
710 1951 : cfun->has_force_vectorize_loops = true;
711 : }
712 11857 : if (striding)
713 : {
714 11857 : r = oacc_thread_numbers (true, mask, &seq);
715 11857 : r = fold_convert (diff_type, r);
716 : }
717 : else
718 : {
719 : tree inner_size = oacc_thread_numbers (false, inner_mask, &seq);
720 : tree outer_size = oacc_thread_numbers (false, outer_mask, &seq);
721 : tree volume = fold_build2 (MULT_EXPR, TREE_TYPE (inner_size),
722 : inner_size, outer_size);
723 :
724 : volume = fold_convert (diff_type, volume);
725 : if (chunking)
726 : chunk_size = fold_convert (diff_type, chunk_size);
727 : else
728 : {
729 : tree per = fold_build2 (MULT_EXPR, diff_type, volume, step);
730 :
731 : chunk_size = build2 (MINUS_EXPR, diff_type, range, dir);
732 : chunk_size = build2 (PLUS_EXPR, diff_type, chunk_size, per);
733 : chunk_size = build2 (TRUNC_DIV_EXPR, diff_type, chunk_size, per);
734 : }
735 :
736 : tree span = build2 (MULT_EXPR, diff_type, chunk_size,
737 : fold_convert (diff_type, inner_size));
738 : r = oacc_thread_numbers (true, outer_mask, &seq);
739 : r = fold_convert (diff_type, r);
740 : r = build2 (MULT_EXPR, diff_type, r, span);
741 :
742 : tree inner = oacc_thread_numbers (true, inner_mask, &seq);
743 : inner = fold_convert (diff_type, inner);
744 : r = fold_build2 (PLUS_EXPR, diff_type, r, inner);
745 :
746 : if (chunking)
747 : {
748 : tree chunk = fold_convert (diff_type, gimple_call_arg (call, 6));
749 : tree per
750 : = fold_build2 (MULT_EXPR, diff_type, volume, chunk_size);
751 : per = build2 (MULT_EXPR, diff_type, per, chunk);
752 :
753 : r = build2 (PLUS_EXPR, diff_type, r, per);
754 : }
755 : }
756 11857 : r = fold_build2 (MULT_EXPR, diff_type, r, step);
757 11857 : if (type != diff_type)
758 178 : r = fold_convert (type, r);
759 : break;
760 :
761 11853 : case IFN_GOACC_LOOP_BOUND:
762 11853 : if (striding)
763 11853 : r = range;
764 : else
765 : {
766 : tree inner_size = oacc_thread_numbers (false, inner_mask, &seq);
767 : tree outer_size = oacc_thread_numbers (false, outer_mask, &seq);
768 : tree volume = fold_build2 (MULT_EXPR, TREE_TYPE (inner_size),
769 : inner_size, outer_size);
770 :
771 : volume = fold_convert (diff_type, volume);
772 : if (chunking)
773 : chunk_size = fold_convert (diff_type, chunk_size);
774 : else
775 : {
776 : tree per = fold_build2 (MULT_EXPR, diff_type, volume, step);
777 :
778 : chunk_size = build2 (MINUS_EXPR, diff_type, range, dir);
779 : chunk_size = build2 (PLUS_EXPR, diff_type, chunk_size, per);
780 : chunk_size = build2 (TRUNC_DIV_EXPR, diff_type, chunk_size, per);
781 : }
782 :
783 : tree span = build2 (MULT_EXPR, diff_type, chunk_size,
784 : fold_convert (diff_type, inner_size));
785 :
786 : r = fold_build2 (MULT_EXPR, diff_type, span, step);
787 :
788 : tree offset = gimple_call_arg (call, 6);
789 : r = build2 (PLUS_EXPR, diff_type, r,
790 : fold_convert (diff_type, offset));
791 : r = build2 (integer_onep (dir) ? MIN_EXPR : MAX_EXPR,
792 : diff_type, r, range);
793 : }
794 11853 : if (diff_type != type)
795 178 : r = fold_convert (type, r);
796 : break;
797 : }
798 :
799 46694 : gimplify_assign (lhs, r, &seq);
800 :
801 46694 : pop_gimplify_context (NULL);
802 :
803 46694 : gsi_replace_with_seq (&gsi, seq, true);
804 : }
805 :
806 : /* Transform a GOACC_TILE call. Determines the element loop span for
807 : the specified loop of the nest. This is 1 if we're not tiling.
808 :
809 : GOACC_TILE (collapse_count, loop_no, tile_arg, gwv_tile, gwv_element); */
810 :
811 : static void
812 284 : oacc_xform_tile (gcall *call)
813 : {
814 284 : gimple_stmt_iterator gsi = gsi_for_stmt (call);
815 284 : unsigned collapse = tree_to_uhwi (gimple_call_arg (call, 0));
816 : /* Inner loops have higher loop_nos. */
817 284 : unsigned loop_no = tree_to_uhwi (gimple_call_arg (call, 1));
818 284 : tree tile_size = gimple_call_arg (call, 2);
819 284 : unsigned e_mask = tree_to_uhwi (gimple_call_arg (call, 4));
820 284 : tree lhs = gimple_call_lhs (call);
821 284 : tree type = TREE_TYPE (lhs);
822 284 : gimple_seq seq = NULL;
823 284 : tree span = build_int_cst (type, 1);
824 :
825 284 : gcc_assert (!(e_mask
826 : & ~(GOMP_DIM_MASK (GOMP_DIM_VECTOR)
827 : | GOMP_DIM_MASK (GOMP_DIM_WORKER))));
828 284 : push_gimplify_context (!seen_error ());
829 :
830 : #ifndef ACCEL_COMPILER
831 : /* Partitioning disabled on host compilers. */
832 284 : e_mask = 0;
833 : #endif
834 284 : if (!e_mask)
835 : /* Not partitioning. */
836 284 : span = integer_one_node;
837 : else if (!integer_zerop (tile_size))
838 : /* User explicitly specified size. */
839 : span = tile_size;
840 : else
841 : {
842 : /* Pick a size based on the partitioning of the element loop and
843 : the number of loop nests. */
844 : tree first_size = NULL_TREE;
845 : tree second_size = NULL_TREE;
846 :
847 : if (e_mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR))
848 : first_size = oacc_dim_call (false, GOMP_DIM_VECTOR, &seq);
849 : if (e_mask & GOMP_DIM_MASK (GOMP_DIM_WORKER))
850 : second_size = oacc_dim_call (false, GOMP_DIM_WORKER, &seq);
851 :
852 : if (!first_size)
853 : {
854 : first_size = second_size;
855 : second_size = NULL_TREE;
856 : }
857 :
858 : if (loop_no + 1 == collapse)
859 : {
860 : span = first_size;
861 : if (!loop_no && second_size)
862 : span = fold_build2 (MULT_EXPR, TREE_TYPE (span),
863 : span, second_size);
864 : }
865 : else if (loop_no + 2 == collapse)
866 : span = second_size;
867 : else
868 : span = NULL_TREE;
869 :
870 : if (!span)
871 : /* There's no obvious element size for this loop. Options
872 : are 1, first_size or some non-unity constant (32 is my
873 : favourite). We should gather some statistics. */
874 : span = first_size;
875 : }
876 :
877 284 : span = fold_convert (type, span);
878 284 : gimplify_assign (lhs, span, &seq);
879 :
880 284 : pop_gimplify_context (NULL);
881 :
882 284 : gsi_replace_with_seq (&gsi, seq, true);
883 284 : }
884 :
885 : /* Default partitioned and minimum partitioned dimensions. */
886 :
887 : static int oacc_default_dims[GOMP_DIM_MAX];
888 : static int oacc_min_dims[GOMP_DIM_MAX];
889 :
890 : int
891 0 : oacc_get_default_dim (int dim)
892 : {
893 0 : gcc_assert (0 <= dim && dim < GOMP_DIM_MAX);
894 0 : return oacc_default_dims[dim];
895 : }
896 :
897 : int
898 0 : oacc_get_min_dim (int dim)
899 : {
900 0 : gcc_assert (0 <= dim && dim < GOMP_DIM_MAX);
901 0 : return oacc_min_dims[dim];
902 : }
903 :
904 : /* Parse the default dimension parameter. This is a set of
905 : :-separated optional compute dimensions. Each specified dimension
906 : is a positive integer. When device type support is added, it is
907 : planned to be a comma separated list of such compute dimensions,
908 : with all but the first prefixed by the colon-terminated device
909 : type. */
910 :
911 : static void
912 2281 : oacc_parse_default_dims (const char *dims)
913 : {
914 2281 : int ix;
915 :
916 9124 : for (ix = GOMP_DIM_MAX; ix--;)
917 : {
918 6843 : oacc_default_dims[ix] = -1;
919 6843 : oacc_min_dims[ix] = 1;
920 : }
921 :
922 : #ifndef ACCEL_COMPILER
923 : /* Cannot be overridden on the host. */
924 2281 : dims = NULL;
925 : #endif
926 2281 : if (dims)
927 : {
928 : const char *pos = dims;
929 :
930 : for (ix = 0; *pos && ix != GOMP_DIM_MAX; ix++)
931 : {
932 : if (ix)
933 : {
934 : if (*pos != ':')
935 : goto malformed;
936 : pos++;
937 : }
938 :
939 : if (*pos != ':')
940 : {
941 : long val;
942 : const char *eptr;
943 :
944 : errno = 0;
945 : val = strtol (pos, const_cast<char **> (&eptr), 10);
946 : if (errno || val <= 0 || (int) val != val)
947 : goto malformed;
948 : pos = eptr;
949 : oacc_default_dims[ix] = (int) val;
950 : }
951 : }
952 : if (*pos)
953 : {
954 : malformed:
955 : error_at (UNKNOWN_LOCATION,
956 : "%<-fopenacc-dim%> operand is malformed at %qs", pos);
957 : }
958 : }
959 :
960 : /* Allow the backend to validate the dimensions. */
961 2281 : targetm.goacc.validate_dims (NULL_TREE, oacc_default_dims, -1, 0);
962 2281 : targetm.goacc.validate_dims (NULL_TREE, oacc_min_dims, -2, 0);
963 2281 : }
964 :
965 : /* Validate and update the dimensions for offloaded FN. ATTRS is the
966 : raw attribute. DIMS is an array of dimensions, which is filled in.
967 : LEVEL is the partitioning level of a routine, or -1 for an offload
968 : region itself. USED is the mask of partitioned execution in the
969 : function. */
970 :
971 : static void
972 9879 : oacc_validate_dims (tree fn, tree attrs, int *dims, int level, unsigned used)
973 : {
974 9879 : tree purpose[GOMP_DIM_MAX];
975 9879 : unsigned ix;
976 9879 : tree pos = TREE_VALUE (attrs);
977 :
978 : /* Make sure the attribute creator attached the dimension
979 : information. */
980 9879 : gcc_assert (pos);
981 :
982 39516 : for (ix = 0; ix != GOMP_DIM_MAX; ix++)
983 : {
984 29637 : purpose[ix] = TREE_PURPOSE (pos);
985 29637 : tree val = TREE_VALUE (pos);
986 29637 : dims[ix] = val ? TREE_INT_CST_LOW (val) : -1;
987 29637 : pos = TREE_CHAIN (pos);
988 : }
989 :
990 9879 : bool check = true;
991 : #ifdef ACCEL_COMPILER
992 : check = false;
993 : #endif
994 9879 : if (check
995 9879 : && warn_openacc_parallelism
996 1371 : && !lookup_attribute ("oacc kernels", DECL_ATTRIBUTES (fn)))
997 : {
998 1268 : static char const *const axes[] =
999 : /* Must be kept in sync with GOMP_DIM enumeration. */
1000 : { "gang", "worker", "vector" };
1001 4775 : for (ix = level >= 0 ? level : 0; ix != GOMP_DIM_MAX; ix++)
1002 3507 : if (dims[ix] < 0)
1003 : ; /* Defaulting axis. */
1004 1970 : else if ((used & GOMP_DIM_MASK (ix)) && dims[ix] == 1)
1005 : /* There is partitioned execution, but the user requested a
1006 : dimension size of 1. They're probably confused. */
1007 94 : warning_at (DECL_SOURCE_LOCATION (fn), OPT_Wopenacc_parallelism,
1008 : "region contains %s partitioned code but"
1009 94 : " is not %s partitioned", axes[ix], axes[ix]);
1010 1876 : else if (!(used & GOMP_DIM_MASK (ix)) && dims[ix] != 1)
1011 : /* The dimension is explicitly partitioned to non-unity, but
1012 : no use is made within the region. */
1013 500 : warning_at (DECL_SOURCE_LOCATION (fn), OPT_Wopenacc_parallelism,
1014 : "region is %s partitioned but"
1015 : " does not contain %s partitioned code",
1016 500 : axes[ix], axes[ix]);
1017 : }
1018 :
1019 9879 : bool changed = targetm.goacc.validate_dims (fn, dims, level, used);
1020 :
1021 : /* Default anything left to 1 or a partitioned default. */
1022 49395 : for (ix = 0; ix != GOMP_DIM_MAX; ix++)
1023 29637 : if (dims[ix] < 0)
1024 : {
1025 : /* The OpenACC spec says 'If the [num_gangs] clause is not
1026 : specified, an implementation-defined default will be used;
1027 : the default may depend on the code within the construct.'
1028 : (2.5.6). Thus an implementation is free to choose
1029 : non-unity default for a parallel region that doesn't have
1030 : any gang-partitioned loops. However, it appears that there
1031 : is a sufficient body of user code that expects non-gang
1032 : partitioned regions to not execute in gang-redundant mode.
1033 : So we (a) don't warn about the non-portability and (b) pick
1034 : the minimum permissible dimension size when there is no
1035 : partitioned execution. Otherwise we pick the global
1036 : default for the dimension, which the user can control. The
1037 : same wording and logic applies to num_workers and
1038 : vector_length, however the worker- or vector- single
1039 : execution doesn't have the same impact as gang-redundant
1040 : execution. (If the minimum gang-level partitioning is not 1,
1041 : the target is probably too confusing.) */
1042 0 : dims[ix] = (used & GOMP_DIM_MASK (ix)
1043 0 : ? oacc_default_dims[ix] : oacc_min_dims[ix]);
1044 0 : changed = true;
1045 : }
1046 :
1047 9879 : if (changed)
1048 : {
1049 : /* Replace the attribute with new values. */
1050 : pos = NULL_TREE;
1051 35428 : for (ix = GOMP_DIM_MAX; ix--;)
1052 26571 : pos = tree_cons (purpose[ix],
1053 26571 : build_int_cst (integer_type_node, dims[ix]), pos);
1054 8857 : oacc_replace_fn_attrib (fn, pos);
1055 : }
1056 9879 : }
1057 :
1058 : /* Create an empty OpenACC loop structure at LOC. */
1059 :
1060 : static oacc_loop *
1061 21347 : new_oacc_loop_raw (oacc_loop *parent, location_t loc)
1062 : {
1063 10835 : oacc_loop *loop = XCNEW (oacc_loop);
1064 :
1065 21347 : loop->parent = parent;
1066 :
1067 10835 : if (parent)
1068 : {
1069 10835 : loop->sibling = parent->child;
1070 10835 : parent->child = loop;
1071 : }
1072 :
1073 21347 : loop->loc = loc;
1074 21347 : return loop;
1075 : }
1076 :
1077 : /* Create an outermost, dummy OpenACC loop for offloaded function
1078 : DECL. */
1079 :
1080 : static oacc_loop *
1081 9879 : new_oacc_loop_outer (tree decl)
1082 : {
1083 9879 : return new_oacc_loop_raw (NULL, DECL_SOURCE_LOCATION (decl));
1084 : }
1085 :
1086 : /* Start a new OpenACC loop structure beginning at head marker HEAD.
1087 : Link into PARENT loop. Return the new loop. */
1088 :
1089 : static oacc_loop *
1090 9634 : new_oacc_loop (oacc_loop *parent, gcall *marker)
1091 : {
1092 9634 : oacc_loop *loop = new_oacc_loop_raw (parent, gimple_location (marker));
1093 :
1094 9634 : loop->marker = marker;
1095 :
1096 : /* TODO: This is where device_type flattening would occur for the loop
1097 : flags. */
1098 :
1099 9634 : loop->flags = TREE_INT_CST_LOW (gimple_call_arg (marker, 3));
1100 :
1101 9634 : tree chunk_size = integer_zero_node;
1102 9634 : if (loop->flags & OLF_GANG_STATIC)
1103 146 : chunk_size = gimple_call_arg (marker, 4);
1104 9634 : loop->chunk_size = chunk_size;
1105 :
1106 9634 : return loop;
1107 : }
1108 :
1109 : /* Create a dummy loop encompassing a call to a openACC routine.
1110 : Extract the routine's partitioning requirements. */
1111 :
1112 : static void
1113 1201 : new_oacc_loop_routine (oacc_loop *parent, gcall *call, tree decl, tree attrs)
1114 : {
1115 1201 : oacc_loop *loop = new_oacc_loop_raw (parent, gimple_location (call));
1116 1201 : int level = oacc_fn_attrib_level (attrs);
1117 :
1118 1201 : gcc_assert (level >= 0);
1119 :
1120 1201 : loop->marker = call;
1121 1201 : loop->routine = decl;
1122 1201 : loop->mask = ((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1)
1123 1201 : ^ (GOMP_DIM_MASK (level) - 1));
1124 1201 : }
1125 :
1126 : /* Finish off the current OpenACC loop ending at tail marker TAIL.
1127 : Return the parent loop. */
1128 :
1129 : static oacc_loop *
1130 9634 : finish_oacc_loop (oacc_loop *loop)
1131 : {
1132 : /* If the loop has been collapsed, don't partition it. */
1133 0 : if (loop->ifns.is_empty ())
1134 0 : loop->mask = loop->flags = 0;
1135 9634 : return loop->parent;
1136 : }
1137 :
1138 : /* Free all OpenACC loop structures within LOOP (inclusive). */
1139 :
1140 : static void
1141 21347 : free_oacc_loop (oacc_loop *loop)
1142 : {
1143 21347 : if (loop->sibling)
1144 2194 : free_oacc_loop (loop->sibling);
1145 21347 : if (loop->child)
1146 8641 : free_oacc_loop (loop->child);
1147 :
1148 21347 : loop->ifns.release ();
1149 21347 : free (loop);
1150 21347 : }
1151 :
1152 : /* Dump out the OpenACC loop head or tail beginning at FROM. */
1153 :
1154 : static void
1155 238 : dump_oacc_loop_part (FILE *file, gcall *from, int depth,
1156 : const char *title, int level)
1157 : {
1158 238 : enum ifn_unique_kind kind
1159 238 : = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (from, 0));
1160 :
1161 238 : fprintf (file, "%*s%s-%d:\n", depth * 2, "", title, level);
1162 238 : for (gimple_stmt_iterator gsi = gsi_for_stmt (from);;)
1163 : {
1164 719 : gimple *stmt = gsi_stmt (gsi);
1165 :
1166 719 : if (gimple_call_internal_p (stmt, IFN_UNIQUE))
1167 : {
1168 719 : enum ifn_unique_kind k
1169 719 : = ((enum ifn_unique_kind) TREE_INT_CST_LOW
1170 719 : (gimple_call_arg (stmt, 0)));
1171 :
1172 719 : if (k == kind && stmt != from)
1173 : break;
1174 : }
1175 481 : print_gimple_stmt (file, stmt, depth * 2 + 2);
1176 :
1177 481 : gsi_next (&gsi);
1178 962 : while (gsi_end_p (gsi))
1179 962 : gsi = gsi_start_bb (single_succ (gsi_bb (gsi)));
1180 : }
1181 238 : }
1182 :
1183 : /* Dump OpenACC loop LOOP, its children, and its siblings. */
1184 :
1185 : static void
1186 183 : dump_oacc_loop (FILE *file, oacc_loop *loop, int depth)
1187 : {
1188 222 : int ix;
1189 :
1190 222 : fprintf (file, "%*sLoop %x(%x) %s:%u\n", depth * 2, "",
1191 : loop->flags, loop->mask,
1192 222 : LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc));
1193 :
1194 222 : if (loop->marker)
1195 108 : print_gimple_stmt (file, loop->marker, depth * 2);
1196 :
1197 222 : if (loop->routine)
1198 48 : fprintf (file, "%*sRoutine %s:%u:%s\n",
1199 48 : depth * 2, "", DECL_SOURCE_FILE (loop->routine),
1200 96 : DECL_SOURCE_LINE (loop->routine),
1201 48 : IDENTIFIER_POINTER (DECL_NAME (loop->routine)));
1202 :
1203 888 : for (ix = GOMP_DIM_GANG; ix != GOMP_DIM_MAX; ix++)
1204 666 : if (loop->heads[ix])
1205 119 : dump_oacc_loop_part (file, loop->heads[ix], depth, "Head", ix);
1206 888 : for (ix = GOMP_DIM_MAX; ix--;)
1207 666 : if (loop->tails[ix])
1208 119 : dump_oacc_loop_part (file, loop->tails[ix], depth, "Tail", ix);
1209 :
1210 222 : if (loop->child)
1211 69 : dump_oacc_loop (file, loop->child, depth + 1);
1212 222 : if (loop->sibling)
1213 : dump_oacc_loop (file, loop->sibling, depth);
1214 183 : }
1215 :
1216 : void debug_oacc_loop (oacc_loop *);
1217 :
1218 : /* Dump loops to stderr. */
1219 :
1220 : DEBUG_FUNCTION void
1221 0 : debug_oacc_loop (oacc_loop *loop)
1222 : {
1223 0 : dump_oacc_loop (stderr, loop, 0);
1224 0 : }
1225 :
1226 : /* Provide diagnostics on OpenACC loop LOOP, its children, and its
1227 : siblings. */
1228 :
1229 : static void
1230 2741 : inform_oacc_loop (const oacc_loop *loop)
1231 : {
1232 1796 : const char *gang
1233 2741 : = loop->mask & GOMP_DIM_MASK (GOMP_DIM_GANG) ? " gang" : "";
1234 2226 : const char *worker
1235 2741 : = loop->mask & GOMP_DIM_MASK (GOMP_DIM_WORKER) ? " worker" : "";
1236 1747 : const char *vector
1237 2741 : = loop->mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR) ? " vector" : "";
1238 2741 : const char *seq = loop->mask == 0 ? " seq" : "";
1239 2741 : const dump_user_location_t loc
1240 2741 : = dump_user_location_t::from_location_t (loop->loc);
1241 2741 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, loc,
1242 : "assigned OpenACC%s%s%s%s loop parallelism\n", gang, worker,
1243 : vector, seq);
1244 :
1245 2741 : if (loop->child)
1246 698 : inform_oacc_loop (loop->child);
1247 2741 : if (loop->sibling)
1248 280 : inform_oacc_loop (loop->sibling);
1249 2741 : }
1250 :
1251 : /* DFS walk of basic blocks BB onwards, creating OpenACC loop
1252 : structures as we go. By construction these loops are properly
1253 : nested. */
1254 :
1255 : static void
1256 176002 : oacc_loop_discover_walk (oacc_loop *loop, basic_block bb)
1257 : {
1258 176002 : int marker = 0;
1259 176002 : int remaining = 0;
1260 :
1261 176002 : if (bb->flags & BB_VISITED)
1262 40061 : return;
1263 :
1264 135941 : follow:
1265 202126 : bb->flags |= BB_VISITED;
1266 :
1267 : /* Scan for loop markers. */
1268 788879 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
1269 384627 : gsi_next (&gsi))
1270 : {
1271 384627 : gimple *stmt = gsi_stmt (gsi);
1272 :
1273 384627 : if (!is_gimple_call (stmt))
1274 220292 : continue;
1275 :
1276 169098 : gcall *call = as_a <gcall *> (stmt);
1277 :
1278 : /* If this is a routine, make a dummy loop for it. */
1279 169098 : if (tree decl = gimple_call_fndecl (call))
1280 4761 : if (tree attrs = oacc_get_fn_attrib (decl))
1281 : {
1282 1201 : gcc_assert (!marker);
1283 1201 : new_oacc_loop_routine (loop, call, decl, attrs);
1284 : }
1285 :
1286 169098 : if (!gimple_call_internal_p (call))
1287 4763 : continue;
1288 :
1289 164335 : switch (gimple_call_internal_fn (call))
1290 : {
1291 : default:
1292 : break;
1293 :
1294 46986 : case IFN_GOACC_LOOP:
1295 46986 : case IFN_GOACC_TILE:
1296 : /* Record the abstraction function, so we can manipulate it
1297 : later. */
1298 46986 : loop->ifns.safe_push (call);
1299 46986 : break;
1300 :
1301 85483 : case IFN_UNIQUE:
1302 85483 : enum ifn_unique_kind kind
1303 85483 : = (enum ifn_unique_kind) (TREE_INT_CST_LOW
1304 85483 : (gimple_call_arg (call, 0)));
1305 85483 : if (kind == IFN_UNIQUE_OACC_HEAD_MARK
1306 85483 : || kind == IFN_UNIQUE_OACC_TAIL_MARK)
1307 : {
1308 52246 : if (gimple_call_num_args (call) == 2)
1309 : {
1310 19268 : gcc_assert (marker && !remaining);
1311 19268 : marker = 0;
1312 19268 : if (kind == IFN_UNIQUE_OACC_TAIL_MARK)
1313 19268 : loop = finish_oacc_loop (loop);
1314 : else
1315 9634 : loop->head_end = call;
1316 : }
1317 : else
1318 : {
1319 32978 : int count = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
1320 :
1321 32978 : if (!marker)
1322 : {
1323 19268 : if (kind == IFN_UNIQUE_OACC_HEAD_MARK)
1324 9634 : loop = new_oacc_loop (loop, call);
1325 : remaining = count;
1326 : }
1327 32978 : gcc_assert (count == remaining);
1328 32978 : if (remaining)
1329 : {
1330 32978 : remaining--;
1331 32978 : if (kind == IFN_UNIQUE_OACC_HEAD_MARK)
1332 16489 : loop->heads[marker] = call;
1333 : else
1334 16489 : loop->tails[remaining] = call;
1335 : }
1336 32978 : marker++;
1337 : }
1338 : }
1339 : }
1340 : }
1341 202126 : if (remaining || marker)
1342 : {
1343 66185 : bb = single_succ (bb);
1344 66185 : gcc_assert (single_pred_p (bb) && !(bb->flags & BB_VISITED));
1345 66185 : goto follow;
1346 : }
1347 :
1348 : /* Walk successor blocks. */
1349 135941 : edge e;
1350 135941 : edge_iterator ei;
1351 :
1352 302064 : FOR_EACH_EDGE (e, ei, bb->succs)
1353 166123 : oacc_loop_discover_walk (loop, e->dest);
1354 : }
1355 :
1356 : /* LOOP is the first sibling. Reverse the order in place and return
1357 : the new first sibling. Recurse to child loops. */
1358 :
1359 : static oacc_loop *
1360 18520 : oacc_loop_sibling_nreverse (oacc_loop *loop)
1361 : {
1362 18520 : oacc_loop *last = NULL;
1363 20714 : do
1364 : {
1365 20714 : if (loop->child)
1366 8641 : loop->child = oacc_loop_sibling_nreverse (loop->child);
1367 :
1368 20714 : oacc_loop *next = loop->sibling;
1369 20714 : loop->sibling = last;
1370 20714 : last = loop;
1371 20714 : loop = next;
1372 : }
1373 20714 : while (loop);
1374 :
1375 18520 : return last;
1376 : }
1377 :
1378 : /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
1379 : the current function. */
1380 :
1381 : static oacc_loop *
1382 9879 : oacc_loop_discovery ()
1383 : {
1384 : /* Clear basic block flags, in particular BB_VISITED which we're going to use
1385 : in the following. */
1386 9879 : clear_bb_flags ();
1387 :
1388 9879 : oacc_loop *top = new_oacc_loop_outer (current_function_decl);
1389 9879 : oacc_loop_discover_walk (top, ENTRY_BLOCK_PTR_FOR_FN (cfun));
1390 :
1391 : /* The siblings were constructed in reverse order, reverse them so
1392 : that diagnostics come out in an unsurprising order. */
1393 9879 : top = oacc_loop_sibling_nreverse (top);
1394 :
1395 9879 : return top;
1396 : }
1397 :
1398 : /* Transform the abstract internal function markers starting at FROM
1399 : to be for partitioning level LEVEL. Stop when we meet another HEAD
1400 : or TAIL marker. */
1401 :
1402 : static void
1403 25614 : oacc_loop_xform_head_tail (gcall *from, int level)
1404 : {
1405 25614 : enum ifn_unique_kind kind
1406 25614 : = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (from, 0));
1407 25614 : tree replacement = build_int_cst (unsigned_type_node, level);
1408 :
1409 25614 : for (gimple_stmt_iterator gsi = gsi_for_stmt (from);;)
1410 : {
1411 107435 : gimple *stmt = gsi_stmt (gsi);
1412 :
1413 107435 : if (gimple_call_internal_p (stmt, IFN_UNIQUE))
1414 : {
1415 77068 : enum ifn_unique_kind k
1416 : = ((enum ifn_unique_kind)
1417 77068 : TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
1418 :
1419 77068 : if (k == IFN_UNIQUE_OACC_FORK
1420 77068 : || k == IFN_UNIQUE_OACC_JOIN
1421 77068 : || k == IFN_UNIQUE_OACC_PRIVATE)
1422 25840 : *gimple_call_arg_ptr (stmt, 2) = replacement;
1423 51228 : else if (k == kind && stmt != from)
1424 : break;
1425 : }
1426 30367 : else if (gimple_call_internal_p (stmt, IFN_GOACC_REDUCTION))
1427 19228 : *gimple_call_arg_ptr (stmt, 3) = replacement;
1428 81821 : update_stmt (stmt);
1429 :
1430 81821 : gsi_next (&gsi);
1431 133275 : while (gsi_end_p (gsi))
1432 102908 : gsi = gsi_start_bb (single_succ (gsi_bb (gsi)));
1433 : }
1434 25614 : }
1435 :
1436 : /* Process the discovered OpenACC loops, setting the correct
1437 : partitioning level etc. */
1438 :
1439 : static void
1440 20714 : oacc_loop_process (oacc_loop *loop, int fn_level)
1441 : {
1442 20714 : if (loop->child)
1443 8641 : oacc_loop_process (loop->child, fn_level);
1444 :
1445 20714 : if (loop->mask && !loop->routine)
1446 : {
1447 8645 : int ix;
1448 8645 : tree mask_arg = build_int_cst (unsigned_type_node, loop->mask);
1449 8645 : tree e_mask_arg = build_int_cst (unsigned_type_node, loop->e_mask);
1450 8645 : tree chunk_arg = loop->chunk_size;
1451 8645 : gcall *call;
1452 :
1453 43708 : for (ix = 0; loop->ifns.iterate (ix, &call); ix++)
1454 : {
1455 35063 : switch (gimple_call_internal_fn (call))
1456 : {
1457 34857 : case IFN_GOACC_LOOP:
1458 34857 : {
1459 34857 : bool is_e = gimple_call_arg (call, 5) == integer_minus_one_node;
1460 69335 : gimple_call_set_arg (call, 5, is_e ? e_mask_arg : mask_arg);
1461 34857 : if (!is_e)
1462 34478 : gimple_call_set_arg (call, 4, chunk_arg);
1463 : }
1464 : break;
1465 :
1466 206 : case IFN_GOACC_TILE:
1467 206 : gimple_call_set_arg (call, 3, mask_arg);
1468 206 : gimple_call_set_arg (call, 4, e_mask_arg);
1469 206 : break;
1470 :
1471 0 : default:
1472 0 : gcc_unreachable ();
1473 : }
1474 35063 : update_stmt (call);
1475 : }
1476 :
1477 8645 : unsigned dim = GOMP_DIM_GANG;
1478 8645 : unsigned mask = loop->mask | loop->e_mask;
1479 21452 : for (ix = 0; ix != GOMP_DIM_MAX && mask; ix++)
1480 : {
1481 25113 : while (!(GOMP_DIM_MASK (dim) & mask))
1482 12306 : dim++;
1483 :
1484 12807 : oacc_loop_xform_head_tail (loop->heads[ix], dim);
1485 12807 : oacc_loop_xform_head_tail (loop->tails[ix], dim);
1486 :
1487 12807 : mask ^= GOMP_DIM_MASK (dim);
1488 : }
1489 : }
1490 :
1491 20714 : if (loop->sibling)
1492 2194 : oacc_loop_process (loop->sibling, fn_level);
1493 :
1494 :
1495 : /* OpenACC 2.6, 2.9.11. "reduction clause" places a restriction such that
1496 : "The 'reduction' clause may not be specified on an orphaned 'loop'
1497 : construct with the 'gang' clause, or on an orphaned 'loop' construct that
1498 : will generate gang parallelism in a procedure that is compiled with the
1499 : 'routine gang' clause." */
1500 20714 : if (fn_level == GOMP_DIM_GANG
1501 624 : && (loop->mask & GOMP_DIM_MASK (GOMP_DIM_GANG))
1502 209 : && (loop->flags & OLF_REDUCTION))
1503 106 : error_at (loop->loc,
1504 : "gang reduction on an orphan loop");
1505 20714 : }
1506 :
1507 : /* Walk the OpenACC loop hierarchy checking and assigning the
1508 : programmer-specified partitionings. OUTER_MASK is the partitioning
1509 : this loop is contained within. Return mask of partitioning
1510 : encountered. If any auto loops are discovered, set GOMP_DIM_MAX
1511 : bit. */
1512 :
1513 : static unsigned
1514 20714 : oacc_loop_fixed_partitions (oacc_loop *loop, unsigned outer_mask)
1515 : {
1516 20714 : unsigned this_mask = loop->mask;
1517 20714 : unsigned mask_all = 0;
1518 20714 : bool noisy = true;
1519 :
1520 : #ifdef ACCEL_COMPILER
1521 : /* When device_type is supported, we want the device compiler to be
1522 : noisy, if the loop parameters are device_type-specific. */
1523 : noisy = false;
1524 : #endif
1525 :
1526 20714 : if (!loop->routine)
1527 : {
1528 19513 : bool auto_par = (loop->flags & OLF_AUTO) != 0;
1529 19513 : bool seq_par = (loop->flags & OLF_SEQ) != 0;
1530 19513 : bool tiling = (loop->flags & OLF_TILE) != 0;
1531 :
1532 19513 : this_mask = ((loop->flags >> OLF_DIM_BASE)
1533 : & (GOMP_DIM_MASK (GOMP_DIM_MAX) - 1));
1534 :
1535 : /* Apply auto partitioning if this is a non-partitioned regular
1536 : loop, or (no more than) single axis tiled loop. */
1537 39026 : bool maybe_auto
1538 19513 : = !seq_par && this_mask == (tiling ? this_mask & -this_mask : 0);
1539 :
1540 19513 : if ((this_mask != 0) + auto_par + seq_par > 1)
1541 : {
1542 170 : if (noisy)
1543 250 : error_at (loop->loc,
1544 : seq_par
1545 : ? G_("%<seq%> overrides other OpenACC loop specifiers")
1546 : : G_("%<auto%> conflicts with other OpenACC loop "
1547 : "specifiers"));
1548 170 : maybe_auto = false;
1549 170 : loop->flags &= ~OLF_AUTO;
1550 170 : if (seq_par)
1551 : {
1552 90 : loop->flags
1553 90 : &= ~((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1) << OLF_DIM_BASE);
1554 90 : this_mask = 0;
1555 : }
1556 : }
1557 :
1558 19433 : if (maybe_auto && (loop->flags & OLF_INDEPENDENT))
1559 : {
1560 5699 : loop->flags |= OLF_AUTO;
1561 5699 : mask_all |= GOMP_DIM_MASK (GOMP_DIM_MAX);
1562 : }
1563 : }
1564 :
1565 20714 : if (this_mask & outer_mask)
1566 : {
1567 248 : const oacc_loop *outer;
1568 350 : for (outer = loop->parent; outer; outer = outer->parent)
1569 248 : if ((outer->mask | outer->e_mask) & this_mask)
1570 : break;
1571 :
1572 248 : if (noisy)
1573 : {
1574 248 : if (outer)
1575 : {
1576 146 : error_at (loop->loc,
1577 146 : loop->routine
1578 : ? G_("routine call uses same OpenACC parallelism"
1579 : " as containing loop")
1580 : : G_("inner loop uses same OpenACC parallelism"
1581 : " as containing loop"));
1582 146 : inform (outer->loc, "containing loop here");
1583 : }
1584 : else
1585 102 : error_at (loop->loc,
1586 102 : loop->routine
1587 : ? G_("routine call uses OpenACC parallelism disallowed"
1588 : " by containing routine")
1589 : : G_("loop uses OpenACC parallelism disallowed"
1590 : " by containing routine"));
1591 :
1592 248 : if (loop->routine)
1593 154 : inform (DECL_SOURCE_LOCATION (loop->routine),
1594 : "routine %qD declared here", loop->routine);
1595 : }
1596 248 : this_mask &= ~outer_mask;
1597 : }
1598 : else
1599 : {
1600 20466 : unsigned outermost = least_bit_hwi (this_mask);
1601 :
1602 20466 : if (outermost && outermost <= outer_mask)
1603 : {
1604 40 : if (noisy)
1605 : {
1606 40 : error_at (loop->loc,
1607 : "incorrectly nested OpenACC loop parallelism");
1608 :
1609 40 : const oacc_loop *outer;
1610 40 : for (outer = loop->parent;
1611 40 : outer->flags && outer->flags < outermost;
1612 0 : outer = outer->parent)
1613 0 : continue;
1614 40 : inform (outer->loc, "containing loop here");
1615 0 : }
1616 :
1617 40 : this_mask &= ~outermost;
1618 : }
1619 : }
1620 :
1621 20714 : mask_all |= this_mask;
1622 :
1623 20714 : if (loop->flags & OLF_TILE)
1624 : {
1625 : /* When tiling, vector goes to the element loop, and failing
1626 : that we put worker there. The std doesn't contemplate
1627 : specifying all three. We choose to put worker and vector on
1628 : the element loops in that case. */
1629 136 : unsigned this_e_mask = this_mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR);
1630 136 : if (!this_e_mask || this_mask & GOMP_DIM_MASK (GOMP_DIM_GANG))
1631 120 : this_e_mask |= this_mask & GOMP_DIM_MASK (GOMP_DIM_WORKER);
1632 :
1633 136 : loop->e_mask = this_e_mask;
1634 136 : this_mask ^= this_e_mask;
1635 : }
1636 :
1637 20714 : loop->mask = this_mask;
1638 :
1639 20714 : if (dump_file)
1640 222 : fprintf (dump_file, "Loop %s:%d user specified %d & %d\n",
1641 444 : LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc),
1642 : loop->mask, loop->e_mask);
1643 :
1644 20714 : if (loop->child)
1645 : {
1646 8641 : unsigned tmp_mask = outer_mask | this_mask | loop->e_mask;
1647 8641 : loop->inner = oacc_loop_fixed_partitions (loop->child, tmp_mask);
1648 8641 : mask_all |= loop->inner;
1649 : }
1650 :
1651 20714 : if (loop->sibling)
1652 2194 : mask_all |= oacc_loop_fixed_partitions (loop->sibling, outer_mask);
1653 :
1654 20714 : return mask_all;
1655 : }
1656 :
1657 : /* Walk the OpenACC loop hierarchy to assign auto-partitioned loops.
1658 : OUTER_MASK is the partitioning this loop is contained within.
1659 : OUTER_ASSIGN is true if an outer loop is being auto-partitioned.
1660 : Return the cumulative partitioning used by this loop, siblings and
1661 : children. */
1662 :
1663 : static unsigned
1664 10192 : oacc_loop_auto_partitions (oacc_loop *loop, unsigned outer_mask,
1665 : bool outer_assign)
1666 : {
1667 10192 : bool assign = (loop->flags & OLF_AUTO) && (loop->flags & OLF_INDEPENDENT);
1668 10192 : bool noisy = true;
1669 10192 : bool tiling = loop->flags & OLF_TILE;
1670 :
1671 : #ifdef ACCEL_COMPILER
1672 : /* When device_type is supported, we want the device compiler to be
1673 : noisy, if the loop parameters are device_type-specific. */
1674 : noisy = false;
1675 : #endif
1676 :
1677 10192 : if (assign && (!outer_assign || loop->inner))
1678 : {
1679 : /* Allocate outermost and non-innermost loops at the outermost
1680 : non-innermost available level. */
1681 : unsigned this_mask = GOMP_DIM_MASK (GOMP_DIM_GANG);
1682 :
1683 : /* Find the first outermost available partition. */
1684 6659 : while (this_mask <= outer_mask)
1685 1923 : this_mask <<= 1;
1686 :
1687 : /* Grab two axes if tiling, and we've not assigned anything */
1688 4736 : if (tiling && !(loop->mask | loop->e_mask))
1689 94 : this_mask |= this_mask << 1;
1690 :
1691 : /* Prohibit the innermost partitioning at the moment. */
1692 4736 : this_mask &= GOMP_DIM_MASK (GOMP_DIM_MAX - 1) - 1;
1693 :
1694 : /* Don't use any dimension explicitly claimed by an inner loop. */
1695 4736 : this_mask &= ~loop->inner;
1696 :
1697 4736 : if (tiling && !loop->e_mask)
1698 : {
1699 : /* If we got two axes, allocate the inner one to the element
1700 : loop. */
1701 98 : loop->e_mask = this_mask & (this_mask << 1);
1702 98 : this_mask ^= loop->e_mask;
1703 : }
1704 :
1705 4736 : loop->mask |= this_mask;
1706 : }
1707 :
1708 10192 : if (loop->child)
1709 : {
1710 5360 : unsigned tmp_mask = outer_mask | loop->mask | loop->e_mask;
1711 5360 : loop->inner = oacc_loop_auto_partitions (loop->child, tmp_mask,
1712 5360 : outer_assign | assign);
1713 : }
1714 :
1715 10192 : if (assign && (!loop->mask || (tiling && !loop->e_mask) || !outer_assign))
1716 : {
1717 : /* Allocate the loop at the innermost available level. Note
1718 : that we do this even if we already assigned this loop the
1719 : outermost available level above. That way we'll partition
1720 : this along 2 axes, if they are available. */
1721 5044 : unsigned this_mask = 0;
1722 :
1723 : /* Determine the outermost partitioning used within this loop. */
1724 5044 : this_mask = loop->inner | GOMP_DIM_MASK (GOMP_DIM_MAX);
1725 5044 : this_mask = least_bit_hwi (this_mask);
1726 :
1727 : /* Pick the partitioning just inside that one. */
1728 5044 : this_mask >>= 1;
1729 :
1730 : /* And avoid picking one use by an outer loop. */
1731 5044 : this_mask &= ~outer_mask;
1732 :
1733 : /* If tiling and we failed completely above, grab the next one
1734 : too. Making sure it doesn't hit an outer loop. */
1735 5044 : if (tiling)
1736 : {
1737 110 : this_mask &= ~(loop->e_mask | loop->mask);
1738 110 : unsigned tile_mask = ((this_mask >> 1)
1739 110 : & ~(outer_mask | loop->e_mask | loop->mask));
1740 :
1741 110 : if (tile_mask || loop->mask)
1742 : {
1743 102 : loop->e_mask |= this_mask;
1744 102 : this_mask = tile_mask;
1745 : }
1746 110 : if (!loop->e_mask && noisy)
1747 8 : warning_at (loop->loc, 0,
1748 : "insufficient partitioning available"
1749 : " to parallelize element loop");
1750 : }
1751 :
1752 5044 : loop->mask |= this_mask;
1753 5044 : if (!loop->mask && noisy)
1754 1078 : warning_at (loop->loc, 0,
1755 : tiling
1756 : ? G_("insufficient partitioning available"
1757 : " to parallelize tile loop")
1758 : : G_("insufficient partitioning available"
1759 : " to parallelize loop"));
1760 : }
1761 :
1762 5699 : if (assign && dump_file)
1763 41 : fprintf (dump_file, "Auto loop %s:%d assigned %d & %d\n",
1764 82 : LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc),
1765 : loop->mask, loop->e_mask);
1766 :
1767 10192 : unsigned inner_mask = 0;
1768 :
1769 10192 : if (loop->sibling)
1770 1714 : inner_mask |= oacc_loop_auto_partitions (loop->sibling,
1771 : outer_mask, outer_assign);
1772 :
1773 10192 : inner_mask |= loop->inner | loop->mask | loop->e_mask;
1774 :
1775 10192 : return inner_mask;
1776 : }
1777 :
1778 : /* Walk the OpenACC loop hierarchy to check and assign partitioning
1779 : axes. Return mask of partitioning. */
1780 :
1781 : static unsigned
1782 9879 : oacc_loop_partition (oacc_loop *loop, unsigned outer_mask)
1783 : {
1784 9879 : unsigned mask_all = oacc_loop_fixed_partitions (loop, outer_mask);
1785 :
1786 9879 : if (mask_all & GOMP_DIM_MASK (GOMP_DIM_MAX))
1787 : {
1788 3118 : mask_all ^= GOMP_DIM_MASK (GOMP_DIM_MAX);
1789 3118 : mask_all |= oacc_loop_auto_partitions (loop, outer_mask, false);
1790 : }
1791 9879 : return mask_all;
1792 : }
1793 :
1794 : /* Default fork/join early expander. Delete the function calls if
1795 : there is no RTL expander. */
1796 :
1797 : bool
1798 25614 : default_goacc_fork_join (gcall *ARG_UNUSED (call),
1799 : const int *ARG_UNUSED (dims), bool is_fork)
1800 : {
1801 25614 : if (is_fork)
1802 12807 : return targetm.have_oacc_fork ();
1803 : else
1804 12807 : return targetm.have_oacc_join ();
1805 : }
1806 :
1807 : /* Default goacc.reduction early expander.
1808 :
1809 : LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
1810 : If RES_PTR is not integer-zerop:
1811 : SETUP - emit 'LHS = *RES_PTR', LHS = NULL
1812 : TEARDOWN - emit '*RES_PTR = VAR'
1813 : If LHS is not NULL
1814 : emit 'LHS = VAR' */
1815 :
1816 : void
1817 30884 : default_goacc_reduction (gcall *call)
1818 : {
1819 30884 : unsigned code = (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call, 0));
1820 30884 : gimple_stmt_iterator gsi = gsi_for_stmt (call);
1821 30884 : tree lhs = gimple_call_lhs (call);
1822 30884 : tree var = gimple_call_arg (call, 2);
1823 30884 : gimple_seq seq = NULL;
1824 :
1825 30884 : if (code == IFN_GOACC_REDUCTION_SETUP
1826 30884 : || code == IFN_GOACC_REDUCTION_TEARDOWN)
1827 : {
1828 : /* Setup and Teardown need to copy from/to the receiver object,
1829 : if there is one. */
1830 15442 : tree ref_to_res = gimple_call_arg (call, 1);
1831 :
1832 15442 : if (!integer_zerop (ref_to_res))
1833 : {
1834 5086 : tree dst = build_simple_mem_ref (ref_to_res);
1835 5086 : tree src = var;
1836 :
1837 5086 : if (code == IFN_GOACC_REDUCTION_SETUP)
1838 : {
1839 2543 : src = dst;
1840 2543 : dst = lhs;
1841 2543 : lhs = NULL;
1842 : }
1843 5086 : gimple_seq_add_stmt (&seq, gimple_build_assign (dst, src));
1844 : }
1845 : }
1846 :
1847 : /* Copy VAR to LHS, if there is an LHS. */
1848 30884 : if (lhs)
1849 26734 : gimple_seq_add_stmt (&seq, gimple_build_assign (lhs, var));
1850 :
1851 30884 : gsi_replace_with_seq (&gsi, seq, true);
1852 30884 : }
1853 :
1854 : struct var_decl_rewrite_info
1855 : {
1856 : gimple *stmt;
1857 : hash_map<tree, tree> *adjusted_vars;
1858 : bool avoid_pointer_conversion;
1859 : bool modified;
1860 : };
1861 :
1862 : /* Helper function for execute_oacc_device_lower. Rewrite VAR_DECLs (by
1863 : themselves or wrapped in various other nodes) according to ADJUSTED_VARS in
1864 : the var_decl_rewrite_info pointed to via DATA. Used as part of coercing
1865 : gang-private variables in OpenACC offload regions to reside in GPU shared
1866 : memory. */
1867 :
1868 : static tree
1869 0 : oacc_rewrite_var_decl (tree *tp, int *walk_subtrees, void *data)
1870 : {
1871 0 : walk_stmt_info *wi = (walk_stmt_info *) data;
1872 0 : var_decl_rewrite_info *info = (var_decl_rewrite_info *) wi->info;
1873 :
1874 0 : if (TREE_CODE (*tp) == ADDR_EXPR)
1875 : {
1876 0 : tree arg = TREE_OPERAND (*tp, 0);
1877 0 : tree *new_arg = info->adjusted_vars->get (arg);
1878 :
1879 0 : if (new_arg)
1880 : {
1881 0 : if (info->avoid_pointer_conversion)
1882 : {
1883 0 : *tp = build_fold_addr_expr (*new_arg);
1884 0 : info->modified = true;
1885 0 : *walk_subtrees = 0;
1886 : }
1887 : else
1888 : {
1889 0 : gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
1890 0 : tree repl = build_fold_addr_expr (*new_arg);
1891 0 : gimple *stmt1
1892 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
1893 0 : tree conv = convert_to_pointer (TREE_TYPE (*tp),
1894 : gimple_assign_lhs (stmt1));
1895 0 : gimple *stmt2
1896 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (*tp)), conv);
1897 0 : gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
1898 0 : gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
1899 0 : *tp = gimple_assign_lhs (stmt2);
1900 0 : info->modified = true;
1901 0 : *walk_subtrees = 0;
1902 : }
1903 : }
1904 : }
1905 0 : else if (TREE_CODE (*tp) == COMPONENT_REF || TREE_CODE (*tp) == ARRAY_REF)
1906 : {
1907 0 : tree *base = &TREE_OPERAND (*tp, 0);
1908 :
1909 0 : while (TREE_CODE (*base) == COMPONENT_REF
1910 0 : || TREE_CODE (*base) == ARRAY_REF)
1911 0 : base = &TREE_OPERAND (*base, 0);
1912 :
1913 0 : if (TREE_CODE (*base) != VAR_DECL)
1914 : return NULL;
1915 :
1916 0 : tree *new_decl = info->adjusted_vars->get (*base);
1917 0 : if (!new_decl)
1918 : return NULL;
1919 :
1920 0 : int base_quals = TYPE_QUALS (TREE_TYPE (*new_decl));
1921 0 : tree field = TREE_OPERAND (*tp, 1);
1922 :
1923 : /* Adjust the type of the field. */
1924 0 : int field_quals = TYPE_QUALS (TREE_TYPE (field));
1925 0 : if (TREE_CODE (field) == FIELD_DECL && field_quals != base_quals)
1926 : {
1927 0 : tree *field_type = &TREE_TYPE (field);
1928 0 : while (TREE_CODE (*field_type) == ARRAY_TYPE)
1929 0 : field_type = &TREE_TYPE (*field_type);
1930 0 : field_quals |= base_quals;
1931 0 : *field_type = build_qualified_type (*field_type, field_quals);
1932 : }
1933 :
1934 : /* Adjust the type of the component ref itself. */
1935 0 : tree comp_type = TREE_TYPE (*tp);
1936 0 : int comp_quals = TYPE_QUALS (comp_type);
1937 0 : if (TREE_CODE (*tp) == COMPONENT_REF && comp_quals != base_quals)
1938 : {
1939 0 : comp_quals |= base_quals;
1940 0 : TREE_TYPE (*tp)
1941 0 : = build_qualified_type (comp_type, comp_quals);
1942 : }
1943 :
1944 0 : *base = *new_decl;
1945 0 : info->modified = true;
1946 0 : }
1947 0 : else if (VAR_P (*tp))
1948 : {
1949 0 : tree *new_decl = info->adjusted_vars->get (*tp);
1950 0 : if (new_decl)
1951 : {
1952 0 : *tp = *new_decl;
1953 0 : info->modified = true;
1954 : }
1955 : }
1956 :
1957 : return NULL_TREE;
1958 : }
1959 :
1960 : /* Return TRUE if CALL is a call to a builtin atomic/sync operation. */
1961 :
1962 : static bool
1963 0 : is_sync_builtin_call (gcall *call)
1964 : {
1965 0 : tree callee = gimple_call_fndecl (call);
1966 :
1967 0 : if (callee != NULL_TREE
1968 0 : && gimple_call_builtin_p (call, BUILT_IN_NORMAL))
1969 0 : switch (DECL_FUNCTION_CODE (callee))
1970 : {
1971 : #undef DEF_SYNC_BUILTIN
1972 : #define DEF_SYNC_BUILTIN(ENUM, NAME, TYPE, ATTRS) case ENUM:
1973 : #include "sync-builtins.def"
1974 : #undef DEF_SYNC_BUILTIN
1975 0 : return true;
1976 :
1977 : default:
1978 : ;
1979 : }
1980 :
1981 : return false;
1982 : }
1983 :
1984 : /* Main entry point for oacc transformations which run on the device
1985 : compiler after LTO, so we know what the target device is at this
1986 : point (including the host fallback). */
1987 :
1988 : static unsigned int
1989 15343 : execute_oacc_loop_designation ()
1990 : {
1991 15343 : tree attrs = oacc_get_fn_attrib (current_function_decl);
1992 :
1993 15343 : if (!attrs)
1994 : /* Not an offloaded function. */
1995 : return 0;
1996 :
1997 : /* Parse the default dim argument exactly once. */
1998 9941 : if ((const void *)flag_openacc_dims != &flag_openacc_dims)
1999 : {
2000 2281 : oacc_parse_default_dims (flag_openacc_dims);
2001 2281 : flag_openacc_dims = (char *)&flag_openacc_dims;
2002 : }
2003 :
2004 9941 : bool is_oacc_parallel
2005 9941 : = (lookup_attribute ("oacc parallel",
2006 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2007 9941 : bool is_oacc_kernels
2008 9941 : = (lookup_attribute ("oacc kernels",
2009 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2010 9941 : bool is_oacc_serial
2011 9941 : = (lookup_attribute ("oacc serial",
2012 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2013 9941 : bool is_oacc_parallel_kernels_parallelized
2014 9941 : = (lookup_attribute ("oacc parallel_kernels_parallelized",
2015 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2016 9941 : bool is_oacc_parallel_kernels_gang_single
2017 9941 : = (lookup_attribute ("oacc parallel_kernels_gang_single",
2018 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2019 9941 : int fn_level = oacc_fn_attrib_level (attrs);
2020 9941 : bool is_oacc_routine = (fn_level >= 0);
2021 9941 : gcc_checking_assert (is_oacc_parallel
2022 : + is_oacc_kernels
2023 : + is_oacc_serial
2024 : + is_oacc_parallel_kernels_parallelized
2025 : + is_oacc_parallel_kernels_gang_single
2026 : + is_oacc_routine
2027 : == 1);
2028 :
2029 9941 : bool is_oacc_kernels_parallelized
2030 9941 : = (lookup_attribute ("oacc kernels parallelized",
2031 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2032 9941 : if (is_oacc_kernels_parallelized)
2033 386 : gcc_checking_assert (is_oacc_kernels);
2034 :
2035 9941 : if (dump_file)
2036 : {
2037 154 : if (is_oacc_parallel)
2038 34 : fprintf (dump_file, "Function is OpenACC parallel offload\n");
2039 120 : else if (is_oacc_kernels)
2040 66 : fprintf (dump_file, "Function is %s OpenACC kernels offload\n",
2041 : (is_oacc_kernels_parallelized
2042 : ? "parallelized" : "unparallelized"));
2043 82 : else if (is_oacc_serial)
2044 26 : fprintf (dump_file, "Function is OpenACC serial offload\n");
2045 56 : else if (is_oacc_parallel_kernels_parallelized)
2046 0 : fprintf (dump_file, "Function is %s OpenACC kernels offload\n",
2047 : "parallel_kernels_parallelized");
2048 56 : else if (is_oacc_parallel_kernels_gang_single)
2049 0 : fprintf (dump_file, "Function is %s OpenACC kernels offload\n",
2050 : "parallel_kernels_gang_single");
2051 56 : else if (is_oacc_routine)
2052 56 : fprintf (dump_file, "Function is OpenACC routine level %d\n",
2053 : fn_level);
2054 : else
2055 0 : gcc_unreachable ();
2056 : }
2057 :
2058 : /* This doesn't belong into 'pass_oacc_loop_designation' conceptually, but
2059 : it's a convenient place, so... */
2060 9941 : if (is_oacc_routine)
2061 : {
2062 553 : tree attr = lookup_attribute ("omp declare target",
2063 553 : DECL_ATTRIBUTES (current_function_decl));
2064 553 : gcc_checking_assert (attr);
2065 553 : tree clauses = TREE_VALUE (attr);
2066 553 : gcc_checking_assert (clauses);
2067 :
2068 : /* Should this OpenACC routine be discarded? */
2069 553 : bool discard = false;
2070 :
2071 553 : tree clause_nohost = omp_find_clause (clauses, OMP_CLAUSE_NOHOST);
2072 553 : if (dump_file)
2073 56 : fprintf (dump_file,
2074 : "OpenACC routine '%s' %s '%s' clause.\n",
2075 56 : lang_hooks.decl_printable_name (current_function_decl, 2),
2076 : clause_nohost ? "has" : "doesn't have",
2077 56 : omp_clause_code_name[OMP_CLAUSE_NOHOST]);
2078 : /* Host compiler, 'nohost' clause? */
2079 : #ifndef ACCEL_COMPILER
2080 553 : if (clause_nohost)
2081 62 : discard = true;
2082 : #endif
2083 :
2084 553 : if (dump_file)
2085 112 : fprintf (dump_file,
2086 : "OpenACC routine '%s' %sdiscarded.\n",
2087 56 : lang_hooks.decl_printable_name (current_function_decl, 2),
2088 : discard ? "" : "not ");
2089 553 : if (discard)
2090 : {
2091 62 : TREE_ASM_WRITTEN (current_function_decl) = 1;
2092 62 : return TODO_discard_function;
2093 : }
2094 : }
2095 :
2096 : /* Unparallelized OpenACC kernels constructs must get launched as 1 x 1 x 1
2097 : kernels, so remove the parallelism dimensions function attributes
2098 : potentially set earlier on. */
2099 9879 : if (is_oacc_kernels && !is_oacc_kernels_parallelized)
2100 : {
2101 1265 : oacc_set_fn_attrib (current_function_decl, NULL, NULL);
2102 1265 : attrs = oacc_get_fn_attrib (current_function_decl);
2103 : }
2104 :
2105 : /* Discover, partition and process the loops. */
2106 9879 : oacc_loop *loops = oacc_loop_discovery ();
2107 :
2108 9879 : unsigned outer_mask = 0;
2109 9879 : if (is_oacc_routine)
2110 491 : outer_mask = GOMP_DIM_MASK (fn_level) - 1;
2111 9879 : unsigned used_mask = oacc_loop_partition (loops, outer_mask);
2112 : /* OpenACC kernels constructs are special: they currently don't use the
2113 : generic oacc_loop infrastructure and attribute/dimension processing. */
2114 9879 : if (is_oacc_kernels && is_oacc_kernels_parallelized)
2115 : {
2116 : /* Parallelized OpenACC kernels constructs use gang parallelism. See
2117 : also tree-parloops.cc:create_parallel_loop. */
2118 386 : used_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
2119 : }
2120 :
2121 9879 : int dims[GOMP_DIM_MAX];
2122 9879 : oacc_validate_dims (current_function_decl, attrs, dims, fn_level, used_mask);
2123 :
2124 9879 : if (dump_file)
2125 : {
2126 : const char *comma = "Compute dimensions [";
2127 456 : for (int ix = 0; ix != GOMP_DIM_MAX; ix++, comma = ", ")
2128 342 : fprintf (dump_file, "%s%d", comma, dims[ix]);
2129 114 : fprintf (dump_file, "]\n");
2130 : }
2131 :
2132 : /* Verify that for OpenACC 'kernels' decomposed "gang-single" parts we launch
2133 : a single gang only. */
2134 9879 : if (is_oacc_parallel_kernels_gang_single)
2135 109 : gcc_checking_assert (dims[GOMP_DIM_GANG] == 1);
2136 :
2137 9879 : oacc_loop_process (loops, fn_level);
2138 9879 : if (dump_file)
2139 : {
2140 114 : fprintf (dump_file, "OpenACC loops\n");
2141 114 : dump_oacc_loop (dump_file, loops, 0);
2142 114 : fprintf (dump_file, "\n");
2143 : }
2144 9879 : if (dump_enabled_p ())
2145 : {
2146 2280 : oacc_loop *l = loops;
2147 : /* OpenACC kernels constructs are special: they currently don't use the
2148 : generic oacc_loop infrastructure. */
2149 2280 : if (is_oacc_kernels)
2150 : {
2151 : /* Create a fake oacc_loop for diagnostic purposes. */
2152 633 : l = new_oacc_loop_raw (NULL,
2153 633 : DECL_SOURCE_LOCATION (current_function_decl));
2154 633 : l->mask = used_mask;
2155 : }
2156 : else
2157 : {
2158 : /* Skip the outermost, dummy OpenACC loop */
2159 1647 : l = l->child;
2160 : }
2161 2280 : if (l)
2162 1763 : inform_oacc_loop (l);
2163 2280 : if (is_oacc_kernels)
2164 633 : free_oacc_loop (l);
2165 : }
2166 :
2167 9879 : free_oacc_loop (loops);
2168 :
2169 9879 : return 0;
2170 : }
2171 :
2172 : static unsigned int
2173 15281 : execute_oacc_device_lower ()
2174 : {
2175 15281 : tree attrs = oacc_get_fn_attrib (current_function_decl);
2176 :
2177 15281 : if (!attrs)
2178 : /* Not an offloaded function. */
2179 : return 0;
2180 :
2181 : int dims[GOMP_DIM_MAX];
2182 39516 : for (unsigned i = 0; i < GOMP_DIM_MAX; i++)
2183 29637 : dims[i] = oacc_get_fn_dim_size (current_function_decl, i);
2184 :
2185 9879 : hash_map<tree, tree> adjusted_vars;
2186 :
2187 : /* Now lower internal loop functions to target-specific code
2188 : sequences. */
2189 9879 : basic_block bb;
2190 185094 : FOR_ALL_BB_FN (bb, cfun)
2191 933543 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
2192 : {
2193 583113 : gimple *stmt = gsi_stmt (gsi);
2194 583113 : if (!is_gimple_call (stmt))
2195 : {
2196 383306 : gsi_next (&gsi);
2197 383306 : continue;
2198 : }
2199 :
2200 199807 : gcall *call = as_a <gcall *> (stmt);
2201 199807 : if (!gimple_call_internal_p (call))
2202 : {
2203 4763 : gsi_next (&gsi);
2204 4763 : continue;
2205 : }
2206 :
2207 : /* Rewind to allow rescan. */
2208 195044 : gsi_prev (&gsi);
2209 195044 : bool rescan = false, remove = false;
2210 195044 : enum internal_fn ifn_code = gimple_call_internal_fn (call);
2211 :
2212 195044 : switch (ifn_code)
2213 : {
2214 : default: break;
2215 :
2216 284 : case IFN_GOACC_TILE:
2217 284 : oacc_xform_tile (call);
2218 284 : rescan = true;
2219 284 : break;
2220 :
2221 46702 : case IFN_GOACC_LOOP:
2222 46702 : oacc_xform_loop (call);
2223 46702 : rescan = true;
2224 46702 : break;
2225 :
2226 30884 : case IFN_GOACC_REDUCTION:
2227 : /* Mark the function for SSA renaming. */
2228 30884 : mark_virtual_operands_for_renaming (cfun);
2229 :
2230 : /* If the level is -1, this ended up being an unused
2231 : axis. Handle as a default. */
2232 30884 : if (integer_minus_onep (gimple_call_arg (call, 3)))
2233 8528 : default_goacc_reduction (call);
2234 : else
2235 22356 : targetm.goacc.reduction (call);
2236 : rescan = true;
2237 : break;
2238 :
2239 85483 : case IFN_UNIQUE:
2240 85483 : {
2241 85483 : enum ifn_unique_kind kind
2242 : = ((enum ifn_unique_kind)
2243 85483 : TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
2244 :
2245 85483 : switch (kind)
2246 : {
2247 : default:
2248 : break;
2249 :
2250 32978 : case IFN_UNIQUE_OACC_FORK:
2251 32978 : case IFN_UNIQUE_OACC_JOIN:
2252 32978 : if (integer_minus_onep (gimple_call_arg (call, 2)))
2253 : remove = true;
2254 25614 : else if (!targetm.goacc.fork_join
2255 25614 : (call, dims, kind == IFN_UNIQUE_OACC_FORK))
2256 85483 : remove = true;
2257 : break;
2258 :
2259 : case IFN_UNIQUE_OACC_HEAD_MARK:
2260 : case IFN_UNIQUE_OACC_TAIL_MARK:
2261 85483 : remove = true;
2262 : break;
2263 :
2264 259 : case IFN_UNIQUE_OACC_PRIVATE:
2265 259 : {
2266 259 : dump_flags_t l_dump_flags
2267 259 : = get_openacc_privatization_dump_flags ();
2268 :
2269 259 : location_t loc = gimple_location (stmt);
2270 259 : if (LOCATION_LOCUS (loc) == UNKNOWN_LOCATION)
2271 30 : loc = DECL_SOURCE_LOCATION (current_function_decl);
2272 259 : const dump_user_location_t d_u_loc
2273 259 : = dump_user_location_t::from_location_t (loc);
2274 :
2275 259 : HOST_WIDE_INT level
2276 259 : = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
2277 259 : gcc_checking_assert (level == -1
2278 : || (level >= 0
2279 : && level < GOMP_DIM_MAX));
2280 339 : for (unsigned i = 3;
2281 598 : i < gimple_call_num_args (call);
2282 : i++)
2283 : {
2284 339 : static char const *const axes[] =
2285 : /* Must be kept in sync with GOMP_DIM enumeration. */
2286 : { "gang", "worker", "vector" };
2287 :
2288 339 : tree arg = gimple_call_arg (call, i);
2289 339 : gcc_checking_assert (TREE_CODE (arg) == ADDR_EXPR);
2290 339 : tree decl = TREE_OPERAND (arg, 0);
2291 339 : if (dump_enabled_p ())
2292 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
2293 : #if __GNUC__ >= 10
2294 318 : # pragma GCC diagnostic push
2295 318 : # pragma GCC diagnostic ignored "-Wformat"
2296 : #endif
2297 318 : dump_printf_loc (l_dump_flags, d_u_loc,
2298 : "variable %<%T%> ought to be"
2299 : " adjusted for OpenACC"
2300 : " privatization level: %qs\n",
2301 : decl,
2302 : (level == -1
2303 : ? "UNKNOWN" : axes[level]));
2304 : #if __GNUC__ >= 10
2305 339 : # pragma GCC diagnostic pop
2306 : #endif
2307 339 : bool adjusted;
2308 339 : if (level == -1)
2309 : adjusted = false;
2310 336 : else if (!targetm.goacc.adjust_private_decl)
2311 : adjusted = false;
2312 0 : else if (level == GOMP_DIM_VECTOR)
2313 : {
2314 : /* That's the default behavior. */
2315 : adjusted = true;
2316 : }
2317 : else
2318 : {
2319 0 : tree oldtype = TREE_TYPE (decl);
2320 0 : tree newdecl
2321 0 : = targetm.goacc.adjust_private_decl (loc, decl,
2322 0 : level);
2323 0 : adjusted = (TREE_TYPE (newdecl) != oldtype
2324 0 : || newdecl != decl);
2325 0 : if (adjusted)
2326 0 : adjusted_vars.put (decl, newdecl);
2327 : }
2328 0 : if (adjusted
2329 0 : && dump_enabled_p ())
2330 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
2331 : #if __GNUC__ >= 10
2332 0 : # pragma GCC diagnostic push
2333 0 : # pragma GCC diagnostic ignored "-Wformat"
2334 : #endif
2335 0 : dump_printf_loc (l_dump_flags, d_u_loc,
2336 : "variable %<%T%> adjusted for"
2337 : " OpenACC privatization level:"
2338 : " %qs\n",
2339 0 : decl, axes[level]);
2340 : #if __GNUC__ >= 10
2341 339 : # pragma GCC diagnostic pop
2342 : #endif
2343 : }
2344 259 : remove = true;
2345 : }
2346 259 : break;
2347 : }
2348 : break;
2349 : }
2350 : }
2351 :
2352 195044 : if (gsi_end_p (gsi))
2353 : /* We rewound past the beginning of the BB. */
2354 188830 : gsi = gsi_start_bb (bb);
2355 : else
2356 : /* Undo the rewind. */
2357 100629 : gsi_next (&gsi);
2358 :
2359 195044 : if (remove)
2360 : {
2361 170966 : if (gimple_vdef (call))
2362 85483 : replace_uses_by (gimple_vdef (call), gimple_vuse (call));
2363 85483 : if (gimple_call_lhs (call))
2364 : {
2365 : /* Propagate the data dependency var. */
2366 79856 : gimple *ass = gimple_build_assign (gimple_call_lhs (call),
2367 : gimple_call_arg (call, 1));
2368 79856 : gsi_replace (&gsi, ass, false);
2369 : }
2370 : else
2371 5627 : gsi_remove (&gsi, true);
2372 : }
2373 109561 : else if (!rescan)
2374 : /* If not rescanning, advance over the call. */
2375 31691 : gsi_next (&gsi);
2376 : }
2377 :
2378 : /* Regarding the OpenACC privatization level, we're currently only looking at
2379 : making the gang-private level work. Regarding that, we have the following
2380 : configurations:
2381 :
2382 : - GCN offloading: 'targetm.goacc.adjust_private_decl' does the work (in
2383 : particular, change 'TREE_TYPE', etc.) and there is no
2384 : 'targetm.goacc.expand_var_decl'.
2385 :
2386 : - nvptx offloading: 'targetm.goacc.adjust_private_decl' only sets a
2387 : marker and then 'targetm.goacc.expand_var_decl' does the work.
2388 :
2389 : Eventually (in particular, for worker-private level?), both
2390 : 'targetm.goacc.adjust_private_decl' and 'targetm.goacc.expand_var_decl'
2391 : may need to do things, but that's currently not meant to be addressed, and
2392 : thus not fully worked out and implemented, and thus untested. Hence,
2393 : 'assert' what currently is implemented/tested, only. */
2394 :
2395 9879 : if (targetm.goacc.expand_var_decl)
2396 0 : gcc_assert (adjusted_vars.is_empty ());
2397 :
2398 : /* Make adjustments to gang-private local variables if required by the
2399 : target, e.g. forcing them into a particular address space. Afterwards,
2400 : ADDR_EXPR nodes which have adjusted variables as their argument need to
2401 : be modified in one of two ways:
2402 :
2403 : 1. They can be recreated, making a pointer to the variable in the new
2404 : address space, or
2405 :
2406 : 2. The address of the variable in the new address space can be taken,
2407 : converted to the default (original) address space, and the result of
2408 : that conversion substituted in place of the original ADDR_EXPR node.
2409 :
2410 : Which of these is done depends on the gimple statement being processed.
2411 : At present atomic operations and inline asms use (1), and everything else
2412 : uses (2). At least on AMD GCN, there are atomic operations that work
2413 : directly in the LDS address space.
2414 :
2415 : COMPONENT_REFS, ARRAY_REFS and plain VAR_DECLs are also rewritten to use
2416 : the new decl, adjusting types of appropriate tree nodes as necessary. */
2417 :
2418 9879 : if (targetm.goacc.adjust_private_decl
2419 9879 : && !adjusted_vars.is_empty ())
2420 : {
2421 0 : FOR_ALL_BB_FN (bb, cfun)
2422 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2423 0 : !gsi_end_p (gsi);
2424 0 : gsi_next (&gsi))
2425 : {
2426 0 : gimple *stmt = gsi_stmt (gsi);
2427 0 : walk_stmt_info wi;
2428 0 : var_decl_rewrite_info info;
2429 :
2430 0 : info.avoid_pointer_conversion
2431 0 : = (is_gimple_call (stmt)
2432 0 : && is_sync_builtin_call (as_a <gcall *> (stmt)))
2433 0 : || gimple_code (stmt) == GIMPLE_ASM;
2434 0 : info.stmt = stmt;
2435 0 : info.modified = false;
2436 0 : info.adjusted_vars = &adjusted_vars;
2437 :
2438 0 : memset (&wi, 0, sizeof (wi));
2439 0 : wi.info = &info;
2440 :
2441 0 : walk_gimple_op (stmt, oacc_rewrite_var_decl, &wi);
2442 :
2443 0 : if (info.modified)
2444 0 : update_stmt (stmt);
2445 : }
2446 : }
2447 :
2448 9879 : return 0;
2449 9879 : }
2450 :
2451 : /* Default launch dimension validator. Force everything to 1. A
2452 : backend that wants to provide larger dimensions must override this
2453 : hook. */
2454 :
2455 : bool
2456 14441 : default_goacc_validate_dims (tree ARG_UNUSED (decl), int *dims,
2457 : int ARG_UNUSED (fn_level),
2458 : unsigned ARG_UNUSED (used))
2459 : {
2460 14441 : bool changed = false;
2461 :
2462 57764 : for (unsigned ix = 0; ix != GOMP_DIM_MAX; ix++)
2463 : {
2464 43323 : if (dims[ix] != 1)
2465 : {
2466 33054 : dims[ix] = 1;
2467 33054 : changed = true;
2468 : }
2469 : }
2470 :
2471 14441 : return changed;
2472 : }
2473 :
2474 : /* Default dimension bound is unknown on accelerator and 1 on host. */
2475 :
2476 : int
2477 0 : default_goacc_dim_limit (int ARG_UNUSED (axis))
2478 : {
2479 : #ifdef ACCEL_COMPILER
2480 : return 0;
2481 : #else
2482 0 : return 1;
2483 : #endif
2484 : }
2485 :
2486 : namespace {
2487 :
2488 : const pass_data pass_data_oacc_loop_designation =
2489 : {
2490 : GIMPLE_PASS, /* type */
2491 : "oaccloops", /* name */
2492 : OPTGROUP_OMP, /* optinfo_flags */
2493 : TV_NONE, /* tv_id */
2494 : PROP_cfg, /* properties_required */
2495 : 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
2496 : 0, /* properties_destroyed */
2497 : 0, /* todo_flags_start */
2498 : TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */
2499 : };
2500 :
2501 : class pass_oacc_loop_designation : public gimple_opt_pass
2502 : {
2503 : public:
2504 294587 : pass_oacc_loop_designation (gcc::context *ctxt)
2505 589174 : : gimple_opt_pass (pass_data_oacc_loop_designation, ctxt)
2506 : {}
2507 :
2508 : /* opt_pass methods: */
2509 1512346 : bool gate (function *) final override { return flag_openacc; };
2510 :
2511 15343 : unsigned int execute (function *) final override
2512 : {
2513 15343 : return execute_oacc_loop_designation ();
2514 : }
2515 :
2516 : }; // class pass_oacc_loop_designation
2517 :
2518 : const pass_data pass_data_oacc_device_lower =
2519 : {
2520 : GIMPLE_PASS, /* type */
2521 : "oaccdevlow", /* name */
2522 : OPTGROUP_OMP, /* optinfo_flags */
2523 : TV_NONE, /* tv_id */
2524 : PROP_cfg, /* properties_required */
2525 : 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
2526 : 0, /* properties_destroyed */
2527 : 0, /* todo_flags_start */
2528 : TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */
2529 : };
2530 :
2531 : class pass_oacc_device_lower : public gimple_opt_pass
2532 : {
2533 : public:
2534 294587 : pass_oacc_device_lower (gcc::context *ctxt)
2535 589174 : : gimple_opt_pass (pass_data_oacc_device_lower, ctxt)
2536 : {}
2537 :
2538 : /* opt_pass methods: */
2539 1512284 : bool gate (function *) final override { return flag_openacc; };
2540 :
2541 15281 : unsigned int execute (function *) final override
2542 : {
2543 15281 : return execute_oacc_device_lower ();
2544 : }
2545 :
2546 : }; // class pass_oacc_device_lower
2547 :
2548 : } // anon namespace
2549 :
2550 : gimple_opt_pass *
2551 294587 : make_pass_oacc_loop_designation (gcc::context *ctxt)
2552 : {
2553 294587 : return new pass_oacc_loop_designation (ctxt);
2554 : }
2555 :
2556 : gimple_opt_pass *
2557 294587 : make_pass_oacc_device_lower (gcc::context *ctxt)
2558 : {
2559 294587 : return new pass_oacc_device_lower (ctxt);
2560 : }
2561 :
2562 :
2563 : /* Rewrite GOMP_SIMT_ENTER_ALLOC call given by GSI and remove the preceding
2564 : GOMP_SIMT_ENTER call identifying the privatized variables, which are
2565 : turned to structure fields and receive a DECL_VALUE_EXPR accordingly.
2566 : Set *REGIMPLIFY to true, except if no privatized variables were seen. */
2567 :
2568 : static void
2569 0 : ompdevlow_adjust_simt_enter (gimple_stmt_iterator *gsi, bool *regimplify)
2570 : {
2571 0 : gimple *alloc_stmt = gsi_stmt (*gsi);
2572 0 : tree simtrec = gimple_call_lhs (alloc_stmt);
2573 0 : tree simduid = gimple_call_arg (alloc_stmt, 0);
2574 0 : gimple *enter_stmt = SSA_NAME_DEF_STMT (simduid);
2575 0 : gcc_assert (gimple_call_internal_p (enter_stmt, IFN_GOMP_SIMT_ENTER));
2576 0 : tree rectype = lang_hooks.types.make_type (RECORD_TYPE);
2577 0 : TYPE_ARTIFICIAL (rectype) = TYPE_NAMELESS (rectype) = 1;
2578 0 : TREE_ADDRESSABLE (rectype) = 1;
2579 0 : TREE_TYPE (simtrec) = build_pointer_type (rectype);
2580 0 : for (unsigned i = 1; i < gimple_call_num_args (enter_stmt); i++)
2581 : {
2582 0 : tree *argp = gimple_call_arg_ptr (enter_stmt, i);
2583 0 : if (*argp == null_pointer_node)
2584 0 : continue;
2585 0 : gcc_assert (TREE_CODE (*argp) == ADDR_EXPR
2586 : && VAR_P (TREE_OPERAND (*argp, 0)));
2587 0 : tree var = TREE_OPERAND (*argp, 0);
2588 :
2589 0 : tree field = build_decl (DECL_SOURCE_LOCATION (var), FIELD_DECL,
2590 0 : DECL_NAME (var), TREE_TYPE (var));
2591 0 : SET_DECL_ALIGN (field, DECL_ALIGN (var));
2592 0 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
2593 0 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
2594 :
2595 0 : insert_field_into_struct (rectype, field);
2596 :
2597 0 : tree t = build_simple_mem_ref (simtrec);
2598 0 : t = build3 (COMPONENT_REF, TREE_TYPE (var), t, field, NULL);
2599 0 : TREE_THIS_VOLATILE (t) = TREE_THIS_VOLATILE (var);
2600 0 : SET_DECL_VALUE_EXPR (var, t);
2601 0 : DECL_HAS_VALUE_EXPR_P (var) = 1;
2602 0 : *regimplify = true;
2603 : }
2604 0 : layout_type (rectype);
2605 0 : tree size = TYPE_SIZE_UNIT (rectype);
2606 0 : tree align = build_int_cst (TREE_TYPE (size), TYPE_ALIGN_UNIT (rectype));
2607 :
2608 0 : alloc_stmt
2609 0 : = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC, 2, size, align);
2610 0 : gimple_call_set_lhs (alloc_stmt, simtrec);
2611 0 : gsi_replace (gsi, alloc_stmt, false);
2612 0 : gimple_stmt_iterator enter_gsi = gsi_for_stmt (enter_stmt);
2613 0 : enter_stmt = gimple_build_assign (simduid, gimple_call_arg (enter_stmt, 0));
2614 0 : gsi_replace (&enter_gsi, enter_stmt, false);
2615 :
2616 0 : use_operand_p use;
2617 0 : gimple *exit_stmt;
2618 0 : if (single_imm_use (simtrec, &use, &exit_stmt))
2619 : {
2620 0 : gcc_assert (gimple_call_internal_p (exit_stmt, IFN_GOMP_SIMT_EXIT));
2621 0 : gimple_stmt_iterator exit_gsi = gsi_for_stmt (exit_stmt);
2622 0 : tree clobber = build_clobber (rectype);
2623 0 : exit_stmt = gimple_build_assign (build_simple_mem_ref (simtrec), clobber);
2624 0 : gsi_insert_before (&exit_gsi, exit_stmt, GSI_SAME_STMT);
2625 : }
2626 : else
2627 0 : gcc_checking_assert (has_zero_uses (simtrec));
2628 0 : }
2629 :
2630 : /* Callback for walk_gimple_stmt used to scan for SIMT-privatized variables. */
2631 :
2632 : static tree
2633 0 : find_simtpriv_var_op (tree *tp, int *walk_subtrees, void *)
2634 : {
2635 0 : tree t = *tp;
2636 :
2637 0 : if (VAR_P (t)
2638 0 : && DECL_HAS_VALUE_EXPR_P (t)
2639 0 : && lookup_attribute ("omp simt private", DECL_ATTRIBUTES (t)))
2640 : {
2641 0 : *walk_subtrees = 0;
2642 0 : return t;
2643 : }
2644 : return NULL_TREE;
2645 : }
2646 :
2647 : /* Helper function for execute_omp_device_lower, invoked via walk_gimple_op.
2648 : Resolve any OMP_TARGET_DEVICE_MATCHES and OMP_NEXT_VARIANT exprs to
2649 : constants. */
2650 : static tree
2651 13877 : resolve_omp_variant_cookies (tree *tp, int *walk_subtrees,
2652 : void *data ATTRIBUTE_UNUSED)
2653 : {
2654 13877 : if (TREE_CODE (*tp) == OMP_TARGET_DEVICE_MATCHES)
2655 : {
2656 0 : *tp = resolve_omp_target_device_matches (*tp);
2657 0 : *walk_subtrees = 0;
2658 0 : return NULL_TREE;
2659 : }
2660 :
2661 13877 : if (TREE_CODE (*tp) != OMP_NEXT_VARIANT)
2662 : return NULL_TREE;
2663 324 : tree index = OMP_NEXT_VARIANT_INDEX (*tp);
2664 324 : tree state = OMP_NEXT_VARIANT_STATE (*tp);
2665 :
2666 : /* State is a triplet of (result-vector, construct_context, selector_vec).
2667 : If result-vector has already been computed, just use it. Otherwise we
2668 : must resolve the variant and fill in that part of the state object.
2669 : All OMP_NEXT_VARIANT exprs for the same variant construct are supposed
2670 : to share the same state object, but if something bad happens and we end
2671 : up with copies, that is OK, it will just cause the result-vector to be
2672 : computed multiple times. */
2673 324 : tree result_vector = TREE_PURPOSE (state);
2674 324 : if (!result_vector)
2675 : {
2676 304 : tree construct_context = TREE_VALUE (state);
2677 304 : tree selectors = TREE_CHAIN (state);
2678 :
2679 304 : vec<struct omp_variant> candidates
2680 304 : = omp_resolve_variant_construct (construct_context, selectors);
2681 304 : int n = TREE_VEC_LENGTH (selectors);
2682 304 : TREE_PURPOSE (state) = result_vector = make_tree_vec (n + 1);
2683 : /* The result vector maps the index of each element of the original
2684 : selectors vector onto the index of the next element of the filtered/
2685 : sorted candidates vector. Since some of the original variants may
2686 : have been discarded as non-matching in candidates, initialize the
2687 : whole array to zero so that we have a placeholder "next" value for
2688 : those elements. Hopefully dead code elimination will take care of
2689 : subsequently discarding the unreachable cases in the already-generated
2690 : switch statement. */
2691 2108 : for (int i = 1; i <= n; i++)
2692 1804 : TREE_VEC_ELT (result_vector, i) = integer_zero_node;
2693 : /* Element 0 is the case label of the first variant in the sorted
2694 : list. */
2695 304 : if (dump_file)
2696 0 : fprintf (dump_file, "Computing case map for variant directive\n");
2697 304 : int j = 0;
2698 1588 : for (unsigned int i = 0; i < candidates.length(); i++)
2699 : {
2700 1284 : if (dump_file)
2701 0 : fprintf (dump_file, " %d -> case %d\n",
2702 0 : j, (int) tree_to_shwi (candidates[i].alternative));
2703 1284 : TREE_VEC_ELT (result_vector, j) = candidates[i].alternative;
2704 1284 : j = (int) tree_to_shwi (candidates[i].alternative);
2705 : }
2706 : }
2707 :
2708 : /* Now just grab the value out of the precomputed array. */
2709 324 : gcc_assert (TREE_CODE (index) == INTEGER_CST);
2710 324 : int indexval = (int) tree_to_shwi (index);
2711 324 : *tp = TREE_VEC_ELT (result_vector, indexval);
2712 324 : *walk_subtrees = 0;
2713 324 : return NULL_TREE;
2714 : }
2715 :
2716 :
2717 : /* Cleanup uses of SIMT placeholder internal functions: on non-SIMT targets,
2718 : VF is 1 and LANE is 0; on SIMT targets, VF is folded to a constant, and
2719 : LANE is kept to be expanded to RTL later on. Also cleanup all other SIMT
2720 : internal functions on non-SIMT targets, and likewise some SIMD internal
2721 : functions on SIMT targets. */
2722 :
2723 : static unsigned int
2724 21182 : execute_omp_device_lower ()
2725 : {
2726 21182 : int vf = targetm.simt.vf ? targetm.simt.vf () : 1;
2727 21182 : bool regimplify = false;
2728 21182 : basic_block bb;
2729 21182 : gimple_stmt_iterator gsi;
2730 : #ifdef ACCEL_COMPILER
2731 : bool omp_redirect_indirect_calls = vec_safe_length (offload_ind_funcs) > 0;
2732 : tree map_ptr_fn
2733 : = builtin_decl_explicit (BUILT_IN_GOMP_TARGET_MAP_INDIRECT_PTR);
2734 : #endif
2735 :
2736 : /* Handle expansion of magic cookies for variant constructs first. */
2737 21182 : if (cgraph_node::get (cfun->decl)->has_omp_variant_constructs)
2738 1822 : FOR_EACH_BB_FN (bb, cfun)
2739 : {
2740 6873 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2741 3409 : walk_gimple_op (gsi_stmt (gsi), resolve_omp_variant_cookies, NULL);
2742 2510 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2743 778 : walk_gimple_op (gsi_stmt (gsi), resolve_omp_variant_cookies, NULL);
2744 : }
2745 :
2746 57233 : FOR_EACH_BB_FN (bb, cfun)
2747 247282 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2748 : {
2749 175180 : gimple *stmt = gsi_stmt (gsi);
2750 175180 : if (!is_gimple_call (stmt))
2751 158245 : continue;
2752 16935 : if (!gimple_call_internal_p (stmt))
2753 : {
2754 : #ifdef ACCEL_COMPILER
2755 : if (omp_redirect_indirect_calls
2756 : && gimple_call_fndecl (stmt) == NULL_TREE)
2757 : {
2758 : gcall *orig_call = dyn_cast <gcall *> (stmt);
2759 : tree call_fn = gimple_call_fn (stmt);
2760 : tree fn_ty = TREE_TYPE (call_fn);
2761 :
2762 : if (TREE_CODE (call_fn) == OBJ_TYPE_REF)
2763 : {
2764 : tree obj_ref = create_tmp_reg (TREE_TYPE (call_fn),
2765 : ".ind_fn_objref");
2766 : gimple *gassign = gimple_build_assign (obj_ref, call_fn);
2767 : gsi_insert_before (&gsi, gassign, GSI_SAME_STMT);
2768 : call_fn = obj_ref;
2769 : }
2770 : tree mapped_fn = create_tmp_reg (fn_ty, ".ind_fn");
2771 : gimple *gcall =
2772 : gimple_build_call (map_ptr_fn, 1, call_fn);
2773 : gimple_set_location (gcall, gimple_location (stmt));
2774 : gimple_call_set_lhs (gcall, mapped_fn);
2775 : gsi_insert_before (&gsi, gcall, GSI_SAME_STMT);
2776 :
2777 : gimple_call_set_fn (orig_call, mapped_fn);
2778 : update_stmt (orig_call);
2779 : }
2780 : #endif
2781 16661 : continue;
2782 : }
2783 274 : tree lhs = gimple_call_lhs (stmt), rhs = NULL_TREE;
2784 274 : tree type = lhs ? TREE_TYPE (lhs) : integer_type_node;
2785 274 : switch (gimple_call_internal_fn (stmt))
2786 : {
2787 0 : case IFN_GOMP_TARGET_REV:
2788 0 : {
2789 : #ifndef ACCEL_COMPILER
2790 0 : gimple_stmt_iterator gsi2 = gsi;
2791 0 : gsi_next (&gsi2);
2792 0 : gcc_assert (!gsi_end_p (gsi2));
2793 0 : gcc_assert (gimple_call_builtin_p (gsi_stmt (gsi2),
2794 : BUILT_IN_GOMP_TARGET));
2795 0 : tree old_decl
2796 0 : = TREE_OPERAND (gimple_call_arg (gsi_stmt (gsi2), 1), 0);
2797 0 : tree new_decl = gimple_call_arg (gsi_stmt (gsi), 0);
2798 0 : gimple_call_set_arg (gsi_stmt (gsi2), 1, new_decl);
2799 0 : update_stmt (gsi_stmt (gsi2));
2800 0 : new_decl = TREE_OPERAND (new_decl, 0);
2801 0 : unsigned i;
2802 0 : unsigned num_funcs = vec_safe_length (offload_funcs);
2803 0 : for (i = 0; i < num_funcs; i++)
2804 : {
2805 0 : if ((*offload_funcs)[i] == old_decl)
2806 : {
2807 0 : (*offload_funcs)[i] = new_decl;
2808 0 : break;
2809 : }
2810 0 : else if ((*offload_funcs)[i] == new_decl)
2811 : break; /* This can happen due to inlining. */
2812 : }
2813 0 : gcc_assert (i < num_funcs);
2814 : #else
2815 : tree old_decl = TREE_OPERAND (gimple_call_arg (gsi_stmt (gsi), 0),
2816 : 0);
2817 : #endif
2818 : /* FIXME: Find a way to actually prevent outputting the empty-body
2819 : old_decl as debug symbol + function in the assembly file. */
2820 0 : cgraph_node *node = cgraph_node::get (old_decl);
2821 0 : node->address_taken = false;
2822 0 : node->need_lto_streaming = false;
2823 0 : node->offloadable = false;
2824 :
2825 0 : unlink_stmt_vdef (stmt);
2826 : }
2827 0 : break;
2828 0 : case IFN_GOMP_USE_SIMT:
2829 0 : rhs = vf == 1 ? integer_zero_node : integer_one_node;
2830 : break;
2831 0 : case IFN_GOMP_SIMT_ENTER:
2832 0 : rhs = vf == 1 ? gimple_call_arg (stmt, 0) : NULL_TREE;
2833 0 : goto simtreg_enter_exit;
2834 0 : case IFN_GOMP_SIMT_ENTER_ALLOC:
2835 0 : if (vf != 1)
2836 0 : ompdevlow_adjust_simt_enter (&gsi, ®implify);
2837 0 : rhs = vf == 1 ? null_pointer_node : NULL_TREE;
2838 0 : goto simtreg_enter_exit;
2839 0 : case IFN_GOMP_SIMT_EXIT:
2840 0 : simtreg_enter_exit:
2841 0 : if (vf != 1)
2842 0 : continue;
2843 0 : unlink_stmt_vdef (stmt);
2844 0 : break;
2845 0 : case IFN_GOMP_SIMT_LANE:
2846 0 : case IFN_GOMP_SIMT_LAST_LANE:
2847 0 : rhs = vf == 1 ? build_zero_cst (type) : NULL_TREE;
2848 : break;
2849 0 : case IFN_GOMP_SIMT_VF:
2850 0 : rhs = build_int_cst (type, vf);
2851 0 : break;
2852 2 : case IFN_GOMP_MAX_VF:
2853 2 : rhs = build_int_cst (type, omp_max_vf (false));
2854 2 : break;
2855 0 : case IFN_GOMP_SIMT_ORDERED_PRED:
2856 0 : rhs = vf == 1 ? integer_zero_node : NULL_TREE;
2857 0 : if (rhs || !lhs)
2858 0 : unlink_stmt_vdef (stmt);
2859 : break;
2860 0 : case IFN_GOMP_SIMT_VOTE_ANY:
2861 0 : case IFN_GOMP_SIMT_XCHG_BFLY:
2862 0 : case IFN_GOMP_SIMT_XCHG_IDX:
2863 0 : rhs = vf == 1 ? gimple_call_arg (stmt, 0) : NULL_TREE;
2864 : break;
2865 0 : case IFN_GOMP_SIMD_LANE:
2866 0 : case IFN_GOMP_SIMD_LAST_LANE:
2867 0 : rhs = vf != 1 ? build_zero_cst (type) : NULL_TREE;
2868 : break;
2869 0 : case IFN_GOMP_SIMD_VF:
2870 0 : rhs = vf != 1 ? build_one_cst (type) : NULL_TREE;
2871 : break;
2872 272 : default:
2873 272 : continue;
2874 272 : }
2875 2 : if (lhs && !rhs)
2876 0 : continue;
2877 2 : stmt = lhs ? gimple_build_assign (lhs, rhs) : gimple_build_nop ();
2878 2 : gsi_replace (&gsi, stmt, false);
2879 : }
2880 21182 : if (regimplify)
2881 0 : FOR_EACH_BB_REVERSE_FN (bb, cfun)
2882 0 : for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
2883 0 : if (walk_gimple_stmt (&gsi, NULL, find_simtpriv_var_op, NULL))
2884 : {
2885 0 : if (gimple_clobber_p (gsi_stmt (gsi)))
2886 0 : gsi_remove (&gsi, true);
2887 : else
2888 0 : gimple_regimplify_operands (gsi_stmt (gsi), &gsi);
2889 : }
2890 21182 : if (vf != 1)
2891 0 : cfun->has_force_vectorize_loops = false;
2892 21182 : return 0;
2893 : }
2894 :
2895 : namespace {
2896 :
2897 : const pass_data pass_data_omp_device_lower =
2898 : {
2899 : GIMPLE_PASS, /* type */
2900 : "ompdevlow", /* name */
2901 : OPTGROUP_OMP, /* optinfo_flags */
2902 : TV_NONE, /* tv_id */
2903 : PROP_cfg, /* properties_required */
2904 : PROP_gimple_lomp_dev, /* properties_provided */
2905 : 0, /* properties_destroyed */
2906 : 0, /* todo_flags_start */
2907 : TODO_update_ssa, /* todo_flags_finish */
2908 : };
2909 :
2910 : class pass_omp_device_lower : public gimple_opt_pass
2911 : {
2912 : public:
2913 294587 : pass_omp_device_lower (gcc::context *ctxt)
2914 589174 : : gimple_opt_pass (pass_data_omp_device_lower, ctxt)
2915 : {}
2916 :
2917 : /* opt_pass methods: */
2918 1512284 : bool gate (function *fun) final override
2919 : {
2920 1512284 : cgraph_node *node = cgraph_node::get (fun->decl);
2921 : #ifdef ACCEL_COMPILER
2922 : bool offload_ind_funcs_p = vec_safe_length (offload_ind_funcs) > 0;
2923 : #else
2924 1512284 : bool offload_ind_funcs_p = false;
2925 : #endif
2926 1512284 : return (!(fun->curr_properties & PROP_gimple_lomp_dev)
2927 1512284 : || (flag_openmp
2928 66150 : && (node->has_omp_variant_constructs || offload_ind_funcs_p)));
2929 : }
2930 21182 : unsigned int execute (function *) final override
2931 : {
2932 21182 : return execute_omp_device_lower ();
2933 : }
2934 :
2935 : }; // class pass_expand_omp_ssa
2936 :
2937 : } // anon namespace
2938 :
2939 : gimple_opt_pass *
2940 294587 : make_pass_omp_device_lower (gcc::context *ctxt)
2941 : {
2942 294587 : return new pass_omp_device_lower (ctxt);
2943 : }
2944 :
2945 : /* "omp declare target link" handling pass. */
2946 :
2947 : namespace {
2948 :
2949 : const pass_data pass_data_omp_target_link =
2950 : {
2951 : GIMPLE_PASS, /* type */
2952 : "omptargetlink", /* name */
2953 : OPTGROUP_OMP, /* optinfo_flags */
2954 : TV_NONE, /* tv_id */
2955 : PROP_ssa, /* properties_required */
2956 : 0, /* properties_provided */
2957 : 0, /* properties_destroyed */
2958 : 0, /* todo_flags_start */
2959 : TODO_update_ssa, /* todo_flags_finish */
2960 : };
2961 :
2962 : class pass_omp_target_link : public gimple_opt_pass
2963 : {
2964 : public:
2965 294587 : pass_omp_target_link (gcc::context *ctxt)
2966 589174 : : gimple_opt_pass (pass_data_omp_target_link, ctxt)
2967 : {}
2968 :
2969 : /* opt_pass methods: */
2970 1512284 : bool gate (function *fun) final override
2971 : {
2972 : #ifdef ACCEL_COMPILER
2973 : return offloading_function_p (fun->decl);
2974 : #else
2975 1512284 : (void) fun;
2976 1512284 : return false;
2977 : #endif
2978 : }
2979 :
2980 : unsigned execute (function *) final override;
2981 : };
2982 :
2983 : /* Callback for walk_gimple_stmt used to scan for link var operands. */
2984 :
2985 : static tree
2986 0 : process_link_var_op (tree *tp, int *walk_subtrees, void *data)
2987 : {
2988 0 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
2989 0 : tree t = *tp;
2990 :
2991 0 : if (VAR_P (t)
2992 0 : && DECL_HAS_VALUE_EXPR_P (t)
2993 0 : && is_global_var (t)
2994 0 : && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t)))
2995 : {
2996 0 : wi->info = *tp = unshare_expr (DECL_VALUE_EXPR (t));
2997 0 : *walk_subtrees = 0;
2998 0 : return NULL_TREE;
2999 : }
3000 :
3001 : return NULL_TREE;
3002 : }
3003 :
3004 : unsigned
3005 0 : pass_omp_target_link::execute (function *fun)
3006 : {
3007 0 : basic_block bb;
3008 0 : FOR_EACH_BB_FN (bb, fun)
3009 : {
3010 0 : gimple_stmt_iterator gsi;
3011 0 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
3012 : {
3013 0 : if (gimple_call_builtin_p (gsi_stmt (gsi), BUILT_IN_GOMP_TARGET))
3014 : {
3015 0 : tree dev = gimple_call_arg (gsi_stmt (gsi), 0);
3016 0 : tree fn = gimple_call_arg (gsi_stmt (gsi), 1);
3017 0 : if (POINTER_TYPE_P (TREE_TYPE (fn)))
3018 0 : fn = TREE_OPERAND (fn, 0);
3019 0 : if (TREE_CODE (dev) == INTEGER_CST
3020 0 : && wi::to_wide (dev) == GOMP_DEVICE_HOST_FALLBACK
3021 0 : && lookup_attribute ("omp target device_ancestor_nohost",
3022 0 : DECL_ATTRIBUTES (fn)) != NULL_TREE)
3023 0 : continue; /* ancestor:1 */
3024 : /* Nullify the second argument of __builtin_GOMP_target_ext. */
3025 0 : gimple_call_set_arg (gsi_stmt (gsi), 1, null_pointer_node);
3026 0 : update_stmt (gsi_stmt (gsi));
3027 : }
3028 0 : struct walk_stmt_info wi;
3029 0 : memset (&wi, 0, sizeof (wi));
3030 0 : walk_gimple_stmt (&gsi, NULL, process_link_var_op, &wi);
3031 0 : if (wi.info)
3032 0 : gimple_regimplify_operands (gsi_stmt (gsi), &gsi);
3033 : }
3034 : }
3035 :
3036 0 : return 0;
3037 : }
3038 :
3039 : } // anon namespace
3040 :
3041 : gimple_opt_pass *
3042 294587 : make_pass_omp_target_link (gcc::context *ctxt)
3043 : {
3044 294587 : return new pass_omp_target_link (ctxt);
3045 : }
|
