Line data Source code
1 : /* Bits of OpenMP and OpenACC handling that is specific to device offloading
2 : and a lowering pass for OpenACC device directives.
3 :
4 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "target.h"
27 : #include "tree.h"
28 : #include "gimple.h"
29 : #include "tree-pass.h"
30 : #include "ssa.h"
31 : #include "cgraph.h"
32 : #include "pretty-print.h"
33 : #include "diagnostic-core.h"
34 : #include "fold-const.h"
35 : #include "internal-fn.h"
36 : #include "langhooks.h"
37 : #include "gimplify.h"
38 : #include "gimple-iterator.h"
39 : #include "gimplify-me.h"
40 : #include "gimple-walk.h"
41 : #include "tree-cfg.h"
42 : #include "tree-into-ssa.h"
43 : #include "tree-nested.h"
44 : #include "stor-layout.h"
45 : #include "common/common-target.h"
46 : #include "omp-general.h"
47 : #include "omp-offload.h"
48 : #include "lto-section-names.h"
49 : #include "gomp-constants.h"
50 : #include "gimple-pretty-print.h"
51 : #include "intl.h"
52 : #include "stringpool.h"
53 : #include "attribs.h"
54 : #include "cfgloop.h"
55 : #include "context.h"
56 : #include "convert.h"
57 : #include "opts.h"
58 :
59 : /* Describe the OpenACC looping structure of a function. The entire
60 : function is held in a 'NULL' loop. */
61 :
62 : struct oacc_loop
63 : {
64 : oacc_loop *parent; /* Containing loop. */
65 :
66 : oacc_loop *child; /* First inner loop. */
67 :
68 : oacc_loop *sibling; /* Next loop within same parent. */
69 :
70 : location_t loc; /* Location of the loop start. */
71 :
72 : gcall *marker; /* Initial head marker. */
73 :
74 : gcall *heads[GOMP_DIM_MAX]; /* Head marker functions. */
75 : gcall *tails[GOMP_DIM_MAX]; /* Tail marker functions. */
76 :
77 : tree routine; /* Pseudo-loop enclosing a routine. */
78 :
79 : unsigned mask; /* Partitioning mask. */
80 : unsigned e_mask; /* Partitioning of element loops (when tiling). */
81 : unsigned inner; /* Partitioning of inner loops. */
82 : unsigned flags; /* Partitioning flags. */
83 : vec<gcall *> ifns; /* Contained loop abstraction functions. */
84 : tree chunk_size; /* Chunk size. */
85 : gcall *head_end; /* Final marker of head sequence. */
86 : };
87 :
88 : /* Holds offload tables with decls. */
89 : vec<tree, va_gc> *offload_funcs, *offload_vars, *offload_ind_funcs;
90 :
91 : /* Return level at which oacc routine may spawn a partitioned loop, or
92 : -1 if it is not a routine (i.e. is an offload fn). */
93 :
94 : int
95 11142 : oacc_fn_attrib_level (tree attr)
96 : {
97 11142 : tree pos = TREE_VALUE (attr);
98 :
99 11142 : if (!TREE_PURPOSE (pos))
100 : return -1;
101 :
102 : int ix = 0;
103 5366 : for (ix = 0; ix != GOMP_DIM_MAX;
104 3612 : ix++, pos = TREE_CHAIN (pos))
105 4398 : if (!integer_zerop (TREE_PURPOSE (pos)))
106 : break;
107 :
108 : return ix;
109 : }
110 :
111 : /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
112 : adds their addresses and sizes to constructor-vector V_CTOR. */
113 :
114 : static void
115 87 : add_decls_addresses_to_decl_constructor (vec<tree, va_gc> *v_decls,
116 : vec<constructor_elt, va_gc> *v_ctor)
117 : {
118 87 : unsigned len = vec_safe_length (v_decls);
119 162 : for (unsigned i = 0; i < len; i++)
120 : {
121 75 : tree it = (*v_decls)[i];
122 75 : bool is_var = VAR_P (it);
123 75 : bool is_link_var
124 : = is_var
125 : #ifdef ACCEL_COMPILER
126 : && DECL_HAS_VALUE_EXPR_P (it)
127 : #endif
128 75 : && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it));
129 :
130 : /* See also omp_finish_file and output_offload_tables in lto-cgraph.cc. */
131 75 : if (!in_lto_p && !symtab_node::get (it))
132 0 : continue;
133 :
134 75 : tree size = NULL_TREE;
135 75 : if (is_var)
136 0 : size = fold_convert (const_ptr_type_node, DECL_SIZE_UNIT (it));
137 :
138 75 : tree addr;
139 75 : if (!is_link_var)
140 75 : addr = build_fold_addr_expr (it);
141 : else
142 : {
143 : #ifdef ACCEL_COMPILER
144 : /* For "omp declare target link" vars add address of the pointer to
145 : the target table, instead of address of the var. */
146 : tree value_expr = DECL_VALUE_EXPR (it);
147 : tree link_ptr_decl = TREE_OPERAND (value_expr, 0);
148 : varpool_node::finalize_decl (link_ptr_decl);
149 : addr = build_fold_addr_expr (link_ptr_decl);
150 : #else
151 0 : addr = build_fold_addr_expr (it);
152 : #endif
153 :
154 : /* Most significant bit of the size marks "omp declare target link"
155 : vars in host and target tables. */
156 0 : unsigned HOST_WIDE_INT isize = tree_to_uhwi (size);
157 0 : isize |= 1ULL << (int_size_in_bytes (const_ptr_type_node)
158 0 : * BITS_PER_UNIT - 1);
159 0 : size = wide_int_to_tree (const_ptr_type_node, isize);
160 : }
161 :
162 75 : CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, addr);
163 75 : if (is_var)
164 0 : CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, size);
165 : }
166 87 : }
167 :
168 : /* Return true if DECL is a function for which its references should be
169 : analyzed. */
170 :
171 : static bool
172 200103 : omp_declare_target_fn_p (tree decl)
173 : {
174 200103 : return (TREE_CODE (decl) == FUNCTION_DECL
175 200103 : && lookup_attribute ("omp declare target", DECL_ATTRIBUTES (decl))
176 24552 : && !lookup_attribute ("omp declare target host",
177 24552 : DECL_ATTRIBUTES (decl))
178 224611 : && (!flag_openacc
179 45 : || oacc_get_fn_attrib (decl) == NULL_TREE));
180 : }
181 :
182 : /* Return true if DECL Is a variable for which its initializer references
183 : should be analyzed. */
184 :
185 : static bool
186 110390 : omp_declare_target_var_p (tree decl)
187 : {
188 110390 : return (VAR_P (decl)
189 110390 : && lookup_attribute ("omp declare target", DECL_ATTRIBUTES (decl))
190 110883 : && !lookup_attribute ("omp declare target link",
191 493 : DECL_ATTRIBUTES (decl)));
192 : }
193 :
194 : /* Helper function for omp_discover_implicit_declare_target, called through
195 : walk_tree. Mark referenced FUNCTION_DECLs implicitly as
196 : declare target to. */
197 :
198 : static tree
199 1007482 : omp_discover_declare_target_tgt_fn_r (tree *tp, int *walk_subtrees, void *data)
200 : {
201 1007482 : if (TREE_CODE (*tp) == CALL_EXPR
202 26188 : && CALL_EXPR_FN (*tp)
203 26142 : && TREE_CODE (CALL_EXPR_FN (*tp)) == ADDR_EXPR
204 26093 : && TREE_CODE (TREE_OPERAND (CALL_EXPR_FN (*tp), 0)) == FUNCTION_DECL
205 1033575 : && lookup_attribute ("omp declare variant base",
206 26093 : DECL_ATTRIBUTES (TREE_OPERAND (CALL_EXPR_FN (*tp),
207 : 0))))
208 : {
209 83 : tree fn = TREE_OPERAND (CALL_EXPR_FN (*tp), 0);
210 196 : for (tree attr = DECL_ATTRIBUTES (fn); attr; attr = TREE_CHAIN (attr))
211 : {
212 114 : attr = lookup_attribute ("omp declare variant base", attr);
213 114 : if (attr == NULL_TREE)
214 : break;
215 113 : tree purpose = TREE_PURPOSE (TREE_VALUE (attr));
216 113 : if (TREE_CODE (purpose) == FUNCTION_DECL)
217 113 : omp_discover_declare_target_tgt_fn_r (&purpose, walk_subtrees, data);
218 : }
219 : }
220 1007399 : else if (TREE_CODE (*tp) == FUNCTION_DECL)
221 : {
222 20751 : tree decl = *tp;
223 20751 : tree id = get_identifier ("omp declare target");
224 20751 : symtab_node *node = symtab_node::get (*tp);
225 20751 : if (node != NULL)
226 : {
227 13828 : while (node->alias_target
228 13828 : && TREE_CODE (node->alias_target) == FUNCTION_DECL)
229 : {
230 4 : if (!omp_declare_target_fn_p (node->decl)
231 8 : && !lookup_attribute ("omp declare target host",
232 4 : DECL_ATTRIBUTES (node->decl)))
233 : {
234 4 : node->offloadable = 1;
235 4 : DECL_ATTRIBUTES (node->decl)
236 8 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (node->decl));
237 : }
238 4 : node = symtab_node::get (node->alias_target);
239 : }
240 13824 : symtab_node *new_node = node->ultimate_alias_target ();
241 13824 : decl = new_node->decl;
242 15555 : while (node != new_node)
243 : {
244 1731 : if (!omp_declare_target_fn_p (node->decl)
245 2616 : && !lookup_attribute ("omp declare target host",
246 885 : DECL_ATTRIBUTES (node->decl)))
247 : {
248 885 : node->offloadable = 1;
249 885 : DECL_ATTRIBUTES (node->decl)
250 1770 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (node->decl));
251 : }
252 1731 : gcc_assert (node->alias && node->analyzed);
253 1731 : node = node->get_alias_target ();
254 : }
255 13824 : node->offloadable = 1;
256 13824 : if (ENABLE_OFFLOADING)
257 : g->have_offload = true;
258 : }
259 20751 : if (omp_declare_target_fn_p (decl)
260 29580 : || lookup_attribute ("omp declare target host",
261 8829 : DECL_ATTRIBUTES (decl)))
262 11922 : return NULL_TREE;
263 :
264 8829 : if (DECL_SAVED_TREE (decl)
265 8829 : && (!DECL_EXTERNAL (decl) || DECL_DECLARED_INLINE_P (decl)))
266 6643 : ((vec<tree> *) data)->safe_push (decl);
267 8829 : DECL_ATTRIBUTES (decl) = tree_cons (id, NULL_TREE,
268 8829 : DECL_ATTRIBUTES (decl));
269 : }
270 986648 : else if (TYPE_P (*tp))
271 99 : *walk_subtrees = 0;
272 986549 : else if (TREE_CODE (*tp) == OMP_TARGET)
273 : {
274 1672 : tree c = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE);
275 1672 : tree c2 = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE_TYPE);
276 43 : if ((c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
277 1672 : || (c2 && (OMP_CLAUSE_DEVICE_TYPE_KIND (c2)
278 : == OMP_CLAUSE_DEVICE_TYPE_HOST)))
279 43 : *walk_subtrees = 0;
280 : }
281 : return NULL_TREE;
282 : }
283 :
284 : /* Similarly, but ignore references outside of OMP_TARGET regions. */
285 :
286 : static tree
287 718994 : omp_discover_declare_target_fn_r (tree *tp, int *walk_subtrees, void *data)
288 : {
289 718994 : if (TREE_CODE (*tp) == OMP_TARGET)
290 : {
291 11987 : tree c = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE);
292 11987 : tree c2 = omp_find_clause (OMP_CLAUSES (*tp), OMP_CLAUSE_DEVICE_TYPE);
293 863 : if ((!c || !OMP_CLAUSE_DEVICE_ANCESTOR (c))
294 12792 : && (!c2 || (OMP_CLAUSE_DEVICE_TYPE_KIND (c2)
295 : != OMP_CLAUSE_DEVICE_TYPE_HOST)))
296 11923 : walk_tree_without_duplicates (&OMP_TARGET_BODY (*tp),
297 : omp_discover_declare_target_tgt_fn_r,
298 : data);
299 11987 : *walk_subtrees = 0;
300 : }
301 707007 : else if (TYPE_P (*tp))
302 371 : *walk_subtrees = 0;
303 718994 : return NULL_TREE;
304 : }
305 :
306 : /* Helper function for omp_discover_implicit_declare_target, called through
307 : walk_tree. Mark referenced FUNCTION_DECLs implicitly as
308 : declare target to. */
309 :
310 : static tree
311 518 : omp_discover_declare_target_var_r (tree *tp, int *walk_subtrees, void *data)
312 : {
313 518 : if (TREE_CODE (*tp) == FUNCTION_DECL)
314 24 : return omp_discover_declare_target_tgt_fn_r (tp, walk_subtrees, data);
315 494 : else if (VAR_P (*tp)
316 54 : && is_global_var (*tp)
317 537 : && !omp_declare_target_var_p (*tp))
318 : {
319 15 : tree id = get_identifier ("omp declare target");
320 15 : if (lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (*tp)))
321 : {
322 0 : error_at (DECL_SOURCE_LOCATION (*tp),
323 : "%qD specified both in declare target %<link%> and "
324 : "implicitly in %<to%> clauses", *tp);
325 0 : DECL_ATTRIBUTES (*tp)
326 0 : = remove_attribute ("omp declare target link", DECL_ATTRIBUTES (*tp));
327 : }
328 15 : if (TREE_STATIC (*tp) && lang_hooks.decls.omp_get_decl_init (*tp))
329 15 : ((vec<tree> *) data)->safe_push (*tp);
330 15 : DECL_ATTRIBUTES (*tp) = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (*tp));
331 15 : symtab_node *node = symtab_node::get (*tp);
332 15 : if (node != NULL && !node->offloadable)
333 : {
334 15 : node->offloadable = 1;
335 15 : if (ENABLE_OFFLOADING)
336 : {
337 : g->have_offload = true;
338 : if (is_a <varpool_node *> (node))
339 : vec_safe_push (offload_vars, node->decl);
340 : }
341 : }
342 : }
343 479 : else if (TYPE_P (*tp))
344 0 : *walk_subtrees = 0;
345 : return NULL_TREE;
346 : }
347 :
348 : /* Perform the OpenMP implicit declare target to discovery. */
349 :
350 : void
351 9655 : omp_discover_implicit_declare_target (void)
352 : {
353 9655 : cgraph_node *node;
354 9655 : varpool_node *vnode;
355 9655 : auto_vec<tree> worklist;
356 :
357 169246 : FOR_EACH_DEFINED_FUNCTION (node)
358 159591 : if (DECL_SAVED_TREE (node->decl))
359 : {
360 159131 : struct cgraph_node *cgn;
361 159131 : if (lookup_attribute ("omp declare target indirect",
362 159131 : DECL_ATTRIBUTES (node->decl)))
363 123 : vec_safe_push (offload_ind_funcs, node->decl);
364 159131 : if (omp_declare_target_fn_p (node->decl))
365 2506 : worklist.safe_push (node->decl);
366 156625 : else if (DECL_STRUCT_FUNCTION (node->decl)
367 156625 : && DECL_STRUCT_FUNCTION (node->decl)->has_omp_target)
368 6758 : worklist.safe_push (node->decl);
369 320374 : for (cgn = first_nested_function (node);
370 161243 : cgn; cgn = next_nested_function (cgn))
371 2112 : if (omp_declare_target_fn_p (cgn->decl))
372 33 : worklist.safe_push (cgn->decl);
373 2079 : else if (DECL_STRUCT_FUNCTION (cgn->decl)
374 2079 : && DECL_STRUCT_FUNCTION (cgn->decl)->has_omp_target)
375 434 : worklist.safe_push (cgn->decl);
376 : }
377 128223 : FOR_EACH_VARIABLE (vnode)
378 118568 : if (lang_hooks.decls.omp_get_decl_init (vnode->decl)
379 118568 : && omp_declare_target_var_p (vnode->decl))
380 465 : worklist.safe_push (vnode->decl);
381 26509 : while (!worklist.is_empty ())
382 : {
383 16854 : tree decl = worklist.pop ();
384 16854 : if (VAR_P (decl))
385 480 : walk_tree_without_duplicates (lang_hooks.decls.omp_get_decl_init (decl),
386 : omp_discover_declare_target_var_r,
387 : &worklist);
388 16374 : else if (omp_declare_target_fn_p (decl))
389 9182 : walk_tree_without_duplicates (&DECL_SAVED_TREE (decl),
390 : omp_discover_declare_target_tgt_fn_r,
391 : &worklist);
392 : else
393 7192 : walk_tree_without_duplicates (&DECL_SAVED_TREE (decl),
394 : omp_discover_declare_target_fn_r,
395 : &worklist);
396 : }
397 :
398 9655 : lang_hooks.decls.omp_finish_decl_inits ();
399 9655 : }
400 :
401 :
402 : /* Create new symbols containing (address, size) pairs for global variables,
403 : marked with "omp declare target" attribute, as well as addresses for the
404 : functions, which are outlined offloading regions. */
405 : void
406 237287 : omp_finish_file (void)
407 : {
408 237287 : unsigned num_funcs = vec_safe_length (offload_funcs);
409 237287 : unsigned num_vars = vec_safe_length (offload_vars);
410 237287 : unsigned num_ind_funcs = vec_safe_length (offload_ind_funcs);
411 :
412 237287 : if (num_funcs == 0 && num_vars == 0 && num_ind_funcs == 0)
413 237287 : return;
414 :
415 29 : if (targetm_common.have_named_sections)
416 : {
417 29 : vec<constructor_elt, va_gc> *v_f, *v_v, *v_if;
418 29 : vec_alloc (v_f, num_funcs);
419 29 : vec_alloc (v_v, num_vars * 2);
420 29 : vec_alloc (v_if, num_ind_funcs);
421 :
422 29 : add_decls_addresses_to_decl_constructor (offload_funcs, v_f);
423 29 : add_decls_addresses_to_decl_constructor (offload_vars, v_v);
424 29 : add_decls_addresses_to_decl_constructor (offload_ind_funcs, v_if);
425 :
426 29 : tree vars_decl_type = build_array_type_nelts (pointer_sized_int_node,
427 29 : vec_safe_length (v_v));
428 29 : tree funcs_decl_type = build_array_type_nelts (pointer_sized_int_node,
429 29 : num_funcs);
430 29 : tree ind_funcs_decl_type = build_array_type_nelts (pointer_sized_int_node,
431 29 : num_ind_funcs);
432 :
433 29 : SET_TYPE_ALIGN (vars_decl_type, TYPE_ALIGN (pointer_sized_int_node));
434 29 : SET_TYPE_ALIGN (funcs_decl_type, TYPE_ALIGN (pointer_sized_int_node));
435 29 : SET_TYPE_ALIGN (ind_funcs_decl_type, TYPE_ALIGN (pointer_sized_int_node));
436 29 : tree ctor_v = build_constructor (vars_decl_type, v_v);
437 29 : tree ctor_f = build_constructor (funcs_decl_type, v_f);
438 29 : tree ctor_if = build_constructor (ind_funcs_decl_type, v_if);
439 29 : TREE_CONSTANT (ctor_v) = TREE_CONSTANT (ctor_f) = TREE_CONSTANT (ctor_if) = 1;
440 29 : TREE_STATIC (ctor_v) = TREE_STATIC (ctor_f) = TREE_STATIC (ctor_if) = 1;
441 29 : tree funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
442 : get_identifier (".offload_func_table"),
443 : funcs_decl_type);
444 29 : tree vars_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
445 : get_identifier (".offload_var_table"),
446 : vars_decl_type);
447 29 : tree ind_funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
448 : get_identifier (".offload_ind_func_table"),
449 : ind_funcs_decl_type);
450 29 : TREE_STATIC (funcs_decl) = TREE_STATIC (ind_funcs_decl) = 1;
451 29 : TREE_STATIC (vars_decl) = 1;
452 : /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
453 : otherwise a joint table in a binary will contain padding between
454 : tables from multiple object files. */
455 29 : DECL_USER_ALIGN (funcs_decl) = DECL_USER_ALIGN (ind_funcs_decl) = 1;
456 29 : DECL_USER_ALIGN (vars_decl) = 1;
457 29 : SET_DECL_ALIGN (funcs_decl, TYPE_ALIGN (funcs_decl_type));
458 29 : SET_DECL_ALIGN (vars_decl, TYPE_ALIGN (vars_decl_type));
459 29 : SET_DECL_ALIGN (ind_funcs_decl, TYPE_ALIGN (ind_funcs_decl_type));
460 29 : DECL_INITIAL (funcs_decl) = ctor_f;
461 29 : DECL_INITIAL (vars_decl) = ctor_v;
462 29 : DECL_INITIAL (ind_funcs_decl) = ctor_if;
463 29 : set_decl_section_name (funcs_decl, OFFLOAD_FUNC_TABLE_SECTION_NAME);
464 29 : set_decl_section_name (vars_decl, OFFLOAD_VAR_TABLE_SECTION_NAME);
465 29 : set_decl_section_name (ind_funcs_decl,
466 : OFFLOAD_IND_FUNC_TABLE_SECTION_NAME);
467 29 : varpool_node::finalize_decl (vars_decl);
468 29 : varpool_node::finalize_decl (funcs_decl);
469 29 : varpool_node::finalize_decl (ind_funcs_decl);
470 : }
471 : else
472 : {
473 0 : for (unsigned i = 0; i < num_funcs; i++)
474 : {
475 0 : tree it = (*offload_funcs)[i];
476 : /* See also add_decls_addresses_to_decl_constructor
477 : and output_offload_tables in lto-cgraph.cc. */
478 0 : if (!in_lto_p && !symtab_node::get (it))
479 0 : continue;
480 0 : targetm.record_offload_symbol (it);
481 : }
482 0 : for (unsigned i = 0; i < num_vars; i++)
483 : {
484 0 : tree it = (*offload_vars)[i];
485 0 : if (!in_lto_p && !symtab_node::get (it))
486 0 : continue;
487 : #ifdef ACCEL_COMPILER
488 : if (DECL_HAS_VALUE_EXPR_P (it)
489 : && lookup_attribute ("omp declare target link",
490 : DECL_ATTRIBUTES (it)))
491 : {
492 : tree value_expr = DECL_VALUE_EXPR (it);
493 : tree link_ptr_decl = TREE_OPERAND (value_expr, 0);
494 : targetm.record_offload_symbol (link_ptr_decl);
495 : varpool_node::finalize_decl (link_ptr_decl);
496 : }
497 : else
498 : #endif
499 0 : targetm.record_offload_symbol (it);
500 : }
501 0 : for (unsigned i = 0; i < num_ind_funcs; i++)
502 : {
503 0 : tree it = (*offload_ind_funcs)[i];
504 : /* See also add_decls_addresses_to_decl_constructor
505 : and output_offload_tables in lto-cgraph.cc. */
506 0 : if (!in_lto_p && !symtab_node::get (it))
507 0 : continue;
508 0 : targetm.record_offload_symbol (it);
509 : }
510 : }
511 : }
512 :
513 : /* Call dim_pos (POS == true) or dim_size (POS == false) builtins for
514 : axis DIM. Return a tmp var holding the result. */
515 :
516 : static tree
517 30709 : oacc_dim_call (bool pos, int dim, gimple_seq *seq)
518 : {
519 30709 : tree arg = build_int_cst (unsigned_type_node, dim);
520 30709 : tree size = create_tmp_var (integer_type_node);
521 30709 : enum internal_fn fn = pos ? IFN_GOACC_DIM_POS : IFN_GOACC_DIM_SIZE;
522 30709 : gimple *call = gimple_build_call_internal (fn, 1, arg);
523 :
524 30709 : gimple_call_set_lhs (call, size);
525 30709 : gimple_seq_add_stmt (seq, call);
526 :
527 30709 : return size;
528 : }
529 :
530 : /* Find the number of threads (POS = false), or thread number (POS =
531 : true) for an OpenACC region partitioned as MASK. Setup code
532 : required for the calculation is added to SEQ. */
533 :
534 : static tree
535 23610 : oacc_thread_numbers (bool pos, int mask, gimple_seq *seq)
536 : {
537 23610 : tree res = pos ? NULL_TREE : build_int_cst (unsigned_type_node, 1);
538 23610 : unsigned ix;
539 :
540 : /* Start at gang level, and examine relevant dimension indices. */
541 94440 : for (ix = GOMP_DIM_GANG; ix != GOMP_DIM_MAX; ix++)
542 70830 : if (GOMP_DIM_MASK (ix) & mask)
543 : {
544 26569 : if (res)
545 : {
546 : /* We had an outer index, so scale that by the size of
547 : this dimension. */
548 17369 : tree n = oacc_dim_call (false, ix, seq);
549 17369 : res = fold_build2 (MULT_EXPR, integer_type_node, res, n);
550 : }
551 26569 : if (pos)
552 : {
553 : /* Determine index in this dimension. */
554 13340 : tree id = oacc_dim_call (true, ix, seq);
555 13340 : if (res)
556 4140 : res = fold_build2 (PLUS_EXPR, integer_type_node, res, id);
557 : else
558 : res = id;
559 : }
560 : }
561 :
562 23610 : if (res == NULL_TREE)
563 2657 : res = integer_zero_node;
564 :
565 23610 : return res;
566 : }
567 :
568 : /* Transform IFN_GOACC_LOOP calls to actual code. See
569 : expand_oacc_for for where these are generated. At the vector
570 : level, we stride loops, such that each member of a warp will
571 : operate on adjacent iterations. At the worker and gang level,
572 : each gang/warp executes a set of contiguous iterations. Chunking
573 : can override this such that each iteration engine executes a
574 : contiguous chunk, and then moves on to stride to the next chunk. */
575 :
576 : static void
577 46702 : oacc_xform_loop (gcall *call)
578 : {
579 46702 : gimple_stmt_iterator gsi = gsi_for_stmt (call);
580 46702 : enum ifn_goacc_loop_kind code
581 46702 : = (enum ifn_goacc_loop_kind) TREE_INT_CST_LOW (gimple_call_arg (call, 0));
582 46702 : tree dir = gimple_call_arg (call, 1);
583 46702 : tree range = gimple_call_arg (call, 2);
584 46702 : tree step = gimple_call_arg (call, 3);
585 46702 : tree chunk_size = NULL_TREE;
586 46702 : unsigned mask = (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call, 5));
587 46702 : tree lhs = gimple_call_lhs (call);
588 46702 : tree type = NULL_TREE;
589 46702 : tree diff_type = TREE_TYPE (range);
590 46702 : tree r = NULL_TREE;
591 46702 : gimple_seq seq = NULL;
592 46702 : bool chunking = false, striding = true;
593 46702 : unsigned outer_mask = mask & (~mask + 1); // Outermost partitioning
594 46702 : unsigned inner_mask = mask & ~outer_mask; // Inner partitioning (if any)
595 :
596 : /* Skip lowering if return value of IFN_GOACC_LOOP call is not used. */
597 46702 : if (!lhs)
598 : {
599 8 : gsi_replace_with_seq (&gsi, seq, true);
600 8 : return;
601 : }
602 :
603 46694 : type = TREE_TYPE (lhs);
604 :
605 : #ifdef ACCEL_COMPILER
606 : chunk_size = gimple_call_arg (call, 4);
607 : if (integer_minus_onep (chunk_size) /* Force static allocation. */
608 : || integer_zerop (chunk_size)) /* Default (also static). */
609 : {
610 : /* If we're at the gang level, we want each to execute a
611 : contiguous run of iterations. Otherwise we want each element
612 : to stride. */
613 : striding = !(outer_mask & GOMP_DIM_MASK (GOMP_DIM_GANG));
614 : chunking = false;
615 : }
616 : else
617 : {
618 : /* Chunk of size 1 is striding. */
619 : striding = integer_onep (chunk_size);
620 : chunking = !striding;
621 : }
622 : #endif
623 :
624 : /* striding=true, chunking=true
625 : -> invalid.
626 : striding=true, chunking=false
627 : -> chunks=1
628 : striding=false,chunking=true
629 : -> chunks=ceil (range/(chunksize*threads*step))
630 : striding=false,chunking=false
631 : -> chunk_size=ceil(range/(threads*step)),chunks=1 */
632 46694 : push_gimplify_context (true);
633 :
634 46694 : switch (code)
635 : {
636 0 : default: gcc_unreachable ();
637 :
638 11231 : case IFN_GOACC_LOOP_CHUNKS:
639 11231 : if (!chunking)
640 11231 : r = build_int_cst (type, 1);
641 : else
642 : {
643 : /* chunk_max
644 : = (range - dir) / (chunks * step * num_threads) + dir */
645 : tree per = oacc_thread_numbers (false, mask, &seq);
646 : per = fold_convert (type, per);
647 : chunk_size = fold_convert (type, chunk_size);
648 : per = fold_build2 (MULT_EXPR, type, per, chunk_size);
649 : per = fold_build2 (MULT_EXPR, type, per, step);
650 : r = build2 (MINUS_EXPR, type, range, dir);
651 : r = build2 (PLUS_EXPR, type, r, per);
652 : r = build2 (TRUNC_DIV_EXPR, type, r, per);
653 : }
654 : break;
655 :
656 11753 : case IFN_GOACC_LOOP_STEP:
657 11753 : {
658 : /* If striding, step by the entire compute volume, otherwise
659 : step by the inner volume. */
660 11753 : unsigned volume = striding ? mask : inner_mask;
661 :
662 11753 : r = oacc_thread_numbers (false, volume, &seq);
663 11753 : r = build2 (MULT_EXPR, type, fold_convert (type, r), step);
664 : }
665 11753 : break;
666 :
667 11857 : case IFN_GOACC_LOOP_OFFSET:
668 : /* Enable vectorization on non-SIMT targets. */
669 11857 : if (!targetm.simt.vf
670 11857 : && outer_mask == GOMP_DIM_MASK (GOMP_DIM_VECTOR)
671 : /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
672 : the loop. */
673 1951 : && (flag_tree_loop_vectorize
674 1463 : || !OPTION_SET_P (flag_tree_loop_vectorize)))
675 : {
676 1951 : basic_block bb = gsi_bb (gsi);
677 1951 : class loop *parent = bb->loop_father;
678 1951 : class loop *body = parent->inner;
679 :
680 1951 : parent->force_vectorize = true;
681 1951 : parent->safelen = INT_MAX;
682 :
683 : /* "Chunking loops" may have inner loops. */
684 1951 : if (parent->inner)
685 : {
686 1939 : body->force_vectorize = true;
687 1939 : body->safelen = INT_MAX;
688 : }
689 :
690 1951 : cfun->has_force_vectorize_loops = true;
691 : }
692 11857 : if (striding)
693 : {
694 11857 : r = oacc_thread_numbers (true, mask, &seq);
695 11857 : r = fold_convert (diff_type, r);
696 : }
697 : else
698 : {
699 : tree inner_size = oacc_thread_numbers (false, inner_mask, &seq);
700 : tree outer_size = oacc_thread_numbers (false, outer_mask, &seq);
701 : tree volume = fold_build2 (MULT_EXPR, TREE_TYPE (inner_size),
702 : inner_size, outer_size);
703 :
704 : volume = fold_convert (diff_type, volume);
705 : if (chunking)
706 : chunk_size = fold_convert (diff_type, chunk_size);
707 : else
708 : {
709 : tree per = fold_build2 (MULT_EXPR, diff_type, volume, step);
710 :
711 : chunk_size = build2 (MINUS_EXPR, diff_type, range, dir);
712 : chunk_size = build2 (PLUS_EXPR, diff_type, chunk_size, per);
713 : chunk_size = build2 (TRUNC_DIV_EXPR, diff_type, chunk_size, per);
714 : }
715 :
716 : tree span = build2 (MULT_EXPR, diff_type, chunk_size,
717 : fold_convert (diff_type, inner_size));
718 : r = oacc_thread_numbers (true, outer_mask, &seq);
719 : r = fold_convert (diff_type, r);
720 : r = build2 (MULT_EXPR, diff_type, r, span);
721 :
722 : tree inner = oacc_thread_numbers (true, inner_mask, &seq);
723 : inner = fold_convert (diff_type, inner);
724 : r = fold_build2 (PLUS_EXPR, diff_type, r, inner);
725 :
726 : if (chunking)
727 : {
728 : tree chunk = fold_convert (diff_type, gimple_call_arg (call, 6));
729 : tree per
730 : = fold_build2 (MULT_EXPR, diff_type, volume, chunk_size);
731 : per = build2 (MULT_EXPR, diff_type, per, chunk);
732 :
733 : r = build2 (PLUS_EXPR, diff_type, r, per);
734 : }
735 : }
736 11857 : r = fold_build2 (MULT_EXPR, diff_type, r, step);
737 11857 : if (type != diff_type)
738 178 : r = fold_convert (type, r);
739 : break;
740 :
741 11853 : case IFN_GOACC_LOOP_BOUND:
742 11853 : if (striding)
743 11853 : r = range;
744 : else
745 : {
746 : tree inner_size = oacc_thread_numbers (false, inner_mask, &seq);
747 : tree outer_size = oacc_thread_numbers (false, outer_mask, &seq);
748 : tree volume = fold_build2 (MULT_EXPR, TREE_TYPE (inner_size),
749 : inner_size, outer_size);
750 :
751 : volume = fold_convert (diff_type, volume);
752 : if (chunking)
753 : chunk_size = fold_convert (diff_type, chunk_size);
754 : else
755 : {
756 : tree per = fold_build2 (MULT_EXPR, diff_type, volume, step);
757 :
758 : chunk_size = build2 (MINUS_EXPR, diff_type, range, dir);
759 : chunk_size = build2 (PLUS_EXPR, diff_type, chunk_size, per);
760 : chunk_size = build2 (TRUNC_DIV_EXPR, diff_type, chunk_size, per);
761 : }
762 :
763 : tree span = build2 (MULT_EXPR, diff_type, chunk_size,
764 : fold_convert (diff_type, inner_size));
765 :
766 : r = fold_build2 (MULT_EXPR, diff_type, span, step);
767 :
768 : tree offset = gimple_call_arg (call, 6);
769 : r = build2 (PLUS_EXPR, diff_type, r,
770 : fold_convert (diff_type, offset));
771 : r = build2 (integer_onep (dir) ? MIN_EXPR : MAX_EXPR,
772 : diff_type, r, range);
773 : }
774 11853 : if (diff_type != type)
775 178 : r = fold_convert (type, r);
776 : break;
777 : }
778 :
779 46694 : gimplify_assign (lhs, r, &seq);
780 :
781 46694 : pop_gimplify_context (NULL);
782 :
783 46694 : gsi_replace_with_seq (&gsi, seq, true);
784 : }
785 :
786 : /* Transform a GOACC_TILE call. Determines the element loop span for
787 : the specified loop of the nest. This is 1 if we're not tiling.
788 :
789 : GOACC_TILE (collapse_count, loop_no, tile_arg, gwv_tile, gwv_element); */
790 :
791 : static void
792 284 : oacc_xform_tile (gcall *call)
793 : {
794 284 : gimple_stmt_iterator gsi = gsi_for_stmt (call);
795 284 : unsigned collapse = tree_to_uhwi (gimple_call_arg (call, 0));
796 : /* Inner loops have higher loop_nos. */
797 284 : unsigned loop_no = tree_to_uhwi (gimple_call_arg (call, 1));
798 284 : tree tile_size = gimple_call_arg (call, 2);
799 284 : unsigned e_mask = tree_to_uhwi (gimple_call_arg (call, 4));
800 284 : tree lhs = gimple_call_lhs (call);
801 284 : tree type = TREE_TYPE (lhs);
802 284 : gimple_seq seq = NULL;
803 284 : tree span = build_int_cst (type, 1);
804 :
805 284 : gcc_assert (!(e_mask
806 : & ~(GOMP_DIM_MASK (GOMP_DIM_VECTOR)
807 : | GOMP_DIM_MASK (GOMP_DIM_WORKER))));
808 284 : push_gimplify_context (!seen_error ());
809 :
810 : #ifndef ACCEL_COMPILER
811 : /* Partitioning disabled on host compilers. */
812 284 : e_mask = 0;
813 : #endif
814 284 : if (!e_mask)
815 : /* Not partitioning. */
816 284 : span = integer_one_node;
817 : else if (!integer_zerop (tile_size))
818 : /* User explicitly specified size. */
819 : span = tile_size;
820 : else
821 : {
822 : /* Pick a size based on the partitioning of the element loop and
823 : the number of loop nests. */
824 : tree first_size = NULL_TREE;
825 : tree second_size = NULL_TREE;
826 :
827 : if (e_mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR))
828 : first_size = oacc_dim_call (false, GOMP_DIM_VECTOR, &seq);
829 : if (e_mask & GOMP_DIM_MASK (GOMP_DIM_WORKER))
830 : second_size = oacc_dim_call (false, GOMP_DIM_WORKER, &seq);
831 :
832 : if (!first_size)
833 : {
834 : first_size = second_size;
835 : second_size = NULL_TREE;
836 : }
837 :
838 : if (loop_no + 1 == collapse)
839 : {
840 : span = first_size;
841 : if (!loop_no && second_size)
842 : span = fold_build2 (MULT_EXPR, TREE_TYPE (span),
843 : span, second_size);
844 : }
845 : else if (loop_no + 2 == collapse)
846 : span = second_size;
847 : else
848 : span = NULL_TREE;
849 :
850 : if (!span)
851 : /* There's no obvious element size for this loop. Options
852 : are 1, first_size or some non-unity constant (32 is my
853 : favourite). We should gather some statistics. */
854 : span = first_size;
855 : }
856 :
857 284 : span = fold_convert (type, span);
858 284 : gimplify_assign (lhs, span, &seq);
859 :
860 284 : pop_gimplify_context (NULL);
861 :
862 284 : gsi_replace_with_seq (&gsi, seq, true);
863 284 : }
864 :
865 : /* Default partitioned and minimum partitioned dimensions. */
866 :
867 : static int oacc_default_dims[GOMP_DIM_MAX];
868 : static int oacc_min_dims[GOMP_DIM_MAX];
869 :
870 : int
871 0 : oacc_get_default_dim (int dim)
872 : {
873 0 : gcc_assert (0 <= dim && dim < GOMP_DIM_MAX);
874 0 : return oacc_default_dims[dim];
875 : }
876 :
877 : int
878 0 : oacc_get_min_dim (int dim)
879 : {
880 0 : gcc_assert (0 <= dim && dim < GOMP_DIM_MAX);
881 0 : return oacc_min_dims[dim];
882 : }
883 :
884 : /* Parse the default dimension parameter. This is a set of
885 : :-separated optional compute dimensions. Each specified dimension
886 : is a positive integer. When device type support is added, it is
887 : planned to be a comma separated list of such compute dimensions,
888 : with all but the first prefixed by the colon-terminated device
889 : type. */
890 :
891 : static void
892 2281 : oacc_parse_default_dims (const char *dims)
893 : {
894 2281 : int ix;
895 :
896 9124 : for (ix = GOMP_DIM_MAX; ix--;)
897 : {
898 6843 : oacc_default_dims[ix] = -1;
899 6843 : oacc_min_dims[ix] = 1;
900 : }
901 :
902 : #ifndef ACCEL_COMPILER
903 : /* Cannot be overridden on the host. */
904 2281 : dims = NULL;
905 : #endif
906 2281 : if (dims)
907 : {
908 : const char *pos = dims;
909 :
910 : for (ix = 0; *pos && ix != GOMP_DIM_MAX; ix++)
911 : {
912 : if (ix)
913 : {
914 : if (*pos != ':')
915 : goto malformed;
916 : pos++;
917 : }
918 :
919 : if (*pos != ':')
920 : {
921 : long val;
922 : const char *eptr;
923 :
924 : errno = 0;
925 : val = strtol (pos, const_cast<char **> (&eptr), 10);
926 : if (errno || val <= 0 || (int) val != val)
927 : goto malformed;
928 : pos = eptr;
929 : oacc_default_dims[ix] = (int) val;
930 : }
931 : }
932 : if (*pos)
933 : {
934 : malformed:
935 : error_at (UNKNOWN_LOCATION,
936 : "%<-fopenacc-dim%> operand is malformed at %qs", pos);
937 : }
938 : }
939 :
940 : /* Allow the backend to validate the dimensions. */
941 2281 : targetm.goacc.validate_dims (NULL_TREE, oacc_default_dims, -1, 0);
942 2281 : targetm.goacc.validate_dims (NULL_TREE, oacc_min_dims, -2, 0);
943 2281 : }
944 :
945 : /* Validate and update the dimensions for offloaded FN. ATTRS is the
946 : raw attribute. DIMS is an array of dimensions, which is filled in.
947 : LEVEL is the partitioning level of a routine, or -1 for an offload
948 : region itself. USED is the mask of partitioned execution in the
949 : function. */
950 :
951 : static void
952 9879 : oacc_validate_dims (tree fn, tree attrs, int *dims, int level, unsigned used)
953 : {
954 9879 : tree purpose[GOMP_DIM_MAX];
955 9879 : unsigned ix;
956 9879 : tree pos = TREE_VALUE (attrs);
957 :
958 : /* Make sure the attribute creator attached the dimension
959 : information. */
960 9879 : gcc_assert (pos);
961 :
962 39516 : for (ix = 0; ix != GOMP_DIM_MAX; ix++)
963 : {
964 29637 : purpose[ix] = TREE_PURPOSE (pos);
965 29637 : tree val = TREE_VALUE (pos);
966 29637 : dims[ix] = val ? TREE_INT_CST_LOW (val) : -1;
967 29637 : pos = TREE_CHAIN (pos);
968 : }
969 :
970 9879 : bool check = true;
971 : #ifdef ACCEL_COMPILER
972 : check = false;
973 : #endif
974 9879 : if (check
975 9879 : && warn_openacc_parallelism
976 1371 : && !lookup_attribute ("oacc kernels", DECL_ATTRIBUTES (fn)))
977 : {
978 1268 : static char const *const axes[] =
979 : /* Must be kept in sync with GOMP_DIM enumeration. */
980 : { "gang", "worker", "vector" };
981 4775 : for (ix = level >= 0 ? level : 0; ix != GOMP_DIM_MAX; ix++)
982 3507 : if (dims[ix] < 0)
983 : ; /* Defaulting axis. */
984 1970 : else if ((used & GOMP_DIM_MASK (ix)) && dims[ix] == 1)
985 : /* There is partitioned execution, but the user requested a
986 : dimension size of 1. They're probably confused. */
987 94 : warning_at (DECL_SOURCE_LOCATION (fn), OPT_Wopenacc_parallelism,
988 : "region contains %s partitioned code but"
989 94 : " is not %s partitioned", axes[ix], axes[ix]);
990 1876 : else if (!(used & GOMP_DIM_MASK (ix)) && dims[ix] != 1)
991 : /* The dimension is explicitly partitioned to non-unity, but
992 : no use is made within the region. */
993 500 : warning_at (DECL_SOURCE_LOCATION (fn), OPT_Wopenacc_parallelism,
994 : "region is %s partitioned but"
995 : " does not contain %s partitioned code",
996 500 : axes[ix], axes[ix]);
997 : }
998 :
999 9879 : bool changed = targetm.goacc.validate_dims (fn, dims, level, used);
1000 :
1001 : /* Default anything left to 1 or a partitioned default. */
1002 49395 : for (ix = 0; ix != GOMP_DIM_MAX; ix++)
1003 29637 : if (dims[ix] < 0)
1004 : {
1005 : /* The OpenACC spec says 'If the [num_gangs] clause is not
1006 : specified, an implementation-defined default will be used;
1007 : the default may depend on the code within the construct.'
1008 : (2.5.6). Thus an implementation is free to choose
1009 : non-unity default for a parallel region that doesn't have
1010 : any gang-partitioned loops. However, it appears that there
1011 : is a sufficient body of user code that expects non-gang
1012 : partitioned regions to not execute in gang-redundant mode.
1013 : So we (a) don't warn about the non-portability and (b) pick
1014 : the minimum permissible dimension size when there is no
1015 : partitioned execution. Otherwise we pick the global
1016 : default for the dimension, which the user can control. The
1017 : same wording and logic applies to num_workers and
1018 : vector_length, however the worker- or vector- single
1019 : execution doesn't have the same impact as gang-redundant
1020 : execution. (If the minimum gang-level partitioning is not 1,
1021 : the target is probably too confusing.) */
1022 0 : dims[ix] = (used & GOMP_DIM_MASK (ix)
1023 0 : ? oacc_default_dims[ix] : oacc_min_dims[ix]);
1024 0 : changed = true;
1025 : }
1026 :
1027 9879 : if (changed)
1028 : {
1029 : /* Replace the attribute with new values. */
1030 : pos = NULL_TREE;
1031 35428 : for (ix = GOMP_DIM_MAX; ix--;)
1032 26571 : pos = tree_cons (purpose[ix],
1033 26571 : build_int_cst (integer_type_node, dims[ix]), pos);
1034 8857 : oacc_replace_fn_attrib (fn, pos);
1035 : }
1036 9879 : }
1037 :
1038 : /* Create an empty OpenACC loop structure at LOC. */
1039 :
1040 : static oacc_loop *
1041 21347 : new_oacc_loop_raw (oacc_loop *parent, location_t loc)
1042 : {
1043 10835 : oacc_loop *loop = XCNEW (oacc_loop);
1044 :
1045 21347 : loop->parent = parent;
1046 :
1047 10835 : if (parent)
1048 : {
1049 10835 : loop->sibling = parent->child;
1050 10835 : parent->child = loop;
1051 : }
1052 :
1053 21347 : loop->loc = loc;
1054 21347 : return loop;
1055 : }
1056 :
1057 : /* Create an outermost, dummy OpenACC loop for offloaded function
1058 : DECL. */
1059 :
1060 : static oacc_loop *
1061 9879 : new_oacc_loop_outer (tree decl)
1062 : {
1063 9879 : return new_oacc_loop_raw (NULL, DECL_SOURCE_LOCATION (decl));
1064 : }
1065 :
1066 : /* Start a new OpenACC loop structure beginning at head marker HEAD.
1067 : Link into PARENT loop. Return the new loop. */
1068 :
1069 : static oacc_loop *
1070 9634 : new_oacc_loop (oacc_loop *parent, gcall *marker)
1071 : {
1072 9634 : oacc_loop *loop = new_oacc_loop_raw (parent, gimple_location (marker));
1073 :
1074 9634 : loop->marker = marker;
1075 :
1076 : /* TODO: This is where device_type flattening would occur for the loop
1077 : flags. */
1078 :
1079 9634 : loop->flags = TREE_INT_CST_LOW (gimple_call_arg (marker, 3));
1080 :
1081 9634 : tree chunk_size = integer_zero_node;
1082 9634 : if (loop->flags & OLF_GANG_STATIC)
1083 146 : chunk_size = gimple_call_arg (marker, 4);
1084 9634 : loop->chunk_size = chunk_size;
1085 :
1086 9634 : return loop;
1087 : }
1088 :
1089 : /* Create a dummy loop encompassing a call to a openACC routine.
1090 : Extract the routine's partitioning requirements. */
1091 :
1092 : static void
1093 1201 : new_oacc_loop_routine (oacc_loop *parent, gcall *call, tree decl, tree attrs)
1094 : {
1095 1201 : oacc_loop *loop = new_oacc_loop_raw (parent, gimple_location (call));
1096 1201 : int level = oacc_fn_attrib_level (attrs);
1097 :
1098 1201 : gcc_assert (level >= 0);
1099 :
1100 1201 : loop->marker = call;
1101 1201 : loop->routine = decl;
1102 1201 : loop->mask = ((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1)
1103 1201 : ^ (GOMP_DIM_MASK (level) - 1));
1104 1201 : }
1105 :
1106 : /* Finish off the current OpenACC loop ending at tail marker TAIL.
1107 : Return the parent loop. */
1108 :
1109 : static oacc_loop *
1110 9634 : finish_oacc_loop (oacc_loop *loop)
1111 : {
1112 : /* If the loop has been collapsed, don't partition it. */
1113 0 : if (loop->ifns.is_empty ())
1114 0 : loop->mask = loop->flags = 0;
1115 9634 : return loop->parent;
1116 : }
1117 :
1118 : /* Free all OpenACC loop structures within LOOP (inclusive). */
1119 :
1120 : static void
1121 21347 : free_oacc_loop (oacc_loop *loop)
1122 : {
1123 21347 : if (loop->sibling)
1124 2194 : free_oacc_loop (loop->sibling);
1125 21347 : if (loop->child)
1126 8641 : free_oacc_loop (loop->child);
1127 :
1128 21347 : loop->ifns.release ();
1129 21347 : free (loop);
1130 21347 : }
1131 :
1132 : /* Dump out the OpenACC loop head or tail beginning at FROM. */
1133 :
1134 : static void
1135 238 : dump_oacc_loop_part (FILE *file, gcall *from, int depth,
1136 : const char *title, int level)
1137 : {
1138 238 : enum ifn_unique_kind kind
1139 238 : = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (from, 0));
1140 :
1141 238 : fprintf (file, "%*s%s-%d:\n", depth * 2, "", title, level);
1142 238 : for (gimple_stmt_iterator gsi = gsi_for_stmt (from);;)
1143 : {
1144 719 : gimple *stmt = gsi_stmt (gsi);
1145 :
1146 719 : if (gimple_call_internal_p (stmt, IFN_UNIQUE))
1147 : {
1148 719 : enum ifn_unique_kind k
1149 719 : = ((enum ifn_unique_kind) TREE_INT_CST_LOW
1150 719 : (gimple_call_arg (stmt, 0)));
1151 :
1152 719 : if (k == kind && stmt != from)
1153 : break;
1154 : }
1155 481 : print_gimple_stmt (file, stmt, depth * 2 + 2);
1156 :
1157 481 : gsi_next (&gsi);
1158 962 : while (gsi_end_p (gsi))
1159 962 : gsi = gsi_start_bb (single_succ (gsi_bb (gsi)));
1160 : }
1161 238 : }
1162 :
1163 : /* Dump OpenACC loop LOOP, its children, and its siblings. */
1164 :
1165 : static void
1166 183 : dump_oacc_loop (FILE *file, oacc_loop *loop, int depth)
1167 : {
1168 222 : int ix;
1169 :
1170 222 : fprintf (file, "%*sLoop %x(%x) %s:%u\n", depth * 2, "",
1171 : loop->flags, loop->mask,
1172 222 : LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc));
1173 :
1174 222 : if (loop->marker)
1175 108 : print_gimple_stmt (file, loop->marker, depth * 2);
1176 :
1177 222 : if (loop->routine)
1178 48 : fprintf (file, "%*sRoutine %s:%u:%s\n",
1179 48 : depth * 2, "", DECL_SOURCE_FILE (loop->routine),
1180 96 : DECL_SOURCE_LINE (loop->routine),
1181 48 : IDENTIFIER_POINTER (DECL_NAME (loop->routine)));
1182 :
1183 888 : for (ix = GOMP_DIM_GANG; ix != GOMP_DIM_MAX; ix++)
1184 666 : if (loop->heads[ix])
1185 119 : dump_oacc_loop_part (file, loop->heads[ix], depth, "Head", ix);
1186 888 : for (ix = GOMP_DIM_MAX; ix--;)
1187 666 : if (loop->tails[ix])
1188 119 : dump_oacc_loop_part (file, loop->tails[ix], depth, "Tail", ix);
1189 :
1190 222 : if (loop->child)
1191 69 : dump_oacc_loop (file, loop->child, depth + 1);
1192 222 : if (loop->sibling)
1193 : dump_oacc_loop (file, loop->sibling, depth);
1194 183 : }
1195 :
1196 : void debug_oacc_loop (oacc_loop *);
1197 :
1198 : /* Dump loops to stderr. */
1199 :
1200 : DEBUG_FUNCTION void
1201 0 : debug_oacc_loop (oacc_loop *loop)
1202 : {
1203 0 : dump_oacc_loop (stderr, loop, 0);
1204 0 : }
1205 :
1206 : /* Provide diagnostics on OpenACC loop LOOP, its children, and its
1207 : siblings. */
1208 :
1209 : static void
1210 2741 : inform_oacc_loop (const oacc_loop *loop)
1211 : {
1212 1796 : const char *gang
1213 2741 : = loop->mask & GOMP_DIM_MASK (GOMP_DIM_GANG) ? " gang" : "";
1214 2226 : const char *worker
1215 2741 : = loop->mask & GOMP_DIM_MASK (GOMP_DIM_WORKER) ? " worker" : "";
1216 1747 : const char *vector
1217 2741 : = loop->mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR) ? " vector" : "";
1218 2741 : const char *seq = loop->mask == 0 ? " seq" : "";
1219 2741 : const dump_user_location_t loc
1220 2741 : = dump_user_location_t::from_location_t (loop->loc);
1221 2741 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, loc,
1222 : "assigned OpenACC%s%s%s%s loop parallelism\n", gang, worker,
1223 : vector, seq);
1224 :
1225 2741 : if (loop->child)
1226 698 : inform_oacc_loop (loop->child);
1227 2741 : if (loop->sibling)
1228 280 : inform_oacc_loop (loop->sibling);
1229 2741 : }
1230 :
1231 : /* DFS walk of basic blocks BB onwards, creating OpenACC loop
1232 : structures as we go. By construction these loops are properly
1233 : nested. */
1234 :
1235 : static void
1236 176002 : oacc_loop_discover_walk (oacc_loop *loop, basic_block bb)
1237 : {
1238 176002 : int marker = 0;
1239 176002 : int remaining = 0;
1240 :
1241 176002 : if (bb->flags & BB_VISITED)
1242 40061 : return;
1243 :
1244 135941 : follow:
1245 202126 : bb->flags |= BB_VISITED;
1246 :
1247 : /* Scan for loop markers. */
1248 788879 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
1249 384627 : gsi_next (&gsi))
1250 : {
1251 384627 : gimple *stmt = gsi_stmt (gsi);
1252 :
1253 384627 : if (!is_gimple_call (stmt))
1254 220292 : continue;
1255 :
1256 169098 : gcall *call = as_a <gcall *> (stmt);
1257 :
1258 : /* If this is a routine, make a dummy loop for it. */
1259 169098 : if (tree decl = gimple_call_fndecl (call))
1260 4761 : if (tree attrs = oacc_get_fn_attrib (decl))
1261 : {
1262 1201 : gcc_assert (!marker);
1263 1201 : new_oacc_loop_routine (loop, call, decl, attrs);
1264 : }
1265 :
1266 169098 : if (!gimple_call_internal_p (call))
1267 4763 : continue;
1268 :
1269 164335 : switch (gimple_call_internal_fn (call))
1270 : {
1271 : default:
1272 : break;
1273 :
1274 46986 : case IFN_GOACC_LOOP:
1275 46986 : case IFN_GOACC_TILE:
1276 : /* Record the abstraction function, so we can manipulate it
1277 : later. */
1278 46986 : loop->ifns.safe_push (call);
1279 46986 : break;
1280 :
1281 85483 : case IFN_UNIQUE:
1282 85483 : enum ifn_unique_kind kind
1283 85483 : = (enum ifn_unique_kind) (TREE_INT_CST_LOW
1284 85483 : (gimple_call_arg (call, 0)));
1285 85483 : if (kind == IFN_UNIQUE_OACC_HEAD_MARK
1286 85483 : || kind == IFN_UNIQUE_OACC_TAIL_MARK)
1287 : {
1288 52246 : if (gimple_call_num_args (call) == 2)
1289 : {
1290 19268 : gcc_assert (marker && !remaining);
1291 19268 : marker = 0;
1292 19268 : if (kind == IFN_UNIQUE_OACC_TAIL_MARK)
1293 19268 : loop = finish_oacc_loop (loop);
1294 : else
1295 9634 : loop->head_end = call;
1296 : }
1297 : else
1298 : {
1299 32978 : int count = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
1300 :
1301 32978 : if (!marker)
1302 : {
1303 19268 : if (kind == IFN_UNIQUE_OACC_HEAD_MARK)
1304 9634 : loop = new_oacc_loop (loop, call);
1305 : remaining = count;
1306 : }
1307 32978 : gcc_assert (count == remaining);
1308 32978 : if (remaining)
1309 : {
1310 32978 : remaining--;
1311 32978 : if (kind == IFN_UNIQUE_OACC_HEAD_MARK)
1312 16489 : loop->heads[marker] = call;
1313 : else
1314 16489 : loop->tails[remaining] = call;
1315 : }
1316 32978 : marker++;
1317 : }
1318 : }
1319 : }
1320 : }
1321 202126 : if (remaining || marker)
1322 : {
1323 66185 : bb = single_succ (bb);
1324 66185 : gcc_assert (single_pred_p (bb) && !(bb->flags & BB_VISITED));
1325 66185 : goto follow;
1326 : }
1327 :
1328 : /* Walk successor blocks. */
1329 135941 : edge e;
1330 135941 : edge_iterator ei;
1331 :
1332 302064 : FOR_EACH_EDGE (e, ei, bb->succs)
1333 166123 : oacc_loop_discover_walk (loop, e->dest);
1334 : }
1335 :
1336 : /* LOOP is the first sibling. Reverse the order in place and return
1337 : the new first sibling. Recurse to child loops. */
1338 :
1339 : static oacc_loop *
1340 18520 : oacc_loop_sibling_nreverse (oacc_loop *loop)
1341 : {
1342 18520 : oacc_loop *last = NULL;
1343 20714 : do
1344 : {
1345 20714 : if (loop->child)
1346 8641 : loop->child = oacc_loop_sibling_nreverse (loop->child);
1347 :
1348 20714 : oacc_loop *next = loop->sibling;
1349 20714 : loop->sibling = last;
1350 20714 : last = loop;
1351 20714 : loop = next;
1352 : }
1353 20714 : while (loop);
1354 :
1355 18520 : return last;
1356 : }
1357 :
1358 : /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
1359 : the current function. */
1360 :
1361 : static oacc_loop *
1362 9879 : oacc_loop_discovery ()
1363 : {
1364 : /* Clear basic block flags, in particular BB_VISITED which we're going to use
1365 : in the following. */
1366 9879 : clear_bb_flags ();
1367 :
1368 9879 : oacc_loop *top = new_oacc_loop_outer (current_function_decl);
1369 9879 : oacc_loop_discover_walk (top, ENTRY_BLOCK_PTR_FOR_FN (cfun));
1370 :
1371 : /* The siblings were constructed in reverse order, reverse them so
1372 : that diagnostics come out in an unsurprising order. */
1373 9879 : top = oacc_loop_sibling_nreverse (top);
1374 :
1375 9879 : return top;
1376 : }
1377 :
1378 : /* Transform the abstract internal function markers starting at FROM
1379 : to be for partitioning level LEVEL. Stop when we meet another HEAD
1380 : or TAIL marker. */
1381 :
1382 : static void
1383 25614 : oacc_loop_xform_head_tail (gcall *from, int level)
1384 : {
1385 25614 : enum ifn_unique_kind kind
1386 25614 : = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (from, 0));
1387 25614 : tree replacement = build_int_cst (unsigned_type_node, level);
1388 :
1389 25614 : for (gimple_stmt_iterator gsi = gsi_for_stmt (from);;)
1390 : {
1391 107435 : gimple *stmt = gsi_stmt (gsi);
1392 :
1393 107435 : if (gimple_call_internal_p (stmt, IFN_UNIQUE))
1394 : {
1395 77068 : enum ifn_unique_kind k
1396 : = ((enum ifn_unique_kind)
1397 77068 : TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
1398 :
1399 77068 : if (k == IFN_UNIQUE_OACC_FORK
1400 77068 : || k == IFN_UNIQUE_OACC_JOIN
1401 77068 : || k == IFN_UNIQUE_OACC_PRIVATE)
1402 25840 : *gimple_call_arg_ptr (stmt, 2) = replacement;
1403 51228 : else if (k == kind && stmt != from)
1404 : break;
1405 : }
1406 30367 : else if (gimple_call_internal_p (stmt, IFN_GOACC_REDUCTION))
1407 19228 : *gimple_call_arg_ptr (stmt, 3) = replacement;
1408 81821 : update_stmt (stmt);
1409 :
1410 81821 : gsi_next (&gsi);
1411 133275 : while (gsi_end_p (gsi))
1412 102908 : gsi = gsi_start_bb (single_succ (gsi_bb (gsi)));
1413 : }
1414 25614 : }
1415 :
1416 : /* Process the discovered OpenACC loops, setting the correct
1417 : partitioning level etc. */
1418 :
1419 : static void
1420 20714 : oacc_loop_process (oacc_loop *loop, int fn_level)
1421 : {
1422 20714 : if (loop->child)
1423 8641 : oacc_loop_process (loop->child, fn_level);
1424 :
1425 20714 : if (loop->mask && !loop->routine)
1426 : {
1427 8645 : int ix;
1428 8645 : tree mask_arg = build_int_cst (unsigned_type_node, loop->mask);
1429 8645 : tree e_mask_arg = build_int_cst (unsigned_type_node, loop->e_mask);
1430 8645 : tree chunk_arg = loop->chunk_size;
1431 8645 : gcall *call;
1432 :
1433 43708 : for (ix = 0; loop->ifns.iterate (ix, &call); ix++)
1434 : {
1435 35063 : switch (gimple_call_internal_fn (call))
1436 : {
1437 34857 : case IFN_GOACC_LOOP:
1438 34857 : {
1439 34857 : bool is_e = gimple_call_arg (call, 5) == integer_minus_one_node;
1440 69335 : gimple_call_set_arg (call, 5, is_e ? e_mask_arg : mask_arg);
1441 34857 : if (!is_e)
1442 34478 : gimple_call_set_arg (call, 4, chunk_arg);
1443 : }
1444 : break;
1445 :
1446 206 : case IFN_GOACC_TILE:
1447 206 : gimple_call_set_arg (call, 3, mask_arg);
1448 206 : gimple_call_set_arg (call, 4, e_mask_arg);
1449 206 : break;
1450 :
1451 0 : default:
1452 0 : gcc_unreachable ();
1453 : }
1454 35063 : update_stmt (call);
1455 : }
1456 :
1457 8645 : unsigned dim = GOMP_DIM_GANG;
1458 8645 : unsigned mask = loop->mask | loop->e_mask;
1459 21452 : for (ix = 0; ix != GOMP_DIM_MAX && mask; ix++)
1460 : {
1461 25113 : while (!(GOMP_DIM_MASK (dim) & mask))
1462 12306 : dim++;
1463 :
1464 12807 : oacc_loop_xform_head_tail (loop->heads[ix], dim);
1465 12807 : oacc_loop_xform_head_tail (loop->tails[ix], dim);
1466 :
1467 12807 : mask ^= GOMP_DIM_MASK (dim);
1468 : }
1469 : }
1470 :
1471 20714 : if (loop->sibling)
1472 2194 : oacc_loop_process (loop->sibling, fn_level);
1473 :
1474 :
1475 : /* OpenACC 2.6, 2.9.11. "reduction clause" places a restriction such that
1476 : "The 'reduction' clause may not be specified on an orphaned 'loop'
1477 : construct with the 'gang' clause, or on an orphaned 'loop' construct that
1478 : will generate gang parallelism in a procedure that is compiled with the
1479 : 'routine gang' clause." */
1480 20714 : if (fn_level == GOMP_DIM_GANG
1481 624 : && (loop->mask & GOMP_DIM_MASK (GOMP_DIM_GANG))
1482 209 : && (loop->flags & OLF_REDUCTION))
1483 106 : error_at (loop->loc,
1484 : "gang reduction on an orphan loop");
1485 20714 : }
1486 :
1487 : /* Walk the OpenACC loop hierarchy checking and assigning the
1488 : programmer-specified partitionings. OUTER_MASK is the partitioning
1489 : this loop is contained within. Return mask of partitioning
1490 : encountered. If any auto loops are discovered, set GOMP_DIM_MAX
1491 : bit. */
1492 :
1493 : static unsigned
1494 20714 : oacc_loop_fixed_partitions (oacc_loop *loop, unsigned outer_mask)
1495 : {
1496 20714 : unsigned this_mask = loop->mask;
1497 20714 : unsigned mask_all = 0;
1498 20714 : bool noisy = true;
1499 :
1500 : #ifdef ACCEL_COMPILER
1501 : /* When device_type is supported, we want the device compiler to be
1502 : noisy, if the loop parameters are device_type-specific. */
1503 : noisy = false;
1504 : #endif
1505 :
1506 20714 : if (!loop->routine)
1507 : {
1508 19513 : bool auto_par = (loop->flags & OLF_AUTO) != 0;
1509 19513 : bool seq_par = (loop->flags & OLF_SEQ) != 0;
1510 19513 : bool tiling = (loop->flags & OLF_TILE) != 0;
1511 :
1512 19513 : this_mask = ((loop->flags >> OLF_DIM_BASE)
1513 : & (GOMP_DIM_MASK (GOMP_DIM_MAX) - 1));
1514 :
1515 : /* Apply auto partitioning if this is a non-partitioned regular
1516 : loop, or (no more than) single axis tiled loop. */
1517 39026 : bool maybe_auto
1518 19513 : = !seq_par && this_mask == (tiling ? this_mask & -this_mask : 0);
1519 :
1520 19513 : if ((this_mask != 0) + auto_par + seq_par > 1)
1521 : {
1522 170 : if (noisy)
1523 250 : error_at (loop->loc,
1524 : seq_par
1525 : ? G_("%<seq%> overrides other OpenACC loop specifiers")
1526 : : G_("%<auto%> conflicts with other OpenACC loop "
1527 : "specifiers"));
1528 170 : maybe_auto = false;
1529 170 : loop->flags &= ~OLF_AUTO;
1530 170 : if (seq_par)
1531 : {
1532 90 : loop->flags
1533 90 : &= ~((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1) << OLF_DIM_BASE);
1534 90 : this_mask = 0;
1535 : }
1536 : }
1537 :
1538 19433 : if (maybe_auto && (loop->flags & OLF_INDEPENDENT))
1539 : {
1540 5699 : loop->flags |= OLF_AUTO;
1541 5699 : mask_all |= GOMP_DIM_MASK (GOMP_DIM_MAX);
1542 : }
1543 : }
1544 :
1545 20714 : if (this_mask & outer_mask)
1546 : {
1547 248 : const oacc_loop *outer;
1548 350 : for (outer = loop->parent; outer; outer = outer->parent)
1549 248 : if ((outer->mask | outer->e_mask) & this_mask)
1550 : break;
1551 :
1552 248 : if (noisy)
1553 : {
1554 248 : if (outer)
1555 : {
1556 146 : error_at (loop->loc,
1557 146 : loop->routine
1558 : ? G_("routine call uses same OpenACC parallelism"
1559 : " as containing loop")
1560 : : G_("inner loop uses same OpenACC parallelism"
1561 : " as containing loop"));
1562 146 : inform (outer->loc, "containing loop here");
1563 : }
1564 : else
1565 102 : error_at (loop->loc,
1566 102 : loop->routine
1567 : ? G_("routine call uses OpenACC parallelism disallowed"
1568 : " by containing routine")
1569 : : G_("loop uses OpenACC parallelism disallowed"
1570 : " by containing routine"));
1571 :
1572 248 : if (loop->routine)
1573 154 : inform (DECL_SOURCE_LOCATION (loop->routine),
1574 : "routine %qD declared here", loop->routine);
1575 : }
1576 248 : this_mask &= ~outer_mask;
1577 : }
1578 : else
1579 : {
1580 20466 : unsigned outermost = least_bit_hwi (this_mask);
1581 :
1582 20466 : if (outermost && outermost <= outer_mask)
1583 : {
1584 40 : if (noisy)
1585 : {
1586 40 : error_at (loop->loc,
1587 : "incorrectly nested OpenACC loop parallelism");
1588 :
1589 40 : const oacc_loop *outer;
1590 40 : for (outer = loop->parent;
1591 40 : outer->flags && outer->flags < outermost;
1592 0 : outer = outer->parent)
1593 0 : continue;
1594 40 : inform (outer->loc, "containing loop here");
1595 0 : }
1596 :
1597 40 : this_mask &= ~outermost;
1598 : }
1599 : }
1600 :
1601 20714 : mask_all |= this_mask;
1602 :
1603 20714 : if (loop->flags & OLF_TILE)
1604 : {
1605 : /* When tiling, vector goes to the element loop, and failing
1606 : that we put worker there. The std doesn't contemplate
1607 : specifying all three. We choose to put worker and vector on
1608 : the element loops in that case. */
1609 136 : unsigned this_e_mask = this_mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR);
1610 136 : if (!this_e_mask || this_mask & GOMP_DIM_MASK (GOMP_DIM_GANG))
1611 120 : this_e_mask |= this_mask & GOMP_DIM_MASK (GOMP_DIM_WORKER);
1612 :
1613 136 : loop->e_mask = this_e_mask;
1614 136 : this_mask ^= this_e_mask;
1615 : }
1616 :
1617 20714 : loop->mask = this_mask;
1618 :
1619 20714 : if (dump_file)
1620 222 : fprintf (dump_file, "Loop %s:%d user specified %d & %d\n",
1621 444 : LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc),
1622 : loop->mask, loop->e_mask);
1623 :
1624 20714 : if (loop->child)
1625 : {
1626 8641 : unsigned tmp_mask = outer_mask | this_mask | loop->e_mask;
1627 8641 : loop->inner = oacc_loop_fixed_partitions (loop->child, tmp_mask);
1628 8641 : mask_all |= loop->inner;
1629 : }
1630 :
1631 20714 : if (loop->sibling)
1632 2194 : mask_all |= oacc_loop_fixed_partitions (loop->sibling, outer_mask);
1633 :
1634 20714 : return mask_all;
1635 : }
1636 :
1637 : /* Walk the OpenACC loop hierarchy to assign auto-partitioned loops.
1638 : OUTER_MASK is the partitioning this loop is contained within.
1639 : OUTER_ASSIGN is true if an outer loop is being auto-partitioned.
1640 : Return the cumulative partitioning used by this loop, siblings and
1641 : children. */
1642 :
1643 : static unsigned
1644 10192 : oacc_loop_auto_partitions (oacc_loop *loop, unsigned outer_mask,
1645 : bool outer_assign)
1646 : {
1647 10192 : bool assign = (loop->flags & OLF_AUTO) && (loop->flags & OLF_INDEPENDENT);
1648 10192 : bool noisy = true;
1649 10192 : bool tiling = loop->flags & OLF_TILE;
1650 :
1651 : #ifdef ACCEL_COMPILER
1652 : /* When device_type is supported, we want the device compiler to be
1653 : noisy, if the loop parameters are device_type-specific. */
1654 : noisy = false;
1655 : #endif
1656 :
1657 10192 : if (assign && (!outer_assign || loop->inner))
1658 : {
1659 : /* Allocate outermost and non-innermost loops at the outermost
1660 : non-innermost available level. */
1661 : unsigned this_mask = GOMP_DIM_MASK (GOMP_DIM_GANG);
1662 :
1663 : /* Find the first outermost available partition. */
1664 6659 : while (this_mask <= outer_mask)
1665 1923 : this_mask <<= 1;
1666 :
1667 : /* Grab two axes if tiling, and we've not assigned anything */
1668 4736 : if (tiling && !(loop->mask | loop->e_mask))
1669 94 : this_mask |= this_mask << 1;
1670 :
1671 : /* Prohibit the innermost partitioning at the moment. */
1672 4736 : this_mask &= GOMP_DIM_MASK (GOMP_DIM_MAX - 1) - 1;
1673 :
1674 : /* Don't use any dimension explicitly claimed by an inner loop. */
1675 4736 : this_mask &= ~loop->inner;
1676 :
1677 4736 : if (tiling && !loop->e_mask)
1678 : {
1679 : /* If we got two axes, allocate the inner one to the element
1680 : loop. */
1681 98 : loop->e_mask = this_mask & (this_mask << 1);
1682 98 : this_mask ^= loop->e_mask;
1683 : }
1684 :
1685 4736 : loop->mask |= this_mask;
1686 : }
1687 :
1688 10192 : if (loop->child)
1689 : {
1690 5360 : unsigned tmp_mask = outer_mask | loop->mask | loop->e_mask;
1691 5360 : loop->inner = oacc_loop_auto_partitions (loop->child, tmp_mask,
1692 5360 : outer_assign | assign);
1693 : }
1694 :
1695 10192 : if (assign && (!loop->mask || (tiling && !loop->e_mask) || !outer_assign))
1696 : {
1697 : /* Allocate the loop at the innermost available level. Note
1698 : that we do this even if we already assigned this loop the
1699 : outermost available level above. That way we'll partition
1700 : this along 2 axes, if they are available. */
1701 5044 : unsigned this_mask = 0;
1702 :
1703 : /* Determine the outermost partitioning used within this loop. */
1704 5044 : this_mask = loop->inner | GOMP_DIM_MASK (GOMP_DIM_MAX);
1705 5044 : this_mask = least_bit_hwi (this_mask);
1706 :
1707 : /* Pick the partitioning just inside that one. */
1708 5044 : this_mask >>= 1;
1709 :
1710 : /* And avoid picking one use by an outer loop. */
1711 5044 : this_mask &= ~outer_mask;
1712 :
1713 : /* If tiling and we failed completely above, grab the next one
1714 : too. Making sure it doesn't hit an outer loop. */
1715 5044 : if (tiling)
1716 : {
1717 110 : this_mask &= ~(loop->e_mask | loop->mask);
1718 110 : unsigned tile_mask = ((this_mask >> 1)
1719 110 : & ~(outer_mask | loop->e_mask | loop->mask));
1720 :
1721 110 : if (tile_mask || loop->mask)
1722 : {
1723 102 : loop->e_mask |= this_mask;
1724 102 : this_mask = tile_mask;
1725 : }
1726 110 : if (!loop->e_mask && noisy)
1727 8 : warning_at (loop->loc, 0,
1728 : "insufficient partitioning available"
1729 : " to parallelize element loop");
1730 : }
1731 :
1732 5044 : loop->mask |= this_mask;
1733 5044 : if (!loop->mask && noisy)
1734 1078 : warning_at (loop->loc, 0,
1735 : tiling
1736 : ? G_("insufficient partitioning available"
1737 : " to parallelize tile loop")
1738 : : G_("insufficient partitioning available"
1739 : " to parallelize loop"));
1740 : }
1741 :
1742 5699 : if (assign && dump_file)
1743 41 : fprintf (dump_file, "Auto loop %s:%d assigned %d & %d\n",
1744 82 : LOCATION_FILE (loop->loc), LOCATION_LINE (loop->loc),
1745 : loop->mask, loop->e_mask);
1746 :
1747 10192 : unsigned inner_mask = 0;
1748 :
1749 10192 : if (loop->sibling)
1750 1714 : inner_mask |= oacc_loop_auto_partitions (loop->sibling,
1751 : outer_mask, outer_assign);
1752 :
1753 10192 : inner_mask |= loop->inner | loop->mask | loop->e_mask;
1754 :
1755 10192 : return inner_mask;
1756 : }
1757 :
1758 : /* Walk the OpenACC loop hierarchy to check and assign partitioning
1759 : axes. Return mask of partitioning. */
1760 :
1761 : static unsigned
1762 9879 : oacc_loop_partition (oacc_loop *loop, unsigned outer_mask)
1763 : {
1764 9879 : unsigned mask_all = oacc_loop_fixed_partitions (loop, outer_mask);
1765 :
1766 9879 : if (mask_all & GOMP_DIM_MASK (GOMP_DIM_MAX))
1767 : {
1768 3118 : mask_all ^= GOMP_DIM_MASK (GOMP_DIM_MAX);
1769 3118 : mask_all |= oacc_loop_auto_partitions (loop, outer_mask, false);
1770 : }
1771 9879 : return mask_all;
1772 : }
1773 :
1774 : /* Default fork/join early expander. Delete the function calls if
1775 : there is no RTL expander. */
1776 :
1777 : bool
1778 25614 : default_goacc_fork_join (gcall *ARG_UNUSED (call),
1779 : const int *ARG_UNUSED (dims), bool is_fork)
1780 : {
1781 25614 : if (is_fork)
1782 12807 : return targetm.have_oacc_fork ();
1783 : else
1784 12807 : return targetm.have_oacc_join ();
1785 : }
1786 :
1787 : /* Default goacc.reduction early expander.
1788 :
1789 : LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
1790 : If RES_PTR is not integer-zerop:
1791 : SETUP - emit 'LHS = *RES_PTR', LHS = NULL
1792 : TEARDOWN - emit '*RES_PTR = VAR'
1793 : If LHS is not NULL
1794 : emit 'LHS = VAR' */
1795 :
1796 : void
1797 30884 : default_goacc_reduction (gcall *call)
1798 : {
1799 30884 : unsigned code = (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call, 0));
1800 30884 : gimple_stmt_iterator gsi = gsi_for_stmt (call);
1801 30884 : tree lhs = gimple_call_lhs (call);
1802 30884 : tree var = gimple_call_arg (call, 2);
1803 30884 : gimple_seq seq = NULL;
1804 :
1805 30884 : if (code == IFN_GOACC_REDUCTION_SETUP
1806 30884 : || code == IFN_GOACC_REDUCTION_TEARDOWN)
1807 : {
1808 : /* Setup and Teardown need to copy from/to the receiver object,
1809 : if there is one. */
1810 15442 : tree ref_to_res = gimple_call_arg (call, 1);
1811 :
1812 15442 : if (!integer_zerop (ref_to_res))
1813 : {
1814 5086 : tree dst = build_simple_mem_ref (ref_to_res);
1815 5086 : tree src = var;
1816 :
1817 5086 : if (code == IFN_GOACC_REDUCTION_SETUP)
1818 : {
1819 2543 : src = dst;
1820 2543 : dst = lhs;
1821 2543 : lhs = NULL;
1822 : }
1823 5086 : gimple_seq_add_stmt (&seq, gimple_build_assign (dst, src));
1824 : }
1825 : }
1826 :
1827 : /* Copy VAR to LHS, if there is an LHS. */
1828 30884 : if (lhs)
1829 26734 : gimple_seq_add_stmt (&seq, gimple_build_assign (lhs, var));
1830 :
1831 30884 : gsi_replace_with_seq (&gsi, seq, true);
1832 30884 : }
1833 :
1834 : struct var_decl_rewrite_info
1835 : {
1836 : gimple *stmt;
1837 : hash_map<tree, tree> *adjusted_vars;
1838 : bool avoid_pointer_conversion;
1839 : bool modified;
1840 : };
1841 :
1842 : /* Helper function for execute_oacc_device_lower. Rewrite VAR_DECLs (by
1843 : themselves or wrapped in various other nodes) according to ADJUSTED_VARS in
1844 : the var_decl_rewrite_info pointed to via DATA. Used as part of coercing
1845 : gang-private variables in OpenACC offload regions to reside in GPU shared
1846 : memory. */
1847 :
1848 : static tree
1849 0 : oacc_rewrite_var_decl (tree *tp, int *walk_subtrees, void *data)
1850 : {
1851 0 : walk_stmt_info *wi = (walk_stmt_info *) data;
1852 0 : var_decl_rewrite_info *info = (var_decl_rewrite_info *) wi->info;
1853 :
1854 0 : if (TREE_CODE (*tp) == ADDR_EXPR)
1855 : {
1856 0 : tree arg = TREE_OPERAND (*tp, 0);
1857 0 : tree *new_arg = info->adjusted_vars->get (arg);
1858 :
1859 0 : if (new_arg)
1860 : {
1861 0 : if (info->avoid_pointer_conversion)
1862 : {
1863 0 : *tp = build_fold_addr_expr (*new_arg);
1864 0 : info->modified = true;
1865 0 : *walk_subtrees = 0;
1866 : }
1867 : else
1868 : {
1869 0 : gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
1870 0 : tree repl = build_fold_addr_expr (*new_arg);
1871 0 : gimple *stmt1
1872 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
1873 0 : tree conv = convert_to_pointer (TREE_TYPE (*tp),
1874 : gimple_assign_lhs (stmt1));
1875 0 : gimple *stmt2
1876 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (*tp)), conv);
1877 0 : gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
1878 0 : gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
1879 0 : *tp = gimple_assign_lhs (stmt2);
1880 0 : info->modified = true;
1881 0 : *walk_subtrees = 0;
1882 : }
1883 : }
1884 : }
1885 0 : else if (TREE_CODE (*tp) == COMPONENT_REF || TREE_CODE (*tp) == ARRAY_REF)
1886 : {
1887 0 : tree *base = &TREE_OPERAND (*tp, 0);
1888 :
1889 0 : while (TREE_CODE (*base) == COMPONENT_REF
1890 0 : || TREE_CODE (*base) == ARRAY_REF)
1891 0 : base = &TREE_OPERAND (*base, 0);
1892 :
1893 0 : if (TREE_CODE (*base) != VAR_DECL)
1894 : return NULL;
1895 :
1896 0 : tree *new_decl = info->adjusted_vars->get (*base);
1897 0 : if (!new_decl)
1898 : return NULL;
1899 :
1900 0 : int base_quals = TYPE_QUALS (TREE_TYPE (*new_decl));
1901 0 : tree field = TREE_OPERAND (*tp, 1);
1902 :
1903 : /* Adjust the type of the field. */
1904 0 : int field_quals = TYPE_QUALS (TREE_TYPE (field));
1905 0 : if (TREE_CODE (field) == FIELD_DECL && field_quals != base_quals)
1906 : {
1907 0 : tree *field_type = &TREE_TYPE (field);
1908 0 : while (TREE_CODE (*field_type) == ARRAY_TYPE)
1909 0 : field_type = &TREE_TYPE (*field_type);
1910 0 : field_quals |= base_quals;
1911 0 : *field_type = build_qualified_type (*field_type, field_quals);
1912 : }
1913 :
1914 : /* Adjust the type of the component ref itself. */
1915 0 : tree comp_type = TREE_TYPE (*tp);
1916 0 : int comp_quals = TYPE_QUALS (comp_type);
1917 0 : if (TREE_CODE (*tp) == COMPONENT_REF && comp_quals != base_quals)
1918 : {
1919 0 : comp_quals |= base_quals;
1920 0 : TREE_TYPE (*tp)
1921 0 : = build_qualified_type (comp_type, comp_quals);
1922 : }
1923 :
1924 0 : *base = *new_decl;
1925 0 : info->modified = true;
1926 0 : }
1927 0 : else if (VAR_P (*tp))
1928 : {
1929 0 : tree *new_decl = info->adjusted_vars->get (*tp);
1930 0 : if (new_decl)
1931 : {
1932 0 : *tp = *new_decl;
1933 0 : info->modified = true;
1934 : }
1935 : }
1936 :
1937 : return NULL_TREE;
1938 : }
1939 :
1940 : /* Return TRUE if CALL is a call to a builtin atomic/sync operation. */
1941 :
1942 : static bool
1943 0 : is_sync_builtin_call (gcall *call)
1944 : {
1945 0 : tree callee = gimple_call_fndecl (call);
1946 :
1947 0 : if (callee != NULL_TREE
1948 0 : && gimple_call_builtin_p (call, BUILT_IN_NORMAL))
1949 0 : switch (DECL_FUNCTION_CODE (callee))
1950 : {
1951 : #undef DEF_SYNC_BUILTIN
1952 : #define DEF_SYNC_BUILTIN(ENUM, NAME, TYPE, ATTRS) case ENUM:
1953 : #include "sync-builtins.def"
1954 : #undef DEF_SYNC_BUILTIN
1955 0 : return true;
1956 :
1957 : default:
1958 : ;
1959 : }
1960 :
1961 : return false;
1962 : }
1963 :
1964 : /* Main entry point for oacc transformations which run on the device
1965 : compiler after LTO, so we know what the target device is at this
1966 : point (including the host fallback). */
1967 :
1968 : static unsigned int
1969 15337 : execute_oacc_loop_designation ()
1970 : {
1971 15337 : tree attrs = oacc_get_fn_attrib (current_function_decl);
1972 :
1973 15337 : if (!attrs)
1974 : /* Not an offloaded function. */
1975 : return 0;
1976 :
1977 : /* Parse the default dim argument exactly once. */
1978 9941 : if ((const void *)flag_openacc_dims != &flag_openacc_dims)
1979 : {
1980 2281 : oacc_parse_default_dims (flag_openacc_dims);
1981 2281 : flag_openacc_dims = (char *)&flag_openacc_dims;
1982 : }
1983 :
1984 9941 : bool is_oacc_parallel
1985 9941 : = (lookup_attribute ("oacc parallel",
1986 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
1987 9941 : bool is_oacc_kernels
1988 9941 : = (lookup_attribute ("oacc kernels",
1989 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
1990 9941 : bool is_oacc_serial
1991 9941 : = (lookup_attribute ("oacc serial",
1992 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
1993 9941 : bool is_oacc_parallel_kernels_parallelized
1994 9941 : = (lookup_attribute ("oacc parallel_kernels_parallelized",
1995 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
1996 9941 : bool is_oacc_parallel_kernels_gang_single
1997 9941 : = (lookup_attribute ("oacc parallel_kernels_gang_single",
1998 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
1999 9941 : int fn_level = oacc_fn_attrib_level (attrs);
2000 9941 : bool is_oacc_routine = (fn_level >= 0);
2001 9941 : gcc_checking_assert (is_oacc_parallel
2002 : + is_oacc_kernels
2003 : + is_oacc_serial
2004 : + is_oacc_parallel_kernels_parallelized
2005 : + is_oacc_parallel_kernels_gang_single
2006 : + is_oacc_routine
2007 : == 1);
2008 :
2009 9941 : bool is_oacc_kernels_parallelized
2010 9941 : = (lookup_attribute ("oacc kernels parallelized",
2011 9941 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
2012 9941 : if (is_oacc_kernels_parallelized)
2013 386 : gcc_checking_assert (is_oacc_kernels);
2014 :
2015 9941 : if (dump_file)
2016 : {
2017 154 : if (is_oacc_parallel)
2018 34 : fprintf (dump_file, "Function is OpenACC parallel offload\n");
2019 120 : else if (is_oacc_kernels)
2020 66 : fprintf (dump_file, "Function is %s OpenACC kernels offload\n",
2021 : (is_oacc_kernels_parallelized
2022 : ? "parallelized" : "unparallelized"));
2023 82 : else if (is_oacc_serial)
2024 26 : fprintf (dump_file, "Function is OpenACC serial offload\n");
2025 56 : else if (is_oacc_parallel_kernels_parallelized)
2026 0 : fprintf (dump_file, "Function is %s OpenACC kernels offload\n",
2027 : "parallel_kernels_parallelized");
2028 56 : else if (is_oacc_parallel_kernels_gang_single)
2029 0 : fprintf (dump_file, "Function is %s OpenACC kernels offload\n",
2030 : "parallel_kernels_gang_single");
2031 56 : else if (is_oacc_routine)
2032 56 : fprintf (dump_file, "Function is OpenACC routine level %d\n",
2033 : fn_level);
2034 : else
2035 0 : gcc_unreachable ();
2036 : }
2037 :
2038 : /* This doesn't belong into 'pass_oacc_loop_designation' conceptually, but
2039 : it's a convenient place, so... */
2040 9941 : if (is_oacc_routine)
2041 : {
2042 553 : tree attr = lookup_attribute ("omp declare target",
2043 553 : DECL_ATTRIBUTES (current_function_decl));
2044 553 : gcc_checking_assert (attr);
2045 553 : tree clauses = TREE_VALUE (attr);
2046 553 : gcc_checking_assert (clauses);
2047 :
2048 : /* Should this OpenACC routine be discarded? */
2049 553 : bool discard = false;
2050 :
2051 553 : tree clause_nohost = omp_find_clause (clauses, OMP_CLAUSE_NOHOST);
2052 553 : if (dump_file)
2053 56 : fprintf (dump_file,
2054 : "OpenACC routine '%s' %s '%s' clause.\n",
2055 56 : lang_hooks.decl_printable_name (current_function_decl, 2),
2056 : clause_nohost ? "has" : "doesn't have",
2057 56 : omp_clause_code_name[OMP_CLAUSE_NOHOST]);
2058 : /* Host compiler, 'nohost' clause? */
2059 : #ifndef ACCEL_COMPILER
2060 553 : if (clause_nohost)
2061 62 : discard = true;
2062 : #endif
2063 :
2064 553 : if (dump_file)
2065 112 : fprintf (dump_file,
2066 : "OpenACC routine '%s' %sdiscarded.\n",
2067 56 : lang_hooks.decl_printable_name (current_function_decl, 2),
2068 : discard ? "" : "not ");
2069 553 : if (discard)
2070 : {
2071 62 : TREE_ASM_WRITTEN (current_function_decl) = 1;
2072 62 : return TODO_discard_function;
2073 : }
2074 : }
2075 :
2076 : /* Unparallelized OpenACC kernels constructs must get launched as 1 x 1 x 1
2077 : kernels, so remove the parallelism dimensions function attributes
2078 : potentially set earlier on. */
2079 9879 : if (is_oacc_kernels && !is_oacc_kernels_parallelized)
2080 : {
2081 1265 : oacc_set_fn_attrib (current_function_decl, NULL, NULL);
2082 1265 : attrs = oacc_get_fn_attrib (current_function_decl);
2083 : }
2084 :
2085 : /* Discover, partition and process the loops. */
2086 9879 : oacc_loop *loops = oacc_loop_discovery ();
2087 :
2088 9879 : unsigned outer_mask = 0;
2089 9879 : if (is_oacc_routine)
2090 491 : outer_mask = GOMP_DIM_MASK (fn_level) - 1;
2091 9879 : unsigned used_mask = oacc_loop_partition (loops, outer_mask);
2092 : /* OpenACC kernels constructs are special: they currently don't use the
2093 : generic oacc_loop infrastructure and attribute/dimension processing. */
2094 9879 : if (is_oacc_kernels && is_oacc_kernels_parallelized)
2095 : {
2096 : /* Parallelized OpenACC kernels constructs use gang parallelism. See
2097 : also tree-parloops.cc:create_parallel_loop. */
2098 386 : used_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
2099 : }
2100 :
2101 9879 : int dims[GOMP_DIM_MAX];
2102 9879 : oacc_validate_dims (current_function_decl, attrs, dims, fn_level, used_mask);
2103 :
2104 9879 : if (dump_file)
2105 : {
2106 : const char *comma = "Compute dimensions [";
2107 456 : for (int ix = 0; ix != GOMP_DIM_MAX; ix++, comma = ", ")
2108 342 : fprintf (dump_file, "%s%d", comma, dims[ix]);
2109 114 : fprintf (dump_file, "]\n");
2110 : }
2111 :
2112 : /* Verify that for OpenACC 'kernels' decomposed "gang-single" parts we launch
2113 : a single gang only. */
2114 9879 : if (is_oacc_parallel_kernels_gang_single)
2115 109 : gcc_checking_assert (dims[GOMP_DIM_GANG] == 1);
2116 :
2117 9879 : oacc_loop_process (loops, fn_level);
2118 9879 : if (dump_file)
2119 : {
2120 114 : fprintf (dump_file, "OpenACC loops\n");
2121 114 : dump_oacc_loop (dump_file, loops, 0);
2122 114 : fprintf (dump_file, "\n");
2123 : }
2124 9879 : if (dump_enabled_p ())
2125 : {
2126 2280 : oacc_loop *l = loops;
2127 : /* OpenACC kernels constructs are special: they currently don't use the
2128 : generic oacc_loop infrastructure. */
2129 2280 : if (is_oacc_kernels)
2130 : {
2131 : /* Create a fake oacc_loop for diagnostic purposes. */
2132 633 : l = new_oacc_loop_raw (NULL,
2133 633 : DECL_SOURCE_LOCATION (current_function_decl));
2134 633 : l->mask = used_mask;
2135 : }
2136 : else
2137 : {
2138 : /* Skip the outermost, dummy OpenACC loop */
2139 1647 : l = l->child;
2140 : }
2141 2280 : if (l)
2142 1763 : inform_oacc_loop (l);
2143 2280 : if (is_oacc_kernels)
2144 633 : free_oacc_loop (l);
2145 : }
2146 :
2147 9879 : free_oacc_loop (loops);
2148 :
2149 9879 : return 0;
2150 : }
2151 :
2152 : static unsigned int
2153 15275 : execute_oacc_device_lower ()
2154 : {
2155 15275 : tree attrs = oacc_get_fn_attrib (current_function_decl);
2156 :
2157 15275 : if (!attrs)
2158 : /* Not an offloaded function. */
2159 : return 0;
2160 :
2161 : int dims[GOMP_DIM_MAX];
2162 39516 : for (unsigned i = 0; i < GOMP_DIM_MAX; i++)
2163 29637 : dims[i] = oacc_get_fn_dim_size (current_function_decl, i);
2164 :
2165 9879 : hash_map<tree, tree> adjusted_vars;
2166 :
2167 : /* Now lower internal loop functions to target-specific code
2168 : sequences. */
2169 9879 : basic_block bb;
2170 185094 : FOR_ALL_BB_FN (bb, cfun)
2171 933543 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
2172 : {
2173 583113 : gimple *stmt = gsi_stmt (gsi);
2174 583113 : if (!is_gimple_call (stmt))
2175 : {
2176 383306 : gsi_next (&gsi);
2177 383306 : continue;
2178 : }
2179 :
2180 199807 : gcall *call = as_a <gcall *> (stmt);
2181 199807 : if (!gimple_call_internal_p (call))
2182 : {
2183 4763 : gsi_next (&gsi);
2184 4763 : continue;
2185 : }
2186 :
2187 : /* Rewind to allow rescan. */
2188 195044 : gsi_prev (&gsi);
2189 195044 : bool rescan = false, remove = false;
2190 195044 : enum internal_fn ifn_code = gimple_call_internal_fn (call);
2191 :
2192 195044 : switch (ifn_code)
2193 : {
2194 : default: break;
2195 :
2196 284 : case IFN_GOACC_TILE:
2197 284 : oacc_xform_tile (call);
2198 284 : rescan = true;
2199 284 : break;
2200 :
2201 46702 : case IFN_GOACC_LOOP:
2202 46702 : oacc_xform_loop (call);
2203 46702 : rescan = true;
2204 46702 : break;
2205 :
2206 30884 : case IFN_GOACC_REDUCTION:
2207 : /* Mark the function for SSA renaming. */
2208 30884 : mark_virtual_operands_for_renaming (cfun);
2209 :
2210 : /* If the level is -1, this ended up being an unused
2211 : axis. Handle as a default. */
2212 30884 : if (integer_minus_onep (gimple_call_arg (call, 3)))
2213 8528 : default_goacc_reduction (call);
2214 : else
2215 22356 : targetm.goacc.reduction (call);
2216 : rescan = true;
2217 : break;
2218 :
2219 85483 : case IFN_UNIQUE:
2220 85483 : {
2221 85483 : enum ifn_unique_kind kind
2222 : = ((enum ifn_unique_kind)
2223 85483 : TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
2224 :
2225 85483 : switch (kind)
2226 : {
2227 : default:
2228 : break;
2229 :
2230 32978 : case IFN_UNIQUE_OACC_FORK:
2231 32978 : case IFN_UNIQUE_OACC_JOIN:
2232 32978 : if (integer_minus_onep (gimple_call_arg (call, 2)))
2233 : remove = true;
2234 25614 : else if (!targetm.goacc.fork_join
2235 25614 : (call, dims, kind == IFN_UNIQUE_OACC_FORK))
2236 85483 : remove = true;
2237 : break;
2238 :
2239 : case IFN_UNIQUE_OACC_HEAD_MARK:
2240 : case IFN_UNIQUE_OACC_TAIL_MARK:
2241 85483 : remove = true;
2242 : break;
2243 :
2244 259 : case IFN_UNIQUE_OACC_PRIVATE:
2245 259 : {
2246 259 : dump_flags_t l_dump_flags
2247 259 : = get_openacc_privatization_dump_flags ();
2248 :
2249 259 : location_t loc = gimple_location (stmt);
2250 259 : if (LOCATION_LOCUS (loc) == UNKNOWN_LOCATION)
2251 30 : loc = DECL_SOURCE_LOCATION (current_function_decl);
2252 259 : const dump_user_location_t d_u_loc
2253 259 : = dump_user_location_t::from_location_t (loc);
2254 :
2255 259 : HOST_WIDE_INT level
2256 259 : = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
2257 259 : gcc_checking_assert (level == -1
2258 : || (level >= 0
2259 : && level < GOMP_DIM_MAX));
2260 339 : for (unsigned i = 3;
2261 598 : i < gimple_call_num_args (call);
2262 : i++)
2263 : {
2264 339 : static char const *const axes[] =
2265 : /* Must be kept in sync with GOMP_DIM enumeration. */
2266 : { "gang", "worker", "vector" };
2267 :
2268 339 : tree arg = gimple_call_arg (call, i);
2269 339 : gcc_checking_assert (TREE_CODE (arg) == ADDR_EXPR);
2270 339 : tree decl = TREE_OPERAND (arg, 0);
2271 339 : if (dump_enabled_p ())
2272 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
2273 : #if __GNUC__ >= 10
2274 318 : # pragma GCC diagnostic push
2275 318 : # pragma GCC diagnostic ignored "-Wformat"
2276 : #endif
2277 318 : dump_printf_loc (l_dump_flags, d_u_loc,
2278 : "variable %<%T%> ought to be"
2279 : " adjusted for OpenACC"
2280 : " privatization level: %qs\n",
2281 : decl,
2282 : (level == -1
2283 : ? "UNKNOWN" : axes[level]));
2284 : #if __GNUC__ >= 10
2285 339 : # pragma GCC diagnostic pop
2286 : #endif
2287 339 : bool adjusted;
2288 339 : if (level == -1)
2289 : adjusted = false;
2290 336 : else if (!targetm.goacc.adjust_private_decl)
2291 : adjusted = false;
2292 0 : else if (level == GOMP_DIM_VECTOR)
2293 : {
2294 : /* That's the default behavior. */
2295 : adjusted = true;
2296 : }
2297 : else
2298 : {
2299 0 : tree oldtype = TREE_TYPE (decl);
2300 0 : tree newdecl
2301 0 : = targetm.goacc.adjust_private_decl (loc, decl,
2302 0 : level);
2303 0 : adjusted = (TREE_TYPE (newdecl) != oldtype
2304 0 : || newdecl != decl);
2305 0 : if (adjusted)
2306 0 : adjusted_vars.put (decl, newdecl);
2307 : }
2308 0 : if (adjusted
2309 0 : && dump_enabled_p ())
2310 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
2311 : #if __GNUC__ >= 10
2312 0 : # pragma GCC diagnostic push
2313 0 : # pragma GCC diagnostic ignored "-Wformat"
2314 : #endif
2315 0 : dump_printf_loc (l_dump_flags, d_u_loc,
2316 : "variable %<%T%> adjusted for"
2317 : " OpenACC privatization level:"
2318 : " %qs\n",
2319 0 : decl, axes[level]);
2320 : #if __GNUC__ >= 10
2321 339 : # pragma GCC diagnostic pop
2322 : #endif
2323 : }
2324 259 : remove = true;
2325 : }
2326 259 : break;
2327 : }
2328 : break;
2329 : }
2330 : }
2331 :
2332 195044 : if (gsi_end_p (gsi))
2333 : /* We rewound past the beginning of the BB. */
2334 188830 : gsi = gsi_start_bb (bb);
2335 : else
2336 : /* Undo the rewind. */
2337 100629 : gsi_next (&gsi);
2338 :
2339 195044 : if (remove)
2340 : {
2341 170966 : if (gimple_vdef (call))
2342 85483 : replace_uses_by (gimple_vdef (call), gimple_vuse (call));
2343 85483 : if (gimple_call_lhs (call))
2344 : {
2345 : /* Propagate the data dependency var. */
2346 79856 : gimple *ass = gimple_build_assign (gimple_call_lhs (call),
2347 : gimple_call_arg (call, 1));
2348 79856 : gsi_replace (&gsi, ass, false);
2349 : }
2350 : else
2351 5627 : gsi_remove (&gsi, true);
2352 : }
2353 109561 : else if (!rescan)
2354 : /* If not rescanning, advance over the call. */
2355 31691 : gsi_next (&gsi);
2356 : }
2357 :
2358 : /* Regarding the OpenACC privatization level, we're currently only looking at
2359 : making the gang-private level work. Regarding that, we have the following
2360 : configurations:
2361 :
2362 : - GCN offloading: 'targetm.goacc.adjust_private_decl' does the work (in
2363 : particular, change 'TREE_TYPE', etc.) and there is no
2364 : 'targetm.goacc.expand_var_decl'.
2365 :
2366 : - nvptx offloading: 'targetm.goacc.adjust_private_decl' only sets a
2367 : marker and then 'targetm.goacc.expand_var_decl' does the work.
2368 :
2369 : Eventually (in particular, for worker-private level?), both
2370 : 'targetm.goacc.adjust_private_decl' and 'targetm.goacc.expand_var_decl'
2371 : may need to do things, but that's currently not meant to be addressed, and
2372 : thus not fully worked out and implemented, and thus untested. Hence,
2373 : 'assert' what currently is implemented/tested, only. */
2374 :
2375 9879 : if (targetm.goacc.expand_var_decl)
2376 0 : gcc_assert (adjusted_vars.is_empty ());
2377 :
2378 : /* Make adjustments to gang-private local variables if required by the
2379 : target, e.g. forcing them into a particular address space. Afterwards,
2380 : ADDR_EXPR nodes which have adjusted variables as their argument need to
2381 : be modified in one of two ways:
2382 :
2383 : 1. They can be recreated, making a pointer to the variable in the new
2384 : address space, or
2385 :
2386 : 2. The address of the variable in the new address space can be taken,
2387 : converted to the default (original) address space, and the result of
2388 : that conversion substituted in place of the original ADDR_EXPR node.
2389 :
2390 : Which of these is done depends on the gimple statement being processed.
2391 : At present atomic operations and inline asms use (1), and everything else
2392 : uses (2). At least on AMD GCN, there are atomic operations that work
2393 : directly in the LDS address space.
2394 :
2395 : COMPONENT_REFS, ARRAY_REFS and plain VAR_DECLs are also rewritten to use
2396 : the new decl, adjusting types of appropriate tree nodes as necessary. */
2397 :
2398 9879 : if (targetm.goacc.adjust_private_decl
2399 9879 : && !adjusted_vars.is_empty ())
2400 : {
2401 0 : FOR_ALL_BB_FN (bb, cfun)
2402 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2403 0 : !gsi_end_p (gsi);
2404 0 : gsi_next (&gsi))
2405 : {
2406 0 : gimple *stmt = gsi_stmt (gsi);
2407 0 : walk_stmt_info wi;
2408 0 : var_decl_rewrite_info info;
2409 :
2410 0 : info.avoid_pointer_conversion
2411 0 : = (is_gimple_call (stmt)
2412 0 : && is_sync_builtin_call (as_a <gcall *> (stmt)))
2413 0 : || gimple_code (stmt) == GIMPLE_ASM;
2414 0 : info.stmt = stmt;
2415 0 : info.modified = false;
2416 0 : info.adjusted_vars = &adjusted_vars;
2417 :
2418 0 : memset (&wi, 0, sizeof (wi));
2419 0 : wi.info = &info;
2420 :
2421 0 : walk_gimple_op (stmt, oacc_rewrite_var_decl, &wi);
2422 :
2423 0 : if (info.modified)
2424 0 : update_stmt (stmt);
2425 : }
2426 : }
2427 :
2428 9879 : return 0;
2429 9879 : }
2430 :
2431 : /* Default launch dimension validator. Force everything to 1. A
2432 : backend that wants to provide larger dimensions must override this
2433 : hook. */
2434 :
2435 : bool
2436 14441 : default_goacc_validate_dims (tree ARG_UNUSED (decl), int *dims,
2437 : int ARG_UNUSED (fn_level),
2438 : unsigned ARG_UNUSED (used))
2439 : {
2440 14441 : bool changed = false;
2441 :
2442 57764 : for (unsigned ix = 0; ix != GOMP_DIM_MAX; ix++)
2443 : {
2444 43323 : if (dims[ix] != 1)
2445 : {
2446 33054 : dims[ix] = 1;
2447 33054 : changed = true;
2448 : }
2449 : }
2450 :
2451 14441 : return changed;
2452 : }
2453 :
2454 : /* Default dimension bound is unknown on accelerator and 1 on host. */
2455 :
2456 : int
2457 0 : default_goacc_dim_limit (int ARG_UNUSED (axis))
2458 : {
2459 : #ifdef ACCEL_COMPILER
2460 : return 0;
2461 : #else
2462 0 : return 1;
2463 : #endif
2464 : }
2465 :
2466 : namespace {
2467 :
2468 : const pass_data pass_data_oacc_loop_designation =
2469 : {
2470 : GIMPLE_PASS, /* type */
2471 : "oaccloops", /* name */
2472 : OPTGROUP_OMP, /* optinfo_flags */
2473 : TV_NONE, /* tv_id */
2474 : PROP_cfg, /* properties_required */
2475 : 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
2476 : 0, /* properties_destroyed */
2477 : 0, /* todo_flags_start */
2478 : TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */
2479 : };
2480 :
2481 : class pass_oacc_loop_designation : public gimple_opt_pass
2482 : {
2483 : public:
2484 294196 : pass_oacc_loop_designation (gcc::context *ctxt)
2485 588392 : : gimple_opt_pass (pass_data_oacc_loop_designation, ctxt)
2486 : {}
2487 :
2488 : /* opt_pass methods: */
2489 1516080 : bool gate (function *) final override { return flag_openacc; };
2490 :
2491 15337 : unsigned int execute (function *) final override
2492 : {
2493 15337 : return execute_oacc_loop_designation ();
2494 : }
2495 :
2496 : }; // class pass_oacc_loop_designation
2497 :
2498 : const pass_data pass_data_oacc_device_lower =
2499 : {
2500 : GIMPLE_PASS, /* type */
2501 : "oaccdevlow", /* name */
2502 : OPTGROUP_OMP, /* optinfo_flags */
2503 : TV_NONE, /* tv_id */
2504 : PROP_cfg, /* properties_required */
2505 : 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
2506 : 0, /* properties_destroyed */
2507 : 0, /* todo_flags_start */
2508 : TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */
2509 : };
2510 :
2511 : class pass_oacc_device_lower : public gimple_opt_pass
2512 : {
2513 : public:
2514 294196 : pass_oacc_device_lower (gcc::context *ctxt)
2515 588392 : : gimple_opt_pass (pass_data_oacc_device_lower, ctxt)
2516 : {}
2517 :
2518 : /* opt_pass methods: */
2519 1516018 : bool gate (function *) final override { return flag_openacc; };
2520 :
2521 15275 : unsigned int execute (function *) final override
2522 : {
2523 15275 : return execute_oacc_device_lower ();
2524 : }
2525 :
2526 : }; // class pass_oacc_device_lower
2527 :
2528 : } // anon namespace
2529 :
2530 : gimple_opt_pass *
2531 294196 : make_pass_oacc_loop_designation (gcc::context *ctxt)
2532 : {
2533 294196 : return new pass_oacc_loop_designation (ctxt);
2534 : }
2535 :
2536 : gimple_opt_pass *
2537 294196 : make_pass_oacc_device_lower (gcc::context *ctxt)
2538 : {
2539 294196 : return new pass_oacc_device_lower (ctxt);
2540 : }
2541 :
2542 :
2543 : /* Rewrite GOMP_SIMT_ENTER_ALLOC call given by GSI and remove the preceding
2544 : GOMP_SIMT_ENTER call identifying the privatized variables, which are
2545 : turned to structure fields and receive a DECL_VALUE_EXPR accordingly.
2546 : Set *REGIMPLIFY to true, except if no privatized variables were seen. */
2547 :
2548 : static void
2549 0 : ompdevlow_adjust_simt_enter (gimple_stmt_iterator *gsi, bool *regimplify)
2550 : {
2551 0 : gimple *alloc_stmt = gsi_stmt (*gsi);
2552 0 : tree simtrec = gimple_call_lhs (alloc_stmt);
2553 0 : tree simduid = gimple_call_arg (alloc_stmt, 0);
2554 0 : gimple *enter_stmt = SSA_NAME_DEF_STMT (simduid);
2555 0 : gcc_assert (gimple_call_internal_p (enter_stmt, IFN_GOMP_SIMT_ENTER));
2556 0 : tree rectype = lang_hooks.types.make_type (RECORD_TYPE);
2557 0 : TYPE_ARTIFICIAL (rectype) = TYPE_NAMELESS (rectype) = 1;
2558 0 : TREE_ADDRESSABLE (rectype) = 1;
2559 0 : TREE_TYPE (simtrec) = build_pointer_type (rectype);
2560 0 : for (unsigned i = 1; i < gimple_call_num_args (enter_stmt); i++)
2561 : {
2562 0 : tree *argp = gimple_call_arg_ptr (enter_stmt, i);
2563 0 : if (*argp == null_pointer_node)
2564 0 : continue;
2565 0 : gcc_assert (TREE_CODE (*argp) == ADDR_EXPR
2566 : && VAR_P (TREE_OPERAND (*argp, 0)));
2567 0 : tree var = TREE_OPERAND (*argp, 0);
2568 :
2569 0 : tree field = build_decl (DECL_SOURCE_LOCATION (var), FIELD_DECL,
2570 0 : DECL_NAME (var), TREE_TYPE (var));
2571 0 : SET_DECL_ALIGN (field, DECL_ALIGN (var));
2572 0 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
2573 0 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
2574 :
2575 0 : insert_field_into_struct (rectype, field);
2576 :
2577 0 : tree t = build_simple_mem_ref (simtrec);
2578 0 : t = build3 (COMPONENT_REF, TREE_TYPE (var), t, field, NULL);
2579 0 : TREE_THIS_VOLATILE (t) = TREE_THIS_VOLATILE (var);
2580 0 : SET_DECL_VALUE_EXPR (var, t);
2581 0 : DECL_HAS_VALUE_EXPR_P (var) = 1;
2582 0 : *regimplify = true;
2583 : }
2584 0 : layout_type (rectype);
2585 0 : tree size = TYPE_SIZE_UNIT (rectype);
2586 0 : tree align = build_int_cst (TREE_TYPE (size), TYPE_ALIGN_UNIT (rectype));
2587 :
2588 0 : alloc_stmt
2589 0 : = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC, 2, size, align);
2590 0 : gimple_call_set_lhs (alloc_stmt, simtrec);
2591 0 : gsi_replace (gsi, alloc_stmt, false);
2592 0 : gimple_stmt_iterator enter_gsi = gsi_for_stmt (enter_stmt);
2593 0 : enter_stmt = gimple_build_assign (simduid, gimple_call_arg (enter_stmt, 0));
2594 0 : gsi_replace (&enter_gsi, enter_stmt, false);
2595 :
2596 0 : use_operand_p use;
2597 0 : gimple *exit_stmt;
2598 0 : if (single_imm_use (simtrec, &use, &exit_stmt))
2599 : {
2600 0 : gcc_assert (gimple_call_internal_p (exit_stmt, IFN_GOMP_SIMT_EXIT));
2601 0 : gimple_stmt_iterator exit_gsi = gsi_for_stmt (exit_stmt);
2602 0 : tree clobber = build_clobber (rectype);
2603 0 : exit_stmt = gimple_build_assign (build_simple_mem_ref (simtrec), clobber);
2604 0 : gsi_insert_before (&exit_gsi, exit_stmt, GSI_SAME_STMT);
2605 : }
2606 : else
2607 0 : gcc_checking_assert (has_zero_uses (simtrec));
2608 0 : }
2609 :
2610 : /* Callback for walk_gimple_stmt used to scan for SIMT-privatized variables. */
2611 :
2612 : static tree
2613 0 : find_simtpriv_var_op (tree *tp, int *walk_subtrees, void *)
2614 : {
2615 0 : tree t = *tp;
2616 :
2617 0 : if (VAR_P (t)
2618 0 : && DECL_HAS_VALUE_EXPR_P (t)
2619 0 : && lookup_attribute ("omp simt private", DECL_ATTRIBUTES (t)))
2620 : {
2621 0 : *walk_subtrees = 0;
2622 0 : return t;
2623 : }
2624 : return NULL_TREE;
2625 : }
2626 :
2627 : /* Helper function for execute_omp_device_lower, invoked via walk_gimple_op.
2628 : Resolve any OMP_TARGET_DEVICE_MATCHES and OMP_NEXT_VARIANT exprs to
2629 : constants. */
2630 : static tree
2631 13877 : resolve_omp_variant_cookies (tree *tp, int *walk_subtrees,
2632 : void *data ATTRIBUTE_UNUSED)
2633 : {
2634 13877 : if (TREE_CODE (*tp) == OMP_TARGET_DEVICE_MATCHES)
2635 : {
2636 0 : *tp = resolve_omp_target_device_matches (*tp);
2637 0 : *walk_subtrees = 0;
2638 0 : return NULL_TREE;
2639 : }
2640 :
2641 13877 : if (TREE_CODE (*tp) != OMP_NEXT_VARIANT)
2642 : return NULL_TREE;
2643 324 : tree index = OMP_NEXT_VARIANT_INDEX (*tp);
2644 324 : tree state = OMP_NEXT_VARIANT_STATE (*tp);
2645 :
2646 : /* State is a triplet of (result-vector, construct_context, selector_vec).
2647 : If result-vector has already been computed, just use it. Otherwise we
2648 : must resolve the variant and fill in that part of the state object.
2649 : All OMP_NEXT_VARIANT exprs for the same variant construct are supposed
2650 : to share the same state object, but if something bad happens and we end
2651 : up with copies, that is OK, it will just cause the result-vector to be
2652 : computed multiple times. */
2653 324 : tree result_vector = TREE_PURPOSE (state);
2654 324 : if (!result_vector)
2655 : {
2656 304 : tree construct_context = TREE_VALUE (state);
2657 304 : tree selectors = TREE_CHAIN (state);
2658 :
2659 304 : vec<struct omp_variant> candidates
2660 304 : = omp_resolve_variant_construct (construct_context, selectors);
2661 304 : int n = TREE_VEC_LENGTH (selectors);
2662 304 : TREE_PURPOSE (state) = result_vector = make_tree_vec (n + 1);
2663 : /* The result vector maps the index of each element of the original
2664 : selectors vector onto the index of the next element of the filtered/
2665 : sorted candidates vector. Since some of the original variants may
2666 : have been discarded as non-matching in candidates, initialize the
2667 : whole array to zero so that we have a placeholder "next" value for
2668 : those elements. Hopefully dead code elimination will take care of
2669 : subsequently discarding the unreachable cases in the already-generated
2670 : switch statement. */
2671 2108 : for (int i = 1; i <= n; i++)
2672 1804 : TREE_VEC_ELT (result_vector, i) = integer_zero_node;
2673 : /* Element 0 is the case label of the first variant in the sorted
2674 : list. */
2675 304 : if (dump_file)
2676 0 : fprintf (dump_file, "Computing case map for variant directive\n");
2677 304 : int j = 0;
2678 1588 : for (unsigned int i = 0; i < candidates.length(); i++)
2679 : {
2680 1284 : if (dump_file)
2681 0 : fprintf (dump_file, " %d -> case %d\n",
2682 0 : j, (int) tree_to_shwi (candidates[i].alternative));
2683 1284 : TREE_VEC_ELT (result_vector, j) = candidates[i].alternative;
2684 1284 : j = (int) tree_to_shwi (candidates[i].alternative);
2685 : }
2686 : }
2687 :
2688 : /* Now just grab the value out of the precomputed array. */
2689 324 : gcc_assert (TREE_CODE (index) == INTEGER_CST);
2690 324 : int indexval = (int) tree_to_shwi (index);
2691 324 : *tp = TREE_VEC_ELT (result_vector, indexval);
2692 324 : *walk_subtrees = 0;
2693 324 : return NULL_TREE;
2694 : }
2695 :
2696 :
2697 : /* Cleanup uses of SIMT placeholder internal functions: on non-SIMT targets,
2698 : VF is 1 and LANE is 0; on SIMT targets, VF is folded to a constant, and
2699 : LANE is kept to be expanded to RTL later on. Also cleanup all other SIMT
2700 : internal functions on non-SIMT targets, and likewise some SIMD internal
2701 : functions on SIMT targets. */
2702 :
2703 : static unsigned int
2704 21006 : execute_omp_device_lower ()
2705 : {
2706 21006 : int vf = targetm.simt.vf ? targetm.simt.vf () : 1;
2707 21006 : bool regimplify = false;
2708 21006 : basic_block bb;
2709 21006 : gimple_stmt_iterator gsi;
2710 : #ifdef ACCEL_COMPILER
2711 : bool omp_redirect_indirect_calls = vec_safe_length (offload_ind_funcs) > 0;
2712 : tree map_ptr_fn
2713 : = builtin_decl_explicit (BUILT_IN_GOMP_TARGET_MAP_INDIRECT_PTR);
2714 : #endif
2715 :
2716 : /* Handle expansion of magic cookies for variant constructs first. */
2717 21006 : if (cgraph_node::get (cfun->decl)->has_omp_variant_constructs)
2718 1822 : FOR_EACH_BB_FN (bb, cfun)
2719 : {
2720 6873 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2721 3409 : walk_gimple_op (gsi_stmt (gsi), resolve_omp_variant_cookies, NULL);
2722 2510 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2723 778 : walk_gimple_op (gsi_stmt (gsi), resolve_omp_variant_cookies, NULL);
2724 : }
2725 :
2726 56566 : FOR_EACH_BB_FN (bb, cfun)
2727 244648 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2728 : {
2729 173528 : gimple *stmt = gsi_stmt (gsi);
2730 173528 : if (!is_gimple_call (stmt))
2731 156646 : continue;
2732 16882 : if (!gimple_call_internal_p (stmt))
2733 : {
2734 : #ifdef ACCEL_COMPILER
2735 : if (omp_redirect_indirect_calls
2736 : && gimple_call_fndecl (stmt) == NULL_TREE)
2737 : {
2738 : gcall *orig_call = dyn_cast <gcall *> (stmt);
2739 : tree call_fn = gimple_call_fn (stmt);
2740 : tree fn_ty = TREE_TYPE (call_fn);
2741 :
2742 : if (TREE_CODE (call_fn) == OBJ_TYPE_REF)
2743 : {
2744 : tree obj_ref = create_tmp_reg (TREE_TYPE (call_fn),
2745 : ".ind_fn_objref");
2746 : gimple *gassign = gimple_build_assign (obj_ref, call_fn);
2747 : gsi_insert_before (&gsi, gassign, GSI_SAME_STMT);
2748 : call_fn = obj_ref;
2749 : }
2750 : tree mapped_fn = create_tmp_reg (fn_ty, ".ind_fn");
2751 : gimple *gcall =
2752 : gimple_build_call (map_ptr_fn, 1, call_fn);
2753 : gimple_set_location (gcall, gimple_location (stmt));
2754 : gimple_call_set_lhs (gcall, mapped_fn);
2755 : gsi_insert_before (&gsi, gcall, GSI_SAME_STMT);
2756 :
2757 : gimple_call_set_fn (orig_call, mapped_fn);
2758 : update_stmt (orig_call);
2759 : }
2760 : #endif
2761 16617 : continue;
2762 : }
2763 265 : tree lhs = gimple_call_lhs (stmt), rhs = NULL_TREE;
2764 265 : tree type = lhs ? TREE_TYPE (lhs) : integer_type_node;
2765 265 : switch (gimple_call_internal_fn (stmt))
2766 : {
2767 0 : case IFN_GOMP_TARGET_REV:
2768 0 : {
2769 : #ifndef ACCEL_COMPILER
2770 0 : gimple_stmt_iterator gsi2 = gsi;
2771 0 : gsi_next (&gsi2);
2772 0 : gcc_assert (!gsi_end_p (gsi2));
2773 0 : gcc_assert (gimple_call_builtin_p (gsi_stmt (gsi2),
2774 : BUILT_IN_GOMP_TARGET));
2775 0 : tree old_decl
2776 0 : = TREE_OPERAND (gimple_call_arg (gsi_stmt (gsi2), 1), 0);
2777 0 : tree new_decl = gimple_call_arg (gsi_stmt (gsi), 0);
2778 0 : gimple_call_set_arg (gsi_stmt (gsi2), 1, new_decl);
2779 0 : update_stmt (gsi_stmt (gsi2));
2780 0 : new_decl = TREE_OPERAND (new_decl, 0);
2781 0 : unsigned i;
2782 0 : unsigned num_funcs = vec_safe_length (offload_funcs);
2783 0 : for (i = 0; i < num_funcs; i++)
2784 : {
2785 0 : if ((*offload_funcs)[i] == old_decl)
2786 : {
2787 0 : (*offload_funcs)[i] = new_decl;
2788 0 : break;
2789 : }
2790 0 : else if ((*offload_funcs)[i] == new_decl)
2791 : break; /* This can happen due to inlining. */
2792 : }
2793 0 : gcc_assert (i < num_funcs);
2794 : #else
2795 : tree old_decl = TREE_OPERAND (gimple_call_arg (gsi_stmt (gsi), 0),
2796 : 0);
2797 : #endif
2798 : /* FIXME: Find a way to actually prevent outputting the empty-body
2799 : old_decl as debug symbol + function in the assembly file. */
2800 0 : cgraph_node *node = cgraph_node::get (old_decl);
2801 0 : node->address_taken = false;
2802 0 : node->need_lto_streaming = false;
2803 0 : node->offloadable = false;
2804 :
2805 0 : unlink_stmt_vdef (stmt);
2806 : }
2807 0 : break;
2808 0 : case IFN_GOMP_USE_SIMT:
2809 0 : rhs = vf == 1 ? integer_zero_node : integer_one_node;
2810 : break;
2811 0 : case IFN_GOMP_SIMT_ENTER:
2812 0 : rhs = vf == 1 ? gimple_call_arg (stmt, 0) : NULL_TREE;
2813 0 : goto simtreg_enter_exit;
2814 0 : case IFN_GOMP_SIMT_ENTER_ALLOC:
2815 0 : if (vf != 1)
2816 0 : ompdevlow_adjust_simt_enter (&gsi, ®implify);
2817 0 : rhs = vf == 1 ? null_pointer_node : NULL_TREE;
2818 0 : goto simtreg_enter_exit;
2819 0 : case IFN_GOMP_SIMT_EXIT:
2820 0 : simtreg_enter_exit:
2821 0 : if (vf != 1)
2822 0 : continue;
2823 0 : unlink_stmt_vdef (stmt);
2824 0 : break;
2825 0 : case IFN_GOMP_SIMT_LANE:
2826 0 : case IFN_GOMP_SIMT_LAST_LANE:
2827 0 : rhs = vf == 1 ? build_zero_cst (type) : NULL_TREE;
2828 : break;
2829 0 : case IFN_GOMP_SIMT_VF:
2830 0 : rhs = build_int_cst (type, vf);
2831 0 : break;
2832 2 : case IFN_GOMP_MAX_VF:
2833 2 : rhs = build_int_cst (type, omp_max_vf (false));
2834 2 : break;
2835 0 : case IFN_GOMP_SIMT_ORDERED_PRED:
2836 0 : rhs = vf == 1 ? integer_zero_node : NULL_TREE;
2837 0 : if (rhs || !lhs)
2838 0 : unlink_stmt_vdef (stmt);
2839 : break;
2840 0 : case IFN_GOMP_SIMT_VOTE_ANY:
2841 0 : case IFN_GOMP_SIMT_XCHG_BFLY:
2842 0 : case IFN_GOMP_SIMT_XCHG_IDX:
2843 0 : rhs = vf == 1 ? gimple_call_arg (stmt, 0) : NULL_TREE;
2844 : break;
2845 0 : case IFN_GOMP_SIMD_LANE:
2846 0 : case IFN_GOMP_SIMD_LAST_LANE:
2847 0 : rhs = vf != 1 ? build_zero_cst (type) : NULL_TREE;
2848 : break;
2849 0 : case IFN_GOMP_SIMD_VF:
2850 0 : rhs = vf != 1 ? build_one_cst (type) : NULL_TREE;
2851 : break;
2852 263 : default:
2853 263 : continue;
2854 263 : }
2855 2 : if (lhs && !rhs)
2856 0 : continue;
2857 2 : stmt = lhs ? gimple_build_assign (lhs, rhs) : gimple_build_nop ();
2858 2 : gsi_replace (&gsi, stmt, false);
2859 : }
2860 21006 : if (regimplify)
2861 0 : FOR_EACH_BB_REVERSE_FN (bb, cfun)
2862 0 : for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
2863 0 : if (walk_gimple_stmt (&gsi, NULL, find_simtpriv_var_op, NULL))
2864 : {
2865 0 : if (gimple_clobber_p (gsi_stmt (gsi)))
2866 0 : gsi_remove (&gsi, true);
2867 : else
2868 0 : gimple_regimplify_operands (gsi_stmt (gsi), &gsi);
2869 : }
2870 21006 : if (vf != 1)
2871 0 : cfun->has_force_vectorize_loops = false;
2872 21006 : return 0;
2873 : }
2874 :
2875 : namespace {
2876 :
2877 : const pass_data pass_data_omp_device_lower =
2878 : {
2879 : GIMPLE_PASS, /* type */
2880 : "ompdevlow", /* name */
2881 : OPTGROUP_OMP, /* optinfo_flags */
2882 : TV_NONE, /* tv_id */
2883 : PROP_cfg, /* properties_required */
2884 : PROP_gimple_lomp_dev, /* properties_provided */
2885 : 0, /* properties_destroyed */
2886 : 0, /* todo_flags_start */
2887 : TODO_update_ssa, /* todo_flags_finish */
2888 : };
2889 :
2890 : class pass_omp_device_lower : public gimple_opt_pass
2891 : {
2892 : public:
2893 294196 : pass_omp_device_lower (gcc::context *ctxt)
2894 588392 : : gimple_opt_pass (pass_data_omp_device_lower, ctxt)
2895 : {}
2896 :
2897 : /* opt_pass methods: */
2898 1516018 : bool gate (function *fun) final override
2899 : {
2900 1516018 : cgraph_node *node = cgraph_node::get (fun->decl);
2901 : #ifdef ACCEL_COMPILER
2902 : bool offload_ind_funcs_p = vec_safe_length (offload_ind_funcs) > 0;
2903 : #else
2904 1516018 : bool offload_ind_funcs_p = false;
2905 : #endif
2906 1516018 : return (!(fun->curr_properties & PROP_gimple_lomp_dev)
2907 1516018 : || (flag_openmp
2908 66026 : && (node->has_omp_variant_constructs || offload_ind_funcs_p)));
2909 : }
2910 21006 : unsigned int execute (function *) final override
2911 : {
2912 21006 : return execute_omp_device_lower ();
2913 : }
2914 :
2915 : }; // class pass_expand_omp_ssa
2916 :
2917 : } // anon namespace
2918 :
2919 : gimple_opt_pass *
2920 294196 : make_pass_omp_device_lower (gcc::context *ctxt)
2921 : {
2922 294196 : return new pass_omp_device_lower (ctxt);
2923 : }
2924 :
2925 : /* "omp declare target link" handling pass. */
2926 :
2927 : namespace {
2928 :
2929 : const pass_data pass_data_omp_target_link =
2930 : {
2931 : GIMPLE_PASS, /* type */
2932 : "omptargetlink", /* name */
2933 : OPTGROUP_OMP, /* optinfo_flags */
2934 : TV_NONE, /* tv_id */
2935 : PROP_ssa, /* properties_required */
2936 : 0, /* properties_provided */
2937 : 0, /* properties_destroyed */
2938 : 0, /* todo_flags_start */
2939 : TODO_update_ssa, /* todo_flags_finish */
2940 : };
2941 :
2942 : class pass_omp_target_link : public gimple_opt_pass
2943 : {
2944 : public:
2945 294196 : pass_omp_target_link (gcc::context *ctxt)
2946 588392 : : gimple_opt_pass (pass_data_omp_target_link, ctxt)
2947 : {}
2948 :
2949 : /* opt_pass methods: */
2950 1516018 : bool gate (function *fun) final override
2951 : {
2952 : #ifdef ACCEL_COMPILER
2953 : return offloading_function_p (fun->decl);
2954 : #else
2955 1516018 : (void) fun;
2956 1516018 : return false;
2957 : #endif
2958 : }
2959 :
2960 : unsigned execute (function *) final override;
2961 : };
2962 :
2963 : /* Callback for walk_gimple_stmt used to scan for link var operands. */
2964 :
2965 : static tree
2966 0 : process_link_var_op (tree *tp, int *walk_subtrees, void *data)
2967 : {
2968 0 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
2969 0 : tree t = *tp;
2970 :
2971 0 : if (VAR_P (t)
2972 0 : && DECL_HAS_VALUE_EXPR_P (t)
2973 0 : && is_global_var (t)
2974 0 : && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t)))
2975 : {
2976 0 : wi->info = *tp = unshare_expr (DECL_VALUE_EXPR (t));
2977 0 : *walk_subtrees = 0;
2978 0 : return NULL_TREE;
2979 : }
2980 :
2981 : return NULL_TREE;
2982 : }
2983 :
2984 : unsigned
2985 0 : pass_omp_target_link::execute (function *fun)
2986 : {
2987 0 : basic_block bb;
2988 0 : FOR_EACH_BB_FN (bb, fun)
2989 : {
2990 0 : gimple_stmt_iterator gsi;
2991 0 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2992 : {
2993 0 : if (gimple_call_builtin_p (gsi_stmt (gsi), BUILT_IN_GOMP_TARGET))
2994 : {
2995 0 : tree dev = gimple_call_arg (gsi_stmt (gsi), 0);
2996 0 : tree fn = gimple_call_arg (gsi_stmt (gsi), 1);
2997 0 : if (POINTER_TYPE_P (TREE_TYPE (fn)))
2998 0 : fn = TREE_OPERAND (fn, 0);
2999 0 : if (TREE_CODE (dev) == INTEGER_CST
3000 0 : && wi::to_wide (dev) == GOMP_DEVICE_HOST_FALLBACK
3001 0 : && lookup_attribute ("omp target device_ancestor_nohost",
3002 0 : DECL_ATTRIBUTES (fn)) != NULL_TREE)
3003 0 : continue; /* ancestor:1 */
3004 : /* Nullify the second argument of __builtin_GOMP_target_ext. */
3005 0 : gimple_call_set_arg (gsi_stmt (gsi), 1, null_pointer_node);
3006 0 : update_stmt (gsi_stmt (gsi));
3007 : }
3008 0 : struct walk_stmt_info wi;
3009 0 : memset (&wi, 0, sizeof (wi));
3010 0 : walk_gimple_stmt (&gsi, NULL, process_link_var_op, &wi);
3011 0 : if (wi.info)
3012 0 : gimple_regimplify_operands (gsi_stmt (gsi), &gsi);
3013 : }
3014 : }
3015 :
3016 0 : return 0;
3017 : }
3018 :
3019 : } // anon namespace
3020 :
3021 : gimple_opt_pass *
3022 294196 : make_pass_omp_target_link (gcc::context *ctxt)
3023 : {
3024 294196 : return new pass_omp_target_link (ctxt);
3025 : }
|
