Line data Source code
1 : /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 : calls to the runtime library (libgomp), data marshalling to implement data
3 : sharing and copying clauses, offloading to accelerators, and more.
4 :
5 : Contributed by Diego Novillo <dnovillo@redhat.com>
6 :
7 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
8 :
9 : This file is part of GCC.
10 :
11 : GCC is free software; you can redistribute it and/or modify it under
12 : the terms of the GNU General Public License as published by the Free
13 : Software Foundation; either version 3, or (at your option) any later
14 : version.
15 :
16 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 : for more details.
20 :
21 : You should have received a copy of the GNU General Public License
22 : along with GCC; see the file COPYING3. If not see
23 : <http://www.gnu.org/licenses/>. */
24 :
25 : #include "config.h"
26 : #include "system.h"
27 : #include "coretypes.h"
28 : #include "backend.h"
29 : #include "target.h"
30 : #include "tree.h"
31 : #include "gimple.h"
32 : #include "tree-pass.h"
33 : #include "ssa.h"
34 : #include "cgraph.h"
35 : #include "pretty-print.h"
36 : #include "diagnostic-core.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "internal-fn.h"
40 : #include "gimple-iterator.h"
41 : #include "gimple-fold.h"
42 : #include "gimplify.h"
43 : #include "gimplify-me.h"
44 : #include "gimple-walk.h"
45 : #include "tree-iterator.h"
46 : #include "tree-inline.h"
47 : #include "langhooks.h"
48 : #include "tree-dfa.h"
49 : #include "tree-ssa.h"
50 : #include "splay-tree.h"
51 : #include "omp-general.h"
52 : #include "omp-low.h"
53 : #include "gimple-low.h"
54 : #include "alloc-pool.h"
55 : #include "symbol-summary.h"
56 : #include "tree-nested.h"
57 : #include "context.h"
58 : #include "gomp-constants.h"
59 : #include "gimple-pretty-print.h"
60 : #include "stringpool.h"
61 : #include "attribs.h"
62 : #include "omp-offload.h"
63 :
64 : /* Lowering of OMP parallel and workshare constructs proceeds in two
65 : phases. The first phase scans the function looking for OMP statements
66 : and then for variables that must be replaced to satisfy data sharing
67 : clauses. The second phase expands code for the constructs, as well as
68 : re-gimplifying things when variables have been replaced with complex
69 : expressions.
70 :
71 : Final code generation is done by pass_expand_omp. The flowgraph is
72 : scanned for regions which are then moved to a new
73 : function, to be invoked by the thread library, or offloaded. */
74 :
75 : /* Context structure. Used to store information about each parallel
76 : directive in the code. */
77 :
78 : struct omp_context
79 : {
80 : /* This field must be at the beginning, as we do "inheritance": Some
81 : callback functions for tree-inline.cc (e.g., omp_copy_decl)
82 : receive a copy_body_data pointer that is up-casted to an
83 : omp_context pointer. */
84 : copy_body_data cb;
85 :
86 : /* The tree of contexts corresponding to the encountered constructs. */
87 : struct omp_context *outer;
88 : gimple *stmt;
89 :
90 : /* Map variables to fields in a structure that allows communication
91 : between sending and receiving threads. */
92 : splay_tree field_map;
93 : tree record_type;
94 : tree sender_decl;
95 : tree receiver_decl;
96 :
97 : /* These are used just by task contexts, if task firstprivate fn is
98 : needed. srecord_type is used to communicate from the thread
99 : that encountered the task construct to task firstprivate fn,
100 : record_type is allocated by GOMP_task, initialized by task firstprivate
101 : fn and passed to the task body fn. */
102 : splay_tree sfield_map;
103 : tree srecord_type;
104 :
105 : /* A chain of variables to add to the top-level block surrounding the
106 : construct. In the case of a parallel, this is in the child function. */
107 : tree block_vars;
108 :
109 : /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
110 : barriers should jump to during omplower pass. */
111 : tree cancel_label;
112 :
113 : /* The sibling GIMPLE_OMP_FOR simd with _simt_ clause or NULL
114 : otherwise. */
115 : gimple *simt_stmt;
116 :
117 : /* For task reductions registered in this context, a vector containing
118 : the length of the private copies block (if constant, otherwise NULL)
119 : and then offsets (if constant, otherwise NULL) for each entry. */
120 : vec<tree> task_reductions;
121 :
122 : /* A hash map from the reduction clauses to the registered array
123 : elts. */
124 : hash_map<tree, unsigned> *task_reduction_map;
125 :
126 : /* And a hash map from the lastprivate(conditional:) variables to their
127 : corresponding tracking loop iteration variables. */
128 : hash_map<tree, tree> *lastprivate_conditional_map;
129 :
130 : /* And a hash map from the allocate variables to their corresponding
131 : allocators. */
132 : hash_map<tree, tree> *allocate_map;
133 :
134 : /* A tree_list of the reduction clauses in this context. This is
135 : only used for checking the consistency of OpenACC reduction
136 : clauses in scan_omp_for and is not guaranteed to contain a valid
137 : value outside of this function. */
138 : tree local_reduction_clauses;
139 :
140 : /* A tree_list of the reduction clauses in outer contexts. This is
141 : only used for checking the consistency of OpenACC reduction
142 : clauses in scan_omp_for and is not guaranteed to contain a valid
143 : value outside of this function. */
144 : tree outer_reduction_clauses;
145 :
146 : /* Nesting depth of this context. Used to beautify error messages re
147 : invalid gotos. The outermost ctx is depth 1, with depth 0 being
148 : reserved for the main body of the function. */
149 : int depth;
150 :
151 : /* True if this parallel directive is nested within another. */
152 : bool is_nested;
153 :
154 : /* True if this construct can be cancelled. */
155 : bool cancellable;
156 :
157 : /* True if lower_omp_1 should look up lastprivate conditional in parent
158 : context. */
159 : bool combined_into_simd_safelen1;
160 :
161 : /* True if there is nested scan context with inclusive clause. */
162 : bool scan_inclusive;
163 :
164 : /* True if there is nested scan context with exclusive clause. */
165 : bool scan_exclusive;
166 :
167 : /* True in the second simd loop of for simd with inscan reductions. */
168 : bool for_simd_scan_phase;
169 :
170 : /* True if there is order(concurrent) clause on the construct. */
171 : bool order_concurrent;
172 :
173 : /* True if there is bind clause on the construct (i.e. a loop construct). */
174 : bool loop_p;
175 :
176 : /* Only used for omp target contexts. True if a teams construct is
177 : strictly nested in it. */
178 : bool teams_nested_p;
179 :
180 : /* Only used for omp target contexts. True if an OpenMP construct other
181 : than teams is strictly nested in it. */
182 : bool nonteams_nested_p;
183 :
184 : /* Candidates for adjusting OpenACC privatization level. */
185 : vec<tree> oacc_privatization_candidates;
186 : };
187 :
188 : static splay_tree all_contexts;
189 : static int taskreg_nesting_level;
190 : static int target_nesting_level;
191 : static bitmap make_addressable_vars;
192 : static bitmap global_nonaddressable_vars;
193 : static vec<omp_context *> taskreg_contexts;
194 : static vec<gomp_task *> task_cpyfns;
195 :
196 : static void scan_omp (gimple_seq *, omp_context *);
197 : static tree scan_omp_1_op (tree *, int *, void *);
198 : static bool omp_maybe_offloaded_ctx (omp_context *ctx);
199 :
200 : #define WALK_SUBSTMTS \
201 : case GIMPLE_BIND: \
202 : case GIMPLE_TRY: \
203 : case GIMPLE_CATCH: \
204 : case GIMPLE_EH_FILTER: \
205 : case GIMPLE_ASSUME: \
206 : case GIMPLE_TRANSACTION: \
207 : /* The sub-statements for these should be walked. */ \
208 : *handled_ops_p = false; \
209 : break;
210 :
211 : /* Return whether CTX represents an OpenACC 'parallel' or 'serial' construct.
212 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
213 :
214 : static bool
215 17772 : is_oacc_parallel_or_serial (omp_context *ctx)
216 : {
217 17772 : enum gimple_code outer_type = gimple_code (ctx->stmt);
218 17772 : return ((outer_type == GIMPLE_OMP_TARGET)
219 17772 : && ((gimple_omp_target_kind (ctx->stmt)
220 : == GF_OMP_TARGET_KIND_OACC_PARALLEL)
221 3344 : || (gimple_omp_target_kind (ctx->stmt)
222 17772 : == GF_OMP_TARGET_KIND_OACC_SERIAL)));
223 : }
224 :
225 : /* Return whether CTX represents an OpenACC 'kernels' construct.
226 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
227 :
228 : static bool
229 49894 : is_oacc_kernels (omp_context *ctx)
230 : {
231 49894 : enum gimple_code outer_type = gimple_code (ctx->stmt);
232 49894 : return ((outer_type == GIMPLE_OMP_TARGET)
233 49894 : && (gimple_omp_target_kind (ctx->stmt)
234 49894 : == GF_OMP_TARGET_KIND_OACC_KERNELS));
235 : }
236 :
237 : /* Return whether CTX represents an OpenACC 'kernels' decomposed part. */
238 :
239 : static bool
240 21645 : is_oacc_kernels_decomposed_part (omp_context *ctx)
241 : {
242 21645 : enum gimple_code outer_type = gimple_code (ctx->stmt);
243 21645 : return ((outer_type == GIMPLE_OMP_TARGET)
244 21645 : && ((gimple_omp_target_kind (ctx->stmt)
245 : == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED)
246 14552 : || (gimple_omp_target_kind (ctx->stmt)
247 : == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE)
248 14552 : || (gimple_omp_target_kind (ctx->stmt)
249 21645 : == GF_OMP_TARGET_KIND_OACC_DATA_KERNELS)));
250 : }
251 :
252 : /* Return true if STMT corresponds to an OpenMP target region. */
253 : static bool
254 42776 : is_omp_target (gimple *stmt)
255 : {
256 42776 : if (gimple_code (stmt) == GIMPLE_OMP_TARGET)
257 : {
258 6446 : int kind = gimple_omp_target_kind (stmt);
259 6446 : return (kind == GF_OMP_TARGET_KIND_REGION
260 6446 : || kind == GF_OMP_TARGET_KIND_DATA
261 6446 : || kind == GF_OMP_TARGET_KIND_ENTER_DATA
262 11029 : || kind == GF_OMP_TARGET_KIND_EXIT_DATA);
263 : }
264 : return false;
265 : }
266 :
267 : /* If DECL is the artificial dummy VAR_DECL created for non-static
268 : data member privatization, return the underlying "this" parameter,
269 : otherwise return NULL. */
270 :
271 : tree
272 531120 : omp_member_access_dummy_var (tree decl)
273 : {
274 531120 : if (!VAR_P (decl)
275 327018 : || !DECL_ARTIFICIAL (decl)
276 160224 : || !DECL_IGNORED_P (decl)
277 154972 : || !DECL_HAS_VALUE_EXPR_P (decl)
278 543701 : || !lang_hooks.decls.omp_disregard_value_expr (decl, false))
279 528860 : return NULL_TREE;
280 :
281 2260 : tree v = DECL_VALUE_EXPR (decl);
282 2260 : if (TREE_CODE (v) != COMPONENT_REF)
283 : return NULL_TREE;
284 :
285 16164 : while (1)
286 9197 : switch (TREE_CODE (v))
287 : {
288 6967 : case COMPONENT_REF:
289 6967 : case MEM_REF:
290 6967 : case INDIRECT_REF:
291 6967 : CASE_CONVERT:
292 6967 : case POINTER_PLUS_EXPR:
293 6967 : v = TREE_OPERAND (v, 0);
294 6967 : continue;
295 2211 : case PARM_DECL:
296 2211 : if (DECL_CONTEXT (v) == current_function_decl
297 2211 : && DECL_ARTIFICIAL (v)
298 4422 : && TREE_CODE (TREE_TYPE (v)) == POINTER_TYPE)
299 : return v;
300 : return NULL_TREE;
301 : default:
302 : return NULL_TREE;
303 : }
304 : }
305 :
306 : /* Helper for unshare_and_remap, called through walk_tree. */
307 :
308 : static tree
309 1579 : unshare_and_remap_1 (tree *tp, int *walk_subtrees, void *data)
310 : {
311 1579 : tree *pair = (tree *) data;
312 1579 : if (*tp == pair[0])
313 : {
314 233 : *tp = unshare_expr (pair[1]);
315 233 : *walk_subtrees = 0;
316 : }
317 1346 : else if (IS_TYPE_OR_DECL_P (*tp))
318 244 : *walk_subtrees = 0;
319 1579 : return NULL_TREE;
320 : }
321 :
322 : /* Return unshare_expr (X) with all occurrences of FROM
323 : replaced with TO. */
324 :
325 : static tree
326 150 : unshare_and_remap (tree x, tree from, tree to)
327 : {
328 150 : tree pair[2] = { from, to };
329 150 : x = unshare_expr (x);
330 150 : walk_tree (&x, unshare_and_remap_1, pair, NULL);
331 150 : return x;
332 : }
333 :
334 : /* Convenience function for calling scan_omp_1_op on tree operands. */
335 :
336 : static inline tree
337 338814 : scan_omp_op (tree *tp, omp_context *ctx)
338 : {
339 338814 : struct walk_stmt_info wi;
340 :
341 338814 : memset (&wi, 0, sizeof (wi));
342 338814 : wi.info = ctx;
343 338814 : wi.want_locations = true;
344 :
345 338814 : return walk_tree (tp, scan_omp_1_op, &wi, NULL);
346 : }
347 :
348 : static void lower_omp (gimple_seq *, omp_context *);
349 : static tree lookup_decl_in_outer_ctx (tree, omp_context *);
350 : static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
351 :
352 : /* Return true if CTX is for an omp parallel. */
353 :
354 : static inline bool
355 775365 : is_parallel_ctx (omp_context *ctx)
356 : {
357 61117 : return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
358 : }
359 :
360 :
361 : /* Return true if CTX is for an omp task. */
362 :
363 : static inline bool
364 235675 : is_task_ctx (omp_context *ctx)
365 : {
366 38735 : return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
367 : }
368 :
369 :
370 : /* Return true if CTX is for an omp taskloop. */
371 :
372 : static inline bool
373 8182 : is_taskloop_ctx (omp_context *ctx)
374 : {
375 8182 : return gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
376 8182 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP;
377 : }
378 :
379 :
380 : /* Return true if CTX is for a host omp teams. */
381 :
382 : static inline bool
383 520582 : is_host_teams_ctx (omp_context *ctx)
384 : {
385 520582 : return gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
386 520582 : && gimple_omp_teams_host (as_a <gomp_teams *> (ctx->stmt));
387 : }
388 :
389 : /* Return true if CTX is for an omp parallel or omp task or host omp teams
390 : (the last one is strictly not a task region in OpenMP speak, but we
391 : need to treat it similarly). */
392 :
393 : static inline bool
394 662819 : is_taskreg_ctx (omp_context *ctx)
395 : {
396 662819 : return is_parallel_ctx (ctx) || is_task_ctx (ctx) || is_host_teams_ctx (ctx);
397 : }
398 :
399 : /* Return true if EXPR is variable sized. If IS_REF is true, then
400 : EXPR is assumed to be a reference and the object it refers
401 : to is checked instead. */
402 :
403 : static inline bool
404 904347 : is_variable_sized (const_tree expr, bool is_ref = false)
405 : {
406 904347 : if (is_ref)
407 2140 : expr = TREE_TYPE (expr);
408 904347 : return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
409 : }
410 :
411 : /* Lookup variables. The "maybe" form
412 : allows for the variable form to not have been entered, otherwise we
413 : assert that the variable must have been entered. */
414 :
415 : static inline tree
416 713585 : lookup_decl (tree var, omp_context *ctx)
417 : {
418 1427071 : tree *n = ctx->cb.decl_map->get (var);
419 713585 : return *n;
420 : }
421 :
422 : static inline tree
423 718780 : maybe_lookup_decl (const_tree var, omp_context *ctx)
424 : {
425 718780 : tree *n = ctx->cb.decl_map->get (const_cast<tree> (var));
426 718780 : return n ? *n : NULL_TREE;
427 : }
428 :
429 : static inline tree
430 126245 : lookup_field (tree var, omp_context *ctx)
431 : {
432 126245 : splay_tree_node n;
433 252490 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
434 126245 : return (tree) n->value;
435 : }
436 :
437 : static inline tree
438 167511 : lookup_sfield (splay_tree_key key, omp_context *ctx)
439 : {
440 167511 : splay_tree_node n;
441 167511 : n = splay_tree_lookup (ctx->sfield_map
442 : ? ctx->sfield_map : ctx->field_map, key);
443 167511 : return (tree) n->value;
444 : }
445 :
446 : static inline tree
447 967 : lookup_sfield (tree var, omp_context *ctx)
448 : {
449 967 : return lookup_sfield ((splay_tree_key) var, ctx);
450 : }
451 :
452 : static inline tree
453 219134 : maybe_lookup_field (splay_tree_key key, omp_context *ctx)
454 : {
455 219134 : splay_tree_node n;
456 219134 : n = splay_tree_lookup (ctx->field_map, key);
457 219134 : return n ? (tree) n->value : NULL_TREE;
458 : }
459 :
460 : static inline tree
461 219134 : maybe_lookup_field (tree var, omp_context *ctx)
462 : {
463 2860 : return maybe_lookup_field ((splay_tree_key) var, ctx);
464 : }
465 :
466 : /* Return true if DECL should be copied by pointer. SHARED_CTX is
467 : the parallel context if DECL is to be shared. */
468 :
469 : static bool
470 232208 : use_pointer_for_field (tree decl, omp_context *shared_ctx)
471 : {
472 452892 : if (AGGREGATE_TYPE_P (TREE_TYPE (decl))
473 215589 : || TYPE_ATOMIC (TREE_TYPE (decl))
474 447797 : || POLY_INT_CST_P (DECL_SIZE (decl)))
475 : return true;
476 :
477 : /* We can only use copy-in/copy-out semantics for shared variables
478 : when we know the value is not accessible from an outer scope. */
479 215559 : if (shared_ctx)
480 : {
481 29394 : gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt));
482 :
483 : /* ??? Trivially accessible from anywhere. But why would we even
484 : be passing an address in this case? Should we simply assert
485 : this to be false, or should we have a cleanup pass that removes
486 : these from the list of mappings? */
487 29394 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, shared_ctx)))
488 : return true;
489 :
490 : /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
491 : without analyzing the expression whether or not its location
492 : is accessible to anyone else. In the case of nested parallel
493 : regions it certainly may be. */
494 29394 : if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
495 : return true;
496 :
497 : /* Do not use copy-in/copy-out for variables that have their
498 : address taken. */
499 28480 : if (is_global_var (decl))
500 : {
501 : /* For file scope vars, track whether we've seen them as
502 : non-addressable initially and in that case, keep the same
503 : answer for the duration of the pass, even when they are made
504 : addressable later on e.g. through reduction expansion. Global
505 : variables which weren't addressable before the pass will not
506 : have their privatized copies address taken. See PR91216. */
507 1232 : if (!TREE_ADDRESSABLE (decl))
508 : {
509 479 : if (!global_nonaddressable_vars)
510 74 : global_nonaddressable_vars = BITMAP_ALLOC (NULL);
511 479 : bitmap_set_bit (global_nonaddressable_vars, DECL_UID (decl));
512 : }
513 753 : else if (!global_nonaddressable_vars
514 986 : || !bitmap_bit_p (global_nonaddressable_vars,
515 233 : DECL_UID (decl)))
516 731 : return true;
517 : }
518 27248 : else if (TREE_ADDRESSABLE (decl))
519 : return true;
520 :
521 : /* lower_send_shared_vars only uses copy-in, but not copy-out
522 : for these. */
523 18344 : if (TREE_READONLY (decl)
524 18344 : || ((TREE_CODE (decl) == RESULT_DECL
525 11761 : || TREE_CODE (decl) == PARM_DECL)
526 673 : && DECL_BY_REFERENCE (decl)))
527 : return false;
528 :
529 : /* Disallow copy-in/out in nested parallel if
530 : decl is shared in outer parallel, otherwise
531 : each thread could store the shared variable
532 : in its own copy-in location, making the
533 : variable no longer really shared. */
534 11529 : if (shared_ctx->is_nested)
535 : {
536 2470 : omp_context *up;
537 :
538 5975 : for (up = shared_ctx->outer; up; up = up->outer)
539 5702 : if ((is_taskreg_ctx (up)
540 3643 : || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
541 411 : && is_gimple_omp_offloaded (up->stmt)))
542 6113 : && maybe_lookup_decl (decl, up))
543 : break;
544 :
545 2470 : if (up)
546 : {
547 2197 : tree c;
548 :
549 2197 : if (gimple_code (up->stmt) == GIMPLE_OMP_TARGET)
550 : {
551 309 : for (c = gimple_omp_target_clauses (up->stmt);
552 2031 : c; c = OMP_CLAUSE_CHAIN (c))
553 1932 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
554 1932 : && OMP_CLAUSE_DECL (c) == decl)
555 : break;
556 : }
557 : else
558 1888 : for (c = gimple_omp_taskreg_clauses (up->stmt);
559 6909 : c; c = OMP_CLAUSE_CHAIN (c))
560 6454 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
561 6454 : && OMP_CLAUSE_DECL (c) == decl)
562 : break;
563 :
564 2197 : if (c)
565 1643 : goto maybe_mark_addressable_and_ret;
566 : }
567 : }
568 :
569 : /* For tasks avoid using copy-in/out. As tasks can be
570 : deferred or executed in different thread, when GOMP_task
571 : returns, the task hasn't necessarily terminated. */
572 9886 : if (is_task_ctx (shared_ctx))
573 : {
574 2193 : tree outer;
575 550 : maybe_mark_addressable_and_ret:
576 2193 : outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
577 2193 : if (is_gimple_reg (outer) && !omp_member_access_dummy_var (outer))
578 : {
579 : /* Taking address of OUTER in lower_send_shared_vars
580 : might need regimplification of everything that uses the
581 : variable. */
582 982 : if (!make_addressable_vars)
583 489 : make_addressable_vars = BITMAP_ALLOC (NULL);
584 982 : bitmap_set_bit (make_addressable_vars, DECL_UID (outer));
585 982 : TREE_ADDRESSABLE (outer) = 1;
586 : }
587 2193 : return true;
588 : }
589 : }
590 :
591 : return false;
592 : }
593 :
594 : /* Construct a new automatic decl similar to VAR. */
595 :
596 : static tree
597 228567 : omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
598 : {
599 228567 : tree copy = copy_var_decl (var, name, type);
600 :
601 228567 : DECL_CONTEXT (copy) = current_function_decl;
602 :
603 228567 : if (ctx)
604 : {
605 228286 : DECL_CHAIN (copy) = ctx->block_vars;
606 228286 : ctx->block_vars = copy;
607 : }
608 : else
609 281 : record_vars (copy);
610 :
611 : /* If VAR is listed in make_addressable_vars, it wasn't
612 : originally addressable, but was only later made so.
613 : We don't need to take address of privatizations
614 : from that var. */
615 228567 : if (TREE_ADDRESSABLE (var)
616 228567 : && ((make_addressable_vars
617 3467 : && bitmap_bit_p (make_addressable_vars, DECL_UID (var)))
618 39149 : || (global_nonaddressable_vars
619 515 : && bitmap_bit_p (global_nonaddressable_vars, DECL_UID (var)))))
620 2077 : TREE_ADDRESSABLE (copy) = 0;
621 :
622 228567 : return copy;
623 : }
624 :
625 : static tree
626 228567 : omp_copy_decl_1 (tree var, omp_context *ctx)
627 : {
628 228567 : return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
629 : }
630 :
631 : /* Build tree nodes to access the field for VAR on the receiver side. */
632 :
633 : static tree
634 121790 : build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
635 : {
636 121790 : tree x, field = lookup_field (var, ctx);
637 :
638 : /* If the receiver record type was remapped in the child function,
639 : remap the field into the new record type. */
640 121790 : x = maybe_lookup_field (field, ctx);
641 121790 : if (x != NULL)
642 11000 : field = x;
643 :
644 121790 : x = build_simple_mem_ref (ctx->receiver_decl);
645 121790 : TREE_THIS_NOTRAP (x) = 1;
646 121790 : x = omp_build_component_ref (x, field);
647 121790 : if (by_ref)
648 : {
649 35534 : x = build_simple_mem_ref (x);
650 35534 : TREE_THIS_NOTRAP (x) = 1;
651 : }
652 :
653 121790 : return x;
654 : }
655 :
656 : /* Build tree nodes to access VAR in the scope outer to CTX. In the case
657 : of a parallel, this is a component reference; for workshare constructs
658 : this is some variable. */
659 :
660 : static tree
661 114318 : build_outer_var_ref (tree var, omp_context *ctx,
662 : enum omp_clause_code code = OMP_CLAUSE_ERROR)
663 : {
664 114318 : tree x;
665 114318 : omp_context *outer = ctx->outer;
666 114707 : for (; outer; outer = outer->outer)
667 : {
668 82468 : if (gimple_code (outer->stmt) == GIMPLE_OMP_TASKGROUP)
669 373 : continue;
670 82111 : if (gimple_code (outer->stmt) == GIMPLE_OMP_SCOPE
671 82095 : && !maybe_lookup_decl (var, outer))
672 16 : continue;
673 : break;
674 : }
675 :
676 114318 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
677 : x = var;
678 106219 : else if (is_variable_sized (var))
679 : {
680 47 : x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
681 47 : x = build_outer_var_ref (x, ctx, code);
682 47 : x = build_simple_mem_ref (x);
683 : }
684 106172 : else if (is_taskreg_ctx (ctx))
685 : {
686 63636 : bool by_ref = use_pointer_for_field (var, NULL);
687 63636 : x = build_receiver_ref (var, by_ref, ctx);
688 : }
689 42536 : else if ((gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
690 41199 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
691 14074 : || ctx->loop_p
692 13301 : || code == OMP_CLAUSE_ALLOCATE
693 55585 : || (code == OMP_CLAUSE_PRIVATE
694 39 : && (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
695 : || gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS
696 : || gimple_code (ctx->stmt) == GIMPLE_OMP_SINGLE)))
697 : {
698 : /* #pragma omp simd isn't a worksharing construct, and can reference
699 : even private vars in its linear etc. clauses.
700 : Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer
701 : to private vars in all worksharing constructs. */
702 29506 : x = NULL_TREE;
703 29506 : if (outer && is_taskreg_ctx (outer))
704 1002 : x = lookup_decl (var, outer);
705 28504 : else if (outer)
706 22314 : x = maybe_lookup_decl_in_outer_ctx (var, ctx);
707 23316 : if (x == NULL_TREE)
708 : x = var;
709 : }
710 13030 : else if (code == OMP_CLAUSE_LASTPRIVATE && is_taskloop_ctx (ctx))
711 : {
712 689 : gcc_assert (outer);
713 689 : splay_tree_node n
714 1378 : = splay_tree_lookup (outer->field_map,
715 689 : (splay_tree_key) &DECL_UID (var));
716 689 : if (n == NULL)
717 : {
718 610 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, outer)))
719 : x = var;
720 : else
721 384 : x = lookup_decl (var, outer);
722 : }
723 : else
724 : {
725 79 : tree field = (tree) n->value;
726 : /* If the receiver record type was remapped in the child function,
727 : remap the field into the new record type. */
728 79 : x = maybe_lookup_field (field, outer);
729 79 : if (x != NULL)
730 0 : field = x;
731 :
732 79 : x = build_simple_mem_ref (outer->receiver_decl);
733 79 : x = omp_build_component_ref (x, field);
734 79 : if (use_pointer_for_field (var, outer))
735 59 : x = build_simple_mem_ref (x);
736 : }
737 : }
738 12341 : else if (outer)
739 12149 : x = lookup_decl (var, outer);
740 192 : else if (omp_privatize_by_reference (var))
741 : /* This can happen with orphaned constructs. If var is reference, it is
742 : possible it is shared and as such valid. */
743 : x = var;
744 94 : else if (omp_member_access_dummy_var (var))
745 : x = var;
746 : else
747 0 : gcc_unreachable ();
748 :
749 114318 : if (x == var)
750 : {
751 16281 : tree t = omp_member_access_dummy_var (var);
752 16281 : if (t)
753 : {
754 220 : x = DECL_VALUE_EXPR (var);
755 220 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
756 220 : if (o != t)
757 54 : x = unshare_and_remap (x, t, o);
758 : else
759 166 : x = unshare_expr (x);
760 : }
761 : }
762 :
763 114318 : if (omp_privatize_by_reference (var))
764 3080 : x = build_simple_mem_ref (x);
765 :
766 114318 : return x;
767 : }
768 :
769 : /* Build tree nodes to access the field for VAR on the sender side. */
770 :
771 : static tree
772 166465 : build_sender_ref (splay_tree_key key, omp_context *ctx)
773 : {
774 166465 : tree field = lookup_sfield (key, ctx);
775 166465 : tree tmp = ctx->sender_decl;
776 166465 : if (POINTER_TYPE_P (TREE_TYPE (tmp)))
777 451 : tmp = build_fold_indirect_ref (tmp);
778 166465 : return omp_build_component_ref (tmp, field);
779 : }
780 :
781 : static tree
782 162319 : build_sender_ref (tree var, omp_context *ctx)
783 : {
784 0 : return build_sender_ref ((splay_tree_key) var, ctx);
785 : }
786 :
787 : /* Add a new field for VAR inside the structure CTX->SENDER_DECL. If
788 : BASE_POINTERS_RESTRICT, declare the field with restrict. */
789 :
790 : static void
791 129969 : install_var_field (tree var, bool by_ref, int mask, omp_context *ctx)
792 : {
793 129969 : tree field, type, sfield = NULL_TREE;
794 129969 : splay_tree_key key = (splay_tree_key) var;
795 :
796 129969 : if ((mask & 16) != 0)
797 : {
798 620 : key = (splay_tree_key) &DECL_NAME (var);
799 620 : gcc_checking_assert (key != (splay_tree_key) var);
800 : }
801 129969 : if ((mask & 8) != 0)
802 : {
803 1615 : key = (splay_tree_key) &DECL_UID (var);
804 1615 : gcc_checking_assert (key != (splay_tree_key) var);
805 : }
806 129969 : gcc_assert ((mask & 1) == 0
807 : || !splay_tree_lookup (ctx->field_map, key));
808 129969 : gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
809 : || !splay_tree_lookup (ctx->sfield_map, key));
810 129969 : gcc_assert ((mask & 3) == 3
811 : || !is_gimple_omp_oacc (ctx->stmt));
812 :
813 129969 : type = TREE_TYPE (var);
814 129969 : if ((mask & 16) != 0)
815 620 : type = lang_hooks.decls.omp_array_data (var, true);
816 :
817 : /* Prevent redeclaring the var in the split-off function with a restrict
818 : pointer type. Note that we only clear type itself, restrict qualifiers in
819 : the pointed-to type will be ignored by points-to analysis. */
820 129969 : if (POINTER_TYPE_P (type)
821 129969 : && TYPE_RESTRICT (type))
822 6914 : type = build_qualified_type (type, TYPE_QUALS (type) & ~TYPE_QUAL_RESTRICT);
823 :
824 129969 : if (mask & 4)
825 : {
826 425 : gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
827 425 : type = build_pointer_type (build_pointer_type (type));
828 : }
829 129544 : else if (by_ref)
830 62944 : type = build_pointer_type (type);
831 66600 : else if ((mask & (32 | 3)) == 1
832 66600 : && omp_privatize_by_reference (var))
833 162 : type = TREE_TYPE (type);
834 :
835 129969 : field = build_decl (DECL_SOURCE_LOCATION (var),
836 129969 : FIELD_DECL, DECL_NAME (var), type);
837 :
838 : /* Remember what variable this field was created for. This does have a
839 : side effect of making dwarf2out ignore this member, so for helpful
840 : debugging we clear it later in delete_omp_context. */
841 129969 : DECL_ABSTRACT_ORIGIN (field) = var;
842 129969 : if ((mask & 16) == 0 && type == TREE_TYPE (var))
843 : {
844 62142 : SET_DECL_ALIGN (field, DECL_ALIGN (var));
845 62142 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
846 62142 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
847 : }
848 : else
849 67827 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
850 :
851 129969 : if ((mask & 3) == 3)
852 : {
853 128917 : insert_field_into_struct (ctx->record_type, field);
854 128917 : if (ctx->srecord_type)
855 : {
856 678 : sfield = build_decl (DECL_SOURCE_LOCATION (var),
857 678 : FIELD_DECL, DECL_NAME (var), type);
858 678 : DECL_ABSTRACT_ORIGIN (sfield) = var;
859 678 : SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
860 678 : DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
861 678 : TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
862 678 : insert_field_into_struct (ctx->srecord_type, sfield);
863 : }
864 : }
865 : else
866 : {
867 1052 : if (ctx->srecord_type == NULL_TREE)
868 : {
869 544 : tree t;
870 :
871 544 : ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
872 544 : ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
873 2027 : for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
874 : {
875 1483 : sfield = build_decl (DECL_SOURCE_LOCATION (t),
876 1483 : FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
877 1483 : DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
878 1483 : insert_field_into_struct (ctx->srecord_type, sfield);
879 2966 : splay_tree_insert (ctx->sfield_map,
880 1483 : (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
881 : (splay_tree_value) sfield);
882 : }
883 : }
884 1052 : sfield = field;
885 1052 : insert_field_into_struct ((mask & 1) ? ctx->record_type
886 : : ctx->srecord_type, field);
887 : }
888 :
889 129969 : if (mask & 1)
890 129582 : splay_tree_insert (ctx->field_map, key, (splay_tree_value) field);
891 129969 : if ((mask & 2) && ctx->sfield_map)
892 1065 : splay_tree_insert (ctx->sfield_map, key, (splay_tree_value) sfield);
893 129969 : }
894 :
895 : static tree
896 228136 : install_var_local (tree var, omp_context *ctx)
897 : {
898 228136 : tree new_var = omp_copy_decl_1 (var, ctx);
899 228136 : insert_decl_map (&ctx->cb, var, new_var);
900 228136 : return new_var;
901 : }
902 :
903 : /* Adjust the replacement for DECL in CTX for the new context. This means
904 : copying the DECL_VALUE_EXPR, and fixing up the type. */
905 :
906 : static void
907 203409 : fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
908 : {
909 203409 : tree new_decl, size;
910 :
911 203409 : new_decl = lookup_decl (decl, ctx);
912 :
913 203409 : TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
914 :
915 405685 : if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
916 203598 : && DECL_HAS_VALUE_EXPR_P (decl))
917 : {
918 1322 : tree ve = DECL_VALUE_EXPR (decl);
919 1322 : walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
920 1322 : SET_DECL_VALUE_EXPR (new_decl, ve);
921 1322 : DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
922 : }
923 :
924 203409 : if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
925 : {
926 1133 : size = remap_decl (DECL_SIZE (decl), &ctx->cb);
927 1133 : if (size == error_mark_node)
928 316 : size = TYPE_SIZE (TREE_TYPE (new_decl));
929 1133 : DECL_SIZE (new_decl) = size;
930 :
931 1133 : size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
932 1133 : if (size == error_mark_node)
933 303 : size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
934 1133 : DECL_SIZE_UNIT (new_decl) = size;
935 : }
936 203409 : }
937 :
938 : /* The callback for remap_decl. Search all containing contexts for a
939 : mapping of the variable; this avoids having to duplicate the splay
940 : tree ahead of time. We know a mapping doesn't already exist in the
941 : given context. Create new mappings to implement default semantics. */
942 :
943 : static tree
944 461274 : omp_copy_decl (tree var, copy_body_data *cb)
945 : {
946 461274 : omp_context *ctx = (omp_context *) cb;
947 461274 : tree new_var;
948 :
949 461274 : if (TREE_CODE (var) == LABEL_DECL)
950 : {
951 186433 : if (FORCED_LABEL (var) || DECL_NONLOCAL (var))
952 : return var;
953 186427 : new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
954 186427 : DECL_CONTEXT (new_var) = current_function_decl;
955 186427 : insert_decl_map (&ctx->cb, var, new_var);
956 186427 : return new_var;
957 : }
958 :
959 356659 : while (!is_taskreg_ctx (ctx))
960 : {
961 337472 : ctx = ctx->outer;
962 337472 : if (ctx == NULL)
963 : return var;
964 279427 : new_var = maybe_lookup_decl (var, ctx);
965 279427 : if (new_var)
966 : return new_var;
967 : }
968 :
969 19187 : if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
970 13354 : return var;
971 :
972 5833 : return error_mark_node;
973 : }
974 :
975 : /* Create a new context, with OUTER_CTX being the surrounding context. */
976 :
977 : static omp_context *
978 136913 : new_omp_context (gimple *stmt, omp_context *outer_ctx)
979 : {
980 136913 : omp_context *ctx = XCNEW (omp_context);
981 :
982 136913 : splay_tree_insert (all_contexts, (splay_tree_key) stmt,
983 : (splay_tree_value) ctx);
984 136913 : ctx->stmt = stmt;
985 :
986 136913 : if (outer_ctx)
987 : {
988 74419 : ctx->outer = outer_ctx;
989 74419 : ctx->cb = outer_ctx->cb;
990 74419 : ctx->cb.block = NULL;
991 74419 : ctx->depth = outer_ctx->depth + 1;
992 : }
993 : else
994 : {
995 62494 : ctx->cb.src_fn = current_function_decl;
996 62494 : ctx->cb.dst_fn = current_function_decl;
997 62494 : ctx->cb.src_node = cgraph_node::get (current_function_decl);
998 62494 : gcc_checking_assert (ctx->cb.src_node);
999 62494 : ctx->cb.dst_node = ctx->cb.src_node;
1000 62494 : ctx->cb.src_cfun = cfun;
1001 62494 : ctx->cb.copy_decl = omp_copy_decl;
1002 62494 : ctx->cb.eh_lp_nr = 0;
1003 62494 : ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1004 62494 : ctx->cb.adjust_array_error_bounds = true;
1005 62494 : ctx->cb.dont_remap_vla_if_no_change = true;
1006 62494 : ctx->depth = 1;
1007 : }
1008 :
1009 136913 : ctx->cb.decl_map = new hash_map<tree, tree>;
1010 :
1011 136913 : return ctx;
1012 : }
1013 :
1014 : static gimple_seq maybe_catch_exception (gimple_seq);
1015 :
1016 : /* Finalize task copyfn. */
1017 :
1018 : static void
1019 504 : finalize_task_copyfn (gomp_task *task_stmt)
1020 : {
1021 504 : struct function *child_cfun;
1022 504 : tree child_fn;
1023 504 : gimple_seq seq = NULL, new_seq;
1024 504 : gbind *bind;
1025 :
1026 504 : child_fn = gimple_omp_task_copy_fn (task_stmt);
1027 504 : if (child_fn == NULL_TREE)
1028 0 : return;
1029 :
1030 504 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1031 504 : DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1032 :
1033 504 : push_cfun (child_cfun);
1034 504 : bind = gimplify_body (child_fn, false);
1035 504 : gimple_seq_add_stmt (&seq, bind);
1036 504 : new_seq = maybe_catch_exception (seq);
1037 504 : if (new_seq != seq)
1038 : {
1039 332 : bind = gimple_build_bind (NULL, new_seq, NULL);
1040 332 : seq = NULL;
1041 332 : gimple_seq_add_stmt (&seq, bind);
1042 : }
1043 504 : gimple_set_body (child_fn, seq);
1044 504 : pop_cfun ();
1045 :
1046 : /* Inform the callgraph about the new function. */
1047 504 : cgraph_node *node = cgraph_node::get_create (child_fn);
1048 504 : node->parallelized_function = 1;
1049 504 : cgraph_node::add_new_function (child_fn, false);
1050 : }
1051 :
1052 : /* Destroy a omp_context data structures. Called through the splay tree
1053 : value delete callback. */
1054 :
1055 : static void
1056 136907 : delete_omp_context (splay_tree_value value)
1057 : {
1058 136907 : omp_context *ctx = (omp_context *) value;
1059 :
1060 273814 : delete ctx->cb.decl_map;
1061 :
1062 136907 : if (ctx->field_map)
1063 68411 : splay_tree_delete (ctx->field_map);
1064 136907 : if (ctx->sfield_map)
1065 544 : splay_tree_delete (ctx->sfield_map);
1066 :
1067 : /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1068 : it produces corrupt debug information. */
1069 136907 : if (ctx->record_type)
1070 : {
1071 52639 : tree t;
1072 219745 : for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1073 167106 : DECL_ABSTRACT_ORIGIN (t) = NULL;
1074 : }
1075 136907 : if (ctx->srecord_type)
1076 : {
1077 544 : tree t;
1078 3092 : for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1079 2548 : DECL_ABSTRACT_ORIGIN (t) = NULL;
1080 : }
1081 :
1082 136907 : if (ctx->task_reduction_map)
1083 : {
1084 1164 : ctx->task_reductions.release ();
1085 2328 : delete ctx->task_reduction_map;
1086 : }
1087 :
1088 137180 : delete ctx->lastprivate_conditional_map;
1089 138286 : delete ctx->allocate_map;
1090 :
1091 136907 : XDELETE (ctx);
1092 136907 : }
1093 :
1094 : /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1095 : context. */
1096 :
1097 : static void
1098 36296 : fixup_child_record_type (omp_context *ctx)
1099 : {
1100 36296 : tree f, type = ctx->record_type;
1101 :
1102 36296 : if (!ctx->receiver_decl)
1103 : return;
1104 : /* ??? It isn't sufficient to just call remap_type here, because
1105 : variably_modified_type_p doesn't work the way we expect for
1106 : record types. Testing each field for whether it needs remapping
1107 : and creating a new record by hand works, however. */
1108 159898 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1109 124838 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1110 : break;
1111 36296 : if (f)
1112 : {
1113 1236 : tree name, new_fields = NULL;
1114 :
1115 1236 : type = lang_hooks.types.make_type (RECORD_TYPE);
1116 1236 : name = DECL_NAME (TYPE_NAME (ctx->record_type));
1117 1236 : name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1118 : TYPE_DECL, name, type);
1119 1236 : TYPE_NAME (type) = name;
1120 :
1121 13195 : for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1122 : {
1123 11959 : tree new_f = copy_node (f);
1124 11959 : DECL_CONTEXT (new_f) = type;
1125 11959 : TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1126 11959 : DECL_CHAIN (new_f) = new_fields;
1127 11959 : walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1128 11959 : walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1129 : &ctx->cb, NULL);
1130 11959 : walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1131 : &ctx->cb, NULL);
1132 11959 : new_fields = new_f;
1133 :
1134 : /* Arrange to be able to look up the receiver field
1135 : given the sender field. */
1136 11959 : splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1137 : (splay_tree_value) new_f);
1138 : }
1139 1236 : TYPE_FIELDS (type) = nreverse (new_fields);
1140 1236 : layout_type (type);
1141 : }
1142 :
1143 : /* In a target region we never modify any of the pointers in *.omp_data_i,
1144 : so attempt to help the optimizers. */
1145 36296 : if (is_gimple_omp_offloaded (ctx->stmt))
1146 17163 : type = build_qualified_type (type, TYPE_QUAL_CONST);
1147 :
1148 36296 : TREE_TYPE (ctx->receiver_decl)
1149 108888 : = build_qualified_type (flexible_array_type_p (type)
1150 107 : ? build_pointer_type (type)
1151 36189 : : build_reference_type (type), TYPE_QUAL_RESTRICT);
1152 : }
1153 :
1154 : /* Instantiate decls as necessary in CTX to satisfy the data sharing
1155 : specified by CLAUSES. */
1156 :
1157 : static void
1158 129076 : scan_sharing_clauses (tree clauses, omp_context *ctx)
1159 : {
1160 129076 : tree c, decl;
1161 129076 : bool scan_array_reductions = false;
1162 129076 : bool flex_array_ptr = false;
1163 :
1164 546534 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1165 417458 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_ALLOCATE
1166 417458 : && (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
1167 : /* omp_default_mem_alloc is 1 */
1168 1284 : || !integer_onep (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1169 572 : || OMP_CLAUSE_ALLOCATE_ALIGN (c) != NULL_TREE))
1170 : {
1171 : /* The allocate clauses that appear on a target construct or on
1172 : constructs in a target region must specify an allocator expression
1173 : unless a requires directive with the dynamic_allocators clause
1174 : is present in the same compilation unit. */
1175 1848 : if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
1176 1092 : && ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0)
1177 2928 : && omp_maybe_offloaded_ctx (ctx))
1178 10 : error_at (OMP_CLAUSE_LOCATION (c), "%<allocate%> clause must"
1179 : " specify an allocator here");
1180 1848 : if (ctx->allocate_map == NULL)
1181 1379 : ctx->allocate_map = new hash_map<tree, tree>;
1182 1848 : tree val = integer_zero_node;
1183 1848 : if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1184 756 : val = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
1185 1848 : if (OMP_CLAUSE_ALLOCATE_ALIGN (c))
1186 287 : val = build_tree_list (val, OMP_CLAUSE_ALLOCATE_ALIGN (c));
1187 1848 : ctx->allocate_map->put (OMP_CLAUSE_DECL (c), val);
1188 : }
1189 :
1190 546534 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1191 : {
1192 417458 : bool by_ref;
1193 :
1194 417458 : switch (OMP_CLAUSE_CODE (c))
1195 : {
1196 73197 : case OMP_CLAUSE_PRIVATE:
1197 73197 : decl = OMP_CLAUSE_DECL (c);
1198 73197 : if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1199 164 : goto do_private;
1200 73033 : else if (!is_variable_sized (decl))
1201 72670 : install_var_local (decl, ctx);
1202 : break;
1203 :
1204 46748 : case OMP_CLAUSE_SHARED:
1205 46748 : decl = OMP_CLAUSE_DECL (c);
1206 46748 : if (ctx->allocate_map && ctx->allocate_map->get (decl))
1207 37 : ctx->allocate_map->remove (decl);
1208 : /* Ignore shared directives in teams construct inside of
1209 : target construct. */
1210 46748 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
1211 46748 : && !is_host_teams_ctx (ctx))
1212 : {
1213 : /* Global variables don't need to be copied,
1214 : the receiver side will use them directly. */
1215 11856 : tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx);
1216 11856 : if (is_global_var (odecl))
1217 : break;
1218 8127 : insert_decl_map (&ctx->cb, decl, odecl);
1219 8127 : break;
1220 : }
1221 34892 : gcc_assert (is_taskreg_ctx (ctx));
1222 34892 : gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1223 : || !is_variable_sized (decl));
1224 : /* Global variables don't need to be copied,
1225 : the receiver side will use them directly. */
1226 34892 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1227 : break;
1228 28839 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1229 : {
1230 86 : use_pointer_for_field (decl, ctx);
1231 86 : break;
1232 : }
1233 28753 : by_ref = use_pointer_for_field (decl, NULL);
1234 55041 : if ((! TREE_READONLY (decl) && !OMP_CLAUSE_SHARED_READONLY (c))
1235 19257 : || TREE_ADDRESSABLE (decl)
1236 18900 : || by_ref
1237 47653 : || omp_privatize_by_reference (decl))
1238 : {
1239 11964 : by_ref = use_pointer_for_field (decl, ctx);
1240 11964 : install_var_field (decl, by_ref, 3, ctx);
1241 11964 : install_var_local (decl, ctx);
1242 11964 : break;
1243 : }
1244 : /* We don't need to copy const scalar vars back. */
1245 16789 : OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1246 16789 : goto do_private;
1247 :
1248 14499 : case OMP_CLAUSE_REDUCTION:
1249 : /* Collect 'reduction' clauses on OpenACC compute construct. */
1250 14499 : if (is_gimple_omp_oacc (ctx->stmt)
1251 14499 : && is_gimple_omp_offloaded (ctx->stmt))
1252 : {
1253 : /* No 'reduction' clauses on OpenACC 'kernels'. */
1254 790 : gcc_checking_assert (!is_oacc_kernels (ctx));
1255 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
1256 790 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
1257 :
1258 790 : ctx->local_reduction_clauses
1259 790 : = tree_cons (NULL, c, ctx->local_reduction_clauses);
1260 : }
1261 : /* FALLTHRU */
1262 :
1263 16573 : case OMP_CLAUSE_IN_REDUCTION:
1264 16573 : decl = OMP_CLAUSE_DECL (c);
1265 16573 : if (ctx->allocate_map
1266 16573 : && ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1267 389 : && (OMP_CLAUSE_REDUCTION_INSCAN (c)
1268 389 : || OMP_CLAUSE_REDUCTION_TASK (c)))
1269 397 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
1270 389 : || is_task_ctx (ctx)))
1271 : {
1272 : /* For now. */
1273 17 : if (ctx->allocate_map->get (decl))
1274 17 : ctx->allocate_map->remove (decl);
1275 : }
1276 16573 : if (TREE_CODE (decl) == MEM_REF)
1277 : {
1278 2366 : tree t = TREE_OPERAND (decl, 0);
1279 2366 : if (TREE_CODE (t) == POINTER_PLUS_EXPR)
1280 436 : t = TREE_OPERAND (t, 0);
1281 2366 : if (INDIRECT_REF_P (t)
1282 2272 : || TREE_CODE (t) == ADDR_EXPR)
1283 1378 : t = TREE_OPERAND (t, 0);
1284 2366 : if (is_omp_target (ctx->stmt))
1285 : {
1286 96 : if (is_variable_sized (t))
1287 : {
1288 32 : gcc_assert (DECL_HAS_VALUE_EXPR_P (t));
1289 32 : t = DECL_VALUE_EXPR (t);
1290 32 : gcc_assert (INDIRECT_REF_P (t));
1291 32 : t = TREE_OPERAND (t, 0);
1292 32 : gcc_assert (DECL_P (t));
1293 : }
1294 96 : tree at = t;
1295 96 : if (ctx->outer)
1296 96 : scan_omp_op (&at, ctx->outer);
1297 96 : tree nt = omp_copy_decl_1 (at, ctx->outer);
1298 192 : splay_tree_insert (ctx->field_map,
1299 96 : (splay_tree_key) &DECL_CONTEXT (t),
1300 : (splay_tree_value) nt);
1301 96 : if (at != t)
1302 96 : splay_tree_insert (ctx->field_map,
1303 48 : (splay_tree_key) &DECL_CONTEXT (at),
1304 : (splay_tree_value) nt);
1305 96 : break;
1306 : }
1307 2270 : install_var_local (t, ctx);
1308 2270 : if (is_taskreg_ctx (ctx)
1309 1822 : && (!is_global_var (maybe_lookup_decl_in_outer_ctx (t, ctx))
1310 387 : || (is_task_ctx (ctx)
1311 310 : && (TREE_CODE (TREE_TYPE (t)) == POINTER_TYPE
1312 234 : || (TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
1313 0 : && (TREE_CODE (TREE_TYPE (TREE_TYPE (t)))
1314 : == POINTER_TYPE)))))
1315 1511 : && !is_variable_sized (t)
1316 3722 : && (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
1317 617 : || (!OMP_CLAUSE_REDUCTION_TASK (c)
1318 578 : && !is_task_ctx (ctx))))
1319 : {
1320 1335 : by_ref = use_pointer_for_field (t, NULL);
1321 1335 : if (is_task_ctx (ctx)
1322 835 : && TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
1323 1495 : && TREE_CODE (TREE_TYPE (TREE_TYPE (t))) == POINTER_TYPE)
1324 : {
1325 32 : install_var_field (t, false, 1, ctx);
1326 32 : install_var_field (t, by_ref, 2, ctx);
1327 : }
1328 : else
1329 1303 : install_var_field (t, by_ref, 3, ctx);
1330 : }
1331 : break;
1332 : }
1333 14207 : if (is_omp_target (ctx->stmt))
1334 : {
1335 335 : tree at = decl;
1336 335 : if (ctx->outer)
1337 54 : scan_omp_op (&at, ctx->outer);
1338 335 : tree nt = omp_copy_decl_1 (at, ctx->outer);
1339 670 : splay_tree_insert (ctx->field_map,
1340 335 : (splay_tree_key) &DECL_CONTEXT (decl),
1341 : (splay_tree_value) nt);
1342 335 : if (at != decl)
1343 24 : splay_tree_insert (ctx->field_map,
1344 12 : (splay_tree_key) &DECL_CONTEXT (at),
1345 : (splay_tree_value) nt);
1346 335 : break;
1347 : }
1348 13872 : if (is_task_ctx (ctx)
1349 26674 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1350 12802 : && OMP_CLAUSE_REDUCTION_TASK (c)
1351 372 : && is_parallel_ctx (ctx)))
1352 : {
1353 : /* Global variables don't need to be copied,
1354 : the receiver side will use them directly. */
1355 1149 : if (!is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1356 : {
1357 461 : by_ref = use_pointer_for_field (decl, ctx);
1358 461 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
1359 304 : install_var_field (decl, by_ref, 3, ctx);
1360 : }
1361 1149 : install_var_local (decl, ctx);
1362 1149 : break;
1363 : }
1364 12723 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1365 12723 : && OMP_CLAUSE_REDUCTION_TASK (c))
1366 : {
1367 293 : install_var_local (decl, ctx);
1368 293 : break;
1369 : }
1370 12430 : goto do_private;
1371 :
1372 22736 : case OMP_CLAUSE_LASTPRIVATE:
1373 : /* Let the corresponding firstprivate clause create
1374 : the variable. */
1375 22736 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1376 : break;
1377 : /* FALLTHRU */
1378 :
1379 58985 : case OMP_CLAUSE_FIRSTPRIVATE:
1380 58985 : case OMP_CLAUSE_LINEAR:
1381 58985 : decl = OMP_CLAUSE_DECL (c);
1382 88756 : do_private:
1383 88756 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1384 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
1385 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1386 45388 : && is_gimple_omp_offloaded (ctx->stmt))
1387 : {
1388 15952 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1389 15952 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR
1390 239 : && lang_hooks.decls.omp_array_data (decl, true)))
1391 : {
1392 15583 : by_ref = !omp_privatize_by_reference (decl);
1393 15583 : install_var_field (decl, by_ref, 3, ctx);
1394 : }
1395 369 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1396 : {
1397 220 : if (INDIRECT_REF_P (decl))
1398 0 : decl = TREE_OPERAND (decl, 0);
1399 220 : install_var_field (decl, true, 3, ctx);
1400 : }
1401 149 : else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1402 8 : install_var_field (decl, true, 3, ctx);
1403 : else
1404 141 : install_var_field (decl, false, 3, ctx);
1405 : }
1406 88756 : if (is_variable_sized (decl))
1407 : {
1408 60 : if (is_task_ctx (ctx))
1409 : {
1410 14 : if (ctx->allocate_map
1411 14 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
1412 : {
1413 : /* For now. */
1414 0 : if (ctx->allocate_map->get (decl))
1415 0 : ctx->allocate_map->remove (decl);
1416 : }
1417 14 : install_var_field (decl, false, 1, ctx);
1418 : }
1419 : break;
1420 : }
1421 88696 : else if (is_taskreg_ctx (ctx))
1422 : {
1423 38258 : bool global
1424 38258 : = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1425 38258 : by_ref = use_pointer_for_field (decl, NULL);
1426 :
1427 38258 : if (is_task_ctx (ctx)
1428 38258 : && (global || by_ref || omp_privatize_by_reference (decl)))
1429 : {
1430 619 : if (ctx->allocate_map
1431 619 : && ctx->allocate_map->get (decl))
1432 59 : install_var_field (decl, by_ref, 32 | 1, ctx);
1433 : else
1434 560 : install_var_field (decl, false, 1, ctx);
1435 619 : if (!global)
1436 355 : install_var_field (decl, by_ref, 2, ctx);
1437 : }
1438 37639 : else if (!global)
1439 36376 : install_var_field (decl, by_ref, 3, ctx);
1440 : }
1441 88696 : install_var_local (decl, ctx);
1442 : /* For descr arrays on target: firstprivatize data + attach ptr. */
1443 88696 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1444 44953 : && is_gimple_omp_offloaded (ctx->stmt)
1445 15558 : && !is_gimple_omp_oacc (ctx->stmt)
1446 100006 : && lang_hooks.decls.omp_array_data (decl, true))
1447 : {
1448 19 : install_var_field (decl, false, 16 | 3, ctx);
1449 19 : install_var_field (decl, true, 8 | 3, ctx);
1450 : }
1451 : break;
1452 :
1453 2118 : case OMP_CLAUSE_USE_DEVICE_PTR:
1454 2118 : case OMP_CLAUSE_USE_DEVICE_ADDR:
1455 2118 : decl = OMP_CLAUSE_DECL (c);
1456 :
1457 : /* Fortran array descriptors. */
1458 2118 : if (lang_hooks.decls.omp_array_data (decl, true))
1459 601 : install_var_field (decl, false, 19, ctx);
1460 1517 : else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
1461 1312 : && !omp_privatize_by_reference (decl)
1462 513 : && !omp_is_allocatable_or_ptr (decl))
1463 2454 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1464 407 : install_var_field (decl, true, 11, ctx);
1465 : else
1466 1110 : install_var_field (decl, false, 11, ctx);
1467 2118 : if (DECL_SIZE (decl)
1468 2118 : && !poly_int_tree_p (DECL_SIZE (decl)))
1469 : {
1470 6 : tree decl2 = DECL_VALUE_EXPR (decl);
1471 6 : gcc_assert (INDIRECT_REF_P (decl2));
1472 6 : decl2 = TREE_OPERAND (decl2, 0);
1473 6 : gcc_assert (DECL_P (decl2));
1474 6 : install_var_local (decl2, ctx);
1475 : }
1476 2118 : install_var_local (decl, ctx);
1477 2118 : break;
1478 :
1479 239 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
1480 239 : decl = OMP_CLAUSE_DECL (c);
1481 239 : while (INDIRECT_REF_P (decl)
1482 256 : || TREE_CODE (decl) == ARRAY_REF)
1483 17 : decl = TREE_OPERAND (decl, 0);
1484 239 : goto do_private;
1485 :
1486 149 : case OMP_CLAUSE_IS_DEVICE_PTR:
1487 149 : decl = OMP_CLAUSE_DECL (c);
1488 149 : goto do_private;
1489 :
1490 20110 : case OMP_CLAUSE__LOOPTEMP_:
1491 20110 : case OMP_CLAUSE__REDUCTEMP_:
1492 20110 : gcc_assert (is_taskreg_ctx (ctx));
1493 20110 : decl = OMP_CLAUSE_DECL (c);
1494 20110 : install_var_field (decl, false, 3, ctx);
1495 20110 : install_var_local (decl, ctx);
1496 20110 : break;
1497 :
1498 892 : case OMP_CLAUSE_COPYPRIVATE:
1499 892 : case OMP_CLAUSE_COPYIN:
1500 892 : decl = OMP_CLAUSE_DECL (c);
1501 892 : by_ref = use_pointer_for_field (decl, NULL);
1502 892 : install_var_field (decl, by_ref, 3, ctx);
1503 892 : break;
1504 :
1505 54463 : case OMP_CLAUSE_FINAL:
1506 54463 : case OMP_CLAUSE_IF:
1507 54463 : case OMP_CLAUSE_SELF:
1508 54463 : case OMP_CLAUSE_NUM_THREADS:
1509 54463 : case OMP_CLAUSE_NUM_TEAMS:
1510 54463 : case OMP_CLAUSE_THREAD_LIMIT:
1511 54463 : case OMP_CLAUSE_DEVICE:
1512 54463 : case OMP_CLAUSE_SCHEDULE:
1513 54463 : case OMP_CLAUSE_DIST_SCHEDULE:
1514 54463 : case OMP_CLAUSE_DEPEND:
1515 54463 : case OMP_CLAUSE_PRIORITY:
1516 54463 : case OMP_CLAUSE_GRAINSIZE:
1517 54463 : case OMP_CLAUSE_NUM_TASKS:
1518 54463 : case OMP_CLAUSE_NUM_GANGS:
1519 54463 : case OMP_CLAUSE_NUM_WORKERS:
1520 54463 : case OMP_CLAUSE_VECTOR_LENGTH:
1521 54463 : case OMP_CLAUSE_DETACH:
1522 54463 : case OMP_CLAUSE_FILTER:
1523 54463 : if (ctx->outer)
1524 17544 : scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1525 : break;
1526 :
1527 86727 : case OMP_CLAUSE_TO:
1528 86727 : case OMP_CLAUSE_FROM:
1529 86727 : case OMP_CLAUSE_MAP:
1530 86727 : if (ctx->outer)
1531 15327 : scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1532 86727 : decl = OMP_CLAUSE_DECL (c);
1533 : /* If requested, make 'decl' addressable. */
1534 86727 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1535 86727 : && OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c))
1536 : {
1537 433 : gcc_checking_assert (DECL_P (decl));
1538 :
1539 433 : bool decl_addressable = TREE_ADDRESSABLE (decl);
1540 433 : if (!decl_addressable)
1541 : {
1542 219 : if (!make_addressable_vars)
1543 131 : make_addressable_vars = BITMAP_ALLOC (NULL);
1544 219 : bitmap_set_bit (make_addressable_vars, DECL_UID (decl));
1545 219 : TREE_ADDRESSABLE (decl) = 1;
1546 : }
1547 :
1548 433 : if (dump_enabled_p ())
1549 : {
1550 421 : location_t loc = OMP_CLAUSE_LOCATION (c);
1551 421 : const dump_user_location_t d_u_loc
1552 421 : = dump_user_location_t::from_location_t (loc);
1553 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
1554 : #if __GNUC__ >= 10
1555 421 : # pragma GCC diagnostic push
1556 421 : # pragma GCC diagnostic ignored "-Wformat"
1557 : #endif
1558 421 : if (!decl_addressable)
1559 207 : dump_printf_loc (MSG_NOTE, d_u_loc,
1560 : "variable %<%T%>"
1561 : " made addressable\n",
1562 : decl);
1563 : else
1564 214 : dump_printf_loc (MSG_NOTE, d_u_loc,
1565 : "variable %<%T%>"
1566 : " already made addressable\n",
1567 : decl);
1568 : #if __GNUC__ >= 10
1569 421 : # pragma GCC diagnostic pop
1570 : #endif
1571 : }
1572 :
1573 : /* Done. */
1574 433 : OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c) = 0;
1575 : }
1576 : /* Global variables with "omp declare target" attribute
1577 : don't need to be copied, the receiver side will use them
1578 : directly. However, global variables with "omp declare target link"
1579 : attribute need to be copied. Or when ALWAYS modifier is used. */
1580 86727 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1581 79070 : && DECL_P (decl)
1582 43427 : && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
1583 39672 : && (OMP_CLAUSE_MAP_KIND (c)
1584 : != GOMP_MAP_FIRSTPRIVATE_REFERENCE)
1585 39247 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
1586 39021 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH)
1587 4466 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1588 40128 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TO
1589 40017 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_FROM
1590 39998 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TOFROM
1591 39510 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TO
1592 39484 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_FROM
1593 39462 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TOFROM
1594 39448 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_TO_PSET
1595 35493 : && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1596 7538 : && varpool_node::get_create (decl)->offloadable
1597 91981 : && !lookup_attribute ("omp declare target link",
1598 5254 : DECL_ATTRIBUTES (decl)))
1599 : break;
1600 81497 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1601 81497 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER)
1602 : {
1603 : /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1604 : not offloaded; there is nothing to map for those. */
1605 10409 : if (!is_gimple_omp_offloaded (ctx->stmt)
1606 4966 : && !POINTER_TYPE_P (TREE_TYPE (decl))
1607 10673 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
1608 : break;
1609 : }
1610 81233 : if (!flex_array_ptr)
1611 81005 : flex_array_ptr = lang_hooks.decls.omp_deep_mapping_p (ctx->stmt, c);
1612 81233 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1613 73576 : && DECL_P (decl)
1614 37933 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
1615 37707 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
1616 81519 : && is_omp_target (ctx->stmt))
1617 : {
1618 : /* If this is an offloaded region, an attach operation should
1619 : only exist when the pointer variable is mapped in a prior
1620 : clause. An exception is if we have a reference (to pointer):
1621 : in that case we should have mapped "*decl" in a previous
1622 : mapping instead of "decl". Skip the assertion in that case.
1623 : If we had an error, we may not have attempted to sort clauses
1624 : properly, so avoid the test. */
1625 252 : if (TREE_CODE (TREE_TYPE (decl)) != REFERENCE_TYPE
1626 220 : && is_gimple_omp_offloaded (ctx->stmt)
1627 322 : && !seen_error ())
1628 62 : gcc_assert
1629 : (maybe_lookup_decl (decl, ctx)
1630 : || (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1631 : && lookup_attribute ("omp declare target",
1632 : DECL_ATTRIBUTES (decl))));
1633 :
1634 : /* By itself, attach/detach is generated as part of pointer
1635 : variable mapping and should not create new variables in the
1636 : offloaded region, however sender refs for it must be created
1637 : for its address to be passed to the runtime. */
1638 252 : tree field
1639 252 : = build_decl (OMP_CLAUSE_LOCATION (c),
1640 : FIELD_DECL, NULL_TREE, ptr_type_node);
1641 252 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
1642 252 : insert_field_into_struct (ctx->record_type, field);
1643 : /* To not clash with a map of the pointer variable itself,
1644 : attach/detach maps have their field looked up by the *clause*
1645 : tree expression, not the decl. */
1646 252 : gcc_assert (!splay_tree_lookup (ctx->field_map,
1647 : (splay_tree_key) c));
1648 252 : splay_tree_insert (ctx->field_map, (splay_tree_key) c,
1649 : (splay_tree_value) field);
1650 252 : break;
1651 : }
1652 80981 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1653 80981 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
1654 69571 : || (OMP_CLAUSE_MAP_KIND (c)
1655 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
1656 : {
1657 4178 : if (TREE_CODE (decl) == COMPONENT_REF
1658 4178 : || (INDIRECT_REF_P (decl)
1659 0 : && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
1660 0 : && (((TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
1661 : == REFERENCE_TYPE)
1662 0 : || (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
1663 : == POINTER_TYPE)))))
1664 : break;
1665 4178 : if (DECL_SIZE (decl)
1666 4178 : && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1667 : {
1668 710 : tree decl2 = DECL_VALUE_EXPR (decl);
1669 710 : gcc_assert (INDIRECT_REF_P (decl2));
1670 710 : decl2 = TREE_OPERAND (decl2, 0);
1671 710 : gcc_assert (DECL_P (decl2));
1672 710 : install_var_local (decl2, ctx);
1673 : }
1674 4178 : install_var_local (decl, ctx);
1675 4178 : break;
1676 : }
1677 76803 : if (DECL_P (decl))
1678 : {
1679 39663 : if (DECL_SIZE (decl)
1680 39663 : && !poly_int_tree_p (DECL_SIZE (decl)))
1681 : {
1682 0 : tree decl2 = DECL_VALUE_EXPR (decl);
1683 0 : gcc_assert (INDIRECT_REF_P (decl2));
1684 0 : decl2 = TREE_OPERAND (decl2, 0);
1685 0 : gcc_assert (DECL_P (decl2));
1686 0 : install_var_field (decl2, true, 3, ctx);
1687 0 : install_var_local (decl2, ctx);
1688 0 : install_var_local (decl, ctx);
1689 : }
1690 : else
1691 : {
1692 39663 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1693 33503 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1694 5157 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1695 44820 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1696 425 : install_var_field (decl, true, 7, ctx);
1697 : else
1698 39238 : install_var_field (decl, true, 3, ctx);
1699 39663 : if (is_gimple_omp_offloaded (ctx->stmt)
1700 39663 : && !(is_gimple_omp_oacc (ctx->stmt)
1701 13841 : && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
1702 23208 : install_var_local (decl, ctx);
1703 : }
1704 : }
1705 : else
1706 : {
1707 37140 : tree base = get_base_address (decl);
1708 37140 : tree nc = OMP_CLAUSE_CHAIN (c);
1709 37140 : if (DECL_P (base)
1710 8794 : && nc != NULL_TREE
1711 6350 : && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1712 5270 : && OMP_CLAUSE_DECL (nc) == base
1713 1148 : && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER
1714 37808 : && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1715 : {
1716 0 : OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1717 0 : OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1718 : }
1719 : else
1720 : {
1721 37140 : if (ctx->outer)
1722 : {
1723 7117 : scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer);
1724 7117 : decl = OMP_CLAUSE_DECL (c);
1725 : }
1726 37140 : gcc_assert (!splay_tree_lookup (ctx->field_map,
1727 : (splay_tree_key) decl));
1728 37140 : tree field
1729 37140 : = build_decl (OMP_CLAUSE_LOCATION (c),
1730 : FIELD_DECL, NULL_TREE, ptr_type_node);
1731 37140 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
1732 37140 : insert_field_into_struct (ctx->record_type, field);
1733 37140 : splay_tree_insert (ctx->field_map, (splay_tree_key) decl,
1734 : (splay_tree_value) field);
1735 : }
1736 : }
1737 : break;
1738 :
1739 3973 : case OMP_CLAUSE_ORDER:
1740 3973 : ctx->order_concurrent = true;
1741 3973 : break;
1742 :
1743 1907 : case OMP_CLAUSE_BIND:
1744 1907 : ctx->loop_p = true;
1745 1907 : break;
1746 :
1747 : case OMP_CLAUSE_NOWAIT:
1748 : case OMP_CLAUSE_ORDERED:
1749 : case OMP_CLAUSE_COLLAPSE:
1750 : case OMP_CLAUSE_UNTIED:
1751 : case OMP_CLAUSE_MERGEABLE:
1752 : case OMP_CLAUSE_PROC_BIND:
1753 : case OMP_CLAUSE_SAFELEN:
1754 : case OMP_CLAUSE_SIMDLEN:
1755 : case OMP_CLAUSE_THREADS:
1756 : case OMP_CLAUSE_SIMD:
1757 : case OMP_CLAUSE_NOGROUP:
1758 : case OMP_CLAUSE_DEFAULTMAP:
1759 : case OMP_CLAUSE_ASYNC:
1760 : case OMP_CLAUSE_WAIT:
1761 : case OMP_CLAUSE_GANG:
1762 : case OMP_CLAUSE_WORKER:
1763 : case OMP_CLAUSE_VECTOR:
1764 : case OMP_CLAUSE_INDEPENDENT:
1765 : case OMP_CLAUSE_AUTO:
1766 : case OMP_CLAUSE_SEQ:
1767 : case OMP_CLAUSE_TILE:
1768 : case OMP_CLAUSE__SIMT_:
1769 : case OMP_CLAUSE_DEFAULT:
1770 : case OMP_CLAUSE_NONTEMPORAL:
1771 : case OMP_CLAUSE_IF_PRESENT:
1772 : case OMP_CLAUSE_FINALIZE:
1773 : case OMP_CLAUSE_TASK_REDUCTION:
1774 : case OMP_CLAUSE_ALLOCATE:
1775 : case OMP_CLAUSE_DEVICE_TYPE:
1776 : break;
1777 :
1778 162 : case OMP_CLAUSE_ALIGNED:
1779 162 : decl = OMP_CLAUSE_DECL (c);
1780 162 : if (is_global_var (decl)
1781 162 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1782 86 : install_var_local (decl, ctx);
1783 : break;
1784 :
1785 361 : case OMP_CLAUSE__CONDTEMP_:
1786 361 : decl = OMP_CLAUSE_DECL (c);
1787 361 : if (is_parallel_ctx (ctx))
1788 : {
1789 118 : install_var_field (decl, false, 3, ctx);
1790 118 : install_var_local (decl, ctx);
1791 : }
1792 243 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1793 243 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
1794 368 : && !OMP_CLAUSE__CONDTEMP__ITER (c))
1795 125 : install_var_local (decl, ctx);
1796 : break;
1797 :
1798 : case OMP_CLAUSE_INIT:
1799 : case OMP_CLAUSE_USE:
1800 : case OMP_CLAUSE_DESTROY:
1801 : break;
1802 :
1803 0 : case OMP_CLAUSE__CACHE_:
1804 0 : case OMP_CLAUSE_NOHOST:
1805 0 : default:
1806 0 : gcc_unreachable ();
1807 : }
1808 : }
1809 :
1810 546534 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1811 : {
1812 417458 : switch (OMP_CLAUSE_CODE (c))
1813 : {
1814 22736 : case OMP_CLAUSE_LASTPRIVATE:
1815 : /* Let the corresponding firstprivate clause create
1816 : the variable. */
1817 22736 : if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1818 7245 : scan_array_reductions = true;
1819 22736 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1820 : break;
1821 : /* FALLTHRU */
1822 :
1823 149359 : case OMP_CLAUSE_FIRSTPRIVATE:
1824 149359 : case OMP_CLAUSE_PRIVATE:
1825 149359 : case OMP_CLAUSE_LINEAR:
1826 149359 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
1827 149359 : case OMP_CLAUSE_IS_DEVICE_PTR:
1828 149359 : decl = OMP_CLAUSE_DECL (c);
1829 149359 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1830 : {
1831 256 : while (INDIRECT_REF_P (decl)
1832 256 : || TREE_CODE (decl) == ARRAY_REF)
1833 17 : decl = TREE_OPERAND (decl, 0);
1834 : }
1835 :
1836 149359 : if (is_variable_sized (decl))
1837 : {
1838 423 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1839 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
1840 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1841 53 : && is_gimple_omp_offloaded (ctx->stmt))
1842 : {
1843 12 : tree decl2 = DECL_VALUE_EXPR (decl);
1844 12 : gcc_assert (INDIRECT_REF_P (decl2));
1845 12 : decl2 = TREE_OPERAND (decl2, 0);
1846 12 : gcc_assert (DECL_P (decl2));
1847 12 : install_var_local (decl2, ctx);
1848 12 : fixup_remapped_decl (decl2, ctx, false);
1849 : }
1850 423 : install_var_local (decl, ctx);
1851 : }
1852 298718 : fixup_remapped_decl (decl, ctx,
1853 149359 : OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1854 149359 : && OMP_CLAUSE_PRIVATE_DEBUG (c));
1855 149359 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
1856 149359 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
1857 : scan_array_reductions = true;
1858 : break;
1859 :
1860 16573 : case OMP_CLAUSE_REDUCTION:
1861 16573 : case OMP_CLAUSE_IN_REDUCTION:
1862 16573 : decl = OMP_CLAUSE_DECL (c);
1863 16573 : if (TREE_CODE (decl) != MEM_REF && !is_omp_target (ctx->stmt))
1864 : {
1865 13872 : if (is_variable_sized (decl))
1866 0 : install_var_local (decl, ctx);
1867 13872 : fixup_remapped_decl (decl, ctx, false);
1868 : }
1869 16573 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1870 2837 : scan_array_reductions = true;
1871 : break;
1872 :
1873 530 : case OMP_CLAUSE_TASK_REDUCTION:
1874 530 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1875 2837 : scan_array_reductions = true;
1876 : break;
1877 :
1878 29959 : case OMP_CLAUSE_SHARED:
1879 : /* Ignore shared directives in teams construct inside of
1880 : target construct. */
1881 29959 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
1882 29959 : && !is_host_teams_ctx (ctx))
1883 : break;
1884 18103 : decl = OMP_CLAUSE_DECL (c);
1885 18103 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1886 : break;
1887 12050 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1888 : {
1889 86 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl,
1890 : ctx->outer)))
1891 : break;
1892 79 : bool by_ref = use_pointer_for_field (decl, ctx);
1893 79 : install_var_field (decl, by_ref, 11, ctx);
1894 79 : break;
1895 : }
1896 11964 : fixup_remapped_decl (decl, ctx, false);
1897 11964 : break;
1898 :
1899 79070 : case OMP_CLAUSE_MAP:
1900 79070 : if (!is_gimple_omp_offloaded (ctx->stmt))
1901 : break;
1902 52839 : decl = OMP_CLAUSE_DECL (c);
1903 52839 : if (DECL_P (decl)
1904 33198 : && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
1905 29611 : && (OMP_CLAUSE_MAP_KIND (c)
1906 : != GOMP_MAP_FIRSTPRIVATE_REFERENCE))
1907 4012 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1908 30353 : && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1909 60221 : && varpool_node::get_create (decl)->offloadable)
1910 : break;
1911 47157 : if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
1912 44971 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
1913 2186 : && is_omp_target (ctx->stmt)
1914 49134 : && !is_gimple_omp_offloaded (ctx->stmt))
1915 : break;
1916 47157 : if (DECL_P (decl))
1917 : {
1918 27516 : if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1919 24498 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER)
1920 6603 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
1921 29105 : && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
1922 : {
1923 24 : tree new_decl = lookup_decl (decl, ctx);
1924 24 : TREE_TYPE (new_decl)
1925 48 : = remap_type (TREE_TYPE (decl), &ctx->cb);
1926 : }
1927 27492 : else if (DECL_SIZE (decl)
1928 27492 : && !poly_int_tree_p (DECL_SIZE (decl)))
1929 : {
1930 710 : tree decl2 = DECL_VALUE_EXPR (decl);
1931 710 : gcc_assert (INDIRECT_REF_P (decl2));
1932 710 : decl2 = TREE_OPERAND (decl2, 0);
1933 710 : gcc_assert (DECL_P (decl2));
1934 710 : fixup_remapped_decl (decl2, ctx, false);
1935 710 : fixup_remapped_decl (decl, ctx, true);
1936 : }
1937 : else
1938 26782 : fixup_remapped_decl (decl, ctx, false);
1939 : }
1940 : break;
1941 :
1942 : case OMP_CLAUSE_COPYPRIVATE:
1943 : case OMP_CLAUSE_COPYIN:
1944 : case OMP_CLAUSE_DEFAULT:
1945 : case OMP_CLAUSE_IF:
1946 : case OMP_CLAUSE_SELF:
1947 : case OMP_CLAUSE_NUM_THREADS:
1948 : case OMP_CLAUSE_NUM_TEAMS:
1949 : case OMP_CLAUSE_THREAD_LIMIT:
1950 : case OMP_CLAUSE_DEVICE:
1951 : case OMP_CLAUSE_SCHEDULE:
1952 : case OMP_CLAUSE_DIST_SCHEDULE:
1953 : case OMP_CLAUSE_NOWAIT:
1954 : case OMP_CLAUSE_ORDERED:
1955 : case OMP_CLAUSE_COLLAPSE:
1956 : case OMP_CLAUSE_UNTIED:
1957 : case OMP_CLAUSE_FINAL:
1958 : case OMP_CLAUSE_MERGEABLE:
1959 : case OMP_CLAUSE_PROC_BIND:
1960 : case OMP_CLAUSE_SAFELEN:
1961 : case OMP_CLAUSE_SIMDLEN:
1962 : case OMP_CLAUSE_ALIGNED:
1963 : case OMP_CLAUSE_DEPEND:
1964 : case OMP_CLAUSE_DETACH:
1965 : case OMP_CLAUSE_ALLOCATE:
1966 : case OMP_CLAUSE__LOOPTEMP_:
1967 : case OMP_CLAUSE__REDUCTEMP_:
1968 : case OMP_CLAUSE_TO:
1969 : case OMP_CLAUSE_FROM:
1970 : case OMP_CLAUSE_PRIORITY:
1971 : case OMP_CLAUSE_GRAINSIZE:
1972 : case OMP_CLAUSE_NUM_TASKS:
1973 : case OMP_CLAUSE_THREADS:
1974 : case OMP_CLAUSE_SIMD:
1975 : case OMP_CLAUSE_NOGROUP:
1976 : case OMP_CLAUSE_DEFAULTMAP:
1977 : case OMP_CLAUSE_ORDER:
1978 : case OMP_CLAUSE_BIND:
1979 : case OMP_CLAUSE_USE_DEVICE_PTR:
1980 : case OMP_CLAUSE_USE_DEVICE_ADDR:
1981 : case OMP_CLAUSE_NONTEMPORAL:
1982 : case OMP_CLAUSE_ASYNC:
1983 : case OMP_CLAUSE_WAIT:
1984 : case OMP_CLAUSE_NUM_GANGS:
1985 : case OMP_CLAUSE_NUM_WORKERS:
1986 : case OMP_CLAUSE_VECTOR_LENGTH:
1987 : case OMP_CLAUSE_GANG:
1988 : case OMP_CLAUSE_WORKER:
1989 : case OMP_CLAUSE_VECTOR:
1990 : case OMP_CLAUSE_INDEPENDENT:
1991 : case OMP_CLAUSE_AUTO:
1992 : case OMP_CLAUSE_SEQ:
1993 : case OMP_CLAUSE_TILE:
1994 : case OMP_CLAUSE__SIMT_:
1995 : case OMP_CLAUSE_IF_PRESENT:
1996 : case OMP_CLAUSE_FINALIZE:
1997 : case OMP_CLAUSE_FILTER:
1998 : case OMP_CLAUSE__CONDTEMP_:
1999 : case OMP_CLAUSE_INIT:
2000 : case OMP_CLAUSE_USE:
2001 : case OMP_CLAUSE_DESTROY:
2002 : case OMP_CLAUSE_DEVICE_TYPE:
2003 : break;
2004 :
2005 0 : case OMP_CLAUSE__CACHE_:
2006 0 : case OMP_CLAUSE_NOHOST:
2007 0 : default:
2008 0 : gcc_unreachable ();
2009 : }
2010 : }
2011 :
2012 129076 : gcc_checking_assert (!scan_array_reductions
2013 : || !is_gimple_omp_oacc (ctx->stmt));
2014 : if (scan_array_reductions)
2015 : {
2016 34621 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2017 29434 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
2018 27055 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
2019 26431 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
2020 30138 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2021 : {
2022 2181 : omp_context *rctx = ctx;
2023 2181 : if (is_omp_target (ctx->stmt))
2024 60 : rctx = ctx->outer;
2025 2181 : scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), rctx);
2026 2181 : scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), rctx);
2027 : }
2028 27253 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
2029 27253 : && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2030 7245 : scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
2031 20008 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2032 20008 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2033 656 : scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
2034 : }
2035 129076 : if (flex_array_ptr)
2036 : {
2037 144 : tree field = build_range_type (size_type_node,
2038 : build_int_cstu (size_type_node, 0),
2039 : NULL_TREE);
2040 144 : field = build_array_type (ptr_type_node, field);
2041 144 : field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE, field);
2042 144 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
2043 144 : DECL_CONTEXT (field) = ctx->record_type;
2044 144 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
2045 144 : TYPE_FIELDS (ctx->record_type) = field;
2046 : }
2047 129076 : }
2048 :
2049 : /* Create a new name for omp child function. Returns an identifier. */
2050 :
2051 : static tree
2052 50980 : create_omp_child_function_name (bool task_copy)
2053 : {
2054 101416 : return clone_function_name_numbered (current_function_decl,
2055 50980 : task_copy ? "_omp_cpyfn" : "_omp_fn");
2056 : }
2057 :
2058 : /* Return true if CTX may belong to offloaded code: either if current function
2059 : is offloaded, or any enclosing context corresponds to a target region. */
2060 :
2061 : static bool
2062 121351 : omp_maybe_offloaded_ctx (omp_context *ctx)
2063 : {
2064 121351 : if (cgraph_node::get (current_function_decl)->offloadable)
2065 : return true;
2066 201728 : for (; ctx; ctx = ctx->outer)
2067 119760 : if (is_gimple_omp_offloaded (ctx->stmt))
2068 : return true;
2069 : return false;
2070 : }
2071 :
2072 : /* Build a decl for the omp child function. It'll not contain a body
2073 : yet, just the bare decl. */
2074 :
2075 : static void
2076 50980 : create_omp_child_function (omp_context *ctx, bool task_copy)
2077 : {
2078 50980 : tree decl, type, name, t;
2079 :
2080 50980 : name = create_omp_child_function_name (task_copy);
2081 50980 : if (task_copy)
2082 544 : type = build_function_type_list (void_type_node, ptr_type_node,
2083 : ptr_type_node, NULL_TREE);
2084 : else
2085 50436 : type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
2086 :
2087 50980 : decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type);
2088 :
2089 50980 : gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt)
2090 : || !task_copy);
2091 39598 : if (!task_copy)
2092 50436 : ctx->cb.dst_fn = decl;
2093 : else
2094 544 : gimple_omp_task_set_copy_fn (ctx->stmt, decl);
2095 :
2096 50980 : TREE_STATIC (decl) = 1;
2097 50980 : TREE_USED (decl) = 1;
2098 50980 : DECL_ARTIFICIAL (decl) = 1;
2099 50980 : DECL_IGNORED_P (decl) = 0;
2100 50980 : TREE_PUBLIC (decl) = 0;
2101 50980 : DECL_UNINLINABLE (decl) = 1;
2102 50980 : DECL_EXTERNAL (decl) = 0;
2103 50980 : DECL_CONTEXT (decl) = NULL_TREE;
2104 50980 : DECL_INITIAL (decl) = make_node (BLOCK);
2105 50980 : BLOCK_SUPERCONTEXT (DECL_INITIAL (decl)) = decl;
2106 50980 : DECL_ATTRIBUTES (decl) = DECL_ATTRIBUTES (current_function_decl);
2107 : /* Remove omp declare simd attribute from the new attributes. */
2108 50980 : if (tree a = lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (decl)))
2109 : {
2110 9 : while (tree a2 = lookup_attribute ("omp declare simd", TREE_CHAIN (a)))
2111 : a = a2;
2112 9 : a = TREE_CHAIN (a);
2113 22 : for (tree *p = &DECL_ATTRIBUTES (decl); *p != a;)
2114 13 : if (is_attribute_p ("omp declare simd", get_attribute_name (*p)))
2115 9 : *p = TREE_CHAIN (*p);
2116 : else
2117 : {
2118 4 : tree chain = TREE_CHAIN (*p);
2119 4 : *p = copy_node (*p);
2120 4 : p = &TREE_CHAIN (*p);
2121 4 : *p = chain;
2122 : }
2123 : }
2124 101960 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (decl)
2125 50980 : = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (current_function_decl);
2126 101960 : DECL_FUNCTION_SPECIFIC_TARGET (decl)
2127 50980 : = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
2128 101960 : DECL_FUNCTION_VERSIONED (decl)
2129 50980 : = DECL_FUNCTION_VERSIONED (current_function_decl);
2130 :
2131 50980 : if (omp_maybe_offloaded_ctx (ctx))
2132 : {
2133 30654 : cgraph_node::get_create (decl)->offloadable = 1;
2134 30654 : if (ENABLE_OFFLOADING)
2135 : g->have_offload = true;
2136 : }
2137 :
2138 50980 : if (cgraph_node::get_create (decl)->offloadable)
2139 : {
2140 30654 : const char *target_attr = (is_gimple_omp_offloaded (ctx->stmt)
2141 30654 : ? "omp target entrypoint"
2142 6198 : : "omp declare target");
2143 30654 : if (lookup_attribute ("omp declare target",
2144 30654 : DECL_ATTRIBUTES (current_function_decl)))
2145 : {
2146 3032 : if (is_gimple_omp_offloaded (ctx->stmt))
2147 1620 : DECL_ATTRIBUTES (decl)
2148 1620 : = remove_attribute ("omp declare target",
2149 1620 : copy_list (DECL_ATTRIBUTES (decl)));
2150 : else
2151 : target_attr = NULL;
2152 : }
2153 1620 : if (target_attr
2154 29242 : && is_gimple_omp_offloaded (ctx->stmt)
2155 24456 : && lookup_attribute ("noclone", DECL_ATTRIBUTES (decl)) == NULL_TREE)
2156 18950 : DECL_ATTRIBUTES (decl) = tree_cons (get_identifier ("noclone"),
2157 18950 : NULL_TREE, DECL_ATTRIBUTES (decl));
2158 30654 : if (target_attr)
2159 29242 : DECL_ATTRIBUTES (decl)
2160 58484 : = tree_cons (get_identifier (target_attr),
2161 29242 : NULL_TREE, DECL_ATTRIBUTES (decl));
2162 : }
2163 :
2164 50980 : t = build_decl (DECL_SOURCE_LOCATION (decl),
2165 : RESULT_DECL, NULL_TREE, void_type_node);
2166 50980 : DECL_ARTIFICIAL (t) = 1;
2167 50980 : DECL_IGNORED_P (t) = 1;
2168 50980 : DECL_CONTEXT (t) = decl;
2169 50980 : DECL_RESULT (decl) = t;
2170 :
2171 50980 : tree data_name = get_identifier (".omp_data_i");
2172 50980 : t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name,
2173 : ptr_type_node);
2174 50980 : DECL_ARTIFICIAL (t) = 1;
2175 50980 : DECL_NAMELESS (t) = 1;
2176 50980 : DECL_ARG_TYPE (t) = ptr_type_node;
2177 50980 : DECL_CONTEXT (t) = current_function_decl;
2178 50980 : TREE_USED (t) = 1;
2179 50980 : TREE_READONLY (t) = 1;
2180 50980 : DECL_ARGUMENTS (decl) = t;
2181 50980 : if (!task_copy)
2182 50436 : ctx->receiver_decl = t;
2183 : else
2184 : {
2185 544 : t = build_decl (DECL_SOURCE_LOCATION (decl),
2186 : PARM_DECL, get_identifier (".omp_data_o"),
2187 : ptr_type_node);
2188 544 : DECL_ARTIFICIAL (t) = 1;
2189 544 : DECL_NAMELESS (t) = 1;
2190 544 : DECL_ARG_TYPE (t) = ptr_type_node;
2191 544 : DECL_CONTEXT (t) = current_function_decl;
2192 544 : TREE_USED (t) = 1;
2193 544 : TREE_ADDRESSABLE (t) = 1;
2194 544 : DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2195 544 : DECL_ARGUMENTS (decl) = t;
2196 : }
2197 :
2198 : /* Allocate memory for the function structure. The call to
2199 : allocate_struct_function clobbers CFUN, so we need to restore
2200 : it afterward. */
2201 50980 : push_struct_function (decl);
2202 50980 : cfun->function_end_locus = gimple_location (ctx->stmt);
2203 50980 : init_tree_ssa (cfun);
2204 50980 : pop_cfun ();
2205 50980 : }
2206 :
2207 : /* Callback for walk_gimple_seq. Check if combined parallel
2208 : contains gimple_omp_for_combined_into_p OMP_FOR. */
2209 :
2210 : tree
2211 35647 : omp_find_combined_for (gimple_stmt_iterator *gsi_p,
2212 : bool *handled_ops_p,
2213 : struct walk_stmt_info *wi)
2214 : {
2215 35647 : gimple *stmt = gsi_stmt (*gsi_p);
2216 :
2217 35647 : *handled_ops_p = true;
2218 35647 : switch (gimple_code (stmt))
2219 : {
2220 19972 : WALK_SUBSTMTS;
2221 :
2222 13502 : case GIMPLE_OMP_FOR:
2223 13502 : if (gimple_omp_for_combined_into_p (stmt)
2224 13502 : && gimple_omp_for_kind (stmt)
2225 7960 : == *(const enum gf_mask *) (wi->info))
2226 : {
2227 7960 : wi->info = stmt;
2228 7960 : return integer_zero_node;
2229 : }
2230 : break;
2231 : default:
2232 : break;
2233 : }
2234 : return NULL;
2235 : }
2236 :
2237 : /* Add _LOOPTEMP_/_REDUCTEMP_ clauses on OpenMP parallel or task. */
2238 :
2239 : static void
2240 14087 : add_taskreg_looptemp_clauses (enum gf_mask msk, gimple *stmt,
2241 : omp_context *outer_ctx)
2242 : {
2243 14087 : struct walk_stmt_info wi;
2244 :
2245 14087 : memset (&wi, 0, sizeof (wi));
2246 14087 : wi.val_only = true;
2247 14087 : wi.info = (void *) &msk;
2248 14087 : walk_gimple_seq (gimple_omp_body (stmt), omp_find_combined_for, NULL, &wi);
2249 14087 : if (wi.info != (void *) &msk)
2250 : {
2251 7960 : gomp_for *for_stmt = as_a <gomp_for *> ((gimple *) wi.info);
2252 7960 : struct omp_for_data fd;
2253 7960 : omp_extract_for_data (for_stmt, &fd, NULL);
2254 : /* We need two temporaries with fd.loop.v type (istart/iend)
2255 : and then (fd.collapse - 1) temporaries with the same
2256 : type for count2 ... countN-1 vars if not constant. */
2257 7960 : size_t count = 2, i;
2258 7960 : tree type = fd.iter_type;
2259 7960 : if (fd.collapse > 1
2260 2648 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
2261 : {
2262 1466 : count += fd.collapse - 1;
2263 : /* If there are lastprivate clauses on the inner
2264 : GIMPLE_OMP_FOR, add one more temporaries for the total number
2265 : of iterations (product of count1 ... countN-1). */
2266 1466 : if (omp_find_clause (gimple_omp_for_clauses (for_stmt),
2267 : OMP_CLAUSE_LASTPRIVATE)
2268 1466 : || (msk == GF_OMP_FOR_KIND_FOR
2269 1335 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
2270 : OMP_CLAUSE_LASTPRIVATE)))
2271 : {
2272 726 : tree temp = create_tmp_var (type);
2273 726 : tree c = build_omp_clause (UNKNOWN_LOCATION,
2274 : OMP_CLAUSE__LOOPTEMP_);
2275 726 : insert_decl_map (&outer_ctx->cb, temp, temp);
2276 726 : OMP_CLAUSE_DECL (c) = temp;
2277 726 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2278 726 : gimple_omp_taskreg_set_clauses (stmt, c);
2279 : }
2280 1466 : if (fd.non_rect
2281 24 : && fd.last_nonrect == fd.first_nonrect + 1)
2282 12 : if (tree v = gimple_omp_for_index (for_stmt, fd.last_nonrect))
2283 12 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
2284 : {
2285 12 : v = gimple_omp_for_index (for_stmt, fd.first_nonrect);
2286 12 : tree type2 = TREE_TYPE (v);
2287 12 : count++;
2288 48 : for (i = 0; i < 3; i++)
2289 : {
2290 36 : tree temp = create_tmp_var (type2);
2291 36 : tree c = build_omp_clause (UNKNOWN_LOCATION,
2292 : OMP_CLAUSE__LOOPTEMP_);
2293 36 : insert_decl_map (&outer_ctx->cb, temp, temp);
2294 36 : OMP_CLAUSE_DECL (c) = temp;
2295 36 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2296 36 : gimple_omp_taskreg_set_clauses (stmt, c);
2297 : }
2298 : }
2299 : }
2300 26725 : for (i = 0; i < count; i++)
2301 : {
2302 18765 : tree temp = create_tmp_var (type);
2303 18765 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
2304 18765 : insert_decl_map (&outer_ctx->cb, temp, temp);
2305 18765 : OMP_CLAUSE_DECL (c) = temp;
2306 18765 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2307 18765 : gimple_omp_taskreg_set_clauses (stmt, c);
2308 : }
2309 : }
2310 14087 : if (msk == GF_OMP_FOR_KIND_TASKLOOP
2311 15654 : && omp_find_clause (gimple_omp_task_clauses (stmt),
2312 : OMP_CLAUSE_REDUCTION))
2313 : {
2314 517 : tree type = build_pointer_type (pointer_sized_int_node);
2315 517 : tree temp = create_tmp_var (type);
2316 517 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
2317 517 : insert_decl_map (&outer_ctx->cb, temp, temp);
2318 517 : OMP_CLAUSE_DECL (c) = temp;
2319 517 : OMP_CLAUSE_CHAIN (c) = gimple_omp_task_clauses (stmt);
2320 517 : gimple_omp_task_set_clauses (stmt, c);
2321 : }
2322 14087 : }
2323 :
2324 : /* Scan an OpenMP parallel directive. */
2325 :
2326 : static void
2327 18143 : scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2328 : {
2329 18143 : omp_context *ctx;
2330 18143 : tree name;
2331 18143 : gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi));
2332 :
2333 : /* Ignore parallel directives with empty bodies, unless there
2334 : are copyin clauses. */
2335 18143 : if (optimize > 0
2336 12460 : && empty_body_p (gimple_omp_body (stmt))
2337 18188 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
2338 : OMP_CLAUSE_COPYIN) == NULL)
2339 : {
2340 39 : gsi_replace (gsi, gimple_build_nop (), false);
2341 39 : return;
2342 : }
2343 :
2344 18104 : if (gimple_omp_parallel_combined_p (stmt))
2345 12520 : add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR, stmt, outer_ctx);
2346 18104 : for (tree c = omp_find_clause (gimple_omp_parallel_clauses (stmt),
2347 : OMP_CLAUSE_REDUCTION);
2348 20932 : c; c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE_REDUCTION))
2349 3906 : if (OMP_CLAUSE_REDUCTION_TASK (c))
2350 : {
2351 66 : tree type = build_pointer_type (pointer_sized_int_node);
2352 66 : tree temp = create_tmp_var (type);
2353 66 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
2354 66 : if (outer_ctx)
2355 17 : insert_decl_map (&outer_ctx->cb, temp, temp);
2356 66 : OMP_CLAUSE_DECL (c) = temp;
2357 66 : OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
2358 66 : gimple_omp_parallel_set_clauses (stmt, c);
2359 66 : break;
2360 : }
2361 3840 : else if (OMP_CLAUSE_CHAIN (c) == NULL_TREE)
2362 : break;
2363 :
2364 18104 : ctx = new_omp_context (stmt, outer_ctx);
2365 18104 : taskreg_contexts.safe_push (ctx);
2366 18104 : if (taskreg_nesting_level > 1)
2367 6190 : ctx->is_nested = true;
2368 18104 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2369 18104 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2370 18104 : name = create_tmp_var_name (".omp_data_s");
2371 18104 : name = build_decl (gimple_location (stmt),
2372 : TYPE_DECL, name, ctx->record_type);
2373 18104 : DECL_ARTIFICIAL (name) = 1;
2374 18104 : DECL_NAMELESS (name) = 1;
2375 18104 : TYPE_NAME (ctx->record_type) = name;
2376 18104 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
2377 18104 : create_omp_child_function (ctx, false);
2378 18104 : gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2379 :
2380 18104 : scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2381 18104 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2382 :
2383 18104 : if (TYPE_FIELDS (ctx->record_type) == NULL)
2384 3565 : ctx->record_type = ctx->receiver_decl = NULL;
2385 : }
2386 :
2387 : /* Scan an OpenMP task directive. */
2388 :
2389 : static void
2390 5367 : scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2391 : {
2392 5367 : omp_context *ctx;
2393 5367 : tree name, t;
2394 5367 : gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi));
2395 :
2396 : /* Ignore task directives with empty bodies, unless they have depend
2397 : clause. */
2398 5367 : if (optimize > 0
2399 2579 : && gimple_omp_body (stmt)
2400 2537 : && empty_body_p (gimple_omp_body (stmt))
2401 5419 : && !omp_find_clause (gimple_omp_task_clauses (stmt), OMP_CLAUSE_DEPEND))
2402 : {
2403 19 : gsi_replace (gsi, gimple_build_nop (), false);
2404 128 : return;
2405 : }
2406 :
2407 5348 : if (gimple_omp_task_taskloop_p (stmt))
2408 1567 : add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP, stmt, outer_ctx);
2409 :
2410 5348 : ctx = new_omp_context (stmt, outer_ctx);
2411 :
2412 5348 : if (gimple_omp_task_taskwait_p (stmt))
2413 : {
2414 90 : scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2415 90 : return;
2416 : }
2417 :
2418 5258 : taskreg_contexts.safe_push (ctx);
2419 5258 : if (taskreg_nesting_level > 1)
2420 2324 : ctx->is_nested = true;
2421 5258 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2422 5258 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2423 5258 : name = create_tmp_var_name (".omp_data_s");
2424 5258 : name = build_decl (gimple_location (stmt),
2425 : TYPE_DECL, name, ctx->record_type);
2426 5258 : DECL_ARTIFICIAL (name) = 1;
2427 5258 : DECL_NAMELESS (name) = 1;
2428 5258 : TYPE_NAME (ctx->record_type) = name;
2429 5258 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
2430 5258 : create_omp_child_function (ctx, false);
2431 5258 : gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2432 :
2433 5258 : scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2434 :
2435 5258 : if (ctx->srecord_type)
2436 : {
2437 544 : name = create_tmp_var_name (".omp_data_a");
2438 544 : name = build_decl (gimple_location (stmt),
2439 : TYPE_DECL, name, ctx->srecord_type);
2440 544 : DECL_ARTIFICIAL (name) = 1;
2441 544 : DECL_NAMELESS (name) = 1;
2442 544 : TYPE_NAME (ctx->srecord_type) = name;
2443 544 : TYPE_ARTIFICIAL (ctx->srecord_type) = 1;
2444 544 : create_omp_child_function (ctx, true);
2445 : }
2446 :
2447 5258 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2448 :
2449 5258 : if (TYPE_FIELDS (ctx->record_type) == NULL)
2450 : {
2451 1836 : ctx->record_type = ctx->receiver_decl = NULL;
2452 1836 : t = build_int_cst (long_integer_type_node, 0);
2453 1836 : gimple_omp_task_set_arg_size (stmt, t);
2454 1836 : t = build_int_cst (long_integer_type_node, 1);
2455 1836 : gimple_omp_task_set_arg_align (stmt, t);
2456 : }
2457 : }
2458 :
2459 : /* Helper function for finish_taskreg_scan, called through walk_tree.
2460 : If maybe_lookup_decl_in_outer_context returns non-NULL for some
2461 : tree, replace it in the expression. */
2462 :
2463 : static tree
2464 276 : finish_taskreg_remap (tree *tp, int *walk_subtrees, void *data)
2465 : {
2466 276 : if (VAR_P (*tp))
2467 : {
2468 96 : omp_context *ctx = (omp_context *) data;
2469 96 : tree t = maybe_lookup_decl_in_outer_ctx (*tp, ctx);
2470 96 : if (t != *tp)
2471 : {
2472 9 : if (DECL_HAS_VALUE_EXPR_P (t))
2473 0 : t = unshare_expr (DECL_VALUE_EXPR (t));
2474 9 : *tp = t;
2475 : }
2476 96 : *walk_subtrees = 0;
2477 : }
2478 180 : else if (IS_TYPE_OR_DECL_P (*tp))
2479 0 : *walk_subtrees = 0;
2480 276 : return NULL_TREE;
2481 : }
2482 :
2483 : /* If any decls have been made addressable during scan_omp,
2484 : adjust their fields if needed, and layout record types
2485 : of parallel/task constructs. */
2486 :
2487 : static void
2488 25980 : finish_taskreg_scan (omp_context *ctx)
2489 : {
2490 25980 : if (ctx->record_type == NULL_TREE)
2491 : return;
2492 :
2493 : /* If any make_addressable_vars were needed, verify all
2494 : OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK,TEAMS}
2495 : statements if use_pointer_for_field hasn't changed
2496 : because of that. If it did, update field types now. */
2497 19133 : if (make_addressable_vars)
2498 : {
2499 2209 : tree c;
2500 :
2501 12338 : for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2502 12338 : c; c = OMP_CLAUSE_CHAIN (c))
2503 10129 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
2504 10129 : && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
2505 : {
2506 2617 : tree decl = OMP_CLAUSE_DECL (c);
2507 :
2508 : /* Global variables don't need to be copied,
2509 : the receiver side will use them directly. */
2510 2617 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2511 281 : continue;
2512 2336 : if (!bitmap_bit_p (make_addressable_vars, DECL_UID (decl))
2513 2336 : || !use_pointer_for_field (decl, ctx))
2514 1787 : continue;
2515 549 : tree field = lookup_field (decl, ctx);
2516 549 : if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE
2517 549 : && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl))
2518 495 : continue;
2519 54 : TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl));
2520 54 : TREE_THIS_VOLATILE (field) = 0;
2521 54 : DECL_USER_ALIGN (field) = 0;
2522 54 : SET_DECL_ALIGN (field, TYPE_ALIGN (TREE_TYPE (field)));
2523 54 : if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field))
2524 16 : SET_TYPE_ALIGN (ctx->record_type, DECL_ALIGN (field));
2525 54 : if (ctx->srecord_type)
2526 : {
2527 0 : tree sfield = lookup_sfield (decl, ctx);
2528 0 : TREE_TYPE (sfield) = TREE_TYPE (field);
2529 0 : TREE_THIS_VOLATILE (sfield) = 0;
2530 0 : DECL_USER_ALIGN (sfield) = 0;
2531 0 : SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
2532 0 : if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield))
2533 0 : SET_TYPE_ALIGN (ctx->srecord_type, DECL_ALIGN (sfield));
2534 : }
2535 : }
2536 : }
2537 :
2538 19133 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2539 : {
2540 14539 : tree clauses = gimple_omp_parallel_clauses (ctx->stmt);
2541 14539 : tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
2542 14539 : if (c)
2543 : {
2544 : /* Move the _reductemp_ clause first. GOMP_parallel_reductions
2545 : expects to find it at the start of data. */
2546 66 : tree f = lookup_field (OMP_CLAUSE_DECL (c), ctx);
2547 66 : tree *p = &TYPE_FIELDS (ctx->record_type);
2548 152 : while (*p)
2549 152 : if (*p == f)
2550 : {
2551 66 : *p = DECL_CHAIN (*p);
2552 66 : break;
2553 : }
2554 : else
2555 86 : p = &DECL_CHAIN (*p);
2556 66 : DECL_CHAIN (f) = TYPE_FIELDS (ctx->record_type);
2557 66 : TYPE_FIELDS (ctx->record_type) = f;
2558 : }
2559 14539 : layout_type (ctx->record_type);
2560 14539 : fixup_child_record_type (ctx);
2561 : }
2562 4594 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2563 : {
2564 1172 : layout_type (ctx->record_type);
2565 1172 : fixup_child_record_type (ctx);
2566 : }
2567 : else
2568 : {
2569 3422 : location_t loc = gimple_location (ctx->stmt);
2570 3422 : tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2571 3422 : tree detach_clause
2572 3422 : = omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
2573 : OMP_CLAUSE_DETACH);
2574 : /* Move VLA fields to the end. */
2575 3422 : p = &TYPE_FIELDS (ctx->record_type);
2576 15319 : while (*p)
2577 11897 : if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2578 11897 : || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2579 : {
2580 96 : *q = *p;
2581 96 : *p = TREE_CHAIN (*p);
2582 96 : TREE_CHAIN (*q) = NULL_TREE;
2583 96 : q = &TREE_CHAIN (*q);
2584 : }
2585 : else
2586 11801 : p = &DECL_CHAIN (*p);
2587 3422 : *p = vla_fields;
2588 3422 : if (gimple_omp_task_taskloop_p (ctx->stmt))
2589 : {
2590 : /* Move fields corresponding to first and second _looptemp_
2591 : clause first. There are filled by GOMP_taskloop
2592 : and thus need to be in specific positions. */
2593 1567 : tree clauses = gimple_omp_task_clauses (ctx->stmt);
2594 1567 : tree c1 = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
2595 1567 : tree c2 = omp_find_clause (OMP_CLAUSE_CHAIN (c1),
2596 : OMP_CLAUSE__LOOPTEMP_);
2597 1567 : tree c3 = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
2598 1567 : tree f1 = lookup_field (OMP_CLAUSE_DECL (c1), ctx);
2599 1567 : tree f2 = lookup_field (OMP_CLAUSE_DECL (c2), ctx);
2600 2084 : tree f3 = c3 ? lookup_field (OMP_CLAUSE_DECL (c3), ctx) : NULL_TREE;
2601 1567 : p = &TYPE_FIELDS (ctx->record_type);
2602 8871 : while (*p)
2603 7304 : if (*p == f1 || *p == f2 || *p == f3)
2604 3651 : *p = DECL_CHAIN (*p);
2605 : else
2606 3653 : p = &DECL_CHAIN (*p);
2607 1567 : DECL_CHAIN (f1) = f2;
2608 1567 : if (c3)
2609 : {
2610 517 : DECL_CHAIN (f2) = f3;
2611 517 : DECL_CHAIN (f3) = TYPE_FIELDS (ctx->record_type);
2612 : }
2613 : else
2614 1050 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->record_type);
2615 1567 : TYPE_FIELDS (ctx->record_type) = f1;
2616 1567 : if (ctx->srecord_type)
2617 : {
2618 389 : f1 = lookup_sfield (OMP_CLAUSE_DECL (c1), ctx);
2619 389 : f2 = lookup_sfield (OMP_CLAUSE_DECL (c2), ctx);
2620 389 : if (c3)
2621 186 : f3 = lookup_sfield (OMP_CLAUSE_DECL (c3), ctx);
2622 389 : p = &TYPE_FIELDS (ctx->srecord_type);
2623 2097 : while (*p)
2624 1708 : if (*p == f1 || *p == f2 || *p == f3)
2625 964 : *p = DECL_CHAIN (*p);
2626 : else
2627 744 : p = &DECL_CHAIN (*p);
2628 389 : DECL_CHAIN (f1) = f2;
2629 389 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2630 389 : if (c3)
2631 : {
2632 186 : DECL_CHAIN (f2) = f3;
2633 186 : DECL_CHAIN (f3) = TYPE_FIELDS (ctx->srecord_type);
2634 : }
2635 : else
2636 203 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2637 389 : TYPE_FIELDS (ctx->srecord_type) = f1;
2638 : }
2639 : }
2640 3422 : if (detach_clause)
2641 : {
2642 189 : tree c, field;
2643 :
2644 : /* Look for a firstprivate clause with the detach event handle. */
2645 540 : for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2646 540 : c; c = OMP_CLAUSE_CHAIN (c))
2647 : {
2648 540 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
2649 347 : continue;
2650 193 : if (maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c), ctx)
2651 193 : == OMP_CLAUSE_DECL (detach_clause))
2652 : break;
2653 : }
2654 :
2655 189 : gcc_assert (c);
2656 189 : field = lookup_field (OMP_CLAUSE_DECL (c), ctx);
2657 :
2658 : /* Move field corresponding to the detach clause first.
2659 : This is filled by GOMP_task and needs to be in a
2660 : specific position. */
2661 189 : p = &TYPE_FIELDS (ctx->record_type);
2662 518 : while (*p)
2663 329 : if (*p == field)
2664 189 : *p = DECL_CHAIN (*p);
2665 : else
2666 140 : p = &DECL_CHAIN (*p);
2667 189 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
2668 189 : TYPE_FIELDS (ctx->record_type) = field;
2669 189 : if (ctx->srecord_type)
2670 : {
2671 3 : field = lookup_sfield (OMP_CLAUSE_DECL (c), ctx);
2672 3 : p = &TYPE_FIELDS (ctx->srecord_type);
2673 9 : while (*p)
2674 6 : if (*p == field)
2675 3 : *p = DECL_CHAIN (*p);
2676 : else
2677 3 : p = &DECL_CHAIN (*p);
2678 3 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->srecord_type);
2679 3 : TYPE_FIELDS (ctx->srecord_type) = field;
2680 : }
2681 : }
2682 3422 : layout_type (ctx->record_type);
2683 3422 : fixup_child_record_type (ctx);
2684 3422 : if (ctx->srecord_type)
2685 544 : layout_type (ctx->srecord_type);
2686 3422 : tree t = fold_convert_loc (loc, long_integer_type_node,
2687 3422 : TYPE_SIZE_UNIT (ctx->record_type));
2688 3422 : if (TREE_CODE (t) != INTEGER_CST)
2689 : {
2690 21 : t = unshare_expr (t);
2691 21 : walk_tree (&t, finish_taskreg_remap, ctx, NULL);
2692 : }
2693 3422 : gimple_omp_task_set_arg_size (ctx->stmt, t);
2694 3422 : t = build_int_cst (long_integer_type_node,
2695 3422 : TYPE_ALIGN_UNIT (ctx->record_type));
2696 3422 : gimple_omp_task_set_arg_align (ctx->stmt, t);
2697 : }
2698 : }
2699 :
2700 : /* Find the enclosing offload context. */
2701 :
2702 : static omp_context *
2703 9665 : enclosing_target_ctx (omp_context *ctx)
2704 : {
2705 43357 : for (; ctx; ctx = ctx->outer)
2706 41743 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET)
2707 : break;
2708 :
2709 22109 : return ctx;
2710 : }
2711 :
2712 : /* Return whether CTX's parent compute construct is an OpenACC 'kernels'
2713 : construct.
2714 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
2715 :
2716 : static bool
2717 11323 : ctx_in_oacc_kernels_region (omp_context *ctx)
2718 : {
2719 39133 : for (;ctx != NULL; ctx = ctx->outer)
2720 : {
2721 29468 : gimple *stmt = ctx->stmt;
2722 29468 : if (gimple_code (stmt) == GIMPLE_OMP_TARGET
2723 29468 : && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS)
2724 : return true;
2725 : }
2726 :
2727 : return false;
2728 : }
2729 :
2730 : /* Check the parallelism clauses inside a OpenACC 'kernels' region.
2731 : (This doesn't include OpenACC 'kernels' decomposed parts.)
2732 : Until kernels handling moves to use the same loop indirection
2733 : scheme as parallel, we need to do this checking early. */
2734 :
2735 : static unsigned
2736 7146 : check_oacc_kernel_gwv (gomp_for *stmt, omp_context *ctx)
2737 : {
2738 7146 : bool checking = true;
2739 7146 : unsigned outer_mask = 0;
2740 7146 : unsigned this_mask = 0;
2741 7146 : bool has_seq = false, has_auto = false;
2742 :
2743 7146 : if (ctx->outer)
2744 4816 : outer_mask = check_oacc_kernel_gwv (NULL, ctx->outer);
2745 7146 : if (!stmt)
2746 : {
2747 4816 : checking = false;
2748 4816 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR)
2749 : return outer_mask;
2750 1730 : stmt = as_a <gomp_for *> (ctx->stmt);
2751 : }
2752 :
2753 11323 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2754 : {
2755 7263 : switch (OMP_CLAUSE_CODE (c))
2756 : {
2757 644 : case OMP_CLAUSE_GANG:
2758 644 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
2759 644 : break;
2760 439 : case OMP_CLAUSE_WORKER:
2761 439 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_WORKER);
2762 439 : break;
2763 342 : case OMP_CLAUSE_VECTOR:
2764 342 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_VECTOR);
2765 342 : break;
2766 63 : case OMP_CLAUSE_SEQ:
2767 63 : has_seq = true;
2768 63 : break;
2769 94 : case OMP_CLAUSE_AUTO:
2770 94 : has_auto = true;
2771 94 : break;
2772 : default:
2773 : break;
2774 : }
2775 : }
2776 :
2777 4060 : if (checking)
2778 : {
2779 2330 : if (has_seq && (this_mask || has_auto))
2780 36 : error_at (gimple_location (stmt), "%<seq%> overrides other"
2781 : " OpenACC loop specifiers");
2782 2294 : else if (has_auto && this_mask)
2783 30 : error_at (gimple_location (stmt), "%<auto%> conflicts with other"
2784 : " OpenACC loop specifiers");
2785 :
2786 2330 : if (this_mask & outer_mask)
2787 15 : error_at (gimple_location (stmt), "inner loop uses same"
2788 : " OpenACC parallelism as containing loop");
2789 : }
2790 :
2791 4060 : return outer_mask | this_mask;
2792 : }
2793 :
2794 : /* Scan a GIMPLE_OMP_FOR. */
2795 :
2796 : static omp_context *
2797 52632 : scan_omp_for (gomp_for *stmt, omp_context *outer_ctx)
2798 : {
2799 52632 : omp_context *ctx;
2800 52632 : size_t i;
2801 52632 : tree clauses = gimple_omp_for_clauses (stmt);
2802 :
2803 52632 : ctx = new_omp_context (stmt, outer_ctx);
2804 :
2805 52632 : if (is_gimple_omp_oacc (stmt))
2806 : {
2807 12444 : omp_context *tgt = enclosing_target_ctx (outer_ctx);
2808 :
2809 12444 : if (!(tgt && is_oacc_kernels (tgt)))
2810 32251 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2811 : {
2812 22137 : tree c_op0;
2813 22137 : switch (OMP_CLAUSE_CODE (c))
2814 : {
2815 1952 : case OMP_CLAUSE_GANG:
2816 1952 : c_op0 = OMP_CLAUSE_GANG_EXPR (c);
2817 1952 : break;
2818 :
2819 1426 : case OMP_CLAUSE_WORKER:
2820 1426 : c_op0 = OMP_CLAUSE_WORKER_EXPR (c);
2821 1426 : break;
2822 :
2823 1627 : case OMP_CLAUSE_VECTOR:
2824 1627 : c_op0 = OMP_CLAUSE_VECTOR_EXPR (c);
2825 1627 : break;
2826 :
2827 17132 : default:
2828 17132 : continue;
2829 : }
2830 :
2831 5005 : if (c_op0)
2832 : {
2833 : /* By construction, this is impossible for OpenACC 'kernels'
2834 : decomposed parts. */
2835 210 : gcc_assert (!(tgt && is_oacc_kernels_decomposed_part (tgt)));
2836 :
2837 210 : error_at (OMP_CLAUSE_LOCATION (c),
2838 : "argument not permitted on %qs clause",
2839 210 : omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
2840 210 : if (tgt)
2841 180 : inform (gimple_location (tgt->stmt),
2842 : "enclosing parent compute construct");
2843 30 : else if (oacc_get_fn_attrib (current_function_decl))
2844 30 : inform (DECL_SOURCE_LOCATION (current_function_decl),
2845 : "enclosing routine");
2846 : else
2847 0 : gcc_unreachable ();
2848 : }
2849 : }
2850 :
2851 12444 : if (tgt && is_oacc_kernels (tgt))
2852 2330 : check_oacc_kernel_gwv (stmt, ctx);
2853 :
2854 : /* Collect all variables named in reductions on this loop. Ensure
2855 : that, if this loop has a reduction on some variable v, and there is
2856 : a reduction on v somewhere in an outer context, then there is a
2857 : reduction on v on all intervening loops as well. */
2858 12444 : tree local_reduction_clauses = NULL;
2859 39367 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2860 : {
2861 26923 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
2862 4712 : local_reduction_clauses
2863 4712 : = tree_cons (NULL, c, local_reduction_clauses);
2864 : }
2865 12444 : if (ctx->outer_reduction_clauses == NULL && ctx->outer != NULL)
2866 11918 : ctx->outer_reduction_clauses
2867 11918 : = chainon (unshare_expr (ctx->outer->local_reduction_clauses),
2868 : ctx->outer->outer_reduction_clauses);
2869 12444 : tree outer_reduction_clauses = ctx->outer_reduction_clauses;
2870 12444 : tree local_iter = local_reduction_clauses;
2871 17156 : for (; local_iter; local_iter = TREE_CHAIN (local_iter))
2872 : {
2873 4712 : tree local_clause = TREE_VALUE (local_iter);
2874 4712 : tree local_var = OMP_CLAUSE_DECL (local_clause);
2875 4712 : tree_code local_op = OMP_CLAUSE_REDUCTION_CODE (local_clause);
2876 4712 : bool have_outer_reduction = false;
2877 4712 : tree ctx_iter = outer_reduction_clauses;
2878 5826 : for (; ctx_iter; ctx_iter = TREE_CHAIN (ctx_iter))
2879 : {
2880 3325 : tree outer_clause = TREE_VALUE (ctx_iter);
2881 3325 : tree outer_var = OMP_CLAUSE_DECL (outer_clause);
2882 3325 : tree_code outer_op = OMP_CLAUSE_REDUCTION_CODE (outer_clause);
2883 3325 : if (outer_var == local_var && outer_op != local_op)
2884 : {
2885 535 : if (warning_at (OMP_CLAUSE_LOCATION (local_clause),
2886 535 : OPT_Wopenmp, "conflicting reduction "
2887 : "operations for %qE",
2888 : local_var))
2889 535 : inform (OMP_CLAUSE_LOCATION (outer_clause),
2890 : "location of the previous reduction for %qE",
2891 : outer_var);
2892 : }
2893 3325 : if (outer_var == local_var)
2894 : {
2895 : have_outer_reduction = true;
2896 : break;
2897 : }
2898 : }
2899 4712 : if (have_outer_reduction)
2900 : {
2901 : /* There is a reduction on outer_var both on this loop and on
2902 : some enclosing loop. Walk up the context tree until such a
2903 : loop with a reduction on outer_var is found, and complain
2904 : about all intervening loops that do not have such a
2905 : reduction. */
2906 2211 : struct omp_context *curr_loop = ctx->outer;
2907 2211 : bool found = false;
2908 2826 : while (curr_loop != NULL)
2909 : {
2910 2826 : tree curr_iter = curr_loop->local_reduction_clauses;
2911 3460 : for (; curr_iter; curr_iter = TREE_CHAIN (curr_iter))
2912 : {
2913 2845 : tree curr_clause = TREE_VALUE (curr_iter);
2914 2845 : tree curr_var = OMP_CLAUSE_DECL (curr_clause);
2915 2845 : if (curr_var == local_var)
2916 : {
2917 : found = true;
2918 : break;
2919 : }
2920 : }
2921 2826 : if (!found)
2922 615 : warning_at (gimple_location (curr_loop->stmt), OPT_Wopenmp,
2923 : "nested loop in reduction needs "
2924 : "reduction clause for %qE",
2925 : local_var);
2926 : else
2927 : break;
2928 615 : curr_loop = curr_loop->outer;
2929 : }
2930 : }
2931 : }
2932 12444 : ctx->local_reduction_clauses = local_reduction_clauses;
2933 12444 : ctx->outer_reduction_clauses
2934 12444 : = chainon (unshare_expr (ctx->local_reduction_clauses),
2935 : ctx->outer_reduction_clauses);
2936 :
2937 12444 : if (tgt && is_oacc_kernels (tgt))
2938 : {
2939 : /* Strip out reductions, as they are not handled yet. */
2940 : tree *prev_ptr = &clauses;
2941 :
2942 7116 : while (tree probe = *prev_ptr)
2943 : {
2944 4786 : tree *next_ptr = &OMP_CLAUSE_CHAIN (probe);
2945 :
2946 4786 : if (OMP_CLAUSE_CODE (probe) == OMP_CLAUSE_REDUCTION)
2947 718 : *prev_ptr = *next_ptr;
2948 : else
2949 : prev_ptr = next_ptr;
2950 : }
2951 :
2952 2330 : gimple_omp_for_set_clauses (stmt, clauses);
2953 : }
2954 : }
2955 :
2956 52632 : scan_sharing_clauses (clauses, ctx);
2957 :
2958 52632 : scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
2959 179933 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
2960 : {
2961 74669 : scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
2962 74669 : scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
2963 74669 : scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
2964 74669 : scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
2965 : }
2966 52632 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2967 52632 : return ctx;
2968 : }
2969 :
2970 : /* Duplicate #pragma omp simd, one for SIMT, another one for SIMD. */
2971 :
2972 : static void
2973 0 : scan_omp_simd (gimple_stmt_iterator *gsi, gomp_for *stmt,
2974 : omp_context *outer_ctx)
2975 : {
2976 0 : gbind *bind = gimple_build_bind (NULL, NULL, NULL);
2977 0 : gsi_replace (gsi, bind, false);
2978 0 : gimple_seq seq = NULL;
2979 0 : gimple *g = gimple_build_call_internal (IFN_GOMP_USE_SIMT, 0);
2980 0 : tree cond = create_tmp_var_raw (integer_type_node);
2981 0 : DECL_CONTEXT (cond) = current_function_decl;
2982 0 : DECL_SEEN_IN_BIND_EXPR_P (cond) = 1;
2983 0 : gimple_bind_set_vars (bind, cond);
2984 0 : gimple_call_set_lhs (g, cond);
2985 0 : gimple_seq_add_stmt (&seq, g);
2986 0 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
2987 0 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
2988 0 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
2989 0 : g = gimple_build_cond (NE_EXPR, cond, integer_zero_node, lab1, lab2);
2990 0 : gimple_seq_add_stmt (&seq, g);
2991 0 : g = gimple_build_label (lab1);
2992 0 : gimple_seq_add_stmt (&seq, g);
2993 0 : gimple_seq new_seq = copy_gimple_seq_and_replace_locals (stmt);
2994 0 : gomp_for *new_stmt = as_a <gomp_for *> (new_seq);
2995 0 : tree clause = build_omp_clause (gimple_location (stmt), OMP_CLAUSE__SIMT_);
2996 0 : OMP_CLAUSE_CHAIN (clause) = gimple_omp_for_clauses (new_stmt);
2997 0 : gimple_omp_for_set_clauses (new_stmt, clause);
2998 0 : gimple_seq_add_stmt (&seq, new_stmt);
2999 0 : g = gimple_build_goto (lab3);
3000 0 : gimple_seq_add_stmt (&seq, g);
3001 0 : g = gimple_build_label (lab2);
3002 0 : gimple_seq_add_stmt (&seq, g);
3003 0 : gimple_seq_add_stmt (&seq, stmt);
3004 0 : g = gimple_build_label (lab3);
3005 0 : gimple_seq_add_stmt (&seq, g);
3006 0 : gimple_bind_set_body (bind, seq);
3007 0 : update_stmt (bind);
3008 0 : scan_omp_for (new_stmt, outer_ctx);
3009 0 : scan_omp_for (stmt, outer_ctx)->simt_stmt = new_stmt;
3010 0 : }
3011 :
3012 : static tree omp_find_scan (gimple_stmt_iterator *, bool *,
3013 : struct walk_stmt_info *);
3014 : static omp_context *maybe_lookup_ctx (gimple *);
3015 :
3016 : /* Duplicate #pragma omp simd, one for the scan input phase loop and one
3017 : for scan phase loop. */
3018 :
3019 : static void
3020 83 : scan_omp_simd_scan (gimple_stmt_iterator *gsi, gomp_for *stmt,
3021 : omp_context *outer_ctx)
3022 : {
3023 : /* The only change between inclusive and exclusive scan will be
3024 : within the first simd loop, so just use inclusive in the
3025 : worksharing loop. */
3026 83 : outer_ctx->scan_inclusive = true;
3027 83 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_INCLUSIVE);
3028 83 : OMP_CLAUSE_DECL (c) = integer_zero_node;
3029 :
3030 83 : gomp_scan *input_stmt = gimple_build_omp_scan (NULL, NULL_TREE);
3031 83 : gomp_scan *scan_stmt = gimple_build_omp_scan (NULL, c);
3032 83 : gsi_replace (gsi, input_stmt, false);
3033 83 : gimple_seq input_body = NULL;
3034 83 : gimple_seq_add_stmt (&input_body, stmt);
3035 83 : gsi_insert_after (gsi, scan_stmt, GSI_NEW_STMT);
3036 :
3037 83 : gimple_stmt_iterator input1_gsi = gsi_none ();
3038 83 : struct walk_stmt_info wi;
3039 83 : memset (&wi, 0, sizeof (wi));
3040 83 : wi.val_only = true;
3041 83 : wi.info = (void *) &input1_gsi;
3042 83 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), omp_find_scan, NULL, &wi);
3043 83 : gcc_assert (!gsi_end_p (input1_gsi));
3044 :
3045 83 : gimple *input_stmt1 = gsi_stmt (input1_gsi);
3046 83 : gsi_next (&input1_gsi);
3047 83 : gimple *scan_stmt1 = gsi_stmt (input1_gsi);
3048 83 : gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
3049 83 : c = gimple_omp_scan_clauses (as_a <gomp_scan *> (scan_stmt1));
3050 166 : if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
3051 : std::swap (input_stmt1, scan_stmt1);
3052 :
3053 83 : gimple_seq input_body1 = gimple_omp_body (input_stmt1);
3054 83 : gimple_omp_set_body (input_stmt1, NULL);
3055 :
3056 83 : gimple_seq scan_body = copy_gimple_seq_and_replace_locals (stmt);
3057 83 : gomp_for *new_stmt = as_a <gomp_for *> (scan_body);
3058 :
3059 83 : gimple_omp_set_body (input_stmt1, input_body1);
3060 83 : gimple_omp_set_body (scan_stmt1, NULL);
3061 :
3062 83 : gimple_stmt_iterator input2_gsi = gsi_none ();
3063 83 : memset (&wi, 0, sizeof (wi));
3064 83 : wi.val_only = true;
3065 83 : wi.info = (void *) &input2_gsi;
3066 83 : walk_gimple_seq_mod (gimple_omp_body_ptr (new_stmt), omp_find_scan,
3067 : NULL, &wi);
3068 83 : gcc_assert (!gsi_end_p (input2_gsi));
3069 :
3070 83 : gimple *input_stmt2 = gsi_stmt (input2_gsi);
3071 83 : gsi_next (&input2_gsi);
3072 83 : gimple *scan_stmt2 = gsi_stmt (input2_gsi);
3073 83 : gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
3074 166 : if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
3075 : std::swap (input_stmt2, scan_stmt2);
3076 :
3077 83 : gimple_omp_set_body (input_stmt2, NULL);
3078 :
3079 83 : gimple_omp_set_body (input_stmt, input_body);
3080 83 : gimple_omp_set_body (scan_stmt, scan_body);
3081 :
3082 83 : omp_context *ctx = new_omp_context (input_stmt, outer_ctx);
3083 83 : scan_omp (gimple_omp_body_ptr (input_stmt), ctx);
3084 :
3085 83 : ctx = new_omp_context (scan_stmt, outer_ctx);
3086 83 : scan_omp (gimple_omp_body_ptr (scan_stmt), ctx);
3087 :
3088 83 : maybe_lookup_ctx (new_stmt)->for_simd_scan_phase = true;
3089 83 : }
3090 :
3091 : /* Scan an OpenMP sections directive. */
3092 :
3093 : static void
3094 581 : scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx)
3095 : {
3096 581 : omp_context *ctx;
3097 :
3098 581 : ctx = new_omp_context (stmt, outer_ctx);
3099 581 : scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
3100 581 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3101 581 : }
3102 :
3103 : /* Scan an OpenMP single directive. */
3104 :
3105 : static void
3106 1228 : scan_omp_single (gomp_single *stmt, omp_context *outer_ctx)
3107 : {
3108 1228 : omp_context *ctx;
3109 1228 : tree name;
3110 :
3111 1228 : ctx = new_omp_context (stmt, outer_ctx);
3112 1228 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3113 1228 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3114 1228 : name = create_tmp_var_name (".omp_copy_s");
3115 1228 : name = build_decl (gimple_location (stmt),
3116 : TYPE_DECL, name, ctx->record_type);
3117 1228 : TYPE_NAME (ctx->record_type) = name;
3118 :
3119 1228 : scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
3120 1228 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3121 :
3122 1228 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3123 1081 : ctx->record_type = NULL;
3124 : else
3125 147 : layout_type (ctx->record_type);
3126 1228 : }
3127 :
3128 : /* Scan a GIMPLE_OMP_TARGET. */
3129 :
3130 : static void
3131 41209 : scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
3132 : {
3133 41209 : omp_context *ctx;
3134 41209 : tree name;
3135 41209 : bool offloaded = is_gimple_omp_offloaded (stmt);
3136 41209 : tree clauses = gimple_omp_target_clauses (stmt);
3137 :
3138 41209 : ctx = new_omp_context (stmt, outer_ctx);
3139 41209 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3140 41209 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3141 41209 : name = create_tmp_var_name (".omp_data_t");
3142 41209 : name = build_decl (gimple_location (stmt),
3143 : TYPE_DECL, name, ctx->record_type);
3144 41209 : DECL_ARTIFICIAL (name) = 1;
3145 41209 : DECL_NAMELESS (name) = 1;
3146 41209 : TYPE_NAME (ctx->record_type) = name;
3147 41209 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
3148 :
3149 41209 : if (offloaded)
3150 : {
3151 24456 : create_omp_child_function (ctx, false);
3152 24456 : gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
3153 : }
3154 :
3155 41209 : scan_sharing_clauses (clauses, ctx);
3156 41209 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3157 :
3158 41209 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3159 7844 : ctx->record_type = ctx->receiver_decl = NULL;
3160 : else
3161 : {
3162 33365 : TYPE_FIELDS (ctx->record_type)
3163 33365 : = nreverse (TYPE_FIELDS (ctx->record_type));
3164 33365 : if (flag_checking)
3165 : {
3166 33365 : unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
3167 33365 : for (tree field = TYPE_FIELDS (ctx->record_type);
3168 128672 : field;
3169 95307 : field = DECL_CHAIN (field))
3170 95307 : gcc_assert (DECL_ALIGN (field) == align);
3171 : }
3172 33365 : layout_type (ctx->record_type);
3173 33365 : if (offloaded)
3174 17163 : fixup_child_record_type (ctx);
3175 : }
3176 :
3177 41209 : if (ctx->teams_nested_p && ctx->nonteams_nested_p)
3178 : {
3179 4 : error_at (gimple_location (stmt),
3180 : "%<target%> construct with nested %<teams%> construct "
3181 : "contains directives outside of the %<teams%> construct");
3182 4 : gimple_omp_set_body (stmt, gimple_build_bind (NULL, NULL, NULL));
3183 : }
3184 41209 : }
3185 :
3186 : /* Scan an OpenMP teams directive. */
3187 :
3188 : static void
3189 8736 : scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
3190 : {
3191 8736 : omp_context *ctx = new_omp_context (stmt, outer_ctx);
3192 :
3193 8736 : if (!gimple_omp_teams_host (stmt))
3194 : {
3195 6118 : scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3196 6118 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3197 6118 : return;
3198 : }
3199 2618 : taskreg_contexts.safe_push (ctx);
3200 2618 : gcc_assert (taskreg_nesting_level == 1);
3201 2618 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3202 2618 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3203 2618 : tree name = create_tmp_var_name (".omp_data_s");
3204 2618 : name = build_decl (gimple_location (stmt),
3205 : TYPE_DECL, name, ctx->record_type);
3206 2618 : DECL_ARTIFICIAL (name) = 1;
3207 2618 : DECL_NAMELESS (name) = 1;
3208 2618 : TYPE_NAME (ctx->record_type) = name;
3209 2618 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
3210 2618 : create_omp_child_function (ctx, false);
3211 2618 : gimple_omp_teams_set_child_fn (stmt, ctx->cb.dst_fn);
3212 :
3213 2618 : scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3214 2618 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3215 :
3216 2618 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3217 1446 : ctx->record_type = ctx->receiver_decl = NULL;
3218 : }
3219 :
3220 : /* Check nesting restrictions. */
3221 : static bool
3222 156193 : check_omp_nesting_restrictions (gimple *stmt, omp_context *ctx)
3223 : {
3224 156193 : tree c;
3225 :
3226 : /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3227 : inside an OpenACC CTX. */
3228 156193 : if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3229 156193 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE)
3230 : /* ..., except for the atomic codes that OpenACC shares with OpenMP. */
3231 : ;
3232 280525 : else if (!(is_gimple_omp (stmt)
3233 138152 : && is_gimple_omp_oacc (stmt)))
3234 : {
3235 112120 : if (oacc_get_fn_attrib (cfun->decl) != NULL)
3236 : {
3237 48 : error_at (gimple_location (stmt),
3238 : "non-OpenACC construct inside of OpenACC routine");
3239 48 : return false;
3240 : }
3241 : else
3242 235489 : for (omp_context *octx = ctx; octx != NULL; octx = octx->outer)
3243 247010 : if (is_gimple_omp (octx->stmt)
3244 123593 : && is_gimple_omp_oacc (octx->stmt))
3245 : {
3246 176 : error_at (gimple_location (stmt),
3247 : "non-OpenACC construct inside of OpenACC region");
3248 176 : return false;
3249 : }
3250 : }
3251 :
3252 155969 : if (ctx != NULL)
3253 : {
3254 84110 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
3255 84110 : && gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_REGION)
3256 : {
3257 8313 : c = omp_find_clause (gimple_omp_target_clauses (ctx->stmt),
3258 : OMP_CLAUSE_DEVICE);
3259 8313 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
3260 : {
3261 9 : error_at (gimple_location (stmt),
3262 : "OpenMP constructs are not allowed in target region "
3263 : "with %<ancestor%>");
3264 9 : return false;
3265 : }
3266 :
3267 8304 : if (gimple_code (stmt) == GIMPLE_OMP_TEAMS && !ctx->teams_nested_p)
3268 6114 : ctx->teams_nested_p = true;
3269 : else
3270 2190 : ctx->nonteams_nested_p = true;
3271 : }
3272 84101 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN
3273 172 : && ctx->outer
3274 84273 : && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
3275 : ctx = ctx->outer;
3276 84101 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3277 28308 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
3278 85910 : && !ctx->loop_p)
3279 : {
3280 1589 : c = NULL_TREE;
3281 1589 : if (ctx->order_concurrent
3282 1589 : && (gimple_code (stmt) == GIMPLE_OMP_ORDERED
3283 280 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3284 190 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE))
3285 : {
3286 210 : error_at (gimple_location (stmt),
3287 : "OpenMP constructs other than %<parallel%>, %<loop%>"
3288 : " or %<simd%> may not be nested inside a region with"
3289 : " the %<order(concurrent)%> clause");
3290 210 : return false;
3291 : }
3292 1379 : if (gimple_code (stmt) == GIMPLE_OMP_ORDERED)
3293 : {
3294 133 : c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3295 133 : if (omp_find_clause (c, OMP_CLAUSE_SIMD))
3296 : {
3297 133 : if (omp_find_clause (c, OMP_CLAUSE_THREADS)
3298 133 : && (ctx->outer == NULL
3299 29 : || !gimple_omp_for_combined_into_p (ctx->stmt)
3300 25 : || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR
3301 25 : || (gimple_omp_for_kind (ctx->outer->stmt)
3302 : != GF_OMP_FOR_KIND_FOR)
3303 25 : || !gimple_omp_for_combined_p (ctx->outer->stmt)))
3304 : {
3305 8 : error_at (gimple_location (stmt),
3306 : "%<ordered simd threads%> must be closely "
3307 : "nested inside of %<%s simd%> region",
3308 8 : lang_GNU_Fortran () ? "do" : "for");
3309 8 : return false;
3310 : }
3311 : return true;
3312 : }
3313 : }
3314 : else if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3315 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE
3316 : || gimple_code (stmt) == GIMPLE_OMP_SCAN
3317 : || gimple_code (stmt) == GIMPLE_OMP_STRUCTURED_BLOCK)
3318 : return true;
3319 : else if (gimple_code (stmt) == GIMPLE_OMP_FOR
3320 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
3321 : return true;
3322 60 : error_at (gimple_location (stmt),
3323 : "OpenMP constructs other than "
3324 : "%<ordered simd%>, %<simd%>, %<loop%> or %<atomic%> may "
3325 : "not be nested inside %<simd%> region");
3326 60 : return false;
3327 : }
3328 82512 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3329 : {
3330 5829 : if ((gimple_code (stmt) != GIMPLE_OMP_FOR
3331 5525 : || (gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_DISTRIBUTE
3332 25 : && omp_find_clause (gimple_omp_for_clauses (stmt),
3333 : OMP_CLAUSE_BIND) == NULL_TREE))
3334 5845 : && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
3335 : {
3336 132 : error_at (gimple_location (stmt),
3337 : "only %<distribute%>, %<parallel%> or %<loop%> "
3338 : "regions are allowed to be strictly nested inside "
3339 : "%<teams%> region");
3340 132 : return false;
3341 : }
3342 : }
3343 76683 : else if (ctx->order_concurrent
3344 1944 : && gimple_code (stmt) != GIMPLE_OMP_PARALLEL
3345 1465 : && (gimple_code (stmt) != GIMPLE_OMP_FOR
3346 1191 : || gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_SIMD)
3347 289 : && gimple_code (stmt) != GIMPLE_OMP_SCAN
3348 76972 : && gimple_code (stmt) != GIMPLE_OMP_STRUCTURED_BLOCK)
3349 : {
3350 285 : if (ctx->loop_p)
3351 135 : error_at (gimple_location (stmt),
3352 : "OpenMP constructs other than %<parallel%>, %<loop%> or "
3353 : "%<simd%> may not be nested inside a %<loop%> region");
3354 : else
3355 150 : error_at (gimple_location (stmt),
3356 : "OpenMP constructs other than %<parallel%>, %<loop%> or "
3357 : "%<simd%> may not be nested inside a region with "
3358 : "the %<order(concurrent)%> clause");
3359 285 : return false;
3360 : }
3361 : }
3362 153954 : switch (gimple_code (stmt))
3363 : {
3364 52751 : case GIMPLE_OMP_FOR:
3365 52751 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD)
3366 : return true;
3367 41993 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
3368 : {
3369 8217 : if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
3370 : {
3371 0 : error_at (gimple_location (stmt),
3372 : "%<distribute%> region must be strictly nested "
3373 : "inside %<teams%> construct");
3374 0 : return false;
3375 : }
3376 : return true;
3377 : }
3378 : /* We split taskloop into task and nested taskloop in it. */
3379 33776 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3380 : return true;
3381 : /* For now, hope this will change and loop bind(parallel) will not
3382 : be allowed in lots of contexts. */
3383 30642 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
3384 48775 : && omp_find_clause (gimple_omp_for_clauses (stmt), OMP_CLAUSE_BIND))
3385 : return true;
3386 30013 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
3387 : {
3388 12509 : bool ok = false;
3389 :
3390 12509 : if (ctx)
3391 11962 : switch (gimple_code (ctx->stmt))
3392 : {
3393 4049 : case GIMPLE_OMP_FOR:
3394 4049 : ok = (gimple_omp_for_kind (ctx->stmt)
3395 : == GF_OMP_FOR_KIND_OACC_LOOP);
3396 4049 : break;
3397 :
3398 7885 : case GIMPLE_OMP_TARGET:
3399 7885 : switch (gimple_omp_target_kind (ctx->stmt))
3400 : {
3401 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3402 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
3403 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
3404 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3405 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3406 : ok = true;
3407 : break;
3408 :
3409 : default:
3410 : break;
3411 : }
3412 :
3413 : default:
3414 : break;
3415 : }
3416 547 : else if (oacc_get_fn_attrib (current_function_decl))
3417 : ok = true;
3418 4049 : if (!ok)
3419 : {
3420 65 : error_at (gimple_location (stmt),
3421 : "OpenACC loop directive must be associated with"
3422 : " an OpenACC compute region");
3423 65 : return false;
3424 : }
3425 : }
3426 : /* FALLTHRU */
3427 34109 : case GIMPLE_CALL:
3428 34109 : if (is_gimple_call (stmt)
3429 34109 : && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3430 : == BUILT_IN_GOMP_CANCEL
3431 3057 : || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3432 : == BUILT_IN_GOMP_CANCELLATION_POINT))
3433 : {
3434 1907 : const char *bad = NULL;
3435 1907 : const char *kind = NULL;
3436 1907 : const char *construct
3437 1907 : = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3438 : == BUILT_IN_GOMP_CANCEL)
3439 1907 : ? "cancel"
3440 803 : : "cancellation point";
3441 1907 : if (ctx == NULL)
3442 : {
3443 41 : error_at (gimple_location (stmt), "orphaned %qs construct",
3444 : construct);
3445 41 : return false;
3446 : }
3447 1866 : switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
3448 1866 : ? tree_to_shwi (gimple_call_arg (stmt, 0))
3449 : : 0)
3450 : {
3451 528 : case 1:
3452 528 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
3453 : bad = "parallel";
3454 208 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3455 : == BUILT_IN_GOMP_CANCEL
3456 208 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3457 164 : ctx->cancellable = true;
3458 164 : kind = "parallel";
3459 164 : break;
3460 427 : case 2:
3461 427 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3462 427 : || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
3463 : bad = "for";
3464 117 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3465 : == BUILT_IN_GOMP_CANCEL
3466 117 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3467 : {
3468 106 : ctx->cancellable = true;
3469 106 : if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3470 : OMP_CLAUSE_NOWAIT))
3471 5 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3472 : "%<cancel for%> inside "
3473 : "%<nowait%> for construct");
3474 106 : if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3475 : OMP_CLAUSE_ORDERED))
3476 5 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3477 : "%<cancel for%> inside "
3478 : "%<ordered%> for construct");
3479 : }
3480 5 : kind = "for";
3481 5 : break;
3482 383 : case 4:
3483 383 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
3484 383 : && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
3485 : bad = "sections";
3486 93 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3487 : == BUILT_IN_GOMP_CANCEL
3488 93 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3489 : {
3490 73 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
3491 : {
3492 0 : ctx->cancellable = true;
3493 0 : if (omp_find_clause (gimple_omp_sections_clauses
3494 0 : (ctx->stmt),
3495 : OMP_CLAUSE_NOWAIT))
3496 0 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3497 : "%<cancel sections%> inside "
3498 : "%<nowait%> sections construct");
3499 : }
3500 : else
3501 : {
3502 73 : gcc_assert (ctx->outer
3503 : && gimple_code (ctx->outer->stmt)
3504 : == GIMPLE_OMP_SECTIONS);
3505 73 : ctx->outer->cancellable = true;
3506 73 : if (omp_find_clause (gimple_omp_sections_clauses
3507 73 : (ctx->outer->stmt),
3508 : OMP_CLAUSE_NOWAIT))
3509 10 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3510 : "%<cancel sections%> inside "
3511 : "%<nowait%> sections construct");
3512 : }
3513 : }
3514 10 : kind = "sections";
3515 10 : break;
3516 528 : case 8:
3517 528 : if (!is_task_ctx (ctx)
3518 528 : && (!is_taskloop_ctx (ctx)
3519 54 : || ctx->outer == NULL
3520 54 : || !is_task_ctx (ctx->outer)))
3521 : bad = "task";
3522 : else
3523 : {
3524 248 : for (omp_context *octx = ctx->outer;
3525 439 : octx; octx = octx->outer)
3526 : {
3527 425 : switch (gimple_code (octx->stmt))
3528 : {
3529 : case GIMPLE_OMP_TASKGROUP:
3530 : break;
3531 40 : case GIMPLE_OMP_TARGET:
3532 40 : if (gimple_omp_target_kind (octx->stmt)
3533 : != GF_OMP_TARGET_KIND_REGION)
3534 20 : continue;
3535 : /* FALLTHRU */
3536 99 : case GIMPLE_OMP_PARALLEL:
3537 99 : case GIMPLE_OMP_TEAMS:
3538 99 : error_at (gimple_location (stmt),
3539 : "%<%s taskgroup%> construct not closely "
3540 : "nested inside of %<taskgroup%> region",
3541 : construct);
3542 99 : return false;
3543 104 : case GIMPLE_OMP_TASK:
3544 104 : if (gimple_omp_task_taskloop_p (octx->stmt)
3545 94 : && octx->outer
3546 198 : && is_taskloop_ctx (octx->outer))
3547 : {
3548 94 : tree clauses
3549 94 : = gimple_omp_for_clauses (octx->outer->stmt);
3550 94 : if (!omp_find_clause (clauses, OMP_CLAUSE_NOGROUP))
3551 : break;
3552 : }
3553 60 : continue;
3554 111 : default:
3555 111 : continue;
3556 171 : }
3557 : break;
3558 : }
3559 149 : ctx->cancellable = true;
3560 : }
3561 149 : kind = "taskgroup";
3562 149 : break;
3563 0 : default:
3564 0 : error_at (gimple_location (stmt), "invalid arguments");
3565 0 : return false;
3566 : }
3567 328 : if (bad)
3568 : {
3569 1200 : error_at (gimple_location (stmt),
3570 : "%<%s %s%> construct not closely nested inside of %qs",
3571 : construct, kind, bad);
3572 1200 : return false;
3573 : }
3574 : }
3575 : /* FALLTHRU */
3576 : case GIMPLE_OMP_SECTIONS:
3577 : case GIMPLE_OMP_SINGLE:
3578 55566 : for (; ctx != NULL; ctx = ctx->outer)
3579 37505 : switch (gimple_code (ctx->stmt))
3580 : {
3581 6873 : case GIMPLE_OMP_FOR:
3582 6873 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3583 6873 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3584 : break;
3585 : /* FALLTHRU */
3586 778 : case GIMPLE_OMP_SECTIONS:
3587 778 : case GIMPLE_OMP_SINGLE:
3588 778 : case GIMPLE_OMP_ORDERED:
3589 778 : case GIMPLE_OMP_MASTER:
3590 778 : case GIMPLE_OMP_MASKED:
3591 778 : case GIMPLE_OMP_TASK:
3592 778 : case GIMPLE_OMP_CRITICAL:
3593 778 : if (is_gimple_call (stmt))
3594 : {
3595 707 : if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3596 : != BUILT_IN_GOMP_BARRIER)
3597 : return true;
3598 23 : error_at (gimple_location (stmt),
3599 : "barrier region may not be closely nested inside "
3600 : "of work-sharing, %<loop%>, %<critical%>, "
3601 : "%<ordered%>, %<master%>, %<masked%>, explicit "
3602 : "%<task%> or %<taskloop%> region");
3603 23 : return false;
3604 : }
3605 71 : error_at (gimple_location (stmt),
3606 : "work-sharing region may not be closely nested inside "
3607 : "of work-sharing, %<loop%>, %<critical%>, %<ordered%>, "
3608 : "%<master%>, %<masked%>, explicit %<task%> or "
3609 : "%<taskloop%> region");
3610 71 : return false;
3611 : case GIMPLE_OMP_PARALLEL:
3612 : case GIMPLE_OMP_TEAMS:
3613 : return true;
3614 13434 : case GIMPLE_OMP_TARGET:
3615 13434 : if (gimple_omp_target_kind (ctx->stmt)
3616 : == GF_OMP_TARGET_KIND_REGION)
3617 : return true;
3618 : break;
3619 : default:
3620 : break;
3621 : }
3622 : break;
3623 : case GIMPLE_OMP_MASTER:
3624 : case GIMPLE_OMP_MASKED:
3625 1513 : for (; ctx != NULL; ctx = ctx->outer)
3626 1008 : switch (gimple_code (ctx->stmt))
3627 : {
3628 14 : case GIMPLE_OMP_FOR:
3629 14 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3630 14 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3631 : break;
3632 : /* FALLTHRU */
3633 30 : case GIMPLE_OMP_SECTIONS:
3634 30 : case GIMPLE_OMP_SINGLE:
3635 30 : case GIMPLE_OMP_TASK:
3636 60 : error_at (gimple_location (stmt),
3637 : "%qs region may not be closely nested inside "
3638 : "of work-sharing, %<loop%>, explicit %<task%> or "
3639 : "%<taskloop%> region",
3640 30 : gimple_code (stmt) == GIMPLE_OMP_MASTER
3641 : ? "master" : "masked");
3642 30 : return false;
3643 : case GIMPLE_OMP_PARALLEL:
3644 : case GIMPLE_OMP_TEAMS:
3645 : return true;
3646 10 : case GIMPLE_OMP_TARGET:
3647 10 : if (gimple_omp_target_kind (ctx->stmt)
3648 : == GF_OMP_TARGET_KIND_REGION)
3649 : return true;
3650 : break;
3651 : default:
3652 : break;
3653 : }
3654 : break;
3655 : case GIMPLE_OMP_SCOPE:
3656 274 : for (; ctx != NULL; ctx = ctx->outer)
3657 175 : switch (gimple_code (ctx->stmt))
3658 : {
3659 7 : case GIMPLE_OMP_FOR:
3660 7 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3661 7 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3662 : break;
3663 : /* FALLTHRU */
3664 25 : case GIMPLE_OMP_SECTIONS:
3665 25 : case GIMPLE_OMP_SINGLE:
3666 25 : case GIMPLE_OMP_TASK:
3667 25 : case GIMPLE_OMP_CRITICAL:
3668 25 : case GIMPLE_OMP_ORDERED:
3669 25 : case GIMPLE_OMP_MASTER:
3670 25 : case GIMPLE_OMP_MASKED:
3671 25 : error_at (gimple_location (stmt),
3672 : "%<scope%> region may not be closely nested inside "
3673 : "of work-sharing, %<loop%>, explicit %<task%>, "
3674 : "%<taskloop%>, %<critical%>, %<ordered%>, %<master%>, "
3675 : "or %<masked%> region");
3676 25 : return false;
3677 : case GIMPLE_OMP_PARALLEL:
3678 : case GIMPLE_OMP_TEAMS:
3679 : return true;
3680 5 : case GIMPLE_OMP_TARGET:
3681 5 : if (gimple_omp_target_kind (ctx->stmt)
3682 : == GF_OMP_TARGET_KIND_REGION)
3683 : return true;
3684 : break;
3685 : default:
3686 : break;
3687 : }
3688 : break;
3689 5371 : case GIMPLE_OMP_TASK:
3690 22916 : for (c = gimple_omp_task_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3691 17549 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
3692 : {
3693 4 : enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
3694 4 : error_at (OMP_CLAUSE_LOCATION (c),
3695 : "%<%s(%s)%> is only allowed in %<omp ordered%>",
3696 4 : OMP_CLAUSE_DOACROSS_DEPEND (c) ? "depend" : "doacross",
3697 : kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
3698 4 : return false;
3699 : }
3700 : break;
3701 1631 : case GIMPLE_OMP_ORDERED:
3702 2873 : for (c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3703 2873 : c; c = OMP_CLAUSE_CHAIN (c))
3704 : {
3705 1258 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS)
3706 : {
3707 169 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
3708 : {
3709 4 : error_at (OMP_CLAUSE_LOCATION (c),
3710 : "invalid depend kind in omp %<ordered%> %<depend%>");
3711 4 : return false;
3712 : }
3713 165 : gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREADS
3714 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SIMD);
3715 165 : continue;
3716 : }
3717 :
3718 1089 : tree oclause;
3719 : /* Look for containing ordered(N) loop. */
3720 1089 : if (ctx == NULL
3721 1077 : || gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3722 1089 : || (oclause
3723 1077 : = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3724 : OMP_CLAUSE_ORDERED)) == NULL_TREE)
3725 : {
3726 12 : error_at (OMP_CLAUSE_LOCATION (c),
3727 : "%<ordered%> construct with %<depend%> clause "
3728 : "must be closely nested inside an %<ordered%> loop");
3729 12 : return false;
3730 : }
3731 : }
3732 1615 : c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3733 1615 : if (omp_find_clause (c, OMP_CLAUSE_SIMD))
3734 : {
3735 : /* ordered simd must be closely nested inside of simd region,
3736 : and simd region must not encounter constructs other than
3737 : ordered simd, therefore ordered simd may be either orphaned,
3738 : or ctx->stmt must be simd. The latter case is handled already
3739 : earlier. */
3740 34 : if (ctx != NULL)
3741 : {
3742 10 : error_at (gimple_location (stmt),
3743 : "%<ordered%> %<simd%> must be closely nested inside "
3744 : "%<simd%> region");
3745 10 : return false;
3746 : }
3747 : }
3748 1605 : for (; ctx != NULL; ctx = ctx->outer)
3749 1474 : switch (gimple_code (ctx->stmt))
3750 : {
3751 38 : case GIMPLE_OMP_CRITICAL:
3752 38 : case GIMPLE_OMP_TASK:
3753 38 : case GIMPLE_OMP_ORDERED:
3754 38 : ordered_in_taskloop:
3755 38 : error_at (gimple_location (stmt),
3756 : "%<ordered%> region may not be closely nested inside "
3757 : "of %<critical%>, %<ordered%>, explicit %<task%> or "
3758 : "%<taskloop%> region");
3759 38 : return false;
3760 1416 : case GIMPLE_OMP_FOR:
3761 1416 : if (gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3762 10 : goto ordered_in_taskloop;
3763 1406 : tree o;
3764 1406 : o = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3765 : OMP_CLAUSE_ORDERED);
3766 1406 : if (o == NULL)
3767 : {
3768 22 : error_at (gimple_location (stmt),
3769 : "%<ordered%> region must be closely nested inside "
3770 : "a loop region with an %<ordered%> clause");
3771 22 : return false;
3772 : }
3773 1384 : if (!gimple_omp_ordered_standalone_p (stmt))
3774 : {
3775 375 : if (OMP_CLAUSE_ORDERED_DOACROSS (o))
3776 : {
3777 10 : error_at (gimple_location (stmt),
3778 : "%<ordered%> construct without %<doacross%> or "
3779 : "%<depend%> clauses must not have the same "
3780 : "binding region as %<ordered%> construct with "
3781 : "those clauses");
3782 10 : return false;
3783 : }
3784 365 : else if (OMP_CLAUSE_ORDERED_EXPR (o))
3785 : {
3786 23 : tree co
3787 23 : = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3788 : OMP_CLAUSE_COLLAPSE);
3789 23 : HOST_WIDE_INT
3790 23 : o_n = tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (o));
3791 23 : HOST_WIDE_INT c_n = 1;
3792 23 : if (co)
3793 5 : c_n = tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (co));
3794 23 : if (o_n != c_n)
3795 : {
3796 10 : error_at (gimple_location (stmt),
3797 : "%<ordered%> construct without %<doacross%> "
3798 : "or %<depend%> clauses binds to loop where "
3799 : "%<collapse%> argument %wd is different from "
3800 : "%<ordered%> argument %wd", c_n, o_n);
3801 10 : return false;
3802 : }
3803 : }
3804 : }
3805 : return true;
3806 10 : case GIMPLE_OMP_TARGET:
3807 10 : if (gimple_omp_target_kind (ctx->stmt)
3808 : != GF_OMP_TARGET_KIND_REGION)
3809 : break;
3810 : /* FALLTHRU */
3811 30 : case GIMPLE_OMP_PARALLEL:
3812 30 : case GIMPLE_OMP_TEAMS:
3813 30 : error_at (gimple_location (stmt),
3814 : "%<ordered%> region must be closely nested inside "
3815 : "a loop region with an %<ordered%> clause");
3816 30 : return false;
3817 : default:
3818 : break;
3819 : }
3820 : break;
3821 462 : case GIMPLE_OMP_CRITICAL:
3822 462 : {
3823 462 : tree this_stmt_name
3824 462 : = gimple_omp_critical_name (as_a <gomp_critical *> (stmt));
3825 888 : for (; ctx != NULL; ctx = ctx->outer)
3826 432 : if (gomp_critical *other_crit
3827 451 : = dyn_cast <gomp_critical *> (ctx->stmt))
3828 25 : if (this_stmt_name == gimple_omp_critical_name (other_crit))
3829 : {
3830 6 : error_at (gimple_location (stmt),
3831 : "%<critical%> region may not be nested inside "
3832 : "a %<critical%> region with the same name");
3833 6 : return false;
3834 : }
3835 : }
3836 : break;
3837 8780 : case GIMPLE_OMP_TEAMS:
3838 8780 : if (ctx == NULL)
3839 : break;
3840 6162 : else if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
3841 6162 : || (gimple_omp_target_kind (ctx->stmt)
3842 : != GF_OMP_TARGET_KIND_REGION))
3843 : {
3844 : /* Teams construct can appear either strictly nested inside of
3845 : target construct with no intervening stmts, or can be encountered
3846 : only by initial task (so must not appear inside any OpenMP
3847 : construct. */
3848 44 : error_at (gimple_location (stmt),
3849 : "%<teams%> construct must be closely nested inside of "
3850 : "%<target%> construct or not nested in any OpenMP "
3851 : "construct");
3852 44 : return false;
3853 : }
3854 : break;
3855 41592 : case GIMPLE_OMP_TARGET:
3856 185943 : for (c = gimple_omp_target_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3857 144355 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
3858 : {
3859 4 : enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
3860 4 : error_at (OMP_CLAUSE_LOCATION (c),
3861 : "%<depend(%s)%> is only allowed in %<omp ordered%>",
3862 : kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
3863 4 : return false;
3864 : }
3865 41588 : if (is_gimple_omp_offloaded (stmt)
3866 41588 : && oacc_get_fn_attrib (cfun->decl) != NULL)
3867 : {
3868 4 : error_at (gimple_location (stmt),
3869 : "OpenACC region inside of OpenACC routine, nested "
3870 : "parallelism not supported yet");
3871 4 : return false;
3872 : }
3873 48987 : for (; ctx != NULL; ctx = ctx->outer)
3874 : {
3875 7809 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
3876 : {
3877 768 : if (is_gimple_omp (stmt)
3878 768 : && is_gimple_omp_oacc (stmt)
3879 242 : && is_gimple_omp (ctx->stmt))
3880 : {
3881 242 : error_at (gimple_location (stmt),
3882 : "OpenACC construct inside of non-OpenACC region");
3883 242 : return false;
3884 : }
3885 526 : continue;
3886 : }
3887 :
3888 7041 : const char *stmt_name, *ctx_stmt_name;
3889 7041 : switch (gimple_omp_target_kind (stmt))
3890 : {
3891 : case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break;
3892 399 : case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break;
3893 1688 : case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break;
3894 12 : case GF_OMP_TARGET_KIND_ENTER_DATA:
3895 12 : stmt_name = "target enter data"; break;
3896 18 : case GF_OMP_TARGET_KIND_EXIT_DATA:
3897 18 : stmt_name = "target exit data"; break;
3898 1442 : case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break;
3899 1618 : case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break;
3900 183 : case GF_OMP_TARGET_KIND_OACC_SERIAL: stmt_name = "serial"; break;
3901 386 : case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break;
3902 256 : case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break;
3903 94 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
3904 94 : stmt_name = "enter data"; break;
3905 86 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
3906 86 : stmt_name = "exit data"; break;
3907 64 : case GF_OMP_TARGET_KIND_OACC_DECLARE: stmt_name = "declare"; break;
3908 128 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA: stmt_name = "host_data";
3909 128 : break;
3910 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3911 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3912 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
3913 : /* OpenACC 'kernels' decomposed parts. */
3914 1618 : stmt_name = "kernels"; break;
3915 0 : default: gcc_unreachable ();
3916 : }
3917 7041 : switch (gimple_omp_target_kind (ctx->stmt))
3918 : {
3919 : case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break;
3920 2704 : case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break;
3921 35 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3922 35 : ctx_stmt_name = "parallel"; break;
3923 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
3924 1034 : ctx_stmt_name = "kernels"; break;
3925 35 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
3926 35 : ctx_stmt_name = "serial"; break;
3927 3117 : case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break;
3928 8 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
3929 8 : ctx_stmt_name = "host_data"; break;
3930 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3931 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3932 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
3933 : /* OpenACC 'kernels' decomposed parts. */
3934 1034 : ctx_stmt_name = "kernels"; break;
3935 0 : default: gcc_unreachable ();
3936 : }
3937 :
3938 : /* OpenACC/OpenMP mismatch? */
3939 14082 : if (is_gimple_omp_oacc (stmt)
3940 7041 : != is_gimple_omp_oacc (ctx->stmt))
3941 : {
3942 56 : error_at (gimple_location (stmt),
3943 : "%s %qs construct inside of %s %qs region",
3944 : (is_gimple_omp_oacc (stmt)
3945 : ? "OpenACC" : "OpenMP"), stmt_name,
3946 : (is_gimple_omp_oacc (ctx->stmt)
3947 : ? "OpenACC" : "OpenMP"), ctx_stmt_name);
3948 28 : return false;
3949 : }
3950 7013 : if (is_gimple_omp_offloaded (ctx->stmt))
3951 : {
3952 : /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3953 185 : if (is_gimple_omp_oacc (ctx->stmt))
3954 : {
3955 105 : error_at (gimple_location (stmt),
3956 : "%qs construct inside of %qs region",
3957 : stmt_name, ctx_stmt_name);
3958 105 : return false;
3959 : }
3960 : else
3961 : {
3962 80 : if ((gimple_omp_target_kind (ctx->stmt)
3963 : == GF_OMP_TARGET_KIND_REGION)
3964 80 : && (gimple_omp_target_kind (stmt)
3965 : == GF_OMP_TARGET_KIND_REGION))
3966 : {
3967 58 : c = omp_find_clause (gimple_omp_target_clauses (stmt),
3968 : OMP_CLAUSE_DEVICE);
3969 58 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
3970 : break;
3971 : }
3972 49 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3973 : "%qs construct inside of %qs region",
3974 : stmt_name, ctx_stmt_name);
3975 : }
3976 : }
3977 : }
3978 : break;
3979 : default:
3980 : break;
3981 : }
3982 : return true;
3983 : }
3984 :
3985 :
3986 : /* Helper function scan_omp.
3987 :
3988 : Callback for walk_tree or operators in walk_gimple_stmt used to
3989 : scan for OMP directives in TP. */
3990 :
3991 : static tree
3992 9688013 : scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
3993 : {
3994 9688013 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
3995 9688013 : omp_context *ctx = (omp_context *) wi->info;
3996 9688013 : tree t = *tp;
3997 9688013 : tree tmp;
3998 :
3999 9688013 : switch (TREE_CODE (t))
4000 : {
4001 4098291 : case VAR_DECL:
4002 4098291 : case PARM_DECL:
4003 4098291 : case LABEL_DECL:
4004 4098291 : case RESULT_DECL:
4005 4098291 : if (ctx)
4006 : {
4007 2105872 : tmp = NULL_TREE;
4008 2105872 : if (TREE_CODE (t) == VAR_DECL
4009 3772532 : && (tmp = lookup_attribute ("omp allocate var",
4010 1666660 : DECL_ATTRIBUTES (t))) != NULL_TREE)
4011 69 : t = TREE_VALUE (TREE_VALUE (tmp));
4012 2105872 : tree repl = remap_decl (t, &ctx->cb);
4013 2105872 : gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
4014 2105872 : if (tmp != NULL_TREE && t != repl)
4015 31 : *tp = build_fold_addr_expr (repl);
4016 2105841 : else if (tmp == NULL_TREE)
4017 2105803 : *tp = repl;
4018 : }
4019 : break;
4020 :
4021 193647 : case INDIRECT_REF:
4022 193647 : case MEM_REF:
4023 193647 : if (ctx
4024 77562 : && TREE_CODE (TREE_OPERAND (t, 0)) == VAR_DECL
4025 252195 : && ((tmp = lookup_attribute ("omp allocate var",
4026 58548 : DECL_ATTRIBUTES (TREE_OPERAND (t, 0))))
4027 : != NULL_TREE))
4028 : {
4029 131 : tmp = TREE_VALUE (TREE_VALUE (tmp));
4030 131 : tree repl = remap_decl (tmp, &ctx->cb);
4031 131 : gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
4032 131 : if (tmp != repl)
4033 120 : *tp = repl;
4034 : break;
4035 : }
4036 5589591 : gcc_fallthrough ();
4037 :
4038 5589591 : default:
4039 5589591 : if (ctx && TYPE_P (t))
4040 5 : *tp = remap_type (t, &ctx->cb);
4041 5589586 : else if (!DECL_P (t))
4042 : {
4043 4640554 : *walk_subtrees = 1;
4044 4640554 : if (ctx)
4045 : {
4046 1418515 : tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
4047 1418515 : if (tem != TREE_TYPE (t))
4048 : {
4049 9873 : if (TREE_CODE (t) == INTEGER_CST)
4050 4165 : *tp = wide_int_to_tree (tem, wi::to_wide (t));
4051 : else
4052 5708 : TREE_TYPE (t) = tem;
4053 : }
4054 : }
4055 : }
4056 : break;
4057 : }
4058 :
4059 9688013 : return NULL_TREE;
4060 : }
4061 :
4062 : /* Return true if FNDECL is a setjmp or a longjmp. */
4063 :
4064 : static bool
4065 3042 : setjmp_or_longjmp_p (const_tree fndecl)
4066 : {
4067 3042 : if (fndecl_built_in_p (fndecl, BUILT_IN_SETJMP, BUILT_IN_LONGJMP))
4068 : return true;
4069 :
4070 3042 : tree declname = DECL_NAME (fndecl);
4071 3042 : if (!declname
4072 3042 : || (DECL_CONTEXT (fndecl) != NULL_TREE
4073 2347 : && TREE_CODE (DECL_CONTEXT (fndecl)) != TRANSLATION_UNIT_DECL)
4074 5833 : || !TREE_PUBLIC (fndecl))
4075 : return false;
4076 :
4077 2629 : const char *name = IDENTIFIER_POINTER (declname);
4078 2629 : return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
4079 : }
4080 :
4081 : /* Helper function for scan_omp.
4082 :
4083 : Callback for walk_gimple_stmt used to scan for OMP directives in
4084 : the current statement in GSI. */
4085 :
4086 : static tree
4087 3363375 : scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
4088 : struct walk_stmt_info *wi)
4089 : {
4090 3363375 : gimple *stmt = gsi_stmt (*gsi);
4091 3363375 : omp_context *ctx = (omp_context *) wi->info;
4092 :
4093 3363375 : if (gimple_has_location (stmt))
4094 2744375 : input_location = gimple_location (stmt);
4095 :
4096 : /* Check the nesting restrictions. */
4097 3363375 : bool remove = false;
4098 3363375 : if (is_gimple_omp (stmt))
4099 151972 : remove = !check_omp_nesting_restrictions (stmt, ctx);
4100 3211403 : else if (is_gimple_call (stmt))
4101 : {
4102 229528 : tree fndecl = gimple_call_fndecl (stmt);
4103 229528 : if (fndecl)
4104 : {
4105 213810 : if (ctx
4106 67578 : && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4107 14654 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
4108 3042 : && setjmp_or_longjmp_p (fndecl)
4109 213818 : && !ctx->loop_p)
4110 : {
4111 4 : remove = true;
4112 4 : error_at (gimple_location (stmt),
4113 : "setjmp/longjmp inside %<simd%> construct");
4114 : }
4115 213806 : else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
4116 62738 : switch (DECL_FUNCTION_CODE (fndecl))
4117 : {
4118 4221 : case BUILT_IN_GOMP_BARRIER:
4119 4221 : case BUILT_IN_GOMP_CANCEL:
4120 4221 : case BUILT_IN_GOMP_CANCELLATION_POINT:
4121 4221 : case BUILT_IN_GOMP_TASKYIELD:
4122 4221 : case BUILT_IN_GOMP_TASKWAIT:
4123 4221 : case BUILT_IN_GOMP_TASKGROUP_START:
4124 4221 : case BUILT_IN_GOMP_TASKGROUP_END:
4125 4221 : remove = !check_omp_nesting_restrictions (stmt, ctx);
4126 4221 : break;
4127 : default:
4128 : break;
4129 : }
4130 151068 : else if (ctx)
4131 : {
4132 45702 : omp_context *octx = ctx;
4133 45702 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN && ctx->outer)
4134 45702 : octx = ctx->outer;
4135 45702 : if (octx->order_concurrent && omp_runtime_api_call (fndecl))
4136 : {
4137 240 : remove = true;
4138 240 : error_at (gimple_location (stmt),
4139 : "OpenMP runtime API call %qD in a region with "
4140 : "%<order(concurrent)%> clause", fndecl);
4141 : }
4142 45702 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
4143 3293 : && omp_runtime_api_call (fndecl)
4144 315 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4145 : != strlen ("omp_get_num_teams"))
4146 182 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4147 : "omp_get_num_teams") != 0)
4148 45850 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4149 : != strlen ("omp_get_team_num"))
4150 121 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4151 : "omp_get_team_num") != 0))
4152 : {
4153 27 : remove = true;
4154 27 : error_at (gimple_location (stmt),
4155 : "OpenMP runtime API call %qD strictly nested in a "
4156 : "%<teams%> region", fndecl);
4157 : }
4158 45702 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
4159 13188 : && (gimple_omp_target_kind (ctx->stmt)
4160 : == GF_OMP_TARGET_KIND_REGION)
4161 50505 : && omp_runtime_api_call (fndecl))
4162 : {
4163 365 : tree tgt_clauses = gimple_omp_target_clauses (ctx->stmt);
4164 365 : tree c = omp_find_clause (tgt_clauses, OMP_CLAUSE_DEVICE);
4165 365 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
4166 5 : error_at (gimple_location (stmt),
4167 : "OpenMP runtime API call %qD in a region with "
4168 : "%<device(ancestor)%> clause", fndecl);
4169 : }
4170 : }
4171 : }
4172 : }
4173 201899 : if (remove)
4174 : {
4175 3326 : stmt = gimple_build_nop ();
4176 3326 : gsi_replace (gsi, stmt, false);
4177 : }
4178 :
4179 3363375 : *handled_ops_p = true;
4180 :
4181 3363375 : switch (gimple_code (stmt))
4182 : {
4183 18143 : case GIMPLE_OMP_PARALLEL:
4184 18143 : taskreg_nesting_level++;
4185 18143 : scan_omp_parallel (gsi, ctx);
4186 18143 : taskreg_nesting_level--;
4187 18143 : break;
4188 :
4189 5367 : case GIMPLE_OMP_TASK:
4190 5367 : taskreg_nesting_level++;
4191 5367 : scan_omp_task (gsi, ctx);
4192 5367 : taskreg_nesting_level--;
4193 5367 : break;
4194 :
4195 52715 : case GIMPLE_OMP_FOR:
4196 52715 : if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
4197 : == GF_OMP_FOR_KIND_SIMD)
4198 10810 : && gimple_omp_for_combined_into_p (stmt)
4199 60407 : && gimple_code (ctx->stmt) != GIMPLE_OMP_SCAN)
4200 : {
4201 7526 : tree clauses = gimple_omp_for_clauses (as_a <gomp_for *> (stmt));
4202 7526 : tree c = omp_find_clause (clauses, OMP_CLAUSE_REDUCTION);
4203 7526 : if (c && OMP_CLAUSE_REDUCTION_INSCAN (c) && !seen_error ())
4204 : {
4205 83 : scan_omp_simd_scan (gsi, as_a <gomp_for *> (stmt), ctx);
4206 83 : break;
4207 : }
4208 : }
4209 52632 : if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
4210 : == GF_OMP_FOR_KIND_SIMD)
4211 10727 : && omp_maybe_offloaded_ctx (ctx)
4212 3641 : && omp_max_simt_vf ()
4213 52632 : && gimple_omp_for_collapse (stmt) == 1)
4214 0 : scan_omp_simd (gsi, as_a <gomp_for *> (stmt), ctx);
4215 : else
4216 52632 : scan_omp_for (as_a <gomp_for *> (stmt), ctx);
4217 : break;
4218 :
4219 184 : case GIMPLE_OMP_SCOPE:
4220 184 : ctx = new_omp_context (stmt, ctx);
4221 184 : scan_sharing_clauses (gimple_omp_scope_clauses (stmt), ctx);
4222 184 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4223 184 : break;
4224 :
4225 852 : case GIMPLE_OMP_DISPATCH:
4226 852 : ctx = new_omp_context (stmt, ctx);
4227 852 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4228 852 : break;
4229 :
4230 612 : case GIMPLE_OMP_INTEROP:
4231 612 : ctx = new_omp_context (stmt, ctx);
4232 612 : break;
4233 :
4234 581 : case GIMPLE_OMP_SECTIONS:
4235 581 : scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
4236 581 : break;
4237 :
4238 1228 : case GIMPLE_OMP_SINGLE:
4239 1228 : scan_omp_single (as_a <gomp_single *> (stmt), ctx);
4240 1228 : break;
4241 :
4242 1284 : case GIMPLE_OMP_SCAN:
4243 1284 : if (tree clauses = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)))
4244 : {
4245 606 : if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_INCLUSIVE)
4246 304 : ctx->scan_inclusive = true;
4247 302 : else if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_EXCLUSIVE)
4248 302 : ctx->scan_exclusive = true;
4249 : }
4250 : /* FALLTHRU */
4251 6207 : case GIMPLE_OMP_SECTION:
4252 6207 : case GIMPLE_OMP_STRUCTURED_BLOCK:
4253 6207 : case GIMPLE_OMP_MASTER:
4254 6207 : case GIMPLE_OMP_ORDERED:
4255 6207 : case GIMPLE_OMP_CRITICAL:
4256 6207 : ctx = new_omp_context (stmt, ctx);
4257 6207 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4258 6207 : break;
4259 :
4260 447 : case GIMPLE_OMP_MASKED:
4261 447 : ctx = new_omp_context (stmt, ctx);
4262 447 : scan_sharing_clauses (gimple_omp_masked_clauses (stmt), ctx);
4263 447 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4264 447 : break;
4265 :
4266 607 : case GIMPLE_OMP_TASKGROUP:
4267 607 : ctx = new_omp_context (stmt, ctx);
4268 607 : scan_sharing_clauses (gimple_omp_taskgroup_clauses (stmt), ctx);
4269 607 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4270 607 : break;
4271 :
4272 41209 : case GIMPLE_OMP_TARGET:
4273 41209 : if (is_gimple_omp_offloaded (stmt))
4274 : {
4275 24456 : taskreg_nesting_level++;
4276 24456 : scan_omp_target (as_a <gomp_target *> (stmt), ctx);
4277 24456 : taskreg_nesting_level--;
4278 : }
4279 : else
4280 16753 : scan_omp_target (as_a <gomp_target *> (stmt), ctx);
4281 : break;
4282 :
4283 8736 : case GIMPLE_OMP_TEAMS:
4284 8736 : if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
4285 : {
4286 2618 : taskreg_nesting_level++;
4287 2618 : scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
4288 2618 : taskreg_nesting_level--;
4289 : }
4290 : else
4291 6118 : scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
4292 : break;
4293 :
4294 278358 : case GIMPLE_BIND:
4295 278358 : {
4296 278358 : tree var;
4297 :
4298 278358 : *handled_ops_p = false;
4299 278358 : if (ctx)
4300 151346 : for (var = gimple_bind_vars (as_a <gbind *> (stmt));
4301 571888 : var ;
4302 420542 : var = DECL_CHAIN (var))
4303 420542 : insert_decl_map (&ctx->cb, var, var);
4304 : }
4305 : break;
4306 2948129 : default:
4307 2948129 : *handled_ops_p = false;
4308 2948129 : break;
4309 : }
4310 :
4311 3363375 : return NULL_TREE;
4312 : }
4313 :
4314 :
4315 : /* Scan all the statements starting at the current statement. CTX
4316 : contains context information about the OMP directives and
4317 : clauses found during the scan. */
4318 :
4319 : static void
4320 257461 : scan_omp (gimple_seq *body_p, omp_context *ctx)
4321 : {
4322 257461 : location_t saved_location;
4323 257461 : struct walk_stmt_info wi;
4324 :
4325 257461 : memset (&wi, 0, sizeof (wi));
4326 257461 : wi.info = ctx;
4327 257461 : wi.want_locations = true;
4328 :
4329 257461 : saved_location = input_location;
4330 257461 : walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
4331 257461 : input_location = saved_location;
4332 257461 : }
4333 :
4334 : /* Re-gimplification and code generation routines. */
4335 :
4336 : /* Remove omp_member_access_dummy_var variables from gimple_bind_vars
4337 : of BIND if in a method. */
4338 :
4339 : static void
4340 255676 : maybe_remove_omp_member_access_dummy_vars (gbind *bind)
4341 : {
4342 255676 : if (DECL_ARGUMENTS (current_function_decl)
4343 96973 : && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
4344 265668 : && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
4345 : == POINTER_TYPE))
4346 : {
4347 3520 : tree vars = gimple_bind_vars (bind);
4348 19340 : for (tree *pvar = &vars; *pvar; )
4349 15820 : if (omp_member_access_dummy_var (*pvar))
4350 719 : *pvar = DECL_CHAIN (*pvar);
4351 : else
4352 15101 : pvar = &DECL_CHAIN (*pvar);
4353 3520 : gimple_bind_set_vars (bind, vars);
4354 : }
4355 255676 : }
4356 :
4357 : /* Remove omp_member_access_dummy_var variables from BLOCK_VARS of
4358 : block and its subblocks. */
4359 :
4360 : static void
4361 12919 : remove_member_access_dummy_vars (tree block)
4362 : {
4363 22143 : for (tree *pvar = &BLOCK_VARS (block); *pvar; )
4364 9224 : if (omp_member_access_dummy_var (*pvar))
4365 108 : *pvar = DECL_CHAIN (*pvar);
4366 : else
4367 9116 : pvar = &DECL_CHAIN (*pvar);
4368 :
4369 16781 : for (block = BLOCK_SUBBLOCKS (block); block; block = BLOCK_CHAIN (block))
4370 3862 : remove_member_access_dummy_vars (block);
4371 12919 : }
4372 :
4373 : /* If a context was created for STMT when it was scanned, return it. */
4374 :
4375 : static omp_context *
4376 120520 : maybe_lookup_ctx (gimple *stmt)
4377 : {
4378 120520 : splay_tree_node n;
4379 120520 : n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
4380 120520 : return n ? (omp_context *) n->value : NULL;
4381 : }
4382 :
4383 :
4384 : /* Find the mapping for DECL in CTX or the immediately enclosing
4385 : context that has a mapping for DECL.
4386 :
4387 : If CTX is a nested parallel directive, we may have to use the decl
4388 : mappings created in CTX's parent context. Suppose that we have the
4389 : following parallel nesting (variable UIDs showed for clarity):
4390 :
4391 : iD.1562 = 0;
4392 : #omp parallel shared(iD.1562) -> outer parallel
4393 : iD.1562 = iD.1562 + 1;
4394 :
4395 : #omp parallel shared (iD.1562) -> inner parallel
4396 : iD.1562 = iD.1562 - 1;
4397 :
4398 : Each parallel structure will create a distinct .omp_data_s structure
4399 : for copying iD.1562 in/out of the directive:
4400 :
4401 : outer parallel .omp_data_s.1.i -> iD.1562
4402 : inner parallel .omp_data_s.2.i -> iD.1562
4403 :
4404 : A shared variable mapping will produce a copy-out operation before
4405 : the parallel directive and a copy-in operation after it. So, in
4406 : this case we would have:
4407 :
4408 : iD.1562 = 0;
4409 : .omp_data_o.1.i = iD.1562;
4410 : #omp parallel shared(iD.1562) -> outer parallel
4411 : .omp_data_i.1 = &.omp_data_o.1
4412 : .omp_data_i.1->i = .omp_data_i.1->i + 1;
4413 :
4414 : .omp_data_o.2.i = iD.1562; -> **
4415 : #omp parallel shared(iD.1562) -> inner parallel
4416 : .omp_data_i.2 = &.omp_data_o.2
4417 : .omp_data_i.2->i = .omp_data_i.2->i - 1;
4418 :
4419 :
4420 : ** This is a problem. The symbol iD.1562 cannot be referenced
4421 : inside the body of the outer parallel region. But since we are
4422 : emitting this copy operation while expanding the inner parallel
4423 : directive, we need to access the CTX structure of the outer
4424 : parallel directive to get the correct mapping:
4425 :
4426 : .omp_data_o.2.i = .omp_data_i.1->i
4427 :
4428 : Since there may be other workshare or parallel directives enclosing
4429 : the parallel directive, it may be necessary to walk up the context
4430 : parent chain. This is not a problem in general because nested
4431 : parallelism happens only rarely. */
4432 :
4433 : static tree
4434 125064 : lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4435 : {
4436 125064 : tree t;
4437 125064 : omp_context *up;
4438 :
4439 181766 : for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4440 56702 : t = maybe_lookup_decl (decl, up);
4441 :
4442 125064 : gcc_assert (!ctx->is_nested || t || is_global_var (decl));
4443 :
4444 96746 : return t ? t : decl;
4445 : }
4446 :
4447 :
4448 : /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4449 : in outer contexts. */
4450 :
4451 : static tree
4452 355263 : maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4453 : {
4454 355263 : tree t = NULL;
4455 355263 : omp_context *up;
4456 :
4457 626334 : for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4458 271071 : t = maybe_lookup_decl (decl, up);
4459 :
4460 355263 : return t ? t : decl;
4461 : }
4462 :
4463 :
4464 : /* Construct the initialization value for reduction operation OP. */
4465 :
4466 : tree
4467 13077 : omp_reduction_init_op (location_t loc, enum tree_code op, tree type)
4468 : {
4469 13077 : switch (op)
4470 : {
4471 10742 : case PLUS_EXPR:
4472 10742 : case MINUS_EXPR:
4473 10742 : case BIT_IOR_EXPR:
4474 10742 : case BIT_XOR_EXPR:
4475 10742 : case TRUTH_OR_EXPR:
4476 10742 : case TRUTH_ORIF_EXPR:
4477 10742 : case TRUTH_XOR_EXPR:
4478 10742 : case NE_EXPR:
4479 10742 : return build_zero_cst (type);
4480 :
4481 1431 : case MULT_EXPR:
4482 1431 : case TRUTH_AND_EXPR:
4483 1431 : case TRUTH_ANDIF_EXPR:
4484 1431 : case EQ_EXPR:
4485 1431 : return fold_convert_loc (loc, type, integer_one_node);
4486 :
4487 208 : case BIT_AND_EXPR:
4488 208 : return fold_convert_loc (loc, type, integer_minus_one_node);
4489 :
4490 388 : case MAX_EXPR:
4491 388 : if (SCALAR_FLOAT_TYPE_P (type))
4492 : {
4493 158 : REAL_VALUE_TYPE min;
4494 158 : if (HONOR_INFINITIES (type))
4495 158 : real_arithmetic (&min, NEGATE_EXPR, &dconstinf, NULL);
4496 : else
4497 0 : real_maxval (&min, 1, TYPE_MODE (type));
4498 158 : return build_real (type, min);
4499 : }
4500 230 : else if (POINTER_TYPE_P (type))
4501 : {
4502 2 : wide_int min
4503 2 : = wi::min_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4504 2 : return wide_int_to_tree (type, min);
4505 2 : }
4506 : else
4507 : {
4508 228 : gcc_assert (INTEGRAL_TYPE_P (type));
4509 228 : return TYPE_MIN_VALUE (type);
4510 : }
4511 :
4512 308 : case MIN_EXPR:
4513 308 : if (SCALAR_FLOAT_TYPE_P (type))
4514 : {
4515 108 : REAL_VALUE_TYPE max;
4516 108 : if (HONOR_INFINITIES (type))
4517 108 : max = dconstinf;
4518 : else
4519 0 : real_maxval (&max, 0, TYPE_MODE (type));
4520 108 : return build_real (type, max);
4521 : }
4522 200 : else if (POINTER_TYPE_P (type))
4523 : {
4524 2 : wide_int max
4525 2 : = wi::max_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4526 2 : return wide_int_to_tree (type, max);
4527 2 : }
4528 : else
4529 : {
4530 198 : gcc_assert (INTEGRAL_TYPE_P (type));
4531 198 : return TYPE_MAX_VALUE (type);
4532 : }
4533 :
4534 0 : default:
4535 0 : gcc_unreachable ();
4536 : }
4537 : }
4538 :
4539 : /* Construct the initialization value for reduction CLAUSE. */
4540 :
4541 : tree
4542 10456 : omp_reduction_init (tree clause, tree type)
4543 : {
4544 10456 : return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause),
4545 10456 : OMP_CLAUSE_REDUCTION_CODE (clause), type);
4546 : }
4547 :
4548 : /* Return alignment to be assumed for var in CLAUSE, which should be
4549 : OMP_CLAUSE_ALIGNED. */
4550 :
4551 : static tree
4552 140 : omp_clause_aligned_alignment (tree clause)
4553 : {
4554 140 : if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
4555 123 : return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
4556 :
4557 : /* Otherwise return implementation defined alignment. */
4558 17 : unsigned int al = 1;
4559 17 : opt_scalar_mode mode_iter;
4560 17 : auto_vector_modes modes;
4561 17 : targetm.vectorize.autovectorize_vector_modes (&modes, true);
4562 17 : static enum mode_class classes[]
4563 : = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
4564 51 : for (int i = 0; i < 4; i += 2)
4565 : /* The for loop above dictates that we only walk through scalar classes. */
4566 255 : FOR_EACH_MODE_IN_CLASS (mode_iter, classes[i])
4567 : {
4568 221 : scalar_mode mode = mode_iter.require ();
4569 221 : machine_mode vmode = targetm.vectorize.preferred_simd_mode (mode);
4570 221 : if (GET_MODE_CLASS (vmode) != classes[i + 1])
4571 323 : continue;
4572 : machine_mode alt_vmode;
4573 476 : for (unsigned int j = 0; j < modes.length (); ++j)
4574 527 : if (related_vector_mode (modes[j], mode).exists (&alt_vmode)
4575 578 : && known_ge (GET_MODE_SIZE (alt_vmode), GET_MODE_SIZE (vmode)))
4576 119 : vmode = alt_vmode;
4577 :
4578 119 : tree type = lang_hooks.types.type_for_mode (mode, 1);
4579 119 : if (type == NULL_TREE || TYPE_MODE (type) != mode)
4580 17 : continue;
4581 102 : type = build_vector_type_for_mode (type, vmode);
4582 102 : if (TYPE_MODE (type) != vmode)
4583 0 : continue;
4584 102 : if (TYPE_ALIGN_UNIT (type) > al)
4585 17 : al = TYPE_ALIGN_UNIT (type);
4586 : }
4587 17 : return build_int_cst (integer_type_node, al);
4588 17 : }
4589 :
4590 :
4591 : /* This structure is part of the interface between lower_rec_simd_input_clauses
4592 : and lower_rec_input_clauses. */
4593 :
4594 : class omplow_simd_context {
4595 : public:
4596 77099 : omplow_simd_context () { memset (this, 0, sizeof (*this)); }
4597 : tree idx;
4598 : tree lane;
4599 : tree lastlane;
4600 : vec<tree, va_heap> simt_eargs;
4601 : gimple_seq simt_dlist;
4602 : poly_uint64 max_vf;
4603 : bool is_simt;
4604 : };
4605 :
4606 : /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4607 : privatization. */
4608 :
4609 : static bool
4610 10228 : lower_rec_simd_input_clauses (tree new_var, omp_context *ctx,
4611 : omplow_simd_context *sctx, tree &ivar,
4612 : tree &lvar, tree *rvar = NULL,
4613 : tree *rvar2 = NULL)
4614 : {
4615 10228 : if (known_eq (sctx->max_vf, 0U))
4616 : {
4617 4790 : sctx->max_vf = (sctx->is_simt ? omp_max_simt_vf ()
4618 4790 : : omp_max_vf (omp_maybe_offloaded_ctx (ctx)));
4619 4790 : if (maybe_gt (sctx->max_vf, 1U))
4620 : {
4621 3459 : tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
4622 : OMP_CLAUSE_SAFELEN);
4623 3459 : if (c)
4624 : {
4625 89 : poly_uint64 safe_len;
4626 89 : if (!poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
4627 89 : || maybe_lt (safe_len, 1U))
4628 6 : sctx->max_vf = 1;
4629 : else
4630 83 : sctx->max_vf = lower_bound (sctx->max_vf, safe_len);
4631 : }
4632 : }
4633 4790 : if (sctx->is_simt && !known_eq (sctx->max_vf, 1U))
4634 : {
4635 0 : for (tree c = gimple_omp_for_clauses (ctx->stmt); c;
4636 0 : c = OMP_CLAUSE_CHAIN (c))
4637 : {
4638 0 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
4639 0 : continue;
4640 :
4641 0 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4642 : {
4643 : /* UDR reductions are not supported yet for SIMT, disable
4644 : SIMT. */
4645 0 : sctx->max_vf = 1;
4646 0 : break;
4647 : }
4648 :
4649 0 : if (truth_value_p (OMP_CLAUSE_REDUCTION_CODE (c))
4650 0 : && !INTEGRAL_TYPE_P (TREE_TYPE (new_var)))
4651 : {
4652 : /* Doing boolean operations on non-integral types is
4653 : for conformance only, it's not worth supporting this
4654 : for SIMT. */
4655 0 : sctx->max_vf = 1;
4656 0 : break;
4657 : }
4658 : }
4659 : }
4660 4790 : if (maybe_gt (sctx->max_vf, 1U))
4661 : {
4662 3440 : sctx->idx = create_tmp_var (unsigned_type_node);
4663 3440 : sctx->lane = create_tmp_var (unsigned_type_node);
4664 : }
4665 : }
4666 10228 : if (known_eq (sctx->max_vf, 1U))
4667 : return false;
4668 :
4669 7189 : if (sctx->is_simt)
4670 : {
4671 0 : if (is_gimple_reg (new_var))
4672 : {
4673 0 : ivar = lvar = new_var;
4674 0 : return true;
4675 : }
4676 0 : tree type = TREE_TYPE (new_var), ptype = build_pointer_type (type);
4677 0 : ivar = lvar = create_tmp_var (type);
4678 0 : TREE_ADDRESSABLE (ivar) = 1;
4679 0 : DECL_ATTRIBUTES (ivar) = tree_cons (get_identifier ("omp simt private"),
4680 0 : NULL, DECL_ATTRIBUTES (ivar));
4681 0 : sctx->simt_eargs.safe_push (build1 (ADDR_EXPR, ptype, ivar));
4682 0 : tree clobber = build_clobber (type);
4683 0 : gimple *g = gimple_build_assign (ivar, clobber);
4684 0 : gimple_seq_add_stmt (&sctx->simt_dlist, g);
4685 : }
4686 : else
4687 : {
4688 7189 : tree atype = build_array_type_nelts (TREE_TYPE (new_var), sctx->max_vf);
4689 7189 : tree avar = create_tmp_var_raw (atype);
4690 7189 : if (TREE_ADDRESSABLE (new_var))
4691 912 : TREE_ADDRESSABLE (avar) = 1;
4692 7189 : DECL_ATTRIBUTES (avar)
4693 7189 : = tree_cons (get_identifier ("omp simd array"), NULL,
4694 7189 : DECL_ATTRIBUTES (avar));
4695 7189 : gimple_add_tmp_var (avar);
4696 7189 : tree iavar = avar;
4697 7189 : if (rvar && !ctx->for_simd_scan_phase)
4698 : {
4699 : /* For inscan reductions, create another array temporary,
4700 : which will hold the reduced value. */
4701 232 : iavar = create_tmp_var_raw (atype);
4702 232 : if (TREE_ADDRESSABLE (new_var))
4703 33 : TREE_ADDRESSABLE (iavar) = 1;
4704 232 : DECL_ATTRIBUTES (iavar)
4705 232 : = tree_cons (get_identifier ("omp simd array"), NULL,
4706 : tree_cons (get_identifier ("omp simd inscan"), NULL,
4707 232 : DECL_ATTRIBUTES (iavar)));
4708 232 : gimple_add_tmp_var (iavar);
4709 232 : ctx->cb.decl_map->put (avar, iavar);
4710 232 : if (sctx->lastlane == NULL_TREE)
4711 184 : sctx->lastlane = create_tmp_var (unsigned_type_node);
4712 232 : *rvar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar,
4713 : sctx->lastlane, NULL_TREE, NULL_TREE);
4714 232 : TREE_THIS_NOTRAP (*rvar) = 1;
4715 :
4716 232 : if (ctx->scan_exclusive)
4717 : {
4718 : /* And for exclusive scan yet another one, which will
4719 : hold the value during the scan phase. */
4720 114 : tree savar = create_tmp_var_raw (atype);
4721 114 : if (TREE_ADDRESSABLE (new_var))
4722 17 : TREE_ADDRESSABLE (savar) = 1;
4723 114 : DECL_ATTRIBUTES (savar)
4724 114 : = tree_cons (get_identifier ("omp simd array"), NULL,
4725 : tree_cons (get_identifier ("omp simd inscan "
4726 : "exclusive"), NULL,
4727 114 : DECL_ATTRIBUTES (savar)));
4728 114 : gimple_add_tmp_var (savar);
4729 114 : ctx->cb.decl_map->put (iavar, savar);
4730 114 : *rvar2 = build4 (ARRAY_REF, TREE_TYPE (new_var), savar,
4731 : sctx->idx, NULL_TREE, NULL_TREE);
4732 114 : TREE_THIS_NOTRAP (*rvar2) = 1;
4733 : }
4734 : }
4735 7189 : ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar, sctx->idx,
4736 : NULL_TREE, NULL_TREE);
4737 7189 : lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, sctx->lane,
4738 : NULL_TREE, NULL_TREE);
4739 7189 : TREE_THIS_NOTRAP (ivar) = 1;
4740 7189 : TREE_THIS_NOTRAP (lvar) = 1;
4741 : }
4742 7189 : if (DECL_P (new_var))
4743 : {
4744 7035 : SET_DECL_VALUE_EXPR (new_var, lvar);
4745 7035 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4746 : }
4747 : return true;
4748 : }
4749 :
4750 : /* Helper function of lower_rec_input_clauses. For a reference
4751 : in simd reduction, add an underlying variable it will reference. */
4752 :
4753 : static void
4754 64 : handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
4755 : {
4756 64 : tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
4757 64 : if (TREE_CONSTANT (z))
4758 : {
4759 64 : z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)),
4760 : get_name (new_vard));
4761 64 : gimple_add_tmp_var (z);
4762 64 : TREE_ADDRESSABLE (z) = 1;
4763 64 : z = build_fold_addr_expr_loc (loc, z);
4764 64 : gimplify_assign (new_vard, z, ilist);
4765 : }
4766 64 : }
4767 :
4768 : /* Helper function for lower_rec_input_clauses. Emit into ilist sequence
4769 : code to emit (type) (tskred_temp[idx]). */
4770 :
4771 : static tree
4772 1119 : task_reduction_read (gimple_seq *ilist, tree tskred_temp, tree type,
4773 : unsigned idx)
4774 : {
4775 1119 : unsigned HOST_WIDE_INT sz
4776 1119 : = tree_to_uhwi (TYPE_SIZE_UNIT (pointer_sized_int_node));
4777 1119 : tree r = build2 (MEM_REF, pointer_sized_int_node,
4778 1119 : tskred_temp, build_int_cst (TREE_TYPE (tskred_temp),
4779 1119 : idx * sz));
4780 1119 : tree v = create_tmp_var (pointer_sized_int_node);
4781 1119 : gimple *g = gimple_build_assign (v, r);
4782 1119 : gimple_seq_add_stmt (ilist, g);
4783 1119 : if (!useless_type_conversion_p (type, pointer_sized_int_node))
4784 : {
4785 871 : v = create_tmp_var (type);
4786 871 : g = gimple_build_assign (v, NOP_EXPR, gimple_assign_lhs (g));
4787 871 : gimple_seq_add_stmt (ilist, g);
4788 : }
4789 1119 : return v;
4790 : }
4791 :
4792 : /* Lower early initialization of privatized variable NEW_VAR
4793 : if it needs an allocator (has allocate clause). */
4794 :
4795 : static bool
4796 144144 : lower_private_allocate (tree var, tree new_var, tree &allocator,
4797 : tree &allocate_ptr, gimple_seq *ilist,
4798 : omp_context *ctx, bool is_ref, tree size)
4799 : {
4800 144144 : if (allocator)
4801 : return false;
4802 144050 : gcc_assert (allocate_ptr == NULL_TREE);
4803 144050 : if (ctx->allocate_map
4804 3082 : && (DECL_P (new_var) || (TYPE_P (new_var) && size)))
4805 3079 : if (tree *allocatorp = ctx->allocate_map->get (var))
4806 1140 : allocator = *allocatorp;
4807 144050 : if (allocator == NULL_TREE)
4808 : return false;
4809 1140 : if (!is_ref && omp_privatize_by_reference (var))
4810 : {
4811 0 : allocator = NULL_TREE;
4812 0 : return false;
4813 : }
4814 :
4815 1140 : unsigned HOST_WIDE_INT ialign = 0;
4816 1140 : if (TREE_CODE (allocator) == TREE_LIST)
4817 : {
4818 236 : ialign = tree_to_uhwi (TREE_VALUE (allocator));
4819 236 : allocator = TREE_PURPOSE (allocator);
4820 : }
4821 1140 : if (TREE_CODE (allocator) != INTEGER_CST)
4822 428 : allocator = build_outer_var_ref (allocator, ctx, OMP_CLAUSE_ALLOCATE);
4823 1140 : allocator = fold_convert (pointer_sized_int_node, allocator);
4824 1140 : if (TREE_CODE (allocator) != INTEGER_CST)
4825 : {
4826 428 : tree var = create_tmp_var (TREE_TYPE (allocator));
4827 428 : gimplify_assign (var, allocator, ilist);
4828 428 : allocator = var;
4829 : }
4830 :
4831 1140 : tree ptr_type, align, sz = size;
4832 1140 : if (TYPE_P (new_var))
4833 : {
4834 225 : ptr_type = build_pointer_type (new_var);
4835 225 : ialign = MAX (ialign, TYPE_ALIGN_UNIT (new_var));
4836 : }
4837 915 : else if (is_ref)
4838 : {
4839 114 : ptr_type = build_pointer_type (TREE_TYPE (TREE_TYPE (new_var)));
4840 114 : ialign = MAX (ialign, TYPE_ALIGN_UNIT (TREE_TYPE (ptr_type)));
4841 : }
4842 : else
4843 : {
4844 801 : ptr_type = build_pointer_type (TREE_TYPE (new_var));
4845 801 : ialign = MAX (ialign, DECL_ALIGN_UNIT (new_var));
4846 801 : if (sz == NULL_TREE)
4847 703 : sz = fold_convert (size_type_node, DECL_SIZE_UNIT (new_var));
4848 : }
4849 1140 : align = build_int_cst (size_type_node, ialign);
4850 1140 : if (TREE_CODE (sz) != INTEGER_CST)
4851 : {
4852 54 : tree szvar = create_tmp_var (size_type_node);
4853 54 : gimplify_assign (szvar, sz, ilist);
4854 54 : sz = szvar;
4855 : }
4856 1140 : allocate_ptr = create_tmp_var (ptr_type);
4857 1140 : tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
4858 1140 : gimple *g = gimple_build_call (a, 3, align, sz, allocator);
4859 1140 : gimple_call_set_lhs (g, allocate_ptr);
4860 1140 : gimple_seq_add_stmt (ilist, g);
4861 1140 : if (!is_ref)
4862 : {
4863 801 : tree x = build_simple_mem_ref (allocate_ptr);
4864 801 : TREE_THIS_NOTRAP (x) = 1;
4865 801 : SET_DECL_VALUE_EXPR (new_var, x);
4866 801 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4867 : }
4868 : return true;
4869 : }
4870 :
4871 : /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4872 : from the receiver (aka child) side and initializers for REFERENCE_TYPE
4873 : private variables. Initialization statements go in ILIST, while calls
4874 : to destructors go in DLIST. */
4875 :
4876 : static void
4877 77099 : lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
4878 : omp_context *ctx, struct omp_for_data *fd)
4879 : {
4880 77099 : tree c, copyin_seq, x, ptr;
4881 77099 : bool copyin_by_ref = false;
4882 77099 : bool lastprivate_firstprivate = false;
4883 77099 : bool reduction_omp_orig_ref = false;
4884 77099 : int pass;
4885 77099 : bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4886 77099 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
4887 77099 : omplow_simd_context sctx = omplow_simd_context ();
4888 77099 : tree simt_lane = NULL_TREE, simtrec = NULL_TREE;
4889 77099 : tree ivar = NULL_TREE, lvar = NULL_TREE, uid = NULL_TREE;
4890 77099 : gimple_seq llist[4] = { };
4891 77099 : tree nonconst_simd_if = NULL_TREE;
4892 :
4893 77099 : copyin_seq = NULL;
4894 77099 : sctx.is_simt = is_simd && omp_find_clause (clauses, OMP_CLAUSE__SIMT_);
4895 :
4896 : /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4897 : with data sharing clauses referencing variable sized vars. That
4898 : is unnecessarily hard to support and very unlikely to result in
4899 : vectorized code anyway. */
4900 9355 : if (is_simd)
4901 61463 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4902 52108 : switch (OMP_CLAUSE_CODE (c))
4903 : {
4904 7579 : case OMP_CLAUSE_LINEAR:
4905 7579 : if (OMP_CLAUSE_LINEAR_ARRAY (c))
4906 216 : sctx.max_vf = 1;
4907 : /* FALLTHRU */
4908 25155 : case OMP_CLAUSE_PRIVATE:
4909 25155 : case OMP_CLAUSE_FIRSTPRIVATE:
4910 25155 : case OMP_CLAUSE_LASTPRIVATE:
4911 25155 : if (is_variable_sized (OMP_CLAUSE_DECL (c)))
4912 0 : sctx.max_vf = 1;
4913 25155 : else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
4914 : {
4915 279 : tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
4916 279 : if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
4917 96 : sctx.max_vf = 1;
4918 : }
4919 : break;
4920 2578 : case OMP_CLAUSE_REDUCTION:
4921 2578 : case OMP_CLAUSE_IN_REDUCTION:
4922 2578 : if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF
4923 2578 : || is_variable_sized (OMP_CLAUSE_DECL (c)))
4924 180 : sctx.max_vf = 1;
4925 2398 : else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
4926 : {
4927 144 : tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
4928 144 : if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
4929 0 : sctx.max_vf = 1;
4930 : }
4931 : break;
4932 759 : case OMP_CLAUSE_IF:
4933 759 : if (integer_zerop (OMP_CLAUSE_IF_EXPR (c)))
4934 124 : sctx.max_vf = 1;
4935 635 : else if (TREE_CODE (OMP_CLAUSE_IF_EXPR (c)) != INTEGER_CST)
4936 625 : nonconst_simd_if = OMP_CLAUSE_IF_EXPR (c);
4937 : break;
4938 626 : case OMP_CLAUSE_SIMDLEN:
4939 626 : if (integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (c)))
4940 110 : sctx.max_vf = 1;
4941 : break;
4942 184 : case OMP_CLAUSE__CONDTEMP_:
4943 : /* FIXME: lastprivate(conditional:) not handled for SIMT yet. */
4944 184 : if (sctx.is_simt)
4945 0 : sctx.max_vf = 1;
4946 : break;
4947 22806 : default:
4948 22806 : continue;
4949 22806 : }
4950 :
4951 : /* Add a placeholder for simduid. */
4952 77099 : if (sctx.is_simt && maybe_ne (sctx.max_vf, 1U))
4953 0 : sctx.simt_eargs.safe_push (NULL_TREE);
4954 :
4955 : unsigned task_reduction_cnt = 0;
4956 : unsigned task_reduction_cntorig = 0;
4957 : unsigned task_reduction_cnt_full = 0;
4958 : unsigned task_reduction_cntorig_full = 0;
4959 : unsigned task_reduction_other_cnt = 0;
4960 234350 : tree tskred_atype = NULL_TREE, tskred_avar = NULL_TREE;
4961 : tree tskred_base = NULL_TREE, tskred_temp = NULL_TREE;
4962 : /* Do all the fixed sized types in the first pass, and the variable sized
4963 : types in the second pass. This makes sure that the scalar arguments to
4964 : the variable sized types are processed before we use them in the
4965 : variable sized operations. For task reductions we use 4 passes, in the
4966 : first two we ignore them, in the third one gather arguments for
4967 : GOMP_task_reduction_remap call and in the last pass actually handle
4968 : the task reductions. */
4969 391601 : for (pass = 0; pass < ((task_reduction_cnt || task_reduction_other_cnt)
4970 234350 : ? 4 : 2); ++pass)
4971 : {
4972 157255 : if (pass == 2 && task_reduction_cnt)
4973 : {
4974 871 : tskred_atype
4975 871 : = build_array_type_nelts (ptr_type_node, task_reduction_cnt
4976 871 : + task_reduction_cntorig);
4977 871 : tskred_avar = create_tmp_var_raw (tskred_atype);
4978 871 : gimple_add_tmp_var (tskred_avar);
4979 871 : TREE_ADDRESSABLE (tskred_avar) = 1;
4980 871 : task_reduction_cnt_full = task_reduction_cnt;
4981 871 : task_reduction_cntorig_full = task_reduction_cntorig;
4982 : }
4983 156384 : else if (pass == 3 && task_reduction_cnt)
4984 : {
4985 871 : x = builtin_decl_explicit (BUILT_IN_GOMP_TASK_REDUCTION_REMAP);
4986 871 : gimple *g
4987 871 : = gimple_build_call (x, 3, size_int (task_reduction_cnt),
4988 871 : size_int (task_reduction_cntorig),
4989 : build_fold_addr_expr (tskred_avar));
4990 871 : gimple_seq_add_stmt (ilist, g);
4991 : }
4992 157255 : if (pass == 3 && task_reduction_other_cnt)
4993 : {
4994 : /* For reduction clauses, build
4995 : tskred_base = (void *) tskred_temp[2]
4996 : + omp_get_thread_num () * tskred_temp[1]
4997 : or if tskred_temp[1] is known to be constant, that constant
4998 : directly. This is the start of the private reduction copy block
4999 : for the current thread. */
5000 837 : tree v = create_tmp_var (integer_type_node);
5001 837 : x = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5002 837 : gimple *g = gimple_build_call (x, 0);
5003 837 : gimple_call_set_lhs (g, v);
5004 837 : gimple_seq_add_stmt (ilist, g);
5005 837 : c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
5006 837 : tskred_temp = OMP_CLAUSE_DECL (c);
5007 837 : if (is_taskreg_ctx (ctx))
5008 556 : tskred_temp = lookup_decl (tskred_temp, ctx);
5009 837 : tree v2 = create_tmp_var (sizetype);
5010 837 : g = gimple_build_assign (v2, NOP_EXPR, v);
5011 837 : gimple_seq_add_stmt (ilist, g);
5012 837 : if (ctx->task_reductions[0])
5013 820 : v = fold_convert (sizetype, ctx->task_reductions[0]);
5014 : else
5015 17 : v = task_reduction_read (ilist, tskred_temp, sizetype, 1);
5016 837 : tree v3 = create_tmp_var (sizetype);
5017 837 : g = gimple_build_assign (v3, MULT_EXPR, v2, v);
5018 837 : gimple_seq_add_stmt (ilist, g);
5019 837 : v = task_reduction_read (ilist, tskred_temp, ptr_type_node, 2);
5020 837 : tskred_base = create_tmp_var (ptr_type_node);
5021 837 : g = gimple_build_assign (tskred_base, POINTER_PLUS_EXPR, v, v3);
5022 837 : gimple_seq_add_stmt (ilist, g);
5023 : }
5024 157255 : task_reduction_cnt = 0;
5025 157255 : task_reduction_cntorig = 0;
5026 157255 : task_reduction_other_cnt = 0;
5027 761050 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
5028 : {
5029 603799 : enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
5030 603799 : tree var, new_var;
5031 603799 : bool by_ref;
5032 603799 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
5033 603799 : bool task_reduction_p = false;
5034 603799 : bool task_reduction_needs_orig_p = false;
5035 603799 : tree cond = NULL_TREE;
5036 603799 : tree allocator, allocate_ptr;
5037 :
5038 603799 : switch (c_kind)
5039 : {
5040 135972 : case OMP_CLAUSE_PRIVATE:
5041 135972 : if (OMP_CLAUSE_PRIVATE_DEBUG (c))
5042 435472 : continue;
5043 : break;
5044 60004 : case OMP_CLAUSE_SHARED:
5045 : /* Ignore shared directives in teams construct inside
5046 : of target construct. */
5047 60004 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
5048 60004 : && !is_host_teams_ctx (ctx))
5049 23496 : continue;
5050 36508 : if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
5051 : {
5052 12068 : gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)
5053 : || is_global_var (OMP_CLAUSE_DECL (c)));
5054 12068 : continue;
5055 : }
5056 : case OMP_CLAUSE_FIRSTPRIVATE:
5057 : case OMP_CLAUSE_COPYIN:
5058 : break;
5059 15612 : case OMP_CLAUSE_LINEAR:
5060 15612 : if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)
5061 15612 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
5062 : lastprivate_firstprivate = true;
5063 : break;
5064 37154 : case OMP_CLAUSE_REDUCTION:
5065 37154 : case OMP_CLAUSE_IN_REDUCTION:
5066 37154 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
5067 28906 : || is_task_ctx (ctx)
5068 63720 : || OMP_CLAUSE_REDUCTION_TASK (c))
5069 : {
5070 12776 : task_reduction_p = true;
5071 12776 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
5072 : {
5073 4528 : task_reduction_other_cnt++;
5074 4528 : if (pass == 2)
5075 1132 : continue;
5076 : }
5077 : else
5078 8248 : task_reduction_cnt++;
5079 11644 : if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5080 : {
5081 696 : var = OMP_CLAUSE_DECL (c);
5082 : /* If var is a global variable that isn't privatized
5083 : in outer contexts, we don't need to look up the
5084 : original address, it is always the address of the
5085 : global variable itself. */
5086 696 : if (!DECL_P (var)
5087 272 : || omp_privatize_by_reference (var)
5088 923 : || !is_global_var
5089 227 : (maybe_lookup_decl_in_outer_ctx (var, ctx)))
5090 : {
5091 594 : task_reduction_needs_orig_p = true;
5092 594 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5093 492 : task_reduction_cntorig++;
5094 : }
5095 : }
5096 : }
5097 24378 : else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5098 357346 : reduction_omp_orig_ref = true;
5099 : break;
5100 3348 : case OMP_CLAUSE__REDUCTEMP_:
5101 3348 : if (!is_taskreg_ctx (ctx))
5102 1124 : continue;
5103 : /* FALLTHRU */
5104 111452 : case OMP_CLAUSE__LOOPTEMP_:
5105 : /* Handle _looptemp_/_reductemp_ clauses only on
5106 : parallel/task. */
5107 111452 : if (fd)
5108 69210 : continue;
5109 : break;
5110 43702 : case OMP_CLAUSE_LASTPRIVATE:
5111 43702 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
5112 : {
5113 2058 : lastprivate_firstprivate = true;
5114 2058 : if (pass != 0 || is_taskloop_ctx (ctx))
5115 1343 : continue;
5116 : }
5117 : /* Even without corresponding firstprivate, if
5118 : decl is Fortran allocatable, it needs outer var
5119 : reference. */
5120 41644 : else if (pass == 0
5121 62430 : && lang_hooks.decls.omp_private_outer_ref
5122 20786 : (OMP_CLAUSE_DECL (c)))
5123 : lastprivate_firstprivate = true;
5124 : break;
5125 280 : case OMP_CLAUSE_ALIGNED:
5126 280 : if (pass != 1)
5127 140 : continue;
5128 140 : var = OMP_CLAUSE_DECL (c);
5129 140 : if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
5130 140 : && !is_global_var (var))
5131 : {
5132 76 : new_var = maybe_lookup_decl (var, ctx);
5133 76 : if (new_var == NULL_TREE)
5134 6 : new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
5135 76 : x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
5136 76 : tree alarg = omp_clause_aligned_alignment (c);
5137 76 : alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
5138 76 : x = build_call_expr_loc (clause_loc, x, 2, new_var, alarg);
5139 76 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5140 76 : x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
5141 76 : gimplify_and_add (x, ilist);
5142 : }
5143 64 : else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
5144 64 : && is_global_var (var))
5145 : {
5146 64 : tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
5147 64 : new_var = lookup_decl (var, ctx);
5148 64 : t = maybe_lookup_decl_in_outer_ctx (var, ctx);
5149 64 : t = build_fold_addr_expr_loc (clause_loc, t);
5150 64 : t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
5151 64 : tree alarg = omp_clause_aligned_alignment (c);
5152 64 : alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
5153 64 : t = build_call_expr_loc (clause_loc, t2, 2, t, alarg);
5154 64 : t = fold_convert_loc (clause_loc, ptype, t);
5155 64 : x = create_tmp_var (ptype);
5156 64 : t = build2 (MODIFY_EXPR, ptype, x, t);
5157 64 : gimplify_and_add (t, ilist);
5158 64 : t = build_simple_mem_ref_loc (clause_loc, x);
5159 64 : SET_DECL_VALUE_EXPR (new_var, t);
5160 64 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5161 : }
5162 140 : continue;
5163 1514 : case OMP_CLAUSE__CONDTEMP_:
5164 1514 : if (is_parallel_ctx (ctx)
5165 1514 : || (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c)))
5166 : break;
5167 1094 : continue;
5168 135784 : default:
5169 135784 : continue;
5170 137018 : }
5171 :
5172 357346 : if (task_reduction_p != (pass >= 2))
5173 15612 : continue;
5174 :
5175 341734 : allocator = NULL_TREE;
5176 341734 : allocate_ptr = NULL_TREE;
5177 341734 : new_var = var = OMP_CLAUSE_DECL (c);
5178 341734 : if ((c_kind == OMP_CLAUSE_REDUCTION
5179 341734 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5180 29580 : && TREE_CODE (var) == MEM_REF)
5181 : {
5182 4356 : var = TREE_OPERAND (var, 0);
5183 4356 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
5184 773 : var = TREE_OPERAND (var, 0);
5185 4356 : if (TREE_CODE (var) == INDIRECT_REF
5186 4168 : || TREE_CODE (var) == ADDR_EXPR)
5187 2506 : var = TREE_OPERAND (var, 0);
5188 4356 : if (is_variable_sized (var))
5189 : {
5190 179 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
5191 179 : var = DECL_VALUE_EXPR (var);
5192 179 : gcc_assert (TREE_CODE (var) == INDIRECT_REF);
5193 179 : var = TREE_OPERAND (var, 0);
5194 179 : gcc_assert (DECL_P (var));
5195 : }
5196 4356 : new_var = var;
5197 : }
5198 29580 : if (c_kind == OMP_CLAUSE_IN_REDUCTION && is_omp_target (ctx->stmt))
5199 : {
5200 844 : splay_tree_key key = (splay_tree_key) &DECL_CONTEXT (var);
5201 844 : new_var = (tree) splay_tree_lookup (ctx->field_map, key)->value;
5202 : }
5203 340890 : else if (c_kind != OMP_CLAUSE_COPYIN)
5204 339870 : new_var = lookup_decl (var, ctx);
5205 :
5206 341734 : if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
5207 : {
5208 24784 : if (pass != 0)
5209 12391 : continue;
5210 : }
5211 : /* C/C++ array section reductions. */
5212 316950 : else if ((c_kind == OMP_CLAUSE_REDUCTION
5213 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5214 316950 : && var != OMP_CLAUSE_DECL (c))
5215 : {
5216 4356 : if (pass == 0)
5217 873 : continue;
5218 :
5219 3483 : tree bias = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
5220 3483 : tree orig_var = TREE_OPERAND (OMP_CLAUSE_DECL (c), 0);
5221 :
5222 3483 : if (TREE_CODE (orig_var) == POINTER_PLUS_EXPR)
5223 : {
5224 723 : tree b = TREE_OPERAND (orig_var, 1);
5225 723 : if (is_omp_target (ctx->stmt))
5226 : b = NULL_TREE;
5227 : else
5228 723 : b = maybe_lookup_decl (b, ctx);
5229 723 : if (b == NULL)
5230 : {
5231 17 : b = TREE_OPERAND (orig_var, 1);
5232 17 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
5233 : }
5234 723 : if (integer_zerop (bias))
5235 : bias = b;
5236 : else
5237 : {
5238 0 : bias = fold_convert_loc (clause_loc,
5239 0 : TREE_TYPE (b), bias);
5240 0 : bias = fold_build2_loc (clause_loc, PLUS_EXPR,
5241 0 : TREE_TYPE (b), b, bias);
5242 : }
5243 723 : orig_var = TREE_OPERAND (orig_var, 0);
5244 : }
5245 3483 : if (pass == 2)
5246 : {
5247 1149 : tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
5248 1149 : if (is_global_var (out)
5249 241 : && TREE_CODE (TREE_TYPE (out)) != POINTER_TYPE
5250 1336 : && (TREE_CODE (TREE_TYPE (out)) != REFERENCE_TYPE
5251 0 : || (TREE_CODE (TREE_TYPE (TREE_TYPE (out)))
5252 : != POINTER_TYPE)))
5253 : x = var;
5254 962 : else if (is_omp_target (ctx->stmt))
5255 : x = out;
5256 : else
5257 : {
5258 866 : bool by_ref = use_pointer_for_field (var, NULL);
5259 866 : x = build_receiver_ref (var, by_ref, ctx);
5260 866 : if (TREE_CODE (TREE_TYPE (var)) == REFERENCE_TYPE
5261 866 : && (TREE_CODE (TREE_TYPE (TREE_TYPE (var)))
5262 : == POINTER_TYPE))
5263 32 : x = build_fold_addr_expr (x);
5264 : }
5265 1149 : if (TREE_CODE (orig_var) == INDIRECT_REF)
5266 40 : x = build_simple_mem_ref (x);
5267 1109 : else if (TREE_CODE (orig_var) == ADDR_EXPR)
5268 : {
5269 646 : if (var == TREE_OPERAND (orig_var, 0))
5270 583 : x = build_fold_addr_expr (x);
5271 : }
5272 1149 : bias = fold_convert (sizetype, bias);
5273 1149 : x = fold_convert (ptr_type_node, x);
5274 1149 : x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
5275 1149 : TREE_TYPE (x), x, bias);
5276 1149 : unsigned cnt = task_reduction_cnt - 1;
5277 1149 : if (!task_reduction_needs_orig_p)
5278 1061 : cnt += (task_reduction_cntorig_full
5279 1061 : - task_reduction_cntorig);
5280 : else
5281 88 : cnt = task_reduction_cntorig - 1;
5282 1149 : tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5283 1149 : size_int (cnt), NULL_TREE, NULL_TREE);
5284 1149 : gimplify_assign (r, x, ilist);
5285 1149 : continue;
5286 1149 : }
5287 :
5288 2334 : if (TREE_CODE (orig_var) == INDIRECT_REF
5289 2240 : || TREE_CODE (orig_var) == ADDR_EXPR)
5290 1362 : orig_var = TREE_OPERAND (orig_var, 0);
5291 2334 : tree d = OMP_CLAUSE_DECL (c);
5292 2334 : tree type = TREE_TYPE (d);
5293 2334 : gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
5294 2334 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
5295 2334 : tree sz = v;
5296 2334 : const char *name = get_name (orig_var);
5297 2334 : if (pass != 3 && !TREE_CONSTANT (v))
5298 : {
5299 93 : tree t;
5300 93 : if (is_omp_target (ctx->stmt))
5301 : t = NULL_TREE;
5302 : else
5303 93 : t = maybe_lookup_decl (v, ctx);
5304 93 : if (t)
5305 88 : v = t;
5306 : else
5307 5 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5308 93 : gimplify_expr (&v, ilist, NULL, is_gimple_val, fb_rvalue);
5309 186 : t = fold_build2_loc (clause_loc, PLUS_EXPR,
5310 93 : TREE_TYPE (v), v,
5311 93 : build_int_cst (TREE_TYPE (v), 1));
5312 93 : sz = fold_build2_loc (clause_loc, MULT_EXPR,
5313 93 : TREE_TYPE (v), t,
5314 93 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5315 : }
5316 2334 : if (pass == 3)
5317 : {
5318 1461 : tree xv = create_tmp_var (ptr_type_node);
5319 1461 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5320 : {
5321 1149 : unsigned cnt = task_reduction_cnt - 1;
5322 1149 : if (!task_reduction_needs_orig_p)
5323 1061 : cnt += (task_reduction_cntorig_full
5324 1061 : - task_reduction_cntorig);
5325 : else
5326 88 : cnt = task_reduction_cntorig - 1;
5327 1149 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5328 1149 : size_int (cnt), NULL_TREE, NULL_TREE);
5329 :
5330 1149 : gimple *g = gimple_build_assign (xv, x);
5331 1149 : gimple_seq_add_stmt (ilist, g);
5332 : }
5333 : else
5334 : {
5335 312 : unsigned int idx = *ctx->task_reduction_map->get (c);
5336 312 : tree off;
5337 312 : if (ctx->task_reductions[1 + idx])
5338 81 : off = fold_convert (sizetype,
5339 : ctx->task_reductions[1 + idx]);
5340 : else
5341 231 : off = task_reduction_read (ilist, tskred_temp, sizetype,
5342 231 : 7 + 3 * idx + 1);
5343 312 : gimple *g = gimple_build_assign (xv, POINTER_PLUS_EXPR,
5344 : tskred_base, off);
5345 312 : gimple_seq_add_stmt (ilist, g);
5346 : }
5347 1461 : x = fold_convert (build_pointer_type (boolean_type_node),
5348 : xv);
5349 1461 : if (TREE_CONSTANT (v))
5350 1065 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x,
5351 : TYPE_SIZE_UNIT (type));
5352 : else
5353 : {
5354 396 : tree t;
5355 396 : if (is_omp_target (ctx->stmt))
5356 : t = NULL_TREE;
5357 : else
5358 336 : t = maybe_lookup_decl (v, ctx);
5359 336 : if (t)
5360 316 : v = t;
5361 : else
5362 80 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5363 396 : gimplify_expr (&v, ilist, NULL, is_gimple_val,
5364 : fb_rvalue);
5365 792 : t = fold_build2_loc (clause_loc, PLUS_EXPR,
5366 396 : TREE_TYPE (v), v,
5367 396 : build_int_cst (TREE_TYPE (v), 1));
5368 396 : t = fold_build2_loc (clause_loc, MULT_EXPR,
5369 396 : TREE_TYPE (v), t,
5370 396 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5371 396 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x, t);
5372 : }
5373 1461 : cond = create_tmp_var (TREE_TYPE (x));
5374 1461 : gimplify_assign (cond, x, ilist);
5375 1461 : x = xv;
5376 : }
5377 873 : else if (lower_private_allocate (var, type, allocator,
5378 : allocate_ptr, ilist, ctx,
5379 : true,
5380 873 : TREE_CONSTANT (v)
5381 780 : ? TYPE_SIZE_UNIT (type)
5382 : : sz))
5383 225 : x = allocate_ptr;
5384 648 : else if (TREE_CONSTANT (v))
5385 : {
5386 560 : x = create_tmp_var_raw (type, name);
5387 560 : gimple_add_tmp_var (x);
5388 560 : TREE_ADDRESSABLE (x) = 1;
5389 560 : x = build_fold_addr_expr_loc (clause_loc, x);
5390 : }
5391 : else
5392 : {
5393 88 : tree atmp
5394 88 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5395 88 : tree al = size_int (TYPE_ALIGN (TREE_TYPE (type)));
5396 88 : x = build_call_expr_loc (clause_loc, atmp, 2, sz, al);
5397 : }
5398 :
5399 2334 : tree ptype = build_pointer_type (TREE_TYPE (type));
5400 2334 : x = fold_convert_loc (clause_loc, ptype, x);
5401 2334 : tree y = create_tmp_var (ptype, name);
5402 2334 : gimplify_assign (y, x, ilist);
5403 2334 : x = y;
5404 2334 : tree yb = y;
5405 :
5406 2334 : if (!integer_zerop (bias))
5407 : {
5408 1512 : bias = fold_convert_loc (clause_loc, pointer_sized_int_node,
5409 : bias);
5410 1512 : yb = fold_convert_loc (clause_loc, pointer_sized_int_node,
5411 : x);
5412 1512 : yb = fold_build2_loc (clause_loc, MINUS_EXPR,
5413 : pointer_sized_int_node, yb, bias);
5414 1512 : x = fold_convert_loc (clause_loc, TREE_TYPE (x), yb);
5415 1512 : yb = create_tmp_var (ptype, name);
5416 1512 : gimplify_assign (yb, x, ilist);
5417 1512 : x = yb;
5418 : }
5419 :
5420 2334 : d = TREE_OPERAND (d, 0);
5421 2334 : if (TREE_CODE (d) == POINTER_PLUS_EXPR)
5422 420 : d = TREE_OPERAND (d, 0);
5423 2334 : if (TREE_CODE (d) == ADDR_EXPR)
5424 : {
5425 1268 : if (orig_var != var)
5426 : {
5427 91 : gcc_assert (is_variable_sized (orig_var));
5428 91 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var),
5429 : x);
5430 91 : gimplify_assign (new_var, x, ilist);
5431 91 : tree new_orig_var = lookup_decl (orig_var, ctx);
5432 91 : tree t = build_fold_indirect_ref (new_var);
5433 91 : DECL_IGNORED_P (new_var) = 0;
5434 91 : TREE_THIS_NOTRAP (t) = 1;
5435 91 : SET_DECL_VALUE_EXPR (new_orig_var, t);
5436 91 : DECL_HAS_VALUE_EXPR_P (new_orig_var) = 1;
5437 : }
5438 : else
5439 : {
5440 1177 : x = build2 (MEM_REF, TREE_TYPE (new_var), x,
5441 : build_int_cst (ptype, 0));
5442 1177 : SET_DECL_VALUE_EXPR (new_var, x);
5443 1177 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5444 : }
5445 : }
5446 : else
5447 : {
5448 1066 : gcc_assert (orig_var == var);
5449 1066 : if (TREE_CODE (d) == INDIRECT_REF)
5450 : {
5451 94 : x = create_tmp_var (ptype, name);
5452 94 : TREE_ADDRESSABLE (x) = 1;
5453 94 : gimplify_assign (x, yb, ilist);
5454 94 : x = build_fold_addr_expr_loc (clause_loc, x);
5455 : }
5456 1066 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5457 1066 : gimplify_assign (new_var, x, ilist);
5458 : }
5459 : /* GOMP_taskgroup_reduction_register memsets the whole
5460 : array to zero. If the initializer is zero, we don't
5461 : need to initialize it again, just mark it as ever
5462 : used unconditionally, i.e. cond = true. */
5463 2334 : if (cond
5464 1461 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE
5465 3267 : && initializer_zerop (omp_reduction_init (c,
5466 933 : TREE_TYPE (type))))
5467 : {
5468 624 : gimple *g = gimple_build_assign (build_simple_mem_ref (cond),
5469 : boolean_true_node);
5470 624 : gimple_seq_add_stmt (ilist, g);
5471 624 : continue;
5472 624 : }
5473 1710 : tree end = create_artificial_label (UNKNOWN_LOCATION);
5474 1710 : if (cond)
5475 : {
5476 837 : gimple *g;
5477 837 : if (!is_parallel_ctx (ctx))
5478 : {
5479 786 : tree condv = create_tmp_var (boolean_type_node);
5480 786 : g = gimple_build_assign (condv,
5481 : build_simple_mem_ref (cond));
5482 786 : gimple_seq_add_stmt (ilist, g);
5483 786 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
5484 786 : g = gimple_build_cond (NE_EXPR, condv,
5485 : boolean_false_node, end, lab1);
5486 786 : gimple_seq_add_stmt (ilist, g);
5487 786 : gimple_seq_add_stmt (ilist, gimple_build_label (lab1));
5488 : }
5489 837 : g = gimple_build_assign (build_simple_mem_ref (cond),
5490 : boolean_true_node);
5491 837 : gimple_seq_add_stmt (ilist, g);
5492 : }
5493 :
5494 1710 : tree y1 = create_tmp_var (ptype);
5495 1710 : gimplify_assign (y1, y, ilist);
5496 1710 : tree i2 = NULL_TREE, y2 = NULL_TREE;
5497 1710 : tree body2 = NULL_TREE, end2 = NULL_TREE;
5498 1710 : tree y3 = NULL_TREE, y4 = NULL_TREE;
5499 1710 : if (task_reduction_needs_orig_p)
5500 : {
5501 112 : y3 = create_tmp_var (ptype);
5502 112 : tree ref;
5503 112 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5504 88 : ref = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5505 88 : size_int (task_reduction_cnt_full
5506 : + task_reduction_cntorig - 1),
5507 : NULL_TREE, NULL_TREE);
5508 : else
5509 : {
5510 24 : unsigned int idx = *ctx->task_reduction_map->get (c);
5511 24 : ref = task_reduction_read (ilist, tskred_temp, ptype,
5512 24 : 7 + 3 * idx);
5513 : }
5514 112 : gimplify_assign (y3, ref, ilist);
5515 : }
5516 1598 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) || is_simd)
5517 : {
5518 668 : if (pass != 3)
5519 : {
5520 252 : y2 = create_tmp_var (ptype);
5521 252 : gimplify_assign (y2, y, ilist);
5522 : }
5523 668 : if (is_simd || OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5524 : {
5525 188 : tree ref = build_outer_var_ref (var, ctx);
5526 : /* For ref build_outer_var_ref already performs this. */
5527 188 : if (TREE_CODE (d) == INDIRECT_REF)
5528 0 : gcc_assert (omp_privatize_by_reference (var));
5529 188 : else if (TREE_CODE (d) == ADDR_EXPR)
5530 96 : ref = build_fold_addr_expr (ref);
5531 92 : else if (omp_privatize_by_reference (var))
5532 8 : ref = build_fold_addr_expr (ref);
5533 188 : ref = fold_convert_loc (clause_loc, ptype, ref);
5534 188 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
5535 188 : && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5536 : {
5537 8 : y3 = create_tmp_var (ptype);
5538 8 : gimplify_assign (y3, unshare_expr (ref), ilist);
5539 : }
5540 188 : if (is_simd)
5541 : {
5542 180 : y4 = create_tmp_var (ptype);
5543 180 : gimplify_assign (y4, ref, dlist);
5544 : }
5545 : }
5546 : }
5547 1710 : tree i = create_tmp_var (TREE_TYPE (v));
5548 1710 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), ilist);
5549 1710 : tree body = create_artificial_label (UNKNOWN_LOCATION);
5550 1710 : gimple_seq_add_stmt (ilist, gimple_build_label (body));
5551 1710 : if (y2)
5552 : {
5553 252 : i2 = create_tmp_var (TREE_TYPE (v));
5554 252 : gimplify_assign (i2, build_int_cst (TREE_TYPE (v), 0), dlist);
5555 252 : body2 = create_artificial_label (UNKNOWN_LOCATION);
5556 252 : end2 = create_artificial_label (UNKNOWN_LOCATION);
5557 252 : gimple_seq_add_stmt (dlist, gimple_build_label (body2));
5558 : }
5559 1710 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5560 : {
5561 600 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
5562 600 : tree decl_placeholder
5563 600 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
5564 600 : SET_DECL_VALUE_EXPR (decl_placeholder,
5565 : build_simple_mem_ref (y1));
5566 600 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
5567 600 : SET_DECL_VALUE_EXPR (placeholder,
5568 : y3 ? build_simple_mem_ref (y3)
5569 : : error_mark_node);
5570 600 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
5571 600 : x = lang_hooks.decls.omp_clause_default_ctor
5572 720 : (c, build_simple_mem_ref (y1),
5573 120 : y3 ? build_simple_mem_ref (y3) : NULL_TREE);
5574 600 : if (x)
5575 152 : gimplify_and_add (x, ilist);
5576 600 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
5577 : {
5578 576 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
5579 576 : lower_omp (&tseq, ctx);
5580 576 : gimple_seq_add_seq (ilist, tseq);
5581 : }
5582 600 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
5583 600 : if (is_simd)
5584 : {
5585 0 : SET_DECL_VALUE_EXPR (decl_placeholder,
5586 : build_simple_mem_ref (y2));
5587 0 : SET_DECL_VALUE_EXPR (placeholder,
5588 : build_simple_mem_ref (y4));
5589 0 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
5590 0 : lower_omp (&tseq, ctx);
5591 0 : gimple_seq_add_seq (dlist, tseq);
5592 0 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
5593 : }
5594 600 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
5595 600 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 0;
5596 600 : if (y2)
5597 : {
5598 72 : x = lang_hooks.decls.omp_clause_dtor
5599 72 : (c, build_simple_mem_ref (y2));
5600 72 : if (x)
5601 40 : gimplify_and_add (x, dlist);
5602 : }
5603 : }
5604 : else
5605 : {
5606 1110 : x = omp_reduction_init (c, TREE_TYPE (type));
5607 1110 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
5608 :
5609 : /* reduction(-:var) sums up the partial results, so it
5610 : acts identically to reduction(+:var). */
5611 1110 : if (code == MINUS_EXPR)
5612 0 : code = PLUS_EXPR;
5613 :
5614 1110 : gimplify_assign (build_simple_mem_ref (y1), x, ilist);
5615 1110 : if (is_simd)
5616 : {
5617 180 : x = build2 (code, TREE_TYPE (type),
5618 : build_simple_mem_ref (y4),
5619 : build_simple_mem_ref (y2));
5620 180 : gimplify_assign (build_simple_mem_ref (y4), x, dlist);
5621 : }
5622 : }
5623 1710 : gimple *g
5624 1710 : = gimple_build_assign (y1, POINTER_PLUS_EXPR, y1,
5625 1710 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5626 1710 : gimple_seq_add_stmt (ilist, g);
5627 1710 : if (y3)
5628 : {
5629 120 : g = gimple_build_assign (y3, POINTER_PLUS_EXPR, y3,
5630 120 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5631 120 : gimple_seq_add_stmt (ilist, g);
5632 : }
5633 1710 : g = gimple_build_assign (i, PLUS_EXPR, i,
5634 1710 : build_int_cst (TREE_TYPE (i), 1));
5635 1710 : gimple_seq_add_stmt (ilist, g);
5636 1710 : g = gimple_build_cond (LE_EXPR, i, v, body, end);
5637 1710 : gimple_seq_add_stmt (ilist, g);
5638 1710 : gimple_seq_add_stmt (ilist, gimple_build_label (end));
5639 1710 : if (y2)
5640 : {
5641 252 : g = gimple_build_assign (y2, POINTER_PLUS_EXPR, y2,
5642 252 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5643 252 : gimple_seq_add_stmt (dlist, g);
5644 252 : if (y4)
5645 : {
5646 180 : g = gimple_build_assign
5647 180 : (y4, POINTER_PLUS_EXPR, y4,
5648 180 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5649 180 : gimple_seq_add_stmt (dlist, g);
5650 : }
5651 252 : g = gimple_build_assign (i2, PLUS_EXPR, i2,
5652 252 : build_int_cst (TREE_TYPE (i2), 1));
5653 252 : gimple_seq_add_stmt (dlist, g);
5654 252 : g = gimple_build_cond (LE_EXPR, i2, v, body2, end2);
5655 252 : gimple_seq_add_stmt (dlist, g);
5656 252 : gimple_seq_add_stmt (dlist, gimple_build_label (end2));
5657 : }
5658 1710 : if (allocator)
5659 : {
5660 225 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
5661 225 : g = gimple_build_call (f, 2, allocate_ptr, allocator);
5662 225 : gimple_seq_add_stmt (dlist, g);
5663 : }
5664 1710 : continue;
5665 1710 : }
5666 312594 : else if (pass == 2)
5667 : {
5668 913 : tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
5669 913 : if (is_global_var (out))
5670 : x = var;
5671 361 : else if (is_omp_target (ctx->stmt))
5672 : x = out;
5673 : else
5674 : {
5675 301 : bool by_ref = use_pointer_for_field (var, ctx);
5676 301 : x = build_receiver_ref (var, by_ref, ctx);
5677 : }
5678 913 : if (!omp_privatize_by_reference (var))
5679 823 : x = build_fold_addr_expr (x);
5680 913 : x = fold_convert (ptr_type_node, x);
5681 913 : unsigned cnt = task_reduction_cnt - 1;
5682 913 : if (!task_reduction_needs_orig_p)
5683 878 : cnt += task_reduction_cntorig_full - task_reduction_cntorig;
5684 : else
5685 35 : cnt = task_reduction_cntorig - 1;
5686 913 : tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5687 913 : size_int (cnt), NULL_TREE, NULL_TREE);
5688 913 : gimplify_assign (r, x, ilist);
5689 913 : continue;
5690 913 : }
5691 311681 : else if (pass == 3)
5692 : {
5693 1733 : tree type = TREE_TYPE (new_var);
5694 1733 : if (!omp_privatize_by_reference (var))
5695 1627 : type = build_pointer_type (type);
5696 1733 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5697 : {
5698 913 : unsigned cnt = task_reduction_cnt - 1;
5699 913 : if (!task_reduction_needs_orig_p)
5700 878 : cnt += (task_reduction_cntorig_full
5701 878 : - task_reduction_cntorig);
5702 : else
5703 35 : cnt = task_reduction_cntorig - 1;
5704 913 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5705 913 : size_int (cnt), NULL_TREE, NULL_TREE);
5706 : }
5707 : else
5708 : {
5709 820 : unsigned int idx = *ctx->task_reduction_map->get (c);
5710 820 : tree off;
5711 820 : if (ctx->task_reductions[1 + idx])
5712 820 : off = fold_convert (sizetype,
5713 : ctx->task_reductions[1 + idx]);
5714 : else
5715 0 : off = task_reduction_read (ilist, tskred_temp, sizetype,
5716 0 : 7 + 3 * idx + 1);
5717 820 : x = fold_build2 (POINTER_PLUS_EXPR, ptr_type_node,
5718 : tskred_base, off);
5719 : }
5720 1733 : x = fold_convert (type, x);
5721 1733 : tree t;
5722 1733 : if (omp_privatize_by_reference (var))
5723 : {
5724 106 : gimplify_assign (new_var, x, ilist);
5725 106 : t = new_var;
5726 106 : new_var = build_simple_mem_ref (new_var);
5727 : }
5728 : else
5729 : {
5730 1627 : t = create_tmp_var (type);
5731 1627 : gimplify_assign (t, x, ilist);
5732 1627 : SET_DECL_VALUE_EXPR (new_var, build_simple_mem_ref (t));
5733 1627 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5734 : }
5735 1733 : t = fold_convert (build_pointer_type (boolean_type_node), t);
5736 1733 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
5737 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5738 1733 : cond = create_tmp_var (TREE_TYPE (t));
5739 1733 : gimplify_assign (cond, t, ilist);
5740 : }
5741 309948 : else if (is_variable_sized (var))
5742 : {
5743 : /* For variable sized types, we need to allocate the
5744 : actual storage here. Call alloca and store the
5745 : result in the pointer decl that we created elsewhere. */
5746 262 : if (pass == 0)
5747 134 : continue;
5748 :
5749 128 : if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
5750 : {
5751 114 : tree tmp;
5752 :
5753 114 : ptr = DECL_VALUE_EXPR (new_var);
5754 114 : gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
5755 114 : ptr = TREE_OPERAND (ptr, 0);
5756 114 : gcc_assert (DECL_P (ptr));
5757 114 : x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
5758 :
5759 114 : if (lower_private_allocate (var, new_var, allocator,
5760 : allocate_ptr, ilist, ctx,
5761 : false, x))
5762 8 : tmp = allocate_ptr;
5763 : else
5764 : {
5765 : /* void *tmp = __builtin_alloca */
5766 106 : tree atmp
5767 106 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5768 106 : gcall *stmt
5769 106 : = gimple_build_call (atmp, 2, x,
5770 106 : size_int (DECL_ALIGN (var)));
5771 106 : cfun->calls_alloca = 1;
5772 106 : tmp = create_tmp_var_raw (ptr_type_node);
5773 106 : gimple_add_tmp_var (tmp);
5774 106 : gimple_call_set_lhs (stmt, tmp);
5775 :
5776 106 : gimple_seq_add_stmt (ilist, stmt);
5777 : }
5778 :
5779 114 : x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
5780 114 : gimplify_assign (ptr, x, ilist);
5781 : }
5782 : }
5783 309686 : else if (omp_privatize_by_reference (var)
5784 309686 : && (c_kind != OMP_CLAUSE_FIRSTPRIVATE
5785 1992 : || !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)))
5786 : {
5787 : /* For references that are being privatized for Fortran,
5788 : allocate new backing storage for the new pointer
5789 : variable. This allows us to avoid changing all the
5790 : code that expects a pointer to something that expects
5791 : a direct variable. */
5792 4869 : if (pass == 0)
5793 2537 : continue;
5794 :
5795 2332 : x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
5796 2332 : if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
5797 : {
5798 146 : x = build_receiver_ref (var, false, ctx);
5799 146 : if (ctx->allocate_map)
5800 16 : if (tree *allocatep = ctx->allocate_map->get (var))
5801 : {
5802 16 : allocator = *allocatep;
5803 16 : if (TREE_CODE (allocator) == TREE_LIST)
5804 0 : allocator = TREE_PURPOSE (allocator);
5805 16 : if (TREE_CODE (allocator) != INTEGER_CST)
5806 8 : allocator = build_outer_var_ref (allocator, ctx);
5807 16 : allocator = fold_convert (pointer_sized_int_node,
5808 : allocator);
5809 16 : allocate_ptr = unshare_expr (x);
5810 : }
5811 146 : if (allocator == NULL_TREE)
5812 130 : x = build_fold_addr_expr_loc (clause_loc, x);
5813 : }
5814 2186 : else if (lower_private_allocate (var, new_var, allocator,
5815 : allocate_ptr,
5816 : ilist, ctx, true, x))
5817 86 : x = allocate_ptr;
5818 2100 : else if (TREE_CONSTANT (x))
5819 : {
5820 : /* For reduction in SIMD loop, defer adding the
5821 : initialization of the reference, because if we decide
5822 : to use SIMD array for it, the initilization could cause
5823 : expansion ICE. Ditto for other privatization clauses. */
5824 1404 : if (is_simd)
5825 : x = NULL_TREE;
5826 : else
5827 : {
5828 1077 : x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
5829 : get_name (var));
5830 1077 : gimple_add_tmp_var (x);
5831 1077 : TREE_ADDRESSABLE (x) = 1;
5832 1077 : x = build_fold_addr_expr_loc (clause_loc, x);
5833 : }
5834 : }
5835 : else
5836 : {
5837 696 : tree atmp
5838 696 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5839 696 : tree rtype = TREE_TYPE (TREE_TYPE (new_var));
5840 696 : tree al = size_int (TYPE_ALIGN (rtype));
5841 696 : x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
5842 : }
5843 :
5844 2005 : if (x)
5845 : {
5846 2005 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5847 2005 : gimplify_assign (new_var, x, ilist);
5848 : }
5849 :
5850 2332 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
5851 : }
5852 304817 : else if ((c_kind == OMP_CLAUSE_REDUCTION
5853 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5854 304817 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5855 : {
5856 1952 : if (pass == 0)
5857 976 : continue;
5858 : }
5859 302865 : else if (pass != 0)
5860 151178 : continue;
5861 :
5862 169249 : switch (OMP_CLAUSE_CODE (c))
5863 : {
5864 11883 : case OMP_CLAUSE_SHARED:
5865 : /* Ignore shared directives in teams construct inside
5866 : target construct. */
5867 11883 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
5868 11883 : && !is_host_teams_ctx (ctx))
5869 0 : continue;
5870 : /* Shared global vars are just accessed directly. */
5871 11883 : if (is_global_var (new_var))
5872 : break;
5873 : /* For taskloop firstprivate/lastprivate, represented
5874 : as firstprivate and shared clause on the task, new_var
5875 : is the firstprivate var. */
5876 11735 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
5877 : break;
5878 : /* Set up the DECL_VALUE_EXPR for shared variables now. This
5879 : needs to be delayed until after fixup_child_record_type so
5880 : that we get the correct type during the dereference. */
5881 11423 : by_ref = use_pointer_for_field (var, ctx);
5882 11423 : x = build_receiver_ref (var, by_ref, ctx);
5883 11423 : SET_DECL_VALUE_EXPR (new_var, x);
5884 11423 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5885 :
5886 : /* ??? If VAR is not passed by reference, and the variable
5887 : hasn't been initialized yet, then we'll get a warning for
5888 : the store into the omp_data_s structure. Ideally, we'd be
5889 : able to notice this and not store anything at all, but
5890 : we're generating code too early. Suppress the warning. */
5891 11423 : if (!by_ref)
5892 5197 : suppress_warning (var, OPT_Wuninitialized);
5893 : break;
5894 :
5895 210 : case OMP_CLAUSE__CONDTEMP_:
5896 210 : if (is_parallel_ctx (ctx))
5897 : {
5898 103 : x = build_receiver_ref (var, false, ctx);
5899 103 : SET_DECL_VALUE_EXPR (new_var, x);
5900 103 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5901 : }
5902 107 : else if (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c))
5903 : {
5904 107 : x = build_zero_cst (TREE_TYPE (var));
5905 107 : goto do_private;
5906 : }
5907 : break;
5908 :
5909 21290 : case OMP_CLAUSE_LASTPRIVATE:
5910 21290 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
5911 : break;
5912 : /* FALLTHRU */
5913 :
5914 87368 : case OMP_CLAUSE_PRIVATE:
5915 87368 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
5916 20786 : x = build_outer_var_ref (var, ctx);
5917 66582 : else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
5918 : {
5919 162 : if (is_task_ctx (ctx))
5920 12 : x = build_receiver_ref (var, false, ctx);
5921 : else
5922 150 : x = build_outer_var_ref (var, ctx, OMP_CLAUSE_PRIVATE);
5923 : }
5924 : else
5925 : x = NULL;
5926 94012 : do_private:
5927 94012 : tree nx;
5928 94012 : bool copy_ctor;
5929 94012 : copy_ctor = false;
5930 94012 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
5931 : ilist, ctx, false, NULL_TREE);
5932 94012 : nx = unshare_expr (new_var);
5933 94012 : if (is_simd
5934 24199 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
5935 101398 : && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c))
5936 22 : copy_ctor = true;
5937 94012 : if (copy_ctor)
5938 22 : nx = lang_hooks.decls.omp_clause_copy_ctor (c, nx, x);
5939 : else
5940 93990 : nx = lang_hooks.decls.omp_clause_default_ctor (c, nx, x);
5941 94012 : if (is_simd)
5942 : {
5943 24199 : tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
5944 23517 : if ((TREE_ADDRESSABLE (new_var) || nx || y
5945 23501 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
5946 7010 : && (gimple_omp_for_collapse (ctx->stmt) != 1
5947 1375 : || (gimple_omp_for_index (ctx->stmt, 0)
5948 : != new_var)))
5949 16793 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_
5950 16686 : || omp_privatize_by_reference (var))
5951 31064 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
5952 : ivar, lvar))
5953 : {
5954 5996 : if (omp_privatize_by_reference (var))
5955 : {
5956 52 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
5957 52 : tree new_vard = TREE_OPERAND (new_var, 0);
5958 52 : gcc_assert (DECL_P (new_vard));
5959 52 : SET_DECL_VALUE_EXPR (new_vard,
5960 : build_fold_addr_expr (lvar));
5961 52 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
5962 : }
5963 :
5964 5996 : if (nx)
5965 : {
5966 30 : tree iv = unshare_expr (ivar);
5967 30 : if (copy_ctor)
5968 22 : x = lang_hooks.decls.omp_clause_copy_ctor (c, iv,
5969 : x);
5970 : else
5971 8 : x = lang_hooks.decls.omp_clause_default_ctor (c,
5972 : iv,
5973 : x);
5974 : }
5975 5966 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_)
5976 : {
5977 37 : x = build2 (MODIFY_EXPR, TREE_TYPE (ivar),
5978 : unshare_expr (ivar), x);
5979 37 : nx = x;
5980 : }
5981 5996 : if (nx && x)
5982 67 : gimplify_and_add (x, &llist[0]);
5983 5996 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
5984 5996 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
5985 : {
5986 37 : tree v = new_var;
5987 37 : if (!DECL_P (v))
5988 : {
5989 4 : gcc_assert (TREE_CODE (v) == MEM_REF);
5990 4 : v = TREE_OPERAND (v, 0);
5991 4 : gcc_assert (DECL_P (v));
5992 : }
5993 37 : v = *ctx->lastprivate_conditional_map->get (v);
5994 37 : tree t = create_tmp_var (TREE_TYPE (v));
5995 37 : tree z = build_zero_cst (TREE_TYPE (v));
5996 37 : tree orig_v
5997 37 : = build_outer_var_ref (var, ctx,
5998 : OMP_CLAUSE_LASTPRIVATE);
5999 37 : gimple_seq_add_stmt (dlist,
6000 37 : gimple_build_assign (t, z));
6001 37 : gcc_assert (DECL_HAS_VALUE_EXPR_P (v));
6002 37 : tree civar = DECL_VALUE_EXPR (v);
6003 37 : gcc_assert (TREE_CODE (civar) == ARRAY_REF);
6004 37 : civar = unshare_expr (civar);
6005 37 : TREE_OPERAND (civar, 1) = sctx.idx;
6006 37 : x = build2 (MODIFY_EXPR, TREE_TYPE (t), t,
6007 : unshare_expr (civar));
6008 74 : x = build2 (COMPOUND_EXPR, TREE_TYPE (orig_v), x,
6009 37 : build2 (MODIFY_EXPR, TREE_TYPE (orig_v),
6010 : orig_v, unshare_expr (ivar)));
6011 37 : tree cond = build2 (LT_EXPR, boolean_type_node, t,
6012 : civar);
6013 37 : x = build3 (COND_EXPR, void_type_node, cond, x,
6014 : void_node);
6015 37 : gimple_seq tseq = NULL;
6016 37 : gimplify_and_add (x, &tseq);
6017 37 : if (ctx->outer)
6018 27 : lower_omp (&tseq, ctx->outer);
6019 37 : gimple_seq_add_seq (&llist[1], tseq);
6020 : }
6021 5996 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6022 5996 : && ctx->for_simd_scan_phase)
6023 : {
6024 9 : x = unshare_expr (ivar);
6025 9 : tree orig_v
6026 9 : = build_outer_var_ref (var, ctx,
6027 : OMP_CLAUSE_LASTPRIVATE);
6028 9 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
6029 : orig_v);
6030 9 : gimplify_and_add (x, &llist[0]);
6031 : }
6032 5996 : if (y)
6033 : {
6034 30 : y = lang_hooks.decls.omp_clause_dtor (c, ivar);
6035 30 : if (y)
6036 30 : gimplify_and_add (y, &llist[1]);
6037 : }
6038 : break;
6039 : }
6040 18203 : if (omp_privatize_by_reference (var))
6041 : {
6042 28 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6043 28 : tree new_vard = TREE_OPERAND (new_var, 0);
6044 28 : gcc_assert (DECL_P (new_vard));
6045 28 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6046 28 : x = TYPE_SIZE_UNIT (type);
6047 28 : if (TREE_CONSTANT (x))
6048 : {
6049 28 : x = create_tmp_var_raw (type, get_name (var));
6050 28 : gimple_add_tmp_var (x);
6051 28 : TREE_ADDRESSABLE (x) = 1;
6052 28 : x = build_fold_addr_expr_loc (clause_loc, x);
6053 28 : x = fold_convert_loc (clause_loc,
6054 28 : TREE_TYPE (new_vard), x);
6055 28 : gimplify_assign (new_vard, x, ilist);
6056 : }
6057 : }
6058 : }
6059 88016 : if (nx)
6060 485 : gimplify_and_add (nx, ilist);
6061 88016 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6062 15148 : && is_simd
6063 89764 : && ctx->for_simd_scan_phase)
6064 : {
6065 5 : tree orig_v = build_outer_var_ref (var, ctx,
6066 : OMP_CLAUSE_LASTPRIVATE);
6067 5 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var,
6068 : orig_v);
6069 5 : gimplify_and_add (x, ilist);
6070 : }
6071 : /* FALLTHRU */
6072 :
6073 118760 : do_dtor:
6074 118760 : x = lang_hooks.decls.omp_clause_dtor (c, new_var);
6075 118760 : if (x)
6076 1125 : gimplify_and_add (x, dlist);
6077 118760 : if (allocator)
6078 : {
6079 847 : if (!is_gimple_val (allocator))
6080 : {
6081 22 : tree avar = create_tmp_var (TREE_TYPE (allocator));
6082 22 : gimplify_assign (avar, allocator, dlist);
6083 22 : allocator = avar;
6084 : }
6085 847 : if (!is_gimple_val (allocate_ptr))
6086 : {
6087 50 : tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
6088 50 : gimplify_assign (apvar, allocate_ptr, dlist);
6089 50 : allocate_ptr = apvar;
6090 : }
6091 847 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
6092 847 : gimple *g
6093 847 : = gimple_build_call (f, 2, allocate_ptr, allocator);
6094 847 : gimple_seq_add_stmt (dlist, g);
6095 : }
6096 : break;
6097 :
6098 7806 : case OMP_CLAUSE_LINEAR:
6099 7806 : if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
6100 1269 : goto do_firstprivate;
6101 6537 : if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
6102 : x = NULL;
6103 : else
6104 3598 : x = build_outer_var_ref (var, ctx);
6105 6537 : goto do_private;
6106 :
6107 28595 : case OMP_CLAUSE_FIRSTPRIVATE:
6108 28595 : if (is_task_ctx (ctx))
6109 : {
6110 4626 : if ((omp_privatize_by_reference (var)
6111 146 : && !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c))
6112 4626 : || is_variable_sized (var))
6113 160 : goto do_dtor;
6114 4466 : else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
6115 : ctx))
6116 4466 : || use_pointer_for_field (var, NULL))
6117 : {
6118 421 : x = build_receiver_ref (var, false, ctx);
6119 421 : if (ctx->allocate_map)
6120 34 : if (tree *allocatep = ctx->allocate_map->get (var))
6121 : {
6122 34 : allocator = *allocatep;
6123 34 : if (TREE_CODE (allocator) == TREE_LIST)
6124 4 : allocator = TREE_PURPOSE (allocator);
6125 34 : if (TREE_CODE (allocator) != INTEGER_CST)
6126 14 : allocator = build_outer_var_ref (allocator, ctx);
6127 34 : allocator = fold_convert (pointer_sized_int_node,
6128 : allocator);
6129 34 : allocate_ptr = unshare_expr (x);
6130 34 : x = build_simple_mem_ref (x);
6131 34 : TREE_THIS_NOTRAP (x) = 1;
6132 : }
6133 421 : SET_DECL_VALUE_EXPR (new_var, x);
6134 421 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
6135 421 : goto do_dtor;
6136 : }
6137 : }
6138 28014 : if (OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)
6139 28014 : && omp_privatize_by_reference (var))
6140 : {
6141 0 : x = build_outer_var_ref (var, ctx);
6142 0 : gcc_assert (TREE_CODE (x) == MEM_REF
6143 : && integer_zerop (TREE_OPERAND (x, 1)));
6144 0 : x = TREE_OPERAND (x, 0);
6145 0 : x = lang_hooks.decls.omp_clause_copy_ctor
6146 0 : (c, unshare_expr (new_var), x);
6147 0 : gimplify_and_add (x, ilist);
6148 0 : goto do_dtor;
6149 : }
6150 29283 : do_firstprivate:
6151 29283 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
6152 : ilist, ctx, false, NULL_TREE);
6153 29283 : x = build_outer_var_ref (var, ctx);
6154 29283 : if (is_simd)
6155 : {
6156 1063 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
6157 1063 : && gimple_omp_for_combined_into_p (ctx->stmt))
6158 : {
6159 668 : tree t = OMP_CLAUSE_LINEAR_STEP (c);
6160 668 : if (DECL_P (t))
6161 114 : t = build_outer_var_ref (t, ctx);
6162 668 : tree stept = TREE_TYPE (t);
6163 668 : tree ct = omp_find_clause (clauses,
6164 : OMP_CLAUSE__LOOPTEMP_);
6165 668 : gcc_assert (ct);
6166 668 : tree l = OMP_CLAUSE_DECL (ct);
6167 668 : tree n1 = fd->loop.n1;
6168 668 : tree step = fd->loop.step;
6169 668 : tree itype = TREE_TYPE (l);
6170 668 : if (POINTER_TYPE_P (itype))
6171 0 : itype = signed_type_for (itype);
6172 668 : l = fold_build2 (MINUS_EXPR, itype, l, n1);
6173 668 : if (TYPE_UNSIGNED (itype)
6174 668 : && fd->loop.cond_code == GT_EXPR)
6175 0 : l = fold_build2 (TRUNC_DIV_EXPR, itype,
6176 : fold_build1 (NEGATE_EXPR, itype, l),
6177 : fold_build1 (NEGATE_EXPR,
6178 : itype, step));
6179 : else
6180 668 : l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
6181 668 : t = fold_build2 (MULT_EXPR, stept,
6182 : fold_convert (stept, l), t);
6183 :
6184 668 : if (OMP_CLAUSE_LINEAR_ARRAY (c))
6185 : {
6186 108 : if (omp_privatize_by_reference (var))
6187 : {
6188 72 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6189 72 : tree new_vard = TREE_OPERAND (new_var, 0);
6190 72 : gcc_assert (DECL_P (new_vard));
6191 72 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6192 72 : nx = TYPE_SIZE_UNIT (type);
6193 72 : if (TREE_CONSTANT (nx))
6194 : {
6195 24 : nx = create_tmp_var_raw (type,
6196 : get_name (var));
6197 24 : gimple_add_tmp_var (nx);
6198 24 : TREE_ADDRESSABLE (nx) = 1;
6199 24 : nx = build_fold_addr_expr_loc (clause_loc,
6200 : nx);
6201 24 : nx = fold_convert_loc (clause_loc,
6202 24 : TREE_TYPE (new_vard),
6203 : nx);
6204 24 : gimplify_assign (new_vard, nx, ilist);
6205 : }
6206 : }
6207 :
6208 108 : x = lang_hooks.decls.omp_clause_linear_ctor
6209 108 : (c, new_var, x, t);
6210 108 : gimplify_and_add (x, ilist);
6211 108 : goto do_dtor;
6212 : }
6213 :
6214 560 : if (POINTER_TYPE_P (TREE_TYPE (x)))
6215 11 : x = fold_build_pointer_plus (x, t);
6216 : else
6217 549 : x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x,
6218 : fold_convert (TREE_TYPE (x), t));
6219 : }
6220 :
6221 955 : if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
6222 934 : || TREE_ADDRESSABLE (new_var)
6223 799 : || omp_privatize_by_reference (var))
6224 1217 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6225 : ivar, lvar))
6226 : {
6227 152 : if (omp_privatize_by_reference (var))
6228 : {
6229 22 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6230 22 : tree new_vard = TREE_OPERAND (new_var, 0);
6231 22 : gcc_assert (DECL_P (new_vard));
6232 22 : SET_DECL_VALUE_EXPR (new_vard,
6233 : build_fold_addr_expr (lvar));
6234 22 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6235 : }
6236 152 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
6237 : {
6238 131 : tree iv = create_tmp_var (TREE_TYPE (new_var));
6239 131 : x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
6240 131 : gimplify_and_add (x, ilist);
6241 131 : gimple_stmt_iterator gsi
6242 131 : = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
6243 131 : gassign *g
6244 131 : = gimple_build_assign (unshare_expr (lvar), iv);
6245 131 : gsi_insert_before_without_update (&gsi, g,
6246 : GSI_SAME_STMT);
6247 131 : tree t = OMP_CLAUSE_LINEAR_STEP (c);
6248 131 : enum tree_code code = PLUS_EXPR;
6249 131 : if (POINTER_TYPE_P (TREE_TYPE (new_var)))
6250 : code = POINTER_PLUS_EXPR;
6251 131 : g = gimple_build_assign (iv, code, iv, t);
6252 131 : gsi_insert_before_without_update (&gsi, g,
6253 : GSI_SAME_STMT);
6254 131 : break;
6255 : }
6256 21 : x = lang_hooks.decls.omp_clause_copy_ctor
6257 21 : (c, unshare_expr (ivar), x);
6258 21 : gimplify_and_add (x, &llist[0]);
6259 21 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6260 21 : if (x)
6261 21 : gimplify_and_add (x, &llist[1]);
6262 : break;
6263 : }
6264 803 : if (omp_privatize_by_reference (var))
6265 : {
6266 105 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6267 105 : tree new_vard = TREE_OPERAND (new_var, 0);
6268 105 : gcc_assert (DECL_P (new_vard));
6269 105 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6270 105 : nx = TYPE_SIZE_UNIT (type);
6271 105 : if (TREE_CONSTANT (nx))
6272 : {
6273 57 : nx = create_tmp_var_raw (type, get_name (var));
6274 57 : gimple_add_tmp_var (nx);
6275 57 : TREE_ADDRESSABLE (nx) = 1;
6276 57 : nx = build_fold_addr_expr_loc (clause_loc, nx);
6277 57 : nx = fold_convert_loc (clause_loc,
6278 57 : TREE_TYPE (new_vard), nx);
6279 57 : gimplify_assign (new_vard, nx, ilist);
6280 : }
6281 : }
6282 : }
6283 29023 : x = lang_hooks.decls.omp_clause_copy_ctor
6284 29023 : (c, unshare_expr (new_var), x);
6285 29019 : gimplify_and_add (x, ilist);
6286 29019 : goto do_dtor;
6287 :
6288 19351 : case OMP_CLAUSE__LOOPTEMP_:
6289 19351 : case OMP_CLAUSE__REDUCTEMP_:
6290 19351 : gcc_assert (is_taskreg_ctx (ctx));
6291 19351 : x = build_outer_var_ref (var, ctx);
6292 19351 : x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
6293 19351 : gimplify_and_add (x, ilist);
6294 19351 : break;
6295 :
6296 510 : case OMP_CLAUSE_COPYIN:
6297 510 : by_ref = use_pointer_for_field (var, NULL);
6298 510 : x = build_receiver_ref (var, by_ref, ctx);
6299 510 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
6300 510 : append_to_statement_list (x, ©in_seq);
6301 510 : copyin_by_ref |= by_ref;
6302 510 : break;
6303 :
6304 13022 : case OMP_CLAUSE_REDUCTION:
6305 13022 : case OMP_CLAUSE_IN_REDUCTION:
6306 : /* OpenACC reductions are initialized using the
6307 : GOACC_REDUCTION internal function. */
6308 13022 : if (is_gimple_omp_oacc (ctx->stmt))
6309 : break;
6310 9040 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
6311 : {
6312 1443 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
6313 1443 : gimple *tseq;
6314 1443 : tree ptype = TREE_TYPE (placeholder);
6315 1443 : if (cond)
6316 : {
6317 294 : x = error_mark_node;
6318 294 : if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)
6319 294 : && !task_reduction_needs_orig_p)
6320 28 : x = var;
6321 266 : else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
6322 : {
6323 45 : tree pptype = build_pointer_type (ptype);
6324 45 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
6325 35 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
6326 35 : size_int (task_reduction_cnt_full
6327 : + task_reduction_cntorig - 1),
6328 : NULL_TREE, NULL_TREE);
6329 : else
6330 : {
6331 10 : unsigned int idx
6332 10 : = *ctx->task_reduction_map->get (c);
6333 10 : x = task_reduction_read (ilist, tskred_temp,
6334 10 : pptype, 7 + 3 * idx);
6335 : }
6336 45 : x = fold_convert (pptype, x);
6337 45 : x = build_simple_mem_ref (x);
6338 : }
6339 : }
6340 : else
6341 : {
6342 1149 : lower_private_allocate (var, new_var, allocator,
6343 : allocate_ptr, ilist, ctx, false,
6344 : NULL_TREE);
6345 1149 : x = build_outer_var_ref (var, ctx);
6346 :
6347 1149 : if (omp_privatize_by_reference (var)
6348 1149 : && !useless_type_conversion_p (ptype, TREE_TYPE (x)))
6349 49 : x = build_fold_addr_expr_loc (clause_loc, x);
6350 : }
6351 1443 : SET_DECL_VALUE_EXPR (placeholder, x);
6352 1443 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
6353 1443 : tree new_vard = new_var;
6354 1443 : if (omp_privatize_by_reference (var))
6355 : {
6356 203 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6357 203 : new_vard = TREE_OPERAND (new_var, 0);
6358 203 : gcc_assert (DECL_P (new_vard));
6359 : }
6360 1443 : tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
6361 1443 : if (is_simd
6362 311 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6363 1754 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6364 : rvarp = &rvar;
6365 1443 : if (is_simd
6366 1443 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6367 : ivar, lvar, rvarp,
6368 : &rvar2))
6369 : {
6370 174 : if (new_vard == new_var)
6371 : {
6372 128 : gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
6373 128 : SET_DECL_VALUE_EXPR (new_var, ivar);
6374 : }
6375 : else
6376 : {
6377 46 : SET_DECL_VALUE_EXPR (new_vard,
6378 : build_fold_addr_expr (ivar));
6379 46 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6380 : }
6381 174 : x = lang_hooks.decls.omp_clause_default_ctor
6382 174 : (c, unshare_expr (ivar),
6383 : build_outer_var_ref (var, ctx));
6384 174 : if (rvarp && ctx->for_simd_scan_phase)
6385 : {
6386 16 : if (x)
6387 8 : gimplify_and_add (x, &llist[0]);
6388 16 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6389 16 : if (x)
6390 8 : gimplify_and_add (x, &llist[1]);
6391 468 : break;
6392 : }
6393 78 : else if (rvarp)
6394 : {
6395 78 : if (x)
6396 : {
6397 38 : gimplify_and_add (x, &llist[0]);
6398 :
6399 38 : tree ivar2 = unshare_expr (lvar);
6400 38 : TREE_OPERAND (ivar2, 1) = sctx.idx;
6401 38 : x = lang_hooks.decls.omp_clause_default_ctor
6402 38 : (c, ivar2, build_outer_var_ref (var, ctx));
6403 38 : gimplify_and_add (x, &llist[0]);
6404 :
6405 38 : if (rvar2)
6406 : {
6407 16 : x = lang_hooks.decls.omp_clause_default_ctor
6408 16 : (c, unshare_expr (rvar2),
6409 : build_outer_var_ref (var, ctx));
6410 16 : gimplify_and_add (x, &llist[0]);
6411 : }
6412 :
6413 : /* For types that need construction, add another
6414 : private var which will be default constructed
6415 : and optionally initialized with
6416 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT, as in the
6417 : loop we want to assign this value instead of
6418 : constructing and destructing it in each
6419 : iteration. */
6420 38 : tree nv = create_tmp_var_raw (TREE_TYPE (ivar));
6421 38 : gimple_add_tmp_var (nv);
6422 60 : ctx->cb.decl_map->put (TREE_OPERAND (rvar2
6423 : ? rvar2
6424 : : ivar, 0),
6425 : nv);
6426 38 : x = lang_hooks.decls.omp_clause_default_ctor
6427 38 : (c, nv, build_outer_var_ref (var, ctx));
6428 38 : gimplify_and_add (x, ilist);
6429 :
6430 38 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6431 : {
6432 19 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6433 19 : x = DECL_VALUE_EXPR (new_vard);
6434 19 : tree vexpr = nv;
6435 19 : if (new_vard != new_var)
6436 11 : vexpr = build_fold_addr_expr (nv);
6437 19 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
6438 19 : lower_omp (&tseq, ctx);
6439 19 : SET_DECL_VALUE_EXPR (new_vard, x);
6440 19 : gimple_seq_add_seq (ilist, tseq);
6441 19 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6442 : }
6443 :
6444 38 : x = lang_hooks.decls.omp_clause_dtor (c, nv);
6445 38 : if (x)
6446 38 : gimplify_and_add (x, dlist);
6447 : }
6448 :
6449 78 : tree ref = build_outer_var_ref (var, ctx);
6450 78 : x = unshare_expr (ivar);
6451 78 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
6452 : ref);
6453 78 : gimplify_and_add (x, &llist[0]);
6454 :
6455 78 : ref = build_outer_var_ref (var, ctx);
6456 78 : x = lang_hooks.decls.omp_clause_assign_op (c, ref,
6457 : rvar);
6458 78 : gimplify_and_add (x, &llist[3]);
6459 :
6460 78 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6461 78 : if (new_vard == new_var)
6462 48 : SET_DECL_VALUE_EXPR (new_var, lvar);
6463 : else
6464 30 : SET_DECL_VALUE_EXPR (new_vard,
6465 : build_fold_addr_expr (lvar));
6466 :
6467 78 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6468 78 : if (x)
6469 38 : gimplify_and_add (x, &llist[1]);
6470 :
6471 78 : tree ivar2 = unshare_expr (lvar);
6472 78 : TREE_OPERAND (ivar2, 1) = sctx.idx;
6473 78 : x = lang_hooks.decls.omp_clause_dtor (c, ivar2);
6474 78 : if (x)
6475 38 : gimplify_and_add (x, &llist[1]);
6476 :
6477 78 : if (rvar2)
6478 : {
6479 36 : x = lang_hooks.decls.omp_clause_dtor (c, rvar2);
6480 36 : if (x)
6481 16 : gimplify_and_add (x, &llist[1]);
6482 : }
6483 : break;
6484 : }
6485 80 : if (x)
6486 34 : gimplify_and_add (x, &llist[0]);
6487 80 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6488 : {
6489 76 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6490 76 : lower_omp (&tseq, ctx);
6491 76 : gimple_seq_add_seq (&llist[0], tseq);
6492 : }
6493 80 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6494 80 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
6495 80 : lower_omp (&tseq, ctx);
6496 80 : gimple_seq_add_seq (&llist[1], tseq);
6497 80 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
6498 80 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6499 80 : if (new_vard == new_var)
6500 70 : SET_DECL_VALUE_EXPR (new_var, lvar);
6501 : else
6502 10 : SET_DECL_VALUE_EXPR (new_vard,
6503 : build_fold_addr_expr (lvar));
6504 80 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6505 80 : if (x)
6506 38 : gimplify_and_add (x, &llist[1]);
6507 : break;
6508 : }
6509 : /* If this is a reference to constant size reduction var
6510 : with placeholder, we haven't emitted the initializer
6511 : for it because it is undesirable if SIMD arrays are used.
6512 : But if they aren't used, we need to emit the deferred
6513 : initialization now. */
6514 1269 : else if (omp_privatize_by_reference (var) && is_simd)
6515 46 : handle_simd_reference (clause_loc, new_vard, ilist);
6516 :
6517 1269 : tree lab2 = NULL_TREE;
6518 1269 : if (cond)
6519 : {
6520 294 : gimple *g;
6521 294 : if (!is_parallel_ctx (ctx))
6522 : {
6523 278 : tree condv = create_tmp_var (boolean_type_node);
6524 278 : tree m = build_simple_mem_ref (cond);
6525 278 : g = gimple_build_assign (condv, m);
6526 278 : gimple_seq_add_stmt (ilist, g);
6527 278 : tree lab1
6528 278 : = create_artificial_label (UNKNOWN_LOCATION);
6529 278 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
6530 278 : g = gimple_build_cond (NE_EXPR, condv,
6531 : boolean_false_node,
6532 : lab2, lab1);
6533 278 : gimple_seq_add_stmt (ilist, g);
6534 278 : gimple_seq_add_stmt (ilist,
6535 278 : gimple_build_label (lab1));
6536 : }
6537 294 : g = gimple_build_assign (build_simple_mem_ref (cond),
6538 : boolean_true_node);
6539 294 : gimple_seq_add_stmt (ilist, g);
6540 : }
6541 294 : x = lang_hooks.decls.omp_clause_default_ctor
6542 1269 : (c, unshare_expr (new_var),
6543 : cond ? NULL_TREE
6544 975 : : build_outer_var_ref (var, ctx));
6545 1269 : if (x)
6546 285 : gimplify_and_add (x, ilist);
6547 :
6548 1269 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6549 1269 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6550 : {
6551 158 : if (ctx->for_simd_scan_phase)
6552 975 : goto do_dtor;
6553 142 : if (x || (!is_simd
6554 32 : && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)))
6555 : {
6556 70 : tree nv = create_tmp_var_raw (TREE_TYPE (new_var));
6557 70 : gimple_add_tmp_var (nv);
6558 70 : ctx->cb.decl_map->put (new_vard, nv);
6559 70 : x = lang_hooks.decls.omp_clause_default_ctor
6560 70 : (c, nv, build_outer_var_ref (var, ctx));
6561 70 : if (x)
6562 70 : gimplify_and_add (x, ilist);
6563 70 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6564 : {
6565 35 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6566 35 : tree vexpr = nv;
6567 35 : if (new_vard != new_var)
6568 19 : vexpr = build_fold_addr_expr (nv);
6569 35 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
6570 35 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6571 35 : lower_omp (&tseq, ctx);
6572 35 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
6573 35 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
6574 35 : gimple_seq_add_seq (ilist, tseq);
6575 : }
6576 70 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6577 70 : if (is_simd && ctx->scan_exclusive)
6578 : {
6579 16 : tree nv2
6580 16 : = create_tmp_var_raw (TREE_TYPE (new_var));
6581 16 : gimple_add_tmp_var (nv2);
6582 16 : ctx->cb.decl_map->put (nv, nv2);
6583 16 : x = lang_hooks.decls.omp_clause_default_ctor
6584 16 : (c, nv2, build_outer_var_ref (var, ctx));
6585 16 : gimplify_and_add (x, ilist);
6586 16 : x = lang_hooks.decls.omp_clause_dtor (c, nv2);
6587 16 : if (x)
6588 16 : gimplify_and_add (x, dlist);
6589 : }
6590 70 : x = lang_hooks.decls.omp_clause_dtor (c, nv);
6591 70 : if (x)
6592 70 : gimplify_and_add (x, dlist);
6593 : }
6594 72 : else if (is_simd
6595 40 : && ctx->scan_exclusive
6596 92 : && TREE_ADDRESSABLE (TREE_TYPE (new_var)))
6597 : {
6598 0 : tree nv2 = create_tmp_var_raw (TREE_TYPE (new_var));
6599 0 : gimple_add_tmp_var (nv2);
6600 0 : ctx->cb.decl_map->put (new_vard, nv2);
6601 0 : x = lang_hooks.decls.omp_clause_dtor (c, nv2);
6602 0 : if (x)
6603 0 : gimplify_and_add (x, dlist);
6604 : }
6605 142 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6606 142 : goto do_dtor;
6607 : }
6608 :
6609 1111 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6610 : {
6611 1063 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6612 1063 : if (c_kind == OMP_CLAUSE_IN_REDUCTION
6613 1063 : && is_omp_target (ctx->stmt))
6614 : {
6615 12 : tree d = maybe_lookup_decl_in_outer_ctx (var, ctx);
6616 12 : tree oldv = NULL_TREE;
6617 12 : gcc_assert (d);
6618 12 : if (DECL_HAS_VALUE_EXPR_P (d))
6619 6 : oldv = DECL_VALUE_EXPR (d);
6620 12 : SET_DECL_VALUE_EXPR (d, new_vard);
6621 12 : DECL_HAS_VALUE_EXPR_P (d) = 1;
6622 12 : lower_omp (&tseq, ctx);
6623 12 : if (oldv)
6624 6 : SET_DECL_VALUE_EXPR (d, oldv);
6625 : else
6626 : {
6627 6 : SET_DECL_VALUE_EXPR (d, NULL_TREE);
6628 6 : DECL_HAS_VALUE_EXPR_P (d) = 0;
6629 : }
6630 : }
6631 : else
6632 1051 : lower_omp (&tseq, ctx);
6633 1063 : gimple_seq_add_seq (ilist, tseq);
6634 : }
6635 1111 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6636 1111 : if (is_simd)
6637 : {
6638 43 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
6639 43 : lower_omp (&tseq, ctx);
6640 43 : gimple_seq_add_seq (dlist, tseq);
6641 43 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
6642 : }
6643 1111 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6644 1111 : if (cond)
6645 : {
6646 294 : if (lab2)
6647 278 : gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
6648 : break;
6649 : }
6650 817 : goto do_dtor;
6651 : }
6652 : else
6653 : {
6654 7597 : x = omp_reduction_init (c, TREE_TYPE (new_var));
6655 7597 : gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
6656 7597 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
6657 :
6658 7597 : if (cond)
6659 : {
6660 1439 : gimple *g;
6661 1439 : tree lab2 = NULL_TREE;
6662 : /* GOMP_taskgroup_reduction_register memsets the whole
6663 : array to zero. If the initializer is zero, we don't
6664 : need to initialize it again, just mark it as ever
6665 : used unconditionally, i.e. cond = true. */
6666 1439 : if (initializer_zerop (x))
6667 : {
6668 1258 : g = gimple_build_assign (build_simple_mem_ref (cond),
6669 : boolean_true_node);
6670 1258 : gimple_seq_add_stmt (ilist, g);
6671 3223 : break;
6672 : }
6673 :
6674 : /* Otherwise, emit
6675 : if (!cond) { cond = true; new_var = x; } */
6676 181 : if (!is_parallel_ctx (ctx))
6677 : {
6678 177 : tree condv = create_tmp_var (boolean_type_node);
6679 177 : tree m = build_simple_mem_ref (cond);
6680 177 : g = gimple_build_assign (condv, m);
6681 177 : gimple_seq_add_stmt (ilist, g);
6682 177 : tree lab1
6683 177 : = create_artificial_label (UNKNOWN_LOCATION);
6684 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
6685 177 : g = gimple_build_cond (NE_EXPR, condv,
6686 : boolean_false_node,
6687 : lab2, lab1);
6688 177 : gimple_seq_add_stmt (ilist, g);
6689 177 : gimple_seq_add_stmt (ilist,
6690 177 : gimple_build_label (lab1));
6691 : }
6692 181 : g = gimple_build_assign (build_simple_mem_ref (cond),
6693 : boolean_true_node);
6694 181 : gimple_seq_add_stmt (ilist, g);
6695 181 : gimplify_assign (new_var, x, ilist);
6696 181 : if (lab2)
6697 177 : gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
6698 : break;
6699 : }
6700 :
6701 : /* reduction(-:var) sums up the partial results, so it
6702 : acts identically to reduction(+:var). */
6703 6158 : if (code == MINUS_EXPR)
6704 33 : code = PLUS_EXPR;
6705 :
6706 6158 : bool is_truth_op
6707 6158 : = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
6708 6158 : tree new_vard = new_var;
6709 6158 : if (is_simd && omp_privatize_by_reference (var))
6710 : {
6711 52 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6712 52 : new_vard = TREE_OPERAND (new_var, 0);
6713 52 : gcc_assert (DECL_P (new_vard));
6714 : }
6715 6158 : tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
6716 6158 : if (is_simd
6717 2087 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6718 8245 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6719 : rvarp = &rvar;
6720 6158 : if (is_simd
6721 6158 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6722 : ivar, lvar, rvarp,
6723 : &rvar2))
6724 : {
6725 867 : if (new_vard != new_var)
6726 : {
6727 34 : SET_DECL_VALUE_EXPR (new_vard,
6728 : build_fold_addr_expr (lvar));
6729 34 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6730 : }
6731 :
6732 867 : tree ref = build_outer_var_ref (var, ctx);
6733 :
6734 867 : if (rvarp)
6735 : {
6736 193 : if (ctx->for_simd_scan_phase)
6737 : break;
6738 154 : gimplify_assign (ivar, ref, &llist[0]);
6739 154 : ref = build_outer_var_ref (var, ctx);
6740 154 : gimplify_assign (ref, rvar, &llist[3]);
6741 154 : break;
6742 : }
6743 :
6744 674 : gimplify_assign (unshare_expr (ivar), x, &llist[0]);
6745 :
6746 674 : if (sctx.is_simt)
6747 : {
6748 0 : if (!simt_lane)
6749 0 : simt_lane = create_tmp_var (unsigned_type_node);
6750 0 : x = build_call_expr_internal_loc
6751 0 : (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_BFLY,
6752 0 : TREE_TYPE (ivar), 2, ivar, simt_lane);
6753 : /* Make sure x is evaluated unconditionally. */
6754 0 : tree bfly_var = create_tmp_var (TREE_TYPE (ivar));
6755 0 : gimplify_assign (bfly_var, x, &llist[2]);
6756 0 : x = build2 (code, TREE_TYPE (ivar), ivar, bfly_var);
6757 0 : gimplify_assign (ivar, x, &llist[2]);
6758 : }
6759 674 : tree ivar2 = ivar;
6760 674 : tree ref2 = ref;
6761 674 : if (is_truth_op)
6762 : {
6763 99 : tree zero = build_zero_cst (TREE_TYPE (ivar));
6764 99 : ivar2 = fold_build2_loc (clause_loc, NE_EXPR,
6765 : boolean_type_node, ivar,
6766 : zero);
6767 99 : ref2 = fold_build2_loc (clause_loc, NE_EXPR,
6768 : boolean_type_node, ref,
6769 : zero);
6770 : }
6771 674 : x = build2 (code, TREE_TYPE (ref), ref2, ivar2);
6772 674 : if (is_truth_op)
6773 99 : x = fold_convert (TREE_TYPE (ref), x);
6774 674 : ref = build_outer_var_ref (var, ctx);
6775 674 : gimplify_assign (ref, x, &llist[1]);
6776 :
6777 : }
6778 : else
6779 : {
6780 5291 : lower_private_allocate (var, new_var, allocator,
6781 : allocate_ptr, ilist, ctx,
6782 : false, NULL_TREE);
6783 5291 : if (omp_privatize_by_reference (var) && is_simd)
6784 18 : handle_simd_reference (clause_loc, new_vard, ilist);
6785 5291 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6786 5291 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6787 : break;
6788 4958 : gimplify_assign (new_var, x, ilist);
6789 4958 : if (is_simd)
6790 : {
6791 1028 : tree ref = build_outer_var_ref (var, ctx);
6792 1028 : tree new_var2 = new_var;
6793 1028 : tree ref2 = ref;
6794 1028 : if (is_truth_op)
6795 : {
6796 24 : tree zero = build_zero_cst (TREE_TYPE (new_var));
6797 24 : new_var2
6798 24 : = fold_build2_loc (clause_loc, NE_EXPR,
6799 : boolean_type_node, new_var,
6800 : zero);
6801 24 : ref2 = fold_build2_loc (clause_loc, NE_EXPR,
6802 : boolean_type_node, ref,
6803 : zero);
6804 : }
6805 1028 : x = build2 (code, TREE_TYPE (ref2), ref2, new_var2);
6806 1028 : if (is_truth_op)
6807 24 : x = fold_convert (TREE_TYPE (new_var), x);
6808 1028 : ref = build_outer_var_ref (var, ctx);
6809 1028 : gimplify_assign (ref, x, dlist);
6810 : }
6811 4958 : if (allocator)
6812 61 : goto do_dtor;
6813 : }
6814 : }
6815 5571 : break;
6816 :
6817 0 : default:
6818 0 : gcc_unreachable ();
6819 : }
6820 : }
6821 : }
6822 77095 : if (tskred_avar)
6823 : {
6824 871 : tree clobber = build_clobber (TREE_TYPE (tskred_avar));
6825 871 : gimple_seq_add_stmt (ilist, gimple_build_assign (tskred_avar, clobber));
6826 : }
6827 :
6828 77095 : if (known_eq (sctx.max_vf, 1U))
6829 : {
6830 1772 : sctx.is_simt = false;
6831 1772 : if (ctx->lastprivate_conditional_map)
6832 : {
6833 52 : if (gimple_omp_for_combined_into_p (ctx->stmt))
6834 : {
6835 : /* Signal to lower_omp_1 that it should use parent context. */
6836 43 : ctx->combined_into_simd_safelen1 = true;
6837 370 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
6838 327 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6839 327 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
6840 : {
6841 57 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
6842 57 : omp_context *outer = ctx->outer;
6843 57 : if (gimple_code (outer->stmt) == GIMPLE_OMP_SCAN)
6844 6 : outer = outer->outer;
6845 57 : tree *v = ctx->lastprivate_conditional_map->get (o);
6846 57 : tree po = lookup_decl (OMP_CLAUSE_DECL (c), outer);
6847 57 : tree *pv = outer->lastprivate_conditional_map->get (po);
6848 57 : *v = *pv;
6849 : }
6850 : }
6851 : else
6852 : {
6853 : /* When not vectorized, treat lastprivate(conditional:) like
6854 : normal lastprivate, as there will be just one simd lane
6855 : writing the privatized variable. */
6856 9 : delete ctx->lastprivate_conditional_map;
6857 9 : ctx->lastprivate_conditional_map = NULL;
6858 : }
6859 : }
6860 : }
6861 :
6862 77095 : if (nonconst_simd_if)
6863 : {
6864 625 : if (sctx.lane == NULL_TREE)
6865 : {
6866 570 : sctx.idx = create_tmp_var (unsigned_type_node);
6867 570 : sctx.lane = create_tmp_var (unsigned_type_node);
6868 : }
6869 : /* FIXME: For now. */
6870 625 : sctx.is_simt = false;
6871 : }
6872 :
6873 77095 : if (sctx.lane || sctx.is_simt)
6874 : {
6875 4010 : uid = create_tmp_var (ptr_type_node, "simduid");
6876 : /* Don't want uninit warnings on simduid, it is always uninitialized,
6877 : but we use it not for the value, but for the DECL_UID only. */
6878 4010 : suppress_warning (uid, OPT_Wuninitialized);
6879 4010 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
6880 4010 : OMP_CLAUSE__SIMDUID__DECL (c) = uid;
6881 4010 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
6882 4010 : gimple_omp_for_set_clauses (ctx->stmt, c);
6883 : }
6884 : /* Emit calls denoting privatized variables and initializing a pointer to
6885 : structure that holds private variables as fields after ompdevlow pass. */
6886 77095 : if (sctx.is_simt)
6887 : {
6888 0 : sctx.simt_eargs[0] = uid;
6889 0 : gimple *g
6890 0 : = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, sctx.simt_eargs);
6891 0 : gimple_call_set_lhs (g, uid);
6892 0 : gimple_seq_add_stmt (ilist, g);
6893 0 : sctx.simt_eargs.release ();
6894 :
6895 0 : simtrec = create_tmp_var (ptr_type_node, ".omp_simt");
6896 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC, 1, uid);
6897 0 : gimple_call_set_lhs (g, simtrec);
6898 0 : gimple_seq_add_stmt (ilist, g);
6899 : }
6900 77095 : if (sctx.lane)
6901 : {
6902 7395 : gimple *g = gimple_build_call_internal (IFN_GOMP_SIMD_LANE,
6903 : 2 + (nonconst_simd_if != NULL),
6904 : uid, integer_zero_node,
6905 : nonconst_simd_if);
6906 4010 : gimple_call_set_lhs (g, sctx.lane);
6907 4010 : gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
6908 4010 : gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
6909 4010 : g = gimple_build_assign (sctx.lane, INTEGER_CST,
6910 : build_int_cst (unsigned_type_node, 0));
6911 4010 : gimple_seq_add_stmt (ilist, g);
6912 4010 : if (sctx.lastlane)
6913 : {
6914 184 : g = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
6915 : 2, uid, sctx.lane);
6916 184 : gimple_call_set_lhs (g, sctx.lastlane);
6917 184 : gimple_seq_add_stmt (dlist, g);
6918 184 : gimple_seq_add_seq (dlist, llist[3]);
6919 : }
6920 : /* Emit reductions across SIMT lanes in log_2(simt_vf) steps. */
6921 4010 : if (llist[2])
6922 : {
6923 0 : tree simt_vf = create_tmp_var (unsigned_type_node);
6924 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_VF, 0);
6925 0 : gimple_call_set_lhs (g, simt_vf);
6926 0 : gimple_seq_add_stmt (dlist, g);
6927 :
6928 0 : tree t = build_int_cst (unsigned_type_node, 1);
6929 0 : g = gimple_build_assign (simt_lane, INTEGER_CST, t);
6930 0 : gimple_seq_add_stmt (dlist, g);
6931 :
6932 0 : t = build_int_cst (unsigned_type_node, 0);
6933 0 : g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
6934 0 : gimple_seq_add_stmt (dlist, g);
6935 :
6936 0 : tree body = create_artificial_label (UNKNOWN_LOCATION);
6937 0 : tree header = create_artificial_label (UNKNOWN_LOCATION);
6938 0 : tree end = create_artificial_label (UNKNOWN_LOCATION);
6939 0 : gimple_seq_add_stmt (dlist, gimple_build_goto (header));
6940 0 : gimple_seq_add_stmt (dlist, gimple_build_label (body));
6941 :
6942 0 : gimple_seq_add_seq (dlist, llist[2]);
6943 :
6944 0 : g = gimple_build_assign (simt_lane, LSHIFT_EXPR, simt_lane, integer_one_node);
6945 0 : gimple_seq_add_stmt (dlist, g);
6946 :
6947 0 : gimple_seq_add_stmt (dlist, gimple_build_label (header));
6948 0 : g = gimple_build_cond (LT_EXPR, simt_lane, simt_vf, body, end);
6949 0 : gimple_seq_add_stmt (dlist, g);
6950 :
6951 0 : gimple_seq_add_stmt (dlist, gimple_build_label (end));
6952 : }
6953 12030 : for (int i = 0; i < 2; i++)
6954 8020 : if (llist[i])
6955 : {
6956 1716 : tree vf = create_tmp_var (unsigned_type_node);
6957 1716 : g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
6958 1716 : gimple_call_set_lhs (g, vf);
6959 1716 : gimple_seq *seq = i == 0 ? ilist : dlist;
6960 1716 : gimple_seq_add_stmt (seq, g);
6961 1716 : tree t = build_int_cst (unsigned_type_node, 0);
6962 1716 : g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
6963 1716 : gimple_seq_add_stmt (seq, g);
6964 1716 : tree body = create_artificial_label (UNKNOWN_LOCATION);
6965 1716 : tree header = create_artificial_label (UNKNOWN_LOCATION);
6966 1716 : tree end = create_artificial_label (UNKNOWN_LOCATION);
6967 1716 : gimple_seq_add_stmt (seq, gimple_build_goto (header));
6968 1716 : gimple_seq_add_stmt (seq, gimple_build_label (body));
6969 1716 : gimple_seq_add_seq (seq, llist[i]);
6970 1716 : t = build_int_cst (unsigned_type_node, 1);
6971 1716 : g = gimple_build_assign (sctx.idx, PLUS_EXPR, sctx.idx, t);
6972 1716 : gimple_seq_add_stmt (seq, g);
6973 1716 : gimple_seq_add_stmt (seq, gimple_build_label (header));
6974 1716 : g = gimple_build_cond (LT_EXPR, sctx.idx, vf, body, end);
6975 1716 : gimple_seq_add_stmt (seq, g);
6976 1716 : gimple_seq_add_stmt (seq, gimple_build_label (end));
6977 : }
6978 : }
6979 77095 : if (sctx.is_simt)
6980 : {
6981 0 : gimple_seq_add_seq (dlist, sctx.simt_dlist);
6982 0 : gimple *g
6983 0 : = gimple_build_call_internal (IFN_GOMP_SIMT_EXIT, 1, simtrec);
6984 0 : gimple_seq_add_stmt (dlist, g);
6985 : }
6986 :
6987 : /* The copyin sequence is not to be executed by the main thread, since
6988 : that would result in self-copies. Perhaps not visible to scalars,
6989 : but it certainly is to C++ operator=. */
6990 77095 : if (copyin_seq)
6991 : {
6992 436 : x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
6993 : 0);
6994 436 : x = build2 (NE_EXPR, boolean_type_node, x,
6995 436 : build_int_cst (TREE_TYPE (x), 0));
6996 436 : x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
6997 436 : gimplify_and_add (x, ilist);
6998 : }
6999 :
7000 : /* If any copyin variable is passed by reference, we must ensure the
7001 : master thread doesn't modify it before it is copied over in all
7002 : threads. Similarly for variables in both firstprivate and
7003 : lastprivate clauses we need to ensure the lastprivate copying
7004 : happens after firstprivate copying in all threads. And similarly
7005 : for UDRs if initializer expression refers to omp_orig. */
7006 77095 : if (copyin_by_ref || lastprivate_firstprivate
7007 75165 : || (reduction_omp_orig_ref
7008 89 : && !ctx->scan_inclusive
7009 89 : && !ctx->scan_exclusive))
7010 : {
7011 : /* Don't add any barrier for #pragma omp simd or
7012 : #pragma omp distribute. */
7013 2019 : if (!is_task_ctx (ctx)
7014 2019 : && (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
7015 1485 : || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR))
7016 882 : gimple_seq_add_stmt (ilist, omp_build_barrier (NULL_TREE));
7017 : }
7018 :
7019 : /* If max_vf is non-zero, then we can use only a vectorization factor
7020 : up to the max_vf we chose. So stick it into the safelen clause. */
7021 77095 : if (maybe_ne (sctx.max_vf, 0U))
7022 : {
7023 5212 : tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
7024 : OMP_CLAUSE_SAFELEN);
7025 5212 : poly_uint64 safe_len;
7026 5212 : if (c == NULL_TREE
7027 5212 : || (poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
7028 613 : && maybe_gt (safe_len, sctx.max_vf)))
7029 : {
7030 5122 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
7031 5122 : OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
7032 5122 : sctx.max_vf);
7033 5122 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
7034 5122 : gimple_omp_for_set_clauses (ctx->stmt, c);
7035 : }
7036 : }
7037 77095 : }
7038 :
7039 : /* Create temporary variables for lastprivate(conditional:) implementation
7040 : in context CTX with CLAUSES. */
7041 :
7042 : static void
7043 47257 : lower_lastprivate_conditional_clauses (tree *clauses, omp_context *ctx)
7044 : {
7045 47257 : tree iter_type = NULL_TREE;
7046 47257 : tree cond_ptr = NULL_TREE;
7047 47257 : tree iter_var = NULL_TREE;
7048 47257 : bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7049 47257 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
7050 47257 : tree next = *clauses;
7051 227024 : for (tree c = *clauses; c; c = OMP_CLAUSE_CHAIN (c))
7052 179767 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7053 179767 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
7054 : {
7055 386 : if (is_simd)
7056 : {
7057 107 : tree cc = omp_find_clause (next, OMP_CLAUSE__CONDTEMP_);
7058 107 : gcc_assert (cc);
7059 107 : if (iter_type == NULL_TREE)
7060 : {
7061 77 : iter_type = TREE_TYPE (OMP_CLAUSE_DECL (cc));
7062 77 : iter_var = create_tmp_var_raw (iter_type);
7063 77 : DECL_CONTEXT (iter_var) = current_function_decl;
7064 77 : DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
7065 77 : DECL_CHAIN (iter_var) = ctx->block_vars;
7066 77 : ctx->block_vars = iter_var;
7067 77 : tree c3
7068 77 : = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
7069 77 : OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
7070 77 : OMP_CLAUSE_DECL (c3) = iter_var;
7071 77 : OMP_CLAUSE_CHAIN (c3) = *clauses;
7072 77 : *clauses = c3;
7073 77 : ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
7074 : }
7075 107 : next = OMP_CLAUSE_CHAIN (cc);
7076 107 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7077 107 : tree v = lookup_decl (OMP_CLAUSE_DECL (cc), ctx);
7078 107 : ctx->lastprivate_conditional_map->put (o, v);
7079 107 : continue;
7080 107 : }
7081 279 : if (iter_type == NULL)
7082 : {
7083 205 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR)
7084 : {
7085 193 : struct omp_for_data fd;
7086 193 : omp_extract_for_data (as_a <gomp_for *> (ctx->stmt), &fd,
7087 : NULL);
7088 193 : iter_type = unsigned_type_for (fd.iter_type);
7089 : }
7090 12 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
7091 12 : iter_type = unsigned_type_node;
7092 205 : tree c2 = omp_find_clause (*clauses, OMP_CLAUSE__CONDTEMP_);
7093 205 : if (c2)
7094 : {
7095 103 : cond_ptr
7096 103 : = lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c2), ctx);
7097 103 : OMP_CLAUSE_DECL (c2) = cond_ptr;
7098 : }
7099 : else
7100 : {
7101 102 : cond_ptr = create_tmp_var_raw (build_pointer_type (iter_type));
7102 102 : DECL_CONTEXT (cond_ptr) = current_function_decl;
7103 102 : DECL_SEEN_IN_BIND_EXPR_P (cond_ptr) = 1;
7104 102 : DECL_CHAIN (cond_ptr) = ctx->block_vars;
7105 102 : ctx->block_vars = cond_ptr;
7106 102 : c2 = build_omp_clause (UNKNOWN_LOCATION,
7107 : OMP_CLAUSE__CONDTEMP_);
7108 102 : OMP_CLAUSE_DECL (c2) = cond_ptr;
7109 102 : OMP_CLAUSE_CHAIN (c2) = *clauses;
7110 102 : *clauses = c2;
7111 : }
7112 205 : iter_var = create_tmp_var_raw (iter_type);
7113 205 : DECL_CONTEXT (iter_var) = current_function_decl;
7114 205 : DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
7115 205 : DECL_CHAIN (iter_var) = ctx->block_vars;
7116 205 : ctx->block_vars = iter_var;
7117 205 : tree c3
7118 205 : = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
7119 205 : OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
7120 205 : OMP_CLAUSE_DECL (c3) = iter_var;
7121 205 : OMP_CLAUSE_CHAIN (c3) = OMP_CLAUSE_CHAIN (c2);
7122 205 : OMP_CLAUSE_CHAIN (c2) = c3;
7123 205 : ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
7124 : }
7125 279 : tree v = create_tmp_var_raw (iter_type);
7126 279 : DECL_CONTEXT (v) = current_function_decl;
7127 279 : DECL_SEEN_IN_BIND_EXPR_P (v) = 1;
7128 279 : DECL_CHAIN (v) = ctx->block_vars;
7129 279 : ctx->block_vars = v;
7130 279 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7131 279 : ctx->lastprivate_conditional_map->put (o, v);
7132 : }
7133 47257 : }
7134 :
7135 :
7136 : /* Generate code to implement the LASTPRIVATE clauses. This is used for
7137 : both parallel and workshare constructs. PREDICATE may be NULL if it's
7138 : always true. BODY_P is the sequence to insert early initialization
7139 : if needed, STMT_LIST is where the non-conditional lastprivate handling
7140 : goes into and CSTMT_LIST is a sequence that needs to be run in a critical
7141 : section. */
7142 :
7143 : static void
7144 47257 : lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *body_p,
7145 : gimple_seq *stmt_list, gimple_seq *cstmt_list,
7146 : omp_context *ctx)
7147 : {
7148 47257 : tree x, c, label = NULL, orig_clauses = clauses;
7149 47257 : bool par_clauses = false;
7150 47257 : tree simduid = NULL, lastlane = NULL, simtcond = NULL, simtlast = NULL;
7151 47257 : unsigned HOST_WIDE_INT conditional_off = 0;
7152 47257 : gimple_seq post_stmt_list = NULL;
7153 :
7154 : /* Early exit if there are no lastprivate or linear clauses. */
7155 196284 : for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
7156 161509 : if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
7157 161509 : || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
7158 6763 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
7159 : break;
7160 47257 : if (clauses == NULL)
7161 : {
7162 : /* If this was a workshare clause, see if it had been combined
7163 : with its parallel. In that case, look for the clauses on the
7164 : parallel statement itself. */
7165 34775 : if (is_parallel_ctx (ctx))
7166 30776 : return;
7167 :
7168 34775 : ctx = ctx->outer;
7169 34775 : if (ctx == NULL || !is_parallel_ctx (ctx))
7170 : return;
7171 :
7172 12249 : clauses = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
7173 : OMP_CLAUSE_LASTPRIVATE);
7174 12249 : if (clauses == NULL)
7175 : return;
7176 : par_clauses = true;
7177 : }
7178 :
7179 16481 : bool maybe_simt = false;
7180 16481 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7181 16481 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
7182 : {
7183 7424 : maybe_simt = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMT_);
7184 7424 : simduid = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
7185 7424 : if (simduid)
7186 3558 : simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
7187 : }
7188 :
7189 16481 : if (predicate)
7190 : {
7191 16362 : gcond *stmt;
7192 16362 : tree label_true, arm1, arm2;
7193 16362 : enum tree_code pred_code = TREE_CODE (predicate);
7194 :
7195 16362 : label = create_artificial_label (UNKNOWN_LOCATION);
7196 16362 : label_true = create_artificial_label (UNKNOWN_LOCATION);
7197 16362 : if (TREE_CODE_CLASS (pred_code) == tcc_comparison)
7198 : {
7199 16362 : arm1 = TREE_OPERAND (predicate, 0);
7200 16362 : arm2 = TREE_OPERAND (predicate, 1);
7201 16362 : gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
7202 16362 : gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
7203 : }
7204 : else
7205 : {
7206 0 : arm1 = predicate;
7207 0 : gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
7208 0 : arm2 = boolean_false_node;
7209 0 : pred_code = NE_EXPR;
7210 : }
7211 16362 : if (maybe_simt)
7212 : {
7213 0 : c = build2 (pred_code, boolean_type_node, arm1, arm2);
7214 0 : c = fold_convert (integer_type_node, c);
7215 0 : simtcond = create_tmp_var (integer_type_node);
7216 0 : gimplify_assign (simtcond, c, stmt_list);
7217 0 : gcall *g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY,
7218 : 1, simtcond);
7219 0 : c = create_tmp_var (integer_type_node);
7220 0 : gimple_call_set_lhs (g, c);
7221 0 : gimple_seq_add_stmt (stmt_list, g);
7222 0 : stmt = gimple_build_cond (NE_EXPR, c, integer_zero_node,
7223 : label_true, label);
7224 : }
7225 : else
7226 16362 : stmt = gimple_build_cond (pred_code, arm1, arm2, label_true, label);
7227 16362 : gimple_seq_add_stmt (stmt_list, stmt);
7228 16362 : gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
7229 : }
7230 :
7231 16481 : tree cond_ptr = NULL_TREE;
7232 61575 : for (c = clauses; c ;)
7233 : {
7234 56348 : tree var, new_var;
7235 56348 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7236 56348 : gimple_seq *this_stmt_list = stmt_list;
7237 56348 : tree lab2 = NULL_TREE;
7238 :
7239 56348 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7240 21813 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
7241 386 : && ctx->lastprivate_conditional_map
7242 56721 : && !ctx->combined_into_simd_safelen1)
7243 : {
7244 316 : gcc_assert (body_p);
7245 316 : if (simduid)
7246 37 : goto next;
7247 279 : if (cond_ptr == NULL_TREE)
7248 : {
7249 205 : cond_ptr = omp_find_clause (orig_clauses, OMP_CLAUSE__CONDTEMP_);
7250 205 : cond_ptr = OMP_CLAUSE_DECL (cond_ptr);
7251 : }
7252 279 : tree type = TREE_TYPE (TREE_TYPE (cond_ptr));
7253 279 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7254 279 : tree v = *ctx->lastprivate_conditional_map->get (o);
7255 279 : gimplify_assign (v, build_zero_cst (type), body_p);
7256 279 : this_stmt_list = cstmt_list;
7257 279 : tree mem;
7258 279 : if (POINTER_TYPE_P (TREE_TYPE (cond_ptr)))
7259 : {
7260 136 : mem = build2 (MEM_REF, type, cond_ptr,
7261 136 : build_int_cst (TREE_TYPE (cond_ptr),
7262 136 : conditional_off));
7263 136 : conditional_off += tree_to_uhwi (TYPE_SIZE_UNIT (type));
7264 : }
7265 : else
7266 143 : mem = build4 (ARRAY_REF, type, cond_ptr,
7267 143 : size_int (conditional_off++), NULL_TREE, NULL_TREE);
7268 279 : tree mem2 = copy_node (mem);
7269 279 : gimple_seq seq = NULL;
7270 279 : mem = force_gimple_operand (mem, &seq, true, NULL_TREE);
7271 279 : gimple_seq_add_seq (this_stmt_list, seq);
7272 279 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
7273 279 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
7274 279 : gimple *g = gimple_build_cond (GT_EXPR, v, mem, lab1, lab2);
7275 279 : gimple_seq_add_stmt (this_stmt_list, g);
7276 279 : gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab1));
7277 279 : gimplify_assign (mem2, v, this_stmt_list);
7278 : }
7279 56032 : else if (predicate
7280 55794 : && ctx->combined_into_simd_safelen1
7281 116 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7282 57 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
7283 56089 : && ctx->lastprivate_conditional_map)
7284 : this_stmt_list = &post_stmt_list;
7285 :
7286 56311 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7287 56311 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
7288 4867 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
7289 : {
7290 26643 : var = OMP_CLAUSE_DECL (c);
7291 26643 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7292 21776 : && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)
7293 27670 : && is_taskloop_ctx (ctx))
7294 : {
7295 312 : gcc_checking_assert (ctx->outer && is_task_ctx (ctx->outer));
7296 312 : new_var = lookup_decl (var, ctx->outer);
7297 : }
7298 : else
7299 : {
7300 26331 : new_var = lookup_decl (var, ctx);
7301 : /* Avoid uninitialized warnings for lastprivate and
7302 : for linear iterators. */
7303 26331 : if (predicate
7304 52473 : && (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7305 4867 : || OMP_CLAUSE_LINEAR_NO_COPYIN (c)))
7306 24873 : suppress_warning (new_var, OPT_Wuninitialized);
7307 : }
7308 :
7309 26643 : if (!maybe_simt && simduid && DECL_HAS_VALUE_EXPR_P (new_var))
7310 : {
7311 5879 : tree val = DECL_VALUE_EXPR (new_var);
7312 5879 : if (TREE_CODE (val) == ARRAY_REF
7313 5831 : && VAR_P (TREE_OPERAND (val, 0))
7314 11710 : && lookup_attribute ("omp simd array",
7315 5831 : DECL_ATTRIBUTES (TREE_OPERAND (val,
7316 : 0))))
7317 : {
7318 5831 : if (lastlane == NULL)
7319 : {
7320 2766 : lastlane = create_tmp_var (unsigned_type_node);
7321 2766 : gcall *g
7322 2766 : = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
7323 : 2, simduid,
7324 2766 : TREE_OPERAND (val, 1));
7325 2766 : gimple_call_set_lhs (g, lastlane);
7326 2766 : gimple_seq_add_stmt (this_stmt_list, g);
7327 : }
7328 5831 : new_var = build4 (ARRAY_REF, TREE_TYPE (val),
7329 5831 : TREE_OPERAND (val, 0), lastlane,
7330 : NULL_TREE, NULL_TREE);
7331 5831 : TREE_THIS_NOTRAP (new_var) = 1;
7332 : }
7333 : }
7334 20764 : else if (maybe_simt)
7335 : {
7336 0 : tree val = (DECL_HAS_VALUE_EXPR_P (new_var)
7337 0 : ? DECL_VALUE_EXPR (new_var)
7338 : : new_var);
7339 0 : if (simtlast == NULL)
7340 : {
7341 0 : simtlast = create_tmp_var (unsigned_type_node);
7342 0 : gcall *g = gimple_build_call_internal
7343 0 : (IFN_GOMP_SIMT_LAST_LANE, 1, simtcond);
7344 0 : gimple_call_set_lhs (g, simtlast);
7345 0 : gimple_seq_add_stmt (this_stmt_list, g);
7346 : }
7347 0 : x = build_call_expr_internal_loc
7348 0 : (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_IDX,
7349 0 : TREE_TYPE (val), 2, val, simtlast);
7350 0 : new_var = unshare_expr (new_var);
7351 0 : gimplify_assign (new_var, x, this_stmt_list);
7352 0 : new_var = unshare_expr (new_var);
7353 : }
7354 :
7355 26643 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7356 26643 : && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
7357 : {
7358 7159 : lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
7359 7159 : gimple_seq_add_seq (this_stmt_list,
7360 7159 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
7361 7159 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
7362 : }
7363 19484 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
7364 19484 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
7365 : {
7366 426 : lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
7367 426 : gimple_seq_add_seq (this_stmt_list,
7368 426 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
7369 426 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
7370 : }
7371 :
7372 26643 : x = NULL_TREE;
7373 26643 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7374 21776 : && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c)
7375 26703 : && is_taskloop_ctx (ctx))
7376 : {
7377 76 : tree ovar = maybe_lookup_decl_in_outer_ctx (var,
7378 38 : ctx->outer->outer);
7379 38 : if (is_global_var (ovar))
7380 7 : x = ovar;
7381 : }
7382 7 : if (!x)
7383 26636 : x = build_outer_var_ref (var, ctx, OMP_CLAUSE_LASTPRIVATE);
7384 26643 : if (omp_privatize_by_reference (var))
7385 657 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7386 26643 : x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
7387 26643 : gimplify_and_add (x, this_stmt_list);
7388 :
7389 26643 : if (lab2)
7390 279 : gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab2));
7391 : }
7392 :
7393 56348 : next:
7394 56348 : c = OMP_CLAUSE_CHAIN (c);
7395 56348 : if (c == NULL && !par_clauses)
7396 : {
7397 : /* If this was a workshare clause, see if it had been combined
7398 : with its parallel. In that case, continue looking for the
7399 : clauses also on the parallel statement itself. */
7400 12482 : if (is_parallel_ctx (ctx))
7401 : break;
7402 :
7403 12482 : ctx = ctx->outer;
7404 12482 : if (ctx == NULL || !is_parallel_ctx (ctx))
7405 : break;
7406 :
7407 1228 : c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
7408 : OMP_CLAUSE_LASTPRIVATE);
7409 1228 : par_clauses = true;
7410 : }
7411 : }
7412 :
7413 16481 : if (label)
7414 16362 : gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
7415 16481 : gimple_seq_add_seq (stmt_list, post_stmt_list);
7416 : }
7417 :
7418 : /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
7419 : (which might be a placeholder). INNER is true if this is an inner
7420 : axis of a multi-axis loop. FORK and JOIN are (optional) fork and
7421 : join markers. Generate the before-loop forking sequence in
7422 : FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
7423 : general form of these sequences is
7424 :
7425 : GOACC_REDUCTION_SETUP
7426 : GOACC_FORK
7427 : GOACC_REDUCTION_INIT
7428 : ...
7429 : GOACC_REDUCTION_FINI
7430 : GOACC_JOIN
7431 : GOACC_REDUCTION_TEARDOWN. */
7432 :
7433 : static void
7434 25942 : lower_oacc_reductions (location_t loc, tree clauses, tree level, bool inner,
7435 : gcall *fork, gcall *private_marker, gcall *join,
7436 : gimple_seq *fork_seq, gimple_seq *join_seq,
7437 : omp_context *ctx)
7438 : {
7439 25942 : gimple_seq before_fork = NULL;
7440 25942 : gimple_seq after_fork = NULL;
7441 25942 : gimple_seq before_join = NULL;
7442 25942 : gimple_seq after_join = NULL;
7443 25942 : tree init_code = NULL_TREE, fini_code = NULL_TREE,
7444 25942 : setup_code = NULL_TREE, teardown_code = NULL_TREE;
7445 25942 : unsigned offset = 0;
7446 :
7447 96490 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
7448 70548 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
7449 : {
7450 : /* No 'reduction' clauses on OpenACC 'kernels'. */
7451 7751 : gcc_checking_assert (!is_oacc_kernels (ctx));
7452 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
7453 7751 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
7454 :
7455 7751 : tree orig = OMP_CLAUSE_DECL (c);
7456 7751 : tree var = maybe_lookup_decl (orig, ctx);
7457 7751 : tree ref_to_res = NULL_TREE;
7458 7751 : tree incoming, outgoing, v1, v2, v3;
7459 7751 : bool is_private = false;
7460 :
7461 7751 : enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
7462 7751 : if (rcode == MINUS_EXPR)
7463 : rcode = PLUS_EXPR;
7464 6609 : else if (rcode == TRUTH_ANDIF_EXPR)
7465 : rcode = BIT_AND_EXPR;
7466 6431 : else if (rcode == TRUTH_ORIF_EXPR)
7467 433 : rcode = BIT_IOR_EXPR;
7468 7751 : tree op = build_int_cst (unsigned_type_node, rcode);
7469 :
7470 7751 : if (!var)
7471 4 : var = orig;
7472 :
7473 7751 : incoming = outgoing = var;
7474 :
7475 7751 : if (!inner)
7476 : {
7477 : /* See if an outer construct also reduces this variable. */
7478 : omp_context *outer = ctx;
7479 :
7480 7385 : while (omp_context *probe = outer->outer)
7481 : {
7482 4708 : enum gimple_code type = gimple_code (probe->stmt);
7483 4708 : tree cls;
7484 :
7485 4708 : switch (type)
7486 : {
7487 2220 : case GIMPLE_OMP_FOR:
7488 2220 : cls = gimple_omp_for_clauses (probe->stmt);
7489 2220 : break;
7490 :
7491 2488 : case GIMPLE_OMP_TARGET:
7492 : /* No 'reduction' clauses inside OpenACC 'kernels'
7493 : regions. */
7494 2488 : gcc_checking_assert (!is_oacc_kernels (probe));
7495 :
7496 2488 : if (!is_gimple_omp_offloaded (probe->stmt))
7497 26 : goto do_lookup;
7498 :
7499 2462 : cls = gimple_omp_target_clauses (probe->stmt);
7500 2462 : break;
7501 :
7502 0 : default:
7503 0 : goto do_lookup;
7504 : }
7505 :
7506 14249 : outer = probe;
7507 14249 : for (; cls; cls = OMP_CLAUSE_CHAIN (cls))
7508 11632 : if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_REDUCTION
7509 11632 : && orig == OMP_CLAUSE_DECL (cls))
7510 : {
7511 1772 : incoming = outgoing = lookup_decl (orig, probe);
7512 1772 : goto has_outer_reduction;
7513 : }
7514 9860 : else if ((OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_FIRSTPRIVATE
7515 8971 : || OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_PRIVATE)
7516 11164 : && orig == OMP_CLAUSE_DECL (cls))
7517 : {
7518 293 : is_private = true;
7519 293 : goto do_lookup;
7520 : }
7521 : }
7522 :
7523 2677 : do_lookup:
7524 : /* This is the outermost construct with this reduction,
7525 : see if there's a mapping for it. */
7526 2996 : if (gimple_code (outer->stmt) == GIMPLE_OMP_TARGET
7527 5856 : && maybe_lookup_field (orig, outer) && !is_private)
7528 : {
7529 2553 : ref_to_res = build_receiver_ref (orig, false, outer);
7530 2553 : if (omp_privatize_by_reference (orig))
7531 79 : ref_to_res = build_simple_mem_ref (ref_to_res);
7532 :
7533 2553 : tree type = TREE_TYPE (var);
7534 2553 : if (POINTER_TYPE_P (type))
7535 79 : type = TREE_TYPE (type);
7536 :
7537 2553 : outgoing = var;
7538 2553 : incoming = omp_reduction_init_op (loc, rcode, type);
7539 : }
7540 : else
7541 : {
7542 : /* Try to look at enclosing contexts for reduction var,
7543 : use original if no mapping found. */
7544 443 : tree t = NULL_TREE;
7545 443 : omp_context *c = ctx->outer;
7546 895 : while (c && !t)
7547 : {
7548 452 : t = maybe_lookup_decl (orig, c);
7549 452 : c = c->outer;
7550 : }
7551 443 : incoming = outgoing = (t ? t : orig);
7552 : }
7553 :
7554 2553 : has_outer_reduction:;
7555 : }
7556 :
7557 4325 : if (!ref_to_res)
7558 5198 : ref_to_res = integer_zero_node;
7559 :
7560 7751 : if (omp_privatize_by_reference (orig))
7561 : {
7562 159 : tree type = TREE_TYPE (var);
7563 159 : const char *id = IDENTIFIER_POINTER (DECL_NAME (var));
7564 :
7565 159 : if (!inner)
7566 : {
7567 110 : tree x = create_tmp_var (TREE_TYPE (type), id);
7568 110 : gimplify_assign (var, build_fold_addr_expr (x), fork_seq);
7569 : }
7570 :
7571 159 : v1 = create_tmp_var (type, id);
7572 159 : v2 = create_tmp_var (type, id);
7573 159 : v3 = create_tmp_var (type, id);
7574 :
7575 159 : gimplify_assign (v1, var, fork_seq);
7576 159 : gimplify_assign (v2, var, fork_seq);
7577 159 : gimplify_assign (v3, var, fork_seq);
7578 :
7579 159 : var = build_simple_mem_ref (var);
7580 159 : v1 = build_simple_mem_ref (v1);
7581 159 : v2 = build_simple_mem_ref (v2);
7582 159 : v3 = build_simple_mem_ref (v3);
7583 159 : outgoing = build_simple_mem_ref (outgoing);
7584 :
7585 159 : if (!TREE_CONSTANT (incoming))
7586 80 : incoming = build_simple_mem_ref (incoming);
7587 : }
7588 : else
7589 : /* Note that 'var' might be a mem ref. */
7590 : v1 = v2 = v3 = var;
7591 :
7592 : /* Determine position in reduction buffer, which may be used
7593 : by target. The parser has ensured that this is not a
7594 : variable-sized type. */
7595 7751 : fixed_size_mode mode
7596 7751 : = as_a <fixed_size_mode> (TYPE_MODE (TREE_TYPE (var)));
7597 7751 : unsigned align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
7598 7751 : offset = (offset + align - 1) & ~(align - 1);
7599 7751 : tree off = build_int_cst (sizetype, offset);
7600 7751 : offset += GET_MODE_SIZE (mode);
7601 :
7602 7751 : if (!init_code)
7603 : {
7604 6969 : init_code = build_int_cst (integer_type_node,
7605 : IFN_GOACC_REDUCTION_INIT);
7606 6969 : fini_code = build_int_cst (integer_type_node,
7607 : IFN_GOACC_REDUCTION_FINI);
7608 6969 : setup_code = build_int_cst (integer_type_node,
7609 : IFN_GOACC_REDUCTION_SETUP);
7610 6969 : teardown_code = build_int_cst (integer_type_node,
7611 : IFN_GOACC_REDUCTION_TEARDOWN);
7612 : }
7613 :
7614 7751 : tree setup_call
7615 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7616 7751 : TREE_TYPE (var), 6, setup_code,
7617 : unshare_expr (ref_to_res),
7618 : unshare_expr (incoming),
7619 : level, op, off);
7620 7751 : tree init_call
7621 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7622 7751 : TREE_TYPE (var), 6, init_code,
7623 : unshare_expr (ref_to_res),
7624 : unshare_expr (v1), level, op, off);
7625 7751 : tree fini_call
7626 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7627 7751 : TREE_TYPE (var), 6, fini_code,
7628 : unshare_expr (ref_to_res),
7629 : unshare_expr (v2), level, op, off);
7630 7751 : tree teardown_call
7631 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7632 7751 : TREE_TYPE (var), 6, teardown_code,
7633 : ref_to_res, unshare_expr (v3),
7634 : level, op, off);
7635 :
7636 7751 : gimplify_assign (unshare_expr (v1), setup_call, &before_fork);
7637 7751 : gimplify_assign (unshare_expr (v2), init_call, &after_fork);
7638 7751 : gimplify_assign (unshare_expr (v3), fini_call, &before_join);
7639 7751 : gimplify_assign (unshare_expr (outgoing), teardown_call, &after_join);
7640 : }
7641 :
7642 : /* Now stitch things together. */
7643 25942 : gimple_seq_add_seq (fork_seq, before_fork);
7644 25942 : if (private_marker)
7645 259 : gimple_seq_add_stmt (fork_seq, private_marker);
7646 25942 : if (fork)
7647 16551 : gimple_seq_add_stmt (fork_seq, fork);
7648 25942 : gimple_seq_add_seq (fork_seq, after_fork);
7649 :
7650 25942 : gimple_seq_add_seq (join_seq, before_join);
7651 25942 : if (join)
7652 16551 : gimple_seq_add_stmt (join_seq, join);
7653 25942 : gimple_seq_add_seq (join_seq, after_join);
7654 25942 : }
7655 :
7656 : /* Generate code to implement the REDUCTION clauses, append it
7657 : to STMT_SEQP. CLIST if non-NULL is a pointer to a sequence
7658 : that should be emitted also inside of the critical section,
7659 : in that case clear *CLIST afterwards, otherwise leave it as is
7660 : and let the caller emit it itself. */
7661 :
7662 : static void
7663 71888 : lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp,
7664 : gimple_seq *clist, omp_context *ctx)
7665 : {
7666 71888 : gimple_seq sub_seq = NULL;
7667 71888 : gimple *stmt;
7668 71888 : tree x, c;
7669 71888 : int count = 0;
7670 :
7671 : /* OpenACC loop reductions are handled elsewhere. */
7672 71888 : if (is_gimple_omp_oacc (ctx->stmt))
7673 70863 : return;
7674 :
7675 : /* SIMD reductions are handled in lower_rec_input_clauses. */
7676 60565 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7677 60565 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
7678 : return;
7679 :
7680 : /* inscan reductions are handled elsewhere. */
7681 51210 : if (ctx->scan_inclusive || ctx->scan_exclusive)
7682 : return;
7683 :
7684 : /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
7685 : update in that case, otherwise use a lock. */
7686 242020 : for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
7687 191831 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
7688 191831 : && !OMP_CLAUSE_REDUCTION_TASK (c))
7689 : {
7690 4581 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
7691 4581 : || TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
7692 : {
7693 : /* Never use OMP_ATOMIC for array reductions or UDRs. */
7694 : count = -1;
7695 : break;
7696 : }
7697 3733 : count++;
7698 : }
7699 :
7700 51037 : if (count == 0)
7701 : return;
7702 :
7703 14912 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
7704 : {
7705 13887 : tree var, ref, new_var, orig_var;
7706 13887 : enum tree_code code;
7707 13887 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7708 :
7709 22381 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
7710 13887 : || OMP_CLAUSE_REDUCTION_TASK (c))
7711 8494 : continue;
7712 :
7713 5393 : enum omp_clause_code ccode = OMP_CLAUSE_REDUCTION;
7714 5393 : orig_var = var = OMP_CLAUSE_DECL (c);
7715 5393 : if (TREE_CODE (var) == MEM_REF)
7716 : {
7717 689 : var = TREE_OPERAND (var, 0);
7718 689 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
7719 50 : var = TREE_OPERAND (var, 0);
7720 689 : if (TREE_CODE (var) == ADDR_EXPR)
7721 304 : var = TREE_OPERAND (var, 0);
7722 : else
7723 : {
7724 : /* If this is a pointer or referenced based array
7725 : section, the var could be private in the outer
7726 : context e.g. on orphaned loop construct. Pretend this
7727 : is private variable's outer reference. */
7728 385 : ccode = OMP_CLAUSE_PRIVATE;
7729 385 : if (INDIRECT_REF_P (var))
7730 54 : var = TREE_OPERAND (var, 0);
7731 : }
7732 689 : orig_var = var;
7733 689 : if (is_variable_sized (var))
7734 : {
7735 25 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
7736 25 : var = DECL_VALUE_EXPR (var);
7737 25 : gcc_assert (INDIRECT_REF_P (var));
7738 25 : var = TREE_OPERAND (var, 0);
7739 25 : gcc_assert (DECL_P (var));
7740 : }
7741 : }
7742 5393 : new_var = lookup_decl (var, ctx);
7743 5393 : if (var == OMP_CLAUSE_DECL (c)
7744 5393 : && omp_privatize_by_reference (var))
7745 152 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7746 5393 : ref = build_outer_var_ref (var, ctx, ccode);
7747 5393 : code = OMP_CLAUSE_REDUCTION_CODE (c);
7748 :
7749 : /* reduction(-:var) sums up the partial results, so it acts
7750 : identically to reduction(+:var). */
7751 5393 : if (code == MINUS_EXPR)
7752 57 : code = PLUS_EXPR;
7753 :
7754 5393 : bool is_truth_op = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
7755 5393 : if (count == 1)
7756 : {
7757 3324 : tree addr = build_fold_addr_expr_loc (clause_loc, ref);
7758 :
7759 3324 : addr = save_expr (addr);
7760 3324 : ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
7761 3324 : tree new_var2 = new_var;
7762 3324 : tree ref2 = ref;
7763 3324 : if (is_truth_op)
7764 : {
7765 784 : tree zero = build_zero_cst (TREE_TYPE (new_var));
7766 784 : new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
7767 : boolean_type_node, new_var, zero);
7768 784 : ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
7769 : ref, zero);
7770 : }
7771 3324 : x = fold_build2_loc (clause_loc, code, TREE_TYPE (new_var2), ref2,
7772 : new_var2);
7773 3324 : if (is_truth_op)
7774 784 : x = fold_convert (TREE_TYPE (new_var), x);
7775 3324 : x = build2 (OMP_ATOMIC, void_type_node, addr, x);
7776 3324 : OMP_ATOMIC_MEMORY_ORDER (x) = OMP_MEMORY_ORDER_RELAXED;
7777 3324 : gimplify_and_add (x, stmt_seqp);
7778 3324 : return;
7779 : }
7780 2069 : else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
7781 : {
7782 689 : tree d = OMP_CLAUSE_DECL (c);
7783 689 : tree type = TREE_TYPE (d);
7784 689 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
7785 689 : tree i = create_tmp_var (TREE_TYPE (v));
7786 689 : tree ptype = build_pointer_type (TREE_TYPE (type));
7787 689 : tree bias = TREE_OPERAND (d, 1);
7788 689 : d = TREE_OPERAND (d, 0);
7789 689 : if (TREE_CODE (d) == POINTER_PLUS_EXPR)
7790 : {
7791 50 : tree b = TREE_OPERAND (d, 1);
7792 50 : b = maybe_lookup_decl (b, ctx);
7793 50 : if (b == NULL)
7794 : {
7795 2 : b = TREE_OPERAND (d, 1);
7796 2 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
7797 : }
7798 50 : if (integer_zerop (bias))
7799 : bias = b;
7800 : else
7801 : {
7802 0 : bias = fold_convert_loc (clause_loc, TREE_TYPE (b), bias);
7803 0 : bias = fold_build2_loc (clause_loc, PLUS_EXPR,
7804 0 : TREE_TYPE (b), b, bias);
7805 : }
7806 50 : d = TREE_OPERAND (d, 0);
7807 : }
7808 : /* For ref build_outer_var_ref already performs this, so
7809 : only new_var needs a dereference. */
7810 689 : if (INDIRECT_REF_P (d))
7811 : {
7812 54 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7813 54 : gcc_assert (omp_privatize_by_reference (var)
7814 : && var == orig_var);
7815 : }
7816 635 : else if (TREE_CODE (d) == ADDR_EXPR)
7817 : {
7818 304 : if (orig_var == var)
7819 : {
7820 279 : new_var = build_fold_addr_expr (new_var);
7821 279 : ref = build_fold_addr_expr (ref);
7822 : }
7823 : }
7824 : else
7825 : {
7826 331 : gcc_assert (orig_var == var);
7827 331 : if (omp_privatize_by_reference (var))
7828 79 : ref = build_fold_addr_expr (ref);
7829 : }
7830 689 : if (DECL_P (v))
7831 : {
7832 93 : tree t = maybe_lookup_decl (v, ctx);
7833 93 : if (t)
7834 88 : v = t;
7835 : else
7836 5 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
7837 93 : gimplify_expr (&v, stmt_seqp, NULL, is_gimple_val, fb_rvalue);
7838 : }
7839 689 : if (!integer_zerop (bias))
7840 : {
7841 402 : bias = fold_convert_loc (clause_loc, sizetype, bias);
7842 804 : new_var = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
7843 402 : TREE_TYPE (new_var), new_var,
7844 : unshare_expr (bias));
7845 402 : ref = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
7846 402 : TREE_TYPE (ref), ref, bias);
7847 : }
7848 689 : new_var = fold_convert_loc (clause_loc, ptype, new_var);
7849 689 : ref = fold_convert_loc (clause_loc, ptype, ref);
7850 689 : tree m = create_tmp_var (ptype);
7851 689 : gimplify_assign (m, new_var, stmt_seqp);
7852 689 : new_var = m;
7853 689 : m = create_tmp_var (ptype);
7854 689 : gimplify_assign (m, ref, stmt_seqp);
7855 689 : ref = m;
7856 689 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), stmt_seqp);
7857 689 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7858 689 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7859 689 : gimple_seq_add_stmt (&sub_seq, gimple_build_label (body));
7860 689 : tree priv = build_simple_mem_ref_loc (clause_loc, new_var);
7861 689 : tree out = build_simple_mem_ref_loc (clause_loc, ref);
7862 689 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
7863 : {
7864 72 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
7865 72 : tree decl_placeholder
7866 72 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
7867 72 : SET_DECL_VALUE_EXPR (placeholder, out);
7868 72 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
7869 72 : SET_DECL_VALUE_EXPR (decl_placeholder, priv);
7870 72 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
7871 72 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
7872 72 : gimple_seq_add_seq (&sub_seq,
7873 72 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
7874 72 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
7875 72 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
7876 72 : OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
7877 : }
7878 : else
7879 : {
7880 617 : tree out2 = out;
7881 617 : tree priv2 = priv;
7882 617 : if (is_truth_op)
7883 : {
7884 0 : tree zero = build_zero_cst (TREE_TYPE (out));
7885 0 : out2 = fold_build2_loc (clause_loc, NE_EXPR,
7886 : boolean_type_node, out, zero);
7887 0 : priv2 = fold_build2_loc (clause_loc, NE_EXPR,
7888 : boolean_type_node, priv, zero);
7889 : }
7890 617 : x = build2 (code, TREE_TYPE (out2), out2, priv2);
7891 617 : if (is_truth_op)
7892 0 : x = fold_convert (TREE_TYPE (out), x);
7893 617 : out = unshare_expr (out);
7894 617 : gimplify_assign (out, x, &sub_seq);
7895 : }
7896 689 : gimple *g = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
7897 689 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
7898 689 : gimple_seq_add_stmt (&sub_seq, g);
7899 689 : g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
7900 689 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
7901 689 : gimple_seq_add_stmt (&sub_seq, g);
7902 689 : g = gimple_build_assign (i, PLUS_EXPR, i,
7903 689 : build_int_cst (TREE_TYPE (i), 1));
7904 689 : gimple_seq_add_stmt (&sub_seq, g);
7905 689 : g = gimple_build_cond (LE_EXPR, i, v, body, end);
7906 689 : gimple_seq_add_stmt (&sub_seq, g);
7907 689 : gimple_seq_add_stmt (&sub_seq, gimple_build_label (end));
7908 : }
7909 1380 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
7910 : {
7911 774 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
7912 :
7913 774 : if (omp_privatize_by_reference (var)
7914 831 : && !useless_type_conversion_p (TREE_TYPE (placeholder),
7915 57 : TREE_TYPE (ref)))
7916 49 : ref = build_fold_addr_expr_loc (clause_loc, ref);
7917 774 : SET_DECL_VALUE_EXPR (placeholder, ref);
7918 774 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
7919 774 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
7920 774 : gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
7921 774 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
7922 774 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
7923 : }
7924 : else
7925 : {
7926 606 : tree new_var2 = new_var;
7927 606 : tree ref2 = ref;
7928 606 : if (is_truth_op)
7929 : {
7930 78 : tree zero = build_zero_cst (TREE_TYPE (new_var));
7931 78 : new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
7932 : boolean_type_node, new_var, zero);
7933 78 : ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
7934 : ref, zero);
7935 : }
7936 606 : x = build2 (code, TREE_TYPE (ref), ref2, new_var2);
7937 606 : if (is_truth_op)
7938 78 : x = fold_convert (TREE_TYPE (new_var), x);
7939 606 : ref = build_outer_var_ref (var, ctx);
7940 606 : gimplify_assign (ref, x, &sub_seq);
7941 : }
7942 : }
7943 :
7944 1025 : stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START),
7945 : 0);
7946 1025 : gimple_seq_add_stmt (stmt_seqp, stmt);
7947 :
7948 1025 : gimple_seq_add_seq (stmt_seqp, sub_seq);
7949 :
7950 1025 : if (clist)
7951 : {
7952 157 : gimple_seq_add_seq (stmt_seqp, *clist);
7953 157 : *clist = NULL;
7954 : }
7955 :
7956 1025 : stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
7957 : 0);
7958 1025 : gimple_seq_add_stmt (stmt_seqp, stmt);
7959 : }
7960 :
7961 :
7962 : /* Generate code to implement the COPYPRIVATE clauses. */
7963 :
7964 : static void
7965 115 : lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
7966 : omp_context *ctx)
7967 : {
7968 115 : tree c;
7969 :
7970 463 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
7971 : {
7972 348 : tree var, new_var, ref, x;
7973 348 : bool by_ref;
7974 348 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7975 :
7976 348 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
7977 6 : continue;
7978 :
7979 342 : var = OMP_CLAUSE_DECL (c);
7980 342 : by_ref = use_pointer_for_field (var, NULL);
7981 :
7982 342 : ref = build_sender_ref (var, ctx);
7983 342 : x = new_var = lookup_decl_in_outer_ctx (var, ctx);
7984 342 : if (by_ref)
7985 : {
7986 74 : x = build_fold_addr_expr_loc (clause_loc, new_var);
7987 74 : x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
7988 : }
7989 342 : gimplify_assign (ref, x, slist);
7990 :
7991 342 : ref = build_receiver_ref (var, false, ctx);
7992 342 : if (by_ref)
7993 : {
7994 74 : ref = fold_convert_loc (clause_loc,
7995 74 : build_pointer_type (TREE_TYPE (new_var)),
7996 : ref);
7997 74 : ref = build_fold_indirect_ref_loc (clause_loc, ref);
7998 : }
7999 342 : if (omp_privatize_by_reference (var))
8000 : {
8001 171 : ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
8002 171 : ref = build_simple_mem_ref_loc (clause_loc, ref);
8003 171 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
8004 : }
8005 342 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
8006 342 : gimplify_and_add (x, rlist);
8007 : }
8008 115 : }
8009 :
8010 :
8011 : /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
8012 : and REDUCTION from the sender (aka parent) side. */
8013 :
8014 : static void
8015 22355 : lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
8016 : omp_context *ctx)
8017 : {
8018 22355 : tree c, t;
8019 22355 : int ignored_looptemp = 0;
8020 22355 : bool is_taskloop = false;
8021 :
8022 : /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
8023 : by GOMP_taskloop. */
8024 22355 : if (is_task_ctx (ctx) && gimple_omp_task_taskloop_p (ctx->stmt))
8025 : {
8026 : ignored_looptemp = 2;
8027 : is_taskloop = true;
8028 : }
8029 :
8030 116783 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
8031 : {
8032 94428 : tree val, ref, x, var;
8033 94428 : bool by_ref, do_in = false, do_out = false;
8034 94428 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
8035 :
8036 94428 : switch (OMP_CLAUSE_CODE (c))
8037 : {
8038 4646 : case OMP_CLAUSE_PRIVATE:
8039 4646 : if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
8040 : break;
8041 4491 : continue;
8042 : case OMP_CLAUSE_FIRSTPRIVATE:
8043 : case OMP_CLAUSE_COPYIN:
8044 : case OMP_CLAUSE_LASTPRIVATE:
8045 : case OMP_CLAUSE_IN_REDUCTION:
8046 : case OMP_CLAUSE__REDUCTEMP_:
8047 : break;
8048 4784 : case OMP_CLAUSE_REDUCTION:
8049 4784 : if (is_task_ctx (ctx) || OMP_CLAUSE_REDUCTION_TASK (c))
8050 744 : continue;
8051 : break;
8052 17541 : case OMP_CLAUSE_SHARED:
8053 17541 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
8054 : break;
8055 17229 : continue;
8056 18795 : case OMP_CLAUSE__LOOPTEMP_:
8057 18795 : if (ignored_looptemp)
8058 : {
8059 2660 : ignored_looptemp--;
8060 2660 : continue;
8061 : }
8062 : break;
8063 11898 : default:
8064 11898 : continue;
8065 : }
8066 :
8067 57406 : val = OMP_CLAUSE_DECL (c);
8068 57406 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
8069 53366 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
8070 59046 : && TREE_CODE (val) == MEM_REF)
8071 : {
8072 1578 : val = TREE_OPERAND (val, 0);
8073 1578 : if (TREE_CODE (val) == POINTER_PLUS_EXPR)
8074 351 : val = TREE_OPERAND (val, 0);
8075 1578 : if (INDIRECT_REF_P (val)
8076 1494 : || TREE_CODE (val) == ADDR_EXPR)
8077 906 : val = TREE_OPERAND (val, 0);
8078 1578 : if (is_variable_sized (val))
8079 56 : continue;
8080 : }
8081 :
8082 : /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
8083 : outer taskloop region. */
8084 57350 : omp_context *ctx_for_o = ctx;
8085 57350 : if (is_taskloop
8086 3911 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
8087 57662 : && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
8088 312 : ctx_for_o = ctx->outer;
8089 :
8090 57350 : var = lookup_decl_in_outer_ctx (val, ctx_for_o);
8091 :
8092 57350 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
8093 56840 : && is_global_var (var)
8094 59649 : && (val == OMP_CLAUSE_DECL (c)
8095 273 : || !is_task_ctx (ctx)
8096 241 : || (TREE_CODE (TREE_TYPE (val)) != POINTER_TYPE
8097 187 : && (TREE_CODE (TREE_TYPE (val)) != REFERENCE_TYPE
8098 0 : || (TREE_CODE (TREE_TYPE (TREE_TYPE (val)))
8099 : != POINTER_TYPE)))))
8100 2245 : continue;
8101 :
8102 55105 : t = omp_member_access_dummy_var (var);
8103 55105 : if (t)
8104 : {
8105 249 : var = DECL_VALUE_EXPR (var);
8106 249 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx_for_o);
8107 249 : if (o != t)
8108 69 : var = unshare_and_remap (var, t, o);
8109 : else
8110 180 : var = unshare_expr (var);
8111 : }
8112 :
8113 55105 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED)
8114 : {
8115 : /* Handle taskloop firstprivate/lastprivate, where the
8116 : lastprivate on GIMPLE_OMP_TASK is represented as
8117 : OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
8118 79 : tree f = lookup_sfield ((splay_tree_key) &DECL_UID (val), ctx);
8119 79 : x = omp_build_component_ref (ctx->sender_decl, f);
8120 79 : if (use_pointer_for_field (val, ctx))
8121 59 : var = build_fold_addr_expr (var);
8122 79 : gimplify_assign (x, var, ilist);
8123 79 : DECL_ABSTRACT_ORIGIN (f) = NULL;
8124 79 : continue;
8125 79 : }
8126 :
8127 55080 : if (((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
8128 51731 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IN_REDUCTION)
8129 4431 : || val == OMP_CLAUSE_DECL (c))
8130 108749 : && is_variable_sized (val))
8131 54 : continue;
8132 54972 : by_ref = use_pointer_for_field (val, NULL);
8133 :
8134 54972 : switch (OMP_CLAUSE_CODE (c))
8135 : {
8136 26723 : case OMP_CLAUSE_FIRSTPRIVATE:
8137 26723 : if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)
8138 7217 : && !by_ref
8139 33888 : && is_task_ctx (ctx))
8140 2328 : suppress_warning (var);
8141 : do_in = true;
8142 : break;
8143 :
8144 : case OMP_CLAUSE_PRIVATE:
8145 : case OMP_CLAUSE_COPYIN:
8146 : case OMP_CLAUSE__LOOPTEMP_:
8147 : case OMP_CLAUSE__REDUCTEMP_:
8148 : do_in = true;
8149 : break;
8150 :
8151 6463 : case OMP_CLAUSE_LASTPRIVATE:
8152 6463 : if (by_ref || omp_privatize_by_reference (val))
8153 : {
8154 383 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
8155 239 : continue;
8156 : do_in = true;
8157 : }
8158 : else
8159 : {
8160 6080 : do_out = true;
8161 6080 : if (lang_hooks.decls.omp_private_outer_ref (val))
8162 : do_in = true;
8163 : }
8164 : break;
8165 :
8166 4431 : case OMP_CLAUSE_REDUCTION:
8167 4431 : case OMP_CLAUSE_IN_REDUCTION:
8168 4431 : do_in = true;
8169 4431 : if (val == OMP_CLAUSE_DECL (c))
8170 : {
8171 3128 : if (is_task_ctx (ctx))
8172 301 : by_ref = use_pointer_for_field (val, ctx);
8173 : else
8174 2827 : do_out = !(by_ref || omp_privatize_by_reference (val));
8175 : }
8176 : else
8177 1303 : by_ref = TREE_CODE (TREE_TYPE (val)) == ARRAY_TYPE;
8178 : break;
8179 :
8180 0 : default:
8181 0 : gcc_unreachable ();
8182 : }
8183 :
8184 1604 : if (do_in)
8185 : {
8186 48677 : ref = build_sender_ref (val, ctx);
8187 48677 : x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
8188 48677 : gimplify_assign (ref, x, ilist);
8189 48677 : if (is_task_ctx (ctx))
8190 6227 : DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
8191 : }
8192 :
8193 54733 : if (do_out)
8194 : {
8195 8374 : ref = build_sender_ref (val, ctx);
8196 8374 : gimplify_assign (var, ref, olist);
8197 : }
8198 : }
8199 22355 : }
8200 :
8201 : /* Generate code to implement SHARED from the sender (aka parent)
8202 : side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
8203 : list things that got automatically shared. */
8204 :
8205 : static void
8206 22355 : lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
8207 : {
8208 22355 : tree var, ovar, nvar, t, f, x, record_type;
8209 :
8210 22355 : if (ctx->record_type == NULL)
8211 4442 : return;
8212 :
8213 17913 : record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
8214 86656 : for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
8215 : {
8216 68743 : ovar = DECL_ABSTRACT_ORIGIN (f);
8217 68743 : if (!ovar || TREE_CODE (ovar) == FIELD_DECL)
8218 6306 : continue;
8219 :
8220 62437 : nvar = maybe_lookup_decl (ovar, ctx);
8221 113247 : if (!nvar
8222 62028 : || !DECL_HAS_VALUE_EXPR_P (nvar)
8223 74245 : || (ctx->allocate_map
8224 274 : && ctx->allocate_map->get (ovar)))
8225 50810 : continue;
8226 :
8227 : /* If CTX is a nested parallel directive. Find the immediately
8228 : enclosing parallel or workshare construct that contains a
8229 : mapping for OVAR. */
8230 11627 : var = lookup_decl_in_outer_ctx (ovar, ctx);
8231 :
8232 11627 : t = omp_member_access_dummy_var (var);
8233 11627 : if (t)
8234 : {
8235 49 : var = DECL_VALUE_EXPR (var);
8236 49 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
8237 49 : if (o != t)
8238 18 : var = unshare_and_remap (var, t, o);
8239 : else
8240 31 : var = unshare_expr (var);
8241 : }
8242 :
8243 11627 : if (use_pointer_for_field (ovar, ctx))
8244 : {
8245 6432 : x = build_sender_ref (ovar, ctx);
8246 6432 : if (TREE_CODE (TREE_TYPE (f)) == ARRAY_TYPE
8247 6432 : && TREE_TYPE (f) == TREE_TYPE (ovar))
8248 : {
8249 103 : gcc_assert (is_parallel_ctx (ctx)
8250 : && DECL_ARTIFICIAL (ovar));
8251 : /* _condtemp_ clause. */
8252 103 : var = build_constructor (TREE_TYPE (x), NULL);
8253 : }
8254 : else
8255 6329 : var = build_fold_addr_expr (var);
8256 6432 : gimplify_assign (x, var, ilist);
8257 : }
8258 : else
8259 : {
8260 5195 : x = build_sender_ref (ovar, ctx);
8261 5195 : gimplify_assign (x, var, ilist);
8262 :
8263 5195 : if (!TREE_READONLY (var)
8264 : /* We don't need to receive a new reference to a result
8265 : or parm decl. In fact we may not store to it as we will
8266 : invalidate any pending RSO and generate wrong gimple
8267 : during inlining. */
8268 5195 : && !((TREE_CODE (var) == RESULT_DECL
8269 3298 : || TREE_CODE (var) == PARM_DECL)
8270 182 : && DECL_BY_REFERENCE (var)))
8271 : {
8272 3227 : x = build_sender_ref (ovar, ctx);
8273 3227 : gimplify_assign (var, x, olist);
8274 : }
8275 : }
8276 : }
8277 : }
8278 :
8279 : /* Emit an OpenACC head marker call, encapulating the partitioning and
8280 : other information that must be processed by the target compiler.
8281 : Return the maximum number of dimensions the associated loop might
8282 : be partitioned over. */
8283 :
8284 : static unsigned
8285 9665 : lower_oacc_head_mark (location_t loc, tree ddvar, tree clauses,
8286 : gimple_seq *seq, omp_context *ctx)
8287 : {
8288 9665 : unsigned levels = 0;
8289 9665 : unsigned tag = 0;
8290 9665 : tree gang_static = NULL_TREE;
8291 9665 : auto_vec<tree, 5> args;
8292 :
8293 19330 : args.quick_push (build_int_cst
8294 9665 : (integer_type_node, IFN_UNIQUE_OACC_HEAD_MARK));
8295 9665 : args.quick_push (ddvar);
8296 30854 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
8297 : {
8298 21189 : switch (OMP_CLAUSE_CODE (c))
8299 : {
8300 1856 : case OMP_CLAUSE_GANG:
8301 1856 : tag |= OLF_DIM_GANG;
8302 1856 : gang_static = OMP_CLAUSE_GANG_STATIC_EXPR (c);
8303 : /* static:* is represented by -1, and we can ignore it, as
8304 : scheduling is always static. */
8305 1856 : if (gang_static && integer_minus_onep (gang_static))
8306 70 : gang_static = NULL_TREE;
8307 1856 : levels++;
8308 1856 : break;
8309 :
8310 1330 : case OMP_CLAUSE_WORKER:
8311 1330 : tag |= OLF_DIM_WORKER;
8312 1330 : levels++;
8313 1330 : break;
8314 :
8315 1523 : case OMP_CLAUSE_VECTOR:
8316 1523 : tag |= OLF_DIM_VECTOR;
8317 1523 : levels++;
8318 1523 : break;
8319 :
8320 314 : case OMP_CLAUSE_SEQ:
8321 314 : tag |= OLF_SEQ;
8322 314 : break;
8323 :
8324 409 : case OMP_CLAUSE_AUTO:
8325 409 : tag |= OLF_AUTO;
8326 409 : break;
8327 :
8328 260 : case OMP_CLAUSE_INDEPENDENT:
8329 260 : tag |= OLF_INDEPENDENT;
8330 260 : break;
8331 :
8332 136 : case OMP_CLAUSE_TILE:
8333 136 : tag |= OLF_TILE;
8334 136 : break;
8335 :
8336 3986 : case OMP_CLAUSE_REDUCTION:
8337 3986 : tag |= OLF_REDUCTION;
8338 3986 : break;
8339 :
8340 11375 : default:
8341 11375 : continue;
8342 : }
8343 : }
8344 :
8345 9665 : if (gang_static)
8346 : {
8347 146 : if (DECL_P (gang_static))
8348 16 : gang_static = build_outer_var_ref (gang_static, ctx);
8349 146 : tag |= OLF_GANG_STATIC;
8350 : }
8351 :
8352 9665 : omp_context *tgt = enclosing_target_ctx (ctx);
8353 9665 : if (!tgt || is_oacc_parallel_or_serial (tgt))
8354 : ;
8355 62 : else if (is_oacc_kernels (tgt))
8356 : /* Not using this loops handling inside OpenACC 'kernels' regions. */
8357 0 : gcc_unreachable ();
8358 62 : else if (is_oacc_kernels_decomposed_part (tgt))
8359 : ;
8360 : else
8361 0 : gcc_unreachable ();
8362 :
8363 : /* In a parallel region, loops are implicitly INDEPENDENT. */
8364 9665 : if (!tgt || is_oacc_parallel_or_serial (tgt))
8365 9603 : tag |= OLF_INDEPENDENT;
8366 :
8367 : /* Loops inside OpenACC 'kernels' decomposed parts' regions are expected to
8368 : have an explicit 'seq' or 'independent' clause, and no 'auto' clause. */
8369 9665 : if (tgt && is_oacc_kernels_decomposed_part (tgt))
8370 : {
8371 62 : gcc_assert (tag & (OLF_SEQ | OLF_INDEPENDENT));
8372 62 : gcc_assert (!(tag & OLF_AUTO));
8373 : }
8374 :
8375 9665 : if (tag & OLF_TILE)
8376 : /* Tiling could use all 3 levels. */
8377 : levels = 3;
8378 : else
8379 : {
8380 : /* A loop lacking SEQ, GANG, WORKER and/or VECTOR could be AUTO.
8381 : Ensure at least one level, or 2 for possible auto
8382 : partitioning */
8383 9529 : bool maybe_auto = !(tag & (((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1)
8384 : << OLF_DIM_BASE) | OLF_SEQ));
8385 :
8386 9529 : if (levels < 1u + maybe_auto)
8387 : levels = 1u + maybe_auto;
8388 : }
8389 :
8390 9665 : args.quick_push (build_int_cst (integer_type_node, levels));
8391 9665 : args.quick_push (build_int_cst (integer_type_node, tag));
8392 9665 : if (gang_static)
8393 146 : args.quick_push (gang_static);
8394 :
8395 9665 : gcall *call = gimple_build_call_internal_vec (IFN_UNIQUE, args);
8396 9665 : gimple_set_location (call, loc);
8397 9665 : gimple_set_lhs (call, ddvar);
8398 9665 : gimple_seq_add_stmt (seq, call);
8399 :
8400 9665 : return levels;
8401 9665 : }
8402 :
8403 : /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
8404 : partitioning level of the enclosed region. */
8405 :
8406 : static void
8407 42767 : lower_oacc_loop_marker (location_t loc, tree ddvar, bool head,
8408 : tree tofollow, gimple_seq *seq)
8409 : {
8410 42767 : int marker_kind = (head ? IFN_UNIQUE_OACC_HEAD_MARK
8411 : : IFN_UNIQUE_OACC_TAIL_MARK);
8412 42767 : tree marker = build_int_cst (integer_type_node, marker_kind);
8413 42767 : int nargs = 2 + (tofollow != NULL_TREE);
8414 42767 : gcall *call = gimple_build_call_internal (IFN_UNIQUE, nargs,
8415 : marker, ddvar, tofollow);
8416 42767 : gimple_set_location (call, loc);
8417 42767 : gimple_set_lhs (call, ddvar);
8418 42767 : gimple_seq_add_stmt (seq, call);
8419 42767 : }
8420 :
8421 : /* Generate the before and after OpenACC loop sequences. CLAUSES are
8422 : the loop clauses, from which we extract reductions. Initialize
8423 : HEAD and TAIL. */
8424 :
8425 : static void
8426 9665 : lower_oacc_head_tail (location_t loc, tree clauses, gcall *private_marker,
8427 : gimple_seq *head, gimple_seq *tail, omp_context *ctx)
8428 : {
8429 9665 : bool inner = false;
8430 9665 : tree ddvar = create_tmp_var (integer_type_node, ".data_dep");
8431 9665 : gimple_seq_add_stmt (head, gimple_build_assign (ddvar, integer_zero_node));
8432 :
8433 9665 : unsigned count = lower_oacc_head_mark (loc, ddvar, clauses, head, ctx);
8434 :
8435 9665 : if (private_marker)
8436 : {
8437 229 : gimple_set_location (private_marker, loc);
8438 229 : gimple_call_set_lhs (private_marker, ddvar);
8439 229 : gimple_call_set_arg (private_marker, 1, ddvar);
8440 : }
8441 :
8442 9665 : tree fork_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_FORK);
8443 9665 : tree join_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_JOIN);
8444 :
8445 9665 : gcc_assert (count);
8446 26216 : for (unsigned done = 1; count; count--, done++)
8447 : {
8448 16551 : gimple_seq fork_seq = NULL;
8449 16551 : gimple_seq join_seq = NULL;
8450 :
8451 16551 : tree place = build_int_cst (integer_type_node, -1);
8452 16551 : gcall *fork = gimple_build_call_internal (IFN_UNIQUE, 3,
8453 : fork_kind, ddvar, place);
8454 16551 : gimple_set_location (fork, loc);
8455 16551 : gimple_set_lhs (fork, ddvar);
8456 :
8457 16551 : gcall *join = gimple_build_call_internal (IFN_UNIQUE, 3,
8458 : join_kind, ddvar, place);
8459 16551 : gimple_set_location (join, loc);
8460 16551 : gimple_set_lhs (join, ddvar);
8461 :
8462 : /* Mark the beginning of this level sequence. */
8463 16551 : if (inner)
8464 6886 : lower_oacc_loop_marker (loc, ddvar, true,
8465 6886 : build_int_cst (integer_type_node, count),
8466 : &fork_seq);
8467 16551 : lower_oacc_loop_marker (loc, ddvar, false,
8468 16551 : build_int_cst (integer_type_node, done),
8469 : &join_seq);
8470 :
8471 23437 : lower_oacc_reductions (loc, clauses, place, inner,
8472 : fork, (count == 1) ? private_marker : NULL,
8473 : join, &fork_seq, &join_seq, ctx);
8474 :
8475 : /* Append this level to head. */
8476 16551 : gimple_seq_add_seq (head, fork_seq);
8477 : /* Prepend it to tail. */
8478 16551 : gimple_seq_add_seq (&join_seq, *tail);
8479 16551 : *tail = join_seq;
8480 :
8481 16551 : inner = true;
8482 : }
8483 :
8484 : /* Mark the end of the sequence. */
8485 9665 : lower_oacc_loop_marker (loc, ddvar, true, NULL_TREE, head);
8486 9665 : lower_oacc_loop_marker (loc, ddvar, false, NULL_TREE, tail);
8487 9665 : }
8488 :
8489 : /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
8490 : catch handler and return it. This prevents programs from violating the
8491 : structured block semantics with throws. */
8492 :
8493 : static gimple_seq
8494 94603 : maybe_catch_exception (gimple_seq body)
8495 : {
8496 94603 : gimple *g;
8497 94603 : tree decl;
8498 :
8499 94603 : if (!flag_exceptions)
8500 : return body;
8501 :
8502 42469 : if (lang_hooks.eh_protect_cleanup_actions != NULL)
8503 42441 : decl = lang_hooks.eh_protect_cleanup_actions ();
8504 : else
8505 28 : decl = builtin_decl_explicit (BUILT_IN_TRAP);
8506 :
8507 42469 : g = gimple_build_eh_must_not_throw (decl);
8508 42469 : g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
8509 : GIMPLE_TRY_CATCH);
8510 :
8511 42469 : return gimple_seq_alloc_with_stmt (g);
8512 : }
8513 :
8514 :
8515 : /* Routines to lower OMP directives into OMP-GIMPLE. */
8516 :
8517 : /* If ctx is a worksharing context inside of a cancellable parallel
8518 : region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8519 : and conditional branch to parallel's cancel_label to handle
8520 : cancellation in the implicit barrier. */
8521 :
8522 : static void
8523 48509 : maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple *omp_return,
8524 : gimple_seq *body)
8525 : {
8526 48509 : gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
8527 48509 : if (gimple_omp_return_nowait_p (omp_return))
8528 : return;
8529 19557 : for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
8530 16008 : if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
8531 16008 : && outer->cancellable)
8532 : {
8533 64 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
8534 64 : tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
8535 64 : tree lhs = create_tmp_var (c_bool_type);
8536 64 : gimple_omp_return_set_lhs (omp_return, lhs);
8537 64 : tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
8538 64 : gimple *g = gimple_build_cond (NE_EXPR, lhs,
8539 : fold_convert (c_bool_type,
8540 : boolean_false_node),
8541 : outer->cancel_label, fallthru_label);
8542 64 : gimple_seq_add_stmt (body, g);
8543 64 : gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
8544 : }
8545 15944 : else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
8546 15944 : && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
8547 : return;
8548 : }
8549 :
8550 : /* Find the first task_reduction or reduction clause or return NULL
8551 : if there are none. */
8552 :
8553 : static inline tree
8554 48763 : omp_task_reductions_find_first (tree clauses, enum tree_code code,
8555 : enum omp_clause_code ccode)
8556 : {
8557 64081 : while (1)
8558 : {
8559 56422 : clauses = omp_find_clause (clauses, ccode);
8560 56422 : if (clauses == NULL_TREE)
8561 : return NULL_TREE;
8562 9104 : if (ccode != OMP_CLAUSE_REDUCTION
8563 9104 : || code == OMP_TASKLOOP
8564 9104 : || OMP_CLAUSE_REDUCTION_TASK (clauses))
8565 : return clauses;
8566 7659 : clauses = OMP_CLAUSE_CHAIN (clauses);
8567 : }
8568 : }
8569 :
8570 : static void lower_omp_task_reductions (omp_context *, enum tree_code, tree,
8571 : gimple_seq *, gimple_seq *);
8572 :
8573 : /* Lower the OpenMP sections directive in the current statement in GSI_P.
8574 : CTX is the enclosing OMP context for the current statement. */
8575 :
8576 : static void
8577 378 : lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8578 : {
8579 378 : tree block, control;
8580 378 : gimple_stmt_iterator tgsi;
8581 378 : gomp_sections *stmt;
8582 378 : gimple *t;
8583 378 : gbind *new_stmt, *bind;
8584 378 : gimple_seq ilist, dlist, olist, tred_dlist = NULL, clist = NULL, new_body;
8585 :
8586 378 : stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p));
8587 :
8588 378 : push_gimplify_context ();
8589 :
8590 378 : dlist = NULL;
8591 378 : ilist = NULL;
8592 :
8593 378 : tree rclauses
8594 378 : = omp_task_reductions_find_first (gimple_omp_sections_clauses (stmt),
8595 : OMP_SECTIONS, OMP_CLAUSE_REDUCTION);
8596 378 : tree rtmp = NULL_TREE;
8597 378 : if (rclauses)
8598 : {
8599 8 : tree type = build_pointer_type (pointer_sized_int_node);
8600 8 : tree temp = create_tmp_var (type);
8601 8 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
8602 8 : OMP_CLAUSE_DECL (c) = temp;
8603 8 : OMP_CLAUSE_CHAIN (c) = gimple_omp_sections_clauses (stmt);
8604 8 : gimple_omp_sections_set_clauses (stmt, c);
8605 8 : lower_omp_task_reductions (ctx, OMP_SECTIONS,
8606 : gimple_omp_sections_clauses (stmt),
8607 : &ilist, &tred_dlist);
8608 8 : rclauses = c;
8609 8 : rtmp = make_ssa_name (type);
8610 8 : gimple_seq_add_stmt (&ilist, gimple_build_assign (rtmp, temp));
8611 : }
8612 :
8613 378 : tree *clauses_ptr = gimple_omp_sections_clauses_ptr (stmt);
8614 378 : lower_lastprivate_conditional_clauses (clauses_ptr, ctx);
8615 :
8616 378 : lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
8617 : &ilist, &dlist, ctx, NULL);
8618 :
8619 378 : control = create_tmp_var (unsigned_type_node, ".section");
8620 378 : gimple_omp_sections_set_control (stmt, control);
8621 :
8622 378 : new_body = gimple_omp_body (stmt);
8623 378 : gimple_omp_set_body (stmt, NULL);
8624 378 : tgsi = gsi_start (new_body);
8625 1233 : for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
8626 : {
8627 855 : omp_context *sctx;
8628 855 : gimple *sec_start;
8629 :
8630 855 : sec_start = gsi_stmt (tgsi);
8631 855 : sctx = maybe_lookup_ctx (sec_start);
8632 855 : gcc_assert (sctx);
8633 :
8634 855 : lower_omp (gimple_omp_body_ptr (sec_start), sctx);
8635 855 : gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
8636 : GSI_CONTINUE_LINKING);
8637 855 : gimple_omp_set_body (sec_start, NULL);
8638 :
8639 855 : if (gsi_one_before_end_p (tgsi))
8640 : {
8641 378 : gimple_seq l = NULL;
8642 378 : lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
8643 : &ilist, &l, &clist, ctx);
8644 378 : gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
8645 378 : gimple_omp_section_set_last (sec_start);
8646 : }
8647 :
8648 855 : gsi_insert_after (&tgsi, gimple_build_omp_return (false),
8649 : GSI_CONTINUE_LINKING);
8650 : }
8651 :
8652 378 : block = make_node (BLOCK);
8653 378 : bind = gimple_build_bind (NULL, new_body, block);
8654 :
8655 378 : olist = NULL;
8656 378 : lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist,
8657 : &clist, ctx);
8658 378 : if (clist)
8659 : {
8660 10 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
8661 10 : gcall *g = gimple_build_call (fndecl, 0);
8662 10 : gimple_seq_add_stmt (&olist, g);
8663 10 : gimple_seq_add_seq (&olist, clist);
8664 10 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
8665 10 : g = gimple_build_call (fndecl, 0);
8666 10 : gimple_seq_add_stmt (&olist, g);
8667 : }
8668 :
8669 378 : block = make_node (BLOCK);
8670 378 : new_stmt = gimple_build_bind (NULL, NULL, block);
8671 378 : gsi_replace (gsi_p, new_stmt, true);
8672 :
8673 378 : pop_gimplify_context (new_stmt);
8674 378 : gimple_bind_append_vars (new_stmt, ctx->block_vars);
8675 378 : BLOCK_VARS (block) = gimple_bind_vars (bind);
8676 378 : if (BLOCK_VARS (block))
8677 0 : TREE_USED (block) = 1;
8678 :
8679 378 : new_body = NULL;
8680 378 : gimple_seq_add_seq (&new_body, ilist);
8681 378 : gimple_seq_add_stmt (&new_body, stmt);
8682 378 : gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
8683 378 : gimple_seq_add_stmt (&new_body, bind);
8684 :
8685 378 : t = gimple_build_omp_continue (control, control);
8686 378 : gimple_seq_add_stmt (&new_body, t);
8687 :
8688 378 : gimple_seq_add_seq (&new_body, olist);
8689 378 : if (ctx->cancellable)
8690 19 : gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
8691 378 : gimple_seq_add_seq (&new_body, dlist);
8692 :
8693 378 : new_body = maybe_catch_exception (new_body);
8694 :
8695 378 : bool nowait = omp_find_clause (gimple_omp_sections_clauses (stmt),
8696 378 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8697 378 : t = gimple_build_omp_return (nowait);
8698 378 : gimple_seq_add_stmt (&new_body, t);
8699 378 : gimple_seq_add_seq (&new_body, tred_dlist);
8700 378 : maybe_add_implicit_barrier_cancel (ctx, t, &new_body);
8701 :
8702 378 : if (rclauses)
8703 8 : OMP_CLAUSE_DECL (rclauses) = rtmp;
8704 :
8705 378 : gimple_bind_set_body (new_stmt, new_body);
8706 378 : }
8707 :
8708 :
8709 : /* A subroutine of lower_omp_single. Expand the simple form of
8710 : a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8711 :
8712 : if (GOMP_single_start ())
8713 : BODY;
8714 : [ GOMP_barrier (); ] -> unless 'nowait' is present.
8715 :
8716 : FIXME. It may be better to delay expanding the logic of this until
8717 : pass_expand_omp. The expanded logic may make the job more difficult
8718 : to a synchronization analysis pass. */
8719 :
8720 : static void
8721 1004 : lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p)
8722 : {
8723 1004 : location_t loc = gimple_location (single_stmt);
8724 1004 : tree tlabel = create_artificial_label (loc);
8725 1004 : tree flabel = create_artificial_label (loc);
8726 1004 : gimple *call, *cond;
8727 1004 : tree lhs, decl;
8728 :
8729 1004 : decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START);
8730 1004 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
8731 1004 : call = gimple_build_call (decl, 0);
8732 1004 : gimple_call_set_lhs (call, lhs);
8733 1004 : gimple_seq_add_stmt (pre_p, call);
8734 :
8735 1004 : cond = gimple_build_cond (EQ_EXPR, lhs,
8736 1004 : fold_convert_loc (loc, TREE_TYPE (lhs),
8737 : boolean_true_node),
8738 : tlabel, flabel);
8739 1004 : gimple_seq_add_stmt (pre_p, cond);
8740 1004 : gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
8741 1004 : gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8742 1004 : gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
8743 1004 : }
8744 :
8745 :
8746 : /* A subroutine of lower_omp_single. Expand the simple form of
8747 : a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8748 :
8749 : #pragma omp single copyprivate (a, b, c)
8750 :
8751 : Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8752 :
8753 : {
8754 : if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8755 : {
8756 : BODY;
8757 : copyout.a = a;
8758 : copyout.b = b;
8759 : copyout.c = c;
8760 : GOMP_single_copy_end (©out);
8761 : }
8762 : else
8763 : {
8764 : a = copyout_p->a;
8765 : b = copyout_p->b;
8766 : c = copyout_p->c;
8767 : }
8768 : GOMP_barrier ();
8769 : }
8770 :
8771 : FIXME. It may be better to delay expanding the logic of this until
8772 : pass_expand_omp. The expanded logic may make the job more difficult
8773 : to a synchronization analysis pass. */
8774 :
8775 : static void
8776 115 : lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p,
8777 : omp_context *ctx)
8778 : {
8779 115 : tree ptr_type, t, l0, l1, l2, bfn_decl;
8780 115 : gimple_seq copyin_seq;
8781 115 : location_t loc = gimple_location (single_stmt);
8782 :
8783 115 : ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
8784 :
8785 115 : ptr_type = build_pointer_type (ctx->record_type);
8786 115 : ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
8787 :
8788 115 : l0 = create_artificial_label (loc);
8789 115 : l1 = create_artificial_label (loc);
8790 115 : l2 = create_artificial_label (loc);
8791 :
8792 115 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
8793 115 : t = build_call_expr_loc (loc, bfn_decl, 0);
8794 115 : t = fold_convert_loc (loc, ptr_type, t);
8795 115 : gimplify_assign (ctx->receiver_decl, t, pre_p);
8796 :
8797 115 : t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
8798 : build_int_cst (ptr_type, 0));
8799 115 : t = build3 (COND_EXPR, void_type_node, t,
8800 : build_and_jump (&l0), build_and_jump (&l1));
8801 115 : gimplify_and_add (t, pre_p);
8802 :
8803 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
8804 :
8805 115 : gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8806 :
8807 115 : copyin_seq = NULL;
8808 115 : lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
8809 : ©in_seq, ctx);
8810 :
8811 115 : t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
8812 115 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
8813 115 : t = build_call_expr_loc (loc, bfn_decl, 1, t);
8814 115 : gimplify_and_add (t, pre_p);
8815 :
8816 115 : t = build_and_jump (&l2);
8817 115 : gimplify_and_add (t, pre_p);
8818 :
8819 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
8820 :
8821 115 : gimple_seq_add_seq (pre_p, copyin_seq);
8822 :
8823 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
8824 115 : }
8825 :
8826 :
8827 : /* Expand code for an OpenMP single directive. */
8828 :
8829 : static void
8830 1119 : lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8831 : {
8832 1119 : tree block;
8833 1119 : gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p));
8834 1119 : gbind *bind;
8835 1119 : gimple_seq bind_body, bind_body_tail = NULL, dlist;
8836 :
8837 1119 : push_gimplify_context ();
8838 :
8839 1119 : block = make_node (BLOCK);
8840 1119 : bind = gimple_build_bind (NULL, NULL, block);
8841 1119 : gsi_replace (gsi_p, bind, true);
8842 1119 : bind_body = NULL;
8843 1119 : dlist = NULL;
8844 1119 : lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
8845 : &bind_body, &dlist, ctx, NULL);
8846 1119 : lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
8847 :
8848 1119 : gimple_seq_add_stmt (&bind_body, single_stmt);
8849 :
8850 1119 : if (ctx->record_type)
8851 115 : lower_omp_single_copy (single_stmt, &bind_body, ctx);
8852 : else
8853 1004 : lower_omp_single_simple (single_stmt, &bind_body);
8854 :
8855 1119 : gimple_omp_set_body (single_stmt, NULL);
8856 :
8857 1119 : gimple_seq_add_seq (&bind_body, dlist);
8858 :
8859 1119 : bind_body = maybe_catch_exception (bind_body);
8860 :
8861 1119 : bool nowait = omp_find_clause (gimple_omp_single_clauses (single_stmt),
8862 1119 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8863 1119 : gimple *g = gimple_build_omp_return (nowait);
8864 1119 : gimple_seq_add_stmt (&bind_body_tail, g);
8865 1119 : maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
8866 1119 : if (ctx->record_type)
8867 : {
8868 115 : gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
8869 115 : tree clobber = build_clobber (ctx->record_type);
8870 115 : gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
8871 : clobber), GSI_SAME_STMT);
8872 : }
8873 1119 : gimple_seq_add_seq (&bind_body, bind_body_tail);
8874 1119 : gimple_bind_set_body (bind, bind_body);
8875 :
8876 1119 : pop_gimplify_context (bind);
8877 :
8878 1119 : gimple_bind_append_vars (bind, ctx->block_vars);
8879 1119 : BLOCK_VARS (block) = ctx->block_vars;
8880 1119 : if (BLOCK_VARS (block))
8881 26 : TREE_USED (block) = 1;
8882 1119 : }
8883 :
8884 :
8885 : /* Lower code for an OMP scope directive. */
8886 :
8887 : static void
8888 133 : lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8889 : {
8890 133 : tree block;
8891 133 : gimple *scope_stmt = gsi_stmt (*gsi_p);
8892 133 : gbind *bind;
8893 133 : gimple_seq bind_body, bind_body_tail = NULL, dlist;
8894 133 : gimple_seq tred_dlist = NULL;
8895 :
8896 133 : push_gimplify_context ();
8897 :
8898 133 : block = make_node (BLOCK);
8899 133 : bind = gimple_build_bind (NULL, NULL, block);
8900 133 : gsi_replace (gsi_p, bind, true);
8901 133 : bind_body = NULL;
8902 133 : dlist = NULL;
8903 :
8904 133 : tree rclauses
8905 133 : = omp_task_reductions_find_first (gimple_omp_scope_clauses (scope_stmt),
8906 : OMP_SCOPE, OMP_CLAUSE_REDUCTION);
8907 133 : if (rclauses)
8908 : {
8909 46 : tree type = build_pointer_type (pointer_sized_int_node);
8910 46 : tree temp = create_tmp_var (type);
8911 46 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
8912 46 : OMP_CLAUSE_DECL (c) = temp;
8913 46 : OMP_CLAUSE_CHAIN (c) = gimple_omp_scope_clauses (scope_stmt);
8914 46 : gimple_omp_scope_set_clauses (scope_stmt, c);
8915 46 : lower_omp_task_reductions (ctx, OMP_SCOPE,
8916 : gimple_omp_scope_clauses (scope_stmt),
8917 : &bind_body, &tred_dlist);
8918 46 : rclauses = c;
8919 46 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_SCOPE_START);
8920 46 : gimple *stmt = gimple_build_call (fndecl, 1, temp);
8921 46 : gimple_seq_add_stmt (&bind_body, stmt);
8922 : }
8923 :
8924 133 : lower_rec_input_clauses (gimple_omp_scope_clauses (scope_stmt),
8925 : &bind_body, &dlist, ctx, NULL);
8926 133 : lower_omp (gimple_omp_body_ptr (scope_stmt), ctx);
8927 :
8928 133 : gimple_seq_add_stmt (&bind_body, scope_stmt);
8929 :
8930 133 : gimple_seq_add_seq (&bind_body, gimple_omp_body (scope_stmt));
8931 :
8932 133 : gimple_omp_set_body (scope_stmt, NULL);
8933 :
8934 133 : gimple_seq clist = NULL;
8935 133 : lower_reduction_clauses (gimple_omp_scope_clauses (scope_stmt),
8936 : &bind_body, &clist, ctx);
8937 133 : if (clist)
8938 : {
8939 0 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
8940 0 : gcall *g = gimple_build_call (fndecl, 0);
8941 0 : gimple_seq_add_stmt (&bind_body, g);
8942 0 : gimple_seq_add_seq (&bind_body, clist);
8943 0 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
8944 0 : g = gimple_build_call (fndecl, 0);
8945 0 : gimple_seq_add_stmt (&bind_body, g);
8946 : }
8947 :
8948 133 : gimple_seq_add_seq (&bind_body, dlist);
8949 :
8950 133 : bind_body = maybe_catch_exception (bind_body);
8951 :
8952 133 : bool nowait = omp_find_clause (gimple_omp_scope_clauses (scope_stmt),
8953 133 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8954 133 : gimple *g = gimple_build_omp_return (nowait);
8955 133 : gimple_seq_add_stmt (&bind_body_tail, g);
8956 133 : gimple_seq_add_seq (&bind_body_tail, tred_dlist);
8957 133 : maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
8958 133 : if (ctx->record_type)
8959 : {
8960 0 : gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
8961 0 : tree clobber = build_clobber (ctx->record_type);
8962 0 : gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
8963 : clobber), GSI_SAME_STMT);
8964 : }
8965 133 : gimple_seq_add_seq (&bind_body, bind_body_tail);
8966 :
8967 133 : gimple_bind_set_body (bind, bind_body);
8968 :
8969 133 : pop_gimplify_context (bind);
8970 :
8971 133 : gimple_bind_append_vars (bind, ctx->block_vars);
8972 133 : BLOCK_VARS (block) = ctx->block_vars;
8973 133 : if (BLOCK_VARS (block))
8974 105 : TREE_USED (block) = 1;
8975 133 : }
8976 :
8977 : /* Lower code for an OMP dispatch directive. */
8978 :
8979 : static void
8980 659 : lower_omp_dispatch (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8981 : {
8982 659 : tree block;
8983 659 : gimple *stmt = gsi_stmt (*gsi_p);
8984 659 : gbind *bind;
8985 :
8986 659 : push_gimplify_context ();
8987 :
8988 659 : block = make_node (BLOCK);
8989 659 : bind = gimple_build_bind (NULL, NULL, block);
8990 659 : gsi_replace (gsi_p, bind, true);
8991 :
8992 659 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
8993 659 : gimple_bind_set_body (bind, maybe_catch_exception (gimple_omp_body (stmt)));
8994 :
8995 659 : pop_gimplify_context (bind);
8996 :
8997 659 : gimple_bind_append_vars (bind, ctx->block_vars);
8998 659 : BLOCK_VARS (block) = ctx->block_vars;
8999 659 : }
9000 :
9001 : /* Expand code for an OpenMP master or masked directive. */
9002 :
9003 : static void
9004 1108 : lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9005 : {
9006 1108 : tree block, lab = NULL, x, bfn_decl;
9007 1108 : gimple *stmt = gsi_stmt (*gsi_p);
9008 1108 : gbind *bind;
9009 1108 : location_t loc = gimple_location (stmt);
9010 1108 : gimple_seq tseq;
9011 1108 : tree filter = integer_zero_node;
9012 :
9013 1108 : push_gimplify_context ();
9014 :
9015 1108 : if (gimple_code (stmt) == GIMPLE_OMP_MASKED)
9016 : {
9017 377 : filter = omp_find_clause (gimple_omp_masked_clauses (stmt),
9018 : OMP_CLAUSE_FILTER);
9019 377 : if (filter)
9020 252 : filter = fold_convert (integer_type_node,
9021 : OMP_CLAUSE_FILTER_EXPR (filter));
9022 : else
9023 125 : filter = integer_zero_node;
9024 : }
9025 1108 : block = make_node (BLOCK);
9026 1108 : bind = gimple_build_bind (NULL, NULL, block);
9027 1108 : gsi_replace (gsi_p, bind, true);
9028 1108 : gimple_bind_add_stmt (bind, stmt);
9029 :
9030 1108 : bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
9031 1108 : x = build_call_expr_loc (loc, bfn_decl, 0);
9032 1108 : x = build2 (EQ_EXPR, boolean_type_node, x, filter);
9033 1108 : x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
9034 1108 : tseq = NULL;
9035 1108 : gimplify_and_add (x, &tseq);
9036 1108 : gimple_bind_add_seq (bind, tseq);
9037 :
9038 1108 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9039 1108 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
9040 1108 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
9041 1108 : gimple_omp_set_body (stmt, NULL);
9042 :
9043 1108 : gimple_bind_add_stmt (bind, gimple_build_label (lab));
9044 :
9045 1108 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
9046 :
9047 1108 : pop_gimplify_context (bind);
9048 :
9049 1108 : gimple_bind_append_vars (bind, ctx->block_vars);
9050 1108 : BLOCK_VARS (block) = ctx->block_vars;
9051 1108 : }
9052 :
9053 : /* Helper function for lower_omp_task_reductions. For a specific PASS
9054 : find out the current clause it should be processed, or return false
9055 : if all have been processed already. */
9056 :
9057 : static inline bool
9058 7946 : omp_task_reduction_iterate (int pass, enum tree_code code,
9059 : enum omp_clause_code ccode, tree *c, tree *decl,
9060 : tree *type, tree *next)
9061 : {
9062 11292 : for (; *c; *c = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode))
9063 : {
9064 6692 : if (ccode == OMP_CLAUSE_REDUCTION
9065 6636 : && code != OMP_TASKLOOP
9066 6636 : && !OMP_CLAUSE_REDUCTION_TASK (*c))
9067 56 : continue;
9068 6580 : *decl = OMP_CLAUSE_DECL (*c);
9069 6580 : *type = TREE_TYPE (*decl);
9070 6580 : if (TREE_CODE (*decl) == MEM_REF)
9071 : {
9072 1940 : if (pass != 1)
9073 970 : continue;
9074 : }
9075 : else
9076 : {
9077 4640 : if (omp_privatize_by_reference (*decl))
9078 152 : *type = TREE_TYPE (*type);
9079 4640 : if (pass != (!TREE_CONSTANT (TYPE_SIZE_UNIT (*type))))
9080 2320 : continue;
9081 : }
9082 3290 : *next = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode);
9083 3290 : return true;
9084 : }
9085 4656 : *decl = NULL_TREE;
9086 4656 : *type = NULL_TREE;
9087 4656 : *next = NULL_TREE;
9088 4656 : return false;
9089 : }
9090 :
9091 : /* Lower task_reduction and reduction clauses (the latter unless CODE is
9092 : OMP_TASKGROUP only with task modifier). Register mapping of those in
9093 : START sequence and reducing them and unregister them in the END sequence. */
9094 :
9095 : static void
9096 1373 : lower_omp_task_reductions (omp_context *ctx, enum tree_code code, tree clauses,
9097 : gimple_seq *start, gimple_seq *end)
9098 : {
9099 837 : enum omp_clause_code ccode
9100 : = (code == OMP_TASKGROUP
9101 1373 : ? OMP_CLAUSE_TASK_REDUCTION : OMP_CLAUSE_REDUCTION);
9102 1373 : tree cancellable = NULL_TREE;
9103 1373 : clauses = omp_task_reductions_find_first (clauses, code, ccode);
9104 1373 : if (clauses == NULL_TREE)
9105 209 : return;
9106 1164 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9107 : {
9108 316 : for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
9109 238 : if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
9110 238 : && outer->cancellable)
9111 : {
9112 14 : cancellable = error_mark_node;
9113 14 : break;
9114 : }
9115 224 : else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
9116 224 : && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
9117 : break;
9118 : }
9119 1164 : tree record_type = lang_hooks.types.make_type (RECORD_TYPE);
9120 1164 : tree *last = &TYPE_FIELDS (record_type);
9121 1164 : unsigned cnt = 0;
9122 1164 : if (cancellable)
9123 : {
9124 14 : tree field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
9125 : ptr_type_node);
9126 14 : tree ifield = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
9127 : integer_type_node);
9128 14 : *last = field;
9129 14 : DECL_CHAIN (field) = ifield;
9130 14 : last = &DECL_CHAIN (ifield);
9131 14 : DECL_CONTEXT (field) = record_type;
9132 14 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
9133 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
9134 14 : DECL_CONTEXT (ifield) = record_type;
9135 14 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (ifield))
9136 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (ifield));
9137 : }
9138 3492 : for (int pass = 0; pass < 2; pass++)
9139 : {
9140 2328 : tree decl, type, next;
9141 2328 : for (tree c = clauses;
9142 3973 : omp_task_reduction_iterate (pass, code, ccode,
9143 1645 : &c, &decl, &type, &next); c = next)
9144 : {
9145 1645 : ++cnt;
9146 1645 : tree new_type = type;
9147 1645 : if (ctx->outer)
9148 1164 : new_type = remap_type (type, &ctx->outer->cb);
9149 1645 : tree field
9150 1645 : = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL,
9151 1645 : DECL_P (decl) ? DECL_NAME (decl) : NULL_TREE,
9152 : new_type);
9153 1645 : if (DECL_P (decl) && type == TREE_TYPE (decl))
9154 : {
9155 1122 : SET_DECL_ALIGN (field, DECL_ALIGN (decl));
9156 1122 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (decl);
9157 1122 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (decl);
9158 : }
9159 : else
9160 523 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
9161 1645 : DECL_CONTEXT (field) = record_type;
9162 1645 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
9163 1186 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
9164 1645 : *last = field;
9165 1645 : last = &DECL_CHAIN (field);
9166 1645 : tree bfield
9167 1645 : = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL, NULL_TREE,
9168 : boolean_type_node);
9169 1645 : DECL_CONTEXT (bfield) = record_type;
9170 1645 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (bfield))
9171 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (bfield));
9172 1645 : *last = bfield;
9173 1645 : last = &DECL_CHAIN (bfield);
9174 : }
9175 : }
9176 1164 : *last = NULL_TREE;
9177 1164 : layout_type (record_type);
9178 :
9179 : /* Build up an array which registers with the runtime all the reductions
9180 : and deregisters them at the end. Format documented in libgomp/task.c. */
9181 1164 : tree atype = build_array_type_nelts (pointer_sized_int_node, 7 + cnt * 3);
9182 1164 : tree avar = create_tmp_var_raw (atype);
9183 1164 : gimple_add_tmp_var (avar);
9184 1164 : TREE_ADDRESSABLE (avar) = 1;
9185 1164 : tree r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_zero_node,
9186 : NULL_TREE, NULL_TREE);
9187 1164 : tree t = build_int_cst (pointer_sized_int_node, cnt);
9188 1164 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9189 1164 : gimple_seq seq = NULL;
9190 1164 : tree sz = fold_convert (pointer_sized_int_node,
9191 : TYPE_SIZE_UNIT (record_type));
9192 1164 : int cachesz = 64;
9193 1164 : sz = fold_build2 (PLUS_EXPR, pointer_sized_int_node, sz,
9194 : build_int_cst (pointer_sized_int_node, cachesz - 1));
9195 1164 : sz = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, sz,
9196 : build_int_cst (pointer_sized_int_node, ~(cachesz - 1)));
9197 1164 : ctx->task_reductions.create (1 + cnt);
9198 1164 : ctx->task_reduction_map = new hash_map<tree, unsigned>;
9199 2328 : ctx->task_reductions.quick_push (TREE_CODE (sz) == INTEGER_CST
9200 1204 : ? sz : NULL_TREE);
9201 1164 : sz = force_gimple_operand (sz, &seq, true, NULL_TREE);
9202 1164 : gimple_seq_add_seq (start, seq);
9203 1164 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_one_node,
9204 : NULL_TREE, NULL_TREE);
9205 1164 : gimple_seq_add_stmt (start, gimple_build_assign (r, sz));
9206 1164 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
9207 : NULL_TREE, NULL_TREE);
9208 1164 : t = build_int_cst (pointer_sized_int_node,
9209 1164 : MAX (TYPE_ALIGN_UNIT (record_type), (unsigned) cachesz));
9210 1164 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9211 1164 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (3),
9212 : NULL_TREE, NULL_TREE);
9213 1164 : t = build_int_cst (pointer_sized_int_node, -1);
9214 1164 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9215 1164 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (4),
9216 : NULL_TREE, NULL_TREE);
9217 1164 : t = build_int_cst (pointer_sized_int_node, 0);
9218 1164 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9219 :
9220 : /* In end, build a loop that iterates from 0 to < omp_get_num_threads ()
9221 : and for each task reduction checks a bool right after the private variable
9222 : within that thread's chunk; if the bool is clear, it hasn't been
9223 : initialized and thus isn't going to be reduced nor destructed, otherwise
9224 : reduce and destruct it. */
9225 1164 : tree idx = create_tmp_var (size_type_node);
9226 1164 : gimple_seq_add_stmt (end, gimple_build_assign (idx, size_zero_node));
9227 1164 : tree num_thr_sz = create_tmp_var (size_type_node);
9228 1164 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
9229 1164 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
9230 1164 : tree lab3 = NULL_TREE, lab7 = NULL_TREE;
9231 1164 : gimple *g;
9232 1164 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9233 : {
9234 : /* For worksharing constructs or scope, only perform it in the master
9235 : thread, with the exception of cancelled implicit barriers - then only
9236 : handle the current thread. */
9237 281 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
9238 281 : t = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
9239 281 : tree thr_num = create_tmp_var (integer_type_node);
9240 281 : g = gimple_build_call (t, 0);
9241 281 : gimple_call_set_lhs (g, thr_num);
9242 281 : gimple_seq_add_stmt (end, g);
9243 281 : if (cancellable)
9244 : {
9245 14 : tree c;
9246 14 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9247 14 : tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
9248 14 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
9249 14 : if (code == OMP_FOR)
9250 4 : c = gimple_omp_for_clauses (ctx->stmt);
9251 10 : else if (code == OMP_SECTIONS)
9252 0 : c = gimple_omp_sections_clauses (ctx->stmt);
9253 : else /* if (code == OMP_SCOPE) */
9254 10 : c = gimple_omp_scope_clauses (ctx->stmt);
9255 14 : c = OMP_CLAUSE_DECL (omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_));
9256 14 : cancellable = c;
9257 14 : g = gimple_build_cond (NE_EXPR, c, build_zero_cst (TREE_TYPE (c)),
9258 : lab5, lab6);
9259 14 : gimple_seq_add_stmt (end, g);
9260 14 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9261 14 : g = gimple_build_assign (idx, NOP_EXPR, thr_num);
9262 14 : gimple_seq_add_stmt (end, g);
9263 14 : g = gimple_build_assign (num_thr_sz, PLUS_EXPR, idx,
9264 14 : build_one_cst (TREE_TYPE (idx)));
9265 14 : gimple_seq_add_stmt (end, g);
9266 14 : gimple_seq_add_stmt (end, gimple_build_goto (lab3));
9267 14 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9268 : }
9269 281 : g = gimple_build_cond (NE_EXPR, thr_num, integer_zero_node, lab2, lab4);
9270 281 : gimple_seq_add_stmt (end, g);
9271 281 : gimple_seq_add_stmt (end, gimple_build_label (lab4));
9272 : }
9273 1164 : if (code != OMP_PARALLEL)
9274 : {
9275 1105 : t = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
9276 1105 : tree num_thr = create_tmp_var (integer_type_node);
9277 1105 : g = gimple_build_call (t, 0);
9278 1105 : gimple_call_set_lhs (g, num_thr);
9279 1105 : gimple_seq_add_stmt (end, g);
9280 1105 : g = gimple_build_assign (num_thr_sz, NOP_EXPR, num_thr);
9281 1105 : gimple_seq_add_stmt (end, g);
9282 1105 : if (cancellable)
9283 14 : gimple_seq_add_stmt (end, gimple_build_label (lab3));
9284 : }
9285 : else
9286 : {
9287 59 : tree c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
9288 : OMP_CLAUSE__REDUCTEMP_);
9289 59 : t = fold_convert (pointer_sized_int_node, OMP_CLAUSE_DECL (c));
9290 59 : t = fold_convert (size_type_node, t);
9291 59 : gimplify_assign (num_thr_sz, t, end);
9292 : }
9293 1164 : t = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
9294 : NULL_TREE, NULL_TREE);
9295 1164 : tree data = create_tmp_var (pointer_sized_int_node);
9296 1164 : gimple_seq_add_stmt (end, gimple_build_assign (data, t));
9297 1164 : if (code == OMP_TASKLOOP)
9298 : {
9299 497 : lab7 = create_artificial_label (UNKNOWN_LOCATION);
9300 497 : g = gimple_build_cond (NE_EXPR, data,
9301 : build_zero_cst (pointer_sized_int_node),
9302 : lab1, lab7);
9303 497 : gimple_seq_add_stmt (end, g);
9304 : }
9305 1164 : gimple_seq_add_stmt (end, gimple_build_label (lab1));
9306 1164 : tree ptr;
9307 1164 : if (TREE_CODE (TYPE_SIZE_UNIT (record_type)) == INTEGER_CST)
9308 1124 : ptr = create_tmp_var (build_pointer_type (record_type));
9309 : else
9310 40 : ptr = create_tmp_var (ptr_type_node);
9311 1164 : gimple_seq_add_stmt (end, gimple_build_assign (ptr, NOP_EXPR, data));
9312 :
9313 1164 : tree field = TYPE_FIELDS (record_type);
9314 1164 : cnt = 0;
9315 1164 : if (cancellable)
9316 14 : field = DECL_CHAIN (DECL_CHAIN (field));
9317 3492 : for (int pass = 0; pass < 2; pass++)
9318 : {
9319 2328 : tree decl, type, next;
9320 2328 : for (tree c = clauses;
9321 3973 : omp_task_reduction_iterate (pass, code, ccode,
9322 1645 : &c, &decl, &type, &next); c = next)
9323 : {
9324 1645 : tree var = decl, ref;
9325 1645 : if (TREE_CODE (decl) == MEM_REF)
9326 : {
9327 485 : var = TREE_OPERAND (var, 0);
9328 485 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
9329 108 : var = TREE_OPERAND (var, 0);
9330 485 : tree v = var;
9331 485 : if (TREE_CODE (var) == ADDR_EXPR)
9332 310 : var = TREE_OPERAND (var, 0);
9333 175 : else if (INDIRECT_REF_P (var))
9334 4 : var = TREE_OPERAND (var, 0);
9335 485 : tree orig_var = var;
9336 485 : if (is_variable_sized (var))
9337 : {
9338 31 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
9339 31 : var = DECL_VALUE_EXPR (var);
9340 31 : gcc_assert (INDIRECT_REF_P (var));
9341 31 : var = TREE_OPERAND (var, 0);
9342 31 : gcc_assert (DECL_P (var));
9343 : }
9344 485 : t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
9345 485 : if (orig_var != var)
9346 31 : gcc_assert (TREE_CODE (v) == ADDR_EXPR);
9347 454 : else if (TREE_CODE (v) == ADDR_EXPR)
9348 279 : t = build_fold_addr_expr (t);
9349 175 : else if (INDIRECT_REF_P (v))
9350 4 : t = build_fold_indirect_ref (t);
9351 485 : if (TREE_CODE (TREE_OPERAND (decl, 0)) == POINTER_PLUS_EXPR)
9352 : {
9353 108 : tree b = TREE_OPERAND (TREE_OPERAND (decl, 0), 1);
9354 108 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
9355 108 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t, b);
9356 : }
9357 485 : if (!integer_zerop (TREE_OPERAND (decl, 1)))
9358 219 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
9359 : fold_convert (size_type_node,
9360 : TREE_OPERAND (decl, 1)));
9361 : }
9362 : else
9363 : {
9364 1160 : t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
9365 1160 : if (!omp_privatize_by_reference (decl))
9366 1122 : t = build_fold_addr_expr (t);
9367 : }
9368 1645 : t = fold_convert (pointer_sized_int_node, t);
9369 1645 : seq = NULL;
9370 1645 : t = force_gimple_operand (t, &seq, true, NULL_TREE);
9371 1645 : gimple_seq_add_seq (start, seq);
9372 1645 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9373 1645 : size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
9374 1645 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9375 1645 : t = unshare_expr (byte_position (field));
9376 1645 : t = fold_convert (pointer_sized_int_node, t);
9377 1645 : ctx->task_reduction_map->put (c, cnt);
9378 3290 : ctx->task_reductions.quick_push (TREE_CODE (t) == INTEGER_CST
9379 2017 : ? t : NULL_TREE);
9380 1645 : seq = NULL;
9381 1645 : t = force_gimple_operand (t, &seq, true, NULL_TREE);
9382 1645 : gimple_seq_add_seq (start, seq);
9383 1645 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9384 1645 : size_int (7 + cnt * 3 + 1), NULL_TREE, NULL_TREE);
9385 1645 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9386 :
9387 1645 : tree bfield = DECL_CHAIN (field);
9388 1645 : tree cond;
9389 1645 : if (code == OMP_PARALLEL
9390 1645 : || code == OMP_FOR
9391 : || code == OMP_SECTIONS
9392 : || code == OMP_SCOPE)
9393 : /* In parallel, worksharing or scope all threads unconditionally
9394 : initialize all their task reduction private variables. */
9395 547 : cond = boolean_true_node;
9396 1098 : else if (TREE_TYPE (ptr) == ptr_type_node)
9397 : {
9398 261 : cond = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
9399 : unshare_expr (byte_position (bfield)));
9400 261 : seq = NULL;
9401 261 : cond = force_gimple_operand (cond, &seq, true, NULL_TREE);
9402 261 : gimple_seq_add_seq (end, seq);
9403 261 : tree pbool = build_pointer_type (TREE_TYPE (bfield));
9404 261 : cond = build2 (MEM_REF, TREE_TYPE (bfield), cond,
9405 : build_int_cst (pbool, 0));
9406 : }
9407 : else
9408 837 : cond = build3 (COMPONENT_REF, TREE_TYPE (bfield),
9409 : build_simple_mem_ref (ptr), bfield, NULL_TREE);
9410 1645 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
9411 1645 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
9412 1645 : tree condv = create_tmp_var (boolean_type_node);
9413 1645 : gimple_seq_add_stmt (end, gimple_build_assign (condv, cond));
9414 1645 : g = gimple_build_cond (NE_EXPR, condv, boolean_false_node,
9415 : lab3, lab4);
9416 1645 : gimple_seq_add_stmt (end, g);
9417 1645 : gimple_seq_add_stmt (end, gimple_build_label (lab3));
9418 1645 : if (cancellable && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE)
9419 : {
9420 : /* If this reduction doesn't need destruction and parallel
9421 : has been cancelled, there is nothing to do for this
9422 : reduction, so jump around the merge operation. */
9423 18 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9424 18 : g = gimple_build_cond (NE_EXPR, cancellable,
9425 18 : build_zero_cst (TREE_TYPE (cancellable)),
9426 : lab4, lab5);
9427 18 : gimple_seq_add_stmt (end, g);
9428 18 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9429 : }
9430 :
9431 1645 : tree new_var;
9432 1645 : if (TREE_TYPE (ptr) == ptr_type_node)
9433 : {
9434 428 : new_var = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
9435 : unshare_expr (byte_position (field)));
9436 428 : seq = NULL;
9437 428 : new_var = force_gimple_operand (new_var, &seq, true, NULL_TREE);
9438 428 : gimple_seq_add_seq (end, seq);
9439 428 : tree pbool = build_pointer_type (TREE_TYPE (field));
9440 428 : new_var = build2 (MEM_REF, TREE_TYPE (field), new_var,
9441 : build_int_cst (pbool, 0));
9442 : }
9443 : else
9444 1217 : new_var = build3 (COMPONENT_REF, TREE_TYPE (field),
9445 : build_simple_mem_ref (ptr), field, NULL_TREE);
9446 :
9447 1645 : enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
9448 1645 : if (TREE_CODE (decl) != MEM_REF
9449 1645 : && omp_privatize_by_reference (decl))
9450 38 : ref = build_simple_mem_ref (ref);
9451 : /* reduction(-:var) sums up the partial results, so it acts
9452 : identically to reduction(+:var). */
9453 1645 : if (rcode == MINUS_EXPR)
9454 0 : rcode = PLUS_EXPR;
9455 1645 : if (TREE_CODE (decl) == MEM_REF)
9456 : {
9457 485 : tree type = TREE_TYPE (new_var);
9458 485 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
9459 485 : tree i = create_tmp_var (TREE_TYPE (v));
9460 485 : tree ptype = build_pointer_type (TREE_TYPE (type));
9461 485 : if (DECL_P (v))
9462 : {
9463 142 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
9464 142 : tree vv = create_tmp_var (TREE_TYPE (v));
9465 142 : gimplify_assign (vv, v, start);
9466 142 : v = vv;
9467 : }
9468 485 : ref = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9469 485 : size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
9470 485 : new_var = build_fold_addr_expr (new_var);
9471 485 : new_var = fold_convert (ptype, new_var);
9472 485 : ref = fold_convert (ptype, ref);
9473 485 : tree m = create_tmp_var (ptype);
9474 485 : gimplify_assign (m, new_var, end);
9475 485 : new_var = m;
9476 485 : m = create_tmp_var (ptype);
9477 485 : gimplify_assign (m, ref, end);
9478 485 : ref = m;
9479 485 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), end);
9480 485 : tree body = create_artificial_label (UNKNOWN_LOCATION);
9481 485 : tree endl = create_artificial_label (UNKNOWN_LOCATION);
9482 485 : gimple_seq_add_stmt (end, gimple_build_label (body));
9483 485 : tree priv = build_simple_mem_ref (new_var);
9484 485 : tree out = build_simple_mem_ref (ref);
9485 485 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
9486 : {
9487 161 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
9488 161 : tree decl_placeholder
9489 161 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
9490 161 : tree lab6 = NULL_TREE;
9491 161 : if (cancellable)
9492 : {
9493 : /* If this reduction needs destruction and parallel
9494 : has been cancelled, jump around the merge operation
9495 : to the destruction. */
9496 4 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9497 4 : lab6 = create_artificial_label (UNKNOWN_LOCATION);
9498 4 : tree zero = build_zero_cst (TREE_TYPE (cancellable));
9499 4 : g = gimple_build_cond (NE_EXPR, cancellable, zero,
9500 : lab6, lab5);
9501 4 : gimple_seq_add_stmt (end, g);
9502 4 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9503 : }
9504 161 : SET_DECL_VALUE_EXPR (placeholder, out);
9505 161 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
9506 161 : SET_DECL_VALUE_EXPR (decl_placeholder, priv);
9507 161 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
9508 161 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
9509 322 : gimple_seq_add_seq (end,
9510 161 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
9511 161 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
9512 161 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
9513 : {
9514 56 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
9515 56 : OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
9516 : }
9517 161 : if (cancellable)
9518 4 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9519 161 : tree x = lang_hooks.decls.omp_clause_dtor (c, priv);
9520 161 : if (x)
9521 : {
9522 137 : gimple_seq tseq = NULL;
9523 137 : gimplify_stmt (&x, &tseq);
9524 137 : gimple_seq_add_seq (end, tseq);
9525 : }
9526 : }
9527 : else
9528 : {
9529 324 : tree x = build2 (rcode, TREE_TYPE (out), out, priv);
9530 324 : out = unshare_expr (out);
9531 324 : gimplify_assign (out, x, end);
9532 : }
9533 485 : gimple *g
9534 485 : = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
9535 485 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
9536 485 : gimple_seq_add_stmt (end, g);
9537 485 : g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
9538 485 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
9539 485 : gimple_seq_add_stmt (end, g);
9540 485 : g = gimple_build_assign (i, PLUS_EXPR, i,
9541 485 : build_int_cst (TREE_TYPE (i), 1));
9542 485 : gimple_seq_add_stmt (end, g);
9543 485 : g = gimple_build_cond (LE_EXPR, i, v, body, endl);
9544 485 : gimple_seq_add_stmt (end, g);
9545 485 : gimple_seq_add_stmt (end, gimple_build_label (endl));
9546 : }
9547 1160 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
9548 : {
9549 126 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
9550 126 : tree oldv = NULL_TREE;
9551 126 : tree lab6 = NULL_TREE;
9552 126 : if (cancellable)
9553 : {
9554 : /* If this reduction needs destruction and parallel
9555 : has been cancelled, jump around the merge operation
9556 : to the destruction. */
9557 4 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9558 4 : lab6 = create_artificial_label (UNKNOWN_LOCATION);
9559 4 : tree zero = build_zero_cst (TREE_TYPE (cancellable));
9560 4 : g = gimple_build_cond (NE_EXPR, cancellable, zero,
9561 : lab6, lab5);
9562 4 : gimple_seq_add_stmt (end, g);
9563 4 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9564 : }
9565 126 : if (omp_privatize_by_reference (decl)
9566 136 : && !useless_type_conversion_p (TREE_TYPE (placeholder),
9567 10 : TREE_TYPE (ref)))
9568 0 : ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
9569 126 : ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
9570 126 : tree refv = create_tmp_var (TREE_TYPE (ref));
9571 126 : gimplify_assign (refv, ref, end);
9572 126 : ref = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), refv);
9573 126 : SET_DECL_VALUE_EXPR (placeholder, ref);
9574 126 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
9575 126 : tree d = maybe_lookup_decl (decl, ctx);
9576 126 : gcc_assert (d);
9577 126 : if (DECL_HAS_VALUE_EXPR_P (d))
9578 7 : oldv = DECL_VALUE_EXPR (d);
9579 126 : if (omp_privatize_by_reference (var))
9580 : {
9581 10 : tree v = fold_convert (TREE_TYPE (d),
9582 : build_fold_addr_expr (new_var));
9583 10 : SET_DECL_VALUE_EXPR (d, v);
9584 : }
9585 : else
9586 116 : SET_DECL_VALUE_EXPR (d, new_var);
9587 126 : DECL_HAS_VALUE_EXPR_P (d) = 1;
9588 126 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
9589 126 : if (oldv)
9590 7 : SET_DECL_VALUE_EXPR (d, oldv);
9591 : else
9592 : {
9593 119 : SET_DECL_VALUE_EXPR (d, NULL_TREE);
9594 119 : DECL_HAS_VALUE_EXPR_P (d) = 0;
9595 : }
9596 126 : gimple_seq_add_seq (end, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
9597 126 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
9598 126 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
9599 24 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
9600 126 : if (cancellable)
9601 4 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9602 126 : tree x = lang_hooks.decls.omp_clause_dtor (c, new_var);
9603 126 : if (x)
9604 : {
9605 28 : gimple_seq tseq = NULL;
9606 28 : gimplify_stmt (&x, &tseq);
9607 28 : gimple_seq_add_seq (end, tseq);
9608 : }
9609 : }
9610 : else
9611 : {
9612 1034 : tree x = build2 (rcode, TREE_TYPE (ref), ref, new_var);
9613 1034 : ref = unshare_expr (ref);
9614 1034 : gimplify_assign (ref, x, end);
9615 : }
9616 1645 : gimple_seq_add_stmt (end, gimple_build_label (lab4));
9617 1645 : ++cnt;
9618 1645 : field = DECL_CHAIN (bfield);
9619 : }
9620 : }
9621 :
9622 1164 : if (code == OMP_TASKGROUP)
9623 : {
9624 327 : t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER);
9625 327 : g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
9626 327 : gimple_seq_add_stmt (start, g);
9627 : }
9628 : else
9629 : {
9630 837 : tree c;
9631 837 : if (code == OMP_FOR)
9632 227 : c = gimple_omp_for_clauses (ctx->stmt);
9633 610 : else if (code == OMP_SECTIONS)
9634 8 : c = gimple_omp_sections_clauses (ctx->stmt);
9635 602 : else if (code == OMP_SCOPE)
9636 46 : c = gimple_omp_scope_clauses (ctx->stmt);
9637 : else
9638 556 : c = gimple_omp_taskreg_clauses (ctx->stmt);
9639 837 : c = omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_);
9640 837 : t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (c)),
9641 : build_fold_addr_expr (avar));
9642 837 : gimplify_assign (OMP_CLAUSE_DECL (c), t, start);
9643 : }
9644 :
9645 1164 : gimple_seq_add_stmt (end, gimple_build_assign (data, PLUS_EXPR, data, sz));
9646 1164 : gimple_seq_add_stmt (end, gimple_build_assign (idx, PLUS_EXPR, idx,
9647 : size_one_node));
9648 1164 : g = gimple_build_cond (NE_EXPR, idx, num_thr_sz, lab1, lab2);
9649 1164 : gimple_seq_add_stmt (end, g);
9650 1164 : gimple_seq_add_stmt (end, gimple_build_label (lab2));
9651 1164 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9652 : {
9653 281 : enum built_in_function bfn
9654 : = BUILT_IN_GOMP_WORKSHARE_TASK_REDUCTION_UNREGISTER;
9655 281 : t = builtin_decl_explicit (bfn);
9656 281 : tree c_bool_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (t)));
9657 281 : tree arg;
9658 281 : if (cancellable)
9659 : {
9660 14 : arg = create_tmp_var (c_bool_type);
9661 14 : gimple_seq_add_stmt (end, gimple_build_assign (arg, NOP_EXPR,
9662 : cancellable));
9663 : }
9664 : else
9665 267 : arg = build_int_cst (c_bool_type, 0);
9666 281 : g = gimple_build_call (t, 1, arg);
9667 281 : }
9668 : else
9669 : {
9670 883 : t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_UNREGISTER);
9671 883 : g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
9672 : }
9673 1164 : gimple_seq_add_stmt (end, g);
9674 1164 : if (lab7)
9675 497 : gimple_seq_add_stmt (end, gimple_build_label (lab7));
9676 1164 : t = build_constructor (atype, NULL);
9677 1164 : TREE_THIS_VOLATILE (t) = 1;
9678 1164 : gimple_seq_add_stmt (end, gimple_build_assign (avar, t));
9679 : }
9680 :
9681 : /* Expand code for an OpenMP taskgroup directive. */
9682 :
9683 : static void
9684 536 : lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9685 : {
9686 536 : gimple *stmt = gsi_stmt (*gsi_p);
9687 536 : gcall *x;
9688 536 : gbind *bind;
9689 536 : gimple_seq dseq = NULL;
9690 536 : tree block = make_node (BLOCK);
9691 :
9692 536 : bind = gimple_build_bind (NULL, NULL, block);
9693 536 : gsi_replace (gsi_p, bind, true);
9694 536 : gimple_bind_add_stmt (bind, stmt);
9695 :
9696 536 : push_gimplify_context ();
9697 :
9698 536 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
9699 : 0);
9700 536 : gimple_bind_add_stmt (bind, x);
9701 :
9702 536 : lower_omp_task_reductions (ctx, OMP_TASKGROUP,
9703 : gimple_omp_taskgroup_clauses (stmt),
9704 : gimple_bind_body_ptr (bind), &dseq);
9705 :
9706 536 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9707 536 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
9708 536 : gimple_omp_set_body (stmt, NULL);
9709 :
9710 536 : gimple_bind_add_seq (bind, dseq);
9711 :
9712 536 : pop_gimplify_context (bind);
9713 :
9714 536 : gimple_bind_append_vars (bind, ctx->block_vars);
9715 536 : BLOCK_VARS (block) = ctx->block_vars;
9716 536 : }
9717 :
9718 :
9719 : /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
9720 :
9721 : static void
9722 0 : lower_omp_ordered_clauses (gimple_stmt_iterator *gsi_p, gomp_ordered *ord_stmt,
9723 : omp_context *ctx)
9724 : {
9725 0 : struct omp_for_data fd;
9726 0 : if (!ctx->outer || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR)
9727 0 : return;
9728 :
9729 0 : unsigned int len = gimple_omp_for_collapse (ctx->outer->stmt);
9730 0 : struct omp_for_data_loop *loops = XALLOCAVEC (struct omp_for_data_loop, len);
9731 0 : omp_extract_for_data (as_a <gomp_for *> (ctx->outer->stmt), &fd, loops);
9732 0 : if (!fd.ordered)
9733 : return;
9734 :
9735 0 : tree *list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
9736 0 : tree c = gimple_omp_ordered_clauses (ord_stmt);
9737 0 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS
9738 0 : && OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK)
9739 : {
9740 : /* Merge depend clauses from multiple adjacent
9741 : #pragma omp ordered depend(sink:...) constructs
9742 : into one #pragma omp ordered depend(sink:...), so that
9743 : we can optimize them together. */
9744 0 : gimple_stmt_iterator gsi = *gsi_p;
9745 0 : gsi_next (&gsi);
9746 0 : while (!gsi_end_p (gsi))
9747 : {
9748 0 : gimple *stmt = gsi_stmt (gsi);
9749 0 : if (is_gimple_debug (stmt)
9750 0 : || gimple_code (stmt) == GIMPLE_NOP)
9751 : {
9752 0 : gsi_next (&gsi);
9753 0 : continue;
9754 : }
9755 0 : if (gimple_code (stmt) != GIMPLE_OMP_ORDERED)
9756 : break;
9757 0 : gomp_ordered *ord_stmt2 = as_a <gomp_ordered *> (stmt);
9758 0 : c = gimple_omp_ordered_clauses (ord_stmt2);
9759 0 : if (c == NULL_TREE
9760 0 : || OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS
9761 0 : || OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
9762 : break;
9763 0 : while (*list_p)
9764 0 : list_p = &OMP_CLAUSE_CHAIN (*list_p);
9765 0 : *list_p = c;
9766 0 : gsi_remove (&gsi, true);
9767 : }
9768 : }
9769 :
9770 : /* Canonicalize sink dependence clauses into one folded clause if
9771 : possible.
9772 :
9773 : The basic algorithm is to create a sink vector whose first
9774 : element is the GCD of all the first elements, and whose remaining
9775 : elements are the minimum of the subsequent columns.
9776 :
9777 : We ignore dependence vectors whose first element is zero because
9778 : such dependencies are known to be executed by the same thread.
9779 :
9780 : We take into account the direction of the loop, so a minimum
9781 : becomes a maximum if the loop is iterating forwards. We also
9782 : ignore sink clauses where the loop direction is unknown, or where
9783 : the offsets are clearly invalid because they are not a multiple
9784 : of the loop increment.
9785 :
9786 : For example:
9787 :
9788 : #pragma omp for ordered(2)
9789 : for (i=0; i < N; ++i)
9790 : for (j=0; j < M; ++j)
9791 : {
9792 : #pragma omp ordered \
9793 : depend(sink:i-8,j-2) \
9794 : depend(sink:i,j-1) \ // Completely ignored because i+0.
9795 : depend(sink:i-4,j-3) \
9796 : depend(sink:i-6,j-4)
9797 : #pragma omp ordered depend(source)
9798 : }
9799 :
9800 : Folded clause is:
9801 :
9802 : depend(sink:-gcd(8,4,6),-min(2,3,4))
9803 : -or-
9804 : depend(sink:-2,-2)
9805 : */
9806 :
9807 : /* FIXME: Computing GCD's where the first element is zero is
9808 : non-trivial in the presence of collapsed loops. Do this later. */
9809 0 : if (fd.collapse > 1)
9810 : return;
9811 :
9812 0 : wide_int *folded_deps = XALLOCAVEC (wide_int, 2 * len - 1);
9813 :
9814 : /* wide_int is not a POD so it must be default-constructed. */
9815 0 : for (unsigned i = 0; i != 2 * len - 1; ++i)
9816 0 : new (static_cast<void*>(folded_deps + i)) wide_int ();
9817 :
9818 : tree folded_dep = NULL_TREE;
9819 : /* TRUE if the first dimension's offset is negative. */
9820 : bool neg_offset_p = false;
9821 :
9822 : list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
9823 : unsigned int i;
9824 0 : while ((c = *list_p) != NULL)
9825 : {
9826 0 : bool remove = false;
9827 :
9828 0 : gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS);
9829 0 : if (OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
9830 0 : goto next_ordered_clause;
9831 :
9832 0 : tree vec;
9833 0 : for (vec = OMP_CLAUSE_DECL (c), i = 0;
9834 0 : vec && TREE_CODE (vec) == TREE_LIST;
9835 0 : vec = TREE_CHAIN (vec), ++i)
9836 : {
9837 0 : gcc_assert (i < len);
9838 :
9839 : /* omp_extract_for_data has canonicalized the condition. */
9840 0 : gcc_assert (fd.loops[i].cond_code == LT_EXPR
9841 : || fd.loops[i].cond_code == GT_EXPR);
9842 0 : bool forward = fd.loops[i].cond_code == LT_EXPR;
9843 0 : bool maybe_lexically_later = true;
9844 :
9845 : /* While the committee makes up its mind, bail if we have any
9846 : non-constant steps. */
9847 0 : if (TREE_CODE (fd.loops[i].step) != INTEGER_CST)
9848 0 : goto lower_omp_ordered_ret;
9849 :
9850 0 : tree itype = TREE_TYPE (TREE_VALUE (vec));
9851 0 : if (POINTER_TYPE_P (itype))
9852 0 : itype = sizetype;
9853 0 : wide_int offset = wide_int::from (wi::to_wide (TREE_PURPOSE (vec)),
9854 0 : TYPE_PRECISION (itype),
9855 0 : TYPE_SIGN (itype));
9856 :
9857 : /* Ignore invalid offsets that are not multiples of the step. */
9858 0 : if (!wi::multiple_of_p (wi::abs (offset),
9859 0 : wi::abs (wi::to_wide (fd.loops[i].step)),
9860 : UNSIGNED))
9861 : {
9862 0 : warning_at (OMP_CLAUSE_LOCATION (c), OPT_Wopenmp,
9863 : "ignoring %<sink%> clause with offset that is not "
9864 : "a multiple of the loop step");
9865 0 : remove = true;
9866 0 : goto next_ordered_clause;
9867 : }
9868 :
9869 : /* Calculate the first dimension. The first dimension of
9870 : the folded dependency vector is the GCD of the first
9871 : elements, while ignoring any first elements whose offset
9872 : is 0. */
9873 0 : if (i == 0)
9874 : {
9875 : /* Ignore dependence vectors whose first dimension is 0. */
9876 0 : if (offset == 0)
9877 : {
9878 0 : remove = true;
9879 0 : goto next_ordered_clause;
9880 : }
9881 : else
9882 : {
9883 0 : if (!TYPE_UNSIGNED (itype) && (forward ^ wi::neg_p (offset)))
9884 : {
9885 0 : error_at (OMP_CLAUSE_LOCATION (c),
9886 : "first offset must be in opposite direction "
9887 : "of loop iterations");
9888 0 : goto lower_omp_ordered_ret;
9889 : }
9890 0 : if (forward)
9891 0 : offset = -offset;
9892 0 : neg_offset_p = forward;
9893 : /* Initialize the first time around. */
9894 0 : if (folded_dep == NULL_TREE)
9895 : {
9896 0 : folded_dep = c;
9897 0 : folded_deps[0] = offset;
9898 : }
9899 : else
9900 0 : folded_deps[0] = wi::gcd (folded_deps[0],
9901 0 : offset, UNSIGNED);
9902 : }
9903 : }
9904 : /* Calculate minimum for the remaining dimensions. */
9905 : else
9906 : {
9907 0 : folded_deps[len + i - 1] = offset;
9908 0 : if (folded_dep == c)
9909 0 : folded_deps[i] = offset;
9910 0 : else if (maybe_lexically_later
9911 0 : && !wi::eq_p (folded_deps[i], offset))
9912 : {
9913 0 : if (forward ^ wi::gts_p (folded_deps[i], offset))
9914 : {
9915 : unsigned int j;
9916 0 : folded_dep = c;
9917 0 : for (j = 1; j <= i; j++)
9918 0 : folded_deps[j] = folded_deps[len + j - 1];
9919 : }
9920 : else
9921 0 : maybe_lexically_later = false;
9922 : }
9923 : }
9924 0 : }
9925 0 : gcc_assert (i == len);
9926 :
9927 : remove = true;
9928 :
9929 0 : next_ordered_clause:
9930 0 : if (remove)
9931 0 : *list_p = OMP_CLAUSE_CHAIN (c);
9932 : else
9933 0 : list_p = &OMP_CLAUSE_CHAIN (c);
9934 : }
9935 :
9936 0 : if (folded_dep)
9937 : {
9938 0 : if (neg_offset_p)
9939 0 : folded_deps[0] = -folded_deps[0];
9940 :
9941 0 : tree itype = TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep)));
9942 0 : if (POINTER_TYPE_P (itype))
9943 0 : itype = sizetype;
9944 :
9945 0 : TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep))
9946 0 : = wide_int_to_tree (itype, folded_deps[0]);
9947 0 : OMP_CLAUSE_CHAIN (folded_dep) = gimple_omp_ordered_clauses (ord_stmt);
9948 0 : *gimple_omp_ordered_clauses_ptr (ord_stmt) = folded_dep;
9949 : }
9950 :
9951 0 : lower_omp_ordered_ret:
9952 :
9953 : /* Ordered without clauses is #pragma omp threads, while we want
9954 : a nop instead if we remove all clauses. */
9955 0 : if (gimple_omp_ordered_clauses (ord_stmt) == NULL_TREE)
9956 0 : gsi_replace (gsi_p, gimple_build_nop (), true);
9957 : }
9958 :
9959 :
9960 : /* Expand code for an OpenMP ordered directive. */
9961 :
9962 : static void
9963 1124 : lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9964 : {
9965 1124 : tree block;
9966 1124 : gimple *stmt = gsi_stmt (*gsi_p), *g;
9967 1124 : gomp_ordered *ord_stmt = as_a <gomp_ordered *> (stmt);
9968 1124 : gcall *x;
9969 1124 : gbind *bind;
9970 1124 : bool simd = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
9971 1124 : OMP_CLAUSE_SIMD);
9972 : /* FIXME: this should check presence of OMP_CLAUSE__SIMT_ on the enclosing
9973 : loop. */
9974 1124 : bool maybe_simt
9975 1124 : = simd && omp_maybe_offloaded_ctx (ctx) && omp_max_simt_vf () > 1;
9976 1124 : bool threads = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
9977 1124 : OMP_CLAUSE_THREADS);
9978 :
9979 1124 : if (gimple_omp_ordered_standalone_p (ord_stmt))
9980 : {
9981 : /* FIXME: This is needs to be moved to the expansion to verify various
9982 : conditions only testable on cfg with dominators computed, and also
9983 : all the depend clauses to be merged still might need to be available
9984 : for the runtime checks. */
9985 : if (0)
9986 : lower_omp_ordered_clauses (gsi_p, ord_stmt, ctx);
9987 1124 : return;
9988 : }
9989 :
9990 411 : push_gimplify_context ();
9991 :
9992 411 : block = make_node (BLOCK);
9993 411 : bind = gimple_build_bind (NULL, NULL, block);
9994 411 : gsi_replace (gsi_p, bind, true);
9995 411 : gimple_bind_add_stmt (bind, stmt);
9996 :
9997 411 : if (simd)
9998 : {
9999 79 : x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START, 1,
10000 79 : build_int_cst (NULL_TREE, threads));
10001 79 : cfun->has_simduid_loops = true;
10002 : }
10003 : else
10004 332 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
10005 : 0);
10006 411 : gimple_bind_add_stmt (bind, x);
10007 :
10008 411 : tree counter = NULL_TREE, test = NULL_TREE, body = NULL_TREE;
10009 411 : if (maybe_simt)
10010 : {
10011 0 : counter = create_tmp_var (integer_type_node);
10012 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_LANE, 0);
10013 0 : gimple_call_set_lhs (g, counter);
10014 0 : gimple_bind_add_stmt (bind, g);
10015 :
10016 0 : body = create_artificial_label (UNKNOWN_LOCATION);
10017 0 : test = create_artificial_label (UNKNOWN_LOCATION);
10018 0 : gimple_bind_add_stmt (bind, gimple_build_label (body));
10019 :
10020 0 : tree simt_pred = create_tmp_var (integer_type_node);
10021 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_ORDERED_PRED, 1, counter);
10022 0 : gimple_call_set_lhs (g, simt_pred);
10023 0 : gimple_bind_add_stmt (bind, g);
10024 :
10025 0 : tree t = create_artificial_label (UNKNOWN_LOCATION);
10026 0 : g = gimple_build_cond (EQ_EXPR, simt_pred, integer_zero_node, t, test);
10027 0 : gimple_bind_add_stmt (bind, g);
10028 :
10029 0 : gimple_bind_add_stmt (bind, gimple_build_label (t));
10030 : }
10031 411 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
10032 411 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10033 411 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10034 411 : gimple_omp_set_body (stmt, NULL);
10035 :
10036 411 : if (maybe_simt)
10037 : {
10038 0 : gimple_bind_add_stmt (bind, gimple_build_label (test));
10039 0 : g = gimple_build_assign (counter, MINUS_EXPR, counter, integer_one_node);
10040 0 : gimple_bind_add_stmt (bind, g);
10041 :
10042 0 : tree c = build2 (GE_EXPR, boolean_type_node, counter, integer_zero_node);
10043 0 : tree nonneg = create_tmp_var (integer_type_node);
10044 0 : gimple_seq tseq = NULL;
10045 0 : gimplify_assign (nonneg, fold_convert (integer_type_node, c), &tseq);
10046 0 : gimple_bind_add_seq (bind, tseq);
10047 :
10048 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY, 1, nonneg);
10049 0 : gimple_call_set_lhs (g, nonneg);
10050 0 : gimple_bind_add_stmt (bind, g);
10051 :
10052 0 : tree end = create_artificial_label (UNKNOWN_LOCATION);
10053 0 : g = gimple_build_cond (NE_EXPR, nonneg, integer_zero_node, body, end);
10054 0 : gimple_bind_add_stmt (bind, g);
10055 :
10056 0 : gimple_bind_add_stmt (bind, gimple_build_label (end));
10057 : }
10058 411 : if (simd)
10059 79 : x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END, 1,
10060 79 : build_int_cst (NULL_TREE, threads));
10061 : else
10062 332 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END),
10063 : 0);
10064 411 : gimple_bind_add_stmt (bind, x);
10065 :
10066 411 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10067 :
10068 411 : pop_gimplify_context (bind);
10069 :
10070 411 : gimple_bind_append_vars (bind, ctx->block_vars);
10071 411 : BLOCK_VARS (block) = gimple_bind_vars (bind);
10072 : }
10073 :
10074 :
10075 : /* Expand code for an OpenMP scan directive and the structured block
10076 : before the scan directive. */
10077 :
10078 : static void
10079 1536 : lower_omp_scan (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10080 : {
10081 1536 : gimple *stmt = gsi_stmt (*gsi_p);
10082 1536 : bool has_clauses
10083 1536 : = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)) != NULL;
10084 1536 : tree lane = NULL_TREE;
10085 1536 : gimple_seq before = NULL;
10086 1536 : omp_context *octx = ctx->outer;
10087 1536 : gcc_assert (octx);
10088 1536 : if (octx->scan_exclusive && !has_clauses)
10089 : {
10090 536 : gimple_stmt_iterator gsi2 = *gsi_p;
10091 536 : gsi_next (&gsi2);
10092 536 : gimple *stmt2 = gsi_stmt (gsi2);
10093 : /* For exclusive scan, swap GIMPLE_OMP_SCAN without clauses
10094 : with following GIMPLE_OMP_SCAN with clauses, so that input_phase,
10095 : the one with exclusive clause(s), comes first. */
10096 536 : if (stmt2
10097 274 : && gimple_code (stmt2) == GIMPLE_OMP_SCAN
10098 804 : && gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt2)) != NULL)
10099 : {
10100 268 : gsi_remove (gsi_p, false);
10101 268 : gsi_insert_after (gsi_p, stmt, GSI_SAME_STMT);
10102 268 : ctx = maybe_lookup_ctx (stmt2);
10103 268 : gcc_assert (ctx);
10104 268 : lower_omp_scan (gsi_p, ctx);
10105 268 : return;
10106 : }
10107 : }
10108 :
10109 1268 : bool input_phase = has_clauses ^ octx->scan_inclusive;
10110 1268 : bool is_simd = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
10111 1268 : && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_SIMD);
10112 1268 : bool is_for = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
10113 1268 : && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_FOR
10114 1614 : && !gimple_omp_for_combined_p (octx->stmt));
10115 1268 : bool is_for_simd = is_simd && gimple_omp_for_combined_into_p (octx->stmt);
10116 332 : if (is_for_simd && octx->for_simd_scan_phase)
10117 : is_simd = false;
10118 1102 : if (is_simd)
10119 756 : if (tree c = omp_find_clause (gimple_omp_for_clauses (octx->stmt),
10120 : OMP_CLAUSE__SIMDUID_))
10121 : {
10122 368 : tree uid = OMP_CLAUSE__SIMDUID__DECL (c);
10123 368 : lane = create_tmp_var (unsigned_type_node);
10124 368 : tree t = build_int_cst (integer_type_node,
10125 552 : input_phase ? 1
10126 184 : : octx->scan_inclusive ? 2 : 3);
10127 368 : gimple *g
10128 368 : = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 2, uid, t);
10129 368 : gimple_call_set_lhs (g, lane);
10130 368 : gimple_seq_add_stmt (&before, g);
10131 : }
10132 :
10133 1268 : if (is_simd || is_for)
10134 : {
10135 4980 : for (tree c = gimple_omp_for_clauses (octx->stmt);
10136 4980 : c; c = OMP_CLAUSE_CHAIN (c))
10137 4044 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
10138 4044 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
10139 : {
10140 1160 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
10141 1160 : tree var = OMP_CLAUSE_DECL (c);
10142 1160 : tree new_var = lookup_decl (var, octx);
10143 1160 : tree val = new_var;
10144 1160 : tree var2 = NULL_TREE;
10145 1160 : tree var3 = NULL_TREE;
10146 1160 : tree var4 = NULL_TREE;
10147 1160 : tree lane0 = NULL_TREE;
10148 1160 : tree new_vard = new_var;
10149 1160 : if (omp_privatize_by_reference (var))
10150 : {
10151 184 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
10152 184 : val = new_var;
10153 : }
10154 1160 : if (DECL_HAS_VALUE_EXPR_P (new_vard))
10155 : {
10156 464 : val = DECL_VALUE_EXPR (new_vard);
10157 464 : if (new_vard != new_var)
10158 : {
10159 76 : gcc_assert (TREE_CODE (val) == ADDR_EXPR);
10160 76 : val = TREE_OPERAND (val, 0);
10161 : }
10162 464 : if (TREE_CODE (val) == ARRAY_REF
10163 464 : && VAR_P (TREE_OPERAND (val, 0)))
10164 : {
10165 464 : tree v = TREE_OPERAND (val, 0);
10166 464 : if (lookup_attribute ("omp simd array",
10167 464 : DECL_ATTRIBUTES (v)))
10168 : {
10169 464 : val = unshare_expr (val);
10170 464 : lane0 = TREE_OPERAND (val, 1);
10171 464 : TREE_OPERAND (val, 1) = lane;
10172 464 : var2 = lookup_decl (v, octx);
10173 464 : if (octx->scan_exclusive)
10174 228 : var4 = lookup_decl (var2, octx);
10175 464 : if (input_phase
10176 464 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10177 120 : var3 = maybe_lookup_decl (var4 ? var4 : var2, octx);
10178 464 : if (!input_phase)
10179 : {
10180 232 : var2 = build4 (ARRAY_REF, TREE_TYPE (val),
10181 : var2, lane, NULL_TREE, NULL_TREE);
10182 232 : TREE_THIS_NOTRAP (var2) = 1;
10183 232 : if (octx->scan_exclusive)
10184 : {
10185 114 : var4 = build4 (ARRAY_REF, TREE_TYPE (val),
10186 : var4, lane, NULL_TREE,
10187 : NULL_TREE);
10188 114 : TREE_THIS_NOTRAP (var4) = 1;
10189 : }
10190 : }
10191 : else
10192 : var2 = val;
10193 : }
10194 : }
10195 464 : gcc_assert (var2);
10196 : }
10197 : else
10198 : {
10199 696 : var2 = build_outer_var_ref (var, octx);
10200 696 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10201 : {
10202 220 : var3 = maybe_lookup_decl (new_vard, octx);
10203 220 : if (var3 == new_vard || var3 == NULL_TREE)
10204 : var3 = NULL_TREE;
10205 108 : else if (is_simd && octx->scan_exclusive && !input_phase)
10206 : {
10207 16 : var4 = maybe_lookup_decl (var3, octx);
10208 16 : if (var4 == var3 || var4 == NULL_TREE)
10209 : {
10210 0 : if (TREE_ADDRESSABLE (TREE_TYPE (new_var)))
10211 : {
10212 : var4 = var3;
10213 : var3 = NULL_TREE;
10214 : }
10215 : else
10216 16 : var4 = NULL_TREE;
10217 : }
10218 : }
10219 : }
10220 664 : if (is_simd
10221 484 : && octx->scan_exclusive
10222 244 : && !input_phase
10223 244 : && var4 == NULL_TREE)
10224 106 : var4 = create_tmp_var (TREE_TYPE (val));
10225 : }
10226 1160 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10227 : {
10228 376 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
10229 376 : if (input_phase)
10230 : {
10231 188 : if (var3)
10232 : {
10233 : /* If we've added a separate identity element
10234 : variable, copy it over into val. */
10235 92 : tree x = lang_hooks.decls.omp_clause_assign_op (c, val,
10236 : var3);
10237 92 : gimplify_and_add (x, &before);
10238 : }
10239 96 : else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
10240 : {
10241 : /* Otherwise, assign to it the identity element. */
10242 96 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
10243 96 : if (is_for)
10244 16 : tseq = copy_gimple_seq_and_replace_locals (tseq);
10245 96 : tree ref = build_outer_var_ref (var, octx);
10246 96 : tree x = (DECL_HAS_VALUE_EXPR_P (new_vard)
10247 96 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
10248 40 : if (x)
10249 : {
10250 40 : if (new_vard != new_var)
10251 8 : val = build_fold_addr_expr_loc (clause_loc, val);
10252 40 : SET_DECL_VALUE_EXPR (new_vard, val);
10253 : }
10254 96 : SET_DECL_VALUE_EXPR (placeholder, ref);
10255 96 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
10256 96 : lower_omp (&tseq, octx);
10257 96 : if (x)
10258 40 : SET_DECL_VALUE_EXPR (new_vard, x);
10259 96 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
10260 96 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
10261 96 : gimple_seq_add_seq (&before, tseq);
10262 96 : if (is_simd)
10263 80 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
10264 : }
10265 : }
10266 188 : else if (is_simd)
10267 : {
10268 156 : tree x;
10269 156 : if (octx->scan_exclusive)
10270 : {
10271 72 : tree v4 = unshare_expr (var4);
10272 72 : tree v2 = unshare_expr (var2);
10273 72 : x = lang_hooks.decls.omp_clause_assign_op (c, v4, v2);
10274 72 : gimplify_and_add (x, &before);
10275 : }
10276 156 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
10277 156 : x = (DECL_HAS_VALUE_EXPR_P (new_vard)
10278 156 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
10279 156 : tree vexpr = val;
10280 156 : if (x && new_vard != new_var)
10281 30 : vexpr = build_fold_addr_expr_loc (clause_loc, val);
10282 156 : if (x)
10283 78 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
10284 156 : SET_DECL_VALUE_EXPR (placeholder, var2);
10285 156 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
10286 156 : lower_omp (&tseq, octx);
10287 156 : gimple_seq_add_seq (&before, tseq);
10288 156 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
10289 156 : if (x)
10290 78 : SET_DECL_VALUE_EXPR (new_vard, x);
10291 156 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
10292 156 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
10293 156 : if (octx->scan_inclusive)
10294 : {
10295 84 : x = lang_hooks.decls.omp_clause_assign_op (c, val,
10296 : var2);
10297 84 : gimplify_and_add (x, &before);
10298 : }
10299 72 : else if (lane0 == NULL_TREE)
10300 : {
10301 36 : x = lang_hooks.decls.omp_clause_assign_op (c, val,
10302 : var4);
10303 36 : gimplify_and_add (x, &before);
10304 : }
10305 : }
10306 : }
10307 : else
10308 : {
10309 784 : if (input_phase)
10310 : {
10311 : /* input phase. Set val to initializer before
10312 : the body. */
10313 392 : tree x = omp_reduction_init (c, TREE_TYPE (new_var));
10314 392 : gimplify_assign (val, x, &before);
10315 : }
10316 392 : else if (is_simd)
10317 : {
10318 : /* scan phase. */
10319 318 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
10320 318 : if (code == MINUS_EXPR)
10321 0 : code = PLUS_EXPR;
10322 :
10323 318 : tree x = build2 (code, TREE_TYPE (var2),
10324 : unshare_expr (var2), unshare_expr (val));
10325 318 : if (octx->scan_inclusive)
10326 : {
10327 154 : gimplify_assign (unshare_expr (var2), x, &before);
10328 154 : gimplify_assign (val, var2, &before);
10329 : }
10330 : else
10331 : {
10332 164 : gimplify_assign (unshare_expr (var4),
10333 : unshare_expr (var2), &before);
10334 164 : gimplify_assign (var2, x, &before);
10335 164 : if (lane0 == NULL_TREE)
10336 86 : gimplify_assign (val, var4, &before);
10337 : }
10338 : }
10339 : }
10340 1160 : if (octx->scan_exclusive && !input_phase && lane0)
10341 : {
10342 114 : tree vexpr = unshare_expr (var4);
10343 114 : TREE_OPERAND (vexpr, 1) = lane0;
10344 114 : if (new_vard != new_var)
10345 16 : vexpr = build_fold_addr_expr_loc (clause_loc, vexpr);
10346 114 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
10347 : }
10348 : }
10349 : }
10350 1268 : if (is_simd && !is_for_simd)
10351 : {
10352 590 : gsi_insert_seq_after (gsi_p, gimple_omp_body (stmt), GSI_SAME_STMT);
10353 590 : gsi_insert_seq_after (gsi_p, before, GSI_SAME_STMT);
10354 590 : gsi_replace (gsi_p, gimple_build_nop (), true);
10355 590 : return;
10356 : }
10357 678 : lower_omp (gimple_omp_body_ptr (stmt), octx);
10358 678 : if (before)
10359 : {
10360 256 : gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (stmt));
10361 256 : gsi_insert_seq_before (&gsi, before, GSI_SAME_STMT);
10362 : }
10363 : }
10364 :
10365 :
10366 : /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10367 : substitution of a couple of function calls. But in the NAMED case,
10368 : requires that languages coordinate a symbol name. It is therefore
10369 : best put here in common code. */
10370 :
10371 : static GTY(()) hash_map<tree, tree> *critical_name_mutexes;
10372 :
10373 : static void
10374 311 : lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10375 : {
10376 311 : tree block;
10377 311 : tree name, lock, unlock;
10378 311 : gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p));
10379 311 : gbind *bind;
10380 311 : location_t loc = gimple_location (stmt);
10381 311 : gimple_seq tbody;
10382 :
10383 311 : name = gimple_omp_critical_name (stmt);
10384 311 : if (name)
10385 : {
10386 88 : tree decl;
10387 :
10388 88 : if (!critical_name_mutexes)
10389 48 : critical_name_mutexes = hash_map<tree, tree>::create_ggc (10);
10390 :
10391 88 : tree *n = critical_name_mutexes->get (name);
10392 88 : if (n == NULL)
10393 : {
10394 70 : char *new_str;
10395 :
10396 70 : decl = create_tmp_var_raw (ptr_type_node);
10397 :
10398 70 : new_str = ACONCAT ((".gomp_critical_user_",
10399 : IDENTIFIER_POINTER (name), NULL));
10400 70 : DECL_NAME (decl) = get_identifier (new_str);
10401 70 : TREE_PUBLIC (decl) = 1;
10402 70 : TREE_STATIC (decl) = 1;
10403 70 : DECL_COMMON (decl) = 1;
10404 70 : DECL_ARTIFICIAL (decl) = 1;
10405 70 : DECL_IGNORED_P (decl) = 1;
10406 :
10407 70 : varpool_node::finalize_decl (decl);
10408 :
10409 70 : critical_name_mutexes->put (name, decl);
10410 : }
10411 : else
10412 18 : decl = *n;
10413 :
10414 : /* If '#pragma omp critical' is inside offloaded region or
10415 : inside function marked as offloadable, the symbol must be
10416 : marked as offloadable too. */
10417 88 : omp_context *octx;
10418 88 : if (cgraph_node::get (current_function_decl)->offloadable)
10419 1 : varpool_node::get_create (decl)->offloadable = 1;
10420 : else
10421 173 : for (octx = ctx->outer; octx; octx = octx->outer)
10422 87 : if (is_gimple_omp_offloaded (octx->stmt))
10423 : {
10424 1 : varpool_node::get_create (decl)->offloadable = 1;
10425 1 : break;
10426 : }
10427 :
10428 88 : lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
10429 88 : lock = build_call_expr_loc (loc, lock, 1,
10430 : build_fold_addr_expr_loc (loc, decl));
10431 :
10432 88 : unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
10433 88 : unlock = build_call_expr_loc (loc, unlock, 1,
10434 : build_fold_addr_expr_loc (loc, decl));
10435 : }
10436 : else
10437 : {
10438 223 : lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
10439 223 : lock = build_call_expr_loc (loc, lock, 0);
10440 :
10441 223 : unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
10442 223 : unlock = build_call_expr_loc (loc, unlock, 0);
10443 : }
10444 :
10445 311 : push_gimplify_context ();
10446 :
10447 311 : block = make_node (BLOCK);
10448 311 : bind = gimple_build_bind (NULL, NULL, block);
10449 311 : gsi_replace (gsi_p, bind, true);
10450 311 : gimple_bind_add_stmt (bind, stmt);
10451 :
10452 311 : tbody = gimple_bind_body (bind);
10453 311 : gimplify_and_add (lock, &tbody);
10454 311 : gimple_bind_set_body (bind, tbody);
10455 :
10456 311 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
10457 311 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10458 311 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10459 311 : gimple_omp_set_body (stmt, NULL);
10460 :
10461 311 : tbody = gimple_bind_body (bind);
10462 311 : gimplify_and_add (unlock, &tbody);
10463 311 : gimple_bind_set_body (bind, tbody);
10464 :
10465 311 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10466 :
10467 311 : pop_gimplify_context (bind);
10468 311 : gimple_bind_append_vars (bind, ctx->block_vars);
10469 311 : BLOCK_VARS (block) = gimple_bind_vars (bind);
10470 311 : }
10471 :
10472 : /* A subroutine of lower_omp_for. Generate code to emit the predicate
10473 : for a lastprivate clause. Given a loop control predicate of (V
10474 : cond N2), we gate the clause on (!(V cond N2)). The lowered form
10475 : is appended to *DLIST, iterator initialization is appended to
10476 : *BODY_P. *CLIST is for lastprivate(conditional:) code that needs
10477 : to be emitted in a critical section. */
10478 :
10479 : static void
10480 46879 : lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
10481 : gimple_seq *dlist, gimple_seq *clist,
10482 : struct omp_context *ctx)
10483 : {
10484 46879 : tree clauses, cond, vinit;
10485 46879 : enum tree_code cond_code;
10486 46879 : gimple_seq stmts;
10487 :
10488 46879 : cond_code = fd->loop.cond_code;
10489 46879 : cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
10490 :
10491 : /* When possible, use a strict equality expression. This can let VRP
10492 : type optimizations deduce the value and remove a copy. */
10493 46879 : if (tree_fits_shwi_p (fd->loop.step))
10494 : {
10495 44640 : HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
10496 44640 : if (step == 1 || step == -1)
10497 46879 : cond_code = EQ_EXPR;
10498 : }
10499 :
10500 46879 : tree n2 = fd->loop.n2;
10501 46879 : if (fd->collapse > 1
10502 11250 : && TREE_CODE (n2) != INTEGER_CST
10503 53093 : && gimple_omp_for_combined_into_p (fd->for_stmt))
10504 : {
10505 2692 : struct omp_context *taskreg_ctx = NULL;
10506 2692 : if (gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
10507 : {
10508 1226 : gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt);
10509 1226 : if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR
10510 1226 : || gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_DISTRIBUTE)
10511 : {
10512 1194 : if (gimple_omp_for_combined_into_p (gfor))
10513 : {
10514 672 : gcc_assert (ctx->outer->outer
10515 : && is_parallel_ctx (ctx->outer->outer));
10516 : taskreg_ctx = ctx->outer->outer;
10517 : }
10518 : else
10519 : {
10520 522 : struct omp_for_data outer_fd;
10521 522 : omp_extract_for_data (gfor, &outer_fd, NULL);
10522 522 : n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2);
10523 : }
10524 : }
10525 32 : else if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_TASKLOOP)
10526 32 : taskreg_ctx = ctx->outer->outer;
10527 : }
10528 1466 : else if (is_taskreg_ctx (ctx->outer))
10529 : taskreg_ctx = ctx->outer;
10530 2692 : if (taskreg_ctx)
10531 : {
10532 2170 : int i;
10533 2170 : tree taskreg_clauses
10534 2170 : = gimple_omp_taskreg_clauses (taskreg_ctx->stmt);
10535 2170 : tree innerc = omp_find_clause (taskreg_clauses,
10536 : OMP_CLAUSE__LOOPTEMP_);
10537 2170 : gcc_assert (innerc);
10538 2170 : int count = fd->collapse;
10539 2170 : if (fd->non_rect
10540 24 : && fd->last_nonrect == fd->first_nonrect + 1)
10541 12 : if (tree v = gimple_omp_for_index (fd->for_stmt, fd->last_nonrect))
10542 12 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
10543 12 : count += 4;
10544 8582 : for (i = 0; i < count; i++)
10545 : {
10546 6412 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
10547 : OMP_CLAUSE__LOOPTEMP_);
10548 6412 : gcc_assert (innerc);
10549 : }
10550 2170 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
10551 : OMP_CLAUSE__LOOPTEMP_);
10552 2170 : if (innerc)
10553 1419 : n2 = fold_convert (TREE_TYPE (n2),
10554 : lookup_decl (OMP_CLAUSE_DECL (innerc),
10555 : taskreg_ctx));
10556 : }
10557 : }
10558 46879 : cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2);
10559 :
10560 46879 : clauses = gimple_omp_for_clauses (fd->for_stmt);
10561 46879 : stmts = NULL;
10562 46879 : lower_lastprivate_clauses (clauses, cond, body_p, &stmts, clist, ctx);
10563 46879 : if (!gimple_seq_empty_p (stmts))
10564 : {
10565 16362 : gimple_seq_add_seq (&stmts, *dlist);
10566 16362 : *dlist = stmts;
10567 :
10568 : /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10569 16362 : vinit = fd->loop.n1;
10570 16362 : if (cond_code == EQ_EXPR
10571 14375 : && tree_fits_shwi_p (fd->loop.n2)
10572 25923 : && ! integer_zerop (fd->loop.n2))
10573 8345 : vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
10574 : else
10575 8017 : vinit = unshare_expr (vinit);
10576 :
10577 : /* Initialize the iterator variable, so that threads that don't execute
10578 : any iterations don't execute the lastprivate clauses by accident. */
10579 16362 : gimplify_assign (fd->loop.v, vinit, body_p);
10580 : }
10581 46879 : }
10582 :
10583 : /* OpenACC privatization.
10584 :
10585 : Or, in other words, *sharing* at the respective OpenACC level of
10586 : parallelism.
10587 :
10588 : From a correctness perspective, a non-addressable variable can't be accessed
10589 : outside the current thread, so it can go in a (faster than shared memory)
10590 : register -- though that register may need to be broadcast in some
10591 : circumstances. A variable can only meaningfully be "shared" across workers
10592 : or vector lanes if its address is taken, e.g. by a call to an atomic
10593 : builtin.
10594 :
10595 : From an optimisation perspective, the answer might be fuzzier: maybe
10596 : sometimes, using shared memory directly would be faster than
10597 : broadcasting. */
10598 :
10599 : static void
10600 18997 : oacc_privatization_begin_diagnose_var (const dump_flags_t l_dump_flags,
10601 : const location_t loc, const tree c,
10602 : const tree decl)
10603 : {
10604 18997 : const dump_user_location_t d_u_loc
10605 18997 : = dump_user_location_t::from_location_t (loc);
10606 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
10607 : #if __GNUC__ >= 10
10608 18997 : # pragma GCC diagnostic push
10609 18997 : # pragma GCC diagnostic ignored "-Wformat"
10610 : #endif
10611 18997 : dump_printf_loc (l_dump_flags, d_u_loc,
10612 : "variable %<%T%> ", decl);
10613 : #if __GNUC__ >= 10
10614 18997 : # pragma GCC diagnostic pop
10615 : #endif
10616 18997 : if (c)
10617 2797 : dump_printf (l_dump_flags,
10618 : "in %qs clause ",
10619 2797 : omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
10620 : else
10621 16200 : dump_printf (l_dump_flags,
10622 : "declared in block ");
10623 18997 : }
10624 :
10625 : static bool
10626 41011 : oacc_privatization_candidate_p (const location_t loc, const tree c,
10627 : const tree decl)
10628 : {
10629 41011 : dump_flags_t l_dump_flags = get_openacc_privatization_dump_flags ();
10630 :
10631 : /* There is some differentiation depending on block vs. clause. */
10632 41011 : bool block = !c;
10633 :
10634 41011 : bool res = true;
10635 :
10636 41011 : if (res && !VAR_P (decl))
10637 : {
10638 : /* A PARM_DECL (appearing in a 'private' clause) is expected to have been
10639 : privatized into a new VAR_DECL. */
10640 712 : gcc_checking_assert (TREE_CODE (decl) != PARM_DECL);
10641 :
10642 712 : res = false;
10643 :
10644 712 : if (dump_enabled_p ())
10645 : {
10646 683 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10647 683 : dump_printf (l_dump_flags,
10648 : "potentially has improper OpenACC privatization level: %qs\n",
10649 683 : get_tree_code_name (TREE_CODE (decl)));
10650 : }
10651 : }
10652 :
10653 40982 : if (res && block && TREE_STATIC (decl))
10654 : {
10655 52 : res = false;
10656 :
10657 52 : if (dump_enabled_p ())
10658 : {
10659 28 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10660 28 : dump_printf (l_dump_flags,
10661 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10662 : "static");
10663 : }
10664 : }
10665 :
10666 40987 : if (res && block && DECL_EXTERNAL (decl))
10667 : {
10668 12 : res = false;
10669 :
10670 12 : if (dump_enabled_p ())
10671 : {
10672 12 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10673 12 : dump_printf (l_dump_flags,
10674 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10675 : "external");
10676 : }
10677 : }
10678 :
10679 41011 : if (res && !TREE_ADDRESSABLE (decl))
10680 : {
10681 39144 : res = false;
10682 :
10683 39144 : if (dump_enabled_p ())
10684 : {
10685 17456 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10686 17456 : dump_printf (l_dump_flags,
10687 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10688 : "not addressable");
10689 : }
10690 : }
10691 :
10692 : /* If an artificial variable has been added to a bind, e.g.
10693 : a compiler-generated temporary structure used by the Fortran front-end, do
10694 : not consider it as a privatization candidate. Note that variables on
10695 : the stack are private per-thread by default: making them "gang-private"
10696 : for OpenACC actually means to share a single instance of a variable
10697 : amongst all workers and threads spawned within each gang.
10698 : At present, no compiler-generated artificial variables require such
10699 : sharing semantics, so this is safe. */
10700 :
10701 18547 : if (res && block && DECL_ARTIFICIAL (decl))
10702 : {
10703 548 : res = false;
10704 :
10705 548 : if (dump_enabled_p ())
10706 : {
10707 360 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10708 360 : dump_printf (l_dump_flags,
10709 : "isn%'t candidate for adjusting OpenACC privatization "
10710 : "level: %s\n", "artificial");
10711 : }
10712 : }
10713 :
10714 40823 : if (res)
10715 : {
10716 543 : if (dump_enabled_p ())
10717 : {
10718 458 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10719 458 : dump_printf (l_dump_flags,
10720 : "is candidate for adjusting OpenACC privatization level\n");
10721 : }
10722 : }
10723 :
10724 41011 : if (dump_file && (dump_flags & TDF_DETAILS))
10725 : {
10726 0 : print_generic_decl (dump_file, decl, dump_flags);
10727 0 : fprintf (dump_file, "\n");
10728 : }
10729 :
10730 41011 : return res;
10731 : }
10732 :
10733 : /* Scan CLAUSES for candidates for adjusting OpenACC privatization level in
10734 : CTX. */
10735 :
10736 : static void
10737 11323 : oacc_privatization_scan_clause_chain (omp_context *ctx, tree clauses)
10738 : {
10739 35216 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
10740 23893 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE)
10741 : {
10742 12982 : tree decl = OMP_CLAUSE_DECL (c);
10743 :
10744 12982 : tree new_decl = lookup_decl (decl, ctx);
10745 :
10746 12982 : if (!oacc_privatization_candidate_p (OMP_CLAUSE_LOCATION (c), c,
10747 : new_decl))
10748 12651 : continue;
10749 :
10750 331 : gcc_checking_assert
10751 : (!ctx->oacc_privatization_candidates.contains (new_decl));
10752 331 : ctx->oacc_privatization_candidates.safe_push (new_decl);
10753 : }
10754 11323 : }
10755 :
10756 : /* Scan DECLS for candidates for adjusting OpenACC privatization level in
10757 : CTX. */
10758 :
10759 : static void
10760 23081 : oacc_privatization_scan_decl_chain (omp_context *ctx, tree decls)
10761 : {
10762 51110 : for (tree decl = decls; decl; decl = DECL_CHAIN (decl))
10763 : {
10764 28029 : tree new_decl = lookup_decl (decl, ctx);
10765 28029 : gcc_checking_assert (new_decl == decl);
10766 :
10767 28029 : if (!oacc_privatization_candidate_p (gimple_location (ctx->stmt), NULL,
10768 : new_decl))
10769 27817 : continue;
10770 :
10771 212 : gcc_checking_assert
10772 : (!ctx->oacc_privatization_candidates.contains (new_decl));
10773 212 : ctx->oacc_privatization_candidates.safe_push (new_decl);
10774 : }
10775 23081 : }
10776 :
10777 : /* Callback for walk_gimple_seq. Find #pragma omp scan statement. */
10778 :
10779 : static tree
10780 2225 : omp_find_scan (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10781 : struct walk_stmt_info *wi)
10782 : {
10783 2225 : gimple *stmt = gsi_stmt (*gsi_p);
10784 :
10785 2225 : *handled_ops_p = true;
10786 2225 : switch (gimple_code (stmt))
10787 : {
10788 418 : WALK_SUBSTMTS;
10789 :
10790 166 : case GIMPLE_OMP_FOR:
10791 166 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD
10792 166 : && gimple_omp_for_combined_into_p (stmt))
10793 166 : *handled_ops_p = false;
10794 : break;
10795 :
10796 678 : case GIMPLE_OMP_SCAN:
10797 678 : *(gimple_stmt_iterator *) (wi->info) = *gsi_p;
10798 678 : return integer_zero_node;
10799 : default:
10800 : break;
10801 : }
10802 : return NULL;
10803 : }
10804 :
10805 : /* Helper function for lower_omp_for, add transformations for a worksharing
10806 : loop with scan directives inside of it.
10807 : For worksharing loop not combined with simd, transform:
10808 : #pragma omp for reduction(inscan,+:r) private(i)
10809 : for (i = 0; i < n; i = i + 1)
10810 : {
10811 : {
10812 : update (r);
10813 : }
10814 : #pragma omp scan inclusive(r)
10815 : {
10816 : use (r);
10817 : }
10818 : }
10819 :
10820 : into two worksharing loops + code to merge results:
10821 :
10822 : num_threads = omp_get_num_threads ();
10823 : thread_num = omp_get_thread_num ();
10824 : if (thread_num == 0) goto <D.2099>; else goto <D.2100>;
10825 : <D.2099>:
10826 : var2 = r;
10827 : goto <D.2101>;
10828 : <D.2100>:
10829 : // For UDRs this is UDR init, or if ctors are needed, copy from
10830 : // var3 that has been constructed to contain the neutral element.
10831 : var2 = 0;
10832 : <D.2101>:
10833 : ivar = 0;
10834 : // The _scantemp_ clauses will arrange for rpriva to be initialized to
10835 : // a shared array with num_threads elements and rprivb to a local array
10836 : // number of elements equal to the number of (contiguous) iterations the
10837 : // current thread will perform. controlb and controlp variables are
10838 : // temporaries to handle deallocation of rprivb at the end of second
10839 : // GOMP_FOR.
10840 : #pragma omp for _scantemp_(rpriva) _scantemp_(rprivb) _scantemp_(controlb) \
10841 : _scantemp_(controlp) reduction(inscan,+:r) private(i) nowait
10842 : for (i = 0; i < n; i = i + 1)
10843 : {
10844 : {
10845 : // For UDRs this is UDR init or copy from var3.
10846 : r = 0;
10847 : // This is the input phase from user code.
10848 : update (r);
10849 : }
10850 : {
10851 : // For UDRs this is UDR merge.
10852 : var2 = var2 + r;
10853 : // Rather than handing it over to the user, save to local thread's
10854 : // array.
10855 : rprivb[ivar] = var2;
10856 : // For exclusive scan, the above two statements are swapped.
10857 : ivar = ivar + 1;
10858 : }
10859 : }
10860 : // And remember the final value from this thread's into the shared
10861 : // rpriva array.
10862 : rpriva[(sizetype) thread_num] = var2;
10863 : // If more than one thread, compute using Work-Efficient prefix sum
10864 : // the inclusive parallel scan of the rpriva array.
10865 : if (num_threads > 1) goto <D.2102>; else goto <D.2103>;
10866 : <D.2102>:
10867 : GOMP_barrier ();
10868 : down = 0;
10869 : k = 1;
10870 : num_threadsu = (unsigned int) num_threads;
10871 : thread_numup1 = (unsigned int) thread_num + 1;
10872 : <D.2108>:
10873 : twok = k << 1;
10874 : if (twok > num_threadsu) goto <D.2110>; else goto <D.2111>;
10875 : <D.2110>:
10876 : down = 4294967295;
10877 : k = k >> 1;
10878 : if (k == num_threadsu) goto <D.2112>; else goto <D.2111>;
10879 : <D.2112>:
10880 : k = k >> 1;
10881 : <D.2111>:
10882 : twok = k << 1;
10883 : cplx = .MUL_OVERFLOW (thread_nump1, twok);
10884 : mul = REALPART_EXPR <cplx>;
10885 : ovf = IMAGPART_EXPR <cplx>;
10886 : if (ovf == 0) goto <D.2116>; else goto <D.2117>;
10887 : <D.2116>:
10888 : andv = k & down;
10889 : andvm1 = andv + 4294967295;
10890 : l = mul + andvm1;
10891 : if (l < num_threadsu) goto <D.2120>; else goto <D.2117>;
10892 : <D.2120>:
10893 : // For UDRs this is UDR merge, performed using var2 variable as temporary,
10894 : // i.e. var2 = rpriva[l - k]; UDR merge (var2, rpriva[l]); rpriva[l] = var2;
10895 : rpriva[l] = rpriva[l - k] + rpriva[l];
10896 : <D.2117>:
10897 : if (down == 0) goto <D.2121>; else goto <D.2122>;
10898 : <D.2121>:
10899 : k = k << 1;
10900 : goto <D.2123>;
10901 : <D.2122>:
10902 : k = k >> 1;
10903 : <D.2123>:
10904 : GOMP_barrier ();
10905 : if (k != 0) goto <D.2108>; else goto <D.2103>;
10906 : <D.2103>:
10907 : if (thread_num == 0) goto <D.2124>; else goto <D.2125>;
10908 : <D.2124>:
10909 : // For UDRs this is UDR init or copy from var3.
10910 : var2 = 0;
10911 : goto <D.2126>;
10912 : <D.2125>:
10913 : var2 = rpriva[thread_num - 1];
10914 : <D.2126>:
10915 : ivar = 0;
10916 : #pragma omp for _scantemp_(controlb) _scantemp_(controlp) \
10917 : reduction(inscan,+:r) private(i)
10918 : for (i = 0; i < n; i = i + 1)
10919 : {
10920 : {
10921 : // For UDRs, this is r = var2; UDR merge (r, rprivb[ivar]);
10922 : r = var2 + rprivb[ivar];
10923 : }
10924 : {
10925 : // This is the scan phase from user code.
10926 : use (r);
10927 : // Plus a bump of the iterator.
10928 : ivar = ivar + 1;
10929 : }
10930 : } */
10931 :
10932 : static void
10933 173 : lower_omp_for_scan (gimple_seq *body_p, gimple_seq *dlist, gomp_for *stmt,
10934 : struct omp_for_data *fd, omp_context *ctx)
10935 : {
10936 173 : bool is_for_simd = gimple_omp_for_combined_p (stmt);
10937 173 : gcc_assert (ctx->scan_inclusive || ctx->scan_exclusive);
10938 :
10939 173 : gimple_seq body = gimple_omp_body (stmt);
10940 173 : gimple_stmt_iterator input1_gsi = gsi_none ();
10941 173 : struct walk_stmt_info wi;
10942 173 : memset (&wi, 0, sizeof (wi));
10943 173 : wi.val_only = true;
10944 173 : wi.info = (void *) &input1_gsi;
10945 173 : walk_gimple_seq_mod (&body, omp_find_scan, NULL, &wi);
10946 173 : gcc_assert (!gsi_end_p (input1_gsi));
10947 :
10948 173 : gimple *input_stmt1 = gsi_stmt (input1_gsi);
10949 173 : gimple_stmt_iterator gsi = input1_gsi;
10950 173 : gsi_next (&gsi);
10951 173 : gimple_stmt_iterator scan1_gsi = gsi;
10952 173 : gimple *scan_stmt1 = gsi_stmt (gsi);
10953 173 : gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
10954 :
10955 173 : gimple_seq input_body = gimple_omp_body (input_stmt1);
10956 173 : gimple_seq scan_body = gimple_omp_body (scan_stmt1);
10957 173 : gimple_omp_set_body (input_stmt1, NULL);
10958 173 : gimple_omp_set_body (scan_stmt1, NULL);
10959 173 : gimple_omp_set_body (stmt, NULL);
10960 :
10961 173 : gomp_for *new_stmt = as_a <gomp_for *> (gimple_copy (stmt));
10962 173 : gimple_seq new_body = copy_gimple_seq_and_replace_locals (body);
10963 173 : gimple_omp_set_body (stmt, body);
10964 173 : gimple_omp_set_body (input_stmt1, input_body);
10965 :
10966 173 : gimple_stmt_iterator input2_gsi = gsi_none ();
10967 173 : memset (&wi, 0, sizeof (wi));
10968 173 : wi.val_only = true;
10969 173 : wi.info = (void *) &input2_gsi;
10970 173 : walk_gimple_seq_mod (&new_body, omp_find_scan, NULL, &wi);
10971 173 : gcc_assert (!gsi_end_p (input2_gsi));
10972 :
10973 173 : gimple *input_stmt2 = gsi_stmt (input2_gsi);
10974 173 : gsi = input2_gsi;
10975 173 : gsi_next (&gsi);
10976 173 : gimple_stmt_iterator scan2_gsi = gsi;
10977 173 : gimple *scan_stmt2 = gsi_stmt (gsi);
10978 173 : gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
10979 173 : gimple_omp_set_body (scan_stmt2, scan_body);
10980 :
10981 173 : gimple_stmt_iterator input3_gsi = gsi_none ();
10982 173 : gimple_stmt_iterator scan3_gsi = gsi_none ();
10983 173 : gimple_stmt_iterator input4_gsi = gsi_none ();
10984 173 : gimple_stmt_iterator scan4_gsi = gsi_none ();
10985 173 : gimple *input_stmt3 = NULL, *scan_stmt3 = NULL;
10986 173 : gimple *input_stmt4 = NULL, *scan_stmt4 = NULL;
10987 173 : omp_context *input_simd_ctx = NULL, *scan_simd_ctx = NULL;
10988 173 : if (is_for_simd)
10989 : {
10990 83 : memset (&wi, 0, sizeof (wi));
10991 83 : wi.val_only = true;
10992 83 : wi.info = (void *) &input3_gsi;
10993 83 : walk_gimple_seq_mod (&input_body, omp_find_scan, NULL, &wi);
10994 83 : gcc_assert (!gsi_end_p (input3_gsi));
10995 :
10996 83 : input_stmt3 = gsi_stmt (input3_gsi);
10997 83 : gsi = input3_gsi;
10998 83 : gsi_next (&gsi);
10999 83 : scan3_gsi = gsi;
11000 83 : scan_stmt3 = gsi_stmt (gsi);
11001 83 : gcc_assert (scan_stmt3 && gimple_code (scan_stmt3) == GIMPLE_OMP_SCAN);
11002 :
11003 83 : memset (&wi, 0, sizeof (wi));
11004 83 : wi.val_only = true;
11005 83 : wi.info = (void *) &input4_gsi;
11006 83 : walk_gimple_seq_mod (&scan_body, omp_find_scan, NULL, &wi);
11007 83 : gcc_assert (!gsi_end_p (input4_gsi));
11008 :
11009 83 : input_stmt4 = gsi_stmt (input4_gsi);
11010 83 : gsi = input4_gsi;
11011 83 : gsi_next (&gsi);
11012 83 : scan4_gsi = gsi;
11013 83 : scan_stmt4 = gsi_stmt (gsi);
11014 83 : gcc_assert (scan_stmt4 && gimple_code (scan_stmt4) == GIMPLE_OMP_SCAN);
11015 :
11016 83 : input_simd_ctx = maybe_lookup_ctx (input_stmt3)->outer;
11017 83 : scan_simd_ctx = maybe_lookup_ctx (input_stmt4)->outer;
11018 : }
11019 :
11020 173 : tree num_threads = create_tmp_var (integer_type_node);
11021 173 : tree thread_num = create_tmp_var (integer_type_node);
11022 173 : tree nthreads_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
11023 173 : tree threadnum_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
11024 173 : gimple *g = gimple_build_call (nthreads_decl, 0);
11025 173 : gimple_call_set_lhs (g, num_threads);
11026 173 : gimple_seq_add_stmt (body_p, g);
11027 173 : g = gimple_build_call (threadnum_decl, 0);
11028 173 : gimple_call_set_lhs (g, thread_num);
11029 173 : gimple_seq_add_stmt (body_p, g);
11030 :
11031 173 : tree ivar = create_tmp_var (sizetype);
11032 173 : tree new_clauses1 = NULL_TREE, new_clauses2 = NULL_TREE;
11033 173 : tree *cp1 = &new_clauses1, *cp2 = &new_clauses2;
11034 173 : tree k = create_tmp_var (unsigned_type_node);
11035 173 : tree l = create_tmp_var (unsigned_type_node);
11036 :
11037 173 : gimple_seq clist = NULL, mdlist = NULL;
11038 173 : gimple_seq thr01_list = NULL, thrn1_list = NULL;
11039 173 : gimple_seq thr02_list = NULL, thrn2_list = NULL;
11040 173 : gimple_seq scan1_list = NULL, input2_list = NULL;
11041 173 : gimple_seq last_list = NULL, reduc_list = NULL;
11042 674 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
11043 501 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
11044 501 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
11045 : {
11046 205 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
11047 205 : tree var = OMP_CLAUSE_DECL (c);
11048 205 : tree new_var = lookup_decl (var, ctx);
11049 205 : tree var3 = NULL_TREE;
11050 205 : tree new_vard = new_var;
11051 205 : if (omp_privatize_by_reference (var))
11052 32 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
11053 205 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
11054 : {
11055 64 : var3 = maybe_lookup_decl (new_vard, ctx);
11056 64 : if (var3 == new_vard)
11057 173 : var3 = NULL_TREE;
11058 : }
11059 :
11060 205 : tree ptype = build_pointer_type (TREE_TYPE (new_var));
11061 205 : tree rpriva = create_tmp_var (ptype);
11062 205 : tree nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
11063 205 : OMP_CLAUSE_DECL (nc) = rpriva;
11064 205 : *cp1 = nc;
11065 205 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11066 :
11067 205 : tree rprivb = create_tmp_var (ptype);
11068 205 : nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
11069 205 : OMP_CLAUSE_DECL (nc) = rprivb;
11070 205 : OMP_CLAUSE__SCANTEMP__ALLOC (nc) = 1;
11071 205 : *cp1 = nc;
11072 205 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11073 :
11074 205 : tree var2 = create_tmp_var_raw (TREE_TYPE (new_var));
11075 205 : if (new_vard != new_var)
11076 32 : TREE_ADDRESSABLE (var2) = 1;
11077 205 : gimple_add_tmp_var (var2);
11078 :
11079 205 : tree x = fold_convert_loc (clause_loc, sizetype, thread_num);
11080 205 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11081 205 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11082 205 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11083 205 : tree rpriva_ref = build_simple_mem_ref_loc (clause_loc, x);
11084 :
11085 205 : x = fold_build2_loc (clause_loc, PLUS_EXPR, integer_type_node,
11086 : thread_num, integer_minus_one_node);
11087 205 : x = fold_convert_loc (clause_loc, sizetype, x);
11088 205 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11089 205 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11090 205 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11091 205 : tree rprivam1_ref = build_simple_mem_ref_loc (clause_loc, x);
11092 :
11093 205 : x = fold_convert_loc (clause_loc, sizetype, l);
11094 205 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11095 205 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11096 205 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11097 205 : tree rprival_ref = build_simple_mem_ref_loc (clause_loc, x);
11098 :
11099 205 : x = fold_build2_loc (clause_loc, MINUS_EXPR, unsigned_type_node, l, k);
11100 205 : x = fold_convert_loc (clause_loc, sizetype, x);
11101 205 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11102 205 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11103 205 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11104 205 : tree rprivalmk_ref = build_simple_mem_ref_loc (clause_loc, x);
11105 :
11106 205 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, ivar,
11107 205 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11108 205 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rprivb), rprivb, x);
11109 205 : tree rprivb_ref = build_simple_mem_ref_loc (clause_loc, x);
11110 :
11111 205 : tree var4 = is_for_simd ? new_var : var2;
11112 99 : tree var5 = NULL_TREE, var6 = NULL_TREE;
11113 99 : if (is_for_simd)
11114 : {
11115 99 : var5 = lookup_decl (var, input_simd_ctx);
11116 99 : var6 = lookup_decl (var, scan_simd_ctx);
11117 99 : if (new_vard != new_var)
11118 : {
11119 16 : var5 = build_simple_mem_ref_loc (clause_loc, var5);
11120 16 : var6 = build_simple_mem_ref_loc (clause_loc, var6);
11121 : }
11122 : }
11123 205 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
11124 : {
11125 64 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
11126 64 : tree val = var2;
11127 :
11128 64 : x = lang_hooks.decls.omp_clause_default_ctor
11129 64 : (c, var2, build_outer_var_ref (var, ctx));
11130 64 : if (x)
11131 32 : gimplify_and_add (x, &clist);
11132 :
11133 64 : x = build_outer_var_ref (var, ctx);
11134 64 : x = lang_hooks.decls.omp_clause_assign_op (c, unshare_expr (var4),
11135 : x);
11136 64 : gimplify_and_add (x, &thr01_list);
11137 :
11138 64 : tree y = (DECL_HAS_VALUE_EXPR_P (new_vard)
11139 64 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
11140 64 : if (var3)
11141 : {
11142 32 : x = unshare_expr (var4);
11143 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
11144 32 : gimplify_and_add (x, &thrn1_list);
11145 32 : x = unshare_expr (var4);
11146 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
11147 32 : gimplify_and_add (x, &thr02_list);
11148 : }
11149 32 : else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
11150 : {
11151 : /* Otherwise, assign to it the identity element. */
11152 32 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
11153 32 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11154 32 : if (!is_for_simd)
11155 : {
11156 16 : if (new_vard != new_var)
11157 4 : val = build_fold_addr_expr_loc (clause_loc, val);
11158 16 : SET_DECL_VALUE_EXPR (new_vard, val);
11159 16 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11160 : }
11161 32 : SET_DECL_VALUE_EXPR (placeholder, error_mark_node);
11162 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11163 32 : lower_omp (&tseq, ctx);
11164 32 : gimple_seq_add_seq (&thrn1_list, tseq);
11165 32 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
11166 32 : lower_omp (&tseq, ctx);
11167 32 : gimple_seq_add_seq (&thr02_list, tseq);
11168 32 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
11169 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11170 32 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
11171 32 : if (y)
11172 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11173 : else
11174 : {
11175 32 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11176 32 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11177 : }
11178 : }
11179 :
11180 64 : x = unshare_expr (var4);
11181 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, rprivam1_ref);
11182 64 : gimplify_and_add (x, &thrn2_list);
11183 :
11184 64 : if (is_for_simd)
11185 : {
11186 32 : x = unshare_expr (rprivb_ref);
11187 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var5);
11188 32 : gimplify_and_add (x, &scan1_list);
11189 : }
11190 : else
11191 : {
11192 32 : if (ctx->scan_exclusive)
11193 : {
11194 16 : x = unshare_expr (rprivb_ref);
11195 16 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
11196 16 : gimplify_and_add (x, &scan1_list);
11197 : }
11198 :
11199 32 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11200 32 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11201 32 : SET_DECL_VALUE_EXPR (placeholder, var2);
11202 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11203 32 : lower_omp (&tseq, ctx);
11204 32 : gimple_seq_add_seq (&scan1_list, tseq);
11205 :
11206 32 : if (ctx->scan_inclusive)
11207 : {
11208 16 : x = unshare_expr (rprivb_ref);
11209 16 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
11210 16 : gimplify_and_add (x, &scan1_list);
11211 : }
11212 : }
11213 :
11214 64 : x = unshare_expr (rpriva_ref);
11215 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
11216 : unshare_expr (var4));
11217 64 : gimplify_and_add (x, &mdlist);
11218 :
11219 96 : x = unshare_expr (is_for_simd ? var6 : new_var);
11220 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var4);
11221 64 : gimplify_and_add (x, &input2_list);
11222 :
11223 64 : val = rprivb_ref;
11224 64 : if (new_vard != new_var)
11225 24 : val = build_fold_addr_expr_loc (clause_loc, val);
11226 :
11227 64 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11228 64 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11229 64 : SET_DECL_VALUE_EXPR (new_vard, val);
11230 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11231 64 : if (is_for_simd)
11232 : {
11233 32 : SET_DECL_VALUE_EXPR (placeholder, var6);
11234 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11235 : }
11236 : else
11237 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11238 64 : lower_omp (&tseq, ctx);
11239 64 : if (y)
11240 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11241 : else
11242 : {
11243 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11244 64 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11245 : }
11246 64 : if (!is_for_simd)
11247 : {
11248 32 : SET_DECL_VALUE_EXPR (placeholder, new_var);
11249 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11250 32 : lower_omp (&tseq, ctx);
11251 : }
11252 64 : gimple_seq_add_seq (&input2_list, tseq);
11253 :
11254 64 : x = build_outer_var_ref (var, ctx);
11255 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, rpriva_ref);
11256 64 : gimplify_and_add (x, &last_list);
11257 :
11258 64 : x = lang_hooks.decls.omp_clause_assign_op (c, var2, rprivalmk_ref);
11259 64 : gimplify_and_add (x, &reduc_list);
11260 64 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11261 64 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11262 64 : val = rprival_ref;
11263 64 : if (new_vard != new_var)
11264 24 : val = build_fold_addr_expr_loc (clause_loc, val);
11265 64 : SET_DECL_VALUE_EXPR (new_vard, val);
11266 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11267 64 : SET_DECL_VALUE_EXPR (placeholder, var2);
11268 64 : lower_omp (&tseq, ctx);
11269 64 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
11270 64 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
11271 64 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11272 64 : if (y)
11273 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11274 : else
11275 : {
11276 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11277 64 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11278 : }
11279 64 : gimple_seq_add_seq (&reduc_list, tseq);
11280 64 : x = lang_hooks.decls.omp_clause_assign_op (c, rprival_ref, var2);
11281 64 : gimplify_and_add (x, &reduc_list);
11282 :
11283 64 : x = lang_hooks.decls.omp_clause_dtor (c, var2);
11284 64 : if (x)
11285 32 : gimplify_and_add (x, dlist);
11286 : }
11287 : else
11288 : {
11289 141 : x = build_outer_var_ref (var, ctx);
11290 141 : gimplify_assign (unshare_expr (var4), x, &thr01_list);
11291 :
11292 141 : x = omp_reduction_init (c, TREE_TYPE (new_var));
11293 141 : gimplify_assign (unshare_expr (var4), unshare_expr (x),
11294 : &thrn1_list);
11295 141 : gimplify_assign (unshare_expr (var4), x, &thr02_list);
11296 :
11297 141 : gimplify_assign (unshare_expr (var4), rprivam1_ref, &thrn2_list);
11298 :
11299 141 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
11300 141 : if (code == MINUS_EXPR)
11301 0 : code = PLUS_EXPR;
11302 :
11303 141 : if (is_for_simd)
11304 67 : gimplify_assign (unshare_expr (rprivb_ref), var5, &scan1_list);
11305 : else
11306 : {
11307 74 : if (ctx->scan_exclusive)
11308 28 : gimplify_assign (unshare_expr (rprivb_ref), var2,
11309 : &scan1_list);
11310 74 : x = build2 (code, TREE_TYPE (new_var), var2, new_var);
11311 74 : gimplify_assign (var2, x, &scan1_list);
11312 74 : if (ctx->scan_inclusive)
11313 46 : gimplify_assign (unshare_expr (rprivb_ref), var2,
11314 : &scan1_list);
11315 : }
11316 :
11317 141 : gimplify_assign (unshare_expr (rpriva_ref), unshare_expr (var4),
11318 : &mdlist);
11319 :
11320 141 : x = build2 (code, TREE_TYPE (new_var), var4, rprivb_ref);
11321 215 : gimplify_assign (is_for_simd ? var6 : new_var, x, &input2_list);
11322 :
11323 141 : gimplify_assign (build_outer_var_ref (var, ctx), rpriva_ref,
11324 : &last_list);
11325 :
11326 141 : x = build2 (code, TREE_TYPE (new_var), rprivalmk_ref,
11327 : unshare_expr (rprival_ref));
11328 141 : gimplify_assign (rprival_ref, x, &reduc_list);
11329 : }
11330 : }
11331 :
11332 173 : g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
11333 173 : gimple_seq_add_stmt (&scan1_list, g);
11334 173 : g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
11335 263 : gimple_seq_add_stmt (gimple_omp_body_ptr (is_for_simd
11336 : ? scan_stmt4 : scan_stmt2), g);
11337 :
11338 173 : tree controlb = create_tmp_var (boolean_type_node);
11339 173 : tree controlp = create_tmp_var (ptr_type_node);
11340 173 : tree nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11341 173 : OMP_CLAUSE_DECL (nc) = controlb;
11342 173 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11343 173 : *cp1 = nc;
11344 173 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11345 173 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11346 173 : OMP_CLAUSE_DECL (nc) = controlp;
11347 173 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11348 173 : *cp1 = nc;
11349 173 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11350 173 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11351 173 : OMP_CLAUSE_DECL (nc) = controlb;
11352 173 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11353 173 : *cp2 = nc;
11354 173 : cp2 = &OMP_CLAUSE_CHAIN (nc);
11355 173 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11356 173 : OMP_CLAUSE_DECL (nc) = controlp;
11357 173 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11358 173 : *cp2 = nc;
11359 173 : cp2 = &OMP_CLAUSE_CHAIN (nc);
11360 :
11361 173 : *cp1 = gimple_omp_for_clauses (stmt);
11362 173 : gimple_omp_for_set_clauses (stmt, new_clauses1);
11363 173 : *cp2 = gimple_omp_for_clauses (new_stmt);
11364 173 : gimple_omp_for_set_clauses (new_stmt, new_clauses2);
11365 :
11366 173 : if (is_for_simd)
11367 : {
11368 83 : gimple_seq_add_seq (gimple_omp_body_ptr (scan_stmt3), scan1_list);
11369 83 : gimple_seq_add_seq (gimple_omp_body_ptr (input_stmt4), input2_list);
11370 :
11371 83 : gsi_insert_seq_after (&input3_gsi, gimple_omp_body (input_stmt3),
11372 : GSI_SAME_STMT);
11373 83 : gsi_remove (&input3_gsi, true);
11374 83 : gsi_insert_seq_after (&scan3_gsi, gimple_omp_body (scan_stmt3),
11375 : GSI_SAME_STMT);
11376 83 : gsi_remove (&scan3_gsi, true);
11377 83 : gsi_insert_seq_after (&input4_gsi, gimple_omp_body (input_stmt4),
11378 : GSI_SAME_STMT);
11379 83 : gsi_remove (&input4_gsi, true);
11380 83 : gsi_insert_seq_after (&scan4_gsi, gimple_omp_body (scan_stmt4),
11381 : GSI_SAME_STMT);
11382 83 : gsi_remove (&scan4_gsi, true);
11383 : }
11384 : else
11385 : {
11386 90 : gimple_omp_set_body (scan_stmt1, scan1_list);
11387 90 : gimple_omp_set_body (input_stmt2, input2_list);
11388 : }
11389 :
11390 173 : gsi_insert_seq_after (&input1_gsi, gimple_omp_body (input_stmt1),
11391 : GSI_SAME_STMT);
11392 173 : gsi_remove (&input1_gsi, true);
11393 173 : gsi_insert_seq_after (&scan1_gsi, gimple_omp_body (scan_stmt1),
11394 : GSI_SAME_STMT);
11395 173 : gsi_remove (&scan1_gsi, true);
11396 173 : gsi_insert_seq_after (&input2_gsi, gimple_omp_body (input_stmt2),
11397 : GSI_SAME_STMT);
11398 173 : gsi_remove (&input2_gsi, true);
11399 173 : gsi_insert_seq_after (&scan2_gsi, gimple_omp_body (scan_stmt2),
11400 : GSI_SAME_STMT);
11401 173 : gsi_remove (&scan2_gsi, true);
11402 :
11403 173 : gimple_seq_add_seq (body_p, clist);
11404 :
11405 173 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
11406 173 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
11407 173 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
11408 173 : g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
11409 173 : gimple_seq_add_stmt (body_p, g);
11410 173 : g = gimple_build_label (lab1);
11411 173 : gimple_seq_add_stmt (body_p, g);
11412 173 : gimple_seq_add_seq (body_p, thr01_list);
11413 173 : g = gimple_build_goto (lab3);
11414 173 : gimple_seq_add_stmt (body_p, g);
11415 173 : g = gimple_build_label (lab2);
11416 173 : gimple_seq_add_stmt (body_p, g);
11417 173 : gimple_seq_add_seq (body_p, thrn1_list);
11418 173 : g = gimple_build_label (lab3);
11419 173 : gimple_seq_add_stmt (body_p, g);
11420 :
11421 173 : g = gimple_build_assign (ivar, size_zero_node);
11422 173 : gimple_seq_add_stmt (body_p, g);
11423 :
11424 173 : gimple_seq_add_stmt (body_p, stmt);
11425 173 : gimple_seq_add_seq (body_p, body);
11426 173 : gimple_seq_add_stmt (body_p, gimple_build_omp_continue (fd->loop.v,
11427 : fd->loop.v));
11428 :
11429 173 : g = gimple_build_omp_return (true);
11430 173 : gimple_seq_add_stmt (body_p, g);
11431 173 : gimple_seq_add_seq (body_p, mdlist);
11432 :
11433 173 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11434 173 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11435 173 : g = gimple_build_cond (GT_EXPR, num_threads, integer_one_node, lab1, lab2);
11436 173 : gimple_seq_add_stmt (body_p, g);
11437 173 : g = gimple_build_label (lab1);
11438 173 : gimple_seq_add_stmt (body_p, g);
11439 :
11440 173 : g = omp_build_barrier (NULL);
11441 173 : gimple_seq_add_stmt (body_p, g);
11442 :
11443 173 : tree down = create_tmp_var (unsigned_type_node);
11444 173 : g = gimple_build_assign (down, build_zero_cst (unsigned_type_node));
11445 173 : gimple_seq_add_stmt (body_p, g);
11446 :
11447 173 : g = gimple_build_assign (k, build_one_cst (unsigned_type_node));
11448 173 : gimple_seq_add_stmt (body_p, g);
11449 :
11450 173 : tree num_threadsu = create_tmp_var (unsigned_type_node);
11451 173 : g = gimple_build_assign (num_threadsu, NOP_EXPR, num_threads);
11452 173 : gimple_seq_add_stmt (body_p, g);
11453 :
11454 173 : tree thread_numu = create_tmp_var (unsigned_type_node);
11455 173 : g = gimple_build_assign (thread_numu, NOP_EXPR, thread_num);
11456 173 : gimple_seq_add_stmt (body_p, g);
11457 :
11458 173 : tree thread_nump1 = create_tmp_var (unsigned_type_node);
11459 173 : g = gimple_build_assign (thread_nump1, PLUS_EXPR, thread_numu,
11460 : build_int_cst (unsigned_type_node, 1));
11461 173 : gimple_seq_add_stmt (body_p, g);
11462 :
11463 173 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
11464 173 : g = gimple_build_label (lab3);
11465 173 : gimple_seq_add_stmt (body_p, g);
11466 :
11467 173 : tree twok = create_tmp_var (unsigned_type_node);
11468 173 : g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
11469 173 : gimple_seq_add_stmt (body_p, g);
11470 :
11471 173 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
11472 173 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
11473 173 : tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
11474 173 : g = gimple_build_cond (GT_EXPR, twok, num_threadsu, lab4, lab5);
11475 173 : gimple_seq_add_stmt (body_p, g);
11476 173 : g = gimple_build_label (lab4);
11477 173 : gimple_seq_add_stmt (body_p, g);
11478 173 : g = gimple_build_assign (down, build_all_ones_cst (unsigned_type_node));
11479 173 : gimple_seq_add_stmt (body_p, g);
11480 173 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11481 173 : gimple_seq_add_stmt (body_p, g);
11482 :
11483 173 : g = gimple_build_cond (EQ_EXPR, k, num_threadsu, lab6, lab5);
11484 173 : gimple_seq_add_stmt (body_p, g);
11485 173 : g = gimple_build_label (lab6);
11486 173 : gimple_seq_add_stmt (body_p, g);
11487 :
11488 173 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11489 173 : gimple_seq_add_stmt (body_p, g);
11490 :
11491 173 : g = gimple_build_label (lab5);
11492 173 : gimple_seq_add_stmt (body_p, g);
11493 :
11494 173 : g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
11495 173 : gimple_seq_add_stmt (body_p, g);
11496 :
11497 173 : tree cplx = create_tmp_var (build_complex_type (unsigned_type_node, false));
11498 173 : g = gimple_build_call_internal (IFN_MUL_OVERFLOW, 2, thread_nump1, twok);
11499 173 : gimple_call_set_lhs (g, cplx);
11500 173 : gimple_seq_add_stmt (body_p, g);
11501 173 : tree mul = create_tmp_var (unsigned_type_node);
11502 173 : g = gimple_build_assign (mul, REALPART_EXPR,
11503 : build1 (REALPART_EXPR, unsigned_type_node, cplx));
11504 173 : gimple_seq_add_stmt (body_p, g);
11505 173 : tree ovf = create_tmp_var (unsigned_type_node);
11506 173 : g = gimple_build_assign (ovf, IMAGPART_EXPR,
11507 : build1 (IMAGPART_EXPR, unsigned_type_node, cplx));
11508 173 : gimple_seq_add_stmt (body_p, g);
11509 :
11510 173 : tree lab7 = create_artificial_label (UNKNOWN_LOCATION);
11511 173 : tree lab8 = create_artificial_label (UNKNOWN_LOCATION);
11512 173 : g = gimple_build_cond (EQ_EXPR, ovf, build_zero_cst (unsigned_type_node),
11513 : lab7, lab8);
11514 173 : gimple_seq_add_stmt (body_p, g);
11515 173 : g = gimple_build_label (lab7);
11516 173 : gimple_seq_add_stmt (body_p, g);
11517 :
11518 173 : tree andv = create_tmp_var (unsigned_type_node);
11519 173 : g = gimple_build_assign (andv, BIT_AND_EXPR, k, down);
11520 173 : gimple_seq_add_stmt (body_p, g);
11521 173 : tree andvm1 = create_tmp_var (unsigned_type_node);
11522 173 : g = gimple_build_assign (andvm1, PLUS_EXPR, andv,
11523 : build_minus_one_cst (unsigned_type_node));
11524 173 : gimple_seq_add_stmt (body_p, g);
11525 :
11526 173 : g = gimple_build_assign (l, PLUS_EXPR, mul, andvm1);
11527 173 : gimple_seq_add_stmt (body_p, g);
11528 :
11529 173 : tree lab9 = create_artificial_label (UNKNOWN_LOCATION);
11530 173 : g = gimple_build_cond (LT_EXPR, l, num_threadsu, lab9, lab8);
11531 173 : gimple_seq_add_stmt (body_p, g);
11532 173 : g = gimple_build_label (lab9);
11533 173 : gimple_seq_add_stmt (body_p, g);
11534 173 : gimple_seq_add_seq (body_p, reduc_list);
11535 173 : g = gimple_build_label (lab8);
11536 173 : gimple_seq_add_stmt (body_p, g);
11537 :
11538 173 : tree lab10 = create_artificial_label (UNKNOWN_LOCATION);
11539 173 : tree lab11 = create_artificial_label (UNKNOWN_LOCATION);
11540 173 : tree lab12 = create_artificial_label (UNKNOWN_LOCATION);
11541 173 : g = gimple_build_cond (EQ_EXPR, down, build_zero_cst (unsigned_type_node),
11542 : lab10, lab11);
11543 173 : gimple_seq_add_stmt (body_p, g);
11544 173 : g = gimple_build_label (lab10);
11545 173 : gimple_seq_add_stmt (body_p, g);
11546 173 : g = gimple_build_assign (k, LSHIFT_EXPR, k, integer_one_node);
11547 173 : gimple_seq_add_stmt (body_p, g);
11548 173 : g = gimple_build_goto (lab12);
11549 173 : gimple_seq_add_stmt (body_p, g);
11550 173 : g = gimple_build_label (lab11);
11551 173 : gimple_seq_add_stmt (body_p, g);
11552 173 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11553 173 : gimple_seq_add_stmt (body_p, g);
11554 173 : g = gimple_build_label (lab12);
11555 173 : gimple_seq_add_stmt (body_p, g);
11556 :
11557 173 : g = omp_build_barrier (NULL);
11558 173 : gimple_seq_add_stmt (body_p, g);
11559 :
11560 173 : g = gimple_build_cond (NE_EXPR, k, build_zero_cst (unsigned_type_node),
11561 : lab3, lab2);
11562 173 : gimple_seq_add_stmt (body_p, g);
11563 :
11564 173 : g = gimple_build_label (lab2);
11565 173 : gimple_seq_add_stmt (body_p, g);
11566 :
11567 173 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11568 173 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11569 173 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
11570 173 : g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
11571 173 : gimple_seq_add_stmt (body_p, g);
11572 173 : g = gimple_build_label (lab1);
11573 173 : gimple_seq_add_stmt (body_p, g);
11574 173 : gimple_seq_add_seq (body_p, thr02_list);
11575 173 : g = gimple_build_goto (lab3);
11576 173 : gimple_seq_add_stmt (body_p, g);
11577 173 : g = gimple_build_label (lab2);
11578 173 : gimple_seq_add_stmt (body_p, g);
11579 173 : gimple_seq_add_seq (body_p, thrn2_list);
11580 173 : g = gimple_build_label (lab3);
11581 173 : gimple_seq_add_stmt (body_p, g);
11582 :
11583 173 : g = gimple_build_assign (ivar, size_zero_node);
11584 173 : gimple_seq_add_stmt (body_p, g);
11585 173 : gimple_seq_add_stmt (body_p, new_stmt);
11586 173 : gimple_seq_add_seq (body_p, new_body);
11587 :
11588 173 : gimple_seq new_dlist = NULL;
11589 173 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11590 173 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11591 173 : tree num_threadsm1 = create_tmp_var (integer_type_node);
11592 173 : g = gimple_build_assign (num_threadsm1, PLUS_EXPR, num_threads,
11593 : integer_minus_one_node);
11594 173 : gimple_seq_add_stmt (&new_dlist, g);
11595 173 : g = gimple_build_cond (EQ_EXPR, thread_num, num_threadsm1, lab1, lab2);
11596 173 : gimple_seq_add_stmt (&new_dlist, g);
11597 173 : g = gimple_build_label (lab1);
11598 173 : gimple_seq_add_stmt (&new_dlist, g);
11599 173 : gimple_seq_add_seq (&new_dlist, last_list);
11600 173 : g = gimple_build_label (lab2);
11601 173 : gimple_seq_add_stmt (&new_dlist, g);
11602 173 : gimple_seq_add_seq (&new_dlist, *dlist);
11603 173 : *dlist = new_dlist;
11604 173 : }
11605 :
11606 : /* Build an internal UNIQUE function with type IFN_UNIQUE_OACC_PRIVATE listing
11607 : the addresses of variables to be made private at the surrounding
11608 : parallelism level. Such functions appear in the gimple code stream in two
11609 : forms, e.g. for a partitioned loop:
11610 :
11611 : .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6, 1, 68);
11612 : .data_dep.6 = .UNIQUE (OACC_PRIVATE, .data_dep.6, -1, &w);
11613 : .data_dep.6 = .UNIQUE (OACC_FORK, .data_dep.6, -1);
11614 : .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6);
11615 :
11616 : or alternatively, OACC_PRIVATE can appear at the top level of a parallel,
11617 : not as part of a HEAD_MARK sequence:
11618 :
11619 : .UNIQUE (OACC_PRIVATE, 0, 0, &w);
11620 :
11621 : For such stand-alone appearances, the 3rd argument is always 0, denoting
11622 : gang partitioning. */
11623 :
11624 : static gcall *
11625 19722 : lower_oacc_private_marker (omp_context *ctx)
11626 : {
11627 19722 : if (ctx->oacc_privatization_candidates.length () == 0)
11628 : return NULL;
11629 :
11630 307 : auto_vec<tree, 5> args;
11631 :
11632 307 : args.quick_push (build_int_cst (integer_type_node, IFN_UNIQUE_OACC_PRIVATE));
11633 307 : args.quick_push (integer_zero_node);
11634 307 : args.quick_push (integer_minus_one_node);
11635 :
11636 307 : int i;
11637 307 : tree decl;
11638 694 : FOR_EACH_VEC_ELT (ctx->oacc_privatization_candidates, i, decl)
11639 : {
11640 387 : gcc_checking_assert (TREE_ADDRESSABLE (decl));
11641 387 : tree addr = build_fold_addr_expr (decl);
11642 387 : args.safe_push (addr);
11643 : }
11644 :
11645 307 : return gimple_build_call_internal_vec (IFN_UNIQUE, args);
11646 307 : }
11647 :
11648 : /* Lower code for an OMP loop directive. */
11649 :
11650 : static void
11651 46879 : lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11652 : {
11653 46879 : tree *rhs_p, block;
11654 46879 : struct omp_for_data fd, *fdp = NULL;
11655 46879 : gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
11656 46879 : gbind *new_stmt;
11657 46879 : gimple_seq omp_for_body, body, dlist, tred_ilist = NULL, tred_dlist = NULL;
11658 46879 : gimple_seq cnt_list = NULL, clist = NULL;
11659 46879 : gimple_seq oacc_head = NULL, oacc_tail = NULL;
11660 46879 : size_t i;
11661 :
11662 46879 : push_gimplify_context ();
11663 :
11664 46879 : if (is_gimple_omp_oacc (ctx->stmt))
11665 11323 : oacc_privatization_scan_clause_chain (ctx, gimple_omp_for_clauses (stmt));
11666 :
11667 46879 : lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
11668 :
11669 46879 : block = make_node (BLOCK);
11670 46879 : new_stmt = gimple_build_bind (NULL, NULL, block);
11671 : /* Replace at gsi right away, so that 'stmt' is no member
11672 : of a sequence anymore as we're going to add to a different
11673 : one below. */
11674 46879 : gsi_replace (gsi_p, new_stmt, true);
11675 :
11676 : /* Move declaration of temporaries in the loop body before we make
11677 : it go away. */
11678 46879 : omp_for_body = gimple_omp_body (stmt);
11679 46879 : if (!gimple_seq_empty_p (omp_for_body)
11680 46879 : && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
11681 : {
11682 19599 : gbind *inner_bind
11683 19599 : = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body));
11684 19599 : tree vars = gimple_bind_vars (inner_bind);
11685 19599 : if (is_gimple_omp_oacc (ctx->stmt))
11686 5303 : oacc_privatization_scan_decl_chain (ctx, vars);
11687 19599 : gimple_bind_append_vars (new_stmt, vars);
11688 : /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
11689 : keep them on the inner_bind and it's block. */
11690 19599 : gimple_bind_set_vars (inner_bind, NULL_TREE);
11691 19599 : if (gimple_bind_block (inner_bind))
11692 17043 : BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
11693 : }
11694 :
11695 46879 : if (gimple_omp_for_combined_into_p (stmt))
11696 : {
11697 14598 : omp_extract_for_data (stmt, &fd, NULL);
11698 14598 : fdp = &fd;
11699 :
11700 : /* We need two temporaries with fd.loop.v type (istart/iend)
11701 : and then (fd.collapse - 1) temporaries with the same
11702 : type for count2 ... countN-1 vars if not constant. */
11703 14598 : size_t count = 2;
11704 14598 : tree type = fd.iter_type;
11705 14598 : if (fd.collapse > 1
11706 4867 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11707 2692 : count += fd.collapse - 1;
11708 14598 : size_t count2 = 0;
11709 14598 : tree type2 = NULL_TREE;
11710 14598 : bool taskreg_for
11711 14598 : = (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
11712 14598 : || gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP);
11713 14598 : tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
11714 14598 : tree simtc = NULL;
11715 14598 : tree clauses = *pc;
11716 14598 : if (fd.collapse > 1
11717 4867 : && fd.non_rect
11718 137 : && fd.last_nonrect == fd.first_nonrect + 1
11719 89 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11720 64 : if (tree v = gimple_omp_for_index (stmt, fd.last_nonrect))
11721 64 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
11722 : {
11723 60 : v = gimple_omp_for_index (stmt, fd.first_nonrect);
11724 60 : type2 = TREE_TYPE (v);
11725 60 : count++;
11726 60 : count2 = 3;
11727 : }
11728 14598 : if (taskreg_for)
11729 7594 : outerc
11730 7594 : = omp_find_clause (gimple_omp_taskreg_clauses (ctx->outer->stmt),
11731 : OMP_CLAUSE__LOOPTEMP_);
11732 14598 : if (ctx->simt_stmt)
11733 0 : simtc = omp_find_clause (gimple_omp_for_clauses (ctx->simt_stmt),
11734 : OMP_CLAUSE__LOOPTEMP_);
11735 49203 : for (i = 0; i < count + count2; i++)
11736 : {
11737 34605 : tree temp;
11738 34605 : if (taskreg_for)
11739 : {
11740 18069 : gcc_assert (outerc);
11741 18069 : temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
11742 18069 : outerc = omp_find_clause (OMP_CLAUSE_CHAIN (outerc),
11743 : OMP_CLAUSE__LOOPTEMP_);
11744 : }
11745 : else
11746 : {
11747 : /* If there are 2 adjacent SIMD stmts, one with _simt_
11748 : clause, another without, make sure they have the same
11749 : decls in _looptemp_ clauses, because the outer stmt
11750 : they are combined into will look up just one inner_stmt. */
11751 16536 : if (ctx->simt_stmt)
11752 0 : temp = OMP_CLAUSE_DECL (simtc);
11753 : else
11754 32928 : temp = create_tmp_var (i >= count ? type2 : type);
11755 16536 : insert_decl_map (&ctx->outer->cb, temp, temp);
11756 : }
11757 34605 : *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
11758 34605 : OMP_CLAUSE_DECL (*pc) = temp;
11759 34605 : pc = &OMP_CLAUSE_CHAIN (*pc);
11760 34605 : if (ctx->simt_stmt)
11761 0 : simtc = omp_find_clause (OMP_CLAUSE_CHAIN (simtc),
11762 : OMP_CLAUSE__LOOPTEMP_);
11763 : }
11764 14598 : *pc = clauses;
11765 : }
11766 :
11767 : /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
11768 46879 : dlist = NULL;
11769 46879 : body = NULL;
11770 46879 : tree rclauses
11771 46879 : = omp_task_reductions_find_first (gimple_omp_for_clauses (stmt), OMP_FOR,
11772 : OMP_CLAUSE_REDUCTION);
11773 46879 : tree rtmp = NULL_TREE;
11774 46879 : if (rclauses)
11775 : {
11776 227 : tree type = build_pointer_type (pointer_sized_int_node);
11777 227 : tree temp = create_tmp_var (type);
11778 227 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
11779 227 : OMP_CLAUSE_DECL (c) = temp;
11780 227 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (stmt);
11781 227 : gimple_omp_for_set_clauses (stmt, c);
11782 227 : lower_omp_task_reductions (ctx, OMP_FOR,
11783 : gimple_omp_for_clauses (stmt),
11784 : &tred_ilist, &tred_dlist);
11785 227 : rclauses = c;
11786 227 : rtmp = make_ssa_name (type);
11787 227 : gimple_seq_add_stmt (&body, gimple_build_assign (rtmp, temp));
11788 : }
11789 :
11790 46879 : lower_lastprivate_conditional_clauses (gimple_omp_for_clauses_ptr (stmt),
11791 : ctx);
11792 :
11793 46879 : lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
11794 : fdp);
11795 93531 : gimple_seq_add_seq (rclauses ? &tred_ilist : &body,
11796 : gimple_omp_for_pre_body (stmt));
11797 :
11798 46879 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
11799 :
11800 46879 : gcall *private_marker = NULL;
11801 46879 : if (is_gimple_omp_oacc (ctx->stmt)
11802 46879 : && !gimple_seq_empty_p (omp_for_body))
11803 10331 : private_marker = lower_oacc_private_marker (ctx);
11804 :
11805 : /* Lower the header expressions. At this point, we can assume that
11806 : the header is of the form:
11807 :
11808 : #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
11809 :
11810 : We just need to make sure that VAL1, VAL2 and VAL3 are lowered
11811 : using the .omp_data_s mapping, if needed. */
11812 115041 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
11813 : {
11814 68162 : rhs_p = gimple_omp_for_initial_ptr (stmt, i);
11815 68162 : if (TREE_CODE (*rhs_p) == TREE_VEC)
11816 : {
11817 639 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
11818 149 : TREE_VEC_ELT (*rhs_p, 1)
11819 298 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
11820 639 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
11821 161 : TREE_VEC_ELT (*rhs_p, 2)
11822 322 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
11823 : }
11824 67523 : else if (!is_gimple_min_invariant (*rhs_p))
11825 11465 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11826 56058 : else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
11827 4713 : recompute_tree_invariant_for_addr_expr (*rhs_p);
11828 :
11829 68162 : rhs_p = gimple_omp_for_final_ptr (stmt, i);
11830 68162 : if (TREE_CODE (*rhs_p) == TREE_VEC)
11831 : {
11832 519 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
11833 142 : TREE_VEC_ELT (*rhs_p, 1)
11834 284 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
11835 519 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
11836 157 : TREE_VEC_ELT (*rhs_p, 2)
11837 314 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
11838 : }
11839 67643 : else if (!is_gimple_min_invariant (*rhs_p))
11840 16787 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11841 50856 : else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
11842 4630 : recompute_tree_invariant_for_addr_expr (*rhs_p);
11843 :
11844 68162 : rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
11845 68162 : if (!is_gimple_min_invariant (*rhs_p))
11846 6318 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11847 : }
11848 46879 : if (rclauses)
11849 227 : gimple_seq_add_seq (&tred_ilist, cnt_list);
11850 : else
11851 46652 : gimple_seq_add_seq (&body, cnt_list);
11852 :
11853 : /* Once lowered, extract the bounds and clauses. */
11854 46879 : omp_extract_for_data (stmt, &fd, NULL);
11855 :
11856 46879 : if (is_gimple_omp_oacc (ctx->stmt)
11857 46879 : && !ctx_in_oacc_kernels_region (ctx))
11858 9665 : lower_oacc_head_tail (gimple_location (stmt),
11859 : gimple_omp_for_clauses (stmt), private_marker,
11860 : &oacc_head, &oacc_tail, ctx);
11861 :
11862 : /* Add OpenACC partitioning and reduction markers just before the loop. */
11863 46879 : if (oacc_head)
11864 9665 : gimple_seq_add_seq (&body, oacc_head);
11865 :
11866 46879 : lower_omp_for_lastprivate (&fd, &body, &dlist, &clist, ctx);
11867 :
11868 46879 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
11869 85779 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
11870 70216 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
11871 70216 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
11872 : {
11873 227 : OMP_CLAUSE_DECL (c) = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
11874 227 : if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c)))
11875 42 : OMP_CLAUSE_LINEAR_STEP (c)
11876 84 : = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c),
11877 : ctx);
11878 : }
11879 :
11880 46513 : if ((ctx->scan_inclusive || ctx->scan_exclusive)
11881 47147 : && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
11882 173 : lower_omp_for_scan (&body, &dlist, stmt, &fd, ctx);
11883 : else
11884 : {
11885 46706 : gimple_seq_add_stmt (&body, stmt);
11886 46706 : gimple_seq_add_seq (&body, gimple_omp_body (stmt));
11887 : }
11888 :
11889 46879 : gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
11890 : fd.loop.v));
11891 :
11892 : /* After the loop, add exit clauses. */
11893 46879 : lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, &clist, ctx);
11894 :
11895 46879 : if (clist)
11896 : {
11897 179 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
11898 179 : gcall *g = gimple_build_call (fndecl, 0);
11899 179 : gimple_seq_add_stmt (&body, g);
11900 179 : gimple_seq_add_seq (&body, clist);
11901 179 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
11902 179 : g = gimple_build_call (fndecl, 0);
11903 179 : gimple_seq_add_stmt (&body, g);
11904 : }
11905 :
11906 46879 : if (ctx->cancellable)
11907 84 : gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
11908 :
11909 46879 : gimple_seq_add_seq (&body, dlist);
11910 :
11911 46879 : if (rclauses)
11912 : {
11913 227 : gimple_seq_add_seq (&tred_ilist, body);
11914 227 : body = tred_ilist;
11915 : }
11916 :
11917 46879 : body = maybe_catch_exception (body);
11918 :
11919 : /* Region exit marker goes at the end of the loop body. */
11920 46879 : gimple *g = gimple_build_omp_return (fd.have_nowait);
11921 46879 : gimple_seq_add_stmt (&body, g);
11922 :
11923 46879 : gimple_seq_add_seq (&body, tred_dlist);
11924 :
11925 46879 : maybe_add_implicit_barrier_cancel (ctx, g, &body);
11926 :
11927 46879 : if (rclauses)
11928 227 : OMP_CLAUSE_DECL (rclauses) = rtmp;
11929 :
11930 : /* Add OpenACC joining and reduction markers just after the loop. */
11931 46879 : if (oacc_tail)
11932 9665 : gimple_seq_add_seq (&body, oacc_tail);
11933 :
11934 46879 : pop_gimplify_context (new_stmt);
11935 :
11936 46879 : gimple_bind_append_vars (new_stmt, ctx->block_vars);
11937 46879 : maybe_remove_omp_member_access_dummy_vars (new_stmt);
11938 46879 : BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
11939 46879 : if (BLOCK_VARS (block))
11940 46242 : TREE_USED (block) = 1;
11941 :
11942 46879 : gimple_bind_set_body (new_stmt, body);
11943 46879 : gimple_omp_set_body (stmt, NULL);
11944 46879 : gimple_omp_for_set_pre_body (stmt, NULL);
11945 46879 : }
11946 :
11947 : /* Callback for walk_stmts. Check if the current statement only contains
11948 : GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
11949 :
11950 : static tree
11951 212883 : check_combined_parallel (gimple_stmt_iterator *gsi_p,
11952 : bool *handled_ops_p,
11953 : struct walk_stmt_info *wi)
11954 : {
11955 212883 : int *info = (int *) wi->info;
11956 212883 : gimple *stmt = gsi_stmt (*gsi_p);
11957 :
11958 212883 : *handled_ops_p = true;
11959 212883 : switch (gimple_code (stmt))
11960 : {
11961 11583 : WALK_SUBSTMTS;
11962 :
11963 : case GIMPLE_DEBUG:
11964 : break;
11965 1901 : case GIMPLE_OMP_FOR:
11966 1901 : case GIMPLE_OMP_SECTIONS:
11967 1901 : *info = *info == 0 ? 1 : -1;
11968 1901 : break;
11969 199360 : default:
11970 199360 : *info = -1;
11971 199360 : break;
11972 : }
11973 212883 : return NULL;
11974 : }
11975 :
11976 : struct omp_taskcopy_context
11977 : {
11978 : /* This field must be at the beginning, as we do "inheritance": Some
11979 : callback functions for tree-inline.cc (e.g., omp_copy_decl)
11980 : receive a copy_body_data pointer that is up-casted to an
11981 : omp_context pointer. */
11982 : copy_body_data cb;
11983 : omp_context *ctx;
11984 : };
11985 :
11986 : static tree
11987 366 : task_copyfn_copy_decl (tree var, copy_body_data *cb)
11988 : {
11989 366 : struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
11990 :
11991 366 : if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
11992 158 : return create_tmp_var (TREE_TYPE (var));
11993 :
11994 : return var;
11995 : }
11996 :
11997 : static tree
11998 50 : task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
11999 : {
12000 50 : tree name, new_fields = NULL, type, f;
12001 :
12002 50 : type = lang_hooks.types.make_type (RECORD_TYPE);
12003 50 : name = DECL_NAME (TYPE_NAME (orig_type));
12004 50 : name = build_decl (gimple_location (tcctx->ctx->stmt),
12005 : TYPE_DECL, name, type);
12006 50 : TYPE_NAME (type) = name;
12007 :
12008 1140 : for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
12009 : {
12010 1090 : tree new_f = copy_node (f);
12011 1090 : DECL_CONTEXT (new_f) = type;
12012 1090 : TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
12013 1090 : TREE_CHAIN (new_f) = new_fields;
12014 1090 : walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
12015 1090 : walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
12016 1090 : walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
12017 : &tcctx->cb, NULL);
12018 1090 : new_fields = new_f;
12019 1090 : tcctx->cb.decl_map->put (f, new_f);
12020 : }
12021 50 : TYPE_FIELDS (type) = nreverse (new_fields);
12022 50 : layout_type (type);
12023 50 : return type;
12024 : }
12025 :
12026 : /* Create task copyfn. */
12027 :
12028 : static void
12029 504 : create_task_copyfn (gomp_task *task_stmt, omp_context *ctx)
12030 : {
12031 504 : struct function *child_cfun;
12032 504 : tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
12033 504 : tree record_type, srecord_type, bind, list;
12034 504 : bool record_needs_remap = false, srecord_needs_remap = false;
12035 504 : splay_tree_node n;
12036 504 : struct omp_taskcopy_context tcctx;
12037 504 : location_t loc = gimple_location (task_stmt);
12038 504 : size_t looptempno = 0;
12039 :
12040 504 : child_fn = gimple_omp_task_copy_fn (task_stmt);
12041 504 : task_cpyfns.safe_push (task_stmt);
12042 504 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
12043 504 : gcc_assert (child_cfun->cfg == NULL);
12044 504 : DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
12045 :
12046 : /* Reset DECL_CONTEXT on function arguments. */
12047 1512 : for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
12048 1008 : DECL_CONTEXT (t) = child_fn;
12049 :
12050 : /* Populate the function. */
12051 504 : push_gimplify_context ();
12052 504 : push_cfun (child_cfun);
12053 :
12054 504 : bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
12055 504 : TREE_SIDE_EFFECTS (bind) = 1;
12056 504 : list = NULL;
12057 504 : DECL_SAVED_TREE (child_fn) = bind;
12058 504 : DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
12059 :
12060 : /* Remap src and dst argument types if needed. */
12061 504 : record_type = ctx->record_type;
12062 504 : srecord_type = ctx->srecord_type;
12063 3100 : for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
12064 2621 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
12065 : {
12066 : record_needs_remap = true;
12067 : break;
12068 : }
12069 2839 : for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
12070 2360 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
12071 : {
12072 : srecord_needs_remap = true;
12073 : break;
12074 : }
12075 :
12076 504 : if (record_needs_remap || srecord_needs_remap)
12077 : {
12078 25 : memset (&tcctx, '\0', sizeof (tcctx));
12079 25 : tcctx.cb.src_fn = ctx->cb.src_fn;
12080 25 : tcctx.cb.dst_fn = child_fn;
12081 25 : tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn);
12082 25 : gcc_checking_assert (tcctx.cb.src_node);
12083 25 : tcctx.cb.dst_node = tcctx.cb.src_node;
12084 25 : tcctx.cb.src_cfun = ctx->cb.src_cfun;
12085 25 : tcctx.cb.copy_decl = task_copyfn_copy_decl;
12086 25 : tcctx.cb.eh_lp_nr = 0;
12087 25 : tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
12088 25 : tcctx.cb.decl_map = new hash_map<tree, tree>;
12089 25 : tcctx.ctx = ctx;
12090 :
12091 25 : if (record_needs_remap)
12092 25 : record_type = task_copyfn_remap_type (&tcctx, record_type);
12093 25 : if (srecord_needs_remap)
12094 25 : srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
12095 : }
12096 : else
12097 479 : tcctx.cb.decl_map = NULL;
12098 :
12099 504 : arg = DECL_ARGUMENTS (child_fn);
12100 504 : TREE_TYPE (arg) = build_pointer_type (record_type);
12101 504 : sarg = DECL_CHAIN (arg);
12102 504 : TREE_TYPE (sarg) = build_pointer_type (srecord_type);
12103 :
12104 : /* First pass: initialize temporaries used in record_type and srecord_type
12105 : sizes and field offsets. */
12106 504 : if (tcctx.cb.decl_map)
12107 631 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12108 606 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12109 : {
12110 497 : tree *p;
12111 :
12112 497 : decl = OMP_CLAUSE_DECL (c);
12113 497 : p = tcctx.cb.decl_map->get (decl);
12114 497 : if (p == NULL)
12115 339 : continue;
12116 158 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12117 158 : sf = (tree) n->value;
12118 158 : sf = *tcctx.cb.decl_map->get (sf);
12119 158 : src = build_simple_mem_ref_loc (loc, sarg);
12120 158 : src = omp_build_component_ref (src, sf);
12121 158 : t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
12122 158 : append_to_statement_list (t, &list);
12123 : }
12124 :
12125 : /* Second pass: copy shared var pointers and copy construct non-VLA
12126 : firstprivate vars. */
12127 5848 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12128 5344 : switch (OMP_CLAUSE_CODE (c))
12129 : {
12130 581 : splay_tree_key key;
12131 581 : case OMP_CLAUSE_SHARED:
12132 581 : decl = OMP_CLAUSE_DECL (c);
12133 581 : key = (splay_tree_key) decl;
12134 581 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
12135 273 : key = (splay_tree_key) &DECL_UID (decl);
12136 581 : n = splay_tree_lookup (ctx->field_map, key);
12137 581 : if (n == NULL)
12138 : break;
12139 147 : f = (tree) n->value;
12140 147 : if (tcctx.cb.decl_map)
12141 12 : f = *tcctx.cb.decl_map->get (f);
12142 147 : n = splay_tree_lookup (ctx->sfield_map, key);
12143 147 : sf = (tree) n->value;
12144 147 : if (tcctx.cb.decl_map)
12145 12 : sf = *tcctx.cb.decl_map->get (sf);
12146 147 : src = build_simple_mem_ref_loc (loc, sarg);
12147 147 : src = omp_build_component_ref (src, sf);
12148 147 : dst = build_simple_mem_ref_loc (loc, arg);
12149 147 : dst = omp_build_component_ref (dst, f);
12150 147 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12151 147 : append_to_statement_list (t, &list);
12152 147 : break;
12153 460 : case OMP_CLAUSE_REDUCTION:
12154 460 : case OMP_CLAUSE_IN_REDUCTION:
12155 460 : decl = OMP_CLAUSE_DECL (c);
12156 460 : if (TREE_CODE (decl) == MEM_REF)
12157 : {
12158 160 : decl = TREE_OPERAND (decl, 0);
12159 160 : if (TREE_CODE (decl) == POINTER_PLUS_EXPR)
12160 38 : decl = TREE_OPERAND (decl, 0);
12161 160 : if (TREE_CODE (decl) == INDIRECT_REF
12162 128 : || TREE_CODE (decl) == ADDR_EXPR)
12163 95 : decl = TREE_OPERAND (decl, 0);
12164 : }
12165 460 : key = (splay_tree_key) decl;
12166 460 : n = splay_tree_lookup (ctx->field_map, key);
12167 460 : if (n == NULL)
12168 : break;
12169 101 : f = (tree) n->value;
12170 101 : if (tcctx.cb.decl_map)
12171 20 : f = *tcctx.cb.decl_map->get (f);
12172 101 : n = splay_tree_lookup (ctx->sfield_map, key);
12173 101 : sf = (tree) n->value;
12174 101 : if (tcctx.cb.decl_map)
12175 20 : sf = *tcctx.cb.decl_map->get (sf);
12176 101 : src = build_simple_mem_ref_loc (loc, sarg);
12177 101 : src = omp_build_component_ref (src, sf);
12178 101 : if (decl != OMP_CLAUSE_DECL (c)
12179 95 : && TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
12180 173 : && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == POINTER_TYPE)
12181 32 : src = build_simple_mem_ref_loc (loc, src);
12182 101 : dst = build_simple_mem_ref_loc (loc, arg);
12183 101 : dst = omp_build_component_ref (dst, f);
12184 101 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12185 101 : append_to_statement_list (t, &list);
12186 101 : break;
12187 754 : case OMP_CLAUSE__LOOPTEMP_:
12188 : /* Fields for first two _looptemp_ clauses are initialized by
12189 : GOMP_taskloop*, the rest are handled like firstprivate. */
12190 754 : if (looptempno < 2)
12191 : {
12192 748 : looptempno++;
12193 748 : break;
12194 : }
12195 : /* FALLTHRU */
12196 1731 : case OMP_CLAUSE__REDUCTEMP_:
12197 1731 : case OMP_CLAUSE_FIRSTPRIVATE:
12198 1731 : decl = OMP_CLAUSE_DECL (c);
12199 1731 : if (is_variable_sized (decl))
12200 : break;
12201 1717 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12202 1717 : if (n == NULL)
12203 : break;
12204 1717 : f = (tree) n->value;
12205 1717 : if (tcctx.cb.decl_map)
12206 490 : f = *tcctx.cb.decl_map->get (f);
12207 1717 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12208 1717 : if (n != NULL)
12209 : {
12210 1467 : sf = (tree) n->value;
12211 1467 : if (tcctx.cb.decl_map)
12212 490 : sf = *tcctx.cb.decl_map->get (sf);
12213 1467 : src = build_simple_mem_ref_loc (loc, sarg);
12214 1467 : src = omp_build_component_ref (src, sf);
12215 1467 : if (use_pointer_for_field (decl, NULL)
12216 1467 : || omp_privatize_by_reference (decl))
12217 317 : src = build_simple_mem_ref_loc (loc, src);
12218 : }
12219 : else
12220 : src = decl;
12221 1717 : dst = build_simple_mem_ref_loc (loc, arg);
12222 1717 : dst = omp_build_component_ref (dst, f);
12223 1717 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
12224 192 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12225 : else
12226 : {
12227 1525 : if (ctx->allocate_map)
12228 84 : if (tree *allocatorp = ctx->allocate_map->get (decl))
12229 : {
12230 50 : tree allocator = *allocatorp;
12231 50 : HOST_WIDE_INT ialign = 0;
12232 50 : if (TREE_CODE (allocator) == TREE_LIST)
12233 : {
12234 4 : ialign = tree_to_uhwi (TREE_VALUE (allocator));
12235 4 : allocator = TREE_PURPOSE (allocator);
12236 : }
12237 50 : if (TREE_CODE (allocator) != INTEGER_CST)
12238 : {
12239 22 : n = splay_tree_lookup (ctx->sfield_map,
12240 : (splay_tree_key) allocator);
12241 22 : allocator = (tree) n->value;
12242 22 : if (tcctx.cb.decl_map)
12243 0 : allocator = *tcctx.cb.decl_map->get (allocator);
12244 22 : tree a = build_simple_mem_ref_loc (loc, sarg);
12245 22 : allocator = omp_build_component_ref (a, allocator);
12246 : }
12247 50 : allocator = fold_convert (pointer_sized_int_node, allocator);
12248 50 : tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
12249 50 : tree align = build_int_cst (size_type_node,
12250 50 : MAX (ialign,
12251 : DECL_ALIGN_UNIT (decl)));
12252 50 : tree sz = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (dst)));
12253 50 : tree ptr = build_call_expr_loc (loc, a, 3, align, sz,
12254 : allocator);
12255 50 : ptr = fold_convert (TREE_TYPE (dst), ptr);
12256 50 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, ptr);
12257 50 : append_to_statement_list (t, &list);
12258 50 : dst = build_simple_mem_ref_loc (loc, dst);
12259 : }
12260 1525 : t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
12261 : }
12262 1717 : append_to_statement_list (t, &list);
12263 1717 : break;
12264 489 : case OMP_CLAUSE_PRIVATE:
12265 489 : if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
12266 : break;
12267 12 : decl = OMP_CLAUSE_DECL (c);
12268 12 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12269 12 : f = (tree) n->value;
12270 12 : if (tcctx.cb.decl_map)
12271 0 : f = *tcctx.cb.decl_map->get (f);
12272 12 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12273 12 : if (n != NULL)
12274 : {
12275 12 : sf = (tree) n->value;
12276 12 : if (tcctx.cb.decl_map)
12277 0 : sf = *tcctx.cb.decl_map->get (sf);
12278 12 : src = build_simple_mem_ref_loc (loc, sarg);
12279 12 : src = omp_build_component_ref (src, sf);
12280 12 : if (use_pointer_for_field (decl, NULL))
12281 12 : src = build_simple_mem_ref_loc (loc, src);
12282 : }
12283 : else
12284 : src = decl;
12285 12 : dst = build_simple_mem_ref_loc (loc, arg);
12286 12 : dst = omp_build_component_ref (dst, f);
12287 12 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12288 12 : append_to_statement_list (t, &list);
12289 12 : break;
12290 : default:
12291 : break;
12292 : }
12293 :
12294 : /* Last pass: handle VLA firstprivates. */
12295 504 : if (tcctx.cb.decl_map)
12296 631 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12297 606 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12298 : {
12299 497 : tree ind, ptr, df;
12300 :
12301 497 : decl = OMP_CLAUSE_DECL (c);
12302 497 : if (!is_variable_sized (decl))
12303 483 : continue;
12304 14 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12305 14 : if (n == NULL)
12306 0 : continue;
12307 14 : f = (tree) n->value;
12308 14 : f = *tcctx.cb.decl_map->get (f);
12309 14 : gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
12310 14 : ind = DECL_VALUE_EXPR (decl);
12311 14 : gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
12312 14 : gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
12313 42 : n = splay_tree_lookup (ctx->sfield_map,
12314 14 : (splay_tree_key) TREE_OPERAND (ind, 0));
12315 14 : sf = (tree) n->value;
12316 14 : sf = *tcctx.cb.decl_map->get (sf);
12317 14 : src = build_simple_mem_ref_loc (loc, sarg);
12318 14 : src = omp_build_component_ref (src, sf);
12319 14 : src = build_simple_mem_ref_loc (loc, src);
12320 14 : dst = build_simple_mem_ref_loc (loc, arg);
12321 14 : dst = omp_build_component_ref (dst, f);
12322 14 : t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
12323 14 : append_to_statement_list (t, &list);
12324 42 : n = splay_tree_lookup (ctx->field_map,
12325 14 : (splay_tree_key) TREE_OPERAND (ind, 0));
12326 14 : df = (tree) n->value;
12327 14 : df = *tcctx.cb.decl_map->get (df);
12328 14 : ptr = build_simple_mem_ref_loc (loc, arg);
12329 14 : ptr = omp_build_component_ref (ptr, df);
12330 14 : t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
12331 : build_fold_addr_expr_loc (loc, dst));
12332 14 : append_to_statement_list (t, &list);
12333 : }
12334 :
12335 504 : t = build1 (RETURN_EXPR, void_type_node, NULL);
12336 504 : append_to_statement_list (t, &list);
12337 :
12338 504 : if (tcctx.cb.decl_map)
12339 25 : delete tcctx.cb.decl_map;
12340 504 : pop_gimplify_context (NULL);
12341 504 : BIND_EXPR_BODY (bind) = list;
12342 504 : pop_cfun ();
12343 504 : }
12344 :
12345 : static void
12346 1546 : lower_depend_clauses (tree *pclauses, gimple_seq *iseq, gimple_seq *oseq)
12347 : {
12348 1546 : tree c, clauses;
12349 1546 : gimple *g;
12350 1546 : size_t cnt[5] = { 0, 0, 0, 0, 0 }, idx = 2, i;
12351 :
12352 1546 : clauses = omp_find_clause (*pclauses, OMP_CLAUSE_DEPEND);
12353 1546 : gcc_assert (clauses);
12354 6305 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12355 4879 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
12356 1706 : switch (OMP_CLAUSE_DEPEND_KIND (c))
12357 : {
12358 120 : case OMP_CLAUSE_DEPEND_LAST:
12359 : /* Lowering already done at gimplification. */
12360 120 : return;
12361 481 : case OMP_CLAUSE_DEPEND_IN:
12362 481 : cnt[2]++;
12363 481 : break;
12364 967 : case OMP_CLAUSE_DEPEND_OUT:
12365 967 : case OMP_CLAUSE_DEPEND_INOUT:
12366 967 : cnt[0]++;
12367 967 : break;
12368 30 : case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
12369 30 : cnt[1]++;
12370 30 : break;
12371 76 : case OMP_CLAUSE_DEPEND_DEPOBJ:
12372 76 : cnt[3]++;
12373 76 : break;
12374 32 : case OMP_CLAUSE_DEPEND_INOUTSET:
12375 32 : cnt[4]++;
12376 32 : break;
12377 0 : default:
12378 0 : gcc_unreachable ();
12379 : }
12380 1426 : if (cnt[1] || cnt[3] || cnt[4])
12381 135 : idx = 5;
12382 1426 : size_t total = cnt[0] + cnt[1] + cnt[2] + cnt[3] + cnt[4];
12383 1426 : size_t inoutidx = total + idx;
12384 1426 : tree type = build_array_type_nelts (ptr_type_node, total + idx + 2 * cnt[4]);
12385 1426 : tree array = create_tmp_var (type);
12386 1426 : TREE_ADDRESSABLE (array) = 1;
12387 1426 : tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
12388 : NULL_TREE);
12389 1426 : if (idx == 5)
12390 : {
12391 135 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, 0));
12392 135 : gimple_seq_add_stmt (iseq, g);
12393 135 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
12394 : NULL_TREE);
12395 : }
12396 1426 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, total));
12397 1426 : gimple_seq_add_stmt (iseq, g);
12398 7130 : for (i = 0; i < (idx == 5 ? 3 : 1); i++)
12399 : {
12400 1696 : r = build4 (ARRAY_REF, ptr_type_node, array,
12401 1696 : size_int (i + 1 + (idx == 5)), NULL_TREE, NULL_TREE);
12402 1696 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, cnt[i]));
12403 1696 : gimple_seq_add_stmt (iseq, g);
12404 : }
12405 8556 : for (i = 0; i < 5; i++)
12406 : {
12407 7130 : if (cnt[i] == 0)
12408 5648 : continue;
12409 6559 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12410 5077 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND)
12411 3352 : continue;
12412 : else
12413 : {
12414 1725 : switch (OMP_CLAUSE_DEPEND_KIND (c))
12415 : {
12416 547 : case OMP_CLAUSE_DEPEND_IN:
12417 547 : if (i != 2)
12418 139 : continue;
12419 : break;
12420 1023 : case OMP_CLAUSE_DEPEND_OUT:
12421 1023 : case OMP_CLAUSE_DEPEND_INOUT:
12422 1023 : if (i != 0)
12423 56 : continue;
12424 : break;
12425 44 : case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
12426 44 : if (i != 1)
12427 14 : continue;
12428 : break;
12429 79 : case OMP_CLAUSE_DEPEND_DEPOBJ:
12430 79 : if (i != 3)
12431 3 : continue;
12432 : break;
12433 32 : case OMP_CLAUSE_DEPEND_INOUTSET:
12434 32 : if (i != 4)
12435 0 : continue;
12436 : break;
12437 0 : default:
12438 0 : gcc_unreachable ();
12439 : }
12440 1586 : tree t = OMP_CLAUSE_DECL (c);
12441 1586 : if (i == 4)
12442 : {
12443 32 : t = build4 (ARRAY_REF, ptr_type_node, array,
12444 32 : size_int (inoutidx), NULL_TREE, NULL_TREE);
12445 32 : t = build_fold_addr_expr (t);
12446 32 : inoutidx += 2;
12447 : }
12448 1586 : t = fold_convert (ptr_type_node, t);
12449 1586 : gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
12450 1586 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12451 : NULL_TREE, NULL_TREE);
12452 1586 : g = gimple_build_assign (r, t);
12453 1586 : gimple_seq_add_stmt (iseq, g);
12454 : }
12455 : }
12456 1426 : if (cnt[4])
12457 81 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12458 49 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
12459 49 : && OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_INOUTSET)
12460 : {
12461 32 : tree t = OMP_CLAUSE_DECL (c);
12462 32 : t = fold_convert (ptr_type_node, t);
12463 32 : gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
12464 32 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12465 : NULL_TREE, NULL_TREE);
12466 32 : g = gimple_build_assign (r, t);
12467 32 : gimple_seq_add_stmt (iseq, g);
12468 32 : t = build_int_cst (ptr_type_node, GOMP_DEPEND_INOUTSET);
12469 32 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12470 : NULL_TREE, NULL_TREE);
12471 32 : g = gimple_build_assign (r, t);
12472 32 : gimple_seq_add_stmt (iseq, g);
12473 : }
12474 :
12475 1426 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
12476 1426 : OMP_CLAUSE_DEPEND_KIND (c) = OMP_CLAUSE_DEPEND_LAST;
12477 1426 : OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
12478 1426 : OMP_CLAUSE_CHAIN (c) = *pclauses;
12479 1426 : *pclauses = c;
12480 1426 : tree clobber = build_clobber (type);
12481 1426 : g = gimple_build_assign (array, clobber);
12482 1426 : gimple_seq_add_stmt (oseq, g);
12483 : }
12484 :
12485 : /* Lower the OpenMP parallel or task directive in the current statement
12486 : in GSI_P. CTX holds context information for the directive. */
12487 :
12488 : static void
12489 22440 : lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12490 : {
12491 22440 : tree clauses;
12492 22440 : tree child_fn, t;
12493 22440 : gimple *stmt = gsi_stmt (*gsi_p);
12494 22440 : gbind *par_bind, *bind, *dep_bind = NULL;
12495 22440 : gimple_seq par_body;
12496 22440 : location_t loc = gimple_location (stmt);
12497 :
12498 22440 : clauses = gimple_omp_taskreg_clauses (stmt);
12499 22440 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12500 22440 : && gimple_omp_task_taskwait_p (stmt))
12501 : {
12502 81 : par_bind = NULL;
12503 81 : par_body = NULL;
12504 : }
12505 : else
12506 : {
12507 22359 : par_bind
12508 22359 : = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
12509 22359 : par_body = gimple_bind_body (par_bind);
12510 : }
12511 22440 : child_fn = ctx->cb.dst_fn;
12512 22440 : if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
12513 22440 : && !gimple_omp_parallel_combined_p (stmt))
12514 : {
12515 4186 : struct walk_stmt_info wi;
12516 4186 : int ws_num = 0;
12517 :
12518 4186 : memset (&wi, 0, sizeof (wi));
12519 4186 : wi.info = &ws_num;
12520 4186 : wi.val_only = true;
12521 4186 : walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
12522 4186 : if (ws_num == 1)
12523 377 : gimple_omp_parallel_set_combined_p (stmt, true);
12524 : }
12525 22440 : gimple_seq dep_ilist = NULL;
12526 22440 : gimple_seq dep_olist = NULL;
12527 22440 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12528 22440 : && omp_find_clause (clauses, OMP_CLAUSE_DEPEND))
12529 : {
12530 1182 : push_gimplify_context ();
12531 1182 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12532 1182 : lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt),
12533 : &dep_ilist, &dep_olist);
12534 : }
12535 :
12536 22440 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12537 22440 : && gimple_omp_task_taskwait_p (stmt))
12538 : {
12539 81 : if (dep_bind)
12540 : {
12541 81 : gsi_replace (gsi_p, dep_bind, true);
12542 81 : gimple_bind_add_seq (dep_bind, dep_ilist);
12543 81 : gimple_bind_add_stmt (dep_bind, stmt);
12544 81 : gimple_bind_add_seq (dep_bind, dep_olist);
12545 81 : pop_gimplify_context (dep_bind);
12546 : }
12547 81 : return;
12548 : }
12549 :
12550 22359 : if (ctx->srecord_type)
12551 504 : create_task_copyfn (as_a <gomp_task *> (stmt), ctx);
12552 :
12553 22359 : gimple_seq tskred_ilist = NULL;
12554 22359 : gimple_seq tskred_olist = NULL;
12555 22359 : if ((is_task_ctx (ctx)
12556 3774 : && gimple_omp_task_taskloop_p (ctx->stmt)
12557 1330 : && omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
12558 : OMP_CLAUSE_REDUCTION))
12559 25636 : || (is_parallel_ctx (ctx)
12560 16089 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
12561 : OMP_CLAUSE__REDUCTEMP_)))
12562 : {
12563 556 : if (dep_bind == NULL)
12564 : {
12565 556 : push_gimplify_context ();
12566 556 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12567 : }
12568 615 : lower_omp_task_reductions (ctx, is_task_ctx (ctx) ? OMP_TASKLOOP
12569 : : OMP_PARALLEL,
12570 556 : gimple_omp_taskreg_clauses (ctx->stmt),
12571 : &tskred_ilist, &tskred_olist);
12572 : }
12573 :
12574 22359 : push_gimplify_context ();
12575 :
12576 22359 : gimple_seq par_olist = NULL;
12577 22359 : gimple_seq par_ilist = NULL;
12578 22359 : gimple_seq par_rlist = NULL;
12579 22359 : lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
12580 22355 : lower_omp (&par_body, ctx);
12581 22355 : if (gimple_code (stmt) != GIMPLE_OMP_TASK)
12582 18581 : lower_reduction_clauses (clauses, &par_rlist, NULL, ctx);
12583 :
12584 : /* Declare all the variables created by mapping and the variables
12585 : declared in the scope of the parallel body. */
12586 22355 : record_vars_into (ctx->block_vars, child_fn);
12587 22355 : maybe_remove_omp_member_access_dummy_vars (par_bind);
12588 22355 : record_vars_into (gimple_bind_vars (par_bind), child_fn);
12589 :
12590 22355 : if (ctx->record_type)
12591 : {
12592 17913 : ctx->sender_decl
12593 35322 : = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
12594 : : ctx->record_type, ".omp_data_o");
12595 17913 : DECL_NAMELESS (ctx->sender_decl) = 1;
12596 17913 : TREE_ADDRESSABLE (ctx->sender_decl) = 1;
12597 17913 : gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
12598 : }
12599 :
12600 22355 : gimple_seq olist = NULL;
12601 22355 : gimple_seq ilist = NULL;
12602 22355 : lower_send_clauses (clauses, &ilist, &olist, ctx);
12603 22355 : lower_send_shared_vars (&ilist, &olist, ctx);
12604 :
12605 22355 : if (ctx->record_type)
12606 : {
12607 17913 : tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
12608 17913 : gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
12609 : clobber));
12610 : }
12611 :
12612 : /* Once all the expansions are done, sequence all the different
12613 : fragments inside gimple_omp_body. */
12614 :
12615 22355 : gimple_seq new_body = NULL;
12616 :
12617 22355 : if (ctx->record_type)
12618 : {
12619 17913 : t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
12620 : /* fixup_child_record_type might have changed receiver_decl's type. */
12621 17913 : t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
12622 17913 : gimple_seq_add_stmt (&new_body,
12623 17913 : gimple_build_assign (ctx->receiver_decl, t));
12624 : }
12625 :
12626 22355 : gimple_seq_add_seq (&new_body, par_ilist);
12627 22355 : gimple_seq_add_seq (&new_body, par_body);
12628 22355 : gimple_seq_add_seq (&new_body, par_rlist);
12629 22355 : if (ctx->cancellable)
12630 135 : gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
12631 22355 : gimple_seq_add_seq (&new_body, par_olist);
12632 22355 : new_body = maybe_catch_exception (new_body);
12633 22355 : if (gimple_code (stmt) == GIMPLE_OMP_TASK)
12634 3774 : gimple_seq_add_stmt (&new_body,
12635 3774 : gimple_build_omp_continue (integer_zero_node,
12636 : integer_zero_node));
12637 22355 : gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
12638 22355 : gimple_omp_set_body (stmt, new_body);
12639 :
12640 22355 : if (dep_bind && gimple_bind_block (par_bind) == NULL_TREE)
12641 497 : bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12642 : else
12643 21858 : bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
12644 43053 : gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
12645 22355 : gimple_bind_add_seq (bind, ilist);
12646 22355 : gimple_bind_add_stmt (bind, stmt);
12647 22355 : gimple_bind_add_seq (bind, olist);
12648 :
12649 22355 : pop_gimplify_context (NULL);
12650 :
12651 22355 : if (dep_bind)
12652 : {
12653 1657 : gimple_bind_add_seq (dep_bind, dep_ilist);
12654 1657 : gimple_bind_add_seq (dep_bind, tskred_ilist);
12655 1657 : gimple_bind_add_stmt (dep_bind, bind);
12656 1657 : gimple_bind_add_seq (dep_bind, tskred_olist);
12657 1657 : gimple_bind_add_seq (dep_bind, dep_olist);
12658 1657 : pop_gimplify_context (dep_bind);
12659 : }
12660 : }
12661 :
12662 : /* Set EXPR as the hostaddr expression that should result from the clause C
12663 : in the target statement STMT. Returns the tree that should be
12664 : passed as the hostaddr (a pointer to the array containing the expanded
12665 : hostaddrs and sizes of the clause). */
12666 :
12667 : static tree
12668 36641 : lower_omp_map_iterator_expr (tree expr, tree c, gomp_target *stmt)
12669 : {
12670 36641 : if (!OMP_CLAUSE_HAS_ITERATORS (c))
12671 : return expr;
12672 :
12673 96 : tree iterator = OMP_CLAUSE_ITERATORS (c);
12674 96 : tree elems = TREE_VEC_ELT (iterator, 7);
12675 96 : tree index = TREE_VEC_ELT (iterator, 8);
12676 96 : gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
12677 :
12678 : /* IN LOOP BODY: */
12679 : /* elems[idx] = <expr>; */
12680 96 : tree lhs = build4 (ARRAY_REF, ptr_type_node, elems, index,
12681 : NULL_TREE, NULL_TREE);
12682 96 : tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
12683 : void_type_node, lhs, expr);
12684 96 : gimplify_and_add (mod_expr, loop_body_p);
12685 96 : exit_omp_iterator_loop_context (c);
12686 :
12687 96 : return build_fold_addr_expr_with_type (elems, ptr_type_node);
12688 : }
12689 :
12690 : /* Set SIZE as the size expression that should result from the clause C
12691 : in the target statement STMT. Returns the tree that should be
12692 : passed as the clause size (a size_int with the value SIZE_MAX, indicating
12693 : that the clause uses an iterator). */
12694 :
12695 : static tree
12696 78625 : lower_omp_map_iterator_size (tree size, tree c, gomp_target *stmt)
12697 : {
12698 78625 : if (!OMP_CLAUSE_HAS_ITERATORS (c))
12699 : return size;
12700 :
12701 96 : tree iterator = OMP_CLAUSE_ITERATORS (c);
12702 96 : tree elems = TREE_VEC_ELT (iterator, 7);
12703 96 : tree index = TREE_VEC_ELT (iterator, 8);
12704 96 : gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
12705 :
12706 : /* IN LOOP BODY: */
12707 : /* elems[idx+1] = <size>; */
12708 96 : tree lhs = build4 (ARRAY_REF, ptr_type_node, elems,
12709 : size_binop (PLUS_EXPR, index, size_int (1)),
12710 : NULL_TREE, NULL_TREE);
12711 96 : tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
12712 : void_type_node, lhs, size);
12713 96 : gimplify_and_add (mod_expr, loop_body_p);
12714 96 : exit_omp_iterator_loop_context (c);
12715 :
12716 96 : return size_int (SIZE_MAX);
12717 : }
12718 :
12719 : /* Lower the GIMPLE_OMP_TARGET in the current statement
12720 : in GSI_P. CTX holds context information for the directive. */
12721 :
12722 : static void
12723 36454 : lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12724 : {
12725 36454 : tree clauses;
12726 36454 : tree child_fn, t, c;
12727 36454 : gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p));
12728 36454 : gbind *tgt_bind, *bind, *dep_bind = NULL;
12729 36454 : gimple_seq tgt_body, olist, ilist, fplist, new_body;
12730 36454 : location_t loc = gimple_location (stmt);
12731 36454 : bool offloaded, data_region;
12732 36454 : unsigned int map_cnt = 0;
12733 36454 : tree in_reduction_clauses = NULL_TREE;
12734 :
12735 36454 : tree deep_map_cnt = NULL_TREE;
12736 36454 : tree deep_map_data = NULL_TREE;
12737 36454 : tree deep_map_offset_data = NULL_TREE;
12738 36454 : tree deep_map_offset = NULL_TREE;
12739 :
12740 36454 : offloaded = is_gimple_omp_offloaded (stmt);
12741 36454 : switch (gimple_omp_target_kind (stmt))
12742 : {
12743 11355 : case GF_OMP_TARGET_KIND_REGION:
12744 11355 : tree *p, *q;
12745 11355 : q = &in_reduction_clauses;
12746 76660 : for (p = gimple_omp_target_clauses_ptr (stmt); *p; )
12747 65305 : if (OMP_CLAUSE_CODE (*p) == OMP_CLAUSE_IN_REDUCTION)
12748 : {
12749 422 : *q = *p;
12750 422 : q = &OMP_CLAUSE_CHAIN (*q);
12751 422 : *p = OMP_CLAUSE_CHAIN (*p);
12752 : }
12753 : else
12754 64883 : p = &OMP_CLAUSE_CHAIN (*p);
12755 11355 : *q = NULL_TREE;
12756 11355 : *p = in_reduction_clauses;
12757 : /* FALLTHRU */
12758 : case GF_OMP_TARGET_KIND_UPDATE:
12759 : case GF_OMP_TARGET_KIND_ENTER_DATA:
12760 : case GF_OMP_TARGET_KIND_EXIT_DATA:
12761 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
12762 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
12763 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
12764 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
12765 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
12766 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
12767 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
12768 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
12769 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
12770 : data_region = false;
12771 : break;
12772 : case GF_OMP_TARGET_KIND_DATA:
12773 : case GF_OMP_TARGET_KIND_OACC_DATA:
12774 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
12775 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
12776 : data_region = true;
12777 : break;
12778 0 : default:
12779 0 : gcc_unreachable ();
12780 : }
12781 :
12782 : /* Ensure that requires map is written via output_offload_tables, even if only
12783 : 'target (enter/exit) data' is used in the translation unit. */
12784 36454 : if (ENABLE_OFFLOADING && (omp_requires_mask & OMP_REQUIRES_TARGET_USED))
12785 : g->have_offload = true;
12786 :
12787 36454 : clauses = gimple_omp_target_clauses (stmt);
12788 :
12789 36454 : gimple_seq dep_ilist = NULL;
12790 36454 : gimple_seq dep_olist = NULL;
12791 36454 : bool has_depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND) != NULL_TREE;
12792 36454 : if (has_depend || in_reduction_clauses)
12793 : {
12794 392 : push_gimplify_context ();
12795 392 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12796 392 : if (has_depend)
12797 329 : lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt),
12798 : &dep_ilist, &dep_olist);
12799 392 : if (in_reduction_clauses)
12800 314 : lower_rec_input_clauses (in_reduction_clauses, &dep_ilist, &dep_olist,
12801 : ctx, NULL);
12802 : }
12803 :
12804 36454 : tgt_bind = NULL;
12805 36454 : tgt_body = NULL;
12806 36454 : if (offloaded)
12807 : {
12808 20746 : tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt));
12809 20746 : tgt_body = gimple_bind_body (tgt_bind);
12810 : }
12811 15708 : else if (data_region)
12812 4007 : tgt_body = gimple_omp_body (stmt);
12813 36454 : child_fn = ctx->cb.dst_fn;
12814 :
12815 36454 : push_gimplify_context ();
12816 36454 : fplist = NULL;
12817 :
12818 36454 : ilist = NULL;
12819 36454 : olist = NULL;
12820 :
12821 36454 : gimple_seq alloc_dlist = NULL;
12822 36454 : hash_map<tree, tree> alloc_map;
12823 72908 : hash_map<tree, gimple_seq> alloc_seq_map;
12824 :
12825 172791 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
12826 136337 : switch (OMP_CLAUSE_CODE (c))
12827 : {
12828 : tree var, x, new_var, allocator, allocate_ptr, size;
12829 : gimple_seq alloc_seq;
12830 : bool by_ref;
12831 :
12832 : default:
12833 90672 : break;
12834 76402 : case OMP_CLAUSE_MAP:
12835 : #if CHECKING_P
12836 : /* First check what we're prepared to handle in the following. */
12837 76402 : switch (OMP_CLAUSE_MAP_KIND (c))
12838 : {
12839 : case GOMP_MAP_ALLOC:
12840 : case GOMP_MAP_TO:
12841 : case GOMP_MAP_FROM:
12842 : case GOMP_MAP_TOFROM:
12843 : case GOMP_MAP_POINTER:
12844 : case GOMP_MAP_TO_PSET:
12845 : case GOMP_MAP_DELETE:
12846 : case GOMP_MAP_RELEASE:
12847 : case GOMP_MAP_ALWAYS_TO:
12848 : case GOMP_MAP_ALWAYS_FROM:
12849 : case GOMP_MAP_ALWAYS_TOFROM:
12850 : case GOMP_MAP_FORCE_PRESENT:
12851 : case GOMP_MAP_ALWAYS_PRESENT_FROM:
12852 : case GOMP_MAP_ALWAYS_PRESENT_TO:
12853 : case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
12854 :
12855 : case GOMP_MAP_FIRSTPRIVATE_POINTER:
12856 : case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
12857 : case GOMP_MAP_STRUCT:
12858 : case GOMP_MAP_STRUCT_UNORD:
12859 : case GOMP_MAP_ALWAYS_POINTER:
12860 : case GOMP_MAP_ATTACH:
12861 : case GOMP_MAP_DETACH:
12862 : case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
12863 : case GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION:
12864 : break;
12865 2578 : case GOMP_MAP_IF_PRESENT:
12866 2578 : case GOMP_MAP_FORCE_ALLOC:
12867 2578 : case GOMP_MAP_FORCE_TO:
12868 2578 : case GOMP_MAP_FORCE_FROM:
12869 2578 : case GOMP_MAP_FORCE_TOFROM:
12870 2578 : case GOMP_MAP_FORCE_DEVICEPTR:
12871 2578 : case GOMP_MAP_DEVICE_RESIDENT:
12872 2578 : case GOMP_MAP_LINK:
12873 2578 : case GOMP_MAP_FORCE_DETACH:
12874 2578 : gcc_assert (is_gimple_omp_oacc (stmt));
12875 : break;
12876 0 : default:
12877 0 : gcc_unreachable ();
12878 : }
12879 : #endif
12880 : /* FALLTHRU */
12881 87846 : case OMP_CLAUSE_TO:
12882 87846 : case OMP_CLAUSE_FROM:
12883 87846 : oacc_firstprivate:
12884 87846 : var = OMP_CLAUSE_DECL (c);
12885 87846 : {
12886 87846 : tree extra = lang_hooks.decls.omp_deep_mapping_cnt (stmt, c, &ilist);
12887 87846 : if (extra != NULL_TREE && deep_map_cnt != NULL_TREE)
12888 59 : deep_map_cnt = fold_build2_loc (OMP_CLAUSE_LOCATION (c), PLUS_EXPR,
12889 : size_type_node, deep_map_cnt,
12890 : extra);
12891 87787 : else if (extra != NULL_TREE)
12892 : deep_map_cnt = extra;
12893 : }
12894 :
12895 87702 : if (deep_map_cnt
12896 88068 : && OMP_CLAUSE_HAS_ITERATORS (c))
12897 0 : sorry ("iterators used together with deep mapping are not "
12898 : "supported yet");
12899 :
12900 87846 : if (!DECL_P (var))
12901 : {
12902 36216 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
12903 36216 : || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
12904 34735 : && (OMP_CLAUSE_MAP_KIND (c)
12905 : != GOMP_MAP_FIRSTPRIVATE_POINTER)))
12906 36216 : map_cnt++;
12907 45665 : continue;
12908 : }
12909 :
12910 51630 : if (DECL_SIZE (var)
12911 51630 : && !poly_int_tree_p (DECL_SIZE (var)))
12912 : {
12913 717 : tree var2 = DECL_VALUE_EXPR (var);
12914 717 : gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
12915 717 : var2 = TREE_OPERAND (var2, 0);
12916 717 : gcc_assert (DECL_P (var2));
12917 : var = var2;
12918 : }
12919 :
12920 51630 : if (offloaded
12921 35986 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
12922 83720 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
12923 28917 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE))
12924 : {
12925 3598 : if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
12926 : {
12927 379 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx))
12928 379 : && varpool_node::get_create (var)->offloadable)
12929 2 : continue;
12930 :
12931 377 : tree type = build_pointer_type (TREE_TYPE (var));
12932 377 : tree new_var = lookup_decl (var, ctx);
12933 377 : x = create_tmp_var_raw (type, get_name (new_var));
12934 377 : gimple_add_tmp_var (x);
12935 377 : x = build_simple_mem_ref (x);
12936 377 : SET_DECL_VALUE_EXPR (new_var, x);
12937 377 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
12938 : }
12939 3596 : continue;
12940 3596 : }
12941 :
12942 48032 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
12943 38069 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
12944 37870 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
12945 48280 : && is_omp_target (stmt))
12946 : {
12947 228 : gcc_assert (maybe_lookup_field (c, ctx));
12948 228 : map_cnt++;
12949 228 : continue;
12950 : }
12951 :
12952 47804 : if (!maybe_lookup_field (var, ctx))
12953 5623 : continue;
12954 :
12955 : /* Don't remap compute constructs' reduction variables, because the
12956 : intermediate result must be local to each gang. */
12957 69395 : if (offloaded && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
12958 23318 : && is_gimple_omp_oacc (ctx->stmt)
12959 13556 : && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
12960 : {
12961 26434 : x = build_receiver_ref (var, true, ctx);
12962 26434 : tree new_var = lookup_decl (var, ctx);
12963 :
12964 26434 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
12965 22538 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
12966 2979 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
12967 29413 : && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
12968 425 : x = build_simple_mem_ref (x);
12969 26434 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12970 : {
12971 3896 : gcc_assert (is_gimple_omp_oacc (ctx->stmt));
12972 3896 : if (omp_privatize_by_reference (new_var)
12973 3896 : && (TREE_CODE (TREE_TYPE (new_var)) != POINTER_TYPE
12974 44 : || DECL_BY_REFERENCE (var)))
12975 : {
12976 : /* Create a local object to hold the instance
12977 : value. */
12978 455 : tree type = TREE_TYPE (TREE_TYPE (new_var));
12979 455 : const char *id = IDENTIFIER_POINTER (DECL_NAME (new_var));
12980 455 : tree inst = create_tmp_var (type, id);
12981 455 : gimplify_assign (inst, fold_indirect_ref (x), &fplist);
12982 455 : x = build_fold_addr_expr (inst);
12983 : }
12984 3896 : gimplify_assign (new_var, x, &fplist);
12985 : }
12986 22538 : else if (DECL_P (new_var))
12987 : {
12988 22538 : SET_DECL_VALUE_EXPR (new_var, x);
12989 22538 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
12990 : }
12991 : else
12992 0 : gcc_unreachable ();
12993 : }
12994 42181 : map_cnt++;
12995 42181 : break;
12996 :
12997 14986 : case OMP_CLAUSE_FIRSTPRIVATE:
12998 14986 : omp_firstprivate_recv:
12999 14986 : gcc_checking_assert (offloaded);
13000 14986 : if (is_gimple_omp_oacc (ctx->stmt))
13001 : {
13002 : /* No 'firstprivate' clauses on OpenACC 'kernels'. */
13003 3896 : gcc_checking_assert (!is_oacc_kernels (ctx));
13004 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
13005 3896 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
13006 :
13007 3896 : goto oacc_firstprivate;
13008 : }
13009 11090 : map_cnt++;
13010 11090 : var = OMP_CLAUSE_DECL (c);
13011 11090 : new_var = lookup_decl (var, ctx);
13012 11090 : allocator = NULL_TREE;
13013 11090 : allocate_ptr = NULL_TREE;
13014 11090 : size = TREE_TYPE (var);
13015 11090 : by_ref = omp_privatize_by_reference (var);
13016 11090 : if (by_ref)
13017 809 : size = TREE_TYPE (size);
13018 11090 : size = TYPE_SIZE_UNIT (size);
13019 11090 : if (is_variable_sized (var, by_ref))
13020 23 : size = lookup_decl (size, ctx);
13021 11090 : alloc_seq = NULL;
13022 11090 : if (lower_private_allocate (var, new_var, allocator, allocate_ptr,
13023 : &alloc_seq, ctx, by_ref, size))
13024 : {
13025 63 : alloc_map.put (new_var, allocate_ptr);
13026 63 : alloc_seq_map.put (new_var, alloc_seq);
13027 : }
13028 11090 : if (!by_ref && !is_gimple_reg_type (TREE_TYPE (var)))
13029 : {
13030 125 : if (is_variable_sized (var))
13031 : {
13032 6 : tree pvar = DECL_VALUE_EXPR (var);
13033 6 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13034 6 : pvar = TREE_OPERAND (pvar, 0);
13035 6 : gcc_assert (DECL_P (pvar));
13036 6 : tree new_pvar = lookup_decl (pvar, ctx);
13037 6 : x = build_fold_indirect_ref (new_pvar);
13038 6 : TREE_THIS_NOTRAP (x) = 1;
13039 : }
13040 119 : else if (allocate_ptr)
13041 19 : x = build_fold_indirect_ref (allocate_ptr);
13042 : else
13043 100 : x = build_receiver_ref (var, true, ctx);
13044 125 : SET_DECL_VALUE_EXPR (new_var, x);
13045 125 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13046 : }
13047 : /* Fortran array descriptors: firstprivate of data + attach. */
13048 11090 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13049 11090 : && lang_hooks.decls.omp_array_data (var, true))
13050 19 : map_cnt += 2;
13051 :
13052 11236 : do_dtor:
13053 11236 : if (allocator)
13054 : {
13055 118 : if (!is_gimple_val (allocator))
13056 : {
13057 0 : tree avar = create_tmp_var (TREE_TYPE (allocator));
13058 0 : gimplify_assign (avar, allocator, &alloc_dlist);
13059 0 : allocator = avar;
13060 : }
13061 118 : if (!is_gimple_val (allocate_ptr))
13062 : {
13063 0 : tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
13064 0 : gimplify_assign (apvar, allocate_ptr, &alloc_dlist);
13065 0 : allocate_ptr = apvar;
13066 : }
13067 118 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
13068 118 : gimple *g = gimple_build_call (f, 2, allocate_ptr, allocator);
13069 118 : gimple_seq_add_stmt (&alloc_dlist, g);
13070 : }
13071 : break;
13072 :
13073 226 : case OMP_CLAUSE_PRIVATE:
13074 226 : gcc_checking_assert (offloaded);
13075 226 : if (is_gimple_omp_oacc (ctx->stmt))
13076 : {
13077 : /* No 'private' clauses on OpenACC 'kernels'. */
13078 80 : gcc_checking_assert (!is_oacc_kernels (ctx));
13079 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
13080 80 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
13081 :
13082 : break;
13083 : }
13084 146 : var = OMP_CLAUSE_DECL (c);
13085 146 : new_var = lookup_decl (var, ctx);
13086 146 : allocator = NULL_TREE;
13087 146 : allocate_ptr = NULL_TREE;
13088 146 : alloc_seq = NULL;
13089 146 : size = TREE_TYPE (var);
13090 146 : by_ref = omp_privatize_by_reference (var);
13091 146 : if (by_ref)
13092 22 : size = TREE_TYPE (size);
13093 146 : size = TYPE_SIZE_UNIT (size);
13094 146 : if (is_variable_sized (var, by_ref))
13095 21 : size = lookup_decl (size, ctx);
13096 146 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
13097 : &alloc_seq, ctx, by_ref, size);
13098 146 : if (allocate_ptr)
13099 : {
13100 55 : alloc_map.put (new_var, allocate_ptr);
13101 55 : alloc_seq_map.put (new_var, alloc_seq);
13102 : }
13103 146 : if (!allocate_ptr && is_variable_sized (var))
13104 : {
13105 1 : tree new_var = lookup_decl (var, ctx);
13106 1 : tree pvar = DECL_VALUE_EXPR (var);
13107 1 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13108 1 : pvar = TREE_OPERAND (pvar, 0);
13109 1 : gcc_assert (DECL_P (pvar));
13110 1 : tree new_pvar = lookup_decl (pvar, ctx);
13111 1 : x = build_fold_indirect_ref (new_pvar);
13112 1 : TREE_THIS_NOTRAP (x) = 1;
13113 1 : SET_DECL_VALUE_EXPR (new_var, x);
13114 1 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13115 : }
13116 146 : goto do_dtor;
13117 :
13118 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
13119 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
13120 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
13121 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
13122 2430 : var = OMP_CLAUSE_DECL (c);
13123 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13124 : {
13125 252 : while (TREE_CODE (var) == INDIRECT_REF
13126 252 : || TREE_CODE (var) == ARRAY_REF)
13127 17 : var = TREE_OPERAND (var, 0);
13128 235 : if (lang_hooks.decls.omp_array_data (var, true))
13129 19 : goto omp_firstprivate_recv;
13130 : }
13131 2411 : map_cnt++;
13132 2411 : if (is_variable_sized (var))
13133 : {
13134 12 : tree new_var = lookup_decl (var, ctx);
13135 12 : tree pvar = DECL_VALUE_EXPR (var);
13136 12 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13137 12 : pvar = TREE_OPERAND (pvar, 0);
13138 12 : gcc_assert (DECL_P (pvar));
13139 12 : tree new_pvar = lookup_decl (pvar, ctx);
13140 12 : x = build_fold_indirect_ref (new_pvar);
13141 12 : TREE_THIS_NOTRAP (x) = 1;
13142 12 : SET_DECL_VALUE_EXPR (new_var, x);
13143 12 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13144 : }
13145 2399 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
13146 542 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13147 2071 : && !omp_privatize_by_reference (var)
13148 703 : && !omp_is_allocatable_or_ptr (var)
13149 564 : && !lang_hooks.decls.omp_array_data (var, true))
13150 4433 : || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13151 : {
13152 389 : tree new_var = lookup_decl (var, ctx);
13153 389 : tree type = build_pointer_type (TREE_TYPE (var));
13154 389 : x = create_tmp_var_raw (type, get_name (new_var));
13155 389 : gimple_add_tmp_var (x);
13156 389 : x = build_simple_mem_ref (x);
13157 389 : SET_DECL_VALUE_EXPR (new_var, x);
13158 389 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13159 : }
13160 : else
13161 : {
13162 2010 : tree new_var = lookup_decl (var, ctx);
13163 2010 : x = create_tmp_var_raw (TREE_TYPE (new_var), get_name (new_var));
13164 2010 : gimple_add_tmp_var (x);
13165 2010 : SET_DECL_VALUE_EXPR (new_var, x);
13166 2010 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13167 : }
13168 : break;
13169 10 : case OMP_CLAUSE_DEVICE_TYPE:
13170 : /* FIXME: Ensure that 'nohost' also has not implied before that
13171 : 'g->have_offload = true' or an implicit declare target. */
13172 10 : if (OMP_CLAUSE_DEVICE_TYPE_KIND (c) != OMP_CLAUSE_DEVICE_TYPE_ANY)
13173 5 : sorry_at (OMP_CLAUSE_LOCATION (c),
13174 : "only the %<device_type(any)%> is supported");
13175 : break;
13176 : }
13177 :
13178 36454 : if (offloaded)
13179 : {
13180 20746 : target_nesting_level++;
13181 20746 : lower_omp (&tgt_body, ctx);
13182 20746 : target_nesting_level--;
13183 : }
13184 15708 : else if (data_region)
13185 4007 : lower_omp (&tgt_body, ctx);
13186 :
13187 36454 : if (offloaded)
13188 : {
13189 : /* Declare all the variables created by mapping and the variables
13190 : declared in the scope of the target body. */
13191 20746 : record_vars_into (ctx->block_vars, child_fn);
13192 20746 : maybe_remove_omp_member_access_dummy_vars (tgt_bind);
13193 20746 : record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
13194 : }
13195 :
13196 36454 : if (ctx->record_type)
13197 : {
13198 31595 : if (deep_map_cnt && TREE_CODE (deep_map_cnt) == INTEGER_CST)
13199 : /* map_cnt = map_cnt + tree_to_hwi (deep_map_cnt); */
13200 : /* deep_map_cnt = NULL_TREE; */
13201 0 : gcc_unreachable ();
13202 144 : else if (deep_map_cnt)
13203 : {
13204 144 : gcc_assert (flexible_array_type_p (ctx->record_type));
13205 144 : tree n = create_tmp_var_raw (size_type_node, "nn_map");
13206 144 : gimple_add_tmp_var (n);
13207 144 : gimplify_assign (n, deep_map_cnt, &ilist);
13208 144 : deep_map_cnt = n;
13209 : }
13210 144 : ctx->sender_decl
13211 31595 : = create_tmp_var (deep_map_cnt ? build_pointer_type (ctx->record_type)
13212 : : ctx->record_type, ".omp_data_arr");
13213 31595 : DECL_NAMELESS (ctx->sender_decl) = 1;
13214 31595 : TREE_ADDRESSABLE (ctx->sender_decl) = 1;
13215 63046 : t = make_tree_vec (deep_map_cnt ? 4 : 3);
13216 31595 : TREE_VEC_ELT (t, 0) = ctx->sender_decl;
13217 31595 : TREE_VEC_ELT (t, 1)
13218 63190 : = create_tmp_var (deep_map_cnt
13219 144 : ? build_pointer_type (size_type_node)
13220 31451 : : build_array_type_nelts (size_type_node, map_cnt),
13221 : ".omp_data_sizes");
13222 31595 : DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
13223 31595 : TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
13224 31595 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
13225 31595 : tree tkind_type = short_unsigned_type_node;
13226 31595 : int talign_shift = 8;
13227 31595 : TREE_VEC_ELT (t, 2)
13228 63190 : = create_tmp_var (deep_map_cnt
13229 144 : ? build_pointer_type (tkind_type)
13230 31451 : : build_array_type_nelts (tkind_type, map_cnt),
13231 : ".omp_data_kinds");
13232 31595 : DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
13233 31595 : TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
13234 31595 : TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
13235 31595 : gimple_omp_target_set_data_arg (stmt, t);
13236 :
13237 31595 : if (deep_map_cnt)
13238 : {
13239 144 : tree tmp, size;
13240 144 : size = create_tmp_var (size_type_node, NULL);
13241 144 : DECL_NAMELESS (size) = 1;
13242 144 : gimplify_assign (size,
13243 : fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
13244 : size_type_node, deep_map_cnt,
13245 : build_int_cst (size_type_node,
13246 144 : map_cnt)), &ilist);
13247 144 : TREE_VEC_ELT (t, 3) = size;
13248 :
13249 144 : tree call = builtin_decl_explicit (BUILT_IN_MALLOC);
13250 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13251 : size_type_node, deep_map_cnt,
13252 144 : TYPE_SIZE_UNIT (ptr_type_node));
13253 144 : size = fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
13254 : size_type_node, size,
13255 144 : TYPE_SIZE_UNIT (ctx->record_type));
13256 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13257 144 : gimplify_assign (ctx->sender_decl, tmp, &ilist);
13258 :
13259 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13260 144 : size_type_node, TREE_VEC_ELT (t, 3),
13261 144 : TYPE_SIZE_UNIT (size_type_node));
13262 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13263 144 : gimplify_assign (TREE_VEC_ELT (t, 1), tmp, &ilist);
13264 :
13265 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13266 144 : size_type_node, TREE_VEC_ELT (t, 3),
13267 144 : TYPE_SIZE_UNIT (tkind_type));
13268 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13269 144 : gimplify_assign (TREE_VEC_ELT (t, 2), tmp, &ilist);
13270 144 : tree field = TYPE_FIELDS (TREE_TYPE (TREE_TYPE (ctx->sender_decl)));
13271 595 : for ( ; DECL_CHAIN (field) != NULL_TREE; field = DECL_CHAIN (field))
13272 : ;
13273 144 : gcc_assert (TREE_CODE (TREE_TYPE (field)));
13274 144 : tmp = build_fold_indirect_ref (ctx->sender_decl);
13275 144 : deep_map_data = omp_build_component_ref (tmp, field);
13276 144 : deep_map_offset_data = create_tmp_var_raw (size_type_node,
13277 : "map_offset_data");
13278 144 : deep_map_offset = create_tmp_var_raw (size_type_node, "map_offset");
13279 144 : gimple_add_tmp_var (deep_map_offset_data);
13280 144 : gimple_add_tmp_var (deep_map_offset);
13281 144 : gimplify_assign (deep_map_offset_data, build_int_cst (size_type_node,
13282 : 0), &ilist);
13283 144 : gimplify_assign (deep_map_offset, build_int_cst (size_type_node,
13284 144 : map_cnt), &ilist);
13285 : }
13286 :
13287 31595 : vec<constructor_elt, va_gc> *vsize;
13288 31595 : vec<constructor_elt, va_gc> *vkind;
13289 31595 : vec_alloc (vsize, map_cnt);
13290 31595 : vec_alloc (vkind, map_cnt);
13291 31595 : unsigned int map_idx = 0;
13292 :
13293 158613 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
13294 127018 : switch (OMP_CLAUSE_CODE (c))
13295 : {
13296 : tree ovar, nc, s, purpose, var, x, type;
13297 : unsigned int talign;
13298 :
13299 : default:
13300 123054 : break;
13301 :
13302 86355 : case OMP_CLAUSE_MAP:
13303 86355 : case OMP_CLAUSE_TO:
13304 86355 : case OMP_CLAUSE_FROM:
13305 86355 : oacc_firstprivate_map:
13306 86355 : nc = c;
13307 86355 : ovar = OMP_CLAUSE_DECL (c);
13308 86355 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13309 86355 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
13310 71570 : || (OMP_CLAUSE_MAP_KIND (c)
13311 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
13312 : break;
13313 82589 : if (deep_map_cnt)
13314 : {
13315 428 : unsigned HOST_WIDE_INT tkind2;
13316 428 : switch (OMP_CLAUSE_CODE (c))
13317 : {
13318 424 : case OMP_CLAUSE_MAP:
13319 424 : tkind2 = OMP_CLAUSE_MAP_KIND (c);
13320 424 : if (OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
13321 424 : && (((tkind2 & GOMP_MAP_FLAG_SPECIAL_BITS)
13322 : & ~GOMP_MAP_IMPLICIT)
13323 : == 0))
13324 : {
13325 : /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
13326 : bits are not interfered by other special bit
13327 : encodings, then turn the GOMP_IMPLICIT_BIT flag on
13328 : for the runtime to see. */
13329 120 : tkind2 |= GOMP_MAP_IMPLICIT;
13330 : }
13331 : break;
13332 : case OMP_CLAUSE_FIRSTPRIVATE: tkind2 = GOMP_MAP_TO; break;
13333 : case OMP_CLAUSE_TO: tkind2 = GOMP_MAP_TO; break;
13334 2 : case OMP_CLAUSE_FROM: tkind2 = GOMP_MAP_FROM; break;
13335 0 : default: gcc_unreachable ();
13336 : }
13337 428 : lang_hooks.decls.omp_deep_mapping (stmt, c, tkind2,
13338 : deep_map_data,
13339 428 : TREE_VEC_ELT (t, 1),
13340 428 : TREE_VEC_ELT (t, 2),
13341 : deep_map_offset_data,
13342 : deep_map_offset, &ilist);
13343 : }
13344 82589 : if (!DECL_P (ovar))
13345 : {
13346 36216 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13347 36216 : && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
13348 : {
13349 0 : nc = OMP_CLAUSE_CHAIN (c);
13350 0 : gcc_checking_assert (OMP_CLAUSE_DECL (nc)
13351 : == get_base_address (ovar));
13352 0 : ovar = OMP_CLAUSE_DECL (nc);
13353 : }
13354 : else
13355 : {
13356 36216 : tree x = build_sender_ref (ovar, ctx);
13357 36216 : tree v = ovar;
13358 36216 : if (in_reduction_clauses
13359 163 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13360 36379 : && OMP_CLAUSE_MAP_IN_REDUCTION (c))
13361 : {
13362 104 : v = unshare_expr (v);
13363 104 : tree *p = &v;
13364 104 : while (handled_component_p (*p)
13365 : || TREE_CODE (*p) == INDIRECT_REF
13366 : || TREE_CODE (*p) == ADDR_EXPR
13367 : || TREE_CODE (*p) == MEM_REF
13368 256 : || TREE_CODE (*p) == NON_LVALUE_EXPR)
13369 152 : p = &TREE_OPERAND (*p, 0);
13370 104 : tree d = *p;
13371 104 : if (is_variable_sized (d))
13372 : {
13373 16 : gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
13374 16 : d = DECL_VALUE_EXPR (d);
13375 16 : gcc_assert (TREE_CODE (d) == INDIRECT_REF);
13376 16 : d = TREE_OPERAND (d, 0);
13377 16 : gcc_assert (DECL_P (d));
13378 : }
13379 104 : splay_tree_key key
13380 104 : = (splay_tree_key) &DECL_CONTEXT (d);
13381 104 : tree nd = (tree) splay_tree_lookup (ctx->field_map,
13382 104 : key)->value;
13383 104 : if (d == *p)
13384 88 : *p = nd;
13385 : else
13386 16 : *p = build_fold_indirect_ref (nd);
13387 : }
13388 36216 : v = build_fold_addr_expr_with_type (v, ptr_type_node);
13389 36216 : v = lower_omp_map_iterator_expr (v, c, stmt);
13390 36216 : gimplify_assign (x, v, &ilist);
13391 36216 : nc = NULL_TREE;
13392 : }
13393 : }
13394 : else
13395 : {
13396 46373 : if (DECL_SIZE (ovar)
13397 46373 : && !poly_int_tree_p (DECL_SIZE (ovar)))
13398 : {
13399 7 : tree ovar2 = DECL_VALUE_EXPR (ovar);
13400 7 : gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
13401 7 : ovar2 = TREE_OPERAND (ovar2, 0);
13402 7 : gcc_assert (DECL_P (ovar2));
13403 : ovar = ovar2;
13404 : }
13405 46373 : if (!maybe_lookup_field (ovar, ctx)
13406 50467 : && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13407 4094 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13408 4006 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)))
13409 3964 : continue;
13410 : }
13411 :
13412 78625 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
13413 78625 : if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
13414 6409 : talign = DECL_ALIGN_UNIT (ovar);
13415 :
13416 78625 : var = NULL_TREE;
13417 78625 : if (nc)
13418 : {
13419 42409 : if (in_reduction_clauses
13420 897 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13421 43306 : && OMP_CLAUSE_MAP_IN_REDUCTION (c))
13422 : {
13423 268 : tree d = ovar;
13424 268 : if (is_variable_sized (d))
13425 : {
13426 0 : gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
13427 0 : d = DECL_VALUE_EXPR (d);
13428 0 : gcc_assert (TREE_CODE (d) == INDIRECT_REF);
13429 0 : d = TREE_OPERAND (d, 0);
13430 0 : gcc_assert (DECL_P (d));
13431 : }
13432 268 : splay_tree_key key
13433 268 : = (splay_tree_key) &DECL_CONTEXT (d);
13434 268 : tree nd = (tree) splay_tree_lookup (ctx->field_map,
13435 268 : key)->value;
13436 268 : if (d == ovar)
13437 : var = nd;
13438 : else
13439 0 : var = build_fold_indirect_ref (nd);
13440 : }
13441 : else
13442 42141 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13443 : }
13444 42409 : if (nc
13445 42409 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13446 32446 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13447 32247 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
13448 248 : && is_omp_target (stmt))
13449 : {
13450 228 : x = build_sender_ref (c, ctx);
13451 228 : gimplify_assign (x, build_fold_addr_expr (var), &ilist);
13452 : }
13453 78397 : else if (nc)
13454 : {
13455 42181 : x = build_sender_ref (ovar, ctx);
13456 :
13457 42181 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13458 32218 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
13459 5137 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13460 47318 : && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
13461 : {
13462 425 : gcc_assert (offloaded);
13463 425 : tree avar = build_fold_addr_expr (var);
13464 425 : if (!OMP_CLAUSE_ITERATORS (c))
13465 : {
13466 425 : tree tmp = create_tmp_var (TREE_TYPE (TREE_TYPE (x)));
13467 425 : mark_addressable (tmp);
13468 425 : gimplify_assign (tmp, avar, &ilist);
13469 425 : avar = tmp;
13470 : }
13471 425 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (x)));
13472 425 : avar = build_fold_addr_expr (avar);
13473 425 : avar = lower_omp_map_iterator_expr (avar, c, stmt);
13474 425 : gimplify_assign (x, avar, &ilist);
13475 : }
13476 41756 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13477 : {
13478 3896 : gcc_assert (is_gimple_omp_oacc (ctx->stmt));
13479 3896 : if (!omp_privatize_by_reference (var))
13480 : {
13481 3441 : if (is_gimple_reg (var)
13482 3441 : && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13483 2578 : suppress_warning (var);
13484 3441 : var = build_fold_addr_expr (var);
13485 : }
13486 : else
13487 455 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13488 3896 : gimplify_assign (x, var, &ilist);
13489 : }
13490 37860 : else if (is_gimple_reg (var))
13491 : {
13492 5121 : gcc_assert (offloaded);
13493 5121 : tree avar = create_tmp_var (TREE_TYPE (var));
13494 5121 : mark_addressable (avar);
13495 5121 : enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c);
13496 5121 : if (GOMP_MAP_COPY_TO_P (map_kind)
13497 : || map_kind == GOMP_MAP_POINTER
13498 475 : || map_kind == GOMP_MAP_TO_PSET
13499 2 : || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
13500 : {
13501 : /* If we need to initialize a temporary
13502 : with VAR because it is not addressable, and
13503 : the variable hasn't been initialized yet, then
13504 : we'll get a warning for the store to avar.
13505 : Don't warn in that case, the mapping might
13506 : be implicit. */
13507 5119 : suppress_warning (var, OPT_Wuninitialized);
13508 5119 : gimplify_assign (avar, var, &ilist);
13509 : }
13510 5121 : avar = build_fold_addr_expr (avar);
13511 5121 : gimplify_assign (x, avar, &ilist);
13512 5121 : if ((GOMP_MAP_COPY_FROM_P (map_kind)
13513 1324 : || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
13514 8918 : && !TYPE_READONLY (TREE_TYPE (var)))
13515 : {
13516 3797 : x = unshare_expr (x);
13517 3797 : x = build_simple_mem_ref (x);
13518 3797 : gimplify_assign (var, x, &olist);
13519 : }
13520 : }
13521 : else
13522 : {
13523 : /* While MAP is handled explicitly by the FE,
13524 : for 'target update', only the identified is passed. */
13525 32739 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM
13526 27615 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TO)
13527 33682 : && (omp_is_allocatable_or_ptr (var)
13528 120 : && omp_check_optional_argument (var, false)))
13529 0 : var = build_fold_indirect_ref (var);
13530 32739 : else if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FROM
13531 27615 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_TO)
13532 33682 : || (!omp_is_allocatable_or_ptr (var)
13533 5947 : && !omp_check_optional_argument (var, false)))
13534 32619 : var = build_fold_addr_expr (var);
13535 32739 : gimplify_assign (x, var, &ilist);
13536 : }
13537 : }
13538 78625 : s = NULL_TREE;
13539 78625 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13540 : {
13541 3896 : gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
13542 3896 : s = TREE_TYPE (ovar);
13543 3896 : if (TREE_CODE (s) == REFERENCE_TYPE
13544 3896 : || omp_check_optional_argument (ovar, false))
13545 449 : s = TREE_TYPE (s);
13546 3896 : s = TYPE_SIZE_UNIT (s);
13547 : }
13548 : else
13549 74729 : s = OMP_CLAUSE_SIZE (c);
13550 78625 : if (s == NULL_TREE)
13551 123 : s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
13552 78625 : s = fold_convert (size_type_node, s);
13553 78625 : s = lower_omp_map_iterator_size (s, c, stmt);
13554 78625 : purpose = size_int (map_idx++);
13555 78625 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13556 78625 : if (TREE_CODE (s) != INTEGER_CST)
13557 15960 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13558 :
13559 78625 : unsigned HOST_WIDE_INT tkind, tkind_zero;
13560 78625 : switch (OMP_CLAUSE_CODE (c))
13561 : {
13562 67181 : case OMP_CLAUSE_MAP:
13563 67181 : tkind = OMP_CLAUSE_MAP_KIND (c);
13564 67181 : tkind_zero = tkind;
13565 67181 : if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c))
13566 5267 : switch (tkind)
13567 : {
13568 : case GOMP_MAP_ALLOC:
13569 : case GOMP_MAP_IF_PRESENT:
13570 : case GOMP_MAP_TO:
13571 : case GOMP_MAP_FROM:
13572 : case GOMP_MAP_TOFROM:
13573 : case GOMP_MAP_ALWAYS_TO:
13574 : case GOMP_MAP_ALWAYS_FROM:
13575 : case GOMP_MAP_ALWAYS_TOFROM:
13576 : case GOMP_MAP_ALWAYS_PRESENT_TO:
13577 : case GOMP_MAP_ALWAYS_PRESENT_FROM:
13578 : case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
13579 : case GOMP_MAP_RELEASE:
13580 : case GOMP_MAP_FORCE_TO:
13581 : case GOMP_MAP_FORCE_FROM:
13582 : case GOMP_MAP_FORCE_TOFROM:
13583 : case GOMP_MAP_FORCE_PRESENT:
13584 67181 : tkind_zero = GOMP_MAP_ZERO_LEN_ARRAY_SECTION;
13585 : break;
13586 20 : case GOMP_MAP_DELETE:
13587 20 : tkind_zero = GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION;
13588 : default:
13589 : break;
13590 : }
13591 67181 : if (tkind_zero != tkind)
13592 : {
13593 5267 : if (integer_zerop (s))
13594 : tkind = tkind_zero;
13595 4541 : else if (integer_nonzerop (s))
13596 67181 : tkind_zero = tkind;
13597 : }
13598 67181 : if (tkind_zero == tkind
13599 65720 : && OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
13600 67181 : && (((tkind & GOMP_MAP_FLAG_SPECIAL_BITS)
13601 5052 : & ~GOMP_MAP_IMPLICIT)
13602 : == 0))
13603 : {
13604 : /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
13605 : bits are not interfered by other special bit encodings,
13606 : then turn the GOMP_IMPLICIT_BIT flag on for the runtime
13607 : to see. */
13608 5009 : tkind |= GOMP_MAP_IMPLICIT;
13609 5009 : tkind_zero = tkind;
13610 : }
13611 : break;
13612 3896 : case OMP_CLAUSE_FIRSTPRIVATE:
13613 3896 : gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
13614 : tkind = GOMP_MAP_TO;
13615 : tkind_zero = tkind;
13616 : break;
13617 1583 : case OMP_CLAUSE_TO:
13618 1583 : tkind
13619 1583 : = (OMP_CLAUSE_MOTION_PRESENT (c)
13620 1583 : ? GOMP_MAP_ALWAYS_PRESENT_TO : GOMP_MAP_TO);
13621 : tkind_zero = tkind;
13622 : break;
13623 5965 : case OMP_CLAUSE_FROM:
13624 5965 : tkind
13625 5965 : = (OMP_CLAUSE_MOTION_PRESENT (c)
13626 5965 : ? GOMP_MAP_ALWAYS_PRESENT_FROM : GOMP_MAP_FROM);
13627 : tkind_zero = tkind;
13628 : break;
13629 0 : default:
13630 0 : gcc_unreachable ();
13631 : }
13632 67181 : gcc_checking_assert (tkind
13633 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13634 78625 : gcc_checking_assert (tkind_zero
13635 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13636 78625 : talign = ceil_log2 (talign);
13637 78625 : tkind |= talign << talign_shift;
13638 78625 : tkind_zero |= talign << talign_shift;
13639 78625 : gcc_checking_assert (tkind
13640 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13641 78625 : gcc_checking_assert (tkind_zero
13642 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13643 78625 : if (tkind == tkind_zero)
13644 77164 : x = build_int_cstu (tkind_type, tkind);
13645 : else
13646 : {
13647 1461 : TREE_STATIC (TREE_VEC_ELT (t, 2)) = 0;
13648 1461 : x = build3 (COND_EXPR, tkind_type,
13649 : fold_build2 (EQ_EXPR, boolean_type_node,
13650 : unshare_expr (s), size_zero_node),
13651 1461 : build_int_cstu (tkind_type, tkind_zero),
13652 1461 : build_int_cstu (tkind_type, tkind));
13653 : }
13654 78625 : CONSTRUCTOR_APPEND_ELT (vkind, purpose, x);
13655 78625 : if (nc && nc != c)
13656 0 : c = nc;
13657 : break;
13658 :
13659 14986 : case OMP_CLAUSE_FIRSTPRIVATE:
13660 14986 : omp_has_device_addr_descr:
13661 14986 : if (is_gimple_omp_oacc (ctx->stmt))
13662 3896 : goto oacc_firstprivate_map;
13663 11090 : ovar = OMP_CLAUSE_DECL (c);
13664 11090 : if (omp_privatize_by_reference (ovar))
13665 809 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13666 : else
13667 10281 : talign = DECL_ALIGN_UNIT (ovar);
13668 11090 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13669 11090 : x = build_sender_ref (ovar, ctx);
13670 11090 : tkind = GOMP_MAP_FIRSTPRIVATE;
13671 11090 : type = TREE_TYPE (ovar);
13672 11090 : if (omp_privatize_by_reference (ovar))
13673 809 : type = TREE_TYPE (type);
13674 11090 : if ((INTEGRAL_TYPE_P (type)
13675 10718 : && TYPE_PRECISION (type) <= POINTER_SIZE)
13676 11100 : || TREE_CODE (type) == POINTER_TYPE)
13677 : {
13678 10863 : tkind = GOMP_MAP_FIRSTPRIVATE_INT;
13679 10863 : tree t = var;
13680 10863 : if (omp_privatize_by_reference (var))
13681 766 : t = build_simple_mem_ref (var);
13682 10097 : else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13683 9069 : suppress_warning (var);
13684 10863 : if (TREE_CODE (type) != POINTER_TYPE)
13685 10708 : t = fold_convert (pointer_sized_int_node, t);
13686 10863 : t = fold_convert (TREE_TYPE (x), t);
13687 10863 : gimplify_assign (x, t, &ilist);
13688 : }
13689 227 : else if (omp_privatize_by_reference (var))
13690 43 : gimplify_assign (x, var, &ilist);
13691 184 : else if (is_gimple_reg (var))
13692 : {
13693 47 : tree avar = create_tmp_var (TREE_TYPE (var));
13694 47 : mark_addressable (avar);
13695 47 : if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13696 40 : suppress_warning (var);
13697 47 : gimplify_assign (avar, var, &ilist);
13698 47 : avar = build_fold_addr_expr (avar);
13699 47 : gimplify_assign (x, avar, &ilist);
13700 : }
13701 : else
13702 : {
13703 137 : var = build_fold_addr_expr (var);
13704 137 : gimplify_assign (x, var, &ilist);
13705 : }
13706 227 : if (tkind == GOMP_MAP_FIRSTPRIVATE_INT)
13707 10863 : s = size_int (0);
13708 227 : else if (omp_privatize_by_reference (ovar))
13709 43 : s = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13710 : else
13711 184 : s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
13712 11090 : s = fold_convert (size_type_node, s);
13713 11090 : purpose = size_int (map_idx++);
13714 11090 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13715 11090 : if (TREE_CODE (s) != INTEGER_CST)
13716 23 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13717 :
13718 11090 : gcc_checking_assert (tkind
13719 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13720 11090 : talign = ceil_log2 (talign);
13721 11090 : tkind |= talign << talign_shift;
13722 11090 : gcc_checking_assert (tkind
13723 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13724 11090 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13725 : build_int_cstu (tkind_type, tkind));
13726 : /* Fortran array descriptors: firstprivate of data + attach. */
13727 11090 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13728 11090 : && lang_hooks.decls.omp_array_data (ovar, true))
13729 : {
13730 19 : tree not_null_lb, null_lb, after_lb;
13731 19 : tree var1, var2, size1, size2;
13732 19 : tree present = omp_check_optional_argument (ovar, true);
13733 19 : if (present)
13734 : {
13735 1 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
13736 1 : not_null_lb = create_artificial_label (clause_loc);
13737 1 : null_lb = create_artificial_label (clause_loc);
13738 1 : after_lb = create_artificial_label (clause_loc);
13739 1 : gimple_seq seq = NULL;
13740 1 : present = force_gimple_operand (present, &seq, true,
13741 : NULL_TREE);
13742 1 : gimple_seq_add_seq (&ilist, seq);
13743 1 : gimple_seq_add_stmt (&ilist,
13744 1 : gimple_build_cond_from_tree (present,
13745 : not_null_lb, null_lb));
13746 1 : gimple_seq_add_stmt (&ilist,
13747 1 : gimple_build_label (not_null_lb));
13748 : }
13749 19 : var1 = lang_hooks.decls.omp_array_data (var, false);
13750 19 : size1 = lang_hooks.decls.omp_array_size (var, &ilist);
13751 19 : var2 = build_fold_addr_expr (x);
13752 19 : if (!POINTER_TYPE_P (TREE_TYPE (var)))
13753 0 : var = build_fold_addr_expr (var);
13754 19 : size2 = fold_build2 (POINTER_DIFF_EXPR, ssizetype,
13755 : build_fold_addr_expr (var1), var);
13756 19 : size2 = fold_convert (sizetype, size2);
13757 19 : if (present)
13758 : {
13759 1 : tree tmp = create_tmp_var (TREE_TYPE (var1));
13760 1 : gimplify_assign (tmp, var1, &ilist);
13761 1 : var1 = tmp;
13762 1 : tmp = create_tmp_var (TREE_TYPE (var2));
13763 1 : gimplify_assign (tmp, var2, &ilist);
13764 1 : var2 = tmp;
13765 1 : tmp = create_tmp_var (TREE_TYPE (size1));
13766 1 : gimplify_assign (tmp, size1, &ilist);
13767 1 : size1 = tmp;
13768 1 : tmp = create_tmp_var (TREE_TYPE (size2));
13769 1 : gimplify_assign (tmp, size2, &ilist);
13770 1 : size2 = tmp;
13771 1 : gimple_seq_add_stmt (&ilist, gimple_build_goto (after_lb));
13772 1 : gimple_seq_add_stmt (&ilist, gimple_build_label (null_lb));
13773 1 : gimplify_assign (var1, null_pointer_node, &ilist);
13774 1 : gimplify_assign (var2, null_pointer_node, &ilist);
13775 1 : gimplify_assign (size1, size_zero_node, &ilist);
13776 1 : gimplify_assign (size2, size_zero_node, &ilist);
13777 1 : gimple_seq_add_stmt (&ilist, gimple_build_label (after_lb));
13778 : }
13779 19 : x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
13780 19 : gimplify_assign (x, var1, &ilist);
13781 19 : tkind = GOMP_MAP_FIRSTPRIVATE;
13782 19 : talign = DECL_ALIGN_UNIT (ovar);
13783 19 : talign = ceil_log2 (talign);
13784 19 : tkind |= talign << talign_shift;
13785 19 : gcc_checking_assert (tkind
13786 : <= tree_to_uhwi (
13787 : TYPE_MAX_VALUE (tkind_type)));
13788 19 : purpose = size_int (map_idx++);
13789 19 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, size1);
13790 19 : if (TREE_CODE (size1) != INTEGER_CST)
13791 19 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13792 19 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13793 : build_int_cstu (tkind_type, tkind));
13794 19 : x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
13795 19 : gimplify_assign (x, var2, &ilist);
13796 19 : tkind = GOMP_MAP_ATTACH;
13797 19 : purpose = size_int (map_idx++);
13798 19 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, size2);
13799 19 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13800 : build_int_cstu (tkind_type, tkind));
13801 : }
13802 : break;
13803 :
13804 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
13805 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
13806 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
13807 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
13808 2430 : ovar = OMP_CLAUSE_DECL (c);
13809 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13810 : {
13811 235 : if (lang_hooks.decls.omp_array_data (ovar, true))
13812 19 : goto omp_has_device_addr_descr;
13813 233 : while (TREE_CODE (ovar) == INDIRECT_REF
13814 233 : || TREE_CODE (ovar) == ARRAY_REF)
13815 17 : ovar = TREE_OPERAND (ovar, 0);
13816 : }
13817 2411 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13818 :
13819 2411 : if (lang_hooks.decls.omp_array_data (ovar, true))
13820 : {
13821 601 : tkind = ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
13822 601 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
13823 601 : ? GOMP_MAP_USE_DEVICE_PTR : GOMP_MAP_FIRSTPRIVATE_INT);
13824 601 : x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
13825 : }
13826 1810 : else if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
13827 1810 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
13828 : {
13829 1453 : tkind = GOMP_MAP_USE_DEVICE_PTR;
13830 1453 : x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
13831 : }
13832 : else
13833 : {
13834 357 : tkind = GOMP_MAP_FIRSTPRIVATE_INT;
13835 357 : x = build_sender_ref (ovar, ctx);
13836 : }
13837 :
13838 2411 : if (is_gimple_omp_oacc (ctx->stmt))
13839 : {
13840 147 : gcc_assert (tkind == GOMP_MAP_USE_DEVICE_PTR);
13841 :
13842 147 : if (OMP_CLAUSE_USE_DEVICE_PTR_IF_PRESENT (c))
13843 57 : tkind = GOMP_MAP_USE_DEVICE_PTR_IF_PRESENT;
13844 : }
13845 :
13846 2411 : type = TREE_TYPE (ovar);
13847 2411 : if (lang_hooks.decls.omp_array_data (ovar, true))
13848 601 : var = lang_hooks.decls.omp_array_data (var, false);
13849 1810 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
13850 542 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13851 1484 : && !omp_privatize_by_reference (ovar)
13852 512 : && !omp_is_allocatable_or_ptr (ovar))
13853 3247 : || TREE_CODE (type) == ARRAY_TYPE)
13854 401 : var = build_fold_addr_expr (var);
13855 : else
13856 : {
13857 1409 : if (omp_privatize_by_reference (ovar)
13858 370 : || omp_check_optional_argument (ovar, false)
13859 1779 : || omp_is_allocatable_or_ptr (ovar))
13860 : {
13861 1212 : type = TREE_TYPE (type);
13862 1212 : if (POINTER_TYPE_P (type)
13863 1212 : && TREE_CODE (type) != ARRAY_TYPE
13864 1571 : && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
13865 36 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13866 28 : && !omp_is_allocatable_or_ptr (ovar))
13867 337 : || (omp_privatize_by_reference (ovar)
13868 337 : && omp_is_allocatable_or_ptr (ovar))))
13869 357 : var = build_simple_mem_ref (var);
13870 1212 : var = fold_convert (TREE_TYPE (x), var);
13871 : }
13872 : }
13873 2411 : tree present;
13874 2411 : present = omp_check_optional_argument (ovar, true);
13875 2411 : if (present)
13876 : {
13877 878 : tree null_label = create_artificial_label (UNKNOWN_LOCATION);
13878 878 : tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
13879 878 : tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
13880 878 : tree new_x = unshare_expr (x);
13881 878 : gimplify_expr (&present, &ilist, NULL, is_gimple_val,
13882 : fb_rvalue);
13883 878 : gcond *cond = gimple_build_cond_from_tree (present,
13884 : notnull_label,
13885 : null_label);
13886 878 : gimple_seq_add_stmt (&ilist, cond);
13887 878 : gimple_seq_add_stmt (&ilist, gimple_build_label (null_label));
13888 878 : gimplify_assign (new_x, null_pointer_node, &ilist);
13889 878 : gimple_seq_add_stmt (&ilist, gimple_build_goto (opt_arg_label));
13890 878 : gimple_seq_add_stmt (&ilist,
13891 878 : gimple_build_label (notnull_label));
13892 878 : gimplify_assign (x, var, &ilist);
13893 878 : gimple_seq_add_stmt (&ilist,
13894 878 : gimple_build_label (opt_arg_label));
13895 : }
13896 : else
13897 1533 : gimplify_assign (x, var, &ilist);
13898 2411 : s = size_int (0);
13899 2411 : purpose = size_int (map_idx++);
13900 2411 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13901 2411 : gcc_checking_assert (tkind
13902 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13903 2411 : gcc_checking_assert (tkind
13904 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13905 2411 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13906 : build_int_cstu (tkind_type, tkind));
13907 2411 : break;
13908 : }
13909 :
13910 31595 : gcc_assert (map_idx == map_cnt);
13911 :
13912 31595 : if (!deep_map_cnt)
13913 : {
13914 31451 : DECL_INITIAL (TREE_VEC_ELT (t, 1))
13915 31451 : = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
13916 31451 : DECL_INITIAL (TREE_VEC_ELT (t, 2))
13917 62902 : = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
13918 : }
13919 94785 : for (int i = 1; i <= 2; i++)
13920 63190 : if (deep_map_cnt || !TREE_STATIC (TREE_VEC_ELT (t, i)))
13921 : {
13922 9495 : tree tmp = TREE_VEC_ELT (t, i);
13923 9495 : if (deep_map_cnt)
13924 : {
13925 288 : const char *prefix = (i == 1 ? ".omp_data_sizes0"
13926 : : ".omp_data_kinds0");
13927 144 : tree type = (i == 1) ? size_type_node : tkind_type;
13928 288 : type = build_array_type_nelts (type, map_cnt);
13929 288 : tree var = create_tmp_var (type, prefix);
13930 288 : DECL_NAMELESS (var) = 1;
13931 288 : TREE_ADDRESSABLE (var) = 1;
13932 288 : TREE_STATIC (var) = TREE_STATIC (tmp);
13933 288 : DECL_INITIAL (var) = build_constructor (type, i == 1
13934 : ? vsize : vkind);
13935 288 : tmp = var;
13936 288 : TREE_STATIC (TREE_VEC_ELT (t, i)) = 0;
13937 : }
13938 :
13939 9495 : gimple_seq initlist = NULL;
13940 9495 : force_gimple_operand (build1 (DECL_EXPR, void_type_node, tmp),
13941 : &initlist, true, NULL_TREE);
13942 9495 : gimple_seq_add_seq (&ilist, initlist);
13943 :
13944 9495 : if (deep_map_cnt)
13945 : {
13946 288 : tree tmp2;
13947 288 : tree call = builtin_decl_explicit (BUILT_IN_MEMCPY);
13948 288 : tmp2 = TYPE_SIZE_UNIT (TREE_TYPE (tmp));
13949 576 : call = build_call_expr_loc (input_location, call, 3,
13950 288 : TREE_VEC_ELT (t, i),
13951 : build_fold_addr_expr (tmp), tmp2);
13952 288 : gimplify_and_add (call, &ilist);
13953 : }
13954 :
13955 9495 : if (!TREE_STATIC (tmp))
13956 : {
13957 9277 : tree clobber = build_clobber (TREE_TYPE (tmp));
13958 9277 : gimple_seq_add_stmt (&olist,
13959 9277 : gimple_build_assign (tmp, clobber));
13960 : }
13961 9495 : if (deep_map_cnt)
13962 : {
13963 288 : tmp = TREE_VEC_ELT (t, i);
13964 288 : tree call = builtin_decl_explicit (BUILT_IN_FREE);
13965 288 : call = build_call_expr_loc (input_location, call, 1, tmp);
13966 288 : gimplify_and_add (call, &olist);
13967 288 : tree clobber = build_clobber (TREE_TYPE (tmp));
13968 288 : gimple_seq_add_stmt (&olist,
13969 288 : gimple_build_assign (tmp, clobber));
13970 : }
13971 : }
13972 53695 : else if (omp_maybe_offloaded_ctx (ctx->outer))
13973 : {
13974 3296 : tree id = get_identifier ("omp declare target");
13975 3296 : tree decl = TREE_VEC_ELT (t, i);
13976 3296 : DECL_ATTRIBUTES (decl)
13977 3296 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (decl));
13978 3296 : varpool_node *node = varpool_node::get (decl);
13979 3296 : if (node)
13980 : {
13981 0 : node->offloadable = 1;
13982 0 : if (ENABLE_OFFLOADING)
13983 : {
13984 : g->have_offload = true;
13985 : vec_safe_push (offload_vars, t);
13986 : }
13987 : }
13988 : }
13989 :
13990 31595 : if (deep_map_cnt)
13991 : {
13992 144 : tree call = builtin_decl_explicit (BUILT_IN_FREE);
13993 144 : call = build_call_expr_loc (input_location, call, 1,
13994 144 : TREE_VEC_ELT (t, 0));
13995 144 : gimplify_and_add (call, &olist);
13996 :
13997 144 : gimplify_expr (&TREE_VEC_ELT (t, 1), &ilist, NULL, is_gimple_val,
13998 : fb_rvalue);
13999 : }
14000 :
14001 31595 : tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
14002 31595 : gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
14003 : clobber));
14004 : }
14005 :
14006 : /* Once all the expansions are done, sequence all the different
14007 : fragments inside gimple_omp_body. */
14008 :
14009 36454 : new_body = NULL;
14010 :
14011 36454 : if (offloaded
14012 20746 : && ctx->record_type)
14013 : {
14014 16116 : t = ctx->sender_decl;
14015 16116 : if (!deep_map_cnt)
14016 16009 : t = build_fold_addr_expr_loc (loc, t);
14017 : /* fixup_child_record_type might have changed receiver_decl's type. */
14018 16116 : t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
14019 16116 : if (!AGGREGATE_TYPE_P (TREE_TYPE (ctx->sender_decl)))
14020 107 : gimplify_assign (ctx->receiver_decl, t, &new_body);
14021 : else
14022 16009 : gimple_seq_add_stmt (&new_body,
14023 16009 : gimple_build_assign (ctx->receiver_decl, t));
14024 : }
14025 36454 : gimple_seq_add_seq (&new_body, fplist);
14026 :
14027 36454 : if (offloaded || data_region)
14028 : {
14029 138290 : tree prev = NULL_TREE;
14030 : bool by_ref;
14031 138290 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
14032 113537 : switch (OMP_CLAUSE_CODE (c))
14033 : {
14034 : tree var, x, new_var, *allocate_ptr;
14035 : default:
14036 : break;
14037 14986 : case OMP_CLAUSE_FIRSTPRIVATE:
14038 14986 : omp_firstprivatize_data_region:
14039 14986 : if (is_gimple_omp_oacc (ctx->stmt))
14040 : break;
14041 11090 : var = OMP_CLAUSE_DECL (c);
14042 11090 : new_var = lookup_decl (var, ctx);
14043 11090 : allocate_ptr = alloc_map.get (new_var);
14044 11090 : by_ref = omp_privatize_by_reference (var);
14045 11090 : if (allocate_ptr)
14046 : {
14047 63 : if (is_variable_sized (var, by_ref))
14048 : /* Handle this in the next pass when the size is
14049 : available. */
14050 : break;
14051 :
14052 48 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14053 48 : gcc_assert (allocate_seq);
14054 48 : gimple_seq_add_seq (&new_body, *allocate_seq);
14055 :
14056 48 : if (by_ref)
14057 : {
14058 4 : x = fold_convert (TREE_TYPE (new_var), *allocate_ptr);
14059 4 : gimplify_assign (new_var, x, &new_body);
14060 4 : new_var = build_fold_indirect_ref (new_var);
14061 : }
14062 : else
14063 44 : new_var = build_simple_mem_ref (*allocate_ptr);
14064 : }
14065 11075 : if (by_ref || is_gimple_reg_type (TREE_TYPE (var)))
14066 : {
14067 10952 : tree type;
14068 10952 : type = TREE_TYPE (var);
14069 10952 : if (by_ref)
14070 796 : type = TREE_TYPE (type);
14071 10952 : if ((INTEGRAL_TYPE_P (type)
14072 10718 : && TYPE_PRECISION (type) <= POINTER_SIZE)
14073 10962 : || TREE_CODE (type) == POINTER_TYPE)
14074 : {
14075 10863 : x = build_receiver_ref (var, false, ctx);
14076 10863 : if (TREE_CODE (type) != POINTER_TYPE)
14077 10708 : x = fold_convert (pointer_sized_int_node, x);
14078 10863 : x = fold_convert (type, x);
14079 10863 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14080 : fb_rvalue);
14081 10863 : if (by_ref && !allocate_ptr)
14082 : {
14083 765 : tree v = create_tmp_var_raw (type, get_name (var));
14084 765 : gimple_add_tmp_var (v);
14085 765 : TREE_ADDRESSABLE (v) = 1;
14086 765 : gimple_seq_add_stmt (&new_body,
14087 765 : gimple_build_assign (v, x));
14088 765 : x = build_fold_addr_expr (v);
14089 : }
14090 10863 : gimplify_assign (new_var, x, &new_body);
14091 : }
14092 : else
14093 : {
14094 89 : x = build_receiver_ref (var, allocate_ptr || !by_ref, ctx);
14095 89 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14096 : fb_rvalue);
14097 89 : gimplify_assign (new_var, x, &new_body);
14098 : }
14099 : }
14100 123 : else if (is_variable_sized (var))
14101 : {
14102 4 : tree pvar = DECL_VALUE_EXPR (var);
14103 4 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14104 4 : pvar = TREE_OPERAND (pvar, 0);
14105 4 : gcc_assert (DECL_P (pvar));
14106 4 : tree new_var = lookup_decl (pvar, ctx);
14107 4 : x = build_receiver_ref (var, false, ctx);
14108 4 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14109 4 : gimple_seq_add_stmt (&new_body,
14110 4 : gimple_build_assign (new_var, x));
14111 : }
14112 119 : else if (allocate_ptr)
14113 : {
14114 19 : x = build_receiver_ref (var, true, ctx);
14115 19 : new_var = unshare_expr (new_var);
14116 19 : x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
14117 19 : gimplify_and_add (x, &new_body);
14118 : }
14119 : break;
14120 226 : case OMP_CLAUSE_PRIVATE:
14121 226 : if (is_gimple_omp_oacc (ctx->stmt))
14122 : break;
14123 146 : var = OMP_CLAUSE_DECL (c);
14124 146 : new_var = lookup_decl (var, ctx);
14125 146 : allocate_ptr = alloc_map.get (new_var);
14126 146 : by_ref = omp_privatize_by_reference (var);
14127 146 : if (allocate_ptr)
14128 : {
14129 55 : if (is_variable_sized (var, by_ref))
14130 : /* Handle this in the next pass when the size is
14131 : available. */
14132 : break;
14133 :
14134 43 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14135 43 : gcc_assert (allocate_seq);
14136 43 : gimple_seq_add_seq (&new_body, *allocate_seq);
14137 :
14138 43 : if (!by_ref)
14139 42 : new_var = build_simple_mem_ref (*allocate_ptr);
14140 : }
14141 133 : if (by_ref)
14142 : {
14143 12 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14144 12 : if (!allocate_ptr)
14145 : {
14146 11 : x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
14147 11 : if (TREE_CONSTANT (x))
14148 : {
14149 3 : x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
14150 : get_name (var));
14151 3 : gimple_add_tmp_var (x);
14152 3 : TREE_ADDRESSABLE (x) = 1;
14153 3 : x = build_fold_addr_expr_loc (clause_loc, x);
14154 : }
14155 : else
14156 : break;
14157 :
14158 3 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
14159 3 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14160 : fb_rvalue);
14161 : }
14162 : else
14163 1 : x = *allocate_ptr;
14164 :
14165 4 : gimple_seq_add_stmt (&new_body,
14166 4 : gimple_build_assign (new_var, x));
14167 : }
14168 : break;
14169 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
14170 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
14171 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
14172 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
14173 2430 : gimple_seq assign_body;
14174 2430 : bool is_array_data;
14175 2430 : bool do_optional_check;
14176 2430 : assign_body = NULL;
14177 2430 : do_optional_check = false;
14178 2430 : var = OMP_CLAUSE_DECL (c);
14179 2430 : is_array_data = lang_hooks.decls.omp_array_data (var, true) != NULL;
14180 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR && is_array_data)
14181 19 : goto omp_firstprivatize_data_region;
14182 :
14183 2411 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
14184 2411 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
14185 4108 : x = build_sender_ref (is_array_data
14186 601 : ? (splay_tree_key) &DECL_NAME (var)
14187 1453 : : (splay_tree_key) &DECL_UID (var), ctx);
14188 : else
14189 : {
14190 357 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14191 : {
14192 233 : while (TREE_CODE (var) == INDIRECT_REF
14193 233 : || TREE_CODE (var) == ARRAY_REF)
14194 17 : var = TREE_OPERAND (var, 0);
14195 : }
14196 357 : x = build_receiver_ref (var, false, ctx);
14197 : }
14198 :
14199 2411 : if (is_array_data)
14200 : {
14201 601 : bool is_ref = omp_privatize_by_reference (var);
14202 601 : do_optional_check = true;
14203 : /* First, we copy the descriptor data from the host; then
14204 : we update its data to point to the target address. */
14205 601 : new_var = lookup_decl (var, ctx);
14206 601 : new_var = DECL_VALUE_EXPR (new_var);
14207 601 : tree v = new_var;
14208 601 : tree v2 = var;
14209 601 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR
14210 601 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR)
14211 601 : v2 = maybe_lookup_decl_in_outer_ctx (var, ctx);
14212 :
14213 601 : if (is_ref)
14214 : {
14215 396 : v2 = build_fold_indirect_ref (v2);
14216 396 : v = create_tmp_var_raw (TREE_TYPE (v2), get_name (var));
14217 396 : gimple_add_tmp_var (v);
14218 396 : TREE_ADDRESSABLE (v) = 1;
14219 396 : gimplify_assign (v, v2, &assign_body);
14220 396 : tree rhs = build_fold_addr_expr (v);
14221 396 : gimple_seq_add_stmt (&assign_body,
14222 396 : gimple_build_assign (new_var, rhs));
14223 : }
14224 : else
14225 205 : gimplify_assign (new_var, v2, &assign_body);
14226 :
14227 601 : v2 = lang_hooks.decls.omp_array_data (unshare_expr (v), false);
14228 601 : gcc_assert (v2);
14229 601 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14230 1202 : gimple_seq_add_stmt (&assign_body,
14231 601 : gimple_build_assign (v2, x));
14232 : }
14233 1810 : else if (is_variable_sized (var))
14234 : {
14235 12 : tree pvar = DECL_VALUE_EXPR (var);
14236 12 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14237 12 : pvar = TREE_OPERAND (pvar, 0);
14238 12 : gcc_assert (DECL_P (pvar));
14239 12 : new_var = lookup_decl (pvar, ctx);
14240 12 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14241 24 : gimple_seq_add_stmt (&assign_body,
14242 12 : gimple_build_assign (new_var, x));
14243 : }
14244 1798 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
14245 536 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14246 1476 : && !omp_privatize_by_reference (var)
14247 504 : && !omp_is_allocatable_or_ptr (var))
14248 3231 : || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
14249 : {
14250 389 : new_var = lookup_decl (var, ctx);
14251 389 : new_var = DECL_VALUE_EXPR (new_var);
14252 389 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
14253 389 : new_var = TREE_OPERAND (new_var, 0);
14254 389 : gcc_assert (DECL_P (new_var));
14255 389 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14256 778 : gimple_seq_add_stmt (&assign_body,
14257 389 : gimple_build_assign (new_var, x));
14258 : }
14259 : else
14260 : {
14261 1409 : tree type = TREE_TYPE (var);
14262 1409 : new_var = lookup_decl (var, ctx);
14263 1409 : if (omp_privatize_by_reference (var))
14264 : {
14265 1039 : type = TREE_TYPE (type);
14266 1039 : if (POINTER_TYPE_P (type)
14267 1039 : && TREE_CODE (type) != ARRAY_TYPE
14268 1398 : && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
14269 36 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
14270 331 : || (omp_privatize_by_reference (var)
14271 331 : && omp_is_allocatable_or_ptr (var))))
14272 : {
14273 357 : tree v = create_tmp_var_raw (type, get_name (var));
14274 357 : gimple_add_tmp_var (v);
14275 357 : TREE_ADDRESSABLE (v) = 1;
14276 357 : x = fold_convert (type, x);
14277 357 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val,
14278 : fb_rvalue);
14279 357 : gimple_seq_add_stmt (&assign_body,
14280 357 : gimple_build_assign (v, x));
14281 357 : x = build_fold_addr_expr (v);
14282 357 : do_optional_check = true;
14283 : }
14284 : }
14285 1409 : new_var = DECL_VALUE_EXPR (new_var);
14286 1409 : x = fold_convert (TREE_TYPE (new_var), x);
14287 1409 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14288 1409 : gimple_seq_add_stmt (&assign_body,
14289 1409 : gimple_build_assign (new_var, x));
14290 : }
14291 2411 : tree present;
14292 1002 : present = ((do_optional_check
14293 958 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
14294 952 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR)
14295 2342 : ? omp_check_optional_argument (OMP_CLAUSE_DECL (c), true)
14296 : : NULL_TREE);
14297 2411 : if (present)
14298 : {
14299 480 : tree null_label = create_artificial_label (UNKNOWN_LOCATION);
14300 480 : tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
14301 480 : tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
14302 480 : glabel *null_glabel = gimple_build_label (null_label);
14303 480 : glabel *notnull_glabel = gimple_build_label (notnull_label);
14304 480 : ggoto *opt_arg_ggoto = gimple_build_goto (opt_arg_label);
14305 480 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14306 : fb_rvalue);
14307 480 : gimplify_expr (&present, &new_body, NULL, is_gimple_val,
14308 : fb_rvalue);
14309 480 : gcond *cond = gimple_build_cond_from_tree (present,
14310 : notnull_label,
14311 : null_label);
14312 480 : gimple_seq_add_stmt (&new_body, cond);
14313 480 : gimple_seq_add_stmt (&new_body, null_glabel);
14314 480 : gimplify_assign (new_var, null_pointer_node, &new_body);
14315 480 : gimple_seq_add_stmt (&new_body, opt_arg_ggoto);
14316 480 : gimple_seq_add_stmt (&new_body, notnull_glabel);
14317 480 : gimple_seq_add_seq (&new_body, assign_body);
14318 480 : gimple_seq_add_stmt (&new_body,
14319 480 : gimple_build_label (opt_arg_label));
14320 : }
14321 : else
14322 1931 : gimple_seq_add_seq (&new_body, assign_body);
14323 : break;
14324 : }
14325 : /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
14326 : so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
14327 : are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
14328 : or references to VLAs. */
14329 138290 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
14330 113537 : switch (OMP_CLAUSE_CODE (c))
14331 : {
14332 : tree var, new_var, *allocate_ptr;
14333 : default:
14334 98578 : break;
14335 62495 : case OMP_CLAUSE_MAP:
14336 62495 : if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
14337 62495 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
14338 : {
14339 3598 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14340 3598 : poly_int64 offset = 0;
14341 3598 : gcc_assert (prev);
14342 3598 : var = OMP_CLAUSE_DECL (c);
14343 3598 : if (DECL_P (var)
14344 3598 : && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
14345 1089 : && is_global_var (maybe_lookup_decl_in_outer_ctx (var,
14346 : ctx))
14347 3696 : && varpool_node::get_create (var)->offloadable)
14348 : break;
14349 3596 : if (TREE_CODE (var) == INDIRECT_REF
14350 3596 : && TREE_CODE (TREE_OPERAND (var, 0)) == COMPONENT_REF)
14351 0 : var = TREE_OPERAND (var, 0);
14352 3596 : if (TREE_CODE (var) == COMPONENT_REF)
14353 : {
14354 0 : var = get_addr_base_and_unit_offset (var, &offset);
14355 0 : gcc_assert (var != NULL_TREE && DECL_P (var));
14356 : }
14357 3596 : else if (DECL_SIZE (var)
14358 3596 : && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
14359 : {
14360 710 : tree var2 = DECL_VALUE_EXPR (var);
14361 710 : gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
14362 710 : var2 = TREE_OPERAND (var2, 0);
14363 710 : gcc_assert (DECL_P (var2));
14364 : var = var2;
14365 : }
14366 3596 : tree new_var = lookup_decl (var, ctx), x;
14367 3596 : tree type = TREE_TYPE (new_var);
14368 3596 : bool is_ref;
14369 3596 : if (TREE_CODE (OMP_CLAUSE_DECL (c)) == INDIRECT_REF
14370 3596 : && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0))
14371 : == COMPONENT_REF))
14372 : {
14373 0 : type = TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0));
14374 0 : is_ref = true;
14375 0 : new_var = build2 (MEM_REF, type,
14376 : build_fold_addr_expr (new_var),
14377 : build_int_cst (build_pointer_type (type),
14378 : offset));
14379 : }
14380 3596 : else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPONENT_REF)
14381 : {
14382 0 : type = TREE_TYPE (OMP_CLAUSE_DECL (c));
14383 0 : is_ref = TREE_CODE (type) == REFERENCE_TYPE;
14384 0 : new_var = build2 (MEM_REF, type,
14385 : build_fold_addr_expr (new_var),
14386 : build_int_cst (build_pointer_type (type),
14387 : offset));
14388 : }
14389 : else
14390 3596 : is_ref = omp_privatize_by_reference (var);
14391 3596 : if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
14392 : is_ref = false;
14393 6591 : bool ref_to_array = false;
14394 6591 : bool ref_to_ptr = false;
14395 3171 : if (is_ref)
14396 : {
14397 176 : type = TREE_TYPE (type);
14398 176 : if (TREE_CODE (type) == ARRAY_TYPE)
14399 : {
14400 170 : type = build_pointer_type (type);
14401 170 : ref_to_array = true;
14402 : }
14403 : }
14404 3420 : else if (TREE_CODE (type) == ARRAY_TYPE)
14405 : {
14406 377 : tree decl2 = DECL_VALUE_EXPR (new_var);
14407 377 : gcc_assert (TREE_CODE (decl2) == MEM_REF);
14408 377 : decl2 = TREE_OPERAND (decl2, 0);
14409 377 : gcc_assert (DECL_P (decl2));
14410 377 : new_var = decl2;
14411 377 : type = TREE_TYPE (new_var);
14412 : }
14413 3043 : else if (TREE_CODE (type) == REFERENCE_TYPE
14414 3043 : && TREE_CODE (TREE_TYPE (type)) == POINTER_TYPE)
14415 : {
14416 156 : type = TREE_TYPE (type);
14417 156 : ref_to_ptr = true;
14418 : }
14419 3596 : x = build_receiver_ref (OMP_CLAUSE_DECL (prev), false, ctx);
14420 3596 : x = fold_convert_loc (clause_loc, type, x);
14421 3596 : if (!integer_zerop (OMP_CLAUSE_SIZE (c)))
14422 : {
14423 702 : tree bias = OMP_CLAUSE_SIZE (c);
14424 702 : if (DECL_P (bias))
14425 304 : bias = lookup_decl (bias, ctx);
14426 702 : bias = fold_convert_loc (clause_loc, sizetype, bias);
14427 702 : bias = fold_build1_loc (clause_loc, NEGATE_EXPR, sizetype,
14428 : bias);
14429 702 : x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
14430 702 : TREE_TYPE (x), x, bias);
14431 : }
14432 3596 : if (ref_to_array)
14433 170 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
14434 3596 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14435 3596 : if ((is_ref && !ref_to_array)
14436 3590 : || ref_to_ptr)
14437 : {
14438 162 : tree t = create_tmp_var_raw (type, get_name (var));
14439 162 : gimple_add_tmp_var (t);
14440 162 : TREE_ADDRESSABLE (t) = 1;
14441 162 : gimple_seq_add_stmt (&new_body,
14442 162 : gimple_build_assign (t, x));
14443 162 : x = build_fold_addr_expr_loc (clause_loc, t);
14444 : }
14445 3596 : gimple_seq_add_stmt (&new_body,
14446 3596 : gimple_build_assign (new_var, x));
14447 3596 : prev = NULL_TREE;
14448 : }
14449 58897 : else if (OMP_CLAUSE_CHAIN (c)
14450 42613 : && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c))
14451 : == OMP_CLAUSE_MAP
14452 96130 : && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
14453 : == GOMP_MAP_FIRSTPRIVATE_POINTER
14454 34060 : || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
14455 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
14456 : prev = c;
14457 : break;
14458 226 : case OMP_CLAUSE_PRIVATE:
14459 226 : var = OMP_CLAUSE_DECL (c);
14460 226 : by_ref = omp_privatize_by_reference (var);
14461 226 : new_var = lookup_decl (var, ctx);
14462 226 : allocate_ptr = alloc_map.get (new_var);
14463 226 : if (is_variable_sized (var, by_ref)
14464 226 : || (!allocate_ptr && by_ref && !is_gimple_omp_oacc (ctx->stmt)))
14465 : {
14466 24 : if (allocate_ptr)
14467 : {
14468 12 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14469 12 : gcc_assert (allocate_seq);
14470 12 : gimple_seq_add_seq (&new_body, *allocate_seq);
14471 : }
14472 24 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14473 24 : tree pvar = var;
14474 24 : if (!by_ref)
14475 : {
14476 3 : pvar = DECL_VALUE_EXPR (var);
14477 3 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14478 3 : pvar = TREE_OPERAND (pvar, 0);
14479 : }
14480 24 : gcc_assert (DECL_P (pvar));
14481 24 : tree new_pvar = lookup_decl (pvar, ctx);
14482 24 : tree x;
14483 24 : if (!allocate_ptr)
14484 : {
14485 12 : tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
14486 12 : tree ty = TREE_TYPE (new_var);
14487 12 : if (by_ref)
14488 11 : ty = TREE_TYPE (ty);
14489 12 : x = TYPE_SIZE_UNIT (ty);
14490 12 : if (TREE_CONSTANT (x))
14491 : break;
14492 9 : tree al = size_int (TYPE_ALIGN (ty));
14493 9 : x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
14494 9 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar), x);
14495 : }
14496 : else
14497 12 : x = *allocate_ptr;
14498 21 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14499 21 : gimple_seq_add_stmt (&new_body,
14500 21 : gimple_build_assign (new_pvar, x));
14501 : }
14502 : break;
14503 14967 : case OMP_CLAUSE_FIRSTPRIVATE:
14504 14967 : var = OMP_CLAUSE_DECL (c);
14505 14967 : by_ref = omp_privatize_by_reference (var);
14506 14967 : if (is_variable_sized (var, by_ref))
14507 : {
14508 23 : tree new_var = lookup_decl (var, ctx);
14509 23 : tree *allocate_ptr = alloc_map.get (new_var);
14510 23 : if (!allocate_ptr)
14511 : break;
14512 15 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14513 15 : gcc_assert (allocate_seq);
14514 15 : gimple_seq_add_seq (&new_body, *allocate_seq);
14515 :
14516 15 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14517 15 : tree pvar = var;
14518 15 : if (!by_ref)
14519 : {
14520 2 : pvar = DECL_VALUE_EXPR (var);
14521 2 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14522 2 : pvar = TREE_OPERAND (pvar, 0);
14523 : }
14524 15 : gcc_assert (DECL_P (pvar));
14525 15 : tree new_pvar = lookup_decl (pvar, ctx);
14526 15 : tree x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar),
14527 15 : *allocate_ptr);
14528 15 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14529 15 : gimple_seq_add_stmt (&new_body,
14530 15 : gimple_build_assign (new_pvar, x));
14531 :
14532 15 : x = build_receiver_ref (var, true, ctx);
14533 15 : new_var = unshare_expr (new_var);
14534 15 : if (by_ref)
14535 13 : new_var = build_fold_indirect_ref (new_var);
14536 15 : x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
14537 15 : gimplify_and_add (x, &new_body);
14538 : }
14539 : }
14540 :
14541 24753 : gimple_seq fork_seq = NULL;
14542 24753 : gimple_seq join_seq = NULL;
14543 :
14544 24753 : if (offloaded && is_gimple_omp_oacc (ctx->stmt))
14545 : {
14546 : /* If there are reductions on the offloaded region itself, treat
14547 : them as a dummy GANG loop. */
14548 9391 : tree level = build_int_cst (integer_type_node, GOMP_DIM_GANG);
14549 :
14550 9391 : gcall *private_marker = lower_oacc_private_marker (ctx);
14551 :
14552 9391 : if (private_marker)
14553 30 : gimple_call_set_arg (private_marker, 2, level);
14554 :
14555 9391 : lower_oacc_reductions (gimple_location (ctx->stmt), clauses, level,
14556 : false, NULL, private_marker, NULL, &fork_seq,
14557 : &join_seq, ctx);
14558 : }
14559 :
14560 24753 : gimple_seq_add_seq (&new_body, fork_seq);
14561 24753 : gimple_seq_add_seq (&new_body, tgt_body);
14562 24753 : gimple_seq_add_seq (&new_body, join_seq);
14563 24753 : gimple_seq_add_seq (&new_body, alloc_dlist);
14564 :
14565 24753 : if (offloaded)
14566 : {
14567 20746 : new_body = maybe_catch_exception (new_body);
14568 20746 : gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
14569 : }
14570 24753 : gimple_omp_set_body (stmt, new_body);
14571 : }
14572 :
14573 36454 : gsi_insert_seq_before (gsi_p, gimple_omp_target_iterator_loops (stmt),
14574 : GSI_SAME_STMT);
14575 36454 : gimple_omp_target_set_iterator_loops (stmt, NULL);
14576 57200 : bind = gimple_build_bind (NULL, NULL,
14577 20746 : tgt_bind ? gimple_bind_block (tgt_bind)
14578 : : NULL_TREE);
14579 72516 : gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
14580 36454 : gimple_bind_add_seq (bind, ilist);
14581 36454 : gimple_bind_add_stmt (bind, stmt);
14582 36454 : gimple_bind_add_seq (bind, olist);
14583 :
14584 36454 : pop_gimplify_context (NULL);
14585 :
14586 36454 : if (dep_bind)
14587 : {
14588 392 : gimple_bind_add_seq (dep_bind, dep_ilist);
14589 392 : gimple_bind_add_stmt (dep_bind, bind);
14590 392 : gimple_bind_add_seq (dep_bind, dep_olist);
14591 392 : pop_gimplify_context (dep_bind);
14592 : }
14593 36454 : }
14594 :
14595 : /* Generate code to implement the action-clauses (destroy, init, use) of an
14596 : OpenMP interop construct. */
14597 :
14598 : static void
14599 492 : lower_omp_interop_action_clauses (gimple_seq *seq, vec<tree> &objs,
14600 : vec<tree> *interop_types = NULL,
14601 : vec<tree> *prefer_types = NULL)
14602 : {
14603 492 : if (objs.length () == 0)
14604 229 : return;
14605 :
14606 263 : enum omp_clause_code action = OMP_CLAUSE_CODE (objs[0]);
14607 263 : if (action == OMP_CLAUSE_INIT)
14608 402 : gcc_checking_assert (objs.length () == interop_types->length ()
14609 : && objs.length () == prefer_types->length ());
14610 : else
14611 129 : gcc_assert (prefer_types == NULL && interop_types == NULL);
14612 :
14613 263 : tree ret_objs = NULL_TREE, ret_interop_types = NULL_TREE,
14614 263 : ret_prefer_types = NULL_TREE;
14615 :
14616 : /* Build an array of interop objects. */
14617 :
14618 263 : tree type_obj_pref = build_array_type_nelts (ptr_type_node, objs.length ());
14619 263 : ret_objs = create_tmp_var (type_obj_pref, "interopobjs");
14620 :
14621 263 : bool have_pref_type = false;
14622 263 : if (action == OMP_CLAUSE_INIT)
14623 : {
14624 592 : for (tree pref_type : prefer_types)
14625 260 : if (pref_type != NULL_TREE)
14626 : {
14627 : have_pref_type = true;
14628 : break;
14629 : }
14630 134 : tree type_tgtsync
14631 134 : = build_array_type_nelts (integer_type_node, objs.length ());
14632 134 : ret_interop_types = create_tmp_var (type_tgtsync, "tgt_tgtsync");
14633 134 : if (have_pref_type)
14634 70 : ret_prefer_types = create_tmp_var (type_obj_pref, "pref_type");
14635 : else
14636 : {
14637 64 : ret_prefer_types = null_pointer_node;
14638 64 : prefer_types->truncate (0);
14639 : }
14640 : }
14641 :
14642 856 : for (size_t i = 0; !objs.is_empty (); i++)
14643 : {
14644 593 : tree offset = build_int_cst (integer_type_node, i);
14645 593 : tree init = build4 (ARRAY_REF, ptr_type_node, ret_objs, offset, NULL_TREE,
14646 : NULL_TREE);
14647 593 : tree obj = OMP_CLAUSE_DECL (objs.pop ());
14648 593 : if (TREE_CODE (TREE_TYPE (obj)) == REFERENCE_TYPE)
14649 6 : obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
14650 593 : if (action != OMP_CLAUSE_USE
14651 593 : && TREE_CODE (TREE_TYPE (obj)) != POINTER_TYPE)
14652 : /* For modifying actions, we need a pointer. */
14653 483 : obj = build_fold_addr_expr (obj);
14654 110 : else if (action == OMP_CLAUSE_USE
14655 110 : && TREE_CODE (TREE_TYPE (obj)) == POINTER_TYPE)
14656 : /* For use action, we need the value. */
14657 2 : obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
14658 593 : init = build2 (MODIFY_EXPR, ptr_type_node, init,
14659 : fold_convert (ptr_type_node, obj));
14660 593 : gimplify_and_add (init, seq);
14661 :
14662 593 : if (action == OMP_CLAUSE_INIT)
14663 : {
14664 384 : init = build4 (ARRAY_REF, integer_type_node, ret_interop_types,
14665 : offset, NULL_TREE, NULL_TREE);
14666 384 : init = build2 (MODIFY_EXPR, integer_type_node, init,
14667 384 : interop_types->pop ());
14668 384 : gimplify_and_add (init, seq);
14669 :
14670 384 : if (have_pref_type)
14671 : {
14672 194 : tree prefer_type = prefer_types->pop ();
14673 194 : tree pref = (prefer_type == NULL_TREE
14674 194 : ? null_pointer_node
14675 194 : : build_fold_addr_expr (prefer_type));
14676 194 : init = build4 (ARRAY_REF, ptr_type_node, ret_prefer_types, offset,
14677 : NULL_TREE, NULL_TREE);
14678 194 : init = build2 (MODIFY_EXPR, ptr_type_node, init, pref);
14679 194 : gimplify_and_add (init, seq);
14680 : }
14681 : }
14682 : }
14683 263 : if (action == OMP_CLAUSE_INIT)
14684 : {
14685 134 : if (have_pref_type)
14686 70 : ret_prefer_types = build_fold_addr_expr (ret_prefer_types);
14687 134 : ret_interop_types = build_fold_addr_expr (ret_interop_types);
14688 : }
14689 263 : ret_objs = build_fold_addr_expr (ret_objs);
14690 :
14691 660 : gcc_assert (objs.is_empty ()
14692 : && (!interop_types || interop_types->is_empty ())
14693 : && (!prefer_types || prefer_types->is_empty ()));
14694 :
14695 263 : objs.safe_push (ret_objs);
14696 263 : if (action == OMP_CLAUSE_INIT)
14697 : {
14698 134 : interop_types->safe_push (ret_interop_types);
14699 134 : prefer_types->safe_push (ret_prefer_types);
14700 : }
14701 : }
14702 :
14703 : /* Lower code for an OpenMP interop directive. */
14704 :
14705 : static void
14706 164 : lower_omp_interop (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14707 : {
14708 164 : push_gimplify_context ();
14709 :
14710 164 : tree block = make_node (BLOCK);
14711 164 : gbind *bind = gimple_build_bind (NULL, NULL, block);
14712 164 : gimple_seq bind_body = NULL;
14713 :
14714 : /* Emit call to GOMP_interop:
14715 : void
14716 : GOMP_interop (int device_num, int n_init, omp_interop_t **init,
14717 : const void *target_targetsync, const void *prefer_type,
14718 : int n_use, omp_interop_t *use, int n_destroy,
14719 : omp_interop_t **destroy, unsigned int flags,
14720 : void **depend) */
14721 :
14722 164 : tree flags = NULL_TREE;
14723 164 : tree depend = null_pointer_node;
14724 164 : tree device_num = NULL_TREE;
14725 :
14726 164 : auto_vec<tree> init_objs, use_objs, destroy_objs, prefer_type,
14727 164 : target_targetsync;
14728 164 : gimple_seq dep_ilist = NULL, dep_olist = NULL;
14729 164 : tree clauses = gimple_omp_interop_clauses (gsi_stmt (*gsi_p));
14730 868 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
14731 : {
14732 704 : switch (OMP_CLAUSE_CODE (c))
14733 : {
14734 384 : case OMP_CLAUSE_INIT:
14735 384 : {
14736 384 : init_objs.safe_push (c);
14737 384 : int target_targetsync_bits = 0;
14738 384 : if (OMP_CLAUSE_INIT_TARGET (c))
14739 229 : target_targetsync_bits |= GOMP_INTEROP_TARGET;
14740 384 : if (OMP_CLAUSE_INIT_TARGETSYNC (c))
14741 204 : target_targetsync_bits |= GOMP_INTEROP_TARGETSYNC;
14742 384 : tree t = build_int_cst (integer_type_node, target_targetsync_bits);
14743 384 : target_targetsync.safe_push (t);
14744 384 : prefer_type.safe_push (OMP_CLAUSE_INIT_PREFER_TYPE (c));
14745 : }
14746 384 : break;
14747 106 : case OMP_CLAUSE_USE:
14748 106 : use_objs.safe_push (c);
14749 106 : break;
14750 103 : case OMP_CLAUSE_DESTROY:
14751 103 : destroy_objs.safe_push (c);
14752 103 : break;
14753 50 : case OMP_CLAUSE_NOWAIT:
14754 50 : flags = build_int_cst (integer_type_node, GOMP_INTEROP_FLAG_NOWAIT);
14755 50 : break;
14756 35 : case OMP_CLAUSE_DEPEND:
14757 35 : {
14758 35 : tree *cp = gimple_omp_interop_clauses_ptr (gsi_stmt (*gsi_p));
14759 35 : lower_depend_clauses (cp, &dep_ilist, &dep_olist);
14760 35 : depend = OMP_CLAUSE_DECL (*cp);
14761 : }
14762 35 : break;
14763 26 : case OMP_CLAUSE_DEVICE:
14764 26 : device_num = OMP_CLAUSE_DEVICE_ID (c);
14765 26 : break;
14766 0 : default:
14767 0 : gcc_unreachable ();
14768 : }
14769 : }
14770 :
14771 164 : if (flags == NULL_TREE)
14772 114 : flags = build_int_cst (integer_type_node, 0);
14773 :
14774 164 : if (device_num == NULL_TREE)
14775 138 : device_num = build_int_cst (integer_type_node, GOMP_DEVICE_DEFAULT_OMP_61);
14776 :
14777 298 : tree n_init = build_int_cst (integer_type_node, init_objs.length ());
14778 234 : tree n_use = build_int_cst (integer_type_node, use_objs.length ());
14779 223 : tree n_destroy = build_int_cst (integer_type_node, destroy_objs.length ());
14780 :
14781 164 : lower_omp_interop_action_clauses (&bind_body, init_objs, &target_targetsync,
14782 : &prefer_type);
14783 164 : lower_omp_interop_action_clauses (&bind_body, use_objs);
14784 164 : lower_omp_interop_action_clauses (&bind_body, destroy_objs);
14785 :
14786 164 : gimple_seq_add_seq (&bind_body, dep_ilist);
14787 164 : tree fn = builtin_decl_explicit (BUILT_IN_GOMP_INTEROP);
14788 298 : tree init_arg = init_objs.length () ? init_objs.pop () : null_pointer_node;
14789 164 : tree target_targetsync_arg = target_targetsync.length ()
14790 134 : ? target_targetsync.pop ()
14791 164 : : null_pointer_node;
14792 164 : tree prefer_type_arg
14793 298 : = prefer_type.length () ? prefer_type.pop () : null_pointer_node;
14794 234 : tree use_arg = use_objs.length () ? use_objs.pop () : null_pointer_node;
14795 164 : tree destroy_arg
14796 223 : = destroy_objs.length () ? destroy_objs.pop () : null_pointer_node;
14797 164 : gcall *call
14798 164 : = gimple_build_call (fn, 11, device_num, n_init, init_arg,
14799 : target_targetsync_arg, prefer_type_arg, n_use, use_arg,
14800 : n_destroy, destroy_arg, flags, depend);
14801 164 : gimple_seq_add_stmt (&bind_body, call);
14802 164 : gimple_seq_add_seq (&bind_body, dep_olist);
14803 :
14804 164 : gsi_replace (gsi_p, bind, true);
14805 164 : gimple_bind_set_body (bind, bind_body);
14806 164 : pop_gimplify_context (bind);
14807 164 : gimple_bind_append_vars (bind, ctx->block_vars);
14808 164 : BLOCK_VARS (block) = ctx->block_vars;
14809 164 : }
14810 :
14811 : /* Expand code for an OpenMP teams directive. */
14812 :
14813 : static void
14814 5917 : lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14815 : {
14816 5917 : gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p));
14817 5917 : push_gimplify_context ();
14818 :
14819 5917 : tree block = make_node (BLOCK);
14820 5917 : gbind *bind = gimple_build_bind (NULL, NULL, block);
14821 5917 : gsi_replace (gsi_p, bind, true);
14822 5917 : gimple_seq bind_body = NULL;
14823 5917 : gimple_seq dlist = NULL;
14824 5917 : gimple_seq olist = NULL;
14825 :
14826 5917 : tree num_teams = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
14827 5917 : OMP_CLAUSE_NUM_TEAMS);
14828 5917 : tree num_teams_lower = NULL_TREE;
14829 5917 : if (num_teams == NULL_TREE)
14830 5300 : num_teams = build_int_cst (unsigned_type_node, 0);
14831 : else
14832 : {
14833 617 : num_teams_lower = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (num_teams);
14834 617 : if (num_teams_lower)
14835 : {
14836 148 : num_teams_lower = fold_convert (unsigned_type_node, num_teams_lower);
14837 148 : gimplify_expr (&num_teams_lower, &bind_body, NULL, is_gimple_val,
14838 : fb_rvalue);
14839 : }
14840 617 : num_teams = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
14841 617 : num_teams = fold_convert (unsigned_type_node, num_teams);
14842 617 : gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
14843 : }
14844 5917 : if (num_teams_lower == NULL_TREE)
14845 5769 : num_teams_lower = num_teams;
14846 5917 : tree thread_limit = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
14847 5917 : OMP_CLAUSE_THREAD_LIMIT);
14848 5917 : if (thread_limit == NULL_TREE)
14849 5446 : thread_limit = build_int_cst (unsigned_type_node, 0);
14850 : else
14851 : {
14852 471 : thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
14853 471 : thread_limit = fold_convert (unsigned_type_node, thread_limit);
14854 471 : gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
14855 : fb_rvalue);
14856 : }
14857 5917 : location_t loc = gimple_location (teams_stmt);
14858 5917 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS4);
14859 5917 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
14860 5917 : tree first = create_tmp_var (rettype);
14861 5917 : gimple_seq_add_stmt (&bind_body,
14862 5917 : gimple_build_assign (first, build_one_cst (rettype)));
14863 5917 : tree llabel = create_artificial_label (loc);
14864 5917 : gimple_seq_add_stmt (&bind_body, gimple_build_label (llabel));
14865 5917 : gimple *call
14866 5917 : = gimple_build_call (decl, 4, num_teams_lower, num_teams, thread_limit,
14867 : first);
14868 5917 : gimple_set_location (call, loc);
14869 5917 : tree temp = create_tmp_var (rettype);
14870 5917 : gimple_call_set_lhs (call, temp);
14871 5917 : gimple_seq_add_stmt (&bind_body, call);
14872 :
14873 5917 : tree tlabel = create_artificial_label (loc);
14874 5917 : tree flabel = create_artificial_label (loc);
14875 5917 : gimple *cond = gimple_build_cond (NE_EXPR, temp, build_zero_cst (rettype),
14876 : tlabel, flabel);
14877 5917 : gimple_seq_add_stmt (&bind_body, cond);
14878 5917 : gimple_seq_add_stmt (&bind_body, gimple_build_label (tlabel));
14879 5917 : gimple_seq_add_stmt (&bind_body,
14880 5917 : gimple_build_assign (first, build_zero_cst (rettype)));
14881 :
14882 5917 : lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
14883 : &bind_body, &dlist, ctx, NULL);
14884 5917 : lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
14885 5917 : lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist,
14886 : NULL, ctx);
14887 5917 : gimple_seq_add_stmt (&bind_body, teams_stmt);
14888 :
14889 5917 : gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
14890 5917 : gimple_omp_set_body (teams_stmt, NULL);
14891 5917 : gimple_seq_add_seq (&bind_body, olist);
14892 5917 : gimple_seq_add_seq (&bind_body, dlist);
14893 5917 : gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
14894 5917 : gimple_seq_add_stmt (&bind_body, gimple_build_goto (llabel));
14895 5917 : gimple_seq_add_stmt (&bind_body, gimple_build_label (flabel));
14896 5917 : gimple_bind_set_body (bind, bind_body);
14897 :
14898 5917 : pop_gimplify_context (bind);
14899 :
14900 5917 : gimple_bind_append_vars (bind, ctx->block_vars);
14901 5917 : BLOCK_VARS (block) = ctx->block_vars;
14902 5917 : if (BLOCK_VARS (block))
14903 687 : TREE_USED (block) = 1;
14904 5917 : }
14905 :
14906 : /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
14907 : regimplified. If DATA is non-NULL, lower_omp_1 is outside
14908 : of OMP context, but with make_addressable_vars set. */
14909 :
14910 : static tree
14911 3180381 : lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
14912 : void *data)
14913 : {
14914 3180381 : tree t = *tp;
14915 :
14916 : /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
14917 3180381 : if ((VAR_P (t) || TREE_CODE (t) == PARM_DECL || TREE_CODE (t) == RESULT_DECL)
14918 1477404 : && data == NULL
14919 1416997 : && DECL_HAS_VALUE_EXPR_P (t))
14920 : return t;
14921 :
14922 3076451 : if (make_addressable_vars
14923 422493 : && DECL_P (t)
14924 3281478 : && bitmap_bit_p (make_addressable_vars, DECL_UID (t)))
14925 : return t;
14926 :
14927 : /* If a global variable has been privatized, TREE_CONSTANT on
14928 : ADDR_EXPR might be wrong. */
14929 3073862 : if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
14930 108395 : recompute_tree_invariant_for_addr_expr (t);
14931 :
14932 3073862 : *walk_subtrees = !IS_TYPE_OR_DECL_P (t);
14933 3073862 : return NULL_TREE;
14934 : }
14935 :
14936 : /* Data to be communicated between lower_omp_regimplify_operands and
14937 : lower_omp_regimplify_operands_p. */
14938 :
14939 : struct lower_omp_regimplify_operands_data
14940 : {
14941 : omp_context *ctx;
14942 : vec<tree> *decls;
14943 : };
14944 :
14945 : /* Helper function for lower_omp_regimplify_operands. Find
14946 : omp_member_access_dummy_var vars and adjust temporarily their
14947 : DECL_VALUE_EXPRs if needed. */
14948 :
14949 : static tree
14950 417917 : lower_omp_regimplify_operands_p (tree *tp, int *walk_subtrees,
14951 : void *data)
14952 : {
14953 417917 : tree t = omp_member_access_dummy_var (*tp);
14954 417917 : if (t)
14955 : {
14956 9 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
14957 9 : lower_omp_regimplify_operands_data *ldata
14958 : = (lower_omp_regimplify_operands_data *) wi->info;
14959 9 : tree o = maybe_lookup_decl (t, ldata->ctx);
14960 9 : if (o == NULL_TREE)
14961 5 : o = maybe_lookup_decl_in_outer_ctx (t, ldata->ctx);
14962 9 : if (o != t)
14963 : {
14964 9 : ldata->decls->safe_push (DECL_VALUE_EXPR (*tp));
14965 9 : ldata->decls->safe_push (*tp);
14966 9 : tree v = unshare_and_remap (DECL_VALUE_EXPR (*tp), t, o);
14967 9 : SET_DECL_VALUE_EXPR (*tp, v);
14968 : }
14969 : }
14970 417917 : *walk_subtrees = !IS_TYPE_OR_DECL_P (*tp);
14971 417917 : return NULL_TREE;
14972 : }
14973 :
14974 : /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
14975 : of omp_member_access_dummy_var vars during regimplification. */
14976 :
14977 : static void
14978 106364 : lower_omp_regimplify_operands (omp_context *ctx, gimple *stmt,
14979 : gimple_stmt_iterator *gsi_p)
14980 : {
14981 106364 : auto_vec<tree, 10> decls;
14982 106364 : if (ctx)
14983 : {
14984 103842 : struct walk_stmt_info wi;
14985 103842 : memset (&wi, '\0', sizeof (wi));
14986 103842 : struct lower_omp_regimplify_operands_data data;
14987 103842 : data.ctx = ctx;
14988 103842 : data.decls = &decls;
14989 103842 : wi.info = &data;
14990 103842 : walk_gimple_op (stmt, lower_omp_regimplify_operands_p, &wi);
14991 : }
14992 106364 : gimple_regimplify_operands (stmt, gsi_p);
14993 319101 : while (!decls.is_empty ())
14994 : {
14995 9 : tree t = decls.pop ();
14996 9 : tree v = decls.pop ();
14997 9 : SET_DECL_VALUE_EXPR (t, v);
14998 : }
14999 106364 : }
15000 :
15001 : static void
15002 2467355 : lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15003 : {
15004 2467355 : gimple *stmt = gsi_stmt (*gsi_p);
15005 2467355 : struct walk_stmt_info wi;
15006 2467355 : gcall *call_stmt;
15007 :
15008 2467355 : if (gimple_has_location (stmt))
15009 2026117 : input_location = gimple_location (stmt);
15010 :
15011 2467355 : if (make_addressable_vars)
15012 154039 : memset (&wi, '\0', sizeof (wi));
15013 :
15014 : /* If we have issued syntax errors, avoid doing any heavy lifting.
15015 : Just replace the OMP directives with a NOP to avoid
15016 : confusing RTL expansion. */
15017 2467355 : if (seen_error () && is_gimple_omp (stmt))
15018 : {
15019 11403 : gsi_replace (gsi_p, gimple_build_nop (), true);
15020 11403 : return;
15021 : }
15022 :
15023 2455952 : switch (gimple_code (stmt))
15024 : {
15025 153505 : case GIMPLE_COND:
15026 153505 : {
15027 153505 : gcond *cond_stmt = as_a <gcond *> (stmt);
15028 86973 : if ((ctx || make_addressable_vars)
15029 157113 : && (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
15030 : lower_omp_regimplify_p,
15031 : ctx ? NULL : &wi, NULL)
15032 134731 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
15033 : lower_omp_regimplify_p,
15034 : ctx ? NULL : &wi, NULL)))
15035 1244 : lower_omp_regimplify_operands (ctx, cond_stmt, gsi_p);
15036 : }
15037 : break;
15038 20 : case GIMPLE_CATCH:
15039 20 : lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx);
15040 20 : break;
15041 0 : case GIMPLE_EH_FILTER:
15042 0 : lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
15043 0 : break;
15044 25470 : case GIMPLE_TRY:
15045 25470 : lower_omp (gimple_try_eval_ptr (stmt), ctx);
15046 25465 : lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
15047 25465 : break;
15048 0 : case GIMPLE_ASSUME:
15049 0 : lower_omp (gimple_assume_body_ptr (stmt), ctx);
15050 0 : break;
15051 5 : case GIMPLE_TRANSACTION:
15052 5 : lower_omp (gimple_transaction_body_ptr (as_a <gtransaction *> (stmt)),
15053 : ctx);
15054 5 : break;
15055 165703 : case GIMPLE_BIND:
15056 165703 : if (ctx && is_gimple_omp_oacc (ctx->stmt))
15057 : {
15058 17778 : tree vars = gimple_bind_vars (as_a <gbind *> (stmt));
15059 17778 : oacc_privatization_scan_decl_chain (ctx, vars);
15060 : }
15061 165703 : lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx);
15062 165696 : maybe_remove_omp_member_access_dummy_vars (as_a <gbind *> (stmt));
15063 165696 : break;
15064 19944 : case GIMPLE_OMP_PARALLEL:
15065 19944 : case GIMPLE_OMP_TASK:
15066 19944 : ctx = maybe_lookup_ctx (stmt);
15067 19944 : gcc_assert (ctx);
15068 19944 : if (ctx->cancellable)
15069 135 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15070 19944 : lower_omp_taskreg (gsi_p, ctx);
15071 19944 : break;
15072 46879 : case GIMPLE_OMP_FOR:
15073 46879 : ctx = maybe_lookup_ctx (stmt);
15074 46879 : gcc_assert (ctx);
15075 46879 : if (ctx->cancellable)
15076 84 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15077 46879 : lower_omp_for (gsi_p, ctx);
15078 46879 : break;
15079 378 : case GIMPLE_OMP_SECTIONS:
15080 378 : ctx = maybe_lookup_ctx (stmt);
15081 378 : gcc_assert (ctx);
15082 378 : if (ctx->cancellable)
15083 19 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15084 378 : lower_omp_sections (gsi_p, ctx);
15085 378 : break;
15086 133 : case GIMPLE_OMP_SCOPE:
15087 133 : ctx = maybe_lookup_ctx (stmt);
15088 133 : gcc_assert (ctx);
15089 133 : lower_omp_scope (gsi_p, ctx);
15090 133 : break;
15091 659 : case GIMPLE_OMP_DISPATCH:
15092 659 : ctx = maybe_lookup_ctx (stmt);
15093 659 : gcc_assert (ctx);
15094 659 : lower_omp_dispatch (gsi_p, ctx);
15095 659 : break;
15096 164 : case GIMPLE_OMP_INTEROP:
15097 164 : ctx = maybe_lookup_ctx (stmt);
15098 164 : gcc_assert (ctx);
15099 164 : lower_omp_interop (gsi_p, ctx);
15100 164 : break;
15101 1119 : case GIMPLE_OMP_SINGLE:
15102 1119 : ctx = maybe_lookup_ctx (stmt);
15103 1119 : gcc_assert (ctx);
15104 1119 : lower_omp_single (gsi_p, ctx);
15105 1119 : break;
15106 658 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15107 : /* We have already done error checking at this point, so these nodes
15108 : can be completely removed and replaced with their body. */
15109 658 : ctx = maybe_lookup_ctx (stmt);
15110 658 : gcc_assert (ctx);
15111 658 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
15112 658 : gsi_replace_with_seq (gsi_p, gimple_omp_body (stmt), true);
15113 658 : break;
15114 1108 : case GIMPLE_OMP_MASTER:
15115 1108 : case GIMPLE_OMP_MASKED:
15116 1108 : ctx = maybe_lookup_ctx (stmt);
15117 1108 : gcc_assert (ctx);
15118 1108 : lower_omp_master (gsi_p, ctx);
15119 1108 : break;
15120 536 : case GIMPLE_OMP_TASKGROUP:
15121 536 : ctx = maybe_lookup_ctx (stmt);
15122 536 : gcc_assert (ctx);
15123 536 : lower_omp_taskgroup (gsi_p, ctx);
15124 536 : break;
15125 1124 : case GIMPLE_OMP_ORDERED:
15126 1124 : ctx = maybe_lookup_ctx (stmt);
15127 1124 : gcc_assert (ctx);
15128 1124 : lower_omp_ordered (gsi_p, ctx);
15129 1124 : break;
15130 1268 : case GIMPLE_OMP_SCAN:
15131 1268 : ctx = maybe_lookup_ctx (stmt);
15132 1268 : gcc_assert (ctx);
15133 1268 : lower_omp_scan (gsi_p, ctx);
15134 1268 : break;
15135 311 : case GIMPLE_OMP_CRITICAL:
15136 311 : ctx = maybe_lookup_ctx (stmt);
15137 311 : gcc_assert (ctx);
15138 311 : lower_omp_critical (gsi_p, ctx);
15139 311 : break;
15140 2516 : case GIMPLE_OMP_ATOMIC_LOAD:
15141 85 : if ((ctx || make_addressable_vars)
15142 2516 : && walk_tree (gimple_omp_atomic_load_rhs_ptr (
15143 : as_a <gomp_atomic_load *> (stmt)),
15144 : lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
15145 1425 : lower_omp_regimplify_operands (ctx, stmt, gsi_p);
15146 : break;
15147 36454 : case GIMPLE_OMP_TARGET:
15148 36454 : ctx = maybe_lookup_ctx (stmt);
15149 36454 : gcc_assert (ctx);
15150 36454 : lower_omp_target (gsi_p, ctx);
15151 36454 : break;
15152 8413 : case GIMPLE_OMP_TEAMS:
15153 8413 : ctx = maybe_lookup_ctx (stmt);
15154 8413 : gcc_assert (ctx);
15155 8413 : if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
15156 2496 : lower_omp_taskreg (gsi_p, ctx);
15157 : else
15158 5917 : lower_omp_teams (gsi_p, ctx);
15159 : break;
15160 133180 : case GIMPLE_CALL:
15161 133180 : tree fndecl;
15162 133180 : call_stmt = as_a <gcall *> (stmt);
15163 133180 : fndecl = gimple_call_fndecl (call_stmt);
15164 133180 : if (fndecl
15165 133180 : && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
15166 38098 : switch (DECL_FUNCTION_CODE (fndecl))
15167 : {
15168 727 : case BUILT_IN_GOMP_BARRIER:
15169 727 : if (ctx == NULL)
15170 : break;
15171 : /* FALLTHRU */
15172 1037 : case BUILT_IN_GOMP_CANCEL:
15173 1037 : case BUILT_IN_GOMP_CANCELLATION_POINT:
15174 1037 : omp_context *cctx;
15175 1037 : cctx = ctx;
15176 1037 : if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
15177 43 : cctx = cctx->outer;
15178 1037 : gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE);
15179 1037 : if (!cctx->cancellable)
15180 : {
15181 679 : if (DECL_FUNCTION_CODE (fndecl)
15182 : == BUILT_IN_GOMP_CANCELLATION_POINT)
15183 : {
15184 0 : stmt = gimple_build_nop ();
15185 0 : gsi_replace (gsi_p, stmt, false);
15186 : }
15187 : break;
15188 : }
15189 358 : if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
15190 : {
15191 14 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
15192 14 : gimple_call_set_fndecl (call_stmt, fndecl);
15193 14 : gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl));
15194 : }
15195 358 : tree lhs;
15196 358 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)));
15197 358 : gimple_call_set_lhs (call_stmt, lhs);
15198 358 : tree fallthru_label;
15199 358 : fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
15200 358 : gimple *g;
15201 358 : g = gimple_build_label (fallthru_label);
15202 358 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15203 358 : g = gimple_build_cond (NE_EXPR, lhs,
15204 358 : fold_convert (TREE_TYPE (lhs),
15205 : boolean_false_node),
15206 : cctx->cancel_label, fallthru_label);
15207 358 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15208 358 : break;
15209 : default:
15210 : break;
15211 : }
15212 133145 : goto regimplify;
15213 :
15214 : case GIMPLE_ASSIGN:
15215 1431215 : for (omp_context *up = ctx; up; up = up->outer)
15216 : {
15217 676271 : if (gimple_code (up->stmt) == GIMPLE_OMP_ORDERED
15218 : || gimple_code (up->stmt) == GIMPLE_OMP_CRITICAL
15219 : || gimple_code (up->stmt) == GIMPLE_OMP_TASKGROUP
15220 : || gimple_code (up->stmt) == GIMPLE_OMP_SCOPE
15221 : || gimple_code (up->stmt) == GIMPLE_OMP_SECTION
15222 : || gimple_code (up->stmt) == GIMPLE_OMP_SCAN
15223 : || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
15224 222699 : && (gimple_omp_target_kind (up->stmt)
15225 : == GF_OMP_TARGET_KIND_DATA)))
15226 90101 : continue;
15227 586170 : else if (!up->lastprivate_conditional_map)
15228 : break;
15229 4204 : tree lhs = get_base_address (gimple_assign_lhs (stmt));
15230 4204 : if (TREE_CODE (lhs) == MEM_REF
15231 26 : && DECL_P (TREE_OPERAND (lhs, 0))
15232 4227 : && TREE_CODE (TREE_TYPE (TREE_OPERAND (lhs,
15233 : 0))) == REFERENCE_TYPE)
15234 19 : lhs = TREE_OPERAND (lhs, 0);
15235 4204 : if (DECL_P (lhs))
15236 3161 : if (tree *v = up->lastprivate_conditional_map->get (lhs))
15237 : {
15238 382 : tree clauses;
15239 382 : if (up->combined_into_simd_safelen1)
15240 : {
15241 54 : up = up->outer;
15242 54 : if (gimple_code (up->stmt) == GIMPLE_OMP_SCAN)
15243 3 : up = up->outer;
15244 : }
15245 382 : if (gimple_code (up->stmt) == GIMPLE_OMP_FOR)
15246 298 : clauses = gimple_omp_for_clauses (up->stmt);
15247 : else
15248 84 : clauses = gimple_omp_sections_clauses (up->stmt);
15249 382 : tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
15250 382 : if (!OMP_CLAUSE__CONDTEMP__ITER (c))
15251 348 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c),
15252 : OMP_CLAUSE__CONDTEMP_);
15253 382 : gcc_assert (OMP_CLAUSE__CONDTEMP__ITER (c));
15254 382 : gimple *g = gimple_build_assign (*v, OMP_CLAUSE_DECL (c));
15255 382 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15256 : }
15257 : }
15258 : /* FALLTHRU */
15259 :
15260 1989585 : default:
15261 1989585 : regimplify:
15262 1044573 : if ((ctx || make_addressable_vars)
15263 2065357 : && walk_gimple_op (stmt, lower_omp_regimplify_p,
15264 : ctx ? NULL : &wi))
15265 : {
15266 : /* Just remove clobbers, this should happen only if we have
15267 : "privatized" local addressable variables in SIMD regions,
15268 : the clobber isn't needed in that case and gimplifying address
15269 : of the ARRAY_REF into a pointer and creating MEM_REF based
15270 : clobber would create worse code than we get with the clobber
15271 : dropped. */
15272 103850 : if (gimple_clobber_p (stmt))
15273 : {
15274 155 : gsi_replace (gsi_p, gimple_build_nop (), true);
15275 155 : break;
15276 : }
15277 103695 : lower_omp_regimplify_operands (ctx, stmt, gsi_p);
15278 : }
15279 : break;
15280 : }
15281 : }
15282 :
15283 : static void
15284 406536 : lower_omp (gimple_seq *body, omp_context *ctx)
15285 : {
15286 406536 : location_t saved_location = input_location;
15287 406536 : gimple_stmt_iterator gsi;
15288 3232287 : for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
15289 2467355 : lower_omp_1 (&gsi, ctx);
15290 : /* During gimplification, we haven't folded statments inside offloading
15291 : or taskreg regions (gimplify.cc:maybe_fold_stmt); do that now. */
15292 406520 : if (target_nesting_level || taskreg_nesting_level)
15293 640976 : for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
15294 430393 : fold_stmt (&gsi);
15295 406520 : input_location = saved_location;
15296 406520 : }
15297 :
15298 : /* Main entry point. */
15299 :
15300 : static unsigned int
15301 2869204 : execute_lower_omp (void)
15302 : {
15303 2869204 : gimple_seq body;
15304 2869204 : int i;
15305 2869204 : omp_context *ctx;
15306 :
15307 : /* This pass always runs, to provide PROP_gimple_lomp.
15308 : But often, there is nothing to do. */
15309 2869204 : if (flag_openacc == 0 && flag_openmp == 0
15310 2815106 : && flag_openmp_simd == 0)
15311 : return 0;
15312 :
15313 56355 : all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
15314 : delete_omp_context);
15315 :
15316 56355 : body = gimple_body (current_function_decl);
15317 :
15318 56355 : scan_omp (&body, NULL);
15319 56355 : gcc_assert (taskreg_nesting_level == 0);
15320 82335 : FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
15321 25980 : finish_taskreg_scan (ctx);
15322 56355 : taskreg_contexts.release ();
15323 :
15324 56355 : if (all_contexts->root)
15325 : {
15326 25477 : if (make_addressable_vars)
15327 620 : push_gimplify_context ();
15328 25477 : lower_omp (&body, NULL);
15329 25473 : if (make_addressable_vars)
15330 620 : pop_gimplify_context (NULL);
15331 : }
15332 :
15333 56351 : if (all_contexts)
15334 : {
15335 56351 : splay_tree_delete (all_contexts);
15336 56351 : all_contexts = NULL;
15337 : }
15338 56351 : BITMAP_FREE (make_addressable_vars);
15339 56351 : BITMAP_FREE (global_nonaddressable_vars);
15340 :
15341 : /* If current function is a method, remove artificial dummy VAR_DECL created
15342 : for non-static data member privatization, they aren't needed for
15343 : debuginfo nor anything else, have been already replaced everywhere in the
15344 : IL and cause problems with LTO. */
15345 56351 : if (DECL_ARGUMENTS (current_function_decl)
15346 32248 : && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
15347 65943 : && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
15348 : == POINTER_TYPE))
15349 9057 : remove_member_access_dummy_vars (DECL_INITIAL (current_function_decl));
15350 :
15351 57327 : for (auto task_stmt : task_cpyfns)
15352 504 : finalize_task_copyfn (task_stmt);
15353 56351 : task_cpyfns.release ();
15354 56351 : return 0;
15355 : }
15356 :
15357 : namespace {
15358 :
15359 : const pass_data pass_data_lower_omp =
15360 : {
15361 : GIMPLE_PASS, /* type */
15362 : "omplower", /* name */
15363 : OPTGROUP_OMP, /* optinfo_flags */
15364 : TV_NONE, /* tv_id */
15365 : PROP_gimple_any, /* properties_required */
15366 : PROP_gimple_lomp | PROP_gimple_lomp_dev, /* properties_provided */
15367 : 0, /* properties_destroyed */
15368 : 0, /* todo_flags_start */
15369 : 0, /* todo_flags_finish */
15370 : };
15371 :
15372 : class pass_lower_omp : public gimple_opt_pass
15373 : {
15374 : public:
15375 285722 : pass_lower_omp (gcc::context *ctxt)
15376 571444 : : gimple_opt_pass (pass_data_lower_omp, ctxt)
15377 : {}
15378 :
15379 : /* opt_pass methods: */
15380 2869204 : unsigned int execute (function *) final override
15381 : {
15382 2869204 : return execute_lower_omp ();
15383 : }
15384 :
15385 : }; // class pass_lower_omp
15386 :
15387 : } // anon namespace
15388 :
15389 : gimple_opt_pass *
15390 285722 : make_pass_lower_omp (gcc::context *ctxt)
15391 : {
15392 285722 : return new pass_lower_omp (ctxt);
15393 : }
15394 :
15395 : /* The following is a utility to diagnose structured block violations.
15396 : It is not part of the "omplower" pass, as that's invoked too late. It
15397 : should be invoked by the respective front ends after gimplification. */
15398 :
15399 : static splay_tree all_labels;
15400 :
15401 : /* Check for mismatched contexts and generate an error if needed. Return
15402 : true if an error is detected. */
15403 :
15404 : static bool
15405 553755 : diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
15406 : gimple *branch_ctx, gimple *label_ctx)
15407 : {
15408 553755 : gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx));
15409 553755 : gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx));
15410 :
15411 553755 : if (label_ctx == branch_ctx)
15412 : return false;
15413 :
15414 180 : const char* kind = NULL;
15415 :
15416 180 : if (flag_openacc)
15417 : {
15418 7 : if ((branch_ctx && is_gimple_omp_oacc (branch_ctx))
15419 16 : || (label_ctx && is_gimple_omp_oacc (label_ctx)))
15420 : {
15421 : gcc_checking_assert (kind == NULL);
15422 : kind = "OpenACC";
15423 : }
15424 : }
15425 : if (kind == NULL)
15426 : {
15427 164 : gcc_checking_assert (flag_openmp || flag_openmp_simd);
15428 : kind = "OpenMP";
15429 : }
15430 :
15431 : /* Previously we kept track of the label's entire context in diagnose_sb_[12]
15432 : so we could traverse it and issue a correct "exit" or "enter" error
15433 : message upon a structured block violation.
15434 :
15435 : We built the context by building a list with tree_cons'ing, but there is
15436 : no easy counterpart in gimple tuples. It seems like far too much work
15437 : for issuing exit/enter error messages. If someone really misses the
15438 : distinct error message... patches welcome. */
15439 :
15440 : #if 0
15441 : /* Try to avoid confusing the user by producing and error message
15442 : with correct "exit" or "enter" verbiage. We prefer "exit"
15443 : unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
15444 : if (branch_ctx == NULL)
15445 : exit_p = false;
15446 : else
15447 : {
15448 : while (label_ctx)
15449 : {
15450 : if (TREE_VALUE (label_ctx) == branch_ctx)
15451 : {
15452 : exit_p = false;
15453 : break;
15454 : }
15455 : label_ctx = TREE_CHAIN (label_ctx);
15456 : }
15457 : }
15458 :
15459 : if (exit_p)
15460 : error ("invalid exit from %s structured block", kind);
15461 : else
15462 : error ("invalid entry to %s structured block", kind);
15463 : #endif
15464 :
15465 : /* If it's obvious we have an invalid entry, be specific about the error. */
15466 180 : if (branch_ctx == NULL)
15467 53 : error ("invalid entry to %s structured block", kind);
15468 : else
15469 : {
15470 : /* Otherwise, be vague and lazy, but efficient. */
15471 127 : error ("invalid branch to/from %s structured block", kind);
15472 : }
15473 :
15474 180 : gsi_replace (gsi_p, gimple_build_nop (), false);
15475 180 : return true;
15476 : }
15477 :
15478 : /* Pass 1: Create a minimal tree of structured blocks, and record
15479 : where each label is found. */
15480 :
15481 : static tree
15482 3301018 : diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
15483 : struct walk_stmt_info *wi)
15484 : {
15485 3301018 : gimple *context = (gimple *) wi->info;
15486 3301018 : gimple *inner_context;
15487 3301018 : gimple *stmt = gsi_stmt (*gsi_p);
15488 :
15489 3301018 : *handled_ops_p = true;
15490 :
15491 3301018 : switch (gimple_code (stmt))
15492 : {
15493 315299 : WALK_SUBSTMTS;
15494 :
15495 82903 : case GIMPLE_OMP_PARALLEL:
15496 82903 : case GIMPLE_OMP_TASK:
15497 82903 : case GIMPLE_OMP_SCOPE:
15498 82903 : case GIMPLE_OMP_SECTIONS:
15499 82903 : case GIMPLE_OMP_SINGLE:
15500 82903 : case GIMPLE_OMP_SECTION:
15501 82903 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15502 82903 : case GIMPLE_OMP_MASTER:
15503 82903 : case GIMPLE_OMP_MASKED:
15504 82903 : case GIMPLE_OMP_ORDERED:
15505 82903 : case GIMPLE_OMP_SCAN:
15506 82903 : case GIMPLE_OMP_CRITICAL:
15507 82903 : case GIMPLE_OMP_TARGET:
15508 82903 : case GIMPLE_OMP_TEAMS:
15509 82903 : case GIMPLE_OMP_TASKGROUP:
15510 : /* The minimal context here is just the current OMP construct. */
15511 82903 : inner_context = stmt;
15512 82903 : wi->info = inner_context;
15513 82903 : walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
15514 82903 : wi->info = context;
15515 82903 : break;
15516 :
15517 52743 : case GIMPLE_OMP_FOR:
15518 52743 : inner_context = stmt;
15519 52743 : wi->info = inner_context;
15520 : /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15521 : walk them. */
15522 52743 : walk_gimple_seq (gimple_omp_for_pre_body (stmt),
15523 : diagnose_sb_1, NULL, wi);
15524 52743 : walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
15525 52743 : wi->info = context;
15526 52743 : break;
15527 :
15528 549889 : case GIMPLE_LABEL:
15529 1099778 : splay_tree_insert (all_labels,
15530 549889 : (splay_tree_key) gimple_label_label (
15531 549889 : as_a <glabel *> (stmt)),
15532 : (splay_tree_value) context);
15533 549889 : break;
15534 :
15535 : default:
15536 : break;
15537 : }
15538 :
15539 3301018 : return NULL_TREE;
15540 : }
15541 :
15542 : /* Pass 2: Check each branch and see if its context differs from that of
15543 : the destination label's context. */
15544 :
15545 : static tree
15546 3301018 : diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
15547 : struct walk_stmt_info *wi)
15548 : {
15549 3301018 : gimple *context = (gimple *) wi->info;
15550 3301018 : splay_tree_node n;
15551 3301018 : gimple *stmt = gsi_stmt (*gsi_p);
15552 :
15553 3301018 : *handled_ops_p = true;
15554 :
15555 3301018 : switch (gimple_code (stmt))
15556 : {
15557 315299 : WALK_SUBSTMTS;
15558 :
15559 82903 : case GIMPLE_OMP_PARALLEL:
15560 82903 : case GIMPLE_OMP_TASK:
15561 82903 : case GIMPLE_OMP_SCOPE:
15562 82903 : case GIMPLE_OMP_SECTIONS:
15563 82903 : case GIMPLE_OMP_SINGLE:
15564 82903 : case GIMPLE_OMP_SECTION:
15565 82903 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15566 82903 : case GIMPLE_OMP_MASTER:
15567 82903 : case GIMPLE_OMP_MASKED:
15568 82903 : case GIMPLE_OMP_ORDERED:
15569 82903 : case GIMPLE_OMP_SCAN:
15570 82903 : case GIMPLE_OMP_CRITICAL:
15571 82903 : case GIMPLE_OMP_TARGET:
15572 82903 : case GIMPLE_OMP_TEAMS:
15573 82903 : case GIMPLE_OMP_TASKGROUP:
15574 82903 : wi->info = stmt;
15575 82903 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
15576 82903 : wi->info = context;
15577 82903 : break;
15578 :
15579 52743 : case GIMPLE_OMP_FOR:
15580 52743 : wi->info = stmt;
15581 : /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15582 : walk them. */
15583 52743 : walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
15584 : diagnose_sb_2, NULL, wi);
15585 52743 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
15586 52743 : wi->info = context;
15587 52743 : break;
15588 :
15589 203840 : case GIMPLE_COND:
15590 203840 : {
15591 203840 : gcond *cond_stmt = as_a <gcond *> (stmt);
15592 203840 : tree lab = gimple_cond_true_label (cond_stmt);
15593 203840 : if (lab)
15594 : {
15595 203840 : n = splay_tree_lookup (all_labels,
15596 : (splay_tree_key) lab);
15597 407680 : diagnose_sb_0 (gsi_p, context,
15598 203840 : n ? (gimple *) n->value : NULL);
15599 : }
15600 203840 : lab = gimple_cond_false_label (cond_stmt);
15601 203840 : if (lab)
15602 : {
15603 203840 : n = splay_tree_lookup (all_labels,
15604 : (splay_tree_key) lab);
15605 407680 : diagnose_sb_0 (gsi_p, context,
15606 203840 : n ? (gimple *) n->value : NULL);
15607 : }
15608 : }
15609 : break;
15610 :
15611 110263 : case GIMPLE_GOTO:
15612 110263 : {
15613 110263 : tree lab = gimple_goto_dest (stmt);
15614 110263 : if (TREE_CODE (lab) != LABEL_DECL)
15615 : break;
15616 :
15617 110263 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15618 110263 : diagnose_sb_0 (gsi_p, context, n ? (gimple *) n->value : NULL);
15619 : }
15620 110263 : break;
15621 :
15622 424 : case GIMPLE_SWITCH:
15623 424 : {
15624 424 : gswitch *switch_stmt = as_a <gswitch *> (stmt);
15625 424 : unsigned int i;
15626 1724 : for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i)
15627 : {
15628 1316 : tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i));
15629 1316 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15630 1316 : if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
15631 : break;
15632 : }
15633 : }
15634 : break;
15635 :
15636 744 : case GIMPLE_ASM:
15637 744 : {
15638 744 : gasm *asm_stmt = as_a <gasm *> (stmt);
15639 759 : for (unsigned i = 0; i < gimple_asm_nlabels (asm_stmt); ++i)
15640 : {
15641 21 : tree lab = TREE_VALUE (gimple_asm_label_op (asm_stmt, i));
15642 21 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15643 21 : if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
15644 : break;
15645 : }
15646 : }
15647 : break;
15648 :
15649 34475 : case GIMPLE_RETURN:
15650 34475 : diagnose_sb_0 (gsi_p, context, NULL);
15651 34475 : break;
15652 :
15653 : default:
15654 : break;
15655 : }
15656 :
15657 3301018 : return NULL_TREE;
15658 : }
15659 :
15660 : static unsigned int
15661 56364 : diagnose_omp_structured_block_errors (void)
15662 : {
15663 56364 : struct walk_stmt_info wi;
15664 56364 : gimple_seq body = gimple_body (current_function_decl);
15665 :
15666 56364 : all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
15667 :
15668 56364 : memset (&wi, 0, sizeof (wi));
15669 56364 : walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
15670 :
15671 56364 : memset (&wi, 0, sizeof (wi));
15672 56364 : wi.want_locations = true;
15673 56364 : walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
15674 :
15675 56364 : gimple_set_body (current_function_decl, body);
15676 :
15677 56364 : splay_tree_delete (all_labels);
15678 56364 : all_labels = NULL;
15679 :
15680 56364 : return 0;
15681 : }
15682 :
15683 : namespace {
15684 :
15685 : const pass_data pass_data_diagnose_omp_blocks =
15686 : {
15687 : GIMPLE_PASS, /* type */
15688 : "*diagnose_omp_blocks", /* name */
15689 : OPTGROUP_OMP, /* optinfo_flags */
15690 : TV_NONE, /* tv_id */
15691 : PROP_gimple_any, /* properties_required */
15692 : 0, /* properties_provided */
15693 : 0, /* properties_destroyed */
15694 : 0, /* todo_flags_start */
15695 : 0, /* todo_flags_finish */
15696 : };
15697 :
15698 : class pass_diagnose_omp_blocks : public gimple_opt_pass
15699 : {
15700 : public:
15701 285722 : pass_diagnose_omp_blocks (gcc::context *ctxt)
15702 571444 : : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
15703 : {}
15704 :
15705 : /* opt_pass methods: */
15706 2869218 : bool gate (function *) final override
15707 : {
15708 2869218 : return flag_openacc || flag_openmp || flag_openmp_simd;
15709 : }
15710 56364 : unsigned int execute (function *) final override
15711 : {
15712 56364 : return diagnose_omp_structured_block_errors ();
15713 : }
15714 :
15715 : }; // class pass_diagnose_omp_blocks
15716 :
15717 : } // anon namespace
15718 :
15719 : gimple_opt_pass *
15720 285722 : make_pass_diagnose_omp_blocks (gcc::context *ctxt)
15721 : {
15722 285722 : return new pass_diagnose_omp_blocks (ctxt);
15723 : }
15724 :
15725 :
15726 : #include "gt-omp-low.h"
|
